Compare commits

..

25 Commits

Author SHA1 Message Date
codex-bot a68e36907f Document exact IJCAI 2018 and 2022 section discrepancies
Metadata integrity checks / metadata (push) Successful in 1m22s
2026-11-08 23:36:50 +08:00
Zayden-Jung 4ec82c8f8f Merge pull request '保留同一来源分批导入后的覆盖统计' (#27) from codex/incremental-coverage into main
Metadata integrity checks / metadata (push) Successful in 1m20s
Reviewed-on: #27
2026-11-08 15:10:16 +00:00
Zayden-Jung bdc9b2aed5 Merge pull request '按证据补充稳定标识并保留原始书目字段' (#26) from codex/identifier-curation into main
Metadata integrity checks / metadata (push) Successful in 1m15s
Reviewed-on: #26
2026-11-08 15:10:10 +00:00
Zayden-Jung 621ceba331 Merge pull request '核对43会议年度、公开决定政策与早期WWW逐来源边界' (#20) from codex/coverage-ledger into main
Metadata integrity checks / metadata (push) Successful in 1m18s
Reviewed-on: #20
2026-11-08 15:07:27 +00:00
Zayden-Jung daf15e21d3 Merge pull request '让稳定论文身份优先并拒绝多候选标题自动合并' (#25) from codex/paper-identity-guards into main
Metadata integrity checks / metadata (push) Successful in 1m16s
Reviewed-on: #25
2026-11-08 15:07:15 +00:00
Zayden-Jung 33fd7bf2e1 Merge pull request '修正早期主系列名称与附属 workshop 的分类边界' (#24) from codex/historical-editions into main
Metadata integrity checks / metadata (push) Successful in 1m11s
Reviewed-on: #24
2026-11-08 14:17:09 +00:00
Zayden-Jung ec6076700b Merge pull request '单列核实身份的 ICLR workshop 公开负面标签' (#22) from codex/iclr-workshop-labels into main
Metadata integrity checks / metadata (push) Successful in 1m14s
Reviewed-on: #22
2026-11-08 14:17:00 +00:00
Zayden-Jung 6b66febba5 Merge pull request '保留缺标题公开决定并识别明确撤稿标签' (#21) from codex/public-negative-holds into main
Metadata integrity checks / metadata (push) Successful in 1m13s
Reviewed-on: #21
2026-11-08 14:10:27 +00:00
Zayden-Jung c8531d8a14 Merge pull request '在 PostgreSQL 报告会议覆盖与完整决定历史' (#19) from codex/postgres-coverage into main
Metadata integrity checks / metadata (push) Successful in 1m13s
Reviewed-on: #19
2026-11-08 14:10:17 +00:00
Zayden-Jung 9726e31677 Merge pull request '规范CVF目录fragment链接并保留歧义检查' (#18) from codex/cvf-fragments into main
Metadata integrity checks / metadata (push) Successful in 1m15s
Reviewed-on: #18
2026-11-08 14:10:09 +00:00
codex-bot 71d3cf0590 Verify incremental coverage rejects a concurrent parent update
Metadata integrity checks / metadata (push) Successful in 1m18s
2026-11-08 17:33:59 +08:00
codex-bot ee140cfd40 Preserve source coverage when importing incremental metadata batches
Metadata integrity checks / metadata (push) Successful in 1m18s
2026-11-08 17:31:12 +08:00
codex-bot 1cf1d228bc Prepare evidence-bound identifier additions without rewriting source records
Metadata integrity checks / metadata (push) Successful in 1m20s
2026-11-08 16:25:33 +08:00
codex-bot 9cc7b5b050 Keep stable paper identities separate from ambiguous title matches
Metadata integrity checks / metadata (push) Successful in 1m16s
2026-11-08 15:47:07 +08:00
codex-bot 3f3dc3f7ad Distinguish historical primary editions from affiliated workshops
Metadata integrity checks / metadata (push) Successful in 1m9s
2026-11-08 15:12:18 +08:00
codex-bot 5fad2fc575 Merge main and preserve coverage and history audit checks
Metadata integrity checks / metadata (push) Successful in 1m17s
2026-11-08 14:53:20 +08:00
Zayden-Jung 1b81d1a30a Merge pull request '保留公开元数据摘要内的 Unicode 分隔符' (#17) from codex/jsonl-lines into main
Metadata integrity checks / metadata (push) Successful in 3m15s
Reviewed-on: #17
2026-11-08 06:26:27 +00:00
Zayden-Jung 6cf74672a5 Merge pull request '只读审计跨会议投稿历史与决定来源关系' (#23) from codex/history-audit into main
Metadata integrity checks / metadata (push) Successful in 2m25s
Reviewed-on: #23
2026-11-08 06:26:09 +00:00
codex-bot 5a925457b0 Audit cross-venue submission histories without inferring chronology
Metadata integrity checks / metadata (push) Successful in 3m0s
2026-11-08 04:30:19 +08:00
codex-bot 8e82742900 Recognize explicit ICLR workshop decisions with scope guards
Metadata integrity checks / metadata (push) Successful in 2m40s
2026-11-08 04:18:09 +08:00
codex-bot f327861a00 Hold untitled public assertions and retain explicit withdrawals
Metadata integrity checks / metadata (push) Successful in 1m17s
2026-11-08 03:23:25 +08:00
codex-bot fd3fdb6cf7 Aggregate coverage paper counts once per snapshot
Metadata integrity checks / metadata (push) Successful in 1m17s
2026-11-08 02:57:09 +08:00
codex-bot 346208a272 Audit current PostgreSQL conference coverage and decision history
Metadata integrity checks / metadata (push) Successful in 3m50s
2026-11-08 02:42:21 +08:00
codex-bot c7cc4d78f7 Deduplicate CVF index links by HTTP resource
Metadata integrity checks / metadata (push) Successful in 3m22s
2026-11-07 16:42:20 +08:00
codex-bot bf8178354f Preserve Unicode separators in public metadata preflight
Metadata integrity checks / metadata (push) Successful in 2m42s
2026-11-07 16:34:54 +08:00
30 changed files with 10658 additions and 56 deletions
+4
View File
@@ -53,8 +53,12 @@ jobs:
subprocess.run([python,'-I','-B','-m','paper_atlas','import','arxiv','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','backfill','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','public-openreview','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','coverage','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','histories','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','curate','identifiers','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-c','from paper_atlas.postgres.writer import PostgresStore, WriteConnection'],cwd=outside,check=True)
smoke = "from paper_atlas.db.store import Store; from paper_atlas.config import DEFAULT_REGISTRY; import json; assert {'COLM','MM','MLSys'} <= {v['acronym'] for v in json.loads(DEFAULT_REGISTRY.read_text())['venues']}; s=Store('installed.sqlite'); s.conn.close()"
subprocess.run([python,'-I','-B','-c',smoke],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','query','--db','installed.sqlite','--stats'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','histories','--db','installed.sqlite','--examples','0'],cwd=outside,check=True)
PY
+2 -1
View File
@@ -51,7 +51,8 @@ PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas query --pg-service fo
PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas query --pg-service foundry-research --paper 123
# 列出全部目标会议,包括尚未入库的项目
paper-atlas coverage --db /data/paper-index/papers.sqlite \
PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas coverage \
--pg-service foundry-research --pg-schema research \
--out /data/paper-index/collected/venue-coverage-current.json
# 采集与导入分开;这里只采集到独立文件
+2
View File
@@ -188,3 +188,5 @@ MM 2026 的[官方 CFP](https://2026.acmmm.org/site/cfp-guidelines.html)说明
WWW早期逐来源核查见 `www-early-proceedings-2026-11-08.json`:1995春季论文三目录均为38个相同条目ID,海报摘要53个条目与作者链接47个条目不齐;1994春季50个Title字段与主页49场报告说法待逐项核对。上述为源目录观察,不是已导入或全部录用数量;主会、海报、教程等继续独立核验。
WWW事件标识生产迁移已独立验收(报告SHA `f5dbb8a8`):保留1399个原届次字段及IDs、59个Fall投稿和完整全库历史哈希。服务器已缓存46个官方HTML来源(约223KB),只作为后续维护式采集器的输入;尚未补导早期WWW论文,也未启用可选注解表。
IJCAI 联办届次的逐条后续核查见 [2018/2022 分区与目录差集](ijcai-sections-2026-11-08.md) 和 [稳定 ID 映射](ijcai-sections-2026-11-08.json)。已定位 345 条非 main 分类及 2022 官方 HTML 缺失但官方文件仍存在的一条记录;生产分类尚未修正,不能把整卷数量作为 main 覆盖率。
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,40 @@
# IJCAI 2018、2022 逐条目录与 track 核查
2026-11-08 的只读生产快照中,edition 867(2018)和 871(2022)已正确属于 IJCAI/ECAI 联办会议,但整本论文集的所有投稿仍归在 `main`。本核查用官方目录的逐条 DOI 和分区定位问题,不修改生产数据。
| 年份 | 库中投稿 | 官方目录条目 | DOI 一对一对应 | 官方 Main track | 明确非 main | 当前目录缺项 |
| --- | ---: | ---: | ---: | ---: | ---: | ---: |
| 2018 | 870 | 870 | 870 | 709 | 161 | 0 |
| 2022 | 863 | 862 | 862 | 678 | 184 | 1 |
2018 官方分区为 Main track 709、Invited Speakers 3、Evolution of the contours of AI 6、Sister Conferences Best Papers 46、Survey track 16、Journal track 24、Early Career 19、Doctoral Consortium 17、Demos 30。
2022 官方分区为 Main Track 678、Special Track on AI, the Arts and Creativity 17、Special Track on AI for Good 33、Sister Conferences Best Papers 26、Survey Track 38、Journal Track 19、Early Career 8、Doctoral Consortium 25、Demo Track 18。Special Track 保留独立范围,不因同在正式论文集就合进 main。
## 862/863 差异已定位,不能删一条凑数
数据库独有条目:`conf/ijcai/Fang22`,题名 **Fixed-Budget Pure Exploration in Multinomial Logit Bandits**,DOI `10.24963/ijcai.2022/409`,页码 2951–2957。submission 333599、source_record 3578445、paper 3442268。
[当前官方总目录](https://ijcai.org/proceedings/2022/) 没有这一条,详情页 `/proceedings/2022/409` 本次 HTTP 404;但[官方论文文件](https://ijcai.org/proceedings/2022/0409.pdf) HEAD 返回 HTTP 200、application/pdf、153657 字节。浏览器读取的官方文件标题、作者 Boli Fang、IJCAI-22 页脚及首页 2951 与数据库对应,共七页。服务器没有下载保存 PDF 正文。
因此这是已定位的“当前 HTML 目录缺项但官方文件仍存在”的来源差异。没有证据证明撤稿、拒稿或数据库多收了一篇;保留原条目与发表证据。其当前官方 track 仍待明确,不根据相邻编号擅自归 main。
## 可复核证据
同目录 JSON 保存两份官方 HTML 的 URL、HTTP 状态、时间、字节数和 SHA256,以及生产只读导出 SHA256。服务器快照目录 `/data/paper-index/collected/ijcai-section-audit-v1/` 保留 HTML、来源回执、原始数据库来源导出和 409 HTTP 回执。
JSON 的 `years[].sections[].records` 每条为 `[submission_id, source_record_id, doi_suffix]`;DOI 为 `10.24963/ijcai.<year>/<doi_suffix>`。这些是现有稳定 ID,不是待插入的新投稿。全部 1732 个当前目录条目均对应唯一数据库来源 DOI,目录与数据库均无重复 DOI,官方目录独有条目为零。
标题仅作辅助核验:NFKC、casefold、去除 Unicode 非 word 字符后,2018 全部一致;2022 有一个拼写差异,DOI `/552` 的 DBLP 标题将 Revisiting、Re-Construction 拼为 Revisting、Re-Constrcution。原始标题均保留,未因此创建或合并论文。
## 未完成事项及修正边界
本次交付只完成来源核查和逐条分区映射,**345 条非 main 的生产分类尚未修正**,第 14 步尚未完成。下一步需要可复用的、带快照前提和验收报告的分区修正流程:
- 保持原 submission、source_record、paper、decision 的 ID 和证据内容;纠正 edition 归属,不制造“新一次投稿”。
- 同步处理依赖 edition 的 submission_identifiers、覆盖记录与联合会议别名;不能只移动 submissions 一个字段。
- 保存官方分区文字、URL、源文件 SHA 和原始分类,防止以后 DBLP 整卷刷新把精确 track 覆盖回 main。
- 对 409 保持单独待核实状态,不推断撤稿、拒稿或当前 main 归属。
- 修正后独立核验完整投稿/决定历史、稳定标识、各 track 行数及实际查询。研究方法类型不能由 track 自动生成。
这是 2018、2022 两届的完整目录差集和分区证据,不代表其他年份已逐条验收,也不替代所有导入结束后的全库去重。
+10
View File
@@ -95,6 +95,12 @@ paper-atlas audit public-openreview --db /data/paper-index/papers.sqlite \
输出按会议、届次及 main、position、datasets_benchmarks 分文件。workshop 默认进入单独的 held 清单;有明确 workshop 名称时可通过 `--include-workshops` 单列采集,不合并到主会。Retraction、未映射的轨道或负面标签也保留在 `held-records.jsonl`,不强改为 Reject 或 Withdrawn。当前决定表没有独立 retracted 状态,因此撤回发表记录仍需后续专门建模。
`parser_version=2` 还会将明确负面标签但标题缺失、空白或 null 的记录放入 `held-records.jsonl`,保留原始状态、forum ID、固定来源版本和行号;不制造标题或导入不可识别的论文,也不让这一条缺失阻断其他可用记录。held 文件哈希与各原因数量保存在 inventory;非文本标题、错误身份、冲突届次和哈希不符仍使整批失败。旧年 ICLR 镜像的明确 `Withdraw` 标签归为 `withdrawn`,`Workshop`、`Poster Workshop`、`Oral Workshop` 均不作为拒稿。采集文件与导入主库仍是两个独立步骤。
`parser_version=3` 增加五组已核实身份的 ICLR workshop 标签:2024 ME-FoMo / GenAI4DM 的 Withdrawn Submission、2025 AgenticAI 的 Reject / LMRL 的 Withdrawn Submission,以及 2026 LLM Reasoning 的 Withdrawn Submission。仅在 `--include-workshops` 下输出,并分别保存 `workshop_me_fomo`、`workshop_genai4dm`、`workshop_agenticai`、`workshop_lmrl`、`workshop_llm_reasoning`;默认仍暂存。标签必须与原文件会议、年份和原始 workshop track 一致,错年/错会议使采集失败,错轨道单独暂存,未知标签继续待核查,不用宽泛的 Reject/Withdraw 正则猜测。
[OpenReview 官方 ICLR 活动目录](https://openreview.net/venue?id=ICLR.cc) 确认上述独立 workshop 身份;[ME-FoMo 2024](https://sites.google.com/view/me-fomo2024/call-for-papers)、[GenAI4DM 2024](https://sites.google.com/view/genai4dm-iclr2024)、[AgenticAI 2025](https://iclragenticai.github.io/) 和 [LLM Reasoning 2026](https://sites.google.com/view/iclr-2026-llmreasoning) 的活动网站也提供对应投稿入口。输出在 metadata 保存 `workshop_venueid` 和身份目录证据,决定仍只标为固定镜像的公开分类断言,保留原始标签、forum、来源行号与哈希,不声称核验原始决定签名。2026-11-08 核查时部分原站记录页要求浏览器验证;身份目录和主办方入口可读,不代表逐篇原始决定已可访问,也不外推其他 workshop 的公开政策。采集后仍须经过 PostgreSQL 身份/决定冲突预检、串行导入和独立验收。
该来源明确标记为 `curated_public_status_assertion`:保留源 commit、文件哈希、行号、原标签和 forum ID,不声称已核验原始 PC 签名。只保存标题、摘要、公开作者名、关键词、领域、索引与决定来源,不缓存镜像中的个人资料、邮箱、性别、评审内容或 PDF。所有输出清单均为 partial;盘点中的零条记录仅描述这个固定镜像。
入库前审计核验文件哈希、数量、轨道和来源声明,并用真实 forum ID 对比同届同轨道已有投稿。存在不同决定时阻止整批导入,交由来源和时序核查;另一会议的后续录用属于独立投稿历史,不构成该次拒稿的冲突。同一 forum 的镜像不新增投稿事件,但论文实体的跨平台去重仍须在全量结束后核验。审计通过的报告包含带 SHA-256 的导入队列;应在单写入维护锁下及时串行导入,主库变化后重新审计。
@@ -180,6 +186,10 @@ paper-atlas collect dblp \
输出按会议/年份/track 分文件,附 manifest、`dblp-normalization.json` 范围审查统计和带哈希的 `dblp-import-queue.json`。不能识别的书目、年份或归属会逐类登记,不默认算作主会。DBLP 不含摘要,因此输出摘要为空、清单为 partial;需要通过有证据的跨来源关联补齐论文实体摘要。书目发表证据与原始审稿决定分别保留,不依据书目缺席推断拒稿。已存在其他出版来源的届次,还须先回填其权威身份并核验镜像记录,不能直接累加投稿数。
早期 AISTATS 1995/1997/1999/2001/2003/2005 和 COLT 1988–1991 的主系列名称本身含 Workshop,不能当作附属 workshop 漏掉。识别要求已核对的完整 proceedings key、会议书目名、原始标题前缀、届数和事件年一致;轨道名为 `main` 表示主系列,保留原始 Workshop 名称及官方目录证据,不改造其历史名称或出版形式。参见 [AISTATS 官方历史目录](https://aistats.org/other.html)、[COLT 协会目录](https://www.learningtheory.org/)。1995 AISTATS 的会议年与1996图书出版年分开;[PMLR R0](https://proceedings.mlr.press/r0/) 是后来重发的 pre-proceedings,不与 DBLP 图书条目盲目逐篇等同。
真正附属的 AISTATS 2010 Active Learning and Experimental Design workshop 仍单列。COLT 2003 与 Kernel Workshop 的 [混合论文集](https://doi.org/10.1007/b12006) 缺少逐篇章节信息,使用 `proceedings_unspecified`,不能整卷改成 main 或附属 workshop。输出保存 `track_scope_note` 与 `track_scope_evidence_url`,导入后仍可追查分类理由;这些出版证据不是真实拒稿或原始决定签名,也不自动修正既有主库记录。
部分早期 SIGIR、CIKM、IJCAI 出版卷使用两位年份键;只有论文集标题明确给出唯一完整年份且与短键相符时才采用,不推测世纪。WWW 特殊兴趣、Alternate track 单列;同卷混合 Applied Data Science、Demo、Industrial 且不能逐篇区分时使用 `proceedings_unspecified`,不把整卷归为某一个具体 track。
书目中的前后页、目录组合和明确会务标题不产生论文录用记录;缺作者的赞助、委员会、活动汇总等标题单独排除。WWW 2019 companion 的匿名合成活动条目归入待核查材料,原始快照仍保留。范围报告分别统计这些原因;单纯缺作者并不排除真实研究论文,也不补造作者。
+38 -2
View File
@@ -97,13 +97,16 @@ paper-atlas audit arxiv-tags --db /data/paper-index/papers.sqlite \
采集文件、已入库的来源记录和去重论文实体是不同的计数。来源可能覆盖同一篇论文,同一篇论文也可能有多次投稿;不能把各批文件数量相加作为独立论文总数。实时统计与覆盖报告使用:
```sh
paper-atlas query --db /data/paper-index/papers.sqlite --stats
paper-atlas coverage --db /data/paper-index/papers.sqlite \
PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas query --pg-service foundry-research --pg-schema research --stats
PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas coverage \
--pg-service foundry-research --pg-schema research \
--out /data/paper-index/collected/venue-coverage-current.json
```
manifest 和采集器状态清单用于确认文件是否齐全及是否缺摘要;`ingest_runs` 和协调器日志用于确认主库导入是否成功。官方程序、书目目录和公开投稿快照分别保留,未出现在某份清单不等于被拒稿;第三方快照也不能代表平台当前全量状态。代码测试成功不能代替最终全库去重、完整性和逐届覆盖核验。
`coverage` 的所有计数来自同一个只读快照。`latest_observed_status_counts` 按每次投稿的最新观察决定统计,`decision_observation_counts` 保留所有历史观察;重复来源不增加投稿数,缺少决定的投稿计入最新状态 unknown,但不凭空制造 unknown 决定。`source_record_counts` 统计来源在该届次的实际记录与缺摘要数,和 `sources` 中逐清单的预期/导入数量分开;身份补充清单与原始论文集清单可以描述同一条记录。来源缺摘要与论文实体从其他来源获得摘要也分别报告。报告仍将历史覆盖标为待独立审计,不能用某个清单 complete 或没有拒稿记录推断全会议覆盖。
当前代码与合并状态以 [Pull Requests](https://gitea.entroia.com/Zayden-Jung/paper-atlas/pulls) 为准;CI 结果以对应提交的 Actions 检查为准。工作分支交付后等待用户手动合并。合并代码不自动部署、不自动启动采集,也不代表数据批次已完成。
## 修正已有 PMLR 投稿的轨道
@@ -120,3 +123,36 @@ paper-atlas maintain pmlr-tracks --db /data/paper-index/papers.sqlite \
检查报告后,在同一命令添加 `--apply` 并使用新的 `--out` 路径执行迁移。计划使用共享维护锁,实际迁移使用独占锁;已有 arXiv 类别修复或其他写入运行时会拒绝启动,不会停止或重启它们。一次事务内核对整个来源集合、原始题目/摘要/作者、forum 身份、已有事件及目标冲突,更新原投稿的 edition 和身份归属,回填官方轨道出处及逐轨道覆盖数,最后复查轨道与外键后提交。论文、来源、投稿、决定的 ID 及决定观察均保留,不重新创建事件、不做实体合并、不改论文正文元数据或标签。
报告记录每个迁移事件的旧 edition、原身份、完整决定观察及新轨道证据。`planned` 是未执行的计划;`complete` 才表示事务已提交;`error` 表示执行失败。若进程在提交前后中断而报告停在 `prepared`,先对照主库和报告核查,不根据日志缺失自行重启任务。新路径重跑会重新核对当前数据库,已正确的轨道不会再次迁移。共享多个来源、重复目标事件或冲突决定会停止迁移,需另行核对,不能直接删掉其中一条。
## 追加同一目录的部分记录
默认 `import records` 把 manifest 当成一份来源快照,覆盖统计使用该文件的数量。
若目录已有2条,之后仅补入1条,不能把这一小批当成完整快照,否则会把目录计数改成1。
为已核验的部分补充批次,在 manifest 显式声明:
```json
{
"coverage_mode": "incremental",
"coverage_previous_count": 2,
"coverage_previous_expected_count": 2,
"coverage_expected_count": 4,
"status": "partial",
"count": 1
}
```
这只是新增字段示例,manifest仍须保留source、venue、year、track、list_url、文件哈希等来源证据。
`count`是本文件行数,`coverage_previous_count`是导入前该来源/届次的实际记录数,
`coverage_previous_expected_count`是旧覆盖行的expected_count(未知时用null),
`coverage_expected_count`是独立目录证据支持的整份来源预期数,不能小于已知旧预期数。
这个例子成功后覆盖统计为预期4、已导入3;文件checkpoint的records_written仍为1。
刷新相同记录不重复累加。初次导入部分来源可声明previous_count为0、previous_expected_count为null。
只支持当前最多一个规范目录URL且旧覆盖状态为partial的来源/届次;多目录、旧状态complete、
旧计数与实际来源记录不符,或前置计数已变化,都会中止,先核对来源范围或重新取得完整快照。
该模式在写入前检查父状态,并在同一文件事务中按实际来源记录重新计数、检查总量界限和乐观更新覆盖行。
失败会回滚本文件的论文、来源、决定、标签和覆盖变化,保留失败运行记录;更早已提交文件不重放。
增量模式始终保留partial,不能靠一个小批次推断完整论文集、摘要或公开拒稿覆盖。
仍需先独立核对新增记录身份与决定,按原生单写入保护串行导入,再独立核验数据与原有历史。
这些计数检查不证明外部目录完整,也不代替最终全库去重。
+76 -6
View File
@@ -2,6 +2,55 @@
先在已确认的远程环境安装项目,使用 `paper-atlas`。从源码运行时,将命令前缀替换为 `python3 -B -m paper_atlas`,其余参数相同。数据目录位于仓库外。
## 按证据补充稳定标识
`paper-atlas curate identifiers --plan /data/review/plan.json --output /data/review/prepared`
只生成新的 JSONL、manifest、证据副本及 `curation-report.json`,不会连接数据库。
输出目录必须不存在,父目录必须已存在;任何输入或断言失败,都不发布本批输出。
原始文件不变,未选中的记录逐字节保留。题名、作者、年份、页码、投稿决定、标签和既有 metadata
均不重写,仅补充 `identifiers` 并追加 `metadata.identifier_curation` 证据链。
计划格式如下(哈希占位符须替换为真实 SHA256,所有路径按命令当前目录解析):
```json
{
"format_version": 1,
"inputs": [{
"file": "/data/review/uai2001.jsonl",
"sha256": "<JSONL 文件 SHA256>",
"manifest_sha256": "<同名 .manifest.json 文件 SHA256>"
}],
"assertions": [{
"source": "dblp",
"external_id": "conf/uai/example",
"record_sha256": "<原始物理行字节 SHA256,包含该行原有换行符>",
"identifiers": {"arxiv": "1408.1485"},
"evidence": [{
"file": "/data/review/primary-metadata.json",
"sha256": "<证据 JSON 文件 SHA256>",
"url": "https://example.org/primary-record",
"checked_at": "2026-11-08T08:00:00Z",
"note": "说明人工核对的题名、作者、出版版本和稳定标识证据"
}]
}]
}
```
输入 manifest 必须包含 `source/venue/year/track/count/list_url/file_sha256`,且与逐条记录一致。
断言必须唯一、准确命中一条记录。仅支持无版本后缀的 arXiv 编号与小写规范 DOI;
已有同类标识不允许覆盖,相同值可以补记证据。证据只接受不超过 16 MiB 的 JSON 元数据文件,
来源 URL 要求 HTTPS 且不含凭据,核查时间必须带时区;不下载论文 PDF。
`verification_scope=input_and_evidence_integrity_only` 和 `semantic_identity_verified=false`
明确表示哈希校验不能证明两条记录属于同一作品。请先独立核对版本关系;仅年份、页码或相同题名
不能作为合并依据。存在来源冲突的行应继续暂缓。新 manifest 保留原来的 partial 状态及摘要缺失数,
不会把补充编号伪装成来源覆盖或摘要补全。证据副本按 SHA256 保存在输出目录 `evidence/`,
原计划也保存在 `plan.json`,供之后复查。
文件生成后仍需另做当前 PostgreSQL 身份、投稿和决定冲突预检;导入运行时必须包含
“稳定标识优先、多个标题候选则整文件回滚”的身份保护,再以 `import records` 独立导入并验收。
该命令不会自动导入或合并实体,也不代表最终全库去重通过。
## PostgreSQL 查询
在远程环境安装 `.[postgres]`,通过权限为 `0600` 的 libpq service 文件连接,不把密码放入 URL、命令行或仓库。
@@ -79,22 +128,43 @@ paper-atlas maintain merge --db /data/paper-index/samples/example.sqlite \
## 4. 查询与分析前检查
```sh
paper-atlas query --db /data/paper-index/papers.sqlite --query '"language model"' --limit 20
paper-atlas query --db /data/paper-index/papers.sqlite --tag arxiv:category=cs.AI
paper-atlas query --db /data/paper-index/papers.sqlite --venue ICLR --year 2025 --status rejected
paper-atlas query --db /data/paper-index/papers.sqlite --paper 123
paper-atlas coverage --db /data/paper-index/papers.sqlite --out /data/paper-index/collected/coverage.json
paper-atlas query --pg-service foundry-research --query '"language model"' --limit 20
paper-atlas query --pg-service foundry-research --tag arxiv:category=cs.AI
paper-atlas query --pg-service foundry-research --venue ICLR --year 2025 --status rejected
paper-atlas query --pg-service foundry-research --paper 123
PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas coverage \
--pg-service foundry-research --pg-schema research --out /data/paper-index/collected/coverage.json
```
`--status rejected` 筛选某次投稿的最新已知决定。它并不表示论文在所有会议均未录用;分析转投情况时查看该论文的完整投稿历史。年份表示会议届次,观察时间和决定日期分别保留。
覆盖报告使用只读一致性快照,标明实际数据库和 schema,并列出零入库目标与注册表外的已入库会议。逐届、逐 track 分别列出最新观察状态的投稿数、全部决定观察数、各来源记录数,以及来源摘要和论文实体摘要的缺失数。摘要缺失指 NULL 或空字符串;有文字不代表内容已经完整核验。同一来源的多个清单 URL 可以重叠,不能把清单计数相加当作来源总量;零条拒稿也不证明该会议没有公开拒稿。SQLite 样本仍可显式使用 `--db`,生产报告应查询当前 PostgreSQL。
例如,查找在 ICLR 有拒稿记录、但在其他已知投稿中已有录用的论文:
```sh
paper-atlas query --db /data/paper-index/papers.sqlite \\
paper-atlas query --pg-service foundry-research \
--venue ICLR --status rejected --outcome accepted
```
`known_acceptances` 列出目前仍为录用状态的各次投稿及会议、年份、track 和决定证据,`publication_outcome` 汇总为 accepted 或 no_known_acceptance。汇总会检查所有会议和 track,不把某次拒稿覆盖到整篇论文;同一次投稿的后续撤稿也会更新该次状态并保留原决定。no_known_acceptance 只表示现有来源中没有查到录用,不能称为“从未录用”。
`--stats` 中 papers 是当前规则下的论文实体数;source_records 是来源记录数;decisions 是决定观察数。一个投稿可能有多个来源和多条观察,不能相加当作论文数。
已有稳定标识或原来源记录匹配时,该身份优先;同标题、同作者的另一 arXiv 存档或扩展稿不会仅因相似标题被并入。没有身份锚点时,作者佐证的精确标题只有一个候选才可匹配;多个候选会报 `Ambiguous paper identity`,整个采集文件回滚并记录失败运行。应补有证据的标识或先核查版本关系,不删除重复候选或任取第一个。一个新来源明确同时携带两个既有稳定标识时,仍可按既有规则归并并保留投稿、标签和合并日志;未归并的同标题关系继续保留为待核查候选。这些规则不证明全库去重完成。
## 5. 投稿历史关系审计
```sh
PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas audit histories \
--pg-service foundry-research --pg-schema research --examples 20 \
--out /data/paper-index/collected/history-audit.json
```
使用一个只读一致性快照,区分论文实体、投稿和决定观察;分别统计同次投稿曾出现负面与录用观察、不同投稿共存负面与当前已知录用、不同会议之间的共存。workshop/main 是不同 track,属于同一会议时不算跨会议;会议别名也不会增加一个会议。当前已知录用使用与 `query --outcome` 相同的“每次投稿最新观察”规则,后续撤稿的该次投稿不再算当前录用,但旧录用决定仍保留。
报告不凭观察时间、届次年份或行顺序断言“拒稿发生在录用之前”。样例保留真实 `decision_date`、`observed_at`、原始标签、证据 URL 和决定来源,供进一步核查;日期缺失仍保持未知。`negative` 统计包含 rejected、desk_rejected 和 withdrawn,其原始状态不互相替换。
总数覆盖快照内全部投稿,样例由 `--examples` 限定为 0–100 篇;每篇最多展示 20 次投稿、每次最多 50 条决定、每条决定最多 100 个来源,并明确标记截断。样例优先保留负面/录用的不同投稿对。四项来源关系检查报告缺失关联和来源论文归属不一致,不自动修复。
这些统计依赖现有论文身份关联,不证明实体合并正确,也不证明原始决定签名、公开拒稿全集、标签类型或最终全库去重已完成。SQLite 独立样本可使用 `--db`;冻结备份不能代表当前 PostgreSQL 的历史情况。
+118 -27
View File
@@ -1,45 +1,136 @@
"""Show every target venue, including those with no imported records."""
"""Describe imported venue coverage without implying a complete census."""
import argparse
from paper_atlas.config import DEFAULT_REGISTRY
from collections import defaultdict
import json
from pathlib import Path
import sqlite3
from paper_atlas.config import DEFAULT_REGISTRY
from paper_atlas.db.backend import database_arguments, open_read_connection
from paper_atlas.db.submissions import timestamp
from paper_atlas.querying import LATEST_DECISION
def report(db,registry):
registry=json.loads(Path(registry).read_text());targets=registry['venues']
names=[row['acronym'].casefold() for row in targets]
if len(names)!=len(set(names)):raise ValueError('Duplicate target venue')
scope='(v.acronym=? COLLATE NOCASE OR EXISTS(SELECT 1 FROM venue_aliases a WHERE a.venue_id=v.id AND a.alias=? COLLATE NOCASE))'
c=sqlite3.connect(Path(db).resolve().as_uri()+'?mode=ro',uri=True);c.row_factory=sqlite3.Row
STATUSES = ('accepted', 'rejected', 'desk_rejected', 'withdrawn', 'under_review', 'unknown')
def _status_counts(connection, *, latest):
if latest:
query = '''SELECT s.edition_id,COALESCE(d.status,'unknown') AS status,count(*) AS count
FROM submissions s LEFT JOIN decisions d ON d.submission_id=s.id AND ''' + LATEST_DECISION
else:
query = '''SELECT s.edition_id,COALESCE(d.status,'unknown') AS status,count(*) AS count
FROM submissions s JOIN decisions d ON d.submission_id=s.id'''
counts = defaultdict(dict)
for row in connection.execute(query + " GROUP BY s.edition_id,COALESCE(d.status,'unknown')"):
counts[row[0]][row[1]] = row[2]
return counts
def report(db, registry, *, pg_service=None, pg_schema=None):
started_at = timestamp()
registry = json.loads(Path(registry).read_text())
targets = registry['venues']
names = [row['acronym'].casefold() for row in targets]
if len(names) != len(set(names)):
raise ValueError('Duplicate target venue')
scope = '''(lower(v.acronym)=lower(?) OR EXISTS(
SELECT 1 FROM venue_aliases a WHERE a.venue_id=v.id AND lower(a.alias)=lower(?)))'''
connection = open_read_connection(db, pg_service=pg_service, pg_schema=pg_schema)
if not pg_service:
connection.row_factory = sqlite3.Row
try:
c.execute('BEGIN');venues=[]
if pg_service:
identity = connection.execute('SELECT current_database(),current_schema()').fetchone()
database, schema = identity[0], identity[1]
else:
database, schema = str(Path(db).resolve()), None
latest = _status_counts(connection, latest=True)
observations = _status_counts(connection, latest=False)
# Aggregate the paper join once for the whole snapshot. Repeating it
# for each registry target caused repeated scans on the production DB.
edition_counts = {}
for row in connection.execute('''SELECT s.edition_id,count(s.id) AS submissions,
count(DISTINCT s.paper_id) AS papers,
count(DISTINCT CASE WHEN p.abstract IS NULL OR p.abstract='' THEN s.paper_id END)
AS papers_without_abstract_text
FROM submissions s LEFT JOIN papers p ON p.id=s.paper_id GROUP BY s.edition_id'''):
entry = dict(row)
edition_counts[entry.pop('edition_id')] = entry
source_counts = defaultdict(list)
for row in connection.execute('''SELECT s.edition_id,so.name AS source,
count(DISTINCT sr.id) AS source_records,
count(DISTINCT CASE WHEN sr.abstract IS NULL OR sr.abstract='' THEN sr.id END)
AS source_records_without_abstract_text
FROM submission_sources ss JOIN submissions s ON s.id=ss.submission_id
JOIN source_records sr ON sr.id=ss.source_record_id JOIN sources so ON so.id=sr.source_id
GROUP BY s.edition_id,so.name ORDER BY s.edition_id,so.name'''):
entry = dict(row)
source_counts[entry.pop('edition_id')].append(entry)
venues, covered_venue_ids = [], set()
for target in targets:
acronym=target['acronym'];params=(acronym,acronym)
editions=[]
for row in c.execute('SELECT e.id,v.acronym AS canonical_venue,e.year,e.track,count(s.id) AS submissions,count(DISTINCT s.paper_id) AS papers FROM editions e JOIN venues v ON v.id=e.venue_id LEFT JOIN submissions s ON s.edition_id=e.id WHERE '+scope+' GROUP BY e.id ORDER BY e.year,e.track',params):
edition=dict(row)
edition['sources']=[dict(source) for source in c.execute('SELECT so.name,es.status,es.expected_count,es.imported_count,es.list_url,es.last_success_at,es.last_error FROM edition_sources es JOIN sources so ON so.id=es.source_id WHERE es.edition_id=? ORDER BY so.name',(row['id'],))]
acronym = target['acronym']
params = (acronym, acronym)
editions = []
for row in connection.execute('''SELECT e.id,v.id AS venue_id,v.acronym AS canonical_venue,
e.year,e.track FROM editions e JOIN venues v ON v.id=e.venue_id
WHERE ''' + scope + '''
ORDER BY e.year,e.track,v.acronym''', params):
edition = dict(row)
edition.update(edition_counts.get(row['id'], {
'submissions': 0, 'papers': 0, 'papers_without_abstract_text': 0}))
covered_venue_ids.add(edition['venue_id'])
edition['latest_observed_status_counts'] = {status: latest[row['id']].get(status, 0) for status in STATUSES}
edition['decision_observation_counts'] = {status: observations[row['id']].get(status, 0) for status in STATUSES}
edition['source_record_counts'] = source_counts[row['id']]
edition['sources'] = [dict(source) for source in connection.execute('''SELECT so.name,
es.status,es.expected_count,es.imported_count,es.list_url,es.last_success_at,es.last_error
FROM edition_sources es JOIN sources so ON so.id=es.source_id
WHERE es.edition_id=? ORDER BY so.name,es.list_url''', (row['id'],))]
editions.append(edition)
totals=c.execute('SELECT count(s.id),count(DISTINCT s.paper_id) FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id WHERE '+scope,params).fetchone()
venues.append({'acronym':acronym,'field':target['field'],'imported_submissions':totals[0],'distinct_papers':totals[1],
'history_status':'partial_history_pending_independent_audit' if totals[0] else 'no_imported_records',
'discovery_sources':target.get('discovery_sources',[]),'editions':editions})
return {'checked_at':timestamp(),'target_venue_count':len(targets),'venues_with_imported_records':sum(bool(v['imported_submissions']) for v in venues),
'scope_note':registry['scope_note'],'interpretation':'All registry targets are listed. Edition source complete means that source file passed import checks; it does not prove full historical or abstract coverage. Papers can appear in multiple venues/years, so do not sum venue paper counts as a global total.',
'venues':venues}
finally:c.close()
totals = connection.execute('''SELECT count(s.id),count(DISTINCT s.paper_id)
FROM submissions s JOIN editions e ON e.id=s.edition_id
JOIN venues v ON v.id=e.venue_id WHERE ''' + scope, params).fetchone()
venues.append({'acronym': acronym, 'field': target['field'],
'imported_submissions': totals[0], 'distinct_papers': totals[1],
'history_status': 'partial_history_pending_independent_audit' if totals[0] else 'no_imported_records',
'discovery_sources': target.get('discovery_sources', []), 'editions': editions})
outside_registry = [dict(row) for row in connection.execute('''SELECT v.id AS venue_id,
v.acronym,count(s.id) AS submissions,count(DISTINCT s.paper_id) AS papers
FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id
GROUP BY v.id,v.acronym ORDER BY v.acronym''') if row[0] not in covered_venue_ids]
return {'report_started_at': started_at, 'checked_at': timestamp(),
'engine': 'postgresql' if pg_service else 'sqlite', 'database': database, 'schema': schema,
'pg_service': pg_service, 'target_venue_count': len(targets),
'venues_with_imported_records': sum(bool(v['imported_submissions']) for v in venues),
'scope_note': registry['scope_note'],
'interpretation': 'All registry targets are listed, including zero imports. '
'Latest status counts use the latest observed decision for each attempt; decision observations retain history. '
'Source record totals are per source and edition, not per list URL; multiple list entries may describe overlapping records. '
'Missing abstract text means NULL or empty string; nonempty text does not prove a complete abstract. '
'An edition source marked complete is not a proof of complete conference history or public rejection coverage. '
'Zero imported rejections does not establish that no rejections are public. '
'Papers can appear in several venues/years; do not sum venue paper counts as a global total.',
'venues': venues, 'imported_venues_outside_registry': outside_registry}
finally:
connection.close()
def main(argv=None, *, prog=None):
parser=argparse.ArgumentParser(prog=prog);parser.add_argument('--db',required=True)
parser.add_argument('--registry',default=str(DEFAULT_REGISTRY));parser.add_argument('--out')
args=parser.parse_args(argv);result=report(args.db,args.registry);text=json.dumps(result,ensure_ascii=False,indent=2)
parser = argparse.ArgumentParser(prog=prog)
database_arguments(parser)
parser.add_argument('--registry', default=str(DEFAULT_REGISTRY))
parser.add_argument('--out')
args = parser.parse_args(argv)
result = report(args.db, args.registry, pg_service=args.pg_service, pg_schema=args.pg_schema)
text = json.dumps(result, ensure_ascii=False, indent=2)
if args.out:
path=Path(args.out);stage=path.with_suffix('.part');stage.write_text(text+'\n');stage.replace(path)
else:print(text)
path = Path(args.out)
stage = path.with_suffix('.part')
stage.write_text(text + '\n')
stage.replace(path)
else:
print(text)
if __name__ == '__main__':
+196
View File
@@ -0,0 +1,196 @@
"""Audit linked submission histories without inferring event chronology."""
import argparse
from collections import Counter
import json
from pathlib import Path
import sqlite3
from paper_atlas.db.backend import database_arguments, open_read_connection
from paper_atlas.db.submissions import timestamp
from paper_atlas.querying import LATEST_DECISION
ATTEMPTS = '''SELECT s.paper_id,s.id AS submission_id,e.venue_id,v.acronym AS venue,e.year,e.track,
count(d.id) AS decision_observations,
MAX(CASE WHEN d.status IN ('rejected','desk_rejected','withdrawn') THEN 1 ELSE 0 END)
AS historical_negative,
MAX(CASE WHEN d.status='accepted' THEN 1 ELSE 0 END) AS historical_acceptance,
COALESCE(MAX(CASE WHEN ''' + LATEST_DECISION + ''' THEN d.status END),'unknown')
AS latest_observed_status
FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id
LEFT JOIN decisions d ON d.submission_id=s.id
GROUP BY s.paper_id,s.id,e.venue_id,v.acronym,e.year,e.track
ORDER BY s.paper_id,e.venue_id,s.id'''
def _paper_counts(attempts):
negative = sum(row['historical_negative'] for row in attempts)
accepted = sum(row['latest_observed_status'] == 'accepted' for row in attempts)
overlap = sum(row['historical_negative'] and row['latest_observed_status'] == 'accepted'
for row in attempts)
venues = {}
for row in attempts:
counts = venues.setdefault(row['venue_id'], [0, 0])
counts[0] += row['historical_negative']
counts[1] += row['latest_observed_status'] == 'accepted'
return {
'papers_with_submissions': 1,
'papers_with_multiple_submissions': int(len(attempts) > 1),
'papers_with_historical_negative': int(negative > 0),
'papers_with_current_known_acceptance': int(accepted > 0),
'papers_with_negative_and_acceptance_in_distinct_attempts': int(negative * accepted > overlap),
'papers_with_negative_and_acceptance_in_distinct_venues':
int(negative * accepted > sum(n * a for n, a in venues.values())),
}
def _example(connection, attempts):
# Keep a negative/accepted pair visible even when a paper has many attempts.
accepted = [row for row in attempts if row['latest_observed_status'] == 'accepted']
first = accepted[0] if accepted else None
other_venue = next((row for row in accepted if row['venue_id'] != first['venue_id']), None)
pair = None
for negative in attempts:
if negative['historical_negative'] and first:
candidate = first if first['venue_id'] != negative['venue_id'] else other_venue
if candidate:
pair = (negative, candidate)
break
if pair is None:
for negative in attempts:
if negative['historical_negative'] and first:
candidate = first if first['submission_id'] != negative['submission_id'] else (
accepted[1] if len(accepted) > 1 else None)
if candidate:
pair = (negative, candidate)
break
selected = {}
for row in list(pair or ()) + attempts:
if len(selected) < 20:
selected.setdefault(row['submission_id'], dict(row))
for row in selected.values():
row['decisions'] = []
for decision in connection.execute('''SELECT id,status,raw_value,evidence_url,
decision_date,observed_at FROM decisions WHERE submission_id=?
ORDER BY observed_at DESC,COALESCE(decision_date,observed_at) DESC,id DESC LIMIT 50''',
(row['submission_id'],)):
entry = dict(decision)
sources = [dict(source) for source in connection.execute('''
SELECT so.name,r.external_id,ds.source_decision_id FROM decision_sources ds
JOIN source_records r ON r.id=ds.source_record_id JOIN sources so ON so.id=r.source_id
WHERE ds.decision_id=? ORDER BY so.name,r.external_id LIMIT 101''', (entry['id'],))]
entry['sources'] = sources[:100]
entry['sources_truncated'] = len(sources) > 100
row['decisions'].append(entry)
row['decisions_truncated'] = row['decision_observations'] > len(row['decisions'])
paper_id = attempts[0]['paper_id']
return {'paper_id': paper_id,
'title': connection.execute('SELECT title FROM papers WHERE id=?', (paper_id,)).fetchone()[0],
'counts': _paper_counts(attempts), 'attempt_count': len(attempts),
'attempts': list(selected.values()), 'attempts_truncated': len(attempts) > len(selected),
'chronology': 'Not inferred from observed_at, edition year, or row order.'}
def report(db=None, *, pg_service=None, pg_schema=None, examples=20):
if not 0 <= examples <= 100:
raise ValueError('Examples must be between 0 and 100')
started_at = timestamp()
connection = open_read_connection(db, pg_service=pg_service, pg_schema=pg_schema)
if not pg_service:
connection.row_factory = sqlite3.Row
try:
if pg_service:
identity = connection.execute('SELECT current_database(),current_schema()').fetchone()
database, schema = identity[0], identity[1]
else:
database, schema = str(Path(db).resolve()), None
names = ['papers_with_submissions', 'papers_with_multiple_submissions',
'papers_with_historical_negative', 'papers_with_current_known_acceptance',
'papers_with_negative_and_acceptance_in_distinct_attempts',
'papers_with_negative_and_acceptance_in_distinct_venues']
totals = Counter({key: 0 for key in names})
totals.update(submissions=0, decision_observations=0,
attempts_with_historical_negative=0,
attempts_with_historical_negative_and_acceptance=0)
latest = Counter()
sample_attempts = []
def consume(rows):
if not rows:
return
counts = _paper_counts(rows)
totals.update(counts)
if counts['papers_with_negative_and_acceptance_in_distinct_attempts'] and len(sample_attempts) < examples:
sample_attempts.append(rows)
current = []
for raw in connection.execute(ATTEMPTS):
row = dict(raw)
if current and row['paper_id'] != current[0]['paper_id']:
consume(current)
current = []
current.append(row)
totals['submissions'] += 1
totals['decision_observations'] += row['decision_observations']
totals['attempts_with_historical_negative'] += row['historical_negative']
totals['attempts_with_historical_negative_and_acceptance'] += int(
row['historical_negative'] and row['historical_acceptance'])
latest[row['latest_observed_status']] += 1
consume(current)
# These are relationship checks, not a substitute for the final entity/FK audit.
integrity = {}
checks = {
'submissions_without_source_links': '''SELECT count(*) FROM submissions s
WHERE NOT EXISTS(SELECT 1 FROM submission_sources ss WHERE ss.submission_id=s.id)''',
'decisions_without_source_links': '''SELECT count(*) FROM decisions d
WHERE NOT EXISTS(SELECT 1 FROM decision_sources ds WHERE ds.decision_id=d.id)''',
'submission_source_paper_mismatches': '''SELECT count(*) FROM submission_sources ss
JOIN submissions s ON s.id=ss.submission_id JOIN source_records r ON r.id=ss.source_record_id
WHERE s.paper_id<>r.paper_id''',
'decision_source_paper_mismatches': '''SELECT count(*) FROM decision_sources ds
JOIN decisions d ON d.id=ds.decision_id JOIN submissions s ON s.id=d.submission_id
JOIN source_records r ON r.id=ds.source_record_id WHERE s.paper_id<>r.paper_id''',
}
for name, query in checks.items():
integrity[name] = connection.execute(query).fetchone()[0]
return {'report_started_at': started_at, 'checked_at': timestamp(),
'engine': 'postgresql' if pg_service else 'sqlite', 'database': database,
'schema': schema, 'pg_service': pg_service,
'status': 'observed_history_audit_not_final_dedup', 'counts': dict(totals),
'latest_attempt_status_counts': dict(sorted(latest.items())),
'evidence_relationship_checks': integrity,
'examples': [_example(connection, rows) for rows in sample_attempts],
'example_limit': examples,
'interpretation': [
'Counts use existing canonical paper links, not independently verified real-world identity.',
'Negative includes rejected, desk_rejected and withdrawn observations; these remain distinct decisions.',
'Known acceptance follows the latest observed decision per attempt, matching query --outcome.',
'Distinct attempts/venues require separate submission/venue IDs; aliases do not create new venues.',
'Coexisting evidence does not prove rejection happened before acceptance. observed_at is collection time.',
'No known acceptance is not proof of never accepted. Source signatures and completeness are not established here.',
'Examples are bounded; counts include all attempts and observations in one read-only snapshot.',
'This does not perform entity merging, final deduplication, or label namespace/type validation.',
]}
finally:
connection.close()
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog)
database_arguments(parser)
parser.add_argument('--examples', type=int, default=20)
parser.add_argument('--out')
args = parser.parse_args(argv)
result = report(args.db, pg_service=args.pg_service, pg_schema=args.pg_schema, examples=args.examples)
text = json.dumps(result, ensure_ascii=False, indent=2)
if args.out:
path = Path(args.out)
stage = path.with_suffix('.part')
stage.write_text(text + '\n')
stage.replace(path)
else:
print(text)
if __name__ == '__main__':
main()
+3 -1
View File
@@ -39,7 +39,9 @@ def audit(db, inventory, out, *, pg_service=None, pg_schema=None):
if json.loads(path.with_suffix('.manifest.json').read_text()) != manifest:
raise ValueError('Collected manifest differs from inventory')
count = 0
for line in path.read_text().splitlines():
# Unicode paragraph/line separators are valid inside JSON strings;
# split physical byte lines, not Unicode text boundaries.
for line in path.read_bytes().splitlines():
row = json.loads(line)
if any(row[key] != manifest[key] for key in ('source', 'venue', 'year', 'track')):
raise ValueError('Record edition differs from inventory')
+3 -1
View File
@@ -6,6 +6,7 @@ from paper_atlas import __version__
COMMANDS = {
('curate','identifiers'): ('paper_atlas.curation.identifiers', {}, '按证据补充稳定标识并生成独立待导入文件'),
('query',): ('paper_atlas.querying', {}, '查询论文、标签和投稿历史'),
('coverage',): ('paper_atlas.audits.coverage', {}, '查看全部候选会议的覆盖情况'),
('download',): ('paper_atlas.io.snapshots', {}, '下载并校验固定版本快照'),
@@ -35,6 +36,7 @@ COMMANDS = {
('audit','arxiv'): ('paper_atlas.audits.arxiv', {}, '核验 arXiv 原始编号、重复行与 ZIP 完整性'),
('audit','arxiv-tags'): ('paper_atlas.audits.arxiv_tags', {}, '逐记录核验 arXiv 类别及出处完整性'),
('audit','public-openreview'): ('paper_atlas.audits.public_openreview', {}, '核对公开负面决定与已有投稿历史'),
('audit','histories'): ('paper_atlas.audits.histories', {}, '只读核对跨会议投稿历史及决定来源关系'),
}
@@ -47,7 +49,7 @@ def main(argv=None):
leaf=commands.add_parser(route[0],help=help_text,add_help=False)
else:
if route[0] not in groups:
group=commands.add_parser(route[0],help={'collect':'采集来源元数据','catalogue':'盘点历史目录','import':'导入数据库','maintain':'维护数据库','audit':'核验数据'}[route[0]])
group=commands.add_parser(route[0],help={'collect':'采集来源元数据','catalogue':'盘点历史目录','import':'导入数据库','maintain':'维护数据库','audit':'核验数据','curate':'按证据整理元数据'}[route[0]])
groups[route[0]]=group.add_subparsers(dest='operation',required=True)
leaf=groups[route[0]].add_parser(route[1],help=help_text,add_help=False)
leaf.set_defaults(handler=(route,module,kwargs))
+2
View File
@@ -90,6 +90,8 @@ def normalize_row(row,volumes,targets,include_workshops=False,through_year=2026)
'ambiguous_identifiers':ambiguous,'publication_evidence':'inproceedings bibliography; not an original decision note'}
if edition.get('track_scope_note'):
metadata['track_scope_note']=edition['track_scope_note']
if edition.get('track_scope_evidence_url'):
metadata['track_scope_evidence_url']=edition['track_scope_evidence_url']
flagged=bool(re.search(r'\b(?:withdrawn|retracted)\b',row.get('publtype') or '',re.I))
result={'source':'dblp','external_id':row['key'],'url':'https://dblp.org/rec/'+row['key']+'.html',
'title':title,'abstract':None,'authors':authors,'identifiers':own,
+39 -1
View File
@@ -38,6 +38,39 @@ BOOKS = {
}
# These are the historical primary series themselves, not co-located workshops.
# Exact proceedings keys, original names, ordinals and event years must agree.
HISTORICAL_WORKSHOP_TITLES = {
**{('aistats', year): prefix + ordinal + ' International Workshop on Artificial Intelligence and Statistics'
for year, ordinal, prefix in [
(1995, 'Fifth', 'Learning from Data - '),
(1997, 'Sixth', 'Proceedings of the '),
(1999, 'Seventh', 'Proceedings of the '),
(2001, 'Eighth', 'Proceedings of the '),
(2003, 'Ninth', 'Proceedings of the '),
(2005, 'Tenth', 'Proceedings of the ')]},
**{('colt', year): 'Proceedings of the ' + ordinal + ' Annual Workshop on Computational Learning Theory'
for year, ordinal in [(1988, 'First'), (1989, 'Second'), (1990, 'Third'), (1991, 'Fourth')]},
}
def historical_track(family, base, suffix, volume_key, year, title):
if suffix or base.casefold() != family or volume_key != f'conf/{family}/{year}':
return None
prefix = HISTORICAL_WORKSHOP_TITLES.get((family, year))
if (prefix and title.casefold().startswith(prefix.casefold() + ',')
and re.search(r'\b' + str(year) + r'\b', title)):
url = 'https://aistats.org/other.html' if family == 'aistats' else 'https://www.learningtheory.org/'
return ('main', 'Historical primary series named Workshop; exact proceedings key, ordinal, title and event year verified.', url)
if (family == 'colt' and year == 2003
and title.startswith('Computational Learning Theory and Kernel Machines, 16th Annual Conference on Computational Learning Theory and 7th Kernel Workshop, COLT/Kernel 2003,')
and title.endswith('Proceedings')):
return ('proceedings_unspecified',
'Combined COLT 2003 and Kernel Workshop volume; per-paper section absent from DBLP XML. Do not infer a main or workshop track.',
'https://doi.org/10.1007/b12006')
return None
def scope(row,volume,through_year=2026):
fields=row['fields'];books=fields.get('booktitle',[])
if len(books)!=1:return None,'ambiguous_booktitle'
@@ -75,7 +108,9 @@ def scope(row,volume,through_year=2026):
if len(bibliographic)!=1 or not re.fullmatch(r'\d{4}',bibliographic[0]):return None,'ambiguous_bibliographic_year'
if int(bibliographic[0])!=year and not re.search(r'\b'+str(year)+r'\b',title):return None,'unverified_year_difference'
text=(suffix+' '+title).casefold();track='main'
if 'findings' in text:track='findings'
historical = historical_track(family, base, suffix, volume['key'], year, title)
if historical:track=historical[0]
elif 'findings' in text:track='findings'
elif 'shared task' in text:track='shared_task'
elif 'student research' in text or 'workshop' in text or 'wokshop' in text or base in ('ICCVW','EWSL'):track='workshop'
elif ('demo track' in text and ('industrial track' in text or 'applied data science track' in text)):track='proceedings_unspecified'
@@ -98,6 +133,9 @@ def scope(row,volume,through_year=2026):
if venue=='AAMAS' and year<2002:return None,'predecessor_not_aamas'
result = {'venue':venue,'venue_aliases':aliases,'year':year,'track':track,'bibliographic_year':int(bibliographic[0]),
'rule':family+':'+base,'proceedings_title':title,'edition_year_basis':year_basis}
if historical:
result['track_scope_note'] = historical[1]
result['track_scope_evidence_url'] = historical[2]
if venue=='MM' and track=='main':
# A main MM volume also includes doctoral, tutorials, companion papers
# and other sections. DBLP XML omits the per-paper TOC section.
+35 -4
View File
@@ -27,11 +27,25 @@ RAW_ROOT = 'https://raw.githubusercontent.com/papercopilot/openreview/' + REVISI
TREE_URL = 'https://api.github.com/repos/papercopilot/openreview/git/trees/' + REVISION + '?recursive=1'
NEGATIVE = {'reject': 'rejected', 'rejected': 'rejected',
'desk reject': 'desk_rejected', 'desk rejected': 'desk_rejected',
'withdrawn': 'withdrawn', 'withdrawal': 'withdrawn'}
'withdraw': 'withdrawn', 'withdrawn': 'withdrawn', 'withdrawal': 'withdrawn'}
TRACKS = {'main': 'main', 'Position': 'position',
'Datasets & Benchmarks': 'datasets_benchmarks'}
WORKSHOP_REJECT_LABELS = {"aim-fm workshop @ neurips'24 reject",
'coginterp @ neurips 2025 reject'}
# Exact curated labels, tied to the year and raw workshop track. The official
# venue directory establishes workshop identity, not an original decision signature.
ICLR_WORKSHOP_LABELS = {
'iclr 2024 workshop mefomo withdrawn submission':
(2024, 'workshop_me_fomo', 'withdrawn', 'ME-FoMo'),
'iclr 2024 workshop genai4dm withdrawn submission':
(2024, 'workshop_genai4dm', 'withdrawn', 'GenAI4DM'),
'iclr 2025 workshop agenticai reject':
(2025, 'workshop_agenticai', 'rejected', 'AgenticAI'),
'iclr 2025 workshop lmrl withdrawn submission':
(2025, 'workshop_lmrl', 'withdrawn', 'LMRL'),
'iclr 2026 workshop llm reasoning withdrawn submission':
(2026, 'workshop_llm_reasoning', 'withdrawn', 'LLM_Reasoning'),
}
def download(url, limit):
@@ -63,6 +77,20 @@ def normalize(row, family, year, provenance, include_workshops=False):
"""Return a normalized assertion or a reason requiring separate review."""
raw_status = row.get('status', '')
status = negative_status(raw_status)
workshop = ICLR_WORKSHOP_LABELS.get(' '.join(raw_status.casefold().split()))
workshop_metadata = {}
if workshop:
expected_year, expected_track, status, group_name = workshop
if family != 'iclr' or year != expected_year:
raise ValueError('Workshop status label conflicts with source conference year')
if row.get('track') != expected_track:
return None, 'conflicting_workshop_track'
if not include_workshops:
return None, 'separate_workshop'
workshop_metadata = {
'workshop_venueid': f'ICLR.cc/{year}/Workshop/{group_name}',
'workshop_identity_evidence_url': 'https://openreview.net/venue?id=ICLR.cc',
}
if status is None:
if re.search(r'\bretract\w*\b', raw_status, re.I):
return None, 'retracted'
@@ -90,7 +118,9 @@ def normalize(row, family, year, provenance, include_workshops=False):
return None, 'unmapped_venue'
title = row.get('title')
abstract = row.get('abstract')
if not isinstance(title, str) or not title.strip():
if title is None or (isinstance(title, str) and not title.strip()):
return None, 'missing_title'
if not isinstance(title, str):
raise ValueError('Public submission lacks a title')
if abstract is not None and not isinstance(abstract, str):
raise ValueError('Abstract must be text or null')
@@ -116,7 +146,8 @@ def normalize(row, family, year, provenance, include_workshops=False):
'venue': venue, 'year': year, 'track': normalized_track,
'status': status, 'raw_decision': raw_status, 'evidence_url': evidence,
'observed_at': AS_OF, 'tags': tags,
'metadata': {**provenance, 'raw_track': track, 'raw_status': raw_status,
'metadata': {**provenance, **workshop_metadata,
'raw_track': track, 'raw_status': raw_status,
'evidence_kind': 'curated_public_status_assertion',
'original_decision_signature_verified': False,
'source_snapshot_upper_bound': AS_OF}}, None
@@ -215,7 +246,7 @@ def _collect_files(out, selected, include_workshops, fetcher):
name = f'public-openreview-{family}-{year}-{track}.jsonl'
path = stage / name
path.write_text(''.join(json.dumps(row, ensure_ascii=False) + '\n' for row in records))
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': records[0]['venue'],
manifest = {'source': SOURCE, 'parser_version': 3, 'venue': records[0]['venue'],
'year': year, 'track': track, 'count': len(records), 'status': 'partial',
'file': str(out / name), 'file_sha256': hashlib.sha256(path.read_bytes()).hexdigest(),
'list_url': REPOSITORY + '/tree/' + REVISION, 'source_revision': REVISION,
+3 -2
View File
@@ -7,7 +7,7 @@ import re
import time
from datetime import date
from urllib.error import HTTPError
from urllib.parse import parse_qs, unquote, urljoin, urlsplit
from urllib.parse import parse_qs, unquote, urldefrag, urljoin, urlsplit
from paper_atlas.io.http import fetch
from paper_atlas.io.html import Document, Node, clean
@@ -147,7 +147,8 @@ def read_index(index,cache):
links=set()
for node in Document(html).root.walk():
if node.tag!='a':continue
link=urljoin(url,node.attrs.get('href',''));day=parse_qs(urlsplit(link).query).get('day',[])
# Fragments select positions within a page, not separate HTTP resources.
link=urldefrag(urljoin(url,node.attrs.get('href','')))[0];day=parse_qs(urlsplit(link).query).get('day',[])
if index_identity(link)!=index_identity(url) or index_identity(url) is None or len(day)!=1:continue
if day[0]!='all':
try:valid=date.fromisoformat(day[0]).year==index['year']
+1
View File
@@ -0,0 +1 @@
"""Offline, evidence-bound preparation of metadata for separate ingestion."""
+212
View File
@@ -0,0 +1,212 @@
"""Add reviewed stable identifiers without rewriting bibliographic source fields.
Hashes establish file integrity, not the truth of an identity assertion. Identity
review and a fresh database preflight remain separate requirements.
"""
import argparse
from datetime import datetime
import hashlib
import json
from pathlib import Path
import re
import tempfile
from urllib.parse import urlsplit
SCOPE = 'input_and_evidence_integrity_only'
MAX_METADATA_BYTES = 16 * 1024 * 1024
def digest(data):
return hashlib.sha256(data).hexdigest()
def object_keys(value, required, optional=()):
if not isinstance(value, dict) or not set(required) <= value.keys() or value.keys() - set(required) - set(optional):
raise ValueError('Missing or unsupported plan fields')
def json_bytes(value):
return (json.dumps(value, ensure_ascii=False, separators=(',', ':'), allow_nan=False) + '\n').encode('utf-8')
def decode(raw):
def unique(pairs):
result = {}
for key, value in pairs:
if key in result:
raise ValueError('Duplicate JSON key: ' + key)
result[key] = value
return result
return json.loads(raw, object_pairs_hook=unique,
parse_constant=lambda value: (_ for _ in ()).throw(ValueError('Non-finite JSON value')))
def metadata(path, expected=None):
with Path(path).open('rb') as stream:
raw = stream.read(MAX_METADATA_BYTES + 1)
if len(raw) > MAX_METADATA_BYTES:
raise ValueError('Metadata exceeds size limit')
if expected is not None:
check_hash(raw, expected)
return raw, decode(raw)
def check_hash(raw, expected):
if not isinstance(expected, str) or not re.fullmatch('[0-9a-f]{64}', expected) or digest(raw) != expected:
raise ValueError('SHA256 mismatch')
def identifier(scheme, value):
if not isinstance(value, str) or any(char.isspace() or ord(char) < 32 or ord(char) == 127 for char in value):
raise ValueError('Invalid identifier')
if scheme == 'arxiv':
pattern = r'(?:\d{2}(?:0[1-9]|1[0-2])\.\d{4,5}|[a-zA-Z][a-zA-Z.-]*/\d{2}(?:0[1-9]|1[0-2])\d{3})'
valid = re.fullmatch(pattern, value)
elif scheme == 'doi':
valid = re.fullmatch(r'10\.\d{4,9}/\S+', value) and value == value.lower()
else:
valid = False
if not valid:
raise ValueError('Unsupported or noncanonical identifier: ' + scheme)
def prepare(plan_path, output):
plan_raw, plan = metadata(plan_path)
object_keys(plan, ('format_version', 'inputs', 'assertions'))
if type(plan['format_version']) is not int or plan['format_version'] != 1:
raise ValueError('Unsupported format version')
if not isinstance(plan['inputs'], list) or not plan['inputs'] or not isinstance(plan['assertions'], list) or not plan['assertions']:
raise ValueError('Inputs and assertions must be nonempty lists')
output = Path(output).absolute()
if output.exists():
raise ValueError('Output directory must not already exist')
if not output.parent.is_dir():
raise ValueError('Output parent directory must exist')
assertions = {}
evidence_files = {}
checked_evidence = {}
for assertion in plan['assertions']:
object_keys(assertion, ('source', 'external_id', 'record_sha256', 'identifiers', 'evidence'))
if any(not isinstance(assertion[key], str) or not assertion[key] for key in ('source', 'external_id')):
raise ValueError('Invalid source identity')
key = (assertion['source'], assertion['external_id'])
if key in assertions:
raise ValueError('Duplicate identity assertion')
ids = assertion['identifiers']
if not isinstance(ids, dict) or not ids:
raise ValueError('Identifiers must be a nonempty object')
for scheme, value in ids.items():
identifier(scheme, value)
if not isinstance(assertion['evidence'], list) or not assertion['evidence']:
raise ValueError('Evidence is required')
references = []
for evidence in assertion['evidence']:
object_keys(evidence, ('file', 'sha256', 'url', 'checked_at', 'note'))
if not all(isinstance(value, str) and value for value in evidence.values()):
raise ValueError('Invalid evidence fields')
url = urlsplit(evidence['url'])
if url.scheme != 'https' or not url.hostname or url.username or url.password:
raise ValueError('Evidence requires an HTTPS URL without credentials')
if datetime.fromisoformat(evidence['checked_at'].replace('Z', '+00:00')).utcoffset() is None:
raise ValueError('Evidence timestamp requires a timezone')
evidence_key = (str(Path(evidence['file']).absolute()), evidence['sha256'])
if evidence_key not in checked_evidence:
checked_evidence[evidence_key] = metadata(evidence['file'], evidence['sha256'])[0]
raw = checked_evidence[evidence_key]
evidence_files[evidence['sha256']] = raw
references.append({**evidence, 'file': 'evidence/' + evidence['sha256'] + '.json'})
assertions[key] = (assertion, references)
seen = set()
applied = set()
names = set()
report = {'format_version': 1, 'plan_sha256': digest(plan_raw),
'verification_scope': SCOPE, 'semantic_identity_verified': False,
'database_written': False, 'files': [], 'assertions_applied': 0, 'identifiers_added': 0}
with tempfile.TemporaryDirectory(prefix='.identifier-curation-', dir=output.parent) as temporary:
staging = Path(temporary) / 'prepared'
staging.mkdir()
(staging / 'evidence').mkdir()
(staging / 'plan.json').write_bytes(plan_raw)
for sha, raw in evidence_files.items():
(staging / 'evidence' / (sha + '.json')).write_bytes(raw)
for entry in plan['inputs']:
object_keys(entry, ('file', 'sha256', 'manifest_sha256'))
path = Path(entry['file'])
if path.suffix != '.jsonl' or path.name in names:
raise ValueError('Inputs require distinct JSONL basenames')
names.add(path.name)
_, manifest = metadata(path.with_suffix('.manifest.json'), entry['manifest_sha256'])
if not isinstance(manifest, dict) or not {'source','venue','year','track','count','list_url','file_sha256'} <= manifest.keys():
raise ValueError('Complete source manifest required')
if manifest['file_sha256'] != entry['sha256']:
raise ValueError('Manifest and plan hashes differ')
before = hashlib.sha256()
after = hashlib.sha256()
count = 0
with path.open('rb') as source, (staging / path.name).open('wb') as target:
for raw in source:
before.update(raw)
row = decode(raw)
if any(row[field] != manifest[field] for field in ('source', 'venue', 'year', 'track')):
raise ValueError('Record differs from source manifest')
key = (row['source'], row['external_id'])
if key in seen:
raise ValueError('Duplicate source identity in input batch')
seen.add(key)
if key in assertions:
assertion, references = assertions[key]
check_hash(raw, assertion['record_sha256'])
existing = row.get('identifiers', {})
if existing is None:
existing = {}
if not isinstance(existing, dict):
raise ValueError('Invalid existing identifiers')
for scheme, value in assertion['identifiers'].items():
if scheme in existing and existing[scheme] != value:
raise ValueError('Existing identifier conflict')
details = row.setdefault('metadata', {})
if not isinstance(details, dict):
raise ValueError('Invalid record metadata')
history = details.setdefault('identifier_curation', [])
if not isinstance(history, list):
raise ValueError('Invalid curation history')
history.append({'plan_sha256': digest(plan_raw), 'input_file_sha256': entry['sha256'],
'record_sha256': digest(raw), 'previous_identifiers': dict(existing),
'identifiers_asserted': assertion['identifiers'], 'evidence': references,
'verification_scope': SCOPE})
report['identifiers_added'] += len(assertion['identifiers'].keys() - existing.keys())
row['identifiers'] = {**existing, **assertion['identifiers']}
raw = json_bytes(row)
applied.add(key)
target.write(raw)
after.update(raw)
count += 1
if before.hexdigest() != entry['sha256'] or not re.fullmatch('[0-9a-f]{64}', entry['sha256']):
raise ValueError('Input SHA256 mismatch')
if not count or type(manifest['count']) is not int or count != manifest['count']:
raise ValueError('Manifest count mismatch')
provenance = {'plan_sha256': digest(plan_raw), 'input_file_sha256': before.hexdigest(),
'input_manifest_sha256': entry['manifest_sha256'], 'verification_scope': SCOPE}
prepared = {**manifest, 'file': str(output / path.name), 'file_sha256': after.hexdigest(),
'identifier_curation': provenance}
(staging / path.with_suffix('.manifest.json').name).write_bytes(json_bytes(prepared))
report['files'].append({'file': str(output / path.name), 'count': count,
'input_sha256': before.hexdigest(), 'sha256': after.hexdigest()})
if applied != assertions.keys():
raise ValueError('Assertion target absent from input batch')
report['assertions_applied'] = len(applied)
(staging / 'curation-report.json').write_bytes(json_bytes(report))
# All files are on the same filesystem. No database or input mutation.
if output.exists():
raise ValueError('Output directory appeared during preparation')
staging.rename(output)
return report
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
parser.add_argument('--plan', required=True, type=Path)
parser.add_argument('--output', required=True, type=Path)
args = parser.parse_args(argv)
print(json.dumps(prepare(args.plan, args.output), ensure_ascii=False))
+13 -7
View File
@@ -95,13 +95,19 @@ class Store:
matches.add(old[0])
key = normalized(title)
title_matches = c.execute('SELECT id,authors_json,abstract FROM papers WHERE title_key=?',(key,)).fetchall()
authors_set = author_keys(authors)
for paper,old_authors,old_abstract in title_matches:
overlap = authors_set & author_keys(json.loads(old_authors))
# Cross-platform title matching needs independent author corroboration.
same_source = c.execute('SELECT 1 FROM source_records WHERE paper_id=? AND source_id=? LIMIT 1',(paper,source_id)).fetchone()
if not same_source and len(key)>=25 and (len(overlap)>=2 or (overlap and abstract and abstract.strip()==(old_abstract or '').strip())):
matches.add(paper)
# An existing identifier/source record anchors the entity. Title
# similarity must not pull in another deposit or an extended version.
if not matches:
title_only_matches = set()
authors_set = author_keys(authors)
for paper,old_authors,old_abstract in title_matches:
overlap = authors_set & author_keys(json.loads(old_authors))
same_source = c.execute('SELECT 1 FROM source_records WHERE paper_id=? AND source_id=? LIMIT 1',(paper,source_id)).fetchone()
if not same_source and len(key)>=25 and (len(overlap)>=2 or (overlap and abstract and abstract.strip()==(old_abstract or '').strip())):
title_only_matches.add(paper)
if len(title_only_matches)>1:
raise ValueError('Ambiguous paper identity: multiple author-corroborated title matches; provide verified identifiers')
matches = title_only_matches
if matches:
paper = min(matches)
for other in sorted(matches-{paper}):
+71 -2
View File
@@ -12,6 +12,55 @@ from paper_atlas.collectors.public_openreview import SOURCE as PUBLIC_SOURCE
from paper_atlas.collectors.icml_public_archive import SOURCE as ARCHIVE_SOURCE
def coverage_options(manifest):
mode = manifest.get('coverage_mode', 'snapshot')
if mode not in ('snapshot', 'incremental'):
raise ValueError('Unsupported coverage mode')
if mode == 'snapshot':
return None
required = ('venue', 'year', 'track', 'count', 'coverage_previous_count',
'coverage_previous_expected_count', 'coverage_expected_count')
if any(key not in manifest for key in required) or manifest.get('status') != 'partial':
raise ValueError('Incremental coverage requires a partial, explicitly scoped manifest')
previous = manifest['coverage_previous_count']
expected = manifest['coverage_expected_count']
prior_expected = manifest['coverage_previous_expected_count']
if (type(previous) is not int or previous < 0 or type(expected) is not int or expected < 1
or type(manifest['count']) is not int or manifest['count'] < 1
or expected < max(previous, manifest['count'])
or (prior_expected is not None and (type(prior_expected) is not int or prior_expected < 0
or expected < prior_expected))):
raise ValueError('Invalid incremental coverage counts')
return previous, prior_expected, expected
def source_record_count(c, source, edition):
return c.execute('''SELECT count(DISTINCT ss.source_record_id) FROM submission_sources ss
JOIN submissions s ON s.id=ss.submission_id
JOIN source_records sr ON sr.id=ss.source_record_id
WHERE s.edition_id=? AND sr.source_id=?''', (edition, source)).fetchone()[0]
def require_incremental_parent(c, source, manifest, options):
edition = c.execute('''SELECT e.id FROM editions e JOIN venues v ON v.id=e.venue_id
WHERE lower(v.acronym)=lower(?) AND e.year=? AND e.track=?''',
(manifest['venue'], manifest['year'], manifest['track'])).fetchone()
entries = c.execute('''SELECT list_url,status,expected_count,imported_count
FROM edition_sources WHERE edition_id=? AND source_id=?''',
(edition[0], source)).fetchall() if edition else []
previous, prior_expected, _ = options
if entries:
if len(entries) != 1 or entries[0][0] != manifest['list_url']:
raise ValueError('Incremental coverage requires one unambiguous source listing')
if tuple(entries[0][1:]) != ('partial', prior_expected, previous):
raise ValueError('Incremental coverage parent changed or is not partial')
elif previous != 0 or prior_expected is not None:
raise ValueError('Incremental coverage parent is absent')
actual = source_record_count(c, source, edition[0]) if edition else 0
if actual != previous:
raise ValueError('Incremental coverage parent differs from actual source records')
def ingest_file(db,path, *, pg_service=None, pg_schema=None):
path = Path(path)
manifest_path = path.with_suffix('.manifest.json')
@@ -24,6 +73,7 @@ def ingest_file(db,path, *, pg_service=None, pg_schema=None):
source_name = manifest.get('source')
if not source_name:
raise ValueError('A source manifest is required')
incremental = coverage_options(manifest)
store = open_store(db, pg_service=pg_service, pg_schema=pg_schema)
c = store.conn
run = None
@@ -35,6 +85,8 @@ def ingest_file(db,path, *, pg_service=None, pg_schema=None):
source_id = store.source(source_name,manifest['list_url'])
run = inserted_id(c,'INSERT INTO ingest_runs(source_id,checkpoint_json) VALUES(?,?)',(source_id,json.dumps({'file':str(path),'expected_sha256':manifest.get('file_sha256')})))
c.commit()
if incremental is not None:
require_incremental_parent(c, source_id, manifest, incremental)
digest = hashlib.sha256()
with path.open('rb') as stream:
for line in stream:
@@ -82,8 +134,25 @@ def ingest_file(db,path, *, pg_service=None, pg_schema=None):
if expected is not None and expected != count:
raise ValueError(f'Manifest count {expected} differs from records {count}')
complete=expected==count and manifest.get('status') not in ('partial','blocked') and not manifest.get('missing_abstract_count',0) and not missing_abstracts
c.execute('INSERT INTO edition_sources(edition_id,source_id,list_url,status,expected_count,imported_count,last_success_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(edition_id,source_id,list_url) DO UPDATE SET status=excluded.status,expected_count=excluded.expected_count,imported_count=excluded.imported_count,last_success_at=excluded.last_success_at,last_error=NULL',(edition,source_id,manifest['list_url'],'complete' if complete else 'partial',expected,count,time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())))
c.execute("UPDATE ingest_runs SET status='complete',finished_at=?,records_seen=?,records_written=?,checkpoint_json=? WHERE id=?",(timestamp(),count,count,json.dumps({'file':str(path),'file_sha256':digest.hexdigest()}),run))
imported = count
condition = ''
condition_params = ()
if incremental is not None:
previous, prior_expected, expected = incremental
imported = source_record_count(c, source_id, edition)
if imported < previous or imported > expected:
raise ValueError('Incremental coverage exceeds the declared source bounds')
complete = False
condition = " WHERE edition_sources.imported_count=? AND COALESCE(edition_sources.expected_count,-1)=? AND edition_sources.status='partial'"
condition_params = (previous, -1 if prior_expected is None else prior_expected)
result = c.execute('INSERT INTO edition_sources(edition_id,source_id,list_url,status,expected_count,imported_count,last_success_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(edition_id,source_id,list_url) DO UPDATE SET status=excluded.status,expected_count=excluded.expected_count,imported_count=excluded.imported_count,last_success_at=excluded.last_success_at,last_error=NULL'+condition,(edition,source_id,manifest['list_url'],'complete' if complete else 'partial',expected,imported,time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()),*condition_params))
if incremental is not None and result.rowcount != 1:
raise ValueError('Incremental coverage parent changed during import')
checkpoint = {'file':str(path),'file_sha256':digest.hexdigest()}
if incremental is not None:
checkpoint['coverage'] = {'mode':'incremental','previous_count':previous,
'expected_count':expected,'imported_count':imported}
c.execute("UPDATE ingest_runs SET status='complete',finished_at=?,records_seen=?,records_written=?,checkpoint_json=? WHERE id=?",(timestamp(),count,count,json.dumps(checkpoint),run))
c.commit()
print(json.dumps({'file':str(path),'status':'imported','records':count,'missing_abstract_count':missing_abstracts}),flush=True)
except BaseException as exc:
+69
View File
@@ -7,6 +7,34 @@ from paper_atlas.audits.coverage import report
from paper_atlas.db.store import Store
def add_coverage_evidence(store):
paper, record = store.record('official_metadata', 'known', 'https://example.org/known',
'Known paper with two source observations', 'Complete canonical abstract', ['Jane Doe', 'John Smith'],
{'openreview': 'coverageForum', 'arxiv': '2401.98765'})
edition = store.decision(paper, record, 'COLING+LREC', 2024, 'main', 'known',
'https://example.org/accept', 'accepted', 'Accept', aliases=['COLING', 'LREC'],
identities=[('openreview', 'coverageForum')], observed_at='2025-05-01T00:00:00Z')
same, mirror = store.record('bibliography', 'mirror', 'https://example.org/mirror',
'Known paper with two source observations', None, ['Jane Doe', 'John Smith'], {'openreview': 'coverageForum'})
store.decision(same, mirror, 'COLING+LREC', 2024, 'main', 'mirror', 'https://example.org/older',
'rejected', 'Reject', identities=[('openreview', 'coverageForum')],
observed_at='2025-05-01T00:00:00Z', decision_date='2025-04-30T00:00:00Z')
unknown, unknown_record = store.record('unknown_catalogue', 'unknown', 'https://example.org/unknown',
'Indexed work without a public decision', None, [])
store.decision(unknown, unknown_record, 'COLING+LREC', 2024, 'main', 'unknown',
'https://example.org/unknown', None, '', identities=[('openreview', 'unknownForum')])
same, later = store.record('official_metadata', 'later', 'https://example.org/later',
'Known paper with two source observations', 'Complete canonical abstract', ['Jane Doe', 'John Smith'],
{'openreview': 'coverageNext', 'arxiv': '2401.98765'})
store.decision(same, later, 'COLING+LREC', 2025, 'findings', 'later', 'https://example.org/later',
'accepted', 'Accept', identities=[('openreview', 'coverageNext')], observed_at='2025-06-01T00:00:00Z')
source = store.source('official_metadata', 'https://example.org')
for url, status in [('https://example.org/list', 'complete'), ('https://example.org/identity-refresh', 'partial')]:
store.conn.execute('''INSERT INTO edition_sources(edition_id,source_id,list_url,status,expected_count,imported_count)
VALUES(?,?,?,?,?,?)''', (edition, source, url, status, 1, 1))
store.conn.commit()
class CoverageTests(unittest.TestCase):
def setUp(self):
self.tmp=tempfile.TemporaryDirectory();self.addCleanup(self.tmp.cleanup)
@@ -31,5 +59,46 @@ class CoverageTests(unittest.TestCase):
self.store.conn.commit();coling=report(self.db,self.registry)['venues'][0]
self.assertEqual(coling['imported_submissions'],2);self.assertEqual(coling['distinct_papers'],1)
def test_observations_attempts_and_missing_source_abstracts_stay_separate(self):
add_coverage_evidence(self.store)
result = report(self.db, self.registry)
self.assertEqual(result['engine'], 'sqlite')
self.assertEqual(result['database'], str(self.db))
coling = result['venues'][0]
self.assertEqual((coling['imported_submissions'], coling['distinct_papers']), (3, 2))
edition, findings = coling['editions']
self.assertEqual((edition['submissions'], edition['papers'], edition['papers_without_abstract_text']), (2, 2, 1))
self.assertEqual(edition['latest_observed_status_counts']['accepted'], 1)
self.assertEqual(edition['latest_observed_status_counts']['rejected'], 0)
self.assertEqual(edition['latest_observed_status_counts']['unknown'], 1)
self.assertEqual(edition['decision_observation_counts']['accepted'], 1)
self.assertEqual(edition['decision_observation_counts']['rejected'], 1)
self.assertEqual(edition['decision_observation_counts']['unknown'], 0)
counts = {row['source']: row for row in edition['source_record_counts']}
self.assertEqual(counts['bibliography']['source_records_without_abstract_text'], 1)
self.assertEqual(counts['official_metadata']['source_records'], 1)
self.assertEqual(len(edition['sources']), 2)
self.assertEqual(findings['track'], 'findings')
self.assertEqual(result['imported_venues_outside_registry'], [])
def test_missing_sqlite_audit_does_not_create_a_database(self):
path = self.root / 'absent.sqlite'
with self.assertRaisesRegex(ValueError, 'Existing database'):
report(path, self.registry)
self.assertFalse(path.exists())
def test_empty_editions_remain_visible_after_shared_aggregation(self):
self.store.conn.execute("INSERT INTO venues(acronym,name,field,venue_type) VALUES('AAAI','AAAI','general_ai','conference')")
venue = self.store.conn.execute("SELECT id FROM venues WHERE acronym='AAAI'").fetchone()[0]
self.store.conn.execute("INSERT INTO editions(venue_id,year,track) VALUES(?,2026,'main')", (venue,))
self.store.conn.commit()
aaai = report(self.db, self.registry)['venues'][1]
self.assertEqual(aaai['history_status'], 'no_imported_records')
self.assertEqual(len(aaai['editions']), 1)
edition = aaai['editions'][0]
self.assertEqual((edition['submissions'], edition['papers'], edition['papers_without_abstract_text']), (0, 0, 0))
self.assertEqual(sum(edition['latest_observed_status_counts'].values()), 0)
self.assertEqual(edition['source_record_counts'], [])
if __name__=='__main__':unittest.main()
+64 -1
View File
@@ -10,7 +10,7 @@ from unittest.mock import patch
from paper_atlas.collectors.dblp import XML_SHA
from paper_atlas.collectors.dblp_records import collect,identifiers,normalize_row,material_reason
from paper_atlas.collectors.dblp_scope import scope
from paper_atlas.collectors.dblp_scope import scope, HISTORICAL_WORKSHOP_TITLES
from paper_atlas.db.store import Store
from paper_atlas.ingestion.records import ingest_file
@@ -70,6 +70,69 @@ class DblpRecordTests(unittest.TestCase):
self.assertEqual(scope(self.row(),self.volume('kdd',2024))[1],'crossref_venue_conflict')
self.assertEqual(scope(self.row(year=2027),self.volume('aaai',2027))[1],'future_edition')
def test_historical_primary_series_workshop_names_require_exact_editions(self):
self.assertEqual(set(HISTORICAL_WORKSHOP_TITLES),
{('aistats',year) for year in [1995,1997,1999,2001,2003,2005]} |
{('colt',year) for year in [1988,1989,1990,1991]})
for (family,year),prefix in HISTORICAL_WORKSHOP_TITLES.items():
with self.subTest(family=family,year=year):
title=prefix+f', {family.upper()} {year}, Proceedings.'
volume=self.volume(family,year,title)
row=self.row(family,family.upper(),year)
if year==1995:
row['fields']['year']=['1996'];volume['fields']['year']=['1996']
edition,reason=scope(row,volume)
self.assertIsNone(reason);self.assertEqual(edition['track'],'main')
self.assertEqual(edition['year'],year)
self.assertIn('Historical primary series',edition['track_scope_note'])
record,reason=normalize_row(row,{volume['key']:volume},{family.upper()})
self.assertIsNone(reason);self.assertEqual(record['metadata']['track_scope_evidence_url'],edition['track_scope_evidence_url'])
self.assertEqual(record['metadata']['proceedings_title'],title)
if year==1995:self.assertEqual(record['metadata']['bibliographic_year'],1996)
def test_historical_exception_does_not_promote_affiliated_or_wrong_scope_workshops(self):
title=HISTORICAL_WORKSHOP_TITLES[('aistats',1995)]+', AISTATS 1995, Proceedings.'
for book,key,changed in [
('AISTATS','conf/aistats/1995w',title),
('AISTATS (Workshops)','conf/aistats/1995',title),
('AISTATS','conf/aistats/1995',title.replace('Fifth','Sixth')),
('AISTATS','conf/aistats/1995',title.replace('1995','1997'))]:
volume=self.volume('aistats',1995,changed);volume['key']=key
row=self.row('aistats',book,1995);row['fields']['crossref']=[key]
self.assertEqual(scope(row,volume)[0]['track'],'workshop')
volume=self.volume('aistats',2010,'Active Learning and Experimental Design workshop, In conjunction with AISTATS 2010')
volume['key']='conf/aistats/2010w'
row=self.row('aistats','AISTATS',2011);row['fields']['crossref']=[volume['key']]
self.assertEqual(scope(row,volume)[0]['track'],'workshop')
def test_combined_colt_kernel_volume_keeps_unknown_per_paper_track(self):
title='Computational Learning Theory and Kernel Machines, 16th Annual Conference on Computational Learning Theory and 7th Kernel Workshop, COLT/Kernel 2003, Washington, DC, USA, August 24-27, 2003, Proceedings'
volume=self.volume('colt',2003,title);row=self.row('colt','COLT',2003)
record,reason=normalize_row(row,{volume['key']:volume},{'COLT'})
self.assertIsNone(reason);self.assertEqual(record['track'],'proceedings_unspecified')
self.assertEqual(record['status'],'accepted')
self.assertIn('per-paper section absent',record['metadata']['track_scope_note'])
self.assertEqual(record['metadata']['track_scope_evidence_url'],'https://doi.org/10.1007/b12006')
volume['key']='conf/colt/2003w';row['fields']['crossref']=[volume['key']]
self.assertEqual(scope(row,volume)[0]['track'],'workshop')
def test_historical_scope_provenance_survives_file_ingestion(self):
title=HISTORICAL_WORKSHOP_TITLES[('colt',1988)]+', COLT 1988, Proceedings.'
row=self.row('colt','COLT',1988);volume=self.volume('colt',1988,title)
record,reason=normalize_row(row,{volume['key']:volume},{'COLT'})
self.assertIsNone(reason)
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);path=root/'history.jsonl'
path.write_text(json.dumps(record)+'\n')
path.with_suffix('.manifest.json').write_text(json.dumps({'source':'dblp','count':1,'list_url':'https://doi.org/10.4230/dblp.xml.2026-11-01','file_sha256':hashlib.sha256(path.read_bytes()).hexdigest()}))
ingest_file(root/'sample.sqlite',path)
with closing(sqlite3.connect(root/'sample.sqlite')) as connection:
track=connection.execute('SELECT track FROM editions').fetchone()[0]
metadata=json.loads(connection.execute('SELECT metadata_json FROM source_records').fetchone()[0])
self.assertEqual(track,'main')
self.assertEqual(metadata['track_scope_evidence_url'],'https://www.learningtheory.org/')
self.assertEqual(metadata['proceedings_title'],title)
def test_only_own_identifier_links_are_used_and_ambiguity_is_preserved(self):
own,ambiguous=identifiers({'ee':['https://doi.org/10.1234/published','https://doi.org/10.48550/arXiv.2401.00001','https://openreview.net/forum?id=realForum'],
'cite':['https://doi.org/10.9999/reference']})
+114
View File
@@ -0,0 +1,114 @@
from pathlib import Path
import tempfile
import unittest
from paper_atlas.audits.histories import report
from paper_atlas.db.store import Store
def add_history_fixture(store):
def event(work, forum, venue, year, track, status, observed):
paper, record = store.record('history_fixture', work + '/' + forum,
'https://example.org/' + forum, work + ' title\x00 preserved', 'Abstract', ['Jane Doe'],
{'arxiv': work})
store.decision(paper, record, venue, year, track, forum,
'https://example.org/' + forum + '/' + str(status), status, status or '',
identities=[('openreview', forum)], observed_at=observed)
return paper
# The negative observation was collected later: observation order is not chronology.
event('2501.00001', 'HistoryA01', 'ICLR', 2024, 'main', 'rejected', '2026-02-01T00:00:00Z')
event('2501.00001', 'HistoryA02', 'ICML', 2025, 'main', 'accepted', '2026-01-01T00:00:00Z')
event('2501.00002', 'HistoryB01', 'ICLR', 2024, 'main', 'rejected', '2026-01-01T00:00:00Z')
event('2501.00002', 'HistoryB01', 'ICLR', 2024, 'main', 'accepted', '2026-02-01T00:00:00Z')
event('2501.00003', 'HistoryC01', 'ICLR', 2024, 'main', 'desk_rejected', '2026-01-01T00:00:00Z')
event('2501.00003', 'HistoryC02', 'ICLR', 2024, 'workshop_example', 'accepted', '2026-02-01T00:00:00Z')
event('2501.00004', 'HistoryD01', 'ICML', 2025, 'main', 'accepted', '2026-01-01T00:00:00Z')
event('2501.00004', 'HistoryD01', 'ICML', 2025, 'main', 'withdrawn', '2026-02-01T00:00:00Z')
event('2501.00004', 'HistoryD02', 'ICLR', 2024, 'main', 'rejected', '2026-01-01T00:00:00Z')
event('2501.00005', 'HistoryE01', 'COLM', 2026, 'main', None, '2026-01-01T00:00:00Z')
store.conn.commit()
class HistoryTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.db = Path(self.tmp.name) / 'sample.sqlite'
self.store = Store(self.db)
self.addCleanup(self.store.conn.close)
def test_distinct_attempts_venues_and_latest_acceptances_are_not_observation_counts(self):
add_history_fixture(self.store)
result = report(self.db)
self.assertEqual(result['counts'], {
'papers_with_submissions': 5, 'papers_with_multiple_submissions': 3,
'papers_with_historical_negative': 4, 'papers_with_current_known_acceptance': 3,
'papers_with_negative_and_acceptance_in_distinct_attempts': 2,
'papers_with_negative_and_acceptance_in_distinct_venues': 1,
'submissions': 8, 'decision_observations': 9,
'attempts_with_historical_negative': 5,
'attempts_with_historical_negative_and_acceptance': 2,
})
self.assertEqual(result['latest_attempt_status_counts'],
{'accepted': 3, 'desk_rejected': 1, 'rejected': 2, 'unknown': 1, 'withdrawn': 1})
self.assertEqual(len(result['examples']), 2)
first = result['examples'][0]
self.assertIn('\x00', first['title'])
self.assertEqual({r['venue'] for r in first['attempts']}, {'ICLR', 'ICML'})
self.assertTrue(all(r['decisions'][0]['sources'] for r in first['attempts']))
self.assertTrue(all(r['decisions'][0]['decision_date'] is None for r in first['attempts']))
self.assertIn('Not inferred', first['chronology'])
self.assertEqual(result['status'], 'observed_history_audit_not_final_dedup')
self.assertTrue(all(v == 0 for v in result['evidence_relationship_checks'].values()))
def test_missing_and_mislinked_source_evidence_is_reported_without_repair(self):
add_history_fixture(self.store)
self.store.conn.execute('DELETE FROM decision_sources WHERE decision_id=1')
self.store.conn.execute('DELETE FROM submission_sources WHERE submission_id=1')
# Record3 belongs to paper2; move it to paper1 to create a real mismatch.
self.store.conn.execute('UPDATE source_records SET paper_id=1 WHERE id=3')
self.store.conn.commit()
result = report(self.db, examples=0)
checks = result['evidence_relationship_checks']
self.assertEqual(checks['decisions_without_source_links'], 1)
self.assertEqual(checks['submissions_without_source_links'], 1)
self.assertEqual(checks['submission_source_paper_mismatches'], 1)
self.assertEqual(checks['decision_source_paper_mismatches'], 2)
self.assertEqual(self.store.conn.execute('SELECT paper_id FROM source_records WHERE id=3').fetchone()[0], 1)
def test_example_limit_does_not_change_global_counts(self):
add_history_fixture(self.store)
none, one = report(self.db, examples=0), report(self.db, examples=1)
self.assertEqual(none['counts'], one['counts'])
self.assertEqual(none['examples'], [])
self.assertEqual(len(one['examples']), 1)
def test_empty_snapshot_and_invalid_limit_do_not_create_missing_database(self):
empty = report(self.db)
self.assertTrue(all(v == 0 for v in empty['counts'].values()))
self.assertEqual(empty['latest_attempt_status_counts'], {})
absent = self.db.parent / 'absent.sqlite'
with self.assertRaisesRegex(ValueError, 'Examples'):
report(absent, examples=-1)
with self.assertRaisesRegex(ValueError, 'Existing database'):
report(absent)
self.assertFalse(absent.exists())
def test_truncated_example_keeps_the_cross_venue_acceptance_visible(self):
paper, record = self.store.record('fixture', 'many', 'https://example.org/many',
'Many attempts', 'Abstract', [])
for index in range(26):
accepted = index == 25
forum = f'BoundedForum{index}'
self.store.decision(paper, record, 'ICML' if accepted else 'ICLR', 2025, 'main', forum,
'https://example.org/' + forum, 'accepted' if accepted else 'rejected',
'Accept' if accepted else 'Reject', identities=[('openreview', forum)])
self.store.conn.commit()
result = report(self.db)
example, = result['examples']
self.assertEqual((example['attempt_count'], len(example['attempts'])), (26, 20))
self.assertTrue(example['attempts_truncated'])
self.assertEqual({row['latest_observed_status'] for row in example['attempts']},
{'accepted', 'rejected'})
self.assertEqual(result['counts']['submissions'], 26)
+185
View File
@@ -0,0 +1,185 @@
import copy
import json
from pathlib import Path
import tempfile
import unittest
from paper_atlas.curation.identifiers import SCOPE, digest, json_bytes, prepare
from paper_atlas.db.store import Store
from paper_atlas.ingestion.records import ingest_file
class IdentifierCurationTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.root = Path(self.tmp.name)
self.output = self.root / 'prepared'
self.row = {'source':'dblp','external_id':'conf/uai/example','url':'https://example.org/paper',
'title':'A sufficiently long example research paper title','abstract':None,
'authors':['Jane Doe','John Smith'],'venue':'UAI','year':2001,'track':'main',
'status':'accepted','tags':[{'namespace':'dblp:topic','value':'version\u2029study\x00'}],
'metadata':{'pages':'203-210','raw_title':'Original\u2029title'}}
self.evidence = self.root / 'evidence.json'
self.evidence.write_bytes(json_bytes({'url':'https://example.org/archive','comment':'Conference version'}))
self.path = self.root / 'records.jsonl'
self.raw = json_bytes(self.row)
self.path.write_bytes(self.raw)
self.manifest = {'source':'dblp','venue':'UAI','year':2001,'track':'main','count':1,
'list_url':'https://example.org/list','file_sha256':digest(self.raw),
'status':'partial','missing_abstract_count':1,'parser_version':2}
self.path.with_suffix('.manifest.json').write_bytes(json_bytes(self.manifest))
self.plan = {'format_version':1,'inputs':[self.input(self.path)],'assertions':[
{'source':'dblp','external_id':self.row['external_id'],'record_sha256':digest(self.raw),
'identifiers':{'arxiv':'1408.1485'},'evidence':[{
'file':str(self.evidence),'sha256':digest(self.evidence.read_bytes()),
'url':'https://example.org/archive','checked_at':'2026-11-08T08:00:00Z',
'note':'Reviewed title, authors, publication and version evidence'}]}]}
def input(self, path):
return {'file':str(path),'sha256':digest(path.read_bytes()),
'manifest_sha256':digest(path.with_suffix('.manifest.json').read_bytes())}
def run_plan(self, plan=None):
path = self.root / 'plan.json'
path.write_bytes(json_bytes(self.plan if plan is None else plan))
return prepare(path, self.output)
def test_only_identifiers_and_provenance_change_and_raw_inputs_survive(self):
result = self.run_plan()
self.assertEqual(self.path.read_bytes(), self.raw)
prepared = json.loads((self.output / self.path.name).read_bytes())
provenance = prepared['metadata'].pop('identifier_curation')
self.assertEqual(prepared.pop('identifiers'), {'arxiv':'1408.1485'})
self.assertEqual(prepared, self.row)
self.assertEqual(provenance[0]['record_sha256'], digest(self.raw))
self.assertEqual((self.output / provenance[0]['evidence'][0]['file']).read_bytes(), self.evidence.read_bytes())
manifest = json.loads((self.output / 'records.manifest.json').read_bytes())
for key, value in self.manifest.items():
if key != 'file_sha256': self.assertEqual(manifest[key], value)
self.assertEqual(manifest['file_sha256'], digest((self.output / self.path.name).read_bytes()))
self.assertEqual(result['assertions_applied'], 1)
self.assertEqual(result['identifiers_added'], 1)
self.assertEqual(result['verification_scope'], SCOPE)
self.assertFalse(result['semantic_identity_verified'])
self.assertFalse(result['database_written'])
self.assertEqual(list(self.output.glob('*.jsonl')), [self.output / self.path.name])
with self.assertRaisesRegex(ValueError, 'already exist'): self.run_plan()
def test_late_invalid_file_publishes_nothing(self):
second = self.root / 'second.jsonl'
second.write_bytes(json_bytes({**self.row,'external_id':'second'}))
manifest = {**self.manifest,'file_sha256':digest(second.read_bytes()),'count':2}
second.with_suffix('.manifest.json').write_bytes(json_bytes(manifest))
self.plan['inputs'].append(self.input(second))
with self.assertRaisesRegex(ValueError, 'count mismatch'): self.run_plan()
self.assertFalse(self.output.exists())
self.assertFalse(list(self.root.glob('.identifier-curation-*')))
self.assertEqual(self.path.read_bytes(), self.raw)
def test_unasserted_rows_keep_exact_physical_bytes_and_doi_is_additive(self):
untouched = (json.dumps({**self.row,'external_id':'unselected'},ensure_ascii=False,indent=None) + ' ').encode()
self.path.write_bytes(self.raw + untouched)
self.path.with_suffix('.manifest.json').write_bytes(json_bytes({**self.manifest,
'file_sha256':digest(self.path.read_bytes()),'count':2,'missing_abstract_count':2}))
self.plan['inputs'] = [self.input(self.path)]
self.plan['assertions'][0]['identifiers']['doi'] = '10.1234/example'
result = self.run_plan()
with (self.output / self.path.name).open('rb') as stream:
first = json.loads(next(stream))
self.assertEqual(next(stream), untouched)
self.assertEqual(first['identifiers'], {'arxiv':'1408.1485','doi':'10.1234/example'})
self.assertEqual(result['identifiers_added'], 2)
def test_missing_duplicate_and_mismatched_assertions_fail_closed(self):
for mode in ('missing','duplicate','line_hash','duplicate_input'):
with self.subTest(mode=mode):
plan = copy.deepcopy(self.plan)
assertion = plan['assertions'][0]
if mode == 'missing': assertion['external_id'] = 'absent'
elif mode == 'duplicate': plan['assertions'].append(copy.deepcopy(assertion))
elif mode == 'line_hash': assertion['record_sha256'] = '0' * 64
else:
other = self.root / 'duplicate.jsonl'
other.write_bytes(self.raw)
other.with_suffix('.manifest.json').write_bytes(json_bytes(self.manifest))
plan['inputs'].append(self.input(other))
with self.assertRaises(ValueError): self.run_plan(plan)
self.assertFalse(self.output.exists())
def test_identifier_conflict_does_not_overwrite_and_same_value_is_auditable(self):
self.row['identifiers'] = {'arxiv':'cs/0307069'}
self.path.write_bytes(json_bytes(self.row))
self.manifest['file_sha256'] = digest(self.path.read_bytes())
self.path.with_suffix('.manifest.json').write_bytes(json_bytes(self.manifest))
self.plan['inputs'] = [self.input(self.path)]
self.plan['assertions'][0]['record_sha256'] = digest(self.path.read_bytes())
with self.assertRaisesRegex(ValueError, 'identifier conflict'): self.run_plan()
self.assertFalse(self.output.exists())
self.plan['assertions'][0]['identifiers']['arxiv'] = 'cs/0307069'
self.assertEqual(self.run_plan()['identifiers_added'], 0)
def test_unsupported_changes_identifiers_and_unbound_evidence_are_rejected(self):
for mode in ('title','bad_month','version','unknown_scheme','doi_case','evidence_hash','credentials','timestamp'):
with self.subTest(mode=mode):
plan = copy.deepcopy(self.plan)
assertion = plan['assertions'][0]
evidence = assertion['evidence'][0]
if mode == 'title': assertion['title'] = 'Rewrite forbidden'
elif mode == 'bad_month': assertion['identifiers'] = {'arxiv':'1413.12345'}
elif mode == 'version': assertion['identifiers'] = {'arxiv':'1408.1485v2'}
elif mode == 'unknown_scheme': assertion['identifiers'] = {'url':'https://example.org'}
elif mode == 'doi_case': assertion['identifiers'] = {'doi':'10.1234/UPPER'}
elif mode == 'evidence_hash': evidence['sha256'] = '0' * 64
elif mode == 'credentials': evidence['url'] = 'https://user:secret@example.org'
else: evidence['checked_at'] = '2026-11-08T08:00:00'
with self.assertRaises(ValueError): self.run_plan(plan)
self.assertFalse(self.output.exists())
def test_input_and_manifest_hashes_and_duplicate_json_keys_are_checked(self):
for mode in ('input','manifest','duplicate_json','invalid_evidence_json'):
with self.subTest(mode=mode):
plan = copy.deepcopy(self.plan)
if mode == 'input':
plan['inputs'][0]['sha256'] = '0' * 64
elif mode == 'manifest':
plan['inputs'][0]['manifest_sha256'] = '0' * 64
elif mode == 'duplicate_json':
raw = self.raw[:-2] + b',"source":"dblp"}\n'
self.path.write_bytes(raw)
self.path.with_suffix('.manifest.json').write_bytes(json_bytes({**self.manifest,'file_sha256':digest(raw)}))
plan['inputs'] = [self.input(self.path)]
else:
self.evidence.write_bytes(b'not json')
plan['assertions'][0]['evidence'][0]['sha256'] = digest(self.evidence.read_bytes())
with self.assertRaises(ValueError): self.run_plan(plan)
self.assertFalse(self.output.exists())
self.path.write_bytes(self.raw)
self.path.with_suffix('.manifest.json').write_bytes(json_bytes(self.manifest))
def test_curated_anchor_import_does_not_merge_extended_version_or_its_history(self):
db = self.root / 'papers.sqlite'
store = Store(db)
try:
original, _ = store.record('arxiv','1408.1485','https://example.org/one',self.row['title'],None,self.row['authors'],{'arxiv':'1408.1485'})
extended, record = store.record('arxiv','cs/0307069','https://example.org/two',self.row['title'],None,self.row['authors'],{'arxiv':'cs/0307069'})
store.decision(extended,record,'ICLR',2025,'main','another-attempt','https://example.org/reject','rejected','Reject')
store.conn.commit()
finally:
store.conn.close()
self.run_plan()
ingest_file(db,self.output / self.path.name)
store = Store(db)
try:
c = store.conn
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0], 2)
self.assertEqual(c.execute('SELECT paper_id FROM source_records WHERE external_id=?',(self.row['external_id'],)).fetchone()[0], original)
self.assertEqual(c.execute('SELECT s.paper_id FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id WHERE v.acronym=?',('ICLR',)).fetchone()[0], extended)
self.assertEqual(c.execute('SELECT count(*) FROM paper_merges').fetchone()[0], 0)
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(), [])
finally:
store.conn.close()
if __name__ == '__main__':
unittest.main()
+116
View File
@@ -0,0 +1,116 @@
from contextlib import redirect_stdout
import hashlib
import io
import json
from pathlib import Path
import sqlite3
import tempfile
import unittest
from paper_atlas.ingestion.records import ingest_file
class IncrementalCoverageTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
self.root = Path(self.tmp.name)
self.db = self.root / 'papers.sqlite'
def file(self, name, external_ids, **options):
rows = [{'source':'example','external_id':key,'url':'https://example.org/'+key,
'venue':'Example','year':2026,'track':'main','status':'accepted',
'title':'Independent research contribution '+key,'abstract':'Known abstract',
'authors':[], 'identifiers':{'doi':'10.1234/'+key}} for key in external_ids]
path = self.root / (name+'.jsonl')
path.write_bytes((''.join(json.dumps(row)+'\n' for row in rows)).encode())
manifest = {key:rows[0][key] for key in ('source','venue','year','track')}
manifest.update(list_url='https://example.org/2026', count=len(rows),status='partial',
file_sha256=hashlib.sha256(path.read_bytes()).hexdigest())
manifest.update(options)
path.with_suffix('.manifest.json').write_text(json.dumps(manifest))
return path
def ingest(self, path):
with redirect_stdout(io.StringIO()):
ingest_file(self.db,path)
def incremental(self, previous=2, prior_expected=2, expected=4):
return {'coverage_mode':'incremental','coverage_previous_count':previous,
'coverage_previous_expected_count':prior_expected,'coverage_expected_count':expected}
def coverage(self):
with sqlite3.connect(self.db) as c:
return c.execute('SELECT status,expected_count,imported_count FROM edition_sources').fetchall()
def test_append_and_refresh_count_actual_source_records_without_erasing_earlier_batch(self):
self.ingest(self.file('first',['a','b']))
self.ingest(self.file('append',['c'],**self.incremental()))
self.assertEqual(self.coverage(),[('partial',4,3)])
self.ingest(self.file('refresh',['c'],**self.incremental(3,4,4)))
self.assertEqual(self.coverage(),[('partial',4,3)])
with sqlite3.connect(self.db) as c:
self.assertEqual(c.execute('SELECT count(*) FROM source_records').fetchone()[0],3)
self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],3)
runs = c.execute('SELECT records_written,checkpoint_json FROM ingest_runs ORDER BY id').fetchall()
self.assertEqual([r[0] for r in runs],[2,1,1])
self.assertEqual(json.loads(runs[-1][1])['coverage'],
{'mode':'incremental','previous_count':3,'expected_count':4,'imported_count':3})
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(),[])
def test_stale_parent_stops_before_source_writes(self):
self.ingest(self.file('first',['a','b']))
with self.assertRaisesRegex(ValueError,'parent changed'):
self.ingest(self.file('stale',['c'],**self.incremental(1,2,4)))
self.assertEqual(self.coverage(),[('partial',2,2)])
with sqlite3.connect(self.db) as c:
self.assertEqual(c.execute("SELECT count(*) FROM source_records WHERE external_id='c'").fetchone()[0],0)
self.assertEqual(c.execute('SELECT status FROM ingest_runs ORDER BY id DESC LIMIT 1').fetchone()[0],'failed')
def test_late_source_bound_failure_rolls_back_new_records_and_coverage(self):
self.ingest(self.file('first',['a','b']))
with self.assertRaisesRegex(ValueError,'source bounds'):
self.ingest(self.file('overflow',['c'],**self.incremental(2,2,2)))
self.assertEqual(self.coverage(),[('partial',2,2)])
with sqlite3.connect(self.db) as c:
for table in ('papers','source_records','submissions','decisions'):
self.assertEqual(c.execute('SELECT count(*) FROM '+table).fetchone()[0],2)
def test_multiple_listings_cannot_be_silently_aggregated(self):
self.ingest(self.file('one',['a']))
self.ingest(self.file('two',['b'],list_url='https://example.org/another-list'))
before=self.coverage()
with self.assertRaisesRegex(ValueError,'unambiguous source listing'):
self.ingest(self.file('ambiguous',['c'],**self.incremental(2,1,4)))
self.assertEqual(self.coverage(),before)
def test_untracked_source_rows_require_reconciliation(self):
self.ingest(self.file('first',['a','b']))
with sqlite3.connect(self.db) as c:
c.execute('UPDATE edition_sources SET imported_count=1')
with self.assertRaisesRegex(ValueError,'actual source records'):
self.ingest(self.file('stale-summary',['c'],**self.incremental(1,2,4)))
self.assertEqual(self.coverage(),[('partial',2,1)])
def test_first_partial_chunk_can_declare_an_independently_known_source_total(self):
self.ingest(self.file('first',['a'],**self.incremental(0,None,4)))
self.assertEqual(self.coverage(),[('partial',4,1)])
def test_complete_listing_is_not_demoted_by_a_partial_chunk(self):
self.ingest(self.file('complete',['a','b'],status='complete'))
with self.assertRaisesRegex(ValueError,'not partial'):
self.ingest(self.file('append',['c'],**self.incremental()))
self.assertEqual(self.coverage(),[('complete',2,2)])
def test_invalid_mode_counts_and_missing_scope_fail_before_database_creation(self):
for options in ({'coverage_mode':'unknown'}, self.incremental(True,2,4),
self.incremental(2,5,4),self.incremental(-1,2,4),
{'coverage_mode':'incremental'},
{**self.incremental(),'status':'complete'}):
with self.subTest(options=options):
with self.assertRaises(ValueError):self.ingest(self.file('invalid',['a'],**options))
self.assertFalse(self.db.exists())
if __name__=='__main__':
unittest.main()
+44
View File
@@ -53,6 +53,50 @@ class StoreTests(unittest.TestCase):
self.assertNotEqual(other,paper)
self.assertEqual(self.store.conn.execute('SELECT count(*) FROM match_candidates').fetchone()[0],1)
def test_identifier_anchor_does_not_absorb_a_same_title_arxiv_version(self):
conference,first=self.record('arxiv','1408.1485')
extended,second=self.record('arxiv','cs/0307069')
self.store.decision(extended,second,'ICLR',2025,'main','separate','https://example.org/reject','rejected','Reject')
resolved,_=self.record('dblp','conf/uai/example',{'arxiv':'1408.1485'})
self.assertEqual(resolved,conference);self.assertNotEqual(conference,extended)
c=self.store.conn
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],2)
self.assertEqual(c.execute('SELECT paper_id FROM submissions').fetchone()[0],extended)
self.assertEqual(c.execute('SELECT count(*) FROM paper_merges').fetchone()[0],0)
self.assertEqual(c.execute('SELECT count(*) FROM match_candidates').fetchone()[0],1)
# A refresh of the anchored original source is equally unambiguous.
self.assertEqual(self.record('arxiv','1408.1485')[0],conference)
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],2)
def test_ambiguous_title_aborts_whole_file_without_merging_existing_entities(self):
self.record('arxiv','1408.1485');self.record('arxiv','cs/0307069')
self.store.conn.commit()
rows=[{'source':'dblp','external_id':'conf/uai/new','url':'https://example.org/new',
'title':'An unrelated earlier record in this file','abstract':None,'authors':[],
'venue':'UAI','year':2001,'track':'main','status':'accepted'},
{'source':'dblp','external_id':'conf/uai/ambiguous','url':'https://example.org/ambiguous',
'title':'A sufficiently long example research paper title','abstract':None,
'authors':['Jane Doe','John Smith'],'identifiers':{'doi':'10.0000/not-yet-known'},
'venue':'UAI','year':2001,'track':'main','status':'accepted'}]
path=Path(self.tmp.name)/'ambiguous.jsonl'
path.write_text(''.join(json.dumps(row)+'\n' for row in rows))
path.with_suffix('.manifest.json').write_text(json.dumps({'source':'dblp','list_url':'https://example.org/list','count':2}))
with self.assertRaisesRegex(ValueError,'Ambiguous paper identity'):ingest_file(self.db,path)
c=self.store.conn
for table,count in [('papers',2),('source_records',2),('submissions',0),('decisions',0),('paper_merges',0)]:
self.assertEqual(c.execute('SELECT count(*) FROM '+table).fetchone()[0],count,table)
self.assertEqual(c.execute('SELECT status FROM ingest_runs').fetchone()[0],'failed')
def test_explicit_bridge_merges_only_its_identifier_matches(self):
first,_=self.record('arxiv','2601.00001',authors=['First Author'])
second,_=self.record('arxiv','2601.00002',{'doi':'10.0000/bridge'},authors=['Second Author'])
unrelated,_=self.record('arxiv','2601.00003')
resolved,_=self.record('official','bridge',{'arxiv':'2601.00001','doi':'10.0000/bridge'})
self.assertEqual(resolved,min(first,second))
self.assertEqual(self.store.conn.execute('SELECT count(*) FROM papers').fetchone()[0],2)
self.assertIsNotNone(self.store.conn.execute('SELECT 1 FROM papers WHERE id=?',(unrelated,)).fetchone())
self.assertEqual(self.store.conn.execute('SELECT count(*) FROM paper_merges').fetchone()[0],1)
def test_manifest_count_mismatch_rolls_back_records(self):
path=Path(self.tmp.name)/'source.jsonl'
path.write_text(json.dumps({'source':'pmlr','external_id':'v1/paper','url':'https://example.org/paper','title':'Paper','abstract':'Abstract','authors':[],'venue':'ICML','year':2023,'track':'main','status':'accepted'})+'\n')
+16
View File
@@ -175,6 +175,22 @@ class PostgreSQLIntegrationTests(unittest.TestCase):
finally:
reader.close()
def test_history_audit_matches_sqlite_for_actual_postgresql_snapshot(self):
from paper_atlas.audits.histories import report
from tests.test_histories import add_history_fixture
store = Store(self.db)
try:
add_history_fixture(store)
finally:
store.conn.close()
expected = report(self.db)
self.assertEqual(self.migrate()['status'], 'complete')
actual = report(pg_service=self.service, pg_schema=self.target)
self.assertEqual(actual['engine'], 'postgresql')
self.assertEqual(actual['schema'], self.target)
for key in ('counts', 'latest_attempt_status_counts', 'evidence_relationship_checks', 'examples'):
self.assertEqual(actual[key], expected[key], key)
def interrupt_after_first_table(self):
calls = 0
def interrupt(*args):
+148
View File
@@ -87,6 +87,82 @@ class PostgreSQLIngestionTests(unittest.TestCase):
with redirect_stdout(io.StringIO()):
ingest_file(None, path, pg_service=self.service, pg_schema=self.schema)
def test_incremental_source_coverage_counts_prior_chunks_and_refreshes_on_postgres(self):
rows = [self.row(source='dblp',external=key,status='accepted',year=2001,venue='UAI')
for key in ('incremental1','incremental2','incremental3')]
listing = 'https://example.org/uai/2001'
def manifest(path, **options):
target=path.with_suffix('.manifest.json')
value=json.loads(target.read_text())
value.update(list_url=listing,status='partial',**options)
target.write_text(json.dumps(value))
first=self.file('initial',rows[:2]);manifest(first);self.ingest(first)
second=self.file('append',[rows[2]])
manifest(second,coverage_mode='incremental',coverage_previous_count=2,
coverage_previous_expected_count=2,coverage_expected_count=4)
self.ingest(second)
manifest(second,coverage_mode='incremental',coverage_previous_count=3,
coverage_previous_expected_count=4,coverage_expected_count=4)
self.ingest(second)
with closing(ReadConnection(self.service,self.schema)) as c:
coverage=c.execute('SELECT status,expected_count,imported_count FROM edition_sources WHERE list_url=?',(listing,)).fetchone()
self.assertEqual(tuple(coverage.values),('partial',4,3))
stored=c.execute('SELECT abstract FROM source_records WHERE external_id=?',('incremental3',)).fetchone()
self.assertEqual(stored[0],rows[2]['abstract'])
self.assertEqual(c.execute("SELECT count(*) FROM source_records r JOIN sources s ON s.id=r.source_id WHERE s.name='dblp'").fetchone()[0],3)
run=c.execute('SELECT records_written,checkpoint_json FROM ingest_runs ORDER BY id DESC LIMIT 1').fetchone()
self.assertEqual(run[0],1)
self.assertEqual(json.loads(run[1])['coverage']['imported_count'],3)
def test_incremental_postgres_bound_failure_rolls_back_rows_and_summary(self):
listing='https://example.org/uai/2001'
rows=[self.row(source='dblp',external=key,status='accepted',year=2001,venue='UAI')
for key in ('bound1','bound2','bound3')]
first=self.file('initial',rows[:2]);target=first.with_suffix('.manifest.json')
value=json.loads(target.read_text());value.update(list_url=listing,status='partial');target.write_text(json.dumps(value))
self.ingest(first)
with closing(ReadConnection(self.service,self.schema)) as c:
before={table:c.execute('SELECT count(*) FROM '+table).fetchone()[0]
for table in ('papers','source_records','submissions','decisions')}
path=self.file('overflow',[rows[2]]);target=path.with_suffix('.manifest.json')
value=json.loads(target.read_text());value.update(list_url=listing,status='partial',coverage_mode='incremental',
coverage_previous_count=2,coverage_previous_expected_count=2,coverage_expected_count=2)
target.write_text(json.dumps(value))
with self.assertRaisesRegex(ValueError,'source bounds'):self.ingest(path)
with closing(ReadConnection(self.service,self.schema)) as c:
self.assertEqual({table:c.execute('SELECT count(*) FROM '+table).fetchone()[0] for table in before},before)
coverage=c.execute('SELECT status,expected_count,imported_count FROM edition_sources WHERE list_url=?',(listing,)).fetchone()
self.assertEqual(tuple(coverage.values),('partial',2,2))
self.assertEqual(c.execute('SELECT status FROM ingest_runs ORDER BY id DESC LIMIT 1').fetchone()[0],'failed')
def test_incremental_postgres_does_not_overwrite_a_concurrently_changed_parent(self):
listing='https://example.org/uai/2001'
rows=[self.row(source='dblp',external=key,status='accepted',year=2001,venue='UAI')
for key in ('concurrent1','concurrent2','concurrent3')]
first=self.file('initial',rows[:2]);target=first.with_suffix('.manifest.json')
value=json.loads(target.read_text());value.update(list_url=listing,status='partial');target.write_text(json.dumps(value))
self.ingest(first)
path=self.file('racing',[rows[2]]);target=path.with_suffix('.manifest.json')
value=json.loads(target.read_text());value.update(list_url=listing,status='partial',coverage_mode='incremental',
coverage_previous_count=2,coverage_previous_expected_count=2,coverage_expected_count=4)
target.write_text(json.dumps(value))
original=Store.record
def change_parent(store,*args,**kwargs):
result=original(store,*args,**kwargs)
# A separate committed SQL writer need not honor the native
# advisory lock. Its update must survive our rejected file.
with connect(self.service) as other:
other.execute('SET search_path TO '+quote(self.schema)+',pg_catalog')
other.execute('UPDATE edition_sources SET expected_count=99 WHERE list_url=%s',(listing,))
return result
with patch.object(Store,'record',change_parent):
with self.assertRaisesRegex(ValueError,'parent changed during import'):self.ingest(path)
with closing(ReadConnection(self.service,self.schema)) as c:
coverage=c.execute('SELECT status,expected_count,imported_count FROM edition_sources WHERE list_url=?',(listing,)).fetchone()
self.assertEqual(tuple(coverage.values),('partial',99,2))
self.assertEqual(c.execute('SELECT count(*) FROM source_records WHERE external_id=?',('concurrent3',)).fetchone()[0],0)
self.assertEqual(c.execute('SELECT status FROM ingest_runs ORDER BY id DESC LIMIT 1').fetchone()[0],'failed')
def test_native_writes_keep_nul_historical_ids_and_later_acceptance(self):
store = self.store()
try:
@@ -110,6 +186,34 @@ class PostgreSQLIngestionTests(unittest.TestCase):
finally:
reader.close()
def test_identity_anchor_keeps_other_arxiv_version_and_ambiguous_file_rolls_back(self):
title='An identical long research title shared by conference and extended versions'
authors=['Jane Doe','John Smith']
store=self.store()
try:
first,_=store.record('arxiv','1408.1485','https://arxiv.org/abs/1408.1485',title,'Conference abstract',authors)
second,_=store.record('arxiv','cs/0307069','https://arxiv.org/abs/cs/0307069',title,'Extended abstract',authors)
resolved,_=store.record('dblp','known','https://example.org/known',title,None,authors,{'arxiv':'1408.1485'})
self.assertEqual(resolved,first);self.assertNotEqual(first,second)
store.conn.commit()
self.assertEqual(store.conn.execute('SELECT count(*) FROM papers').fetchone()[0],3)
self.assertEqual(store.conn.execute('SELECT count(*) FROM paper_merges').fetchone()[0],1) # migration fixture only
self.assertEqual(store.conn.execute('SELECT count(*) FROM match_candidates').fetchone()[0],1)
finally:store.conn.close()
rows=[self.row('official','earlier',status='accepted',venue='UAI',year=2001),
dict(self.row('official','ambiguous',status='accepted',venue='UAI',year=2001),
title=title,abstract=None,authors=authors,identifiers={})]
with self.assertRaisesRegex(ValueError,'Ambiguous paper identity'):self.ingest(self.file('ambiguous-identity',rows))
store=self.store()
try:
self.assertEqual(store.conn.execute('SELECT count(*) FROM papers').fetchone()[0],3)
self.assertEqual(store.conn.execute('SELECT count(*) FROM source_records').fetchone()[0],4)
self.assertEqual(store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0],1)
self.assertEqual(store.conn.execute('SELECT count(*) FROM paper_merges').fetchone()[0],1)
self.assertEqual(store.conn.execute('SELECT status FROM ingest_runs ORDER BY id DESC LIMIT 1').fetchone()[0],'failed')
self.assertEqual(store.conn.execute("SELECT count(*) FROM source_records WHERE external_id IN (?,?)",('earlier','ambiguous')).fetchone()[0],0)
finally:store.conn.close()
def test_file_import_is_idempotent_and_refreshes_only_own_tags(self):
row = self.row(); row['tags'] = [{'namespace': 'openreview_snapshot:area', 'value': 'theory'}]
path = self.file('reject', [row]); self.ingest(path)
@@ -243,6 +347,50 @@ class PostgreSQLIngestionTests(unittest.TestCase):
finally:
reader.close()
def test_coverage_matches_sqlite_with_history_aliases_and_missing_source_abstracts(self):
from paper_atlas.audits.coverage import report
from tests.test_coverage_report import add_coverage_evidence
registry = self.root / 'venues.json'
registry.write_text(json.dumps({'scope_note': 'Candidate scope', 'venues': [
{'acronym': 'coling', 'field': 'language'}, {'acronym': 'AAAI', 'field': 'general_ai'}]}))
sample = Store(self.root / 'seed.sqlite')
try:
add_coverage_evidence(sample)
finally:
sample.conn.close()
store = self.store()
try:
add_coverage_evidence(store)
finally:
store.conn.close()
sqlite_report = report(self.root / 'seed.sqlite', registry)
observed = []
def reader(*args, **kwargs):
connection = open_read_connection(*args, **kwargs)
observed.append((connection.execute('SHOW transaction_read_only').fetchone()[0],
connection.execute('SHOW transaction_isolation').fetchone()[0]))
return connection
with patch('paper_atlas.audits.coverage.open_read_connection', side_effect=reader):
actual = report(None, registry, pg_service=self.service, pg_schema=self.schema)
self.assertEqual(observed, [('on', 'repeatable read')])
self.assertEqual(actual['engine'], 'postgresql')
self.assertEqual(actual['schema'], self.schema)
self.assertEqual(actual['pg_service'], self.service)
def content(result):
values = json.loads(json.dumps(result['venues']))
for venue in values:
for edition in venue['editions']:
for key in ('id', 'venue_id'):
edition.pop(key)
return values
self.assertEqual(content(actual), content(sqlite_report))
self.assertEqual([r['acronym'] for r in actual['imported_venues_outside_registry']], ['ICLR'])
edition = actual['venues'][0]['editions'][0]
self.assertEqual(edition['latest_observed_status_counts']['accepted'], 1)
self.assertEqual(edition['latest_observed_status_counts']['unknown'], 1)
self.assertEqual(edition['decision_observation_counts']['rejected'], 1)
self.assertEqual(edition['papers_without_abstract_text'], 1)
def test_backfill_merges_evidence_without_reimporting_or_losing_history(self):
files = []
for source, external, title in [('official', 'legacy', 'First bibliographic name'), ('openreview_snapshot', 'forum', 'Renamed submission')]:
+147 -1
View File
@@ -45,10 +45,12 @@ class PublicOpenReviewTests(unittest.TestCase):
def test_only_explicit_decisions_and_verified_scope_become_events(self):
for value in ('Pending', 'Active', 'Submitted to NeurIPS 2025', 'TBD', '',
'Low score', 'ICML 2025 Retracted by Authors', 'Do not reject'):
'Low score', 'ICML 2025 Retracted by Authors', 'Do not reject',
'Workshop', 'Poster Workshop', 'Oral Workshop'):
self.assertIsNone(negative_status(value), value)
self.assertEqual(negative_status('Reject'), 'rejected')
self.assertEqual(negative_status('Desk rejected'), 'desk_rejected')
self.assertEqual(negative_status('Withdraw'), 'withdrawn')
self.assertEqual(negative_status('NeurIPS 2023 Conference Withdrawn Submission'), 'withdrawn')
for raw, expected in [('main', 'main'), ('Position', 'position'),
('Datasets & Benchmarks', 'datasets_benchmarks')]:
@@ -98,6 +100,121 @@ class PublicOpenReviewTests(unittest.TestCase):
self.assertEqual(list(Path(tmp).glob('*.jsonl')), [])
self.assertFalse((Path(tmp) / 'public-openreview-inventory.json').exists())
def test_missing_title_is_held_without_losing_valid_negative_records(self):
with tempfile.TemporaryDirectory() as tmp:
result = self.collect(tmp, {'venues/iclr/iclr2017.jsonl': [
self.row(id='SJUdkecgx', title=''),
self.row(id='NoTitle001', title=None),
self.row(id='Whitespace', title=' '),
self.row(id='ValidReject'),
self.row(id='Withdraw01', status='Withdraw'),
self.row(id='Workshop01', status='Workshop'),
]}, families=['iclr'], years=[2017])
self.assertEqual(result['input_rows'], 6)
self.assertEqual(result['normalized_records'], 2)
self.assertEqual(result['source_files'][0]['excluded_or_held'],
{'missing_title': 3, 'not_explicit_negative': 1})
held_path = Path(result['held_file'])
self.assertEqual(hashlib.sha256(held_path.read_bytes()).hexdigest(),
result['held_file_sha256'])
held = [json.loads(line) for line in held_path.read_text().splitlines()]
self.assertEqual([row['id'] for row in held], ['SJUdkecgx', 'NoTitle001', 'Whitespace'])
self.assertTrue(all(row['reason'] == 'missing_title' and row['status'] == 'Reject'
for row in held))
self.assertEqual([row['provenance']['source_line'] for row in held], [1, 2, 3])
self.assertNotIn('private@example.org', held_path.read_text())
self.assertNotIn('private review', held_path.read_text())
manifest = result['files'][0]
self.assertEqual((manifest['parser_version'], manifest['status']), (3, 'partial'))
rows = [json.loads(line) for line in Path(manifest['file']).read_text().splitlines()]
self.assertEqual([(row['identifiers']['openreview'], row['status']) for row in rows],
[('ValidReject', 'rejected'), ('Withdraw01', 'withdrawn')])
for row in held:
self.assertEqual(row['provenance']['source_revision'], REVISION)
def test_nontext_title_still_fails_collection(self):
with tempfile.TemporaryDirectory() as tmp:
with self.assertRaisesRegex(ValueError, 'lacks a title'):
self.collect(tmp, {'venues/iclr/iclr2017.jsonl': [self.row(title=['invalid'])]},
families=['iclr'], years=[2017])
self.assertFalse((Path(tmp) / 'public-openreview-inventory.json').exists())
self.assertEqual(list(Path(tmp).glob('*.jsonl')), [])
def workshop_cases(self):
return [
(2024, 'workshop_me_fomo', 'ICLR 2024 Workshop MEFoMo Withdrawn Submission', 'withdrawn', 'ME-FoMo'),
(2024, 'workshop_genai4dm', 'ICLR 2024 Workshop GenAI4DM Withdrawn Submission', 'withdrawn', 'GenAI4DM'),
(2025, 'workshop_agenticai', 'ICLR 2025 Workshop AgenticAI Reject', 'rejected', 'AgenticAI'),
(2025, 'workshop_lmrl', 'ICLR 2025 Workshop LMRL Withdrawn Submission', 'withdrawn', 'LMRL'),
(2026, 'workshop_llm_reasoning', 'ICLR 2026 Workshop LLM Reasoning Withdrawn Submission', 'withdrawn', 'LLM_Reasoning'),
]
def test_iclr_workshop_labels_require_opt_in_and_retain_identity_evidence(self):
for year, track, label, status, group in self.workshop_cases():
with self.subTest(label=label):
raw = self.row(track=track, status=label)
self.assertEqual(normalize(raw, 'iclr', year, self.provenance()),
(None, 'separate_workshop'))
row, reason = normalize(raw, 'iclr', year, self.provenance(), True)
self.assertIsNone(reason)
self.assertEqual((row['venue'], row['year'], row['track'], row['status']),
('ICLR', year, track, status))
self.assertEqual(row['raw_decision'], label)
self.assertEqual(row['metadata']['workshop_venueid'],
f'ICLR.cc/{year}/Workshop/{group}')
self.assertFalse(row['metadata']['original_decision_signature_verified'])
self.assertEqual(row['evidence_url'], 'https://example.org/pinned.jsonl#L7')
self.assertNotIn('private review', json.dumps(row))
def test_workshop_label_scope_conflicts_never_become_main_events(self):
for year, track, label, _, _ in self.workshop_cases():
for enabled in (False, True):
for wrong_track in ('main', 'workshop_other'):
self.assertEqual(normalize(self.row(track=wrong_track, status=label),
'iclr', year, self.provenance(), enabled),
(None, 'conflicting_workshop_track'))
for family, wrong_year in [('icml', year), ('iclr', year + 1)]:
with self.assertRaisesRegex(ValueError, 'conference year'):
normalize(self.row(track=track, status=label), family,
wrong_year, self.provenance(), enabled)
row, reason = normalize(self.row(track='workshop_unknown',
status='ICLR 2026 Workshop Unknown Reject'),
'iclr', 2026, self.provenance(), True)
self.assertIsNone(row)
self.assertEqual(reason, 'unmapped_negative_label')
def test_workshop_collection_and_import_keep_five_tracks_separate_from_main(self):
files = {}
for index, (year, track, label, _, _) in enumerate(self.workshop_cases()):
files.setdefault(f'venues/iclr/iclr{year}.jsonl', []).append(
self.row(id=f'Workshop{index:02}', track=track, status=label))
files['venues/iclr/iclr2024.jsonl'].append(self.row())
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
default = self.collect(root / 'default', files, families=['iclr'])
self.assertEqual(default['normalized_records'], 1)
held = [json.loads(line) for line in Path(default['held_file']).read_bytes().split(b'\n') if line]
self.assertEqual(len(held), 5)
self.assertTrue(all(row['reason'] == 'separate_workshop' for row in held))
selected = self.collect(root / 'selected', files, families=['iclr'], include_workshops=True)
self.assertEqual(selected['normalized_records'], 6)
self.assertEqual(len(selected['files']), 6)
self.assertEqual(Path(selected['held_file']).read_bytes(), b'')
db = root / 'db.sqlite'
for manifest in selected['files']:
self.assertEqual(manifest['parser_version'], 3)
ingest_file(db, manifest['file'])
store = Store(db)
try:
scopes = set(store.conn.execute('SELECT e.year,e.track,d.status FROM submissions s '
'JOIN editions e ON e.id=s.edition_id '
'JOIN decisions d ON d.submission_id=s.id'))
expected = {(year, track, status) for year, track, _, status, _ in self.workshop_cases()}
self.assertEqual(scopes, expected | {(2024, 'main', 'rejected')})
self.assertEqual(store.conn.execute('PRAGMA foreign_key_check').fetchall(), [])
finally:
store.conn.close()
def test_mirror_reimport_preserves_one_attempt_and_later_acceptance_elsewhere(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
@@ -152,6 +269,35 @@ class PublicOpenReviewTests(unittest.TestCase):
finally:
store.conn.close()
def test_unicode_separators_in_abstract_survive_audit_and_import(self):
for separator in ('\u0085', '\u2028', '\u2029'):
with self.subTest(separator=repr(separator)), tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
abstract = 'First paragraph' + separator + 'Second paragraph'
result = self.collect(root / 'source', {
'venues/icml/icml2025.jsonl': [self.row(abstract=abstract)]})
path = Path(result['files'][0]['file'])
data = path.read_bytes()
self.assertIn(separator.encode(), data)
self.assertEqual(data.count(b'\n'), 1)
db = root / 'db.sqlite'
store = Store(db)
store.conn.close()
inventory = root / 'source/public-openreview-inventory.json'
before = audit(db, inventory, root / 'before.json')
self.assertEqual(before['status'], 'ready_for_serial_import')
self.assertEqual(before['records_checked'], 1)
ingest_file(db, path)
after = audit(db, inventory, root / 'after.json')
self.assertEqual(after['status'], 'ready_for_serial_import')
self.assertEqual(after['existing_same_edition_attempts'], 1)
store = Store(db)
try:
self.assertEqual(store.conn.execute(
'SELECT abstract FROM source_records').fetchone()[0], abstract)
finally:
store.conn.close()
if __name__ == '__main__':
unittest.main()
+21
View File
@@ -37,6 +37,27 @@ class VisionTests(unittest.TestCase):
collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp)
self.assertFalse(list(Path(tmp).glob('*.jsonl')))
def test_fragment_variants_share_one_all_papers_resource(self):
root,papers,entry=self.cvf_fixture();all_url=root+'?day=all'
pages={root:''.join(f'<a href="{all_url}{fragment}">All</a>' for fragment in ('','#','#papers')),
all_url:'<dl>'+entry(papers[0])+entry(papers[1])+'</dl>'}
pages.update({paper:'<div id="papertitle">Paper</div><div id="abstract">Abstract</div>' for paper in papers[:2]})
with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',side_effect=lambda url,*a,**kw:pages[url].encode()) as fetch:
manifest=collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp)
self.assertEqual(manifest['count'],2)
self.assertEqual(manifest['index_urls'],[all_url])
self.assertEqual([call.args[0] for call in fetch.call_args_list].count(all_url),1)
self.assertFalse(any('#' in call.args[0] for call in fetch.call_args_list))
def test_distinct_all_papers_queries_still_fail_before_publishing(self):
root,_,_=self.cvf_fixture();urls=[root+'?day=all',root+'?day=all&view=alternate']
html=''.join(f'<a href="{url}">All</a>' for url in urls).encode()
with tempfile.TemporaryDirectory() as tmp,patch('paper_atlas.collectors.vision.fetch',return_value=html) as fetch:
with self.assertRaisesRegex(ValueError,'Ambiguous all-papers index'):
collect_index('cvf',{'venue':'CVPR','year':2018,'track':'main','index_url':root},tmp,tmp)
self.assertEqual(fetch.call_count,1)
self.assertFalse(list(Path(tmp).glob('*.jsonl')))
def test_missing_abstract_page_keeps_official_index_metadata_and_marks_partial(self):
root,papers,entry=self.cvf_fixture();error=HTTPError(papers[0],404,'Not Found',{},io.BytesIO())
def fetch(url,*args,**kwargs):