Compare commits

...

21 Commits

Author SHA1 Message Date
codex-bot 25ed4559ca docs: set 2020 backfill scope and preserve recovery inventory since 2000
Metadata integrity checks / metadata (push) Successful in 2m9s
2026-11-10 00:50:15 +08:00
Zayden-Jung b226b3b98b Merge pull request '限定ACL1978与TINLAP卷的来源关系及非论文条目' (#42) from codex/acl-early into main
Metadata integrity checks / metadata (push) Successful in 2m18s
Reviewed-on: #42
2026-11-09 13:42:18 +00:00
Zayden-Jung 05ada8daf8 Merge pull request '支持 AAMAS 2025 官方出版目录元数据采集' (#44) from codex/aamas2025 into main
Metadata integrity checks / metadata (push) Successful in 3m20s
Reviewed-on: #44
2026-11-09 13:42:00 +00:00
codex-bot 13408869c2 Add reviewed AAMAS 2025 publisher metadata adapter
Metadata integrity checks / metadata (push) Successful in 2m11s
2026-11-09 20:50:29 +08:00
codex-bot 443a79dce2 Preserve edition-specific ACL and TINLAP 1978 source evidence
Metadata integrity checks / metadata (push) Successful in 2m6s
2026-11-09 17:20:27 +08:00
Zayden-Jung ec9edd88d4 Merge pull request '保留ICLR历史录用档案的轨道与完整来源证据' (#41) from codex/iclr-archive into main
Metadata integrity checks / metadata (push) Successful in 2m54s
Reviewed-on: #41
2026-11-09 09:02:49 +00:00
codex-bot b5c0618be5 Handle standalone italic author blocks in the historical archive
Metadata integrity checks / metadata (push) Successful in 2m6s
2026-11-09 16:24:42 +08:00
codex-bot 81864e48ae Preserve complete fragmented ICLR 2014 author paragraphs
Metadata integrity checks / metadata (push) Successful in 2m9s
2026-11-09 16:19:19 +08:00
Zayden-Jung 8ad052f577 Merge pull request '核验学习与规划会议前身卷并补齐发现范围' (#40) from codex/historical-predecessors into main
Metadata integrity checks / metadata (push) Successful in 2m12s
Reviewed-on: #40
2026-11-09 08:13:52 +00:00
codex-bot 348ebe0695 Collect archived ICLR accepted lists with track and presentation evidence
Metadata integrity checks / metadata (push) Successful in 2m15s
2026-11-09 16:09:13 +08:00
codex-bot aa696bd7b5 Isolate tiny predecessor fixtures from production disk reserve
Metadata integrity checks / metadata (push) Successful in 2m11s
2026-11-09 15:15:48 +08:00
codex-bot 90bd534edd Preserve reviewed historical learning and planning predecessor series
Metadata integrity checks / metadata (push) Failing after 2m0s
2026-11-09 15:01:12 +08:00
Zayden-Jung 02a408ba8f Merge pull request '保留 AAMAS 发表来源的同届 DOI 投稿身份' (#39) from codex/aamas-publication-identity into main
Metadata integrity checks / metadata (push) Successful in 2m15s
Reviewed-on: #39
2026-11-09 06:43:15 +00:00
codex-bot 714f745bd5 Assert normalized identity tuples and exact negative-status boundaries
Metadata integrity checks / metadata (push) Successful in 2m13s
2026-11-09 14:13:12 +08:00
Zayden-Jung cc78e837d2 Merge pull request '按发表形式采集 AAMAS 官方论文集' (#38) from codex/aamas-proceedings into main
Metadata integrity checks / metadata (push) Successful in 2m26s
Reviewed-on: #38
2026-11-09 05:52:57 +00:00
codex-bot c0b2146ae8 Recognize AAMAS publisher DOI identities within submission scopes
Metadata integrity checks / metadata (push) Failing after 2m1s
2026-11-09 13:51:49 +08:00
codex-bot ae88367fac Scope AAMAS tags to their exact source namespace
Metadata integrity checks / metadata (push) Successful in 8m56s
2026-11-09 13:31:46 +08:00
codex-bot a305d00c75 Collect evidenced AAMAS proceedings metadata by publication format
Metadata integrity checks / metadata (push) Failing after 2m2s
2026-11-09 13:27:23 +08:00
Zayden-Jung 57fcfc7693 Merge pull request '按官方证据识别早期 ICML 与 IROS 主系列' (#37) from codex/historical-primary-series into main
Metadata integrity checks / metadata (push) Successful in 2m58s
Reviewed-on: #37
2026-11-09 05:19:49 +00:00
Zayden-Jung a07b1e42a8 Merge pull request '修正早期 ACL 来源误匹配并记录官方目录缺口' (#36) from codex/history-source-scope into main
Metadata integrity checks / metadata (push) Successful in 2m48s
Reviewed-on: #36
2026-11-09 05:19:43 +00:00
codex-bot 7c6f4c9f1d Recognize evidenced early ICML and IROS primary editions
Metadata integrity checks / metadata (push) Successful in 9m26s
2026-11-09 02:39:18 +08:00
34 changed files with 18376 additions and 8 deletions
+4
View File
@@ -49,6 +49,10 @@ jobs:
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','www-archive','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--year','2025','--snapshots',str(src/'tests/fixtures/aamas2025'),'--out',str(root/'installed-aamas2025')],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','iclr-archive','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','acl','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','postgres','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','edition-tracks','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','joint-editions','--help'],cwd=outside,check=True)
+12
View File
@@ -6,8 +6,14 @@ Paper Atlas 将 arXiv、AI 会议论文集及公开投稿决定整理到可追
项目仍在建设中。候选范围包含 43 个会议(含 ACM MM、MLSys),尚未完成全部历史数据和最终去重复核。实际进度以数据库、来源清单和覆盖报告为准,而不是仓库文件数量。
当前主动补漏从 **2020 年**起;2000 年起已发现的名单、缺口与待办保存在[补漏恢复清单](docs/coverage/backfill-handoff-2026-11-10.md),下次从这里接续。已有旧数据保留,范围排除不算补齐。
历史 ICLR 2014–2016 官方录用页由 `collect iclr-archive` 适配,保留主会/workshop及重复展示证据;运行和身份核验边界见 [历史档案说明](docs/coverage/iclr-archive-2014-2016.md)。
## 项目结构
ACL 1978 的年会与 TINLAP-2 来源关系按精确卷适配,保留原系列和不明逐篇角色;1965 的替代关系不转成全局别名。来源、排除条目和恢复边界见 [ACL 早期来源核查](docs/coverage/acl-early-sources-2026-11-09.md)。
```text
paper_atlas/
cli.py 统一命令入口
@@ -60,6 +66,12 @@ paper-atlas collect colm --cache /data/paper-index/raw/http-cache \
--out /data/paper-index/collected
```
早期主系列的Workshop名称按[ICML/IROS精确届次证据](docs/coverage/historical-primary-series-2026-11-09.md)识别,保留原名与范围证据;代码支持不代表已补齐生产数据。
EWSL、EPS、ECP、EWSP 的[具体前身卷与完整快照要求](docs/coverage/historical-predecessors-2026-11-09.md)单独核验,保留原始系列及事件/出版年份;旧候选文件缺少 ECP/EWSP 时会明确提示,避免静默遗漏。
AAMAS 的[官方论文集采集器](docs/aamas-proceedings.md)支持已核验的 2012、2025 版式,可从缓存或已校验快照采集元数据。2025 年目录含 476 条非 keynote 出版物,保留完整作者和机构,HTML 未提供摘要时留空;发表形式与投稿 track 分开,不根据主题推定主会轨道或研究类型。
`paper-atlas collect --help` 查看来源;`paper-atlas import --help` 查看导入方式。完整批次步骤见[使用指南](docs/usage.md)。
共享实例名为 `foundry-postgres`,数据位于 `foundry_data` 库的 `research` schema;其他项目可按权限复用同一组表。库名通过 libpq service 配置、schema 通过 search_path 或 `--pg-schema` 指定,不写死在查询中。PostgreSQL 凭据放在服务器权限为 `0600` 的 libpq service 文件中;命令行只使用服务名称。`paper-atlas maintain postgres --help` 提供迁移入口:先完整核验 21 张业务表,再发布目标 schema。SQLite FTS5 内部表由 PostgreSQL GIN 索引替代。切换后 SQLite 为只读备份;`import records`、`import arxiv` 和 `maintain backfill` 使用 `--pg-service foundry-research` 写入主库,整个写入会话只允许一个协调写入者,会议文件失败时整体回滚并保留失败批次记录。
+51
View File
@@ -0,0 +1,51 @@
# AAMAS 官方论文集元数据
`paper_atlas.collectors.aamas` 是按需运行的采集适配器,通过统一入口 `collect aamas-proceedings` 使用。支持已独立核验的 **AAMAS 2012、2025** HTML 版式;其他年份需先核对官方目录和逐篇结构,再增加适配,不能仅替换年份。2025 解析单独封装在 `paper_atlas.collectors.aamas2025`。
2025 版式使用[官方首页](https://www.ifaamas.org/Proceedings/aamas2025/)和[出版目录](https://www.ifaamas.org/Proceedings/aamas2025/forms/contents.htm)。原始目录共 479 条:3 keynote、260 research paper、148 extended abstract、12 Blue Sky、25 doctoral、22 Demo 栏目、9 JAAMAS。排除 keynote,输出 476 条;Demo 栏目中的 p3000 是明确标题的 COMP24 竞赛报告,发表年份仍为 2025,格式记 `competition_report`、投稿 track 留作 `proceedings_unspecified`,保留原 Demo 栏目证据。其余输出按 track 分组为 409 unspecified、12 blue_sky、25 doctoral、21 demo、9 jaamas。
作者按逐篇 BR 行解析,斜体机构原文另存,保留 Unicode、顺序及完整名单。HTML 没有提供摘要或逐篇 DOI,故 `abstract=null`、标识字典为空;只保存 PDF 链接,不下载研究 PDF。不能把 Research Paper Track 的所有条目推成 main,不能把目录缺席推为 rejected/withdrawn,也不能把出版格式标签当研究类型。
2025 用法:`paper-atlas collect aamas-proceedings --year 2025 --snapshots /data/review/aamas2025 --out /data/paper-index/collected/aamas-2025`;在线模式改用 `--cache /data/paper-index/cache/aamas2025`,只请求上述两个 HTML 页面。快照目录包含原始 `index.html`、`contents.html` 和 `receipts.json`,凭据必须列出各原始 URL、文件、字节数、SHA256、`http_status=200` 和带时区 `at`。已恢复的凭据需保留 `at_precision=filesystem_write_time_recovered` 等说明,不能伪造 HTTP 头或请求时间。2026-11-09 保存的两个页面通过哈希校验;独立录用清单 URL 返回 403,失败保留,不绕过或依赖其完成采集。
2025 命令同样使用目录锁、临时目录、最后写入采集报告。五个 JSONL/manifest 均保持 `partial`,记录摘要缺失数量;它不连接数据库。首次采集需独立核验全部标题/作者/栏目和 source IDs,随后做 PostgreSQL 身份预检、必要的逐条版本审核、原生导入与独立验收。若失败且没有完成报告,应先检查原始响应和错误,再使用新输出目录重跑,不重放已提交批次。代码验证、原始目录盘点和采集成功均不代表已导入生产或最终去重完成。
## 来源与范围
[官方论文集介绍](https://www.ifaamas.org/Proceedings/aamas2012/)及其 Full Papers、Extended Abstracts、Demos、Doctorial Consortium Abstracts 四个栏目分别列出 136、146、28、24 条,共 334 条。旧 DBLP 候选卷的 310 条不能替代这四类官方目录,也不能全部标记为主会全文。栏目数量来自 2026-11-09 冻结的原始页面;页面结构或数量发生变化时,采集器停止,留待重新核验。
采集标题、逐篇作者、页面实际提供的摘要、官方条目 ID、目录锚点和 PDF 地址。**只保存 PDF 链接,不下载 PDF。** 全文页面的摘要藏在标题链接内部的 `abstract` 元素,作者链接里的 “has N papers” 是界面徽标,二者不会混入标题或姓名。未提供的摘要保存为 `null`,不会用标题、说明文字或其他论文填充。
官方介绍说明了 general、robotics、virtual agents、innovative applications 等投稿轨道,但逐篇目录的主题会话不能独立证明某篇论文的投稿轨道。因此全文和扩展摘要均使用 `proceedings_unspecified`;演示、博士栏目使用 `demo`、`doctoral`。发表形式保留在 `metadata.publication_format` 和 `aamas_proceedings:publication_format` 标签中,会话原文单独保存。它们不是研究类型标注,也不会写入自定义 annotation。
目录条目仅提供发表录用证据,不是原始决定信。`observed_at` 是页面采集时间,不能当作 2012 年的决定日期;官方介绍中的投稿、撤稿、录用汇总数不能识别逐篇拒稿或撤稿,采集器不会由总量相减生成这些决定。
## 运行
在已确认的远程运行环境中,使用 HTTP 缓存采集:
```sh
paper-atlas collect aamas-proceedings --year 2012 \
--cache /data/paper-index/cache/aamas \
--out /data/paper-index/collected/aamas-2012
```
也可使用已冻结的官方页面:
```sh
paper-atlas collect aamas-proceedings --year 2012 \
--snapshots /data/paper-index/collected/historical-next-source-evidence-v1 \
--out /data/paper-index/collected/aamas-2012
```
快照目录需含 `receipts.json`,其中每个响应包含原始 `url`、HTTP `status`、相对 `file`、`bytes`、`sha256` 和 `finished_at`。要求五个准确官方 URL 均唯一匹配且返回 200;验证全部字节数和哈希后才解析,不接受目录穿越或绝对文件路径。快照中其他来源的响应不会被当成 AAMAS 页面。
输出为按数据库 track 分组的三个 JSONL 文件和对应 manifest、五份原始 HTML 证据,以及最后写出的 `aamas-proceedings-2012.json` 采集报告。锁文件避免同一输出目录并发运行。manifest 始终为 `partial`,记录发表形式数量和缺失摘要数;**采集命令不连接数据库,也不导入。**
## 复核与恢复
每条记录携带对应官方页面的 URL、采集时间、字节数、SHA-256 与输出内证据路径。重新运行前先检查已有采集报告和文件哈希;采集中断不等于主库已导入,报告缺失或哈希不一致的目录不能直接作为导入队列。输出中的原始证据应随文件一起保存。
生产导入前仍需单独与主库核验稳定标识、标题、作者及已有投稿,按证据补充 DOI/arXiv 等标识;目录页上的作者计数徽标和会话 ID 不能作为这些标识。需要另建经过核验的增量清单、保护旧历史,完成独立验收后才能计入覆盖。跨来源同一论文不得仅因同题就新增或合并投稿。
本功能的代码、测试或采集成功均不代表 AAMAS 2012 已导入生产,也不证明拒稿全集完整。全库最终去重、摘要和来源覆盖验收属于独立后续阶段。
+2
View File
@@ -1,5 +1,7 @@
# 会议覆盖与公开决定核查
**当前工作范围:2020 年起主动补漏。** 2000 年起已发现资料保留,恢复工作从 [补漏范围与恢复清单(2026-11-10)](backfill-handoff-2026-11-10.md) 开始;下文旧日期快照不改写,较早年份排除不算补齐。
这是 2026-11-08 的首份核查基线,**完整逐年政策普查和最终去重仍未完成**。收录 43 个目标,不能把“有记录”当作“完整覆盖”。结构化清单见 [inventory-2026-11-08.json](inventory-2026-11-08.json)。清单中的 `policy_checks` 均限定所列年份和轨道;空数组表示尚未核实,绝不表示没有公开拒稿。
## 快照与复核
@@ -0,0 +1,52 @@
# ACL 早期年份:会议存在、出版目录与逐篇资料分开核验
固定普查中 ACL 1963–1978 仍有年份缺口。不能因为会议年表连续,就假设每年都有独立的正式论文集;也不能因为当前目录始于 1979,就断言此前没有摘要、通知或其他公开资料。本次不修改固定的 1,266 个目标年份基准。
## 1978 年:限定 T78-1 的来源关系
[同期年会通知](https://aclanthology.org/J77-4001.pdf)的 PDF 第 9–10 页(印刷页码 9–10)明确说明,1978 年 ACL 年会采用第二届 Theoretical Issues in Natural Language Processing 的形式,地点为 Urbana/Champaign,日期为 7 月 25–27 日。通知中的 ACM 联合赞助确认仍处于待确认状态,不能将该预告改写为已完成的行政决定。
[1982 年 Donald E. Walker 的历史回顾](https://aclanthology.org/www.mt-archive.info/70/ACL-1982-Walker-1.pdf)第 3 页(印刷页码 91)的年会表及缩写说明,独立对应了 1978 年的时间、地点与 TNLP/TINLAP-2 关系。两份扫描件相关页已渲染并逐页目视核对,不仅依靠 OCR。
[官方卷目录](https://aclanthology.org/volumes/T78-1/)与固定 XML `T78.xml` 保留原系列名 TINLAP、完整卷名、1978 年及编辑 David L. Waltz。适配仅接受精确的 collection `T78`、volume `1` 和上述字段组合;关键元数据变化会拒绝处理,等待重新核查。
- 规范事件名使用 `ACL+TINLAP`,原系列名 `TINLAP`、原始 XML 的 `venue_codes: [tinlap]` 及证据均保留。
- ACL/TINLAP 别名只挂在这一规范事件系列上,不把所有 TINLAP 年份都算成 ACL。T75、T87 及 T78 其他卷没有此例外。
- 通知描述了邀请与提交论文两种参与方式,XML 没有逐篇角色。统一暂存为 `proceedings_unspecified`,不猜测每篇属于 invited、main 或 workshop。
- XML 共 36 条条目,其中 T78-1036 是无作者的场次讨论问题列表。精确识别并在 manifest 的 `excluded_materials` 保留其标识、标题与原因;其余 35 条作为具名贡献整理。该排除不推广到所有无作者条目,已识别条目发生变化也须重新核查。
- 全部 35 条 XML 均无摘要。出版目录中的 accepted 是来源观察;通知提及录用/不录用通知时间,不提供任何可识别的拒稿决定,不能制造 rejected 或 withdrawn。
采集器仅生成元数据文件,不写业务数据库。真实来源逐条验收、PostgreSQL 身份预检、串行导入与独立验收仍是后续门槛。35 条来源不等于 35 个新增论文实体,更不等于完成全库去重。
## 1965 年:替代关系不改写为别名
Walker 回顾第 1 页(印刷页码 89)说明,1965 年 ICCL(后来称 COLING)替代了当年的年会。[官方 C65-1 目录](https://aclanthology.org/volumes/C65-1/)及 XML 仍以 COLING 标记该卷。这是限定年份的历史关系,不证明 COLING 与 ACL 是两个可全局互换的名字。
本次保留 C65 原有 `COLING` 归属,不添加 ACL 全局别名、不重放已导入记录、不将某一年的日历关系推广到整个系列,也不据此悄悄减少固定普查缺口。其他 1963–1977 年资料仍须逐年核查;本次不能证明这些年份整体属于非归档例外。
## 固定来源与恢复
服务器证据目录:`/data/paper-index/collected/acl-early-source-evidence-v1`。五份官方响应均 HTTP 200,另有两份来自既有官方 XML checkout 的文件,已与 Git 对象逐字节核对。未下载论文全文集合;两份 PDF 是本次历史/行政取证资料,不进入论文全文库。
| 文件 | SHA256 |
|---|---|
| `acl-venue.html` | `c60f64cb4d72e677aeb84a5844cde28a8d50579e1995f08e8c7801eed1e3fac5` |
| `acl-history-1982.pdf` | `35c21bd4983975bcaccbb94bd8ab91c9019fc190feec1791255407cf18a7e49a` |
| `acl-1978-notice.pdf` | `14b5deb471c55afb07d789529b95d0462a2a1a77df3dfba7fa1f5d3ab75eb411` |
| `tinlap1978-volume.html` | `7e20d709f72f415819e4d17e3674b19dc7d871eabcf0cf8b4935f0392d775729` |
| `coling1965-volume.html` | `0d2f2f49f2a311f9c0b80a8832f6476eb32f94f35abf36f6d2f2e10063ac880f` |
| `T78.xml` | `f712f94cff0cba05f786b7ac31dad2ea9408fde2a17a04531130df623f57dd84` |
| `C65.xml` | `572791610e20b7c6045d984d968e73a18a144bcb5c3da64112001c781826808e` |
XML commit:`a7be623d01c48c61405dec31c871121cdbed5a01`。测试 fixture `tests/fixtures/acl-t78.xml` 是该原始文件,不是手工模拟论文;不包含 PDF。PDF 哈希建立完整性,不单独证明语义,语义依据是上述具体页面。
在服务器使用既有命令,固定 checkout 并输出到新目录:
```sh
paper-atlas collect acl \
--checkout /data/paper-index/raw/metadata-checkouts/acl-anthology \
--collection T78 \
--out /data/paper-index/collected/acl-t78-source-v1
```
该命令会校验/更新官方 metadata checkout,实际 commit 必须以生成清单为准;严格重现本次取证时,使用已冻结的 XML 和上述 commit,不把更新后的内容冒充旧快照。不要将无 collection 限制的全部历史输出直接作为增量队列。采集完成不代表导入完成,恢复前先检查当前单元和报告,不因日志安静重启已执行的写入。
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,52 @@
# 补漏范围与恢复清单(2026-11-10)
用户确定:**后续主动补漏从 2020 年起**。2000 年起已经发现的名单、缺口、资料组合和身份核查记录保留在项目中,供下次接着做;已有旧数据继续保留。停止补采的年份标为范围外或延期,不算作已补齐。
本页是下一次工作的入口,结构化记录见 [backfill-handoff-2026-11-10.json](backfill-handoff-2026-11-10.json)。它保存 43 个会议的 2000–2026 已观察年份、缺年、官方目录文件入口、近期待办与恢复证据。旧的日期快照不改写,也不能当实时数据库。
## 当前范围和年份差集
依据生产只读快照 `2026-11-09 16:26:41.688509+00:00`,原全历史口径为 1266 个目标年份槽位:1208 已观察、43 个历史缺年、14 个 2026 缺年,以及 1 个 SysML 2018 非存档例外。原 135 个历史缺口中有 92 个通过数据补入解决;其余 43 个现在停止补采,不能因此宣称全部解决。
- **2020–2025**:没有整年完全缺记录的目标槽位,但逐篇、track、摘要、公开负面决定仍不完整。
- **2026**:NeurIPS、ECCV、RecSys、CIKM、ICDM、SDM、IROS、RSS、CoRL、ICAPS、AAMAS、ICASSP、ISMIR、ACM MM,共 14 个槽位。须逐个核查是否已经公开出版;不能将尚未发布的资料说成漏采。
- **2000–2019 延期线索**:WWW 2000、ICAPS 2001 两个整年缺口;官方历届记录分别见 [WWW](www-editions-2026-11-08.json)、[ICAPS](icaps-editions-2026-11-08.json)。SysML 2018 是非存档首届,见 [MLSys](mlsys-editions-2026-11-08.json),不能制造正式出版记录。
- **2000 年前**:剩余 41 个历史缺年停止补采。旧来源文件继续保留作证据,不再作为当前工作队列。
以上只判断目标会议年份是否至少有一条投稿记录,绝不等于目录完整、独立论文数量或最终去重验收。
## 近期资料与恢复位置
服务器采集根目录为 `/data/paper-index/collected`,本地证据备份为 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index`。这些目录提供大体积原始文件,仓库 JSON 提供路径、状态和可用哈希;恢复前先验证文件存在、哈希和生产实际状态。
| 资料 | 当前状态 | 下一步与证据入口 |
| --- | --- | --- |
| AAMAS 2026 | 官方目录 643 条,3 个 keynote 排除;640 个非 keynote 中含 1 个明确撤稿。尚未导入 | `aamas2026-primary-evidence-v1/` 保存官方 HTML、回执、逐项目录、审计和校验清单。八个标题差异/撤稿前缀条目需核查。先扩展正式 collector 并远程验证,再做新 PostgreSQL 身份预检。 |
| CVPR 2026 Findings | 941 条摘要已核验,尚未导入 | `cvf-2026-verified/recovery-audit.json`;原 404 与 RVLF/FVLF 路径更正证据保留。先核对 manifests、track 和实体关系。 |
| AAMAS 2025 | 476 个非 keynote 来源中,240 已验收、236 待身份核查 | `aamas2025-import225-v1/` 与 `aamas2025-import15-v1/` 是已消费批次,禁止重放。15 条最近验收报告哈希见 JSON。余下资料从当前身份候选与新只读基线继续。 |
| RSS 2025 | 163 条目录中 162 已验收,`2025/p117` 待核查 | 保留 spinal/RMCC 关系问题;查看工作状态键 `rss2025_held7_primary_evidence_v1`,不能重跑已消费的 6/88/54/14 批次。 |
| 近期公开负面决定首批 | 1625 条已验收:1624 rejected、1 withdrawn | `public-negative-postgresql-import-postcheck-v1.json`;仅该批次验收,不是公开拒稿全集,不要重放。 |
AAMAS 2026 官方目录分区:research 338、extended abstracts 185、AAAI 19、Blue Sky 17、doctoral 34、demo 44、JAAMAS 3,另有 keynote 3。JAAMAS 是期刊展示关系,不能凭目录再造一次独立审稿录用。官方撤稿条目在 research 分区,不能把全目录都标为当前 accepted。
AAAI 转投政策有年份冲突:[详细征稿页](https://cyprusconferences.org/aamas2026/call-for-papers-aaai-track/) 指 AAAI-26 rejected,[总览](https://cyprusconferences.org/aamas2026/calls/) 写 AAAI-25。保留两份证据;19 个 AAAI track 条目不能直接转成 19 条有原始 note 的拒稿决定。原投稿 ID、逐篇决定和版本关系仍需核实。
## 2000–2019 已发现但延期的身份与名单工作
- **AAMAS 2012**:官方 334 条,319 已独立验收、15 个身份条目保留待查。验收报告 `aamas2012-import-v1/postgresql-postcheck-v1.json`,哈希在 JSON;已验收 319 条及被替代的 316 条计划均不得重放。来源布局与用法见 [AAMAS 适配说明](../aamas-proceedings.md)。
- **UAI 2008**:官方 programme 与归档资料确认了四篇独立论文,但部分原始 DBLP 键的标题、作者、页码、出版链接对应仍有冲突。JSON 保留四组候选、arXiv ID、官方题名及归档报告哈希。之前仅根据页码推断题名/作者错误的结论已撤回,未应用纠正;恢复时不得据此覆盖或合并。
- **其他名单**:本目录全部 `*-editions-2026-11-08.json` 保留各会议官方年份及来源;[历史缺口](historical-gaps-2026-11-09.md)、[来源候选](historical-source-candidates-2026-11-09.json)、[来源范围](historical-source-scope-2026-11-09.md)、[主系列](historical-primary-series-2026-11-09.md)、[前身会议](historical-predecessors-2026-11-09.md)、[ICLR 档案](iclr-archive-2014-2016.md)继续留档。这些旧报告的“缺失”可能已被后续导入解决,恢复时必须对照当前状态,不能直接当可执行队列。
## 公开拒稿资料不能遗漏
结构化清单保留最近已查的政策 URL、限定年份和可用性结论。优先继续 UAI 2024–2026 的 opt-in 公开拒稿来源、EMNLP 2023 的公开集合范围及其他目标的逐年核查;平台使用 OpenReview 不能证明 rejected 公开。访问受限如实记录,不绕过 Challenge。
保留 rejected、desk rejected、withdrawn、retracted、resubmit 的区别;评审建议不等于最终决定。COLM 2026 的不公开政策只适用于已核实年份,不能推广到其他会议。PaperCopilot 与原 OpenReview、镜像间有重叠,候选计数不能相加。论文先被 A 拒绝再被 B 录用时,两次投稿、决定、来源与证据全部保留。
## 下次启动顺序
1. 读取本页和 JSON,再核实服务器工作状态、明确的 systemd invocation 与终态报告。权威工作状态 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/work-state.json` 中部分旧字段会过期;以最新验收和实际查询为准。
2. 当前主动范围为 2020 起,2000–2019 只恢复线索,不自动启动其采集。先检查新来源是否已导入,验收过的队列禁止重放。
3. 使用现有 `foundry-postgres` 的 `foundry_data.research`。当前已验证 writer 为提交 `714f745bd5c3fbc742be1efae0b31ab37a55e21f` 的固定 release;恢复时重新确认,不使用旧 `/data/paper-index/code` 替代。服务配置仅由服务器受保护文件读取,不复制密码。
4. 对待导入资料做新只读身份预检,处理稳定标识、短标题、作者/版本差异、track 与跨会历史,再生成小范围增量队列。每批独立验收,SQLite 只读备份保持保护。
5. 编译、测试或 wheel 构建仅在已确认 `ces-control:host` runner 执行。代码按工作分支交付 PR,等待用户手动合并。范围内导入结束后仍须全库去重、近似标题/作者冲突、来源-年份-track、摘要、约束、实际全文查询验收;不能用单批导入或 CI 成功替代。
@@ -0,0 +1,53 @@
# EWSL 与规划会议前身的来源范围
本项扩展 DBLP 元数据适配:核对具体历史卷,保留 EWSL、EPS、ECP、EWSP 原始系列名称,并使用已有会议别名关系关联 ECML-PKDD 或 ICAPS。它不把历史论文的原始会议名称替换成后来的会议,也不从卷的存在推断有逐篇数据。
## 证据与已核验边界
[ECML-PKDD 2011 官方历史页](https://ecmlpkdd2011.org/pastConferences.php)与 [ICAPS 官方系列历史](https://www.icaps-conference.org/conference-series/)分别提供系列前身关系。2026-11-09 冻结页面 SHA-256 为:
- ECML-PKDD:`c3ecdb032aeacf6a365aace5457773827d1405872ad9bc6be6fc05851d5e68d6`。
- ICAPS:`de1d217660cb819825f58112cdaf774d0f68da6ed87e3e90a59055e52f152081`。
与固定 DBLP 2026-11-01 快照的完整 proceedings 目录对照后,仅对下列具体卷识别前身主系列:
| 原始系列 | 事件年 | DBLP 卷 key | 卷出版年 | 原始逐篇书目计数 |
|---|---:|---|---:|---:|
| EWSL | 1987 | `conf/ecml/1987` | 1987 | 16 |
| EWSL | 1988 | `conf/ecml/1988` | 1988 | 21 |
| EWSL | 1989 | `conf/ecml/1989` | 1989 | 0 |
| EWSL | 1991 | `conf/ecml/1991` | 1991 | 44 |
| EPS | 1990 | `conf/aips/1990` | 1990 | 48 |
| ECP | 1997 | `conf/ecp/1997` | 1997 | 35 |
| ECP | 1999 | `conf/ecp/1999` | 2000 | 28 |
| EWSP | 1991 | `conf/ewsp/1991` | 1991 | 10 |
合计 202 条是原始 `inproceedings` 书目计数,尚未经过前言等材料排除、身份核查与生产验收,不能作为新论文数量。EWSL 1989 在当前 catalogue 中有卷信息,但未匹配到 `inproceedings` 条目;其他记录类型和其他来源仍待核查。代码支持它的范围规则不表示已补上这一年的论文。
核验同时要求来源 family、booktitle 基名、空轨道后缀、完整 proceedings key、完整标题及卷出版年与固定证据一致。邻近年份、标题附加内容、`w` 卷后缀或工作坊 booktitle 均不能继承此范围。ECP、EWSP 及 EPS 的未知卷保持待审。原有 AIPS、ICML/IROS、AISTATS/COLT 规则保持各自边界。
EWSL 的主轨道仍属于 `EWSL`,通过 `ECML-PKDD` 别名纳入家族查询;EPS、ECP、EWSP 保留原名,通过 `ICAPS` 别名关联。`main` 在此表示该原始系列的主轨道。原始 booktitle、作者、论文书目年、卷标题、卷出版年、事件年及历史页 URL 分开保存在 metadata 中。例如 ECP 1999 的事件年 1999 与卷出版年 2000 都保留,不用其中一个覆盖另一个。
## 候选文件与完整快照
旧的窄范围 `dblp-ai-candidates.jsonl` 未纳入 `ecp`、`ewsp` family,但完整 catalogue 的 proceedings 与 volume_counts 中有它们。新的 catalogue 导出会纳入这两个 family。
选择 ICAPS 且截止年达到 EWSP 1991 或 ECP 1997 时,若候选 manifest 缺少相应 family,`collect dblp --candidates` 会明确报错,要求使用原始完整 `--archive`。错误发生在写输出之前,不会把遗漏当作成功采集。选择其他会议或截止 EPS 1990 的旧候选采集不受这项新家族检查影响。
在已确认的远程运行环境中使用完整快照:
```sh
paper-atlas collect dblp \
--catalogue /data/paper-index/collected/dblp/dblp-catalogue.sqlite \
--archive /path/to/pinned-dblp.xml.gz --dtd /path/to/pinned-dblp.dtd \
--venue ECML-PKDD --venue ICAPS --through-year 2001 \
--out /data/paper-index/collected/historical-predecessors
```
完整快照与 DTD 均沿用现有固定身份校验。该命令也会输出已覆盖的其他年度;生产导入前须单独筛选目标卷与年度,检查 raw source、前言材料、稳定标识、标题、作者及现有投稿,不能直接导入整个输出目录。不会下载论文 PDF。
## 验证和当前限制
回归检查使用独立保存的八卷原始目录 fixture,合成论文只用于隔离测试;不代表真实逐篇来源。检查原始系列/别名、完整标题和年份边界、旧候选文件显式失败、扩展 family 的 catalogue 导出、原生导入后的元数据与关系。
代码、测试、采集和生产导入是不同阶段。新范围不表示本批数据已采集或已导入,更不表示公开拒稿、摘要或最终去重完整。原始书目仅提供发表条目证据,不是原始决定信;没有记录不能推断为拒稿或未举办。后续各批需保留独立验收报告,保护既有跨会投稿、决定与标签历史。
@@ -0,0 +1,33 @@
# ICML 与 IROS 早期主系列的来源范围
本轮修正采集阶段的届次识别,不修改任何现有生产投稿轨道或历史。正式主系列曾以Workshop命名,不等于附属workshop;也不能反过来把所有旧workshop升为主会。
## 已核实的精确范围
| 会议 | 论文集键 | 依据与行为 |
| --- | --- | --- |
| ICML | `conf/icml/1988`、`1989`、`1990`、`1991`、`1992` | [官方历史](https://icml.cc/Conferences/2008/past_icmls.shtml.html)将其列为第5–9届,年份与举办地对应固定目录中的完整标题。1989/1991/1992的Workshop名称不再导致主系列被默认排除;1988/1990保留main并增加来源范围证据 |
| IROS | `conf/iros/1988`、`1989`、`1990` | [IEEE RAS官方历届目录](https://www.ieee-ras.org/conferences-workshops/financially-co-sponsored/iros/iros-past-and-future-venues/)列出三届与论文集链接,地点及日期对应固定目录标题;这些是IROS主系列 |
规则同时要求family、booktitle基础别名(ML或IROS)、空轨道后缀、完整精确论文集键、匹配的事件年份及固定完整标题。仍执行原有书目年份核查:与事件年不同且标题无明确事件年时拒绝推断。ML91原始标题没有四位年份,使用精确1991卷键、完整标题、逐篇书目年和官方第8届记录,不从两位标题代码猜世纪。
`1991w`/`1988w`等卷键、booktitle中的Workshops后缀、不同届次/地点/完整标题、未列入的年份均不会获得上述主系列豁免。AISTATS/COLT既有精确规则保留;真正的附属workshop仍受默认不采集workshop的边界约束。
## 原始字段、决定与后续导入
采集器仍保存原始proceedings标题(包括Workshop)、书目年、原始作者名、完整来源书目字段及稳定标识;新增 `track_scope_note` 与官方 `track_scope_evidence_url`。接受状态仍只源自该论文的inproceedings书目证据;不从历史目录生成占位论文或拒稿,也不重新导入已经存在的记录。
新规则的输出必须重新做原始来源/标题/作者/稳定标识与现有库冲突预检,再按增量导入。此前被隔离的ICML1988/1990记录不能仅凭代码规则而直接解除全部冲突。当前正在运行的历史批次使用原固定版本和文件,不能在运行中替换其源码或输入。
## 验证与证据
新增回归覆盖:ICML三届Workshop名称、IROS最初三届、ICML原Conference名称的两届、错误卷键/后缀/标题/未知年份,以及归一化记录经过原生文件导入后保留轨道证据。构建与测试仅在已授权的ces-control远程runner执行,本机仅做语法和文本/JSON一致性检查;结果以相应commit的CI回执为准。
官方HTML在工作证据目录 `paper-index/historical-primary-series-evidence-v1/`,附响应状态、字节数、时间与SHA256;仅取元数据网页,没有下载论文全文。
| 来源 | UTC采样时间 | HTTP | 字节数 | SHA256 |
| --- | --- | ---: | ---: | --- |
| [icml-history](https://icml.cc/Conferences/2008/past_icmls.shtml.html) | 2026-11-08T18:36:43.043135+00:00 | 200 | 11993 | `80cce886fe985551339a52b13c4077acf827e8fcc01555f854dc24077bdebdf0` |
| [iros-history](https://www.ieee-ras.org/conferences-workshops/financially-co-sponsored/iros/iros-past-and-future-venues/) | 2026-11-08T18:36:45.687171+00:00 | 200 | 466059 | `fdb4594621e68567e1099f5c88aae53de835b7b8146025148c3b1968df990697` |
本次不关闭历史年度缺口,也不替代全部导入后的全库去重、来源/年度/track覆盖、摘要和实际查询验收。
+44
View File
@@ -0,0 +1,44 @@
# ICLR 2014–2016 官方历史录用档案
此适配器是长期维护工具的一部分,按需运行。只识别已经独立核对的三种页面布局;后续页面发生变化必须重新核查,不能放松计数或轨道边界以使采集成功。
| 官方页面 | 页面链接次数 | 整理后的不同记录 |
|---|---:|---:|
| [2014 Conference Proceedings](https://iclr.cc/archive/2014/conference-proceedings/) | 35 | main 35 |
| [2015 accepted-main](https://iclr.cc/archive/www/doku.php%3Fid=iclr2015:accepted-main.html) | 116 | main 31、workshop 74 |
| [2016 accepted-main](https://iclr.cc/archive/www/doku.php%3Fid=iclr2016:accepted-main.html) | 80 | main 80 |
2015 的 11 篇 oral 全部再次出现在 poster 段落中。相同年份、轨道及 arXiv 标识的重复展示只生成一条来源记录,但每次展示的标题、作者显示字符串、链接、版本和段落均保留在 `source_variants`。两处标题或解析作者冲突时停止采集、等待人工核查。不同年份或不同轨道不合并投稿。
2014 的作者以分号列出,一条有末尾分号,一条标题和作者斜体块不在普通段落中,部分作者段落拆成多个格式片段;必须读取完整作者段落或独立斜体块,不能只读首个格式片段。2016 的代码链接不属于作者。保留原文,解析结果不替代来源证据。oral、poster 作为 `iclr_archive:presentation` 标签,不改成独立投稿轨道。arXiv 版本留在来源变体中,稳定标识使用去版本号的编号。
## 来源快照与运行
2026-11-09 已冻结的官方响应 SHA256:
- 2014:`a33858176fbffae0066ed3c10310f7dd98a66d12a36334b6339ef1f915ef07af`
- 2015:`131491663742cb084662a4804009b8318e0fa7bc184d4a9044d16dd28f05b888`
- 2016:`7320b8652431793b6ea0743f871ee446bf08b6361ccd4e421b6b53449e71f4f6`
在服务器运行,输出新目录,不下载论文 PDF:
```sh
paper-atlas collect iclr-archive --year 2015 \
--snapshots /data/paper-index/collected/historical-next-source-evidence-v1 \
--out /data/paper-index/collected/iclr-archive-source-v1/2015
# 或重新下载官方 HTML,并保留 HTTP 缓存与收据
paper-atlas collect iclr-archive --year 2016 \
--cache /data/paper-index/cache/iclr-archive \
--out /data/paper-index/collected/iclr-archive-source-v1/2016
```
冻结输入目录必须包含 `receipts.json`,每项有精确 `url`、HTTP `status: 200`、相对 `file`、`bytes`、`sha256` 和带时区的 `finished_at`。读取时重新核对尺寸、哈希和路径;拒绝越界及指向目录外的软链接。已完成输出不可覆盖,应检查报告后另建明确版本目录。解析或计数失败时不生成完成报告,也不导入数据库。
输出分轨道 JSONL、manifest、原始 HTML 证据和年度采集报告。仅支持这三年已审查的页面。2014 workshop、其他未列轨道、逐篇摘要及公开拒稿范围尚未通过这些页面核验,不能据此声称完整。缺席不代表 rejected、withdrawn 或从未被录用。
## 导入前后的边界
这三页合计 220 条不同来源记录,不等于新增 220 篇论文;同一工作可能已存在于 arXiv 或其他会议。先核验来源字段与原始页面,再执行 PostgreSQL 只读预检,比较稳定标识、标题、作者和原始来源。标题/作者差异、版本关联及冲突必须保留证据并明确评估;不能仅靠标题猜测身份,也不能覆盖原有摘要。
采集器不连接数据库。正式导入使用既有 `import records`,必须有新的生产保护基线和独立验收;全部导入结束后仍需全库去重和历史覆盖审计。来自官方录用清单的 accepted 是清单观察,不是原始审稿决定签名或决定日期。
+2
View File
@@ -13,6 +13,7 @@ COMMANDS = {
('download',): ('paper_atlas.io.snapshots', {}, '下载并校验固定版本快照'),
('collect','pmlr'): ('paper_atlas.collectors.pmlr', {}, 'PMLR 官方元数据'),
('collect','acl'): ('paper_atlas.collectors.acl', {}, 'ACL Anthology XML'),
('collect','iclr-archive'): ('paper_atlas.collectors.iclr_archive', {}, '核对ICLR 2014–2016官方历史录用档案'),
('collect','iclr-program'): ('paper_atlas.collectors.iclr', {}, 'ICLR 官方公开程序'),
('collect','iclr-snapshot'): ('paper_atlas.collectors.openreview', {}, 'ICLR 公开投稿与决定快照'),
('collect','public-openreview'): ('paper_atlas.collectors.public_openreview', {}, '核验其他会议的公开拒稿及撤稿标签'),
@@ -23,6 +24,7 @@ COMMANDS = {
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
('collect','www-archive'): ('paper_atlas.collectors.www_archive', {}, '核验早期WWW官方档案并分开待核查条目'),
('collect','dblp'): ('paper_atlas.collectors.dblp_records', {}, '核对并整理 DBLP 历史发表记录'),
('catalogue','dblp'): ('paper_atlas.collectors.dblp', {}, '盘点 DBLP 历史会议目录'),
+213
View File
@@ -0,0 +1,213 @@
"""Official AAMAS proceedings metadata, with publication format separate from track."""
import argparse
from collections import Counter, defaultdict
import fcntl
import hashlib
import json
from pathlib import Path
import re
import tempfile
from urllib.parse import urljoin, urlsplit
from paper_atlas.db.submissions import timestamp
from paper_atlas.io.html import Document, Node, clean
from paper_atlas.io.http import fetch
SOURCE = 'aamas_proceedings'
ROOT = 'https://www.ifaamas.org/Proceedings/aamas2012/'
# Counts independently checked against the frozen publisher sections. This
# adapter deliberately refuses other editions/layouts until they are reviewed.
SECTIONS = {
'fullpapers': ('Full Papers', 'full_paper', 'proceedings_unspecified', 136),
'extabstracts': ('Extended Abstracts', 'extended_abstract', 'proceedings_unspecified', 146),
'demos': ('Demos', 'demo', 'demo', 28),
'doctorals': ('Doctorial Consortium Abstracts', 'doctoral_abstract', 'doctoral', 24),
}
def classes(node):
return set(node.attrs.get('class', '').split())
def text_without(node, omitted):
if node.tag in ('script', 'style') or classes(node) & omitted:
return ''
parts = [text_without(child, omitted) if isinstance(child, Node) else child
for child in node.children]
value = ''.join(parts)
return ' '+value+' ' if node.tag in ('p', 'div', 'section', 'li', 'br') else value
def one(nodes, description):
if len(nodes) != 1:
raise ValueError('Expected one '+description)
return nodes[0]
def parse_section(html, section, observed_at, evidence):
heading, publication_format, track, _ = SECTIONS[section]
document = Document(html)
titles = [clean(n.text()) for n in document.root.walk() if n.tag == 'title']
if titles != [heading+' - AAMAS 2012']:
raise ValueError('Unexpected proceedings edition or section title')
headings = [clean(n.text()) for n in document.root.walk() if n.tag == 'h1']
if headings != [heading]:
raise ValueError('Unexpected proceedings section heading')
url = ROOT+'resources/'+section+'.html'
records = []
seen = set()
session = None
for node in document.root.walk():
if node.tag == 'h2' and 'section' in classes(node):
session = {'id': node.attrs.get('id'), 'title': clean(node.text())}
if node.tag != 'div' or 'paper' not in classes(node):
continue
paper_id = node.attrs.get('id', '')
if not re.fullmatch(r'[A-Za-z0-9]+_[0-9]+', paper_id) or paper_id in seen:
raise ValueError('Missing or repeated publisher paper ID')
seen.add(paper_id)
title_node = one([n for n in node.walk() if n.tag == 'span' and 'title' in classes(n)], 'paper title')
link = one([n for n in title_node.children if isinstance(n, Node) and n.tag == 'a'], 'paper link')
paper_url = urljoin(url, link.attrs.get('href', ''))
if paper_url != ROOT+'papers/'+paper_id+'.pdf':
raise ValueError('Paper link differs from publisher ID or official host')
title = clean(text_without(link, {'abstract'}))
abstract_nodes = [n for n in link.walk() if n.tag == 'span' and 'abstract' in classes(n)]
if len(abstract_nodes) > 1:
raise ValueError('Multiple abstracts for one paper')
abstract = (clean(abstract_nodes[0].text()) or None) if abstract_nodes else None
author_node = one([n for n in node.walk() if n.tag == 'span' and 'authors' in classes(n)], 'author list')
author_display = clean(text_without(author_node, {'amount'}))
authors = [clean(name) for name in author_display.split(',')]
if not title or not author_display or any(not name for name in authors):
raise ValueError('Empty paper title or author')
tags = [{'namespace': 'aamas_proceedings:publication_format', 'value': publication_format}]
if session:
tags.append({'namespace': 'aamas_proceedings:session', 'value': session['title']})
records.append({
'source': SOURCE, 'external_id': '2012/'+paper_id,
'url': url+'#'+paper_id, 'title': title, 'abstract': abstract,
'authors': authors, 'identifiers': {}, 'venue': 'AAMAS', 'venue_aliases': [],
'year': 2012, 'track': track, 'status': 'accepted',
'observed_at': observed_at,
'raw_decision': 'listed in official AAMAS 2012 proceedings '+heading,
'metadata': {'publisher_paper_id': paper_id, 'paper_url': paper_url,
'section_url': url, 'section_heading': heading,
'publication_format': publication_format, 'session': session,
'authors_display': author_display, 'source_evidence': evidence,
'publication_evidence': 'official published proceedings entry; not an original decision note',
'track_scope_note': ('Full papers and extended abstracts include general and special submission tracks; '
'session topics do not establish an individual submission track.'
if track == 'proceedings_unspecified' else 'Explicit publisher section.'),
'decision_date_precision': 'not_supplied'},
'tags': tags})
if not records:
raise ValueError('No publisher paper entries found')
return records
def read_pages(cache=None, snapshots=None):
if bool(cache) == bool(snapshots):
raise ValueError('Choose exactly one HTTP cache or frozen snapshot directory')
urls = {'introduction': ROOT, **{key: ROOT+'resources/'+key+'.html' for key in SECTIONS}}
pages = {}
receipts = []
frozen = json.loads((Path(snapshots)/'receipts.json').read_text())['receipts'] if snapshots else []
for key, url in urls.items():
if snapshots:
receipt = one([r for r in frozen if r['url'] == url], 'frozen response receipt')
filename = Path(receipt['file'])
if filename.is_absolute() or '..' in filename.parts or receipt.get('status') != 200:
raise ValueError('Unsafe or unsuccessful frozen response')
data = (Path(snapshots)/filename).read_bytes()
if len(data) > 8*1024**2 or hashlib.sha256(data).hexdigest() != receipt['sha256'] or len(data) != receipt['bytes']:
raise ValueError('Frozen publisher response checksum or size mismatch')
observed = receipt['finished_at']
else:
data = fetch(url, cache, 8*1024**2, delay=1)
receipt = json.loads((Path(cache)/hashlib.sha256(url.encode()).hexdigest()).with_suffix('.json').read_text())
observed = receipt['retrieved_at']
evidence = {'url': url, 'sha256': hashlib.sha256(data).hexdigest(), 'bytes': len(data),
'file': 'evidence/'+hashlib.sha256(data).hexdigest()+'.html', 'observed_at': observed,
'http_status': 200, 'response_receipt': receipt}
pages[key] = (data, evidence)
receipts.append(evidence)
return pages, receipts
def _collect(year, out, *, cache=None, snapshots=None):
if year == 2025:
from paper_atlas.collectors.aamas2025 import collect as collect_2025
return collect_2025(out, cache=cache, snapshots=snapshots)
if year != 2012:
raise ValueError('Only the independently reviewed AAMAS 2012 layout is supported')
pages, receipts = read_pages(cache, snapshots)
intro = Document(pages['introduction'][0].decode('utf-8'))
if [clean(n.text()) for n in intro.root.walk() if n.tag == 'title'] != ['AAMAS 2012']:
raise ValueError('Unexpected proceedings introduction')
records = []
for section, (_, _, _, expected) in SECTIONS.items():
data, evidence = pages[section]
group = parse_section(data.decode('utf-8'), section, evidence['observed_at'], evidence)
if len(group) != expected:
raise ValueError('Publisher section count differs from reviewed inventory: '+section)
records.extend(group)
if len({r['external_id'] for r in records}) != len(records):
raise ValueError('Paper IDs collide across publisher sections')
grouped = defaultdict(list)
for record in records:
grouped[record['track']].append(record)
out = Path(out)
out.mkdir(parents=True, exist_ok=True)
files = []
with tempfile.TemporaryDirectory(prefix='.aamas-', dir=out) as temporary:
stage = Path(temporary)
(stage/'evidence').mkdir()
for data, evidence in pages.values():
(stage/evidence['file']).write_bytes(data)
for track, group in sorted(grouped.items()):
path = stage/f'aamas-proceedings-2012-{track}.jsonl'
path.write_text(''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in group))
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'AAMAS', 'year': 2012, 'track': track,
'count': len(group), 'file': str(out/path.name), 'file_sha256': hashlib.sha256(path.read_bytes()).hexdigest(),
'list_url': ROOT, 'status': 'partial', 'missing_abstract_count': sum(not r['abstract'] for r in group),
'publication_format_counts': dict(Counter(r['metadata']['publication_format'] for r in group)),
'note': 'Publisher sections verified; individual full/extended submission track, decision history and independent identities still need review.'}
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, ensure_ascii=False, indent=2)+'\n')
files.append(manifest)
result = {'source': SOURCE, 'year': year, 'finished_at': timestamp(), 'status': 'collected_pending_identity_review',
'records': len(records), 'files': files, 'source_receipts': receipts,
'publication_format_counts': dict(Counter(r['metadata']['publication_format'] for r in records)),
'database_written': False, 'pdfs_downloaded': False,
'note': 'Aggregate submission/withdrawal totals do not identify individual rejected or withdrawn papers.'}
(stage/'aamas-proceedings-2012.json').write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n')
# Validated files first; write the completed collection report last.
(out/'evidence').mkdir(exist_ok=True)
for path in (stage/'evidence').iterdir():
path.replace(out/'evidence'/path.name)
for path in stage.iterdir():
if path.is_file() and path.name != 'aamas-proceedings-2012.json':
path.replace(out/path.name)
(stage/'aamas-proceedings-2012.json').replace(out/'aamas-proceedings-2012.json')
return result
def collect(year, out, *, cache=None, snapshots=None):
if year not in (2012, 2025):
raise ValueError('Only independently reviewed AAMAS 2012/2025 layouts are supported')
out = Path(out)
out.mkdir(parents=True, exist_ok=True)
with (out/'aamas.collect.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
return _collect(year, out, cache=cache, snapshots=snapshots)
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog)
parser.add_argument('--year', type=int, choices=[2012, 2025], required=True)
parser.add_argument('--out', required=True)
source = parser.add_mutually_exclusive_group(required=True)
source.add_argument('--cache')
source.add_argument('--snapshots')
args = parser.parse_args(argv)
print(json.dumps(collect(args.year, args.out, cache=args.cache, snapshots=args.snapshots), ensure_ascii=False))
+183
View File
@@ -0,0 +1,183 @@
"""Reviewed AAMAS 2025 publisher TOC; HTML metadata only, never paper PDFs."""
from collections import Counter, defaultdict
import hashlib
import json
from pathlib import Path
import re
import tempfile
from paper_atlas.db.submissions import timestamp
from paper_atlas.io.html import Document, Node, clean
from paper_atlas.io.http import fetch
ROOT = 'https://www.ifaamas.org/Proceedings/aamas2025/'
URLS = {'index': ROOT, 'contents': ROOT+'forms/contents.htm'}
TITLE = 'AAMAS 2025: Proceedings of the 2025 International Conference on Autonomous Agents and Multiagent Systems'
SECTIONS = {
'1': ('Keynote Talks', 'keynote', 'keynote', 3),
'2': ('Research Paper Track', 'full_paper', 'proceedings_unspecified', 260),
'3': ('Extended Abstracts', 'extended_abstract', 'proceedings_unspecified', 148),
'4': ('Blue Sky Ideas', 'blue_sky_paper', 'blue_sky', 12),
'5': ('Doctoral Consortium', 'doctoral_abstract', 'doctoral', 25),
'6': ('Demo Track', 'demo', 'demo', 22),
'7': ('JAAMAS Track', 'journal_presentation', 'jaamas', 9),
}
COMPETITION_TITLE = '[COMP24] The Automated Negotiating Agents Competition (ANAC) 2024 Challenges and Results'
PAPER = re.compile(r'\.\./pdfs/p([0-9]+)\.pdf')
def one(items, label):
if len(items) != 1:
raise ValueError('Expected one '+label)
return items[0]
def parse_contents(html, evidence):
doc = Document(html)
if [clean(n.text()) for n in doc.root.walk() if n.tag == 'title'] != [TITLE]:
raise ValueError('Unexpected AAMAS proceedings title/year')
for paragraph in doc.root.walk():
if paragraph.tag != 'p' or any(n is not paragraph and n.tag == 'p' for n in paragraph.walk()):
continue
anchors = [n.attrs['name'] for n in paragraph.walk() if n.tag == 'a' and n.attrs.get('name') in SECTIONS]
if anchors and (len(anchors) != 1 or clean(paragraph.text()) != SECTIONS[anchors[0]][0]):
raise ValueError('Publisher section heading differs from reviewed layout')
section = None
headers = []
records, excluded, counts, seen = [], [], Counter(), set()
for node in doc.root.walk():
if node.tag == 'a' and node.attrs.get('name') in SECTIONS:
section = node.attrs['name']
headers.append(section)
if node.tag != 'p' or any(n is not node and n.tag == 'p' for n in node.walk()):
continue
links = [n for n in node.walk() if n.tag == 'a' and PAPER.fullmatch(n.attrs.get('href', ''))]
if not links:
continue
link = one(links, 'paper link per paragraph')
if section not in SECTIONS:
raise ValueError('Paper outside a reviewed publisher section')
page = int(PAPER.fullmatch(link.attrs['href'])[1])
if page in seen:
raise ValueError('Repeated publisher paper/page ID')
seen.add(page)
title = clean(link.text())
if not title:
raise ValueError('Empty paper title')
# Names are outside italicized affiliations and separated by explicit BRs.
font = one([n for n in node.walk() if n.tag == 'font' and n.attrs.get('color', '').lower() == '#222222'], 'ordered author block')
authors, affiliations = [], []
name, affiliation = '', ''
def flush():
nonlocal name, affiliation
if clean(name) or clean(affiliation):
if not clean(name):
raise ValueError('Affiliation without author')
authors.append(clean(name)); affiliations.append(clean(affiliation))
name, affiliation = '', ''
for child in font.children:
if isinstance(child, str):
name += child
elif child.tag == 'br':
flush()
elif child.tag == 'i':
affiliation += child.text()
else:
raise ValueError('Unreviewed author markup: '+child.tag)
flush()
if not authors:
raise ValueError('Empty author list')
heading, publication_format, track, _ = SECTIONS[section]
counts[section] += 1
if section == '1':
excluded.append({'publisher_page': page, 'title': title, 'authors': authors, 'reason': 'publisher keynote talk, not a research-paper submission'})
continue
note = 'Explicit publisher section; section topics are not research types.'
if track == 'proceedings_unspecified':
note = 'Published format is known; individual main/special submission track is not provided.'
if page == 3000:
if section != '6' or title != COMPETITION_TITLE:
raise ValueError('Competition report differs from reviewed publisher entry')
publication_format, track = 'competition_report', 'proceedings_unspecified'
note = 'Publisher places this competition report in Demo Track; no individual demonstration submission is inferred.'
records.append({
'source': 'aamas_proceedings', 'external_id': '2025/p'+str(page),
'url': URLS['contents']+'#'+section, 'title': title, 'abstract': None,
'authors': authors, 'identifiers': {}, 'venue': 'AAMAS', 'venue_aliases': [],
'year': 2025, 'track': track, 'status': 'accepted',
'observed_at': evidence['observed_at'],
'raw_decision': 'listed in official AAMAS 2025 published proceedings '+heading,
'tags': [{'namespace': 'aamas_proceedings:publication_format', 'value': publication_format}],
'metadata': {'publisher_page': page, 'paper_url': ROOT+'pdfs/p'+str(page)+'.pdf',
'section_url': URLS['contents'], 'section_anchor': section,
'section_heading': heading, 'publication_format': publication_format,
'author_affiliations': affiliations, 'source_evidence': evidence,
'track_scope_note': note, 'decision_date_precision': 'not_supplied',
'missing_abstract_reason': 'Publisher HTML supplies no abstract; research PDF not downloaded.',
'publication_evidence': 'Published proceedings entry, not an original decision note.'}})
if headers != list(SECTIONS) or dict(counts) != {k: v[3] for k, v in SECTIONS.items()}:
raise ValueError('Publisher section inventory differs from reviewed 2025 layout')
if len(records) != 476 or len(excluded) != 3 or sum(r['metadata']['publication_format'] == 'competition_report' for r in records) != 1:
raise ValueError('Unexpected publication partition')
return records, excluded
def read_pages(cache, snapshots):
if bool(cache) == bool(snapshots):
raise ValueError('Choose exactly one cache or frozen snapshot directory')
frozen = json.loads((Path(snapshots)/'receipts.json').read_text())['receipts'] if snapshots else []
pages = {}
for key, url in URLS.items():
if snapshots:
r = one([r for r in frozen if r['url'] == url], 'frozen response receipt')
filename = Path(r['file']); base = Path(snapshots).resolve()
if filename.is_absolute() or '..' in filename.parts or not (base/filename).resolve().is_relative_to(base) or r.get('http_status') != 200:
raise ValueError('Unsafe or unsuccessful publisher response')
data = (base/filename).read_bytes()
if len(data) > 8*1024**2 or len(data) != r['bytes'] or hashlib.sha256(data).hexdigest() != r['sha256']:
raise ValueError('Frozen publisher response checksum/size mismatch')
observed = r['at']
else:
data = fetch(url, cache, 8*1024**2, delay=1)
r = json.loads((Path(cache)/hashlib.sha256(url.encode()).hexdigest()).with_suffix('.json').read_text())
observed = r['retrieved_at']
timestamp(observed)
pages[key] = data, {'url': url, 'sha256': hashlib.sha256(data).hexdigest(), 'bytes': len(data),
'file': 'evidence/'+hashlib.sha256(data).hexdigest()+'.html',
'observed_at': observed, 'http_status': 200, 'response_receipt': r}
return pages
def collect(out, *, cache=None, snapshots=None):
pages = read_pages(cache, snapshots)
if [clean(n.text()) for n in Document(pages['index'][0].decode('utf-8')).root.walk() if n.tag == 'title'] != [TITLE]:
raise ValueError('Unexpected AAMAS 2025 introduction')
records, excluded = parse_contents(pages['contents'][0].decode('utf-8'), pages['contents'][1])
grouped = defaultdict(list)
for row in records: grouped[row['track']].append(row)
out = Path(out); out.mkdir(parents=True, exist_ok=True); files = []
with tempfile.TemporaryDirectory(prefix='.aamas2025-', dir=out) as temporary:
stage = Path(temporary); (stage/'evidence').mkdir()
for data, e in pages.values(): (stage/e['file']).write_bytes(data)
for track, group in sorted(grouped.items()):
p = stage/f'aamas-proceedings-2025-{track}.jsonl'
p.write_text(''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in group))
m = {'source': 'aamas_proceedings', 'parser_version': 2, 'venue': 'AAMAS', 'year': 2025,
'track': track, 'count': len(group), 'file': str(out/p.name), 'file_sha256': hashlib.sha256(p.read_bytes()).hexdigest(),
'list_url': URLS['contents'], 'status': 'partial', 'missing_abstract_count': len(group),
'publication_format_counts': dict(Counter(r['metadata']['publication_format'] for r in group)),
'note': 'Complete published TOC partition; HTML abstracts absent, identities and submission-track scope require independent review.'}
p.with_suffix('.manifest.json').write_text(json.dumps(m, ensure_ascii=False, indent=2)+'\n'); files.append(m)
result = {'source': 'aamas_proceedings', 'year': 2025, 'finished_at': timestamp(),
'status': 'collected_pending_identity_review', 'records': len(records), 'files': files,
'excluded_publications': excluded, 'source_receipts': [e for _, e in pages.values()],
'publication_format_counts': dict(Counter(r['metadata']['publication_format'] for r in records)),
'database_written': False, 'pdfs_downloaded': False,
'note': 'Neither absence from proceedings nor aggregate rejection/withdrawal counts identify an individual negative decision.'}
report = 'aamas-proceedings-2025.json'; (stage/report).write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n')
(out/'evidence').mkdir(exist_ok=True)
for p in (stage/'evidence').iterdir(): p.replace(out/'evidence'/p.name)
for p in stage.iterdir():
if p.is_file() and p.name != report: p.replace(out/p.name)
(stage/report).replace(out/report)
return result
+15 -3
View File
@@ -6,6 +6,7 @@ import xml.etree.ElementTree as ET
from pathlib import Path
from paper_atlas.io.git_metadata import checkout_metadata
from paper_atlas.collectors.acl_historical import event_scope, material_reason
VENUES={'acl':'ACL','emnlp':'EMNLP','naacl':'NAACL','eacl':'EACL','coling':'COLING','conll':'CoNLL'}
@@ -29,7 +30,10 @@ def paper_id(collection,volume,number):
return f'{collection}-{int(volume):0{width}d}{int(number):0{digits}d}'
def volume_scope(meta,volume):
def volume_scope(meta,volume,collection=None):
historical=event_scope(meta,volume,collection)
if historical:
return historical['canonical_venue'],historical['venue_aliases'],historical['track']
codes=[text(n).lower() for n in meta.findall('venue')]
book=text(meta.find('booktitle'))
lower=book.lower()
@@ -59,7 +63,7 @@ def collect_xml(path,out,commit):
vid=volume.attrib['id']
meta=volume.find('meta')
if meta is None:raise ValueError('Volume metadata missing')
scope=volume_scope(meta,vid)
scope=volume_scope(meta,vid,collection)
key=f'{collection}-{vid}'
if scope is None:
results.append({'volume':key,'status':'excluded_by_venue','venue_codes':[text(n) for n in meta.findall('venue')],'booktitle':text(meta.find('booktitle')),'year':text(meta.find('year'))})
@@ -70,12 +74,18 @@ def collect_xml(path,out,commit):
year=int(year_text)
records=[]
frontmatter=0
excluded_materials=[]
historical=event_scope(meta,vid,collection)
for paper in volume.findall('paper'):
pid=paper.attrib['id']
if int(pid)==0:
frontmatter+=1
continue
identifier=paper_id(collection,vid,pid)
reason=material_reason(collection,vid,paper)
if reason:
excluded_materials.append({'external_id':identifier,'title':text(paper.find('title')),'reason':reason})
continue
doi=text(paper.find('doi'))
if doi.startswith('10.18653/v1/') and doi.split('10.18653/v1/',1)[1].casefold()!=identifier.casefold():
raise ValueError('Official DOI disagrees with constructed Anthology ID: '+identifier)
@@ -83,6 +93,8 @@ def collect_xml(path,out,commit):
if not title:raise ValueError('Missing paper title: '+identifier)
authors=[{'given':text(a.find('first')),'family':text(a.find('last'))} for a in paper.findall('author')]
records.append({'source':'acl_anthology','external_id':identifier,'url':f'https://aclanthology.org/{identifier}/','title':title,'abstract':text(paper.find('abstract')) or None,'authors':authors,'identifiers':{'acl':identifier,'doi':doi},'venue':venue,'venue_aliases':aliases if len(aliases)>1 else [],'year':year,'track':track,'status':'accepted','raw_decision':'listed in authoritative ACL Anthology proceedings XML','metadata':{'collection':collection,'volume':vid,'booktitle':text(meta.find('booktitle')),'venue_codes':[text(n) for n in meta.findall('venue')],'pages':text(paper.find('pages')),'publisher':text(meta.find('publisher')),'metadata_commit':commit,'source_xml':f'https://github.com/acl-org/acl-anthology/blob/{commit}/data/xml/{collection}.xml'}})
if historical:
records[-1]['metadata']['historical_event_scope']=historical
if not records:
results.append({'volume':key,'status':'unavailable','reason':'no research paper elements after excluding front matter'})
continue
@@ -93,7 +105,7 @@ def collect_xml(path,out,commit):
for record in records:output.write(json.dumps(record,ensure_ascii=False)+'\n')
part.replace(target)
missing=sum(not row['abstract'] for row in records)
manifest={'source':'acl_anthology','venue':venue,'year':year,'track':track,'count':len(records),'xml_paper_elements':len(volume.findall('paper')),'excluded_frontmatter':frontmatter,'missing_abstract_count':missing,'list_url':f'https://aclanthology.org/{key}/','metadata_commit':commit,'file':str(target),'status':'partial' if missing else 'collected'}
manifest={'source':'acl_anthology','venue':venue,'year':year,'track':track,'count':len(records),'xml_paper_elements':len(volume.findall('paper')),'excluded_frontmatter':frontmatter,'excluded_materials':excluded_materials,'missing_abstract_count':missing,'list_url':f'https://aclanthology.org/{key}/','metadata_commit':commit,'file':str(target),'status':'partial' if missing else 'collected'}
target.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2))
results.append(manifest)
print(json.dumps(manifest),flush=True)
+45
View File
@@ -0,0 +1,45 @@
"""Edition-specific historical scope; never alias every TINLAP year to ACL."""
HISTORY_URL = 'https://aclanthology.org/www.mt-archive.info/70/ACL-1982-Walker-1.pdf'
NOTICE_URL = 'https://aclanthology.org/J77-4001.pdf'
BOOK = 'Theoretical Issues in Natural Language Processing-2'
MATERIAL_TITLE = 'List of questions suggested for consideration in each session'
def text(node):
return ' '.join(''.join(node.itertext()).split()) if node is not None else ''
def event_scope(meta, volume, collection):
if (collection, volume) != ('T78', '1'):
return None
if ([text(n) for n in meta.findall('venue')] != ['tinlap']
or [text(n) for n in meta.findall('year')] != ['1978']
or [text(n) for n in meta.findall('booktitle')] != [BOOK]
or [(text(n.find('first')), text(n.find('last'))) for n in meta.findall('editor')]
!= [('David L.', 'Waltz')]):
raise ValueError('Reviewed T78-1 scope metadata changed; independent review required')
return {
'collection': 'T78', 'volume': '1', 'original_venue': 'TINLAP',
'event_year': 1978, 'event_dates': ['1978-07-25', '1978-07-27'],
'relationship': 'ACL annual meeting took the form of TINLAP-2',
'canonical_venue': 'ACL+TINLAP', 'venue_aliases': ['ACL', 'TINLAP'],
'track': 'proceedings_unspecified',
'track_scope_note': 'Invited and submitted contributions are described at volume level; individual roles are not identified in XML.',
'evidence': [
{'url': NOTICE_URL, 'sha256': '14b5deb471c55afb07d789529b95d0462a2a1a77df3dfba7fa1f5d3ab75eb411',
'pdf_pages': [9, 10], 'printed_pages': ['9', '10'],
'scope': 'Contemporary annual meeting notice, not individual decision notes; ACM confirmation was pending in this notice.'},
{'url': HISTORY_URL, 'sha256': '35c21bd4983975bcaccbb94bd8ab91c9019fc190feec1791255407cf18a7e49a',
'pdf_pages': [3], 'printed_pages': ['91'],
'scope': 'Retrospective annual meeting table and TNLP abbreviation confirm the 1978 event association.'},
],
}
def material_reason(collection, volume, paper):
if (collection, volume, paper.attrib.get('id')) != ('T78', '1', '36'):
return None
if text(paper.find('title')) != MATERIAL_TITLE or paper.findall('author'):
raise ValueError('Reviewed T78-1036 material changed; independent review required')
return 'Session question list, not an authored research contribution; no decision inferred.'
+1 -1
View File
@@ -20,7 +20,7 @@ DTD_MD5='96e4283e71886dec2173718957293648'
DTD_NAMES={'dblp.dtd','dblp-2023-06-28.dtd'}
RELEASE='https://doi.org/10.4230/dblp.xml.2026-11-01'
# Broad discovery candidates, including historical names; not a venue mapping.
FAMILIES=set('nips icml iclr colm colt aistats uai clear acl emnlp naacl eacl coling conll cvpr iccv eccv aaai ijcai ecai kdd sigir www wsdm recsys cikm icdm sdm ecml pkdd ecmlpkdd icra iros rss corl aips icaps kr atal aamas icassp interspeech eurospeech icslp ismir miccai fat facct lrec mm mlsys'.split())
FAMILIES=set('nips icml iclr colm colt aistats uai clear acl emnlp naacl eacl coling conll cvpr iccv eccv aaai ijcai ecai kdd sigir www wsdm recsys cikm icdm sdm ecml pkdd ecmlpkdd icra iros rss corl aips icaps ecp ewsp kr atal aamas icassp interspeech eurospeech icslp ismir miccai fat facct lrec mm mlsys'.split())
def records(archive,dtd_bytes):
@@ -0,0 +1,45 @@
"""Exact predecessor volumes, with original names and independent family histories."""
ECML_HISTORY = 'https://ecmlpkdd2011.org/pastConferences.php'
ICAPS_HISTORY = 'https://www.icaps-conference.org/conference-series/'
# family/year: booktitle, original series, target family, publication year, full title
VOLUMES = {
('ecml', 1987): ('EWSL', 'EWSL', 'ECML-PKDD', 1987,
'Progress in Machine Learning - Proceedings of EWSL 87: 2nd European Working Session on Learning, Bled, Yugoslavia, May 1987'),
('ecml', 1988): ('EWSL', 'EWSL', 'ECML-PKDD', 1988,
'Proceedings of the Third European Working Session on Learning, EWSL 1988, Turing Institute, Glasgow, UK, October 3-5, 1988'),
('ecml', 1989): ('EWSL', 'EWSL', 'ECML-PKDD', 1989,
'Proceedings of the 4th European Working Session on Learning, Montpellier, France, 4-6 December 1989'),
('ecml', 1991): ('EWSL', 'EWSL', 'ECML-PKDD', 1991,
'Machine Learning - EWSL-91, European Working Session on Learning, Porto, Portugal, March 6-8, 1991, Proceedings'),
('aips', 1990): ('Expert Planning Systems', 'EPS', 'ICAPS', 1990,
'Proceedings of the First International Conference on Expert Planning Systems, Brighton, UK, June 27-29, 1990'),
('ecp', 1997): ('ECP', 'ECP', 'ICAPS', 1997,
"Recent Advances in AI Planning, 4th European Conference on Planning, ECP'97, Toulouse, France, September 24-26, 1997, Proceedings"),
('ecp', 1999): ('ECP', 'ECP', 'ICAPS', 2000,
"Recent Advances in AI Planning, 5th European Conference on Planning, ECP'99, Durham, UK, September 8-10, 1999, Proceedings"),
('ewsp', 1991): ('EWSP', 'EWSP', 'ICAPS', 1991,
"European Workshop on Planning, EWSP '91, Sankt Augustin, Germany, March 18-19, 1991, Proceedings"),
}
def predecessor_scope(family, base, suffix, volume, year, title):
definition = VOLUMES.get((family, year))
if definition is None:
return None
book, original, target, publication_year, expected_title = definition
if (suffix or base != book or volume['key'] != f'conf/{family}/{year}'
or title != expected_title or volume['fields'].get('year') != [str(publication_year)]):
return None
return {'original_series': original, 'target_family': target,
'history_url': ECML_HISTORY if target == 'ECML-PKDD' else ICAPS_HISTORY,
'proceedings_key': volume['key'], 'proceedings_title': title,
'event_year': year, 'proceedings_year_fields': list(volume['fields']['year'])}
def missing_candidate_families(targets, families, through_year):
# These two families were absent from the earlier narrowed candidate JSONL,
# although their proceedings and volume_counts exist in the full catalogue.
required = {family for (family, year), definition in VOLUMES.items()
if family in ('ecp', 'ewsp') and definition[2] in targets and year <= through_year}
return sorted(family for family in required if not families.get(family))
+5 -1
View File
@@ -15,6 +15,7 @@ from urllib.parse import parse_qs,unquote,urlsplit
from paper_atlas.config import DEFAULT_REGISTRY
from paper_atlas.collectors.dblp import XML_SHA,DTD_MD5,RELEASE,records
from paper_atlas.collectors.dblp_scope import scope
from paper_atlas.collectors.dblp_predecessors import missing_candidate_families
from paper_atlas.db.submissions import timestamp
from paper_atlas.io.snapshots import digest_file
from paper_atlas.io.publication_ids import mlsys_id
@@ -88,6 +89,8 @@ def normalize_row(row,volumes,targets,include_workshops=False,through_year=2026)
'scope_rule':edition['rule'],'author_names_raw':fields.get('author',[]),'publtype':row.get('publtype'),
'bibliography':{key:value for key,value in fields.items() if key in ('booktitle','year','ee','url','pages','volume','number','publisher','series','isbn')},
'ambiguous_identifiers':ambiguous,'publication_evidence':'inproceedings bibliography; not an original decision note'}
if edition.get('historical_predecessor_evidence'):
metadata['historical_predecessor_evidence'] = edition['historical_predecessor_evidence']
if edition.get('joint_edition_evidence'):
metadata['joint_edition_evidence'] = edition['joint_edition_evidence']
if edition.get('track_scope_note'):
@@ -124,8 +127,9 @@ def collect(catalogue,out,venues=None,candidates=None,archive=None,dtd=None,incl
families = manifest.get('candidate_family_counts', {})
missing = [venue for venue, family in [('MM', 'mm'), ('MLSys', 'mlsys')]
if venue in targets and not families.get(family)]
missing += ['ICAPS/'+family for family in missing_candidate_families(targets, families, through_year)]
if missing:
raise ValueError('Candidate file predates requested families ' + ', '.join(missing) + '; use the original --archive')
raise ValueError('Candidate file omits requested families ' + ', '.join(missing) + '; use the original --archive')
out=Path(out);out.mkdir(parents=True,exist_ok=True)
with (out/'dblp.normalize.lock').open('a') as lock:
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
+38 -2
View File
@@ -1,6 +1,8 @@
"""Explicit bibliographic aliases; a directory family alone is not venue evidence."""
import re
from paper_atlas.collectors.dblp_predecessors import predecessor_scope
# Ordered long/joint names precede their shorter forms.
BOOKS = {
@@ -27,7 +29,8 @@ BOOKS = {
'pkdd':[('ECML/PKDD','ECML-PKDD',[]),('ECML PKDD','ECML-PKDD',[]),('PKDD','PKDD',['ECML-PKDD'])],
'icra':[('ICRA','ICRA',[])], 'iros':[('IROS','IROS',[])],
'rss':[('Robotics: Science and Systems','RSS',[])], 'corl':[('CoRL','CoRL',[])],
'aips':[('ICAPS','ICAPS',[]),('AIPS','AIPS',['ICAPS'])], 'icaps':[('ICAPS','ICAPS',[])],
'aips':[('ICAPS','ICAPS',[]),('AIPS','AIPS',['ICAPS']),('Expert Planning Systems','EPS',['ICAPS'])], 'icaps':[('ICAPS','ICAPS',[])],
'ecp':[('ECP','ECP',['ICAPS'])], 'ewsp':[('EWSP','EWSP',['ICAPS'])],
'kr':[('KR','KR',[])], 'atal':[('AAMAS','AAMAS',[])],
'icassp':[('ICASSP','ICASSP',[])],
'interspeech':[('INTERSPEECH','INTERSPEECH',[]),('EUROSPEECH','EUROSPEECH',['INTERSPEECH']),('ICSLP','ICSLP',['INTERSPEECH']),('ECST','ECST',['INTERSPEECH'])],
@@ -54,7 +57,34 @@ HISTORICAL_WORKSHOP_TITLES = {
}
# Exact pinned volume titles corroborated by official series histories. The
# original title (including Workshop) remains in source metadata. In particular
# ML91 has no four-digit year in its title; its exact key/title and bibliographic
# year are checked independently, rather than guessing a title year.
HISTORICAL_PRIMARY_EVIDENCE = {
'icml': ('ML', 'https://icml.cc/Conferences/2008/past_icmls.shtml.html'),
'iros': ('IROS', 'https://www.ieee-ras.org/conferences-workshops/financially-co-sponsored/iros/iros-past-and-future-venues/'),
}
HISTORICAL_PRIMARY_TITLES = {
('icml', 1988): 'Machine Learning, Proceedings of the Fifth International Conference on Machine Learning, Ann Arbor, Michigan, USA, June 12-14, 1988',
('icml', 1989): 'Proceedings of the Sixth International Workshop on Machine Learning (ML 1989), Cornell University, Ithaca, New York, USA, June 26-27, 1989',
('icml', 1990): 'Machine Learning, Proceedings of the Seventh International Conference on Machine Learning, Austin, Texas, USA, June 21-23, 1990',
('icml', 1991): 'Proceedings of the Eighth International Workshop (ML91), Northwestern University, Evanston, Illinois, USA',
('icml', 1992): 'Proceedings of the Ninth International Workshop on Machine Learning (ML 1992), Aberdeen, Scotland, UK, July 1-3, 1992',
('iros', 1988): "IEEE International Workshop on Intelligent Robots and Systems '88, Proceedings. IROS 1988, Tokyo, Japan, October 31 - November 2, 1988",
('iros', 1989): "IEEE International Workshop on Intelligent Robots and Systems '89, Proceedings. IROS 1989, September 4-6, 1989, Tsukuba, Japan",
('iros', 1990): "IEEE International Workshop on Intelligent Robots and Systems '90, Towards a New Frontier of Applications, Proceedings. IROS 1990, July 3-6, 1990, Ibaraki, Japan",
}
def historical_track(family, base, suffix, volume_key, year, title):
primary = HISTORICAL_PRIMARY_EVIDENCE.get(family)
if (primary and not suffix and base.casefold() == primary[0].casefold()
and volume_key == f'conf/{family}/{year}'
and title == HISTORICAL_PRIMARY_TITLES.get((family, year))):
return ('main',
'Historical primary series; exact proceedings key, complete title and bibliographic/event year corroborated by official series history. Workshop in this original title is not an affiliated track.',
primary[1])
if suffix or base.casefold() != family or volume_key != f'conf/{family}/{year}':
return None
prefix = HISTORICAL_WORKSHOP_TITLES.get((family, year))
@@ -108,7 +138,11 @@ def scope(row,volume,through_year=2026):
if len(bibliographic)!=1 or not re.fullmatch(r'\d{4}',bibliographic[0]):return None,'ambiguous_bibliographic_year'
if int(bibliographic[0])!=year and not re.search(r'\b'+str(year)+r'\b',title):return None,'unverified_year_difference'
text=(suffix+' '+title).casefold();track='main'
historical = historical_track(family, base, suffix, volume['key'], year, title)
predecessor = predecessor_scope(family, base, suffix, volume, year, title)
if (family in ('ecp', 'ewsp') or (family == 'aips' and base == 'Expert Planning Systems')) and not predecessor:
return None, 'unverified_historical_predecessor'
historical = (('main', 'Historical predecessor; exact volume, full title, event and publication years verified; original series retained.', predecessor['history_url'])
if predecessor else historical_track(family, base, suffix, volume['key'], year, title))
if historical:track=historical[0]
elif 'findings' in text:track='findings'
elif 'shared task' in text:track='shared_task'
@@ -138,6 +172,8 @@ def scope(row,volume,through_year=2026):
joint_evidence = 'https://www.eurai.org/ecai/history'
result = {'venue':venue,'venue_aliases':aliases,'year':year,'track':track,'bibliographic_year':int(bibliographic[0]),
'rule':family+':'+base,'proceedings_title':title,'edition_year_basis':year_basis}
if predecessor:
result['historical_predecessor_evidence'] = predecessor
if joint_evidence:
result['joint_edition_evidence'] = joint_evidence
if historical:
+225
View File
@@ -0,0 +1,225 @@
"""Frozen official ICLR 2014–2016 lists; presentation evidence is not a track."""
import argparse
from collections import Counter, defaultdict
import fcntl
import hashlib
import json
from pathlib import Path
import re
import tempfile
from urllib.parse import urlsplit
from paper_atlas.db.submissions import timestamp
from paper_atlas.io.html import Document, Node, clean
from paper_atlas.io.http import fetch
SOURCE = 'iclr_archive'
PAGES = {
2014: ('https://iclr.cc/archive/2014/conference-proceedings/', {'main': 35}),
2015: ('https://iclr.cc/archive/www/doku.php%3Fid=iclr2015:accepted-main.html', {'main': 31, 'workshop': 74}),
2016: ('https://iclr.cc/archive/www/doku.php%3Fid=iclr2016:accepted-main.html', {'main': 80}),
}
SECTIONS = {
2015: {'Main Conference - Oral Presentations': ('main', 'oral'),
'Main Conference - Poster Presentations': ('main', 'poster'),
'Workshop Papers': ('workshop', 'workshop')},
2016: {'Oral Presentations': ('main', 'oral'), 'Poster Presentations': ('main', 'poster')},
}
def arxiv_link(url):
parsed = urlsplit(url)
if parsed.hostname != 'arxiv.org':
return None
match = re.fullmatch(r'/(?:abs|pdf)/(\d{4}\.\d{4,5})(v[1-9]\d*)?(?:\.pdf)?', parsed.path)
if (parsed.scheme not in ('http', 'https') or parsed.netloc != 'arxiv.org'
or parsed.query or parsed.fragment or not match):
raise ValueError('Unexpected official arXiv link')
return match[1], match[2] or None
def without_links(node):
if node.tag in ('a', 'script', 'style'):
return ''
value = ''.join(without_links(n) if isinstance(n, Node) else n for n in node.children)
return ' '+value+' ' if node.tag in ('div', 'li', 'p', 'br') else value
def authors(display, year):
parts = display.rstrip(';').split(';') if year == 2014 else re.split(r',\s*(?:and\s+)?|\s+and\s+', display)
names = [clean(p) for p in parts]
if not names or any(not name for name in names):
raise ValueError('Missing author or unsupported author delimiter')
return names
def parse_page(html, year, evidence):
if year not in PAGES:
raise ValueError('Unreviewed ICLR archive year')
document = Document(html)
nodes = list(document.root.walk())
headings = [clean(n.text()) for n in nodes if n.tag in ('h1', 'h2', 'h3')]
if year == 2014:
if 'Conference Proceedings' not in headings or 'conference papers accepted' not in clean(document.root.text()):
raise ValueError('Unexpected ICLR 2014 accepted list')
elif (set(SECTIONS[year]) - set(headings)
or (year == 2016 and 'Accepted Papers (Conference Track)' not in headings)):
raise ValueError('Unexpected ICLR accepted sections')
entries = []
section = None
for index, node in enumerate(nodes):
if node.tag in ('h1', 'h2', 'h3'):
section = clean(node.text())
if year == 2014:
if node.tag != 'a' or not arxiv_link(node.attrs.get('href', '')):
continue
link = node
# This archived Google Sites page has an entry outside a paragraph.
# Authors can span several differently formatted italic spans.
# Read their whole paragraph, never only the first styled fragment.
display = None
for following in nodes[index+sum(1 for _ in link.walk()):]:
if following.tag == 'a' and arxiv_link(following.attrs.get('href', '')):
break
if following.tag in ('i', 'em') or (following.tag == 'p' and any(
n.tag in ('i', 'em') or 'font-style:italic' in n.attrs.get('style', '').replace(' ', '')
for n in following.walk())):
text = clean(following.text())
if text:
display = text
break
if not display:
raise ValueError('Missing explicit ICLR 2014 author span')
track, presentation = 'main', 'conference'
else:
if node.tag != 'li':
continue
links = [n for n in node.walk() if n.tag == 'a' and arxiv_link(n.attrs.get('href', ''))]
if not links:
continue
if len(links) != 1 or section not in SECTIONS[year]:
raise ValueError('Ambiguous paper link or unknown accepted section')
link, = links
display = clean(without_links(node)).lstrip(',').strip()
track, presentation = SECTIONS[year][section]
arxiv, version = arxiv_link(link.attrs['href'])
title = clean(link.text())
if not title:
raise ValueError('Empty official title')
entries.append({'arxiv': arxiv, 'version': version, 'title': title,
'authors': authors(display, year), 'authors_display': display,
'paper_url': link.attrs['href'], 'track': track,
'section': section, 'presentation': presentation})
# Check every arXiv occurrence, including links outside the expected layout.
occurrences = sum(n.tag == 'a' and bool(arxiv_link(n.attrs.get('href', ''))) for n in nodes)
if len(entries) != occurrences or not entries:
raise ValueError('Some official paper links were not parsed')
grouped = defaultdict(list)
for entry in entries:
grouped[(entry['track'], entry['arxiv'])].append(entry)
result = []
for (track, arxiv), variants in grouped.items():
first = variants[0]
if any((v['title'], v['authors']) != (first['title'], first['authors']) for v in variants):
raise ValueError('Repeated presentation has conflicting title or authors; review required')
presentations = sorted({v['presentation'] for v in variants})
result.append({'source': SOURCE, 'external_id': f'{year}/{track}/{arxiv}',
'url': PAGES[year][0], 'title': first['title'], 'abstract': None,
'authors': first['authors'], 'identifiers': {'arxiv': arxiv},
'venue': 'ICLR', 'venue_aliases': [], 'year': year, 'track': track,
'status': 'accepted', 'observed_at': evidence['observed_at'],
'raw_decision': 'listed in official ICLR accepted '+track+' list',
'metadata': {'source_evidence': evidence, 'source_variants': variants,
'presentations': presentations, 'decision_date_precision': 'not_supplied',
'identity_note': 'Official arXiv link, not title matching; validate canonical identity before import.',
'decision_evidence': 'Archived accepted list, not an original decision note.'},
'tags': [{'namespace': SOURCE+':presentation', 'value': value} for value in presentations]})
return result
def read_page(year, *, cache=None, snapshots=None):
if bool(cache) == bool(snapshots):
raise ValueError('Choose exactly one cache or frozen snapshot directory')
url = PAGES[year][0]
if snapshots:
root = Path(snapshots).resolve()
receipts = json.loads((root/'receipts.json').read_bytes())['receipts']
matches = [r for r in receipts if r['url'] == url]
if len(matches) != 1:
raise ValueError('Expected one exact official response receipt')
receipt, = matches
filename = Path(receipt['file'])
path = root/filename
if (filename.is_absolute() or '..' in filename.parts or not path.resolve().is_relative_to(root)
or receipt['status'] != 200):
raise ValueError('Unsafe or unsuccessful archive snapshot')
data = path.read_bytes()
if (len(data) > 8*1024**2 or len(data) != receipt['bytes']
or hashlib.sha256(data).hexdigest() != receipt['sha256']):
raise ValueError('Archive snapshot checksum or size differs')
observed = timestamp(receipt['finished_at'])
else:
data = fetch(url, cache, 8*1024**2, delay=1)
receipt = json.loads((Path(cache)/hashlib.sha256(url.encode()).hexdigest()).with_suffix('.json').read_bytes())
observed = timestamp(receipt['retrieved_at'])
evidence = {'url': url, 'sha256': hashlib.sha256(data).hexdigest(), 'bytes': len(data),
'file': 'evidence/'+hashlib.sha256(data).hexdigest()+'.html',
'observed_at': observed, 'http_status': 200, 'response_receipt': receipt}
return data, evidence
def collect(year, out, *, cache=None, snapshots=None):
if year not in PAGES:
raise ValueError('Only independently reviewed ICLR archive years are supported')
out = Path(out)
out.mkdir(parents=True, exist_ok=True)
with (out/'iclr-archive.collect.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
report = out/f'iclr-archive-{year}.json'
if report.exists():
raise FileExistsError('Existing completed archive output must be inspected, not overwritten')
data, evidence = read_page(year, cache=cache, snapshots=snapshots)
records = parse_page(data.decode('utf-8'), year, evidence)
counts = Counter(r['track'] for r in records)
if dict(counts) != PAGES[year][1]:
raise ValueError('Distinct archive entries differ from reviewed inventory')
files = []
with tempfile.TemporaryDirectory(prefix='.iclr-archive-', dir=out) as temporary:
stage = Path(temporary)
(stage/'evidence').mkdir()
(stage/evidence['file']).write_bytes(data)
for track in sorted(counts):
group = [r for r in records if r['track'] == track]
path = stage/f'iclr-archive-{year}-{track}.jsonl'
path.write_text(''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in group))
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICLR', 'year': year,
'track': track, 'count': len(group), 'file': str(out/path.name),
'file_sha256': hashlib.sha256(path.read_bytes()).hexdigest(),
'list_url': PAGES[year][0], 'status': 'partial', 'missing_abstract_count': len(group)}
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, ensure_ascii=False, indent=2)+'\n')
files.append(manifest)
result = {'source': SOURCE, 'year': year, 'records': len(records), 'files': files,
'source_evidence': evidence, 'database_written': False, 'pdfs_downloaded': False,
'status': 'collected_pending_identity_review',
'note': 'Accepted list only. Oral/poster are presentation evidence; absent papers are not rejected. '
'ICLR2014 workshop and other unlisted tracks remain outside these reviewed pages.'}
(stage/report.name).write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n')
(out/'evidence').mkdir(exist_ok=True)
for path in (stage/'evidence').iterdir():
path.replace(out/'evidence'/path.name)
for path in stage.iterdir():
if path.is_file() and path.name != report.name:
path.replace(out/path.name)
(stage/report.name).replace(report)
return result
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog)
parser.add_argument('--year', type=int, choices=sorted(PAGES), required=True)
parser.add_argument('--out', required=True)
source = parser.add_mutually_exclusive_group(required=True)
source.add_argument('--cache')
source.add_argument('--snapshots')
args = parser.parse_args(argv)
print(json.dumps(collect(args.year, args.out, cache=args.cache, snapshots=args.snapshots), ensure_ascii=False))
+1 -1
View File
@@ -4,7 +4,7 @@ import re
from paper_atlas.io.publication_ids import mlsys_id
PUBLICATIONS={'pmlr','acl_anthology','colm','neurips','mlsys','cvf','ecva','dblp'}
PUBLICATIONS={'pmlr','acl_anthology','colm','neurips','mlsys','cvf','ecva','dblp','aamas_proceedings'}
def timestamp(value=None):
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+54
View File
@@ -0,0 +1,54 @@
<HTML><HEAD><TITLE>AAMAS 2025: Proceedings of the 2025 International Conference on Autonomous Agents and Multiagent Systems</TITLE>
<META http-equiv=Content-Type content="text/html; charset=utf-8"><LINK
href="forms/index_files/factemplate.css" type=text/css rel=stylesheet>
<META content="MSHTML 6.00.2900.3562" name=GENERATOR></HEAD>
<BODY bgcolor="#010101" bgproperties="fixed"><DIV style="TEXT-ALIGN: center" >
<table width="724" border="0" align="center" cellpadding="0" cellspacing="0" valign="top">
<td colspan="2" align="center" valign="top" bgcolor="#FFFFFF"><img src="forms/index_files/aamas25.png" width="750" height="300" alt=""/></td>
</tr>
<tr bgcolor="#FFFFFF">
<td width="125" align="center" valign="top" bgcolor="#FFFFFF"><table width="125" border="1" align="center" cellpadding="1" cellspacing="1" bordercolor="#444444" style="margin-bottom: 2">
<tr>
<td bgcolor="#FFFFFF"><p style="text-indent: 0; margin-left: 6; margin-right: 6; margin-top: 5; margin-bottom: 4" align="left"><A href="forms/index.htm">Main
Page</A></p></td>
</tr>
<tr>
<td bgcolor="#FFFFFF"><p style="text-indent: 0; margin-left: 6; margin-right: 6; margin-top: 5; margin-bottom: 4" align="left"><A href="forms/contents.htm">Table
of Contents</A></p></td>
</tr>
</table></td>
<td width="599"><p style="text-indent: 0; margin-left: 12; margin-right: 14; margin-top: 12; margin-bottom: 12" align="left"><font color="#010101" size="4" face="Arial, Helvetica, sans-serif"><strong>Welcome
to the Proceedings of AAMAS-2025</strong></font></p>
<p style="text-indent: 0; margin-left: 12; margin-right: 14; margin-top: 12; margin-bottom: 12;" align="left"><font color="#333333" size="2" face="Times New Roman, Times, serif">These
are the proceedings of the 24th International Conference on Autonomous
Agents and Multiagent Systems (AAMAS-2025). They are published by the
International Foundation for Autonomous Agents and Multiagent Systems
(IFAAMAS).</font></p></td>
</tr>
<tr bgcolor="#FFFFFF">
<td align="center" valign="middle" bgcolor="#FFFFFF">&nbsp;</td>
<td><table width="96%" border="0" cellspacing="2" cellpadding="2">
<tr align="left" valign="top">
<td> <p style="text-indent: 0; margin-left: 12; margin-right: 6; margin-top: 0; margin-bottom: 6;" align="left"><font color="#333333" face="Times New Roman, Times, serif"><font size="2"><strong>ISBN: 979-8-4007-1426-9</strong></font></font></p></td>
<td> <p style="text-indent: 0; margin-left: 12; margin-right: 6; margin-top: 0; margin-bottom: 6;" align="left"><font color="#333333" face="Times New Roman, Times, serif"></font></p></td>
</tr>
<tr>
<td width="42%"><div align="left"><img src="forms/index_files/ifaamas.jpg" width="256" height="104"></div></td>
<td width="58%"><div align="center"><img src="forms/index_files/ACM-In-Cooperation_medium.jpg" width="285" height="86"></div></td>
</tr>
</table>
<p style="text-indent: 0; margin-left: 12; margin-right: 14; margin-top: 12; margin-bottom: 12;" align="left"><font color="#333333" size="2" face="Times New Roman, Times, serif">Copyright
&copy; 2025 by International Foundation for Autonomous Agents and Multiagent
Systems (IFAAMAS). Permission to make digital or hard copies of portions
of this work for personal or classroom use is granted without fee provided
that copies are not made or distributed for profit or commercial advantage
and that copies bear this notice and the full citation on the first page.
Copyright for components of this work owned by others than IFAAMAS must
be honored. Abstracting with credit is permitted. To copy otherwise, to
republish, to post on servers or to redistribute to lists, requires prior
specific permission and/or a fee. </font></p></td>
</tr>
</table>
<p>&nbsp;</p>
</BODY></HTML>
+38
View File
@@ -0,0 +1,38 @@
{
"status": "two_successful_official_metadata_pages_recovered_third403_not_retried",
"receipts": [
{
"file": "index.html",
"url": "https://www.ifaamas.org/Proceedings/aamas2025/",
"bytes": 4107,
"sha256": "850b7825c66d0147ac8a8bbf03a554590e4a9e8f4b5c576e043999bbd43317f4",
"at": "2026-11-09T12:31:49.347577+00:00",
"at_precision": "filesystem_write_time_recovered",
"http_status": 200,
"content_type": "text/html",
"receipt_recovered": true,
"scope": "Collectorasserted200/htmlbeforewrite;request3acceptedprogram403;noheaders/finalredirectURLpersisted;no refetch or rawbytesmutation."
},
{
"file": "contents.html",
"url": "https://www.ifaamas.org/Proceedings/aamas2025/forms/contents.htm",
"bytes": 370650,
"sha256": "732847b51972f34c721a49d8bd963a80a029a48f3ec08022c9a0d5ddc2945549",
"at": "2026-11-09T12:31:52.006642+00:00",
"at_precision": "filesystem_write_time_recovered",
"http_status": 200,
"content_type": "text/html",
"receipt_recovered": true,
"scope": "Collectorasserted200/htmlbeforewrite;request3acceptedprogram403;noheaders/finalredirectURLpersisted;no refetch or rawbytesmutation."
}
],
"failed_url": "https://aamas2025.org/index.php/conference/program/accepted-papers/",
"not_attempted_url": "https://aamas2025.org/index.php/conference/calls/submission-instructions-main-technical-track/",
"original_error": {
"error": "HTTP Error 403: Forbidden",
"type": "HTTPError",
"downloaded": 2
},
"research_fulltext_downloaded": false,
"database_written": false
}
+273
View File
@@ -0,0 +1,273 @@
<?xml version='1.0' encoding='UTF-8'?>
<collection id="T78">
<volume id="1" type="proceedings">
<!-- https://aclanthology.org/T78-1/ -->
<meta>
<booktitle>Theoretical Issues in Natural Language <fixed-case>P</fixed-case>rocessing-2</booktitle>
<editor id="david-l-waltz"><first>David L.</first><last>Waltz</last></editor>
<venue>tinlap</venue>
<year>1978</year>
</meta>
<frontmatter>
<!-- https://aclanthology.org/T78-1000/ -->
<pdf hash="fb6164ad"/>
<bibkey>tinlap-1978-theoretical</bibkey>
</frontmatter>
<paper id="1">
<!-- https://aclanthology.org/T78-1001/ -->
<title>Testing The Psychological Reality of a Representational Model</title>
<author><first>Dedre</first><last>Gentner</last></author>
<pdf hash="f723a956"/>
<bibkey>gentner-1978-testing-psychological</bibkey>
</paper>
<paper id="2">
<!-- https://aclanthology.org/T78-1002/ -->
<title>What Makes Something “Ad Hoc”</title>
<author id="roger-c-schank"><first>Roger C.</first><last>Schank</last></author>
<pdf hash="636853f9"/>
<bibkey>schank-1978-makes-something</bibkey>
</paper>
<paper id="3">
<!-- https://aclanthology.org/T78-1003/ -->
<title>The Relation of Grammar to Cognition–a Synopsis</title>
<author><first>Leonard</first><last>Talmy</last></author>
<pdf hash="101c67dc"/>
<bibkey>talmy-1978-relation-grammar</bibkey>
</paper>
<paper id="4">
<!-- https://aclanthology.org/T78-1004/ -->
<title>On primitives, prototypes, and other semantic anomalies</title>
<author><first>Terry</first><last>Winograd</last></author>
<pdf hash="f8e47023"/>
<bibkey>winograd-1978-primitives-prototypes</bibkey>
</paper>
<paper id="5">
<!-- https://aclanthology.org/T78-1005/ -->
<title>Taxonomic Lattice Structures for Situation Recognition</title>
<author id="william-a-woods"><first>William A.</first><last>Woods</last></author>
<pdf hash="92957ddf"/>
<bibkey>woods-1978-taxonomic-lattice</bibkey>
</paper>
<paper id="6">
<!-- https://aclanthology.org/T78-1006/ -->
<title>Description Formation and Discourse Model Synthesis</title>
<author id="bonnie-webber"><first>Bonnie Lynn</first><last>Webber</last></author>
<pdf hash="ffded1d2"/>
<bibkey>webber-1978-description-formation</bibkey>
</paper>
<paper id="7">
<!-- https://aclanthology.org/T78-1007/ -->
<title>The Processing of Referring Expressions within a Semantic Network</title>
<author><first>John R.</first><last>Anderson</last></author>
<pdf hash="8bcc083a"/>
<bibkey>anderson-1978-processing-referring</bibkey>
</paper>
<paper id="8">
<!-- https://aclanthology.org/T78-1008/ -->
<title>Reference Diaries</title>
<author><first>Herbert H.</first><last>Clark</last></author>
<author><first>Catherine</first><last>Marshall</last></author>
<pdf hash="d06b7ea2"/>
<bibkey>clark-marshall-1978-reference-diaries</bibkey>
</paper>
<paper id="9">
<!-- https://aclanthology.org/T78-1009/ -->
<title>Subsequent Reference: Syntactic and Rhetorical Constraints</title>
<author id="david-d-mcdonald"><first>David D.</first><last>McDonald</last></author>
<pdf hash="2d424559"/>
<bibkey>mcdonald-1978-subsequent-reference</bibkey>
</paper>
<paper id="10">
<!-- https://aclanthology.org/T78-1010/ -->
<title>Some Psycholinguistic Constraints on the Construction and Interpretation of Definite Descriptions</title>
<author><first>Andrew</first><last>Ortony</last></author>
<pdf hash="69807727"/>
<bibkey>ortony-1978-psycholinguistic-constraints</bibkey>
</paper>
<paper id="11">
<!-- https://aclanthology.org/T78-1011/ -->
<title>Bound Variables and Other Anaphors</title>
<author id="barbara-h-partee"><first>Barbara H.</first><last>Partee</last></author>
<pdf hash="e8e6a510"/>
<bibkey>partee-1978-bound-variables</bibkey>
</paper>
<paper id="12">
<!-- https://aclanthology.org/T78-1012/ -->
<title>The Use of Focus as a Tool for Disambiguation of Definite Noun Phrases</title>
<author id="candace-l-sidner"><first>Candace L.</first><last>Sidner</last></author>
<pdf hash="7b96d2b7"/>
<bibkey>sidner-1978-use-focus</bibkey>
</paper>
<paper id="13">
<!-- https://aclanthology.org/T78-1013/ -->
<title>Focusing in Dialog</title>
<author id="barbara-j-grosz"><first>Barbara J.</first><last>Grosz</last></author>
<pdf hash="8c8d4c2e"/>
<bibkey>grosz-1978-focusing-dialog</bibkey>
</paper>
<paper id="14">
<!-- https://aclanthology.org/T78-1014/ -->
<title>Topic Levels</title>
<author><first>Joseph E.</first><last>Grimes</last></author>
<pdf hash="f485b2ba"/>
<bibkey>grimes-1978-topic-levels</bibkey>
</paper>
<paper id="15">
<!-- https://aclanthology.org/T78-1015/ -->
<title>Toward a Rational Model of Discourse Comprehension</title>
<author><first>Jerry L.</first><last>Morgan</last></author>
<pdf hash="5e12a597"/>
<bibkey>morgan-1978-toward-rational</bibkey>
</paper>
<paper id="16">
<!-- https://aclanthology.org/T78-1016/ -->
<title>Assent and Compliance in Children’s Language Comprehension</title>
<author><first>David R.</first><last>Olson</last></author>
<pdf hash="708bc4a0"/>
<bibkey>olson-1978-assent-compliance</bibkey>
</paper>
<paper id="17">
<!-- https://aclanthology.org/T78-1017/ -->
<title>Speech Acts as a Basis for Understanding Dialogue Coherence</title>
<author id="c-raymond-perrault"><first>C. Raymond</first><last>Perrault</last></author>
<author id="james-allen"><first>James F.</first><last>Allen</last></author>
<pdf hash="d111402b"/>
<bibkey>perrault-allen-1978-speech</bibkey>
</paper>
<paper id="18">
<!-- https://aclanthology.org/T78-1018/ -->
<title>A Framework for Comparing Language Experiences (with particular emphasis on: The Effect of Audience on Discourse Models)</title>
<author><first>Andee</first><last>Rubin</last></author>
<pdf hash="5d9a6150"/>
<bibkey>rubin-1978-framework-comparing</bibkey>
</paper>
<paper id="19">
<!-- https://aclanthology.org/T78-1019/ -->
<title>Intentlonallty and Human Conversations</title>
<author id="jaime-g-carbonell"><first>Jaime G.</first><last>Carbonell Jr</last></author>
<pdf hash="c50fc619"/>
<bibkey>carbonell-jr-1978-intentlonallty-human</bibkey>
</paper>
<paper id="20">
<!-- https://aclanthology.org/T78-1020/ -->
<title>On the Interdependence of Language and Perception</title>
<author id="david-l-waltz"><first>David L.</first><last>Waltz</last></author>
<pdf hash="3c708191"/>
<bibkey>waltz-1978-interdependence-language</bibkey>
</paper>
<paper id="21">
<!-- https://aclanthology.org/T78-1021/ -->
<title>The Problem of Naming Shapes: Vision-Language Interface</title>
<author id="ruzena-bajcsy"><first>R.</first><last>Bajcsy</last></author>
<author id="aravind-joshi"><first>A. K.</first><last>Joshi</last></author>
<pdf hash="50331b4e"/>
<bibkey>bajcsy-joshi-1978-problem-naming</bibkey>
</paper>
<paper id="22">
<!-- https://aclanthology.org/T78-1022/ -->
<title>An Argument about the Composition of Conceptual Structure</title>
<author><first>Ray</first><last>Jackendoff</last></author>
<pdf hash="57d001f8"/>
<bibkey>jackendoff-1978-argument-composition</bibkey>
</paper>
<paper id="23">
<!-- https://aclanthology.org/T78-1023/ -->
<title>On the Ontological Status of Visual Mental Images</title>
<author><first>Stephen Michael</first><last>Kosslyn</last></author>
<pdf hash="fd06e265"/>
<bibkey>kosslyn-1978-ontological-status</bibkey>
</paper>
<paper id="24">
<!-- https://aclanthology.org/T78-1024/ -->
<title>What has language to do with perception? Some speculations on the <i>Lingua Mentis</i></title>
<author><first>Zenon W.</first><last>Pylyshyn</last></author>
<pdf hash="a2f3a044"/>
<bibkey>pylyshyn-1978-language-perception</bibkey>
</paper>
<paper id="25">
<!-- https://aclanthology.org/T78-1025/ -->
<title>Semantic Primitives in Language and Vision</title>
<author id="yorick-wilks"><first>Yorick</first><last>Wilks</last></author>
<pdf hash="b4e13896"/>
<bibkey>wilks-1978-semantic-primitives</bibkey>
</paper>
<paper id="26">
<!-- https://aclanthology.org/T78-1026/ -->
<title>A Note on Partial Match of Descriptions. Can One Simultaneously Question (Retrieve) and Inform (Update)?</title>
<author id="aravind-joshi"><first>Aravind K.</first><last>Joshi</last></author>
<pdf hash="74969243"/>
<bibkey>joshi-1978-note-partial</bibkey>
</paper>
<paper id="27">
<!-- https://aclanthology.org/T78-1027/ -->
<title>With a Spoon in Hand This Must Be the Eating Frame</title>
<author><first>Eugene</first><last>Charniak</last></author>
<pdf hash="f1be68bb"/>
<bibkey>charniak-1978-spoon-hand</bibkey>
</paper>
<paper id="28">
<!-- https://aclanthology.org/T78-1028/ -->
<title>Fragments of a Theory of Human Plausible Reasoning</title>
<author><first>Allan</first><last>Collins</last></author>
<pdf hash="67b4bca2"/>
<bibkey>collins-1978-fragments-theory</bibkey>
</paper>
<paper id="29">
<!-- https://aclanthology.org/T78-1029/ -->
<title>Indirect Responses to Loaded Questions</title>
<author><first>S. Jerrold</first><last>Kaplan</last></author>
<pdf hash="135037f9"/>
<bibkey>kaplan-1978-indirect-responses</bibkey>
</paper>
<paper id="30">
<!-- https://aclanthology.org/T78-1030/ -->
<title>On Reasoning by Default</title>
<author><first>Raymond</first><last>Reiter</last></author>
<pdf hash="0de53d33"/>
<bibkey>reiter-1978-reasoning-default</bibkey>
</paper>
<paper id="31">
<!-- https://aclanthology.org/T78-1031/ -->
<title>Path-Based and Node-Based Inference in Semantic Networks</title>
<author id="stuart-c-shapiro"><first>Stuart C.</first><last>Shapiro</last></author>
<pdf hash="2ddc5535"/>
<bibkey>shapiro-1978-path-based</bibkey>
</paper>
<paper id="32">
<!-- https://aclanthology.org/T78-1032/ -->
<title>The Representation of Derivable Information in Memory: When What Might Have Been Left Unsaid Is Said</title>
<author><first>Rand J.</first><last>Spiro</last></author>
<author><first>Joseph</first><last>Esposito</last></author>
<author><first>Richard J.</first><last>Vondruska</last></author>
<pdf hash="7758fb45"/>
<bibkey>spiro-etal-1978-representation-derivable</bibkey>
</paper>
<paper id="33">
<!-- https://aclanthology.org/T78-1033/ -->
<title>A Heuristic for Paradigms</title>
<author><first>Joseph E.</first><last>Grimes</last></author>
<pdf hash="53b42c16"/>
<bibkey>grimes-1978-heuristic-paradigms</bibkey>
</paper>
<paper id="34">
<!-- https://aclanthology.org/T78-1034/ -->
<title>A Computational Account of Some Constraints on Language</title>
<author id="mitch-marcus"><first>Mitchell</first><last>Marcus</last></author>
<pdf hash="c9724c43"/>
<bibkey>marcus-1978-computational-account</bibkey>
</paper>
<paper id="35">
<!-- https://aclanthology.org/T78-1035/ -->
<title>Remarks on Processing, Constraints, and the Lexicon</title>
<author id="thomas-wasow"><first>Thomas</first><last>Wasow</last></author>
<pdf hash="8beed275"/>
<bibkey>wasow-1978-remarks-processing</bibkey>
</paper>
<paper id="36">
<!-- https://aclanthology.org/T78-1036/ -->
<title>List of questions suggested for consideration in each session</title>
<pdf hash="1950b908"/>
<bibkey>nn-1978-list-questions</bibkey>
</paper>
</volume>
</collection>
+314
View File
@@ -0,0 +1,314 @@
{
"source_snapshot_sha256": "0376ad019b216553faff0a14e1757b2d4c7a3b84c1beacf0511ef83695211b18",
"scope": "Pinned proceedings metadata, not authentic example paper records or evidence that all volumes contain indexed papers. Test papers are synthetic.",
"volumes": [
{
"key": "conf/ecml/1987",
"fields": {
"editor": [
"Ivan Bratko",
"Nada Lavrac"
],
"title": [
"Progress in Machine Learning - Proceedings of EWSL 87: 2nd European Working Session on Learning, Bled, Yugoslavia, May 1987"
],
"booktitle": [
"EWSL"
],
"publisher": [
"Sigma Press, Wilmslow"
],
"year": [
"1987"
],
"isbn": [
"1-85058-088-X"
],
"url": [
"db/conf/ecml/ewsl87.html"
]
},
"bibliographic_counts": [
{
"year": "1987",
"booktitle": "EWSL",
"record_type": "inproceedings",
"count": 16
}
],
"expected_original_series": "EWSL",
"expected_target_family": "ECML-PKDD"
},
{
"key": "conf/ecml/1988",
"fields": {
"editor": [
"Derek H. Sleeman"
],
"title": [
"Proceedings of the Third European Working Session on Learning, EWSL 1988, Turing Institute, Glasgow, UK, October 3-5, 1988"
],
"booktitle": [
"EWSL"
],
"publisher": [
"Pitman Publishing"
],
"year": [
"1988"
],
"isbn": [
"0-273-08800-9"
],
"url": [
"db/conf/ecml/ewsl88.html"
]
},
"bibliographic_counts": [
{
"year": "1988",
"booktitle": "EWSL",
"record_type": "inproceedings",
"count": 21
}
],
"expected_original_series": "EWSL",
"expected_target_family": "ECML-PKDD"
},
{
"key": "conf/ecml/1989",
"fields": {
"editor": [
"Katharina Morik"
],
"title": [
"Proceedings of the 4th European Working Session on Learning, Montpellier, France, 4-6 December 1989"
],
"booktitle": [
"EWSL"
],
"publisher": [
"Pitman and Morgan Kaufman Publishers"
],
"year": [
"1989"
]
},
"bibliographic_counts": [],
"expected_original_series": "EWSL",
"expected_target_family": "ECML-PKDD"
},
{
"key": "conf/ecml/1991",
"fields": {
"editor": [
"Yves Kodratoff"
],
"title": [
"Machine Learning - EWSL-91, European Working Session on Learning, Porto, Portugal, March 6-8, 1991, Proceedings"
],
"booktitle": [
"EWSL"
],
"series": [
"Lecture Notes in Computer Science"
],
"volume": [
"482"
],
"publisher": [
"Springer"
],
"year": [
"1991"
],
"isbn": [
"3-540-53816-X"
],
"ee": [
"https://doi.org/10.1007/BFb0016999"
],
"url": [
"db/conf/ecml/ewsl91.html"
]
},
"bibliographic_counts": [
{
"year": "1991",
"booktitle": "EWSL",
"record_type": "inproceedings",
"count": 44
}
],
"expected_original_series": "EWSL",
"expected_target_family": "ECML-PKDD"
},
{
"key": "conf/ecp/1997",
"fields": {
"editor": [
"Sam Steel",
"Rachid Alami 0001"
],
"title": [
"Recent Advances in AI Planning, 4th European Conference on Planning, ECP'97, Toulouse, France, September 24-26, 1997, Proceedings"
],
"booktitle": [
"ECP"
],
"series": [
"Lecture Notes in Computer Science"
],
"volume": [
"1348"
],
"publisher": [
"Springer"
],
"year": [
"1997"
],
"isbn": [
"3-540-63912-8"
],
"ee": [
"https://doi.org/10.1007/3-540-63912-8"
],
"url": [
"db/conf/ecp/ecp1997.html"
]
},
"bibliographic_counts": [
{
"year": "1997",
"booktitle": "ECP",
"record_type": "inproceedings",
"count": 35
}
],
"expected_original_series": "ECP",
"expected_target_family": "ICAPS"
},
{
"key": "conf/ecp/1999",
"fields": {
"editor": [
"Susanne Biundo",
"Maria Fox 0001"
],
"title": [
"Recent Advances in AI Planning, 5th European Conference on Planning, ECP'99, Durham, UK, September 8-10, 1999, Proceedings"
],
"booktitle": [
"ECP"
],
"series": [
"Lecture Notes in Computer Science"
],
"volume": [
"1809"
],
"publisher": [
"Springer"
],
"year": [
"2000"
],
"isbn": [
"3-540-67866-2"
],
"ee": [
"https://doi.org/10.1007/10720246"
],
"url": [
"db/conf/ecp/ecp1999.html"
]
},
"bibliographic_counts": [
{
"year": "1999",
"booktitle": "ECP",
"record_type": "inproceedings",
"count": 28
}
],
"expected_original_series": "ECP",
"expected_target_family": "ICAPS"
},
{
"key": "conf/ewsp/1991",
"fields": {
"editor": [
"Joachim Hertzberg"
],
"title": [
"European Workshop on Planning, EWSP '91, Sankt Augustin, Germany, March 18-19, 1991, Proceedings"
],
"booktitle": [
"EWSP"
],
"series": [
"Lecture Notes in Computer Science"
],
"volume": [
"522"
],
"publisher": [
"Springer"
],
"year": [
"1991"
],
"isbn": [
"3-540-54364-3"
],
"ee": [
"https://doi.org/10.1007/BFb0052944"
],
"url": [
"db/conf/ewsp/ewsp1991.html"
]
},
"bibliographic_counts": [
{
"year": "1991",
"booktitle": "EWSP",
"record_type": "inproceedings",
"count": 10
}
],
"expected_original_series": "EWSP",
"expected_target_family": "ICAPS"
},
{
"key": "conf/aips/1990",
"fields": {
"title": [
"Proceedings of the First International Conference on Expert Planning Systems, Brighton, UK, June 27-29, 1990"
],
"booktitle": [
"Expert Planning Systems"
],
"publisher": [
"AAAI"
],
"year": [
"1990"
],
"url": [
"db/conf/aips/aips1990.html"
]
},
"bibliographic_counts": [
{
"year": "1990",
"booktitle": "Expert Planning Systems",
"record_type": "inproceedings",
"count": 48
}
],
"expected_original_series": "EPS",
"expected_target_family": "ICAPS"
}
]
}
+85
View File
@@ -0,0 +1,85 @@
import hashlib
import json
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from paper_atlas.collectors import aamas, aamas2025
FIXTURE = Path(__file__).parent/'fixtures/aamas2025'
class Aamas2025Tests(unittest.TestCase):
def parse(self, html=None):
return aamas2025.parse_contents(html or (FIXTURE/'contents.html').read_text(),
{'observed_at': '2026-11-09T12:31:50+00:00', 'url': aamas2025.URLS['contents']})
def test_real_toc_preserves_every_title_author_and_affiliation(self):
expected = json.loads((FIXTURE/'expected-authors.json').read_text())
self.assertEqual(hashlib.sha256((FIXTURE/'contents.html').read_bytes()).hexdigest(), expected['source_sha256'])
rows, excluded = self.parse()
self.assertEqual((len(rows), len(excluded)), (476, 3))
by_url = {r['paper_url']: r for r in expected['entries']}
for row in rows:
original = by_url[row['metadata']['paper_url']]
self.assertEqual(row['title'], original['title'])
self.assertEqual(row['authors'], [a['name'] for a in original['authors']])
self.assertEqual(row['metadata']['author_affiliations'], [a['affiliation'] for a in original['authors']])
self.assertIsNone(row['abstract'])
self.assertEqual(row['identifiers'], {})
self.assertEqual(row['status'], 'accepted')
self.assertNotIn('decision_date', row)
self.assertTrue(all(t['namespace'] == 'aamas_proceedings:publication_format' for t in row['tags']))
def test_publication_format_and_track_are_separate(self):
rows, excluded = self.parse()
groups = {k: sum(r['track'] == k for r in rows) for k in {r['track'] for r in rows}}
self.assertEqual(groups, {'proceedings_unspecified': 409, 'blue_sky': 12, 'doctoral': 25, 'demo': 21, 'jaamas': 9})
competition = next(r for r in rows if r['external_id'] == '2025/p3000')
self.assertEqual(competition['year'], 2025)
self.assertEqual(competition['metadata']['publication_format'], 'competition_report')
self.assertEqual(competition['metadata']['section_heading'], 'Demo Track')
self.assertEqual(competition['track'], 'proceedings_unspecified')
self.assertEqual(len({r['external_id'] for r in rows}), 476)
self.assertEqual([r['publisher_page'] for r in excluded], [1, 3, 4])
def test_refuses_changed_edition_inventory_or_author_markup(self):
original = (FIXTURE/'contents.html').read_text()
for html in [original.replace('AAMAS 2025:', 'AAMAS 2026:', 1),
original.replace('../pdfs/p5.pdf', '../pdfs/p14.pdf', 1),
original.replace('Paper Track</font>', 'Workshop Track</font>'),
original.replace('Amir Abolfazli', '', 1),
original.replace('[COMP24]', '[CHANGED]', 1)]:
with self.subTest(change=html[:60]):
with self.assertRaises(ValueError): self.parse(html)
def test_frozen_collection_is_partial_and_never_fetches_pdfs(self):
with tempfile.TemporaryDirectory() as tmp, patch.object(aamas2025, 'fetch', side_effect=AssertionError('No HTTP in frozen mode')):
out = Path(tmp)/'out'
report = aamas.collect(2025, out, snapshots=FIXTURE)
self.assertEqual(report['records'], 476)
self.assertFalse(report['database_written'])
self.assertFalse(report['pdfs_downloaded'])
self.assertEqual(len(report['files']), 5)
for m in report['files']:
self.assertEqual(m['status'], 'partial')
self.assertEqual(m['missing_abstract_count'], m['count'])
self.assertEqual(hashlib.sha256(Path(m['file']).read_bytes()).hexdigest(), m['file_sha256'])
receipt = report['source_receipts'][0]['response_receipt']
self.assertEqual(receipt['at_precision'], 'filesystem_write_time_recovered')
self.assertTrue((out/'aamas-proceedings-2025.json').is_file())
def test_rejects_tampered_or_unsafe_frozen_response(self):
import shutil
for mode in ['tamper', 'path', 'status']:
with self.subTest(mode=mode), tempfile.TemporaryDirectory() as tmp:
root = Path(tmp); source = root/'source'; shutil.copytree(FIXTURE, source)
p = source/'receipts.json'; r = json.loads(p.read_text())
if mode == 'tamper': (source/'contents.html').write_text('bad')
elif mode == 'path': r['receipts'][0]['file'] = '../index.html'
else: r['receipts'][0]['http_status'] = 403
p.write_text(json.dumps(r))
with self.assertRaises(ValueError): aamas.collect(2025, root/'out', snapshots=source)
self.assertFalse((root/'out/aamas-proceedings-2025.json').exists())
+151
View File
@@ -0,0 +1,151 @@
"""Publisher evidence, section boundaries and native-ingestion regression checks."""
import hashlib
import json
import re
from pathlib import Path
import sqlite3
import tempfile
import unittest
from unittest.mock import patch
from paper_atlas.collectors import aamas
from paper_atlas.ingestion.records import ingest_file
def paper(pid, abstract=True):
summary = '<span class="abstract">Research with <i>evidence</i>.<br>More detail.</span>' if abstract else ''
return f'''<div class="paper" id="{pid}"><span class="id">{pid}</span>
<p class="paper"><span class="title"><a href="../papers/{pid}.pdf">{summary}
A structured &amp; auditable paper {pid}</a></span><span class="authors">
<a href="authors.html#Jane">Jane Doe<span class="amount">has 7 papers</span></a>, Bob Smith
</span></p></div>'''
def page(section, pid, abstract=True):
heading = aamas.SECTIONS[section][0]
return (f'<html><head><title>{heading} - AAMAS 2012</title></head><body>'
f'<h1>{heading}</h1><h2 class="section" id="s-1A">Session 1A – Robotics I</h2>'
+paper(pid, abstract)+'</body></html>')
class AamasProceedingsTests(unittest.TestCase):
def snapshots(self, root):
root.mkdir()
receipts = []
items = {'introduction': ('<title>AAMAS 2012</title><p>Aggregate withdrawals do not identify papers.</p>', aamas.ROOT)}
for index, section in enumerate(aamas.SECTIONS):
items[section] = (page(section, f'P{index}_1', section != 'extabstracts'), aamas.ROOT+'resources/'+section+'.html')
for key, (html, url) in items.items():
data = html.encode()
(root/(key+'.html')).write_bytes(data)
receipts.append({'url': url, 'status': 200, 'file': key+'.html', 'bytes': len(data),
'sha256': hashlib.sha256(data).hexdigest(), 'finished_at': '2026-11-09T00:00:00+00:00'})
(root/'receipts.json').write_text(json.dumps({'receipts': receipts}))
return root
def test_hidden_abstract_and_author_badges_are_not_title_or_authors(self):
rows = aamas.parse_section(page('fullpapers', '1A_1'), 'fullpapers', '2026-11-09T00:00:00Z', {'sha256': 'proof'})
row, = rows
self.assertEqual(row['title'], 'A structured & auditable paper 1A_1')
self.assertEqual(row['abstract'], 'Research with evidence. More detail.')
self.assertEqual(row['authors'], ['Jane Doe', 'Bob Smith'])
self.assertEqual(row['metadata']['authors_display'], 'Jane Doe, Bob Smith')
self.assertEqual(row['url'], aamas.ROOT+'resources/fullpapers.html#1A_1')
self.assertEqual(row['metadata']['paper_url'], aamas.ROOT+'papers/1A_1.pdf')
self.assertEqual(row['metadata']['source_evidence'], {'sha256': 'proof'})
def test_publication_formats_and_session_topics_do_not_invent_submission_tracks(self):
for section, (_, fmt, track, _) in aamas.SECTIONS.items():
row, = aamas.parse_section(page(section, 'P_1', False), section, '2026-11-09T00:00:00Z', {})
self.assertEqual((row['track'], row['metadata']['publication_format']), (track, fmt))
self.assertIsNone(row['abstract'])
self.assertEqual(row['status'], 'accepted')
self.assertEqual(row['metadata']['decision_date_precision'], 'not_supplied')
self.assertEqual(row['metadata']['session']['title'], 'Session 1A – Robotics I')
self.assertIn({'namespace': 'aamas_proceedings:publication_format', 'value': fmt}, row['tags'])
self.assertEqual(aamas.SECTIONS['fullpapers'][2], 'proceedings_unspecified')
self.assertEqual(aamas.SECTIONS['extabstracts'][2], 'proceedings_unspecified')
def test_changed_edition_section_duplicate_id_or_pdf_link_is_rejected(self):
html = page('fullpapers', '1A_1')
for changed in [html.replace('AAMAS 2012', 'AAMAS 2025'),
html.replace('<h1>Full Papers</h1>', '<h1>Rejected Papers</h1>'),
html.replace('</body>', paper('1A_1')+'</body>'),
html.replace('../papers/1A_1.pdf', 'https://example.org/1A_1.pdf'),
html.replace('../papers/1A_1.pdf', '../papers/1A_2.pdf')]:
with self.subTest(changed=changed):
with self.assertRaises(ValueError):
aamas.parse_section(changed, 'fullpapers', '2026-11-09T00:00:00Z', {})
def test_frozen_responses_require_checksums_success_and_safe_paths(self):
for failure in ['checksum', 'status', 'path']:
with self.subTest(failure=failure), tempfile.TemporaryDirectory() as temporary:
root = self.snapshots(Path(temporary)/'source')
path = root/'receipts.json'
receipts = json.loads(path.read_text())
item = receipts['receipts'][0]
if failure == 'checksum': item['sha256'] = '0'*64
elif failure == 'status': item['status'] = 404
else: item['file'] = '../outside.html'
path.write_text(json.dumps(receipts))
with self.assertRaises(ValueError):
aamas.read_pages(snapshots=root)
def test_reviewed_section_counts_and_year_are_required(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
source = self.snapshots(root/'source')
with self.assertRaises(ValueError):
aamas.collect(2012, root/'out', snapshots=source)
self.assertFalse((root/'out/aamas-proceedings-2012.json').exists())
with self.assertRaises(ValueError):
aamas.collect(2026, root/'future', snapshots=source)
self.assertFalse((root/'future').exists())
def test_ids_colliding_across_sections_do_not_produce_a_completed_collection(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
source = self.snapshots(root/'source')
receipts_path = source/'receipts.json'
receipts = json.loads(receipts_path.read_text())
for item in receipts['receipts'][1:]:
path = source/item['file']
data = re.sub(r'P[0-9]_1', 'SAME_1', path.read_text()).encode()
path.write_bytes(data)
item.update(bytes=len(data), sha256=hashlib.sha256(data).hexdigest())
receipts_path.write_text(json.dumps(receipts))
small = {key: (*value[:3], 1) for key, value in aamas.SECTIONS.items()}
with patch.dict(aamas.SECTIONS, small, clear=True), self.assertRaises(ValueError):
aamas.collect(2012, root/'out', snapshots=source)
self.assertFalse((root/'out/aamas-proceedings-2012.json').exists())
def test_collection_and_native_ingestion_preserve_formats_evidence_and_missing_abstracts(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
source = self.snapshots(root/'source')
small = {key: (*value[:3], 1) for key, value in aamas.SECTIONS.items()}
with patch.dict(aamas.SECTIONS, small, clear=True):
result = aamas.collect(2012, root/'out', snapshots=source)
self.assertEqual(result['records'], 4)
self.assertEqual(len(result['files']), 3)
self.assertFalse(result['database_written'])
self.assertFalse(result['pdfs_downloaded'])
db = root/'sample.sqlite'
for manifest in result['files']:
path = Path(manifest['file'])
self.assertEqual(hashlib.sha256(path.read_bytes()).hexdigest(), manifest['file_sha256'])
for raw in path.read_text().splitlines():
record = json.loads(raw)
evidence = record['metadata']['source_evidence']
self.assertEqual(hashlib.sha256((root/'out'/evidence['file']).read_bytes()).hexdigest(), evidence['sha256'])
ingest_file(db, path)
with sqlite3.connect(db) as connection:
self.assertEqual(connection.execute('SELECT count(*) FROM papers').fetchone()[0], 4)
self.assertEqual(connection.execute('SELECT count(*) FROM submissions').fetchone()[0], 4)
self.assertEqual(connection.execute("SELECT count(*) FROM decisions WHERE status='accepted'").fetchone()[0], 4)
self.assertEqual(connection.execute('SELECT count(*) FROM papers WHERE abstract IS NULL').fetchone()[0], 1)
metadata = [json.loads(r[0]) for r in connection.execute('SELECT metadata_json FROM source_records')]
self.assertEqual({r['publication_format'] for r in metadata}, {'full_paper','extended_abstract','demo','doctoral_abstract'})
namespaces = {r[0] for r in connection.execute('SELECT DISTINCT namespace FROM tags')}
self.assertEqual(namespaces, {'aamas_proceedings:publication_format', 'aamas_proceedings:session'})
self.assertEqual(connection.execute('PRAGMA foreign_key_check').fetchall(), [])
+89
View File
@@ -0,0 +1,89 @@
"""Cross-source publisher identities must preserve attempts and decision history."""
import hashlib
import json
from pathlib import Path
import sqlite3
import tempfile
import unittest
from paper_atlas.db.submissions import identities_for_row
from paper_atlas.ingestion.records import ingest_file
DOI = '10.1234/aamas.identity.example'
def row(source, external_id, *, venue='AAMAS', year=2012, track='demo', status='accepted', identifiers=None):
return {'source': source, 'external_id': external_id,
'url': 'https://example.org/'+source+'/'+external_id,
'title': 'A shared publication with independently recorded evidence',
'abstract': 'An independently recorded source abstract.',
'authors': ['Jane Doe', 'Bob Smith'],
'identifiers': identifiers if identifiers is not None else {'doi': DOI},
'venue': venue, 'year': year, 'track': track, 'status': status,
'observed_at': '2026-11-09T00:00:00Z',
'raw_decision': source+' observed '+status,
'metadata': {'original_source': source}, 'tags': []}
def ingest(db, root, record):
path = root/(record['source']+'-'+record['external_id'].replace('/', '-')+'.jsonl')
path.write_text(json.dumps(record)+'\n')
path.with_suffix('.manifest.json').write_text(json.dumps({
'source': record['source'], 'parser_version': 1, 'venue': record['venue'],
'year': record['year'], 'track': record['track'], 'count': 1,
'list_url': 'https://example.org/'+record['source'], 'status': 'partial',
'file_sha256': hashlib.sha256(path.read_bytes()).hexdigest(),
'missing_abstract_count': 0}))
ingest_file(db, path)
class AamasPublicationIdentityTests(unittest.TestCase):
def test_only_accepted_known_publisher_rows_supply_publication_doi_identity(self):
record = row('aamas_proceedings', '2012/D1_1', identifiers={'doi': 'https://doi.org/'+DOI.upper()})
self.assertEqual(identities_for_row(record), [('publication_doi', DOI)])
for status in ['rejected', 'desk_rejected', 'withdrawn', 'under_review', 'unknown']:
with self.subTest(status=status):
self.assertEqual(identities_for_row({**record, 'status': status}), [])
self.assertEqual(identities_for_row({**record, 'source': 'unverified_source'}), [])
def test_same_edition_doi_joins_sources_in_both_orders_without_duplicate_attempts(self):
publisher = row('aamas_proceedings', '2012/D1_1')
bibliography = row('dblp', 'conf/atal/Example12')
for order in [(publisher, bibliography), (bibliography, publisher)]:
with self.subTest(first=order[0]['source']), tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
db = root/'sample.sqlite'
for record in order:
ingest(db, root, record)
ingest(db, root, publisher)
with sqlite3.connect(db) as connection:
self.assertEqual(connection.execute('SELECT count(*) FROM papers').fetchone()[0], 1)
self.assertEqual(connection.execute('SELECT count(*) FROM source_records').fetchone()[0], 2)
self.assertEqual(connection.execute('SELECT count(*) FROM submissions').fetchone()[0], 1)
self.assertEqual(connection.execute('SELECT count(*) FROM submission_sources').fetchone()[0], 2)
self.assertEqual(connection.execute('SELECT count(*) FROM decision_sources').fetchone()[0], 2)
self.assertEqual(connection.execute('SELECT scheme,value FROM submission_identifiers').fetchall(), [('publication_doi', DOI)])
metadata = [json.loads(r[0]) for r in connection.execute('SELECT metadata_json FROM source_records')]
self.assertEqual({r['original_source'] for r in metadata}, {'aamas_proceedings', 'dblp'})
self.assertEqual(connection.execute('PRAGMA foreign_key_check').fetchall(), [])
def test_cross_venue_rejection_and_distinct_tracks_remain_separate_attempts(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
db = root/'sample.sqlite'
rejected = row('openreview_snapshot', 'original-forum', venue='AAAI', year=2011,
track='main', status='rejected', identifiers={'arxiv': '1101.00001', 'openreview': 'original-forum'})
accepted = row('aamas_proceedings', '2012/D1_1', identifiers={'arxiv': '1101.00001', 'doi': DOI})
another_track = row('aamas_proceedings', '2012/F1_1', track='proceedings_unspecified', identifiers={'doi': DOI})
for record in [rejected, accepted, another_track]:
ingest(db, root, record)
with sqlite3.connect(db) as connection:
self.assertEqual(connection.execute('SELECT count(*) FROM papers').fetchone()[0], 1)
self.assertEqual(connection.execute('SELECT count(*) FROM submissions').fetchone()[0], 3)
history = connection.execute('''SELECT v.acronym,e.year,e.track,d.status FROM decisions d
JOIN submissions s ON s.id=d.submission_id JOIN editions e ON e.id=s.edition_id
JOIN venues v ON v.id=e.venue_id ORDER BY v.acronym,e.track''').fetchall()
self.assertEqual(set(history), {('AAAI',2011,'main','rejected'),('AAMAS',2012,'demo','accepted'),('AAMAS',2012,'proceedings_unspecified','accepted')})
self.assertEqual(connection.execute('SELECT count(*) FROM decision_sources').fetchone()[0], 3)
self.assertEqual(connection.execute('PRAGMA foreign_key_check').fetchall(), [])
+114
View File
@@ -0,0 +1,114 @@
"""Pinned T78 evidence, edition-local aliases and an explicitly identified material."""
import copy
import hashlib
import json
from pathlib import Path
import sqlite3
import tempfile
import unittest
import xml.etree.ElementTree as ET
from paper_atlas.collectors.acl import collect_xml, volume_scope
from paper_atlas.collectors.acl_historical import event_scope, material_reason
from paper_atlas.db.store import Store
from paper_atlas.ingestion.records import ingest_file
FIXTURE = Path(__file__).parent/'fixtures/acl-t78.xml'
COMMIT = 'a7be623d01c48c61405dec31c871121cdbed5a01'
class AclHistoricalTests(unittest.TestCase):
def setUp(self):
self.root = ET.parse(FIXTURE).getroot()
self.volume = self.root.find('volume')
self.meta = self.volume.find('meta')
def test_frozen_volume_matches_official_snapshot(self):
self.assertEqual(hashlib.sha256(FIXTURE.read_bytes()).hexdigest(), 'f712f94cff0cba05f786b7ac31dad2ea9408fde2a17a04531130df623f57dd84')
self.assertEqual(len(self.volume.findall('paper')), 36)
self.assertEqual(volume_scope(self.meta, '1', 'T78'), ('ACL+TINLAP', ['ACL', 'TINLAP'], 'proceedings_unspecified'))
def test_unknown_year_collection_or_volume_is_not_globally_aliased(self):
for collection, volume in [('T75', '1'), ('T87', '1'), ('T78', '2'), ('C65', '1')]:
with self.subTest(collection=collection, volume=volume):
self.assertIsNone(event_scope(self.meta, volume, collection))
self.assertIsNone(volume_scope(self.meta, volume, collection))
self.assertIsNone(volume_scope(self.meta, '1'))
def test_known_volume_scope_rejects_critical_metadata_drift(self):
for field, replacement in [('venue', 'acl'), ('year', '1979'), ('booktitle', 'Another TINLAP'), ('editor/first', 'Another')]:
changed = copy.deepcopy(self.meta)
changed.find(field).text = replacement
with self.subTest(field=field), self.assertRaises(ValueError):
event_scope(changed, '1', 'T78')
changed = copy.deepcopy(self.meta)
ET.SubElement(changed, 'year').text = '1978'
with self.assertRaises(ValueError):
event_scope(changed, '1', 'T78')
def test_evidence_retains_original_series_dates_and_role_boundary(self):
scope = event_scope(self.meta, '1', 'T78')
self.assertEqual(scope['original_venue'], 'TINLAP')
self.assertEqual(scope['event_dates'], ['1978-07-25', '1978-07-27'])
self.assertEqual([e['pdf_pages'] for e in scope['evidence']], [[9, 10], [3]])
self.assertIn('not identified', scope['track_scope_note'])
self.assertIn('pending', scope['evidence'][0]['scope'])
def test_only_exact_session_questions_material_is_excluded(self):
paper = copy.deepcopy(self.volume.findall('paper')[-1])
self.assertIn('Session question list', material_reason('T78', '1', paper))
self.assertIsNone(material_reason('C65', '1', paper))
self.assertIsNone(material_reason('T78', '2', paper))
paper.set('id', '37')
self.assertIsNone(material_reason('T78', '1', paper))
paper.set('id', '36')
ET.SubElement(paper, 'author')
with self.assertRaises(ValueError):
material_reason('T78', '1', paper)
def test_all_authored_source_fields_and_material_receipt_survive_collection(self):
with tempfile.TemporaryDirectory() as temporary:
manifest, = collect_xml(FIXTURE, temporary, COMMIT)
rows = [json.loads(raw) for raw in Path(manifest['file']).read_text().splitlines()]
self.assertEqual(manifest['count'], 35)
self.assertEqual(manifest['xml_paper_elements'], 36)
self.assertEqual(manifest['missing_abstract_count'], 35)
self.assertEqual(manifest['excluded_materials'][0]['external_id'], 'T78-1036')
for number, row in enumerate(rows, 1):
paper = self.volume.findall('paper')[number-1]
self.assertEqual(row['external_id'], f'T78-1{number:03d}')
self.assertEqual(row['title'], ' '.join(''.join(paper.find('title').itertext()).split()))
self.assertEqual(row['metadata']['venue_codes'], ['tinlap'])
self.assertEqual(row['metadata']['historical_event_scope']['original_venue'], 'TINLAP')
self.assertEqual(row['metadata']['metadata_commit'], COMMIT)
self.assertEqual((row['status'], row['track'], row['abstract']), ('accepted', 'proceedings_unspecified', None))
def test_old_coling_scope_is_unchanged_by_calendar_replacement_evidence(self):
meta = ET.fromstring('<meta><venue>coling</venue><year>1965</year><booktitle>COLING 1965</booktitle></meta>')
self.assertEqual(volume_scope(meta, '1', 'C65'), ('COLING', ['COLING'], 'main'))
self.assertIsNone(event_scope(meta, '1', 'C65'))
def test_native_ingestion_does_not_alias_other_tinlap_years_or_infer_negative_status(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
manifest, = collect_xml(FIXTURE, root/'source', COMMIT)
db = root/'sample.sqlite'
store = Store(db)
pid, rid = store.record('test_control', 'tinlap1975', 'https://example.org/tinlap1975',
'An earlier TINLAP contribution', None, ['Jane Doe'])
store.decision(pid, rid, 'TINLAP', 1975, 'main', 'tinlap1975', 'https://example.org/tinlap1975', 'accepted', 'test control')
store.conn.commit()
store.conn.close()
ingest_file(db, manifest['file'])
ingest_file(db, manifest['file'])
with sqlite3.connect(db) as c:
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0], 36)
self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0], 36)
self.assertEqual(c.execute('SELECT count(*) FROM decisions').fetchone()[0], 36)
self.assertEqual(c.execute("SELECT count(*) FROM decisions WHERE status!='accepted'").fetchone()[0], 0)
self.assertIsNone(c.execute("SELECT 1 FROM source_records WHERE external_id='T78-1036'").fetchone())
aliases = c.execute('SELECT v.acronym,a.alias FROM venue_aliases a JOIN venues v ON v.id=a.venue_id ORDER BY a.alias').fetchall()
self.assertEqual(aliases, [('ACL+TINLAP', 'ACL'), ('ACL+TINLAP', 'TINLAP')])
metadata = json.loads(c.execute("SELECT metadata_json FROM source_records WHERE external_id='T78-1001'").fetchone()[0])
self.assertEqual(metadata['historical_event_scope']['event_year'], 1978)
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(), [])
+151
View File
@@ -0,0 +1,151 @@
"""Original series, exact edition boundaries, and complete candidate discovery."""
from contextlib import closing
from copy import deepcopy
import gzip
import hashlib
import json
from pathlib import Path
import sqlite3
import tempfile
import unittest
from types import SimpleNamespace
from unittest.mock import patch
from paper_atlas.collectors.dblp import catalogue, XML_SHA
from paper_atlas.collectors.dblp_records import collect, normalize_row
from paper_atlas.ingestion.records import ingest_file
FIXTURE = json.loads((Path(__file__).parent/'fixtures/historical_predecessors.json').read_text())
EXAMPLES = {row['key']: row for row in FIXTURE['volumes']}
def example(key, *, bibliographic_year=None):
fixed = EXAMPLES[key]
family = key.split('/')[1]
year = int(key.split('/')[-1])
volume = {'record_type': 'proceedings', 'key': key, 'fields': deepcopy(fixed['fields'])}
row = {'record_type': 'inproceedings', 'key': f'conf/{family}/SyntheticExample{year}', 'publtype': None,
'fields': {'title': [f'Synthetic historical research example {family} {year}'],
'author': ['Jane Doe 0001', 'John Smith'], 'booktitle': fixed['fields']['booktitle'],
'year': [str(bibliographic_year or year)], 'crossref': [key],
'ee': [f'https://doi.org/10.1234/synthetic.{family}.{year}']}}
return row, volume
class HistoricalPredecessorTests(unittest.TestCase):
def normalize(self, key, **kwargs):
row, volume = example(key, **kwargs)
fixed = EXAMPLES[key]
result, reason = normalize_row(row, {key: volume}, {fixed['expected_target_family']})
self.assertIsNone(reason)
self.assertIsNotNone(result)
return result
def test_ewsl_primary_series_retains_original_name_and_family_alias(self):
for year in [1987, 1988, 1989, 1991]:
with self.subTest(year=year):
record = self.normalize(f'conf/ecml/{year}')
self.assertEqual((record['venue'],record['venue_aliases'],record['year'],record['track']), ('EWSL',['ECML-PKDD'],year,'main'))
proof = record['metadata']['historical_predecessor_evidence']
self.assertEqual(proof['original_series'], 'EWSL')
self.assertEqual(proof['history_url'], 'https://ecmlpkdd2011.org/pastConferences.php')
self.assertEqual(record['metadata']['bibliography']['booktitle'], ['EWSL'])
self.assertEqual(record['metadata']['author_names_raw'], ['Jane Doe 0001','John Smith'])
self.assertEqual(record['authors'], ['Jane Doe','John Smith'])
self.assertEqual(EXAMPLES['conf/ecml/1989']['bibliographic_counts'], [])
def test_eps_ecp_ewsp_keep_original_series_instead_of_renaming_to_icaps(self):
for key in ['conf/aips/1990','conf/ecp/1997','conf/ecp/1999','conf/ewsp/1991']:
with self.subTest(key=key):
record = self.normalize(key)
fixed = EXAMPLES[key]
self.assertEqual(record['venue'], fixed['expected_original_series'])
self.assertEqual(record['venue_aliases'], ['ICAPS'])
self.assertEqual(record['track'], 'main')
proof = record['metadata']['historical_predecessor_evidence']
self.assertEqual(proof['history_url'], 'https://www.icaps-conference.org/conference-series/')
self.assertEqual(proof['proceedings_title'], fixed['fields']['title'][0])
self.assertEqual(proof['proceedings_year_fields'], fixed['fields']['year'])
def test_altered_titles_keys_suffixes_publication_years_and_unreviewed_editions_stay_held(self):
for key in ['conf/ecml/1988','conf/aips/1990','conf/ecp/1997','conf/ewsp/1991']:
row, volume = example(key)
for changed in ['title','key','suffix','publication_year']:
with self.subTest(key=key, changed=changed):
r, v = deepcopy(row), deepcopy(volume)
if changed == 'title': v['fields']['title'][0] += ', Affiliated Workshop'
elif changed == 'key': v['key'] += 'w'; r['fields']['crossref'] = [v['key']]
elif changed == 'suffix': r['fields']['booktitle'][0] += ' (Workshop)'
else: v['fields']['year'] = ['2001']
result, _ = normalize_row(r, {v['key']: v}, {EXAMPLES[key]['expected_target_family']})
self.assertIsNone(result)
row, volume = example('conf/ecp/1997')
row['fields']['year'] = ['2001']; row['fields']['crossref'] = ['conf/ecp/2001']
volume['key'] = 'conf/ecp/2001'; volume['fields']['year'] = ['2001']
self.assertIsNone(normalize_row(row, {volume['key']:volume}, {'ICAPS'})[0])
def test_ecp1999_event_and_bibliographic_years_are_independently_retained(self):
record = self.normalize('conf/ecp/1999', bibliographic_year=2000)
self.assertEqual(record['year'], 1999)
self.assertEqual(record['metadata']['bibliographic_year'], 2000)
self.assertEqual(record['metadata']['bibliography']['year'], ['2000'])
self.assertEqual(record['metadata']['historical_predecessor_evidence']['proceedings_year_fields'], ['2000'])
self.assertEqual(record['metadata']['historical_predecessor_evidence']['event_year'], 1999)
def test_old_candidates_require_new_families_only_when_requested_years_need_them(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary); row, volume = example('conf/aips/1990')
db = root/'dblp-catalogue.sqlite'
with closing(sqlite3.connect(db)) as connection, connection:
connection.execute('CREATE TABLE proceedings(key TEXT PRIMARY KEY,metadata_json TEXT)')
connection.execute('INSERT INTO proceedings VALUES(?,?)', (volume['key'],json.dumps(volume)))
candidate = root/'candidate.jsonl'; candidate.write_text(json.dumps(row)+'\n')
db.with_name('dblp-ai-candidates.manifest.json').write_text(json.dumps({
'snapshot_sha256': XML_SHA, 'candidate_sha256': hashlib.sha256(candidate.read_bytes()).hexdigest(),
'candidate_records': 1, 'candidate_family_counts': {'aips':1},
'record_types': {'proceedings':1,'inproceedings':1}}))
# Tiny CI fixtures may live on tmpfs; production reserve is tested separately.
with patch('paper_atlas.collectors.dblp_records.shutil.disk_usage', return_value=SimpleNamespace(free=16*1024**3)):
result = collect(db, root/'early', ['ICAPS'], candidates=candidate, through_year=1990)
self.assertEqual((result['normalized_records'],result['files'][0]['venue']), (1,'EPS'))
for year, missing in [(1991,'ICAPS/ewsp'),(1997,'ICAPS/ecp')]:
out = root/str(year); out.mkdir(); sentinel=out/'previous.txt';sentinel.write_text('accepted previous output')
with self.subTest(year=year), self.assertRaisesRegex(ValueError, missing):
collect(db, out, ['ICAPS'], candidates=candidate, through_year=year)
self.assertEqual(sentinel.read_text(), 'accepted previous output')
self.assertFalse((out/'dblp-normalization.json').exists())
def test_new_catalogue_includes_previously_missing_families_without_decision_inference(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary);dtd=root/'dblp.dtd';dtd.write_bytes(b'<!ENTITY eacute "&#233;">')
archive=root/'dblp.xml.gz'
body=''.join(f'<inproceedings key="conf/{family}/Synthetic"><title>Synthetic example</title><year>{year}</year><booktitle>{book}</booktitle><crossref>conf/{family}/{year}</crossref></inproceedings>' for family,year,book in [('ecp',1997,'ECP'),('ewsp',1991,'EWSP')])
with gzip.open(archive,'wb') as stream:stream.write(('<?xml version="1.0"?><!DOCTYPE dblp SYSTEM "dblp.dtd"><dblp>'+body+'</dblp>').encode())
result=catalogue(archive,dtd,root/'out',hashlib.sha256(archive.read_bytes()).hexdigest(),hashlib.md5(dtd.read_bytes()).hexdigest())
rows=[json.loads(line)for line in Path(result['file']).read_text().splitlines()]
self.assertEqual({r['key'] for r in rows}, {'conf/ecp/Synthetic','conf/ewsp/Synthetic'})
self.assertEqual(result['candidate_records'], 2)
self.assertTrue(all('status' not in r and 'venue' not in r for r in rows))
def test_native_import_keeps_original_series_aliases_and_full_source_evidence(self):
with tempfile.TemporaryDirectory() as temporary:
root=Path(temporary);db=root/'sample.sqlite'
for key in ['conf/ecml/1988','conf/ecp/1999']:
record=self.normalize(key);path=root/(key.replace('/','-')+'.jsonl')
path.write_text(json.dumps(record,ensure_ascii=False)+'\n')
path.with_suffix('.manifest.json').write_text(json.dumps({'source':'dblp','venue':record['venue'],'year':record['year'],'track':'main','count':1,'list_url':'https://doi.org/10.4230/dblp.xml.2026-11-01','file_sha256':hashlib.sha256(path.read_bytes()).hexdigest(),'status':'partial','missing_abstract_count':1}))
ingest_file(db,path)
with closing(sqlite3.connect(db)) as connection:
rows=connection.execute('SELECT v.acronym,a.alias,e.year FROM editions e JOIN venues v ON v.id=e.venue_id JOIN venue_aliases a ON a.venue_id=v.id ORDER BY v.acronym').fetchall()
self.assertEqual(rows,[('ECP','ICAPS',1999),('EWSL','ECML-PKDD',1988)])
metadata=[json.loads(r[0])for r in connection.execute('SELECT metadata_json FROM source_records')]
self.assertEqual({r['historical_predecessor_evidence']['original_series'] for r in metadata}, {'ECP','EWSL'})
self.assertEqual(connection.execute('SELECT count(*) FROM decisions WHERE status=?',('accepted',)).fetchone()[0],2)
self.assertEqual(connection.execute('PRAGMA foreign_key_check').fetchall(),[])
def test_existing_aips_normalization_remains_original_aips(self):
row, volume=example('conf/aips/1990');row['fields']['booktitle']=['AIPS'];row['fields']['year']=['1994'];row['fields']['crossref']=['conf/aips/1994']
volume['key']='conf/aips/1994';volume['fields']['title']=['AIPS 1994 Proceedings'];volume['fields']['year']=['1994']
record,reason=normalize_row(row,{volume['key']:volume},{'ICAPS'})
self.assertIsNone(reason);self.assertEqual((record['venue'],record['venue_aliases']),('AIPS',['ICAPS']))
self.assertNotIn('historical_predecessor_evidence',record['metadata'])
+115
View File
@@ -0,0 +1,115 @@
"""Regression coverage for primary series once named Workshop, with strict scope boundaries."""
from contextlib import closing
import hashlib
import json
from pathlib import Path
import sqlite3
import tempfile
import unittest
from paper_atlas.collectors.dblp_records import normalize_row
from paper_atlas.collectors.dblp_scope import scope
from paper_atlas.ingestion.records import ingest_file
ICML_HISTORY = 'https://icml.cc/Conferences/2008/past_icmls.shtml.html'
IROS_HISTORY = 'https://www.ieee-ras.org/conferences-workshops/financially-co-sponsored/iros/iros-past-and-future-venues/'
class HistoricalPrimarySeriesTests(unittest.TestCase):
def example(self, family, year, title, book=None, key=None):
key = key or f'conf/{family}/{year}'
book = book or ('ML' if family == 'icml' else 'IROS')
row = {'record_type':'inproceedings', 'key':f'conf/{family}/Paper{year}', 'publtype':None,
'fields':{'title':['An independently identified historical research paper'],
'author':['Jane Doe'], 'booktitle':[book], 'year':[str(year)],
'crossref':[key], 'ee':[f'https://doi.org/10.1234/{family}.{year}']}}
volume = {'record_type':'proceedings', 'key':key,
'fields':{'title':[title], 'year':[str(year)]}}
return row, volume
def test_icml_workshop_primary_editions_are_not_silently_filtered(self):
examples = {
1989:'Proceedings of the Sixth International Workshop on Machine Learning (ML 1989), Cornell University, Ithaca, New York, USA, June 26-27, 1989',
1991:'Proceedings of the Eighth International Workshop (ML91), Northwestern University, Evanston, Illinois, USA',
1992:'Proceedings of the Ninth International Workshop on Machine Learning (ML 1992), Aberdeen, Scotland, UK, July 1-3, 1992',
}
for year, title in examples.items():
with self.subTest(year=year):
row, volume = self.example('icml', year, title)
record, reason = normalize_row(row, {volume['key']:volume}, {'ICML'})
self.assertIsNone(reason)
self.assertEqual((record['venue'],record['year'],record['track'],record['status']), ('ICML',year,'main','accepted'))
self.assertEqual(record['metadata']['proceedings_title'], title)
self.assertEqual(record['metadata']['track_scope_evidence_url'], ICML_HISTORY)
self.assertEqual(record['metadata']['bibliographic_year'], year)
def test_iros_first_three_editions_are_primary_series(self):
examples = {
1988:"IEEE International Workshop on Intelligent Robots and Systems '88, Proceedings. IROS 1988, Tokyo, Japan, October 31 - November 2, 1988",
1989:"IEEE International Workshop on Intelligent Robots and Systems '89, Proceedings. IROS 1989, September 4-6, 1989, Tsukuba, Japan",
1990:"IEEE International Workshop on Intelligent Robots and Systems '90, Towards a New Frontier of Applications, Proceedings. IROS 1990, July 3-6, 1990, Ibaraki, Japan",
}
for year, title in examples.items():
with self.subTest(year=year):
row, volume = self.example('iros', year, title)
record, reason = normalize_row(row, {volume['key']:volume}, {'IROS'})
self.assertIsNone(reason)
self.assertEqual((record['venue'],record['year'],record['track']), ('IROS',year,'main'))
self.assertEqual(record['metadata']['proceedings_title'], title)
self.assertEqual(record['metadata']['track_scope_evidence_url'], IROS_HISTORY)
def test_main_named_icml_predecessors_also_carry_primary_scope_provenance(self):
examples = {
1988:'Machine Learning, Proceedings of the Fifth International Conference on Machine Learning, Ann Arbor, Michigan, USA, June 12-14, 1988',
1990:'Machine Learning, Proceedings of the Seventh International Conference on Machine Learning, Austin, Texas, USA, June 21-23, 1990',
}
for year, title in examples.items():
row, volume = self.example('icml', year, title)
edition, reason = scope(row, volume)
self.assertIsNone(reason)
self.assertEqual(edition['track'], 'main')
self.assertEqual(edition['track_scope_evidence_url'], ICML_HISTORY)
def test_wrong_keys_titles_suffixes_and_unknown_years_keep_workshop_boundary(self):
title = 'Proceedings of the Eighth International Workshop (ML91), Northwestern University, Evanston, Illinois, USA'
for year, changed, book, key in [
(1991,title,'ML','conf/icml/1991w'),
(1991,title,'ML (Workshops)','conf/icml/1991'),
(1991,title.replace('Eighth','Ninth'),'ML','conf/icml/1991'),
(1991,title + ', Affiliated Workshop','ML','conf/icml/1991'),
(1987,'Machine Learning Workshop 1987, Irvine','ML','conf/icml/1987'),
]:
with self.subTest(year=year, book=book, key=key, title=changed):
row, volume = self.example('icml',year,changed,book,key)
record, reason = normalize_row(row,{volume['key']:volume},{'ICML'})
self.assertIsNone(record)
self.assertEqual(reason,'separate_workshop')
self.assertNotIn('track_scope_evidence_url',scope(row,volume)[0])
title = "IEEE International Workshop on Intelligent Robots and Systems '88, Proceedings. IROS 1988, Tokyo, Japan, October 31 - November 2, 1988"
for changed, book, key in [
(title,'IROS','conf/iros/1988w'),
(title,'IROS (Workshops)','conf/iros/1988'),
(title.replace('Tokyo','Another city'),'IROS','conf/iros/1988'),
]:
row, volume = self.example('iros',1988,changed,book,key)
self.assertEqual(normalize_row(row,{volume['key']:volume},{'IROS'})[1],'separate_workshop')
def test_primary_scope_proof_survives_native_file_ingestion(self):
title = 'Proceedings of the Eighth International Workshop (ML91), Northwestern University, Evanston, Illinois, USA'
row, volume = self.example('icml',1991,title)
record, reason = normalize_row(row,{volume['key']:volume},{'ICML'})
self.assertIsNone(reason)
with tempfile.TemporaryDirectory() as temporary:
root=Path(temporary);path=root/'records.jsonl';path.write_text(json.dumps(record)+'\n')
path.with_suffix('.manifest.json').write_text(json.dumps({'source':'dblp','count':1,'list_url':ICML_HISTORY,'file_sha256':hashlib.sha256(path.read_bytes()).hexdigest()}))
ingest_file(root/'test.sqlite',path)
with closing(sqlite3.connect(root/'test.sqlite')) as connection:
self.assertEqual(connection.execute('SELECT track FROM editions').fetchone()[0],'main')
self.assertEqual(connection.execute('SELECT status FROM decisions').fetchone()[0],'accepted')
metadata=json.loads(connection.execute('SELECT metadata_json FROM source_records').fetchone()[0])
self.assertEqual(metadata['proceedings_title'],title)
self.assertEqual(metadata['track_scope_evidence_url'],ICML_HISTORY)
if __name__ == '__main__':
unittest.main()
+148
View File
@@ -0,0 +1,148 @@
"""Historical accepted lists, repeated presentations and preserved work histories."""
import hashlib
import json
from pathlib import Path
import sqlite3
import tempfile
import unittest
from unittest.mock import patch
from paper_atlas.collectors import iclr_archive as archive
from paper_atlas.db.store import Store
from paper_atlas.ingestion.records import ingest_file
OBSERVED = '2026-11-09T00:00:00Z'
def entry(identifier, title='A shared research paper', display='Jane Doe and Bob Smith'):
return f'<li><div><a href="http://arxiv.org/abs/{identifier}">{title}</a>, {display}</div></li>'
def page2015():
return ('<h3>Main Conference - Oral Presentations</h3><ol>'+entry('1412.0001v2')+'</ol>'
'<h3>Main Conference - Poster Presentations</h3><ol>'+entry('1412.0001v2')+'</ol>'
'<h3>Workshop Papers</h3><ol>'+entry('1412.0002', 'A workshop paper')+'</ol>')
class IclrArchiveTests(unittest.TestCase):
def frozen(self, root, year, html):
root.mkdir()
data = html.encode()
(root/'page.html').write_bytes(data)
receipt = {'url': archive.PAGES[year][0], 'file': 'page.html', 'status': 200,
'bytes': len(data), 'sha256': hashlib.sha256(data).hexdigest(), 'finished_at': OBSERVED}
(root/'receipts.json').write_text(json.dumps({'receipts': [receipt]}))
return root
def test_oral_poster_repeat_is_one_attempt_with_all_source_variants(self):
rows = archive.parse_page(page2015(), 2015, {'observed_at': OBSERVED})
main, workshop = rows
self.assertEqual((main['track'], workshop['track']), ('main', 'workshop'))
self.assertEqual(main['identifiers'], {'arxiv': '1412.0001'})
self.assertEqual(main['external_id'], '2015/main/1412.0001')
self.assertEqual(main['metadata']['presentations'], ['oral', 'poster'])
self.assertEqual(len(main['metadata']['source_variants']), 2)
self.assertEqual(main['metadata']['source_variants'][0]['version'], 'v2')
self.assertEqual(main['authors'], ['Jane Doe', 'Bob Smith'])
self.assertTrue(all(r['status'] == 'accepted' and r['abstract'] is None for r in rows))
def test_2014_author_span_outside_paragraph_and_trailing_separator(self):
html = ('<h3>Conference Proceedings</h3><p>conference papers accepted</p>'
'<div><b><a href="http://arxiv.org/abs/1312.0001">A <i>research</i> title</a></b></div>'
'<p><span style="font-style: italic">Jane Doe; </span>'
'<span style="font-style:italic">Bob Smith; Carol Wu;</span></p>'
'<div><a href="http://arxiv.org/abs/1312.0002">Another title</a>'
'<div><i><span>First Author; </span><span>Second Author</span></i></div></div>')
row, unwrapped = archive.parse_page(html, 2014, {'observed_at': OBSERVED})
self.assertEqual(row['title'], 'A research title')
self.assertEqual(row['authors'], ['Jane Doe', 'Bob Smith', 'Carol Wu'])
self.assertEqual(row['metadata']['source_variants'][0]['authors_display'], 'Jane Doe; Bob Smith; Carol Wu;')
self.assertEqual(row['track'], 'main')
self.assertEqual(unwrapped['authors'], ['First Author', 'Second Author'])
def test_same_arxiv_in_different_tracks_keeps_separate_source_attempts(self):
html = page2015().replace('1412.0002', '1412.0001v3')
rows = archive.parse_page(html, 2015, {'observed_at': OBSERVED})
self.assertEqual(len(rows), 2)
self.assertEqual({r['external_id'] for r in rows}, {'2015/main/1412.0001', '2015/workshop/1412.0001'})
self.assertEqual(rows[1]['metadata']['source_variants'][0]['version'], 'v3')
def test_2016_code_links_and_breaks_do_not_become_authors(self):
html = ('<h1>Accepted Papers (Conference Track)</h1><h3>Oral Presentations</h3><ol>'
'<li><a href="http://arxiv.org/abs/1511.00001">A title</a>'
'<a href="https://example.org/code">[code]</a><br>Jane Doe, Bob Smith</li></ol>'
'<h3>Poster Presentations</h3>')
row, = archive.parse_page(html, 2016, {'observed_at': OBSERVED})
self.assertEqual(row['authors'], ['Jane Doe', 'Bob Smith'])
self.assertEqual(row['metadata']['presentations'], ['oral'])
def test_conflicting_repeat_unknown_sections_and_unparsed_links_fail(self):
variants = [page2015().replace('Poster Presentations', 'Other Presentations'),
page2015().replace('</ol>', '</ol><a href="http://arxiv.org/abs/1412.0003">stray</a>', 1),
page2015().replace('Jane Doe and Bob Smith', 'Different Author', 1)]
for html in variants:
with self.subTest(html=html), self.assertRaises(ValueError):
archive.parse_page(html, 2015, {'observed_at': OBSERVED})
def test_arxiv_versions_are_preserved_and_unsafe_links_rejected(self):
self.assertEqual(archive.arxiv_link('https://arxiv.org/pdf/1511.00001v3.pdf'), ('1511.00001', 'v3'))
self.assertIsNone(archive.arxiv_link('https://arxiv.org.example.org/abs/1511.00001'))
for url in ['https://arxiv.org/abs/1511.00001?other=1', 'ftp://arxiv.org/abs/1511.00001',
'https://user@arxiv.org/abs/1511.00001', 'https://arxiv.org/abs/not-a-paper']:
with self.subTest(url=url), self.assertRaises(ValueError):
archive.arxiv_link(url)
def test_receipt_hash_status_path_and_symlink_cannot_escape_snapshot(self):
for failure in ['hash', 'status', 'path', 'symlink']:
with self.subTest(failure=failure), tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
source = self.frozen(root/'source', 2015, page2015())
file = source/'receipts.json'
data = json.loads(file.read_text())
receipt = data['receipts'][0]
if failure == 'hash': receipt['sha256'] = '0'*64
elif failure == 'status': receipt['status'] = 404
elif failure == 'path': receipt['file'] = '../outside.html'
else:
(root/'outside.html').write_text(page2015())
(source/'page.html').unlink()
(source/'page.html').symlink_to(root/'outside.html')
file.write_text(json.dumps(data))
with self.assertRaises(ValueError):
archive.read_page(2015, snapshots=source)
def test_changed_inventory_leaves_no_completed_output(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
source = self.frozen(root/'source', 2015, page2015())
with self.assertRaises(ValueError):
archive.collect(2015, root/'out', snapshots=source)
self.assertFalse((root/'out/iclr-archive-2015.json').exists())
def test_native_import_keeps_rejected_to_accepted_history_and_abstract(self):
with tempfile.TemporaryDirectory() as temporary:
root = Path(temporary)
source = self.frozen(root/'source', 2015, page2015())
with patch.dict(archive.PAGES, {2015: (archive.PAGES[2015][0], {'main': 1, 'workshop': 1})}):
result = archive.collect(2015, root/'out', snapshots=source)
with self.assertRaises(FileExistsError):
archive.collect(2015, root/'out', snapshots=source)
db = root/'sample.sqlite'
store = Store(db)
pid, rid = store.record('arxiv', '1412.0001', 'https://arxiv.org/abs/1412.0001',
'A shared research paper', 'Keep this canonical abstract.',
['Jane Doe', 'Bob Smith'], {'arxiv': '1412.0001'}, {})
store.decision(pid, rid, 'Earlier Conference', 2014, 'main', 'original-attempt',
'https://example.org/rejected', 'rejected', 'explicit earlier decision', observed_at=OBSERVED)
store.conn.commit()
store.conn.close()
for manifest in result['files']:
ingest_file(db, manifest['file'])
ingest_file(db, manifest['file'])
with sqlite3.connect(db) as c:
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0], 2)
self.assertEqual(c.execute('SELECT abstract FROM papers WHERE id=?', (pid,)).fetchone()[0], 'Keep this canonical abstract.')
history = c.execute('SELECT v.acronym,e.year,d.status FROM decisions d JOIN submissions s ON s.id=d.submission_id JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id WHERE s.paper_id=? ORDER BY e.year', (pid,)).fetchall()
self.assertEqual(history, [('Earlier Conference', 2014, 'rejected'), ('ICLR', 2015, 'accepted')])
self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0], 3)
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(), [])