Compare commits
11 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 5c78a36fd5 | |||
| 7facde6c94 | |||
| c06981b244 | |||
| b226b3b98b | |||
| 05ada8daf8 | |||
| 13408869c2 | |||
| 443a79dce2 | |||
| ec9edd88d4 | |||
| 8ad052f577 | |||
| aa696bd7b5 | |||
| 90bd534edd |
@@ -12,7 +12,7 @@ jobs:
|
||||
- name: Verify metadata storage and query commands
|
||||
run: |
|
||||
python3 -B - <<'PY'
|
||||
import io, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request
|
||||
import gzip, io, json, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request
|
||||
sha = os.environ['ARCHIVE_SHA']
|
||||
if not re.fullmatch(r'[0-9a-fA-F]{40,64}', sha):
|
||||
raise SystemExit('Invalid source commit')
|
||||
@@ -50,7 +50,21 @@ jobs:
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','www-archive','--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--year','2025','--snapshots',str(src/'tests/fixtures/aamas2025'),'--out',str(root/'installed-aamas2025')],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','iclr-archive','--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','acl','--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','springer-selected','--help'],cwd=outside,check=True)
|
||||
springer = json.loads(gzip.decompress((src/'tests/fixtures/springer-aistats1993-fixture.json.gz').read_bytes()))
|
||||
for group in ('toc','chapters'):
|
||||
folder = root/'springer-snapshots'/group
|
||||
folder.mkdir(parents=True)
|
||||
for name, content in springer[group].items():
|
||||
(folder/name).write_text(content)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','springer-selected','--toc-snapshots',str(root/'springer-snapshots/toc'),'--chapter-snapshots',str(root/'springer-snapshots/chapters'),'--out',str(root/'installed-springer')],cwd=outside,check=True)
|
||||
springer_report = json.loads((root/'installed-springer/collection-report.json').read_bytes())
|
||||
assert springer_report['records'] == 49 and not springer_report['database_written']
|
||||
springer_manifest, = springer_report['files']
|
||||
assert springer_manifest['status'] == 'partial' and springer_manifest['missing_abstract_count'] == 0
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','postgres','--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','edition-tracks','--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','joint-editions','--help'],cwd=outside,check=True)
|
||||
|
||||
@@ -8,8 +8,12 @@ Paper Atlas 将 arXiv、AI 会议论文集及公开投稿决定整理到可追
|
||||
|
||||
历史 ICLR 2014–2016 官方录用页由 `collect iclr-archive` 适配,保留主会/workshop及重复展示证据;运行和身份核验边界见 [历史档案说明](docs/coverage/iclr-archive-2014-2016.md)。
|
||||
|
||||
`collect springer-selected` 核验限定的会后选集,分开保存会议年份、出版年份及选集事件身份。目前支持 AISTATS 1993 对应的 1994 年出版选集,原始会议决定仍须单独核查;配置、证据和恢复见 [会后选集说明](docs/springer-selected-volumes.md)。
|
||||
|
||||
## 项目结构
|
||||
|
||||
ACL 1978 的年会与 TINLAP-2 来源关系按精确卷适配,保留原系列和不明逐篇角色;1965 的替代关系不转成全局别名。来源、排除条目和恢复边界见 [ACL 早期来源核查](docs/coverage/acl-early-sources-2026-11-09.md)。
|
||||
|
||||
```text
|
||||
paper_atlas/
|
||||
cli.py 统一命令入口
|
||||
@@ -64,7 +68,9 @@ paper-atlas collect colm --cache /data/paper-index/raw/http-cache \
|
||||
|
||||
早期主系列的Workshop名称按[ICML/IROS精确届次证据](docs/coverage/historical-primary-series-2026-11-09.md)识别,保留原名与范围证据;代码支持不代表已补齐生产数据。
|
||||
|
||||
AAMAS 的[官方论文集采集器](docs/aamas-proceedings.md)目前核验了 2012 版式,可从缓存或已校验快照采集标题、作者及页面提供的摘要;发表形式单独保存,全文/扩展摘要中未明确的投稿 track 保持待明确,不根据会话主题推定主会轨道。
|
||||
EWSL、EPS、ECP、EWSP 的[具体前身卷与完整快照要求](docs/coverage/historical-predecessors-2026-11-09.md)单独核验,保留原始系列及事件/出版年份;旧候选文件缺少 ECP/EWSP 时会明确提示,避免静默遗漏。
|
||||
|
||||
AAMAS 的[官方论文集采集器](docs/aamas-proceedings.md)支持已核验的 2012、2025 版式,可从缓存或已校验快照采集元数据。2025 年目录含 476 条非 keynote 出版物,保留完整作者和机构,HTML 未提供摘要时留空;发表形式与投稿 track 分开,不根据主题推定主会轨道或研究类型。
|
||||
|
||||
`paper-atlas collect --help` 查看来源;`paper-atlas import --help` 查看导入方式。完整批次步骤见[使用指南](docs/usage.md)。
|
||||
|
||||
|
||||
@@ -1,6 +1,14 @@
|
||||
# AAMAS 官方论文集元数据
|
||||
|
||||
`paper_atlas.collectors.aamas` 是按需运行的采集适配器,通过统一入口 `collect aamas-proceedings` 使用。目前只支持已独立核验的 **AAMAS 2012** HTML 版式;其他年份需先核对官方目录和逐篇结构,再增加适配,不能仅替换年份。
|
||||
`paper_atlas.collectors.aamas` 是按需运行的采集适配器,通过统一入口 `collect aamas-proceedings` 使用。支持已独立核验的 **AAMAS 2012、2025** HTML 版式;其他年份需先核对官方目录和逐篇结构,再增加适配,不能仅替换年份。2025 解析单独封装在 `paper_atlas.collectors.aamas2025`。
|
||||
|
||||
2025 版式使用[官方首页](https://www.ifaamas.org/Proceedings/aamas2025/)和[出版目录](https://www.ifaamas.org/Proceedings/aamas2025/forms/contents.htm)。原始目录共 479 条:3 keynote、260 research paper、148 extended abstract、12 Blue Sky、25 doctoral、22 Demo 栏目、9 JAAMAS。排除 keynote,输出 476 条;Demo 栏目中的 p3000 是明确标题的 COMP24 竞赛报告,发表年份仍为 2025,格式记 `competition_report`、投稿 track 留作 `proceedings_unspecified`,保留原 Demo 栏目证据。其余输出按 track 分组为 409 unspecified、12 blue_sky、25 doctoral、21 demo、9 jaamas。
|
||||
|
||||
作者按逐篇 BR 行解析,斜体机构原文另存,保留 Unicode、顺序及完整名单。HTML 没有提供摘要或逐篇 DOI,故 `abstract=null`、标识字典为空;只保存 PDF 链接,不下载研究 PDF。不能把 Research Paper Track 的所有条目推成 main,不能把目录缺席推为 rejected/withdrawn,也不能把出版格式标签当研究类型。
|
||||
|
||||
2025 用法:`paper-atlas collect aamas-proceedings --year 2025 --snapshots /data/review/aamas2025 --out /data/paper-index/collected/aamas-2025`;在线模式改用 `--cache /data/paper-index/cache/aamas2025`,只请求上述两个 HTML 页面。快照目录包含原始 `index.html`、`contents.html` 和 `receipts.json`,凭据必须列出各原始 URL、文件、字节数、SHA256、`http_status=200` 和带时区 `at`。已恢复的凭据需保留 `at_precision=filesystem_write_time_recovered` 等说明,不能伪造 HTTP 头或请求时间。2026-11-09 保存的两个页面通过哈希校验;独立录用清单 URL 返回 403,失败保留,不绕过或依赖其完成采集。
|
||||
|
||||
2025 命令同样使用目录锁、临时目录、最后写入采集报告。五个 JSONL/manifest 均保持 `partial`,记录摘要缺失数量;它不连接数据库。首次采集需独立核验全部标题/作者/栏目和 source IDs,随后做 PostgreSQL 身份预检、必要的逐条版本审核、原生导入与独立验收。若失败且没有完成报告,应先检查原始响应和错误,再使用新输出目录重跑,不重放已提交批次。代码验证、原始目录盘点和采集成功均不代表已导入生产或最终去重完成。
|
||||
|
||||
## 来源与范围
|
||||
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
# ACL 早期年份:会议存在、出版目录与逐篇资料分开核验
|
||||
|
||||
固定普查中 ACL 1963–1978 仍有年份缺口。不能因为会议年表连续,就假设每年都有独立的正式论文集;也不能因为当前目录始于 1979,就断言此前没有摘要、通知或其他公开资料。本次不修改固定的 1,266 个目标年份基准。
|
||||
|
||||
## 1978 年:限定 T78-1 的来源关系
|
||||
|
||||
[同期年会通知](https://aclanthology.org/J77-4001.pdf)的 PDF 第 9–10 页(印刷页码 9–10)明确说明,1978 年 ACL 年会采用第二届 Theoretical Issues in Natural Language Processing 的形式,地点为 Urbana/Champaign,日期为 7 月 25–27 日。通知中的 ACM 联合赞助确认仍处于待确认状态,不能将该预告改写为已完成的行政决定。
|
||||
|
||||
[1982 年 Donald E. Walker 的历史回顾](https://aclanthology.org/www.mt-archive.info/70/ACL-1982-Walker-1.pdf)第 3 页(印刷页码 91)的年会表及缩写说明,独立对应了 1978 年的时间、地点与 TNLP/TINLAP-2 关系。两份扫描件相关页已渲染并逐页目视核对,不仅依靠 OCR。
|
||||
|
||||
[官方卷目录](https://aclanthology.org/volumes/T78-1/)与固定 XML `T78.xml` 保留原系列名 TINLAP、完整卷名、1978 年及编辑 David L. Waltz。适配仅接受精确的 collection `T78`、volume `1` 和上述字段组合;关键元数据变化会拒绝处理,等待重新核查。
|
||||
|
||||
- 规范事件名使用 `ACL+TINLAP`,原系列名 `TINLAP`、原始 XML 的 `venue_codes: [tinlap]` 及证据均保留。
|
||||
- ACL/TINLAP 别名只挂在这一规范事件系列上,不把所有 TINLAP 年份都算成 ACL。T75、T87 及 T78 其他卷没有此例外。
|
||||
- 通知描述了邀请与提交论文两种参与方式,XML 没有逐篇角色。统一暂存为 `proceedings_unspecified`,不猜测每篇属于 invited、main 或 workshop。
|
||||
- XML 共 36 条条目,其中 T78-1036 是无作者的场次讨论问题列表。精确识别并在 manifest 的 `excluded_materials` 保留其标识、标题与原因;其余 35 条作为具名贡献整理。该排除不推广到所有无作者条目,已识别条目发生变化也须重新核查。
|
||||
- 全部 35 条 XML 均无摘要。出版目录中的 accepted 是来源观察;通知提及录用/不录用通知时间,不提供任何可识别的拒稿决定,不能制造 rejected 或 withdrawn。
|
||||
|
||||
采集器仅生成元数据文件,不写业务数据库。真实来源逐条验收、PostgreSQL 身份预检、串行导入与独立验收仍是后续门槛。35 条来源不等于 35 个新增论文实体,更不等于完成全库去重。
|
||||
|
||||
## 1965 年:替代关系不改写为别名
|
||||
|
||||
Walker 回顾第 1 页(印刷页码 89)说明,1965 年 ICCL(后来称 COLING)替代了当年的年会。[官方 C65-1 目录](https://aclanthology.org/volumes/C65-1/)及 XML 仍以 COLING 标记该卷。这是限定年份的历史关系,不证明 COLING 与 ACL 是两个可全局互换的名字。
|
||||
|
||||
本次保留 C65 原有 `COLING` 归属,不添加 ACL 全局别名、不重放已导入记录、不将某一年的日历关系推广到整个系列,也不据此悄悄减少固定普查缺口。其他 1963–1977 年资料仍须逐年核查;本次不能证明这些年份整体属于非归档例外。
|
||||
|
||||
## 固定来源与恢复
|
||||
|
||||
服务器证据目录:`/data/paper-index/collected/acl-early-source-evidence-v1`。五份官方响应均 HTTP 200,另有两份来自既有官方 XML checkout 的文件,已与 Git 对象逐字节核对。未下载论文全文集合;两份 PDF 是本次历史/行政取证资料,不进入论文全文库。
|
||||
|
||||
| 文件 | SHA256 |
|
||||
|---|---|
|
||||
| `acl-venue.html` | `c60f64cb4d72e677aeb84a5844cde28a8d50579e1995f08e8c7801eed1e3fac5` |
|
||||
| `acl-history-1982.pdf` | `35c21bd4983975bcaccbb94bd8ab91c9019fc190feec1791255407cf18a7e49a` |
|
||||
| `acl-1978-notice.pdf` | `14b5deb471c55afb07d789529b95d0462a2a1a77df3dfba7fa1f5d3ab75eb411` |
|
||||
| `tinlap1978-volume.html` | `7e20d709f72f415819e4d17e3674b19dc7d871eabcf0cf8b4935f0392d775729` |
|
||||
| `coling1965-volume.html` | `0d2f2f49f2a311f9c0b80a8832f6476eb32f94f35abf36f6d2f2e10063ac880f` |
|
||||
| `T78.xml` | `f712f94cff0cba05f786b7ac31dad2ea9408fde2a17a04531130df623f57dd84` |
|
||||
| `C65.xml` | `572791610e20b7c6045d984d968e73a18a144bcb5c3da64112001c781826808e` |
|
||||
|
||||
XML commit:`a7be623d01c48c61405dec31c871121cdbed5a01`。测试 fixture `tests/fixtures/acl-t78.xml` 是该原始文件,不是手工模拟论文;不包含 PDF。PDF 哈希建立完整性,不单独证明语义,语义依据是上述具体页面。
|
||||
|
||||
在服务器使用既有命令,固定 checkout 并输出到新目录:
|
||||
|
||||
```sh
|
||||
paper-atlas collect acl \
|
||||
--checkout /data/paper-index/raw/metadata-checkouts/acl-anthology \
|
||||
--collection T78 \
|
||||
--out /data/paper-index/collected/acl-t78-source-v1
|
||||
```
|
||||
|
||||
该命令会校验/更新官方 metadata checkout,实际 commit 必须以生成清单为准;严格重现本次取证时,使用已冻结的 XML 和上述 commit,不把更新后的内容冒充旧快照。不要将无 collection 限制的全部历史输出直接作为增量队列。采集完成不代表导入完成,恢复前先检查当前单元和报告,不因日志安静重启已执行的写入。
|
||||
@@ -0,0 +1,53 @@
|
||||
# EWSL 与规划会议前身的来源范围
|
||||
|
||||
本项扩展 DBLP 元数据适配:核对具体历史卷,保留 EWSL、EPS、ECP、EWSP 原始系列名称,并使用已有会议别名关系关联 ECML-PKDD 或 ICAPS。它不把历史论文的原始会议名称替换成后来的会议,也不从卷的存在推断有逐篇数据。
|
||||
|
||||
## 证据与已核验边界
|
||||
|
||||
[ECML-PKDD 2011 官方历史页](https://ecmlpkdd2011.org/pastConferences.php)与 [ICAPS 官方系列历史](https://www.icaps-conference.org/conference-series/)分别提供系列前身关系。2026-11-09 冻结页面 SHA-256 为:
|
||||
|
||||
- ECML-PKDD:`c3ecdb032aeacf6a365aace5457773827d1405872ad9bc6be6fc05851d5e68d6`。
|
||||
- ICAPS:`de1d217660cb819825f58112cdaf774d0f68da6ed87e3e90a59055e52f152081`。
|
||||
|
||||
与固定 DBLP 2026-11-01 快照的完整 proceedings 目录对照后,仅对下列具体卷识别前身主系列:
|
||||
|
||||
| 原始系列 | 事件年 | DBLP 卷 key | 卷出版年 | 原始逐篇书目计数 |
|
||||
|---|---:|---|---:|---:|
|
||||
| EWSL | 1987 | `conf/ecml/1987` | 1987 | 16 |
|
||||
| EWSL | 1988 | `conf/ecml/1988` | 1988 | 21 |
|
||||
| EWSL | 1989 | `conf/ecml/1989` | 1989 | 0 |
|
||||
| EWSL | 1991 | `conf/ecml/1991` | 1991 | 44 |
|
||||
| EPS | 1990 | `conf/aips/1990` | 1990 | 48 |
|
||||
| ECP | 1997 | `conf/ecp/1997` | 1997 | 35 |
|
||||
| ECP | 1999 | `conf/ecp/1999` | 2000 | 28 |
|
||||
| EWSP | 1991 | `conf/ewsp/1991` | 1991 | 10 |
|
||||
|
||||
合计 202 条是原始 `inproceedings` 书目计数,尚未经过前言等材料排除、身份核查与生产验收,不能作为新论文数量。EWSL 1989 在当前 catalogue 中有卷信息,但未匹配到 `inproceedings` 条目;其他记录类型和其他来源仍待核查。代码支持它的范围规则不表示已补上这一年的论文。
|
||||
|
||||
核验同时要求来源 family、booktitle 基名、空轨道后缀、完整 proceedings key、完整标题及卷出版年与固定证据一致。邻近年份、标题附加内容、`w` 卷后缀或工作坊 booktitle 均不能继承此范围。ECP、EWSP 及 EPS 的未知卷保持待审。原有 AIPS、ICML/IROS、AISTATS/COLT 规则保持各自边界。
|
||||
|
||||
EWSL 的主轨道仍属于 `EWSL`,通过 `ECML-PKDD` 别名纳入家族查询;EPS、ECP、EWSP 保留原名,通过 `ICAPS` 别名关联。`main` 在此表示该原始系列的主轨道。原始 booktitle、作者、论文书目年、卷标题、卷出版年、事件年及历史页 URL 分开保存在 metadata 中。例如 ECP 1999 的事件年 1999 与卷出版年 2000 都保留,不用其中一个覆盖另一个。
|
||||
|
||||
## 候选文件与完整快照
|
||||
|
||||
旧的窄范围 `dblp-ai-candidates.jsonl` 未纳入 `ecp`、`ewsp` family,但完整 catalogue 的 proceedings 与 volume_counts 中有它们。新的 catalogue 导出会纳入这两个 family。
|
||||
|
||||
选择 ICAPS 且截止年达到 EWSP 1991 或 ECP 1997 时,若候选 manifest 缺少相应 family,`collect dblp --candidates` 会明确报错,要求使用原始完整 `--archive`。错误发生在写输出之前,不会把遗漏当作成功采集。选择其他会议或截止 EPS 1990 的旧候选采集不受这项新家族检查影响。
|
||||
|
||||
在已确认的远程运行环境中使用完整快照:
|
||||
|
||||
```sh
|
||||
paper-atlas collect dblp \
|
||||
--catalogue /data/paper-index/collected/dblp/dblp-catalogue.sqlite \
|
||||
--archive /path/to/pinned-dblp.xml.gz --dtd /path/to/pinned-dblp.dtd \
|
||||
--venue ECML-PKDD --venue ICAPS --through-year 2001 \
|
||||
--out /data/paper-index/collected/historical-predecessors
|
||||
```
|
||||
|
||||
完整快照与 DTD 均沿用现有固定身份校验。该命令也会输出已覆盖的其他年度;生产导入前须单独筛选目标卷与年度,检查 raw source、前言材料、稳定标识、标题、作者及现有投稿,不能直接导入整个输出目录。不会下载论文 PDF。
|
||||
|
||||
## 验证和当前限制
|
||||
|
||||
回归检查使用独立保存的八卷原始目录 fixture,合成论文只用于隔离测试;不代表真实逐篇来源。检查原始系列/别名、完整标题和年份边界、旧候选文件显式失败、扩展 family 的 catalogue 导出、原生导入后的元数据与关系。
|
||||
|
||||
代码、测试、采集和生产导入是不同阶段。新范围不表示本批数据已采集或已导入,更不表示公开拒稿、摘要或最终去重完整。原始书目仅提供发表条目证据,不是原始决定信;没有记录不能推断为拒稿或未举办。后续各批需保留独立验收报告,保护既有跨会投稿、决定与标签历史。
|
||||
@@ -0,0 +1,36 @@
|
||||
# 会后选集的元数据采集
|
||||
|
||||
`collect springer-selected` 读取已冻结、带 HTTP 回执的公开 HTML,不访问 PDF、不写业务库。目前仅支持核验过的 *Selecting Models from Data: Artificial Intelligence and Statistics IV*,书籍 DOI 为 `10.1007/978-1-4612-2660-4`。增加其他卷前必须另行核对会议关系、目录、年份和字段格式。
|
||||
|
||||
[会议官方旧站](https://www.gatsby.ucl.ac.uk/aistats/workshops.html)区分现场分发的 Preliminary Papers 与之后修订、再次评审的选集。[出版社卷说明](https://link.springer.com/book/10.1007/978-1-4612-2660-4)将本书关联到 1993 年 1 月的第四届 workshop,书籍出版年份为 1994。三页目录共有 49 个章节 DOI;分区 Front Matter 不作为章节论文导入。
|
||||
|
||||
规范记录保留:
|
||||
|
||||
- 会议年份 1993、独立 track `selected_volume` 与事件标识 `aistats-1993-selected-volume-1994`;出版年份单独记为 1994。
|
||||
- 章节本身的 DOI、作者原始引用写法及顺序、姓名首字母、标题、摘要、页码和逐页原始证据。不扩写作者首字母。
|
||||
- 明确提供本章 `publication_doi` 投稿出版标识,不把它转移到旧的原始会议评审记录,也不因短标题相同而强制合并实体。
|
||||
- `springer_selected_volume:publication_format` 标签与独立来源 namespace。
|
||||
- `accepted` 仅表示章节已列入这本出版选集,原始 1993 投稿评审决定、决定日期和全部录用清单均未知。后续原会议 main、拒稿或撤稿证据可以独立保留。
|
||||
|
||||
不能据此宣称会议全部录用、全部投稿或公开拒稿覆盖完整,也不能把出版社机器添加的关键词视为作者标签。未命中选集不等于被拒。
|
||||
|
||||
## 按需运行与恢复
|
||||
|
||||
输入分别是卷目录/旧站说明和章节页的快照目录。每个目录的 `receipts.json` 包含 `responses` 列表,每条保留 `url`、`final_url`、`file`、`http_status`、`headers`、`bytes`、`sha256`、`at`。需要三页目录、官方旧站说明及全部 49 个章节页面。
|
||||
|
||||
```sh
|
||||
paper-atlas collect springer-selected \
|
||||
--toc-snapshots /data/paper-index/collected/aistats1993-book-evidence-v1 \
|
||||
--chapter-snapshots /data/paper-index/collected/aistats1993-chapter-evidence-v1 \
|
||||
--out /data/paper-index/collected/aistats1993-source-v1
|
||||
```
|
||||
|
||||
采集器验证回执和内容哈希、文件路径、HTTP 类型、目录/章节 DOI 和标题、作者格式、出版年份及页码。Springer 对公开元数据返回的同路径 `cookies_not_supported` 诊断参数会原样保留;不接受其他域、路径、分页变化或未知查询参数。官方旧站说明固定哈希为 `6db205b9dbc82b4a096556efd2aa67e065cad8e0bd6351176b1962ffd1fca179`,变化时须重新审核。
|
||||
|
||||
全部文件先核验,输出使用排他锁和暂存目录,完成报告最后发布。已有输出拒绝覆盖;发现不完整或已消费文件,应先查运行状态与报告,另建明确版本目录,不能重放生产导入。
|
||||
|
||||
输出 JSONL 配套 manifest 始终为 `partial`。收集完成后还必须做 PostgreSQL 稳定标识/标题/作者身份预检、排除已导入记录、准备限定增量文件、保存旧数据保护基线、原生导入和独立验收。49 条来源不代表 49 个新增论文实体,全部来源处理后仍须全库去重审计。
|
||||
|
||||
## 固定测试资料
|
||||
|
||||
`tests/fixtures/springer-aistats1993-fixture.json.gz` 保存 4 份范围/目录 HTML、49 份章节 HTML、原始请求回执及独立原始字段审计的预期元数据,SHA256 为 `94028fd37080f9d8027e5790e379cc151ee6eab17ed0d1dcc0f59e6e75f9d4d0`。摘要按 HTML 段落边界分隔;未修改原始 HTML 或回执。它是压缩的公开元数据测试资料,不包含论文 PDF。测试覆盖全部真实章节字段、缺失和漂移输入、来源安全边界、幂等导入及旧拒稿历史保留。构建和测试只在已授权远程 runner 执行。
|
||||
@@ -13,6 +13,7 @@ COMMANDS = {
|
||||
('download',): ('paper_atlas.io.snapshots', {}, '下载并校验固定版本快照'),
|
||||
('collect','pmlr'): ('paper_atlas.collectors.pmlr', {}, 'PMLR 官方元数据'),
|
||||
('collect','acl'): ('paper_atlas.collectors.acl', {}, 'ACL Anthology XML'),
|
||||
('collect','springer-selected'): ('paper_atlas.collectors.springer_selected', {}, '核验限定的会后选集章节与原会议关系'),
|
||||
('collect','iclr-archive'): ('paper_atlas.collectors.iclr_archive', {}, '核对ICLR 2014–2016官方历史录用档案'),
|
||||
('collect','iclr-program'): ('paper_atlas.collectors.iclr', {}, 'ICLR 官方公开程序'),
|
||||
('collect','iclr-snapshot'): ('paper_atlas.collectors.openreview', {}, 'ICLR 公开投稿与决定快照'),
|
||||
|
||||
@@ -136,6 +136,9 @@ def read_pages(cache=None, snapshots=None):
|
||||
|
||||
|
||||
def _collect(year, out, *, cache=None, snapshots=None):
|
||||
if year == 2025:
|
||||
from paper_atlas.collectors.aamas2025 import collect as collect_2025
|
||||
return collect_2025(out, cache=cache, snapshots=snapshots)
|
||||
if year != 2012:
|
||||
raise ValueError('Only the independently reviewed AAMAS 2012 layout is supported')
|
||||
pages, receipts = read_pages(cache, snapshots)
|
||||
@@ -190,8 +193,8 @@ def _collect(year, out, *, cache=None, snapshots=None):
|
||||
|
||||
|
||||
def collect(year, out, *, cache=None, snapshots=None):
|
||||
if year != 2012:
|
||||
raise ValueError('Only the independently reviewed AAMAS 2012 layout is supported')
|
||||
if year not in (2012, 2025):
|
||||
raise ValueError('Only independently reviewed AAMAS 2012/2025 layouts are supported')
|
||||
out = Path(out)
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
with (out/'aamas.collect.lock').open('a') as lock:
|
||||
@@ -201,7 +204,7 @@ def collect(year, out, *, cache=None, snapshots=None):
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser = argparse.ArgumentParser(prog=prog)
|
||||
parser.add_argument('--year', type=int, choices=[2012], required=True)
|
||||
parser.add_argument('--year', type=int, choices=[2012, 2025], required=True)
|
||||
parser.add_argument('--out', required=True)
|
||||
source = parser.add_mutually_exclusive_group(required=True)
|
||||
source.add_argument('--cache')
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
"""Reviewed AAMAS 2025 publisher TOC; HTML metadata only, never paper PDFs."""
|
||||
from collections import Counter, defaultdict
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import tempfile
|
||||
|
||||
from paper_atlas.db.submissions import timestamp
|
||||
from paper_atlas.io.html import Document, Node, clean
|
||||
from paper_atlas.io.http import fetch
|
||||
|
||||
ROOT = 'https://www.ifaamas.org/Proceedings/aamas2025/'
|
||||
URLS = {'index': ROOT, 'contents': ROOT+'forms/contents.htm'}
|
||||
TITLE = 'AAMAS 2025: Proceedings of the 2025 International Conference on Autonomous Agents and Multiagent Systems'
|
||||
SECTIONS = {
|
||||
'1': ('Keynote Talks', 'keynote', 'keynote', 3),
|
||||
'2': ('Research Paper Track', 'full_paper', 'proceedings_unspecified', 260),
|
||||
'3': ('Extended Abstracts', 'extended_abstract', 'proceedings_unspecified', 148),
|
||||
'4': ('Blue Sky Ideas', 'blue_sky_paper', 'blue_sky', 12),
|
||||
'5': ('Doctoral Consortium', 'doctoral_abstract', 'doctoral', 25),
|
||||
'6': ('Demo Track', 'demo', 'demo', 22),
|
||||
'7': ('JAAMAS Track', 'journal_presentation', 'jaamas', 9),
|
||||
}
|
||||
COMPETITION_TITLE = '[COMP24] The Automated Negotiating Agents Competition (ANAC) 2024 Challenges and Results'
|
||||
PAPER = re.compile(r'\.\./pdfs/p([0-9]+)\.pdf')
|
||||
|
||||
|
||||
def one(items, label):
|
||||
if len(items) != 1:
|
||||
raise ValueError('Expected one '+label)
|
||||
return items[0]
|
||||
|
||||
|
||||
def parse_contents(html, evidence):
|
||||
doc = Document(html)
|
||||
if [clean(n.text()) for n in doc.root.walk() if n.tag == 'title'] != [TITLE]:
|
||||
raise ValueError('Unexpected AAMAS proceedings title/year')
|
||||
for paragraph in doc.root.walk():
|
||||
if paragraph.tag != 'p' or any(n is not paragraph and n.tag == 'p' for n in paragraph.walk()):
|
||||
continue
|
||||
anchors = [n.attrs['name'] for n in paragraph.walk() if n.tag == 'a' and n.attrs.get('name') in SECTIONS]
|
||||
if anchors and (len(anchors) != 1 or clean(paragraph.text()) != SECTIONS[anchors[0]][0]):
|
||||
raise ValueError('Publisher section heading differs from reviewed layout')
|
||||
section = None
|
||||
headers = []
|
||||
records, excluded, counts, seen = [], [], Counter(), set()
|
||||
for node in doc.root.walk():
|
||||
if node.tag == 'a' and node.attrs.get('name') in SECTIONS:
|
||||
section = node.attrs['name']
|
||||
headers.append(section)
|
||||
if node.tag != 'p' or any(n is not node and n.tag == 'p' for n in node.walk()):
|
||||
continue
|
||||
links = [n for n in node.walk() if n.tag == 'a' and PAPER.fullmatch(n.attrs.get('href', ''))]
|
||||
if not links:
|
||||
continue
|
||||
link = one(links, 'paper link per paragraph')
|
||||
if section not in SECTIONS:
|
||||
raise ValueError('Paper outside a reviewed publisher section')
|
||||
page = int(PAPER.fullmatch(link.attrs['href'])[1])
|
||||
if page in seen:
|
||||
raise ValueError('Repeated publisher paper/page ID')
|
||||
seen.add(page)
|
||||
title = clean(link.text())
|
||||
if not title:
|
||||
raise ValueError('Empty paper title')
|
||||
# Names are outside italicized affiliations and separated by explicit BRs.
|
||||
font = one([n for n in node.walk() if n.tag == 'font' and n.attrs.get('color', '').lower() == '#222222'], 'ordered author block')
|
||||
authors, affiliations = [], []
|
||||
name, affiliation = '', ''
|
||||
def flush():
|
||||
nonlocal name, affiliation
|
||||
if clean(name) or clean(affiliation):
|
||||
if not clean(name):
|
||||
raise ValueError('Affiliation without author')
|
||||
authors.append(clean(name)); affiliations.append(clean(affiliation))
|
||||
name, affiliation = '', ''
|
||||
for child in font.children:
|
||||
if isinstance(child, str):
|
||||
name += child
|
||||
elif child.tag == 'br':
|
||||
flush()
|
||||
elif child.tag == 'i':
|
||||
affiliation += child.text()
|
||||
else:
|
||||
raise ValueError('Unreviewed author markup: '+child.tag)
|
||||
flush()
|
||||
if not authors:
|
||||
raise ValueError('Empty author list')
|
||||
heading, publication_format, track, _ = SECTIONS[section]
|
||||
counts[section] += 1
|
||||
if section == '1':
|
||||
excluded.append({'publisher_page': page, 'title': title, 'authors': authors, 'reason': 'publisher keynote talk, not a research-paper submission'})
|
||||
continue
|
||||
note = 'Explicit publisher section; section topics are not research types.'
|
||||
if track == 'proceedings_unspecified':
|
||||
note = 'Published format is known; individual main/special submission track is not provided.'
|
||||
if page == 3000:
|
||||
if section != '6' or title != COMPETITION_TITLE:
|
||||
raise ValueError('Competition report differs from reviewed publisher entry')
|
||||
publication_format, track = 'competition_report', 'proceedings_unspecified'
|
||||
note = 'Publisher places this competition report in Demo Track; no individual demonstration submission is inferred.'
|
||||
records.append({
|
||||
'source': 'aamas_proceedings', 'external_id': '2025/p'+str(page),
|
||||
'url': URLS['contents']+'#'+section, 'title': title, 'abstract': None,
|
||||
'authors': authors, 'identifiers': {}, 'venue': 'AAMAS', 'venue_aliases': [],
|
||||
'year': 2025, 'track': track, 'status': 'accepted',
|
||||
'observed_at': evidence['observed_at'],
|
||||
'raw_decision': 'listed in official AAMAS 2025 published proceedings '+heading,
|
||||
'tags': [{'namespace': 'aamas_proceedings:publication_format', 'value': publication_format}],
|
||||
'metadata': {'publisher_page': page, 'paper_url': ROOT+'pdfs/p'+str(page)+'.pdf',
|
||||
'section_url': URLS['contents'], 'section_anchor': section,
|
||||
'section_heading': heading, 'publication_format': publication_format,
|
||||
'author_affiliations': affiliations, 'source_evidence': evidence,
|
||||
'track_scope_note': note, 'decision_date_precision': 'not_supplied',
|
||||
'missing_abstract_reason': 'Publisher HTML supplies no abstract; research PDF not downloaded.',
|
||||
'publication_evidence': 'Published proceedings entry, not an original decision note.'}})
|
||||
if headers != list(SECTIONS) or dict(counts) != {k: v[3] for k, v in SECTIONS.items()}:
|
||||
raise ValueError('Publisher section inventory differs from reviewed 2025 layout')
|
||||
if len(records) != 476 or len(excluded) != 3 or sum(r['metadata']['publication_format'] == 'competition_report' for r in records) != 1:
|
||||
raise ValueError('Unexpected publication partition')
|
||||
return records, excluded
|
||||
|
||||
|
||||
def read_pages(cache, snapshots):
|
||||
if bool(cache) == bool(snapshots):
|
||||
raise ValueError('Choose exactly one cache or frozen snapshot directory')
|
||||
frozen = json.loads((Path(snapshots)/'receipts.json').read_text())['receipts'] if snapshots else []
|
||||
pages = {}
|
||||
for key, url in URLS.items():
|
||||
if snapshots:
|
||||
r = one([r for r in frozen if r['url'] == url], 'frozen response receipt')
|
||||
filename = Path(r['file']); base = Path(snapshots).resolve()
|
||||
if filename.is_absolute() or '..' in filename.parts or not (base/filename).resolve().is_relative_to(base) or r.get('http_status') != 200:
|
||||
raise ValueError('Unsafe or unsuccessful publisher response')
|
||||
data = (base/filename).read_bytes()
|
||||
if len(data) > 8*1024**2 or len(data) != r['bytes'] or hashlib.sha256(data).hexdigest() != r['sha256']:
|
||||
raise ValueError('Frozen publisher response checksum/size mismatch')
|
||||
observed = r['at']
|
||||
else:
|
||||
data = fetch(url, cache, 8*1024**2, delay=1)
|
||||
r = json.loads((Path(cache)/hashlib.sha256(url.encode()).hexdigest()).with_suffix('.json').read_text())
|
||||
observed = r['retrieved_at']
|
||||
timestamp(observed)
|
||||
pages[key] = data, {'url': url, 'sha256': hashlib.sha256(data).hexdigest(), 'bytes': len(data),
|
||||
'file': 'evidence/'+hashlib.sha256(data).hexdigest()+'.html',
|
||||
'observed_at': observed, 'http_status': 200, 'response_receipt': r}
|
||||
return pages
|
||||
|
||||
|
||||
def collect(out, *, cache=None, snapshots=None):
|
||||
pages = read_pages(cache, snapshots)
|
||||
if [clean(n.text()) for n in Document(pages['index'][0].decode('utf-8')).root.walk() if n.tag == 'title'] != [TITLE]:
|
||||
raise ValueError('Unexpected AAMAS 2025 introduction')
|
||||
records, excluded = parse_contents(pages['contents'][0].decode('utf-8'), pages['contents'][1])
|
||||
grouped = defaultdict(list)
|
||||
for row in records: grouped[row['track']].append(row)
|
||||
out = Path(out); out.mkdir(parents=True, exist_ok=True); files = []
|
||||
with tempfile.TemporaryDirectory(prefix='.aamas2025-', dir=out) as temporary:
|
||||
stage = Path(temporary); (stage/'evidence').mkdir()
|
||||
for data, e in pages.values(): (stage/e['file']).write_bytes(data)
|
||||
for track, group in sorted(grouped.items()):
|
||||
p = stage/f'aamas-proceedings-2025-{track}.jsonl'
|
||||
p.write_text(''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in group))
|
||||
m = {'source': 'aamas_proceedings', 'parser_version': 2, 'venue': 'AAMAS', 'year': 2025,
|
||||
'track': track, 'count': len(group), 'file': str(out/p.name), 'file_sha256': hashlib.sha256(p.read_bytes()).hexdigest(),
|
||||
'list_url': URLS['contents'], 'status': 'partial', 'missing_abstract_count': len(group),
|
||||
'publication_format_counts': dict(Counter(r['metadata']['publication_format'] for r in group)),
|
||||
'note': 'Complete published TOC partition; HTML abstracts absent, identities and submission-track scope require independent review.'}
|
||||
p.with_suffix('.manifest.json').write_text(json.dumps(m, ensure_ascii=False, indent=2)+'\n'); files.append(m)
|
||||
result = {'source': 'aamas_proceedings', 'year': 2025, 'finished_at': timestamp(),
|
||||
'status': 'collected_pending_identity_review', 'records': len(records), 'files': files,
|
||||
'excluded_publications': excluded, 'source_receipts': [e for _, e in pages.values()],
|
||||
'publication_format_counts': dict(Counter(r['metadata']['publication_format'] for r in records)),
|
||||
'database_written': False, 'pdfs_downloaded': False,
|
||||
'note': 'Neither absence from proceedings nor aggregate rejection/withdrawal counts identify an individual negative decision.'}
|
||||
report = 'aamas-proceedings-2025.json'; (stage/report).write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n')
|
||||
(out/'evidence').mkdir(exist_ok=True)
|
||||
for p in (stage/'evidence').iterdir(): p.replace(out/'evidence'/p.name)
|
||||
for p in stage.iterdir():
|
||||
if p.is_file() and p.name != report: p.replace(out/p.name)
|
||||
(stage/report).replace(out/report)
|
||||
return result
|
||||
@@ -6,6 +6,7 @@ import xml.etree.ElementTree as ET
|
||||
from pathlib import Path
|
||||
|
||||
from paper_atlas.io.git_metadata import checkout_metadata
|
||||
from paper_atlas.collectors.acl_historical import event_scope, material_reason
|
||||
|
||||
VENUES={'acl':'ACL','emnlp':'EMNLP','naacl':'NAACL','eacl':'EACL','coling':'COLING','conll':'CoNLL'}
|
||||
|
||||
@@ -29,7 +30,10 @@ def paper_id(collection,volume,number):
|
||||
return f'{collection}-{int(volume):0{width}d}{int(number):0{digits}d}'
|
||||
|
||||
|
||||
def volume_scope(meta,volume):
|
||||
def volume_scope(meta,volume,collection=None):
|
||||
historical=event_scope(meta,volume,collection)
|
||||
if historical:
|
||||
return historical['canonical_venue'],historical['venue_aliases'],historical['track']
|
||||
codes=[text(n).lower() for n in meta.findall('venue')]
|
||||
book=text(meta.find('booktitle'))
|
||||
lower=book.lower()
|
||||
@@ -59,7 +63,7 @@ def collect_xml(path,out,commit):
|
||||
vid=volume.attrib['id']
|
||||
meta=volume.find('meta')
|
||||
if meta is None:raise ValueError('Volume metadata missing')
|
||||
scope=volume_scope(meta,vid)
|
||||
scope=volume_scope(meta,vid,collection)
|
||||
key=f'{collection}-{vid}'
|
||||
if scope is None:
|
||||
results.append({'volume':key,'status':'excluded_by_venue','venue_codes':[text(n) for n in meta.findall('venue')],'booktitle':text(meta.find('booktitle')),'year':text(meta.find('year'))})
|
||||
@@ -70,12 +74,18 @@ def collect_xml(path,out,commit):
|
||||
year=int(year_text)
|
||||
records=[]
|
||||
frontmatter=0
|
||||
excluded_materials=[]
|
||||
historical=event_scope(meta,vid,collection)
|
||||
for paper in volume.findall('paper'):
|
||||
pid=paper.attrib['id']
|
||||
if int(pid)==0:
|
||||
frontmatter+=1
|
||||
continue
|
||||
identifier=paper_id(collection,vid,pid)
|
||||
reason=material_reason(collection,vid,paper)
|
||||
if reason:
|
||||
excluded_materials.append({'external_id':identifier,'title':text(paper.find('title')),'reason':reason})
|
||||
continue
|
||||
doi=text(paper.find('doi'))
|
||||
if doi.startswith('10.18653/v1/') and doi.split('10.18653/v1/',1)[1].casefold()!=identifier.casefold():
|
||||
raise ValueError('Official DOI disagrees with constructed Anthology ID: '+identifier)
|
||||
@@ -83,6 +93,8 @@ def collect_xml(path,out,commit):
|
||||
if not title:raise ValueError('Missing paper title: '+identifier)
|
||||
authors=[{'given':text(a.find('first')),'family':text(a.find('last'))} for a in paper.findall('author')]
|
||||
records.append({'source':'acl_anthology','external_id':identifier,'url':f'https://aclanthology.org/{identifier}/','title':title,'abstract':text(paper.find('abstract')) or None,'authors':authors,'identifiers':{'acl':identifier,'doi':doi},'venue':venue,'venue_aliases':aliases if len(aliases)>1 else [],'year':year,'track':track,'status':'accepted','raw_decision':'listed in authoritative ACL Anthology proceedings XML','metadata':{'collection':collection,'volume':vid,'booktitle':text(meta.find('booktitle')),'venue_codes':[text(n) for n in meta.findall('venue')],'pages':text(paper.find('pages')),'publisher':text(meta.find('publisher')),'metadata_commit':commit,'source_xml':f'https://github.com/acl-org/acl-anthology/blob/{commit}/data/xml/{collection}.xml'}})
|
||||
if historical:
|
||||
records[-1]['metadata']['historical_event_scope']=historical
|
||||
if not records:
|
||||
results.append({'volume':key,'status':'unavailable','reason':'no research paper elements after excluding front matter'})
|
||||
continue
|
||||
@@ -93,7 +105,7 @@ def collect_xml(path,out,commit):
|
||||
for record in records:output.write(json.dumps(record,ensure_ascii=False)+'\n')
|
||||
part.replace(target)
|
||||
missing=sum(not row['abstract'] for row in records)
|
||||
manifest={'source':'acl_anthology','venue':venue,'year':year,'track':track,'count':len(records),'xml_paper_elements':len(volume.findall('paper')),'excluded_frontmatter':frontmatter,'missing_abstract_count':missing,'list_url':f'https://aclanthology.org/{key}/','metadata_commit':commit,'file':str(target),'status':'partial' if missing else 'collected'}
|
||||
manifest={'source':'acl_anthology','venue':venue,'year':year,'track':track,'count':len(records),'xml_paper_elements':len(volume.findall('paper')),'excluded_frontmatter':frontmatter,'excluded_materials':excluded_materials,'missing_abstract_count':missing,'list_url':f'https://aclanthology.org/{key}/','metadata_commit':commit,'file':str(target),'status':'partial' if missing else 'collected'}
|
||||
target.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2))
|
||||
results.append(manifest)
|
||||
print(json.dumps(manifest),flush=True)
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Edition-specific historical scope; never alias every TINLAP year to ACL."""
|
||||
|
||||
HISTORY_URL = 'https://aclanthology.org/www.mt-archive.info/70/ACL-1982-Walker-1.pdf'
|
||||
NOTICE_URL = 'https://aclanthology.org/J77-4001.pdf'
|
||||
BOOK = 'Theoretical Issues in Natural Language Processing-2'
|
||||
MATERIAL_TITLE = 'List of questions suggested for consideration in each session'
|
||||
|
||||
|
||||
def text(node):
|
||||
return ' '.join(''.join(node.itertext()).split()) if node is not None else ''
|
||||
|
||||
|
||||
def event_scope(meta, volume, collection):
|
||||
if (collection, volume) != ('T78', '1'):
|
||||
return None
|
||||
if ([text(n) for n in meta.findall('venue')] != ['tinlap']
|
||||
or [text(n) for n in meta.findall('year')] != ['1978']
|
||||
or [text(n) for n in meta.findall('booktitle')] != [BOOK]
|
||||
or [(text(n.find('first')), text(n.find('last'))) for n in meta.findall('editor')]
|
||||
!= [('David L.', 'Waltz')]):
|
||||
raise ValueError('Reviewed T78-1 scope metadata changed; independent review required')
|
||||
return {
|
||||
'collection': 'T78', 'volume': '1', 'original_venue': 'TINLAP',
|
||||
'event_year': 1978, 'event_dates': ['1978-07-25', '1978-07-27'],
|
||||
'relationship': 'ACL annual meeting took the form of TINLAP-2',
|
||||
'canonical_venue': 'ACL+TINLAP', 'venue_aliases': ['ACL', 'TINLAP'],
|
||||
'track': 'proceedings_unspecified',
|
||||
'track_scope_note': 'Invited and submitted contributions are described at volume level; individual roles are not identified in XML.',
|
||||
'evidence': [
|
||||
{'url': NOTICE_URL, 'sha256': '14b5deb471c55afb07d789529b95d0462a2a1a77df3dfba7fa1f5d3ab75eb411',
|
||||
'pdf_pages': [9, 10], 'printed_pages': ['9', '10'],
|
||||
'scope': 'Contemporary annual meeting notice, not individual decision notes; ACM confirmation was pending in this notice.'},
|
||||
{'url': HISTORY_URL, 'sha256': '35c21bd4983975bcaccbb94bd8ab91c9019fc190feec1791255407cf18a7e49a',
|
||||
'pdf_pages': [3], 'printed_pages': ['91'],
|
||||
'scope': 'Retrospective annual meeting table and TNLP abbreviation confirm the 1978 event association.'},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def material_reason(collection, volume, paper):
|
||||
if (collection, volume, paper.attrib.get('id')) != ('T78', '1', '36'):
|
||||
return None
|
||||
if text(paper.find('title')) != MATERIAL_TITLE or paper.findall('author'):
|
||||
raise ValueError('Reviewed T78-1036 material changed; independent review required')
|
||||
return 'Session question list, not an authored research contribution; no decision inferred.'
|
||||
@@ -20,7 +20,7 @@ DTD_MD5='96e4283e71886dec2173718957293648'
|
||||
DTD_NAMES={'dblp.dtd','dblp-2023-06-28.dtd'}
|
||||
RELEASE='https://doi.org/10.4230/dblp.xml.2026-11-01'
|
||||
# Broad discovery candidates, including historical names; not a venue mapping.
|
||||
FAMILIES=set('nips icml iclr colm colt aistats uai clear acl emnlp naacl eacl coling conll cvpr iccv eccv aaai ijcai ecai kdd sigir www wsdm recsys cikm icdm sdm ecml pkdd ecmlpkdd icra iros rss corl aips icaps kr atal aamas icassp interspeech eurospeech icslp ismir miccai fat facct lrec mm mlsys'.split())
|
||||
FAMILIES=set('nips icml iclr colm colt aistats uai clear acl emnlp naacl eacl coling conll cvpr iccv eccv aaai ijcai ecai kdd sigir www wsdm recsys cikm icdm sdm ecml pkdd ecmlpkdd icra iros rss corl aips icaps ecp ewsp kr atal aamas icassp interspeech eurospeech icslp ismir miccai fat facct lrec mm mlsys'.split())
|
||||
|
||||
|
||||
def records(archive,dtd_bytes):
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
"""Exact predecessor volumes, with original names and independent family histories."""
|
||||
ECML_HISTORY = 'https://ecmlpkdd2011.org/pastConferences.php'
|
||||
ICAPS_HISTORY = 'https://www.icaps-conference.org/conference-series/'
|
||||
|
||||
# family/year: booktitle, original series, target family, publication year, full title
|
||||
VOLUMES = {
|
||||
('ecml', 1987): ('EWSL', 'EWSL', 'ECML-PKDD', 1987,
|
||||
'Progress in Machine Learning - Proceedings of EWSL 87: 2nd European Working Session on Learning, Bled, Yugoslavia, May 1987'),
|
||||
('ecml', 1988): ('EWSL', 'EWSL', 'ECML-PKDD', 1988,
|
||||
'Proceedings of the Third European Working Session on Learning, EWSL 1988, Turing Institute, Glasgow, UK, October 3-5, 1988'),
|
||||
('ecml', 1989): ('EWSL', 'EWSL', 'ECML-PKDD', 1989,
|
||||
'Proceedings of the 4th European Working Session on Learning, Montpellier, France, 4-6 December 1989'),
|
||||
('ecml', 1991): ('EWSL', 'EWSL', 'ECML-PKDD', 1991,
|
||||
'Machine Learning - EWSL-91, European Working Session on Learning, Porto, Portugal, March 6-8, 1991, Proceedings'),
|
||||
('aips', 1990): ('Expert Planning Systems', 'EPS', 'ICAPS', 1990,
|
||||
'Proceedings of the First International Conference on Expert Planning Systems, Brighton, UK, June 27-29, 1990'),
|
||||
('ecp', 1997): ('ECP', 'ECP', 'ICAPS', 1997,
|
||||
"Recent Advances in AI Planning, 4th European Conference on Planning, ECP'97, Toulouse, France, September 24-26, 1997, Proceedings"),
|
||||
('ecp', 1999): ('ECP', 'ECP', 'ICAPS', 2000,
|
||||
"Recent Advances in AI Planning, 5th European Conference on Planning, ECP'99, Durham, UK, September 8-10, 1999, Proceedings"),
|
||||
('ewsp', 1991): ('EWSP', 'EWSP', 'ICAPS', 1991,
|
||||
"European Workshop on Planning, EWSP '91, Sankt Augustin, Germany, March 18-19, 1991, Proceedings"),
|
||||
}
|
||||
|
||||
|
||||
def predecessor_scope(family, base, suffix, volume, year, title):
|
||||
definition = VOLUMES.get((family, year))
|
||||
if definition is None:
|
||||
return None
|
||||
book, original, target, publication_year, expected_title = definition
|
||||
if (suffix or base != book or volume['key'] != f'conf/{family}/{year}'
|
||||
or title != expected_title or volume['fields'].get('year') != [str(publication_year)]):
|
||||
return None
|
||||
return {'original_series': original, 'target_family': target,
|
||||
'history_url': ECML_HISTORY if target == 'ECML-PKDD' else ICAPS_HISTORY,
|
||||
'proceedings_key': volume['key'], 'proceedings_title': title,
|
||||
'event_year': year, 'proceedings_year_fields': list(volume['fields']['year'])}
|
||||
|
||||
|
||||
def missing_candidate_families(targets, families, through_year):
|
||||
# These two families were absent from the earlier narrowed candidate JSONL,
|
||||
# although their proceedings and volume_counts exist in the full catalogue.
|
||||
required = {family for (family, year), definition in VOLUMES.items()
|
||||
if family in ('ecp', 'ewsp') and definition[2] in targets and year <= through_year}
|
||||
return sorted(family for family in required if not families.get(family))
|
||||
@@ -15,6 +15,7 @@ from urllib.parse import parse_qs,unquote,urlsplit
|
||||
from paper_atlas.config import DEFAULT_REGISTRY
|
||||
from paper_atlas.collectors.dblp import XML_SHA,DTD_MD5,RELEASE,records
|
||||
from paper_atlas.collectors.dblp_scope import scope
|
||||
from paper_atlas.collectors.dblp_predecessors import missing_candidate_families
|
||||
from paper_atlas.db.submissions import timestamp
|
||||
from paper_atlas.io.snapshots import digest_file
|
||||
from paper_atlas.io.publication_ids import mlsys_id
|
||||
@@ -88,6 +89,8 @@ def normalize_row(row,volumes,targets,include_workshops=False,through_year=2026)
|
||||
'scope_rule':edition['rule'],'author_names_raw':fields.get('author',[]),'publtype':row.get('publtype'),
|
||||
'bibliography':{key:value for key,value in fields.items() if key in ('booktitle','year','ee','url','pages','volume','number','publisher','series','isbn')},
|
||||
'ambiguous_identifiers':ambiguous,'publication_evidence':'inproceedings bibliography; not an original decision note'}
|
||||
if edition.get('historical_predecessor_evidence'):
|
||||
metadata['historical_predecessor_evidence'] = edition['historical_predecessor_evidence']
|
||||
if edition.get('joint_edition_evidence'):
|
||||
metadata['joint_edition_evidence'] = edition['joint_edition_evidence']
|
||||
if edition.get('track_scope_note'):
|
||||
@@ -124,8 +127,9 @@ def collect(catalogue,out,venues=None,candidates=None,archive=None,dtd=None,incl
|
||||
families = manifest.get('candidate_family_counts', {})
|
||||
missing = [venue for venue, family in [('MM', 'mm'), ('MLSys', 'mlsys')]
|
||||
if venue in targets and not families.get(family)]
|
||||
missing += ['ICAPS/'+family for family in missing_candidate_families(targets, families, through_year)]
|
||||
if missing:
|
||||
raise ValueError('Candidate file predates requested families ' + ', '.join(missing) + '; use the original --archive')
|
||||
raise ValueError('Candidate file omits requested families ' + ', '.join(missing) + '; use the original --archive')
|
||||
out=Path(out);out.mkdir(parents=True,exist_ok=True)
|
||||
with (out/'dblp.normalize.lock').open('a') as lock:
|
||||
fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
"""Explicit bibliographic aliases; a directory family alone is not venue evidence."""
|
||||
import re
|
||||
|
||||
from paper_atlas.collectors.dblp_predecessors import predecessor_scope
|
||||
|
||||
|
||||
# Ordered long/joint names precede their shorter forms.
|
||||
BOOKS = {
|
||||
@@ -27,7 +29,8 @@ BOOKS = {
|
||||
'pkdd':[('ECML/PKDD','ECML-PKDD',[]),('ECML PKDD','ECML-PKDD',[]),('PKDD','PKDD',['ECML-PKDD'])],
|
||||
'icra':[('ICRA','ICRA',[])], 'iros':[('IROS','IROS',[])],
|
||||
'rss':[('Robotics: Science and Systems','RSS',[])], 'corl':[('CoRL','CoRL',[])],
|
||||
'aips':[('ICAPS','ICAPS',[]),('AIPS','AIPS',['ICAPS'])], 'icaps':[('ICAPS','ICAPS',[])],
|
||||
'aips':[('ICAPS','ICAPS',[]),('AIPS','AIPS',['ICAPS']),('Expert Planning Systems','EPS',['ICAPS'])], 'icaps':[('ICAPS','ICAPS',[])],
|
||||
'ecp':[('ECP','ECP',['ICAPS'])], 'ewsp':[('EWSP','EWSP',['ICAPS'])],
|
||||
'kr':[('KR','KR',[])], 'atal':[('AAMAS','AAMAS',[])],
|
||||
'icassp':[('ICASSP','ICASSP',[])],
|
||||
'interspeech':[('INTERSPEECH','INTERSPEECH',[]),('EUROSPEECH','EUROSPEECH',['INTERSPEECH']),('ICSLP','ICSLP',['INTERSPEECH']),('ECST','ECST',['INTERSPEECH'])],
|
||||
@@ -135,7 +138,11 @@ def scope(row,volume,through_year=2026):
|
||||
if len(bibliographic)!=1 or not re.fullmatch(r'\d{4}',bibliographic[0]):return None,'ambiguous_bibliographic_year'
|
||||
if int(bibliographic[0])!=year and not re.search(r'\b'+str(year)+r'\b',title):return None,'unverified_year_difference'
|
||||
text=(suffix+' '+title).casefold();track='main'
|
||||
historical = historical_track(family, base, suffix, volume['key'], year, title)
|
||||
predecessor = predecessor_scope(family, base, suffix, volume, year, title)
|
||||
if (family in ('ecp', 'ewsp') or (family == 'aips' and base == 'Expert Planning Systems')) and not predecessor:
|
||||
return None, 'unverified_historical_predecessor'
|
||||
historical = (('main', 'Historical predecessor; exact volume, full title, event and publication years verified; original series retained.', predecessor['history_url'])
|
||||
if predecessor else historical_track(family, base, suffix, volume['key'], year, title))
|
||||
if historical:track=historical[0]
|
||||
elif 'findings' in text:track='findings'
|
||||
elif 'shared task' in text:track='shared_task'
|
||||
@@ -165,6 +172,8 @@ def scope(row,volume,through_year=2026):
|
||||
joint_evidence = 'https://www.eurai.org/ecai/history'
|
||||
result = {'venue':venue,'venue_aliases':aliases,'year':year,'track':track,'bibliographic_year':int(bibliographic[0]),
|
||||
'rule':family+':'+base,'proceedings_title':title,'edition_year_basis':year_basis}
|
||||
if predecessor:
|
||||
result['historical_predecessor_evidence'] = predecessor
|
||||
if joint_evidence:
|
||||
result['joint_edition_evidence'] = joint_evidence
|
||||
if historical:
|
||||
|
||||
@@ -0,0 +1,231 @@
|
||||
"""Reviewed postconference selected volumes from frozen public HTML metadata."""
|
||||
import argparse
|
||||
from collections import defaultdict
|
||||
import fcntl
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import tempfile
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from paper_atlas.db.submissions import timestamp
|
||||
from paper_atlas.io.html import Document, clean
|
||||
|
||||
SOURCE = 'springer_selected_volume'
|
||||
BOOK = '10.1007/978-1-4612-2660-4'
|
||||
BOOK_URL = 'https://link.springer.com/book/'+BOOK
|
||||
HISTORY_URL = 'https://www.gatsby.ucl.ac.uk/aistats/workshops.html'
|
||||
HISTORY_SHA = '6db205b9dbc82b4a096556efd2aa67e065cad8e0bd6351176b1962ffd1fca179'
|
||||
SCOPE = {
|
||||
'venue': 'AISTATS', 'year': 1993, 'track': 'selected_volume',
|
||||
'event_key': 'aistats-1993-selected-volume-1994',
|
||||
'publication_year': 1994, 'book_doi': BOOK,
|
||||
'book_title': 'Selecting Models from Data', 'expected_chapters': 49,
|
||||
'relationship': 'Selected workshop papers were revised and rereviewed for later publication; this is not the original complete workshop decision list.',
|
||||
}
|
||||
|
||||
|
||||
def snapshot_index(root):
|
||||
root = Path(root).resolve()
|
||||
receipts = json.loads((root/'receipts.json').read_bytes())['responses']
|
||||
index = {}
|
||||
for receipt in receipts:
|
||||
url = receipt['url']
|
||||
if url in index:
|
||||
raise ValueError('Duplicate snapshot URL')
|
||||
index[url] = (root, receipt)
|
||||
return index
|
||||
|
||||
|
||||
def read_page(index, url):
|
||||
if url not in index:
|
||||
raise ValueError('Missing required public HTML snapshot: '+url)
|
||||
root, receipt = index[url]
|
||||
filename = Path(receipt['file'])
|
||||
path = root/filename
|
||||
content_type = {k.lower():v for k,v in receipt.get('headers', {}).items()}.get('content-type', '')
|
||||
requested, final = urlsplit(url), urlsplit(receipt.get('final_url', ''))
|
||||
query = parse_qs(final.query, keep_blank_values=True)
|
||||
original_query = parse_qs(requested.query, keep_blank_values=True)
|
||||
# Springer returns the public metadata with an explicit cookie diagnostic.
|
||||
# Retain that final URL verbatim; accept no other redirect or changed page.
|
||||
if query != original_query and requested.netloc == 'link.springer.com':
|
||||
if (query.pop('error', None) != ['cookies_not_supported']
|
||||
or not re.fullmatch(r'[0-9a-f-]{36}', ''.join(query.pop('code', [])))):
|
||||
raise ValueError('Unreviewed Springer redirect query')
|
||||
if (filename.is_absolute() or '..' in filename.parts or not path.resolve().is_relative_to(root)
|
||||
or receipt.get('http_status') != 200
|
||||
or (final.scheme, final.netloc, final.path) != (requested.scheme, requested.netloc, requested.path)
|
||||
or final.fragment or query != original_query
|
||||
or 'text/html' not in content_type.lower()):
|
||||
raise ValueError('Unsafe, redirected or unsuccessful public HTML snapshot')
|
||||
raw = path.read_bytes()
|
||||
digest = hashlib.sha256(raw).hexdigest()
|
||||
if len(raw) > 2_000_000 or len(raw) != receipt['bytes'] or digest != receipt['sha256']:
|
||||
raise ValueError('Snapshot checksum or size differs')
|
||||
evidence = {'url':url, 'sha256':digest, 'bytes':len(raw),
|
||||
'file':'evidence/'+digest+'.html', 'http_status':200,
|
||||
'observed_at':timestamp(receipt['at']), 'response_receipt':receipt}
|
||||
return raw, evidence
|
||||
|
||||
|
||||
def parse_toc(html, *, excluded_materials=None):
|
||||
entries = {}
|
||||
for node in Document(html).root.walk():
|
||||
if node.tag != 'li' or node.attrs.get('data-test') != 'chapter':
|
||||
continue
|
||||
links = [n for n in node.walk() if n.tag == 'a' and re.fullmatch(
|
||||
r'/chapter/'+re.escape(BOOK)+r'_[1-9]\d*', n.attrs.get('href', ''))]
|
||||
if not links:
|
||||
headings = [clean(n.text()) for n in node.walk() if n.tag in ('h3','h4')]
|
||||
frontmatter = [n.attrs.get('href','') for n in node.walk()
|
||||
if n.tag == 'a' and n.attrs.get('data-test') == 'front-matter-pdf']
|
||||
if headings == ['Front Matter'] and len(frontmatter) == 1:
|
||||
if excluded_materials is not None:
|
||||
excluded_materials.append({'title':'Front Matter', 'url':frontmatter[0],
|
||||
'reason':'Section front matter without a chapter DOI; not an authored chapter.'})
|
||||
continue
|
||||
if len(links) != 1:
|
||||
raise ValueError('Chapter entry lacks one reviewed-book DOI')
|
||||
link, = links
|
||||
doi = link.attrs['href'].removeprefix('/chapter/')
|
||||
if doi in entries or not clean(link.text()):
|
||||
raise ValueError('Duplicate DOI or missing chapter title')
|
||||
entries[doi] = clean(link.text())
|
||||
return entries
|
||||
|
||||
|
||||
def parse_chapter(html, doi):
|
||||
nodes = list(Document(html).root.walk())
|
||||
meta = defaultdict(list)
|
||||
for node in nodes:
|
||||
if node.tag == 'meta' and node.attrs.get('name', '').startswith('citation_'):
|
||||
meta[node.attrs['name']].append(node.attrs.get('content', ''))
|
||||
def one(key):
|
||||
values = meta[key]
|
||||
if len(values) != 1 or not values[0]:
|
||||
raise ValueError('Missing or ambiguous citation field: '+key)
|
||||
return values[0]
|
||||
if (one('citation_doi') != doi or one('citation_inbook_title') != SCOPE['book_title']
|
||||
or one('citation_publication_date') != str(SCOPE['publication_year'])):
|
||||
raise ValueError('Chapter DOI, book or publication year differs from reviewed scope')
|
||||
raw_authors = meta['citation_author']
|
||||
if not raw_authors:
|
||||
raise ValueError('Missing named chapter authors')
|
||||
authors = []
|
||||
for raw in raw_authors:
|
||||
if raw.count(',') != 1:
|
||||
raise ValueError('Unreviewed citation author format')
|
||||
family, given = [clean(s) for s in raw.split(',', 1)]
|
||||
if not family or not given:
|
||||
raise ValueError('Incomplete citation author')
|
||||
authors.append({'given':given, 'family':family})
|
||||
abstract_nodes = [n for n in nodes if n.attrs.get('id') == 'Abs1-content']
|
||||
if len(abstract_nodes) > 1:
|
||||
raise ValueError('Ambiguous abstract section')
|
||||
abstract = clean(abstract_nodes[0].text()) if abstract_nodes else None
|
||||
first, last = one('citation_firstpage'), one('citation_lastpage')
|
||||
if not first.isdigit() or not last.isdigit() or int(first) > int(last):
|
||||
raise ValueError('Invalid chapter pages')
|
||||
return {'doi':doi, 'title':one('citation_title'), 'authors':authors,
|
||||
'authors_citation_raw':list(raw_authors), 'abstract':abstract or None,
|
||||
'first_page':first, 'last_page':last}
|
||||
|
||||
|
||||
def record_for(chapter, evidence, scope_evidence):
|
||||
return {
|
||||
'source':SOURCE, 'external_id':chapter['doi'],
|
||||
'url':'https://link.springer.com/chapter/'+chapter['doi'],
|
||||
'title':chapter['title'], 'abstract':chapter['abstract'], 'authors':chapter['authors'],
|
||||
'identifiers':{'doi':chapter['doi']},
|
||||
'submission_identifiers':[{'scheme':'publication_doi','value':chapter['doi']}],
|
||||
**{k:SCOPE[k] for k in ('venue','year','track','event_key')},
|
||||
'status':'accepted', 'observed_at':evidence['observed_at'],
|
||||
'raw_decision':'listed in the 1994 selected postconference volume; original 1993 review decision not supplied',
|
||||
'tags':[{'namespace':SOURCE+':publication_format','value':'selected_volume_chapter'}],
|
||||
'metadata':{
|
||||
'publication_format':'selected_volume_chapter', 'bibliographic_year':1994,
|
||||
'related_workshop_year':1993, 'book_doi':BOOK, 'book_title':SCOPE['book_title'],
|
||||
'year_basis':'Reviewed workshop year; bibliographic publication year is separately retained.',
|
||||
'first_page':chapter['first_page'], 'last_page':chapter['last_page'],
|
||||
'authors_citation_raw':chapter['authors_citation_raw'],
|
||||
'publication_evidence':'Published selected-volume chapter, not an original workshop decision note.',
|
||||
'scope_relationship':SCOPE['relationship'], 'decision_date_precision':'not_supplied',
|
||||
'source_evidence':evidence, 'scope_evidence':scope_evidence,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def collect(out, *, toc_snapshots, chapter_snapshots):
|
||||
toc_index, chapter_index = snapshot_index(toc_snapshots), snapshot_index(chapter_snapshots)
|
||||
frozen = {}
|
||||
def read(index, url):
|
||||
raw, evidence = read_page(index, url)
|
||||
frozen[evidence['file']] = raw
|
||||
return raw.decode('utf-8'), evidence
|
||||
history, history_evidence = read(toc_index, HISTORY_URL)
|
||||
if hashlib.sha256(history.encode()).hexdigest() != HISTORY_SHA:
|
||||
raise ValueError('Reviewed selected-volume relationship evidence changed')
|
||||
scope_evidence = [history_evidence]
|
||||
titles, excluded = {}, []
|
||||
for page in range(1, 4):
|
||||
html, evidence = read(toc_index, BOOK_URL+('?page='+str(page) if page > 1 else ''))
|
||||
scope_evidence.append(evidence)
|
||||
entries = parse_toc(html, excluded_materials=excluded)
|
||||
if titles.keys() & entries.keys():
|
||||
raise ValueError('DOI repeated across TOC pages')
|
||||
titles.update(entries)
|
||||
expected = {BOOK+'_'+str(i) for i in range(1, 50)}
|
||||
if titles.keys() != expected:
|
||||
raise ValueError('Reviewed 49-chapter inventory changed')
|
||||
chapter_urls = {'https://link.springer.com/chapter/'+doi for doi in expected}
|
||||
if chapter_index.keys() != chapter_urls:
|
||||
raise ValueError('Missing or extraneous chapter snapshot')
|
||||
records = []
|
||||
for doi in sorted(expected, key=lambda s:int(s.rsplit('_',1)[1])):
|
||||
html, evidence = read(chapter_index, 'https://link.springer.com/chapter/'+doi)
|
||||
chapter = parse_chapter(html, doi)
|
||||
if chapter['title'] != titles[doi]:
|
||||
raise ValueError('Chapter title disagrees with official TOC')
|
||||
records.append(record_for(chapter, evidence, scope_evidence))
|
||||
out = Path(out)
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
with (out/'springer-selected.collect.lock').open('a') as lock:
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
if any(p.name != 'springer-selected.collect.lock' for p in out.iterdir()):
|
||||
raise FileExistsError('Existing output must be inspected, not overwritten')
|
||||
with tempfile.TemporaryDirectory(prefix='.springer-selected-', dir=out) as temporary:
|
||||
stage = Path(temporary)
|
||||
(stage/'evidence').mkdir()
|
||||
for filename, raw in frozen.items():
|
||||
(stage/filename).write_bytes(raw)
|
||||
path = stage/'springer-aistats-1993-selected-volume.jsonl'
|
||||
path.write_text(''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in records))
|
||||
manifest = {'source':SOURCE, 'parser_version':1,
|
||||
**{k:SCOPE[k] for k in ('venue','year','track','event_key')},
|
||||
'count':len(records), 'file':str(out/path.name),
|
||||
'file_sha256':hashlib.sha256(path.read_bytes()).hexdigest(),
|
||||
'list_url':BOOK_URL, 'status':'partial',
|
||||
'excluded_materials':excluded,
|
||||
'missing_abstract_count':sum(not r['abstract'] for r in records)}
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)+'\n')
|
||||
report = {'status':'collected_pending_identity_review', 'records':len(records),
|
||||
'files':[manifest], 'reviewed_scope':SCOPE, 'database_written':False,
|
||||
'pdfs_downloaded':False, 'scope_evidence':scope_evidence}
|
||||
(stage/'collection-report.json').write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n')
|
||||
for item in stage.iterdir():
|
||||
if item.name != 'collection-report.json':
|
||||
item.replace(out/item.name)
|
||||
(stage/'collection-report.json').replace(out/'collection-report.json')
|
||||
return report
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser = argparse.ArgumentParser(prog=prog)
|
||||
parser.add_argument('--toc-snapshots', required=True)
|
||||
parser.add_argument('--chapter-snapshots', required=True)
|
||||
parser.add_argument('--out', required=True)
|
||||
args = parser.parse_args(argv)
|
||||
print(json.dumps(collect(args.out, toc_snapshots=args.toc_snapshots,
|
||||
chapter_snapshots=args.chapter_snapshots),ensure_ascii=False))
|
||||
+4281
File diff suppressed because it is too large
Load Diff
+9968
File diff suppressed because it is too large
Load Diff
Vendored
+54
@@ -0,0 +1,54 @@
|
||||
<HTML><HEAD><TITLE>AAMAS 2025: Proceedings of the 2025 International Conference on Autonomous Agents and Multiagent Systems</TITLE>
|
||||
<META http-equiv=Content-Type content="text/html; charset=utf-8"><LINK
|
||||
href="forms/index_files/factemplate.css" type=text/css rel=stylesheet>
|
||||
<META content="MSHTML 6.00.2900.3562" name=GENERATOR></HEAD>
|
||||
<BODY bgcolor="#010101" bgproperties="fixed"><DIV style="TEXT-ALIGN: center" >
|
||||
<table width="724" border="0" align="center" cellpadding="0" cellspacing="0" valign="top">
|
||||
<td colspan="2" align="center" valign="top" bgcolor="#FFFFFF"><img src="forms/index_files/aamas25.png" width="750" height="300" alt=""/></td>
|
||||
</tr>
|
||||
<tr bgcolor="#FFFFFF">
|
||||
<td width="125" align="center" valign="top" bgcolor="#FFFFFF"><table width="125" border="1" align="center" cellpadding="1" cellspacing="1" bordercolor="#444444" style="margin-bottom: 2">
|
||||
<tr>
|
||||
<td bgcolor="#FFFFFF"><p style="text-indent: 0; margin-left: 6; margin-right: 6; margin-top: 5; margin-bottom: 4" align="left"><A href="forms/index.htm">Main
|
||||
Page</A></p></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td bgcolor="#FFFFFF"><p style="text-indent: 0; margin-left: 6; margin-right: 6; margin-top: 5; margin-bottom: 4" align="left"><A href="forms/contents.htm">Table
|
||||
of Contents</A></p></td>
|
||||
</tr>
|
||||
</table></td>
|
||||
<td width="599"><p style="text-indent: 0; margin-left: 12; margin-right: 14; margin-top: 12; margin-bottom: 12" align="left"><font color="#010101" size="4" face="Arial, Helvetica, sans-serif"><strong>Welcome
|
||||
to the Proceedings of AAMAS-2025</strong></font></p>
|
||||
<p style="text-indent: 0; margin-left: 12; margin-right: 14; margin-top: 12; margin-bottom: 12;" align="left"><font color="#333333" size="2" face="Times New Roman, Times, serif">These
|
||||
are the proceedings of the 24th International Conference on Autonomous
|
||||
Agents and Multiagent Systems (AAMAS-2025). They are published by the
|
||||
International Foundation for Autonomous Agents and Multiagent Systems
|
||||
(IFAAMAS).</font></p></td>
|
||||
</tr>
|
||||
<tr bgcolor="#FFFFFF">
|
||||
<td align="center" valign="middle" bgcolor="#FFFFFF"> </td>
|
||||
<td><table width="96%" border="0" cellspacing="2" cellpadding="2">
|
||||
<tr align="left" valign="top">
|
||||
<td> <p style="text-indent: 0; margin-left: 12; margin-right: 6; margin-top: 0; margin-bottom: 6;" align="left"><font color="#333333" face="Times New Roman, Times, serif"><font size="2"><strong>ISBN: 979-8-4007-1426-9</strong></font></font></p></td>
|
||||
<td> <p style="text-indent: 0; margin-left: 12; margin-right: 6; margin-top: 0; margin-bottom: 6;" align="left"><font color="#333333" face="Times New Roman, Times, serif"></font></p></td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td width="42%"><div align="left"><img src="forms/index_files/ifaamas.jpg" width="256" height="104"></div></td>
|
||||
<td width="58%"><div align="center"><img src="forms/index_files/ACM-In-Cooperation_medium.jpg" width="285" height="86"></div></td>
|
||||
</tr>
|
||||
</table>
|
||||
<p style="text-indent: 0; margin-left: 12; margin-right: 14; margin-top: 12; margin-bottom: 12;" align="left"><font color="#333333" size="2" face="Times New Roman, Times, serif">Copyright
|
||||
© 2025 by International Foundation for Autonomous Agents and Multiagent
|
||||
Systems (IFAAMAS). Permission to make digital or hard copies of portions
|
||||
of this work for personal or classroom use is granted without fee provided
|
||||
that copies are not made or distributed for profit or commercial advantage
|
||||
and that copies bear this notice and the full citation on the first page.
|
||||
Copyright for components of this work owned by others than IFAAMAS must
|
||||
be honored. Abstracting with credit is permitted. To copy otherwise, to
|
||||
republish, to post on servers or to redistribute to lists, requires prior
|
||||
specific permission and/or a fee. </font></p></td>
|
||||
</tr>
|
||||
</table>
|
||||
<p> </p>
|
||||
|
||||
</BODY></HTML>
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
{
|
||||
"status": "two_successful_official_metadata_pages_recovered_third403_not_retried",
|
||||
"receipts": [
|
||||
{
|
||||
"file": "index.html",
|
||||
"url": "https://www.ifaamas.org/Proceedings/aamas2025/",
|
||||
"bytes": 4107,
|
||||
"sha256": "850b7825c66d0147ac8a8bbf03a554590e4a9e8f4b5c576e043999bbd43317f4",
|
||||
"at": "2026-11-09T12:31:49.347577+00:00",
|
||||
"at_precision": "filesystem_write_time_recovered",
|
||||
"http_status": 200,
|
||||
"content_type": "text/html",
|
||||
"receipt_recovered": true,
|
||||
"scope": "Collectorasserted200/htmlbeforewrite;request3acceptedprogram403;noheaders/finalredirectURLpersisted;no refetch or rawbytesmutation."
|
||||
},
|
||||
{
|
||||
"file": "contents.html",
|
||||
"url": "https://www.ifaamas.org/Proceedings/aamas2025/forms/contents.htm",
|
||||
"bytes": 370650,
|
||||
"sha256": "732847b51972f34c721a49d8bd963a80a029a48f3ec08022c9a0d5ddc2945549",
|
||||
"at": "2026-11-09T12:31:52.006642+00:00",
|
||||
"at_precision": "filesystem_write_time_recovered",
|
||||
"http_status": 200,
|
||||
"content_type": "text/html",
|
||||
"receipt_recovered": true,
|
||||
"scope": "Collectorasserted200/htmlbeforewrite;request3acceptedprogram403;noheaders/finalredirectURLpersisted;no refetch or rawbytesmutation."
|
||||
}
|
||||
],
|
||||
"failed_url": "https://aamas2025.org/index.php/conference/program/accepted-papers/",
|
||||
"not_attempted_url": "https://aamas2025.org/index.php/conference/calls/submission-instructions-main-technical-track/",
|
||||
"original_error": {
|
||||
"error": "HTTP Error 403: Forbidden",
|
||||
"type": "HTTPError",
|
||||
"downloaded": 2
|
||||
},
|
||||
"research_fulltext_downloaded": false,
|
||||
"database_written": false
|
||||
}
|
||||
Vendored
+273
@@ -0,0 +1,273 @@
|
||||
<?xml version='1.0' encoding='UTF-8'?>
|
||||
<collection id="T78">
|
||||
<volume id="1" type="proceedings">
|
||||
<!-- https://aclanthology.org/T78-1/ -->
|
||||
<meta>
|
||||
<booktitle>Theoretical Issues in Natural Language <fixed-case>P</fixed-case>rocessing-2</booktitle>
|
||||
<editor id="david-l-waltz"><first>David L.</first><last>Waltz</last></editor>
|
||||
<venue>tinlap</venue>
|
||||
<year>1978</year>
|
||||
</meta>
|
||||
<frontmatter>
|
||||
<!-- https://aclanthology.org/T78-1000/ -->
|
||||
<pdf hash="fb6164ad"/>
|
||||
<bibkey>tinlap-1978-theoretical</bibkey>
|
||||
</frontmatter>
|
||||
<paper id="1">
|
||||
<!-- https://aclanthology.org/T78-1001/ -->
|
||||
<title>Testing The Psychological Reality of a Representational Model</title>
|
||||
<author><first>Dedre</first><last>Gentner</last></author>
|
||||
<pdf hash="f723a956"/>
|
||||
<bibkey>gentner-1978-testing-psychological</bibkey>
|
||||
</paper>
|
||||
<paper id="2">
|
||||
<!-- https://aclanthology.org/T78-1002/ -->
|
||||
<title>What Makes Something “Ad Hoc”</title>
|
||||
<author id="roger-c-schank"><first>Roger C.</first><last>Schank</last></author>
|
||||
<pdf hash="636853f9"/>
|
||||
<bibkey>schank-1978-makes-something</bibkey>
|
||||
</paper>
|
||||
<paper id="3">
|
||||
<!-- https://aclanthology.org/T78-1003/ -->
|
||||
<title>The Relation of Grammar to Cognition–a Synopsis</title>
|
||||
<author><first>Leonard</first><last>Talmy</last></author>
|
||||
<pdf hash="101c67dc"/>
|
||||
<bibkey>talmy-1978-relation-grammar</bibkey>
|
||||
</paper>
|
||||
<paper id="4">
|
||||
<!-- https://aclanthology.org/T78-1004/ -->
|
||||
<title>On primitives, prototypes, and other semantic anomalies</title>
|
||||
<author><first>Terry</first><last>Winograd</last></author>
|
||||
<pdf hash="f8e47023"/>
|
||||
<bibkey>winograd-1978-primitives-prototypes</bibkey>
|
||||
</paper>
|
||||
<paper id="5">
|
||||
<!-- https://aclanthology.org/T78-1005/ -->
|
||||
<title>Taxonomic Lattice Structures for Situation Recognition</title>
|
||||
<author id="william-a-woods"><first>William A.</first><last>Woods</last></author>
|
||||
<pdf hash="92957ddf"/>
|
||||
<bibkey>woods-1978-taxonomic-lattice</bibkey>
|
||||
</paper>
|
||||
<paper id="6">
|
||||
<!-- https://aclanthology.org/T78-1006/ -->
|
||||
<title>Description Formation and Discourse Model Synthesis</title>
|
||||
<author id="bonnie-webber"><first>Bonnie Lynn</first><last>Webber</last></author>
|
||||
<pdf hash="ffded1d2"/>
|
||||
<bibkey>webber-1978-description-formation</bibkey>
|
||||
</paper>
|
||||
<paper id="7">
|
||||
<!-- https://aclanthology.org/T78-1007/ -->
|
||||
<title>The Processing of Referring Expressions within a Semantic Network</title>
|
||||
<author><first>John R.</first><last>Anderson</last></author>
|
||||
<pdf hash="8bcc083a"/>
|
||||
<bibkey>anderson-1978-processing-referring</bibkey>
|
||||
</paper>
|
||||
<paper id="8">
|
||||
<!-- https://aclanthology.org/T78-1008/ -->
|
||||
<title>Reference Diaries</title>
|
||||
<author><first>Herbert H.</first><last>Clark</last></author>
|
||||
<author><first>Catherine</first><last>Marshall</last></author>
|
||||
<pdf hash="d06b7ea2"/>
|
||||
<bibkey>clark-marshall-1978-reference-diaries</bibkey>
|
||||
</paper>
|
||||
<paper id="9">
|
||||
<!-- https://aclanthology.org/T78-1009/ -->
|
||||
<title>Subsequent Reference: Syntactic and Rhetorical Constraints</title>
|
||||
<author id="david-d-mcdonald"><first>David D.</first><last>McDonald</last></author>
|
||||
<pdf hash="2d424559"/>
|
||||
<bibkey>mcdonald-1978-subsequent-reference</bibkey>
|
||||
</paper>
|
||||
<paper id="10">
|
||||
<!-- https://aclanthology.org/T78-1010/ -->
|
||||
<title>Some Psycholinguistic Constraints on the Construction and Interpretation of Definite Descriptions</title>
|
||||
<author><first>Andrew</first><last>Ortony</last></author>
|
||||
<pdf hash="69807727"/>
|
||||
<bibkey>ortony-1978-psycholinguistic-constraints</bibkey>
|
||||
</paper>
|
||||
<paper id="11">
|
||||
<!-- https://aclanthology.org/T78-1011/ -->
|
||||
<title>Bound Variables and Other Anaphors</title>
|
||||
<author id="barbara-h-partee"><first>Barbara H.</first><last>Partee</last></author>
|
||||
<pdf hash="e8e6a510"/>
|
||||
<bibkey>partee-1978-bound-variables</bibkey>
|
||||
</paper>
|
||||
<paper id="12">
|
||||
<!-- https://aclanthology.org/T78-1012/ -->
|
||||
<title>The Use of Focus as a Tool for Disambiguation of Definite Noun Phrases</title>
|
||||
<author id="candace-l-sidner"><first>Candace L.</first><last>Sidner</last></author>
|
||||
<pdf hash="7b96d2b7"/>
|
||||
<bibkey>sidner-1978-use-focus</bibkey>
|
||||
</paper>
|
||||
<paper id="13">
|
||||
<!-- https://aclanthology.org/T78-1013/ -->
|
||||
<title>Focusing in Dialog</title>
|
||||
<author id="barbara-j-grosz"><first>Barbara J.</first><last>Grosz</last></author>
|
||||
<pdf hash="8c8d4c2e"/>
|
||||
<bibkey>grosz-1978-focusing-dialog</bibkey>
|
||||
</paper>
|
||||
<paper id="14">
|
||||
<!-- https://aclanthology.org/T78-1014/ -->
|
||||
<title>Topic Levels</title>
|
||||
<author><first>Joseph E.</first><last>Grimes</last></author>
|
||||
<pdf hash="f485b2ba"/>
|
||||
<bibkey>grimes-1978-topic-levels</bibkey>
|
||||
</paper>
|
||||
<paper id="15">
|
||||
<!-- https://aclanthology.org/T78-1015/ -->
|
||||
<title>Toward a Rational Model of Discourse Comprehension</title>
|
||||
<author><first>Jerry L.</first><last>Morgan</last></author>
|
||||
<pdf hash="5e12a597"/>
|
||||
<bibkey>morgan-1978-toward-rational</bibkey>
|
||||
</paper>
|
||||
<paper id="16">
|
||||
<!-- https://aclanthology.org/T78-1016/ -->
|
||||
<title>Assent and Compliance in Children’s Language Comprehension</title>
|
||||
<author><first>David R.</first><last>Olson</last></author>
|
||||
<pdf hash="708bc4a0"/>
|
||||
<bibkey>olson-1978-assent-compliance</bibkey>
|
||||
</paper>
|
||||
<paper id="17">
|
||||
<!-- https://aclanthology.org/T78-1017/ -->
|
||||
<title>Speech Acts as a Basis for Understanding Dialogue Coherence</title>
|
||||
<author id="c-raymond-perrault"><first>C. Raymond</first><last>Perrault</last></author>
|
||||
<author id="james-allen"><first>James F.</first><last>Allen</last></author>
|
||||
<pdf hash="d111402b"/>
|
||||
<bibkey>perrault-allen-1978-speech</bibkey>
|
||||
</paper>
|
||||
<paper id="18">
|
||||
<!-- https://aclanthology.org/T78-1018/ -->
|
||||
<title>A Framework for Comparing Language Experiences (with particular emphasis on: The Effect of Audience on Discourse Models)</title>
|
||||
<author><first>Andee</first><last>Rubin</last></author>
|
||||
<pdf hash="5d9a6150"/>
|
||||
<bibkey>rubin-1978-framework-comparing</bibkey>
|
||||
</paper>
|
||||
<paper id="19">
|
||||
<!-- https://aclanthology.org/T78-1019/ -->
|
||||
<title>Intentlonallty and Human Conversations</title>
|
||||
<author id="jaime-g-carbonell"><first>Jaime G.</first><last>Carbonell Jr</last></author>
|
||||
<pdf hash="c50fc619"/>
|
||||
<bibkey>carbonell-jr-1978-intentlonallty-human</bibkey>
|
||||
</paper>
|
||||
<paper id="20">
|
||||
<!-- https://aclanthology.org/T78-1020/ -->
|
||||
<title>On the Interdependence of Language and Perception</title>
|
||||
<author id="david-l-waltz"><first>David L.</first><last>Waltz</last></author>
|
||||
<pdf hash="3c708191"/>
|
||||
<bibkey>waltz-1978-interdependence-language</bibkey>
|
||||
</paper>
|
||||
<paper id="21">
|
||||
<!-- https://aclanthology.org/T78-1021/ -->
|
||||
<title>The Problem of Naming Shapes: Vision-Language Interface</title>
|
||||
<author id="ruzena-bajcsy"><first>R.</first><last>Bajcsy</last></author>
|
||||
<author id="aravind-joshi"><first>A. K.</first><last>Joshi</last></author>
|
||||
<pdf hash="50331b4e"/>
|
||||
<bibkey>bajcsy-joshi-1978-problem-naming</bibkey>
|
||||
</paper>
|
||||
<paper id="22">
|
||||
<!-- https://aclanthology.org/T78-1022/ -->
|
||||
<title>An Argument about the Composition of Conceptual Structure</title>
|
||||
<author><first>Ray</first><last>Jackendoff</last></author>
|
||||
<pdf hash="57d001f8"/>
|
||||
<bibkey>jackendoff-1978-argument-composition</bibkey>
|
||||
</paper>
|
||||
<paper id="23">
|
||||
<!-- https://aclanthology.org/T78-1023/ -->
|
||||
<title>On the Ontological Status of Visual Mental Images</title>
|
||||
<author><first>Stephen Michael</first><last>Kosslyn</last></author>
|
||||
<pdf hash="fd06e265"/>
|
||||
<bibkey>kosslyn-1978-ontological-status</bibkey>
|
||||
</paper>
|
||||
<paper id="24">
|
||||
<!-- https://aclanthology.org/T78-1024/ -->
|
||||
<title>What has language to do with perception? Some speculations on the <i>Lingua Mentis</i></title>
|
||||
<author><first>Zenon W.</first><last>Pylyshyn</last></author>
|
||||
<pdf hash="a2f3a044"/>
|
||||
<bibkey>pylyshyn-1978-language-perception</bibkey>
|
||||
</paper>
|
||||
<paper id="25">
|
||||
<!-- https://aclanthology.org/T78-1025/ -->
|
||||
<title>Semantic Primitives in Language and Vision</title>
|
||||
<author id="yorick-wilks"><first>Yorick</first><last>Wilks</last></author>
|
||||
<pdf hash="b4e13896"/>
|
||||
<bibkey>wilks-1978-semantic-primitives</bibkey>
|
||||
</paper>
|
||||
<paper id="26">
|
||||
<!-- https://aclanthology.org/T78-1026/ -->
|
||||
<title>A Note on Partial Match of Descriptions. Can One Simultaneously Question (Retrieve) and Inform (Update)?</title>
|
||||
<author id="aravind-joshi"><first>Aravind K.</first><last>Joshi</last></author>
|
||||
<pdf hash="74969243"/>
|
||||
<bibkey>joshi-1978-note-partial</bibkey>
|
||||
</paper>
|
||||
<paper id="27">
|
||||
<!-- https://aclanthology.org/T78-1027/ -->
|
||||
<title>With a Spoon in Hand This Must Be the Eating Frame</title>
|
||||
<author><first>Eugene</first><last>Charniak</last></author>
|
||||
<pdf hash="f1be68bb"/>
|
||||
<bibkey>charniak-1978-spoon-hand</bibkey>
|
||||
</paper>
|
||||
<paper id="28">
|
||||
<!-- https://aclanthology.org/T78-1028/ -->
|
||||
<title>Fragments of a Theory of Human Plausible Reasoning</title>
|
||||
<author><first>Allan</first><last>Collins</last></author>
|
||||
<pdf hash="67b4bca2"/>
|
||||
<bibkey>collins-1978-fragments-theory</bibkey>
|
||||
</paper>
|
||||
<paper id="29">
|
||||
<!-- https://aclanthology.org/T78-1029/ -->
|
||||
<title>Indirect Responses to Loaded Questions</title>
|
||||
<author><first>S. Jerrold</first><last>Kaplan</last></author>
|
||||
<pdf hash="135037f9"/>
|
||||
<bibkey>kaplan-1978-indirect-responses</bibkey>
|
||||
</paper>
|
||||
<paper id="30">
|
||||
<!-- https://aclanthology.org/T78-1030/ -->
|
||||
<title>On Reasoning by Default</title>
|
||||
<author><first>Raymond</first><last>Reiter</last></author>
|
||||
<pdf hash="0de53d33"/>
|
||||
<bibkey>reiter-1978-reasoning-default</bibkey>
|
||||
</paper>
|
||||
<paper id="31">
|
||||
<!-- https://aclanthology.org/T78-1031/ -->
|
||||
<title>Path-Based and Node-Based Inference in Semantic Networks</title>
|
||||
<author id="stuart-c-shapiro"><first>Stuart C.</first><last>Shapiro</last></author>
|
||||
<pdf hash="2ddc5535"/>
|
||||
<bibkey>shapiro-1978-path-based</bibkey>
|
||||
</paper>
|
||||
<paper id="32">
|
||||
<!-- https://aclanthology.org/T78-1032/ -->
|
||||
<title>The Representation of Derivable Information in Memory: When What Might Have Been Left Unsaid Is Said</title>
|
||||
<author><first>Rand J.</first><last>Spiro</last></author>
|
||||
<author><first>Joseph</first><last>Esposito</last></author>
|
||||
<author><first>Richard J.</first><last>Vondruska</last></author>
|
||||
<pdf hash="7758fb45"/>
|
||||
<bibkey>spiro-etal-1978-representation-derivable</bibkey>
|
||||
</paper>
|
||||
<paper id="33">
|
||||
<!-- https://aclanthology.org/T78-1033/ -->
|
||||
<title>A Heuristic for Paradigms</title>
|
||||
<author><first>Joseph E.</first><last>Grimes</last></author>
|
||||
<pdf hash="53b42c16"/>
|
||||
<bibkey>grimes-1978-heuristic-paradigms</bibkey>
|
||||
</paper>
|
||||
<paper id="34">
|
||||
<!-- https://aclanthology.org/T78-1034/ -->
|
||||
<title>A Computational Account of Some Constraints on Language</title>
|
||||
<author id="mitch-marcus"><first>Mitchell</first><last>Marcus</last></author>
|
||||
<pdf hash="c9724c43"/>
|
||||
<bibkey>marcus-1978-computational-account</bibkey>
|
||||
</paper>
|
||||
<paper id="35">
|
||||
<!-- https://aclanthology.org/T78-1035/ -->
|
||||
<title>Remarks on Processing, Constraints, and the Lexicon</title>
|
||||
<author id="thomas-wasow"><first>Thomas</first><last>Wasow</last></author>
|
||||
<pdf hash="8beed275"/>
|
||||
<bibkey>wasow-1978-remarks-processing</bibkey>
|
||||
</paper>
|
||||
<paper id="36">
|
||||
<!-- https://aclanthology.org/T78-1036/ -->
|
||||
<title>List of questions suggested for consideration in each session</title>
|
||||
<pdf hash="1950b908"/>
|
||||
<bibkey>nn-1978-list-questions</bibkey>
|
||||
</paper>
|
||||
</volume>
|
||||
</collection>
|
||||
+314
@@ -0,0 +1,314 @@
|
||||
{
|
||||
"source_snapshot_sha256": "0376ad019b216553faff0a14e1757b2d4c7a3b84c1beacf0511ef83695211b18",
|
||||
"scope": "Pinned proceedings metadata, not authentic example paper records or evidence that all volumes contain indexed papers. Test papers are synthetic.",
|
||||
"volumes": [
|
||||
{
|
||||
"key": "conf/ecml/1987",
|
||||
"fields": {
|
||||
"editor": [
|
||||
"Ivan Bratko",
|
||||
"Nada Lavrac"
|
||||
],
|
||||
"title": [
|
||||
"Progress in Machine Learning - Proceedings of EWSL 87: 2nd European Working Session on Learning, Bled, Yugoslavia, May 1987"
|
||||
],
|
||||
"booktitle": [
|
||||
"EWSL"
|
||||
],
|
||||
"publisher": [
|
||||
"Sigma Press, Wilmslow"
|
||||
],
|
||||
"year": [
|
||||
"1987"
|
||||
],
|
||||
"isbn": [
|
||||
"1-85058-088-X"
|
||||
],
|
||||
"url": [
|
||||
"db/conf/ecml/ewsl87.html"
|
||||
]
|
||||
},
|
||||
"bibliographic_counts": [
|
||||
{
|
||||
"year": "1987",
|
||||
"booktitle": "EWSL",
|
||||
"record_type": "inproceedings",
|
||||
"count": 16
|
||||
}
|
||||
],
|
||||
"expected_original_series": "EWSL",
|
||||
"expected_target_family": "ECML-PKDD"
|
||||
},
|
||||
{
|
||||
"key": "conf/ecml/1988",
|
||||
"fields": {
|
||||
"editor": [
|
||||
"Derek H. Sleeman"
|
||||
],
|
||||
"title": [
|
||||
"Proceedings of the Third European Working Session on Learning, EWSL 1988, Turing Institute, Glasgow, UK, October 3-5, 1988"
|
||||
],
|
||||
"booktitle": [
|
||||
"EWSL"
|
||||
],
|
||||
"publisher": [
|
||||
"Pitman Publishing"
|
||||
],
|
||||
"year": [
|
||||
"1988"
|
||||
],
|
||||
"isbn": [
|
||||
"0-273-08800-9"
|
||||
],
|
||||
"url": [
|
||||
"db/conf/ecml/ewsl88.html"
|
||||
]
|
||||
},
|
||||
"bibliographic_counts": [
|
||||
{
|
||||
"year": "1988",
|
||||
"booktitle": "EWSL",
|
||||
"record_type": "inproceedings",
|
||||
"count": 21
|
||||
}
|
||||
],
|
||||
"expected_original_series": "EWSL",
|
||||
"expected_target_family": "ECML-PKDD"
|
||||
},
|
||||
{
|
||||
"key": "conf/ecml/1989",
|
||||
"fields": {
|
||||
"editor": [
|
||||
"Katharina Morik"
|
||||
],
|
||||
"title": [
|
||||
"Proceedings of the 4th European Working Session on Learning, Montpellier, France, 4-6 December 1989"
|
||||
],
|
||||
"booktitle": [
|
||||
"EWSL"
|
||||
],
|
||||
"publisher": [
|
||||
"Pitman and Morgan Kaufman Publishers"
|
||||
],
|
||||
"year": [
|
||||
"1989"
|
||||
]
|
||||
},
|
||||
"bibliographic_counts": [],
|
||||
"expected_original_series": "EWSL",
|
||||
"expected_target_family": "ECML-PKDD"
|
||||
},
|
||||
{
|
||||
"key": "conf/ecml/1991",
|
||||
"fields": {
|
||||
"editor": [
|
||||
"Yves Kodratoff"
|
||||
],
|
||||
"title": [
|
||||
"Machine Learning - EWSL-91, European Working Session on Learning, Porto, Portugal, March 6-8, 1991, Proceedings"
|
||||
],
|
||||
"booktitle": [
|
||||
"EWSL"
|
||||
],
|
||||
"series": [
|
||||
"Lecture Notes in Computer Science"
|
||||
],
|
||||
"volume": [
|
||||
"482"
|
||||
],
|
||||
"publisher": [
|
||||
"Springer"
|
||||
],
|
||||
"year": [
|
||||
"1991"
|
||||
],
|
||||
"isbn": [
|
||||
"3-540-53816-X"
|
||||
],
|
||||
"ee": [
|
||||
"https://doi.org/10.1007/BFb0016999"
|
||||
],
|
||||
"url": [
|
||||
"db/conf/ecml/ewsl91.html"
|
||||
]
|
||||
},
|
||||
"bibliographic_counts": [
|
||||
{
|
||||
"year": "1991",
|
||||
"booktitle": "EWSL",
|
||||
"record_type": "inproceedings",
|
||||
"count": 44
|
||||
}
|
||||
],
|
||||
"expected_original_series": "EWSL",
|
||||
"expected_target_family": "ECML-PKDD"
|
||||
},
|
||||
{
|
||||
"key": "conf/ecp/1997",
|
||||
"fields": {
|
||||
"editor": [
|
||||
"Sam Steel",
|
||||
"Rachid Alami 0001"
|
||||
],
|
||||
"title": [
|
||||
"Recent Advances in AI Planning, 4th European Conference on Planning, ECP'97, Toulouse, France, September 24-26, 1997, Proceedings"
|
||||
],
|
||||
"booktitle": [
|
||||
"ECP"
|
||||
],
|
||||
"series": [
|
||||
"Lecture Notes in Computer Science"
|
||||
],
|
||||
"volume": [
|
||||
"1348"
|
||||
],
|
||||
"publisher": [
|
||||
"Springer"
|
||||
],
|
||||
"year": [
|
||||
"1997"
|
||||
],
|
||||
"isbn": [
|
||||
"3-540-63912-8"
|
||||
],
|
||||
"ee": [
|
||||
"https://doi.org/10.1007/3-540-63912-8"
|
||||
],
|
||||
"url": [
|
||||
"db/conf/ecp/ecp1997.html"
|
||||
]
|
||||
},
|
||||
"bibliographic_counts": [
|
||||
{
|
||||
"year": "1997",
|
||||
"booktitle": "ECP",
|
||||
"record_type": "inproceedings",
|
||||
"count": 35
|
||||
}
|
||||
],
|
||||
"expected_original_series": "ECP",
|
||||
"expected_target_family": "ICAPS"
|
||||
},
|
||||
{
|
||||
"key": "conf/ecp/1999",
|
||||
"fields": {
|
||||
"editor": [
|
||||
"Susanne Biundo",
|
||||
"Maria Fox 0001"
|
||||
],
|
||||
"title": [
|
||||
"Recent Advances in AI Planning, 5th European Conference on Planning, ECP'99, Durham, UK, September 8-10, 1999, Proceedings"
|
||||
],
|
||||
"booktitle": [
|
||||
"ECP"
|
||||
],
|
||||
"series": [
|
||||
"Lecture Notes in Computer Science"
|
||||
],
|
||||
"volume": [
|
||||
"1809"
|
||||
],
|
||||
"publisher": [
|
||||
"Springer"
|
||||
],
|
||||
"year": [
|
||||
"2000"
|
||||
],
|
||||
"isbn": [
|
||||
"3-540-67866-2"
|
||||
],
|
||||
"ee": [
|
||||
"https://doi.org/10.1007/10720246"
|
||||
],
|
||||
"url": [
|
||||
"db/conf/ecp/ecp1999.html"
|
||||
]
|
||||
},
|
||||
"bibliographic_counts": [
|
||||
{
|
||||
"year": "1999",
|
||||
"booktitle": "ECP",
|
||||
"record_type": "inproceedings",
|
||||
"count": 28
|
||||
}
|
||||
],
|
||||
"expected_original_series": "ECP",
|
||||
"expected_target_family": "ICAPS"
|
||||
},
|
||||
{
|
||||
"key": "conf/ewsp/1991",
|
||||
"fields": {
|
||||
"editor": [
|
||||
"Joachim Hertzberg"
|
||||
],
|
||||
"title": [
|
||||
"European Workshop on Planning, EWSP '91, Sankt Augustin, Germany, March 18-19, 1991, Proceedings"
|
||||
],
|
||||
"booktitle": [
|
||||
"EWSP"
|
||||
],
|
||||
"series": [
|
||||
"Lecture Notes in Computer Science"
|
||||
],
|
||||
"volume": [
|
||||
"522"
|
||||
],
|
||||
"publisher": [
|
||||
"Springer"
|
||||
],
|
||||
"year": [
|
||||
"1991"
|
||||
],
|
||||
"isbn": [
|
||||
"3-540-54364-3"
|
||||
],
|
||||
"ee": [
|
||||
"https://doi.org/10.1007/BFb0052944"
|
||||
],
|
||||
"url": [
|
||||
"db/conf/ewsp/ewsp1991.html"
|
||||
]
|
||||
},
|
||||
"bibliographic_counts": [
|
||||
{
|
||||
"year": "1991",
|
||||
"booktitle": "EWSP",
|
||||
"record_type": "inproceedings",
|
||||
"count": 10
|
||||
}
|
||||
],
|
||||
"expected_original_series": "EWSP",
|
||||
"expected_target_family": "ICAPS"
|
||||
},
|
||||
{
|
||||
"key": "conf/aips/1990",
|
||||
"fields": {
|
||||
"title": [
|
||||
"Proceedings of the First International Conference on Expert Planning Systems, Brighton, UK, June 27-29, 1990"
|
||||
],
|
||||
"booktitle": [
|
||||
"Expert Planning Systems"
|
||||
],
|
||||
"publisher": [
|
||||
"AAAI"
|
||||
],
|
||||
"year": [
|
||||
"1990"
|
||||
],
|
||||
"url": [
|
||||
"db/conf/aips/aips1990.html"
|
||||
]
|
||||
},
|
||||
"bibliographic_counts": [
|
||||
{
|
||||
"year": "1990",
|
||||
"booktitle": "Expert Planning Systems",
|
||||
"record_type": "inproceedings",
|
||||
"count": 48
|
||||
}
|
||||
],
|
||||
"expected_original_series": "EPS",
|
||||
"expected_target_family": "ICAPS"
|
||||
}
|
||||
]
|
||||
}
|
||||
Binary file not shown.
@@ -0,0 +1,85 @@
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from paper_atlas.collectors import aamas, aamas2025
|
||||
|
||||
|
||||
FIXTURE = Path(__file__).parent/'fixtures/aamas2025'
|
||||
|
||||
|
||||
class Aamas2025Tests(unittest.TestCase):
|
||||
def parse(self, html=None):
|
||||
return aamas2025.parse_contents(html or (FIXTURE/'contents.html').read_text(),
|
||||
{'observed_at': '2026-11-09T12:31:50+00:00', 'url': aamas2025.URLS['contents']})
|
||||
|
||||
def test_real_toc_preserves_every_title_author_and_affiliation(self):
|
||||
expected = json.loads((FIXTURE/'expected-authors.json').read_text())
|
||||
self.assertEqual(hashlib.sha256((FIXTURE/'contents.html').read_bytes()).hexdigest(), expected['source_sha256'])
|
||||
rows, excluded = self.parse()
|
||||
self.assertEqual((len(rows), len(excluded)), (476, 3))
|
||||
by_url = {r['paper_url']: r for r in expected['entries']}
|
||||
for row in rows:
|
||||
original = by_url[row['metadata']['paper_url']]
|
||||
self.assertEqual(row['title'], original['title'])
|
||||
self.assertEqual(row['authors'], [a['name'] for a in original['authors']])
|
||||
self.assertEqual(row['metadata']['author_affiliations'], [a['affiliation'] for a in original['authors']])
|
||||
self.assertIsNone(row['abstract'])
|
||||
self.assertEqual(row['identifiers'], {})
|
||||
self.assertEqual(row['status'], 'accepted')
|
||||
self.assertNotIn('decision_date', row)
|
||||
self.assertTrue(all(t['namespace'] == 'aamas_proceedings:publication_format' for t in row['tags']))
|
||||
|
||||
def test_publication_format_and_track_are_separate(self):
|
||||
rows, excluded = self.parse()
|
||||
groups = {k: sum(r['track'] == k for r in rows) for k in {r['track'] for r in rows}}
|
||||
self.assertEqual(groups, {'proceedings_unspecified': 409, 'blue_sky': 12, 'doctoral': 25, 'demo': 21, 'jaamas': 9})
|
||||
competition = next(r for r in rows if r['external_id'] == '2025/p3000')
|
||||
self.assertEqual(competition['year'], 2025)
|
||||
self.assertEqual(competition['metadata']['publication_format'], 'competition_report')
|
||||
self.assertEqual(competition['metadata']['section_heading'], 'Demo Track')
|
||||
self.assertEqual(competition['track'], 'proceedings_unspecified')
|
||||
self.assertEqual(len({r['external_id'] for r in rows}), 476)
|
||||
self.assertEqual([r['publisher_page'] for r in excluded], [1, 3, 4])
|
||||
|
||||
def test_refuses_changed_edition_inventory_or_author_markup(self):
|
||||
original = (FIXTURE/'contents.html').read_text()
|
||||
for html in [original.replace('AAMAS 2025:', 'AAMAS 2026:', 1),
|
||||
original.replace('../pdfs/p5.pdf', '../pdfs/p14.pdf', 1),
|
||||
original.replace('Paper Track</font>', 'Workshop Track</font>'),
|
||||
original.replace('Amir Abolfazli', '', 1),
|
||||
original.replace('[COMP24]', '[CHANGED]', 1)]:
|
||||
with self.subTest(change=html[:60]):
|
||||
with self.assertRaises(ValueError): self.parse(html)
|
||||
|
||||
def test_frozen_collection_is_partial_and_never_fetches_pdfs(self):
|
||||
with tempfile.TemporaryDirectory() as tmp, patch.object(aamas2025, 'fetch', side_effect=AssertionError('No HTTP in frozen mode')):
|
||||
out = Path(tmp)/'out'
|
||||
report = aamas.collect(2025, out, snapshots=FIXTURE)
|
||||
self.assertEqual(report['records'], 476)
|
||||
self.assertFalse(report['database_written'])
|
||||
self.assertFalse(report['pdfs_downloaded'])
|
||||
self.assertEqual(len(report['files']), 5)
|
||||
for m in report['files']:
|
||||
self.assertEqual(m['status'], 'partial')
|
||||
self.assertEqual(m['missing_abstract_count'], m['count'])
|
||||
self.assertEqual(hashlib.sha256(Path(m['file']).read_bytes()).hexdigest(), m['file_sha256'])
|
||||
receipt = report['source_receipts'][0]['response_receipt']
|
||||
self.assertEqual(receipt['at_precision'], 'filesystem_write_time_recovered')
|
||||
self.assertTrue((out/'aamas-proceedings-2025.json').is_file())
|
||||
|
||||
def test_rejects_tampered_or_unsafe_frozen_response(self):
|
||||
import shutil
|
||||
for mode in ['tamper', 'path', 'status']:
|
||||
with self.subTest(mode=mode), tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); source = root/'source'; shutil.copytree(FIXTURE, source)
|
||||
p = source/'receipts.json'; r = json.loads(p.read_text())
|
||||
if mode == 'tamper': (source/'contents.html').write_text('bad')
|
||||
elif mode == 'path': r['receipts'][0]['file'] = '../index.html'
|
||||
else: r['receipts'][0]['http_status'] = 403
|
||||
p.write_text(json.dumps(r))
|
||||
with self.assertRaises(ValueError): aamas.collect(2025, root/'out', snapshots=source)
|
||||
self.assertFalse((root/'out/aamas-proceedings-2025.json').exists())
|
||||
@@ -99,7 +99,7 @@ class AamasProceedingsTests(unittest.TestCase):
|
||||
aamas.collect(2012, root/'out', snapshots=source)
|
||||
self.assertFalse((root/'out/aamas-proceedings-2012.json').exists())
|
||||
with self.assertRaises(ValueError):
|
||||
aamas.collect(2025, root/'future', snapshots=source)
|
||||
aamas.collect(2026, root/'future', snapshots=source)
|
||||
self.assertFalse((root/'future').exists())
|
||||
|
||||
def test_ids_colliding_across_sections_do_not_produce_a_completed_collection(self):
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
"""Pinned T78 evidence, edition-local aliases and an explicitly identified material."""
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
from paper_atlas.collectors.acl import collect_xml, volume_scope
|
||||
from paper_atlas.collectors.acl_historical import event_scope, material_reason
|
||||
from paper_atlas.db.store import Store
|
||||
from paper_atlas.ingestion.records import ingest_file
|
||||
|
||||
FIXTURE = Path(__file__).parent/'fixtures/acl-t78.xml'
|
||||
COMMIT = 'a7be623d01c48c61405dec31c871121cdbed5a01'
|
||||
|
||||
|
||||
class AclHistoricalTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.root = ET.parse(FIXTURE).getroot()
|
||||
self.volume = self.root.find('volume')
|
||||
self.meta = self.volume.find('meta')
|
||||
|
||||
def test_frozen_volume_matches_official_snapshot(self):
|
||||
self.assertEqual(hashlib.sha256(FIXTURE.read_bytes()).hexdigest(), 'f712f94cff0cba05f786b7ac31dad2ea9408fde2a17a04531130df623f57dd84')
|
||||
self.assertEqual(len(self.volume.findall('paper')), 36)
|
||||
self.assertEqual(volume_scope(self.meta, '1', 'T78'), ('ACL+TINLAP', ['ACL', 'TINLAP'], 'proceedings_unspecified'))
|
||||
|
||||
def test_unknown_year_collection_or_volume_is_not_globally_aliased(self):
|
||||
for collection, volume in [('T75', '1'), ('T87', '1'), ('T78', '2'), ('C65', '1')]:
|
||||
with self.subTest(collection=collection, volume=volume):
|
||||
self.assertIsNone(event_scope(self.meta, volume, collection))
|
||||
self.assertIsNone(volume_scope(self.meta, volume, collection))
|
||||
self.assertIsNone(volume_scope(self.meta, '1'))
|
||||
|
||||
def test_known_volume_scope_rejects_critical_metadata_drift(self):
|
||||
for field, replacement in [('venue', 'acl'), ('year', '1979'), ('booktitle', 'Another TINLAP'), ('editor/first', 'Another')]:
|
||||
changed = copy.deepcopy(self.meta)
|
||||
changed.find(field).text = replacement
|
||||
with self.subTest(field=field), self.assertRaises(ValueError):
|
||||
event_scope(changed, '1', 'T78')
|
||||
changed = copy.deepcopy(self.meta)
|
||||
ET.SubElement(changed, 'year').text = '1978'
|
||||
with self.assertRaises(ValueError):
|
||||
event_scope(changed, '1', 'T78')
|
||||
|
||||
def test_evidence_retains_original_series_dates_and_role_boundary(self):
|
||||
scope = event_scope(self.meta, '1', 'T78')
|
||||
self.assertEqual(scope['original_venue'], 'TINLAP')
|
||||
self.assertEqual(scope['event_dates'], ['1978-07-25', '1978-07-27'])
|
||||
self.assertEqual([e['pdf_pages'] for e in scope['evidence']], [[9, 10], [3]])
|
||||
self.assertIn('not identified', scope['track_scope_note'])
|
||||
self.assertIn('pending', scope['evidence'][0]['scope'])
|
||||
|
||||
def test_only_exact_session_questions_material_is_excluded(self):
|
||||
paper = copy.deepcopy(self.volume.findall('paper')[-1])
|
||||
self.assertIn('Session question list', material_reason('T78', '1', paper))
|
||||
self.assertIsNone(material_reason('C65', '1', paper))
|
||||
self.assertIsNone(material_reason('T78', '2', paper))
|
||||
paper.set('id', '37')
|
||||
self.assertIsNone(material_reason('T78', '1', paper))
|
||||
paper.set('id', '36')
|
||||
ET.SubElement(paper, 'author')
|
||||
with self.assertRaises(ValueError):
|
||||
material_reason('T78', '1', paper)
|
||||
|
||||
def test_all_authored_source_fields_and_material_receipt_survive_collection(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
manifest, = collect_xml(FIXTURE, temporary, COMMIT)
|
||||
rows = [json.loads(raw) for raw in Path(manifest['file']).read_text().splitlines()]
|
||||
self.assertEqual(manifest['count'], 35)
|
||||
self.assertEqual(manifest['xml_paper_elements'], 36)
|
||||
self.assertEqual(manifest['missing_abstract_count'], 35)
|
||||
self.assertEqual(manifest['excluded_materials'][0]['external_id'], 'T78-1036')
|
||||
for number, row in enumerate(rows, 1):
|
||||
paper = self.volume.findall('paper')[number-1]
|
||||
self.assertEqual(row['external_id'], f'T78-1{number:03d}')
|
||||
self.assertEqual(row['title'], ' '.join(''.join(paper.find('title').itertext()).split()))
|
||||
self.assertEqual(row['metadata']['venue_codes'], ['tinlap'])
|
||||
self.assertEqual(row['metadata']['historical_event_scope']['original_venue'], 'TINLAP')
|
||||
self.assertEqual(row['metadata']['metadata_commit'], COMMIT)
|
||||
self.assertEqual((row['status'], row['track'], row['abstract']), ('accepted', 'proceedings_unspecified', None))
|
||||
|
||||
def test_old_coling_scope_is_unchanged_by_calendar_replacement_evidence(self):
|
||||
meta = ET.fromstring('<meta><venue>coling</venue><year>1965</year><booktitle>COLING 1965</booktitle></meta>')
|
||||
self.assertEqual(volume_scope(meta, '1', 'C65'), ('COLING', ['COLING'], 'main'))
|
||||
self.assertIsNone(event_scope(meta, '1', 'C65'))
|
||||
|
||||
def test_native_ingestion_does_not_alias_other_tinlap_years_or_infer_negative_status(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary)
|
||||
manifest, = collect_xml(FIXTURE, root/'source', COMMIT)
|
||||
db = root/'sample.sqlite'
|
||||
store = Store(db)
|
||||
pid, rid = store.record('test_control', 'tinlap1975', 'https://example.org/tinlap1975',
|
||||
'An earlier TINLAP contribution', None, ['Jane Doe'])
|
||||
store.decision(pid, rid, 'TINLAP', 1975, 'main', 'tinlap1975', 'https://example.org/tinlap1975', 'accepted', 'test control')
|
||||
store.conn.commit()
|
||||
store.conn.close()
|
||||
ingest_file(db, manifest['file'])
|
||||
ingest_file(db, manifest['file'])
|
||||
with sqlite3.connect(db) as c:
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0], 36)
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0], 36)
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM decisions').fetchone()[0], 36)
|
||||
self.assertEqual(c.execute("SELECT count(*) FROM decisions WHERE status!='accepted'").fetchone()[0], 0)
|
||||
self.assertIsNone(c.execute("SELECT 1 FROM source_records WHERE external_id='T78-1036'").fetchone())
|
||||
aliases = c.execute('SELECT v.acronym,a.alias FROM venue_aliases a JOIN venues v ON v.id=a.venue_id ORDER BY a.alias').fetchall()
|
||||
self.assertEqual(aliases, [('ACL+TINLAP', 'ACL'), ('ACL+TINLAP', 'TINLAP')])
|
||||
metadata = json.loads(c.execute("SELECT metadata_json FROM source_records WHERE external_id='T78-1001'").fetchone()[0])
|
||||
self.assertEqual(metadata['historical_event_scope']['event_year'], 1978)
|
||||
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(), [])
|
||||
@@ -0,0 +1,151 @@
|
||||
"""Original series, exact edition boundaries, and complete candidate discovery."""
|
||||
from contextlib import closing
|
||||
from copy import deepcopy
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import patch
|
||||
|
||||
from paper_atlas.collectors.dblp import catalogue, XML_SHA
|
||||
from paper_atlas.collectors.dblp_records import collect, normalize_row
|
||||
from paper_atlas.ingestion.records import ingest_file
|
||||
|
||||
FIXTURE = json.loads((Path(__file__).parent/'fixtures/historical_predecessors.json').read_text())
|
||||
EXAMPLES = {row['key']: row for row in FIXTURE['volumes']}
|
||||
|
||||
|
||||
def example(key, *, bibliographic_year=None):
|
||||
fixed = EXAMPLES[key]
|
||||
family = key.split('/')[1]
|
||||
year = int(key.split('/')[-1])
|
||||
volume = {'record_type': 'proceedings', 'key': key, 'fields': deepcopy(fixed['fields'])}
|
||||
row = {'record_type': 'inproceedings', 'key': f'conf/{family}/SyntheticExample{year}', 'publtype': None,
|
||||
'fields': {'title': [f'Synthetic historical research example {family} {year}'],
|
||||
'author': ['Jane Doe 0001', 'John Smith'], 'booktitle': fixed['fields']['booktitle'],
|
||||
'year': [str(bibliographic_year or year)], 'crossref': [key],
|
||||
'ee': [f'https://doi.org/10.1234/synthetic.{family}.{year}']}}
|
||||
return row, volume
|
||||
|
||||
|
||||
class HistoricalPredecessorTests(unittest.TestCase):
|
||||
def normalize(self, key, **kwargs):
|
||||
row, volume = example(key, **kwargs)
|
||||
fixed = EXAMPLES[key]
|
||||
result, reason = normalize_row(row, {key: volume}, {fixed['expected_target_family']})
|
||||
self.assertIsNone(reason)
|
||||
self.assertIsNotNone(result)
|
||||
return result
|
||||
|
||||
def test_ewsl_primary_series_retains_original_name_and_family_alias(self):
|
||||
for year in [1987, 1988, 1989, 1991]:
|
||||
with self.subTest(year=year):
|
||||
record = self.normalize(f'conf/ecml/{year}')
|
||||
self.assertEqual((record['venue'],record['venue_aliases'],record['year'],record['track']), ('EWSL',['ECML-PKDD'],year,'main'))
|
||||
proof = record['metadata']['historical_predecessor_evidence']
|
||||
self.assertEqual(proof['original_series'], 'EWSL')
|
||||
self.assertEqual(proof['history_url'], 'https://ecmlpkdd2011.org/pastConferences.php')
|
||||
self.assertEqual(record['metadata']['bibliography']['booktitle'], ['EWSL'])
|
||||
self.assertEqual(record['metadata']['author_names_raw'], ['Jane Doe 0001','John Smith'])
|
||||
self.assertEqual(record['authors'], ['Jane Doe','John Smith'])
|
||||
self.assertEqual(EXAMPLES['conf/ecml/1989']['bibliographic_counts'], [])
|
||||
|
||||
def test_eps_ecp_ewsp_keep_original_series_instead_of_renaming_to_icaps(self):
|
||||
for key in ['conf/aips/1990','conf/ecp/1997','conf/ecp/1999','conf/ewsp/1991']:
|
||||
with self.subTest(key=key):
|
||||
record = self.normalize(key)
|
||||
fixed = EXAMPLES[key]
|
||||
self.assertEqual(record['venue'], fixed['expected_original_series'])
|
||||
self.assertEqual(record['venue_aliases'], ['ICAPS'])
|
||||
self.assertEqual(record['track'], 'main')
|
||||
proof = record['metadata']['historical_predecessor_evidence']
|
||||
self.assertEqual(proof['history_url'], 'https://www.icaps-conference.org/conference-series/')
|
||||
self.assertEqual(proof['proceedings_title'], fixed['fields']['title'][0])
|
||||
self.assertEqual(proof['proceedings_year_fields'], fixed['fields']['year'])
|
||||
|
||||
def test_altered_titles_keys_suffixes_publication_years_and_unreviewed_editions_stay_held(self):
|
||||
for key in ['conf/ecml/1988','conf/aips/1990','conf/ecp/1997','conf/ewsp/1991']:
|
||||
row, volume = example(key)
|
||||
for changed in ['title','key','suffix','publication_year']:
|
||||
with self.subTest(key=key, changed=changed):
|
||||
r, v = deepcopy(row), deepcopy(volume)
|
||||
if changed == 'title': v['fields']['title'][0] += ', Affiliated Workshop'
|
||||
elif changed == 'key': v['key'] += 'w'; r['fields']['crossref'] = [v['key']]
|
||||
elif changed == 'suffix': r['fields']['booktitle'][0] += ' (Workshop)'
|
||||
else: v['fields']['year'] = ['2001']
|
||||
result, _ = normalize_row(r, {v['key']: v}, {EXAMPLES[key]['expected_target_family']})
|
||||
self.assertIsNone(result)
|
||||
row, volume = example('conf/ecp/1997')
|
||||
row['fields']['year'] = ['2001']; row['fields']['crossref'] = ['conf/ecp/2001']
|
||||
volume['key'] = 'conf/ecp/2001'; volume['fields']['year'] = ['2001']
|
||||
self.assertIsNone(normalize_row(row, {volume['key']:volume}, {'ICAPS'})[0])
|
||||
|
||||
def test_ecp1999_event_and_bibliographic_years_are_independently_retained(self):
|
||||
record = self.normalize('conf/ecp/1999', bibliographic_year=2000)
|
||||
self.assertEqual(record['year'], 1999)
|
||||
self.assertEqual(record['metadata']['bibliographic_year'], 2000)
|
||||
self.assertEqual(record['metadata']['bibliography']['year'], ['2000'])
|
||||
self.assertEqual(record['metadata']['historical_predecessor_evidence']['proceedings_year_fields'], ['2000'])
|
||||
self.assertEqual(record['metadata']['historical_predecessor_evidence']['event_year'], 1999)
|
||||
|
||||
def test_old_candidates_require_new_families_only_when_requested_years_need_them(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary); row, volume = example('conf/aips/1990')
|
||||
db = root/'dblp-catalogue.sqlite'
|
||||
with closing(sqlite3.connect(db)) as connection, connection:
|
||||
connection.execute('CREATE TABLE proceedings(key TEXT PRIMARY KEY,metadata_json TEXT)')
|
||||
connection.execute('INSERT INTO proceedings VALUES(?,?)', (volume['key'],json.dumps(volume)))
|
||||
candidate = root/'candidate.jsonl'; candidate.write_text(json.dumps(row)+'\n')
|
||||
db.with_name('dblp-ai-candidates.manifest.json').write_text(json.dumps({
|
||||
'snapshot_sha256': XML_SHA, 'candidate_sha256': hashlib.sha256(candidate.read_bytes()).hexdigest(),
|
||||
'candidate_records': 1, 'candidate_family_counts': {'aips':1},
|
||||
'record_types': {'proceedings':1,'inproceedings':1}}))
|
||||
# Tiny CI fixtures may live on tmpfs; production reserve is tested separately.
|
||||
with patch('paper_atlas.collectors.dblp_records.shutil.disk_usage', return_value=SimpleNamespace(free=16*1024**3)):
|
||||
result = collect(db, root/'early', ['ICAPS'], candidates=candidate, through_year=1990)
|
||||
self.assertEqual((result['normalized_records'],result['files'][0]['venue']), (1,'EPS'))
|
||||
for year, missing in [(1991,'ICAPS/ewsp'),(1997,'ICAPS/ecp')]:
|
||||
out = root/str(year); out.mkdir(); sentinel=out/'previous.txt';sentinel.write_text('accepted previous output')
|
||||
with self.subTest(year=year), self.assertRaisesRegex(ValueError, missing):
|
||||
collect(db, out, ['ICAPS'], candidates=candidate, through_year=year)
|
||||
self.assertEqual(sentinel.read_text(), 'accepted previous output')
|
||||
self.assertFalse((out/'dblp-normalization.json').exists())
|
||||
|
||||
def test_new_catalogue_includes_previously_missing_families_without_decision_inference(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root = Path(temporary);dtd=root/'dblp.dtd';dtd.write_bytes(b'<!ENTITY eacute "é">')
|
||||
archive=root/'dblp.xml.gz'
|
||||
body=''.join(f'<inproceedings key="conf/{family}/Synthetic"><title>Synthetic example</title><year>{year}</year><booktitle>{book}</booktitle><crossref>conf/{family}/{year}</crossref></inproceedings>' for family,year,book in [('ecp',1997,'ECP'),('ewsp',1991,'EWSP')])
|
||||
with gzip.open(archive,'wb') as stream:stream.write(('<?xml version="1.0"?><!DOCTYPE dblp SYSTEM "dblp.dtd"><dblp>'+body+'</dblp>').encode())
|
||||
result=catalogue(archive,dtd,root/'out',hashlib.sha256(archive.read_bytes()).hexdigest(),hashlib.md5(dtd.read_bytes()).hexdigest())
|
||||
rows=[json.loads(line)for line in Path(result['file']).read_text().splitlines()]
|
||||
self.assertEqual({r['key'] for r in rows}, {'conf/ecp/Synthetic','conf/ewsp/Synthetic'})
|
||||
self.assertEqual(result['candidate_records'], 2)
|
||||
self.assertTrue(all('status' not in r and 'venue' not in r for r in rows))
|
||||
|
||||
def test_native_import_keeps_original_series_aliases_and_full_source_evidence(self):
|
||||
with tempfile.TemporaryDirectory() as temporary:
|
||||
root=Path(temporary);db=root/'sample.sqlite'
|
||||
for key in ['conf/ecml/1988','conf/ecp/1999']:
|
||||
record=self.normalize(key);path=root/(key.replace('/','-')+'.jsonl')
|
||||
path.write_text(json.dumps(record,ensure_ascii=False)+'\n')
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps({'source':'dblp','venue':record['venue'],'year':record['year'],'track':'main','count':1,'list_url':'https://doi.org/10.4230/dblp.xml.2026-11-01','file_sha256':hashlib.sha256(path.read_bytes()).hexdigest(),'status':'partial','missing_abstract_count':1}))
|
||||
ingest_file(db,path)
|
||||
with closing(sqlite3.connect(db)) as connection:
|
||||
rows=connection.execute('SELECT v.acronym,a.alias,e.year FROM editions e JOIN venues v ON v.id=e.venue_id JOIN venue_aliases a ON a.venue_id=v.id ORDER BY v.acronym').fetchall()
|
||||
self.assertEqual(rows,[('ECP','ICAPS',1999),('EWSL','ECML-PKDD',1988)])
|
||||
metadata=[json.loads(r[0])for r in connection.execute('SELECT metadata_json FROM source_records')]
|
||||
self.assertEqual({r['historical_predecessor_evidence']['original_series'] for r in metadata}, {'ECP','EWSL'})
|
||||
self.assertEqual(connection.execute('SELECT count(*) FROM decisions WHERE status=?',('accepted',)).fetchone()[0],2)
|
||||
self.assertEqual(connection.execute('PRAGMA foreign_key_check').fetchall(),[])
|
||||
|
||||
def test_existing_aips_normalization_remains_original_aips(self):
|
||||
row, volume=example('conf/aips/1990');row['fields']['booktitle']=['AIPS'];row['fields']['year']=['1994'];row['fields']['crossref']=['conf/aips/1994']
|
||||
volume['key']='conf/aips/1994';volume['fields']['title']=['AIPS 1994 Proceedings'];volume['fields']['year']=['1994']
|
||||
record,reason=normalize_row(row,{volume['key']:volume},{'ICAPS'})
|
||||
self.assertIsNone(reason);self.assertEqual((record['venue'],record['venue_aliases']),('AIPS',['ICAPS']))
|
||||
self.assertNotIn('historical_predecessor_evidence',record['metadata'])
|
||||
@@ -0,0 +1,160 @@
|
||||
"""Real 49-chapter snapshots compared with a separately frozen metadata audit."""
|
||||
import copy
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sqlite3
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from paper_atlas.collectors.springer_selected import (
|
||||
BOOK, SOURCE, collect, parse_chapter, parse_toc, read_page, snapshot_index,
|
||||
)
|
||||
from paper_atlas.db.store import Store
|
||||
from paper_atlas.ingestion.records import ingest_file
|
||||
|
||||
FIXTURE = Path(__file__).parent/'fixtures/springer-aistats1993-fixture.json.gz'
|
||||
|
||||
|
||||
class SpringerSelectedTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
raw = FIXTURE.read_bytes()
|
||||
if hashlib.sha256(raw).hexdigest() != '94028fd37080f9d8027e5790e379cc151ee6eab17ed0d1dcc0f59e6e75f9d4d0':
|
||||
raise AssertionError('Fixed source fixture differs')
|
||||
cls.fixture = json.loads(gzip.decompress(raw))
|
||||
|
||||
def snapshots(self, root):
|
||||
for group in ['toc','chapters']:
|
||||
folder = root/group
|
||||
folder.mkdir()
|
||||
for name, text in self.fixture[group].items():
|
||||
(folder/name).write_text(text)
|
||||
return {'toc_snapshots':root/'toc','chapter_snapshots':root/'chapters'}
|
||||
|
||||
def test_all49_original_fields_against_independent_audit(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
report = collect(root/'out', **self.snapshots(root))
|
||||
manifest, = report['files']
|
||||
rows = [json.loads(s) for s in Path(manifest['file']).read_text().splitlines()]
|
||||
self.assertEqual(len(rows), 49)
|
||||
self.assertEqual(manifest['status'], 'partial')
|
||||
self.assertEqual(manifest['missing_abstract_count'], 0)
|
||||
self.assertTrue(manifest['excluded_materials'])
|
||||
self.assertTrue(all(x['title'] == 'Front Matter' for x in manifest['excluded_materials']))
|
||||
for row, expected in zip(rows, self.fixture['expected']):
|
||||
for field in ['title','abstract']:
|
||||
self.assertEqual(row[field], expected[field])
|
||||
self.assertEqual(row['identifiers'], {'doi':expected['doi']})
|
||||
self.assertEqual(row['submission_identifiers'], [{'scheme':'publication_doi','value':expected['doi']}])
|
||||
self.assertEqual(row['external_id'], expected['doi'])
|
||||
self.assertEqual(row['metadata']['authors_citation_raw'], expected['authors_citation_raw'])
|
||||
self.assertEqual(row['authors'], [{'family':s.split(',',1)[0].strip(), 'given':s.split(',',1)[1].strip()} for s in expected['authors_citation_raw']])
|
||||
self.assertEqual(row['metadata']['first_page'], expected['first_page'])
|
||||
self.assertEqual(row['metadata']['last_page'], expected['last_page'])
|
||||
self.assertEqual((row['year'],row['metadata']['bibliographic_year']), (1993,1994))
|
||||
self.assertEqual(row['track'], 'selected_volume')
|
||||
self.assertEqual(row['event_key'], 'aistats-1993-selected-volume-1994')
|
||||
self.assertNotIn('decision_date', row)
|
||||
self.assertIn('original 1993 review decision not supplied', row['raw_decision'])
|
||||
for e in [row['metadata']['source_evidence'], *row['metadata']['scope_evidence']]:
|
||||
self.assertEqual(hashlib.sha256((root/'out'/e['file']).read_bytes()).hexdigest(), e['sha256'])
|
||||
self.assertFalse(report['database_written'])
|
||||
with self.assertRaises(FileExistsError):
|
||||
collect(root/'out', toc_snapshots=root/'toc', chapter_snapshots=root/'chapters')
|
||||
|
||||
def test_native_history_namespaces_and_idempotent_import(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
report = collect(root/'out', **self.snapshots(root))
|
||||
manifest, = report['files']
|
||||
first = json.loads(Path(manifest['file']).read_text().splitlines()[0])
|
||||
db = root/'sample.sqlite'
|
||||
store = Store(db)
|
||||
pid, rid = store.record('prior_public','earlier','https://example.org/earlier',first['title'],first['abstract'],first['authors'])
|
||||
store.decision(pid,rid,'AISTATS',1993,'main','earlier','https://example.org/earlier','rejected','original public rejection')
|
||||
# Explicitly verified work identity; the short title alone must not
|
||||
# force a merge or attach this publication DOI to the old attempt.
|
||||
store.conn.execute('INSERT INTO identifiers VALUES(?,?,?)', ('doi',first['identifiers']['doi'],pid))
|
||||
store.conn.commit()
|
||||
old = store.conn.execute('SELECT * FROM decisions').fetchall()
|
||||
store.conn.close()
|
||||
ingest_file(db, manifest['file'])
|
||||
ingest_file(db, manifest['file'])
|
||||
with sqlite3.connect(db) as c:
|
||||
self.assertEqual(c.execute('SELECT * FROM decisions WHERE status=\'rejected\'').fetchall(), [tuple(r) for r in old])
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0], 49)
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0], 50)
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM decisions').fetchone()[0], 50)
|
||||
self.assertEqual(c.execute('SELECT e.year,e.track,e.event_key FROM editions e ORDER BY e.id').fetchall(), [(1993,'main',''),(1993,'selected_volume','aistats-1993-selected-volume-1994')])
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id WHERE t.namespace=?', (SOURCE+':publication_format',)).fetchone()[0], 49)
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM submission_identifiers WHERE scheme=\'publication_doi\'').fetchone()[0], 49)
|
||||
self.assertIsNone(c.execute("SELECT 1 FROM submission_identifiers si JOIN submissions s ON s.id=si.submission_id JOIN editions e ON e.id=s.edition_id WHERE e.track='main'").fetchone())
|
||||
|
||||
def test_short_unanchored_title_keeps_identity_unconfirmed(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
store = Store(Path(tmp)/'identity.sqlite')
|
||||
title = 'Statistical strategy: step 1'
|
||||
authors = [{'given':'D. J.','family':'Hand'}]
|
||||
old, _ = store.record('prior_public','earlier','https://example.org/old',title,'Same abstract',authors)
|
||||
new, _ = store.record(SOURCE,BOOK+'_1','https://link.springer.com/chapter/'+BOOK+'_1',title,'Same abstract',authors,{'doi':BOOK+'_1'})
|
||||
self.assertNotEqual(old, new)
|
||||
self.assertEqual(store.conn.execute('SELECT count(*) FROM match_candidates WHERE method=\'exact_title_unconfirmed\'').fetchone()[0], 1)
|
||||
store.conn.close()
|
||||
|
||||
def test_receipt_hash_path_redirect_and_content_type_guards(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
self.snapshots(root)
|
||||
index = snapshot_index(root/'chapters')
|
||||
url = 'https://link.springer.com/chapter/'+BOOK+'_1'
|
||||
self.assertTrue(read_page(index,url)[0])
|
||||
for key, value in [('sha256','0'*64),('file','../escape.html'),('final_url','https://example.org/chapter'),('http_status',403),('headers',{'Content-Type':'application/pdf'})]:
|
||||
changed = copy.deepcopy(index)
|
||||
changed[url][1][key] = value
|
||||
with self.subTest(key=key), self.assertRaises((ValueError,KeyError)):
|
||||
read_page(changed,url)
|
||||
changed = copy.deepcopy(index)
|
||||
changed[url][1]['final_url'] = url+'?page=2'
|
||||
with self.assertRaises(ValueError):
|
||||
read_page(changed,url)
|
||||
|
||||
def test_missing_or_duplicate_snapshot_stops_collection(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
args = self.snapshots(root)
|
||||
path = root/'chapters/receipts.json'
|
||||
d = json.loads(path.read_bytes())
|
||||
original = copy.deepcopy(d)
|
||||
d['responses'].pop()
|
||||
path.write_text(json.dumps(d))
|
||||
with self.assertRaises(ValueError):
|
||||
collect(root/'out', **args)
|
||||
self.assertFalse((root/'out').exists())
|
||||
original['responses'].append(original['responses'][0])
|
||||
path.write_text(json.dumps(original))
|
||||
with self.assertRaises(ValueError):
|
||||
snapshot_index(root/'chapters')
|
||||
|
||||
def test_book_doi_year_and_duplicate_citation_fields_reject_drift(self):
|
||||
html = self.fixture['chapters']['chapter-01.html']
|
||||
for key, value in [('citation_doi',BOOK+'_2'),('citation_inbook_title','Another Book'),('citation_publication_date','1993')]:
|
||||
import re
|
||||
changed, count = re.subn(r'(<meta[^>]*name="'+key+r'"[^>]*content=")[^"]*', lambda m:m[1]+value, html)
|
||||
self.assertEqual(count, 1)
|
||||
with self.subTest(key=key), self.assertRaises(ValueError):
|
||||
parse_chapter(changed, BOOK+'_1')
|
||||
with self.assertRaises(ValueError):
|
||||
parse_chapter(html+'<meta name="citation_doi" content="'+BOOK+'_1">', BOOK+'_1')
|
||||
|
||||
def test_missing_abstract_is_explicit_and_does_not_use_other_page_text(self):
|
||||
html = self.fixture['chapters']['chapter-01.html'].replace('id="Abs1-content"','id="not-an-abstract"')
|
||||
self.assertIsNone(parse_chapter(html, BOOK+'_1')['abstract'])
|
||||
|
||||
def test_toc_ignores_frontmatter_but_rejects_duplicate_publications(self):
|
||||
html = self.fixture['toc']['springer-toc1.html']
|
||||
self.assertEqual(len(parse_toc(html)), 17)
|
||||
with self.assertRaises(ValueError):
|
||||
parse_toc(html+html)
|
||||
Reference in New Issue
Block a user