diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml new file mode 100644 index 0000000..5fbeb95 --- /dev/null +++ b/.gitea/workflows/check.yml @@ -0,0 +1,35 @@ +name: Metadata integrity checks +on: + push: + branches: [main, 'codex/**'] +jobs: + metadata: + runs-on: ces-control + env: + ARCHIVE_SHA: ${{ github.sha }} + PYTHONDONTWRITEBYTECODE: '1' + steps: + - name: Verify metadata storage and query commands + run: | + python3 -B - <<'PY' + import io, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request + sha = os.environ['ARCHIVE_SHA'] + if not re.fullmatch(r'[0-9a-fA-F]{40,64}', sha): + raise SystemExit('Invalid source commit') + # Public repository archive: no owner credentials or runner SSH keys. + url = 'https://gitea.entroia.com/Zayden-Jung/paper-atlas/archive/' + sha + '.tar.gz' + with urllib.request.urlopen(url, timeout=60) as response: + archive = response.read(16*1024*1024+1) + if len(archive)>16*1024*1024: + raise SystemExit('Unexpectedly large source archive') + with tempfile.TemporaryDirectory(prefix='paper-atlas-ci-') as tmp: + root = pathlib.Path(tmp) + with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar: + tar.extractall(root, filter='data') + candidates = list(root.glob('*/test_import_arxiv.py')) + if len(candidates)!=1: + raise SystemExit('Unexpected repository archive layout') + src = candidates[0].parent + subprocess.run(['python3','-B','-m','unittest','discover','-v'],cwd=src,check=True) + subprocess.run(['python3','-B','query.py','--help'],cwd=src,check=True) + PY diff --git a/.gitignore b/.gitignore index adfd07a..b0a58ff 100644 --- a/.gitignore +++ b/.gitignore @@ -3,8 +3,15 @@ __pycache__/ *.sqlite *.sqlite-* *.zip +*.gz +*.bz2 +*.jsonl *.part +*.part.json +*.download.lock +*.maintenance.lock .env .DS_Store raw/ data/ +collected/ diff --git a/README.md b/README.md index 17edc73..02d83d0 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Paper Atlas 是一个**长期维护、按需或按批次运行的研究数据工 目标包括：完整历史元数据导入、逐年会议目录、跨平台论文去重、公开拒稿/撤稿信息、平台标签与自定义标签隔离、可恢复更新和覆盖率报告。 -**目前仍在开发，不能视为完整数据集。** 已实现 SQLite 数据结构、arXiv 元数据 ZIP 流式导入、断点恢复、DOI/标识合并和只读查询脚本。服务器上进行过合成样本验证；全量快照下载与历史导入尚未完成。会议适配器、跨平台模糊匹配复核流程、完整覆盖率核验与 CI 仍待实现。`venues.json` 的 41 个会议是候选覆盖范围，不代表已完成采集，也不是与 COLM 比较的客观排名。 +**目前仍在开发，不能视为完整数据集。** 已实现 SQLite 数据结构、arXiv 元数据 ZIP 流式导入、断点恢复、跨平台标识合并及审计、PMLR、COLM、NeurIPS、ACL Anthology、ICLR 官方公开程序、CVF/ECVA 视觉会议元数据采集、带清单核验的会议导入和只读查询。PeerRead 提供公开 ICLR 2017 明确决定的历史补充；近期 ICLR 公开决定快照已解析，实际主库进度以运行日志与统计为准。arXiv 元数据 ZIP 已完整下载（1,878,831,586 字节），主批次读取 3,233,620 行至 EOF；主库有 3,233,527 条 arXiv 来源记录，93 行差额仍需按原始 ID 独立核验。这些不是去重后的全库论文数，最终全库审计尚未完成。其他会议适配器、历史缺失届次、完整投稿历史、跨平台疑似重复复核和覆盖率核验仍待完成。回归检查在远端执行，已配置 ces-control:host 对应的 CI 工作流，实际 CI 结果以 Gitea Actions 为准。`venues.json` 的 41 个会议是候选覆盖范围，不代表已完成采集，也不是与 COLM 比较的客观排名。 ## 数据范围与来源 @@ -30,6 +30,22 @@ Paper Atlas 是一个**长期维护、按需或按批次运行的研究数据工 - [AAAI](https://ojs.aaai.org/index.php/AAAI/issue/archive) - [DBLP](https://dblp.org/db/conf/) +## 候选会议覆盖目录 + +| 领域 | 会议系列 | 主要发现入口 | +| --- | --- | --- | +| 机器学习、理论 | NeurIPS、ICML、ICLR、COLM、COLT、AISTATS、UAI、CLeaR | OpenReview、PMLR、NeurIPS、COLM 官方论文集 | +| 自然语言 | ACL、EMNLP、NAACL、EACL、COLING、CoNLL | ACL Anthology | +| 视觉 | CVPR、ICCV、ECCV | CVF、ECVA | +| 综合 AI | AAAI、IJCAI、ECAI | 官方论文集、DBLP | +| 检索与数据 | KDD、SIGIR、WWW、WSDM、RecSys、CIKM、ICDM、SDM、ECML-PKDD | 官方出版方、DBLP，具体年份入口待逐项登记 | +| 机器人 | ICRA、IROS、RSS、CoRL | 官方论文集、PMLR、DBLP | +| 规划、知识与智能体 | ICAPS、KR、AAMAS | 官方论文集、DBLP | +| 语音与音频 | ICASSP、INTERSPEECH、ISMIR | 官方论文集、ISCA、DBLP | +| 相关专门领域 | MICCAI、FAccT | 官方出版方、DBLP | + +这些是领域覆盖候选，不构成统一排名，也不能据此声称已全部入库。逐年目录以真实举办届次和正式出版清单为准：双年会议不补造不存在的年份；主会、Findings、workshop 分开。`edition_sources` 保存实际采集状态，不能用候选名单代替覆盖率报告。 + ## 数据模型 | 表 | 用途 | @@ -37,22 +53,24 @@ Paper Atlas 是一个**长期维护、按需或按批次运行的研究数据工 | `papers` / `papers_fts` | 稳定论文实体、代表性标题摘要、FTS5 全文检索 | | `identifiers` | arXiv ID、DOI 等唯一外部标识 | | `source_records` / `sources` | 平台记录、原平台标题摘要与索引元数据、来源 URL 和抓取时间 | -| `venues` / `editions` | 会议或期刊系列、年份与 track，区分主会、Findings 和 workshop | +| `venues` / `venue_aliases` / `editions` | 会议或期刊系列、年份与 track，区分主会、Findings 和 workshop，联合会议用别名查询，不重复生成事件 | | `edition_sources` | 每届每个来源的覆盖状态、预计/已导入数量、错误与断点 | -| `submissions` / `decisions` | 多次投稿及带证据的决定历史，支持改投、撤稿和决定变更 | +| `submissions` / `submission_identifiers` / `submission_sources` | 投稿事件、按届次限定的原平台强标识、多来源证据；同稿的不同投稿尝试分别保留 | +| `decisions` / `decision_sources` | 明确决定、原始决定日期、来源观察日期及证据，不把缺失决定编造成拒稿 | +| `submission_merges` / `decision_merges` | 投稿事件与重复决定观察的合并审计，保留来源关联并避免重用已合并的编号 | | `tags` / `paper_tags` | 平台标签与人工标签，使用命名空间隔离 | -| `match_candidates` | 证据不充分的潜在重复记录及人工复核状态 | +| `match_candidates` / `paper_merges` | 潜在重复复核状态、确定合并审计 | | `ingest_runs` | 导入批次、状态、错误和恢复位置 | 决定状态为 `accepted`、`rejected`、`desk_rejected`、`withdrawn`、`under_review`、`unknown`。一篇论文可以对应多次投稿和多个决定，不能用单一“最终会议”字段覆盖历史。查询中按观察时间选择最近状态；后续分析需要同时检查来源覆盖率。 -当前确定性去重使用 arXiv ID 和单个 DOI。保留各平台源记录，避免合并后丢失出处。规范化标题只建检索索引，尚未实现模糊匹配流程；**现阶段不保证跨平台去重已经完成**。不同平台标签不自动等同，自定义标签应使用 `custom:*` 命名空间。 +当前确定性去重使用 arXiv ID、单个 DOI 和 OpenReview forum ID；跨平台同标题还需至少两位作者吻合，或作者吻合且摘要相同，才自动合并。保留各平台源记录，避免合并后丢失出处。规范化标题只建检索索引，尚未实现模糊匹配流程；**现阶段不保证跨平台去重已经完成**。不同平台标签不自动等同，自定义标签应使用 `custom:*` 命名空间。 ## 运行与查询 -依赖 Python 3 标准库和带 FTS5 的 SQLite，目前无第三方 Python 依赖，无项目构建步骤。服务器已检查为 Python 3.14.4。所有运行数据放在仓库外，不提交数据库、下载包或访问凭据。 +基础导入和查询依赖 Python 3 标准库和带 FTS5 的 SQLite；PMLR 采集另需 `requirements.txt` 中锁定的 PyYAML 6.0.3。无项目构建步骤。服务器已检查为 Python 3.14.4。所有运行数据放在仓库外，不提交数据库、下载包或访问凭据。 -服务器暂存目录：`/data/paper-index/`，快照目录：`/data/paper-index/raw/`，运行代码目录：`/data/paper-index/code/`。这些路径不是对外访问的 API。 +服务器暂存目录：`/data/paper-index/`，快照目录：`/data/paper-index/raw/`，运行代码目录：`/data/paper-index/code/`。这些路径不是对外访问的 API。本地工作区只用于编辑代码、文档和 Git；采集、全量导入和运行检查均在用户确认的服务器执行。 下载完成并验证 ZIP 后，在服务器运行： @@ -80,7 +98,201 @@ python3 -B query.py --db /data/paper-index/papers.sqlite --tag arxiv:category=cs python3 -B query.py --db /data/paper-index/papers.sqlite --venue COLM --year 2025 --status rejected ``` -查询连接使用 SQLite 只读模式，结果输出 JSON。当前没有公共 Web 页面或网络查询服务。 +查询连接使用 SQLite 只读模式，结果输出 JSON。`--stats` 中 `papers` 是按当前规则去重后的论文实体数，`records_by_source` 是各平台已入库记录数，`decision_events_by_status` 是决定事件数。采集文件 manifest 的 `count` 是已采集数量，不能等同已入库数量，也不能把各平台记录数相加当成独立论文数。 + +全部目标会议的覆盖报告会保留尚无入库记录的项目，避免只展示已经做完的会议： + +```sh +python3 -B coverage_report.py --db /data/paper-index/papers.sqlite \ + --out /data/paper-index/collected/venue-coverage-current.json +``` + +报告按会议、年份和 track 列出投稿数、独立论文数及来源清单状态，包含联合会议别名。没有数据的目标明确标为 no_imported_records；已有数据也保持“历史待独立核验”，不会因单个文件导入通过就宣称历年覆盖完成。同一论文可以出现于多个会议或年份，各会议论文数也不能直接相加作为全库数量。 + +用 `--paper 531782` 可查看单篇论文的所有已知来源、平台标签、投稿届次及带证据的决定历史。会议筛选不区分大小写，状态筛选使用该次投稿的最新已知决定。当前没有公共 Web 页面或网络查询服务。 + +## 官方会议元数据采集 + +```sh +# 单卷示例：ICML 2025，读取官方 gh-pages 元数据，不下载论文正文 +python3 -B collect_pmlr.py --cache /data/paper-index/raw/http-cache \ + --out /data/paper-index/collected --volume 267 + +# 待主库完成 arXiv 导入、且没有其他写入进程后，合并会议记录 +python3 -B import_records.py --db /data/paper-index/papers.sqlite \ + /data/paper-index/collected/pmlr-v267.jsonl +``` + +采集输出 JSONL 和对应 manifest，只有明确来源标签才产生相应决定：官方论文集记录为 accepted；PeerRead 使用其明确的 accepted 布尔标签，缺失标签保留 unknown。清单数目不一致时整个文件导入回滚，记录失败批次，不能标为覆盖完成。多个历史卷的全量覆盖仍需单独核验。旧卷可能把 PDF 放入归档；采集器对前 50 卷使用 Git partial clone 和 sparse checkout，仅取 `_posts` 和卷配置，不执行出版方项目代码。较新归档若超出 64 MiB 限额，也转为此方式。已对原本超限的 AISTATS 2009（第 5 卷）验证采集 84 条且工作区无 PDF。旧 YAML 的 `!binary` 文本、空文档尾标记会安全解析；带年份的 AISTATS/COLT 等历史 shortname 会规范化，避免名称别名漏届；缺失 year 时仅从官方会议 start 日期提取年份。`pmlr-volume-status.json` 逐卷保存结果和候选范围归属，`pmlr-last-run.json` 记录本批实际请求范围，局部重试不能证明全历史完成。缓存默认有效 24 小时；同一次批次可复用缓存，后续更新会重新读取动态入口。 + +PMLR 的同一出版 ID 可能对应多份 Git 元数据，例如 CoRL 2023 的两篇论文分别存在不同日期的副本。只有论文内容与标识完全一致、差异限于元数据日期时才合为一条，并保留源文件路径和日期版本；冲突副本会使该卷采集失败，保留上一份有效文件。manifest 分别记录 metadata_post_count、duplicate_post_count 和论文 count。 + +会议导入器还会独立检查每条记录的会议、年份、track 与 manifest 是否一致，并从实际记录计算缺失摘要数；旧 manifest 即使声称摘要完整，也不能让缺失摘要的来源被标为 complete。缺失的源摘要与合并后论文实体是否从其他来源补得摘要分别统计。 + +PeerRead 采集仅选择作者公开发布的 ICLR 2017 投稿元数据，固定到源仓库 commit。只依据明确的布尔 `accepted` 标签形成 accepted/rejected；空标签为 unknown，不使用其他数据分区中的 probably-rejected 推断。不会保存 reviews 或 parsed_pdfs 正文。 + +```sh +python3 -B collect_peerread.py --cache /data/paper-index/raw/http-cache \ + --out /data/paper-index/collected +``` + +数据任务以一次性 systemd 服务运行：`paper-atlas-arxiv-import`、`paper-atlas-pmlr-collect`、`paper-atlas-peerread-collect`；`paper-atlas-pmlr-verify` 用于全索引复核，`paper-atlas-conference-merge` 用于单写入合并。它们不是定时任务。主库只允许一个写入进程；会议采集先写独立 JSONL，待 arXiv 主批次结束后再合并；也可以使用 `merge_conferences.py` 在保存断点处暂停本项目的 arXiv unit、顺序合并会议清单并恢复。该操作应作为独立的一次性 systemd 任务运行，避免 SSH 断开影响恢复；不能同时启动多个合并任务。运行日志位于 `/data/paper-index/*-import.log`、`/data/paper-index/*-collect.log`。任何服务启动成功都不等于批次已完成，需检查批次状态、清单与完整性结果。标签按来源刷新使用 `paper_tags_source_evidence` 索引，避免每条论文都扫描整张标签表。运行中的 arXiv 可以通过同一协调脚本的 `--schema-only` 模式在断点更新索引后恢复；此模式要求已有 arXiv unit 正在运行。 + +### 投稿事件身份与旧数据回填 + +论文实体与投稿事件分开去重。同一篇论文可以在同一年向同一会议投稿多次，不能仅凭 paper_id、会议、标题相同就把投稿历史压成一次。跨来源只有原平台 forum ID、已验证的投稿编号，或正式发表稿的同届同 track 出版 DOI 等强证据才能关联同一事件。程序展示的内部编号仅在经过独立核验并带有 program_submission_number_verified 标记时才能用作投稿编号。 + +新版 ICLR manifest 的 parser_version 为 2，明确区别投稿编号、forum 和展示事件编号。旧版文件应先用缓存重新采集；事件证据回填只接受已入库、与 manifest 的会议/年份/track 和数量一致的文件。队列 JSON 包含 files 及对应的 file_sha256，文件变化后必须重新核验，不能继续使用旧哈希。 + +```sh +python3 -B merge_conferences.py --db /data/paper-index/papers.sqlite \ + --archive /data/paper-index/raw/arxiv-metadata.zip \ + --backfill-queue /data/paper-index/collected/submission-evidence-queue.json +``` + +协调器只维护本项目的 arXiv unit，并对维护过程加互斥锁。若 arXiv 正在运行，会保存断点、应用 schema、回填来源身份、导入本次附加文件、检查外键后恢复；若批次已经读至 EOF 并标记 complete，则不再启动它。若导入停下但未完成，会报错要求先调查，不以“unit 不存在”推断成功或重新全量运行。应以一次性 systemd 任务运行该命令，保持单写入。 + +首次升级会给已有投稿与决定补上来源关联；身份回填不重新导入标题摘要，不更改原抓取时间。确认同一次投稿后，可以合并镜像事件并保留所有决定及来源。当前稿件的原始强标识也会写入 source_records 元数据，后续源 URL 更新会同步保存。 + +通用 JSONL 导入支持 decision_events 数组及 observed_at、decision_date、submitted_at。没有取得明确决定时，可用 unknown 状态和空数组仅保存投稿，不生成虚假的决定；`--status unknown` 也能查询这些投稿。历史快照使用其实际观察时间，不能因今天导入就覆盖较新来源的状态；同一观察时间内按原始决定日期排序。反复导入相同证据只补来源关联，不重复制造决定事件。全量结束后仍须单独核验重复候选、冲突决定及来源覆盖，迁移检查不能代替最终全库审计。 + +### 固定版本的补充元数据快照 + +`download_snapshot.py` 流式下载到 `.part`，核对完整 HTTP 响应、预期大小和可信来源发布的校验和后，才发布正式文件及 manifest。已存在的正式文件会先重新核验；错误校验和不会覆盖它。默认保留 8 GiB 空闲空间，同一输出只允许一个下载进程。 + +DBLP 官方月度快照用于补充出版目录和历史届次，元数据按 CC0 发布。固定版本可复现，不把每日变动下载链接当作同一快照： + +```sh +python3 -B download_snapshot.py \ + --url https://drops.dagstuhl.de/storage/artifacts/dblp/xml/2026/dblp-2026-11-01.xml.gz \ + --md5 18ad5db59592406ec74289975c4b44cb \ + --output /data/paper-index/raw/dblp-2026-11-01.xml.gz +``` + +[官方版本与校验值](https://doi.org/10.4230/dblp.xml.2026-11-01)。下载器另外计算 SHA-256，记录来源 URL、大小和核验时间。DBLP 没有提供的摘要不能编造，出版目录也不能作为拒稿证据。 + +DBLP 目录盘点先核验固定快照与[官方 DTD](https://doi.org/10.4230/dblp.xml.dtd.2023-06-28)，再流式读取 XML，直至 gzip EOF 检查通过： + +```sh +python3 -B download_snapshot.py \ + --url https://drops.dagstuhl.de/storage/artifacts/dblp/xml/2023/dblp-2023-06-28.dtd \ + --md5 96e4283e71886dec2173718957293648 \ + --output /data/paper-index/raw/dblp-2023-06-28.dtd +python3 -B catalogue_dblp.py \ + --archive /data/paper-index/raw/dblp-2026-11-01.xml.gz \ + --dtd /data/paper-index/raw/dblp-2023-06-28.dtd \ + --out /data/paper-index/collected/dblp +``` + +`dblp-catalogue.sqlite` 保存论文集目录、按 crossref/年份/booktitle 的记录数量、各目录族数量及少量样本；`dblp-ai-candidates.jsonl` 保存候选会议族的原始索引元数据。存在多个出版卷字段的记录另存 `volume_anomalies`，保留全部值供归属核验；统计中仍只计一条，不任取第一个字段或展开成多篇。候选范围包含历史名称与 workshop，尚未赋予主会归属或录用状态。后续须依据具体论文集和届次核对目标范围、track 与出版标识，才能生成入库文件；不会把同一目录下的所有记录都视为主会。该盘点不下载 PDF，也不更改论文主库。 + +XML 只允许校验过的本地 DTD；官方通用名 `dblp.dtd` 与月度快照使用的 `dblp-2023-06-28.dtd` 均解析到同一份已校验文件。拒绝其他外部实体和内部 DTD 子集，不访问 XML 声明的任意网络地址，并遵循源 XML 的字符编码。临时目录库或 gzip 检查失败时不发布正式候选文件，需先调查残留的 `.part` 状态，再决定是否重新运行。 + +公开 ICLR 投稿快照的固定 Release 供后续筛选投稿和明确决定；它是第三方保存的 OpenReview 历史副本，不等于最新官方全量数据： + +```sh +python3 -B download_snapshot.py \ + --url https://github.com/qhjqhj00/iclr-openreview-reviews/releases/download/v1.0/iclr.tar.bz2 \ + --sha256 350071daf25ebf1aef854ee1d5dc3a81834e1c468917c82a4f78af5548de34fc \ + --expected-bytes 512940722 \ + --output /data/paper-index/raw/iclr-openreview-v1.0.tar.bz2 +``` + +[快照来源与复现脚本](https://github.com/qhjqhj00/iclr-openreview-reviews)。原包包含评审回复；解析只筛选公开投稿的索引、标题、摘要、平台标签及带证据的明确决定，不提取评审正文、作者邮件或论文全文。必须核对原始 invitation、readers、签名及决定字段，不能根据评分或录用名单缺席推断拒稿。 + +优先处理近期三届，再单独补全 2020–2023；不以 2017 年的 PeerRead 样本代表近期拒稿覆盖： + +```sh +python3 -B collect_openreview_snapshot.py \ + --archive /data/paper-index/raw/iclr-openreview-v1.0.tar.bz2 \ + --out /data/paper-index/collected --year 2024 --year 2025 --year 2026 +``` + +解析器先核验固定 SHA-256，再流式读取 tar，以临时 SQLite 按年份和 forum 关联投稿及决定，兼容回复先于投稿的归档顺序。只接受明确对 everyone 公开的投稿和具有会议主席签名的 Decision；保留原始决定值、note ID、决定日期、invitation 和签名证据。无明确决定的投稿保留 unknown，不产生拒稿推断。无法关联到公开投稿元数据的公开决定另存 unmatched-decisions 文件，供复核，不擅自创建论文。输出清单分别统计投稿状态、决定事件、缺失摘要和排除原因；重复副本去重，内容冲突会中止。导入时若同一编号指向不同 forum，或同一 forum 指向不同投稿编号，会报冲突并回滚，不强制合并。 + +该版本是 2026-05-08 发布的第三方历史副本，不代表平台当前状态或全部投稿；缺失决定及公开性筛选均需核验。采集成功也不等于主库已更新，后续应通过单写入协调器导入，并核对同届官方程序与公开决定的差异。 + +入库前按真实 forum 对照同届官方程序，生成逐年状态、名单缺口、历史决定差异和文本版本差异报告，以及带 SHA-256 的现有程序身份回填队列： + +```sh +python3 -B audit_iclr_snapshot.py --root /data/paper-index/collected \ + --year 2024 --year 2025 --year 2026 \ + --report /data/paper-index/collected/iclr-recent-audit.json +``` + +该检查要求清单和记录的届次、数量、摘要缺失统计一致，拒绝重复 forum、编号冲突及缺失公开决定证据。报告中的名单差异和 unknown 不会转换成拒稿。不同时间来源的决定保留各自证据，强 forum 标识负责关联同一次投稿；报告通过不等于平台全量覆盖或最终全库去重已经完成。生成的 `*-backfill-queue.json` 仅用于已入库程序记录的身份回填，快照文件仍须由协调器单独导入。 + +下载异常后先检查对应进程或 CI 任务确已终止，再使用同一参数增加 `--resume`。只有断点身份一致且服务器实际返回正确 Range 响应才续传；不支持时保留断点并报错，不自动重下。正式文件与下载清单均放在仓库外，不提交大型来源包。 + +### COLM 与 NeurIPS 官方摘要 + +```sh +python3 -B collect_proceedings.py --source colm \ + --cache /data/paper-index/raw/http-cache --out /data/paper-index/collected +python3 -B collect_proceedings.py --source neurips \ + --cache /data/paper-index/raw/http-cache --out /data/paper-index/collected +``` + +公共元数据请求对连接重置、超时及不完整响应最多尝试三次，保留原有有效缓存；HTTP 403 等访问拒绝会直接记录失败。声明 Content-Length 与实际响应不符时不发布缓存，已有缓存也受字节上限约束。旧页面没有赋值的 HTML 属性按空值处理，避免某一个空 class 属性阻断整届采集。 + +采集器从官方根目录发现实际发表届次，仅访问索引和 Abstract HTML 页面，不请求 PDF、补充材料或源码。可用重复 `--year` 限定实际存在的届次；不存在的年份会报错，局部重试保留其他届次的已有状态。每个来源串行请求，实际网络响应后等待 1 秒；缓存命中不额外等待。 + +COLM 当前目录提供 2024–2026 三届 pre-proceedings，按会议年份保存，保留平台实际出版日期和出版状态，不把晚补刊日期当作会议年份。NeurIPS 当前目录提供 1987–2025，并包含 2025 独立卷及 2021 独立 Datasets and Benchmarks 索引。主会、datasets_benchmarks、position、creative_ai 分开出文件，不混为同一个 track。跨平台自标识仅从出版方 paper-actions 区域链接与 citation 元数据提取，不把摘要里的引用链接当作当前论文 ID。manifest 的 `parser_version` 至少为 2 才允许入库；旧文件须利用缓存重新采集，避免错误身份合并。首次真实样本核对：COLM 2024 官方目录 299 个摘要入口；NeurIPS 1987 为 90 个，作者与摘要均已验证解析。样本成功不代表所有页面采集完成。 + +`colm-index-status.json`、`neurips-index-status.json` 保存逐索引状态；对应 `*-last-run.json` 保存本次实际选择范围。每个 JSONL 的 manifest 记录索引入口数、track 记录数和缺失摘要数。论文数量与官方页面声明不一致会失败，缺失摘要会标记 partial；不会把页脚、姓名政策或其他正文当作摘要。 + +这两个采集器只写采集文件，合并仍通过单写入协调流程。全量历史覆盖和所有未实现会议仍待最终核验。 + +### ACL Anthology 权威 XML + +```sh +python3 -B collect_acl.py \ + --checkout /data/paper-index/raw/metadata-checkouts/acl-anthology \ + --out /data/paper-index/collected +``` + +`metadata_checkout.py` 对官方仓库做只包含 `data/xml` 的 partial clone/sparse checkout，不获取 PDF，不执行仓库 hooks 或构建。每批固定到实际 Git commit。遍历 XML 的 volume venue 字段选择 ACL、EMNLP、NAACL、EACL、COLING、CoNLL；Findings 按所属会议独立成 track，不按文件名把其他同场 workshop 冒充主会。 + +新旧 Anthology ID 按官方规则构造，并核对源 DOI。保留 C69、D19-50 至 D19-66、W 系列的历史编号例外，排除编号 0 的前言。XML 混合标记通过 itertext 提取；无摘要保留为空并标记 partial。主会、Findings、workshop、demo、tutorial、industry、shared_task 分开。 + +联合举办会议保存一个会议事件及多个名称别名。例如 LREC-COLING 的论文可用 COLING 或 LREC 查询，仍只有一次该届投稿记录。可用 `--track main` 限定主会，避免把 Findings 或演示算入主会；别名数据在正常单写入合并时应用 schema。 + +`acl-collection-status.json` 记录每个 XML 及其 volume 的处理结果和排除原因；`acl-last-run.json` 记录源 commit 与本批范围。`--collection P17` 可限定一个确实存在的 collection。已验证真实 ACL 2017 长文 195 条、短文 107 条，二者摘要完整；其他 track 按源实际缺失情况记录，不能凭样本宣称整个历史完成。 + +### ICLR 官方公开程序元数据 + +```sh +python3 -B collect_iclr.py --cache /data/paper-index/raw/http-cache \ + --out /data/paper-index/collected +# 入库后按官方主题检索；主题值以实际来源为准 +python3 -B query.py --db /data/paper-index/papers.sqlite \ + --venue ICLR --tag 'iclr_program:topic=Reinforcement Learning->Deep RL' +``` + +从 ICLR 官方年份导航登记届次，读取对应 Papers 页面明确引用的 orals-posters/abstracts JSON。核对声明数量、分页状态及唯一 event ID；只下载公开程序元数据，不访问用户资料、PDF、海报、幻灯片或视频。`--year` 可限制年份；清单中的已采集数与数据库已入库数分别统计。 + +官方程序中的 event 数不是论文数：同一篇文章可以有 poster 和 oral 两场展示。程序还会给负 sourceid 的 oral 展示生成占位 OpenReview 链接，这些链接不能作为真实论文标识。采集器核对原投稿编号、标题与作者后合并展示，优先使用正编号记录的真实论文链接；原展示编号和类型保留在源元数据中。 + +同一投稿编号的展示标题可能有小幅修订。只有负编号 oral 与正编号原稿对应、至少两位作者完全一致且标题相似度满足严格阈值时，才保留为同稿的标题变体；该规则仅在一届官方程序内部生效，不用于跨平台模糊合并。 + +只将 Conference 来源的论文视为 ICLR 主会录用，排除 Blog 和期刊转会展示，并在审计清单保存排除计数。2020/2021 旧数据允许依据官方主会 paper program 形成录用证据；无 forum ID 时保留 program 身份和匹配候选，不伪造标识。历史年度返回 404 会登记 unavailable，未来未发表届次登记 not_yet_published，其他访问或解析失败登记 blocked；这些状态都不能宣称该年已覆盖。 + +主题、关键词使用 `iclr_program:topic`、`iclr_program:keyword` 命名空间。通用导入器按源记录刷新标签证据，保留同一论文其他源记录及 `custom:*` 标签。每年的 JSONL、manifest、`iclr-program-year-status.json` 和 `iclr-program-last-run.json` 保存实际范围、独立展示数、主会展示数、论文数、缺失摘要与缺失 forum 数。录用程序不能提供拒稿全集，公开投稿及拒稿补充仍需独立来源。 + +### CVF / ECVA 视觉会议元数据 + +```sh +python3 -B collect_vision.py --source cvf \ + --cache /data/paper-index/raw/http-cache --out /data/paper-index/collected +python3 -B collect_vision.py --source ecva \ + --cache /data/paper-index/raw/http-cache --out /data/paper-index/collected +``` + +CVF 从官方根目录发现 CVPR、ICCV 主会与 Findings，分别保存 track；不把 workshop 或 WACV 算入这些会议。优先跟随页面明确提供的 All Papers 链接，避免只收录默认显示的第一天。ECVA 从官方总目录按论文 URL 的会议与年份分组，当前总目录列有 ECCV 2018、2020、2022、2024。不存在于来源的 `--venue`、`--year` 或二者组合会报错，不补造双年会议届次，也不能因此推断旧届次未举办。 + +只访问论文索引与摘要 HTML/PHP 页面。作者优先取 citation 元数据，旧页面尊重声明编码；无 citation 作者时取索引中的作者行，排除论文集出版信息。DOI 只取该论文条目所属的 DOI 链接或 citation 字段；冲突会失败，不从摘要引用中抽取当前论文 ID。不请求 PDF、补充材料、视频或代码。 + +每份清单记录索引入口数、论文数、缺失摘要与作者数，缺失内容保留为空并标记 partial。存在未被标题条目解析的论文链接时会失败，防止静默漏项。逐来源的 discovery、index-status、last-run 文件分别保存发现目录、逐届结果和实际请求范围。可重复使用 `--year` 或 `--venue` 限定重试；其他届次状态会保留。官方开放目录之外的早期届次、尚未刊出的新届次和独立出版方历史数据仍需后续补齐，采集器完成不代表全历史覆盖已完成。 ## 长期维护与更新 @@ -96,4 +308,6 @@ python3 -B query.py --db /data/paper-index/papers.sqlite --venue COLM --year 202 2026-11-06 从目标服务器测试：Kaggle 元数据 GET 可下载；OpenReview 会议 group 接口可读，但 notes 接口返回 `ChallengeRequiredError`。拒稿记录导入尚受此限制，不能用录用名单替代全部投稿，也不能声称已覆盖全部拒稿。需通过平台支持的访问方式或有合法来源的公开数据导出继续补全。 -全量导入、会议适配器、正式 runner 配置与 PR 交付状态应随实现及时更新本 README。 +已核验的采集文件计数：ICLR 2020–2026 主会 15,534 条、COLM 2024–2026 共 1,573 条，摘要均已取得；候选范围内的 PMLR 文件为 31,428 条，已剔除两份重复出版元数据。ACL Anthology 已选文件为 63,297 条，其中 15,147 条源记录缺摘要。这些是采集文件的来源记录数，不能相加当作去重论文数，也不能代替主库实时统计；实际入库量使用 query.py --stats 查看。 + +代码在 `codex/metadata-index` 工作分支交付，[Draft PR #1](https://gitea.entroia.com/Zayden-Jung/paper-atlas/pulls/1) 尚未合并，等待全量就绪后用户手动合并。各提交分别由远程 CI 验证，结果以对应提交的 Gitea Actions 检查为准。全量导入、其他会议适配器、去重复核和覆盖率核验继续进行，应随实际结果更新此说明。 diff --git a/audit_iclr_snapshot.py b/audit_iclr_snapshot.py new file mode 100644 index 0000000..8501d78 --- /dev/null +++ b/audit_iclr_snapshot.py @@ -0,0 +1,91 @@ +"""Cross-check collected public snapshots against same-edition official programs.""" +import argparse +from collections import Counter +import hashlib +import json +from pathlib import Path + +from index_core import author_keys,normalized +from submission_evidence import identities_for_row,timestamp + + +def load(path,source,year): + path=Path(path);manifest=json.loads(path.with_suffix('.manifest.json').read_text()) + minimum=2 if source=='iclr_program' else 1 + if manifest.get('parser_version',0)=3 and parts[0]=='conf' else '' + + +def catalogue(archive,dtd,out,expected_sha=XML_SHA,expected_dtd_md5=DTD_MD5): + archive=Path(archive);out=Path(out);dtd=Path(dtd) + identity=digest_file(archive);dtd_identity=digest_file(dtd) + if identity['sha256']!=expected_sha:raise ValueError('DBLP snapshot checksum mismatch') + if dtd_identity['md5']!=expected_dtd_md5:raise ValueError('DBLP DTD checksum mismatch') + if dtd.stat().st_size>128*1024:raise ValueError('Unexpectedly large DTD') + out.mkdir(parents=True,exist_ok=True) + target=out/'dblp-catalogue.sqlite';stage=out/'dblp-catalogue.sqlite.part' + candidate=out/'dblp-ai-candidates.jsonl';part=candidate.with_suffix('.part') + if stage.exists():raise ValueError('Unfinished catalogue exists; inspect it before another run') + c=sqlite3.connect(stage) + c.executescript('PRAGMA journal_mode=DELETE; PRAGMA synchronous=NORMAL; CREATE TABLE proceedings(key TEXT PRIMARY KEY,family TEXT,metadata_json TEXT); CREATE INDEX proceedings_family ON proceedings(family); CREATE TABLE volume_counts(crossref TEXT,year TEXT,booktitle TEXT,record_type TEXT,count INTEGER,PRIMARY KEY(crossref,year,booktitle,record_type)); CREATE TABLE family_counts(family TEXT,record_type TEXT,count INTEGER,PRIMARY KEY(family,record_type)); CREATE TABLE family_samples(family TEXT,seq INTEGER,metadata_json TEXT,PRIMARY KEY(family,seq)); CREATE TABLE volume_anomalies(key TEXT PRIMARY KEY,metadata_json TEXT);') + totals=Counter();families=Counter();volumes=Counter();samples=Counter();staged=Counter();seen=0 + + def flush(): + c.executemany('INSERT INTO volume_counts VALUES(?,?,?,?,?) ON CONFLICT(crossref,year,booktitle,record_type) DO UPDATE SET count=count+excluded.count',((*key,count) for key,count in volumes.items())) + volumes.clear();c.commit() + + try: + with part.open('w') as output: + for row in records(archive,dtd.read_bytes()): + seen+=1;kind=row['record_type'];key=row['key'];fields=row['fields'];prefix=family(key) + if not key:raise ValueError('Bibliographic record lacks a key') + totals[kind]+=1 + if kind=='proceedings': + c.execute('INSERT INTO proceedings VALUES(?,?,?)',(key,prefix,json.dumps(row,ensure_ascii=False))) + if prefix: + families[(prefix,kind)]+=1 + if kind=='inproceedings': + crossrefs=fields.get('crossref') or [''];years=fields.get('year') or [''];books=fields.get('booktitle') or [''] + if any(len(values)!=1 for values in (crossrefs,years,books)): + c.execute('INSERT INTO volume_anomalies VALUES(?,?)',(key,json.dumps(row,ensure_ascii=False))) + totals['multiple_volume_fields']+=1 + group=tuple(values[0] if len(values)==1 else '@multiple:'+json.dumps(values,ensure_ascii=False) for values in (crossrefs,years,books)) + volumes[(*group,kind)]+=1 + if samples[prefix]<2: + c.execute('INSERT INTO family_samples VALUES(?,?,?)',(prefix,samples[prefix],json.dumps(row,ensure_ascii=False)));samples[prefix]+=1 + if prefix in FAMILIES: + output.write(json.dumps(row,ensure_ascii=False)+'\n');staged[prefix]+=1 + if seen%50000==0: + flush();print(json.dumps({'source':'dblp','records_scanned':seen,'candidate_records':sum(staged.values())}),flush=True) + flush() + c.executemany('INSERT INTO family_counts VALUES(?,?,?)',((*key,count) for key,count in families.items()));c.commit() + if c.execute('PRAGMA quick_check').fetchall()!=[('ok',)]:raise ValueError('Catalogue integrity failed') + c.close();c=None + manifest={'source':'dblp','status':'catalogued_candidates_pending_scope_review','source_release':RELEASE,'snapshot_sha256':identity['sha256'], + 'dtd_md5':dtd_identity['md5'],'finished_at':timestamp(),'records_scanned':seen,'record_types':{kind:totals[kind] for kind in ('proceedings','inproceedings')}, + 'multiple_volume_field_records':totals['multiple_volume_fields'], + 'candidate_records':sum(staged.values()),'candidate_family_counts':dict(staged),'candidate_sha256':digest_file(part)['sha256'], + 'file':str(candidate),'catalogue':str(target),'note':'Candidate families include workshops and historical names. No acceptance labels or normalized venue assignments have been inferred.'} + stage.replace(target);part.replace(candidate) + manifest_path=candidate.with_suffix('.manifest.json');manifest_part=manifest_path.with_suffix('.part') + manifest_part.write_text(json.dumps(manifest,indent=2));manifest_part.replace(manifest_path) + print(json.dumps(manifest),flush=True);return manifest + finally: + if c is not None:c.close() + + +if __name__=='__main__': + parser=argparse.ArgumentParser();parser.add_argument('--archive',required=True);parser.add_argument('--dtd',required=True);parser.add_argument('--out',required=True) + args=parser.parse_args();catalogue(args.archive,args.dtd,args.out) diff --git a/collect_acl.py b/collect_acl.py new file mode 100644 index 0000000..cfd9976 --- /dev/null +++ b/collect_acl.py @@ -0,0 +1,131 @@ +"""Collect authoritative ACL Anthology XML metadata, never PDFs.""" +import argparse +import json +import re +import xml.etree.ElementTree as ET +from pathlib import Path + +from metadata_checkout import checkout_metadata + +VENUES={'acl':'ACL','emnlp':'EMNLP','naacl':'NAACL','eacl':'EACL','coling':'COLING','conll':'CoNLL'} + + +def text(element): + return ' '.join(''.join(element.itertext()).split()) if element is not None else '' + + +def paper_id(collection,volume,number): + if not str(number).isdigit() or not re.fullmatch(r'[A-Za-z0-9_-]+',str(volume)): + raise ValueError('Invalid Anthology paper numbering') + if re.fullmatch(r'\d{4}\.[a-z0-9]+',collection): + return f'{collection}-{volume}.{int(number)}' + if not re.fullmatch(r'[A-Z]\d{2}',collection) or not str(volume).isdigit(): + raise ValueError('Unsupported historical identifier') + small=collection.startswith('W') or collection=='C69' or (collection=='D19' and 50<=int(volume)<=66) + width=2 if small else 1 + digits=2 if small else 3 + if len(str(int(volume)))>width or len(str(int(number)))>digits: + raise ValueError('Historical identifier exceeds documented width') + return f'{collection}-{int(volume):0{width}d}{int(number):0{digits}d}' + + +def volume_scope(meta,volume): + codes=[text(n).lower() for n in meta.findall('venue')] + book=text(meta.find('booktitle')) + lower=book.lower() + selected=sorted({VENUES[v] for v in codes if v in VENUES}) + if 'findings' in codes: + associated=volume.split('-')[0].lower() + if associated in VENUES:selected=[VENUES[associated]] + if not selected: + return None + joint_names=sorted({(VENUES|{'lrec':'LREC'})[v] for v in codes if v in VENUES or v=='lrec'}) + if len(joint_names)>1:selected=joint_names + track='main' + if 'findings' in codes or 'findings of' in lower:track='findings' + elif 'shared task' in lower:track='shared_task' + elif 'workshop' in lower or 'student research' in lower:track='workshop' + elif 'demonstration' in lower or 'system demonstrations' in lower:track='demo' + elif 'tutorial' in lower:track='tutorial' + elif 'industry' in lower:track='industry' + return '+'.join(selected),selected,track + + +def collect_xml(path,out,commit): + root=ET.parse(path).getroot() + collection=root.attrib['id'] + results=[] + for volume in root.findall('volume'): + vid=volume.attrib['id'] + meta=volume.find('meta') + if meta is None:raise ValueError('Volume metadata missing') + scope=volume_scope(meta,vid) + key=f'{collection}-{vid}' + if scope is None: + results.append({'volume':key,'status':'excluded_by_venue','venue_codes':[text(n) for n in meta.findall('venue')],'booktitle':text(meta.find('booktitle')),'year':text(meta.find('year'))}) + continue + venue,aliases,track=scope + year_text=text(meta.find('year')) + if not re.fullmatch(r'\d{4}',year_text):raise ValueError('No explicit official volume year: '+key) + year=int(year_text) + records=[] + frontmatter=0 + for paper in volume.findall('paper'): + pid=paper.attrib['id'] + if int(pid)==0: + frontmatter+=1 + continue + identifier=paper_id(collection,vid,pid) + doi=text(paper.find('doi')) + if doi.startswith('10.18653/v1/') and doi.split('10.18653/v1/',1)[1].casefold()!=identifier.casefold(): + raise ValueError('Official DOI disagrees with constructed Anthology ID: '+identifier) + title=text(paper.find('title')) + if not title:raise ValueError('Missing paper title: '+identifier) + authors=[{'given':text(a.find('first')),'family':text(a.find('last'))} for a in paper.findall('author')] + records.append({'source':'acl_anthology','external_id':identifier,'url':f'https://aclanthology.org/{identifier}/','title':title,'abstract':text(paper.find('abstract')) or None,'authors':authors,'identifiers':{'acl':identifier,'doi':doi},'venue':venue,'venue_aliases':aliases if len(aliases)>1 else [],'year':year,'track':track,'status':'accepted','raw_decision':'listed in authoritative ACL Anthology proceedings XML','metadata':{'collection':collection,'volume':vid,'booktitle':text(meta.find('booktitle')),'venue_codes':[text(n) for n in meta.findall('venue')],'pages':text(paper.find('pages')),'publisher':text(meta.find('publisher')),'metadata_commit':commit,'source_xml':f'https://github.com/acl-org/acl-anthology/blob/{commit}/data/xml/{collection}.xml'}}) + if not records: + results.append({'volume':key,'status':'unavailable','reason':'no research paper elements after excluding front matter'}) + continue + target=Path(out)/f'acl-{key}.jsonl' + target.parent.mkdir(parents=True,exist_ok=True) + part=target.with_suffix('.part') + with part.open('w') as output: + for record in records:output.write(json.dumps(record,ensure_ascii=False)+'\n') + part.replace(target) + missing=sum(not row['abstract'] for row in records) + manifest={'source':'acl_anthology','venue':venue,'year':year,'track':track,'count':len(records),'xml_paper_elements':len(volume.findall('paper')),'excluded_frontmatter':frontmatter,'missing_abstract_count':missing,'list_url':f'https://aclanthology.org/{key}/','metadata_commit':commit,'file':str(target),'status':'partial' if missing else 'collected'} + target.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)) + results.append(manifest) + print(json.dumps(manifest),flush=True) + return results + + +if __name__=='__main__': + parser=argparse.ArgumentParser() + parser.add_argument('--checkout',required=True) + parser.add_argument('--out',required=True) + parser.add_argument('--collection',action='append') + args=parser.parse_args() + commit=checkout_metadata('https://github.com/acl-org/acl-anthology.git',args.checkout,['/data/xml/']) + out=Path(args.out);out.mkdir(parents=True,exist_ok=True) + status_file=out/'acl-collection-status.json' + catalogue=json.loads(status_file.read_text()) if status_file.exists() else {} + paths=sorted((Path(args.checkout)/'data/xml').glob('*.xml')) + if not paths:raise SystemExit('No official XML files found') + if args.collection and set(args.collection)-{p.stem for p in paths}:parser.error('Requested collection is absent') + errors=[] + processed=0 + for path in paths: + if args.collection and path.stem not in args.collection:continue + try:catalogue[path.stem]={'status':'processed','metadata_commit':commit,'volumes':collect_xml(path,args.out,commit)} + except Exception as exc: + catalogue[path.stem]={'status':'blocked','metadata_commit':commit,'error':str(exc)} + errors.append(path.stem) + print(json.dumps({'collection':path.stem,'error':str(exc)}),flush=True) + processed+=1 + if processed%25==0: + part=status_file.with_suffix('.part');part.write_text(json.dumps(catalogue,indent=2));part.replace(status_file) + print(json.dumps({'xml_collections_processed':processed}),flush=True) + part=status_file.with_suffix('.part');part.write_text(json.dumps(catalogue,indent=2));part.replace(status_file) + (out/'acl-last-run.json').write_text(json.dumps({'metadata_commit':commit,'collection_filter':args.collection,'blocked_collections':errors},indent=2)) + if errors:raise SystemExit(1) diff --git a/collect_iclr.py b/collect_iclr.py new file mode 100644 index 0000000..6eb4eb0 --- /dev/null +++ b/collect_iclr.py @@ -0,0 +1,247 @@ +"""Collect public ICLR program metadata, excluding blog/journal presentations.""" +import argparse +from collections import Counter, defaultdict +from datetime import datetime, timezone +from difflib import SequenceMatcher +import json +from pathlib import Path +import re +from urllib.error import HTTPError +from urllib.parse import parse_qs, urljoin, urlsplit + +from collect_pmlr import fetch +from collect_proceedings import Document, clean +from index_core import author_keys, normalized + +ROOT = 'https://iclr.cc/' +SOURCE = 'iclr_program' + + +class NotYetPublished(ValueError): + pass + + +def discover(html): + years = {} + for node in Document(html).root.walk(): + text = clean(node.text()) + if node.tag == 'a' and re.fullmatch(r'20\d{2}', text): + url = urljoin(ROOT, node.attrs.get('href', '')) + if urlsplit(url).netloc == 'iclr.cc': + years[int(text)] = url + if not years: + raise ValueError('No ICLR years in the official navigation') + return sorted(years.items()) + + +def asset_urls(html, page_url, year): + refs = re.findall(r'''["']([^"'\s]+\.json)["']''', html) + result = {} + for kind in ('orals-posters', 'abstracts'): + matches = {urljoin(page_url, ref) for ref in refs + if urlsplit(urljoin(page_url, ref)).netloc == 'iclr.cc' + and urlsplit(urljoin(page_url, ref)).path == f'/static/virtual/data/iclr-{year}-{kind}.json'} + if len(matches) > 1: + raise ValueError('Ambiguous official program asset') + if matches: + result[kind] = matches.pop() + if 'orals-posters' not in result: + raise ValueError('Official page has no supported program JSON reference') + return result + + +def main_event(row, year): + source = row.get('sourceurl') + if source: + parsed = urlsplit(source) + return (parsed.netloc == 'openreview.net' and parsed.path == '/group' + and parse_qs(parsed.query).get('id') == [f'ICLR.cc/{year}/Conference']) + # These historical official paper lists predate sourceurl/decision fields. + return year in (2020, 2021) and row.get('eventtype') in ('Poster', 'Oral', 'Spotlight') + + +def source_number(row): + value = row.get('sourceid') + return int(value) if re.fullmatch(r'-?[1-9]\d*', str(value)) else None + + +def identifiers(row): + # The program creates negative-ID oral events with fabricated forum URLs. + if (source_number(row) or 0) < 0: + return {} + links = [row.get('paper_url'), row.get('paper_pdf_url')] + links += [media.get('uri') for media in row.get('eventmedia', []) + if str(media.get('name') or '').casefold() in ('openreview', 'paper')] + forums = set() + for link in links: + if not isinstance(link, str): + continue + parsed = urlsplit(link) + if parsed.netloc == 'openreview.net' and parsed.path in ('/forum', '/pdf'): + values = parse_qs(parsed.query).get('id', []) + if len(values) == 1 and re.fullmatch(r'[\w-]+', values[0]) and not re.match(r'20\d{2}-Oral-', values[0]): + forums.add(values[0]) + if len(forums) > 1: + raise ValueError('Conflicting paper self-links in one program event') + return {'openreview': forums.pop()} if forums else {} + + +def parse_program(data, abstracts, year, index_url): + if not isinstance(data, dict) or not isinstance(data.get('results'), list): + raise ValueError('Expected a program results array with a declared count') + events = data['results'] + if data.get('next') or data.get('previous') or data.get('count') != len(events): + raise ValueError('Program JSON is paginated or its count differs from events') + if not isinstance(abstracts, dict): + raise ValueError('Expected an abstract mapping keyed by event ID') + event_ids = [str(row['id']) for row in events] + if len(event_ids) != len(set(event_ids)): + raise ValueError('Duplicate event IDs in program JSON') + grouped = defaultdict(list) + excluded = Counter() + for row in events: + if not main_event(row, year): + excluded[str(row.get('sourceurl') or 'unclassified')] += 1 + continue + decision = row.get('decision') + if decision and not (str(decision).casefold().startswith('accept') or decision in ('Poster', 'Oral', 'Spotlight')): + raise ValueError('Unexpected decision in an accepted conference program') + number = source_number(row) + identity = ('submission', abs(number)) if number else ('forum', identifiers(row).get('openreview')) + if identity == ('forum', None): + identity = ('event', row['id']) + grouped[identity].append(row) + records = [] + missing_identity = 0 + for identity, group in grouped.items(): + # Verify the negative oral event really describes its positive counterpart. + group.sort(key=lambda r: ((source_number(r) or 0) < 0, str(r['id']))) + primary = group[0] + title = normalized(primary.get('name', '')) + authors = [author_keys([a['fullname'] for a in row.get('authors', [])]) for row in group] + title_match = all(normalized(row.get('name', '')) == title or ( + identity[0] == 'submission' and (source_number(row) or 0) < 0 + and (source_number(primary) or 0) > 0 and len(authors[0]) >= 2 + and SequenceMatcher(None, title, normalized(row.get('name', ''))).ratio() >= 0.85) + for row in group) + if not title or not title_match or any(a != authors[0] for a in authors): + raise ValueError('Program submission ID collides across different papers') + own_ids = {} + for row in group: + for scheme, value in identifiers(row).items(): + if scheme in own_ids and own_ids[scheme] != value: + raise ValueError('Conflicting identifiers across presentation events') + own_ids[scheme] = value + abstract = next((text for row in group + if (text := row.get('abstract') or abstracts.get(str(row['id'])))), None) + if abstract is not None and not isinstance(abstract, str): + raise ValueError('Nontext abstract') + presentation_ids = [row['id'] for row in group] + forum = own_ids.get('openreview') + external = f'{year}/{forum}' if forum else f'{year}/program-{identity[1]}' + missing_identity += not bool(forum) + page = urljoin(ROOT, primary.get('virtualsite_url') or f'/virtual/{year}/{str(primary.get("eventtype", "Poster")).lower()}/{primary["id"]}') + topics = sorted({str(row['topic']).strip() for row in group if row.get('topic')}) + keywords = sorted({str(keyword).strip() for row in group for keyword in row.get('keywords', []) if keyword}) + records.append({'source': SOURCE, 'external_id': external, 'url': page, + 'title': clean(primary['name']), 'abstract': abstract.strip() if abstract else None, + 'authors': [a['fullname'] for a in primary.get('authors', [])], + 'identifiers': own_ids, 'venue': 'ICLR', 'year': year, 'track': 'main', + 'status': 'accepted', 'raw_decision': str(primary.get('decision') or 'listed in official conference paper program'), + 'evidence_url': index_url, 'list_url': index_url, + 'tags': [{'namespace': SOURCE+':'+kind, 'value': value} + for kind, values in [('topic', topics), ('keyword', keywords)] for value in values], + 'metadata': {'program_event_ids': presentation_ids, + 'program_identity_type': identity[0], + 'presentation_titles': sorted({clean(r['name']) for r in group}), + 'program_source_id': identity[1], 'presentation_types': sorted({str(r.get('event_type') or r.get('eventtype')) for r in group}), + 'raw_decisions': sorted({str(r.get('decision')) for r in group if r.get('decision')}), + 'identity_state': 'forum' if forum else 'program_only'}}) + # Different source numbers can still identify the same forum. Keep one record. + by_id = {} + for row in records: + previous = by_id.get(row['external_id']) + if previous: + if normalized(previous['title']) != normalized(row['title']) or author_keys(previous['authors']) != author_keys(row['authors']): + raise ValueError('One forum identifies different program papers') + previous['metadata']['program_event_ids'] += row['metadata']['program_event_ids'] + previous['metadata']['presentation_types'] = sorted(set(previous['metadata']['presentation_types'] + row['metadata']['presentation_types'])) + previous['abstract'] = previous['abstract'] or row['abstract'] + else: + by_id[row['external_id']] = row + result = list(by_id.values()) + if not result: + raise ValueError('No verified main-conference papers in program') + return result, {'program_event_count': len(events), 'main_event_count': sum(map(len, grouped.values())), + 'excluded_events_by_source': dict(excluded), 'missing_forum_count': missing_identity, + 'missing_abstract_count': sum(not r['abstract'] for r in result)} + + +def collect_year(year, cache, out): + page = f'https://iclr.cc/virtual/{year}/papers.html' + assets = asset_urls(fetch(page, cache, 16*1024**2, delay=1).decode(), page, year) + data = json.loads(fetch(assets['orals-posters'], cache, 32*1024**2, delay=1)) + if data.get('count') == 0 and data.get('results') == [] and year > datetime.now(timezone.utc).year: + raise NotYetPublished('Official program is empty for a future conference') + abstracts = {} + abstract_asset_status = 'not_referenced' + if 'abstracts' in assets: + try: + abstracts = json.loads(fetch(assets['abstracts'], cache, 32*1024**2, delay=1)) + abstract_asset_status = 'available' + except HTTPError as exc: + if exc.code != 404: + raise + # Historical programs embed abstracts but retain a stale split-file URL. + exc.close() + abstract_asset_status = 'unavailable_404' + records, audit = parse_program(data, abstracts, year, page) + path = Path(out)/f'iclr-program-{year}-main.jsonl' + part = path.with_suffix('.part') + part.write_text(''.join(json.dumps(row, ensure_ascii=False)+'\n' for row in records)) + part.replace(path) + manifest = {'source': SOURCE, 'parser_version': 2, 'venue': 'ICLR', 'year': year, 'track': 'main', + 'count': len(records), 'list_url': page, 'asset_urls': assets, + 'abstract_asset_status': abstract_asset_status, 'file': str(path), + 'status': 'partial' if audit['missing_abstract_count'] else 'collected', **audit} + path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)) + return manifest + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--cache', required=True) + parser.add_argument('--out', required=True) + parser.add_argument('--year', type=int, action='append') + args = parser.parse_args() + out = Path(args.out); out.mkdir(parents=True, exist_ok=True) + years = discover(fetch(ROOT, args.cache, 4*1024**2, delay=1).decode()) + if args.year and set(args.year)-{year for year, _ in years}: + parser.error('Requested year is absent from the official navigation') + status_path = out/'iclr-program-year-status.json' + catalogue = json.loads(status_path.read_text()) if status_path.exists() else {} + requested = [] + for year, conference_url in years: + if args.year and year not in args.year: + continue + requested.append(year) + try: + result = collect_year(year, args.cache, out) + except NotYetPublished as exc: + result = {'year': year, 'status': 'not_yet_published', 'error': str(exc)} + except HTTPError as exc: + result = {'year': year, 'status': 'not_yet_published' if year > datetime.now(timezone.utc).year and exc.code == 404 else 'unavailable' if exc.code == 404 else 'blocked', 'error': str(exc)} + exc.close() + except Exception as exc: + result = {'year': year, 'status': 'blocked', 'error': str(exc)} + result['conference_url'] = conference_url + catalogue[str(year)] = result + part = status_path.with_suffix('.part'); part.write_text(json.dumps(catalogue, indent=2)); part.replace(status_path) + print(json.dumps(result), flush=True) + (out/'iclr-program-last-run.json').write_text(json.dumps({'requested_years': requested, 'year_filter': args.year}, indent=2)) + if any(catalogue[str(y)]['status'] == 'blocked' for y in requested): + raise SystemExit(1) + + +if __name__ == '__main__': + main() diff --git a/collect_openreview_snapshot.py b/collect_openreview_snapshot.py new file mode 100644 index 0000000..b2c3302 --- /dev/null +++ b/collect_openreview_snapshot.py @@ -0,0 +1,200 @@ +"""Extract public paper metadata and signed Decision notes from a pinned release.""" +import argparse +from collections import Counter,defaultdict +from contextlib import closing +from datetime import datetime,timezone +import json +from pathlib import Path +import re +import sqlite3 +import tarfile +import tempfile + +from download_snapshot import digest_file + + +SOURCE='openreview_snapshot' +SHA256='350071daf25ebf1aef854ee1d5dc3a81834e1c468917c82a4f78af5548de34fc' +RELEASE='https://github.com/qhjqhj00/iclr-openreview-reviews/releases/tag/v1.0' +AS_OF='2026-05-08T04:50:23.000Z' +EXPECTED={2020:2213,2021:2594,2022:2617,2023:3792,2024:7404,2025:11672,2026:19814} + + +def value(content,key,default=None): + result=content.get(key,default) + return result.get('value',default) if isinstance(result,dict) else result + + +def invitations(note): + result=note.get('invitations') or [] + if isinstance(result,str):result=[result] + if note.get('invitation'):result=list(result)+[note['invitation']] + return [i for i in result if isinstance(i,str)] + + +def public(note): + return isinstance(note.get('readers'),list) and 'everyone' in note['readers'] + + +def date_ms(raw): + if raw is None:return None + return datetime.fromtimestamp(int(raw)/1000,timezone.utc).isoformat(timespec='milliseconds').replace('+00:00','Z') + + +def decision_status(raw): + key=' '.join(raw.casefold().replace('_',' ').split()) + if key.startswith(('desk reject','desk-reject')):return 'desk_rejected' + if key in ('accept','accepted','poster','oral','spotlight') or key.startswith(('accept (','accept:','accept -')):return 'accepted' + if key in ('reject','rejected','reject and resubmit') or key.startswith(('reject (','reject:','reject -')):return 'rejected' + if key in ('withdraw','withdrawn','withdrawal'):return 'withdrawn' + return 'unknown' + + +def paper_record(note,year): + if not public(note):return None,'not_public' + base=f'ICLR.cc/{year}/Conference/-/' + if not any(i in (base+'Submission',base+'Blind_Submission') for i in invitations(note)):return None,'wrong_invitation' + forum=note.get('id') + if not isinstance(forum,str) or not re.fullmatch(r'[A-Za-z0-9_-]+',forum) or note.get('forum') not in (None,forum):return None,'not_submission_root' + content=note.get('content') or {};title=value(content,'title');abstract=value(content,'abstract') + if not isinstance(title,str) or not title.strip():return None,'missing_title' + if abstract is not None and not isinstance(abstract,str):return None,'nontext_abstract' + authors=value(content,'authors',[]) or [] + if isinstance(authors,str):authors=[a.strip() for a in authors.split(',')] + if not isinstance(authors,list):return None,'invalid_authors' + authors=[a.strip() for a in authors if isinstance(a,str) and a.strip() and not re.fullmatch(r'anonymous(?: authors?)?',a.strip(),re.I)] + number=note.get('number');number=int(number) if re.fullmatch(r'[1-9]\d*',str(number)) else None + identifiers={'openreview':forum} + for field in ('doi','arxiv','arxiv_id'): + own=value(content,field) + if not isinstance(own,str):continue + own=own.strip() + if field=='doi': + own=re.sub(r'^https?://(?:dx\.)?doi\.org/','',own,flags=re.I).lower() + if re.fullmatch(r'10\.\d{4,9}/\S+',own):identifiers['doi']=own + else: + own=re.sub(r'^https?://arxiv.org/(?:abs|pdf)/','',own).removesuffix('.pdf') + if re.fullmatch(r'(?:\d{4}\.\d{4,5}|[a-zA-Z.-]+/\d{7})(?:v\d+)?',own):identifiers['arxiv']=own + tags=[];index_fields={} + for field in ('keywords','primary_area','secondary_area','Please_choose_the_closest_area_that_your_submission_falls_into'): + raw=value(content,field) + if raw is None:continue + index_fields[field]=raw + terms=raw if isinstance(raw,list) else re.split(r'[,;\n]',raw) if field=='keywords' and isinstance(raw,str) else [raw] + namespace=SOURCE+':'+('keyword' if field=='keywords' else field) + tags.extend({'namespace':namespace,'value':term.strip()} for term in terms if isinstance(term,str) and term.strip()) + ids=[{'scheme':'openreview','value':forum}] + if number:ids.append({'scheme':'iclr_number','value':str(number)}) + return {'source':SOURCE,'external_id':f'{year}/{forum}','url':'https://openreview.net/forum?id='+forum, + 'title':' '.join(title.split()),'abstract':abstract.strip() if abstract else None,'authors':authors, + 'identifiers':identifiers,'submission_identifiers':ids,'venue':'ICLR','year':year,'track':'main', + 'submitted_at':date_ms(note.get('cdate') or note.get('tcdate')),'observed_at':AS_OF,'tags':tags, + 'metadata':{'forum':forum,'source_submission_number':number,'original_invitations':invitations(note), + 'public_readers_verified':True,'index_fields':index_fields,'venue':value(content,'venue'), + 'venueid':value(content,'venueid'),'snapshot_release':RELEASE, + 'snapshot_observation_upper_bound':AS_OF}},None + + +def decision_record(note,year): + if not public(note):return None,'not_public' + base=f'ICLR.cc/{year}/Conference/' + if not any(i.startswith(base) and i.endswith('/Decision') for i in invitations(note)):return None,'not_decision' + if not any(isinstance(s,str) and s.startswith(base) and s.rsplit('/',1)[-1] in ('Program_Chairs','Area_Chairs','Senior_Area_Chairs') for s in note.get('signatures',[])): + return None,'unverified_signature' + if note.get('ddate'):return None,'deleted_decision' + forum=note.get('forum');identifier=note.get('id') + if not all(isinstance(v,str) and re.fullmatch(r'[A-Za-z0-9_-]+',v) for v in (forum,identifier)):return None,'invalid_identity' + if note.get('_paper_forum',forum)!=forum:return None,'forum_mismatch' + raw=value(note.get('content') or {},'decision') + if not isinstance(raw,str) or not raw.strip():return None,'missing_decision_value' + return {'id':identifier,'forum':forum,'status':decision_status(raw),'raw_value':raw, + 'evidence_url':f'https://openreview.net/forum?id={forum}¬eId={identifier}', + 'observed_at':AS_OF,'decision_date':date_ms(note.get('cdate') or note.get('tcdate') or note.get('mdate')), + 'source_decision_id':identifier,'authority':{'invitations':invitations(note), + 'signatures':note.get('signatures',[]),'public_readers_verified':True}},None + + +def collect(archive,out,years,expected_sha=SHA256): + actual=digest_file(Path(archive)) + if actual['sha256']!=expected_sha:raise ValueError('Snapshot checksum mismatch before parsing') + if expected_sha==SHA256 and actual['bytes']!=512940722:raise ValueError('Unexpected pinned snapshot size') + out=Path(out);out.mkdir(parents=True,exist_ok=True);years=set(years) + counts=defaultdict(Counter);members=set() + with tempfile.TemporaryDirectory(prefix='openreview-stage-',dir=out) as tmp,closing(sqlite3.connect(Path(tmp)/'metadata.sqlite')) as c: + c.executescript('PRAGMA synchronous=OFF; CREATE TABLE papers(year INTEGER,forum TEXT,payload TEXT,PRIMARY KEY(year,forum)); CREATE TABLE decisions(year INTEGER,id TEXT,forum TEXT,payload TEXT,PRIMARY KEY(year,id)); CREATE INDEX decisions_forum ON decisions(year,forum);') + with tarfile.open(archive,mode='r|bz2') as tar: + for member in tar: + match=re.fullmatch(r'(?:\./)?data/iclr_(20\d{2})/(papers|reviews)\.jsonl',member.name) + if not match or int(match[1]) not in years:continue + year=int(match[1]);kind=match[2] + if not member.isfile() or (year,kind) in members:raise ValueError('Invalid or repeated metadata member') + members.add((year,kind));lines=0 + with tar.extractfile(member) as stream: + while line:=stream.readline(16*1024**2+1): + if len(line)>16*1024**2:raise ValueError('Unexpectedly large metadata line') + if not line.strip():continue + lines+=1;counts[year][kind+'_lines']+=1 + if kind=='reviews' and b'/Decision' not in line:continue + note=json.loads(line) + row,reason=paper_record(note,year) if kind=='papers' else decision_record(note,year) + if row is None: + counts[year][kind+'_excluded_'+reason]+=1;continue + if kind=='papers': + forum=row['metadata']['forum'];payload=json.dumps(row,ensure_ascii=False) + old=c.execute('SELECT payload FROM papers WHERE year=? AND forum=?',(year,forum)).fetchone() + if old and old[0]!=payload:raise ValueError('Conflicting copies of one submission') + if old:counts[year]['duplicate_papers']+=1 + else:c.execute('INSERT INTO papers VALUES(?,?,?)',(year,forum,payload)) + else: + payload=json.dumps(row,ensure_ascii=False);old=c.execute('SELECT payload FROM decisions WHERE year=? AND id=?',(year,row['id'])).fetchone() + if old and old[0]!=payload:raise ValueError('Conflicting copies of one decision note') + if old:counts[year]['duplicate_decisions']+=1 + else:c.execute('INSERT INTO decisions VALUES(?,?,?,?)',(year,row['id'],row['forum'],payload)) + if lines%5000==0:print(json.dumps({'source':SOURCE,'year':year,'member':kind,'lines':lines}),flush=True) + c.commit();print(json.dumps({'source':SOURCE,'year':year,'member':kind,'lines':lines,'status':'member_processed'}),flush=True) + if members!={(year,kind) for year in years for kind in ('papers','reviews')}:raise ValueError('Selected year metadata members are missing') + if expected_sha==SHA256: + for year in years: + if counts[year]['papers_lines']!=EXPECTED.get(year):raise ValueError('Pinned release paper count differs from published inventory') + status_path=out/'openreview-snapshot-year-status.json';catalogue=json.loads(status_path.read_text()) if status_path.exists() else {} + for year in sorted(years,reverse=True): + target=out/f'openreview-snapshot-{year}-main.jsonl';part=target.with_suffix('.part') + statuses=Counter();event_counts=Counter();missing=0;written=0 + with part.open('w') as output: + for forum,payload in c.execute('SELECT forum,payload FROM papers WHERE year=? ORDER BY forum',(year,)): + row=json.loads(payload) + events=[json.loads(r[0]) for r in c.execute('SELECT payload FROM decisions WHERE year=? AND forum=?',(year,forum))] + events.sort(key=lambda event:(event['decision_date'] or '',event['id'])) + row['decision_events']=events;row['status']=events[-1]['status'] if events else 'unknown' + row['metadata']['snapshot_sha256']=actual['sha256'];row['metadata']['decision_note_ids']=[e['id'] for e in events] + row['metadata']['decision_authority']=[{'note_id':e['id'],**e['authority']} for e in events] + output.write(json.dumps(row,ensure_ascii=False)+'\n');written+=1;missing+=not bool(row['abstract']);statuses[row['status']]+=1 + event_counts.update(e['status'] for e in events) + if not written:raise ValueError('No verified public submissions for selected year') + unmatched=c.execute('SELECT count(*) FROM decisions d WHERE d.year=? AND NOT EXISTS(SELECT 1 FROM papers p WHERE p.year=d.year AND p.forum=d.forum)',(year,)).fetchone()[0] + unmatched_path=None + if unmatched: + unresolved=out/f'openreview-snapshot-{year}-unmatched-decisions.jsonl' + with unresolved.with_suffix('.part').open('w') as output: + for payload, in c.execute('SELECT payload FROM decisions d WHERE d.year=? AND NOT EXISTS(SELECT 1 FROM papers p WHERE p.year=d.year AND p.forum=d.forum)',(year,)): + event=json.loads(payload);event.update(year=year,reason='public decision lacks public submission metadata in snapshot',snapshot_sha256=actual['sha256']) + output.write(json.dumps(event,ensure_ascii=False)+'\n') + unresolved.with_suffix('.part').replace(unresolved);unmatched_path=str(unresolved) + part.replace(target) + manifest={'source':SOURCE,'parser_version':1,'venue':'ICLR','year':year,'track':'main','count':written, + 'list_url':RELEASE,'snapshot_sha256':actual['sha256'],'snapshot_observation_upper_bound':AS_OF, + 'missing_abstract_count':missing,'submission_status_counts':dict(statuses),'decision_event_counts':dict(event_counts), + 'unmatched_public_decisions':unmatched,'archive_counts':dict(counts[year]),'file':str(target), + 'unmatched_evidence_file':unmatched_path, + 'status':'partial' if missing or statuses['unknown'] or unmatched or any('_excluded_' in k for k in counts[year]) else 'collected'} + target.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2));catalogue[str(year)]=manifest + print(json.dumps(manifest),flush=True) + stage=status_path.with_suffix('.part');stage.write_text(json.dumps(catalogue,indent=2));stage.replace(status_path) + (out/'openreview-snapshot-last-run.json').write_text(json.dumps({'requested_years':sorted(years),'sha256':actual['sha256'],'release':RELEASE,'observed_at_upper_bound':AS_OF},indent=2)) + return {year:catalogue[str(year)] for year in years} + + +if __name__=='__main__': + parser=argparse.ArgumentParser();parser.add_argument('--archive',required=True);parser.add_argument('--out',required=True) + parser.add_argument('--year',type=int,action='append');args=parser.parse_args() + collect(args.archive,args.out,args.year or sorted(EXPECTED)) diff --git a/collect_peerread.py b/collect_peerread.py new file mode 100644 index 0000000..ca18641 --- /dev/null +++ b/collect_peerread.py @@ -0,0 +1,71 @@ +"""Collect explicit public ICLR decisions from the authors' PeerRead release. + +Review bodies and parsed full paper text are deliberately not persisted. +""" +import argparse +import json +import time +import urllib.request +from collections import defaultdict +from pathlib import Path + +from collect_pmlr import fetch + + +def collect(cache,out): + root = Path(out) + root.mkdir(parents=True,exist_ok=True) + commit = json.loads(fetch('https://api.github.com/repos/allenai/PeerRead/commits/master',cache))['sha'] + tree = json.loads(fetch(f'https://api.github.com/repos/allenai/PeerRead/git/trees/{commit}?recursive=1',cache)) + if tree.get('truncated'): + raise RuntimeError('Truncated repository tree') + paths = [v['path'] for v in tree['tree'] if v['path'].startswith('data/iclr_2017/') and '/reviews/' in v['path'] and v['path'].endswith('.json')] + if not paths: + raise RuntimeError('No explicit ICLR2017 metadata files found') + cache_dir = root/'peerread-normalized'/commit + cache_dir.mkdir(parents=True,exist_ok=True) + grouped = defaultdict(list) + for number,path in enumerate(paths,1): + external = '/'.join(path.split('/')[1:]) + normalized_cache = cache_dir/(path.replace('/','_')) + if normalized_cache.exists(): + record = json.loads(normalized_cache.read_text()) + else: + raw_url = f'https://raw.githubusercontent.com/allenai/PeerRead/{commit}/{path}' + with urllib.request.urlopen(urllib.request.Request(raw_url,headers={'User-Agent':'PaperAtlas/0.1 metadata research'}),timeout=60) as response: + payload = response.read(3*1024**2+1) + if len(payload)>3*1024**2: + raise RuntimeError('Unexpected metadata size') + data = json.loads(payload) + conference = data.get('conference','') + if 'iclr' not in conference.lower() or '2017' not in conference: + raise RuntimeError('Unexpected conference metadata: '+conference) + accepted = data.get('accepted') + status = ('accepted' if accepted else 'rejected') if isinstance(accepted,bool) else 'unknown' + authors = data.get('authors') or [] + if isinstance(authors,str): + authors = [author.strip() for author in authors.split(',') if author.strip()] + record = {'source':'peerread','external_id':external,'url':f'https://github.com/allenai/PeerRead/blob/{commit}/{path}','title':data['title'],'abstract':data.get('abstract'),'authors':authors,'identifiers':{},'venue':'ICLR','year':2017,'track':'workshop' if 'workshop' in conference.lower() else 'main','status':status,'raw_decision':json.dumps({'accepted':accepted}),'metadata':{'conference':conference,'dataset_commit':commit,'dataset_path':path,'original_id':data.get('id'),'accepted':accepted,'decision_provenance':'PeerRead authors public release; explicit label, not inferred from publication absence'}} + normalized_cache.write_text(json.dumps(record,ensure_ascii=False)) + time.sleep(0.25) + grouped[record['track']].append(record) + if number%25==0: + print(json.dumps({'source':'peerread','collected':number,'expected':len(paths)}),flush=True) + for track,records in grouped.items(): + target = root/f'peerread-iclr2017-{track}.jsonl' + part = target.with_suffix('.part') + with part.open('w') as output: + for record in records: + output.write(json.dumps(record,ensure_ascii=False)+'\n') + part.replace(target) + manifest = {'source':'peerread','venue':'ICLR','year':2017,'track':track,'count':len(records),'list_url':f'https://github.com/allenai/PeerRead/tree/{commit}/data/iclr_2017','dataset_commit':commit,'file':str(target),'decision_counts':{status:sum(r['status']==status for r in records) for status in ('accepted','rejected','unknown')}} + target.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)) + print(json.dumps(manifest),flush=True) + + +if __name__=='__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--cache',required=True) + parser.add_argument('--out',required=True) + args = parser.parse_args() + collect(args.cache,args.out) diff --git a/collect_pmlr.py b/collect_pmlr.py new file mode 100644 index 0000000..a92422f --- /dev/null +++ b/collect_pmlr.py @@ -0,0 +1,236 @@ +"""Collect official PMLR metadata, never the PDFs linked by the metadata.""" +import argparse +import hashlib +from http.client import IncompleteRead +import io +import json +import re +import os +import subprocess +import ssl +import tarfile +import time +import urllib.error +import urllib.request +from pathlib import Path + +import yaml + + +class MetadataLoader(yaml.SafeLoader): + pass + + +def binary_text(loader,node): + return loader.construct_yaml_binary(node).decode('utf-8') + + +MetadataLoader.add_constructor('!binary',binary_text) +MetadataLoader.add_constructor('tag:yaml.org,2002:binary',binary_text) + + +def load_metadata(value): + documents=[doc for doc in yaml.load_all(value,Loader=MetadataLoader) if doc is not None] + if len(documents)!=1 or not isinstance(documents[0],dict): + raise ValueError('Expected exactly one nonempty metadata mapping') + return documents[0] + + +def venue_key(shortname): + return re.sub(r'[\s_-]*(?:19|20)\d{2}$','',str(shortname).strip().upper()) + + +def fetch(url,cache,limit=64*1024**2,delay=0.5): + cache = Path(cache) + cache.mkdir(parents=True,exist_ok=True) + path = cache/hashlib.sha256(url.encode()).hexdigest() + if path.exists() and time.time()-path.stat().st_mtime < 24*3600: + if path.stat().st_size>limit:raise RuntimeError('Metadata response exceeds byte limit') + return path.read_bytes() + for attempt in range(3): + try: + request = urllib.request.Request(url,headers={'User-Agent':'PaperAtlas/0.1 (public scholarly metadata research)'}) + with urllib.request.urlopen(request,timeout=90) as response: + if response.status != 200: + raise RuntimeError('Expected complete HTTP 200 response') + expected=int(response.headers.get('Content-Length','0')) + if expected>limit: + raise RuntimeError('Metadata response exceeds byte limit') + data = response.read(limit+1) + if len(data)>limit: + raise RuntimeError('Metadata response exceeds byte limit') + if expected and len(data)!=expected: + raise IncompleteRead(data,max(0,expected-len(data))) + part = path.with_suffix('.part') + part.write_bytes(data) + part.replace(path) + path.with_suffix('.json').write_text(json.dumps({'url':url,'sha256':hashlib.sha256(data).hexdigest(),'bytes':len(data),'retrieved_at':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())})) + time.sleep(delay) + return data + except urllib.error.HTTPError as exc: + if exc.code not in (429,500,502,503,504) or attempt==2: + exc.close() + raise + retry_after=exc.headers.get('Retry-After','1');exc.close() + try:retry_after=int(retry_after) + except ValueError:retry_after=5 + time.sleep(min(30,max(2**attempt,retry_after))) + except (urllib.error.URLError,TimeoutError,ConnectionError,IncompleteRead,ssl.SSLError) as exc: + reason=getattr(exc,'reason',exc) + if isinstance(reason,ssl.SSLCertVerificationError) or attempt==2:raise + time.sleep(min(30,2**(attempt+1))) + + +def metadata_texts(volume,cache): + archive = None + if volume>50: + try: + archive = fetch(f'https://codeload.github.com/mlresearch/v{volume}/tar.gz/refs/heads/gh-pages',cache) + except RuntimeError as exc: + if 'exceeds byte limit' not in str(exc): + raise + if archive is not None: + with tarfile.open(fileobj=io.BytesIO(archive),mode='r:gz') as tar: + for member in tar: + if member.isfile() and '/_posts/' in member.name and member.name.endswith(('.md','.markdown','.html')): + if member.size>8*1024**2: + raise RuntimeError('Unexpectedly large metadata member') + yield member.name,tar.extractfile(member).read().decode('utf-8-sig') + return + # Old volumes may contain PDFs. Partial clone plus sparse checkout retrieves + # trees and metadata blobs only, without checking out or executing publisher code. + checkout = Path(cache)/'metadata-checkouts'/f'pmlr-v{volume}' + checkout.parent.mkdir(parents=True,exist_ok=True) + env = dict(os.environ,GIT_TERMINAL_PROMPT='0') + def git(*args): + result = subprocess.run(['git','-c','credential.helper=',*args],env=env,capture_output=True,text=True,timeout=180) + if result.returncode: + raise RuntimeError('Public metadata Git operation failed: '+result.stderr[-500:]) + return result.stdout + if not checkout.exists(): + git('clone','--filter=blob:none','--depth','1','--single-branch','--branch','gh-pages','--no-checkout',f'https://github.com/mlresearch/v{volume}.git',str(checkout)) + git('-C',str(checkout),'sparse-checkout','set','--no-cone','/_posts/','/_config.yml') + git('-C',str(checkout),'checkout','gh-pages') + else: + if git('-C',str(checkout),'status','--porcelain').strip(): + raise RuntimeError('Metadata checkout has local changes; refusing to overwrite') + git('-C',str(checkout),'fetch','--filter=blob:none','--depth','1','origin','gh-pages') + git('-C',str(checkout),'checkout','--detach','FETCH_HEAD') + posts = checkout/'_posts' + for path in sorted(posts.rglob('*')): + if path.is_file() and path.suffix in ('.md','.markdown','.html'): + if path.stat().st_size>8*1024**2: + raise RuntimeError('Unexpectedly large metadata member') + yield str(path.relative_to(checkout)),path.read_text(encoding='utf-8-sig') + + +def collect_volume(volume,cache,out,allowed=None): + config_url = f'https://raw.githubusercontent.com/mlresearch/v{volume}/gh-pages/_config.yml' + config = load_metadata(fetch(config_url,cache)) + venue = venue_key(config.get('shortname','')) + if allowed and venue not in allowed: + return {'source':'pmlr','volume':volume,'status':'excluded_by_registry','raw_shortname':venue,'booktitle':str(config.get('booktitle',''))} + year_value = config.get('year') + if not year_value: + year_value = str(config.get('start',''))[:4] + if not re.fullmatch(r'\d{4}',str(year_value)): + raise ValueError('No explicit year or conference start date') + year = int(year_value) + title = str(config.get('booktitle','')) + track = 'workshop' if 'workshop' in title.lower() else 'main' + path = Path(out)/f'pmlr-v{volume}.jsonl' + path.parent.mkdir(parents=True,exist_ok=True) + part = path.with_suffix('.part') + records = {} + duplicate_posts = 0 + with part.open('w') as output: + for member_name,content in metadata_texts(volume,cache): + parts = re.split(r'(?m)^---\s*$',content,maxsplit=2) + if len(parts)<3 or parts[0].strip(): + raise RuntimeError('Missing YAML frontmatter: '+member_name) + row = load_metadata(parts[1]) + if row.get('layout') not in ('inproceedings','article'): + continue + if not row.get('title'): + raise ValueError('Paper metadata has no title: '+member_name) + if not row.get('id'): + raise ValueError('Paper metadata has no publication ID: '+member_name) + external = f"v{volume}/{row['id']}" + ids = {} + for key in ('doi','openreview','arxiv'): + value = row.get(key) + if value: + value = str(value) + if key=='arxiv': + value = re.sub(r'^https?://arxiv.org/(abs|pdf)/','',value).removesuffix('.pdf') + if key=='openreview': + value = value.split('id=')[-1].split('&')[0] + ids[key] = value + record = {'source':'pmlr','external_id':external,'url':f"https://proceedings.mlr.press/v{volume}/{row.get('id')}.html",'title':row['title'],'abstract':row.get('abstract'),'authors':row.get('author',[]),'identifiers':ids,'venue':venue,'year':year,'track':track,'status':'accepted','list_url':f'https://proceedings.mlr.press/v{volume}/','metadata':{key:row[key] for key in ('id','volume','date','page','doi','openreview','arxiv','publisher') if key in row}} + version={'path':member_name,'date':row.get('date')} + if external in records: + previous=records[external] + if ({k:v for k,v in previous.items() if k!='metadata'} != {k:v for k,v in record.items() if k!='metadata'} + or {k:v for k,v in previous['metadata'].items() if k not in ('date','source_versions')} != {k:v for k,v in record['metadata'].items() if k!='date'}): + raise ValueError('Conflicting metadata posts share a publication ID: '+external) + previous['metadata']['source_versions'].append(version) + if str(row.get('date') or '')>str(previous['metadata'].get('date') or ''): + previous['metadata']['date']=row['date'] + duplicate_posts+=1 + continue + record['metadata']['source_versions']=[version] + records[external]=record + for record in records.values(): + output.write(json.dumps(record,ensure_ascii=False,default=str)+'\n') + count=len(records) + missing_abstracts=sum(not (record.get('abstract') or '').strip() for record in records.values()) + if count==0: + part.unlink(missing_ok=True) + raise RuntimeError('No papers parsed; refusing an empty successful collection') + part.replace(path) + result = {'source':'pmlr','parser_version':2,'venue':venue,'year':year,'track':track,'volume':volume,'count':count,'metadata_post_count':count+duplicate_posts,'duplicate_post_count':duplicate_posts,'missing_abstract_count':missing_abstracts,'list_url':f'https://proceedings.mlr.press/v{volume}/','file':str(path),'status':'partial' if missing_abstracts else 'collected'} + path.with_suffix('.manifest.json').write_text(json.dumps(result,indent=2)) + print(json.dumps(result),flush=True) + return result + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--cache',required=True) + parser.add_argument('--out',required=True) + parser.add_argument('--volume',type=int,action='append') + parser.add_argument('--registry',default=str(Path(__file__).with_name('venues.json'))) + args = parser.parse_args() + allowed = {venue_key(v['acronym']) for v in json.loads(Path(args.registry).read_text())['venues']} + volumes = args.volume + if not volumes: + index = fetch('https://proceedings.mlr.press/',args.cache).decode() + volumes = sorted(set(int(v) for v in re.findall(r'href=["\'](?:https?://proceedings\.mlr\.press/|\./|/)?v(\d+)/?["\']',index))) + if not volumes: + raise RuntimeError('PMLR index yielded no volumes') + Path(args.out).mkdir(parents=True,exist_ok=True) + catalogue_path=Path(args.out)/'pmlr-volume-status.json' + catalogue=json.loads(catalogue_path.read_text()) if catalogue_path.exists() else {} + errors = [] + for volume in volumes: + try: + result=collect_volume(volume,args.cache,args.out,None if args.volume else allowed) + result['selection_mode']='explicit' if args.volume else 'registry' + result['selected_by_registry']=result.get('venue',result.get('raw_shortname')) in allowed + catalogue[str(volume)]=result + except Exception as exc: + error = {'volume':volume,'status':'blocked','error':str(exc)} + errors.append(error) + catalogue[str(volume)]=error + print(json.dumps(error),flush=True) + part=catalogue_path.with_suffix('.part') + part.write_text(json.dumps(catalogue,indent=2)) + part.replace(catalogue_path) + (Path(args.out)/'pmlr-errors.json').write_text(json.dumps(errors,indent=2)) + (Path(args.out)/'pmlr-last-run.json').write_text(json.dumps({'requested_volumes':volumes,'selection_mode':'explicit' if args.volume else 'registry','errors':errors},indent=2)) + if errors: + raise SystemExit(1) + + +if __name__=='__main__': + main() diff --git a/collect_proceedings.py b/collect_proceedings.py new file mode 100644 index 0000000..6c19bf3 --- /dev/null +++ b/collect_proceedings.py @@ -0,0 +1,216 @@ +"""Collect COLM/NeurIPS official abstract pages without downloading PDFs.""" +import argparse +import json +import re +import time +from collections import defaultdict +from html.parser import HTMLParser +from pathlib import Path +from urllib.parse import urljoin,urlsplit + +from collect_pmlr import fetch + +ROOTS={'colm':'https://proceedings.colm.cc/','neurips':'https://proceedings.neurips.cc/'} + + +class Node: + def __init__(self,tag='',attrs=None): + self.tag=tag + self.attrs={key:value or '' for key,value in attrs or []} + self.children=[] + + def text(self): + if self.tag in ('script','style'): + return '' + content=''.join(child.text() if isinstance(child,Node) else child for child in self.children) + return ' '+content+' ' if self.tag in ('p','div','section','li','br') else content + + def walk(self): + yield self + for child in self.children: + if isinstance(child,Node): + yield from child.walk() + + +class Document(HTMLParser): + VOID={'area','base','br','col','embed','hr','img','input','link','meta','param','source','track','wbr'} + + def __init__(self,html): + super().__init__(convert_charrefs=True) + self.root=Node() + self.stack=[self.root] + self.feed(html) + + def handle_starttag(self,tag,attrs): + node=Node(tag,attrs) + self.stack[-1].children.append(node) + if tag not in self.VOID: + self.stack.append(node) + + def handle_endtag(self,tag): + for index in range(len(self.stack)-1,0,-1): + if self.stack[index].tag==tag: + del self.stack[index:] + break + + def handle_data(self,data): + self.stack[-1].children.append(data) + + +def clean(text): + return ' '.join(text.split()) + + +def track_key(text): + text=text.lower() + if 'dataset' in text or 'benchmark' in text: + return 'datasets_benchmarks' + if 'creative' in text: + return 'creative_ai' + if 'position' in text: + return 'position' + return 'main' + + +def paper_links(html,index_url): + doc=Document(html) + links=sorted({urljoin(index_url,n.attrs['href']) for n in doc.root.walk() if n.tag=='a' and '-Abstract' in n.attrs.get('href','') and urlsplit(urljoin(index_url,n.attrs['href'])).path.endswith('.html') and urlsplit(urljoin(index_url,n.attrs['href'])).netloc==urlsplit(index_url).netloc}) + declared=next((match for n in doc.root.walk() if n.tag in ('p','span','div') and not any(child.tag=='a' for child in n.walk()) and (match:=re.fullmatch(r'([\d,]+)\s+papers',clean(n.text()),re.I))),None) + if not links: + raise ValueError('Official index contains no abstract-page links') + if declared and int(declared[1].replace(',',''))!=len(links): + raise ValueError('Declared paper count differs from unique abstract-page links') + return links + + +def parse_paper(html,url,source,year,index_url): + nodes=list(Document(html).root.walk()) + meta=defaultdict(list) + for node in nodes: + if node.tag=='meta': + meta[node.attrs.get('name','').lower()].append(node.attrs.get('content','')) + title=next(iter(meta['citation_title']),None) + if not title: + title=next((clean(n.text()) for n in nodes if n.tag=='h1'),None) + abstract=next((clean(n.text()) for n in nodes if 'paper-abstract' in n.attrs.get('class','').split()),None) + if not abstract: + for index,node in enumerate(nodes): + if node.tag in ('h2','h3','h4','h5') and clean(node.text()).lower()=='abstract': + following=[] + covered=set() + for candidate in nodes[index+1:]: + if candidate.tag in ('h1','h2','h3','h4','h5'): + break + if candidate.tag=='p' and candidate not in covered: + following.append(clean(candidate.text())) + covered.update(candidate.walk()) + abstract=' '.join(following).strip() or None + break + if not title: + raise ValueError('Missing title on '+url) + # Never fall back to footer/policy text when the abstract is absent. + authors=[] + for author in meta['citation_author']: + if ',' in author: + family,given=author.split(',',1) + author=given.strip()+' '+family.strip() + authors.append(clean(author)) + track_text=' '.join(clean(n.text()) for n in nodes if 'paper-track' in n.attrs.get('class','').split()) + track=track_key(track_text or url) + identifiers={} + if meta['citation_doi']: + identifiers['doi']=meta['citation_doi'][0] + action_nodes=[] + for container in nodes: + if 'paper-actions' in container.attrs.get('class','').split(): + action_nodes.extend(container.walk()) + # Abstract links can cite other papers; only labelled self links identify this one. + for node in action_nodes: + if node.tag!='a': + continue + link=urljoin(url,node.attrs.get('href','')) + parsed=urlsplit(link) + if parsed.netloc in ('arxiv.org','www.arxiv.org') and re.match(r'^/(abs|pdf)/',parsed.path): + identifiers['arxiv']=re.sub(r'^/(abs|pdf)/','',parsed.path).removesuffix('.pdf') + if parsed.netloc=='openreview.net' and parsed.path=='/forum': + from urllib.parse import parse_qs + forum=parse_qs(parsed.query).get('id') + if forum: identifiers['openreview']=forum[0] + venue='COLM' if source=='colm' else 'NeurIPS' + return {'source':source,'external_id':urlsplit(url).path,'url':url,'title':clean(title),'abstract':abstract,'authors':authors,'identifiers':identifiers,'venue':venue,'year':year,'track':track,'status':'accepted','raw_decision':'listed in official proceedings','list_url':index_url,'metadata':{'citation':dict(meta),'publication_state':'pre-proceedings' if 'pre-proceedings' in html.lower() else 'proceedings'}} + + +def discover(source,cache): + root=ROOTS[source] + html=fetch(root,cache,8*1024**2,delay=1.0).decode() + result={} + for node in Document(html).root.walk(): + if node.tag!='a':continue + url=urljoin(root,node.attrs.get('href','')) + parsed=urlsplit(url) + match=re.search(r'/paper(?:_files/paper)?/((?:19|20)\d{2})(?:/|$)',parsed.path) + if parsed.netloc==urlsplit(root).netloc and match and not any(term in parsed.path for term in ('/hash/','/file/','.pdf','-Abstract')): + result[url]=int(match[1]) + if source=='neurips' and parsed.netloc=='datasets-benchmarks-proceedings.neurips.cc' and match: + result[url]=int(match[1]) + if not result: + raise ValueError('No official year indexes found') + return sorted(result.items(),key=lambda entry:(entry[1],entry[0])) + + +def collect_index(source,index_url,year,cache,out): + html=fetch(index_url,cache,16*1024**2,delay=1.0).decode() + links=paper_links(html,index_url) + grouped=defaultdict(list) + missing=[] + for number,url in enumerate(links,1): + record=parse_paper(fetch(url,cache,2*1024**2,delay=1.0).decode(),url,source,year,index_url) + if not record['abstract']:missing.append(url) + grouped[record['track']].append(record) + if number%25==0: + print(json.dumps({'source':source,'year':year,'collected':number,'expected':len(links)}),flush=True) + results=[] + for track,records in grouped.items(): + # Distinct official index URLs may publish multiple volumes in one year. + import hashlib + suffix=hashlib.sha256(index_url.encode()).hexdigest()[:10] + path=Path(out)/f'{source}-{year}-{track}-{suffix}.jsonl' + path.parent.mkdir(parents=True,exist_ok=True) + part=path.with_suffix('.part') + with part.open('w') as output: + for record in records:output.write(json.dumps(record,ensure_ascii=False)+'\n') + part.replace(path) + missing_count=sum(not r['abstract'] for r in records) + result={'source':source,'parser_version':2,'venue':records[0]['venue'],'year':year,'track':track,'count':len(records),'index_link_count':len(links),'index_url':index_url,'list_url':index_url,'missing_abstract_count':missing_count,'file':str(path),'status':'collected' if not missing_count else 'partial'} + path.with_suffix('.manifest.json').write_text(json.dumps(result,indent=2)) + results.append(result) + print(json.dumps(result),flush=True) + return results + + +if __name__=='__main__': + parser=argparse.ArgumentParser() + parser.add_argument('--source',choices=ROOTS,required=True) + parser.add_argument('--cache',required=True) + parser.add_argument('--out',required=True) + parser.add_argument('--year',type=int,action='append') + args=parser.parse_args() + out=Path(args.out);out.mkdir(parents=True,exist_ok=True) + status_path=out/f'{args.source}-index-status.json' + catalogue=json.loads(status_path.read_text()) if status_path.exists() else {} + indexes=discover(args.source,args.cache) + if args.year and set(args.year)-{year for _,year in indexes}: + parser.error('Requested year is absent from the current official proceedings index') + requested=[] + for url,year in indexes: + if args.year and year not in args.year:continue + requested.append(url) + try: + files=collect_index(args.source,url,year,args.cache,args.out) + catalogue[url]={'status':'partial' if any(f['status']=='partial' for f in files) else 'collected','files':files} + except Exception as exc: + catalogue[url]={'status':'blocked','error':str(exc),'year':year} + print(json.dumps(catalogue[url]),flush=True) + part=status_path.with_suffix('.part');part.write_text(json.dumps(catalogue,indent=2));part.replace(status_path) + (out/f'{args.source}-last-run.json').write_text(json.dumps({'requested_indexes':requested,'year_filter':args.year},indent=2)) + if any(catalogue[url]['status']=='blocked' for url in requested):raise SystemExit(1) diff --git a/collect_vision.py b/collect_vision.py new file mode 100644 index 0000000..9709222 --- /dev/null +++ b/collect_vision.py @@ -0,0 +1,189 @@ +"""Collect CVF/ECVA public titles, abstracts and bibliographic identifiers.""" +import argparse +from collections import defaultdict +import json +from pathlib import Path +import re +import time +from urllib.parse import parse_qs, unquote, urljoin, urlsplit + +from collect_pmlr import fetch +from collect_proceedings import Document, Node, clean + +ROOTS={'cvf':'https://openaccess.thecvf.com/','ecva':'https://www.ecva.net/papers.php'} +HOSTS={'openaccess.thecvf.com','www.ecva.net','ecva.net'} + + +def decode_html(data): + try: + return data.decode('utf-8-sig') + except UnicodeDecodeError: + charset=re.search(br'charset\s*=\s*["\x27]?([\w-]+)',data[:4096],re.I) + if not charset: + raise + return data.decode(charset[1].decode('ascii')) + + +def publication(url): + parsed=urlsplit(url) + match=re.search(r'(CVPR|ICCV|ECCV)[_/]?(20\d{2})',parsed.path,re.I) + if parsed.netloc in HOSTS and '/html/' in parsed.path and parsed.path.endswith(('.html','.php')) and match: + return match[1].upper(),int(match[2]) + + +def doi_link(url): + parsed=urlsplit(url) + value=None + if parsed.netloc in ('doi.org','dx.doi.org'): + value=unquote(parsed.path.lstrip('/')) + elif parsed.netloc=='link.springer.com' and parsed.path.startswith('/chapter/'): + value=unquote(parsed.path.removeprefix('/chapter/')) + return value.lower() if value and re.fullmatch(r'10\.\d{4,9}/\S+',value) else None + + +def index_entries(html,index_url): + records={} + raw_count=0 + for dl in (n for n in Document(html).root.walk() if n.tag=='dl'): + current=None + for node in (n for n in dl.children if isinstance(n,Node)): + if node.tag=='dt': + current=None + links=[urljoin(index_url,n.attrs['href']) for n in node.walk() if n.tag=='a' and n.attrs.get('href') and publication(urljoin(index_url,n.attrs['href']))] + if not links: + continue + if len(set(links))!=1: + raise ValueError('Ambiguous publication links in one index title') + url=links[0];raw_count+=1 + if url in records and records[url]['index_title']!=clean(node.text()): + raise ValueError('Different index titles share a publication URL') + current=records.setdefault(url,{'url':url,'index_title':clean(node.text()),'index_authors':None,'doi':None}) + elif node.tag=='dd' and current is not None: + text=clean(node.text()) + # Author rows precede download/bibliography rows. + if current['index_authors'] is None and text and not text.startswith('['): + current['index_authors']=text + for link in (n for n in node.walk() if n.tag=='a' and clean(n.text()).casefold()=='doi'): + value=doi_link(urljoin(index_url,link.attrs.get('href',''))) + if value: + if current['doi'] and current['doi']!=value: + raise ValueError('Conflicting DOI links in one index entry') + current['doi']=value + if not records: + raise ValueError('No official paper entries found') + all_links={urljoin(index_url,n.attrs['href']) for n in Document(html).root.walk() if n.tag=='a' and n.attrs.get('href') and publication(urljoin(index_url,n.attrs['href']))} + if all_links!=set(records):raise ValueError('Paper links exist outside parsed title entries') + return list(records.values()),raw_count + + +def discover(source,cache): + root=ROOTS[source] + html=decode_html(fetch(root,cache,32*1024**2,delay=1)) + if source=='ecva': + entries,_=index_entries(html,root) + groups=defaultdict(list) + for entry in entries: + venue,year=publication(entry['url']) + if venue=='ECCV':groups[(venue,year)].append(entry) + return [{'venue':venue,'year':year,'track':'main','index_url':root,'entries':entries} + for (venue,year),entries in sorted(groups.items(),key=lambda item:item[0][1])] + indexes={} + for node in Document(html).root.walk(): + if node.tag!='a':continue + url=urljoin(root,node.attrs.get('href',''));parsed=urlsplit(url) + match=re.fullmatch(r'/(CVPR|ICCV)(20\d{2})(_findings)?(?:\.py)?/?',parsed.path) + if parsed.netloc=='openaccess.thecvf.com' and match: + indexes[url]={'venue':match[1],'year':int(match[2]),'track':'findings' if match[3] else 'main','index_url':url} + if not indexes:raise ValueError('No official CVPR/ICCV indexes found') + return sorted(indexes.values(),key=lambda row:(row['year'],row['venue'],row['track']!='main',row['track'])) + + +def parse_paper(html,entry,source,venue,year,track,index_url): + nodes=list(Document(html).root.walk());meta=defaultdict(list) + for node in nodes: + if node.tag=='meta':meta[node.attrs.get('name','').lower()].append(node.attrs.get('content','')) + title=next(iter(meta['citation_title']),None) or next((clean(n.text()) for n in nodes if n.attrs.get('id')=='papertitle'),None) + if not title:raise ValueError('Missing official paper title: '+entry['url']) + abstract=next((clean(n.text()) or None for n in nodes if n.attrs.get('id')=='abstract'),None) + authors=[] + for author in meta['citation_author']: + if ',' in author: + family,given=author.split(',',1);author=given.strip()+' '+family.strip() + if clean(author):authors.append(clean(author)) + if not authors: + raw=next((clean(n.text()) for n in nodes if n.attrs.get('id')=='authors'),'') or entry.get('index_authors') or '' + raw=raw.split(';',1)[0] + if ' and ' in raw and all(',' in name for name in raw.split(' and ')): + names=[' '.join(reversed([part.strip() for part in name.split(',',1)])) for name in raw.split(' and ')] + else:names=raw.split(',') + authors=[clean(name).removesuffix('*').strip() for name in names if clean(name).removesuffix('*').strip()] + dois={value for doi in meta['citation_doi'] if doi and (value:=doi_link(doi) or doi.strip().lower()) and re.fullmatch(r'10\.\d{4,9}/\S+',value)} + if entry.get('doi'):dois.add(entry['doi']) + if len(dois)>1:raise ValueError('Citation DOI differs from the official index') + identifiers={'doi':dois.pop()} if dois else {} + # Abstract/reference links are never interpreted as this paper's identifiers. + return {'source':source,'external_id':urlsplit(entry['url']).path,'url':entry['url'], + 'title':clean(title),'abstract':abstract,'authors':authors,'identifiers':identifiers, + 'venue':venue,'year':year,'track':track,'status':'accepted', + 'raw_decision':'listed in official proceedings','list_url':index_url, + 'metadata':{'citation':dict(meta),'index_title':entry['index_title'],'index_authors':entry.get('index_authors')}} + + +def collect_index(source,index,cache,out): + url=index['index_url'];venue=index['venue'];year=index['year'];track=index['track'] + entries=index.get('entries') + raw_count=len(entries) if entries is not None else None + if entries is None: + html=decode_html(fetch(url,cache,32*1024**2,delay=1)) + # Some defaults contain only day one; prefer the explicitly linked all-papers page. + all_pages={urljoin(url,n.attrs.get('href','')) for n in Document(html).root.walk() if n.tag=='a' and parse_qs(urlsplit(urljoin(url,n.attrs.get('href',''))).query).get('day')==['all'] and urlsplit(urljoin(url,n.attrs.get('href',''))).netloc==urlsplit(url).netloc and urlsplit(urljoin(url,n.attrs.get('href',''))).path==urlsplit(url).path} + if len(all_pages)>1:raise ValueError('Ambiguous all-papers index') + if all_pages: + url=all_pages.pop();html=decode_html(fetch(url,cache,32*1024**2,delay=1)) + entries,raw_count=index_entries(html,url) + records=[] + for number,entry in enumerate(entries,1): + if publication(entry['url'])!=(venue,year):raise ValueError('Publication path differs from requested edition') + record=parse_paper(decode_html(fetch(entry['url'],cache,2*1024**2,delay=1)),entry,source,venue,year,track,url) + records.append(record) + if number%25==0:print(json.dumps({'source':source,'venue':venue,'year':year,'track':track,'collected':number,'expected':len(entries)}),flush=True) + if not records:raise ValueError('Empty publication list') + path=Path(out)/f'{source}-{venue}{year}-{track}.jsonl';path.parent.mkdir(parents=True,exist_ok=True) + part=path.with_suffix('.part');part.write_text(''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in records));part.replace(path) + missing=sum(not r['abstract'] for r in records);missing_authors=sum(not r['authors'] for r in records) + manifest={'source':source,'parser_version':1,'venue':venue,'year':year,'track':track, + 'count':len(records),'index_link_count':len(entries),'raw_index_entry_count':raw_count, + 'missing_abstract_count':missing,'missing_authors_count':missing_authors,'file':str(path), + 'list_url':url,'status':'partial' if missing or missing_authors else 'collected'} + path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2));print(json.dumps(manifest),flush=True) + return manifest + + +def main(): + parser=argparse.ArgumentParser();parser.add_argument('--source',choices=ROOTS,required=True) + parser.add_argument('--cache',required=True);parser.add_argument('--out',required=True) + parser.add_argument('--year',type=int,action='append');parser.add_argument('--venue',choices=['CVPR','ICCV','ECCV'],action='append') + args=parser.parse_args();out=Path(args.out);out.mkdir(parents=True,exist_ok=True) + indexes=discover(args.source,args.cache) + if args.year and set(args.year)-{r['year'] for r in indexes}:parser.error('Requested year is absent from the official source') + if args.venue and set(args.venue)-{r['venue'] for r in indexes}:parser.error('Requested venue is absent from this source') + if not any((not args.year or r['year'] in args.year) and (not args.venue or r['venue'] in args.venue) for r in indexes):parser.error('Requested venue/year combination is absent from the official source') + path=out/f'{args.source}-index-status.json';catalogue=json.loads(path.read_text()) if path.exists() else {};requested=[] + (out/f'{args.source}-discovery.json').write_text(json.dumps({'source':args.source,'root_url':ROOTS[args.source], + 'discovered_at':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()),'indexes':[ + {**{k:index[k] for k in ('venue','year','track','index_url')}, + 'index_link_count':len(index['entries']) if 'entries' in index else None} for index in indexes]},indent=2)) + for index in indexes: + if args.year and index['year'] not in args.year or args.venue and index['venue'] not in args.venue:continue + key=f'{index["venue"]}:{index["year"]}:{index["track"]}';requested.append(key) + try: + manifest=collect_index(args.source,index,args.cache,args.out);catalogue[key]={'status':manifest['status'],'files':[manifest]} + except Exception as exc: + catalogue[key]={'status':'blocked','year':index['year'],'venue':index['venue'],'track':index['track'],'index_url':index['index_url'],'error':str(exc)} + print(json.dumps(catalogue[key]),flush=True) + part=path.with_suffix('.part');part.write_text(json.dumps(catalogue,indent=2));part.replace(path) + (out/f'{args.source}-last-run.json').write_text(json.dumps({'requested_indexes':requested,'year_filter':args.year,'venue_filter':args.venue},indent=2)) + if any(catalogue[key]['status']=='blocked' for key in requested):raise SystemExit(1) + + +if __name__=='__main__':main() diff --git a/coverage_report.py b/coverage_report.py new file mode 100644 index 0000000..11cdbac --- /dev/null +++ b/coverage_report.py @@ -0,0 +1,41 @@ +"""Show every target venue, including those with no imported records.""" +import argparse +import json +from pathlib import Path +import sqlite3 + +from submission_evidence import timestamp + + +def report(db,registry): + registry=json.loads(Path(registry).read_text());targets=registry['venues'] + names=[row['acronym'].casefold() for row in targets] + if len(names)!=len(set(names)):raise ValueError('Duplicate target venue') + scope='(v.acronym=? COLLATE NOCASE OR EXISTS(SELECT 1 FROM venue_aliases a WHERE a.venue_id=v.id AND a.alias=? COLLATE NOCASE))' + c=sqlite3.connect(Path(db).resolve().as_uri()+'?mode=ro',uri=True);c.row_factory=sqlite3.Row + try: + c.execute('BEGIN');venues=[] + for target in targets: + acronym=target['acronym'];params=(acronym,acronym) + editions=[] + for row in c.execute('SELECT e.id,v.acronym AS canonical_venue,e.year,e.track,count(s.id) AS submissions,count(DISTINCT s.paper_id) AS papers FROM editions e JOIN venues v ON v.id=e.venue_id LEFT JOIN submissions s ON s.edition_id=e.id WHERE '+scope+' GROUP BY e.id ORDER BY e.year,e.track',params): + edition=dict(row) + edition['sources']=[dict(source) for source in c.execute('SELECT so.name,es.status,es.expected_count,es.imported_count,es.list_url,es.last_success_at,es.last_error FROM edition_sources es JOIN sources so ON so.id=es.source_id WHERE es.edition_id=? ORDER BY so.name',(row['id'],))] + editions.append(edition) + totals=c.execute('SELECT count(s.id),count(DISTINCT s.paper_id) FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id WHERE '+scope,params).fetchone() + venues.append({'acronym':acronym,'field':target['field'],'imported_submissions':totals[0],'distinct_papers':totals[1], + 'history_status':'partial_history_pending_independent_audit' if totals[0] else 'no_imported_records', + 'discovery_sources':target.get('discovery_sources',[]),'editions':editions}) + return {'checked_at':timestamp(),'target_venue_count':len(targets),'venues_with_imported_records':sum(bool(v['imported_submissions']) for v in venues), + 'scope_note':registry['scope_note'],'interpretation':'All registry targets are listed. Edition source complete means that source file passed import checks; it does not prove full historical or abstract coverage. Papers can appear in multiple venues/years, so do not sum venue paper counts as a global total.', + 'venues':venues} + finally:c.close() + + +if __name__=='__main__': + parser=argparse.ArgumentParser();parser.add_argument('--db',required=True) + parser.add_argument('--registry',default=str(Path(__file__).with_name('venues.json')));parser.add_argument('--out') + args=parser.parse_args();result=report(args.db,args.registry);text=json.dumps(result,ensure_ascii=False,indent=2) + if args.out: + path=Path(args.out);stage=path.with_suffix('.part');stage.write_text(text+'\n');stage.replace(path) + else:print(text) diff --git a/download_snapshot.py b/download_snapshot.py new file mode 100644 index 0000000..4f40848 --- /dev/null +++ b/download_snapshot.py @@ -0,0 +1,106 @@ +"""Checksum-verified streaming downloads; existing valid snapshots stay intact.""" +import argparse +import fcntl +import hashlib +import json +from pathlib import Path +import re +import shutil +import time +from urllib.parse import urlsplit +import urllib.request + + +def digest_file(path): + sha=hashlib.sha256();md5=hashlib.md5(usedforsecurity=False);size=0 + with path.open('rb') as stream: + while block:=stream.read(1024**2): + sha.update(block);md5.update(block);size+=len(block) + return {'sha256':sha.hexdigest(),'md5':md5.hexdigest(),'bytes':size} + + +def verify(actual,sha256,md5,expected_bytes): + if expected_bytes is not None and actual['bytes']!=expected_bytes: + raise ValueError('Snapshot size differs from trusted expected size') + if sha256 and actual['sha256']!=sha256: + raise ValueError('Snapshot SHA256 mismatch; partial file retained') + if md5 and actual['md5']!=md5: + raise ValueError('Snapshot MD5 mismatch; partial file retained') + + +def write_manifest(output,url,actual): + manifest={'status':'verified','file':str(output),'url':url,**actual, + 'verified_at':time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime())} + metadata=Path(str(output)+'.manifest.json');stage=Path(str(metadata)+'.part') + stage.write_text(json.dumps(manifest,indent=2));stage.replace(metadata) + return manifest + + +def download(url,output,sha256=None,md5=None,expected_bytes=None,resume=False,min_free=8*1024**3): + parsed=urlsplit(url) + if parsed.scheme!='https' or not parsed.netloc or parsed.username or parsed.password: + raise ValueError('A public HTTPS URL without embedded credentials is required') + if not (sha256 or md5):raise ValueError('A trusted published checksum is required') + if expected_bytes is not None and expected_bytes<=0:raise ValueError('Expected byte count must be positive') + if min_free<0:raise ValueError('Disk reserve must not be negative') + for name,value,length in [('SHA256',sha256,64),('MD5',md5,32)]: + if value and not re.fullmatch('[0-9a-f]{'+str(length)+'}',value):raise ValueError('Invalid '+name) + output=Path(output);output.parent.mkdir(parents=True,exist_ok=True) + if output.is_symlink():raise ValueError('Refusing a symlink output') + intent={'url':url,'expected_sha256':sha256,'expected_md5':md5,'expected_bytes':expected_bytes} + with Path(str(output)+'.download.lock').open('a') as lock: + fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + if output.exists(): + actual=digest_file(output);verify(actual,sha256,md5,expected_bytes) + if not Path(str(output)+'.manifest.json').exists():write_manifest(output,url,actual) + return {'status':'already_verified','file':str(output),**actual} + part=Path(str(output)+'.part');intent_path=Path(str(part)+'.json') + offset=0 + if part.exists(): + if not resume or not intent_path.is_file() or json.loads(intent_path.read_text())!=intent: + raise ValueError('Existing partial requires --resume and the same snapshot identity') + offset=part.stat().st_size + else: + intent_path.write_text(json.dumps(intent,indent=2)) + remaining=max(0,(expected_bytes or 0)-offset) + if shutil.disk_usage(output.parent).freetotal:raise ValueError('HTTP response exceeds advertised size') + if written-reported>=64*1024**2: + print(json.dumps({'status':'downloading','file':str(output),'bytes':written,'expected_bytes':total}),flush=True);reported=written + if shutil.disk_usage(output.parent).free= 20: + print(json.dumps({'run':run,'rows':line_number}),flush=True) + last = time.monotonic() + conn.execute("UPDATE ingest_runs SET status='complete',finished_at=CURRENT_TIMESTAMP,records_seen=?,records_written=?,checkpoint_json=? WHERE id=?",(line_number,line_number,json.dumps({'archive':fingerprint,'line':line_number}),run)) + conn.commit() + print(json.dumps({'status':'complete','rows':line_number}),flush=True) + except BaseException as exc: + conn.rollback() + conn.execute("UPDATE ingest_runs SET status='failed',error=? WHERE id=?",(str(exc),run)) + conn.commit() + raise + finally: + conn.close() + + +if __name__ == '__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--db', required=True) + parser.add_argument('--archive', required=True) + args = parser.parse_args() + ingest(args.db,args.archive) diff --git a/import_records.py b/import_records.py new file mode 100644 index 0000000..fd5c61d --- /dev/null +++ b/import_records.py @@ -0,0 +1,93 @@ +"""Import normalized conference JSONL files with coverage evidence.""" +import argparse +import json +import time +from pathlib import Path + +from index_core import Store +from submission_evidence import identities_for_row,timestamp + + +def ingest_file(db,path): + path = Path(path) + manifest_path = path.with_suffix('.manifest.json') + manifest = json.loads(manifest_path.read_text()) if manifest_path.exists() else {} + if manifest.get('source') in ('colm','neurips') and manifest.get('parser_version',0)<2: + raise ValueError('Recollect proceedings with parser_version>=2 before identity matching') + if manifest.get('source')=='iclr_program' and manifest.get('parser_version',0)<2: + raise ValueError('Recollect ICLR with parser_version>=2 before submission identity matching') + store = Store(db) + c = store.conn + source_name = manifest.get('source') + if not source_name: + raise ValueError('A source manifest is required') + source_id = store.source(source_name,manifest['list_url']) + run = c.execute('INSERT INTO ingest_runs(source_id,checkpoint_json) VALUES(?,?)',(source_id,json.dumps({'file':str(path)}))).lastrowid + c.commit() + count = 0 + missing_abstracts = 0 + seen = set() + editions = set() + try: + with path.open() as stream: + for line in stream: + row = json.loads(line) + if row['source'] != source_name: + raise ValueError('Source differs from manifest') + if any(key in manifest and manifest[key]!=row[key] for key in ('venue','year','track')): + raise ValueError('Conference edition differs from manifest') + external = row['external_id'] + if external in seen: + raise ValueError('Duplicate external ID in one collected file: '+external) + seen.add(external) + paper,record = store.record(source_name,external,row['url'],row['title'],row.get('abstract'),row.get('authors',[]),row.get('identifiers'),row.get('metadata')) + if 'tags' in row: + store.record_tags(paper,record,source_name,row['tags']) + if row.get('status') in ('accepted','rejected','desk_rejected','withdrawn','under_review','unknown'): + identities=identities_for_row(row) + events=row.get('decision_events') + if events is None: + events=[{'status':row['status'],'raw_value':row.get('raw_decision','listed in official proceedings' if source_name=='pmlr' else row['status']),'evidence_url':row.get('evidence_url',row['url'])}] + if not isinstance(events,list):raise ValueError('Decision events must be a list') + if not events and row['status']!='unknown':raise ValueError('Known status requires decision evidence') + if row['status']!='unknown' and not any(event.get('status')==row['status'] for event in events):raise ValueError('Summary status has no corresponding decision evidence') + row_observed=row.get('observed_at') or timestamp() + for event in events or [{'status':None}]: + status=event['status'] + if events and status is None:raise ValueError('Explicit decision event requires a status') + if status is not None and status not in ('accepted','rejected','desk_rejected','withdrawn','under_review','unknown'):raise ValueError('Unsupported decision event') + edition=store.decision(paper,record,row['venue'],row['year'],row['track'],external,event.get('evidence_url',row['url']),status,event.get('raw_value',''),row.get('venue_aliases'),identities=identities,observed_at=event.get('observed_at') or row_observed,decision_date=event.get('decision_date'),source_decision_id=event.get('source_decision_id',''),submitted_at=row.get('submitted_at')) + editions.add(edition) + else: + raise ValueError('This importer requires an explicit supported decision status') + count += 1 + missing_abstracts += not bool((row.get('abstract') or '').strip()) + if not count: + raise ValueError('Empty metadata file') + if len(editions) != 1: + raise ValueError('A manifest must describe exactly one conference edition and track') + edition = next(iter(editions)) + expected = manifest.get('count') + if expected is not None and expected != count: + raise ValueError(f'Manifest count {expected} differs from records {count}') + complete=expected==count and manifest.get('status') not in ('partial','blocked') and not manifest.get('missing_abstract_count',0) and not missing_abstracts + c.execute('INSERT INTO edition_sources(edition_id,source_id,list_url,status,expected_count,imported_count,last_success_at) VALUES(?,?,?,?,?,?,?) ON CONFLICT(edition_id,source_id,list_url) DO UPDATE SET status=excluded.status,expected_count=excluded.expected_count,imported_count=excluded.imported_count,last_success_at=excluded.last_success_at,last_error=NULL',(edition,source_id,manifest['list_url'],'complete' if complete else 'partial',expected,count,time.strftime('%Y-%m-%dT%H:%M:%SZ',time.gmtime()))) + c.execute("UPDATE ingest_runs SET status='complete',finished_at=CURRENT_TIMESTAMP,records_seen=?,records_written=? WHERE id=?",(count,count,run)) + c.commit() + print(json.dumps({'file':str(path),'status':'imported','records':count,'missing_abstract_count':missing_abstracts}),flush=True) + except BaseException as exc: + c.rollback() + c.execute("UPDATE ingest_runs SET status='failed',finished_at=CURRENT_TIMESTAMP,error=? WHERE id=?",(str(exc),run)) + c.commit() + raise + finally: + c.close() + + +if __name__=='__main__': + parser = argparse.ArgumentParser() + parser.add_argument('--db',required=True) + parser.add_argument('files',nargs='+') + args = parser.parse_args() + for filename in args.files: + ingest_file(args.db,filename) diff --git a/index_core.py b/index_core.py new file mode 100644 index 0000000..df7ce7f --- /dev/null +++ b/index_core.py @@ -0,0 +1,155 @@ +"""Shared identity resolution and provenance-preserving metadata storage.""" +import json +import re +import sqlite3 +import unicodedata +from pathlib import Path +from submission_evidence import attach_submission,migrate_sources,write_decision + + +def normalized(value): + return ''.join(c for c in unicodedata.normalize('NFKD',value).casefold() if c.isalnum()) + + +def author_keys(authors): + if not isinstance(authors,list): + return set() + result = set() + for author in authors: + if isinstance(author,list): + author = ' '.join(str(x) for x in author[1:2]+author[:1]) + elif isinstance(author,dict): + author = ' '.join(str(author.get(k,'')) for k in ('given','family')) + key = normalized(str(author)) + if key: + result.add(key) + return result + + +class Store: + def __init__(self,path): + self.conn = sqlite3.connect(path,timeout=60) + self.conn.executescript(Path(__file__).with_name('schema.sql').read_text()) + self.conn.execute('PRAGMA synchronous=NORMAL') + self.conn.execute('PRAGMA cache_size=-32768') + migrate_sources(self.conn) + + def source(self,name,url): + self.conn.execute('INSERT OR IGNORE INTO sources(name,homepage) VALUES(?,?)',(name,url)) + return self.conn.execute('SELECT id FROM sources WHERE name=?',(name,)).fetchone()[0] + + def merge(self,target,other,reason): + c = self.conn + if target == other: + return + columns=('abstract','authors_json','first_published') + target_values=c.execute('SELECT abstract,authors_json,first_published FROM papers WHERE id=?',(target,)).fetchone() + other_values=c.execute('SELECT abstract,authors_json,first_published FROM papers WHERE id=?',(other,)).fetchone() + if not target_values or not other_values:raise ValueError('Missing paper entity during merge') + fill=[(column,value) for column,current,value in zip(columns,target_values,other_values) if current in (None,'','[]') and value not in (None,'','[]')] + if fill:c.execute('UPDATE papers SET '+','.join(column+'=?' for column,_ in fill)+' WHERE id=?',(*[value for _,value in fill],target)) + c.execute('INSERT INTO paper_merges(former_paper_id,surviving_paper_id,reason) VALUES(?,?,?)',(other,target,reason)) + c.execute('UPDATE identifiers SET paper_id=? WHERE paper_id=?',(target,other)) + c.execute('UPDATE source_records SET paper_id=? WHERE paper_id=?',(target,other)) + c.execute('UPDATE submissions SET paper_id=? WHERE paper_id=?',(target,other)) + c.execute('INSERT OR IGNORE INTO paper_tags SELECT ?,tag_id,source_record_id,evidence_key FROM paper_tags WHERE paper_id=?',(target,other)) + c.execute('DELETE FROM paper_tags WHERE paper_id=?',(other,)) + c.execute('UPDATE paper_merges SET surviving_paper_id=? WHERE surviving_paper_id=?',(target,other)) + # Reattach pending candidate evidence without losing colliding rows. + candidates = c.execute('SELECT left_paper_id,right_paper_id,method,score,evidence_json,status FROM match_candidates WHERE left_paper_id=? OR right_paper_id=?',(other,other)).fetchall() + c.execute('DELETE FROM match_candidates WHERE left_paper_id=? OR right_paper_id=?',(other,other)) + for left,right,method,score,evidence,status in candidates: + left = target if left == other else left + right = target if right == other else right + if left != right: + c.execute('INSERT OR IGNORE INTO match_candidates VALUES(?,?,?,?,?,?)',(min(left,right),max(left,right),method,score,evidence,status)) + c.execute('DELETE FROM papers WHERE id=?',(other,)) + + def record(self,source,external_id,url,title,abstract,authors,identifiers=None,metadata=None,published=None): + c = self.conn + identifiers = dict(identifiers or {}) + # External IDs are scoped to platforms; an OpenReview ID is a forum ID. + identifiers[source] = external_id + ids = [] + for scheme,value in identifiers.items(): + if not value: + continue + value = str(value).strip() + if scheme == 'doi': + value = re.sub(r'^https?://(?:dx\.)?doi\.org/','',value,flags=re.I).lower() + if not re.fullmatch(r'10\.\d{4,9}/\S+',value): + continue + if scheme == 'arxiv': + value = re.sub(r'v\d+$','',value) + ids.append((scheme,value)) + matches = set() + for scheme,value in ids: + found = c.execute('SELECT paper_id FROM identifiers WHERE scheme=? AND value=?',(scheme,value)).fetchone() + if found: + matches.add(found[0]) + old = c.execute('SELECT paper_id,title,abstract FROM source_records WHERE source_id=? AND external_id=?',(source_id:=self.source(source,url),external_id)).fetchone() + if old: + matches.add(old[0]) + key = normalized(title) + title_matches = c.execute('SELECT id,authors_json,abstract FROM papers WHERE title_key=?',(key,)).fetchall() + authors_set = author_keys(authors) + for paper,old_authors,old_abstract in title_matches: + overlap = authors_set & author_keys(json.loads(old_authors)) + # Cross-platform title matching needs independent author corroboration. + same_source = c.execute('SELECT 1 FROM source_records WHERE paper_id=? AND source_id=? LIMIT 1',(paper,source_id)).fetchone() + if not same_source and len(key)>=25 and (len(overlap)>=2 or (overlap and abstract and abstract.strip()==(old_abstract or '').strip())): + matches.add(paper) + if matches: + paper = min(matches) + for other in sorted(matches-{paper}): + self.merge(paper,other,'explicit identifier or exact title corroborated by authors') + if old: + c.execute("UPDATE papers SET title=?,abstract=?,title_key=?,authors_json=? WHERE id=? AND title=? AND COALESCE(abstract,'')=COALESCE(?,'')",(title,abstract,key,json.dumps(authors,ensure_ascii=False),paper,old[1],old[2])) + c.execute("UPDATE papers SET abstract=CASE WHEN abstract IS NULL OR abstract='' THEN ? ELSE abstract END WHERE id=?",(abstract,paper)) + else: + paper = c.execute('INSERT INTO papers(title,abstract,title_key,authors_json,first_published) VALUES(?,?,?,?,?)',(title,abstract,key,json.dumps(authors,ensure_ascii=False),published)).lastrowid + for scheme,value in ids: + c.execute('INSERT OR IGNORE INTO identifiers VALUES(?,?,?)',(scheme,value,paper)) + metadata=dict(metadata or {});metadata['record_identifiers']=dict(ids) + c.execute('INSERT INTO source_records(source_id,external_id,paper_id,url,title,abstract,authors_json,metadata_json) VALUES(?,?,?,?,?,?,?,?) ON CONFLICT(source_id,external_id) DO UPDATE SET paper_id=excluded.paper_id,url=excluded.url,title=excluded.title,abstract=excluded.abstract,authors_json=excluded.authors_json,metadata_json=excluded.metadata_json,retrieved_at=CURRENT_TIMESTAMP',(source_id,external_id,paper,url,title,abstract,json.dumps(authors,ensure_ascii=False),json.dumps(metadata,ensure_ascii=False,default=str))) + record = c.execute('SELECT id FROM source_records WHERE source_id=? AND external_id=?',(source_id,external_id)).fetchone()[0] + for other,_,_ in title_matches: + if other != paper and c.execute('SELECT 1 FROM papers WHERE id=?',(other,)).fetchone(): + c.execute('INSERT OR IGNORE INTO match_candidates VALUES(?,?,?,?,?,?)',(min(paper,other),max(paper,other),'exact_title_unconfirmed',None,json.dumps({'title':title}), 'pending')) + return paper,record + + def record_tags(self,paper,record,source,tags): + """Refresh this record's platform tags without removing other evidence.""" + values=set() + for tag in tags: + namespace,value=tag['namespace'],tag['value'] + if not isinstance(namespace,str) or not namespace.startswith(source+':') or not isinstance(value,str) or not value.strip(): + raise ValueError('Platform tag must have its own source namespace and a nonempty text value') + values.add((namespace,value.strip())) + evidence=f'{source}:record:{record}:tags' + c=self.conn + c.execute('DELETE FROM paper_tags WHERE source_record_id=? AND evidence_key=?',(record,evidence)) + for namespace,value in sorted(values): + c.execute('INSERT OR IGNORE INTO tags(namespace,value) VALUES(?,?)',(namespace,value)) + tag=c.execute('SELECT id FROM tags WHERE namespace=? AND value=?',(namespace,value)).fetchone()[0] + c.execute('INSERT OR IGNORE INTO paper_tags VALUES(?,?,?,?)',(paper,tag,record,evidence)) + + def decision(self,paper,record,venue,year,track,external_id,url,status,raw_value,aliases=None,identities=None,observed_at=None,decision_date=None,source_decision_id='',submitted_at=None): + c = self.conn + c.execute("INSERT OR IGNORE INTO venues(acronym,name,field,venue_type) VALUES(?,?,?,'conference')",(venue,venue,'unclassified')) + venue_id = c.execute('SELECT id FROM venues WHERE acronym=?',(venue,)).fetchone()[0] + for alias in aliases or []: + c.execute('INSERT OR IGNORE INTO venue_aliases VALUES(?,?)',(venue_id,alias)) + c.execute('INSERT OR IGNORE INTO editions(venue_id,year,track) VALUES(?,?,?)',(venue_id,int(year),track)) + edition = c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=?',(venue_id,int(year),track)).fetchone()[0] + # A previous event in this row may already have merged the input paper. + paper=c.execute('SELECT paper_id FROM source_records WHERE id=?',(record,)).fetchone()[0] + submission=attach_submission(self,edition,paper,record,external_id,identities or []) + if submitted_at: + from submission_evidence import timestamp + c.execute('UPDATE submissions SET submitted_at=COALESCE(submitted_at,?) WHERE id=?',(timestamp(submitted_at),submission)) + if status is not None:write_decision(self,submission,record,status,raw_value,url,observed_at,decision_date,source_decision_id) + return edition + + def accepted(self,paper,record,venue,year,track,external_id,url): + return self.decision(paper,record,venue,year,track,external_id,url,'accepted','listed in official proceedings') diff --git a/merge_conferences.py b/merge_conferences.py new file mode 100644 index 0000000..0d07a41 --- /dev/null +++ b/merge_conferences.py @@ -0,0 +1,91 @@ +"""One-shot server operation: merge collected files at an arXiv checkpoint. + +Run as a transient systemd job so an SSH disconnect cannot strand the importer. +Only the Paper Atlas arXiv unit is controlled; business services are untouched. +""" +import argparse +from contextlib import closing +import json +import fcntl +import sqlite3 +import subprocess +from pathlib import Path + +from import_records import ingest_file +from index_core import Store +from backfill_submission_evidence import backfill,queue_files + + +UNIT='paper-atlas-arxiv-import.service' + + +def property_value(name): + return subprocess.check_output(['systemctl','show',UNIT,'-p',name,'--value'],text=True).strip() + + +def merge_locked(db,archive,files,backfill_queue=None): + # Validate inputs before stopping the existing writer. + for filename in files: + path=Path(filename) + if not path.is_file() or not path.with_suffix('.manifest.json').is_file(): + raise ValueError('Missing collected file or manifest: '+filename) + evidence_files=queue_files(backfill_queue) if backfill_queue else [] + if not Path(archive).is_file(): + raise ValueError('Missing arXiv archive') + if not Path(db).is_file(): + raise ValueError('Missing existing database') + resume_arxiv=property_value('ActiveState')=='active' + if not resume_arxiv: + with closing(sqlite3.connect(Path(db).resolve().as_uri()+'?mode=ro',uri=True)) as c: + last=c.execute("SELECT status FROM ingest_runs WHERE source_id=(SELECT id FROM sources WHERE name='arxiv') ORDER BY id DESC LIMIT 1").fetchone() + if not last or last[0]!='complete': + raise RuntimeError('arXiv is neither active nor verified complete; inspect its state before maintenance') + try: + if resume_arxiv:subprocess.run(['systemctl','stop',UNIT],check=True) + with closing(sqlite3.connect(db)) as c, c: + checkpoint=c.execute("SELECT records_written,status FROM ingest_runs WHERE source_id=(SELECT id FROM sources WHERE name='arxiv') ORDER BY id DESC LIMIT 1").fetchone() + print('Saved arxiv checkpoint:',checkpoint,flush=True) + if checkpoint and checkpoint[1]=='complete':resume_arxiv=False + schema_store=Store(db) + schema_store.conn.close() + if evidence_files:backfill(db,evidence_files) + for filename in files: + ingest_file(db,filename) + with closing(sqlite3.connect(db)) as c, c: + c.execute('PRAGMA cache_size=-32768') + if files or evidence_files: + errors=c.execute('PRAGMA foreign_key_check').fetchall() + if errors: + raise RuntimeError('Foreign key violations after merge') + else: + print('Tag refresh plan:',c.execute('EXPLAIN QUERY PLAN DELETE FROM paper_tags WHERE source_record_id=? AND evidence_key=?',(1,'arxiv:categories')).fetchall(),flush=True) + print('Decision totals:',c.execute('SELECT status,count(*) FROM decisions GROUP BY status').fetchall(),flush=True) + finally: + if resume_arxiv: + if property_value('LoadState')=='loaded': + subprocess.run(['systemctl','start',UNIT],check=True) + else: + subprocess.run(['systemd-run','--unit='+UNIT,'--property=WorkingDirectory='+str(Path(__file__).parent),'--property=CPUQuota=50%','--property=MemoryMax=512M','--property=Nice=15','--property=IOWeight=10','--property=StandardOutput=append:/data/paper-index/arxiv-import.log','--property=StandardError=append:/data/paper-index/arxiv-import.log','/usr/bin/env','PYTHONDONTWRITEBYTECODE=1','python3','-B',str(Path(__file__).with_name('import_arxiv.py')),'--db',db,'--archive',archive],check=True) + if property_value('ActiveState')!='active': + raise RuntimeError('arXiv unit did not resume; inspect the logs') + print('ARXIV_RESUMED',flush=True) + else:print('ARXIV_ALREADY_COMPLETE',flush=True) + + +def merge(db,archive,files,backfill_queue=None): + with Path(str(db)+'.maintenance.lock').open('a') as lock: + fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB) + return merge_locked(db,archive,files,backfill_queue) + + +if __name__=='__main__': + parser=argparse.ArgumentParser() + parser.add_argument('--db',required=True) + parser.add_argument('--archive',required=True) + parser.add_argument('--schema-only',action='store_true',help='Apply idempotent schema/index updates at a saved checkpoint') + parser.add_argument('--backfill-queue',help='Audited existing-source queue for canonical submission evidence') + parser.add_argument('files',nargs='*') + args=parser.parse_args() + if not args.files and not args.schema_only and not args.backfill_queue: + parser.error('Provide collected files or --schema-only') + merge(args.db,args.archive,args.files,args.backfill_queue) diff --git a/metadata_checkout.py b/metadata_checkout.py new file mode 100644 index 0000000..4d19294 --- /dev/null +++ b/metadata_checkout.py @@ -0,0 +1,29 @@ +"""Public metadata-only Git checkouts, without credentials or project hooks.""" +import os +import subprocess +import tempfile +from pathlib import Path + + +def checkout_metadata(url,destination,patterns,branch='master'): + destination=Path(destination) + destination.parent.mkdir(parents=True,exist_ok=True) + env=dict(os.environ,GIT_TERMINAL_PROMPT='0',GIT_LFS_SKIP_SMUDGE='1') + def git(*args): + result=subprocess.run(['git','-c','credential.helper=','-c','http.extraHeader=','-c','core.hooksPath=/dev/null','-c','core.deltaBaseCacheLimit=32m',*args],env=env,capture_output=True,text=True,timeout=900) + if result.returncode: + raise RuntimeError('Metadata checkout failed: '+result.stderr[-500:]) + return result.stdout.strip() + if not destination.exists(): + with tempfile.TemporaryDirectory(prefix='metadata-clone-',dir=destination.parent) as tmp: + stage=Path(tmp)/'repo' + git('clone','--filter=blob:none','--depth','1','--single-branch','--branch',branch,'--no-checkout',url,str(stage)) + git('-C',str(stage),'sparse-checkout','set','--no-cone',*patterns) + git('-C',str(stage),'checkout',branch) + stage.rename(destination) + else: + if git('-C',str(destination),'status','--porcelain'): + raise RuntimeError('Metadata checkout has local changes; refusing to overwrite') + git('-C',str(destination),'fetch','--filter=blob:none','--depth','1','origin',branch) + git('-C',str(destination),'checkout','--detach','FETCH_HEAD') + return git('-C',str(destination),'rev-parse','HEAD') diff --git a/query.py b/query.py new file mode 100644 index 0000000..04354fb --- /dev/null +++ b/query.py @@ -0,0 +1,90 @@ +"""Read-only command-line queries for the paper index.""" +import argparse +import json +import sqlite3 +from pathlib import Path + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--db', required=True) + parser.add_argument('--query', help='SQLite FTS5 query over title and abstract') + parser.add_argument('--paper',type=int,help='Paper ID; include sources, tags and submission history') + parser.add_argument('--tag', help='Filter as namespace=value, e.g. arxiv:category=cs.AI') + parser.add_argument('--venue', help='Conference acronym') + parser.add_argument('--year', type=int) + parser.add_argument('--track',help='Exact track, e.g. main, findings, workshop') + parser.add_argument('--status', choices=['accepted','rejected','desk_rejected','withdrawn','under_review','unknown']) + parser.add_argument('--limit', type=int, default=20) + parser.add_argument('--stats', action='store_true') + args = parser.parse_args() + conn = sqlite3.connect(Path(args.db).resolve().as_uri()+'?mode=ro', uri=True) + conn.row_factory = sqlite3.Row + conn.execute('BEGIN') # One consistent read snapshot while ingestion continues. + if args.stats: + result = {name:conn.execute('SELECT count(*) FROM '+name).fetchone()[0] for name in ('papers','source_records','submissions','decisions','tags','match_candidates')} + result['records_by_source'] = [dict(r) for r in conn.execute('SELECT s.name,count(r.id) AS imported_records FROM sources s LEFT JOIN source_records r ON r.source_id=s.id GROUP BY s.id ORDER BY s.name')] + result['decision_events_by_status'] = [dict(r) for r in conn.execute('SELECT status,count(*) AS events FROM decisions GROUP BY status')] + result['coverage'] = [dict(r) for r in conn.execute('SELECT status,count(*) AS sources FROM edition_sources GROUP BY status')] + result['runs'] = [dict(r) for r in conn.execute('SELECT * FROM ingest_runs ORDER BY id DESC LIMIT 10')] + else: + where = []; params = [] + if args.paper is not None: + where.append('p.id=?'); params.append(args.paper) + if args.query: + where.append('p.id IN (SELECT rowid FROM papers_fts WHERE papers_fts MATCH ?)'); params.append(args.query) + if args.tag: + namespace,value = args.tag.split('=',1) + where.append('EXISTS(SELECT 1 FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id WHERE pt.paper_id=p.id AND t.namespace=? AND t.value=?)'); params.extend([namespace,value]) + if args.venue or args.year or args.status or args.track: + clause = ['1=1'] + if args.venue: + if conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='venue_aliases'").fetchone(): + clause.append('(v.acronym=? COLLATE NOCASE OR EXISTS(SELECT 1 FROM venue_aliases va WHERE va.venue_id=v.id AND va.alias=? COLLATE NOCASE))'); params.extend([args.venue,args.venue]) + else: + clause.append('v.acronym=? COLLATE NOCASE');params.append(args.venue) + if args.track: + clause.append('e.track=?');params.append(args.track) + if args.year: + clause.append('e.year=?'); params.append(args.year) + if args.status: + latest='''EXISTS ( + SELECT 1 FROM decisions d WHERE d.submission_id=s.id AND d.status=? + AND NOT EXISTS ( + SELECT 1 FROM decisions newer WHERE newer.submission_id=d.submission_id + AND (newer.observed_at,COALESCE(newer.decision_date,newer.observed_at),newer.id) + > (d.observed_at,COALESCE(d.decision_date,d.observed_at),d.id) + ) + )''' + clause.append('(NOT EXISTS(SELECT 1 FROM decisions d WHERE d.submission_id=s.id) OR '+latest+')' if args.status=='unknown' else latest); params.append(args.status) + where.append('p.id IN (SELECT s.paper_id FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id WHERE '+' AND '.join(clause)+')') + sql = 'SELECT p.* FROM papers p'+(' WHERE '+' AND '.join(where) if where else '')+' ORDER BY p.id LIMIT ?' + params.append(max(1,min(args.limit,200))) + result = [] + for row in conn.execute(sql,params): + paper = dict(row) + paper['authors'] = json.loads(paper.pop('authors_json')) + paper['identifiers'] = [dict(r) for r in conn.execute('SELECT scheme,value FROM identifiers WHERE paper_id=?',(row['id'],))] + if args.paper is not None: + paper['tags'] = [dict(r) for r in conn.execute('SELECT t.namespace,t.value,pt.evidence_key FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id WHERE pt.paper_id=?',(row['id'],))] + paper['sources'] = [dict(r) for r in conn.execute('SELECT s.name,r.external_id,r.url,r.title,r.abstract,r.retrieved_at FROM source_records r JOIN sources s ON s.id=r.source_id WHERE r.paper_id=?',(row['id'],))] + submissions = [] + for sub in conn.execute('SELECT s.id,s.source_submission_id,s.submitted_at,v.acronym,e.year,e.track FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id WHERE s.paper_id=? ORDER BY e.year,s.id',(row['id'],)): + submission = dict(sub) + submission['decisions'] = [] + for d in conn.execute('SELECT id,status,raw_value,evidence_url,observed_at,decision_date FROM decisions WHERE submission_id=? ORDER BY observed_at,COALESCE(decision_date,observed_at),id',(sub['id'],)): + decision=dict(d) + if conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='decision_sources'").fetchone(): + decision['sources']=[dict(r) for r in conn.execute('SELECT s.name,r.external_id,ds.source_decision_id FROM decision_sources ds JOIN source_records r ON r.id=ds.source_record_id JOIN sources s ON s.id=r.source_id WHERE ds.decision_id=?',(d['id'],))] + submission['decisions'].append(decision) + if conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name='submission_sources'").fetchone(): + submission['sources']=[dict(r) for r in conn.execute('SELECT so.name,r.external_id,ss.source_submission_id FROM submission_sources ss JOIN source_records r ON r.id=ss.source_record_id JOIN sources so ON so.id=r.source_id WHERE ss.submission_id=?',(sub['id'],))] + submissions.append(submission) + paper['submissions'] = submissions + result.append(paper) + print(json.dumps(result,ensure_ascii=False,indent=2)) + conn.close() + + +if __name__ == '__main__': + main() diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..f62ce0c --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +PyYAML==6.0.3 diff --git a/schema.sql b/schema.sql new file mode 100644 index 0000000..5a0b268 --- /dev/null +++ b/schema.sql @@ -0,0 +1,125 @@ +PRAGMA foreign_keys=ON; +PRAGMA journal_mode=WAL; +CREATE TABLE IF NOT EXISTS sources ( + id INTEGER PRIMARY KEY, name TEXT NOT NULL UNIQUE, homepage TEXT NOT NULL +); +CREATE TABLE IF NOT EXISTS papers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, title TEXT NOT NULL, abstract TEXT, + title_key TEXT NOT NULL, authors_json TEXT NOT NULL DEFAULT '[]', + first_published TEXT, created_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE INDEX IF NOT EXISTS papers_title_key ON papers(title_key); +CREATE TABLE IF NOT EXISTS identifiers ( + scheme TEXT NOT NULL, value TEXT NOT NULL, paper_id INTEGER NOT NULL REFERENCES papers(id), + PRIMARY KEY(scheme,value) +); +CREATE INDEX IF NOT EXISTS identifiers_paper ON identifiers(paper_id); +CREATE TABLE IF NOT EXISTS source_records ( + id INTEGER PRIMARY KEY, source_id INTEGER NOT NULL REFERENCES sources(id), + external_id TEXT NOT NULL, paper_id INTEGER NOT NULL REFERENCES papers(id), + url TEXT NOT NULL, title TEXT NOT NULL, abstract TEXT, authors_json TEXT NOT NULL, + metadata_json TEXT NOT NULL, retrieved_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + UNIQUE(source_id,external_id) +); +CREATE INDEX IF NOT EXISTS source_records_paper ON source_records(paper_id); +CREATE TABLE IF NOT EXISTS venues ( + id INTEGER PRIMARY KEY, acronym TEXT NOT NULL UNIQUE, name TEXT NOT NULL, + field TEXT NOT NULL, venue_type TEXT NOT NULL CHECK(venue_type IN ('conference','journal','review_platform')) +); +CREATE TABLE IF NOT EXISTS editions ( + id INTEGER PRIMARY KEY, venue_id INTEGER NOT NULL REFERENCES venues(id), + year INTEGER NOT NULL, track TEXT NOT NULL DEFAULT 'main', + UNIQUE(venue_id,year,track) +); +CREATE TABLE IF NOT EXISTS venue_aliases ( + venue_id INTEGER NOT NULL REFERENCES venues(id), alias TEXT NOT NULL, + PRIMARY KEY(venue_id,alias) +); +CREATE TABLE IF NOT EXISTS edition_sources ( + edition_id INTEGER NOT NULL REFERENCES editions(id), source_id INTEGER NOT NULL REFERENCES sources(id), + list_url TEXT NOT NULL, status TEXT NOT NULL DEFAULT 'pending' + CHECK(status IN ('pending','running','complete','partial','blocked','unavailable','not_yet_published')), + expected_count INTEGER, imported_count INTEGER NOT NULL DEFAULT 0, + cursor TEXT, last_success_at TEXT, last_error TEXT, + PRIMARY KEY(edition_id,source_id,list_url) +); +CREATE TABLE IF NOT EXISTS submissions ( + id INTEGER PRIMARY KEY, paper_id INTEGER NOT NULL REFERENCES papers(id), + edition_id INTEGER NOT NULL REFERENCES editions(id), + source_record_id INTEGER NOT NULL REFERENCES source_records(id), + source_submission_id TEXT NOT NULL, submitted_at TEXT, + UNIQUE(source_record_id,edition_id,source_submission_id) +); +CREATE INDEX IF NOT EXISTS submissions_paper ON submissions(paper_id); +CREATE TABLE IF NOT EXISTS decisions ( + id INTEGER PRIMARY KEY, submission_id INTEGER NOT NULL REFERENCES submissions(id), + status TEXT NOT NULL CHECK(status IN ('accepted','rejected','desk_rejected','withdrawn','under_review','unknown')), + raw_value TEXT NOT NULL, evidence_url TEXT NOT NULL, observed_at TEXT NOT NULL, + decision_date TEXT, UNIQUE(submission_id,status,raw_value,evidence_url,observed_at) +); +CREATE INDEX IF NOT EXISTS decisions_submission_order ON decisions(submission_id,observed_at DESC,id DESC); +CREATE TABLE IF NOT EXISTS submission_sources ( + submission_id INTEGER NOT NULL REFERENCES submissions(id), + source_record_id INTEGER NOT NULL REFERENCES source_records(id), source_submission_id TEXT NOT NULL, + PRIMARY KEY(submission_id,source_record_id,source_submission_id) +); +CREATE INDEX IF NOT EXISTS submission_sources_record ON submission_sources(source_record_id,source_submission_id); +CREATE TABLE IF NOT EXISTS submission_identifiers ( + edition_id INTEGER NOT NULL REFERENCES editions(id), scheme TEXT NOT NULL, value TEXT NOT NULL, + submission_id INTEGER NOT NULL REFERENCES submissions(id), PRIMARY KEY(edition_id,scheme,value) +); +CREATE INDEX IF NOT EXISTS submission_identifiers_submission ON submission_identifiers(submission_id); +CREATE TABLE IF NOT EXISTS decision_sources ( + decision_id INTEGER NOT NULL REFERENCES decisions(id), + source_record_id INTEGER NOT NULL REFERENCES source_records(id), source_decision_id TEXT NOT NULL DEFAULT '', + PRIMARY KEY(decision_id,source_record_id,source_decision_id) +); +CREATE TABLE IF NOT EXISTS submission_merges ( + former_submission_id INTEGER PRIMARY KEY, surviving_submission_id INTEGER NOT NULL REFERENCES submissions(id), + reason TEXT NOT NULL, merged_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE TABLE IF NOT EXISTS decision_merges ( + former_decision_id INTEGER PRIMARY KEY, surviving_decision_id INTEGER NOT NULL REFERENCES decisions(id), + reason TEXT NOT NULL, merged_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE TABLE IF NOT EXISTS schema_migrations ( + name TEXT PRIMARY KEY, applied_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE TABLE IF NOT EXISTS tags ( + id INTEGER PRIMARY KEY, namespace TEXT NOT NULL, value TEXT NOT NULL, + description TEXT, UNIQUE(namespace,value) +); +CREATE TABLE IF NOT EXISTS paper_tags ( + paper_id INTEGER NOT NULL REFERENCES papers(id), tag_id INTEGER NOT NULL REFERENCES tags(id), + source_record_id INTEGER REFERENCES source_records(id), + evidence_key TEXT NOT NULL, PRIMARY KEY(paper_id,tag_id,evidence_key) +); +CREATE INDEX IF NOT EXISTS paper_tags_source_evidence ON paper_tags(source_record_id,evidence_key); +CREATE TABLE IF NOT EXISTS match_candidates ( + left_paper_id INTEGER NOT NULL REFERENCES papers(id), right_paper_id INTEGER NOT NULL REFERENCES papers(id), + method TEXT NOT NULL, score REAL, evidence_json TEXT NOT NULL, + status TEXT NOT NULL DEFAULT 'pending' CHECK(status IN ('pending','merged','distinct')), + PRIMARY KEY(left_paper_id,right_paper_id,method) +); +CREATE TABLE IF NOT EXISTS paper_merges ( + id INTEGER PRIMARY KEY, former_paper_id INTEGER NOT NULL, + surviving_paper_id INTEGER NOT NULL REFERENCES papers(id), + reason TEXT NOT NULL, merged_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP +); +CREATE TABLE IF NOT EXISTS ingest_runs ( + id INTEGER PRIMARY KEY, source_id INTEGER NOT NULL REFERENCES sources(id), + started_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, finished_at TEXT, + status TEXT NOT NULL DEFAULT 'running', records_seen INTEGER NOT NULL DEFAULT 0, + records_written INTEGER NOT NULL DEFAULT 0, error TEXT, checkpoint_json TEXT +); +CREATE VIRTUAL TABLE IF NOT EXISTS papers_fts USING fts5(title,abstract,content='papers',content_rowid='id'); +CREATE TRIGGER IF NOT EXISTS papers_insert AFTER INSERT ON papers BEGIN + INSERT INTO papers_fts(rowid,title,abstract) VALUES(new.id,new.title,new.abstract); +END; +CREATE TRIGGER IF NOT EXISTS papers_delete AFTER DELETE ON papers BEGIN + INSERT INTO papers_fts(papers_fts,rowid,title,abstract) VALUES('delete',old.id,old.title,old.abstract); +END; +CREATE TRIGGER IF NOT EXISTS papers_update AFTER UPDATE OF title,abstract ON papers BEGIN + INSERT INTO papers_fts(papers_fts,rowid,title,abstract) VALUES('delete',old.id,old.title,old.abstract); + INSERT INTO papers_fts(rowid,title,abstract) VALUES(new.id,new.title,new.abstract); +END; diff --git a/submission_evidence.py b/submission_evidence.py new file mode 100644 index 0000000..c1d350b --- /dev/null +++ b/submission_evidence.py @@ -0,0 +1,115 @@ +"""Canonical submission events with independent source and decision evidence.""" +from datetime import datetime, timezone +import re + + +PUBLICATIONS={'pmlr','acl_anthology','colm','neurips','cvf','ecva','dblp'} + + +def timestamp(value=None): + date=datetime.now(timezone.utc) if value is None else datetime.fromisoformat(value.replace('Z','+00:00')) + if date.tzinfo is None:raise ValueError('Decision timestamp requires a timezone') + return date.astimezone(timezone.utc).isoformat(timespec='milliseconds').replace('+00:00','Z') + + +def identities_for_row(row): + identities=list(row.get('submission_identifiers') or []) + own=row.get('identifiers') or {} + if own.get('openreview'): + identities.append({'scheme':'openreview','value':own['openreview']}) + if row['source'] in PUBLICATIONS and row.get('status')=='accepted' and own.get('doi'): + doi=re.sub(r'^https?://(?:dx\.)?doi\.org/','',str(own['doi']),flags=re.I).strip().lower() + if re.fullmatch(r'10\.\d{4,9}/\S+',doi):identities.append({'scheme':'publication_doi','value':doi}) + meta=row.get('metadata') or {} + if row['source']=='iclr_program' and row['venue']=='ICLR' and meta.get('program_identity_type')=='submission' and meta.get('program_submission_number_verified') is True: + identities.append({'scheme':'iclr_number','value':str(meta['program_source_id'])}) + result=set() + for identity in identities: + scheme,value=identity['scheme'],str(identity['value']).strip() + if scheme=='openreview': + valid=re.fullmatch(r'[A-Za-z0-9_-]+',value) and not re.match(r'20\d{2}-Oral-',value) + elif scheme=='iclr_number':valid=re.fullmatch(r'[1-9]\d*',value) and row['venue']=='ICLR' + elif scheme=='publication_doi':valid=re.fullmatch(r'10\.\d{4,9}/\S+',value) + else:valid=False + if not valid:raise ValueError('Invalid authoritative submission identity') + result.add((scheme,value)) + return sorted(result) + + +def migrate_sources(c): + if c.execute("SELECT 1 FROM schema_migrations WHERE name='submission_source_baseline_v1'").fetchone():return + with c: + c.execute('INSERT OR IGNORE INTO submission_sources SELECT id,source_record_id,source_submission_id FROM submissions') + c.execute("INSERT OR IGNORE INTO decision_sources SELECT d.id,s.source_record_id,'' FROM decisions d JOIN submissions s ON s.id=d.submission_id") + c.execute("INSERT INTO schema_migrations(name) VALUES('submission_source_baseline_v1')") + + +def next_id(c,table,audit,former): + # Merged IDs are kept in audit; never allocate them to a later unrelated event. + return max(c.execute('SELECT COALESCE(max(id),0) FROM '+table).fetchone()[0],c.execute('SELECT COALESCE(max('+former+'),0) FROM '+audit).fetchone()[0])+1 + + +def merge_submissions(c,target,other): + if target==other:return + editions=c.execute('SELECT id,edition_id FROM submissions WHERE id IN (?,?)',(target,other)).fetchall() + if len(editions)!=2 or editions[0][1]!=editions[1][1]:raise ValueError('Cannot combine submissions from different editions') + c.execute('INSERT OR IGNORE INTO submission_sources SELECT ?,source_record_id,source_submission_id FROM submission_sources WHERE submission_id=?',(target,other)) + c.execute('DELETE FROM submission_sources WHERE submission_id=?',(other,)) + for decision,status,raw,url,observed,date in c.execute('SELECT id,status,raw_value,evidence_url,observed_at,decision_date FROM decisions WHERE submission_id=?',(other,)).fetchall(): + duplicate=c.execute('SELECT id,decision_date FROM decisions WHERE submission_id=? AND status=? AND raw_value=? AND evidence_url=? AND observed_at=?',(target,status,raw,url,observed)).fetchone() + if duplicate: + surviving=duplicate[0] + if date and duplicate[1] and date!=duplicate[1]:raise ValueError('Conflicting dates for the same decision observation') + if date and not duplicate[1]:c.execute('UPDATE decisions SET decision_date=? WHERE id=?',(date,surviving)) + c.execute('INSERT OR IGNORE INTO decision_sources SELECT ?,source_record_id,source_decision_id FROM decision_sources WHERE decision_id=?',(surviving,decision)) + c.execute('DELETE FROM decision_sources WHERE decision_id=?',(decision,)) + c.execute('UPDATE decision_merges SET surviving_decision_id=? WHERE surviving_decision_id=?',(surviving,decision)) + c.execute("INSERT INTO decision_merges(former_decision_id,surviving_decision_id,reason) VALUES(?,?,'same decision observation on merged submission')",(decision,surviving)) + c.execute('DELETE FROM decisions WHERE id=?',(decision,)) + else:c.execute('UPDATE decisions SET submission_id=? WHERE id=?',(target,decision)) + c.execute('UPDATE submission_identifiers SET submission_id=? WHERE submission_id=?',(target,other)) + c.execute('UPDATE submission_merges SET surviving_submission_id=? WHERE surviving_submission_id=?',(target,other)) + c.execute("INSERT INTO submission_merges(former_submission_id,surviving_submission_id,reason) VALUES(?,?,'shared authoritative submission identity')",(other,target)) + c.execute('DELETE FROM submissions WHERE id=?',(other,)) + + +def attach_submission(store,edition,paper,record,external_id,identities): + c=store.conn + matches={r[0] for r in c.execute('SELECT s.id FROM submission_sources p JOIN submissions s ON s.id=p.submission_id WHERE p.source_record_id=? AND p.source_submission_id=? AND s.edition_id=?',(record,external_id,edition))} + matches.update(r[0] for r in c.execute('SELECT id FROM submissions WHERE source_record_id=? AND source_submission_id=? AND edition_id=?',(record,external_id,edition))) + for scheme,value in identities: + found=c.execute('SELECT submission_id FROM submission_identifiers WHERE edition_id=? AND scheme=? AND value=?',(edition,scheme,value)).fetchone() + if found:matches.add(found[0]) + for scheme in ('openreview','iclr_number'): + values={value for kind,value in identities if kind==scheme} + for sid in matches: + values.update(value for value, in c.execute('SELECT value FROM submission_identifiers WHERE submission_id=? AND scheme=?',(sid,scheme))) + if len(values)>1:raise ValueError('Conflicting authoritative submission identities: '+scheme) + if matches: + papers={paper}|{c.execute('SELECT paper_id FROM submissions WHERE id=?',(sid,)).fetchone()[0] for sid in matches} + surviving=min(papers) + for other in sorted(papers-{surviving}):store.merge(surviving,other,'shared authoritative conference submission identity') + submission=min(matches) + for other in sorted(matches-{submission}):merge_submissions(c,submission,other) + c.execute('UPDATE submissions SET paper_id=? WHERE id=?',(surviving,submission)) + else: + submission=next_id(c,'submissions','submission_merges','former_submission_id') + c.execute('INSERT INTO submissions(id,paper_id,edition_id,source_record_id,source_submission_id) VALUES(?,?,?,?,?)',(submission,paper,edition,record,external_id)) + c.execute('INSERT OR IGNORE INTO submission_sources VALUES(?,?,?)',(submission,record,external_id)) + for scheme,value in identities: + c.execute('INSERT OR IGNORE INTO submission_identifiers VALUES(?,?,?,?)',(edition,scheme,value,submission)) + return submission + + +def write_decision(store,submission,record,status,raw,url,observed_at=None,decision_date=None,source_decision_id=''): + c=store.conn + observed=timestamp(observed_at) + date=timestamp(decision_date) if decision_date else None + # Re-observing unchanged evidence adds provenance, not a new decision event. + old=c.execute('SELECT id FROM decisions WHERE submission_id=? AND status=? AND raw_value=? AND evidence_url=? AND COALESCE(decision_date,\'\')=COALESCE(?,\'\')',(submission,status,raw,url,date)).fetchone() + if old:decision=old[0] + else: + decision=next_id(c,'decisions','decision_merges','former_decision_id') + c.execute('INSERT INTO decisions(id,submission_id,status,raw_value,evidence_url,observed_at,decision_date) VALUES(?,?,?,?,?,?,?)',(decision,submission,status,raw,url,observed,date)) + c.execute('INSERT OR IGNORE INTO decision_sources VALUES(?,?,?)',(decision,record,source_decision_id or '')) + return decision diff --git a/test_acl.py b/test_acl.py new file mode 100644 index 0000000..99775ba --- /dev/null +++ b/test_acl.py @@ -0,0 +1,40 @@ +import json +import tempfile +import unittest +import xml.etree.ElementTree as ET +from pathlib import Path + +from collect_acl import paper_id,volume_scope,collect_xml + + +class AnthologyTests(unittest.TestCase): + def test_documented_identifier_formats_and_exceptions(self): + for args,expected in [(('P17','1','1'),'P17-1001'),(('2025.acl','long','1'),'2025.acl-long.1'),(('C69','1','1'),'C69-0101'),(('D19','50','1'),'D19-5001'),(('W17','1','1'),'W17-0101')]: + self.assertEqual(paper_id(*args),expected) + + def test_findings_uses_associated_conference_and_separate_track(self): + meta=ET.fromstring('findingsFindings of ACL: EMNLP 2025') + self.assertEqual(volume_scope(meta,'emnlp'),('EMNLP',['EMNLP'],'findings')) + + def test_joint_event_has_one_identity_and_two_aliases(self): + meta=ET.fromstring('lreccolingLREC-COLING 2024') + self.assertEqual(volume_scope(meta,'main'),('COLING+LREC',['COLING','LREC'],'main')) + + def test_xml_mixed_text_frontmatter_and_missing_abstract(self): + xml='2017aclACLFront matterA <fixed-case>Mixed</fixed-case> TitleJaneDoe10.18653/v1/P17-1001' + with tempfile.TemporaryDirectory() as tmp: + path=Path(tmp)/'P17.xml';path.write_text(xml) + result=collect_xml(path,tmp,'a'*40)[0] + row=json.loads(Path(result['file']).read_text()) + self.assertEqual(result['count'],1) + self.assertEqual(result['excluded_frontmatter'],1) + self.assertEqual(result['status'],'partial') + self.assertEqual(row['external_id'],'P17-1001') + self.assertEqual(row['title'],'A Mixed Title') + + def test_non_target_workshop_is_not_main_conference(self): + meta=ET.fromstring('some-workshopwsWorkshop at ACL') + self.assertIsNone(volume_scope(meta,'1')) + + +if __name__=='__main__':unittest.main() diff --git a/test_coverage_report.py b/test_coverage_report.py new file mode 100644 index 0000000..c2c4e94 --- /dev/null +++ b/test_coverage_report.py @@ -0,0 +1,35 @@ +import json +from pathlib import Path +import tempfile +import unittest + +from coverage_report import report +from index_core import Store + + +class CoverageTests(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.addCleanup(self.tmp.cleanup) + self.root=Path(self.tmp.name);self.db=self.root/'papers.sqlite';self.store=Store(self.db);self.addCleanup(self.store.conn.close) + self.registry=self.root/'venues.json';self.registry.write_text(json.dumps({'scope_note':'Candidate scope','venues':[{'acronym':'COLING','field':'language'},{'acronym':'AAAI','field':'general_ai'}]})) + + def test_zero_coverage_targets_are_listed_and_joint_alias_is_counted_once(self): + paper,record=self.store.record('acl_anthology','joint-paper','https://example.org','Joint paper','Abstract',['Jane Doe']) + edition=self.store.decision(paper,record,'COLING+LREC',2024,'main','joint-paper','https://example.org','accepted','Accepted',aliases=['COLING','LREC']) + self.store.decision(paper,record,'COLING+LREC',2024,'main','joint-paper','https://example.org/another-observation','accepted','Accepted',aliases=['COLING']) + self.store.conn.commit();result=report(self.db,self.registry) + self.assertEqual(result['target_venue_count'],2);self.assertEqual(result['venues_with_imported_records'],1) + coling,aaai=result['venues'];self.assertEqual(coling['imported_submissions'],1) + self.assertEqual(coling['editions'][0]['canonical_venue'],'COLING+LREC') + self.assertEqual(aaai['history_status'],'no_imported_records');self.assertEqual(aaai['editions'],[]) + self.assertNotEqual(coling['history_status'],'complete') + + def test_same_paper_across_years_is_not_two_distinct_papers(self): + paper,record=self.store.record('source','one','https://example.org','A paper','Abstract',[]) + for year in (2024,2025): + self.store.decision(paper,record,'COLING',year,'main',str(year),'https://example.org','accepted','Accepted') + self.store.conn.commit();coling=report(self.db,self.registry)['venues'][0] + self.assertEqual(coling['imported_submissions'],2);self.assertEqual(coling['distinct_papers'],1) + + +if __name__=='__main__':unittest.main() diff --git a/test_dblp_catalogue.py b/test_dblp_catalogue.py new file mode 100644 index 0000000..67bc23a --- /dev/null +++ b/test_dblp_catalogue.py @@ -0,0 +1,68 @@ +import gzip +import hashlib +import json +from pathlib import Path +import sqlite3 +import tempfile +import unittest + +from catalogue_dblp import catalogue,records + + +class DblpCatalogueTests(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.addCleanup(self.tmp.cleanup);self.root=Path(self.tmp.name) + self.dtd=self.root/'dblp.dtd';self.dtd.write_bytes(b'') + + def archive(self,body,doctype=''): + path=self.root/'dblp.xml.gz' + with gzip.open(path,'wb') as stream:stream.write((''+doctype+''+body+'').encode()) + return path + + def test_streamed_entities_nested_titles_and_candidate_scope_remain_explicit(self): + path=self.archive('AAAI Workshop proceedings2025RenéLearn <i>deep</i> models2025AAAI Workshopconf/aaai/2025https://doi.org/10.1234/exampleOther2020') + manifest=catalogue(path,self.dtd,self.root/'out',hashlib.sha256(path.read_bytes()).hexdigest(),hashlib.md5(self.dtd.read_bytes()).hexdigest()) + self.assertEqual(manifest['records_scanned'],3);self.assertEqual(manifest['candidate_records'],1) + row=json.loads(Path(manifest['file']).read_text()) + self.assertEqual(row['fields']['author'],['René']);self.assertEqual(row['fields']['title'],['Learn deep models']) + self.assertNotIn('status',row);self.assertNotIn('venue',row) + with sqlite3.connect(manifest['catalogue']) as connection: + self.assertEqual(connection.execute('SELECT sum(count) FROM volume_counts').fetchone()[0],2) + self.assertEqual(connection.execute('SELECT count(*) FROM proceedings').fetchone()[0],1) + + def test_remote_external_dtd_is_rejected_without_fetching(self): + path=self.archive('',doctype='') + with self.assertRaisesRegex(ValueError,'pinned external'):list(records(path,self.dtd.read_bytes())) + + def test_real_monthly_dtd_name_and_latin1_encoding(self): + path=self.root/'monthly.xml.gz' + xml='RenéMetadata' + with gzip.open(path,'wb') as stream:stream.write(xml.encode('latin1')) + self.assertEqual(list(records(path,self.dtd.read_bytes()))[0]['fields']['author'],['René']) + + def test_multiple_volume_fields_remain_reviewable_without_double_counting(self): + path=self.archive('Joint publication2024First meetingSecond meetingconf/aaai/joint2024') + manifest=catalogue(path,self.dtd,self.root/'out',hashlib.sha256(path.read_bytes()).hexdigest(),hashlib.md5(self.dtd.read_bytes()).hexdigest()) + with sqlite3.connect(manifest['catalogue']) as connection: + raw=json.loads(connection.execute('SELECT metadata_json FROM volume_anomalies').fetchone()[0]) + self.assertEqual(raw['fields']['booktitle'],['First meeting','Second meeting']) + self.assertEqual(connection.execute('SELECT sum(count) FROM volume_counts').fetchone()[0],1) + + def test_internal_entities_are_rejected(self): + path=self.archive('',doctype=']>') + with self.assertRaisesRegex(ValueError,'pinned external'):list(records(path,self.dtd.read_bytes())) + + def test_checksum_failure_preserves_existing_output(self): + path=self.archive('');out=self.root/'out';out.mkdir();target=out/'dblp-ai-candidates.jsonl';target.write_text('previous') + with self.assertRaisesRegex(ValueError,'snapshot checksum'):catalogue(path,self.dtd,out,'0'*64) + self.assertEqual(target.read_text(),'previous') + + def test_truncated_gzip_cannot_publish_a_catalogue(self): + path=self.archive('One') + path.write_bytes(path.read_bytes()[:-8]);out=self.root/'out' + with self.assertRaises((EOFError,OSError)): + catalogue(path,self.dtd,out,hashlib.sha256(path.read_bytes()).hexdigest(),hashlib.md5(self.dtd.read_bytes()).hexdigest()) + self.assertFalse((out/'dblp-ai-candidates.jsonl').exists()) + + +if __name__=='__main__':unittest.main() diff --git a/test_download_snapshot.py b/test_download_snapshot.py new file mode 100644 index 0000000..301c7aa --- /dev/null +++ b/test_download_snapshot.py @@ -0,0 +1,71 @@ +import hashlib +import io +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from download_snapshot import download + + +class Response(io.BytesIO): + def __init__(self,body,status=200,headers=None): + super().__init__(body);self.status=status;self.headers=headers or {'Content-Length':str(len(body))} + + +class DownloadTests(unittest.TestCase): + def test_verified_download_is_reused_without_replacing_it(self): + body=b'public metadata';sha=hashlib.sha256(body).hexdigest() + with tempfile.TemporaryDirectory() as tmp: + path=Path(tmp)/'snapshot.gz' + with patch('download_snapshot.urllib.request.urlopen',return_value=Response(body)): + result=download('https://example.org/snapshot.gz',path,sha256=sha,expected_bytes=len(body),min_free=0) + self.assertEqual(result['status'],'verified') + with patch('download_snapshot.urllib.request.urlopen',side_effect=AssertionError('No network expected')): + reused=download('https://example.org/snapshot.gz',path,sha256=sha,min_free=0) + self.assertEqual(reused['status'],'already_verified') + self.assertEqual(json.loads(Path(str(path)+'.manifest.json').read_text())['sha256'],sha) + + def test_bad_checksum_never_publishes_a_snapshot(self): + with tempfile.TemporaryDirectory() as tmp,patch('download_snapshot.urllib.request.urlopen',return_value=Response(b'wrong bytes')): + path=Path(tmp)/'snapshot.gz' + with self.assertRaisesRegex(ValueError,'SHA256 mismatch'): + download('https://example.org/snapshot.gz',path,sha256='0'*64,min_free=0) + self.assertFalse(path.exists());self.assertTrue(Path(str(path)+'.part').exists()) + + def test_resume_requires_the_same_identity_and_a_real_partial_response(self): + body=b'abcdef';sha=hashlib.sha256(body).hexdigest();url='https://example.org/snapshot.gz' + with tempfile.TemporaryDirectory() as tmp: + path=Path(tmp)/'snapshot.gz';part=Path(str(path)+'.part');part.write_bytes(body[:3]) + Path(str(part)+'.json').write_text(json.dumps({'url':url,'expected_sha256':sha,'expected_md5':None,'expected_bytes':6})) + with self.assertRaisesRegex(ValueError,'same snapshot'): + download(url,path,sha256=sha,expected_bytes=6,min_free=0) + with patch('download_snapshot.urllib.request.urlopen',return_value=Response(body)): + with self.assertRaisesRegex(ValueError,'resume range'): + download(url,path,sha256=sha,expected_bytes=6,resume=True,min_free=0) + self.assertEqual(part.read_bytes(),b'abc') + with patch('download_snapshot.urllib.request.urlopen',return_value=Response(b'def',206,{'Content-Range':'bytes 3-5/6'})): + download(url,path,sha256=sha,expected_bytes=6,resume=True,min_free=0) + self.assertEqual(path.read_bytes(),body) + + def test_wrong_http_length_is_rejected_before_writing(self): + with tempfile.TemporaryDirectory() as tmp,patch('download_snapshot.urllib.request.urlopen',return_value=Response(b'abc')): + path=Path(tmp)/'snapshot.gz' + with self.assertRaisesRegex(ValueError,'HTTP size'): + download('https://example.org/snapshot.gz',path,sha256='0'*64,expected_bytes=4,min_free=0) + self.assertFalse(path.exists());self.assertFalse(Path(str(path)+'.part').exists()) + + def test_empty_partial_and_missing_final_manifest_can_recover(self): + body=b'abc';sha=hashlib.sha256(body).hexdigest();url='https://example.org/snapshot.gz' + with tempfile.TemporaryDirectory() as tmp: + path=Path(tmp)/'snapshot.gz';part=Path(str(path)+'.part');part.write_bytes(b'') + Path(str(part)+'.json').write_text(json.dumps({'url':url,'expected_sha256':sha,'expected_md5':None,'expected_bytes':3})) + with patch('download_snapshot.urllib.request.urlopen',return_value=Response(body)): + download(url,path,sha256=sha,expected_bytes=3,resume=True,min_free=0) + metadata=Path(str(path)+'.manifest.json');metadata.unlink() + download(url,path,sha256=sha,expected_bytes=3,min_free=0) + self.assertEqual(json.loads(metadata.read_text())['sha256'],sha) + + +if __name__=='__main__':unittest.main() diff --git a/test_fetch.py b/test_fetch.py new file mode 100644 index 0000000..720bf4b --- /dev/null +++ b/test_fetch.py @@ -0,0 +1,49 @@ +import hashlib +import io +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch +from urllib.error import HTTPError,URLError + +from collect_pmlr import fetch + + +class Response(io.BytesIO): + def __init__(self,body,expected=None): + super().__init__(body);self.status=200;self.headers={'Content-Length':str(expected if expected is not None else len(body))} + + +class FetchTests(unittest.TestCase): + def test_transient_connection_failure_is_retried(self): + with tempfile.TemporaryDirectory() as tmp,patch('collect_pmlr.time.sleep'),patch('collect_pmlr.urllib.request.urlopen',side_effect=[URLError(ConnectionResetError('reset')),Response(b'complete')]) as network: + self.assertEqual(fetch('https://example.org/metadata',tmp),b'complete') + self.assertEqual(network.call_count,2) + + def test_truncated_response_is_not_cached_as_complete(self): + with tempfile.TemporaryDirectory() as tmp,patch('collect_pmlr.time.sleep'),patch('collect_pmlr.urllib.request.urlopen',side_effect=[Response(b'bad',expected=8),Response(b'complete')]): + self.assertEqual(fetch('https://example.org/metadata',tmp),b'complete') + path=Path(tmp)/hashlib.sha256(b'https://example.org/metadata').hexdigest() + self.assertEqual(path.read_bytes(),b'complete') + + def test_forbidden_response_is_not_retried_or_bypassed(self): + body=io.BytesIO(b'Forbidden');error=HTTPError('https://example.org/metadata',403,'Forbidden',{},body) + with tempfile.TemporaryDirectory() as tmp,patch('collect_pmlr.time.sleep'),patch('collect_pmlr.urllib.request.urlopen',side_effect=error) as network: + with self.assertRaises(HTTPError):fetch('https://example.org/metadata',tmp) + self.assertEqual(network.call_count,1) + self.assertTrue(body.closed) + + def test_transport_retries_are_bounded(self): + with tempfile.TemporaryDirectory() as tmp,patch('collect_pmlr.time.sleep'),patch('collect_pmlr.urllib.request.urlopen',side_effect=ConnectionResetError('reset')) as network: + with self.assertRaises(ConnectionResetError):fetch('https://example.org/metadata',tmp) + self.assertEqual(network.call_count,3) + + def test_cached_response_still_obeys_the_size_limit(self): + with tempfile.TemporaryDirectory() as tmp,patch('collect_pmlr.urllib.request.urlopen') as network: + path=Path(tmp)/hashlib.sha256(b'https://example.org/metadata').hexdigest();path.write_bytes(b'too large') + with self.assertRaisesRegex(RuntimeError,'exceeds byte limit'): + fetch('https://example.org/metadata',tmp,limit=2) + network.assert_not_called() + + +if __name__=='__main__':unittest.main() diff --git a/test_iclr.py b/test_iclr.py new file mode 100644 index 0000000..fb9b983 --- /dev/null +++ b/test_iclr.py @@ -0,0 +1,98 @@ +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch +from urllib.error import HTTPError + +from collect_iclr import asset_urls, collect_year, discover, parse_program +from import_records import ingest_file +from index_core import Store + + +class ICLRTests(unittest.TestCase): + def event(self,identifier=1,sourceid=10,**updates): + row={'id':identifier,'sourceid':sourceid,'sourceurl':'https://openreview.net/group?id=ICLR.cc/2026/Conference', + 'name':'A reliable paper on abstract metadata indexing','authors':[{'fullname':'Jane Doe'},{'fullname':'John Smith'}], + 'eventtype':'Poster','event_type':'Poster','decision':'Accept (Oral)', + 'paper_url':'https://openreview.net/forum?id=realForum','topic':'Learning->Theory','keywords':[]} + return dict(row,**updates) + + def parse(self,events,abstracts=None,year=2026): + return parse_program({'count':len(events),'next':None,'previous':None,'results':events},abstracts or {},year,f'https://iclr.cc/virtual/{year}/papers.html') + + def test_oral_placeholder_is_not_an_identifier_or_another_submission(self): + poster=self.event() + oral=self.event(2,-10,eventtype='Oral',event_type='Oral',paper_url='https://openreview.net/forum?id=2026-Oral--10-placeholder') + rows,audit=self.parse([oral,poster],{'1':'The real abstract.'}) + self.assertEqual(len(rows),1) + self.assertEqual(rows[0]['identifiers'],{'openreview':'realForum'}) + self.assertEqual(rows[0]['abstract'],'The real abstract.') + self.assertEqual(rows[0]['metadata']['presentation_types'],['Oral','Poster']) + self.assertEqual(audit['main_event_count'],2) + + def test_blog_and_journal_presentations_are_not_main_acceptances(self): + rows,audit=self.parse([self.event(),self.event(2,20,sourceurl='https://openreview.net/group?id=ICLR.cc/2026/BlogPosts'),self.event(3,30,sourceurl='2026-TMLR-J2C-Spreadsheet')]) + self.assertEqual(len(rows),1) + self.assertEqual(sum(audit['excluded_events_by_source'].values()),2) + self.assertEqual(audit['missing_abstract_count'],1) + self.assertIsNone(rows[0]['abstract']) + + def test_title_revision_is_scoped_to_matching_signed_submission_and_authors(self): + poster=self.event(name='AlphaEdit: Null-Space Constrained Knowledge Editing for Language Models') + oral=self.event(2,-10,eventtype='Oral',name='AlphaEdit: Null-Space Constrained Model Editing for Language Models') + rows,_=self.parse([oral,poster]) + self.assertEqual(len(rows),1) + self.assertEqual(rows[0]['title'],poster['name']) + self.assertEqual(len(rows[0]['metadata']['presentation_titles']),2) + with self.assertRaisesRegex(ValueError,'collides'): + self.parse([poster,dict(oral,authors=[{'fullname':'Unrelated Author'}])]) + + def test_truncation_and_identity_collision_fail(self): + with self.assertRaisesRegex(ValueError,'count differs'): + parse_program({'count':2,'results':[self.event()]},{},2026,'https://iclr.cc/') + with self.assertRaisesRegex(ValueError,'collides'): + self.parse([self.event(),self.event(2,-10,name='A different paper')]) + with self.assertRaisesRegex(ValueError,'Conflicting identifiers'): + self.parse([self.event(),self.event(2,10,paper_url='https://openreview.net/forum?id=otherForum')]) + + def test_legacy_embedded_abstracts_and_paper_links(self): + row=self.event(sourceurl=None,abstract='Historical abstract',paper_url=None,paper_pdf_url='https://openreview.net/forum?id=historicalForum',decision='Poster') + rows,_=self.parse([row],year=2021) + self.assertEqual(rows[0]['abstract'],'Historical abstract') + self.assertEqual(rows[0]['identifiers'],{'openreview':'historicalForum'}) + rows,_=self.parse([dict(row,paper_pdf_url=None)],year=2020) + self.assertEqual(rows[0]['metadata']['identity_state'],'program_only') + self.assertEqual(rows[0]['identifiers'],{}) + + def test_only_referenced_official_assets_and_real_navigation_years(self): + html='201320262025' + self.assertEqual([y for y,_ in discover(html)],[2013,2026]) + html='start("/static/virtual/data/iclr-2026-orals-posters.json",true,"https://other.example/abstracts.json")' + self.assertEqual(asset_urls(html,'https://iclr.cc/virtual/2026/papers.html',2026),{'orals-posters':'https://iclr.cc/static/virtual/data/iclr-2026-orals-posters.json'}) + + def test_missing_split_abstract_asset_does_not_discard_embedded_abstract(self): + html=b'start("/static/virtual/data/iclr-2021-orals-posters.json",true,"/static/virtual/data/iclr-2021-abstracts.json")' + event=self.event(sourceurl=None,decision='Poster',abstract='Embedded historical abstract') + responses=[html,json.dumps({'count':1,'results':[event]}).encode(),HTTPError('https://iclr.cc/abstracts',404,'Not Found',None,None)] + with tempfile.TemporaryDirectory() as tmp,patch('collect_iclr.fetch',side_effect=responses): + manifest=collect_year(2021,tmp,tmp) + self.assertEqual(manifest['status'],'collected') + self.assertEqual(manifest['missing_abstract_count'],0) + self.assertEqual(manifest['abstract_asset_status'],'unavailable_404') + + def test_import_repeat_preserves_one_submission_and_queryable_platform_tags(self): + rows,_=self.parse([self.event(),self.event(2,-10,eventtype='Oral')],{'1':'A searchable abstract.'}) + with tempfile.TemporaryDirectory() as tmp: + path=Path(tmp)/'iclr.jsonl';db=Path(tmp)/'papers.sqlite' + path.write_text(''.join(json.dumps(r)+'\n' for r in rows)) + path.with_suffix('.manifest.json').write_text(json.dumps({'source':'iclr_program','parser_version':2,'list_url':'https://iclr.cc/virtual/2026/papers.html','count':1,'status':'collected'})) + ingest_file(db,path);ingest_file(db,path) + store=Store(db);self.addCleanup(store.conn.close) + self.assertEqual(store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0],1) + self.assertEqual(store.conn.execute('SELECT count(*) FROM decisions').fetchone()[0],1) + self.assertEqual(store.conn.execute('SELECT namespace,value FROM tags').fetchall(),[('iclr_program:topic','Learning->Theory')]) + self.assertEqual(store.conn.execute('PRAGMA foreign_key_check').fetchall(),[]) + + +if __name__=='__main__':unittest.main() diff --git a/test_iclr_snapshot_audit.py b/test_iclr_snapshot_audit.py new file mode 100644 index 0000000..71d0a05 --- /dev/null +++ b/test_iclr_snapshot_audit.py @@ -0,0 +1,58 @@ +import json +from pathlib import Path +import tempfile +import unittest + +from audit_iclr_snapshot import audit + + +class SnapshotAuditTests(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.addCleanup(self.tmp.cleanup);self.root=Path(self.tmp.name) + + def write(self,source,rows): + prefix='iclr-program' if source=='iclr_program' else 'openreview-snapshot' + path=self.root/f'{prefix}-2026-main.jsonl' + path.write_text(''.join(json.dumps({'source':source,'venue':'ICLR','year':2026,'track':'main',**row})+'\n' for row in rows)) + manifest={'source':source,'venue':'ICLR','year':2026,'track':'main','parser_version':2,'count':len(rows),'missing_abstract_count':0} + if source=='openreview_snapshot': + manifest['submission_status_counts']={status:sum(r['status']==status for r in rows) for status in set(r['status'] for r in rows)} + path.with_suffix('.manifest.json').write_text(json.dumps(manifest));return path + + def row(self,forum,status='accepted',number=None): + ids=[{'scheme':'openreview','value':forum}] + if number:ids.append({'scheme':'iclr_number','value':str(number)}) + return {'external_id':forum,'identifiers':{'openreview':forum},'submission_identifiers':ids, + 'title':'Title '+forum,'abstract':'Abstract','authors':['Jane Doe'],'status':status, + 'metadata':{'public_readers_verified':True},'decision_events':[] if status=='unknown' else [ + {'status':status,'source_decision_id':'note-'+forum,'authority':{'public_readers_verified':True}}]} + + def test_gaps_and_historical_status_differences_are_explicit_not_inferred(self): + self.write('iclr_program',[self.row('shared'),self.row('programOnly')]) + self.write('openreview_snapshot',[self.row('shared','rejected',1),self.row('snapshotOnly','accepted',2),self.row('unknown','unknown',3)]) + result=audit(self.root,[2026],self.root/'audit.json');report=result['years'][0] + self.assertEqual(report['program_only_forums'],['programOnly']) + self.assertEqual(report['snapshot_accepted_not_in_program'],['snapshotOnly']) + self.assertEqual(report['program_snapshot_status_differences'][0]['snapshot_status'],'rejected') + self.assertEqual(report['snapshot_status_counts']['rejected'],1) + self.assertTrue((self.root/'audit-backfill-queue.json').is_file()) + + def test_conflicting_numbers_fail_before_publishing_a_queue(self): + self.write('iclr_program',[self.row('one')]) + self.write('openreview_snapshot',[self.row('one',number=1),self.row('two',number=1)]) + with self.assertRaisesRegex(ValueError,'different forums'):audit(self.root,[2026],self.root/'audit.json') + self.assertFalse((self.root/'audit-backfill-queue.json').exists()) + + def test_missing_public_decision_evidence_is_rejected(self): + self.write('iclr_program',[self.row('one')]) + row=self.row('one');row['decision_events'][0]['authority']={} + self.write('openreview_snapshot',[row]) + with self.assertRaisesRegex(ValueError,'public decision authority'):audit(self.root,[2026],self.root/'audit.json') + + def test_manifest_counts_cannot_silently_hide_missing_records(self): + path=self.write('iclr_program',[self.row('one')]);path.write_text('') + self.write('openreview_snapshot',[self.row('one')]) + with self.assertRaisesRegex(ValueError,'count mismatch'):audit(self.root,[2026],self.root/'audit.json') + + +if __name__=='__main__':unittest.main() diff --git a/test_import_arxiv.py b/test_import_arxiv.py new file mode 100644 index 0000000..d973cd4 --- /dev/null +++ b/test_import_arxiv.py @@ -0,0 +1,58 @@ +import json +from contextlib import closing +import sqlite3 +import tempfile +import unittest +import zipfile +from pathlib import Path + +from import_arxiv import ingest + + +class ImportTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.db = self.root/'papers.sqlite' + self.archive = self.root/'metadata.zip' + + def write(self, rows): + with zipfile.ZipFile(self.archive,'w') as z: + z.writestr('arxiv-metadata-oai-snapshot.json','\n'.join(json.dumps(r) for r in rows)) + + def row(self, identifier, doi=None): + return dict(id=identifier,title='A shared title',abstract='Searchable abstract',authors_parsed=[['Doe','Jane','']],categories='cs.AI cs.LG',doi=doi,versions=[]) + + def test_identity_provenance_and_idempotency(self): + self.write([self.row('2601.00001','10.0000/example'),self.row('2601.00002','10.0000/example')]) + ingest(self.db,self.archive) + ingest(self.db,self.archive) + with closing(sqlite3.connect(self.db)) as c, c: + self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],1) + self.assertEqual(c.execute('SELECT count(*) FROM source_records').fetchone()[0],2) + self.assertEqual(c.execute("SELECT count(*) FROM papers_fts WHERE papers_fts MATCH 'Searchable'").fetchone()[0],1) + self.assertEqual(c.execute('SELECT count(*) FROM decisions').fetchone()[0],0) + self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(),[]) + self.assertEqual(c.execute('PRAGMA integrity_check').fetchone()[0],'ok') + + def test_title_alone_does_not_merge(self): + self.write([self.row('2601.00001'),self.row('2601.00002')]) + ingest(self.db,self.archive) + with closing(sqlite3.connect(self.db)) as c, c: + self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],2) + + def test_platform_and_custom_tags_remain_distinct(self): + self.write([self.row('2601.00001')]) + ingest(self.db,self.archive) + with closing(sqlite3.connect(self.db)) as c, c: + paper = c.execute('SELECT id FROM papers').fetchone()[0] + tag = c.execute("INSERT INTO tags(namespace,value) VALUES('custom:topic','cs.AI')").lastrowid + c.execute('INSERT INTO paper_tags VALUES(?,?,NULL,?)',(paper,tag,'manual:1')) + ingest(self.db,self.archive) + with closing(sqlite3.connect(self.db)) as c, c: + self.assertEqual(c.execute("SELECT count(*) FROM paper_tags pt JOIN tags t ON pt.tag_id=t.id WHERE t.value='cs.AI'").fetchone()[0],2) + + +if __name__ == '__main__': + unittest.main() diff --git a/test_index_core.py b/test_index_core.py new file mode 100644 index 0000000..c55c331 --- /dev/null +++ b/test_index_core.py @@ -0,0 +1,110 @@ +import json +import sqlite3 +import tempfile +import unittest +from pathlib import Path + +from index_core import Store +from import_records import ingest_file + + +class StoreTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.db = Path(self.tmp.name)/'papers.sqlite' + self.store = Store(self.db) + self.addCleanup(self.store.conn.close) + + def record(self,source,identifier,ids=None,authors=None): + return self.store.record(source,identifier,'https://example.org/'+identifier,'A sufficiently long example research paper title','Example abstract',authors or ['Jane Doe','John Smith'],ids) + + def test_bridge_identifiers_preserves_submission_and_tags(self): + first,record = self.record('arxiv','2601.00001',{'arxiv':'2601.00001'},['First Author']) + second,other_record = self.record('pmlr','v1/other',{'doi':'10.0000/example'},['Other Author']) + self.store.accepted(second,other_record,'ICML',2025,'main','v1/other','https://example.org/other') + tag = self.store.conn.execute("INSERT INTO tags(namespace,value) VALUES('custom:topic','test')").lastrowid + self.store.conn.execute('INSERT INTO paper_tags VALUES(?,?,?,?)',(second,tag,other_record,'manual')) + resolved,_ = self.record('openreview','forum1',{'arxiv':'2601.00001','doi':'10.0000/example'},['Anonymous']) + self.assertEqual(resolved,first) + c=self.store.conn + self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],1) + self.assertEqual(c.execute('SELECT paper_id FROM submissions').fetchone()[0],first) + self.assertEqual(c.execute('SELECT paper_id FROM paper_tags').fetchone()[0],first) + self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(),[]) + self.assertEqual(c.execute('SELECT count(*) FROM paper_merges').fetchone()[0],1) + + def test_cross_platform_exact_title_needs_author_corroboration(self): + paper,_=self.record('arxiv','2601.00001') + same,_=self.record('pmlr','v1/paper') + other,_=self.record('acl','2025.acl.1',authors=['Unrelated Person']) + self.assertEqual(same,paper) + self.assertNotEqual(other,paper) + self.assertEqual(self.store.conn.execute('SELECT count(*) FROM match_candidates').fetchone()[0],1) + + def test_manifest_count_mismatch_rolls_back_records(self): + path=Path(self.tmp.name)/'source.jsonl' + path.write_text(json.dumps({'source':'pmlr','external_id':'v1/paper','url':'https://example.org/paper','title':'Paper','abstract':'Abstract','authors':[],'venue':'ICML','year':2025,'track':'main','status':'accepted'})+'\n') + path.with_suffix('.manifest.json').write_text(json.dumps({'source':'pmlr','list_url':'https://example.org/list','count':2})) + with self.assertRaisesRegex(ValueError,'differs'): + ingest_file(self.db,path) + self.assertEqual(self.store.conn.execute('SELECT count(*) FROM source_records').fetchone()[0],0) + self.assertEqual(self.store.conn.execute('SELECT status FROM ingest_runs').fetchone()[0],'failed') + + def test_source_update_refreshes_canonical_text_and_fts(self): + paper,_=self.record('arxiv','2601.00001') + updated,_=self.store.record('arxiv','2601.00001','https://arxiv.org/abs/2601.00001','Updated research title','New searchable content',['Jane Doe']) + self.assertEqual(updated,paper) + self.assertEqual(self.store.conn.execute('SELECT title FROM papers').fetchone()[0],'Updated research title') + self.assertEqual(self.store.conn.execute('SELECT url FROM source_records').fetchone()[0],'https://arxiv.org/abs/2601.00001') + self.assertEqual(self.store.conn.execute("SELECT count(*) FROM papers_fts WHERE papers_fts MATCH 'searchable'").fetchone()[0],1) + + def test_tag_refresh_keeps_other_record_evidence_and_custom_tags(self): + paper,first=self.record('iclr_program','2025/a',{'openreview':'sharedForum'}) + same,second=self.record('iclr_program','2025/b',{'openreview':'sharedForum'}) + self.store.record_tags(paper,first,'iclr_program',[{'namespace':'iclr_program:topic','value':'Shared'}]) + self.store.record_tags(same,second,'iclr_program',[{'namespace':'iclr_program:topic','value':'Shared'}]) + tag=self.store.conn.execute("INSERT INTO tags(namespace,value) VALUES('custom:topic','Manual')").lastrowid + self.store.conn.execute('INSERT INTO paper_tags VALUES(?,?,?,?)',(paper,tag,None,'manual')) + self.store.record_tags(paper,first,'iclr_program',[]) + self.assertEqual(self.store.conn.execute('SELECT source_record_id FROM paper_tags ORDER BY source_record_id').fetchall(),[(None,),(second,)]) + with self.assertRaisesRegex(ValueError,'namespace'): + self.store.record_tags(paper,second,'iclr_program',[{'namespace':'custom:topic','value':'No overwrite'}]) + self.assertEqual(self.store.conn.execute('SELECT count(*) FROM paper_tags').fetchone()[0],2) + + def test_missing_abstract_does_not_become_complete_coverage(self): + path=Path(self.tmp.name)/'partial.jsonl' + path.write_text(json.dumps({'source':'colm','external_id':'paper1','url':'https://example.org/paper1','title':'Paper','abstract':None,'authors':[],'venue':'COLM','year':2024,'track':'main','status':'accepted'})+'\n') + path.with_suffix('.manifest.json').write_text(json.dumps({'source':'colm','parser_version':2,'list_url':'https://example.org/list','count':1,'status':'partial','missing_abstract_count':1})) + ingest_file(self.db,path) + self.assertEqual(self.store.conn.execute('SELECT status FROM edition_sources').fetchone()[0],'partial') + + def test_legacy_proceedings_manifest_is_rejected_before_import(self): + path=Path(self.tmp.name)/'old.jsonl' + path.write_text('') + path.with_suffix('.manifest.json').write_text(json.dumps({'source':'colm','count':1})) + with self.assertRaisesRegex(ValueError,'parser_version'): + ingest_file(self.db,path) + self.assertEqual(self.store.conn.execute('SELECT count(*) FROM source_records').fetchone()[0],0) + + def test_missing_abstract_is_checked_even_when_manifest_claims_complete(self): + path=Path(self.tmp.name)/'incomplete.jsonl' + row={'source':'pmlr','external_id':'v1/paper','url':'https://example.org/paper','title':'Paper','abstract':None,'authors':[],'venue':'ICML','year':2025,'track':'main','status':'accepted'} + path.write_text(json.dumps(row)+'\n') + path.with_suffix('.manifest.json').write_text(json.dumps({'source':'pmlr','list_url':'https://example.org/list','count':1,'status':'collected','missing_abstract_count':0})) + ingest_file(self.db,path) + self.assertEqual(self.store.conn.execute('SELECT status FROM edition_sources').fetchone()[0],'partial') + + def test_manifest_edition_mismatch_rolls_back_instead_of_marking_coverage(self): + path=Path(self.tmp.name)/'wrong-edition.jsonl' + row={'source':'pmlr','external_id':'v1/paper','url':'https://example.org/paper','title':'Paper','abstract':'Abstract','authors':[],'venue':'ICML','year':2025,'track':'main','status':'accepted'} + path.write_text(json.dumps(row)+'\n') + path.with_suffix('.manifest.json').write_text(json.dumps({'source':'pmlr','list_url':'https://example.org/list','count':1,'year':2024})) + with self.assertRaisesRegex(ValueError,'edition differs'): + ingest_file(self.db,path) + self.assertEqual(self.store.conn.execute('SELECT count(*) FROM source_records').fetchone()[0],0) + self.assertEqual(self.store.conn.execute('SELECT count(*) FROM edition_sources').fetchone()[0],0) + + +if __name__=='__main__': + unittest.main() diff --git a/test_maintenance.py b/test_maintenance.py new file mode 100644 index 0000000..057f9d2 --- /dev/null +++ b/test_maintenance.py @@ -0,0 +1,41 @@ +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from index_core import Store +from merge_conferences import merge + + +class MaintenanceTests(unittest.TestCase): + def fixture(self,tmp,status): + db=Path(tmp)/'papers.sqlite';archive=Path(tmp)/'metadata.zip';archive.write_bytes(b'not read by schema-only maintenance') + store=Store(db);source=store.source('arxiv','https://arxiv.org') + store.conn.execute('INSERT INTO ingest_runs(source_id,status) VALUES(?,?)',(source,status));store.conn.commit();store.conn.close() + return db,archive + + def test_completed_archive_is_not_started_again_after_maintenance(self): + with tempfile.TemporaryDirectory() as tmp: + db,archive=self.fixture(tmp,'complete') + with patch('merge_conferences.property_value',return_value='inactive'),patch('merge_conferences.subprocess.run') as run: + merge(db,archive,[]) + run.assert_not_called() + + def test_inactive_incomplete_arxiv_requires_investigation(self): + with tempfile.TemporaryDirectory() as tmp: + db,archive=self.fixture(tmp,'failed') + with patch('merge_conferences.property_value',return_value='inactive'),patch('merge_conferences.subprocess.run') as run: + with self.assertRaisesRegex(RuntimeError,'neither active nor verified complete'): + merge(db,archive,[]) + run.assert_not_called() + + def test_arxiv_finishing_at_the_stop_boundary_is_not_restarted(self): + with tempfile.TemporaryDirectory() as tmp: + db,archive=self.fixture(tmp,'complete') + with patch('merge_conferences.property_value',return_value='active'),patch('merge_conferences.subprocess.run') as run: + merge(db,archive,[]) + self.assertEqual(run.call_args_list[0].args[0],['systemctl','stop','paper-atlas-arxiv-import.service']) + self.assertEqual(run.call_count,1) + + +if __name__=='__main__':unittest.main() diff --git a/test_openreview_snapshot.py b/test_openreview_snapshot.py new file mode 100644 index 0000000..46e1640 --- /dev/null +++ b/test_openreview_snapshot.py @@ -0,0 +1,108 @@ +import hashlib +import io +import json +from pathlib import Path +import sqlite3 +import tarfile +import tempfile +import unittest + +from collect_openreview_snapshot import collect,decision_record,paper_record +from import_records import ingest_file + + +class SnapshotTests(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.addCleanup(self.tmp.cleanup) + self.root=Path(self.tmp.name) + + def paper(self,forum='paper1',v2=True): + content={'title':'A public paper','abstract':'A public abstract','authors':['Jane Doe'], + 'authorids':['private@example.org'],'keywords':['learning']} + if v2:content={key:{'value':value} for key,value in content.items()} + number=int(forum[-1]) if forum[-1].isdigit() else 42 + return {'id':forum,'forum':forum,'number':number,'readers':['everyone'],'cdate':1760000000000, + 'invitations':['ICLR.cc/2026/Conference/-/Submission'],'content':content} + + def decision(self,forum='paper1',identifier='decision1',value='Reject'): + return {'id':identifier,'forum':forum,'_paper_forum':forum,'readers':['everyone'], + 'invitations':['ICLR.cc/2026/Conference/Submission42/-/Decision'], + 'signatures':['ICLR.cc/2026/Conference/Program_Chairs'],'cdate':1770000000000, + 'content':{'decision':{'value':value},'comment':{'value':'PRIVATE REVIEW BODY'}}} + + def archive(self,papers,reviews): + path=self.root/'snapshot.tar.bz2' + with tarfile.open(path,'w:bz2') as archive: + # Real release order is not guaranteed; decisions precede submissions. + for kind,rows in [('reviews',reviews),('papers',papers)]: + data=(''.join(json.dumps(row)+'\n' for row in rows)).encode() + entry=tarfile.TarInfo('data/iclr_2026/'+kind+'.jsonl');entry.size=len(data) + archive.addfile(entry,io.BytesIO(data)) + return path,hashlib.sha256(path.read_bytes()).hexdigest() + + def test_public_v1_and_v2_metadata_omit_authorids_and_review_bodies(self): + for v2 in (False,True): + row,reason=paper_record(self.paper(v2=v2),2026) + self.assertIsNone(reason);self.assertEqual(row['authors'],['Jane Doe']) + self.assertNotIn('private@example.org',json.dumps(row)) + self.assertEqual(row['identifiers']['openreview'],'paper1') + note=self.paper();note['readers']=['chairs'] + self.assertEqual(paper_record(note,2026)[1],'not_public') + + def test_only_public_signed_formal_decisions_count(self): + note=self.decision();row,reason=decision_record(note,2026) + self.assertIsNone(reason);self.assertEqual(row['status'],'rejected') + self.assertNotIn('PRIVATE REVIEW BODY',json.dumps(row)) + note['invitations']=['ICLR.cc/2026/Conference/Submission42/-/Official_Review'] + self.assertEqual(decision_record(note,2026)[1],'not_decision') + note=self.decision();note['signatures']=['ICLR.cc/2026/Conference/Submission42/Reviewer_1'] + self.assertEqual(decision_record(note,2026)[1],'unverified_signature') + note=self.decision();note['readers']=['chairs'] + self.assertEqual(decision_record(note,2026)[1],'not_public') + note=self.decision(value='Borderline Accept') + self.assertEqual(decision_record(note,2026)[0]['status'],'unknown') + + def test_unordered_snapshot_import_is_idempotent_and_preserves_unknown_and_orphans(self): + path,sha=self.archive([self.paper(),self.paper('paper2'),self.paper('paper3')], + [self.decision(),self.decision('paper2','decision2','Accept (poster)'),self.decision('absent','orphan')]) + out=self.root/'out';manifest=collect(path,out,[2026],sha)[2026] + self.assertEqual(manifest['submission_status_counts'],{'rejected':1,'accepted':1,'unknown':1}) + self.assertEqual(manifest['unmatched_public_decisions'],1) + self.assertEqual(len(Path(manifest['unmatched_evidence_file']).read_text().splitlines()),1) + rows=[json.loads(line) for line in Path(manifest['file']).read_text().splitlines()] + self.assertEqual(rows[2]['decision_events'],[]) + self.assertNotIn('PRIVATE REVIEW BODY',json.dumps(rows)) + db=self.root/'db.sqlite' + ingest_file(db,manifest['file']);ingest_file(db,manifest['file']) + with sqlite3.connect(db) as connection: + self.assertEqual(connection.execute('SELECT count(*) FROM submissions').fetchone()[0],3) + self.assertEqual(connection.execute('SELECT count(*) FROM decisions').fetchone()[0],2) + self.assertEqual(connection.execute('PRAGMA foreign_key_check').fetchall(),[]) + + def test_identical_copies_coalesce_and_conflicting_copies_fail(self): + paper=self.paper();decision=self.decision() + path,sha=self.archive([paper,paper],[decision,decision]) + manifest=collect(path,self.root/'out',[2026],sha)[2026] + self.assertEqual(manifest['count'],1) + self.assertEqual(manifest['archive_counts']['duplicate_decisions'],1) + different=self.paper();different['content']['title']={'value':'Conflicting title'} + path,sha=self.archive([paper,different],[decision]) + with self.assertRaisesRegex(ValueError,'Conflicting copies'): + collect(path,self.root/'bad',[2026],sha) + self.assertFalse((self.root/'bad'/'openreview-snapshot-2026-main.jsonl').exists()) + + def test_checksum_failure_does_not_publish_output(self): + path,sha=self.archive([self.paper()],[self.decision()]) + with self.assertRaisesRegex(ValueError,'checksum mismatch'): + collect(path,self.root/'out',[2026],'0'*64) + self.assertFalse((self.root/'out').exists()) + + def test_legacy_invitation_and_plain_decision_value(self): + note=self.decision();note['invitation']=note.pop('invitations')[0] + note['content']['decision']='Desk Reject' + self.assertEqual(decision_record(note,2026)[0]['status'],'desk_rejected') + note['_paper_forum']='wrong' + self.assertEqual(decision_record(note,2026)[1],'forum_mismatch') + + +if __name__=='__main__':unittest.main() diff --git a/test_peerread.py b/test_peerread.py new file mode 100644 index 0000000..40ac185 --- /dev/null +++ b/test_peerread.py @@ -0,0 +1,33 @@ +import io +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from collect_peerread import collect +from import_records import ingest_file +from index_core import Store + + +class PeerReadTests(unittest.TestCase): + def test_explicit_rejection_and_no_review_body_storage(self): + tree={'sha':'a'*40,'tree':[{'path':'data/iclr_2017/test/reviews/123.json'}]} + paper={'conference':'ICLR 2017 conference submission','id':'123','title':'Public paper','abstract':'Abstract','authors':'Jane Doe, John Smith','accepted':False,'reviews':[{'text':'Do not persist this review body'}]} + with tempfile.TemporaryDirectory() as tmp: + root=Path(tmp) + with patch('collect_peerread.fetch',side_effect=[json.dumps({'sha':'a'*40}).encode(),json.dumps(tree).encode()]),patch('collect_peerread.urllib.request.urlopen',return_value=io.BytesIO(json.dumps(paper).encode())),patch('collect_peerread.time.sleep'): + collect(root/'cache',root/'out') + path=root/'out/peerread-iclr2017-main.jsonl' + record=json.loads(path.read_text()) + self.assertEqual(record['status'],'rejected') + self.assertNotIn('Do not persist',path.read_text()) + ingest_file(root/'db.sqlite',path) + store=Store(root/'db.sqlite') + self.addCleanup(store.conn.close) + self.assertEqual(store.conn.execute('SELECT status FROM decisions').fetchone()[0],'rejected') + self.assertEqual(store.conn.execute('SELECT status FROM edition_sources').fetchone()[0],'complete') + + +if __name__=='__main__': + unittest.main() diff --git a/test_pmlr.py b/test_pmlr.py new file mode 100644 index 0000000..0af571d --- /dev/null +++ b/test_pmlr.py @@ -0,0 +1,60 @@ +import unittest +import tempfile +import json +from pathlib import Path +from unittest.mock import patch + +from collect_pmlr import load_metadata,venue_key,collect_volume + + +class MetadataTests(unittest.TestCase): + def test_legacy_binary_scalar_is_decoded_as_text(self): + self.assertEqual(load_metadata('title: !binary |\n SGVsbG8=\n')['title'],'Hello') + + def test_trailing_empty_document_is_allowed(self): + self.assertEqual(load_metadata('---\nyear: 2024\n---\n'),{'year':2024}) + + def test_conflicting_documents_are_not_silently_combined(self): + with self.assertRaisesRegex(ValueError,'exactly one'): + load_metadata('year: 2024\n---\nyear: 2025\n') + + def test_historical_conference_name_aliases(self): + self.assertEqual(venue_key('AISTATS 2010'),'AISTATS') + self.assertEqual(venue_key('AISTATS 2012'),'AISTATS') + self.assertEqual(venue_key('COLT 2012'),'COLT') + self.assertEqual(venue_key('ECML-PKDD'),'ECML-PKDD') + + def test_missing_abstract_is_reported_as_partial_collection(self): + config=b'shortname: ICML\nyear: 2025\nbooktitle: ICML\n' + paper='---\nid: paper1\nlayout: inproceedings\ntitle: Paper\nabstract:\n---\n' + with tempfile.TemporaryDirectory() as tmp,patch('collect_pmlr.fetch',return_value=config),patch('collect_pmlr.metadata_texts',return_value=iter([('paper.md',paper)])): + result=collect_volume(999,tmp,tmp) + self.assertEqual(result['count'],1) + self.assertEqual(result['missing_abstract_count'],1) + self.assertEqual(result['status'],'partial') + + def test_duplicate_publication_posts_preserve_date_evidence_once(self): + config=b'shortname: CoRL\nyear: 2023\nbooktitle: CoRL\n' + paper='---\nid: paper1\nlayout: inproceedings\ntitle: Paper\nabstract: Real abstract\ndate: {date}\n---\n' + posts=[('2023-11-13-paper1.md',paper.format(date='2023-11-13')),('2023-12-02-paper1.md',paper.format(date='2023-12-02'))] + with tempfile.TemporaryDirectory() as tmp,patch('collect_pmlr.fetch',return_value=config),patch('collect_pmlr.metadata_texts',return_value=iter(posts)): + result=collect_volume(229,tmp,tmp) + rows=[json.loads(line) for line in Path(result['file']).read_text().splitlines()] + self.assertEqual(result['count'],1) + self.assertEqual(result['metadata_post_count'],2) + self.assertEqual(result['duplicate_post_count'],1) + self.assertEqual(len(rows[0]['metadata']['source_versions']),2) + self.assertEqual(rows[0]['metadata']['date'],'2023-12-02') + + def test_conflicting_duplicate_posts_do_not_replace_previous_collection(self): + config=b'shortname: CoRL\nyear: 2023\nbooktitle: CoRL\n' + paper='---\nid: paper1\nlayout: inproceedings\ntitle: {title}\nabstract: Abstract\n---\n' + with tempfile.TemporaryDirectory() as tmp,patch('collect_pmlr.fetch',return_value=config),patch('collect_pmlr.metadata_texts',return_value=iter([('a.md',paper.format(title='First paper')),('b.md',paper.format(title='Other paper'))])): + previous=Path(tmp)/'pmlr-v229.jsonl';previous.write_text('previous valid collection') + with self.assertRaisesRegex(ValueError,'Conflicting metadata posts'): + collect_volume(229,tmp,tmp) + self.assertEqual(previous.read_text(),'previous valid collection') + + +if __name__=='__main__': + unittest.main() diff --git a/test_proceedings.py b/test_proceedings.py new file mode 100644 index 0000000..4c9f607 --- /dev/null +++ b/test_proceedings.py @@ -0,0 +1,66 @@ +import unittest +import json +import runpy +import tempfile +from pathlib import Path +from unittest.mock import patch + +from collect_proceedings import paper_links,parse_paper + + +class ProceedingsTests(unittest.TestCase): + def test_legacy_empty_class_attribute_does_not_break_a_paper(self): + html='
Legacy formatting

Paper

Real abstract.

' + row=parse_paper(html,'https://proceedings.neurips.cc/paper/2000/hash/a-Abstract.html','neurips',2000,'https://proceedings.neurips.cc/paper/2000') + self.assertEqual(row['abstract'],'Real abstract.') + + def test_colm_nested_abstract_and_citation_authors(self): + html='''

Abstract

Real abstract text.

Name Change Policy

Do not ingest footer.

''' + row=parse_paper(html,'https://proceedings.colm.cc/paper_files/paper/2024/hash/a-Abstract-Conference.html','colm',2024,'https://proceedings.colm.cc/paper_files/paper/2024') + self.assertEqual(row['abstract'],'Real abstract text.') + self.assertEqual(row['authors'],['Jane Doe']) + self.assertEqual(row['track'],'main') + self.assertEqual(row['year'],2024) + + def test_declared_index_count_detects_truncation(self): + html='

2 papers

Paper' + with self.assertRaisesRegex(ValueError,'count differs'): + paper_links(html,'https://proceedings.colm.cc/paper/2024') + + def test_missing_abstract_does_not_capture_footer(self): + row=parse_paper('

Paper

Name Change Policy

Footer

','https://example.org/a','colm',2024,'https://example.org/') + self.assertIsNone(row['abstract']) + + def test_paper_title_is_not_mistaken_for_index_paper_count(self): + html='
  • Lessons from 1000 papers
    • ' + self.assertEqual(len(paper_links(html,'https://datasets-benchmarks-proceedings.neurips.cc/paper/2021')),1) + + def test_neurips_track_and_legacy_abstract(self): + html='

    Paper

    Datasets and Benchmarks Track

    Abstract

    Actual abstract.

    Name Change Policy

    Footer

    ' + row=parse_paper(html,'https://proceedings.neurips.cc/a','neurips',2025,'https://proceedings.neurips.cc/') + self.assertEqual(row['abstract'],'Actual abstract.') + self.assertEqual(row['track'],'datasets_benchmarks') + + def test_focused_retry_preserves_other_index_status(self): + root='https://proceedings.colm.cc/' + url=root+'paper_files/paper/2024' + other=root+'paper_files/paper/2025' + paper=root+'paper_files/paper/2024/hash/a-Abstract-Conference.html' + documents={root:f'20242025',url:f'

    1 papers

    Paper',paper:'

    Paper

    Abstract

    '} + with tempfile.TemporaryDirectory() as tmp: + status=Path(tmp)/'colm-index-status.json' + status.write_text(json.dumps({other:{'status':'collected','files':[]}})) + argv=['collect_proceedings.py','--source','colm','--cache',tmp,'--out',tmp,'--year','2024'] + with patch('sys.argv',argv),patch('collect_pmlr.fetch',side_effect=lambda u,*a,**kw:documents[u].encode()): + runpy.run_path(str(Path(__file__).with_name('collect_proceedings.py')),run_name='__main__') + result=json.loads(status.read_text()) + self.assertEqual(result[other]['status'],'collected') + self.assertEqual(result[url]['files'][0]['count'],1) + + def test_abstract_citations_are_not_used_as_self_identifiers(self): + html='
    OpenReview

    Related work and another paper.

    ' + row=parse_paper(html,'https://proceedings.colm.cc/paper','colm',2024,'https://proceedings.colm.cc/') + self.assertEqual(row['identifiers'],{'openreview':'selfForum'}) + + +if __name__=='__main__':unittest.main() diff --git a/test_query.py b/test_query.py new file mode 100644 index 0000000..068d4b9 --- /dev/null +++ b/test_query.py @@ -0,0 +1,51 @@ +import json +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +from index_core import Store + + +class QueryTests(unittest.TestCase): + def test_joint_conference_alias_does_not_duplicate_submission(self): + with tempfile.TemporaryDirectory() as tmp: + db=Path(tmp)/'papers.sqlite';store=Store(db);self.addCleanup(store.conn.close) + paper,record=store.record('acl_anthology','paper1','https://example.org/1','Joint event paper','Abstract',[]) + store.decision(paper,record,'COLING+LREC',2024,'main','paper1','https://example.org/1','accepted','official XML',['COLING','LREC']) + store.conn.commit() + for venue in ('COLING','LREC'): + command=[sys.executable,'-B',str(Path(__file__).with_name('query.py')),'--db',str(db),'--venue',venue,'--track','main'] + rows=json.loads(subprocess.run(command,check=True,capture_output=True,text=True).stdout) + self.assertEqual([r['id'] for r in rows],[paper]) + self.assertEqual(store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0],1) + + def test_venue_filter_is_case_insensitive_and_uses_latest_decision(self): + with tempfile.TemporaryDirectory() as tmp: + db=Path(tmp)/'papers.sqlite' + store=Store(db) + self.addCleanup(store.conn.close) + paper,record=store.record('peerread','paper1','https://example.org/paper1','A real submission','Abstract',['Jane Doe']) + store.decision(paper,record,'ICLR',2017,'main','paper1','https://example.org/paper1','rejected','explicit rejection') + store.record('arxiv','2601.00001','https://example.org/other','Another paper','Abstract',['John Smith']) + store.conn.commit() + command=[sys.executable,'-B',str(Path(__file__).with_name('query.py')),'--db',str(db),'--venue','iclr','--year','2017','--status','rejected'] + result=json.loads(subprocess.run(command,check=True,capture_output=True,text=True).stdout) + self.assertEqual([r['id'] for r in result],[paper]) + details_command=[sys.executable,'-B',str(Path(__file__).with_name('query.py')),'--db',str(db),'--paper',str(paper)] + details=json.loads(subprocess.run(details_command,check=True,capture_output=True,text=True).stdout)[0] + self.assertEqual(details['authors'],['Jane Doe']) + self.assertEqual(details['submissions'][0]['decisions'][0]['status'],'rejected') + self.assertEqual(details['sources'][0]['external_id'],'paper1') + stats_command=[sys.executable,'-B',str(Path(__file__).with_name('query.py')),'--db',str(db),'--stats'] + stats=json.loads(subprocess.run(stats_command,check=True,capture_output=True,text=True).stdout) + self.assertEqual(stats['papers'],2) + self.assertEqual({r['name']:r['imported_records'] for r in stats['records_by_source']},{'arxiv':1,'peerread':1}) + store.decision(paper,record,'ICLR',2017,'main','paper1','https://example.org/paper1','accepted','changed decision') + store.conn.commit() + self.assertEqual(json.loads(subprocess.run(command,check=True,capture_output=True,text=True).stdout),[]) + + +if __name__=='__main__': + unittest.main() diff --git a/test_submission_evidence.py b/test_submission_evidence.py new file mode 100644 index 0000000..71e195c --- /dev/null +++ b/test_submission_evidence.py @@ -0,0 +1,173 @@ +import json +from pathlib import Path +import sqlite3 +import subprocess +import sys +import tempfile +import unittest + +from import_records import ingest_file +from index_core import Store +from submission_evidence import identities_for_row +from backfill_submission_evidence import backfill,queue_files + + +class SubmissionTests(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory();self.addCleanup(self.tmp.cleanup) + self.db=Path(self.tmp.name)/'papers.sqlite';self.store=Store(self.db);self.addCleanup(self.store.conn.close) + + def record(self,source,external,authors): + return self.store.record(source,external,'https://example.org/'+external,'A long paper about reliable metadata and historical events','Abstract',authors) + + def decision(self,paper,record,external,identities,status='accepted',raw='Accept',url='https://example.org/decision',observed='2026-05-08T00:00:00Z',date=None): + return self.store.decision(paper,record,'ICLR',2026,'main',external,url,status,raw,identities=identities,observed_at=observed,decision_date=date) + + def query(self,*args): + self.store.conn.commit() + command=[sys.executable,'-B',str(Path(__file__).with_name('query.py')),'--db',str(self.db),*args] + result=subprocess.run(command,capture_output=True,text=True) + self.assertEqual(result.returncode,0,result.stderr) + return json.loads(result.stdout) + + def test_same_submission_across_sources_has_one_event_and_both_provenances(self): + first,one=self.store.record('iclr_program','2026/number-42','https://example.org/program','A long paper about reliable metadata and historical events',None,['Jane Doe','John Smith']) + second,two=self.record('openreview_snapshot','realForum',['Anonymous']) + self.assertNotEqual(first,second) + self.decision(first,one,'2026/number-42',[('iclr_number','42')]) + self.decision(second,two,'realForum',[('iclr_number','42'),('openreview','realForum')],url='https://openreview.net/forum?id=realForum¬eId=note1') + c=self.store.conn + self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],1) + self.assertEqual(c.execute('SELECT abstract FROM papers').fetchone()[0],'Abstract') + self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],1) + self.assertEqual(c.execute('SELECT count(*) FROM submission_sources').fetchone()[0],2) + details=self.query('--paper',str(first))[0]['submissions'][0] + self.assertEqual({r['name'] for r in details['sources']},{'iclr_program','openreview_snapshot'}) + self.assertEqual(len(details['decisions']),2) + self.assertTrue(all(d['sources'] for d in details['decisions'])) + self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(),[]) + + def test_same_paper_can_have_two_distinct_submission_attempts(self): + paper,one=self.store.record('openreview_snapshot','forum1','https://example.org/1','First version','Abstract',[],{'arxiv':'2601.00001','openreview':'forum1'}) + other,two=self.store.record('openreview_snapshot','forum2','https://example.org/2','Another version','Abstract',[],{'arxiv':'2601.00001','openreview':'forum2'}) + self.assertEqual(paper,other) + self.decision(paper,one,'forum1',[('openreview','forum1'),('iclr_number','1')],status='rejected',raw='Reject') + self.decision(paper,two,'forum2',[('openreview','forum2'),('iclr_number','2')]) + self.assertEqual(self.store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0],2) + + def test_conflicting_forums_cannot_be_bridged_by_one_submission_number(self): + paper,one=self.record('snapshot','realForum',['Jane Doe']) + other,two=self.record('snapshot','differentForum',['Other Author']) + self.decision(paper,one,'realForum',[('openreview','realForum'),('iclr_number','42')]) + with self.assertRaisesRegex(ValueError,'Conflicting authoritative submission identities'): + self.decision(other,two,'differentForum',[('openreview','differentForum'),('iclr_number','42')]) + self.assertEqual(self.store.conn.execute('SELECT count(*) FROM papers').fetchone()[0],2) + self.assertEqual(self.store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0],1) + self.assertEqual(self.store.conn.execute('SELECT count(*) FROM submission_merges').fetchone()[0],0) + + def test_late_identity_bridge_preserves_decisions_and_does_not_reuse_merged_ids(self): + first,one=self.record('iclr_program','number-42',['Jane Doe','John Smith']) + second,two=self.record('openreview_snapshot','realForum',['Anonymous']) + self.decision(first,one,'number-42',[('iclr_number','42')]) + self.decision(second,two,'realForum',[('openreview','realForum')]) + self.decision(second,two,'realForum',[('openreview','realForum'),('iclr_number','42')]) + c=self.store.conn + self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],1) + self.assertEqual(c.execute('SELECT count(*) FROM decisions').fetchone()[0],1) + self.assertEqual(c.execute('SELECT count(*) FROM decision_sources').fetchone()[0],2) + self.assertEqual(c.execute('SELECT count(*) FROM submission_merges').fetchone()[0],1) + self.assertEqual(c.execute('SELECT count(*) FROM decision_merges').fetchone()[0],1) + third,record=self.record('other','different',['Other Author']) + self.decision(third,record,'different',[('openreview','differentForum')],status='rejected',raw='Reject') + self.assertGreater(c.execute('SELECT max(id) FROM submissions').fetchone()[0],2) + self.assertGreater(c.execute('SELECT max(id) FROM decisions').fetchone()[0],2) + self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(),[]) + + def test_old_snapshot_does_not_override_a_newer_official_observation(self): + paper,one=self.record('iclr_program','number-42',['Jane Doe','John Smith']) + _,two=self.record('openreview_snapshot','realForum',['Jane Doe','John Smith']) + self.decision(paper,one,'number-42',[('iclr_number','42')],observed='2026-11-06T00:00:00Z') + self.decision(paper,two,'realForum',[('iclr_number','42')],status='rejected',raw='Reject',observed='2026-05-08T00:00:00Z') + self.assertEqual(len(self.query('--venue','ICLR','--status','accepted')),1) + self.assertEqual(self.query('--venue','ICLR','--status','rejected'),[]) + + def test_decision_date_orders_events_with_the_same_snapshot_time(self): + paper,record=self.record('openreview_snapshot','realForum',['Jane Doe']) + self.decision(paper,record,'realForum',[('openreview','realForum')],date='2026-02-01T00:00:00Z') + self.decision(paper,record,'realForum',[('openreview','realForum')],status='rejected',raw='Reject',date='2026-01-01T00:00:00Z') + self.assertEqual(len(self.query('--venue','ICLR','--status','accepted')),1) + self.assertEqual(self.query('--venue','ICLR','--status','rejected'),[]) + + def test_unknown_without_a_decision_is_queryable_without_invented_verdict(self): + path=Path(self.tmp.name)/'unknown.jsonl' + row={'source':'openreview_snapshot','external_id':'realForum','url':'https://example.org/paper','title':'Unknown result','abstract':'Abstract','authors':[],'identifiers':{'openreview':'realForum'},'venue':'ICLR','year':2026,'track':'main','status':'unknown','decision_events':[]} + path.write_text(json.dumps(row)+'\n');path.with_suffix('.manifest.json').write_text(json.dumps({'source':row['source'],'list_url':'https://example.org/list','count':1})) + ingest_file(self.db,path) + self.assertEqual(self.store.conn.execute('SELECT count(*) FROM decisions').fetchone()[0],0) + self.assertEqual(len(self.query('--venue','ICLR','--status','unknown')),1) + + def test_known_summary_without_corresponding_decision_rolls_back(self): + path=Path(self.tmp.name)/'missing-evidence.jsonl' + row={'source':'openreview_snapshot','external_id':'realForum','url':'https://example.org/paper','title':'Unsupported result','abstract':'Abstract','authors':[],'venue':'ICLR','year':2026,'track':'main','status':'accepted','decision_events':[{'status':None}]} + path.write_text(json.dumps(row)+'\n');path.with_suffix('.manifest.json').write_text(json.dumps({'source':row['source'],'list_url':'https://example.org/list','count':1})) + with self.assertRaisesRegex(ValueError,'no corresponding decision evidence'): + ingest_file(self.db,path) + self.assertEqual(self.store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0],0) + + def test_event_number_is_not_assumed_to_be_a_submission_number(self): + row={'source':'iclr_program','venue':'ICLR','metadata':{'program_source_id':42,'program_identity_type':'event'},'identifiers':{}} + self.assertEqual(identities_for_row(row),[]) + row['metadata']['program_identity_type']='submission' + self.assertEqual(identities_for_row(row),[]) + row['metadata']['program_submission_number_verified']=True + self.assertEqual(identities_for_row(row),[('iclr_number','42')]) + with self.assertRaisesRegex(ValueError,'Invalid authoritative'): + identities_for_row({'source':'snapshot','venue':'ICLR','submission_identifiers':[{'scheme':'openreview','value':'2026-Oral--42-placeholder'}]}) + + def test_legacy_database_migration_preserves_source_and_decision_links(self): + legacy=Path(self.tmp.name)/'legacy.sqlite';sql=Path(__file__).with_name('schema.sql').read_text() + sql=sql[:sql.index('CREATE TABLE IF NOT EXISTS submission_sources')]+sql[sql.index('CREATE TABLE IF NOT EXISTS tags'):] + c=sqlite3.connect(legacy);c.executescript(sql) + c.execute("INSERT INTO sources VALUES(1,'legacy','https://example.org')") + c.execute("INSERT INTO papers(id,title,title_key) VALUES(1,'Paper','paper')") + c.execute("INSERT INTO source_records VALUES(1,1,'paper',1,'https://example.org/p','Paper','Abstract','[]','{}',CURRENT_TIMESTAMP)") + c.execute("INSERT INTO venues VALUES(1,'ICLR','ICLR','machine_learning','conference')") + c.execute("INSERT INTO editions VALUES(1,1,2026,'main')") + c.execute("INSERT INTO submissions VALUES(1,1,1,1,'paper',NULL)") + c.execute("INSERT INTO decisions VALUES(1,1,'accepted','Accept','https://example.org/decision','2026-01-01T00:00:00Z',NULL)") + c.commit();c.close() + store=Store(legacy);self.addCleanup(store.conn.close) + self.assertEqual(store.conn.execute('SELECT * FROM submission_sources').fetchall(),[(1,1,'paper')]) + self.assertEqual(store.conn.execute('SELECT * FROM decision_sources').fetchall(),[(1,1,'')]) + self.assertEqual(store.conn.execute('PRAGMA foreign_key_check').fetchall(),[]) + + def test_existing_program_identity_backfill_is_idempotent_and_preserves_text(self): + paper,record=self.record('iclr_program','2026/program-42',['Jane Doe','John Smith']) + self.decision(paper,record,'2026/program-42',[]) + self.store.conn.commit() + original=self.store.conn.execute('SELECT title,abstract,retrieved_at FROM source_records WHERE id=?',(record,)).fetchone() + path=Path(self.tmp.name)/'program.jsonl' + row={'source':'iclr_program','external_id':'2026/program-42','venue':'ICLR','year':2026,'track':'main','status':'accepted','identifiers':{},'metadata':{'program_identity_type':'submission','program_source_id':42,'program_submission_number_verified':True}} + path.write_text(json.dumps(row)+'\n');path.with_suffix('.manifest.json').write_text(json.dumps({'source':'iclr_program','parser_version':2,'venue':'ICLR','year':2026,'track':'main','count':1})) + first=backfill(self.db,[path]);second=backfill(self.db,[path]) + self.assertEqual(first,second) + self.assertEqual(self.store.conn.execute('SELECT scheme,value FROM submission_identifiers').fetchall(),[('iclr_number','42')]) + self.assertEqual(self.store.conn.execute('SELECT title,abstract,retrieved_at FROM source_records WHERE id=?',(record,)).fetchone(),original) + other,snapshot=self.record('openreview_snapshot','realForum',['Anonymous']) + self.decision(other,snapshot,'realForum',[('iclr_number','42'),('openreview','realForum')]) + self.assertEqual(self.store.conn.execute('SELECT count(*) FROM submissions').fetchone()[0],1) + + def test_evidence_queue_rejects_changed_file_before_mutation(self): + path=Path(self.tmp.name)/'program.jsonl';path.write_text('changed bytes') + queue=Path(self.tmp.name)/'queue.json';queue.write_text(json.dumps({'files':[str(path)],'file_sha256':{str(path):'0'*64}})) + with self.assertRaisesRegex(ValueError,'differs from audited'): + queue_files(queue) + + def test_old_untyped_program_files_require_recollection(self): + path=Path(self.tmp.name)/'old.jsonl';path.write_text('') + path.with_suffix('.manifest.json').write_text(json.dumps({'source':'iclr_program','parser_version':1})) + with self.assertRaisesRegex(ValueError,'Recollect ICLR'): + ingest_file(self.db,path) + + +if __name__=='__main__':unittest.main() diff --git a/test_vision.py b/test_vision.py new file mode 100644 index 0000000..e236bc4 --- /dev/null +++ b/test_vision.py @@ -0,0 +1,72 @@ +import tempfile +import unittest +from unittest.mock import patch + +from collect_vision import collect_index, decode_html, discover, index_entries, parse_paper + + +class VisionTests(unittest.TestCase): + def entry(self): + return {'url':'https://www.ecva.net/papers/eccv_2024/papers_ECCV/html/1_ECCV_2024_paper.php','index_title':'Paper','index_authors':'Jane Doe*, John Smith','doi':'10.1007/example_1'} + + def test_ecva_doi_belongs_to_its_own_title_entry(self): + html='
    First paper
    Jane Doe*, John Smith
    DOI
    Second paper
    Other Author
    DOI
    ' + entries,count=index_entries(html,'https://www.ecva.net/papers.php') + self.assertEqual(count,2) + self.assertEqual([e['doi'] for e in entries],['10.1007/example_1','10.1007/example_2']) + html='
    Paper
    Abstract citing another paper and a reference.
    ' + row=parse_paper(html,entries[0],'ecva','ECCV',2024,'main','https://www.ecva.net/papers.php') + self.assertEqual(row['identifiers'],{'doi':'10.1007/example_1'}) + self.assertEqual(row['authors'],['Jane Doe','John Smith']) + + def test_cvf_meta_authors_and_missing_abstract_never_capture_footer(self): + html='
    Do not include the proceedings citation

    Copyright footer

    ' + entry=dict(self.entry(),doi=None) + row=parse_paper(html,entry,'cvf','CVPR',2013,'main','https://openaccess.thecvf.com/CVPR2013') + self.assertEqual(row['authors'],['Jane Doe']) + self.assertIsNone(row['abstract']) + self.assertEqual(row['title'],'A vision paper') + + def test_conflicting_self_doi_is_rejected(self): + html='' + with self.assertRaisesRegex(ValueError,'DOI differs'): + parse_paper(html,self.entry(),'ecva','ECCV',2024,'main','https://www.ecva.net/papers.php') + + def test_legacy_ecva_author_formats_do_not_become_extra_people(self): + entry=dict(self.entry(),index_authors='Doe, Jane and Smith, John') + page='
    Paper
    Jane Doe, John Smith; The European Conference on Computer Vision (ECCV), 2018, pp. 1-12
    ' + row=parse_paper(page,entry,'ecva','ECCV',2018,'main','https://www.ecva.net/papers.php') + self.assertEqual(row['authors'],['Jane Doe','John Smith']) + row=parse_paper('
    Paper
    ',entry,'ecva','ECCV',2018,'main','https://www.ecva.net/papers.php') + self.assertEqual(row['authors'],['Jane Doe','John Smith']) + + def test_cvpr_day_one_is_replaced_by_explicit_all_papers_index(self): + root='https://openaccess.thecvf.com/CVPR2026';all_url=root+'?day=all' + first='https://openaccess.thecvf.com/content/CVPR2026/html/First_CVPR_2026_paper.html' + second='https://openaccess.thecvf.com/content/CVPR2026/html/Second_CVPR_2026_paper.html' + def item(url):return f'
    Paper
    Jane Doe, John Smith
    [pdf]
    ' + pages={root:f'All papers
    {item(first)}
    ',all_url:f'
    {item(first)}{item(second)}
    ',first:'
    First paper
    First abstract
    ',second:'
    Second paper

    Footer without an abstract

    '} + with tempfile.TemporaryDirectory() as tmp,patch('collect_vision.fetch',side_effect=lambda url,*a,**kw:pages[url].encode()) as fetch: + manifest=collect_index('cvf',{'venue':'CVPR','year':2026,'track':'main','index_url':root},tmp,tmp) + self.assertEqual(manifest['count'],2) + self.assertEqual(manifest['list_url'],all_url) + self.assertEqual(manifest['status'],'partial') + self.assertEqual(manifest['missing_abstract_count'],1) + self.assertEqual({call.args[0] for call in fetch.call_args_list},{root,all_url,first,second}) + + def test_discovery_keeps_findings_separate_and_excludes_workshops(self): + html='MainFindingsMainLegacy mainWorkshopsWACV' + with patch('collect_vision.fetch',return_value=html.encode()):indexes=discover('cvf','unused') + self.assertEqual({(x['venue'],x['year'],x['track']) for x in indexes},{('CVPR',2013,'main'),('CVPR',2026,'main'),('CVPR',2026,'findings'),('ICCV',2025,'main')}) + + def test_unparsed_publication_link_does_not_silently_reduce_coverage(self): + html='Other paper
    Paper
    ' + with self.assertRaisesRegex(ValueError,'outside parsed'): + index_entries(html,'https://openaccess.thecvf.com/') + + def test_legacy_declared_encoding_is_respected(self): + html='
    André
    ' + self.assertIn('André',decode_html(html.encode('iso-8859-1'))) + + +if __name__=='__main__':unittest.main() diff --git a/venues.json b/venues.json new file mode 100644 index 0000000..1784f8a --- /dev/null +++ b/venues.json @@ -0,0 +1,658 @@ +{ + "scope_note": "Broad field coverage, not a claim of objective ranking relative to COLM. Main tracks, Findings, workshops, journals remain distinct. Missing editions must be reported, not silently skipped.", + "sources": { + "arxiv": "https://info.arxiv.org/help/bulk_data/index.html", + "openreview": "https://docs.openreview.net/getting-started/using-the-api", + "acl_anthology": "https://github.com/acl-org/acl-anthology/tree/master/data/xml", + "pmlr": "https://proceedings.mlr.press/", + "neurips": "https://proceedings.neurips.cc/", + "cvf": "https://openaccess.thecvf.com/", + "ecva": "https://www.ecva.net/papers.php", + "aaai": "https://ojs.aaai.org/index.php/AAAI/issue/archive", + "ijcai": "https://www.ijcai.org/all_proceedings/", + "dblp": "https://dblp.org/db/conf/", + "colm": "https://proceedings.colm.cc/", + "iclr_program": "https://iclr.cc/virtual/2026/papers.html" + }, + "venues": [ + { + "acronym": "NeurIPS", + "field": "machine_learning", + "venue_type": "conference", + "discovery_sources": [ + "openreview", + "neurips" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "ICML", + "field": "machine_learning", + "venue_type": "conference", + "discovery_sources": [ + "openreview", + "pmlr" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "ICLR", + "field": "machine_learning", + "venue_type": "conference", + "discovery_sources": [ + "openreview", + "iclr_program" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "COLM", + "field": "machine_learning", + "venue_type": "conference", + "discovery_sources": [ + "openreview", + "colm" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "COLT", + "field": "machine_learning", + "venue_type": "conference", + "discovery_sources": [ + "openreview", + "pmlr", + "neurips" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "AISTATS", + "field": "machine_learning", + "venue_type": "conference", + "discovery_sources": [ + "openreview", + "pmlr", + "neurips" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "UAI", + "field": "machine_learning", + "venue_type": "conference", + "discovery_sources": [ + "openreview", + "pmlr", + "neurips" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "CLeaR", + "field": "machine_learning", + "venue_type": "conference", + "discovery_sources": [ + "openreview", + "pmlr", + "neurips" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "ACL", + "field": "language", + "venue_type": "conference", + "discovery_sources": [ + "acl_anthology", + "openreview" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "EMNLP", + "field": "language", + "venue_type": "conference", + "discovery_sources": [ + "acl_anthology", + "openreview" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "NAACL", + "field": "language", + "venue_type": "conference", + "discovery_sources": [ + "acl_anthology", + "openreview" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "EACL", + "field": "language", + "venue_type": "conference", + "discovery_sources": [ + "acl_anthology", + "openreview" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "COLING", + "field": "language", + "venue_type": "conference", + "discovery_sources": [ + "acl_anthology", + "openreview" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "CoNLL", + "field": "language", + "venue_type": "conference", + "discovery_sources": [ + "acl_anthology", + "openreview" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "CVPR", + "field": "vision", + "venue_type": "conference", + "discovery_sources": [ + "cvf", + "dblp" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "ICCV", + "field": "vision", + "venue_type": "conference", + "discovery_sources": [ + "cvf", + "dblp" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "ECCV", + "field": "vision", + "venue_type": "conference", + "discovery_sources": [ + "ecva", + "dblp" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "AAAI", + "field": "general_ai", + "venue_type": "conference", + "discovery_sources": [ + "aaai", + "ijcai", + "dblp" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "IJCAI", + "field": "general_ai", + "venue_type": "conference", + "discovery_sources": [ + "aaai", + "ijcai", + "dblp" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "ECAI", + "field": "general_ai", + "venue_type": "conference", + "discovery_sources": [ + "aaai", + "ijcai", + "dblp" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "KDD", + "field": "retrieval_data", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "publisher" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "SIGIR", + "field": "retrieval_data", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "publisher" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "WWW", + "field": "retrieval_data", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "publisher" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "WSDM", + "field": "retrieval_data", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "publisher" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "RecSys", + "field": "retrieval_data", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "publisher" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "CIKM", + "field": "retrieval_data", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "publisher" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "ICDM", + "field": "retrieval_data", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "publisher" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "SDM", + "field": "retrieval_data", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "publisher" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "ECML-PKDD", + "field": "retrieval_data", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "publisher" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "ICRA", + "field": "robotics", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "pmlr", + "rss" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "IROS", + "field": "robotics", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "pmlr", + "rss" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "RSS", + "field": "robotics", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "pmlr", + "rss" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "CoRL", + "field": "robotics", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "pmlr", + "rss" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "ICAPS", + "field": "reasoning_agents", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "proceedings" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "KR", + "field": "reasoning_agents", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "proceedings" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "AAMAS", + "field": "reasoning_agents", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "proceedings" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "ICASSP", + "field": "speech_audio", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "isca", + "ismir" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "INTERSPEECH", + "field": "speech_audio", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "isca", + "ismir" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "ISMIR", + "field": "speech_audio", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "isca", + "ismir" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "MICCAI", + "field": "specialist", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "publisher" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + }, + { + "acronym": "FAccT", + "field": "specialist", + "venue_type": "conference", + "discovery_sources": [ + "dblp", + "publisher" + ], + "scope": "candidate_pending_scope_review", + "years": "all_publicly_available", + "tracks": [ + "main" + ], + "coverage": "pending" + } + ], + "additional_tracks": [ + "Findings", + "workshop" + ], + "additional_nonconference_venues": [ + "TMLR", + "JMLR", + "TACL", + "ACL ARR" + ] +}