Compare commits
5 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 33bb378369 | |||
| 7ed5966d3a | |||
| 871b2aa81e | |||
| b14193063a | |||
| f139c7e49d |
@@ -12,7 +12,7 @@ jobs:
|
|||||||
- name: Verify metadata storage and query commands
|
- name: Verify metadata storage and query commands
|
||||||
run: |
|
run: |
|
||||||
python3 -B - <<'PY'
|
python3 -B - <<'PY'
|
||||||
import gzip, io, json, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request
|
import base64, gzip, io, json, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request
|
||||||
sha = os.environ['ARCHIVE_SHA']
|
sha = os.environ['ARCHIVE_SHA']
|
||||||
if not re.fullmatch(r'[0-9a-fA-F]{40,64}', sha):
|
if not re.fullmatch(r'[0-9a-fA-F]{40,64}', sha):
|
||||||
raise SystemExit('Invalid source commit')
|
raise SystemExit('Invalid source commit')
|
||||||
@@ -135,6 +135,21 @@ jobs:
|
|||||||
(rss_snapshots/'receipts.json').write_text(json.dumps(rss_fixture['receipts']))
|
(rss_snapshots/'receipts.json').write_text(json.dumps(rss_fixture['receipts']))
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--snapshots',str(rss_snapshots),'--out',str(root/'installed-rss2026')],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--snapshots',str(rss_snapshots),'--out',str(root/'installed-rss2026')],cwd=outside,check=True)
|
||||||
assert json.loads((root/'installed-rss2026/collection-report.json').read_text())['records'] == 210
|
assert json.loads((root/'installed-rss2026/collection-report.json').read_text())['records'] == 210
|
||||||
|
recsys_fixture = json.loads(gzip.decompress((src/'tests/fixtures/recsys2026-source.json.gz').read_bytes()))
|
||||||
|
recsys_snapshots = root/'recsys2026-snapshots'
|
||||||
|
recsys_snapshots.mkdir()
|
||||||
|
for name,encoded in recsys_fixture['files'].items():
|
||||||
|
path = recsys_snapshots/name
|
||||||
|
path.parent.mkdir(parents=True,exist_ok=True)
|
||||||
|
path.write_bytes(base64.b64decode(encoded))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','recsys2026-program','--snapshots',str(recsys_snapshots),'--out',str(root/'installed-recsys2026')],cwd=outside,check=True)
|
||||||
|
recsys_report = json.loads((root/'installed-recsys2026/collection-report.json').read_text())
|
||||||
|
assert (recsys_report['records'],recsys_report['unique_dois'],recsys_report['missing_abstract_count']) == (263,260,0)
|
||||||
|
assert recsys_report['duplicate_doi_observations'] == 6 and len(recsys_report['repeated_dois']) == 3
|
||||||
|
assert not recsys_report['database_written'] and recsys_report['network_requests'] == 0 and recsys_report['approved_for_import'] == 0
|
||||||
|
recsys_rows = [json.loads(line) for manifest in recsys_report['files'] for line in pathlib.Path(manifest['file']).read_text().splitlines()]
|
||||||
|
gold_keys = recsys_fixture['independent_gold'][0].keys()
|
||||||
|
assert [{k:r['metadata']['program_fields'][k] for k in gold_keys} for r in recsys_rows] == recsys_fixture['independent_gold']
|
||||||
corl_fixture = json.loads(gzip.decompress((src/'tests/fixtures/corl2026-source.json.gz').read_bytes()))
|
corl_fixture = json.loads(gzip.decompress((src/'tests/fixtures/corl2026-source.json.gz').read_bytes()))
|
||||||
corl_official = root/'corl2026-official'
|
corl_official = root/'corl2026-official'
|
||||||
corl_archive = root/'corl2026-archive'
|
corl_archive = root/'corl2026-archive'
|
||||||
|
|||||||
@@ -134,3 +134,5 @@ ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目
|
|||||||
`collect sdm2026-proceedings` 核验 53 个 Crossref 论文章节,并单独保留 53 项日程观察及 52 页摘要。作者差异、取消展示和摘要缺失分别保留,不自动拼接来源或绑定作品;见 [SDM 2026 使用与恢复说明](docs/sdm2026-proceedings.md)。
|
`collect sdm2026-proceedings` 核验 53 个 Crossref 论文章节,并单独保留 53 项日程观察及 52 页摘要。作者差异、取消展示和摘要缺失分别保留,不自动拼接来源或绑定作品;见 [SDM 2026 使用与恢复说明](docs/sdm2026-proceedings.md)。
|
||||||
|
|
||||||
`collect icdm2026-program` 离线核验 324 条 ICDM2026 官方线下日程,按明确的研究/应用标签输出 241/83 条;展示长短、缺失摘要及未核实出版身份分别保留,不自动入库。见 [ICDM2026 使用与恢复说明](docs/icdm2026-program.md)。
|
`collect icdm2026-program` 离线核验 324 条 ICDM2026 官方线下日程,按明确的研究/应用标签输出 241/83 条;展示长短、缺失摘要及未核实出版身份分别保留,不自动入库。见 [ICDM2026 使用与恢复说明](docs/icdm2026-program.md)。
|
||||||
|
|
||||||
|
`collect recsys2026-program` 核验九个 track 的 263 项官方录用贡献观察(260 个不同 DOI),保留摘要、原始作者和重复 DOI 的来源版本。见 [RecSys 2026 使用与恢复说明](docs/recsys2026-program.md)。
|
||||||
|
|||||||
@@ -1,5 +1,7 @@
|
|||||||
# 会议覆盖与公开决定核查
|
# 会议覆盖与公开决定核查
|
||||||
|
|
||||||
|
最新生产进展和剩余工作见 [2026-11-17 补漏状态](backfill-status-2026-11-17.md):固定 278 个会议年份槽位中,270 个有记录、8 个仍为空;这不代表完整覆盖,第 14 步仍未完成。
|
||||||
|
|
||||||
**当前工作范围:2020 年起主动补漏。** 2000 年起已发现资料保留,恢复工作从 [补漏范围与恢复清单(2026-11-10)](backfill-handoff-2026-11-10.md) 开始;下文旧日期快照不改写,较早年份排除不算补齐。
|
**当前工作范围:2020 年起主动补漏。** 2000 年起已发现资料保留,恢复工作从 [补漏范围与恢复清单(2026-11-10)](backfill-handoff-2026-11-10.md) 开始;下文旧日期快照不改写,较早年份排除不算补齐。
|
||||||
|
|
||||||
这是 2026-11-08 的首份核查基线,**完整逐年政策普查和最终去重仍未完成**。收录 43 个目标,不能把“有记录”当作“完整覆盖”。结构化清单见 [inventory-2026-11-08.json](inventory-2026-11-08.json)。清单中的 `policy_checks` 均限定所列年份和轨道;空数组表示尚未核实,绝不表示没有公开拒稿。
|
这是 2026-11-08 的首份核查基线,**完整逐年政策普查和最终去重仍未完成**。收录 43 个目标,不能把“有记录”当作“完整覆盖”。结构化清单见 [inventory-2026-11-08.json](inventory-2026-11-08.json)。清单中的 `policy_checks` 均限定所列年份和轨道;空数组表示尚未核实,绝不表示没有公开拒稿。
|
||||||
|
|||||||
@@ -0,0 +1,47 @@
|
|||||||
|
# 2020 起补漏状态:2026-11-17
|
||||||
|
|
||||||
|
**总体仍为 13/17,第 14 步未完成。** 本页补充新的生产结果,不回写 [2026-11-10 恢复清单](backfill-handoff-2026-11-10.md) 或更早快照。用户确定的范围不变:2020 年起主动补漏;43 个更早历史缺口延期而非补齐,2000–2019 已发现的名单和身份线索继续保留。
|
||||||
|
|
||||||
|
## 最新数据库盘点
|
||||||
|
|
||||||
|
只读快照 `2026-11-17 09:38:31.677743+00`,固定 43 个目标会议、278 个应有会议年份槽位,其中 270 个至少有一条投稿记录,8 个仍为空,均为 2026 年:**CIKM、ECCV、ICASSP、ICDM、IROS、ISMIR、MM、RecSys**。本次 SDM2026 新增记录使空槽从 9 减到 8。
|
||||||
|
|
||||||
|
这只说明有无记录,**不表示 270 个槽位已经逐篇、逐 track、摘要或公开拒稿覆盖完整**;不能据此宣布第 14 步完成。2020–2025 没有整年空槽,仍有逐来源完整性和公开负面决定缺口。SDM2026 仅导入 25/53 条,状态为 partial,同样不能称为整届补齐。
|
||||||
|
|
||||||
|
| 生产文件(相对 `/data/paper-index/`) | SHA-256 |
|
||||||
|
| --- | --- |
|
||||||
|
| collected/active-scope-census-20261117-v17/summary.json | bfb1db8b50619c3756abb91ad0f73fa58ff998c7db58a216661342f12047749f |
|
||||||
|
| collected/active-scope-census-20261117-v17/snapshot.json | b5389205f8b34f7ed57946416f49c253e0aed4a05e45eb818801f18fca6a8cb4 |
|
||||||
|
| validation/active-scope-fixed278-v1.json | a835bef5b9c3a0f76069d840cecdb9a6bcf65d3a38796d70dbfdfd4b49242264 |
|
||||||
|
|
||||||
|
## 已推进的两批来源
|
||||||
|
|
||||||
|
- **SDM2026**:53 个正式注册章节、53 个独立日程观察和 52 个公开摘要页已冻结并规范化。25 条发表记录已在 PostgreSQL 独立验收,新增论文实体为 0;28 条待身份/版本核查。完整旧历史和 5,653 篇受保护论文的资料、来源、标签和既有标识保留。见 [生产验收与恢复](../sdm2026-identity-review.md)。这批队列已经消费,禁止重放。
|
||||||
|
- **ECCV2026**:2,834 个唯一初步日程条目及公开摘要已全量采集并独立验收;新的只读身份预检和 1,997 个候选历史证据包已保存,尚未入库。初步日程不等于已核验出版社最终论文集;负面决定元数据仍受匿名访问限制。见 [来源状态](../eccv2026-source-status.md)。
|
||||||
|
|
||||||
|
SDM 新增事件使 SeBA 和 AdamG 同时具有较早 ICLR 拒稿与后来的 SDM 发表事实。旧的两次 ICLR 投稿及四条决定来源观察仍保留,没有被较晚 accepted 覆盖;多个来源观察不直接累加成多次拒稿。
|
||||||
|
|
||||||
|
## 其余七个会议的最新入口核查
|
||||||
|
|
||||||
|
已对下表七个会议的 14 个明确官方 URL 保存原始响应和回执:13 个 HTTP 200,IROS 官方论文索引 HTTP 403。所有请求均匿名,不绕过访问限制,没有下载论文 PDF;保存入口不是完成逐篇采集或负面决定普查。
|
||||||
|
|
||||||
|
| 会议 | 已保存的明确官方入口 | 尚未完成 |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| ICDM | [详细日程](https://icdm2026.neu.edu.cn/DetailedSessions/list.htm)、[研究轨道政策](https://icdm2026.neu.edu.cn/CallforResearchTrack/list.htm)、[应用轨道政策](https://icdm2026.neu.edu.cn/CallforAppliedTrack/list.htm) | 官方页面称为线下展示列表;需要核对逐篇范围、研究/应用轨道及演讲长短标签,不能把展示长短自动等同于论文类型;摘要和正式出版身份仍待核查 |
|
||||||
|
| ICASSP | [录用标题清单](https://cmsworkshops.com/ICASSP2026/papers/accepted_papers.php)、[作者指南](https://2026.ieeeicassp.org/author-guidelines/) | 取得逐篇完整作者/摘要/出版记录,区别会议投稿与期刊展示,不能只导入标题或从缺席推断拒稿 |
|
||||||
|
| IROS | [官方论文索引](https://2026.ieee-iros.org/program/paper_index/) | 当前服务器返回 403,保留缺口并停止该入口;第三方浏览器页面不能替代独立核验的官方原文 |
|
||||||
|
| ISMIR | [官网](https://ismir2026.ismir.net/)、[官方虚拟平台](https://ismir2026program.ismir.net/)、[作者指南](https://ismir2026.ismir.net/authors/author-guidelines) | 核查平台公开静态数据与正式开放论文集的逐篇映射;首页可读取不代表论文目录已经取得 |
|
||||||
|
| CIKM | [录用页](https://cikm2026.diag.uniroma1.it/accepted-papers/)、[日程页](https://cikm2026.diag.uniroma1.it/program/) | 网页提取的录用页仍显示 TBD;需要检查原文引用的节目数据与独立正式出版目录,不能当成没有论文 |
|
||||||
|
| RecSys | [官网](https://recsys.acm.org/recsys26/)、[投稿说明](https://recsys.acm.org/recsys26/call/) | 继续取得可核验的逐篇正式记录,并分开主轨、工业、复现、demo、challenge/workshop |
|
||||||
|
| MM | [官网](https://2026.acmmm.org/) | 继续核对官方完整日程和出版目录,分开研究论文与 grand challenge、workshop 等;首页响应不证明覆盖完整 |
|
||||||
|
|
||||||
|
冻结目录:`/data/paper-index/collected/remaining2026-official-leads-v1`。manifest SHA-256:`0e13313d94e3bff17f071d3a1384f6ed83ece156239fe5bc4633cc7a34fb8e04`。恢复包:`/data/paper-index/validation/remaining2026-official-leads-v1-recovery.tar.gz`,SHA-256:`0be8c84bb5c591e5afaaa5f95dd5334bf13a9be1671cb942e46e998f502495bb`。HTTP 成功仅是入口可读性,不能作为可导入记录数量。
|
||||||
|
|
||||||
|
## 第 14 步仍需做什么
|
||||||
|
|
||||||
|
1. 取得并规范化其余可公开取得的完整逐篇来源,记录应有范围、排除项和版本;对缺失、受限或尚未公开的入口保存明确证据,不能伪造“已补齐”。优先推进当前可读的 ICDM 详细日程、ICASSP 清单及其他会议的正式目录。
|
||||||
|
2. 完成来源与年度、track、摘要完整性的对应核查;已经有若干记录的年份也要看目录差集,不能只消除空槽。
|
||||||
|
3. 继续近期和历史年度实际公开的 rejected、desk_rejected、withdrawn 来源核查,特别是 UAI2024–2026 的 opt-in 拒稿、EMNLP2023 的实际公开范围以及其他会议的逐年限定政策。公开评审和公开决定不同;没有找到不等于零条或未曾拒稿。
|
||||||
|
4. 通过身份核查的记录再执行新快照守护、恢复备份和增量导入,保留较早决定。深层作品版本、跨会历史及类型/标签协调继续列为第 15 步;不能为减少空槽创建重复论文,也不能只盯住少量身份疑难项而停止其他来源采集。
|
||||||
|
|
||||||
|
第 16 步仍是所有导入结束后的独立全库去重与完整审计,第 17 步是最终长期运行/恢复和交付整理。上述任务均未因单批验收或 CI 成功而完成;总体 goal 继续保持 active。
|
||||||
@@ -37,3 +37,23 @@ group 的 `rejected_venue_id` 实际配置为 `thecvf.com/ECCV/2026/Conference/R
|
|||||||
| anonymous-decision-probes/manifest.json | a522756ed3ec2880e5d942a211d7ad13819b18c8acada551dd3a8718b5e4d63b |
|
| anonymous-decision-probes/manifest.json | a522756ed3ec2880e5d942a211d7ad13819b18c8acada551dd3a8718b5e4d63b |
|
||||||
|
|
||||||
这里的补充证据与采集器的三份冻结节目/政策输入分开保存,不能替换采集器固定版本的 manifest。下次补漏先核查官方是否提供公开决定导出或明确的公开元数据说明;访问仍受限时保留缺口。若发现允许公开访问的新入口,另存带时间与哈希的快照,先审阅读者权限、决定语义和逐篇身份,再准备入库队列。
|
这里的补充证据与采集器的三份冻结节目/政策输入分开保存,不能替换采集器固定版本的 manifest。下次补漏先核查官方是否提供公开决定导出或明确的公开元数据说明;访问仍受限时保留缺口。若发现允许公开访问的新入口,另存带时间与哈希的快照,先审阅读者权限、决定语义和逐篇身份,再准备入库队列。
|
||||||
|
|
||||||
|
|
||||||
|
## 2026-11-17 全量采集与只读预检
|
||||||
|
|
||||||
|
第三次采集 `paper-atlas-eccv2026-collect-pass03.service`(invocation `147c8779a3af4223a4d5d60dbdc88264`)已终态成功:2,834 个唯一日程条目全部取得公开摘要,新取 1,775 页,复用已验收缓存 1,059 页;2,851 次日程出现中的 17 处重复保留原始位置。独立原文/回执/字段审计通过,缺少条目和摘要均为 0,5 处 JSON-LD 显示文字规范化仍留待逐篇审阅。这里仍是官方初步日程版本,不改称出版社最终论文集。
|
||||||
|
|
||||||
|
| 服务器证据 | SHA-256 |
|
||||||
|
| --- | --- |
|
||||||
|
| validation/eccv2026-pass03-audit-v1.json | ad90199e08a234c0b2a01f44e1edfbecb2a9101b45de87e00a64eadfd5c92848 |
|
||||||
|
| collected/eccv2026-pass03/eccv-program-2026-unclassified.jsonl | d0a80745fc6f58163f8722af351bbba86803af97857ea30554a0f3062d8ab2ef |
|
||||||
|
| validation/eccv2026-pass03-recovery-v1.tar.gz | df55493ad5f8d6413dca86e9b9aca1d8b544ac1dc222b3849f95b04f94accba9 |
|
||||||
|
| collected/eccv2026-pass03-preflight-v1/postgresql-preflight.json | 97bf57828cc67d05de0221188a9d988eef35631c845b18372186a6249f64e5da |
|
||||||
|
| validation/eccv2026-pass03-preflight-packets-check-v1.json | 8a4f44f5d2c3a15d05b5a1df96fe465a96a2fe8914d7ed68958e6978c30985d4 |
|
||||||
|
| validation/eccv2026-pass03-preflight-recovery-v1.tar.gz | 69c84fd0f24024ca56bfa4f848c977c578267bfc53c713105987e030df3f92a2 |
|
||||||
|
|
||||||
|
全量原文恢复包为 85,430,941 字节,保留输出、全部缓存原文与回执、官方输入和固定源码/CI 凭证。失败的第一次打包任务在创建文件前因命令引用错误退出;改用文件脚本的新单元 `paper-atlas-eccv2026-pass03-recovery-v2.service` 成功,未重启或重跑采集器。
|
||||||
|
|
||||||
|
新的 PostgreSQL 只读预检覆盖全部 2,834 条:1,519 条精确标题和作者集合一致、468 条仅部分作者重合、840 条无精确匹配、4 条没有作者重合、3 条命中多个同标题实体。1,997 个候选实体包的哈希独立核对通过,来源、投稿、决定与来源关系未触发截断。该结果只生成候选,`database_written=false`、`approved_for_import=0`;没有批准合并、创建新论文或导入 ECCV 事件。无精确匹配不等于新作品,相同作者也不证明人物或作品身份。
|
||||||
|
|
||||||
|
后续先保留这批完整来源并推进其他会议来源补漏,再做逐篇内容与版本审阅、当前快照守护和保护基线。当前全局最新基线来自 SDM25 验收,保护 5,653 篇论文;不能回退到 ICAPS34 之前的计数或重放已消费批次。最终全库去重仍须另行执行。
|
||||||
|
|||||||
@@ -0,0 +1,53 @@
|
|||||||
|
# RecSys 2026 公开录用贡献采集
|
||||||
|
|
||||||
|
这是长期维护项目中的离线采集器,按需处理冻结的官方元数据,不连接数据库、不访问论文链接或下载 PDF。输出须经过作品、DOI 和投稿历史审查才能导入。
|
||||||
|
|
||||||
|
## 使用
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect recsys2026-program \
|
||||||
|
--snapshots /data/paper-index/collected \
|
||||||
|
--out /data/paper-index/collected/recsys2026-normalized-<new-run>
|
||||||
|
```
|
||||||
|
|
||||||
|
输入父目录应包含 `remaining2026-listings-research-v2/manifest.json` 与 `recsys-contributions.html`。采集器检查固定清单 SHA256、官方 URL、HTTP 200、采集时区、原始字节哈希、九个 track 的菜单及完整条目数。现有输出目录不可覆盖;失败保留 `failed.json`,成功才发布 `collection-report.json`。再次运行前先检查原任务是否结束。
|
||||||
|
|
||||||
|
## 数据范围与重复项
|
||||||
|
|
||||||
|
来源是 [RecSys 官方录用贡献页面](https://recsys.acm.org/recsys26/contributions/),冻结时间为 2026-11-17。九个来源 track:
|
||||||
|
|
||||||
|
| track | 观察数 |
|
||||||
|
| --- | ---: |
|
||||||
|
| long | 65 |
|
||||||
|
| short | 29 |
|
||||||
|
| past_present_future | 13 |
|
||||||
|
| reproducibility | 4 |
|
||||||
|
| resource | 7 |
|
||||||
|
| industry | 83 |
|
||||||
|
| research_practice_notes | 34 |
|
||||||
|
| demo | 18 |
|
||||||
|
| doctoral_symposium | 10 |
|
||||||
|
|
||||||
|
**263 项来源观察对应 260 个不同 DOI,并非 263 篇不同论文。** 每条记录用官方页面中的 accordion locator 标识,分别保留来源标题、作者原文、摘要和位置,不假设 locator 是原始投稿 ID。
|
||||||
|
|
||||||
|
以下 DOI 各出现两次,六个观察均显式标记待审查:
|
||||||
|
|
||||||
|
- `10.1145/3773078.3831839`:Dart 两条来源记录的标题、作者和摘要都有差异;不得静默选一个覆盖另一个。
|
||||||
|
- `10.1145/3773078.3831819`:标题与作者相同、摘要版本不同。
|
||||||
|
- `10.1145/3773078.3831861`:重复展示的文字相同,保留两个来源位置。
|
||||||
|
|
||||||
|
所有观察都有公开摘要。正文只转换为 HTML 展示文字并规范空白,原始页面完整保留;遇到未审查的数学或图片结构会停止,不把上标扁平化成数字。作者按来源的逗号和最后一个 and 分隔,原始名单同时保留;这不是已验证的作者人物身份或别名。
|
||||||
|
|
||||||
|
`accepted` 仅指官方页面明示录用,不是原始签名评审决定。DOI 来自官方超链接,尚未独立核查实时出版商版本;归入 `publication_doi`,不复制其他会议的投稿 ID。track 明确来自页面区块;`recsys_program:contribution_type` 仅保存页面原始类型标签(Research、PastPresentFuture、Doctoral Symposium),各 track 的 RES/IND/RPN 等缩写同时保留在来源字段,不统一推断研究方法类型。
|
||||||
|
|
||||||
|
本页只含录用贡献,不能据此推断未出现的稿件被拒。公开拒稿导出仍为 unknown,征稿页的 desk rejection 政策不等于具体拒稿记录。不得将本批采集成功当成会议全部来源、负面决定或全库去重完成。
|
||||||
|
|
||||||
|
## 输出与恢复
|
||||||
|
|
||||||
|
输出九份 JSONL、对应部分覆盖清单、原始 HTML 和清单证据,以及记录观察数和不同 DOI 数的报告;批准导入数为零。
|
||||||
|
|
||||||
|
清单 SHA256:`fea7b710587636db06794ee195cd083634c41e9054a217859d9dd0ccdd1afab9`。
|
||||||
|
|
||||||
|
服务器恢复包 `/data/paper-index/validation/remaining2026-listings-research-v2-recovery.tar.gz`,本地备份 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/remaining2026-listings-research-v2-recovery.tar.gz`,SHA256 为 `f5ef7c2e73d46712146e4d6711ad5ad562e16b8b55c0cd75d1041cd0b81142cb`。安全恢复原始目录、核对哈希后重跑上述命令。
|
||||||
|
|
||||||
|
完整来源与独立 263 项基准已保存在测试夹具,所有测试、构建和安装验证交给指定的 `ces-control:host` runner。生产导入前还需独立核验规范化结果,并检查当前数据库已有 DOI 绑定及历次投稿。最终全库去重审计仍是独立必做阶段。
|
||||||
@@ -1,6 +1,6 @@
|
|||||||
# SDM 2026:生产规范化与身份核查记录
|
# SDM 2026:生产规范化与身份核查记录
|
||||||
|
|
||||||
本页记录 2026-11-17 的来源验收和只读身份核查,供下次按需运行时恢复。**本轮没有写入 PostgreSQL,53 条记录均未获得导入批准。** 核查完成、存在候选、内容支持同一作品和最终允许入库是不同状态。
|
本页记录 2026-11-17 的来源验收、早期只读核查和随后的生产导入。**当前 53 条注册记录中,25 条已入库并独立验收,28 条仍待处理;没有新增论文实体。** 下文早期报告保持原始状态,不将它们改写成事后批准;最终结果见“25 条生产导入验收”。
|
||||||
|
|
||||||
## 已核实的生产输出
|
## 已核实的生产输出
|
||||||
|
|
||||||
@@ -52,7 +52,7 @@
|
|||||||
| Towards Reliability of Parameter-Free Optimization | 3282961 | ICLR 2025 rejected |
|
| Towards Reliability of Parameter-Free Optimization | 3282961 | ICLR 2025 rejected |
|
||||||
| Scaffold-Aware Generative Augmentation and Reranking for Enhanced Virtual Screening | 2506693 | ICLR 2026 desk_rejected |
|
| Scaffold-Aware Generative Augmentation and Reranking for Enhanced Virtual Screening | 2506693 | ICLR 2026 desk_rejected |
|
||||||
|
|
||||||
前两条已有内容支持,但没有导入 SDM 事件;第三条仍保留作者版本问题。如果后续通过身份核查,SDM 发表事实应作为独立会议事件加入,保留旧投稿、旧决定及各自来源。两个来源对同一拒稿的观察不能直接统计成两次独立投稿或拒稿。不能覆盖旧状态,也不能因这批录用候选就宣称完整投稿历史已补齐。
|
前两条在后续生产验收中各自新增独立 SDM 发表事件;第三条仍保留作者版本问题,未导入。旧投稿、旧决定及各自来源均完整保留。两个来源对同一拒稿的观察不能直接统计成两次独立投稿或拒稿。不能覆盖旧状态,也不能因这批录用候选就宣称完整投稿历史已补齐。
|
||||||
|
|
||||||
## 未精确匹配的补充检索
|
## 未精确匹配的补充检索
|
||||||
|
|
||||||
@@ -70,9 +70,28 @@
|
|||||||
|
|
||||||
两个原报告的 SHA-256 分别为 `b6ab2e9228b8e8e9548b27df70834faf25bfda551844979f8410a74b6e6aadd0` 和 `c1f902b4e4f33997eb182b10dc78332e182ceb6051f1a7c14b21465c8878263d`。前者保留 1 个进一步审阅的完整候选包,后者保留全部 113 个前缀命中的候选包;这些数量不是论文批准数。
|
两个原报告的 SHA-256 分别为 `b6ab2e9228b8e8e9548b27df70834faf25bfda551844979f8410a74b6e6aadd0` 和 `c1f902b4e4f33997eb182b10dc78332e182ceb6051f1a7c14b21465c8878263d`。前者保留 1 个进一步审阅的完整候选包,后者保留全部 113 个前缀命中的候选包;这些数量不是论文批准数。
|
||||||
|
|
||||||
|
## 25 条生产导入验收
|
||||||
|
|
||||||
|
`paper-atlas-sdm2026-import25-v1.service`,invocation `003b490a2f2c449da078a2f8625b31b8`,终态成功。执行前核验 81 个源码文件,并确认 24 个核心写入文件与此前已验收的 canonical writer 相同;复用原生 `paper-atlas import records`,未修改默认运行目录。
|
||||||
|
|
||||||
|
- 新增 25 条来源、25 次会议事件、25 条发表决定观察及 25 个 `publication_doi` 投稿标识,新增论文实体为 0。
|
||||||
|
- 24 篇使用已核实的 arXiv 标识补注;AdamG 没有 arXiv 标识,采用单独审阅并实时守护的唯一精确标题、四名完整作者和具体方法证据。旧 OpenReview ID 未复制为 SDM 投稿标识。
|
||||||
|
- 28 条待处理记录全部排除;25 个注册来源摘要仍为空,52 份日程摘要保持独立来源版本。覆盖状态为 `partial`,应有 53 条、当前导入 25 条,track 保留 `proceedings_unclassified`。
|
||||||
|
- 全局旧原始投稿与决定历史、5,653 篇受保护论文的原资料/来源/标签/既有标识均通过导入后比较。SeBA、AdamG 的两次较早 ICLR 投稿及四条 rejected 来源观察保留,没有将四条观察说成四次投稿。
|
||||||
|
- 约束有效、实际全文查询与 GIN 计划通过;foundry-core `/readyz` 返回 200,受保护 SQLite 快照保持只读。
|
||||||
|
|
||||||
|
| 生产证据 | SHA-256 |
|
||||||
|
| --- | --- |
|
||||||
|
| collected/sdm2026-import25-v1/postgresql-postcheck-v1.json | 0c861722f0d455462a2686a42d634e0b963d075cf58f90fa9f48e1ae1fc82c27 |
|
||||||
|
| collected/sdm2026-import25-baseline-v1/baseline.json | 42ffbaf6f843a91ebf64f0bfefa4af36f6ac72f044b0e8587e65c5e6648c0931 |
|
||||||
|
| collected/sdm2026-prepared25-v1/independent-preparation-check.json | 8ea8eb59f4b1601a484d1bc0535b289cb950b83747ed5f82b0eac55e527c4a7c |
|
||||||
|
| validation/sdm2026-reviewed25-recovery-v1.tar.gz | 13447944ec34e0e2a6eee07a1d9820bfb8066756aa11016673e416c0d6f8b2b2 |
|
||||||
|
|
||||||
|
恢复包包含 263 个文件;本机已独立核对队列、70 份内容寻址证据和 28 个早期候选证据包。**25 条队列已消费,禁止重放。** 28 条剩余记录需要新的导入后快照;先前候选包不能当作当前数据库状态。后续批次须沿用最新 5,653 篇保护基线,并加上各自新目标。
|
||||||
|
|
||||||
## 下次恢复
|
## 下次恢复
|
||||||
|
|
||||||
1. 先核对实际服务终态、来源及报告哈希,再读取上述证据;终态成功的研究任务无需重启。
|
1. 先核对实际服务终态、来源及报告哈希,再读取上述证据;终态成功的研究任务无需重启。
|
||||||
2. 继续解决 3 条作者版本问题和 25 条未精确匹配记录;不把缺少候选当作新作品证明,不把源端 403 当作零条公开拒稿。
|
2. 保留已验收的 25 条,不重放;继续解决 3 条作者版本问题和 25 条未精确匹配记录;不把缺少候选当作新作品证明,不把源端 403 当作零条公开拒稿。
|
||||||
3. 对已获内容支持的候选另做新快照检查:作品与 DOI 绑定、来源是否已导入、已有会议事件及全部历史保护。建立新的明确入库队列和可恢复备份后再执行写入,不能直接使用本次候选报告当导入计划。
|
3. 对后续新获内容支持的候选另做新快照检查:作品与 DOI 绑定、来源是否已导入、已有会议事件及全部历史保护。建立新的明确入库队列和可恢复备份后再执行写入,不能直接使用本次候选报告当导入计划。
|
||||||
4. 导入后逐项核验新事件、旧历史、来源关系和精确数量变化;注册摘要缺失仍如实保留。完整来源范围结束后另做全库去重、标识冲突、近似标题/作者、覆盖、摘要、约束和查询验收。
|
4. 导入后逐项核验新事件、旧历史、来源关系和精确数量变化;注册摘要缺失仍如实保留。完整来源范围结束后另做全库去重、标识冲突、近似标题/作者、覆盖、摘要、约束和查询验收。
|
||||||
|
|||||||
@@ -29,6 +29,7 @@ COMMANDS = {
|
|||||||
('collect','sdm2026-proceedings'): ('paper_atlas.collectors.sdm2026', {}, '核验 SDM 2026 注册论文并单独保留公开日程版本'),
|
('collect','sdm2026-proceedings'): ('paper_atlas.collectors.sdm2026', {}, '核验 SDM 2026 注册论文并单独保留公开日程版本'),
|
||||||
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
|
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
|
||||||
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
||||||
|
('collect','recsys2026-program'): ('paper_atlas.collectors.recsys2026', {}, '核验 RecSys 2026 公开录用贡献并保留 DOI 来源版本'),
|
||||||
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
||||||
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
|
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
|
||||||
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
|
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
|
||||||
|
|||||||
@@ -0,0 +1,172 @@
|
|||||||
|
"""Frozen RecSys 2026 accepted-contribution observations, preserving source variants."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, Node, clean
|
||||||
|
|
||||||
|
SOURCE = 'recsys_program'
|
||||||
|
SNAPSHOT = 'remaining2026-listings-research-v2'
|
||||||
|
MANIFEST_SHA = 'fea7b710587636db06794ee195cd083634c41e9054a217859d9dd0ccdd1afab9'
|
||||||
|
URL = 'https://recsys.acm.org/recsys26/contributions/'
|
||||||
|
TRACKS = [('long', 'Long Papers', 65), ('short', 'Short Papers', 29),
|
||||||
|
('past_present_future', 'PPF Papers', 13), ('reproducibility', 'Reproducibility', 4),
|
||||||
|
('resource', 'Resource', 7), ('industry', 'Industry', 83),
|
||||||
|
('research_practice_notes', 'R&P Notes', 34), ('demo', 'Demos', 18),
|
||||||
|
('doctoral_symposium', 'Doctoral', 10)]
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def one(items):
|
||||||
|
if len(items) != 1:
|
||||||
|
raise ValueError('Expected exactly one reviewed source field')
|
||||||
|
return items[0]
|
||||||
|
|
||||||
|
|
||||||
|
def load_snapshot(root):
|
||||||
|
root = Path(root); directory = root/SNAPSHOT; path = directory/'manifest.json'
|
||||||
|
if root.is_symlink() or directory.is_symlink() or path.is_symlink() or path.stat().st_size > 128*1024:
|
||||||
|
raise ValueError('Unsafe snapshot manifest')
|
||||||
|
raw = path.read_bytes()
|
||||||
|
if digest(raw) != MANIFEST_SHA:
|
||||||
|
raise ValueError('Reviewed manifest hash changed')
|
||||||
|
manifest = json.loads(raw)
|
||||||
|
if manifest['status'] != 'research_completed_not_coverage_or_import_approval' or manifest['database_written'] is not False:
|
||||||
|
raise ValueError('Unreviewed snapshot scope')
|
||||||
|
receipt = one([e for e in manifest['entries'] if e['venue'] == 'RecSys' and e['label'] == 'contributions'])
|
||||||
|
name = receipt['file']; path = directory/name
|
||||||
|
if Path(name).name != name or path.is_symlink() or path.stat().st_size > 4*1024**2:
|
||||||
|
raise ValueError('Unsafe snapshot body')
|
||||||
|
body = path.read_bytes()
|
||||||
|
if (receipt['url'] != URL or receipt['final_url'] != URL or receipt['http_status'] != 200
|
||||||
|
or not receipt['content_type'].startswith('text/html') or receipt['bytes'] != len(body)
|
||||||
|
or receipt['sha256'] != digest(body) or datetime.fromisoformat(receipt['at']).utcoffset() is None):
|
||||||
|
raise ValueError('Snapshot receipt/body disagree')
|
||||||
|
return receipt, {'manifest.json': raw, name: body}, body
|
||||||
|
|
||||||
|
|
||||||
|
def body_text(node, doi_url):
|
||||||
|
if not isinstance(node, Node):
|
||||||
|
return node
|
||||||
|
if node.tag in ('script', 'style') or (node.tag == 'a' and node.attrs.get('href') == doi_url):
|
||||||
|
return ''
|
||||||
|
text = ''.join(body_text(c, doi_url) for c in node.children)
|
||||||
|
return ' '+text+' ' if node.tag in ('p','div','li','br','section') else text
|
||||||
|
|
||||||
|
|
||||||
|
def parse_program(raw):
|
||||||
|
doc = Document(raw.decode('utf-8')); nodes = list(doc.root.walk())
|
||||||
|
if not any(n.tag == 'title' and 'RecSys 26 - Accepted Contributions' in n.text() for n in nodes):
|
||||||
|
raise ValueError('Wrong conference edition')
|
||||||
|
menu = one([n for n in nodes if n.tag == 'ul' and n.attrs.get('id') == 'tab-1'])
|
||||||
|
labels = [(n.attrs.get('href'), clean(n.text())) for n in menu.walk() if n.tag == 'a']
|
||||||
|
if labels != [(f'#content-tab-1-{i}', label) for i, (_,label,_) in enumerate(TRACKS)]:
|
||||||
|
raise ValueError('Unreviewed track menu')
|
||||||
|
observations = []; seen = set()
|
||||||
|
for tab, (track, label, count) in enumerate(TRACKS):
|
||||||
|
section = one([n for n in nodes if n.tag == 'div' and n.attrs.get('id') == f'content-tab-1-{tab}'])
|
||||||
|
listing = one([n for n in section.walk() if n.tag == 'ul' and n.attrs.get('id') == f'accordion-{tab+1}'])
|
||||||
|
rows = [n for n in listing.children if isinstance(n, Node) and n.tag == 'li']
|
||||||
|
if len(rows) != count:
|
||||||
|
raise ValueError('Incomplete reviewed track census')
|
||||||
|
for position, row in enumerate(rows, 1):
|
||||||
|
anchor = one([n for n in row.children if isinstance(n, Node) and n.tag == 'a'])
|
||||||
|
locator = anchor.attrs.get('rel', '').removeprefix('#')
|
||||||
|
if not re.fullmatch(f'accordion-{tab+1}-slide-[0-9]+', locator) or locator in seen:
|
||||||
|
raise ValueError('Missing or repeated program locator')
|
||||||
|
seen.add(locator)
|
||||||
|
types = [n for n in anchor.walk() if n.tag == 'span' and n.attrs.get('class') == 'paper-type']
|
||||||
|
type_node = one(types); type_literal = type_node.attrs.get('title'); abbreviation = clean(type_node.text())
|
||||||
|
expected_type = [('Research','RES'),('Research','RES'),('PastPresentFuture','PPF'),
|
||||||
|
('Research','REPR'),('Research','RSC'),('Research','IND'),('Research','RPN'),('Research','DEMO'),('Doctoral Symposium','DS')][tab]
|
||||||
|
if (type_literal,abbreviation) != expected_type:
|
||||||
|
raise ValueError('Unreviewed source contribution type')
|
||||||
|
title = clean(''.join(c.text() if isinstance(c, Node) else c for c in anchor.children
|
||||||
|
if not isinstance(c, Node) or c.tag not in ('span','em','br')))
|
||||||
|
authors_literal = clean(one([n for n in anchor.walk() if n.tag == 'em']).text())
|
||||||
|
if not authors_literal.startswith('by '):
|
||||||
|
raise ValueError('Missing program author prefix')
|
||||||
|
authors_literal = authors_literal[3:]
|
||||||
|
authors = [n.strip() for n in re.split(r',\s+|\s+and\s+', authors_literal)]
|
||||||
|
if not title or any(not n or '@' in n for n in authors):
|
||||||
|
raise ValueError('Missing title or program author')
|
||||||
|
body = one([n for n in row.children if isinstance(n, Node) and n.tag == 'div'])
|
||||||
|
doi_url = one([n.attrs.get('href') for n in body.walk() if n.tag == 'a'
|
||||||
|
and n.attrs.get('href', '').startswith('https://doi.org/')])
|
||||||
|
if not re.fullmatch(r'https://doi.org/10\.1145/3773078\.[0-9]+', doi_url):
|
||||||
|
raise ValueError('Wrong publication DOI family')
|
||||||
|
abstract = clean(body_text(body, doi_url))
|
||||||
|
if len(abstract) < 100 or any(n.tag in ('math','sup','sub','img','svg') for n in body.walk()):
|
||||||
|
raise ValueError('Missing abstract or unreviewed structured markup')
|
||||||
|
observations.append({'doi': doi_url.removeprefix('https://doi.org/'), 'tab': tab, 'position': position,
|
||||||
|
'locator': locator, 'title': title, 'authors': authors, 'authors_literal': authors_literal,
|
||||||
|
'abstract': abstract, 'type_labels': [type_literal], 'source_type_abbreviation': abbreviation,
|
||||||
|
'track': track, 'track_label': label})
|
||||||
|
counts = Counter(r['doi'] for r in observations)
|
||||||
|
if len(observations) != 263 or len(counts) != 260 or {d:n for d,n in counts.items() if n>1} != {
|
||||||
|
'10.1145/3773078.3831839':2,'10.1145/3773078.3831819':2,'10.1145/3773078.3831861':2}:
|
||||||
|
raise ValueError('Unreviewed DOI observation census')
|
||||||
|
for item in observations:
|
||||||
|
item['review_flags'] = ['repeated_program_doi_preserve_versions_requires_review'] if counts[item['doi']] > 1 else []
|
||||||
|
return observations
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
receipt, files, body = load_snapshot(snapshots); observations = parse_program(body)
|
||||||
|
evidence = out/'evidence'; evidence.mkdir()
|
||||||
|
for name, raw in files.items():
|
||||||
|
(evidence/name).write_bytes(raw)
|
||||||
|
outputs = []
|
||||||
|
for track,label,count in TRACKS:
|
||||||
|
rows = []
|
||||||
|
for item in observations:
|
||||||
|
if item['track'] != track:
|
||||||
|
continue
|
||||||
|
doi = item['doi']; url = URL+'#'+item['locator']
|
||||||
|
rows.append({'source': SOURCE, 'external_id': '2026/'+item['locator'], 'url': url,
|
||||||
|
'title': item['title'], 'authors': item['authors'], 'abstract': item['abstract'],
|
||||||
|
'identifiers': {'doi': doi}, 'submission_identifiers': [{'scheme':'publication_doi','value':doi}],
|
||||||
|
'venue':'RecSys', 'year':2026, 'track':track, 'status':'accepted', 'observed_at':receipt['at'],
|
||||||
|
'evidence_url':URL, 'raw_decision':'Listed as an accepted contribution in the official RecSys2026 '+label+' section; not a signed review decision.',
|
||||||
|
'tags':[{'namespace':SOURCE+':contribution_type','value':item['type_labels'][0]}],
|
||||||
|
'metadata':{'program_fields':item, 'program_receipt':receipt,
|
||||||
|
'source_evidence':{'file':'evidence/'+receipt['file'],'sha256':receipt['sha256'],'url':URL},
|
||||||
|
'source_manifest':{'file':'evidence/manifest.json','sha256':MANIFEST_SHA},
|
||||||
|
'original_decision_signature_verified':False,
|
||||||
|
'author_scope':'Source-reported comma/and-delimited names; original literal retained. Not verified publisher person identities or aliases.',
|
||||||
|
'doi_scope':'DOI asserted by official program hyperlink, not independently verified live publisher metadata.',
|
||||||
|
'abstract_transform':'HTML display text and whitespace normalization; source retained. No PDF or cross-source fusion.',
|
||||||
|
'scope':'Program observation keyed by source locator. Repeated DOI variants retained separately and flagged, not counted as distinct scientific works. Requires DOI/work/history review before import.'}})
|
||||||
|
path = out/f'recsys-program-2026-{track}.jsonl';raw=''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in rows).encode();path.write_bytes(raw)
|
||||||
|
manifest={'source':SOURCE,'parser_version':1,'venue':'RecSys','year':2026,'track':track,'count':len(rows),
|
||||||
|
'file':str(path),'file_sha256':digest(raw),'list_url':URL,'status':'partial','missing_abstract_count':0,
|
||||||
|
'scope':'Official accepted-contribution observations; unique DOI counts differ from observation counts.'}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)+'\n');outputs.append(manifest)
|
||||||
|
report={'status':'collected_pending_identity_review','records':263,'unique_dois':260,'files':outputs,
|
||||||
|
'track_counts':{track:count for track,_,count in TRACKS},'missing_abstract_count':0,
|
||||||
|
'duplicate_doi_observations':6,'repeated_dois':{d:n for d,n in Counter(r['doi'] for r in observations).items() if n>1},
|
||||||
|
'source_manifest_sha256':MANIFEST_SHA,'database_written':False,'network_requests':0,'pdfs_downloaded':False,
|
||||||
|
'approved_for_import':0,'public_rejected_availability':'unknown_no_public_export_verified',
|
||||||
|
'scope':'263 program observations for260 DOIs, not263 unique papers or final work identities. Track/type labels are source scoped. Not complete public-negative coverage or a final whole-database audit.'}
|
||||||
|
part=out/'collection-report.json.part';part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n');part.replace(out/'collection-report.json');return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser=argparse.ArgumentParser(prog=prog,description=__doc__)
|
||||||
|
parser.add_argument('--snapshots',required=True);parser.add_argument('--out',required=True)
|
||||||
|
args=parser.parse_args(argv);print(json.dumps(collect(args.out,snapshots=args.snapshots),ensure_ascii=False))
|
||||||
BIN
Binary file not shown.
@@ -0,0 +1,79 @@
|
|||||||
|
import base64
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import recsys2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class RecSys2026Test(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture=json.loads(gzip.decompress((Path(__file__).parent/'fixtures/recsys2026-source.json.gz').read_bytes()))
|
||||||
|
cls.files={n:base64.b64decode(raw) for n,raw in cls.fixture['files'].items()}
|
||||||
|
cls.body=cls.files[c.SNAPSHOT+'/recsys-contributions.html']
|
||||||
|
|
||||||
|
def materialize(self,root):
|
||||||
|
for name,raw in self.files.items():
|
||||||
|
path=root/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(raw)
|
||||||
|
|
||||||
|
def test_all263_observations_against_independent_gold(self):
|
||||||
|
items=c.parse_program(self.body)
|
||||||
|
keys=self.fixture['independent_gold'][0].keys()
|
||||||
|
self.assertEqual([{k:r[k] for k in keys} for r in items],self.fixture['independent_gold'])
|
||||||
|
self.assertEqual(len(items),263);self.assertEqual(len({r['doi'] for r in items}),260)
|
||||||
|
self.assertEqual(sum(bool(r['review_flags']) for r in items),6)
|
||||||
|
variants=[r for r in items if r['doi']=='10.1145/3773078.3831839']
|
||||||
|
self.assertNotEqual(variants[0]['title'],variants[1]['title']);self.assertNotEqual(variants[0]['authors'],variants[1]['authors'])
|
||||||
|
self.assertNotEqual(variants[0]['abstract'],variants[1]['abstract'])
|
||||||
|
|
||||||
|
def test_installed_format_tracks_evidence_and_original_fields(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);src=root/'src';self.materialize(src);out=root/'out'
|
||||||
|
report=c.collect(out,snapshots=src)
|
||||||
|
self.assertEqual((report['records'],report['unique_dois'],report['missing_abstract_count']),(263,260,0))
|
||||||
|
self.assertFalse(report['database_written']);self.assertFalse(report['pdfs_downloaded']);self.assertEqual(report['network_requests'],0);self.assertEqual(report['approved_for_import'],0)
|
||||||
|
self.assertEqual(report['track_counts'],{k:n for k,_,n in c.TRACKS});self.assertEqual(report['duplicate_doi_observations'],6)
|
||||||
|
gold={r['locator']:r for r in self.fixture['independent_gold']};seen=set()
|
||||||
|
for manifest in report['files']:
|
||||||
|
path=Path(manifest['file']);raw=path.read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),manifest['file_sha256'])
|
||||||
|
rows=[json.loads(l) for l in raw.splitlines()];self.assertEqual(len(rows),manifest['count']);self.assertEqual(manifest['status'],'partial')
|
||||||
|
for r in rows:
|
||||||
|
item=r['metadata']['program_fields'];locator=item['locator'];seen.add(locator);expected=gold[locator]
|
||||||
|
for key in expected:self.assertEqual(item[key],expected[key])
|
||||||
|
self.assertEqual(r['external_id'],'2026/'+locator);self.assertEqual(r['title'],expected['title']);self.assertEqual(r['authors'],expected['authors']);self.assertEqual(r['abstract'],expected['abstract'])
|
||||||
|
self.assertEqual(r['track'],manifest['track']);self.assertEqual(r['identifiers'],{'doi':expected['doi']});self.assertEqual(r['submission_identifiers'],[{'scheme':'publication_doi','value':expected['doi']}])
|
||||||
|
self.assertEqual(r['tags'],[{'namespace':'recsys_program:contribution_type','value':expected['type_labels'][0]}]);self.assertFalse(r['metadata']['original_decision_signature_verified'])
|
||||||
|
self.assertEqual(seen,set(gold));self.assertEqual((out/'evidence/recsys-contributions.html').read_bytes(),self.body)
|
||||||
|
with self.assertRaisesRegex(ValueError,'Output exists'):c.collect(out,snapshots=src)
|
||||||
|
|
||||||
|
def test_edition_and_track_label_drift_rejected(self):
|
||||||
|
for old,new in [(b'RecSys 26 - Accepted Contributions',b'RecSys 25 - Accepted Contributions'),(b'>Long Papers</a>',b'>Workshops</a>'),(b"title='Research'",b"title='Unreviewed'")]:
|
||||||
|
with self.subTest(old=old),self.assertRaises(ValueError):c.parse_program(self.body.replace(old,new,1))
|
||||||
|
|
||||||
|
def test_duplicate_locator_missing_author_and_foreign_doi_rejected(self):
|
||||||
|
for old,new in [(b'accordion-1-slide-2"',b'accordion-1-slide-1"'),(b'by Md Aminul Islam, Kathryn Vasilaky and Elena Zheleva',b'by '),(b'https://doi.org/10.1145/3773078.3831824',b'https://doi.org/10.1145/9999999.3831824')]:
|
||||||
|
with self.subTest(old=old),self.assertRaises(ValueError):c.parse_program(self.body.replace(old,new,1))
|
||||||
|
|
||||||
|
def test_math_markup_requires_review_and_is_not_flattened(self):
|
||||||
|
changed=self.body.replace(b'Learning-to-rank (LTR)',b'Learning-to-rank <sup>2</sup> (LTR)',1)
|
||||||
|
with self.assertRaisesRegex(ValueError,'markup'):c.parse_program(changed)
|
||||||
|
|
||||||
|
def test_corruption_and_missing_receipt_body_preserve_failure(self):
|
||||||
|
for kind in ('manifest','body','missing'):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);src=root/'src';self.materialize(src);path=src/c.SNAPSHOT/('manifest.json' if kind=='manifest' else 'recsys-contributions.html')
|
||||||
|
if kind=='missing':path.unlink()
|
||||||
|
else:path.write_bytes(path.read_bytes()+b' ')
|
||||||
|
with self.assertRaises((ValueError,FileNotFoundError)):c.collect(root/'out',snapshots=src)
|
||||||
|
self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_symlink_source_and_root_rejected(self):
|
||||||
|
for kind in ('source','root'):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);src=root/'src';self.materialize(src);path=src/c.SNAPSHOT/'recsys-contributions.html' if kind=='source' else src
|
||||||
|
original=path.with_name(path.name+'-original');path.rename(original);path.symlink_to(original,target_is_directory=kind=='root')
|
||||||
|
with self.assertRaisesRegex(ValueError,'Unsafe'):c.collect(root/'out',snapshots=src)
|
||||||
Reference in New Issue
Block a user