Compare commits

..

11 Commits

Author SHA1 Message Date
codex-bot b14193063a Collect RecSys 2026 source observations with explicit duplicate DOI versions
Metadata integrity checks / metadata (push) Successful in 3m20s
2026-11-17 19:34:03 +08:00
Zayden-Jung 6a89f5f6c2 Merge pull request '封装 ECCV2026 公开元数据与不可变断点缓存' (#56) from codex/eccv2026 into main
Metadata integrity checks / metadata (push) Successful in 5m55s
Reviewed-on: #56
2026-11-17 09:16:10 +00:00
Zayden-Jung cb3d97ed92 Merge pull request '记录 SDM2026 身份核查、历史保护与恢复入口' (#58) from codex/sdm2026-review into main
Metadata integrity checks / metadata (push) Successful in 3m22s
Reviewed-on: #58
2026-11-17 09:16:00 +00:00
codex-bot 06467835c7 docs: record SDM2026 identity evidence and recovery gates
Metadata integrity checks / metadata (push) Successful in 3m0s
2026-11-17 16:59:15 +08:00
codex-bot f2753bc741 Merge main and preserve both ECCV and SDM validation paths
Metadata integrity checks / metadata (push) Successful in 3m5s
2026-11-17 16:13:06 +08:00
Zayden-Jung 1ea17c810d Merge pull request '封装 SDM2026 注册论文与独立日程版本' (#57) from codex/sdm2026 into main
Metadata integrity checks / metadata (push) Successful in 2m53s
Reviewed-on: #57
2026-11-17 08:07:51 +00:00
codex-bot f800c6ecb4 fix: preserve already matching ECCV literal text before entity decoding
Metadata integrity checks / metadata (push) Successful in 3m8s
2026-11-17 16:07:06 +08:00
codex-bot 279d6a2735 fix: preserve and validate encoded ECCV structured author text
Metadata integrity checks / metadata (push) Successful in 3m9s
2026-11-17 14:19:21 +08:00
codex-bot 8996a21e09 docs: record ECCV contribution types and rejected metadata access evidence
Metadata integrity checks / metadata (push) Successful in 3m19s
2026-11-17 13:49:32 +08:00
codex-bot e9bcbc3aef Preserve exact HTTP bytes when replaying ECCV fixtures
Metadata integrity checks / metadata (push) Successful in 3m7s
2026-11-17 05:20:10 +08:00
codex-bot c97ea94b49 Collect ECCV 2026 public metadata with immutable resumable cache
Metadata integrity checks / metadata (push) Failing after 2m44s
2026-11-17 05:12:31 +08:00
16 changed files with 1097 additions and 1 deletions
+44 -1
View File
@@ -12,7 +12,7 @@ jobs:
- name: Verify metadata storage and query commands
run: |
python3 -B - <<'PY'
import gzip, io, json, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request
import base64, gzip, io, json, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request
sha = os.environ['ARCHIVE_SHA']
if not re.fullmatch(r'[0-9a-fA-F]{40,64}', sha):
raise SystemExit('Invalid source commit')
@@ -49,6 +49,49 @@ jobs:
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--help'],cwd=outside,check=True)
eccv_fixture = json.loads(gzip.decompress((src/'tests/fixtures/eccv2026-source.json.gz').read_bytes()))
eccv_official = root/'eccv2026-official'
eccv_papers = root/'eccv2026-paper-samples'
eccv_official.mkdir(); eccv_papers.mkdir()
for name, html in eccv_fixture['official_pages'].items():
(eccv_official/name).write_bytes(html.encode('utf-8'))
(eccv_official/'manifest.json').write_bytes(eccv_fixture['official_manifest'].encode('utf-8'))
for pid, html in eccv_fixture['paper_pages'].items():
(eccv_papers/(pid+'.html')).write_bytes(html.encode('utf-8'))
eccv_encoding = eccv_fixture['encoding_sample']
(eccv_papers/(eccv_encoding['paper_id']+'.html')).write_bytes(eccv_encoding['html'].encode('utf-8'))
eccv_fixture['paper_manifest']['records'].append({**eccv_encoding['receipt'], 'paper_id': eccv_encoding['paper_id'], 'file': eccv_encoding['paper_id']+'.html'})
eccv_literal = eccv_fixture['literal_title_sample']
(eccv_papers/(eccv_literal['paper_id']+'.html')).write_bytes(eccv_literal['html'].encode('utf-8'))
eccv_fixture['paper_manifest']['records'].append({**eccv_literal['receipt'], 'paper_id': eccv_literal['paper_id'], 'file': eccv_literal['paper_id']+'.html'})
(eccv_papers/'manifest.json').write_text(json.dumps(eccv_fixture['paper_manifest']))
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','eccv2026-program','--official-snapshots',str(eccv_official),'--paper-snapshots',str(eccv_papers),'--out',str(root/'installed-eccv2026')],cwd=outside,check=True)
eccv_report = json.loads((root/'installed-eccv2026/collection-report.json').read_text())
assert eccv_report['records'] == 5 and eccv_report['expected_program_ids'] == 2834
assert eccv_report['duplicate_program_occurrences'] == 17 and len(eccv_report['uncollected_program_ids']) == 2829
assert not eccv_report['database_written'] and eccv_report['files'][0]['track'] == 'program_unclassified'
eccv_rows = [json.loads(line) for line in pathlib.Path(eccv_report['files'][0]['file']).read_text().splitlines()]
eccv_encoded_row = next(row for row in eccv_rows if row['external_id'] == '2026/4199')
assert eccv_encoded_row['authors'] == eccv_encoding['gold']['authors']
assert 'jsonld_display_text_normalized_requires_review' in eccv_encoded_row['metadata']['public_page_fields']['review_flags']
eccv_literal_row = next(row for row in eccv_rows if row['external_id'] == '2026/3844')
assert eccv_literal_row['title'] == eccv_literal['gold']['visible_title']
assert '&amp;' in eccv_literal_row['title']
recsys_fixture = json.loads(gzip.decompress((src/'tests/fixtures/recsys2026-source.json.gz').read_bytes()))
recsys_snapshots = root/'recsys2026-snapshots'
recsys_snapshots.mkdir()
for name,encoded in recsys_fixture['files'].items():
path = recsys_snapshots/name
path.parent.mkdir(parents=True,exist_ok=True)
path.write_bytes(base64.b64decode(encoded))
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','recsys2026-program','--snapshots',str(recsys_snapshots),'--out',str(root/'installed-recsys2026')],cwd=outside,check=True)
recsys_report = json.loads((root/'installed-recsys2026/collection-report.json').read_text())
assert (recsys_report['records'],recsys_report['unique_dois'],recsys_report['missing_abstract_count']) == (263,260,0)
assert recsys_report['duplicate_doi_observations'] == 6 and len(recsys_report['repeated_dois']) == 3
assert not recsys_report['database_written'] and recsys_report['network_requests'] == 0 and recsys_report['approved_for_import'] == 0
recsys_rows = [json.loads(line) for manifest in recsys_report['files'] for line in pathlib.Path(manifest['file']).read_text().splitlines()]
gold_keys = recsys_fixture['independent_gold'][0].keys()
assert [{k:r['metadata']['program_fields'][k] for k in gold_keys} for r in recsys_rows] == recsys_fixture['independent_gold']
sdm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/sdm2026-source.json.gz').read_bytes()))
sdm_snapshots = root/'sdm2026-snapshots'
for name, text in sdm_fixture['files'].items():
+4
View File
@@ -129,4 +129,8 @@ ICAPS 2026 也提供独立的 `collect icaps2026-registry`,保留 Crossref 卷
`collect corl2026-program` 将官方 687 个论坛与固定公开档案按 ID 对照,保留 34 条标题版本差异及分字段来源。不会自动接受 arXiv 匹配或推断拒稿;详情见 [CoRL 2026 使用与恢复说明](docs/corl2026-program.md)。
ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目 ID,并以不可变缓存断点采集公开元数据;保留摘要原文、来源关键词及未分类 track,始终待身份审核,不自动入库。
`collect recsys2026-program` 核验九个 track 的 263 项官方录用贡献观察(260 个不同 DOI),保留摘要、原始作者和重复 DOI 的来源版本。见 [RecSys 2026 使用与恢复说明](docs/recsys2026-program.md)。
`collect sdm2026-proceedings` 核验 53 个 Crossref 论文章节,并单独保留 53 项日程观察及 52 页摘要。作者差异、取消展示和摘要缺失分别保留,不自动拼接来源或绑定作品;见 [SDM 2026 使用与恢复说明](docs/sdm2026-proceedings.md)。
+35
View File
@@ -0,0 +1,35 @@
# ECCV 2026:暂定节目与公开元数据
这是长期维护、按需运行的来源适配器。它读取已核验的官方节目快照,采集公开论文页的标题、完整作者和摘要,输出待身份审核文件;不连接数据库,不下载 PDF、评审、个人档案或受保护的直播内容。
```sh
paper-atlas collect eccv2026-program \
--official-snapshots /data/paper-index/collected/eccv2026-official-research-v1 \
--cache /data/paper-index/cache/eccv2026-public-pages \
--max-new-pages 200 \
--out /data/paper-index/collected/eccv2026-pass01
```
官方目录包含 `accepted.html`、`policies.html`、`faq.html` 和冻结的 `manifest.json`。当前适配器核验固定版本:节目 2,851 行、2,834 个节目 ID;17 个 ID 各重复一次,标题、作者、关键词、展示信息均一致。输出按节目 ID 合并这些重复行,并保留全部行序号。它不是跨来源作品去重。
`--max-new-pages` 限制本轮新请求数;不指定时遍历整个冻结名单。再次运行时使用同一 `--cache` 和**全新** `--out`,复用已验证缓存并继续读取缺失页。缓存不按 24 小时过期,不自动刷新或覆盖已成功的原始响应。每次运行输出包含本轮能核验的全部缓存条目,并明确列出尚未采集的 ID;不能未经审核把多轮重叠输出全部重复导入。
公开请求按固定来源和路径限制,只接受 HTTP 200、原 URL、HTML 和限定大小的完整响应。请求失败、重定向、访问挑战、非 HTML 或异常响应留下阻断标记,不自动重试或更换账号、域名、代理绕过限制。记录的 `Retry-After` 供人工核查使用。网络超时等未产生成功缓存的情况可在确认进程终止后用新输出目录继续。缓存数据与回执须成对且哈希一致;遇到孤立文件或 `.part` 时保留现场,先核查再把本任务的残留文件移至恢复目录,不静默丢弃。写同一缓存时持有非阻塞单写锁。
离线重放可改用 `--paper-snapshots /path/to/papers`,该目录包含 `<节目ID>.html` 和 `manifest.json`(带时区的 `at`、`records`,各条含文件、URL、最终 URL、HTTP 状态、类型、字节数、SHA-256)。缺少的公开页会明确列为未采集;三篇样本只会生成三条 partial 记录,不会冒充全量成功。
## 来源边界
[官方名单](https://eccv.ecva.net/Conferences/2026/AcceptedPapers)明确注明 **preliminary pending publisher checks**。这里的 accepted 表示冻结时列入暂定官方名单,不代表已核验 Springer 最终发表或原始签名决定。所有输出始终 partial。节目 ID 是官方节目定位符,不冒充原始 OpenReview 投稿号;尚未核实跨平台标识时作品标识和投稿标识均为空。
公开页的 canonical URL、可见标题/作者和 JSON-LD 相互核对;页面版本与冻结节目文字有差异时分别保留并标记待审阅,不制造人名别名。JSON-LD 日期只是网站条目时间,不当作论文发表日期或原始决定日期。摘要保留公开页 DOM 文本(规范空白),包括源头的断词、缺空格;不自动修补。原始 HTML 单独留存;遇到公式、上下标或图片结构会标记待审阅,不能把扁平化文本当作无损数学表达。页面上的登录提示对应直播内容,不用于访问受保护内容。
第 4199 页的 JSON-LD 作者姓名存在双重 HTML 实体编码;第 3844 页的可见标题和 JSON-LD 则已经一致地包含字面 `&amp;`。解析器优先保留已经一致的文字,仅在字段不一致时尝试最多两层实体解码来核对可见文本,不把已经一致的标题再次解码。完整作者序列仍须精确一致,真实差异或更多编码层次仍停止。`jsonld_raw_text` 保留原始结构化标题/姓名,发生文本规范化时增加 `jsonld_display_text_normalized_requires_review`,不生成姓名别名。恢复必须使用校验过的修复版本、新输出目录和原缓存;保留旧失败目录,不重新请求已有成功页面。
官方关键词保存在独立 `eccv_program:keyword` 命名空间,并保留原始单元格和证据。关键词、展示形式、贡献/研究类型、自定义标签相互独立,投稿 track 暂用 `program_unclassified`,不按主题推断 main、findings、workshop 或 position。
[2026 投稿政策](https://eccv.ecva.net/Conferences/2026/SubmissionPolicies)明确评审和作者回复不公开;这不能代替对公开 rejected、desk rejected、withdrawn **决定元数据**的单独核查。当前拒稿决定可见性仍记为 unknown,不能从录用表缺席推断拒稿。
贡献类型的官方定义、OpenReview 状态入口与匿名访问核查见[来源核查记录](eccv2026-source-status.md)。这是独立于冻结采集输入的证据,不会改变已核验节目版本,也不能用类型定义替论文推断标签。
失败输出保留 `failed.json`,成功报告最后发布;恢复前核查既有进程、输出及缓存,不因日志安静重启。身份、来源版本和完整投稿历史审阅后,另行准备专用入库队列、保护基线和导入后验收;全部来源导入结束后仍须全库去重、覆盖、摘要、约束及查询审计。测试、构建及安装验证只在已授权 `ces-control:host` runner 执行。
+39
View File
@@ -0,0 +1,39 @@
# ECCV 2026 来源核查记录
核查时间:2026-11-17,以下状态是当时的公开可见性。长期补漏时先核实来源是否变化,保留旧快照,不覆盖旧结论或重复导入。
## 贡献类型与 track
[作者贡献类型说明](https://eccv.ecva.net/Conferences/2026/AuthorContributionTypes)通过官方导航确认。它要求作者选择一项主要贡献类型;官方 FAQ 明确贡献类型不是独立 track。公开 OpenReview group 的 `source_submissions_query_mapping` 也包含五项对应值:
| 官方 group 字段值 | 含义摘要 |
| --- | --- |
| Algorithms/General | 方法、模型或算法;也是边界情形的默认选择 |
| Theory/Foundational | 理论、形式分析或基础理解 |
| Applied/Systems | 实际系统、部署与工程约束 |
| Datasets/Benchmarks | 数据集、基准或评估协议 |
| Concept & Feasibility | 新概念及可行性证据 |
这些是类型定义和配置值,**尚未核验逐篇论文的作者所选类型**。不能根据标题、摘要、主题关键词或展示形式补写类型;公开节目采集器继续输出 `program_unclassified`,主题关键词继续使用 `eccv_program:keyword`。以后获得逐篇类型证据时,保留独立命名空间、原始值、来源和版本,不把它写成投稿 track。
## 公开负面决定元数据
[投稿指南](https://eccv.ecva.net/Conferences/2026/AuthorGuide)链接的会议 group 是 [`thecvf.com/ECCV/2026/Conference`](https://openreview.net/group?id=thecvf.com/ECCV/2026/Conference)。匿名 API 读取 group 成功(HTTP 200),返回 `readers: ["everyone"]`。这只说明 group 配置公开,不证明论文或决定公开。
group 的 `rejected_venue_id` 实际配置为 `thecvf.com/ECCV/2026/Conference/Rejected_Submission`。使用该值查询公开 notes(`content.venueid`,`limit=1`)返回 HTTP 403,随后停止;没有自动重试、切换账号、代理或其他绕过操作。其余 desk-rejected、withdrawn 和 submission 状态查询没有执行,不能报告为已核查或零条。
结论仍为 **拒稿决定元数据覆盖未核实,当前匿名查询受限**。403 的具体原因未确定;不能由此推断真实投稿数量、全局不存在公开拒稿数据,或某篇论文遭拒。[投稿政策](https://eccv.ecva.net/Conferences/2026/SubmissionPolicies)关于评审与作者回复不公开的规定也不能代替决定元数据核查。
## 可恢复证据与后续入口
服务器证据目录:`/data/paper-index/collected/eccv2026-policy-research-v2`。本机工作记录目录保留同名备份;三份成功响应均已逐文件校验。证据不含登录凭据。
| 文件 | SHA-256 |
| --- | --- |
| manifest.json | 5a04e52bf6ea34ab25fe1d91fa9a1df18c9be5abb4f554586ee28fe9aa566e6e |
| contribution-types.html | 6a5128b1f47a275411acacbb08ab91320a767bdcc48aaad09fe1be9e31dff00f |
| author-guide.html | 779196716f5b37fcd10ddc191e9cd0530d7e9d43e5bc66b16a5546cd152959c4 |
| openreview-group.json | 055bcf99d412d77f79a760890739d4de13a20ea099857154ee3e714c0647e763 |
| anonymous-decision-probes/manifest.json | a522756ed3ec2880e5d942a211d7ad13819b18c8acada551dd3a8718b5e4d63b |
这里的补充证据与采集器的三份冻结节目/政策输入分开保存,不能替换采集器固定版本的 manifest。下次补漏先核查官方是否提供公开决定导出或明确的公开元数据说明;访问仍受限时保留缺口。若发现允许公开访问的新入口,另存带时间与哈希的快照,先审阅读者权限、决定语义和逐篇身份,再准备入库队列。
+53
View File
@@ -0,0 +1,53 @@
# RecSys 2026 公开录用贡献采集
这是长期维护项目中的离线采集器,按需处理冻结的官方元数据,不连接数据库、不访问论文链接或下载 PDF。输出须经过作品、DOI 和投稿历史审查才能导入。
## 使用
```sh
paper-atlas collect recsys2026-program \
--snapshots /data/paper-index/collected \
--out /data/paper-index/collected/recsys2026-normalized-<new-run>
```
输入父目录应包含 `remaining2026-listings-research-v2/manifest.json` 与 `recsys-contributions.html`。采集器检查固定清单 SHA256、官方 URL、HTTP 200、采集时区、原始字节哈希、九个 track 的菜单及完整条目数。现有输出目录不可覆盖;失败保留 `failed.json`,成功才发布 `collection-report.json`。再次运行前先检查原任务是否结束。
## 数据范围与重复项
来源是 [RecSys 官方录用贡献页面](https://recsys.acm.org/recsys26/contributions/),冻结时间为 2026-11-17。九个来源 track:
| track | 观察数 |
| --- | ---: |
| long | 65 |
| short | 29 |
| past_present_future | 13 |
| reproducibility | 4 |
| resource | 7 |
| industry | 83 |
| research_practice_notes | 34 |
| demo | 18 |
| doctoral_symposium | 10 |
**263 项来源观察对应 260 个不同 DOI,并非 263 篇不同论文。** 每条记录用官方页面中的 accordion locator 标识,分别保留来源标题、作者原文、摘要和位置,不假设 locator 是原始投稿 ID。
以下 DOI 各出现两次,六个观察均显式标记待审查:
- `10.1145/3773078.3831839`:Dart 两条来源记录的标题、作者和摘要都有差异;不得静默选一个覆盖另一个。
- `10.1145/3773078.3831819`:标题与作者相同、摘要版本不同。
- `10.1145/3773078.3831861`:重复展示的文字相同,保留两个来源位置。
所有观察都有公开摘要。正文只转换为 HTML 展示文字并规范空白,原始页面完整保留;遇到未审查的数学或图片结构会停止,不把上标扁平化成数字。作者按来源的逗号和最后一个 and 分隔,原始名单同时保留;这不是已验证的作者人物身份或别名。
`accepted` 仅指官方页面明示录用,不是原始签名评审决定。DOI 来自官方超链接,尚未独立核查实时出版商版本;归入 `publication_doi`,不复制其他会议的投稿 ID。track 明确来自页面区块;`recsys_program:contribution_type` 仅保存页面原始类型标签(Research、PastPresentFuture、Doctoral Symposium),各 track 的 RES/IND/RPN 等缩写同时保留在来源字段,不统一推断研究方法类型。
本页只含录用贡献,不能据此推断未出现的稿件被拒。公开拒稿导出仍为 unknown,征稿页的 desk rejection 政策不等于具体拒稿记录。不得将本批采集成功当成会议全部来源、负面决定或全库去重完成。
## 输出与恢复
输出九份 JSONL、对应部分覆盖清单、原始 HTML 和清单证据,以及记录观察数和不同 DOI 数的报告;批准导入数为零。
清单 SHA256:`fea7b710587636db06794ee195cd083634c41e9054a217859d9dd0ccdd1afab9`。
服务器恢复包 `/data/paper-index/validation/remaining2026-listings-research-v2-recovery.tar.gz`,本地备份 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/remaining2026-listings-research-v2-recovery.tar.gz`,SHA256 为 `f5ef7c2e73d46712146e4d6711ad5ad562e16b8b55c0cd75d1041cd0b81142cb`。安全恢复原始目录、核对哈希后重跑上述命令。
完整来源与独立 263 项基准已保存在测试夹具,所有测试、构建和安装验证交给指定的 `ces-control:host` runner。生产导入前还需独立核验规范化结果,并检查当前数据库已有 DOI 绑定及历次投稿。最终全库去重审计仍是独立必做阶段。
+78
View File
@@ -0,0 +1,78 @@
# SDM 2026:生产规范化与身份核查记录
本页记录 2026-11-17 的来源验收和只读身份核查,供下次按需运行时恢复。**本轮没有写入 PostgreSQL,53 条记录均未获得导入批准。** 核查完成、存在候选、内容支持同一作品和最终允许入库是不同状态。
## 已核实的生产输出
生产规范化使用通过远程 CI 1216 的源码版本 `f2753bc741d0fff06134cf64019dac38002fae37`,对应 494 项测试、wheel 构建及安装后 CLI 检查。该版本包含已合并的 SDM 适配器;这里没有替换服务器默认代码或运行环境。本页是文档更新,不能把此前 CI 结果当作本次提交的 CI 结果。
`/data/paper-index/collected/sdm2026-normalized-v1` 保留:
- 53 条 Crossref 论文章节记录,全部没有注册摘要;排除论文集和前言。
- 53 条独立日程观察,其中 52 条有公开摘要;20 处作者序列差异仍保留。
- 取消展示的条目与已注册发表记录分别保留,不生成撤回决定。
规范化服务 `paper-atlas-sdm2026-normalize-v1.service` 的 invocation 为 `3021248cae4044ac88eb80fb8b964a26`,终态成功。独立审计逐条核对冻结注册字段、完整作者顺序、原始日程和摘要证据,未以适配器自身统计替代验收。
| 证据文件 | SHA-256 |
| --- | --- |
| validation/sdm2026-normalized-v1-audit.json | 66d4e698ff8828731ccb657e273fb2f0d19cbce8fd91ade2c81ed68124c6d3a3 |
| collected/sdm2026-normalized-v1/sdm-crossref-registry-2026-unclassified.jsonl | 7edd78170ced9b79e4405eb16d63f8a9a38d38dfc906e892580f0ac4be165b2c |
| collected/sdm2026-normalized-v1/program-observations.jsonl | 421a7187369817fa90aa3484575e8a8a2f3e008b9d152bed6beca678b1e12204 |
表中服务器相对路径均位于 `/data/paper-index/`。本机工作记录根目录为 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`,保存同名输出及审计备份;数据证据不作为项目源码提交。
## 当前数据库只读预检
`paper-atlas-sdm2026-preflight-v1.service`,invocation `9be7227930354633abf8f41cdbd0dd73`,终态成功。数据库为 `foundry_data.research`,快照时间 `2026-11-17 08:34:34.270409+00`。报告核实事务为 READ ONLY,`database_written=false`、`approved_for_import=0`。
| 候选类别 | 记录数 | 含义 |
| --- | ---: | --- |
| 精确标题与作者集合一致 | 26 | 需要进一步核对作品、版本和已有标识 |
| 精确标题但仅部分作者重合 | 2 | 作者版本冲突尚未解决 |
| 没有精确匹配 | 25 | 不是新作品证明,不能直接创建论文 |
报告位于 `collected/sdm2026-preflight-v1/postgresql-preflight.json`,SHA-256 为 `eb0cc45fa5bf01159133af0536ba6621a2403914ccd9591a5d1a6a230bc3791e`。28 个候选论文压缩证据包的哈希均核对通过;来源记录、投稿、决定及其来源关系均未触发报告的截断标志。这里的完整性仅限这些候选包,不代表全库审计完成。
随后逐项阅读来源摘要、既有摘要、注册作者和已有标识,为 25 个候选记录了支持同一作品的具体方法、数据集、实验结果或摘要一致性证据,仍需最新数据库保护基线和导入前检查。相似度只用于安排审阅顺序,不作为自动批准阈值。以下三条保留作者版本问题:
| 章节 DOI 尾号 | 既有 paper_id | 尚存问题 |
| --- | ---: | --- |
| .10 | 2656770 | 注册姓名 Tiejun Chen 与既有及日程姓名 Tiejin Chen 不同;不生成别名 |
| .15 | 2745754 | 注册 Sarang Patil 与既有 Sarang Rajendra Patil 不同;日程另有作者顺序与姓名缩写变化 |
| .49 | 2506693 | 注册和既有完整作者集合相同,但日程列出不同名单;ScaffAug 内容一致性仍不能消除来源矛盾 |
这里的尾号均属于 `10.1137/1.9781611979329`。逐条内容审阅文件为本机工作记录根目录的 `sdm2026-candidate-evidence-review-v1.json`,SHA-256 为 `6ff791f87eafeb50fa853766c632e91e3f0377ab3f619365117d557f6d56100d`。其中 25 条为 `same_work_supported_pending_fresh_import_guards`,3 条为 `hold_author_version_conflict`;所有条目的 `approved_for_import` 仍为 false。不能据此直接重放来源、绑定 DOI 或拼接日程摘要。
## 需要保护的较早拒稿历史
| SDM 候选作品 | paper_id | 已有较早决定 |
| --- | ---: | --- |
| SeBA | 2672776 | ICLR 2026 rejected |
| Towards Reliability of Parameter-Free Optimization | 3282961 | ICLR 2025 rejected |
| Scaffold-Aware Generative Augmentation and Reranking for Enhanced Virtual Screening | 2506693 | ICLR 2026 desk_rejected |
前两条已有内容支持,但没有导入 SDM 事件;第三条仍保留作者版本问题。如果后续通过身份核查,SDM 发表事实应作为独立会议事件加入,保留旧投稿、旧决定及各自来源。两个来源对同一拒稿的观察不能直接统计成两次独立投稿或拒稿。不能覆盖旧状态,也不能因这批录用候选就宣称完整投稿历史已补齐。
## 未精确匹配的补充检索
对另外 25 条运行了有时限、数量上限和只读快照的补充检索,使用现有索引,不新增数据库索引或改变全局配置。
- 全文检索 `paper-atlas-sdm2026-title-variants-v1.service`,invocation `29c5d2a288804ce7a9c3e2f7db4dca6f`,终态成功。每项使用完整标题及明确记录的锚点词,8 条至少有一次查询达到候选数量上限,不能据此排除重复。
- 标题前缀范围检索 `paper-atlas-sdm2026-title-prefix-v2.service`,invocation `3087f1c0f252427485a661a63c50b27f`,终态成功。25 条均使用既有标题索引,没有查询错误或数量截断。18 字符前缀只是补充方法,不是全库模糊匹配验收,也不会消除前一轮查询的限制。
全文候选中的既有论文 `2025464` 与 SDM .9 共享作者,但前者讨论 CLOUD、离散/混合/连续变量,后者讨论 SLICE、类别数据与 HCR 模型。标题前缀候选 `938886` 与 SDM .28 只共享一名作者,既有版本讨论随机游走相似度、有效电阻和社交网络推断。不能凭研究主题或作者重合将这些论文合并。25 条仍保留为待进一步查证,没有获准创建新作品。
| 服务器恢复文件 | SHA-256 |
| --- | --- |
| validation/sdm2026-title-variants-v1-recovery.tar.gz | 1f387f2ff4d5684aa72abd419bddeb7e972624f6a56f6e9a943168aa944b9137 |
| validation/sdm2026-title-prefix-v2-recovery.tar.gz | 4941343bd6152c5046778f7c82588e3c6ad41c9bb938e0107f8b2b426ac10772 |
两个原报告的 SHA-256 分别为 `b6ab2e9228b8e8e9548b27df70834faf25bfda551844979f8410a74b6e6aadd0` 和 `c1f902b4e4f33997eb182b10dc78332e182ceb6051f1a7c14b21465c8878263d`。前者保留 1 个进一步审阅的完整候选包,后者保留全部 113 个前缀命中的候选包;这些数量不是论文批准数。
## 下次恢复
1. 先核对实际服务终态、来源及报告哈希,再读取上述证据;终态成功的研究任务无需重启。
2. 继续解决 3 条作者版本问题和 25 条未精确匹配记录;不把缺少候选当作新作品证明,不把源端 403 当作零条公开拒稿。
3. 对已获内容支持的候选另做新快照检查:作品与 DOI 绑定、来源是否已导入、已有会议事件及全部历史保护。建立新的明确入库队列和可恢复备份后再执行写入,不能直接使用本次候选报告当导入计划。
4. 导入后逐项核验新事件、旧历史、来源关系和精确数量变化;注册摘要缺失仍如实保留。完整来源范围结束后另做全库去重、标识冲突、近似标题/作者、覆盖、摘要、约束和查询验收。
+2
View File
@@ -42,3 +42,5 @@ paper-atlas collect sdm2026-proceedings \
服务器恢复包:`/data/paper-index/validation/sdm2026-sources-recovery-v1.tar.gz`,SHA-256 为 `d6d14c0e4bb2c26e16a4888e3b150e44f982d689351638b1b9a28644736c4387`;本机工作记录目录有同名备份。包内保留原始来源、独立盘点及逐页版本比较记录,含义是可复现来源证据,不是已完成导入或身份批准。
下次继续时,先核对来源版本和访问状态,再做当前数据库的 DOI 绑定、标题/作者近似候选与完整投稿历史核查,逐条解决作者及摘要版本问题。批准后另建入库队列、保护基线和导入后验收;所有来源结束后仍需全库去重、覆盖、摘要、约束和实际查询验收。测试、构建和安装验证只在已授权 `ces-control:host` 远程 runner 执行。
生产规范化验收、当前只读候选数量、已有拒稿历史保护及下次恢复入口见 [SDM2026 身份核查记录](sdm2026-identity-review.md)。该记录不代表已批准或完成导入。
+2
View File
@@ -24,6 +24,8 @@ COMMANDS = {
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'),
('collect','eccv2026-program'): ('paper_atlas.collectors.eccv2026', {}, '核验ECCV2026暂定节目并断点采集公开元数据'),
('collect','recsys2026-program'): ('paper_atlas.collectors.recsys2026', {}, '核验 RecSys 2026 公开录用贡献并保留 DOI 来源版本'),
('collect','sdm2026-proceedings'): ('paper_atlas.collectors.sdm2026', {}, '核验 SDM 2026 注册论文并单独保留公开日程版本'),
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
+254
View File
@@ -0,0 +1,254 @@
"""ECCV 2026 frozen preliminary program and resumable public metadata pages."""
import argparse
from collections import Counter
from datetime import datetime, timezone
import fcntl
from html import unescape
import json
from pathlib import Path
import re
from paper_atlas.io.html import Document, Node, clean
from paper_atlas.io.html_cache import PublicHTMLCache, checked_html, digest
SOURCE = 'eccv_program'
ORIGIN = 'https://eccv.ecva.net'
ACCEPTED = ORIGIN+'/Conferences/2026/AcceptedPapers'
TRACK = 'program_unclassified'
MANIFEST_SHA = '04f9734783bbbfa2e19f936d8f5a7aecfb524b0ceef33fc5d9fb6276c7b6678a'
PAGES = {'accepted.html': ACCEPTED, 'policies.html': ORIGIN+'/Conferences/2026/SubmissionPolicies',
'faq.html': ORIGIN+'/Conferences/2026/FAQs'}
def one(items):
if len(items) != 1:
raise ValueError('Expected one reviewed metadata field')
return items[0]
def has_class(node, value):
return value in node.attrs.get('class', '').split()
def parse_program(raw):
nodes = list(Document(raw).root.walk())
if len([n for n in nodes if n.tag == 'h2' and clean(n.text()) == 'ECCV 2026 Accepted Papers:']) != 1:
raise ValueError('Unexpected ECCV program edition')
if 'This paper list is preliminary pending publisher checks.' not in clean(Document(raw).root.text()):
raise ValueError('Reviewed provisional-publication notice changed')
table = one([n for n in nodes if n.tag == 'table' and n.attrs.get('id') == 'event-list-2026-poster-nodates-filter-vslinks-table'])
papers = {}; occurrences = 0
for tr in (n for n in table.walk() if n.tag == 'tr'):
cells = [n for n in tr.children if isinstance(n, Node) and n.tag == 'td']
if not cells:
continue
if len(cells) != 3:
raise ValueError('Unexpected program columns')
links = [n for n in cells[0].walk() if n.tag == 'a' and re.fullmatch(r'/virtual/2026/poster/[1-9][0-9]*', n.attrs.get('href', ''))]
link = one(links); pid = link.attrs['href'].rsplit('/', 1)[1]
authors = clean(one([n for n in cells[0].walk() if n.tag == 'i']).text())
row = {'title': clean(link.text()), 'authors_literal': authors,
'keywords_literal': clean(cells[1].text()), 'presentation_literal': clean(cells[2].text())}
if not row['title'] or not authors:
raise ValueError('Incomplete public program fields')
occurrences += 1
if pid in papers:
if papers[pid]['listing'] != row:
raise ValueError('Conflicting entries share a program ID')
papers[pid]['row_ordinals'].append(occurrences)
else:
papers[pid] = {'url': ORIGIN+link.attrs['href'], 'listing': row, 'row_ordinals': [occurrences]}
if occurrences != 2851 or len(papers) != 2834 or Counter(len(v['row_ordinals']) for v in papers.values()) != {1: 2817, 2: 17}:
raise ValueError('Incomplete reviewed program census')
return papers
def parse_paper(raw, url, program):
nodes = list(Document(raw).root.walk())
title = clean(one([n for n in nodes if n.tag == 'h1' and has_class(n, 'event-title')]).text())
authors_literal = clean(one([n for n in nodes if n.tag == 'div' and has_class(n, 'event-organizers')]).text())
abstract_node = one([n for n in nodes if n.tag == 'div' and has_class(n, 'abstract-text-inner')])
abstract = clean(abstract_node.text())
markup = sorted({n.tag for n in abstract_node.walk() if n.tag in ('math','sup','sub','svg','img')})
canonical = one([n.attrs.get('href') for n in nodes if n.tag == 'link' and n.attrs.get('rel') == 'canonical'])
scripts = [n for n in nodes if n.tag == 'script' and n.attrs.get('type') == 'application/ld+json']
ld = one([json.loads(''.join(x for x in n.children if isinstance(x, str))) for n in scripts])
authors = ld.get('author')
if (canonical != url or ld.get('@type') != 'CreativeWork' or ld.get('creditText') != 'ECCV 2026'
or not title or not isinstance(ld.get('name'), str) or not abstract
or not isinstance(authors, list) or not authors
or any(not isinstance(a, dict) or a.get('@type') != 'Person' or not isinstance(a.get('name'), str)
or not clean(a['name']) or '@' in a['name'] for a in authors)):
raise ValueError('Public visible/structured metadata identity or edition disagrees')
# Prefer source text that already agrees. Poster 3844 intentionally displays
# literal '&amp;', while 4199 double-encodes an author's apostrophe in JSON-LD.
# Decode only to test agreement with visible text, never to rewrite it.
def agrees_with_display(value, expected):
if clean(value) == expected:
return True
for _ in range(2):
value = unescape(value)
if clean(value) == expected:
return True
return False
raw_names = [a['name'] for a in authors]
names = [clean(name) for name in raw_names]
if not agrees_with_display(ld['name'], title):
raise ValueError('Public visible and structured titles disagree')
if ' ⋅ '.join(names) != authors_literal:
visible_names = authors_literal.split(' ⋅ ')
if len(visible_names) != len(raw_names) or any(
not agrees_with_display(raw_name, visible_name)
for raw_name, visible_name in zip(raw_names, visible_names)):
raise ValueError('Complete visible and structured authors disagree')
names = visible_names
if any('@' in name for name in names) or ' ⋅ '.join(names) != authors_literal:
raise ValueError('Complete visible and structured authors disagree')
flags = ['structured_abstract_markup_requires_review'] if markup else []
if raw_names != names or ld['name'] != title:
flags.append('jsonld_display_text_normalized_requires_review')
if title != program['listing']['title']:
flags.append('program_and_page_title_versions_differ_requires_review')
if authors_literal != program['listing']['authors_literal']:
flags.append('program_and_page_author_versions_differ_requires_review')
return {'title': title, 'authors': names, 'authors_literal': authors_literal, 'abstract': abstract,
'page_metadata_dates': {k: ld.get(k) for k in ('datePublished', 'dateModified')}, 'review_flags': flags, 'abstract_markup_tags': markup,
'jsonld_raw_text': {'name': ld['name'], 'author_names': raw_names}}
def official_snapshot(root):
root = Path(root); manifest = root/'manifest.json'
if root.is_symlink() or manifest.is_symlink() or digest(manifest.read_bytes()) != MANIFEST_SHA:
raise ValueError('Reviewed official snapshot manifest changed')
m = json.loads(manifest.read_bytes())
if datetime.fromisoformat(m['at'].replace('Z', '+00:00')).utcoffset() is None:
raise ValueError('Official snapshot requires a timezone')
if len(m['sources']) != 3 or {r['file'] for r in m['sources']} != set(PAGES):
raise ValueError('Official source set changed')
pages = {}; evidence = {}
for r in m['sources']:
name = r['file']; p = root/name
if p.is_symlink() or p.stat().st_size > 8*1024**2:
raise ValueError('Unsafe official snapshot')
raw = p.read_bytes()
if r['url'] != PAGES[name] or r['final_url'] != r['url'] or r['http_status'] != 200 or r['bytes'] != len(raw) or r['sha256'] != digest(raw):
raise ValueError('Invalid official source receipt')
pages[name] = raw; evidence[name] = {'url': r['url'], 'sha256': digest(raw)}
program = parse_program(pages['accepted.html'].decode('utf-8'))
return m, pages, evidence, program
def paper_snapshots(root, program):
root = Path(root); path = root/'manifest.json'
if root.is_symlink() or path.is_symlink():
raise ValueError('Paper snapshot symlinks rejected')
if path.stat().st_size > 4*1024**2:
raise ValueError('Oversized paper manifest')
m = json.loads(path.read_bytes()); result = {}
for r in m['records']:
pid = r['paper_id']
if pid not in program or pid in result or r['file'] != pid+'.html':
raise ValueError('Foreign or repeated paper snapshot')
p = root/r['file']
if p.is_symlink() or p.stat().st_size > 2*1024**2:
raise ValueError('Unsafe paper snapshot')
receipt = {**r, 'retrieved_at': r.get('retrieved_at', m['at'])}
result[pid] = checked_html(p.read_bytes(), receipt, program[pid]['url'], 2*1024**2)
return result
def collect(out, *, official_snapshots, paper_snapshots_dir=None, cache=None, max_new_pages=None):
if (paper_snapshots_dir is None) == (cache is None):
raise ValueError('Choose paper snapshots or public metadata cache')
if max_new_pages is not None and (type(max_new_pages) is not int or max_new_pages < 0 or cache is None):
raise ValueError('New-page budget requires cache mode and a nonnegative integer')
out = Path(out)
if out.exists():
raise ValueError('Output exists; inspect previous operation before retrying')
out.mkdir(parents=True)
lock = None
try:
official, pages, source_evidence, program = official_snapshot(official_snapshots)
directory = out/'evidence'; directory.mkdir()
for name, raw in pages.items():
rel = 'evidence/'+digest(raw)+'.html'; (out/rel).write_bytes(raw); source_evidence[name]['file'] = rel
snapshots = paper_snapshots(paper_snapshots_dir, program) if paper_snapshots_dir is not None else None
client = None
if cache is not None:
cache = Path(cache); cache.mkdir(parents=True, exist_ok=True)
if cache.is_symlink() or (cache/'.collector.lock').is_symlink():
raise ValueError('Cache lock symlinks rejected')
lock = (cache/'.collector.lock').open('a+')
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
client = PublicHTMLCache(cache, ORIGIN)
rows = []; missing = []; fetched = 0; reused = 0
for pid in sorted(program, key=int):
p = program[pid]
if snapshots is not None:
response = snapshots.get(pid)
else:
response = client.cached(p['url'])
if response is not None:
reused += 1
elif max_new_pages is None or fetched < max_new_pages:
response = client.get(p['url']); fetched += 1
if response is None:
missing.append(pid); continue
raw, receipt = response; fields = parse_paper(raw.decode('utf-8'), p['url'], p)
rel = 'evidence/'+digest(raw)+'.html'; (out/rel).write_bytes(raw)
keyword = p['listing']['keywords_literal']
rows.append({'source': SOURCE, 'external_id': '2026/'+pid, 'url': p['url'],
'title': fields['title'], 'authors': fields['authors'], 'abstract': fields['abstract'],
'identifiers': {}, 'submission_identifiers': [], 'venue': 'ECCV', 'year': 2026,
'track': TRACK, 'status': 'accepted', 'observed_at': official['at'],
'raw_decision': 'listed in frozen preliminary ECCV 2026 accepted program, pending publisher checks',
'evidence_url': ACCEPTED,
'tags': [{'namespace': SOURCE+':keyword', 'value': keyword}] if keyword else [],
'metadata': {'official_program': p, 'public_page_fields': fields,
'public_page_receipt': receipt, 'public_page_evidence': {'file': rel, 'sha256': digest(raw), 'url': p['url']},
'program_source_evidence': source_evidence, 'publication_state': 'preliminary_pending_publisher_checks',
'original_decision_signature_verified': False,
'program_id_semantics': 'Official program locator, not a verified original submission number or cross-platform identifier.',
'abstract_quality': 'Public display literal retained, including broken hyphenation/missing spaces; no automatic text repair.',
'date_semantics': 'JSON-LD dates describe the website entry, not verified paper publication or decision dates.',
'label_semantics': 'Official keyword namespace independent of track, contribution/research type and custom labels.',
'scope': 'Public metadata only; no protected live content, reviews, person profiles or PDFs. Requires work identity, metadata version and complete history review before import.'}})
if len(rows) % 10 == 0:
print(json.dumps({'phase': 'public_metadata', 'records': len(rows), 'new_pages': fetched,
'cached_pages': reused, 'expected_program_ids': 2834, 'at': datetime.now(timezone.utc).isoformat()}), flush=True)
files = []
if rows:
path = out/'eccv-program-2026-unclassified.jsonl'; payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ECCV', 'year': 2026, 'track': TRACK,
'count': len(rows), 'file': str(path), 'file_sha256': digest(payload), 'list_url': ACCEPTED,
'status': 'partial', 'missing_abstract_count': 0, 'source_expected_count': 2834}
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n'); files.append(manifest)
report = {'status': 'collected_pending_identity_review' if rows else 'no_public_metadata_collected',
'records': len(rows), 'files': files, 'expected_program_ids': 2834, 'program_occurrences': 2851,
'duplicate_program_occurrences': 17, 'uncollected_program_ids': missing,
'new_pages': fetched, 'reused_pages': reused, 'database_written': False,
'public_review_visibility': 'not_public_per_2026_policy', 'public_rejected_decision_metadata': 'unknown',
'scope': 'Always partial: preliminary program, unresolved track/identity/version coverage; not final whole-database deduplication.'}
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
return report
except Exception as error:
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__,
'database_written': False, 'scope': 'Preserve this output and verified cache; resume with a new output directory after inspection.'})+'\n')
raise
finally:
if lock is not None:
lock.close()
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
parser.add_argument('--official-snapshots', required=True)
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('--paper-snapshots')
group.add_argument('--cache')
parser.add_argument('--max-new-pages', type=int)
parser.add_argument('--out', required=True)
args = parser.parse_args(argv)
print(json.dumps(collect(args.out, official_snapshots=args.official_snapshots,
paper_snapshots_dir=args.paper_snapshots, cache=args.cache, max_new_pages=args.max_new_pages), ensure_ascii=False))
+172
View File
@@ -0,0 +1,172 @@
"""Frozen RecSys 2026 accepted-contribution observations, preserving source variants."""
import argparse
from collections import Counter
from datetime import datetime
import hashlib
import json
from pathlib import Path
import re
from paper_atlas.io.html import Document, Node, clean
SOURCE = 'recsys_program'
SNAPSHOT = 'remaining2026-listings-research-v2'
MANIFEST_SHA = 'fea7b710587636db06794ee195cd083634c41e9054a217859d9dd0ccdd1afab9'
URL = 'https://recsys.acm.org/recsys26/contributions/'
TRACKS = [('long', 'Long Papers', 65), ('short', 'Short Papers', 29),
('past_present_future', 'PPF Papers', 13), ('reproducibility', 'Reproducibility', 4),
('resource', 'Resource', 7), ('industry', 'Industry', 83),
('research_practice_notes', 'R&P Notes', 34), ('demo', 'Demos', 18),
('doctoral_symposium', 'Doctoral', 10)]
def digest(raw):
return hashlib.sha256(raw).hexdigest()
def one(items):
if len(items) != 1:
raise ValueError('Expected exactly one reviewed source field')
return items[0]
def load_snapshot(root):
root = Path(root); directory = root/SNAPSHOT; path = directory/'manifest.json'
if root.is_symlink() or directory.is_symlink() or path.is_symlink() or path.stat().st_size > 128*1024:
raise ValueError('Unsafe snapshot manifest')
raw = path.read_bytes()
if digest(raw) != MANIFEST_SHA:
raise ValueError('Reviewed manifest hash changed')
manifest = json.loads(raw)
if manifest['status'] != 'research_completed_not_coverage_or_import_approval' or manifest['database_written'] is not False:
raise ValueError('Unreviewed snapshot scope')
receipt = one([e for e in manifest['entries'] if e['venue'] == 'RecSys' and e['label'] == 'contributions'])
name = receipt['file']; path = directory/name
if Path(name).name != name or path.is_symlink() or path.stat().st_size > 4*1024**2:
raise ValueError('Unsafe snapshot body')
body = path.read_bytes()
if (receipt['url'] != URL or receipt['final_url'] != URL or receipt['http_status'] != 200
or not receipt['content_type'].startswith('text/html') or receipt['bytes'] != len(body)
or receipt['sha256'] != digest(body) or datetime.fromisoformat(receipt['at']).utcoffset() is None):
raise ValueError('Snapshot receipt/body disagree')
return receipt, {'manifest.json': raw, name: body}, body
def body_text(node, doi_url):
if not isinstance(node, Node):
return node
if node.tag in ('script', 'style') or (node.tag == 'a' and node.attrs.get('href') == doi_url):
return ''
text = ''.join(body_text(c, doi_url) for c in node.children)
return ' '+text+' ' if node.tag in ('p','div','li','br','section') else text
def parse_program(raw):
doc = Document(raw.decode('utf-8')); nodes = list(doc.root.walk())
if not any(n.tag == 'title' and 'RecSys 26 - Accepted Contributions' in n.text() for n in nodes):
raise ValueError('Wrong conference edition')
menu = one([n for n in nodes if n.tag == 'ul' and n.attrs.get('id') == 'tab-1'])
labels = [(n.attrs.get('href'), clean(n.text())) for n in menu.walk() if n.tag == 'a']
if labels != [(f'#content-tab-1-{i}', label) for i, (_,label,_) in enumerate(TRACKS)]:
raise ValueError('Unreviewed track menu')
observations = []; seen = set()
for tab, (track, label, count) in enumerate(TRACKS):
section = one([n for n in nodes if n.tag == 'div' and n.attrs.get('id') == f'content-tab-1-{tab}'])
listing = one([n for n in section.walk() if n.tag == 'ul' and n.attrs.get('id') == f'accordion-{tab+1}'])
rows = [n for n in listing.children if isinstance(n, Node) and n.tag == 'li']
if len(rows) != count:
raise ValueError('Incomplete reviewed track census')
for position, row in enumerate(rows, 1):
anchor = one([n for n in row.children if isinstance(n, Node) and n.tag == 'a'])
locator = anchor.attrs.get('rel', '').removeprefix('#')
if not re.fullmatch(f'accordion-{tab+1}-slide-[0-9]+', locator) or locator in seen:
raise ValueError('Missing or repeated program locator')
seen.add(locator)
types = [n for n in anchor.walk() if n.tag == 'span' and n.attrs.get('class') == 'paper-type']
type_node = one(types); type_literal = type_node.attrs.get('title'); abbreviation = clean(type_node.text())
expected_type = [('Research','RES'),('Research','RES'),('PastPresentFuture','PPF'),
('Research','REPR'),('Research','RSC'),('Research','IND'),('Research','RPN'),('Research','DEMO'),('Doctoral Symposium','DS')][tab]
if (type_literal,abbreviation) != expected_type:
raise ValueError('Unreviewed source contribution type')
title = clean(''.join(c.text() if isinstance(c, Node) else c for c in anchor.children
if not isinstance(c, Node) or c.tag not in ('span','em','br')))
authors_literal = clean(one([n for n in anchor.walk() if n.tag == 'em']).text())
if not authors_literal.startswith('by '):
raise ValueError('Missing program author prefix')
authors_literal = authors_literal[3:]
authors = [n.strip() for n in re.split(r',\s+|\s+and\s+', authors_literal)]
if not title or any(not n or '@' in n for n in authors):
raise ValueError('Missing title or program author')
body = one([n for n in row.children if isinstance(n, Node) and n.tag == 'div'])
doi_url = one([n.attrs.get('href') for n in body.walk() if n.tag == 'a'
and n.attrs.get('href', '').startswith('https://doi.org/')])
if not re.fullmatch(r'https://doi.org/10\.1145/3773078\.[0-9]+', doi_url):
raise ValueError('Wrong publication DOI family')
abstract = clean(body_text(body, doi_url))
if len(abstract) < 100 or any(n.tag in ('math','sup','sub','img','svg') for n in body.walk()):
raise ValueError('Missing abstract or unreviewed structured markup')
observations.append({'doi': doi_url.removeprefix('https://doi.org/'), 'tab': tab, 'position': position,
'locator': locator, 'title': title, 'authors': authors, 'authors_literal': authors_literal,
'abstract': abstract, 'type_labels': [type_literal], 'source_type_abbreviation': abbreviation,
'track': track, 'track_label': label})
counts = Counter(r['doi'] for r in observations)
if len(observations) != 263 or len(counts) != 260 or {d:n for d,n in counts.items() if n>1} != {
'10.1145/3773078.3831839':2,'10.1145/3773078.3831819':2,'10.1145/3773078.3831861':2}:
raise ValueError('Unreviewed DOI observation census')
for item in observations:
item['review_flags'] = ['repeated_program_doi_preserve_versions_requires_review'] if counts[item['doi']] > 1 else []
return observations
def collect(out, *, snapshots):
out = Path(out)
if out.exists():
raise ValueError('Output exists; inspect previous operation before retrying')
out.mkdir(parents=True)
try:
receipt, files, body = load_snapshot(snapshots); observations = parse_program(body)
evidence = out/'evidence'; evidence.mkdir()
for name, raw in files.items():
(evidence/name).write_bytes(raw)
outputs = []
for track,label,count in TRACKS:
rows = []
for item in observations:
if item['track'] != track:
continue
doi = item['doi']; url = URL+'#'+item['locator']
rows.append({'source': SOURCE, 'external_id': '2026/'+item['locator'], 'url': url,
'title': item['title'], 'authors': item['authors'], 'abstract': item['abstract'],
'identifiers': {'doi': doi}, 'submission_identifiers': [{'scheme':'publication_doi','value':doi}],
'venue':'RecSys', 'year':2026, 'track':track, 'status':'accepted', 'observed_at':receipt['at'],
'evidence_url':URL, 'raw_decision':'Listed as an accepted contribution in the official RecSys2026 '+label+' section; not a signed review decision.',
'tags':[{'namespace':SOURCE+':contribution_type','value':item['type_labels'][0]}],
'metadata':{'program_fields':item, 'program_receipt':receipt,
'source_evidence':{'file':'evidence/'+receipt['file'],'sha256':receipt['sha256'],'url':URL},
'source_manifest':{'file':'evidence/manifest.json','sha256':MANIFEST_SHA},
'original_decision_signature_verified':False,
'author_scope':'Source-reported comma/and-delimited names; original literal retained. Not verified publisher person identities or aliases.',
'doi_scope':'DOI asserted by official program hyperlink, not independently verified live publisher metadata.',
'abstract_transform':'HTML display text and whitespace normalization; source retained. No PDF or cross-source fusion.',
'scope':'Program observation keyed by source locator. Repeated DOI variants retained separately and flagged, not counted as distinct scientific works. Requires DOI/work/history review before import.'}})
path = out/f'recsys-program-2026-{track}.jsonl';raw=''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in rows).encode();path.write_bytes(raw)
manifest={'source':SOURCE,'parser_version':1,'venue':'RecSys','year':2026,'track':track,'count':len(rows),
'file':str(path),'file_sha256':digest(raw),'list_url':URL,'status':'partial','missing_abstract_count':0,
'scope':'Official accepted-contribution observations; unique DOI counts differ from observation counts.'}
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)+'\n');outputs.append(manifest)
report={'status':'collected_pending_identity_review','records':263,'unique_dois':260,'files':outputs,
'track_counts':{track:count for track,_,count in TRACKS},'missing_abstract_count':0,
'duplicate_doi_observations':6,'repeated_dois':{d:n for d,n in Counter(r['doi'] for r in observations).items() if n>1},
'source_manifest_sha256':MANIFEST_SHA,'database_written':False,'network_requests':0,'pdfs_downloaded':False,
'approved_for_import':0,'public_rejected_availability':'unknown_no_public_export_verified',
'scope':'263 program observations for260 DOIs, not263 unique papers or final work identities. Track/type labels are source scoped. Not complete public-negative coverage or a final whole-database audit.'}
part=out/'collection-report.json.part';part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n');part.replace(out/'collection-report.json');return report
except Exception as error:
(out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n')
raise
def main(argv=None, *, prog=None):
parser=argparse.ArgumentParser(prog=prog,description=__doc__)
parser.add_argument('--snapshots',required=True);parser.add_argument('--out',required=True)
args=parser.parse_args(argv);print(json.dumps(collect(args.out,snapshots=args.snapshots),ensure_ascii=False))
+106
View File
@@ -0,0 +1,106 @@
"""Immutable, receipt-checked public HTML cache; no expiry or silent refresh."""
from datetime import datetime, timezone
import hashlib
import json
from pathlib import Path
import time
import urllib.request
import urllib.error
from urllib.parse import urlsplit
def digest(raw):
return hashlib.sha256(raw).hexdigest()
def checked_html(raw, receipt, url, limit):
if (len(raw) > limit or receipt.get('url') != url or receipt.get('final_url') != url
or receipt.get('http_status') != 200 or receipt.get('bytes') != len(raw)
or receipt.get('sha256') != digest(raw)
or not str(receipt.get('content_type', '')).lower().startswith('text/html')
or datetime.fromisoformat(receipt['retrieved_at'].replace('Z', '+00:00')).utcoffset() is None):
raise ValueError('Invalid public HTML receipt, response or timestamp')
if any(v in raw.lower() for v in (b'<title>just a moment', b'cf-chl-', b'botstopper')):
raise ValueError('Access challenge; stop without bypass')
return raw, receipt
class NoRedirect(urllib.request.HTTPRedirectHandler):
def redirect_request(self, *args, **kwargs):
return None
class PublicHTMLCache:
"""Caller holds a single-writer lock and restricts the allowed URL paths."""
def __init__(self, directory, origin, *, limit=2*1024**2, delay=1):
self.root = Path(directory)
self.root.mkdir(parents=True, exist_ok=True)
if self.root.is_symlink():
raise ValueError('Cache symlinks rejected')
self.origin = origin
self.limit = limit
self.delay = delay
self.opener = urllib.request.build_opener(NoRedirect())
def paths(self, url):
u = urlsplit(url)
if u.scheme != 'https' or u.username or u.password or u.fragment or u.scheme+'://'+u.netloc != self.origin:
raise ValueError('Unapproved public metadata origin')
key = digest(url.encode())
return self.root/(key+'.html'), self.root/(key+'.json'), self.root/(key+'.failed.json')
def cached(self, url):
body, receipt, failure = self.paths(url)
parts = (body.with_suffix('.html.part'), receipt.with_suffix('.json.part'))
if any(p.is_symlink() for p in (body, receipt, failure, *parts)):
raise ValueError('Cache symlinks rejected')
if any(p.exists() for p in parts):
raise ValueError('Incomplete cache part; preserve and inspect orphan before recovery')
if failure.exists():
raise ValueError('Previous access failure requires inspection; no automatic retry')
if body.exists() != receipt.exists():
raise ValueError('Incomplete cache pair; preserve and inspect orphan before recovery')
if not body.exists():
return None
if body.stat().st_size > self.limit or receipt.stat().st_size > 16*1024:
raise ValueError('Oversized cache entry')
return checked_html(body.read_bytes(), json.loads(receipt.read_bytes()), url, self.limit)
def get(self, url):
cached = self.cached(url)
if cached is not None:
return cached
body, receipt, failure = self.paths(url)
try:
request = urllib.request.Request(url, headers={'User-Agent': 'PaperAtlas/0.1 (public scholarly metadata research)'})
with self.opener.open(request, timeout=30) as response:
rec = {'url': url, 'final_url': response.geturl(), 'http_status': response.status,
'content_type': response.headers.get('Content-Type', '')}
if rec['final_url'] != url or rec['http_status'] != 200 or not rec['content_type'].lower().startswith('text/html'):
raise ValueError('Unexpected metadata redirect, HTTP status or content type')
raw = response.read(self.limit+1)
if len(raw) > self.limit:
raise ValueError('Metadata response exceeds size limit')
expected = response.headers.get('Content-Length')
if expected is not None and int(expected) != len(raw):
raise ConnectionError('Incomplete metadata response')
rec.update(bytes=len(raw), sha256=digest(raw), retrieved_at=datetime.now(timezone.utc).isoformat())
checked_html(raw, rec, url, self.limit)
except (urllib.error.HTTPError, ValueError) as error:
if isinstance(error, urllib.error.HTTPError):
status = error.code
retry_after = error.headers.get('Retry-After') if error.headers else None
error.close()
else:
status = None
retry_after = None
# Access errors remain sticky until the operator verifies availability.
failure.write_text(json.dumps({'url': url, 'http_status': status, 'retry_after': retry_after, 'error_type': type(error).__name__,
'scope': 'Inspect access/redirect/response failure before moving this marker aside; no bypass or automatic retry.'})+'\n')
raise
part = body.with_suffix('.html.part')
part.write_bytes(raw); part.replace(body)
part = receipt.with_suffix('.json.part')
part.write_text(json.dumps(rec, indent=2)+'\n'); part.replace(receipt)
time.sleep(self.delay)
return raw, rec
Binary file not shown.
Binary file not shown.
+139
View File
@@ -0,0 +1,139 @@
import copy
import gzip
import json
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
from paper_atlas.collectors import eccv2026 as c
from paper_atlas.io.html_cache import PublicHTMLCache
class ECCV2026Tests(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.bundle = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/eccv2026-source.json.gz').read_bytes()))
cls.program = c.parse_program(cls.bundle['official_pages']['accepted.html'])
def snapshots(self, root):
official = root/'official'; paper = root/'paper'; official.mkdir(); paper.mkdir()
for name, raw in self.bundle['official_pages'].items():
(official/name).write_bytes(raw.encode('utf-8'))
(official/'manifest.json').write_bytes(self.bundle['official_manifest'].encode('utf-8'))
for pid, raw in self.bundle['paper_pages'].items():
(paper/(pid+'.html')).write_bytes(raw.encode('utf-8'))
(paper/'manifest.json').write_text(json.dumps(self.bundle['paper_manifest']))
return official, paper
def test_complete_program_and_duplicate_occurrences_match_gold(self):
self.assertEqual(len(self.program), 2834)
self.assertEqual(sum(len(p['row_ordinals']) for p in self.program.values()), 2851)
for ordinal, gold in enumerate(self.bundle['inventory']['records'], 1):
item = self.program[gold['paper_id']]
self.assertEqual(item['listing'], {k: gold[k] for k in ('title','authors_literal','keywords_literal','presentation_literal')})
self.assertIn(ordinal, item['row_ordinals'])
def test_conflicting_duplicate_not_silently_folded(self):
title = 'Transferability Between Understanding and Generation in Unified Multimodal Models'
with self.assertRaisesRegex(ValueError, 'Conflicting'):
c.parse_program(self.bundle['official_pages']['accepted.html'].replace('href="/virtual/2026/poster/3137">'+title, 'href="/virtual/2026/poster/3137">Conflicting title', 1))
def test_missing_member_and_changed_provisional_status_rejected(self):
html = self.bundle['official_pages']['accepted.html']
for changed in [html.replace('/virtual/2026/poster/4048', '/virtual/2024/poster/4048', 1),
html.replace('This paper list is preliminary pending publisher checks.', 'Final archival papers.')]:
with self.assertRaises(ValueError):
c.parse_program(changed)
def test_public_page_fields_match_independent_sample_review(self):
for gold in self.bundle['paper_gold']['records']:
pid = gold['paper_id']; fields = c.parse_paper(self.bundle['paper_pages'][pid], gold['url'], self.program[pid])
self.assertEqual(fields['title'], gold['title']); self.assertEqual(fields['authors'], gold['authors'])
self.assertEqual(fields['abstract'], gold['abstract_literal']); self.assertFalse(fields['review_flags'])
self.assertIn('sig-nal', c.parse_paper(self.bundle['paper_pages']['4048'], self.program['4048']['url'], self.program['4048'])['abstract'])
def test_foreign_canonical_and_inconsistent_jsonld_author_rejected(self):
raw = self.bundle['paper_pages']['4048']; url = self.program['4048']['url']
for changed in [raw.replace(url, 'https://example.org/other'), raw.replace('Suhyeon Ha', 'Other Author', 1)]:
with self.assertRaises(ValueError):
c.parse_paper(changed, url, self.program['4048'])
def test_program_and_page_versions_flagged_without_aliases(self):
program = copy.deepcopy(self.program['4048']); program['listing']['title'] = 'Earlier source title'
fields = c.parse_paper(self.bundle['paper_pages']['4048'], program['url'], program)
self.assertEqual(fields['review_flags'], ['program_and_page_title_versions_differ_requires_review'])
self.assertEqual(fields['title'], self.program['4048']['listing']['title'])
def test_partial_snapshot_collection_has_no_inferred_submission_ids(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp); official, paper = self.snapshots(root)
report = c.collect(root/'out', official_snapshots=official, paper_snapshots_dir=paper)
self.assertEqual(report['records'], 3); self.assertEqual(len(report['uncollected_program_ids']), 2831)
self.assertEqual(report['public_rejected_decision_metadata'], 'unknown'); self.assertFalse(report['database_written'])
manifest, = report['files']; self.assertEqual(manifest['status'], 'partial'); self.assertEqual(manifest['source_expected_count'], 2834)
rows = [json.loads(l) for l in Path(manifest['file']).read_text().splitlines()]
self.assertEqual({r['external_id'] for r in rows}, {'2026/4048','2026/3137','2026/3652'})
self.assertTrue(all(not r['identifiers'] and not r['submission_identifiers'] and r['track']=='program_unclassified' for r in rows))
self.assertTrue(all(r['metadata']['publication_state']=='preliminary_pending_publisher_checks' and not r['metadata']['original_decision_signature_verified'] for r in rows))
self.assertTrue(all(t['namespace']=='eccv_program:keyword' for r in rows for t in r['tags']))
with self.assertRaisesRegex(ValueError, 'Output exists'):
c.collect(root/'out', official_snapshots=official, paper_snapshots_dir=paper)
def test_zero_network_budget_reuses_immutable_cache_only(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp); official, _ = self.snapshots(root); client = PublicHTMLCache(root/'cache', c.ORIGIN)
for r in self.bundle['paper_manifest']['records']:
body, receipt, _ = client.paths(r['url']); body.write_text(self.bundle['paper_pages'][r['paper_id']]); receipt.write_text(json.dumps({**r,'retrieved_at':self.bundle['paper_manifest']['at']}))
with patch.object(PublicHTMLCache, 'get', side_effect=AssertionError('network forbidden')):
report = c.collect(root/'out', official_snapshots=official, cache=root/'cache', max_new_pages=0)
self.assertEqual((report['records'],report['new_pages'],report['reused_pages']), (3,0,3))
def test_tampered_snapshot_preserves_failure_without_success(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp); official, paper = self.snapshots(root); (paper/'4048.html').write_text('tampered')
with self.assertRaises(ValueError):
c.collect(root/'out', official_snapshots=official, paper_snapshots_dir=paper)
self.assertTrue((root/'out/failed.json').exists()); self.assertFalse((root/'out/collection-report.json').exists())
def test_structured_math_markup_requires_explicit_review(self):
raw = self.bundle['paper_pages']['4048']
self.assertIn('<p>Proactive tamper', raw)
fields = c.parse_paper(raw.replace('<p>Proactive tamper', '<p>10<sup>6</sup> Proactive tamper', 1), self.program['4048']['url'], self.program['4048'])
self.assertIn('structured_abstract_markup_requires_review', fields['review_flags'])
self.assertEqual(fields['abstract_markup_tags'], ['sup'])
def test_fixture_preserves_exact_http_bytes_including_crlf(self):
import hashlib
for receipt in json.loads(self.bundle['official_manifest'])['sources']:
raw = self.bundle['official_pages'][receipt['file']].encode('utf-8')
self.assertEqual(len(raw), receipt['bytes'])
self.assertEqual(hashlib.sha256(raw).hexdigest(), receipt['sha256'])
for receipt in self.bundle['paper_manifest']['records']:
raw = self.bundle['paper_pages'][receipt['paper_id']].encode('utf-8')
self.assertEqual(hashlib.sha256(raw).hexdigest(), receipt['sha256'])
def test_real_double_encoded_author_requires_exact_visible_agreement(self):
import hashlib
sample = self.bundle['encoding_sample']; pid = sample['paper_id']; raw = sample['html']
self.assertEqual(hashlib.sha256(raw.encode('utf-8')).hexdigest(), sample['receipt']['sha256'])
self.assertEqual(sample['receipt']['sha256'], 'b575318fa569e83eea5e96f937dd60f078fbed482877318013f797c16a02ff09')
fields = c.parse_paper(raw, sample['receipt']['url'], self.program[pid])
self.assertEqual(fields['authors'], sample['gold']['authors'])
self.assertEqual(fields['jsonld_raw_text']['author_names'][0], 'Zi&amp;#x27;ang Lu')
self.assertIn('jsonld_display_text_normalized_requires_review', fields['review_flags'])
# Real changes to the author or an extra encoding layer remain errors.
for replacement in ('Other Author', 'Zi&amp;amp;#x27;ang Lu'):
with self.assertRaisesRegex(ValueError, 'authors disagree'):
c.parse_paper(raw.replace('Zi&amp;#x27;ang Lu', replacement), sample['receipt']['url'], self.program[pid])
def test_real_already_agreeing_literal_title_is_not_decoded_again(self):
import hashlib
sample = self.bundle['literal_title_sample']; pid = sample['paper_id']; raw = sample['html']
self.assertEqual(hashlib.sha256(raw.encode('utf-8')).hexdigest(), 'c208525ad5f02b4ca274dff7e24a0392dabdc55bb3620497079a84df81a02a72')
fields = c.parse_paper(raw, sample['receipt']['url'], self.program[pid])
self.assertIn('&amp;', fields['title'])
self.assertEqual(fields['title'], sample['gold']['visible_title'])
self.assertEqual(fields['jsonld_raw_text']['name'], sample['gold']['jsonld_title_raw'])
self.assertNotEqual(fields['title'], sample['gold']['blind_double_decoded_title'])
self.assertNotIn('jsonld_display_text_normalized_requires_review', fields['review_flags'])
+90
View File
@@ -0,0 +1,90 @@
from datetime import datetime, timezone
import json
import os
from pathlib import Path
import tempfile
import unittest
from unittest.mock import MagicMock
import urllib.error
from paper_atlas.io.html_cache import PublicHTMLCache, digest, NoRedirect
URL = 'https://eccv.ecva.net/virtual/2026/poster/4048'
class PublicHTMLCacheTests(unittest.TestCase):
def receipt(self, raw):
return {'url':URL,'final_url':URL,'http_status':200,'content_type':'text/html; charset=utf-8',
'bytes':len(raw),'sha256':digest(raw),'retrieved_at':datetime.now(timezone.utc).isoformat()}
def test_old_cache_reused_without_network_or_expiry(self):
with tempfile.TemporaryDirectory() as tmp:
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0); raw = b'<html>metadata</html>'
body, receipt, _ = c.paths(URL); body.write_bytes(raw); receipt.write_text(json.dumps(self.receipt(raw))); os.utime(body,(0,0))
c.opener = MagicMock(); self.assertEqual(c.get(URL)[0], raw); c.opener.open.assert_not_called()
def test_pair_tampering_and_orphan_parts_fail_without_refetch(self):
for kind in ('hash','orphan','part'):
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp:
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0); body, receipt, _ = c.paths(URL); body.write_bytes(b'original')
if kind == 'hash': receipt.write_text(json.dumps(self.receipt(b'different')))
if kind == 'part': body.unlink(); body.with_suffix('.html.part').write_bytes(b'partial')
c.opener = MagicMock()
with self.assertRaises(ValueError): c.get(URL)
c.opener.open.assert_not_called()
def test_http_access_failure_sticky_without_automatic_retry(self):
with tempfile.TemporaryDirectory() as tmp:
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0); c.opener = MagicMock()
c.opener.open.side_effect = urllib.error.HTTPError(URL,403,'Forbidden',{},None)
with self.assertRaises(urllib.error.HTTPError): c.get(URL)
with self.assertRaisesRegex(ValueError,'Previous access'): c.get(URL)
self.assertEqual(c.opener.open.call_count,1)
def test_redirect_nonhtml_challenge_and_oversize_rejected(self):
for kind in ('redirect','type','challenge','size'):
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp:
c = PublicHTMLCache(tmp,'https://eccv.ecva.net',limit=100,delay=0); c.opener=MagicMock(); r=c.opener.open.return_value.__enter__.return_value
r.status=200; r.geturl.return_value=URL; r.headers={'Content-Type':'text/html'}; r.read.return_value=b'<html>metadata</html>'
if kind=='redirect':r.geturl.return_value='https://example.org/login'
if kind=='type':r.headers={'Content-Type':'application/pdf'}
if kind=='challenge':r.read.return_value=b'<title>Just a moment</title>'
if kind=='size':r.read.return_value=b'x'*101
with self.assertRaises(ValueError):c.get(URL)
self.assertFalse(c.paths(URL)[0].exists())
self.assertIsNone(NoRedirect().redirect_request(None,None,None,None,None,None))
def test_valid_response_has_atomic_body_and_receipt(self):
with tempfile.TemporaryDirectory() as tmp:
c=PublicHTMLCache(tmp,'https://eccv.ecva.net',delay=0);c.opener=MagicMock();r=c.opener.open.return_value.__enter__.return_value;r.status=200;r.geturl.return_value=URL;r.headers={'Content-Type':'text/html','Content-Length':'4'};r.read.return_value=b'data'
raw, receipt=c.get(URL);self.assertEqual(raw,b'data');self.assertEqual(receipt['sha256'],digest(raw));self.assertEqual(c.cached(URL),(raw,receipt));self.assertFalse(list(Path(tmp).glob('*.part')))
def test_foreign_origin_and_cache_symlink_rejected(self):
with tempfile.TemporaryDirectory() as tmp:
c=PublicHTMLCache(tmp,'https://eccv.ecva.net',delay=0)
with self.assertRaises(ValueError):c.get('https://example.org/other')
body,_,_=c.paths(URL);target=Path(tmp)/'outside';target.write_bytes(b'other');body.symlink_to(target)
with self.assertRaises(ValueError):c.get(URL)
def test_successful_fetch_publishes_receipt_and_reuses_without_refetch(self):
with tempfile.TemporaryDirectory() as tmp:
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0)
c.opener = MagicMock(); response = c.opener.open.return_value.__enter__.return_value
response.status = 200; response.geturl.return_value = URL
raw = b'<html>public metadata</html>'
response.headers = {'Content-Type':'text/html','Content-Length':str(len(raw))}; response.read.return_value = raw
self.assertEqual(c.get(URL)[0], raw); self.assertEqual(c.get(URL)[0], raw)
self.assertEqual(c.opener.open.call_count, 1)
body, receipt, _ = c.paths(URL)
self.assertTrue(body.exists()); self.assertTrue(receipt.exists())
self.assertFalse(body.with_suffix('.html.part').exists())
self.assertFalse(receipt.with_suffix('.json.part').exists())
def test_changed_url_is_rejected_before_reading_body(self):
with tempfile.TemporaryDirectory() as tmp:
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0)
c.opener = MagicMock(); response = c.opener.open.return_value.__enter__.return_value
response.status = 200; response.geturl.return_value = 'https://example.org/login'
response.headers = {'Content-Type':'text/html'}
with self.assertRaises(ValueError): c.get(URL)
response.read.assert_not_called()
+79
View File
@@ -0,0 +1,79 @@
import base64
import gzip
import hashlib
import json
from pathlib import Path
import tempfile
import unittest
from paper_atlas.collectors import recsys2026 as c
class RecSys2026Test(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.fixture=json.loads(gzip.decompress((Path(__file__).parent/'fixtures/recsys2026-source.json.gz').read_bytes()))
cls.files={n:base64.b64decode(raw) for n,raw in cls.fixture['files'].items()}
cls.body=cls.files[c.SNAPSHOT+'/recsys-contributions.html']
def materialize(self,root):
for name,raw in self.files.items():
path=root/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(raw)
def test_all263_observations_against_independent_gold(self):
items=c.parse_program(self.body)
keys=self.fixture['independent_gold'][0].keys()
self.assertEqual([{k:r[k] for k in keys} for r in items],self.fixture['independent_gold'])
self.assertEqual(len(items),263);self.assertEqual(len({r['doi'] for r in items}),260)
self.assertEqual(sum(bool(r['review_flags']) for r in items),6)
variants=[r for r in items if r['doi']=='10.1145/3773078.3831839']
self.assertNotEqual(variants[0]['title'],variants[1]['title']);self.assertNotEqual(variants[0]['authors'],variants[1]['authors'])
self.assertNotEqual(variants[0]['abstract'],variants[1]['abstract'])
def test_installed_format_tracks_evidence_and_original_fields(self):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);src=root/'src';self.materialize(src);out=root/'out'
report=c.collect(out,snapshots=src)
self.assertEqual((report['records'],report['unique_dois'],report['missing_abstract_count']),(263,260,0))
self.assertFalse(report['database_written']);self.assertFalse(report['pdfs_downloaded']);self.assertEqual(report['network_requests'],0);self.assertEqual(report['approved_for_import'],0)
self.assertEqual(report['track_counts'],{k:n for k,_,n in c.TRACKS});self.assertEqual(report['duplicate_doi_observations'],6)
gold={r['locator']:r for r in self.fixture['independent_gold']};seen=set()
for manifest in report['files']:
path=Path(manifest['file']);raw=path.read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),manifest['file_sha256'])
rows=[json.loads(l) for l in raw.splitlines()];self.assertEqual(len(rows),manifest['count']);self.assertEqual(manifest['status'],'partial')
for r in rows:
item=r['metadata']['program_fields'];locator=item['locator'];seen.add(locator);expected=gold[locator]
for key in expected:self.assertEqual(item[key],expected[key])
self.assertEqual(r['external_id'],'2026/'+locator);self.assertEqual(r['title'],expected['title']);self.assertEqual(r['authors'],expected['authors']);self.assertEqual(r['abstract'],expected['abstract'])
self.assertEqual(r['track'],manifest['track']);self.assertEqual(r['identifiers'],{'doi':expected['doi']});self.assertEqual(r['submission_identifiers'],[{'scheme':'publication_doi','value':expected['doi']}])
self.assertEqual(r['tags'],[{'namespace':'recsys_program:contribution_type','value':expected['type_labels'][0]}]);self.assertFalse(r['metadata']['original_decision_signature_verified'])
self.assertEqual(seen,set(gold));self.assertEqual((out/'evidence/recsys-contributions.html').read_bytes(),self.body)
with self.assertRaisesRegex(ValueError,'Output exists'):c.collect(out,snapshots=src)
def test_edition_and_track_label_drift_rejected(self):
for old,new in [(b'RecSys 26 - Accepted Contributions',b'RecSys 25 - Accepted Contributions'),(b'>Long Papers</a>',b'>Workshops</a>'),(b"title='Research'",b"title='Unreviewed'")]:
with self.subTest(old=old),self.assertRaises(ValueError):c.parse_program(self.body.replace(old,new,1))
def test_duplicate_locator_missing_author_and_foreign_doi_rejected(self):
for old,new in [(b'accordion-1-slide-2"',b'accordion-1-slide-1"'),(b'by Md Aminul Islam, Kathryn Vasilaky and Elena Zheleva',b'by '),(b'https://doi.org/10.1145/3773078.3831824',b'https://doi.org/10.1145/9999999.3831824')]:
with self.subTest(old=old),self.assertRaises(ValueError):c.parse_program(self.body.replace(old,new,1))
def test_math_markup_requires_review_and_is_not_flattened(self):
changed=self.body.replace(b'Learning-to-rank (LTR)',b'Learning-to-rank <sup>2</sup> (LTR)',1)
with self.assertRaisesRegex(ValueError,'markup'):c.parse_program(changed)
def test_corruption_and_missing_receipt_body_preserve_failure(self):
for kind in ('manifest','body','missing'):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);src=root/'src';self.materialize(src);path=src/c.SNAPSHOT/('manifest.json' if kind=='manifest' else 'recsys-contributions.html')
if kind=='missing':path.unlink()
else:path.write_bytes(path.read_bytes()+b' ')
with self.assertRaises((ValueError,FileNotFoundError)):c.collect(root/'out',snapshots=src)
self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists())
def test_symlink_source_and_root_rejected(self):
for kind in ('source','root'):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);src=root/'src';self.materialize(src);path=src/c.SNAPSHOT/'recsys-contributions.html' if kind=='source' else src
original=path.with_name(path.name+'-original');path.rename(original);path.symlink_to(original,target_is_directory=kind=='root')
with self.assertRaisesRegex(ValueError,'Unsafe'):c.collect(root/'out',snapshots=src)