Compare commits
9 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| f139c7e49d | |||
| 6a89f5f6c2 | |||
| cb3d97ed92 | |||
| f2753bc741 | |||
| f800c6ecb4 | |||
| 279d6a2735 | |||
| 8996a21e09 | |||
| e9bcbc3aef | |||
| c97ea94b49 |
@@ -49,6 +49,34 @@ jobs:
|
||||
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--help'],cwd=outside,check=True)
|
||||
eccv_fixture = json.loads(gzip.decompress((src/'tests/fixtures/eccv2026-source.json.gz').read_bytes()))
|
||||
eccv_official = root/'eccv2026-official'
|
||||
eccv_papers = root/'eccv2026-paper-samples'
|
||||
eccv_official.mkdir(); eccv_papers.mkdir()
|
||||
for name, html in eccv_fixture['official_pages'].items():
|
||||
(eccv_official/name).write_bytes(html.encode('utf-8'))
|
||||
(eccv_official/'manifest.json').write_bytes(eccv_fixture['official_manifest'].encode('utf-8'))
|
||||
for pid, html in eccv_fixture['paper_pages'].items():
|
||||
(eccv_papers/(pid+'.html')).write_bytes(html.encode('utf-8'))
|
||||
eccv_encoding = eccv_fixture['encoding_sample']
|
||||
(eccv_papers/(eccv_encoding['paper_id']+'.html')).write_bytes(eccv_encoding['html'].encode('utf-8'))
|
||||
eccv_fixture['paper_manifest']['records'].append({**eccv_encoding['receipt'], 'paper_id': eccv_encoding['paper_id'], 'file': eccv_encoding['paper_id']+'.html'})
|
||||
eccv_literal = eccv_fixture['literal_title_sample']
|
||||
(eccv_papers/(eccv_literal['paper_id']+'.html')).write_bytes(eccv_literal['html'].encode('utf-8'))
|
||||
eccv_fixture['paper_manifest']['records'].append({**eccv_literal['receipt'], 'paper_id': eccv_literal['paper_id'], 'file': eccv_literal['paper_id']+'.html'})
|
||||
(eccv_papers/'manifest.json').write_text(json.dumps(eccv_fixture['paper_manifest']))
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','eccv2026-program','--official-snapshots',str(eccv_official),'--paper-snapshots',str(eccv_papers),'--out',str(root/'installed-eccv2026')],cwd=outside,check=True)
|
||||
eccv_report = json.loads((root/'installed-eccv2026/collection-report.json').read_text())
|
||||
assert eccv_report['records'] == 5 and eccv_report['expected_program_ids'] == 2834
|
||||
assert eccv_report['duplicate_program_occurrences'] == 17 and len(eccv_report['uncollected_program_ids']) == 2829
|
||||
assert not eccv_report['database_written'] and eccv_report['files'][0]['track'] == 'program_unclassified'
|
||||
eccv_rows = [json.loads(line) for line in pathlib.Path(eccv_report['files'][0]['file']).read_text().splitlines()]
|
||||
eccv_encoded_row = next(row for row in eccv_rows if row['external_id'] == '2026/4199')
|
||||
assert eccv_encoded_row['authors'] == eccv_encoding['gold']['authors']
|
||||
assert 'jsonld_display_text_normalized_requires_review' in eccv_encoded_row['metadata']['public_page_fields']['review_flags']
|
||||
eccv_literal_row = next(row for row in eccv_rows if row['external_id'] == '2026/3844')
|
||||
assert eccv_literal_row['title'] == eccv_literal['gold']['visible_title']
|
||||
assert '&' in eccv_literal_row['title']
|
||||
sdm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/sdm2026-source.json.gz').read_bytes()))
|
||||
sdm_snapshots = root/'sdm2026-snapshots'
|
||||
for name, text in sdm_fixture['files'].items():
|
||||
|
||||
@@ -129,4 +129,6 @@ ICAPS 2026 也提供独立的 `collect icaps2026-registry`,保留 Crossref 卷
|
||||
|
||||
`collect corl2026-program` 将官方 687 个论坛与固定公开档案按 ID 对照,保留 34 条标题版本差异及分字段来源。不会自动接受 arXiv 匹配或推断拒稿;详情见 [CoRL 2026 使用与恢复说明](docs/corl2026-program.md)。
|
||||
|
||||
ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目 ID,并以不可变缓存断点采集公开元数据;保留摘要原文、来源关键词及未分类 track,始终待身份审核,不自动入库。
|
||||
|
||||
`collect sdm2026-proceedings` 核验 53 个 Crossref 论文章节,并单独保留 53 项日程观察及 52 页摘要。作者差异、取消展示和摘要缺失分别保留,不自动拼接来源或绑定作品;见 [SDM 2026 使用与恢复说明](docs/sdm2026-proceedings.md)。
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
# 会议覆盖与公开决定核查
|
||||
|
||||
最新生产进展和剩余工作见 [2026-11-17 补漏状态](backfill-status-2026-11-17.md):固定 278 个会议年份槽位中,270 个有记录、8 个仍为空;这不代表完整覆盖,第 14 步仍未完成。
|
||||
|
||||
**当前工作范围:2020 年起主动补漏。** 2000 年起已发现资料保留,恢复工作从 [补漏范围与恢复清单(2026-11-10)](backfill-handoff-2026-11-10.md) 开始;下文旧日期快照不改写,较早年份排除不算补齐。
|
||||
|
||||
这是 2026-11-08 的首份核查基线,**完整逐年政策普查和最终去重仍未完成**。收录 43 个目标,不能把“有记录”当作“完整覆盖”。结构化清单见 [inventory-2026-11-08.json](inventory-2026-11-08.json)。清单中的 `policy_checks` 均限定所列年份和轨道;空数组表示尚未核实,绝不表示没有公开拒稿。
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
# 2020 起补漏状态:2026-11-17
|
||||
|
||||
**总体仍为 13/17,第 14 步未完成。** 本页补充新的生产结果,不回写 [2026-11-10 恢复清单](backfill-handoff-2026-11-10.md) 或更早快照。用户确定的范围不变:2020 年起主动补漏;43 个更早历史缺口延期而非补齐,2000–2019 已发现的名单和身份线索继续保留。
|
||||
|
||||
## 最新数据库盘点
|
||||
|
||||
只读快照 `2026-11-17 09:38:31.677743+00`,固定 43 个目标会议、278 个应有会议年份槽位,其中 270 个至少有一条投稿记录,8 个仍为空,均为 2026 年:**CIKM、ECCV、ICASSP、ICDM、IROS、ISMIR、MM、RecSys**。本次 SDM2026 新增记录使空槽从 9 减到 8。
|
||||
|
||||
这只说明有无记录,**不表示 270 个槽位已经逐篇、逐 track、摘要或公开拒稿覆盖完整**;不能据此宣布第 14 步完成。2020–2025 没有整年空槽,仍有逐来源完整性和公开负面决定缺口。SDM2026 仅导入 25/53 条,状态为 partial,同样不能称为整届补齐。
|
||||
|
||||
| 生产文件(相对 `/data/paper-index/`) | SHA-256 |
|
||||
| --- | --- |
|
||||
| collected/active-scope-census-20261117-v17/summary.json | bfb1db8b50619c3756abb91ad0f73fa58ff998c7db58a216661342f12047749f |
|
||||
| collected/active-scope-census-20261117-v17/snapshot.json | b5389205f8b34f7ed57946416f49c253e0aed4a05e45eb818801f18fca6a8cb4 |
|
||||
| validation/active-scope-fixed278-v1.json | a835bef5b9c3a0f76069d840cecdb9a6bcf65d3a38796d70dbfdfd4b49242264 |
|
||||
|
||||
## 已推进的两批来源
|
||||
|
||||
- **SDM2026**:53 个正式注册章节、53 个独立日程观察和 52 个公开摘要页已冻结并规范化。25 条发表记录已在 PostgreSQL 独立验收,新增论文实体为 0;28 条待身份/版本核查。完整旧历史和 5,653 篇受保护论文的资料、来源、标签和既有标识保留。见 [生产验收与恢复](../sdm2026-identity-review.md)。这批队列已经消费,禁止重放。
|
||||
- **ECCV2026**:2,834 个唯一初步日程条目及公开摘要已全量采集并独立验收;新的只读身份预检和 1,997 个候选历史证据包已保存,尚未入库。初步日程不等于已核验出版社最终论文集;负面决定元数据仍受匿名访问限制。见 [来源状态](../eccv2026-source-status.md)。
|
||||
|
||||
SDM 新增事件使 SeBA 和 AdamG 同时具有较早 ICLR 拒稿与后来的 SDM 发表事实。旧的两次 ICLR 投稿及四条决定来源观察仍保留,没有被较晚 accepted 覆盖;多个来源观察不直接累加成多次拒稿。
|
||||
|
||||
## 其余七个会议的最新入口核查
|
||||
|
||||
已对下表七个会议的 14 个明确官方 URL 保存原始响应和回执:13 个 HTTP 200,IROS 官方论文索引 HTTP 403。所有请求均匿名,不绕过访问限制,没有下载论文 PDF;保存入口不是完成逐篇采集或负面决定普查。
|
||||
|
||||
| 会议 | 已保存的明确官方入口 | 尚未完成 |
|
||||
| --- | --- | --- |
|
||||
| ICDM | [详细日程](https://icdm2026.neu.edu.cn/DetailedSessions/list.htm)、[研究轨道政策](https://icdm2026.neu.edu.cn/CallforResearchTrack/list.htm)、[应用轨道政策](https://icdm2026.neu.edu.cn/CallforAppliedTrack/list.htm) | 官方页面称为线下展示列表;需要核对逐篇范围、研究/应用轨道及演讲长短标签,不能把展示长短自动等同于论文类型;摘要和正式出版身份仍待核查 |
|
||||
| ICASSP | [录用标题清单](https://cmsworkshops.com/ICASSP2026/papers/accepted_papers.php)、[作者指南](https://2026.ieeeicassp.org/author-guidelines/) | 取得逐篇完整作者/摘要/出版记录,区别会议投稿与期刊展示,不能只导入标题或从缺席推断拒稿 |
|
||||
| IROS | [官方论文索引](https://2026.ieee-iros.org/program/paper_index/) | 当前服务器返回 403,保留缺口并停止该入口;第三方浏览器页面不能替代独立核验的官方原文 |
|
||||
| ISMIR | [官网](https://ismir2026.ismir.net/)、[官方虚拟平台](https://ismir2026program.ismir.net/)、[作者指南](https://ismir2026.ismir.net/authors/author-guidelines) | 核查平台公开静态数据与正式开放论文集的逐篇映射;首页可读取不代表论文目录已经取得 |
|
||||
| CIKM | [录用页](https://cikm2026.diag.uniroma1.it/accepted-papers/)、[日程页](https://cikm2026.diag.uniroma1.it/program/) | 网页提取的录用页仍显示 TBD;需要检查原文引用的节目数据与独立正式出版目录,不能当成没有论文 |
|
||||
| RecSys | [官网](https://recsys.acm.org/recsys26/)、[投稿说明](https://recsys.acm.org/recsys26/call/) | 继续取得可核验的逐篇正式记录,并分开主轨、工业、复现、demo、challenge/workshop |
|
||||
| MM | [官网](https://2026.acmmm.org/) | 继续核对官方完整日程和出版目录,分开研究论文与 grand challenge、workshop 等;首页响应不证明覆盖完整 |
|
||||
|
||||
冻结目录:`/data/paper-index/collected/remaining2026-official-leads-v1`。manifest SHA-256:`0e13313d94e3bff17f071d3a1384f6ed83ece156239fe5bc4633cc7a34fb8e04`。恢复包:`/data/paper-index/validation/remaining2026-official-leads-v1-recovery.tar.gz`,SHA-256:`0be8c84bb5c591e5afaaa5f95dd5334bf13a9be1671cb942e46e998f502495bb`。HTTP 成功仅是入口可读性,不能作为可导入记录数量。
|
||||
|
||||
## 第 14 步仍需做什么
|
||||
|
||||
1. 取得并规范化其余可公开取得的完整逐篇来源,记录应有范围、排除项和版本;对缺失、受限或尚未公开的入口保存明确证据,不能伪造“已补齐”。优先推进当前可读的 ICDM 详细日程、ICASSP 清单及其他会议的正式目录。
|
||||
2. 完成来源与年度、track、摘要完整性的对应核查;已经有若干记录的年份也要看目录差集,不能只消除空槽。
|
||||
3. 继续近期和历史年度实际公开的 rejected、desk_rejected、withdrawn 来源核查,特别是 UAI2024–2026 的 opt-in 拒稿、EMNLP2023 的实际公开范围以及其他会议的逐年限定政策。公开评审和公开决定不同;没有找到不等于零条或未曾拒稿。
|
||||
4. 通过身份核查的记录再执行新快照守护、恢复备份和增量导入,保留较早决定。深层作品版本、跨会历史及类型/标签协调继续列为第 15 步;不能为减少空槽创建重复论文,也不能只盯住少量身份疑难项而停止其他来源采集。
|
||||
|
||||
第 16 步仍是所有导入结束后的独立全库去重与完整审计,第 17 步是最终长期运行/恢复和交付整理。上述任务均未因单批验收或 CI 成功而完成;总体 goal 继续保持 active。
|
||||
@@ -0,0 +1,35 @@
|
||||
# ECCV 2026:暂定节目与公开元数据
|
||||
|
||||
这是长期维护、按需运行的来源适配器。它读取已核验的官方节目快照,采集公开论文页的标题、完整作者和摘要,输出待身份审核文件;不连接数据库,不下载 PDF、评审、个人档案或受保护的直播内容。
|
||||
|
||||
```sh
|
||||
paper-atlas collect eccv2026-program \
|
||||
--official-snapshots /data/paper-index/collected/eccv2026-official-research-v1 \
|
||||
--cache /data/paper-index/cache/eccv2026-public-pages \
|
||||
--max-new-pages 200 \
|
||||
--out /data/paper-index/collected/eccv2026-pass01
|
||||
```
|
||||
|
||||
官方目录包含 `accepted.html`、`policies.html`、`faq.html` 和冻结的 `manifest.json`。当前适配器核验固定版本:节目 2,851 行、2,834 个节目 ID;17 个 ID 各重复一次,标题、作者、关键词、展示信息均一致。输出按节目 ID 合并这些重复行,并保留全部行序号。它不是跨来源作品去重。
|
||||
|
||||
`--max-new-pages` 限制本轮新请求数;不指定时遍历整个冻结名单。再次运行时使用同一 `--cache` 和**全新** `--out`,复用已验证缓存并继续读取缺失页。缓存不按 24 小时过期,不自动刷新或覆盖已成功的原始响应。每次运行输出包含本轮能核验的全部缓存条目,并明确列出尚未采集的 ID;不能未经审核把多轮重叠输出全部重复导入。
|
||||
|
||||
公开请求按固定来源和路径限制,只接受 HTTP 200、原 URL、HTML 和限定大小的完整响应。请求失败、重定向、访问挑战、非 HTML 或异常响应留下阻断标记,不自动重试或更换账号、域名、代理绕过限制。记录的 `Retry-After` 供人工核查使用。网络超时等未产生成功缓存的情况可在确认进程终止后用新输出目录继续。缓存数据与回执须成对且哈希一致;遇到孤立文件或 `.part` 时保留现场,先核查再把本任务的残留文件移至恢复目录,不静默丢弃。写同一缓存时持有非阻塞单写锁。
|
||||
|
||||
离线重放可改用 `--paper-snapshots /path/to/papers`,该目录包含 `<节目ID>.html` 和 `manifest.json`(带时区的 `at`、`records`,各条含文件、URL、最终 URL、HTTP 状态、类型、字节数、SHA-256)。缺少的公开页会明确列为未采集;三篇样本只会生成三条 partial 记录,不会冒充全量成功。
|
||||
|
||||
## 来源边界
|
||||
|
||||
[官方名单](https://eccv.ecva.net/Conferences/2026/AcceptedPapers)明确注明 **preliminary pending publisher checks**。这里的 accepted 表示冻结时列入暂定官方名单,不代表已核验 Springer 最终发表或原始签名决定。所有输出始终 partial。节目 ID 是官方节目定位符,不冒充原始 OpenReview 投稿号;尚未核实跨平台标识时作品标识和投稿标识均为空。
|
||||
|
||||
公开页的 canonical URL、可见标题/作者和 JSON-LD 相互核对;页面版本与冻结节目文字有差异时分别保留并标记待审阅,不制造人名别名。JSON-LD 日期只是网站条目时间,不当作论文发表日期或原始决定日期。摘要保留公开页 DOM 文本(规范空白),包括源头的断词、缺空格;不自动修补。原始 HTML 单独留存;遇到公式、上下标或图片结构会标记待审阅,不能把扁平化文本当作无损数学表达。页面上的登录提示对应直播内容,不用于访问受保护内容。
|
||||
|
||||
第 4199 页的 JSON-LD 作者姓名存在双重 HTML 实体编码;第 3844 页的可见标题和 JSON-LD 则已经一致地包含字面 `&`。解析器优先保留已经一致的文字,仅在字段不一致时尝试最多两层实体解码来核对可见文本,不把已经一致的标题再次解码。完整作者序列仍须精确一致,真实差异或更多编码层次仍停止。`jsonld_raw_text` 保留原始结构化标题/姓名,发生文本规范化时增加 `jsonld_display_text_normalized_requires_review`,不生成姓名别名。恢复必须使用校验过的修复版本、新输出目录和原缓存;保留旧失败目录,不重新请求已有成功页面。
|
||||
|
||||
官方关键词保存在独立 `eccv_program:keyword` 命名空间,并保留原始单元格和证据。关键词、展示形式、贡献/研究类型、自定义标签相互独立,投稿 track 暂用 `program_unclassified`,不按主题推断 main、findings、workshop 或 position。
|
||||
|
||||
[2026 投稿政策](https://eccv.ecva.net/Conferences/2026/SubmissionPolicies)明确评审和作者回复不公开;这不能代替对公开 rejected、desk rejected、withdrawn **决定元数据**的单独核查。当前拒稿决定可见性仍记为 unknown,不能从录用表缺席推断拒稿。
|
||||
|
||||
贡献类型的官方定义、OpenReview 状态入口与匿名访问核查见[来源核查记录](eccv2026-source-status.md)。这是独立于冻结采集输入的证据,不会改变已核验节目版本,也不能用类型定义替论文推断标签。
|
||||
|
||||
失败输出保留 `failed.json`,成功报告最后发布;恢复前核查既有进程、输出及缓存,不因日志安静重启。身份、来源版本和完整投稿历史审阅后,另行准备专用入库队列、保护基线和导入后验收;全部来源导入结束后仍须全库去重、覆盖、摘要、约束及查询审计。测试、构建及安装验证只在已授权 `ces-control:host` runner 执行。
|
||||
@@ -0,0 +1,59 @@
|
||||
# ECCV 2026 来源核查记录
|
||||
|
||||
核查时间:2026-11-17,以下状态是当时的公开可见性。长期补漏时先核实来源是否变化,保留旧快照,不覆盖旧结论或重复导入。
|
||||
|
||||
## 贡献类型与 track
|
||||
|
||||
[作者贡献类型说明](https://eccv.ecva.net/Conferences/2026/AuthorContributionTypes)通过官方导航确认。它要求作者选择一项主要贡献类型;官方 FAQ 明确贡献类型不是独立 track。公开 OpenReview group 的 `source_submissions_query_mapping` 也包含五项对应值:
|
||||
|
||||
| 官方 group 字段值 | 含义摘要 |
|
||||
| --- | --- |
|
||||
| Algorithms/General | 方法、模型或算法;也是边界情形的默认选择 |
|
||||
| Theory/Foundational | 理论、形式分析或基础理解 |
|
||||
| Applied/Systems | 实际系统、部署与工程约束 |
|
||||
| Datasets/Benchmarks | 数据集、基准或评估协议 |
|
||||
| Concept & Feasibility | 新概念及可行性证据 |
|
||||
|
||||
这些是类型定义和配置值,**尚未核验逐篇论文的作者所选类型**。不能根据标题、摘要、主题关键词或展示形式补写类型;公开节目采集器继续输出 `program_unclassified`,主题关键词继续使用 `eccv_program:keyword`。以后获得逐篇类型证据时,保留独立命名空间、原始值、来源和版本,不把它写成投稿 track。
|
||||
|
||||
## 公开负面决定元数据
|
||||
|
||||
[投稿指南](https://eccv.ecva.net/Conferences/2026/AuthorGuide)链接的会议 group 是 [`thecvf.com/ECCV/2026/Conference`](https://openreview.net/group?id=thecvf.com/ECCV/2026/Conference)。匿名 API 读取 group 成功(HTTP 200),返回 `readers: ["everyone"]`。这只说明 group 配置公开,不证明论文或决定公开。
|
||||
|
||||
group 的 `rejected_venue_id` 实际配置为 `thecvf.com/ECCV/2026/Conference/Rejected_Submission`。使用该值查询公开 notes(`content.venueid`,`limit=1`)返回 HTTP 403,随后停止;没有自动重试、切换账号、代理或其他绕过操作。其余 desk-rejected、withdrawn 和 submission 状态查询没有执行,不能报告为已核查或零条。
|
||||
|
||||
结论仍为 **拒稿决定元数据覆盖未核实,当前匿名查询受限**。403 的具体原因未确定;不能由此推断真实投稿数量、全局不存在公开拒稿数据,或某篇论文遭拒。[投稿政策](https://eccv.ecva.net/Conferences/2026/SubmissionPolicies)关于评审与作者回复不公开的规定也不能代替决定元数据核查。
|
||||
|
||||
## 可恢复证据与后续入口
|
||||
|
||||
服务器证据目录:`/data/paper-index/collected/eccv2026-policy-research-v2`。本机工作记录目录保留同名备份;三份成功响应均已逐文件校验。证据不含登录凭据。
|
||||
|
||||
| 文件 | SHA-256 |
|
||||
| --- | --- |
|
||||
| manifest.json | 5a04e52bf6ea34ab25fe1d91fa9a1df18c9be5abb4f554586ee28fe9aa566e6e |
|
||||
| contribution-types.html | 6a5128b1f47a275411acacbb08ab91320a767bdcc48aaad09fe1be9e31dff00f |
|
||||
| author-guide.html | 779196716f5b37fcd10ddc191e9cd0530d7e9d43e5bc66b16a5546cd152959c4 |
|
||||
| openreview-group.json | 055bcf99d412d77f79a760890739d4de13a20ea099857154ee3e714c0647e763 |
|
||||
| anonymous-decision-probes/manifest.json | a522756ed3ec2880e5d942a211d7ad13819b18c8acada551dd3a8718b5e4d63b |
|
||||
|
||||
这里的补充证据与采集器的三份冻结节目/政策输入分开保存,不能替换采集器固定版本的 manifest。下次补漏先核查官方是否提供公开决定导出或明确的公开元数据说明;访问仍受限时保留缺口。若发现允许公开访问的新入口,另存带时间与哈希的快照,先审阅读者权限、决定语义和逐篇身份,再准备入库队列。
|
||||
|
||||
|
||||
## 2026-11-17 全量采集与只读预检
|
||||
|
||||
第三次采集 `paper-atlas-eccv2026-collect-pass03.service`(invocation `147c8779a3af4223a4d5d60dbdc88264`)已终态成功:2,834 个唯一日程条目全部取得公开摘要,新取 1,775 页,复用已验收缓存 1,059 页;2,851 次日程出现中的 17 处重复保留原始位置。独立原文/回执/字段审计通过,缺少条目和摘要均为 0,5 处 JSON-LD 显示文字规范化仍留待逐篇审阅。这里仍是官方初步日程版本,不改称出版社最终论文集。
|
||||
|
||||
| 服务器证据 | SHA-256 |
|
||||
| --- | --- |
|
||||
| validation/eccv2026-pass03-audit-v1.json | ad90199e08a234c0b2a01f44e1edfbecb2a9101b45de87e00a64eadfd5c92848 |
|
||||
| collected/eccv2026-pass03/eccv-program-2026-unclassified.jsonl | d0a80745fc6f58163f8722af351bbba86803af97857ea30554a0f3062d8ab2ef |
|
||||
| validation/eccv2026-pass03-recovery-v1.tar.gz | df55493ad5f8d6413dca86e9b9aca1d8b544ac1dc222b3849f95b04f94accba9 |
|
||||
| collected/eccv2026-pass03-preflight-v1/postgresql-preflight.json | 97bf57828cc67d05de0221188a9d988eef35631c845b18372186a6249f64e5da |
|
||||
| validation/eccv2026-pass03-preflight-packets-check-v1.json | 8a4f44f5d2c3a15d05b5a1df96fe465a96a2fe8914d7ed68958e6978c30985d4 |
|
||||
| validation/eccv2026-pass03-preflight-recovery-v1.tar.gz | 69c84fd0f24024ca56bfa4f848c977c578267bfc53c713105987e030df3f92a2 |
|
||||
|
||||
全量原文恢复包为 85,430,941 字节,保留输出、全部缓存原文与回执、官方输入和固定源码/CI 凭证。失败的第一次打包任务在创建文件前因命令引用错误退出;改用文件脚本的新单元 `paper-atlas-eccv2026-pass03-recovery-v2.service` 成功,未重启或重跑采集器。
|
||||
|
||||
新的 PostgreSQL 只读预检覆盖全部 2,834 条:1,519 条精确标题和作者集合一致、468 条仅部分作者重合、840 条无精确匹配、4 条没有作者重合、3 条命中多个同标题实体。1,997 个候选实体包的哈希独立核对通过,来源、投稿、决定与来源关系未触发截断。该结果只生成候选,`database_written=false`、`approved_for_import=0`;没有批准合并、创建新论文或导入 ECCV 事件。无精确匹配不等于新作品,相同作者也不证明人物或作品身份。
|
||||
|
||||
后续先保留这批完整来源并推进其他会议来源补漏,再做逐篇内容与版本审阅、当前快照守护和保护基线。当前全局最新基线来自 SDM25 验收,保护 5,653 篇论文;不能回退到 ICAPS34 之前的计数或重放已消费批次。最终全库去重仍须另行执行。
|
||||
@@ -1,6 +1,6 @@
|
||||
# SDM 2026:生产规范化与身份核查记录
|
||||
|
||||
本页记录 2026-11-17 的来源验收和只读身份核查,供下次按需运行时恢复。**本轮没有写入 PostgreSQL,53 条记录均未获得导入批准。** 核查完成、存在候选、内容支持同一作品和最终允许入库是不同状态。
|
||||
本页记录 2026-11-17 的来源验收、早期只读核查和随后的生产导入。**当前 53 条注册记录中,25 条已入库并独立验收,28 条仍待处理;没有新增论文实体。** 下文早期报告保持原始状态,不将它们改写成事后批准;最终结果见“25 条生产导入验收”。
|
||||
|
||||
## 已核实的生产输出
|
||||
|
||||
@@ -52,7 +52,7 @@
|
||||
| Towards Reliability of Parameter-Free Optimization | 3282961 | ICLR 2025 rejected |
|
||||
| Scaffold-Aware Generative Augmentation and Reranking for Enhanced Virtual Screening | 2506693 | ICLR 2026 desk_rejected |
|
||||
|
||||
前两条已有内容支持,但没有导入 SDM 事件;第三条仍保留作者版本问题。如果后续通过身份核查,SDM 发表事实应作为独立会议事件加入,保留旧投稿、旧决定及各自来源。两个来源对同一拒稿的观察不能直接统计成两次独立投稿或拒稿。不能覆盖旧状态,也不能因这批录用候选就宣称完整投稿历史已补齐。
|
||||
前两条在后续生产验收中各自新增独立 SDM 发表事件;第三条仍保留作者版本问题,未导入。旧投稿、旧决定及各自来源均完整保留。两个来源对同一拒稿的观察不能直接统计成两次独立投稿或拒稿。不能覆盖旧状态,也不能因这批录用候选就宣称完整投稿历史已补齐。
|
||||
|
||||
## 未精确匹配的补充检索
|
||||
|
||||
@@ -70,9 +70,28 @@
|
||||
|
||||
两个原报告的 SHA-256 分别为 `b6ab2e9228b8e8e9548b27df70834faf25bfda551844979f8410a74b6e6aadd0` 和 `c1f902b4e4f33997eb182b10dc78332e182ceb6051f1a7c14b21465c8878263d`。前者保留 1 个进一步审阅的完整候选包,后者保留全部 113 个前缀命中的候选包;这些数量不是论文批准数。
|
||||
|
||||
## 25 条生产导入验收
|
||||
|
||||
`paper-atlas-sdm2026-import25-v1.service`,invocation `003b490a2f2c449da078a2f8625b31b8`,终态成功。执行前核验 81 个源码文件,并确认 24 个核心写入文件与此前已验收的 canonical writer 相同;复用原生 `paper-atlas import records`,未修改默认运行目录。
|
||||
|
||||
- 新增 25 条来源、25 次会议事件、25 条发表决定观察及 25 个 `publication_doi` 投稿标识,新增论文实体为 0。
|
||||
- 24 篇使用已核实的 arXiv 标识补注;AdamG 没有 arXiv 标识,采用单独审阅并实时守护的唯一精确标题、四名完整作者和具体方法证据。旧 OpenReview ID 未复制为 SDM 投稿标识。
|
||||
- 28 条待处理记录全部排除;25 个注册来源摘要仍为空,52 份日程摘要保持独立来源版本。覆盖状态为 `partial`,应有 53 条、当前导入 25 条,track 保留 `proceedings_unclassified`。
|
||||
- 全局旧原始投稿与决定历史、5,653 篇受保护论文的原资料/来源/标签/既有标识均通过导入后比较。SeBA、AdamG 的两次较早 ICLR 投稿及四条 rejected 来源观察保留,没有将四条观察说成四次投稿。
|
||||
- 约束有效、实际全文查询与 GIN 计划通过;foundry-core `/readyz` 返回 200,受保护 SQLite 快照保持只读。
|
||||
|
||||
| 生产证据 | SHA-256 |
|
||||
| --- | --- |
|
||||
| collected/sdm2026-import25-v1/postgresql-postcheck-v1.json | 0c861722f0d455462a2686a42d634e0b963d075cf58f90fa9f48e1ae1fc82c27 |
|
||||
| collected/sdm2026-import25-baseline-v1/baseline.json | 42ffbaf6f843a91ebf64f0bfefa4af36f6ac72f044b0e8587e65c5e6648c0931 |
|
||||
| collected/sdm2026-prepared25-v1/independent-preparation-check.json | 8ea8eb59f4b1601a484d1bc0535b289cb950b83747ed5f82b0eac55e527c4a7c |
|
||||
| validation/sdm2026-reviewed25-recovery-v1.tar.gz | 13447944ec34e0e2a6eee07a1d9820bfb8066756aa11016673e416c0d6f8b2b2 |
|
||||
|
||||
恢复包包含 263 个文件;本机已独立核对队列、70 份内容寻址证据和 28 个早期候选证据包。**25 条队列已消费,禁止重放。** 28 条剩余记录需要新的导入后快照;先前候选包不能当作当前数据库状态。后续批次须沿用最新 5,653 篇保护基线,并加上各自新目标。
|
||||
|
||||
## 下次恢复
|
||||
|
||||
1. 先核对实际服务终态、来源及报告哈希,再读取上述证据;终态成功的研究任务无需重启。
|
||||
2. 继续解决 3 条作者版本问题和 25 条未精确匹配记录;不把缺少候选当作新作品证明,不把源端 403 当作零条公开拒稿。
|
||||
3. 对已获内容支持的候选另做新快照检查:作品与 DOI 绑定、来源是否已导入、已有会议事件及全部历史保护。建立新的明确入库队列和可恢复备份后再执行写入,不能直接使用本次候选报告当导入计划。
|
||||
2. 保留已验收的 25 条,不重放;继续解决 3 条作者版本问题和 25 条未精确匹配记录;不把缺少候选当作新作品证明,不把源端 403 当作零条公开拒稿。
|
||||
3. 对后续新获内容支持的候选另做新快照检查:作品与 DOI 绑定、来源是否已导入、已有会议事件及全部历史保护。建立新的明确入库队列和可恢复备份后再执行写入,不能直接使用本次候选报告当导入计划。
|
||||
4. 导入后逐项核验新事件、旧历史、来源关系和精确数量变化;注册摘要缺失仍如实保留。完整来源范围结束后另做全库去重、标识冲突、近似标题/作者、覆盖、摘要、约束和查询验收。
|
||||
|
||||
@@ -24,6 +24,7 @@ COMMANDS = {
|
||||
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
|
||||
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
|
||||
('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'),
|
||||
('collect','eccv2026-program'): ('paper_atlas.collectors.eccv2026', {}, '核验ECCV2026暂定节目并断点采集公开元数据'),
|
||||
('collect','sdm2026-proceedings'): ('paper_atlas.collectors.sdm2026', {}, '核验 SDM 2026 注册论文并单独保留公开日程版本'),
|
||||
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
|
||||
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
"""ECCV 2026 frozen preliminary program and resumable public metadata pages."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from datetime import datetime, timezone
|
||||
import fcntl
|
||||
from html import unescape
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from paper_atlas.io.html import Document, Node, clean
|
||||
from paper_atlas.io.html_cache import PublicHTMLCache, checked_html, digest
|
||||
|
||||
SOURCE = 'eccv_program'
|
||||
ORIGIN = 'https://eccv.ecva.net'
|
||||
ACCEPTED = ORIGIN+'/Conferences/2026/AcceptedPapers'
|
||||
TRACK = 'program_unclassified'
|
||||
MANIFEST_SHA = '04f9734783bbbfa2e19f936d8f5a7aecfb524b0ceef33fc5d9fb6276c7b6678a'
|
||||
PAGES = {'accepted.html': ACCEPTED, 'policies.html': ORIGIN+'/Conferences/2026/SubmissionPolicies',
|
||||
'faq.html': ORIGIN+'/Conferences/2026/FAQs'}
|
||||
|
||||
|
||||
def one(items):
|
||||
if len(items) != 1:
|
||||
raise ValueError('Expected one reviewed metadata field')
|
||||
return items[0]
|
||||
|
||||
|
||||
def has_class(node, value):
|
||||
return value in node.attrs.get('class', '').split()
|
||||
|
||||
|
||||
def parse_program(raw):
|
||||
nodes = list(Document(raw).root.walk())
|
||||
if len([n for n in nodes if n.tag == 'h2' and clean(n.text()) == 'ECCV 2026 Accepted Papers:']) != 1:
|
||||
raise ValueError('Unexpected ECCV program edition')
|
||||
if 'This paper list is preliminary pending publisher checks.' not in clean(Document(raw).root.text()):
|
||||
raise ValueError('Reviewed provisional-publication notice changed')
|
||||
table = one([n for n in nodes if n.tag == 'table' and n.attrs.get('id') == 'event-list-2026-poster-nodates-filter-vslinks-table'])
|
||||
papers = {}; occurrences = 0
|
||||
for tr in (n for n in table.walk() if n.tag == 'tr'):
|
||||
cells = [n for n in tr.children if isinstance(n, Node) and n.tag == 'td']
|
||||
if not cells:
|
||||
continue
|
||||
if len(cells) != 3:
|
||||
raise ValueError('Unexpected program columns')
|
||||
links = [n for n in cells[0].walk() if n.tag == 'a' and re.fullmatch(r'/virtual/2026/poster/[1-9][0-9]*', n.attrs.get('href', ''))]
|
||||
link = one(links); pid = link.attrs['href'].rsplit('/', 1)[1]
|
||||
authors = clean(one([n for n in cells[0].walk() if n.tag == 'i']).text())
|
||||
row = {'title': clean(link.text()), 'authors_literal': authors,
|
||||
'keywords_literal': clean(cells[1].text()), 'presentation_literal': clean(cells[2].text())}
|
||||
if not row['title'] or not authors:
|
||||
raise ValueError('Incomplete public program fields')
|
||||
occurrences += 1
|
||||
if pid in papers:
|
||||
if papers[pid]['listing'] != row:
|
||||
raise ValueError('Conflicting entries share a program ID')
|
||||
papers[pid]['row_ordinals'].append(occurrences)
|
||||
else:
|
||||
papers[pid] = {'url': ORIGIN+link.attrs['href'], 'listing': row, 'row_ordinals': [occurrences]}
|
||||
if occurrences != 2851 or len(papers) != 2834 or Counter(len(v['row_ordinals']) for v in papers.values()) != {1: 2817, 2: 17}:
|
||||
raise ValueError('Incomplete reviewed program census')
|
||||
return papers
|
||||
|
||||
|
||||
def parse_paper(raw, url, program):
|
||||
nodes = list(Document(raw).root.walk())
|
||||
title = clean(one([n for n in nodes if n.tag == 'h1' and has_class(n, 'event-title')]).text())
|
||||
authors_literal = clean(one([n for n in nodes if n.tag == 'div' and has_class(n, 'event-organizers')]).text())
|
||||
abstract_node = one([n for n in nodes if n.tag == 'div' and has_class(n, 'abstract-text-inner')])
|
||||
abstract = clean(abstract_node.text())
|
||||
markup = sorted({n.tag for n in abstract_node.walk() if n.tag in ('math','sup','sub','svg','img')})
|
||||
canonical = one([n.attrs.get('href') for n in nodes if n.tag == 'link' and n.attrs.get('rel') == 'canonical'])
|
||||
scripts = [n for n in nodes if n.tag == 'script' and n.attrs.get('type') == 'application/ld+json']
|
||||
ld = one([json.loads(''.join(x for x in n.children if isinstance(x, str))) for n in scripts])
|
||||
authors = ld.get('author')
|
||||
if (canonical != url or ld.get('@type') != 'CreativeWork' or ld.get('creditText') != 'ECCV 2026'
|
||||
or not title or not isinstance(ld.get('name'), str) or not abstract
|
||||
or not isinstance(authors, list) or not authors
|
||||
or any(not isinstance(a, dict) or a.get('@type') != 'Person' or not isinstance(a.get('name'), str)
|
||||
or not clean(a['name']) or '@' in a['name'] for a in authors)):
|
||||
raise ValueError('Public visible/structured metadata identity or edition disagrees')
|
||||
# Prefer source text that already agrees. Poster 3844 intentionally displays
|
||||
# literal '&', while 4199 double-encodes an author's apostrophe in JSON-LD.
|
||||
# Decode only to test agreement with visible text, never to rewrite it.
|
||||
def agrees_with_display(value, expected):
|
||||
if clean(value) == expected:
|
||||
return True
|
||||
for _ in range(2):
|
||||
value = unescape(value)
|
||||
if clean(value) == expected:
|
||||
return True
|
||||
return False
|
||||
|
||||
raw_names = [a['name'] for a in authors]
|
||||
names = [clean(name) for name in raw_names]
|
||||
if not agrees_with_display(ld['name'], title):
|
||||
raise ValueError('Public visible and structured titles disagree')
|
||||
if ' ⋅ '.join(names) != authors_literal:
|
||||
visible_names = authors_literal.split(' ⋅ ')
|
||||
if len(visible_names) != len(raw_names) or any(
|
||||
not agrees_with_display(raw_name, visible_name)
|
||||
for raw_name, visible_name in zip(raw_names, visible_names)):
|
||||
raise ValueError('Complete visible and structured authors disagree')
|
||||
names = visible_names
|
||||
if any('@' in name for name in names) or ' ⋅ '.join(names) != authors_literal:
|
||||
raise ValueError('Complete visible and structured authors disagree')
|
||||
flags = ['structured_abstract_markup_requires_review'] if markup else []
|
||||
if raw_names != names or ld['name'] != title:
|
||||
flags.append('jsonld_display_text_normalized_requires_review')
|
||||
if title != program['listing']['title']:
|
||||
flags.append('program_and_page_title_versions_differ_requires_review')
|
||||
if authors_literal != program['listing']['authors_literal']:
|
||||
flags.append('program_and_page_author_versions_differ_requires_review')
|
||||
return {'title': title, 'authors': names, 'authors_literal': authors_literal, 'abstract': abstract,
|
||||
'page_metadata_dates': {k: ld.get(k) for k in ('datePublished', 'dateModified')}, 'review_flags': flags, 'abstract_markup_tags': markup,
|
||||
'jsonld_raw_text': {'name': ld['name'], 'author_names': raw_names}}
|
||||
|
||||
|
||||
def official_snapshot(root):
|
||||
root = Path(root); manifest = root/'manifest.json'
|
||||
if root.is_symlink() or manifest.is_symlink() or digest(manifest.read_bytes()) != MANIFEST_SHA:
|
||||
raise ValueError('Reviewed official snapshot manifest changed')
|
||||
m = json.loads(manifest.read_bytes())
|
||||
if datetime.fromisoformat(m['at'].replace('Z', '+00:00')).utcoffset() is None:
|
||||
raise ValueError('Official snapshot requires a timezone')
|
||||
if len(m['sources']) != 3 or {r['file'] for r in m['sources']} != set(PAGES):
|
||||
raise ValueError('Official source set changed')
|
||||
pages = {}; evidence = {}
|
||||
for r in m['sources']:
|
||||
name = r['file']; p = root/name
|
||||
if p.is_symlink() or p.stat().st_size > 8*1024**2:
|
||||
raise ValueError('Unsafe official snapshot')
|
||||
raw = p.read_bytes()
|
||||
if r['url'] != PAGES[name] or r['final_url'] != r['url'] or r['http_status'] != 200 or r['bytes'] != len(raw) or r['sha256'] != digest(raw):
|
||||
raise ValueError('Invalid official source receipt')
|
||||
pages[name] = raw; evidence[name] = {'url': r['url'], 'sha256': digest(raw)}
|
||||
program = parse_program(pages['accepted.html'].decode('utf-8'))
|
||||
return m, pages, evidence, program
|
||||
|
||||
|
||||
def paper_snapshots(root, program):
|
||||
root = Path(root); path = root/'manifest.json'
|
||||
if root.is_symlink() or path.is_symlink():
|
||||
raise ValueError('Paper snapshot symlinks rejected')
|
||||
if path.stat().st_size > 4*1024**2:
|
||||
raise ValueError('Oversized paper manifest')
|
||||
m = json.loads(path.read_bytes()); result = {}
|
||||
for r in m['records']:
|
||||
pid = r['paper_id']
|
||||
if pid not in program or pid in result or r['file'] != pid+'.html':
|
||||
raise ValueError('Foreign or repeated paper snapshot')
|
||||
p = root/r['file']
|
||||
if p.is_symlink() or p.stat().st_size > 2*1024**2:
|
||||
raise ValueError('Unsafe paper snapshot')
|
||||
receipt = {**r, 'retrieved_at': r.get('retrieved_at', m['at'])}
|
||||
result[pid] = checked_html(p.read_bytes(), receipt, program[pid]['url'], 2*1024**2)
|
||||
return result
|
||||
|
||||
|
||||
def collect(out, *, official_snapshots, paper_snapshots_dir=None, cache=None, max_new_pages=None):
|
||||
if (paper_snapshots_dir is None) == (cache is None):
|
||||
raise ValueError('Choose paper snapshots or public metadata cache')
|
||||
if max_new_pages is not None and (type(max_new_pages) is not int or max_new_pages < 0 or cache is None):
|
||||
raise ValueError('New-page budget requires cache mode and a nonnegative integer')
|
||||
out = Path(out)
|
||||
if out.exists():
|
||||
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||
out.mkdir(parents=True)
|
||||
lock = None
|
||||
try:
|
||||
official, pages, source_evidence, program = official_snapshot(official_snapshots)
|
||||
directory = out/'evidence'; directory.mkdir()
|
||||
for name, raw in pages.items():
|
||||
rel = 'evidence/'+digest(raw)+'.html'; (out/rel).write_bytes(raw); source_evidence[name]['file'] = rel
|
||||
snapshots = paper_snapshots(paper_snapshots_dir, program) if paper_snapshots_dir is not None else None
|
||||
client = None
|
||||
if cache is not None:
|
||||
cache = Path(cache); cache.mkdir(parents=True, exist_ok=True)
|
||||
if cache.is_symlink() or (cache/'.collector.lock').is_symlink():
|
||||
raise ValueError('Cache lock symlinks rejected')
|
||||
lock = (cache/'.collector.lock').open('a+')
|
||||
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||
client = PublicHTMLCache(cache, ORIGIN)
|
||||
rows = []; missing = []; fetched = 0; reused = 0
|
||||
for pid in sorted(program, key=int):
|
||||
p = program[pid]
|
||||
if snapshots is not None:
|
||||
response = snapshots.get(pid)
|
||||
else:
|
||||
response = client.cached(p['url'])
|
||||
if response is not None:
|
||||
reused += 1
|
||||
elif max_new_pages is None or fetched < max_new_pages:
|
||||
response = client.get(p['url']); fetched += 1
|
||||
if response is None:
|
||||
missing.append(pid); continue
|
||||
raw, receipt = response; fields = parse_paper(raw.decode('utf-8'), p['url'], p)
|
||||
rel = 'evidence/'+digest(raw)+'.html'; (out/rel).write_bytes(raw)
|
||||
keyword = p['listing']['keywords_literal']
|
||||
rows.append({'source': SOURCE, 'external_id': '2026/'+pid, 'url': p['url'],
|
||||
'title': fields['title'], 'authors': fields['authors'], 'abstract': fields['abstract'],
|
||||
'identifiers': {}, 'submission_identifiers': [], 'venue': 'ECCV', 'year': 2026,
|
||||
'track': TRACK, 'status': 'accepted', 'observed_at': official['at'],
|
||||
'raw_decision': 'listed in frozen preliminary ECCV 2026 accepted program, pending publisher checks',
|
||||
'evidence_url': ACCEPTED,
|
||||
'tags': [{'namespace': SOURCE+':keyword', 'value': keyword}] if keyword else [],
|
||||
'metadata': {'official_program': p, 'public_page_fields': fields,
|
||||
'public_page_receipt': receipt, 'public_page_evidence': {'file': rel, 'sha256': digest(raw), 'url': p['url']},
|
||||
'program_source_evidence': source_evidence, 'publication_state': 'preliminary_pending_publisher_checks',
|
||||
'original_decision_signature_verified': False,
|
||||
'program_id_semantics': 'Official program locator, not a verified original submission number or cross-platform identifier.',
|
||||
'abstract_quality': 'Public display literal retained, including broken hyphenation/missing spaces; no automatic text repair.',
|
||||
'date_semantics': 'JSON-LD dates describe the website entry, not verified paper publication or decision dates.',
|
||||
'label_semantics': 'Official keyword namespace independent of track, contribution/research type and custom labels.',
|
||||
'scope': 'Public metadata only; no protected live content, reviews, person profiles or PDFs. Requires work identity, metadata version and complete history review before import.'}})
|
||||
if len(rows) % 10 == 0:
|
||||
print(json.dumps({'phase': 'public_metadata', 'records': len(rows), 'new_pages': fetched,
|
||||
'cached_pages': reused, 'expected_program_ids': 2834, 'at': datetime.now(timezone.utc).isoformat()}), flush=True)
|
||||
files = []
|
||||
if rows:
|
||||
path = out/'eccv-program-2026-unclassified.jsonl'; payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
|
||||
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ECCV', 'year': 2026, 'track': TRACK,
|
||||
'count': len(rows), 'file': str(path), 'file_sha256': digest(payload), 'list_url': ACCEPTED,
|
||||
'status': 'partial', 'missing_abstract_count': 0, 'source_expected_count': 2834}
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n'); files.append(manifest)
|
||||
report = {'status': 'collected_pending_identity_review' if rows else 'no_public_metadata_collected',
|
||||
'records': len(rows), 'files': files, 'expected_program_ids': 2834, 'program_occurrences': 2851,
|
||||
'duplicate_program_occurrences': 17, 'uncollected_program_ids': missing,
|
||||
'new_pages': fetched, 'reused_pages': reused, 'database_written': False,
|
||||
'public_review_visibility': 'not_public_per_2026_policy', 'public_rejected_decision_metadata': 'unknown',
|
||||
'scope': 'Always partial: preliminary program, unresolved track/identity/version coverage; not final whole-database deduplication.'}
|
||||
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||
return report
|
||||
except Exception as error:
|
||||
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__,
|
||||
'database_written': False, 'scope': 'Preserve this output and verified cache; resume with a new output directory after inspection.'})+'\n')
|
||||
raise
|
||||
finally:
|
||||
if lock is not None:
|
||||
lock.close()
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||
parser.add_argument('--official-snapshots', required=True)
|
||||
group = parser.add_mutually_exclusive_group(required=True)
|
||||
group.add_argument('--paper-snapshots')
|
||||
group.add_argument('--cache')
|
||||
parser.add_argument('--max-new-pages', type=int)
|
||||
parser.add_argument('--out', required=True)
|
||||
args = parser.parse_args(argv)
|
||||
print(json.dumps(collect(args.out, official_snapshots=args.official_snapshots,
|
||||
paper_snapshots_dir=args.paper_snapshots, cache=args.cache, max_new_pages=args.max_new_pages), ensure_ascii=False))
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Immutable, receipt-checked public HTML cache; no expiry or silent refresh."""
|
||||
from datetime import datetime, timezone
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import time
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
|
||||
def digest(raw):
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def checked_html(raw, receipt, url, limit):
|
||||
if (len(raw) > limit or receipt.get('url') != url or receipt.get('final_url') != url
|
||||
or receipt.get('http_status') != 200 or receipt.get('bytes') != len(raw)
|
||||
or receipt.get('sha256') != digest(raw)
|
||||
or not str(receipt.get('content_type', '')).lower().startswith('text/html')
|
||||
or datetime.fromisoformat(receipt['retrieved_at'].replace('Z', '+00:00')).utcoffset() is None):
|
||||
raise ValueError('Invalid public HTML receipt, response or timestamp')
|
||||
if any(v in raw.lower() for v in (b'<title>just a moment', b'cf-chl-', b'botstopper')):
|
||||
raise ValueError('Access challenge; stop without bypass')
|
||||
return raw, receipt
|
||||
|
||||
|
||||
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||
def redirect_request(self, *args, **kwargs):
|
||||
return None
|
||||
|
||||
|
||||
class PublicHTMLCache:
|
||||
"""Caller holds a single-writer lock and restricts the allowed URL paths."""
|
||||
def __init__(self, directory, origin, *, limit=2*1024**2, delay=1):
|
||||
self.root = Path(directory)
|
||||
self.root.mkdir(parents=True, exist_ok=True)
|
||||
if self.root.is_symlink():
|
||||
raise ValueError('Cache symlinks rejected')
|
||||
self.origin = origin
|
||||
self.limit = limit
|
||||
self.delay = delay
|
||||
self.opener = urllib.request.build_opener(NoRedirect())
|
||||
|
||||
def paths(self, url):
|
||||
u = urlsplit(url)
|
||||
if u.scheme != 'https' or u.username or u.password or u.fragment or u.scheme+'://'+u.netloc != self.origin:
|
||||
raise ValueError('Unapproved public metadata origin')
|
||||
key = digest(url.encode())
|
||||
return self.root/(key+'.html'), self.root/(key+'.json'), self.root/(key+'.failed.json')
|
||||
|
||||
def cached(self, url):
|
||||
body, receipt, failure = self.paths(url)
|
||||
parts = (body.with_suffix('.html.part'), receipt.with_suffix('.json.part'))
|
||||
if any(p.is_symlink() for p in (body, receipt, failure, *parts)):
|
||||
raise ValueError('Cache symlinks rejected')
|
||||
if any(p.exists() for p in parts):
|
||||
raise ValueError('Incomplete cache part; preserve and inspect orphan before recovery')
|
||||
if failure.exists():
|
||||
raise ValueError('Previous access failure requires inspection; no automatic retry')
|
||||
if body.exists() != receipt.exists():
|
||||
raise ValueError('Incomplete cache pair; preserve and inspect orphan before recovery')
|
||||
if not body.exists():
|
||||
return None
|
||||
if body.stat().st_size > self.limit or receipt.stat().st_size > 16*1024:
|
||||
raise ValueError('Oversized cache entry')
|
||||
return checked_html(body.read_bytes(), json.loads(receipt.read_bytes()), url, self.limit)
|
||||
|
||||
def get(self, url):
|
||||
cached = self.cached(url)
|
||||
if cached is not None:
|
||||
return cached
|
||||
body, receipt, failure = self.paths(url)
|
||||
try:
|
||||
request = urllib.request.Request(url, headers={'User-Agent': 'PaperAtlas/0.1 (public scholarly metadata research)'})
|
||||
with self.opener.open(request, timeout=30) as response:
|
||||
rec = {'url': url, 'final_url': response.geturl(), 'http_status': response.status,
|
||||
'content_type': response.headers.get('Content-Type', '')}
|
||||
if rec['final_url'] != url or rec['http_status'] != 200 or not rec['content_type'].lower().startswith('text/html'):
|
||||
raise ValueError('Unexpected metadata redirect, HTTP status or content type')
|
||||
raw = response.read(self.limit+1)
|
||||
if len(raw) > self.limit:
|
||||
raise ValueError('Metadata response exceeds size limit')
|
||||
expected = response.headers.get('Content-Length')
|
||||
if expected is not None and int(expected) != len(raw):
|
||||
raise ConnectionError('Incomplete metadata response')
|
||||
rec.update(bytes=len(raw), sha256=digest(raw), retrieved_at=datetime.now(timezone.utc).isoformat())
|
||||
checked_html(raw, rec, url, self.limit)
|
||||
except (urllib.error.HTTPError, ValueError) as error:
|
||||
if isinstance(error, urllib.error.HTTPError):
|
||||
status = error.code
|
||||
retry_after = error.headers.get('Retry-After') if error.headers else None
|
||||
error.close()
|
||||
else:
|
||||
status = None
|
||||
retry_after = None
|
||||
# Access errors remain sticky until the operator verifies availability.
|
||||
failure.write_text(json.dumps({'url': url, 'http_status': status, 'retry_after': retry_after, 'error_type': type(error).__name__,
|
||||
'scope': 'Inspect access/redirect/response failure before moving this marker aside; no bypass or automatic retry.'})+'\n')
|
||||
raise
|
||||
part = body.with_suffix('.html.part')
|
||||
part.write_bytes(raw); part.replace(body)
|
||||
part = receipt.with_suffix('.json.part')
|
||||
part.write_text(json.dumps(rec, indent=2)+'\n'); part.replace(receipt)
|
||||
time.sleep(self.delay)
|
||||
return raw, rec
|
||||
BIN
Binary file not shown.
@@ -0,0 +1,139 @@
|
||||
import copy
|
||||
import gzip
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from paper_atlas.collectors import eccv2026 as c
|
||||
from paper_atlas.io.html_cache import PublicHTMLCache
|
||||
|
||||
|
||||
class ECCV2026Tests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.bundle = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/eccv2026-source.json.gz').read_bytes()))
|
||||
cls.program = c.parse_program(cls.bundle['official_pages']['accepted.html'])
|
||||
|
||||
def snapshots(self, root):
|
||||
official = root/'official'; paper = root/'paper'; official.mkdir(); paper.mkdir()
|
||||
for name, raw in self.bundle['official_pages'].items():
|
||||
(official/name).write_bytes(raw.encode('utf-8'))
|
||||
(official/'manifest.json').write_bytes(self.bundle['official_manifest'].encode('utf-8'))
|
||||
for pid, raw in self.bundle['paper_pages'].items():
|
||||
(paper/(pid+'.html')).write_bytes(raw.encode('utf-8'))
|
||||
(paper/'manifest.json').write_text(json.dumps(self.bundle['paper_manifest']))
|
||||
return official, paper
|
||||
|
||||
def test_complete_program_and_duplicate_occurrences_match_gold(self):
|
||||
self.assertEqual(len(self.program), 2834)
|
||||
self.assertEqual(sum(len(p['row_ordinals']) for p in self.program.values()), 2851)
|
||||
for ordinal, gold in enumerate(self.bundle['inventory']['records'], 1):
|
||||
item = self.program[gold['paper_id']]
|
||||
self.assertEqual(item['listing'], {k: gold[k] for k in ('title','authors_literal','keywords_literal','presentation_literal')})
|
||||
self.assertIn(ordinal, item['row_ordinals'])
|
||||
|
||||
def test_conflicting_duplicate_not_silently_folded(self):
|
||||
title = 'Transferability Between Understanding and Generation in Unified Multimodal Models'
|
||||
with self.assertRaisesRegex(ValueError, 'Conflicting'):
|
||||
c.parse_program(self.bundle['official_pages']['accepted.html'].replace('href="/virtual/2026/poster/3137">'+title, 'href="/virtual/2026/poster/3137">Conflicting title', 1))
|
||||
|
||||
def test_missing_member_and_changed_provisional_status_rejected(self):
|
||||
html = self.bundle['official_pages']['accepted.html']
|
||||
for changed in [html.replace('/virtual/2026/poster/4048', '/virtual/2024/poster/4048', 1),
|
||||
html.replace('This paper list is preliminary pending publisher checks.', 'Final archival papers.')]:
|
||||
with self.assertRaises(ValueError):
|
||||
c.parse_program(changed)
|
||||
|
||||
def test_public_page_fields_match_independent_sample_review(self):
|
||||
for gold in self.bundle['paper_gold']['records']:
|
||||
pid = gold['paper_id']; fields = c.parse_paper(self.bundle['paper_pages'][pid], gold['url'], self.program[pid])
|
||||
self.assertEqual(fields['title'], gold['title']); self.assertEqual(fields['authors'], gold['authors'])
|
||||
self.assertEqual(fields['abstract'], gold['abstract_literal']); self.assertFalse(fields['review_flags'])
|
||||
self.assertIn('sig-nal', c.parse_paper(self.bundle['paper_pages']['4048'], self.program['4048']['url'], self.program['4048'])['abstract'])
|
||||
|
||||
def test_foreign_canonical_and_inconsistent_jsonld_author_rejected(self):
|
||||
raw = self.bundle['paper_pages']['4048']; url = self.program['4048']['url']
|
||||
for changed in [raw.replace(url, 'https://example.org/other'), raw.replace('Suhyeon Ha', 'Other Author', 1)]:
|
||||
with self.assertRaises(ValueError):
|
||||
c.parse_paper(changed, url, self.program['4048'])
|
||||
|
||||
def test_program_and_page_versions_flagged_without_aliases(self):
|
||||
program = copy.deepcopy(self.program['4048']); program['listing']['title'] = 'Earlier source title'
|
||||
fields = c.parse_paper(self.bundle['paper_pages']['4048'], program['url'], program)
|
||||
self.assertEqual(fields['review_flags'], ['program_and_page_title_versions_differ_requires_review'])
|
||||
self.assertEqual(fields['title'], self.program['4048']['listing']['title'])
|
||||
|
||||
def test_partial_snapshot_collection_has_no_inferred_submission_ids(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); official, paper = self.snapshots(root)
|
||||
report = c.collect(root/'out', official_snapshots=official, paper_snapshots_dir=paper)
|
||||
self.assertEqual(report['records'], 3); self.assertEqual(len(report['uncollected_program_ids']), 2831)
|
||||
self.assertEqual(report['public_rejected_decision_metadata'], 'unknown'); self.assertFalse(report['database_written'])
|
||||
manifest, = report['files']; self.assertEqual(manifest['status'], 'partial'); self.assertEqual(manifest['source_expected_count'], 2834)
|
||||
rows = [json.loads(l) for l in Path(manifest['file']).read_text().splitlines()]
|
||||
self.assertEqual({r['external_id'] for r in rows}, {'2026/4048','2026/3137','2026/3652'})
|
||||
self.assertTrue(all(not r['identifiers'] and not r['submission_identifiers'] and r['track']=='program_unclassified' for r in rows))
|
||||
self.assertTrue(all(r['metadata']['publication_state']=='preliminary_pending_publisher_checks' and not r['metadata']['original_decision_signature_verified'] for r in rows))
|
||||
self.assertTrue(all(t['namespace']=='eccv_program:keyword' for r in rows for t in r['tags']))
|
||||
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||
c.collect(root/'out', official_snapshots=official, paper_snapshots_dir=paper)
|
||||
|
||||
def test_zero_network_budget_reuses_immutable_cache_only(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); official, _ = self.snapshots(root); client = PublicHTMLCache(root/'cache', c.ORIGIN)
|
||||
for r in self.bundle['paper_manifest']['records']:
|
||||
body, receipt, _ = client.paths(r['url']); body.write_text(self.bundle['paper_pages'][r['paper_id']]); receipt.write_text(json.dumps({**r,'retrieved_at':self.bundle['paper_manifest']['at']}))
|
||||
with patch.object(PublicHTMLCache, 'get', side_effect=AssertionError('network forbidden')):
|
||||
report = c.collect(root/'out', official_snapshots=official, cache=root/'cache', max_new_pages=0)
|
||||
self.assertEqual((report['records'],report['new_pages'],report['reused_pages']), (3,0,3))
|
||||
|
||||
def test_tampered_snapshot_preserves_failure_without_success(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); official, paper = self.snapshots(root); (paper/'4048.html').write_text('tampered')
|
||||
with self.assertRaises(ValueError):
|
||||
c.collect(root/'out', official_snapshots=official, paper_snapshots_dir=paper)
|
||||
self.assertTrue((root/'out/failed.json').exists()); self.assertFalse((root/'out/collection-report.json').exists())
|
||||
|
||||
def test_structured_math_markup_requires_explicit_review(self):
|
||||
raw = self.bundle['paper_pages']['4048']
|
||||
self.assertIn('<p>Proactive tamper', raw)
|
||||
fields = c.parse_paper(raw.replace('<p>Proactive tamper', '<p>10<sup>6</sup> Proactive tamper', 1), self.program['4048']['url'], self.program['4048'])
|
||||
self.assertIn('structured_abstract_markup_requires_review', fields['review_flags'])
|
||||
self.assertEqual(fields['abstract_markup_tags'], ['sup'])
|
||||
|
||||
def test_fixture_preserves_exact_http_bytes_including_crlf(self):
|
||||
import hashlib
|
||||
for receipt in json.loads(self.bundle['official_manifest'])['sources']:
|
||||
raw = self.bundle['official_pages'][receipt['file']].encode('utf-8')
|
||||
self.assertEqual(len(raw), receipt['bytes'])
|
||||
self.assertEqual(hashlib.sha256(raw).hexdigest(), receipt['sha256'])
|
||||
for receipt in self.bundle['paper_manifest']['records']:
|
||||
raw = self.bundle['paper_pages'][receipt['paper_id']].encode('utf-8')
|
||||
self.assertEqual(hashlib.sha256(raw).hexdigest(), receipt['sha256'])
|
||||
|
||||
def test_real_double_encoded_author_requires_exact_visible_agreement(self):
|
||||
import hashlib
|
||||
sample = self.bundle['encoding_sample']; pid = sample['paper_id']; raw = sample['html']
|
||||
self.assertEqual(hashlib.sha256(raw.encode('utf-8')).hexdigest(), sample['receipt']['sha256'])
|
||||
self.assertEqual(sample['receipt']['sha256'], 'b575318fa569e83eea5e96f937dd60f078fbed482877318013f797c16a02ff09')
|
||||
fields = c.parse_paper(raw, sample['receipt']['url'], self.program[pid])
|
||||
self.assertEqual(fields['authors'], sample['gold']['authors'])
|
||||
self.assertEqual(fields['jsonld_raw_text']['author_names'][0], 'Zi&#x27;ang Lu')
|
||||
self.assertIn('jsonld_display_text_normalized_requires_review', fields['review_flags'])
|
||||
# Real changes to the author or an extra encoding layer remain errors.
|
||||
for replacement in ('Other Author', 'Zi&amp;#x27;ang Lu'):
|
||||
with self.assertRaisesRegex(ValueError, 'authors disagree'):
|
||||
c.parse_paper(raw.replace('Zi&#x27;ang Lu', replacement), sample['receipt']['url'], self.program[pid])
|
||||
|
||||
def test_real_already_agreeing_literal_title_is_not_decoded_again(self):
|
||||
import hashlib
|
||||
sample = self.bundle['literal_title_sample']; pid = sample['paper_id']; raw = sample['html']
|
||||
self.assertEqual(hashlib.sha256(raw.encode('utf-8')).hexdigest(), 'c208525ad5f02b4ca274dff7e24a0392dabdc55bb3620497079a84df81a02a72')
|
||||
fields = c.parse_paper(raw, sample['receipt']['url'], self.program[pid])
|
||||
self.assertIn('&', fields['title'])
|
||||
self.assertEqual(fields['title'], sample['gold']['visible_title'])
|
||||
self.assertEqual(fields['jsonld_raw_text']['name'], sample['gold']['jsonld_title_raw'])
|
||||
self.assertNotEqual(fields['title'], sample['gold']['blind_double_decoded_title'])
|
||||
self.assertNotIn('jsonld_display_text_normalized_requires_review', fields['review_flags'])
|
||||
@@ -0,0 +1,90 @@
|
||||
from datetime import datetime, timezone
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import MagicMock
|
||||
import urllib.error
|
||||
|
||||
from paper_atlas.io.html_cache import PublicHTMLCache, digest, NoRedirect
|
||||
|
||||
URL = 'https://eccv.ecva.net/virtual/2026/poster/4048'
|
||||
|
||||
|
||||
class PublicHTMLCacheTests(unittest.TestCase):
|
||||
def receipt(self, raw):
|
||||
return {'url':URL,'final_url':URL,'http_status':200,'content_type':'text/html; charset=utf-8',
|
||||
'bytes':len(raw),'sha256':digest(raw),'retrieved_at':datetime.now(timezone.utc).isoformat()}
|
||||
|
||||
def test_old_cache_reused_without_network_or_expiry(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0); raw = b'<html>metadata</html>'
|
||||
body, receipt, _ = c.paths(URL); body.write_bytes(raw); receipt.write_text(json.dumps(self.receipt(raw))); os.utime(body,(0,0))
|
||||
c.opener = MagicMock(); self.assertEqual(c.get(URL)[0], raw); c.opener.open.assert_not_called()
|
||||
|
||||
def test_pair_tampering_and_orphan_parts_fail_without_refetch(self):
|
||||
for kind in ('hash','orphan','part'):
|
||||
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp:
|
||||
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0); body, receipt, _ = c.paths(URL); body.write_bytes(b'original')
|
||||
if kind == 'hash': receipt.write_text(json.dumps(self.receipt(b'different')))
|
||||
if kind == 'part': body.unlink(); body.with_suffix('.html.part').write_bytes(b'partial')
|
||||
c.opener = MagicMock()
|
||||
with self.assertRaises(ValueError): c.get(URL)
|
||||
c.opener.open.assert_not_called()
|
||||
|
||||
def test_http_access_failure_sticky_without_automatic_retry(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0); c.opener = MagicMock()
|
||||
c.opener.open.side_effect = urllib.error.HTTPError(URL,403,'Forbidden',{},None)
|
||||
with self.assertRaises(urllib.error.HTTPError): c.get(URL)
|
||||
with self.assertRaisesRegex(ValueError,'Previous access'): c.get(URL)
|
||||
self.assertEqual(c.opener.open.call_count,1)
|
||||
|
||||
def test_redirect_nonhtml_challenge_and_oversize_rejected(self):
|
||||
for kind in ('redirect','type','challenge','size'):
|
||||
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp:
|
||||
c = PublicHTMLCache(tmp,'https://eccv.ecva.net',limit=100,delay=0); c.opener=MagicMock(); r=c.opener.open.return_value.__enter__.return_value
|
||||
r.status=200; r.geturl.return_value=URL; r.headers={'Content-Type':'text/html'}; r.read.return_value=b'<html>metadata</html>'
|
||||
if kind=='redirect':r.geturl.return_value='https://example.org/login'
|
||||
if kind=='type':r.headers={'Content-Type':'application/pdf'}
|
||||
if kind=='challenge':r.read.return_value=b'<title>Just a moment</title>'
|
||||
if kind=='size':r.read.return_value=b'x'*101
|
||||
with self.assertRaises(ValueError):c.get(URL)
|
||||
self.assertFalse(c.paths(URL)[0].exists())
|
||||
self.assertIsNone(NoRedirect().redirect_request(None,None,None,None,None,None))
|
||||
|
||||
def test_valid_response_has_atomic_body_and_receipt(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
c=PublicHTMLCache(tmp,'https://eccv.ecva.net',delay=0);c.opener=MagicMock();r=c.opener.open.return_value.__enter__.return_value;r.status=200;r.geturl.return_value=URL;r.headers={'Content-Type':'text/html','Content-Length':'4'};r.read.return_value=b'data'
|
||||
raw, receipt=c.get(URL);self.assertEqual(raw,b'data');self.assertEqual(receipt['sha256'],digest(raw));self.assertEqual(c.cached(URL),(raw,receipt));self.assertFalse(list(Path(tmp).glob('*.part')))
|
||||
|
||||
def test_foreign_origin_and_cache_symlink_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
c=PublicHTMLCache(tmp,'https://eccv.ecva.net',delay=0)
|
||||
with self.assertRaises(ValueError):c.get('https://example.org/other')
|
||||
body,_,_=c.paths(URL);target=Path(tmp)/'outside';target.write_bytes(b'other');body.symlink_to(target)
|
||||
with self.assertRaises(ValueError):c.get(URL)
|
||||
|
||||
def test_successful_fetch_publishes_receipt_and_reuses_without_refetch(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0)
|
||||
c.opener = MagicMock(); response = c.opener.open.return_value.__enter__.return_value
|
||||
response.status = 200; response.geturl.return_value = URL
|
||||
raw = b'<html>public metadata</html>'
|
||||
response.headers = {'Content-Type':'text/html','Content-Length':str(len(raw))}; response.read.return_value = raw
|
||||
self.assertEqual(c.get(URL)[0], raw); self.assertEqual(c.get(URL)[0], raw)
|
||||
self.assertEqual(c.opener.open.call_count, 1)
|
||||
body, receipt, _ = c.paths(URL)
|
||||
self.assertTrue(body.exists()); self.assertTrue(receipt.exists())
|
||||
self.assertFalse(body.with_suffix('.html.part').exists())
|
||||
self.assertFalse(receipt.with_suffix('.json.part').exists())
|
||||
|
||||
def test_changed_url_is_rejected_before_reading_body(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0)
|
||||
c.opener = MagicMock(); response = c.opener.open.return_value.__enter__.return_value
|
||||
response.status = 200; response.geturl.return_value = 'https://example.org/login'
|
||||
response.headers = {'Content-Type':'text/html'}
|
||||
with self.assertRaises(ValueError): c.get(URL)
|
||||
response.read.assert_not_called()
|
||||
Reference in New Issue
Block a user