diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index 8f9c652..99b66b1 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -49,6 +49,34 @@ jobs: subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--help'],cwd=outside,check=True) + eccv_fixture = json.loads(gzip.decompress((src/'tests/fixtures/eccv2026-source.json.gz').read_bytes())) + eccv_official = root/'eccv2026-official' + eccv_papers = root/'eccv2026-paper-samples' + eccv_official.mkdir(); eccv_papers.mkdir() + for name, html in eccv_fixture['official_pages'].items(): + (eccv_official/name).write_bytes(html.encode('utf-8')) + (eccv_official/'manifest.json').write_bytes(eccv_fixture['official_manifest'].encode('utf-8')) + for pid, html in eccv_fixture['paper_pages'].items(): + (eccv_papers/(pid+'.html')).write_bytes(html.encode('utf-8')) + eccv_encoding = eccv_fixture['encoding_sample'] + (eccv_papers/(eccv_encoding['paper_id']+'.html')).write_bytes(eccv_encoding['html'].encode('utf-8')) + eccv_fixture['paper_manifest']['records'].append({**eccv_encoding['receipt'], 'paper_id': eccv_encoding['paper_id'], 'file': eccv_encoding['paper_id']+'.html'}) + eccv_literal = eccv_fixture['literal_title_sample'] + (eccv_papers/(eccv_literal['paper_id']+'.html')).write_bytes(eccv_literal['html'].encode('utf-8')) + eccv_fixture['paper_manifest']['records'].append({**eccv_literal['receipt'], 'paper_id': eccv_literal['paper_id'], 'file': eccv_literal['paper_id']+'.html'}) + (eccv_papers/'manifest.json').write_text(json.dumps(eccv_fixture['paper_manifest'])) + subprocess.run([python,'-I','-B','-m','paper_atlas','collect','eccv2026-program','--official-snapshots',str(eccv_official),'--paper-snapshots',str(eccv_papers),'--out',str(root/'installed-eccv2026')],cwd=outside,check=True) + eccv_report = json.loads((root/'installed-eccv2026/collection-report.json').read_text()) + assert eccv_report['records'] == 5 and eccv_report['expected_program_ids'] == 2834 + assert eccv_report['duplicate_program_occurrences'] == 17 and len(eccv_report['uncollected_program_ids']) == 2829 + assert not eccv_report['database_written'] and eccv_report['files'][0]['track'] == 'program_unclassified' + eccv_rows = [json.loads(line) for line in pathlib.Path(eccv_report['files'][0]['file']).read_text().splitlines()] + eccv_encoded_row = next(row for row in eccv_rows if row['external_id'] == '2026/4199') + assert eccv_encoded_row['authors'] == eccv_encoding['gold']['authors'] + assert 'jsonld_display_text_normalized_requires_review' in eccv_encoded_row['metadata']['public_page_fields']['review_flags'] + eccv_literal_row = next(row for row in eccv_rows if row['external_id'] == '2026/3844') + assert eccv_literal_row['title'] == eccv_literal['gold']['visible_title'] + assert '&' in eccv_literal_row['title'] sdm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/sdm2026-source.json.gz').read_bytes())) sdm_snapshots = root/'sdm2026-snapshots' for name, text in sdm_fixture['files'].items(): diff --git a/README.md b/README.md index 28cd310..e8b9d96 100644 --- a/README.md +++ b/README.md @@ -129,4 +129,6 @@ ICAPS 2026 也提供独立的 `collect icaps2026-registry`,保留 Crossref 卷 `collect corl2026-program` 将官方 687 个论坛与固定公开档案按 ID 对照,保留 34 条标题版本差异及分字段来源。不会自动接受 arXiv 匹配或推断拒稿;详情见 [CoRL 2026 使用与恢复说明](docs/corl2026-program.md)。 +ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目 ID,并以不可变缓存断点采集公开元数据;保留摘要原文、来源关键词及未分类 track,始终待身份审核,不自动入库。 + `collect sdm2026-proceedings` 核验 53 个 Crossref 论文章节,并单独保留 53 项日程观察及 52 页摘要。作者差异、取消展示和摘要缺失分别保留,不自动拼接来源或绑定作品;见 [SDM 2026 使用与恢复说明](docs/sdm2026-proceedings.md)。 diff --git a/docs/eccv2026-program.md b/docs/eccv2026-program.md new file mode 100644 index 0000000..4d10616 --- /dev/null +++ b/docs/eccv2026-program.md @@ -0,0 +1,35 @@ +# ECCV 2026:暂定节目与公开元数据 + +这是长期维护、按需运行的来源适配器。它读取已核验的官方节目快照,采集公开论文页的标题、完整作者和摘要,输出待身份审核文件;不连接数据库,不下载 PDF、评审、个人档案或受保护的直播内容。 + +```sh +paper-atlas collect eccv2026-program \ + --official-snapshots /data/paper-index/collected/eccv2026-official-research-v1 \ + --cache /data/paper-index/cache/eccv2026-public-pages \ + --max-new-pages 200 \ + --out /data/paper-index/collected/eccv2026-pass01 +``` + +官方目录包含 `accepted.html`、`policies.html`、`faq.html` 和冻结的 `manifest.json`。当前适配器核验固定版本:节目 2,851 行、2,834 个节目 ID;17 个 ID 各重复一次,标题、作者、关键词、展示信息均一致。输出按节目 ID 合并这些重复行,并保留全部行序号。它不是跨来源作品去重。 + +`--max-new-pages` 限制本轮新请求数;不指定时遍历整个冻结名单。再次运行时使用同一 `--cache` 和**全新** `--out`,复用已验证缓存并继续读取缺失页。缓存不按 24 小时过期,不自动刷新或覆盖已成功的原始响应。每次运行输出包含本轮能核验的全部缓存条目,并明确列出尚未采集的 ID;不能未经审核把多轮重叠输出全部重复导入。 + +公开请求按固定来源和路径限制,只接受 HTTP 200、原 URL、HTML 和限定大小的完整响应。请求失败、重定向、访问挑战、非 HTML 或异常响应留下阻断标记,不自动重试或更换账号、域名、代理绕过限制。记录的 `Retry-After` 供人工核查使用。网络超时等未产生成功缓存的情况可在确认进程终止后用新输出目录继续。缓存数据与回执须成对且哈希一致;遇到孤立文件或 `.part` 时保留现场,先核查再把本任务的残留文件移至恢复目录,不静默丢弃。写同一缓存时持有非阻塞单写锁。 + +离线重放可改用 `--paper-snapshots /path/to/papers`,该目录包含 `<节目ID>.html` 和 `manifest.json`(带时区的 `at`、`records`,各条含文件、URL、最终 URL、HTTP 状态、类型、字节数、SHA-256)。缺少的公开页会明确列为未采集;三篇样本只会生成三条 partial 记录,不会冒充全量成功。 + +## 来源边界 + +[官方名单](https://eccv.ecva.net/Conferences/2026/AcceptedPapers)明确注明 **preliminary pending publisher checks**。这里的 accepted 表示冻结时列入暂定官方名单,不代表已核验 Springer 最终发表或原始签名决定。所有输出始终 partial。节目 ID 是官方节目定位符,不冒充原始 OpenReview 投稿号;尚未核实跨平台标识时作品标识和投稿标识均为空。 + +公开页的 canonical URL、可见标题/作者和 JSON-LD 相互核对;页面版本与冻结节目文字有差异时分别保留并标记待审阅,不制造人名别名。JSON-LD 日期只是网站条目时间,不当作论文发表日期或原始决定日期。摘要保留公开页 DOM 文本(规范空白),包括源头的断词、缺空格;不自动修补。原始 HTML 单独留存;遇到公式、上下标或图片结构会标记待审阅,不能把扁平化文本当作无损数学表达。页面上的登录提示对应直播内容,不用于访问受保护内容。 + +第 4199 页的 JSON-LD 作者姓名存在双重 HTML 实体编码;第 3844 页的可见标题和 JSON-LD 则已经一致地包含字面 `&`。解析器优先保留已经一致的文字,仅在字段不一致时尝试最多两层实体解码来核对可见文本,不把已经一致的标题再次解码。完整作者序列仍须精确一致,真实差异或更多编码层次仍停止。`jsonld_raw_text` 保留原始结构化标题/姓名,发生文本规范化时增加 `jsonld_display_text_normalized_requires_review`,不生成姓名别名。恢复必须使用校验过的修复版本、新输出目录和原缓存;保留旧失败目录,不重新请求已有成功页面。 + +官方关键词保存在独立 `eccv_program:keyword` 命名空间,并保留原始单元格和证据。关键词、展示形式、贡献/研究类型、自定义标签相互独立,投稿 track 暂用 `program_unclassified`,不按主题推断 main、findings、workshop 或 position。 + +[2026 投稿政策](https://eccv.ecva.net/Conferences/2026/SubmissionPolicies)明确评审和作者回复不公开;这不能代替对公开 rejected、desk rejected、withdrawn **决定元数据**的单独核查。当前拒稿决定可见性仍记为 unknown,不能从录用表缺席推断拒稿。 + +贡献类型的官方定义、OpenReview 状态入口与匿名访问核查见[来源核查记录](eccv2026-source-status.md)。这是独立于冻结采集输入的证据,不会改变已核验节目版本,也不能用类型定义替论文推断标签。 + +失败输出保留 `failed.json`,成功报告最后发布;恢复前核查既有进程、输出及缓存,不因日志安静重启。身份、来源版本和完整投稿历史审阅后,另行准备专用入库队列、保护基线和导入后验收;全部来源导入结束后仍须全库去重、覆盖、摘要、约束及查询审计。测试、构建及安装验证只在已授权 `ces-control:host` runner 执行。 diff --git a/docs/eccv2026-source-status.md b/docs/eccv2026-source-status.md new file mode 100644 index 0000000..0492ec8 --- /dev/null +++ b/docs/eccv2026-source-status.md @@ -0,0 +1,39 @@ +# ECCV 2026 来源核查记录 + +核查时间:2026-11-17,以下状态是当时的公开可见性。长期补漏时先核实来源是否变化,保留旧快照,不覆盖旧结论或重复导入。 + +## 贡献类型与 track + +[作者贡献类型说明](https://eccv.ecva.net/Conferences/2026/AuthorContributionTypes)通过官方导航确认。它要求作者选择一项主要贡献类型;官方 FAQ 明确贡献类型不是独立 track。公开 OpenReview group 的 `source_submissions_query_mapping` 也包含五项对应值: + +| 官方 group 字段值 | 含义摘要 | +| --- | --- | +| Algorithms/General | 方法、模型或算法;也是边界情形的默认选择 | +| Theory/Foundational | 理论、形式分析或基础理解 | +| Applied/Systems | 实际系统、部署与工程约束 | +| Datasets/Benchmarks | 数据集、基准或评估协议 | +| Concept & Feasibility | 新概念及可行性证据 | + +这些是类型定义和配置值,**尚未核验逐篇论文的作者所选类型**。不能根据标题、摘要、主题关键词或展示形式补写类型;公开节目采集器继续输出 `program_unclassified`,主题关键词继续使用 `eccv_program:keyword`。以后获得逐篇类型证据时,保留独立命名空间、原始值、来源和版本,不把它写成投稿 track。 + +## 公开负面决定元数据 + +[投稿指南](https://eccv.ecva.net/Conferences/2026/AuthorGuide)链接的会议 group 是 [`thecvf.com/ECCV/2026/Conference`](https://openreview.net/group?id=thecvf.com/ECCV/2026/Conference)。匿名 API 读取 group 成功(HTTP 200),返回 `readers: ["everyone"]`。这只说明 group 配置公开,不证明论文或决定公开。 + +group 的 `rejected_venue_id` 实际配置为 `thecvf.com/ECCV/2026/Conference/Rejected_Submission`。使用该值查询公开 notes(`content.venueid`,`limit=1`)返回 HTTP 403,随后停止;没有自动重试、切换账号、代理或其他绕过操作。其余 desk-rejected、withdrawn 和 submission 状态查询没有执行,不能报告为已核查或零条。 + +结论仍为 **拒稿决定元数据覆盖未核实,当前匿名查询受限**。403 的具体原因未确定;不能由此推断真实投稿数量、全局不存在公开拒稿数据,或某篇论文遭拒。[投稿政策](https://eccv.ecva.net/Conferences/2026/SubmissionPolicies)关于评审与作者回复不公开的规定也不能代替决定元数据核查。 + +## 可恢复证据与后续入口 + +服务器证据目录:`/data/paper-index/collected/eccv2026-policy-research-v2`。本机工作记录目录保留同名备份;三份成功响应均已逐文件校验。证据不含登录凭据。 + +| 文件 | SHA-256 | +| --- | --- | +| manifest.json | 5a04e52bf6ea34ab25fe1d91fa9a1df18c9be5abb4f554586ee28fe9aa566e6e | +| contribution-types.html | 6a5128b1f47a275411acacbb08ab91320a767bdcc48aaad09fe1be9e31dff00f | +| author-guide.html | 779196716f5b37fcd10ddc191e9cd0530d7e9d43e5bc66b16a5546cd152959c4 | +| openreview-group.json | 055bcf99d412d77f79a760890739d4de13a20ea099857154ee3e714c0647e763 | +| anonymous-decision-probes/manifest.json | a522756ed3ec2880e5d942a211d7ad13819b18c8acada551dd3a8718b5e4d63b | + +这里的补充证据与采集器的三份冻结节目/政策输入分开保存,不能替换采集器固定版本的 manifest。下次补漏先核查官方是否提供公开决定导出或明确的公开元数据说明;访问仍受限时保留缺口。若发现允许公开访问的新入口,另存带时间与哈希的快照,先审阅读者权限、决定语义和逐篇身份,再准备入库队列。 diff --git a/paper_atlas/cli.py b/paper_atlas/cli.py index 15d8793..0c867bf 100644 --- a/paper_atlas/cli.py +++ b/paper_atlas/cli.py @@ -24,6 +24,7 @@ COMMANDS = { ('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'), ('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'), ('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'), + ('collect','eccv2026-program'): ('paper_atlas.collectors.eccv2026', {}, '核验ECCV2026暂定节目并断点采集公开元数据'), ('collect','sdm2026-proceedings'): ('paper_atlas.collectors.sdm2026', {}, '核验 SDM 2026 注册论文并单独保留公开日程版本'), ('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'), ('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'), diff --git a/paper_atlas/collectors/eccv2026.py b/paper_atlas/collectors/eccv2026.py new file mode 100644 index 0000000..f8e8c30 --- /dev/null +++ b/paper_atlas/collectors/eccv2026.py @@ -0,0 +1,254 @@ +"""ECCV 2026 frozen preliminary program and resumable public metadata pages.""" +import argparse +from collections import Counter +from datetime import datetime, timezone +import fcntl +from html import unescape +import json +from pathlib import Path +import re + +from paper_atlas.io.html import Document, Node, clean +from paper_atlas.io.html_cache import PublicHTMLCache, checked_html, digest + +SOURCE = 'eccv_program' +ORIGIN = 'https://eccv.ecva.net' +ACCEPTED = ORIGIN+'/Conferences/2026/AcceptedPapers' +TRACK = 'program_unclassified' +MANIFEST_SHA = '04f9734783bbbfa2e19f936d8f5a7aecfb524b0ceef33fc5d9fb6276c7b6678a' +PAGES = {'accepted.html': ACCEPTED, 'policies.html': ORIGIN+'/Conferences/2026/SubmissionPolicies', + 'faq.html': ORIGIN+'/Conferences/2026/FAQs'} + + +def one(items): + if len(items) != 1: + raise ValueError('Expected one reviewed metadata field') + return items[0] + + +def has_class(node, value): + return value in node.attrs.get('class', '').split() + + +def parse_program(raw): + nodes = list(Document(raw).root.walk()) + if len([n for n in nodes if n.tag == 'h2' and clean(n.text()) == 'ECCV 2026 Accepted Papers:']) != 1: + raise ValueError('Unexpected ECCV program edition') + if 'This paper list is preliminary pending publisher checks.' not in clean(Document(raw).root.text()): + raise ValueError('Reviewed provisional-publication notice changed') + table = one([n for n in nodes if n.tag == 'table' and n.attrs.get('id') == 'event-list-2026-poster-nodates-filter-vslinks-table']) + papers = {}; occurrences = 0 + for tr in (n for n in table.walk() if n.tag == 'tr'): + cells = [n for n in tr.children if isinstance(n, Node) and n.tag == 'td'] + if not cells: + continue + if len(cells) != 3: + raise ValueError('Unexpected program columns') + links = [n for n in cells[0].walk() if n.tag == 'a' and re.fullmatch(r'/virtual/2026/poster/[1-9][0-9]*', n.attrs.get('href', ''))] + link = one(links); pid = link.attrs['href'].rsplit('/', 1)[1] + authors = clean(one([n for n in cells[0].walk() if n.tag == 'i']).text()) + row = {'title': clean(link.text()), 'authors_literal': authors, + 'keywords_literal': clean(cells[1].text()), 'presentation_literal': clean(cells[2].text())} + if not row['title'] or not authors: + raise ValueError('Incomplete public program fields') + occurrences += 1 + if pid in papers: + if papers[pid]['listing'] != row: + raise ValueError('Conflicting entries share a program ID') + papers[pid]['row_ordinals'].append(occurrences) + else: + papers[pid] = {'url': ORIGIN+link.attrs['href'], 'listing': row, 'row_ordinals': [occurrences]} + if occurrences != 2851 or len(papers) != 2834 or Counter(len(v['row_ordinals']) for v in papers.values()) != {1: 2817, 2: 17}: + raise ValueError('Incomplete reviewed program census') + return papers + + +def parse_paper(raw, url, program): + nodes = list(Document(raw).root.walk()) + title = clean(one([n for n in nodes if n.tag == 'h1' and has_class(n, 'event-title')]).text()) + authors_literal = clean(one([n for n in nodes if n.tag == 'div' and has_class(n, 'event-organizers')]).text()) + abstract_node = one([n for n in nodes if n.tag == 'div' and has_class(n, 'abstract-text-inner')]) + abstract = clean(abstract_node.text()) + markup = sorted({n.tag for n in abstract_node.walk() if n.tag in ('math','sup','sub','svg','img')}) + canonical = one([n.attrs.get('href') for n in nodes if n.tag == 'link' and n.attrs.get('rel') == 'canonical']) + scripts = [n for n in nodes if n.tag == 'script' and n.attrs.get('type') == 'application/ld+json'] + ld = one([json.loads(''.join(x for x in n.children if isinstance(x, str))) for n in scripts]) + authors = ld.get('author') + if (canonical != url or ld.get('@type') != 'CreativeWork' or ld.get('creditText') != 'ECCV 2026' + or not title or not isinstance(ld.get('name'), str) or not abstract + or not isinstance(authors, list) or not authors + or any(not isinstance(a, dict) or a.get('@type') != 'Person' or not isinstance(a.get('name'), str) + or not clean(a['name']) or '@' in a['name'] for a in authors)): + raise ValueError('Public visible/structured metadata identity or edition disagrees') + # Prefer source text that already agrees. Poster 3844 intentionally displays + # literal '&', while 4199 double-encodes an author's apostrophe in JSON-LD. + # Decode only to test agreement with visible text, never to rewrite it. + def agrees_with_display(value, expected): + if clean(value) == expected: + return True + for _ in range(2): + value = unescape(value) + if clean(value) == expected: + return True + return False + + raw_names = [a['name'] for a in authors] + names = [clean(name) for name in raw_names] + if not agrees_with_display(ld['name'], title): + raise ValueError('Public visible and structured titles disagree') + if ' ⋅ '.join(names) != authors_literal: + visible_names = authors_literal.split(' ⋅ ') + if len(visible_names) != len(raw_names) or any( + not agrees_with_display(raw_name, visible_name) + for raw_name, visible_name in zip(raw_names, visible_names)): + raise ValueError('Complete visible and structured authors disagree') + names = visible_names + if any('@' in name for name in names) or ' ⋅ '.join(names) != authors_literal: + raise ValueError('Complete visible and structured authors disagree') + flags = ['structured_abstract_markup_requires_review'] if markup else [] + if raw_names != names or ld['name'] != title: + flags.append('jsonld_display_text_normalized_requires_review') + if title != program['listing']['title']: + flags.append('program_and_page_title_versions_differ_requires_review') + if authors_literal != program['listing']['authors_literal']: + flags.append('program_and_page_author_versions_differ_requires_review') + return {'title': title, 'authors': names, 'authors_literal': authors_literal, 'abstract': abstract, + 'page_metadata_dates': {k: ld.get(k) for k in ('datePublished', 'dateModified')}, 'review_flags': flags, 'abstract_markup_tags': markup, + 'jsonld_raw_text': {'name': ld['name'], 'author_names': raw_names}} + + +def official_snapshot(root): + root = Path(root); manifest = root/'manifest.json' + if root.is_symlink() or manifest.is_symlink() or digest(manifest.read_bytes()) != MANIFEST_SHA: + raise ValueError('Reviewed official snapshot manifest changed') + m = json.loads(manifest.read_bytes()) + if datetime.fromisoformat(m['at'].replace('Z', '+00:00')).utcoffset() is None: + raise ValueError('Official snapshot requires a timezone') + if len(m['sources']) != 3 or {r['file'] for r in m['sources']} != set(PAGES): + raise ValueError('Official source set changed') + pages = {}; evidence = {} + for r in m['sources']: + name = r['file']; p = root/name + if p.is_symlink() or p.stat().st_size > 8*1024**2: + raise ValueError('Unsafe official snapshot') + raw = p.read_bytes() + if r['url'] != PAGES[name] or r['final_url'] != r['url'] or r['http_status'] != 200 or r['bytes'] != len(raw) or r['sha256'] != digest(raw): + raise ValueError('Invalid official source receipt') + pages[name] = raw; evidence[name] = {'url': r['url'], 'sha256': digest(raw)} + program = parse_program(pages['accepted.html'].decode('utf-8')) + return m, pages, evidence, program + + +def paper_snapshots(root, program): + root = Path(root); path = root/'manifest.json' + if root.is_symlink() or path.is_symlink(): + raise ValueError('Paper snapshot symlinks rejected') + if path.stat().st_size > 4*1024**2: + raise ValueError('Oversized paper manifest') + m = json.loads(path.read_bytes()); result = {} + for r in m['records']: + pid = r['paper_id'] + if pid not in program or pid in result or r['file'] != pid+'.html': + raise ValueError('Foreign or repeated paper snapshot') + p = root/r['file'] + if p.is_symlink() or p.stat().st_size > 2*1024**2: + raise ValueError('Unsafe paper snapshot') + receipt = {**r, 'retrieved_at': r.get('retrieved_at', m['at'])} + result[pid] = checked_html(p.read_bytes(), receipt, program[pid]['url'], 2*1024**2) + return result + + +def collect(out, *, official_snapshots, paper_snapshots_dir=None, cache=None, max_new_pages=None): + if (paper_snapshots_dir is None) == (cache is None): + raise ValueError('Choose paper snapshots or public metadata cache') + if max_new_pages is not None and (type(max_new_pages) is not int or max_new_pages < 0 or cache is None): + raise ValueError('New-page budget requires cache mode and a nonnegative integer') + out = Path(out) + if out.exists(): + raise ValueError('Output exists; inspect previous operation before retrying') + out.mkdir(parents=True) + lock = None + try: + official, pages, source_evidence, program = official_snapshot(official_snapshots) + directory = out/'evidence'; directory.mkdir() + for name, raw in pages.items(): + rel = 'evidence/'+digest(raw)+'.html'; (out/rel).write_bytes(raw); source_evidence[name]['file'] = rel + snapshots = paper_snapshots(paper_snapshots_dir, program) if paper_snapshots_dir is not None else None + client = None + if cache is not None: + cache = Path(cache); cache.mkdir(parents=True, exist_ok=True) + if cache.is_symlink() or (cache/'.collector.lock').is_symlink(): + raise ValueError('Cache lock symlinks rejected') + lock = (cache/'.collector.lock').open('a+') + fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) + client = PublicHTMLCache(cache, ORIGIN) + rows = []; missing = []; fetched = 0; reused = 0 + for pid in sorted(program, key=int): + p = program[pid] + if snapshots is not None: + response = snapshots.get(pid) + else: + response = client.cached(p['url']) + if response is not None: + reused += 1 + elif max_new_pages is None or fetched < max_new_pages: + response = client.get(p['url']); fetched += 1 + if response is None: + missing.append(pid); continue + raw, receipt = response; fields = parse_paper(raw.decode('utf-8'), p['url'], p) + rel = 'evidence/'+digest(raw)+'.html'; (out/rel).write_bytes(raw) + keyword = p['listing']['keywords_literal'] + rows.append({'source': SOURCE, 'external_id': '2026/'+pid, 'url': p['url'], + 'title': fields['title'], 'authors': fields['authors'], 'abstract': fields['abstract'], + 'identifiers': {}, 'submission_identifiers': [], 'venue': 'ECCV', 'year': 2026, + 'track': TRACK, 'status': 'accepted', 'observed_at': official['at'], + 'raw_decision': 'listed in frozen preliminary ECCV 2026 accepted program, pending publisher checks', + 'evidence_url': ACCEPTED, + 'tags': [{'namespace': SOURCE+':keyword', 'value': keyword}] if keyword else [], + 'metadata': {'official_program': p, 'public_page_fields': fields, + 'public_page_receipt': receipt, 'public_page_evidence': {'file': rel, 'sha256': digest(raw), 'url': p['url']}, + 'program_source_evidence': source_evidence, 'publication_state': 'preliminary_pending_publisher_checks', + 'original_decision_signature_verified': False, + 'program_id_semantics': 'Official program locator, not a verified original submission number or cross-platform identifier.', + 'abstract_quality': 'Public display literal retained, including broken hyphenation/missing spaces; no automatic text repair.', + 'date_semantics': 'JSON-LD dates describe the website entry, not verified paper publication or decision dates.', + 'label_semantics': 'Official keyword namespace independent of track, contribution/research type and custom labels.', + 'scope': 'Public metadata only; no protected live content, reviews, person profiles or PDFs. Requires work identity, metadata version and complete history review before import.'}}) + if len(rows) % 10 == 0: + print(json.dumps({'phase': 'public_metadata', 'records': len(rows), 'new_pages': fetched, + 'cached_pages': reused, 'expected_program_ids': 2834, 'at': datetime.now(timezone.utc).isoformat()}), flush=True) + files = [] + if rows: + path = out/'eccv-program-2026-unclassified.jsonl'; payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload) + manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ECCV', 'year': 2026, 'track': TRACK, + 'count': len(rows), 'file': str(path), 'file_sha256': digest(payload), 'list_url': ACCEPTED, + 'status': 'partial', 'missing_abstract_count': 0, 'source_expected_count': 2834} + path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n'); files.append(manifest) + report = {'status': 'collected_pending_identity_review' if rows else 'no_public_metadata_collected', + 'records': len(rows), 'files': files, 'expected_program_ids': 2834, 'program_occurrences': 2851, + 'duplicate_program_occurrences': 17, 'uncollected_program_ids': missing, + 'new_pages': fetched, 'reused_pages': reused, 'database_written': False, + 'public_review_visibility': 'not_public_per_2026_policy', 'public_rejected_decision_metadata': 'unknown', + 'scope': 'Always partial: preliminary program, unresolved track/identity/version coverage; not final whole-database deduplication.'} + part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json') + return report + except Exception as error: + (out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, + 'database_written': False, 'scope': 'Preserve this output and verified cache; resume with a new output directory after inspection.'})+'\n') + raise + finally: + if lock is not None: + lock.close() + + +def main(argv=None, *, prog=None): + parser = argparse.ArgumentParser(prog=prog, description=__doc__) + parser.add_argument('--official-snapshots', required=True) + group = parser.add_mutually_exclusive_group(required=True) + group.add_argument('--paper-snapshots') + group.add_argument('--cache') + parser.add_argument('--max-new-pages', type=int) + parser.add_argument('--out', required=True) + args = parser.parse_args(argv) + print(json.dumps(collect(args.out, official_snapshots=args.official_snapshots, + paper_snapshots_dir=args.paper_snapshots, cache=args.cache, max_new_pages=args.max_new_pages), ensure_ascii=False)) diff --git a/paper_atlas/io/html_cache.py b/paper_atlas/io/html_cache.py new file mode 100644 index 0000000..2b25649 --- /dev/null +++ b/paper_atlas/io/html_cache.py @@ -0,0 +1,106 @@ +"""Immutable, receipt-checked public HTML cache; no expiry or silent refresh.""" +from datetime import datetime, timezone +import hashlib +import json +from pathlib import Path +import time +import urllib.request +import urllib.error +from urllib.parse import urlsplit + + +def digest(raw): + return hashlib.sha256(raw).hexdigest() + + +def checked_html(raw, receipt, url, limit): + if (len(raw) > limit or receipt.get('url') != url or receipt.get('final_url') != url + or receipt.get('http_status') != 200 or receipt.get('bytes') != len(raw) + or receipt.get('sha256') != digest(raw) + or not str(receipt.get('content_type', '')).lower().startswith('text/html') + or datetime.fromisoformat(receipt['retrieved_at'].replace('Z', '+00:00')).utcoffset() is None): + raise ValueError('Invalid public HTML receipt, response or timestamp') + if any(v in raw.lower() for v in (b'just a moment', b'cf-chl-', b'botstopper')): + raise ValueError('Access challenge; stop without bypass') + return raw, receipt + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, *args, **kwargs): + return None + + +class PublicHTMLCache: + """Caller holds a single-writer lock and restricts the allowed URL paths.""" + def __init__(self, directory, origin, *, limit=2*1024**2, delay=1): + self.root = Path(directory) + self.root.mkdir(parents=True, exist_ok=True) + if self.root.is_symlink(): + raise ValueError('Cache symlinks rejected') + self.origin = origin + self.limit = limit + self.delay = delay + self.opener = urllib.request.build_opener(NoRedirect()) + + def paths(self, url): + u = urlsplit(url) + if u.scheme != 'https' or u.username or u.password or u.fragment or u.scheme+'://'+u.netloc != self.origin: + raise ValueError('Unapproved public metadata origin') + key = digest(url.encode()) + return self.root/(key+'.html'), self.root/(key+'.json'), self.root/(key+'.failed.json') + + def cached(self, url): + body, receipt, failure = self.paths(url) + parts = (body.with_suffix('.html.part'), receipt.with_suffix('.json.part')) + if any(p.is_symlink() for p in (body, receipt, failure, *parts)): + raise ValueError('Cache symlinks rejected') + if any(p.exists() for p in parts): + raise ValueError('Incomplete cache part; preserve and inspect orphan before recovery') + if failure.exists(): + raise ValueError('Previous access failure requires inspection; no automatic retry') + if body.exists() != receipt.exists(): + raise ValueError('Incomplete cache pair; preserve and inspect orphan before recovery') + if not body.exists(): + return None + if body.stat().st_size > self.limit or receipt.stat().st_size > 16*1024: + raise ValueError('Oversized cache entry') + return checked_html(body.read_bytes(), json.loads(receipt.read_bytes()), url, self.limit) + + def get(self, url): + cached = self.cached(url) + if cached is not None: + return cached + body, receipt, failure = self.paths(url) + try: + request = urllib.request.Request(url, headers={'User-Agent': 'PaperAtlas/0.1 (public scholarly metadata research)'}) + with self.opener.open(request, timeout=30) as response: + rec = {'url': url, 'final_url': response.geturl(), 'http_status': response.status, + 'content_type': response.headers.get('Content-Type', '')} + if rec['final_url'] != url or rec['http_status'] != 200 or not rec['content_type'].lower().startswith('text/html'): + raise ValueError('Unexpected metadata redirect, HTTP status or content type') + raw = response.read(self.limit+1) + if len(raw) > self.limit: + raise ValueError('Metadata response exceeds size limit') + expected = response.headers.get('Content-Length') + if expected is not None and int(expected) != len(raw): + raise ConnectionError('Incomplete metadata response') + rec.update(bytes=len(raw), sha256=digest(raw), retrieved_at=datetime.now(timezone.utc).isoformat()) + checked_html(raw, rec, url, self.limit) + except (urllib.error.HTTPError, ValueError) as error: + if isinstance(error, urllib.error.HTTPError): + status = error.code + retry_after = error.headers.get('Retry-After') if error.headers else None + error.close() + else: + status = None + retry_after = None + # Access errors remain sticky until the operator verifies availability. + failure.write_text(json.dumps({'url': url, 'http_status': status, 'retry_after': retry_after, 'error_type': type(error).__name__, + 'scope': 'Inspect access/redirect/response failure before moving this marker aside; no bypass or automatic retry.'})+'\n') + raise + part = body.with_suffix('.html.part') + part.write_bytes(raw); part.replace(body) + part = receipt.with_suffix('.json.part') + part.write_text(json.dumps(rec, indent=2)+'\n'); part.replace(receipt) + time.sleep(self.delay) + return raw, rec diff --git a/tests/fixtures/eccv2026-source.json.gz b/tests/fixtures/eccv2026-source.json.gz new file mode 100644 index 0000000..c4377e6 Binary files /dev/null and b/tests/fixtures/eccv2026-source.json.gz differ diff --git a/tests/test_eccv2026.py b/tests/test_eccv2026.py new file mode 100644 index 0000000..171e8de --- /dev/null +++ b/tests/test_eccv2026.py @@ -0,0 +1,139 @@ +import copy +import gzip +import json +from pathlib import Path +import tempfile +import unittest +from unittest.mock import patch + +from paper_atlas.collectors import eccv2026 as c +from paper_atlas.io.html_cache import PublicHTMLCache + + +class ECCV2026Tests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.bundle = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/eccv2026-source.json.gz').read_bytes())) + cls.program = c.parse_program(cls.bundle['official_pages']['accepted.html']) + + def snapshots(self, root): + official = root/'official'; paper = root/'paper'; official.mkdir(); paper.mkdir() + for name, raw in self.bundle['official_pages'].items(): + (official/name).write_bytes(raw.encode('utf-8')) + (official/'manifest.json').write_bytes(self.bundle['official_manifest'].encode('utf-8')) + for pid, raw in self.bundle['paper_pages'].items(): + (paper/(pid+'.html')).write_bytes(raw.encode('utf-8')) + (paper/'manifest.json').write_text(json.dumps(self.bundle['paper_manifest'])) + return official, paper + + def test_complete_program_and_duplicate_occurrences_match_gold(self): + self.assertEqual(len(self.program), 2834) + self.assertEqual(sum(len(p['row_ordinals']) for p in self.program.values()), 2851) + for ordinal, gold in enumerate(self.bundle['inventory']['records'], 1): + item = self.program[gold['paper_id']] + self.assertEqual(item['listing'], {k: gold[k] for k in ('title','authors_literal','keywords_literal','presentation_literal')}) + self.assertIn(ordinal, item['row_ordinals']) + + def test_conflicting_duplicate_not_silently_folded(self): + title = 'Transferability Between Understanding and Generation in Unified Multimodal Models' + with self.assertRaisesRegex(ValueError, 'Conflicting'): + c.parse_program(self.bundle['official_pages']['accepted.html'].replace('href="/virtual/2026/poster/3137">'+title, 'href="/virtual/2026/poster/3137">Conflicting title', 1)) + + def test_missing_member_and_changed_provisional_status_rejected(self): + html = self.bundle['official_pages']['accepted.html'] + for changed in [html.replace('/virtual/2026/poster/4048', '/virtual/2024/poster/4048', 1), + html.replace('This paper list is preliminary pending publisher checks.', 'Final archival papers.')]: + with self.assertRaises(ValueError): + c.parse_program(changed) + + def test_public_page_fields_match_independent_sample_review(self): + for gold in self.bundle['paper_gold']['records']: + pid = gold['paper_id']; fields = c.parse_paper(self.bundle['paper_pages'][pid], gold['url'], self.program[pid]) + self.assertEqual(fields['title'], gold['title']); self.assertEqual(fields['authors'], gold['authors']) + self.assertEqual(fields['abstract'], gold['abstract_literal']); self.assertFalse(fields['review_flags']) + self.assertIn('sig-nal', c.parse_paper(self.bundle['paper_pages']['4048'], self.program['4048']['url'], self.program['4048'])['abstract']) + + def test_foreign_canonical_and_inconsistent_jsonld_author_rejected(self): + raw = self.bundle['paper_pages']['4048']; url = self.program['4048']['url'] + for changed in [raw.replace(url, 'https://example.org/other'), raw.replace('Suhyeon Ha', 'Other Author', 1)]: + with self.assertRaises(ValueError): + c.parse_paper(changed, url, self.program['4048']) + + def test_program_and_page_versions_flagged_without_aliases(self): + program = copy.deepcopy(self.program['4048']); program['listing']['title'] = 'Earlier source title' + fields = c.parse_paper(self.bundle['paper_pages']['4048'], program['url'], program) + self.assertEqual(fields['review_flags'], ['program_and_page_title_versions_differ_requires_review']) + self.assertEqual(fields['title'], self.program['4048']['listing']['title']) + + def test_partial_snapshot_collection_has_no_inferred_submission_ids(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); official, paper = self.snapshots(root) + report = c.collect(root/'out', official_snapshots=official, paper_snapshots_dir=paper) + self.assertEqual(report['records'], 3); self.assertEqual(len(report['uncollected_program_ids']), 2831) + self.assertEqual(report['public_rejected_decision_metadata'], 'unknown'); self.assertFalse(report['database_written']) + manifest, = report['files']; self.assertEqual(manifest['status'], 'partial'); self.assertEqual(manifest['source_expected_count'], 2834) + rows = [json.loads(l) for l in Path(manifest['file']).read_text().splitlines()] + self.assertEqual({r['external_id'] for r in rows}, {'2026/4048','2026/3137','2026/3652'}) + self.assertTrue(all(not r['identifiers'] and not r['submission_identifiers'] and r['track']=='program_unclassified' for r in rows)) + self.assertTrue(all(r['metadata']['publication_state']=='preliminary_pending_publisher_checks' and not r['metadata']['original_decision_signature_verified'] for r in rows)) + self.assertTrue(all(t['namespace']=='eccv_program:keyword' for r in rows for t in r['tags'])) + with self.assertRaisesRegex(ValueError, 'Output exists'): + c.collect(root/'out', official_snapshots=official, paper_snapshots_dir=paper) + + def test_zero_network_budget_reuses_immutable_cache_only(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); official, _ = self.snapshots(root); client = PublicHTMLCache(root/'cache', c.ORIGIN) + for r in self.bundle['paper_manifest']['records']: + body, receipt, _ = client.paths(r['url']); body.write_text(self.bundle['paper_pages'][r['paper_id']]); receipt.write_text(json.dumps({**r,'retrieved_at':self.bundle['paper_manifest']['at']})) + with patch.object(PublicHTMLCache, 'get', side_effect=AssertionError('network forbidden')): + report = c.collect(root/'out', official_snapshots=official, cache=root/'cache', max_new_pages=0) + self.assertEqual((report['records'],report['new_pages'],report['reused_pages']), (3,0,3)) + + def test_tampered_snapshot_preserves_failure_without_success(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); official, paper = self.snapshots(root); (paper/'4048.html').write_text('tampered') + with self.assertRaises(ValueError): + c.collect(root/'out', official_snapshots=official, paper_snapshots_dir=paper) + self.assertTrue((root/'out/failed.json').exists()); self.assertFalse((root/'out/collection-report.json').exists()) + + def test_structured_math_markup_requires_explicit_review(self): + raw = self.bundle['paper_pages']['4048'] + self.assertIn('<p>Proactive tamper', raw) + fields = c.parse_paper(raw.replace('<p>Proactive tamper', '<p>10<sup>6</sup> Proactive tamper', 1), self.program['4048']['url'], self.program['4048']) + self.assertIn('structured_abstract_markup_requires_review', fields['review_flags']) + self.assertEqual(fields['abstract_markup_tags'], ['sup']) + + def test_fixture_preserves_exact_http_bytes_including_crlf(self): + import hashlib + for receipt in json.loads(self.bundle['official_manifest'])['sources']: + raw = self.bundle['official_pages'][receipt['file']].encode('utf-8') + self.assertEqual(len(raw), receipt['bytes']) + self.assertEqual(hashlib.sha256(raw).hexdigest(), receipt['sha256']) + for receipt in self.bundle['paper_manifest']['records']: + raw = self.bundle['paper_pages'][receipt['paper_id']].encode('utf-8') + self.assertEqual(hashlib.sha256(raw).hexdigest(), receipt['sha256']) + + def test_real_double_encoded_author_requires_exact_visible_agreement(self): + import hashlib + sample = self.bundle['encoding_sample']; pid = sample['paper_id']; raw = sample['html'] + self.assertEqual(hashlib.sha256(raw.encode('utf-8')).hexdigest(), sample['receipt']['sha256']) + self.assertEqual(sample['receipt']['sha256'], 'b575318fa569e83eea5e96f937dd60f078fbed482877318013f797c16a02ff09') + fields = c.parse_paper(raw, sample['receipt']['url'], self.program[pid]) + self.assertEqual(fields['authors'], sample['gold']['authors']) + self.assertEqual(fields['jsonld_raw_text']['author_names'][0], 'Zi&#x27;ang Lu') + self.assertIn('jsonld_display_text_normalized_requires_review', fields['review_flags']) + # Real changes to the author or an extra encoding layer remain errors. + for replacement in ('Other Author', 'Zi&amp;#x27;ang Lu'): + with self.assertRaisesRegex(ValueError, 'authors disagree'): + c.parse_paper(raw.replace('Zi&#x27;ang Lu', replacement), sample['receipt']['url'], self.program[pid]) + + def test_real_already_agreeing_literal_title_is_not_decoded_again(self): + import hashlib + sample = self.bundle['literal_title_sample']; pid = sample['paper_id']; raw = sample['html'] + self.assertEqual(hashlib.sha256(raw.encode('utf-8')).hexdigest(), 'c208525ad5f02b4ca274dff7e24a0392dabdc55bb3620497079a84df81a02a72') + fields = c.parse_paper(raw, sample['receipt']['url'], self.program[pid]) + self.assertIn('&', fields['title']) + self.assertEqual(fields['title'], sample['gold']['visible_title']) + self.assertEqual(fields['jsonld_raw_text']['name'], sample['gold']['jsonld_title_raw']) + self.assertNotEqual(fields['title'], sample['gold']['blind_double_decoded_title']) + self.assertNotIn('jsonld_display_text_normalized_requires_review', fields['review_flags']) diff --git a/tests/test_html_cache.py b/tests/test_html_cache.py new file mode 100644 index 0000000..abdf73d --- /dev/null +++ b/tests/test_html_cache.py @@ -0,0 +1,90 @@ +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import tempfile +import unittest +from unittest.mock import MagicMock +import urllib.error + +from paper_atlas.io.html_cache import PublicHTMLCache, digest, NoRedirect + +URL = 'https://eccv.ecva.net/virtual/2026/poster/4048' + + +class PublicHTMLCacheTests(unittest.TestCase): + def receipt(self, raw): + return {'url':URL,'final_url':URL,'http_status':200,'content_type':'text/html; charset=utf-8', + 'bytes':len(raw),'sha256':digest(raw),'retrieved_at':datetime.now(timezone.utc).isoformat()} + + def test_old_cache_reused_without_network_or_expiry(self): + with tempfile.TemporaryDirectory() as tmp: + c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0); raw = b'<html>metadata</html>' + body, receipt, _ = c.paths(URL); body.write_bytes(raw); receipt.write_text(json.dumps(self.receipt(raw))); os.utime(body,(0,0)) + c.opener = MagicMock(); self.assertEqual(c.get(URL)[0], raw); c.opener.open.assert_not_called() + + def test_pair_tampering_and_orphan_parts_fail_without_refetch(self): + for kind in ('hash','orphan','part'): + with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp: + c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0); body, receipt, _ = c.paths(URL); body.write_bytes(b'original') + if kind == 'hash': receipt.write_text(json.dumps(self.receipt(b'different'))) + if kind == 'part': body.unlink(); body.with_suffix('.html.part').write_bytes(b'partial') + c.opener = MagicMock() + with self.assertRaises(ValueError): c.get(URL) + c.opener.open.assert_not_called() + + def test_http_access_failure_sticky_without_automatic_retry(self): + with tempfile.TemporaryDirectory() as tmp: + c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0); c.opener = MagicMock() + c.opener.open.side_effect = urllib.error.HTTPError(URL,403,'Forbidden',{},None) + with self.assertRaises(urllib.error.HTTPError): c.get(URL) + with self.assertRaisesRegex(ValueError,'Previous access'): c.get(URL) + self.assertEqual(c.opener.open.call_count,1) + + def test_redirect_nonhtml_challenge_and_oversize_rejected(self): + for kind in ('redirect','type','challenge','size'): + with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp: + c = PublicHTMLCache(tmp,'https://eccv.ecva.net',limit=100,delay=0); c.opener=MagicMock(); r=c.opener.open.return_value.__enter__.return_value + r.status=200; r.geturl.return_value=URL; r.headers={'Content-Type':'text/html'}; r.read.return_value=b'<html>metadata</html>' + if kind=='redirect':r.geturl.return_value='https://example.org/login' + if kind=='type':r.headers={'Content-Type':'application/pdf'} + if kind=='challenge':r.read.return_value=b'<title>Just a moment' + if kind=='size':r.read.return_value=b'x'*101 + with self.assertRaises(ValueError):c.get(URL) + self.assertFalse(c.paths(URL)[0].exists()) + self.assertIsNone(NoRedirect().redirect_request(None,None,None,None,None,None)) + + def test_valid_response_has_atomic_body_and_receipt(self): + with tempfile.TemporaryDirectory() as tmp: + c=PublicHTMLCache(tmp,'https://eccv.ecva.net',delay=0);c.opener=MagicMock();r=c.opener.open.return_value.__enter__.return_value;r.status=200;r.geturl.return_value=URL;r.headers={'Content-Type':'text/html','Content-Length':'4'};r.read.return_value=b'data' + raw, receipt=c.get(URL);self.assertEqual(raw,b'data');self.assertEqual(receipt['sha256'],digest(raw));self.assertEqual(c.cached(URL),(raw,receipt));self.assertFalse(list(Path(tmp).glob('*.part'))) + + def test_foreign_origin_and_cache_symlink_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + c=PublicHTMLCache(tmp,'https://eccv.ecva.net',delay=0) + with self.assertRaises(ValueError):c.get('https://example.org/other') + body,_,_=c.paths(URL);target=Path(tmp)/'outside';target.write_bytes(b'other');body.symlink_to(target) + with self.assertRaises(ValueError):c.get(URL) + + def test_successful_fetch_publishes_receipt_and_reuses_without_refetch(self): + with tempfile.TemporaryDirectory() as tmp: + c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0) + c.opener = MagicMock(); response = c.opener.open.return_value.__enter__.return_value + response.status = 200; response.geturl.return_value = URL + raw = b'public metadata' + response.headers = {'Content-Type':'text/html','Content-Length':str(len(raw))}; response.read.return_value = raw + self.assertEqual(c.get(URL)[0], raw); self.assertEqual(c.get(URL)[0], raw) + self.assertEqual(c.opener.open.call_count, 1) + body, receipt, _ = c.paths(URL) + self.assertTrue(body.exists()); self.assertTrue(receipt.exists()) + self.assertFalse(body.with_suffix('.html.part').exists()) + self.assertFalse(receipt.with_suffix('.json.part').exists()) + + def test_changed_url_is_rejected_before_reading_body(self): + with tempfile.TemporaryDirectory() as tmp: + c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0) + c.opener = MagicMock(); response = c.opener.open.return_value.__enter__.return_value + response.status = 200; response.geturl.return_value = 'https://example.org/login' + response.headers = {'Content-Type':'text/html'} + with self.assertRaises(ValueError): c.get(URL) + response.read.assert_not_called()