diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index f051978..e8d1b42 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -61,16 +61,22 @@ jobs: eccv_encoding = eccv_fixture['encoding_sample'] (eccv_papers/(eccv_encoding['paper_id']+'.html')).write_bytes(eccv_encoding['html'].encode('utf-8')) eccv_fixture['paper_manifest']['records'].append({**eccv_encoding['receipt'], 'paper_id': eccv_encoding['paper_id'], 'file': eccv_encoding['paper_id']+'.html'}) + eccv_literal = eccv_fixture['literal_title_sample'] + (eccv_papers/(eccv_literal['paper_id']+'.html')).write_bytes(eccv_literal['html'].encode('utf-8')) + eccv_fixture['paper_manifest']['records'].append({**eccv_literal['receipt'], 'paper_id': eccv_literal['paper_id'], 'file': eccv_literal['paper_id']+'.html'}) (eccv_papers/'manifest.json').write_text(json.dumps(eccv_fixture['paper_manifest'])) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','eccv2026-program','--official-snapshots',str(eccv_official),'--paper-snapshots',str(eccv_papers),'--out',str(root/'installed-eccv2026')],cwd=outside,check=True) eccv_report = json.loads((root/'installed-eccv2026/collection-report.json').read_text()) - assert eccv_report['records'] == 4 and eccv_report['expected_program_ids'] == 2834 - assert eccv_report['duplicate_program_occurrences'] == 17 and len(eccv_report['uncollected_program_ids']) == 2830 + assert eccv_report['records'] == 5 and eccv_report['expected_program_ids'] == 2834 + assert eccv_report['duplicate_program_occurrences'] == 17 and len(eccv_report['uncollected_program_ids']) == 2829 assert not eccv_report['database_written'] and eccv_report['files'][0]['track'] == 'program_unclassified' eccv_rows = [json.loads(line) for line in pathlib.Path(eccv_report['files'][0]['file']).read_text().splitlines()] eccv_encoded_row = next(row for row in eccv_rows if row['external_id'] == '2026/4199') assert eccv_encoded_row['authors'] == eccv_encoding['gold']['authors'] assert 'jsonld_display_text_normalized_requires_review' in eccv_encoded_row['metadata']['public_page_fields']['review_flags'] + eccv_literal_row = next(row for row in eccv_rows if row['external_id'] == '2026/3844') + assert eccv_literal_row['title'] == eccv_literal['gold']['visible_title'] + assert '&' in eccv_literal_row['title'] icaps_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icaps2026-source.json.gz').read_bytes())) icaps_snapshots = root/'icaps2026-snapshots' icaps_snapshots.mkdir() diff --git a/docs/eccv2026-program.md b/docs/eccv2026-program.md index 34a6eef..4d10616 100644 --- a/docs/eccv2026-program.md +++ b/docs/eccv2026-program.md @@ -24,7 +24,7 @@ paper-atlas collect eccv2026-program \ 公开页的 canonical URL、可见标题/作者和 JSON-LD 相互核对;页面版本与冻结节目文字有差异时分别保留并标记待审阅,不制造人名别名。JSON-LD 日期只是网站条目时间,不当作论文发表日期或原始决定日期。摘要保留公开页 DOM 文本(规范空白),包括源头的断词、缺空格;不自动修补。原始 HTML 单独留存;遇到公式、上下标或图片结构会标记待审阅,不能把扁平化文本当作无损数学表达。页面上的登录提示对应直播内容,不用于访问受保护内容。 -第 4199 页的 JSON-LD 作者姓名存在双重 HTML 实体编码。解析器最多解码两层文本实体并规范空白,且仍要求标题、完整作者序列与可见页面精确一致;真实不一致或更多编码层次仍停止。`jsonld_raw_text` 保留原始结构化标题/姓名,发生文本规范化时增加 `jsonld_display_text_normalized_requires_review`,不生成姓名别名。恢复必须使用校验过的修复版本、新输出目录和原缓存;保留旧失败目录,不重新请求已有成功页面。 +第 4199 页的 JSON-LD 作者姓名存在双重 HTML 实体编码;第 3844 页的可见标题和 JSON-LD 则已经一致地包含字面 `&`。解析器优先保留已经一致的文字,仅在字段不一致时尝试最多两层实体解码来核对可见文本,不把已经一致的标题再次解码。完整作者序列仍须精确一致,真实差异或更多编码层次仍停止。`jsonld_raw_text` 保留原始结构化标题/姓名,发生文本规范化时增加 `jsonld_display_text_normalized_requires_review`,不生成姓名别名。恢复必须使用校验过的修复版本、新输出目录和原缓存;保留旧失败目录,不重新请求已有成功页面。 官方关键词保存在独立 `eccv_program:keyword` 命名空间,并保留原始单元格和证据。关键词、展示形式、贡献/研究类型、自定义标签相互独立,投稿 track 暂用 `program_unclassified`,不按主题推断 main、findings、workshop 或 position。 diff --git a/paper_atlas/collectors/eccv2026.py b/paper_atlas/collectors/eccv2026.py index e68d3ae..f8e8c30 100644 --- a/paper_atlas/collectors/eccv2026.py +++ b/paper_atlas/collectors/eccv2026.py @@ -80,18 +80,29 @@ def parse_paper(raw, url, program): or any(not isinstance(a, dict) or a.get('@type') != 'Person' or not isinstance(a.get('name'), str) or not clean(a['name']) or '@' in a['name'] for a in authors)): raise ValueError('Public visible/structured metadata identity or edition disagrees') - # The real poster 4199 double-encodes an apostrophe in JSON-LD. Decode a - # bounded number of text-entity layers, then require exact visible agreement. - # Keep literal structured fields as evidence; this does not create aliases. - def display_text(value): + # Prefer source text that already agrees. Poster 3844 intentionally displays + # literal '&', while 4199 double-encodes an author's apostrophe in JSON-LD. + # Decode only to test agreement with visible text, never to rewrite it. + def agrees_with_display(value, expected): + if clean(value) == expected: + return True for _ in range(2): value = unescape(value) - return clean(value) + if clean(value) == expected: + return True + return False raw_names = [a['name'] for a in authors] - names = [display_text(name) for name in raw_names] - if display_text(ld['name']) != title: + names = [clean(name) for name in raw_names] + if not agrees_with_display(ld['name'], title): raise ValueError('Public visible and structured titles disagree') + if ' ⋅ '.join(names) != authors_literal: + visible_names = authors_literal.split(' ⋅ ') + if len(visible_names) != len(raw_names) or any( + not agrees_with_display(raw_name, visible_name) + for raw_name, visible_name in zip(raw_names, visible_names)): + raise ValueError('Complete visible and structured authors disagree') + names = visible_names if any('@' in name for name in names) or ' ⋅ '.join(names) != authors_literal: raise ValueError('Complete visible and structured authors disagree') flags = ['structured_abstract_markup_requires_review'] if markup else [] diff --git a/tests/fixtures/eccv2026-source.json.gz b/tests/fixtures/eccv2026-source.json.gz index b547829..c4377e6 100644 Binary files a/tests/fixtures/eccv2026-source.json.gz and b/tests/fixtures/eccv2026-source.json.gz differ diff --git a/tests/test_eccv2026.py b/tests/test_eccv2026.py index 63fdb91..171e8de 100644 --- a/tests/test_eccv2026.py +++ b/tests/test_eccv2026.py @@ -126,3 +126,14 @@ class ECCV2026Tests(unittest.TestCase): for replacement in ('Other Author', 'Zi'ang Lu'): with self.assertRaisesRegex(ValueError, 'authors disagree'): c.parse_paper(raw.replace('Zi'ang Lu', replacement), sample['receipt']['url'], self.program[pid]) + + def test_real_already_agreeing_literal_title_is_not_decoded_again(self): + import hashlib + sample = self.bundle['literal_title_sample']; pid = sample['paper_id']; raw = sample['html'] + self.assertEqual(hashlib.sha256(raw.encode('utf-8')).hexdigest(), 'c208525ad5f02b4ca274dff7e24a0392dabdc55bb3620497079a84df81a02a72') + fields = c.parse_paper(raw, sample['receipt']['url'], self.program[pid]) + self.assertIn('&', fields['title']) + self.assertEqual(fields['title'], sample['gold']['visible_title']) + self.assertEqual(fields['jsonld_raw_text']['name'], sample['gold']['jsonld_title_raw']) + self.assertNotEqual(fields['title'], sample['gold']['blind_double_decoded_title']) + self.assertNotIn('jsonld_display_text_normalized_requires_review', fields['review_flags'])