fix: preserve already matching ECCV literal text before entity decoding
Metadata integrity checks / metadata (push) Successful in 3m8s
Metadata integrity checks / metadata (push) Successful in 3m8s
This commit is contained in:
@@ -61,16 +61,22 @@ jobs:
|
||||
eccv_encoding = eccv_fixture['encoding_sample']
|
||||
(eccv_papers/(eccv_encoding['paper_id']+'.html')).write_bytes(eccv_encoding['html'].encode('utf-8'))
|
||||
eccv_fixture['paper_manifest']['records'].append({**eccv_encoding['receipt'], 'paper_id': eccv_encoding['paper_id'], 'file': eccv_encoding['paper_id']+'.html'})
|
||||
eccv_literal = eccv_fixture['literal_title_sample']
|
||||
(eccv_papers/(eccv_literal['paper_id']+'.html')).write_bytes(eccv_literal['html'].encode('utf-8'))
|
||||
eccv_fixture['paper_manifest']['records'].append({**eccv_literal['receipt'], 'paper_id': eccv_literal['paper_id'], 'file': eccv_literal['paper_id']+'.html'})
|
||||
(eccv_papers/'manifest.json').write_text(json.dumps(eccv_fixture['paper_manifest']))
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','eccv2026-program','--official-snapshots',str(eccv_official),'--paper-snapshots',str(eccv_papers),'--out',str(root/'installed-eccv2026')],cwd=outside,check=True)
|
||||
eccv_report = json.loads((root/'installed-eccv2026/collection-report.json').read_text())
|
||||
assert eccv_report['records'] == 4 and eccv_report['expected_program_ids'] == 2834
|
||||
assert eccv_report['duplicate_program_occurrences'] == 17 and len(eccv_report['uncollected_program_ids']) == 2830
|
||||
assert eccv_report['records'] == 5 and eccv_report['expected_program_ids'] == 2834
|
||||
assert eccv_report['duplicate_program_occurrences'] == 17 and len(eccv_report['uncollected_program_ids']) == 2829
|
||||
assert not eccv_report['database_written'] and eccv_report['files'][0]['track'] == 'program_unclassified'
|
||||
eccv_rows = [json.loads(line) for line in pathlib.Path(eccv_report['files'][0]['file']).read_text().splitlines()]
|
||||
eccv_encoded_row = next(row for row in eccv_rows if row['external_id'] == '2026/4199')
|
||||
assert eccv_encoded_row['authors'] == eccv_encoding['gold']['authors']
|
||||
assert 'jsonld_display_text_normalized_requires_review' in eccv_encoded_row['metadata']['public_page_fields']['review_flags']
|
||||
eccv_literal_row = next(row for row in eccv_rows if row['external_id'] == '2026/3844')
|
||||
assert eccv_literal_row['title'] == eccv_literal['gold']['visible_title']
|
||||
assert '&' in eccv_literal_row['title']
|
||||
icaps_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icaps2026-source.json.gz').read_bytes()))
|
||||
icaps_snapshots = root/'icaps2026-snapshots'
|
||||
icaps_snapshots.mkdir()
|
||||
|
||||
@@ -24,7 +24,7 @@ paper-atlas collect eccv2026-program \
|
||||
|
||||
公开页的 canonical URL、可见标题/作者和 JSON-LD 相互核对;页面版本与冻结节目文字有差异时分别保留并标记待审阅,不制造人名别名。JSON-LD 日期只是网站条目时间,不当作论文发表日期或原始决定日期。摘要保留公开页 DOM 文本(规范空白),包括源头的断词、缺空格;不自动修补。原始 HTML 单独留存;遇到公式、上下标或图片结构会标记待审阅,不能把扁平化文本当作无损数学表达。页面上的登录提示对应直播内容,不用于访问受保护内容。
|
||||
|
||||
第 4199 页的 JSON-LD 作者姓名存在双重 HTML 实体编码。解析器最多解码两层文本实体并规范空白,且仍要求标题、完整作者序列与可见页面精确一致;真实不一致或更多编码层次仍停止。`jsonld_raw_text` 保留原始结构化标题/姓名,发生文本规范化时增加 `jsonld_display_text_normalized_requires_review`,不生成姓名别名。恢复必须使用校验过的修复版本、新输出目录和原缓存;保留旧失败目录,不重新请求已有成功页面。
|
||||
第 4199 页的 JSON-LD 作者姓名存在双重 HTML 实体编码;第 3844 页的可见标题和 JSON-LD 则已经一致地包含字面 `&`。解析器优先保留已经一致的文字,仅在字段不一致时尝试最多两层实体解码来核对可见文本,不把已经一致的标题再次解码。完整作者序列仍须精确一致,真实差异或更多编码层次仍停止。`jsonld_raw_text` 保留原始结构化标题/姓名,发生文本规范化时增加 `jsonld_display_text_normalized_requires_review`,不生成姓名别名。恢复必须使用校验过的修复版本、新输出目录和原缓存;保留旧失败目录,不重新请求已有成功页面。
|
||||
|
||||
官方关键词保存在独立 `eccv_program:keyword` 命名空间,并保留原始单元格和证据。关键词、展示形式、贡献/研究类型、自定义标签相互独立,投稿 track 暂用 `program_unclassified`,不按主题推断 main、findings、workshop 或 position。
|
||||
|
||||
|
||||
@@ -80,18 +80,29 @@ def parse_paper(raw, url, program):
|
||||
or any(not isinstance(a, dict) or a.get('@type') != 'Person' or not isinstance(a.get('name'), str)
|
||||
or not clean(a['name']) or '@' in a['name'] for a in authors)):
|
||||
raise ValueError('Public visible/structured metadata identity or edition disagrees')
|
||||
# The real poster 4199 double-encodes an apostrophe in JSON-LD. Decode a
|
||||
# bounded number of text-entity layers, then require exact visible agreement.
|
||||
# Keep literal structured fields as evidence; this does not create aliases.
|
||||
def display_text(value):
|
||||
# Prefer source text that already agrees. Poster 3844 intentionally displays
|
||||
# literal '&', while 4199 double-encodes an author's apostrophe in JSON-LD.
|
||||
# Decode only to test agreement with visible text, never to rewrite it.
|
||||
def agrees_with_display(value, expected):
|
||||
if clean(value) == expected:
|
||||
return True
|
||||
for _ in range(2):
|
||||
value = unescape(value)
|
||||
return clean(value)
|
||||
if clean(value) == expected:
|
||||
return True
|
||||
return False
|
||||
|
||||
raw_names = [a['name'] for a in authors]
|
||||
names = [display_text(name) for name in raw_names]
|
||||
if display_text(ld['name']) != title:
|
||||
names = [clean(name) for name in raw_names]
|
||||
if not agrees_with_display(ld['name'], title):
|
||||
raise ValueError('Public visible and structured titles disagree')
|
||||
if ' ⋅ '.join(names) != authors_literal:
|
||||
visible_names = authors_literal.split(' ⋅ ')
|
||||
if len(visible_names) != len(raw_names) or any(
|
||||
not agrees_with_display(raw_name, visible_name)
|
||||
for raw_name, visible_name in zip(raw_names, visible_names)):
|
||||
raise ValueError('Complete visible and structured authors disagree')
|
||||
names = visible_names
|
||||
if any('@' in name for name in names) or ' ⋅ '.join(names) != authors_literal:
|
||||
raise ValueError('Complete visible and structured authors disagree')
|
||||
flags = ['structured_abstract_markup_requires_review'] if markup else []
|
||||
|
||||
BIN
Binary file not shown.
@@ -126,3 +126,14 @@ class ECCV2026Tests(unittest.TestCase):
|
||||
for replacement in ('Other Author', 'Zi'ang Lu'):
|
||||
with self.assertRaisesRegex(ValueError, 'authors disagree'):
|
||||
c.parse_paper(raw.replace('Zi'ang Lu', replacement), sample['receipt']['url'], self.program[pid])
|
||||
|
||||
def test_real_already_agreeing_literal_title_is_not_decoded_again(self):
|
||||
import hashlib
|
||||
sample = self.bundle['literal_title_sample']; pid = sample['paper_id']; raw = sample['html']
|
||||
self.assertEqual(hashlib.sha256(raw.encode('utf-8')).hexdigest(), 'c208525ad5f02b4ca274dff7e24a0392dabdc55bb3620497079a84df81a02a72')
|
||||
fields = c.parse_paper(raw, sample['receipt']['url'], self.program[pid])
|
||||
self.assertIn('&', fields['title'])
|
||||
self.assertEqual(fields['title'], sample['gold']['visible_title'])
|
||||
self.assertEqual(fields['jsonld_raw_text']['name'], sample['gold']['jsonld_title_raw'])
|
||||
self.assertNotEqual(fields['title'], sample['gold']['blind_double_decoded_title'])
|
||||
self.assertNotIn('jsonld_display_text_normalized_requires_review', fields['review_flags'])
|
||||
|
||||
Reference in New Issue
Block a user