Handle section front matter and preserve abstract paragraph boundaries
Metadata integrity checks / metadata (push) Failing after 2m3s

This commit is contained in:
2026-11-10 00:00:30 +08:00
parent c06981b244
commit 7facde6c94
4 changed files with 17 additions and 5 deletions
+1 -1
View File
@@ -32,4 +32,4 @@ paper-atlas collect springer-selected \
## 固定测试资料
`tests/fixtures/springer-aistats1993-fixture.json.gz` 保存 4 份范围/目录 HTML、49 份章节 HTML、原始请求回执及先于本采集器独立生成的预期元数据,SHA256 为 `d9ad54008598e69a92d51c7bfccb072c8ee2f97f40ae436e7f795c59c9cb8f02`。它是压缩的公开元数据测试资料,不包含论文 PDF。测试覆盖全部真实章节字段、缺失和漂移输入、来源安全边界、幂等导入及旧拒稿历史保留。构建和测试只在已授权远程 runner 执行。
`tests/fixtures/springer-aistats1993-fixture.json.gz` 保存 4 份范围/目录 HTML、49 份章节 HTML、原始请求回执及独立原始字段审计的预期元数据,SHA256 为 `94028fd37080f9d8027e5790e379cc151ee6eab17ed0d1dcc0f59e6e75f9d4d0`。摘要按 HTML 段落边界分隔;未修改原始 HTML 或回执。它是压缩的公开元数据测试资料,不包含论文 PDF。测试覆盖全部真实章节字段、缺失和漂移输入、来源安全边界、幂等导入及旧拒稿历史保留。构建和测试只在已授权远程 runner 执行。
+13 -3
View File
@@ -70,13 +70,22 @@ def read_page(index, url):
return raw, evidence
def parse_toc(html):
def parse_toc(html, *, excluded_materials=None):
entries = {}
for node in Document(html).root.walk():
if node.tag != 'li' or node.attrs.get('data-test') != 'chapter':
continue
links = [n for n in node.walk() if n.tag == 'a' and re.fullmatch(
r'/chapter/'+re.escape(BOOK)+r'_[1-9]\d*', n.attrs.get('href', ''))]
if not links:
headings = [clean(n.text()) for n in node.walk() if n.tag in ('h3','h4')]
frontmatter = [n.attrs.get('href','') for n in node.walk()
if n.tag == 'a' and n.attrs.get('data-test') == 'front-matter-pdf']
if headings == ['Front Matter'] and len(frontmatter) == 1:
if excluded_materials is not None:
excluded_materials.append({'title':'Front Matter', 'url':frontmatter[0],
'reason':'Section front matter without a chapter DOI; not an authored chapter.'})
continue
if len(links) != 1:
raise ValueError('Chapter entry lacks one reviewed-book DOI')
link, = links
@@ -158,11 +167,11 @@ def collect(out, *, toc_snapshots, chapter_snapshots):
if hashlib.sha256(history.encode()).hexdigest() != HISTORY_SHA:
raise ValueError('Reviewed selected-volume relationship evidence changed')
scope_evidence = [history_evidence]
titles = {}
titles, excluded = {}, []
for page in range(1, 4):
html, evidence = read(toc_index, BOOK_URL+('?page='+str(page) if page > 1 else ''))
scope_evidence.append(evidence)
entries = parse_toc(html)
entries = parse_toc(html, excluded_materials=excluded)
if titles.keys() & entries.keys():
raise ValueError('DOI repeated across TOC pages')
titles.update(entries)
@@ -197,6 +206,7 @@ def collect(out, *, toc_snapshots, chapter_snapshots):
'count':len(records), 'file':str(out/path.name),
'file_sha256':hashlib.sha256(path.read_bytes()).hexdigest(),
'list_url':BOOK_URL, 'status':'partial',
'excluded_materials':excluded,
'missing_abstract_count':sum(not r['abstract'] for r in records)}
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)+'\n')
report = {'status':'collected_pending_identity_review', 'records':len(records),
Binary file not shown.
+3 -1
View File
@@ -21,7 +21,7 @@ class SpringerSelectedTests(unittest.TestCase):
@classmethod
def setUpClass(cls):
raw = FIXTURE.read_bytes()
if hashlib.sha256(raw).hexdigest() != 'd9ad54008598e69a92d51c7bfccb072c8ee2f97f40ae436e7f795c59c9cb8f02':
if hashlib.sha256(raw).hexdigest() != '94028fd37080f9d8027e5790e379cc151ee6eab17ed0d1dcc0f59e6e75f9d4d0':
raise AssertionError('Fixed source fixture differs')
cls.fixture = json.loads(gzip.decompress(raw))
@@ -42,6 +42,8 @@ class SpringerSelectedTests(unittest.TestCase):
self.assertEqual(len(rows), 49)
self.assertEqual(manifest['status'], 'partial')
self.assertEqual(manifest['missing_abstract_count'], 0)
self.assertTrue(manifest['excluded_materials'])
self.assertTrue(all(x['title'] == 'Front Matter' for x in manifest['excluded_materials']))
for row, expected in zip(rows, self.fixture['expected']):
for field in ['title','abstract']:
self.assertEqual(row[field], expected[field])