Handle section front matter and preserve abstract paragraph boundaries
Metadata integrity checks / metadata (push) Failing after 2m3s
Metadata integrity checks / metadata (push) Failing after 2m3s
This commit is contained in:
@@ -32,4 +32,4 @@ paper-atlas collect springer-selected \
|
||||
|
||||
## 固定测试资料
|
||||
|
||||
`tests/fixtures/springer-aistats1993-fixture.json.gz` 保存 4 份范围/目录 HTML、49 份章节 HTML、原始请求回执及先于本采集器独立生成的预期元数据,SHA256 为 `d9ad54008598e69a92d51c7bfccb072c8ee2f97f40ae436e7f795c59c9cb8f02`。它是压缩的公开元数据测试资料,不包含论文 PDF。测试覆盖全部真实章节字段、缺失和漂移输入、来源安全边界、幂等导入及旧拒稿历史保留。构建和测试只在已授权远程 runner 执行。
|
||||
`tests/fixtures/springer-aistats1993-fixture.json.gz` 保存 4 份范围/目录 HTML、49 份章节 HTML、原始请求回执及独立原始字段审计的预期元数据,SHA256 为 `94028fd37080f9d8027e5790e379cc151ee6eab17ed0d1dcc0f59e6e75f9d4d0`。摘要按 HTML 段落边界分隔;未修改原始 HTML 或回执。它是压缩的公开元数据测试资料,不包含论文 PDF。测试覆盖全部真实章节字段、缺失和漂移输入、来源安全边界、幂等导入及旧拒稿历史保留。构建和测试只在已授权远程 runner 执行。
|
||||
|
||||
@@ -70,13 +70,22 @@ def read_page(index, url):
|
||||
return raw, evidence
|
||||
|
||||
|
||||
def parse_toc(html):
|
||||
def parse_toc(html, *, excluded_materials=None):
|
||||
entries = {}
|
||||
for node in Document(html).root.walk():
|
||||
if node.tag != 'li' or node.attrs.get('data-test') != 'chapter':
|
||||
continue
|
||||
links = [n for n in node.walk() if n.tag == 'a' and re.fullmatch(
|
||||
r'/chapter/'+re.escape(BOOK)+r'_[1-9]\d*', n.attrs.get('href', ''))]
|
||||
if not links:
|
||||
headings = [clean(n.text()) for n in node.walk() if n.tag in ('h3','h4')]
|
||||
frontmatter = [n.attrs.get('href','') for n in node.walk()
|
||||
if n.tag == 'a' and n.attrs.get('data-test') == 'front-matter-pdf']
|
||||
if headings == ['Front Matter'] and len(frontmatter) == 1:
|
||||
if excluded_materials is not None:
|
||||
excluded_materials.append({'title':'Front Matter', 'url':frontmatter[0],
|
||||
'reason':'Section front matter without a chapter DOI; not an authored chapter.'})
|
||||
continue
|
||||
if len(links) != 1:
|
||||
raise ValueError('Chapter entry lacks one reviewed-book DOI')
|
||||
link, = links
|
||||
@@ -158,11 +167,11 @@ def collect(out, *, toc_snapshots, chapter_snapshots):
|
||||
if hashlib.sha256(history.encode()).hexdigest() != HISTORY_SHA:
|
||||
raise ValueError('Reviewed selected-volume relationship evidence changed')
|
||||
scope_evidence = [history_evidence]
|
||||
titles = {}
|
||||
titles, excluded = {}, []
|
||||
for page in range(1, 4):
|
||||
html, evidence = read(toc_index, BOOK_URL+('?page='+str(page) if page > 1 else ''))
|
||||
scope_evidence.append(evidence)
|
||||
entries = parse_toc(html)
|
||||
entries = parse_toc(html, excluded_materials=excluded)
|
||||
if titles.keys() & entries.keys():
|
||||
raise ValueError('DOI repeated across TOC pages')
|
||||
titles.update(entries)
|
||||
@@ -197,6 +206,7 @@ def collect(out, *, toc_snapshots, chapter_snapshots):
|
||||
'count':len(records), 'file':str(out/path.name),
|
||||
'file_sha256':hashlib.sha256(path.read_bytes()).hexdigest(),
|
||||
'list_url':BOOK_URL, 'status':'partial',
|
||||
'excluded_materials':excluded,
|
||||
'missing_abstract_count':sum(not r['abstract'] for r in records)}
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)+'\n')
|
||||
report = {'status':'collected_pending_identity_review', 'records':len(records),
|
||||
|
||||
Binary file not shown.
@@ -21,7 +21,7 @@ class SpringerSelectedTests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
raw = FIXTURE.read_bytes()
|
||||
if hashlib.sha256(raw).hexdigest() != 'd9ad54008598e69a92d51c7bfccb072c8ee2f97f40ae436e7f795c59c9cb8f02':
|
||||
if hashlib.sha256(raw).hexdigest() != '94028fd37080f9d8027e5790e379cc151ee6eab17ed0d1dcc0f59e6e75f9d4d0':
|
||||
raise AssertionError('Fixed source fixture differs')
|
||||
cls.fixture = json.loads(gzip.decompress(raw))
|
||||
|
||||
@@ -42,6 +42,8 @@ class SpringerSelectedTests(unittest.TestCase):
|
||||
self.assertEqual(len(rows), 49)
|
||||
self.assertEqual(manifest['status'], 'partial')
|
||||
self.assertEqual(manifest['missing_abstract_count'], 0)
|
||||
self.assertTrue(manifest['excluded_materials'])
|
||||
self.assertTrue(all(x['title'] == 'Front Matter' for x in manifest['excluded_materials']))
|
||||
for row, expected in zip(rows, self.fixture['expected']):
|
||||
for field in ['title','abstract']:
|
||||
self.assertEqual(row[field], expected[field])
|
||||
|
||||
Reference in New Issue
Block a user