diff --git a/docs/springer-selected-volumes.md b/docs/springer-selected-volumes.md index 3e36c07..c5cda9e 100644 --- a/docs/springer-selected-volumes.md +++ b/docs/springer-selected-volumes.md @@ -32,4 +32,4 @@ paper-atlas collect springer-selected \ ## 固定测试资料 -`tests/fixtures/springer-aistats1993-fixture.json.gz` 保存 4 份范围/目录 HTML、49 份章节 HTML、原始请求回执及先于本采集器独立生成的预期元数据,SHA256 为 `d9ad54008598e69a92d51c7bfccb072c8ee2f97f40ae436e7f795c59c9cb8f02`。它是压缩的公开元数据测试资料,不包含论文 PDF。测试覆盖全部真实章节字段、缺失和漂移输入、来源安全边界、幂等导入及旧拒稿历史保留。构建和测试只在已授权远程 runner 执行。 +`tests/fixtures/springer-aistats1993-fixture.json.gz` 保存 4 份范围/目录 HTML、49 份章节 HTML、原始请求回执及独立原始字段审计的预期元数据,SHA256 为 `94028fd37080f9d8027e5790e379cc151ee6eab17ed0d1dcc0f59e6e75f9d4d0`。摘要按 HTML 段落边界分隔;未修改原始 HTML 或回执。它是压缩的公开元数据测试资料,不包含论文 PDF。测试覆盖全部真实章节字段、缺失和漂移输入、来源安全边界、幂等导入及旧拒稿历史保留。构建和测试只在已授权远程 runner 执行。 diff --git a/paper_atlas/collectors/springer_selected.py b/paper_atlas/collectors/springer_selected.py index b30edff..0817a9b 100644 --- a/paper_atlas/collectors/springer_selected.py +++ b/paper_atlas/collectors/springer_selected.py @@ -70,13 +70,22 @@ def read_page(index, url): return raw, evidence -def parse_toc(html): +def parse_toc(html, *, excluded_materials=None): entries = {} for node in Document(html).root.walk(): if node.tag != 'li' or node.attrs.get('data-test') != 'chapter': continue links = [n for n in node.walk() if n.tag == 'a' and re.fullmatch( r'/chapter/'+re.escape(BOOK)+r'_[1-9]\d*', n.attrs.get('href', ''))] + if not links: + headings = [clean(n.text()) for n in node.walk() if n.tag in ('h3','h4')] + frontmatter = [n.attrs.get('href','') for n in node.walk() + if n.tag == 'a' and n.attrs.get('data-test') == 'front-matter-pdf'] + if headings == ['Front Matter'] and len(frontmatter) == 1: + if excluded_materials is not None: + excluded_materials.append({'title':'Front Matter', 'url':frontmatter[0], + 'reason':'Section front matter without a chapter DOI; not an authored chapter.'}) + continue if len(links) != 1: raise ValueError('Chapter entry lacks one reviewed-book DOI') link, = links @@ -158,11 +167,11 @@ def collect(out, *, toc_snapshots, chapter_snapshots): if hashlib.sha256(history.encode()).hexdigest() != HISTORY_SHA: raise ValueError('Reviewed selected-volume relationship evidence changed') scope_evidence = [history_evidence] - titles = {} + titles, excluded = {}, [] for page in range(1, 4): html, evidence = read(toc_index, BOOK_URL+('?page='+str(page) if page > 1 else '')) scope_evidence.append(evidence) - entries = parse_toc(html) + entries = parse_toc(html, excluded_materials=excluded) if titles.keys() & entries.keys(): raise ValueError('DOI repeated across TOC pages') titles.update(entries) @@ -197,6 +206,7 @@ def collect(out, *, toc_snapshots, chapter_snapshots): 'count':len(records), 'file':str(out/path.name), 'file_sha256':hashlib.sha256(path.read_bytes()).hexdigest(), 'list_url':BOOK_URL, 'status':'partial', + 'excluded_materials':excluded, 'missing_abstract_count':sum(not r['abstract'] for r in records)} path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)+'\n') report = {'status':'collected_pending_identity_review', 'records':len(records), diff --git a/tests/fixtures/springer-aistats1993-fixture.json.gz b/tests/fixtures/springer-aistats1993-fixture.json.gz index de409ba..e8dfb47 100644 Binary files a/tests/fixtures/springer-aistats1993-fixture.json.gz and b/tests/fixtures/springer-aistats1993-fixture.json.gz differ diff --git a/tests/test_springer_selected.py b/tests/test_springer_selected.py index 4587435..02e3cf1 100644 --- a/tests/test_springer_selected.py +++ b/tests/test_springer_selected.py @@ -21,7 +21,7 @@ class SpringerSelectedTests(unittest.TestCase): @classmethod def setUpClass(cls): raw = FIXTURE.read_bytes() - if hashlib.sha256(raw).hexdigest() != 'd9ad54008598e69a92d51c7bfccb072c8ee2f97f40ae436e7f795c59c9cb8f02': + if hashlib.sha256(raw).hexdigest() != '94028fd37080f9d8027e5790e379cc151ee6eab17ed0d1dcc0f59e6e75f9d4d0': raise AssertionError('Fixed source fixture differs') cls.fixture = json.loads(gzip.decompress(raw)) @@ -42,6 +42,8 @@ class SpringerSelectedTests(unittest.TestCase): self.assertEqual(len(rows), 49) self.assertEqual(manifest['status'], 'partial') self.assertEqual(manifest['missing_abstract_count'], 0) + self.assertTrue(manifest['excluded_materials']) + self.assertTrue(all(x['title'] == 'Front Matter' for x in manifest['excluded_materials'])) for row, expected in zip(rows, self.fixture['expected']): for field in ['title','abstract']: self.assertEqual(row[field], expected[field])