Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4d97822d9e |
@@ -11,6 +11,6 @@ paper-atlas collect eccv2026-publications \
|
|||||||
|
|
||||||
目录归档含 `receipts.jsonl`、`volume-checkpoint.json`、`summary.json` 及 156 页 HTML。摘要在章节详情页读取,章节归档沿用 `springer-selected` 的 `receipts.json` 格式。两类归档都按响应大小和 SHA-256 校验。目录卷号、分页、章节 DOI 归属、标题及 2026 年出处需要一致。已有输出目录需另查回执或换新空目录后运行。
|
目录归档含 `receipts.jsonl`、`volume-checkpoint.json`、`summary.json` 及 156 页 HTML。摘要在章节详情页读取,章节归档沿用 `springer-selected` 的 `receipts.json` 格式。两类归档都按响应大小和 SHA-256 校验。目录卷号、分页、章节 DOI 归属、标题及 2026 年出处需要一致。已有输出目录需另查回执或换新空目录后运行。
|
||||||
|
|
||||||
输出含规范化 JSONL、manifest、采集回执以及所选章节的内容寻址存证。每条保留出版 DOI、卷号、页码、按顺序的完整作者及公开摘要。目录证据用 URL 和哈希连接到输入归档,保存归档以便复核。主会出版观察依据 Springer 正式论文集,原始投稿号、决定日期及签名仍未知。工具仅处理已归档的公开 HTML。
|
输出含规范化 JSONL、manifest、采集回执以及所选章节的内容寻址存证。每条保留出版 DOI、卷号、页码、按顺序的完整作者及公开摘要。单名署名须由同页的结构化作者列表按顺序确认,确认后按原名保存为字符串,引用元数据仍保留原文。目录证据用 URL 和哈希连接到输入归档,保存归档以便复核。主会出版观察依据 Springer 正式论文集,原始投稿号、决定日期及签名仍未知。工具仅处理已归档的公开 HTML。
|
||||||
|
|
||||||
生成的文件待做论文身份复核后方可导入。准备生产批次时核实每个 DOI 与现有 arXiv 标识的绑定,保持两者的来源证据。对已存在的同届节目投稿需先核对投稿身份及轨道,再附加出版来源,避免因直接导入创建重复投稿。具有 ICLR 等既往决定的论文须保留原记录。新的增量导入 manifest 需以当前实际来源覆盖为父版本,留存待核差集和独立验收回执。
|
生成的文件待做论文身份复核后方可导入。准备生产批次时核实每个 DOI 与现有 arXiv 标识的绑定,保持两者的来源证据。对已存在的同届节目投稿需先核对投稿身份及轨道,再附加出版来源,避免因直接导入创建重复投稿。具有 ICLR 等既往决定的论文须保留原记录。新的增量导入 manifest 需以当前实际来源覆盖为父版本,留存待核差集和独立验收回执。
|
||||||
|
|||||||
@@ -54,6 +54,27 @@ def read_page(index, url):
|
|||||||
return raw, evidence
|
return raw, evidence
|
||||||
|
|
||||||
|
|
||||||
|
def require_display_authors(nodes, authors):
|
||||||
|
matching_article_lists = []
|
||||||
|
for node in nodes:
|
||||||
|
if node.tag != 'script' or node.attrs.get('type') != 'application/ld+json':
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
article = json.loads(''.join(node.children))
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
continue
|
||||||
|
if not isinstance(article, dict) or article.get('@type') != 'ScholarlyArticle':
|
||||||
|
continue
|
||||||
|
entries = article.get('author')
|
||||||
|
if not isinstance(entries, list) or not all(
|
||||||
|
isinstance(entry, dict) and isinstance(entry.get('name'), str) for entry in entries
|
||||||
|
):
|
||||||
|
raise ValueError('Unreviewed Springer article author list')
|
||||||
|
matching_article_lists.append([clean(entry['name']) for entry in entries])
|
||||||
|
if matching_article_lists != [authors]:
|
||||||
|
raise ValueError('Single-name author requires matching Springer display author list')
|
||||||
|
|
||||||
|
|
||||||
def parse_chapter(html, doi, *, book_title, publication_year):
|
def parse_chapter(html, doi, *, book_title, publication_year):
|
||||||
nodes = list(Document(html).root.walk())
|
nodes = list(Document(html).root.walk())
|
||||||
meta = defaultdict(list)
|
meta = defaultdict(list)
|
||||||
@@ -73,12 +94,22 @@ def parse_chapter(html, doi, *, book_title, publication_year):
|
|||||||
raise ValueError('Missing named chapter authors')
|
raise ValueError('Missing named chapter authors')
|
||||||
authors = []
|
authors = []
|
||||||
for raw in raw_authors:
|
for raw in raw_authors:
|
||||||
|
if ',' not in raw:
|
||||||
|
name = clean(raw)
|
||||||
|
if len(name.split()) != 1 or not any(character.isalpha() for character in name):
|
||||||
|
raise ValueError('Unreviewed citation author format')
|
||||||
|
authors.append(name)
|
||||||
|
continue
|
||||||
if raw.count(',') != 1:
|
if raw.count(',') != 1:
|
||||||
raise ValueError('Unreviewed citation author format')
|
raise ValueError('Unreviewed citation author format')
|
||||||
family, given = [clean(s) for s in raw.split(',', 1)]
|
family, given = [clean(s) for s in raw.split(',', 1)]
|
||||||
if not family or not given:
|
if not family or not given:
|
||||||
raise ValueError('Incomplete citation author')
|
raise ValueError('Incomplete citation author')
|
||||||
authors.append({'given':given, 'family':family})
|
authors.append({'given':given, 'family':family})
|
||||||
|
if any(isinstance(author, str) for author in authors):
|
||||||
|
display_names = [author if isinstance(author, str) else author['given']+' '+author['family']
|
||||||
|
for author in authors]
|
||||||
|
require_display_authors(nodes, display_names)
|
||||||
abstract_nodes = [n for n in nodes if n.attrs.get('id') == 'Abs1-content']
|
abstract_nodes = [n for n in nodes if n.attrs.get('id') == 'Abs1-content']
|
||||||
if len(abstract_nodes) > 1:
|
if len(abstract_nodes) > 1:
|
||||||
raise ValueError('Ambiguous abstract section')
|
raise ValueError('Ambiguous abstract section')
|
||||||
|
|||||||
@@ -9,7 +9,7 @@ import tempfile
|
|||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from paper_atlas.collectors.springer_selected import (
|
from paper_atlas.collectors.springer_selected import (
|
||||||
BOOK, SOURCE, collect, parse_chapter, parse_toc, read_page, snapshot_index,
|
BOOK, SCOPE, SOURCE, collect, parse_chapter, parse_toc, read_page, snapshot_index,
|
||||||
)
|
)
|
||||||
from paper_atlas.db.store import Store
|
from paper_atlas.db.store import Store
|
||||||
from paper_atlas.ingestion.records import ingest_file
|
from paper_atlas.ingestion.records import ingest_file
|
||||||
@@ -153,6 +153,28 @@ class SpringerSelectedTests(unittest.TestCase):
|
|||||||
html = self.fixture['chapters']['chapter-01.html'].replace('id="Abs1-content"','id="not-an-abstract"')
|
html = self.fixture['chapters']['chapter-01.html'].replace('id="Abs1-content"','id="not-an-abstract"')
|
||||||
self.assertIsNone(parse_chapter(html, BOOK+'_1')['abstract'])
|
self.assertIsNone(parse_chapter(html, BOOK+'_1')['abstract'])
|
||||||
|
|
||||||
|
def test_single_name_preserves_literal_and_requires_ordered_display_confirmation(self):
|
||||||
|
html = f"""
|
||||||
|
<meta name="citation_doi" content="{BOOK}_1">
|
||||||
|
<meta name="citation_inbook_title" content="{SCOPE['book_title']}">
|
||||||
|
<meta name="citation_publication_date" content="{SCOPE['publication_year']}">
|
||||||
|
<meta name="citation_title" content="Example">
|
||||||
|
<meta name="citation_author" content="Rajkumar">
|
||||||
|
<meta name="citation_author" content="Doe, Jane">
|
||||||
|
<meta name="citation_firstpage" content="1">
|
||||||
|
<meta name="citation_lastpage" content="5">
|
||||||
|
"""
|
||||||
|
article = {'@type':'ScholarlyArticle', 'author':[{'name':'Rajkumar'}, {'name':'Jane Doe'}]}
|
||||||
|
script = '<script type="application/ld+json">{}</script>'
|
||||||
|
chapter = parse_chapter(html+script.format(json.dumps(article)), BOOK+'_1')
|
||||||
|
self.assertEqual(chapter['authors'], ['Rajkumar', {'family':'Doe', 'given':'Jane'}])
|
||||||
|
self.assertEqual(chapter['authors_citation_raw'], ['Rajkumar','Doe, Jane'])
|
||||||
|
with self.assertRaisesRegex(ValueError, 'matching Springer display author list'):
|
||||||
|
parse_chapter(html, BOOK+'_1')
|
||||||
|
article['author'].reverse()
|
||||||
|
with self.assertRaisesRegex(ValueError, 'matching Springer display author list'):
|
||||||
|
parse_chapter(html+script.format(json.dumps(article)), BOOK+'_1')
|
||||||
|
|
||||||
def test_toc_ignores_frontmatter_but_rejects_duplicate_publications(self):
|
def test_toc_ignores_frontmatter_but_rejects_duplicate_publications(self):
|
||||||
html = self.fixture['toc']['springer-toc1.html']
|
html = self.fixture['toc']['springer-toc1.html']
|
||||||
self.assertEqual(len(parse_toc(html)), 17)
|
self.assertEqual(len(parse_toc(html)), 17)
|
||||||
|
|||||||
Reference in New Issue
Block a user