Files
paper-atlas/paper_atlas/io/springer.py
T
codex-bot 4d97822d9e
Metadata integrity checks / metadata (push) Successful in 8m28s
Preserve confirmed single-name Springer authors
2026-11-18 20:30:11 +08:00

123 lines
5.8 KiB
Python

"""Frozen Springer public HTML receipts and chapter citation metadata."""
from collections import defaultdict
import hashlib
from pathlib import Path
import json
import re
from urllib.parse import parse_qs, urlsplit
from paper_atlas.db.submissions import timestamp
from paper_atlas.io.html import Document, clean
def snapshot_index(root):
root = Path(root).resolve()
receipts = json.loads((root/'receipts.json').read_bytes())['responses']
index = {}
for receipt in receipts:
url = receipt['url']
if url in index:
raise ValueError('Duplicate snapshot URL')
index[url] = (root, receipt)
return index
def read_page(index, url):
if url not in index:
raise ValueError('Missing required public HTML snapshot: '+url)
root, receipt = index[url]
filename = Path(receipt['file'])
path = root/filename
content_type = {k.lower():v for k,v in receipt.get('headers', {}).items()}.get('content-type', '')
requested, final = urlsplit(url), urlsplit(receipt.get('final_url', ''))
query = parse_qs(final.query, keep_blank_values=True)
original_query = parse_qs(requested.query, keep_blank_values=True)
# Springer returns the public metadata with an explicit cookie diagnostic.
# Retain that final URL verbatim; accept no other redirect or changed page.
if query != original_query and requested.netloc == 'link.springer.com':
if (query.pop('error', None) != ['cookies_not_supported']
or not re.fullmatch(r'[0-9a-f-]{36}', ''.join(query.pop('code', [])))):
raise ValueError('Unreviewed Springer redirect query')
if (filename.is_absolute() or '..' in filename.parts or not path.resolve().is_relative_to(root)
or receipt.get('http_status') != 200
or (final.scheme, final.netloc, final.path) != (requested.scheme, requested.netloc, requested.path)
or final.fragment or query != original_query
or 'text/html' not in content_type.lower()):
raise ValueError('Unsafe, redirected or unsuccessful public HTML snapshot')
raw = path.read_bytes()
digest = hashlib.sha256(raw).hexdigest()
if len(raw) > 2_000_000 or len(raw) != receipt['bytes'] or digest != receipt['sha256']:
raise ValueError('Snapshot checksum or size differs')
evidence = {'url':url, 'sha256':digest, 'bytes':len(raw),
'file':'evidence/'+digest+'.html', 'http_status':200,
'observed_at':timestamp(receipt['at']), 'response_receipt':receipt}
return raw, evidence
def require_display_authors(nodes, authors):
matching_article_lists = []
for node in nodes:
if node.tag != 'script' or node.attrs.get('type') != 'application/ld+json':
continue
try:
article = json.loads(''.join(node.children))
except json.JSONDecodeError:
continue
if not isinstance(article, dict) or article.get('@type') != 'ScholarlyArticle':
continue
entries = article.get('author')
if not isinstance(entries, list) or not all(
isinstance(entry, dict) and isinstance(entry.get('name'), str) for entry in entries
):
raise ValueError('Unreviewed Springer article author list')
matching_article_lists.append([clean(entry['name']) for entry in entries])
if matching_article_lists != [authors]:
raise ValueError('Single-name author requires matching Springer display author list')
def parse_chapter(html, doi, *, book_title, publication_year):
nodes = list(Document(html).root.walk())
meta = defaultdict(list)
for node in nodes:
if node.tag == 'meta' and node.attrs.get('name', '').startswith('citation_'):
meta[node.attrs['name']].append(node.attrs.get('content', ''))
def one(key):
values = meta[key]
if len(values) != 1 or not values[0]:
raise ValueError('Missing or ambiguous citation field: '+key)
return values[0]
if (one('citation_doi') != doi or one('citation_inbook_title') != book_title
or one('citation_publication_date') != str(publication_year)):
raise ValueError('Chapter DOI, book or publication year differs from reviewed scope')
raw_authors = meta['citation_author']
if not raw_authors:
raise ValueError('Missing named chapter authors')
authors = []
for raw in raw_authors:
if ',' not in raw:
name = clean(raw)
if len(name.split()) != 1 or not any(character.isalpha() for character in name):
raise ValueError('Unreviewed citation author format')
authors.append(name)
continue
if raw.count(',') != 1:
raise ValueError('Unreviewed citation author format')
family, given = [clean(s) for s in raw.split(',', 1)]
if not family or not given:
raise ValueError('Incomplete citation author')
authors.append({'given':given, 'family':family})
if any(isinstance(author, str) for author in authors):
display_names = [author if isinstance(author, str) else author['given']+' '+author['family']
for author in authors]
require_display_authors(nodes, display_names)
abstract_nodes = [n for n in nodes if n.attrs.get('id') == 'Abs1-content']
if len(abstract_nodes) > 1:
raise ValueError('Ambiguous abstract section')
abstract = clean(abstract_nodes[0].text()) if abstract_nodes else None
first, last = one('citation_firstpage'), one('citation_lastpage')
if not first.isdigit() or not last.isdigit() or int(first) > int(last):
raise ValueError('Invalid chapter pages')
return {'doi':doi, 'title':one('citation_title'), 'authors':authors,
'authors_citation_raw':list(raw_authors), 'abstract':abstract or None,
'first_page':first, 'last_page':last}