"""Frozen Springer public HTML receipts and chapter citation metadata.""" from collections import defaultdict import hashlib from pathlib import Path import json import re from urllib.parse import parse_qs, urlsplit from paper_atlas.db.submissions import timestamp from paper_atlas.io.html import Document, clean def snapshot_index(root): root = Path(root).resolve() receipts = json.loads((root/'receipts.json').read_bytes())['responses'] index = {} for receipt in receipts: url = receipt['url'] if url in index: raise ValueError('Duplicate snapshot URL') index[url] = (root, receipt) return index def read_page(index, url): if url not in index: raise ValueError('Missing required public HTML snapshot: '+url) root, receipt = index[url] filename = Path(receipt['file']) path = root/filename content_type = {k.lower():v for k,v in receipt.get('headers', {}).items()}.get('content-type', '') requested, final = urlsplit(url), urlsplit(receipt.get('final_url', '')) query = parse_qs(final.query, keep_blank_values=True) original_query = parse_qs(requested.query, keep_blank_values=True) # Springer returns the public metadata with an explicit cookie diagnostic. # Retain that final URL verbatim; accept no other redirect or changed page. if query != original_query and requested.netloc == 'link.springer.com': if (query.pop('error', None) != ['cookies_not_supported'] or not re.fullmatch(r'[0-9a-f-]{36}', ''.join(query.pop('code', [])))): raise ValueError('Unreviewed Springer redirect query') if (filename.is_absolute() or '..' in filename.parts or not path.resolve().is_relative_to(root) or receipt.get('http_status') != 200 or (final.scheme, final.netloc, final.path) != (requested.scheme, requested.netloc, requested.path) or final.fragment or query != original_query or 'text/html' not in content_type.lower()): raise ValueError('Unsafe, redirected or unsuccessful public HTML snapshot') raw = path.read_bytes() digest = hashlib.sha256(raw).hexdigest() if len(raw) > 2_000_000 or len(raw) != receipt['bytes'] or digest != receipt['sha256']: raise ValueError('Snapshot checksum or size differs') evidence = {'url':url, 'sha256':digest, 'bytes':len(raw), 'file':'evidence/'+digest+'.html', 'http_status':200, 'observed_at':timestamp(receipt['at']), 'response_receipt':receipt} return raw, evidence def parse_chapter(html, doi, *, book_title, publication_year): nodes = list(Document(html).root.walk()) meta = defaultdict(list) for node in nodes: if node.tag == 'meta' and node.attrs.get('name', '').startswith('citation_'): meta[node.attrs['name']].append(node.attrs.get('content', '')) def one(key): values = meta[key] if len(values) != 1 or not values[0]: raise ValueError('Missing or ambiguous citation field: '+key) return values[0] if (one('citation_doi') != doi or one('citation_inbook_title') != book_title or one('citation_publication_date') != str(publication_year)): raise ValueError('Chapter DOI, book or publication year differs from reviewed scope') raw_authors = meta['citation_author'] if not raw_authors: raise ValueError('Missing named chapter authors') authors = [] for raw in raw_authors: if raw.count(',') != 1: raise ValueError('Unreviewed citation author format') family, given = [clean(s) for s in raw.split(',', 1)] if not family or not given: raise ValueError('Incomplete citation author') authors.append({'given':given, 'family':family}) abstract_nodes = [n for n in nodes if n.attrs.get('id') == 'Abs1-content'] if len(abstract_nodes) > 1: raise ValueError('Ambiguous abstract section') abstract = clean(abstract_nodes[0].text()) if abstract_nodes else None first, last = one('citation_firstpage'), one('citation_lastpage') if not first.isdigit() or not last.isdigit() or int(first) > int(last): raise ValueError('Invalid chapter pages') return {'doi':doi, 'title':one('citation_title'), 'authors':authors, 'authors_citation_raw':list(raw_authors), 'abstract':abstract or None, 'first_page':first, 'last_page':last}