92 lines
4.4 KiB
Python
92 lines
4.4 KiB
Python
"""Frozen Springer public HTML receipts and chapter citation metadata."""
|
|
from collections import defaultdict
|
|
import hashlib
|
|
from pathlib import Path
|
|
import json
|
|
import re
|
|
from urllib.parse import parse_qs, urlsplit
|
|
|
|
from paper_atlas.db.submissions import timestamp
|
|
from paper_atlas.io.html import Document, clean
|
|
|
|
|
|
def snapshot_index(root):
|
|
root = Path(root).resolve()
|
|
receipts = json.loads((root/'receipts.json').read_bytes())['responses']
|
|
index = {}
|
|
for receipt in receipts:
|
|
url = receipt['url']
|
|
if url in index:
|
|
raise ValueError('Duplicate snapshot URL')
|
|
index[url] = (root, receipt)
|
|
return index
|
|
|
|
|
|
def read_page(index, url):
|
|
if url not in index:
|
|
raise ValueError('Missing required public HTML snapshot: '+url)
|
|
root, receipt = index[url]
|
|
filename = Path(receipt['file'])
|
|
path = root/filename
|
|
content_type = {k.lower():v for k,v in receipt.get('headers', {}).items()}.get('content-type', '')
|
|
requested, final = urlsplit(url), urlsplit(receipt.get('final_url', ''))
|
|
query = parse_qs(final.query, keep_blank_values=True)
|
|
original_query = parse_qs(requested.query, keep_blank_values=True)
|
|
# Springer returns the public metadata with an explicit cookie diagnostic.
|
|
# Retain that final URL verbatim; accept no other redirect or changed page.
|
|
if query != original_query and requested.netloc == 'link.springer.com':
|
|
if (query.pop('error', None) != ['cookies_not_supported']
|
|
or not re.fullmatch(r'[0-9a-f-]{36}', ''.join(query.pop('code', [])))):
|
|
raise ValueError('Unreviewed Springer redirect query')
|
|
if (filename.is_absolute() or '..' in filename.parts or not path.resolve().is_relative_to(root)
|
|
or receipt.get('http_status') != 200
|
|
or (final.scheme, final.netloc, final.path) != (requested.scheme, requested.netloc, requested.path)
|
|
or final.fragment or query != original_query
|
|
or 'text/html' not in content_type.lower()):
|
|
raise ValueError('Unsafe, redirected or unsuccessful public HTML snapshot')
|
|
raw = path.read_bytes()
|
|
digest = hashlib.sha256(raw).hexdigest()
|
|
if len(raw) > 2_000_000 or len(raw) != receipt['bytes'] or digest != receipt['sha256']:
|
|
raise ValueError('Snapshot checksum or size differs')
|
|
evidence = {'url':url, 'sha256':digest, 'bytes':len(raw),
|
|
'file':'evidence/'+digest+'.html', 'http_status':200,
|
|
'observed_at':timestamp(receipt['at']), 'response_receipt':receipt}
|
|
return raw, evidence
|
|
|
|
|
|
def parse_chapter(html, doi, *, book_title, publication_year):
|
|
nodes = list(Document(html).root.walk())
|
|
meta = defaultdict(list)
|
|
for node in nodes:
|
|
if node.tag == 'meta' and node.attrs.get('name', '').startswith('citation_'):
|
|
meta[node.attrs['name']].append(node.attrs.get('content', ''))
|
|
def one(key):
|
|
values = meta[key]
|
|
if len(values) != 1 or not values[0]:
|
|
raise ValueError('Missing or ambiguous citation field: '+key)
|
|
return values[0]
|
|
if (one('citation_doi') != doi or one('citation_inbook_title') != book_title
|
|
or one('citation_publication_date') != str(publication_year)):
|
|
raise ValueError('Chapter DOI, book or publication year differs from reviewed scope')
|
|
raw_authors = meta['citation_author']
|
|
if not raw_authors:
|
|
raise ValueError('Missing named chapter authors')
|
|
authors = []
|
|
for raw in raw_authors:
|
|
if raw.count(',') != 1:
|
|
raise ValueError('Unreviewed citation author format')
|
|
family, given = [clean(s) for s in raw.split(',', 1)]
|
|
if not family or not given:
|
|
raise ValueError('Incomplete citation author')
|
|
authors.append({'given':given, 'family':family})
|
|
abstract_nodes = [n for n in nodes if n.attrs.get('id') == 'Abs1-content']
|
|
if len(abstract_nodes) > 1:
|
|
raise ValueError('Ambiguous abstract section')
|
|
abstract = clean(abstract_nodes[0].text()) if abstract_nodes else None
|
|
first, last = one('citation_firstpage'), one('citation_lastpage')
|
|
if not first.isdigit() or not last.isdigit() or int(first) > int(last):
|
|
raise ValueError('Invalid chapter pages')
|
|
return {'doi':doi, 'title':one('citation_title'), 'authors':authors,
|
|
'authors_citation_raw':list(raw_authors), 'abstract':abstract or None,
|
|
'first_page':first, 'last_page':last}
|