123 lines
5.8 KiB
Python
123 lines
5.8 KiB
Python
"""Frozen Springer public HTML receipts and chapter citation metadata."""
|
|
from collections import defaultdict
|
|
import hashlib
|
|
from pathlib import Path
|
|
import json
|
|
import re
|
|
from urllib.parse import parse_qs, urlsplit
|
|
|
|
from paper_atlas.db.submissions import timestamp
|
|
from paper_atlas.io.html import Document, clean
|
|
|
|
|
|
def snapshot_index(root):
|
|
root = Path(root).resolve()
|
|
receipts = json.loads((root/'receipts.json').read_bytes())['responses']
|
|
index = {}
|
|
for receipt in receipts:
|
|
url = receipt['url']
|
|
if url in index:
|
|
raise ValueError('Duplicate snapshot URL')
|
|
index[url] = (root, receipt)
|
|
return index
|
|
|
|
|
|
def read_page(index, url):
|
|
if url not in index:
|
|
raise ValueError('Missing required public HTML snapshot: '+url)
|
|
root, receipt = index[url]
|
|
filename = Path(receipt['file'])
|
|
path = root/filename
|
|
content_type = {k.lower():v for k,v in receipt.get('headers', {}).items()}.get('content-type', '')
|
|
requested, final = urlsplit(url), urlsplit(receipt.get('final_url', ''))
|
|
query = parse_qs(final.query, keep_blank_values=True)
|
|
original_query = parse_qs(requested.query, keep_blank_values=True)
|
|
# Springer returns the public metadata with an explicit cookie diagnostic.
|
|
# Retain that final URL verbatim; accept no other redirect or changed page.
|
|
if query != original_query and requested.netloc == 'link.springer.com':
|
|
if (query.pop('error', None) != ['cookies_not_supported']
|
|
or not re.fullmatch(r'[0-9a-f-]{36}', ''.join(query.pop('code', [])))):
|
|
raise ValueError('Unreviewed Springer redirect query')
|
|
if (filename.is_absolute() or '..' in filename.parts or not path.resolve().is_relative_to(root)
|
|
or receipt.get('http_status') != 200
|
|
or (final.scheme, final.netloc, final.path) != (requested.scheme, requested.netloc, requested.path)
|
|
or final.fragment or query != original_query
|
|
or 'text/html' not in content_type.lower()):
|
|
raise ValueError('Unsafe, redirected or unsuccessful public HTML snapshot')
|
|
raw = path.read_bytes()
|
|
digest = hashlib.sha256(raw).hexdigest()
|
|
if len(raw) > 2_000_000 or len(raw) != receipt['bytes'] or digest != receipt['sha256']:
|
|
raise ValueError('Snapshot checksum or size differs')
|
|
evidence = {'url':url, 'sha256':digest, 'bytes':len(raw),
|
|
'file':'evidence/'+digest+'.html', 'http_status':200,
|
|
'observed_at':timestamp(receipt['at']), 'response_receipt':receipt}
|
|
return raw, evidence
|
|
|
|
|
|
def require_display_authors(nodes, authors):
|
|
matching_article_lists = []
|
|
for node in nodes:
|
|
if node.tag != 'script' or node.attrs.get('type') != 'application/ld+json':
|
|
continue
|
|
try:
|
|
article = json.loads(''.join(node.children))
|
|
except json.JSONDecodeError:
|
|
continue
|
|
if not isinstance(article, dict) or article.get('@type') != 'ScholarlyArticle':
|
|
continue
|
|
entries = article.get('author')
|
|
if not isinstance(entries, list) or not all(
|
|
isinstance(entry, dict) and isinstance(entry.get('name'), str) for entry in entries
|
|
):
|
|
raise ValueError('Unreviewed Springer article author list')
|
|
matching_article_lists.append([clean(entry['name']) for entry in entries])
|
|
if matching_article_lists != [authors]:
|
|
raise ValueError('Single-name author requires matching Springer display author list')
|
|
|
|
|
|
def parse_chapter(html, doi, *, book_title, publication_year):
|
|
nodes = list(Document(html).root.walk())
|
|
meta = defaultdict(list)
|
|
for node in nodes:
|
|
if node.tag == 'meta' and node.attrs.get('name', '').startswith('citation_'):
|
|
meta[node.attrs['name']].append(node.attrs.get('content', ''))
|
|
def one(key):
|
|
values = meta[key]
|
|
if len(values) != 1 or not values[0]:
|
|
raise ValueError('Missing or ambiguous citation field: '+key)
|
|
return values[0]
|
|
if (one('citation_doi') != doi or one('citation_inbook_title') != book_title
|
|
or one('citation_publication_date') != str(publication_year)):
|
|
raise ValueError('Chapter DOI, book or publication year differs from reviewed scope')
|
|
raw_authors = meta['citation_author']
|
|
if not raw_authors:
|
|
raise ValueError('Missing named chapter authors')
|
|
authors = []
|
|
for raw in raw_authors:
|
|
if ',' not in raw:
|
|
name = clean(raw)
|
|
if len(name.split()) != 1 or not any(character.isalpha() for character in name):
|
|
raise ValueError('Unreviewed citation author format')
|
|
authors.append(name)
|
|
continue
|
|
if raw.count(',') != 1:
|
|
raise ValueError('Unreviewed citation author format')
|
|
family, given = [clean(s) for s in raw.split(',', 1)]
|
|
if not family or not given:
|
|
raise ValueError('Incomplete citation author')
|
|
authors.append({'given':given, 'family':family})
|
|
if any(isinstance(author, str) for author in authors):
|
|
display_names = [author if isinstance(author, str) else author['given']+' '+author['family']
|
|
for author in authors]
|
|
require_display_authors(nodes, display_names)
|
|
abstract_nodes = [n for n in nodes if n.attrs.get('id') == 'Abs1-content']
|
|
if len(abstract_nodes) > 1:
|
|
raise ValueError('Ambiguous abstract section')
|
|
abstract = clean(abstract_nodes[0].text()) if abstract_nodes else None
|
|
first, last = one('citation_firstpage'), one('citation_lastpage')
|
|
if not first.isdigit() or not last.isdigit() or int(first) > int(last):
|
|
raise ValueError('Invalid chapter pages')
|
|
return {'doi':doi, 'title':one('citation_title'), 'authors':authors,
|
|
'authors_citation_raw':list(raw_authors), 'abstract':abstract or None,
|
|
'first_page':first, 'last_page':last}
|