232 lines
12 KiB
Python
232 lines
12 KiB
Python
"""Reviewed postconference selected volumes from frozen public HTML metadata."""
|
|
import argparse
|
|
from collections import defaultdict
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import tempfile
|
|
from urllib.parse import parse_qs, urlsplit
|
|
|
|
from paper_atlas.db.submissions import timestamp
|
|
from paper_atlas.io.html import Document, clean
|
|
|
|
SOURCE = 'springer_selected_volume'
|
|
BOOK = '10.1007/978-1-4612-2660-4'
|
|
BOOK_URL = 'https://link.springer.com/book/'+BOOK
|
|
HISTORY_URL = 'https://www.gatsby.ucl.ac.uk/aistats/workshops.html'
|
|
HISTORY_SHA = '6db205b9dbc82b4a096556efd2aa67e065cad8e0bd6351176b1962ffd1fca179'
|
|
SCOPE = {
|
|
'venue': 'AISTATS', 'year': 1993, 'track': 'selected_volume',
|
|
'event_key': 'aistats-1993-selected-volume-1994',
|
|
'publication_year': 1994, 'book_doi': BOOK,
|
|
'book_title': 'Selecting Models from Data', 'expected_chapters': 49,
|
|
'relationship': 'Selected workshop papers were revised and rereviewed for later publication; this is not the original complete workshop decision list.',
|
|
}
|
|
|
|
|
|
def snapshot_index(root):
|
|
root = Path(root).resolve()
|
|
receipts = json.loads((root/'receipts.json').read_bytes())['responses']
|
|
index = {}
|
|
for receipt in receipts:
|
|
url = receipt['url']
|
|
if url in index:
|
|
raise ValueError('Duplicate snapshot URL')
|
|
index[url] = (root, receipt)
|
|
return index
|
|
|
|
|
|
def read_page(index, url):
|
|
if url not in index:
|
|
raise ValueError('Missing required public HTML snapshot: '+url)
|
|
root, receipt = index[url]
|
|
filename = Path(receipt['file'])
|
|
path = root/filename
|
|
content_type = {k.lower():v for k,v in receipt.get('headers', {}).items()}.get('content-type', '')
|
|
requested, final = urlsplit(url), urlsplit(receipt.get('final_url', ''))
|
|
query = parse_qs(final.query, keep_blank_values=True)
|
|
original_query = parse_qs(requested.query, keep_blank_values=True)
|
|
# Springer returns the public metadata with an explicit cookie diagnostic.
|
|
# Retain that final URL verbatim; accept no other redirect or changed page.
|
|
if query != original_query and requested.netloc == 'link.springer.com':
|
|
if (query.pop('error', None) != ['cookies_not_supported']
|
|
or not re.fullmatch(r'[0-9a-f-]{36}', ''.join(query.pop('code', [])))):
|
|
raise ValueError('Unreviewed Springer redirect query')
|
|
if (filename.is_absolute() or '..' in filename.parts or not path.resolve().is_relative_to(root)
|
|
or receipt.get('http_status') != 200
|
|
or (final.scheme, final.netloc, final.path) != (requested.scheme, requested.netloc, requested.path)
|
|
or final.fragment or query != original_query
|
|
or 'text/html' not in content_type.lower()):
|
|
raise ValueError('Unsafe, redirected or unsuccessful public HTML snapshot')
|
|
raw = path.read_bytes()
|
|
digest = hashlib.sha256(raw).hexdigest()
|
|
if len(raw) > 2_000_000 or len(raw) != receipt['bytes'] or digest != receipt['sha256']:
|
|
raise ValueError('Snapshot checksum or size differs')
|
|
evidence = {'url':url, 'sha256':digest, 'bytes':len(raw),
|
|
'file':'evidence/'+digest+'.html', 'http_status':200,
|
|
'observed_at':timestamp(receipt['at']), 'response_receipt':receipt}
|
|
return raw, evidence
|
|
|
|
|
|
def parse_toc(html, *, excluded_materials=None):
|
|
entries = {}
|
|
for node in Document(html).root.walk():
|
|
if node.tag != 'li' or node.attrs.get('data-test') != 'chapter':
|
|
continue
|
|
links = [n for n in node.walk() if n.tag == 'a' and re.fullmatch(
|
|
r'/chapter/'+re.escape(BOOK)+r'_[1-9]\d*', n.attrs.get('href', ''))]
|
|
if not links:
|
|
headings = [clean(n.text()) for n in node.walk() if n.tag in ('h3','h4')]
|
|
frontmatter = [n.attrs.get('href','') for n in node.walk()
|
|
if n.tag == 'a' and n.attrs.get('data-test') == 'front-matter-pdf']
|
|
if headings == ['Front Matter'] and len(frontmatter) == 1:
|
|
if excluded_materials is not None:
|
|
excluded_materials.append({'title':'Front Matter', 'url':frontmatter[0],
|
|
'reason':'Section front matter without a chapter DOI; not an authored chapter.'})
|
|
continue
|
|
if len(links) != 1:
|
|
raise ValueError('Chapter entry lacks one reviewed-book DOI')
|
|
link, = links
|
|
doi = link.attrs['href'].removeprefix('/chapter/')
|
|
if doi in entries or not clean(link.text()):
|
|
raise ValueError('Duplicate DOI or missing chapter title')
|
|
entries[doi] = clean(link.text())
|
|
return entries
|
|
|
|
|
|
def parse_chapter(html, doi):
|
|
nodes = list(Document(html).root.walk())
|
|
meta = defaultdict(list)
|
|
for node in nodes:
|
|
if node.tag == 'meta' and node.attrs.get('name', '').startswith('citation_'):
|
|
meta[node.attrs['name']].append(node.attrs.get('content', ''))
|
|
def one(key):
|
|
values = meta[key]
|
|
if len(values) != 1 or not values[0]:
|
|
raise ValueError('Missing or ambiguous citation field: '+key)
|
|
return values[0]
|
|
if (one('citation_doi') != doi or one('citation_inbook_title') != SCOPE['book_title']
|
|
or one('citation_publication_date') != str(SCOPE['publication_year'])):
|
|
raise ValueError('Chapter DOI, book or publication year differs from reviewed scope')
|
|
raw_authors = meta['citation_author']
|
|
if not raw_authors:
|
|
raise ValueError('Missing named chapter authors')
|
|
authors = []
|
|
for raw in raw_authors:
|
|
if raw.count(',') != 1:
|
|
raise ValueError('Unreviewed citation author format')
|
|
family, given = [clean(s) for s in raw.split(',', 1)]
|
|
if not family or not given:
|
|
raise ValueError('Incomplete citation author')
|
|
authors.append({'given':given, 'family':family})
|
|
abstract_nodes = [n for n in nodes if n.attrs.get('id') == 'Abs1-content']
|
|
if len(abstract_nodes) > 1:
|
|
raise ValueError('Ambiguous abstract section')
|
|
abstract = clean(abstract_nodes[0].text()) if abstract_nodes else None
|
|
first, last = one('citation_firstpage'), one('citation_lastpage')
|
|
if not first.isdigit() or not last.isdigit() or int(first) > int(last):
|
|
raise ValueError('Invalid chapter pages')
|
|
return {'doi':doi, 'title':one('citation_title'), 'authors':authors,
|
|
'authors_citation_raw':list(raw_authors), 'abstract':abstract or None,
|
|
'first_page':first, 'last_page':last}
|
|
|
|
|
|
def record_for(chapter, evidence, scope_evidence):
|
|
return {
|
|
'source':SOURCE, 'external_id':chapter['doi'],
|
|
'url':'https://link.springer.com/chapter/'+chapter['doi'],
|
|
'title':chapter['title'], 'abstract':chapter['abstract'], 'authors':chapter['authors'],
|
|
'identifiers':{'doi':chapter['doi']},
|
|
'submission_identifiers':[{'scheme':'publication_doi','value':chapter['doi']}],
|
|
**{k:SCOPE[k] for k in ('venue','year','track','event_key')},
|
|
'status':'accepted', 'observed_at':evidence['observed_at'],
|
|
'raw_decision':'listed in the 1994 selected postconference volume; original 1993 review decision not supplied',
|
|
'tags':[{'namespace':SOURCE+':publication_format','value':'selected_volume_chapter'}],
|
|
'metadata':{
|
|
'publication_format':'selected_volume_chapter', 'bibliographic_year':1994,
|
|
'related_workshop_year':1993, 'book_doi':BOOK, 'book_title':SCOPE['book_title'],
|
|
'year_basis':'Reviewed workshop year; bibliographic publication year is separately retained.',
|
|
'first_page':chapter['first_page'], 'last_page':chapter['last_page'],
|
|
'authors_citation_raw':chapter['authors_citation_raw'],
|
|
'publication_evidence':'Published selected-volume chapter, not an original workshop decision note.',
|
|
'scope_relationship':SCOPE['relationship'], 'decision_date_precision':'not_supplied',
|
|
'source_evidence':evidence, 'scope_evidence':scope_evidence,
|
|
},
|
|
}
|
|
|
|
|
|
def collect(out, *, toc_snapshots, chapter_snapshots):
|
|
toc_index, chapter_index = snapshot_index(toc_snapshots), snapshot_index(chapter_snapshots)
|
|
frozen = {}
|
|
def read(index, url):
|
|
raw, evidence = read_page(index, url)
|
|
frozen[evidence['file']] = raw
|
|
return raw.decode('utf-8'), evidence
|
|
history, history_evidence = read(toc_index, HISTORY_URL)
|
|
if hashlib.sha256(history.encode()).hexdigest() != HISTORY_SHA:
|
|
raise ValueError('Reviewed selected-volume relationship evidence changed')
|
|
scope_evidence = [history_evidence]
|
|
titles, excluded = {}, []
|
|
for page in range(1, 4):
|
|
html, evidence = read(toc_index, BOOK_URL+('?page='+str(page) if page > 1 else ''))
|
|
scope_evidence.append(evidence)
|
|
entries = parse_toc(html, excluded_materials=excluded)
|
|
if titles.keys() & entries.keys():
|
|
raise ValueError('DOI repeated across TOC pages')
|
|
titles.update(entries)
|
|
expected = {BOOK+'_'+str(i) for i in range(1, 50)}
|
|
if titles.keys() != expected:
|
|
raise ValueError('Reviewed 49-chapter inventory changed')
|
|
chapter_urls = {'https://link.springer.com/chapter/'+doi for doi in expected}
|
|
if chapter_index.keys() != chapter_urls:
|
|
raise ValueError('Missing or extraneous chapter snapshot')
|
|
records = []
|
|
for doi in sorted(expected, key=lambda s:int(s.rsplit('_',1)[1])):
|
|
html, evidence = read(chapter_index, 'https://link.springer.com/chapter/'+doi)
|
|
chapter = parse_chapter(html, doi)
|
|
if chapter['title'] != titles[doi]:
|
|
raise ValueError('Chapter title disagrees with official TOC')
|
|
records.append(record_for(chapter, evidence, scope_evidence))
|
|
out = Path(out)
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
with (out/'springer-selected.collect.lock').open('a') as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
if any(p.name != 'springer-selected.collect.lock' for p in out.iterdir()):
|
|
raise FileExistsError('Existing output must be inspected, not overwritten')
|
|
with tempfile.TemporaryDirectory(prefix='.springer-selected-', dir=out) as temporary:
|
|
stage = Path(temporary)
|
|
(stage/'evidence').mkdir()
|
|
for filename, raw in frozen.items():
|
|
(stage/filename).write_bytes(raw)
|
|
path = stage/'springer-aistats-1993-selected-volume.jsonl'
|
|
path.write_text(''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in records))
|
|
manifest = {'source':SOURCE, 'parser_version':1,
|
|
**{k:SCOPE[k] for k in ('venue','year','track','event_key')},
|
|
'count':len(records), 'file':str(out/path.name),
|
|
'file_sha256':hashlib.sha256(path.read_bytes()).hexdigest(),
|
|
'list_url':BOOK_URL, 'status':'partial',
|
|
'excluded_materials':excluded,
|
|
'missing_abstract_count':sum(not r['abstract'] for r in records)}
|
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)+'\n')
|
|
report = {'status':'collected_pending_identity_review', 'records':len(records),
|
|
'files':[manifest], 'reviewed_scope':SCOPE, 'database_written':False,
|
|
'pdfs_downloaded':False, 'scope_evidence':scope_evidence}
|
|
(stage/'collection-report.json').write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n')
|
|
for item in stage.iterdir():
|
|
if item.name != 'collection-report.json':
|
|
item.replace(out/item.name)
|
|
(stage/'collection-report.json').replace(out/'collection-report.json')
|
|
return report
|
|
|
|
|
|
def main(argv=None, *, prog=None):
|
|
parser = argparse.ArgumentParser(prog=prog)
|
|
parser.add_argument('--toc-snapshots', required=True)
|
|
parser.add_argument('--chapter-snapshots', required=True)
|
|
parser.add_argument('--out', required=True)
|
|
args = parser.parse_args(argv)
|
|
print(json.dumps(collect(args.out, toc_snapshots=args.toc_snapshots,
|
|
chapter_snapshots=args.chapter_snapshots),ensure_ascii=False))
|