Files
paper-atlas/paper_atlas/collectors/springer_selected.py
codex-bot 5c78a36fd5
Metadata integrity checks / metadata (push) Successful in 2m20s
Scope chapter publication identities and test explicit work anchors
2026-11-10 00:04:35 +08:00

232 lines
12 KiB
Python

"""Reviewed postconference selected volumes from frozen public HTML metadata."""
import argparse
from collections import defaultdict
import fcntl
import hashlib
import json
from pathlib import Path
import re
import tempfile
from urllib.parse import parse_qs, urlsplit
from paper_atlas.db.submissions import timestamp
from paper_atlas.io.html import Document, clean
SOURCE = 'springer_selected_volume'
BOOK = '10.1007/978-1-4612-2660-4'
BOOK_URL = 'https://link.springer.com/book/'+BOOK
HISTORY_URL = 'https://www.gatsby.ucl.ac.uk/aistats/workshops.html'
HISTORY_SHA = '6db205b9dbc82b4a096556efd2aa67e065cad8e0bd6351176b1962ffd1fca179'
SCOPE = {
'venue': 'AISTATS', 'year': 1993, 'track': 'selected_volume',
'event_key': 'aistats-1993-selected-volume-1994',
'publication_year': 1994, 'book_doi': BOOK,
'book_title': 'Selecting Models from Data', 'expected_chapters': 49,
'relationship': 'Selected workshop papers were revised and rereviewed for later publication; this is not the original complete workshop decision list.',
}
def snapshot_index(root):
root = Path(root).resolve()
receipts = json.loads((root/'receipts.json').read_bytes())['responses']
index = {}
for receipt in receipts:
url = receipt['url']
if url in index:
raise ValueError('Duplicate snapshot URL')
index[url] = (root, receipt)
return index
def read_page(index, url):
if url not in index:
raise ValueError('Missing required public HTML snapshot: '+url)
root, receipt = index[url]
filename = Path(receipt['file'])
path = root/filename
content_type = {k.lower():v for k,v in receipt.get('headers', {}).items()}.get('content-type', '')
requested, final = urlsplit(url), urlsplit(receipt.get('final_url', ''))
query = parse_qs(final.query, keep_blank_values=True)
original_query = parse_qs(requested.query, keep_blank_values=True)
# Springer returns the public metadata with an explicit cookie diagnostic.
# Retain that final URL verbatim; accept no other redirect or changed page.
if query != original_query and requested.netloc == 'link.springer.com':
if (query.pop('error', None) != ['cookies_not_supported']
or not re.fullmatch(r'[0-9a-f-]{36}', ''.join(query.pop('code', [])))):
raise ValueError('Unreviewed Springer redirect query')
if (filename.is_absolute() or '..' in filename.parts or not path.resolve().is_relative_to(root)
or receipt.get('http_status') != 200
or (final.scheme, final.netloc, final.path) != (requested.scheme, requested.netloc, requested.path)
or final.fragment or query != original_query
or 'text/html' not in content_type.lower()):
raise ValueError('Unsafe, redirected or unsuccessful public HTML snapshot')
raw = path.read_bytes()
digest = hashlib.sha256(raw).hexdigest()
if len(raw) > 2_000_000 or len(raw) != receipt['bytes'] or digest != receipt['sha256']:
raise ValueError('Snapshot checksum or size differs')
evidence = {'url':url, 'sha256':digest, 'bytes':len(raw),
'file':'evidence/'+digest+'.html', 'http_status':200,
'observed_at':timestamp(receipt['at']), 'response_receipt':receipt}
return raw, evidence
def parse_toc(html, *, excluded_materials=None):
entries = {}
for node in Document(html).root.walk():
if node.tag != 'li' or node.attrs.get('data-test') != 'chapter':
continue
links = [n for n in node.walk() if n.tag == 'a' and re.fullmatch(
r'/chapter/'+re.escape(BOOK)+r'_[1-9]\d*', n.attrs.get('href', ''))]
if not links:
headings = [clean(n.text()) for n in node.walk() if n.tag in ('h3','h4')]
frontmatter = [n.attrs.get('href','') for n in node.walk()
if n.tag == 'a' and n.attrs.get('data-test') == 'front-matter-pdf']
if headings == ['Front Matter'] and len(frontmatter) == 1:
if excluded_materials is not None:
excluded_materials.append({'title':'Front Matter', 'url':frontmatter[0],
'reason':'Section front matter without a chapter DOI; not an authored chapter.'})
continue
if len(links) != 1:
raise ValueError('Chapter entry lacks one reviewed-book DOI')
link, = links
doi = link.attrs['href'].removeprefix('/chapter/')
if doi in entries or not clean(link.text()):
raise ValueError('Duplicate DOI or missing chapter title')
entries[doi] = clean(link.text())
return entries
def parse_chapter(html, doi):
nodes = list(Document(html).root.walk())
meta = defaultdict(list)
for node in nodes:
if node.tag == 'meta' and node.attrs.get('name', '').startswith('citation_'):
meta[node.attrs['name']].append(node.attrs.get('content', ''))
def one(key):
values = meta[key]
if len(values) != 1 or not values[0]:
raise ValueError('Missing or ambiguous citation field: '+key)
return values[0]
if (one('citation_doi') != doi or one('citation_inbook_title') != SCOPE['book_title']
or one('citation_publication_date') != str(SCOPE['publication_year'])):
raise ValueError('Chapter DOI, book or publication year differs from reviewed scope')
raw_authors = meta['citation_author']
if not raw_authors:
raise ValueError('Missing named chapter authors')
authors = []
for raw in raw_authors:
if raw.count(',') != 1:
raise ValueError('Unreviewed citation author format')
family, given = [clean(s) for s in raw.split(',', 1)]
if not family or not given:
raise ValueError('Incomplete citation author')
authors.append({'given':given, 'family':family})
abstract_nodes = [n for n in nodes if n.attrs.get('id') == 'Abs1-content']
if len(abstract_nodes) > 1:
raise ValueError('Ambiguous abstract section')
abstract = clean(abstract_nodes[0].text()) if abstract_nodes else None
first, last = one('citation_firstpage'), one('citation_lastpage')
if not first.isdigit() or not last.isdigit() or int(first) > int(last):
raise ValueError('Invalid chapter pages')
return {'doi':doi, 'title':one('citation_title'), 'authors':authors,
'authors_citation_raw':list(raw_authors), 'abstract':abstract or None,
'first_page':first, 'last_page':last}
def record_for(chapter, evidence, scope_evidence):
return {
'source':SOURCE, 'external_id':chapter['doi'],
'url':'https://link.springer.com/chapter/'+chapter['doi'],
'title':chapter['title'], 'abstract':chapter['abstract'], 'authors':chapter['authors'],
'identifiers':{'doi':chapter['doi']},
'submission_identifiers':[{'scheme':'publication_doi','value':chapter['doi']}],
**{k:SCOPE[k] for k in ('venue','year','track','event_key')},
'status':'accepted', 'observed_at':evidence['observed_at'],
'raw_decision':'listed in the 1994 selected postconference volume; original 1993 review decision not supplied',
'tags':[{'namespace':SOURCE+':publication_format','value':'selected_volume_chapter'}],
'metadata':{
'publication_format':'selected_volume_chapter', 'bibliographic_year':1994,
'related_workshop_year':1993, 'book_doi':BOOK, 'book_title':SCOPE['book_title'],
'year_basis':'Reviewed workshop year; bibliographic publication year is separately retained.',
'first_page':chapter['first_page'], 'last_page':chapter['last_page'],
'authors_citation_raw':chapter['authors_citation_raw'],
'publication_evidence':'Published selected-volume chapter, not an original workshop decision note.',
'scope_relationship':SCOPE['relationship'], 'decision_date_precision':'not_supplied',
'source_evidence':evidence, 'scope_evidence':scope_evidence,
},
}
def collect(out, *, toc_snapshots, chapter_snapshots):
toc_index, chapter_index = snapshot_index(toc_snapshots), snapshot_index(chapter_snapshots)
frozen = {}
def read(index, url):
raw, evidence = read_page(index, url)
frozen[evidence['file']] = raw
return raw.decode('utf-8'), evidence
history, history_evidence = read(toc_index, HISTORY_URL)
if hashlib.sha256(history.encode()).hexdigest() != HISTORY_SHA:
raise ValueError('Reviewed selected-volume relationship evidence changed')
scope_evidence = [history_evidence]
titles, excluded = {}, []
for page in range(1, 4):
html, evidence = read(toc_index, BOOK_URL+('?page='+str(page) if page > 1 else ''))
scope_evidence.append(evidence)
entries = parse_toc(html, excluded_materials=excluded)
if titles.keys() & entries.keys():
raise ValueError('DOI repeated across TOC pages')
titles.update(entries)
expected = {BOOK+'_'+str(i) for i in range(1, 50)}
if titles.keys() != expected:
raise ValueError('Reviewed 49-chapter inventory changed')
chapter_urls = {'https://link.springer.com/chapter/'+doi for doi in expected}
if chapter_index.keys() != chapter_urls:
raise ValueError('Missing or extraneous chapter snapshot')
records = []
for doi in sorted(expected, key=lambda s:int(s.rsplit('_',1)[1])):
html, evidence = read(chapter_index, 'https://link.springer.com/chapter/'+doi)
chapter = parse_chapter(html, doi)
if chapter['title'] != titles[doi]:
raise ValueError('Chapter title disagrees with official TOC')
records.append(record_for(chapter, evidence, scope_evidence))
out = Path(out)
out.mkdir(parents=True, exist_ok=True)
with (out/'springer-selected.collect.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if any(p.name != 'springer-selected.collect.lock' for p in out.iterdir()):
raise FileExistsError('Existing output must be inspected, not overwritten')
with tempfile.TemporaryDirectory(prefix='.springer-selected-', dir=out) as temporary:
stage = Path(temporary)
(stage/'evidence').mkdir()
for filename, raw in frozen.items():
(stage/filename).write_bytes(raw)
path = stage/'springer-aistats-1993-selected-volume.jsonl'
path.write_text(''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in records))
manifest = {'source':SOURCE, 'parser_version':1,
**{k:SCOPE[k] for k in ('venue','year','track','event_key')},
'count':len(records), 'file':str(out/path.name),
'file_sha256':hashlib.sha256(path.read_bytes()).hexdigest(),
'list_url':BOOK_URL, 'status':'partial',
'excluded_materials':excluded,
'missing_abstract_count':sum(not r['abstract'] for r in records)}
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)+'\n')
report = {'status':'collected_pending_identity_review', 'records':len(records),
'files':[manifest], 'reviewed_scope':SCOPE, 'database_written':False,
'pdfs_downloaded':False, 'scope_evidence':scope_evidence}
(stage/'collection-report.json').write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n')
for item in stage.iterdir():
if item.name != 'collection-report.json':
item.replace(out/item.name)
(stage/'collection-report.json').replace(out/'collection-report.json')
return report
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog)
parser.add_argument('--toc-snapshots', required=True)
parser.add_argument('--chapter-snapshots', required=True)
parser.add_argument('--out', required=True)
args = parser.parse_args(argv)
print(json.dumps(collect(args.out, toc_snapshots=args.toc_snapshots,
chapter_snapshots=args.chapter_snapshots),ensure_ascii=False))