Files
paper-atlas/paper_atlas/collectors/springer_selected.py
T
codex-bot 6d35a2e567
Metadata integrity checks / metadata (push) Successful in 7m51s
Collect verified ECCV 2026 Springer proceedings metadata
2026-11-18 18:30:01 +08:00

154 lines
8.1 KiB
Python

"""Reviewed postconference selected volumes from frozen public HTML metadata."""
import argparse
import fcntl
import hashlib
import json
from pathlib import Path
import re
import tempfile
from paper_atlas.io.html import Document, clean
from paper_atlas.io.springer import parse_chapter as parse_citation_chapter, read_page, snapshot_index
SOURCE = 'springer_selected_volume'
BOOK = '10.1007/978-1-4612-2660-4'
BOOK_URL = 'https://link.springer.com/book/'+BOOK
HISTORY_URL = 'https://www.gatsby.ucl.ac.uk/aistats/workshops.html'
HISTORY_SHA = '6db205b9dbc82b4a096556efd2aa67e065cad8e0bd6351176b1962ffd1fca179'
SCOPE = {
'venue': 'AISTATS', 'year': 1993, 'track': 'selected_volume',
'event_key': 'aistats-1993-selected-volume-1994',
'publication_year': 1994, 'book_doi': BOOK,
'book_title': 'Selecting Models from Data', 'expected_chapters': 49,
'relationship': 'Selected workshop papers were revised and rereviewed for later publication; this is not the original complete workshop decision list.',
}
def parse_toc(html, *, excluded_materials=None):
entries = {}
for node in Document(html).root.walk():
if node.tag != 'li' or node.attrs.get('data-test') != 'chapter':
continue
links = [n for n in node.walk() if n.tag == 'a' and re.fullmatch(
r'/chapter/'+re.escape(BOOK)+r'_[1-9]\d*', n.attrs.get('href', ''))]
if not links:
headings = [clean(n.text()) for n in node.walk() if n.tag in ('h3','h4')]
frontmatter = [n.attrs.get('href','') for n in node.walk()
if n.tag == 'a' and n.attrs.get('data-test') == 'front-matter-pdf']
if headings == ['Front Matter'] and len(frontmatter) == 1:
if excluded_materials is not None:
excluded_materials.append({'title':'Front Matter', 'url':frontmatter[0],
'reason':'Section front matter without a chapter DOI; not an authored chapter.'})
continue
if len(links) != 1:
raise ValueError('Chapter entry lacks one reviewed-book DOI')
link, = links
doi = link.attrs['href'].removeprefix('/chapter/')
if doi in entries or not clean(link.text()):
raise ValueError('Duplicate DOI or missing chapter title')
entries[doi] = clean(link.text())
return entries
def parse_chapter(html, doi):
return parse_citation_chapter(html, doi, book_title=SCOPE['book_title'],
publication_year=SCOPE['publication_year'])
def record_for(chapter, evidence, scope_evidence):
return {
'source':SOURCE, 'external_id':chapter['doi'],
'url':'https://link.springer.com/chapter/'+chapter['doi'],
'title':chapter['title'], 'abstract':chapter['abstract'], 'authors':chapter['authors'],
'identifiers':{'doi':chapter['doi']},
'submission_identifiers':[{'scheme':'publication_doi','value':chapter['doi']}],
**{k:SCOPE[k] for k in ('venue','year','track','event_key')},
'status':'accepted', 'observed_at':evidence['observed_at'],
'raw_decision':'listed in the 1994 selected postconference volume; original 1993 review decision not supplied',
'tags':[{'namespace':SOURCE+':publication_format','value':'selected_volume_chapter'}],
'metadata':{
'publication_format':'selected_volume_chapter', 'bibliographic_year':1994,
'related_workshop_year':1993, 'book_doi':BOOK, 'book_title':SCOPE['book_title'],
'year_basis':'Reviewed workshop year; bibliographic publication year is separately retained.',
'first_page':chapter['first_page'], 'last_page':chapter['last_page'],
'authors_citation_raw':chapter['authors_citation_raw'],
'publication_evidence':'Published selected-volume chapter, not an original workshop decision note.',
'scope_relationship':SCOPE['relationship'], 'decision_date_precision':'not_supplied',
'source_evidence':evidence, 'scope_evidence':scope_evidence,
},
}
def collect(out, *, toc_snapshots, chapter_snapshots):
toc_index, chapter_index = snapshot_index(toc_snapshots), snapshot_index(chapter_snapshots)
frozen = {}
def read(index, url):
raw, evidence = read_page(index, url)
frozen[evidence['file']] = raw
return raw.decode('utf-8'), evidence
history, history_evidence = read(toc_index, HISTORY_URL)
if hashlib.sha256(history.encode()).hexdigest() != HISTORY_SHA:
raise ValueError('Reviewed selected-volume relationship evidence changed')
scope_evidence = [history_evidence]
titles, excluded = {}, []
for page in range(1, 4):
html, evidence = read(toc_index, BOOK_URL+('?page='+str(page) if page > 1 else ''))
scope_evidence.append(evidence)
entries = parse_toc(html, excluded_materials=excluded)
if titles.keys() & entries.keys():
raise ValueError('DOI repeated across TOC pages')
titles.update(entries)
expected = {BOOK+'_'+str(i) for i in range(1, 50)}
if titles.keys() != expected:
raise ValueError('Reviewed 49-chapter inventory changed')
chapter_urls = {'https://link.springer.com/chapter/'+doi for doi in expected}
if chapter_index.keys() != chapter_urls:
raise ValueError('Missing or extraneous chapter snapshot')
records = []
for doi in sorted(expected, key=lambda s:int(s.rsplit('_',1)[1])):
html, evidence = read(chapter_index, 'https://link.springer.com/chapter/'+doi)
chapter = parse_chapter(html, doi)
if chapter['title'] != titles[doi]:
raise ValueError('Chapter title disagrees with official TOC')
records.append(record_for(chapter, evidence, scope_evidence))
out = Path(out)
out.mkdir(parents=True, exist_ok=True)
with (out/'springer-selected.collect.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if any(p.name != 'springer-selected.collect.lock' for p in out.iterdir()):
raise FileExistsError('Existing output must be inspected, not overwritten')
with tempfile.TemporaryDirectory(prefix='.springer-selected-', dir=out) as temporary:
stage = Path(temporary)
(stage/'evidence').mkdir()
for filename, raw in frozen.items():
(stage/filename).write_bytes(raw)
path = stage/'springer-aistats-1993-selected-volume.jsonl'
path.write_text(''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in records))
manifest = {'source':SOURCE, 'parser_version':1,
**{k:SCOPE[k] for k in ('venue','year','track','event_key')},
'count':len(records), 'file':str(out/path.name),
'file_sha256':hashlib.sha256(path.read_bytes()).hexdigest(),
'list_url':BOOK_URL, 'status':'partial',
'excluded_materials':excluded,
'missing_abstract_count':sum(not r['abstract'] for r in records)}
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)+'\n')
report = {'status':'collected_pending_identity_review', 'records':len(records),
'files':[manifest], 'reviewed_scope':SCOPE, 'database_written':False,
'pdfs_downloaded':False, 'scope_evidence':scope_evidence}
(stage/'collection-report.json').write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n')
for item in stage.iterdir():
if item.name != 'collection-report.json':
item.replace(out/item.name)
(stage/'collection-report.json').replace(out/'collection-report.json')
return report
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog)
parser.add_argument('--toc-snapshots', required=True)
parser.add_argument('--chapter-snapshots', required=True)
parser.add_argument('--out', required=True)
args = parser.parse_args(argv)
print(json.dumps(collect(args.out, toc_snapshots=args.toc_snapshots,
chapter_snapshots=args.chapter_snapshots),ensure_ascii=False))