223 lines
13 KiB
Python
223 lines
13 KiB
Python
"""Reviewed RSS 2026 publisher metadata, with separate program evidence."""
|
|
REVIEWED_AUTHOR_VARIANTS = {16: '04de7e813b7c68b5c1764e90fe41bcd2e45259d46749bb2cc54bac5150c8f25b', 27: '975f6adc4d15045f9c8829689a759c5e3a44eecb3410a18ff99b37226d9c8b97', 78: '8208d0950468295c2b73dae28658566b1c726f65a4167f490039cf3449aa2db4', 132: '31eee6289de404f3d859dd0d9de4dd40e968f86b89312de41df87ae01c529711', 199: 'ced0bc9e38b28b4a490799fd94315f104ece0df28be428b3cc417db8ea797b9d', 204: '7c6959e6668bf44af09959388f6b9e2d883a76106bf7f191dc9949368c969ee5'}
|
|
import argparse
|
|
from datetime import datetime, date
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import unicodedata
|
|
|
|
from paper_atlas.io.html import Document, Node, clean
|
|
from paper_atlas.io.http import fetch
|
|
|
|
SOURCE = 'rss_proceedings'
|
|
ROOT = 'https://www.roboticsproceedings.org/rss22/'
|
|
HEADING = 'Robotics: Science and Systems XXII'
|
|
PAGES = {'index.html': ROOT,
|
|
'accepted.html': 'https://roboticsconference.org/2026/program/papers/',
|
|
'cfp.html': 'https://roboticsconference.org/2026/information/cfp/',
|
|
'sponsor-demos.html': 'https://roboticsconference.org/2026/program/demos/'}
|
|
|
|
|
|
def digest(raw):
|
|
return hashlib.sha256(raw).hexdigest()
|
|
|
|
|
|
def one(values):
|
|
if len(values) != 1:
|
|
raise ValueError('Expected one reviewed HTML field')
|
|
return values[0]
|
|
|
|
|
|
def comparable(value):
|
|
return unicodedata.normalize('NFC', clean(value))
|
|
|
|
|
|
def parse_lists(index_html, program_html, policy_html, sponsors_html):
|
|
policy = clean(Document(policy_html).root.text())
|
|
for text in ('RSS 2026 will not have a separate category for demo papers.',
|
|
'papers will only be public after acceptance', 'reviews will not be publicly visible'):
|
|
if text not in policy:
|
|
raise ValueError('Reviewed 2026 publication/demo/privacy policy changed')
|
|
if [clean(n.text()) for n in Document(sponsors_html).root.walk() if n.tag == 'h1'] != ['Sponsor Demos']:
|
|
raise ValueError('Sponsor activity page changed; never classify these as papers')
|
|
nodes = list(Document(index_html).root.walk())
|
|
if [clean(n.text()) for n in nodes if n.tag == 'h1'] != [HEADING]:
|
|
raise ValueError('Unexpected publisher edition')
|
|
index = {}
|
|
for row in (n for n in nodes if n.tag == 'tr'):
|
|
links = [n for n in row.walk() if n.tag == 'a' and re.fullmatch(r'p[0-9]{3}\.html', n.attrs.get('href', ''))]
|
|
if not links:
|
|
continue
|
|
link = one(links); pid = int(link.attrs['href'][1:4])
|
|
if pid in index:
|
|
raise ValueError('Repeated publisher ID')
|
|
index[pid] = {'title': clean(link.text()), 'authors': clean(one([n for n in row.walk() if n.tag == 'i']).text())}
|
|
program = {}
|
|
if [clean(n.text()) for n in Document(program_html).root.walk() if n.tag == 'h1'] != ['Accepted Papers']:
|
|
raise ValueError('Unexpected accepted program')
|
|
for row in (n for n in Document(program_html).root.walk() if n.tag == 'tr'):
|
|
cells = [n for n in row.children if isinstance(n, Node) and n.tag == 'td']
|
|
if not cells:
|
|
continue
|
|
if len(cells) != 4 or not re.fullmatch('[1-9][0-9]*', clean(cells[0].text())):
|
|
raise ValueError('Unexpected program row')
|
|
pid = int(clean(cells[0].text()))
|
|
if pid in program:
|
|
raise ValueError('Repeated program ID')
|
|
preview = clean(' '.join(n for n in cells[3].children if isinstance(n, str)))
|
|
full = one([clean(n.text()) for n in cells[3].walk() if n.tag == 'div'
|
|
and 'display:none' in n.attrs.get('style', '').replace(' ', '')])
|
|
if not preview or not full.startswith(preview):
|
|
raise ValueError('Author preview is not a prefix of complete author evidence')
|
|
links = [n.attrs.get('href') for n in cells[2].walk() if n.tag == 'a']
|
|
if links != [f'/2026/program/papers/{pid}/']:
|
|
raise ValueError('Program display link changed')
|
|
program[pid] = {'display_id': pid, 'title': clean(cells[2].text()), 'authors_complete': full,
|
|
'authors_preview': preview, 'session': clean(cells[1].text()),
|
|
'program_url': 'https://roboticsconference.org'+links[0]}
|
|
if set(index) != set(program) or set(index) != set(range(1, 211)):
|
|
raise ValueError('Reviewed 210-entry source census changed')
|
|
return index, program
|
|
|
|
|
|
def parse_paper(html, pid, index, program):
|
|
nodes = list(Document(html).root.walk()); meta = {}
|
|
for n in nodes:
|
|
if n.tag == 'meta' and n.attrs.get('name', '').startswith('citation_'):
|
|
meta.setdefault(n.attrs['name'], []).append(n.attrs.get('content', ''))
|
|
title = one(meta.get('citation_title', [])); authors = meta.get('citation_author', [])
|
|
day = one(meta.get('citation_publication_date', []))
|
|
if not re.fullmatch(r'2026/[0-9]{2}/[0-9]{2}', day):
|
|
raise ValueError('Unexpected publication date')
|
|
date.fromisoformat(day.replace('/', '-'))
|
|
if (meta.get('citation_volume') != ['22'] or meta.get('citation_conference_title') != [HEADING]
|
|
or [clean(n.text()) for n in nodes if n.tag == 'h1'] != [HEADING]
|
|
or not authors or any(not clean(a) for a in authors)
|
|
or comparable(title) != comparable(index['title'])
|
|
or comparable(', '.join(authors)) != comparable(index['authors'])
|
|
or [comparable(n.text()) for n in nodes if n.tag == 'h3'] != [comparable(title)]):
|
|
raise ValueError('Publisher citation/index/visible metadata disagree')
|
|
pdf = one(meta.get('citation_pdf_url', []))
|
|
if pdf not in (ROOT+f'p{pid:03d}.pdf', ROOT.replace('https:', 'http:')+f'p{pid:03d}.pdf'):
|
|
raise ValueError('Citation PDF ID mismatch; PDFs are never fetched')
|
|
bib = one([n.text() for n in nodes if n.tag == 'pre'])
|
|
doi = one(re.findall(r'^\s*DOI\s*=\s*\{([^{}]+)\}', bib, re.I | re.M))
|
|
year = one(re.findall(r'^\s*YEAR\s*=\s*\{([^{}]+)\}', bib, re.I | re.M))
|
|
if doi != f'10.15607/RSS.2026.XXII.{pid:03d}' or year != '2026':
|
|
raise ValueError('BibTeX DOI/year mismatch')
|
|
variants = []
|
|
if comparable(', '.join(authors)) != comparable(program['authors_complete']):
|
|
key = digest(json.dumps([authors, program['authors_complete']], ensure_ascii=False, separators=(',', ':')).encode())
|
|
if REVIEWED_AUTHOR_VARIANTS.get(pid) != key:
|
|
raise ValueError('Unreviewed complete author variant')
|
|
variants.append('reviewed_program_author_name_variant_not_person_alias')
|
|
if comparable(title) != comparable(program['title']):
|
|
if (pid, title, program['title']) != (89,
|
|
'RLinf-VLA: A Unified and Efficient Framework for Reinforcement Learning of Vision-Language-Action Models',
|
|
'RLux-VLA: A Unified and Efficient Framework for Reinforcement Learning of Vision-Language-Action Models'):
|
|
raise ValueError('Unreviewed program title variant')
|
|
variants.append('reviewed_program_title_variant_not_erratum')
|
|
inside = False; parts = []; markers = []
|
|
for n in (n for n in nodes if n.tag == 'p'):
|
|
labels = [clean(b.text()) for b in n.walk() if b.tag == 'b']
|
|
if 'Abstract:' in labels:
|
|
inside = True; markers.append('start'); value = clean(n.text()).removeprefix('Abstract:').strip()
|
|
if value: parts.append(value)
|
|
elif 'Download:' in labels:
|
|
inside = False; markers.append('end')
|
|
elif inside:
|
|
parts.append(clean(n.text()))
|
|
if markers != ['start', 'end'] or not any(parts):
|
|
raise ValueError('Missing or changed publisher abstract layout')
|
|
return {'title': title, 'authors': authors, 'abstract': '\n\n'.join(v for v in parts if v),
|
|
'doi_raw': doi, 'publication_date': day, 'bibtex': bib, 'program_variants': variants}
|
|
|
|
|
|
def collect(out, *, snapshots=None, cache=None):
|
|
if (snapshots is None) == (cache is None):
|
|
raise ValueError('Specify snapshots or cache')
|
|
out = Path(out)
|
|
if out.exists():
|
|
raise ValueError('Output directory must not exist; inspect partial/completed runs first')
|
|
out.parent.mkdir(parents=True, exist_ok=True)
|
|
out.mkdir()
|
|
evidence = {}; receipts = {}
|
|
if snapshots is not None:
|
|
root = Path(snapshots)
|
|
try:
|
|
for r in json.loads((root/'receipts.json').read_bytes()):
|
|
name = r['file']
|
|
if not isinstance(name, str) or Path(name).name != name or name in receipts:
|
|
raise ValueError('Unsafe or repeated snapshot receipt')
|
|
receipts[name] = r
|
|
except Exception as error:
|
|
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
|
raise
|
|
def page(name, url):
|
|
if snapshots is not None:
|
|
path = root/name
|
|
if path.is_symlink(): raise ValueError('Snapshot symlink rejected')
|
|
raw = path.read_bytes(); r = receipts[name]
|
|
if (r['url'] != url or r.get('final_url', url) != url or r['http_status'] != 200
|
|
or r['bytes'] != len(raw) or r['sha256'] != digest(raw)):
|
|
raise ValueError('Snapshot hash/receipt mismatch')
|
|
at = r['at']
|
|
else:
|
|
raw = fetch(url, cache, limit=8*1024*1024)
|
|
sidecar = json.loads((Path(cache)/digest(url.encode())).with_suffix('.json').read_bytes())
|
|
if sidecar['sha256'] != digest(raw): raise ValueError('Cache receipt mismatch')
|
|
at = sidecar['retrieved_at']
|
|
if len(raw) > 8*1024*1024 or datetime.fromisoformat(at.replace('Z', '+00:00')).utcoffset() is None:
|
|
raise ValueError('Invalid source size/timezone')
|
|
relative = 'evidence/'+digest(raw)+'.html'; target = out/relative
|
|
target.parent.mkdir(exist_ok=True); target.write_bytes(raw)
|
|
evidence[name] = {'file': relative, 'sha256': digest(raw), 'url': url, 'observed_at': at}
|
|
return raw.decode('utf-8')
|
|
try:
|
|
pages = {name: page(name, url) for name, url in PAGES.items()}
|
|
index, program = parse_lists(pages['index.html'], pages['accepted.html'], pages['cfp.html'], pages['sponsor-demos.html'])
|
|
records = []
|
|
for pid in sorted(index):
|
|
name = f'p{pid:03d}.html'; fields = parse_paper(page(name, ROOT+name), pid, index[pid], program[pid])
|
|
row = {'source': SOURCE, 'external_id': f'2026/p{pid:03d}', 'url': ROOT+name,
|
|
'title': fields['title'], 'abstract': fields['abstract'], 'authors': fields['authors'],
|
|
'identifiers': {'doi': fields['doi_raw'].lower()}, 'venue': 'RSS', 'venue_aliases': [],
|
|
'year': 2026, 'track': 'main', 'status': 'accepted',
|
|
'raw_decision': 'listed in official RSS 2026 proceedings and accepted papers',
|
|
'observed_at': evidence[name]['observed_at'],
|
|
'metadata': {'publisher_date': fields['publication_date'], 'publisher_doi_raw': fields['doi_raw'],
|
|
'original_bibtex': fields['bibtex'], 'program': program[pid],
|
|
'program_variants': fields['program_variants'], 'source_evidence': evidence[name],
|
|
'index_evidence': evidence['index.html'], 'program_evidence': evidence['accepted.html'],
|
|
'policy_evidence': evidence['cfp.html'],
|
|
'scope': '2026 Science/Systems papers; no separate demo-paper category. Program display ID is not an original submission ID.'}}
|
|
records.append(row)
|
|
path = out/'rss-proceedings-2026-main.jsonl'
|
|
raw = b''.join((json.dumps(r, ensure_ascii=False)+'\n').encode() for r in records)
|
|
path.write_bytes(raw)
|
|
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'RSS', 'year': 2026, 'track': 'main',
|
|
'count': 210, 'file': str(path), 'file_sha256': digest(raw), 'list_url': ROOT,
|
|
'status': 'partial', 'missing_abstract_count': 0}
|
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
|
|
report = {'status': 'collected_pending_identity_review', 'source': SOURCE, 'records': 210,
|
|
'files': [manifest], 'source_evidence': evidence, 'database_written': False, 'pdfs_downloaded': False,
|
|
'limitations': 'Publisher metadata only; not database import, scientific-work deduplication, private reviews or a public rejection set.'}
|
|
temporary = out/'collection-report.json.part'; temporary.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n')
|
|
temporary.replace(out/'collection-report.json')
|
|
return report
|
|
except Exception as error:
|
|
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
|
raise
|
|
|
|
|
|
def main(argv=None, *, prog=None):
|
|
p = argparse.ArgumentParser(prog=prog, description=__doc__)
|
|
p.add_argument('--out', required=True, type=Path)
|
|
g = p.add_mutually_exclusive_group(required=True)
|
|
g.add_argument('--snapshots', type=Path); g.add_argument('--cache', type=Path)
|
|
args = p.parse_args(argv)
|
|
report = collect(args.out, snapshots=args.snapshots, cache=args.cache)
|
|
print(json.dumps({'status': report['status'], 'records': report['records'], 'database_written': False}))
|