280 lines
15 KiB
Python
280 lines
15 KiB
Python
"""Reviewed RSS 2025 proceedings HTML; no PDF or private review access."""
|
|
import argparse
|
|
from collections import Counter
|
|
from datetime import date
|
|
import fcntl
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import tempfile
|
|
import unicodedata
|
|
|
|
from paper_atlas.db.submissions import timestamp
|
|
from paper_atlas.io.html import Document, Node, clean
|
|
from paper_atlas.io.http import fetch
|
|
|
|
SOURCE = 'rss_proceedings'
|
|
ROOT = 'https://www.roboticsproceedings.org/rss21/'
|
|
HEADING = 'Robotics: Science and Systems XXI'
|
|
PAGES = {'index': ROOT,
|
|
'accepted': 'https://roboticsconference.org/2025/program/papers/',
|
|
'demos': 'https://roboticsconference.org/2025/program/demos/',
|
|
'authorinfo': 'https://roboticsconference.org/2025/information/authorinfo/'}
|
|
EXPECTED = 163
|
|
DEMO_IDS = {2, 20, 21, 33, 58, 59, 62, 84, 86, 92, 93, 107, 129, 132, 141}
|
|
|
|
|
|
def one(values, description):
|
|
if len(values) != 1:
|
|
raise ValueError('Expected one '+description)
|
|
return values[0]
|
|
|
|
|
|
def comparable(text):
|
|
return unicodedata.normalize('NFC', clean(text))
|
|
|
|
|
|
def parse_index(html):
|
|
document = Document(html)
|
|
nodes = list(document.root.walk())
|
|
if [clean(n.text()) for n in nodes if n.tag == 'h1'] != [HEADING]:
|
|
raise ValueError('Unexpected RSS proceedings edition')
|
|
result = {}
|
|
for row in (n for n in nodes if n.tag == 'tr'):
|
|
links = [n for n in row.walk() if n.tag == 'a'
|
|
and re.fullmatch(r'p[0-9]{3}\.html', n.attrs.get('href', ''))]
|
|
if not links:
|
|
continue
|
|
link = one(links, 'proceedings entry link')
|
|
pid = int(link.attrs['href'][1:4])
|
|
title = clean(link.text())
|
|
authors = clean(one([n for n in row.walk() if n.tag == 'i'], 'index authors').text())
|
|
if pid in result or pid < 1 or not title or not authors:
|
|
raise ValueError('Empty or repeated RSS index entry')
|
|
result[pid] = {'title': title, 'authors_display': authors,
|
|
'url': ROOT+link.attrs['href']}
|
|
occurrences = sum(n.tag == 'a' and bool(re.fullmatch(r'p[0-9]{3}\.html', n.attrs.get('href', '')))
|
|
for n in nodes)
|
|
if not result or len(result) != occurrences:
|
|
raise ValueError('Unparsed or duplicate proceedings links')
|
|
return result
|
|
|
|
|
|
def parse_program(html, heading, columns):
|
|
nodes = list(Document(html).root.walk())
|
|
if [clean(n.text()) for n in nodes if n.tag == 'h1'] != [heading]:
|
|
raise ValueError('Unexpected RSS program page')
|
|
headers = [[clean(n.text()) for n in row.children if isinstance(n, Node) and n.tag == 'th']
|
|
for row in nodes if row.tag == 'tr']
|
|
if list(columns) not in headers:
|
|
raise ValueError('Unexpected RSS program columns')
|
|
result = {}
|
|
for row in (n for n in nodes if n.tag == 'tr'):
|
|
cells = [n for n in row.children if isinstance(n, Node) and n.tag == 'td']
|
|
if not cells:
|
|
continue
|
|
if len(cells) != len(columns):
|
|
raise ValueError('Unexpected RSS program row width')
|
|
raw_id = clean(cells[0].text())
|
|
if not re.fullmatch('[1-9][0-9]*', raw_id):
|
|
raise ValueError('Invalid program ID')
|
|
pid = int(raw_id)
|
|
if pid in result:
|
|
raise ValueError('Duplicate program ID')
|
|
result[pid] = {key: clean(node.text()) for key, node in zip(columns, cells)}
|
|
if not result[pid]['Title']:
|
|
raise ValueError('Missing program title')
|
|
if not result:
|
|
raise ValueError('No RSS program rows')
|
|
return result
|
|
|
|
|
|
def validate_program(index, accepted, demos, authorinfo):
|
|
if set(index) != set(accepted) or len(index) != EXPECTED or set(demos) != DEMO_IDS:
|
|
raise ValueError('RSS program and reviewed proceedings census differ')
|
|
text = clean(Document(authorinfo).root.text())
|
|
if ('There are two types of submissions to RSS' not in text
|
|
or 'Both types of papers appear in the RSS proceedings.' not in text):
|
|
raise ValueError('Submission category evidence changed')
|
|
for pid, item in index.items():
|
|
variants = [accepted[pid]['Title']] + ([demos[pid]['Title']] if pid in demos else [])
|
|
if any(comparable(title) != comparable(item['title']) for title in variants):
|
|
raise ValueError('Conflicting official program title for '+str(pid))
|
|
|
|
|
|
def parse_paper(html, pid, entry, accepted, demo, evidence):
|
|
nodes = list(Document(html).root.walk())
|
|
meta = {}
|
|
for node in nodes:
|
|
if node.tag == 'meta' and node.attrs.get('name', '').startswith('citation_'):
|
|
meta.setdefault(node.attrs['name'], []).append(node.attrs.get('content', ''))
|
|
title = one(meta.get('citation_title', []), 'citation title')
|
|
authors = meta.get('citation_author', [])
|
|
publication_date = one(meta.get('citation_publication_date', []), 'publication date')
|
|
if not re.fullmatch(r'2025/[0-9]{2}/[0-9]{2}', publication_date):
|
|
raise ValueError('Unexpected publication year/date')
|
|
date.fromisoformat(publication_date.replace('/', '-'))
|
|
if (one(meta.get('citation_conference_title', []), 'conference') != HEADING
|
|
or one(meta.get('citation_volume', []), 'volume') != '21'
|
|
or [clean(n.text()) for n in nodes if n.tag == 'h1'] != [HEADING]
|
|
or [comparable(n.text()) for n in nodes if n.tag == 'h3'] != [comparable(title)]
|
|
or comparable(title) != comparable(entry['title']) or not authors
|
|
or any(not clean(author) for author in authors)
|
|
or comparable(', '.join(authors)) != comparable(entry['authors_display'])):
|
|
raise ValueError('Proceedings/citation title, authors or edition disagree')
|
|
pdf = one(meta.get('citation_pdf_url', []), 'citation PDF URL')
|
|
if pdf not in (ROOT+f'p{pid:03d}.pdf', ROOT.replace('https:', 'http:')+f'p{pid:03d}.pdf'):
|
|
raise ValueError('Citation PDF ID differs; URL is metadata only')
|
|
bibtex = one([n.text() for n in nodes if n.tag == 'pre'], 'original BibTeX')
|
|
doi = one(re.findall(r'^\s*DOI\s*=\s*\{([^{}]+)\}', bibtex, re.I | re.M), 'BibTeX DOI')
|
|
year = one(re.findall(r'^\s*YEAR\s*=\s*\{([^{}]+)\}', bibtex, re.I | re.M), 'BibTeX year')
|
|
if doi != f'10.15607/RSS.2025.XXI.{pid:03d}' or year != '2025':
|
|
raise ValueError('BibTeX DOI/year differs from volume and paper ID')
|
|
abstract_parts = []
|
|
inside = False
|
|
markers = []
|
|
for node in (n for n in nodes if n.tag == 'p'):
|
|
labels = [clean(n.text()) for n in node.walk() if n.tag == 'b']
|
|
if 'Abstract:' in labels:
|
|
markers.append('start')
|
|
inside = True
|
|
value = clean(node.text()).removeprefix('Abstract:').strip()
|
|
if value:
|
|
abstract_parts.append(value)
|
|
elif 'Download:' in labels:
|
|
markers.append('end')
|
|
inside = False
|
|
elif inside:
|
|
abstract_parts.append(clean(node.text()))
|
|
if markers != ['start', 'end']:
|
|
raise ValueError('Abstract boundaries missing or repeated')
|
|
abstract = clean(' '.join(abstract_parts)) or None
|
|
track = 'demo' if demo else 'proceedings_unspecified'
|
|
tags = [{'namespace': SOURCE+':session', 'value': accepted['Session']}]
|
|
if demo:
|
|
tags.append({'namespace': SOURCE+':program_role', 'value': 'demonstration'})
|
|
return {'source': SOURCE, 'external_id': f'2025/p{pid:03d}', 'url': entry['url'],
|
|
'title': title, 'abstract': abstract, 'authors': authors,
|
|
'identifiers': {'doi': doi.lower()},
|
|
'submission_identifiers': [{'scheme': 'publication_doi', 'value': doi.lower()}],
|
|
'venue': 'RSS', 'venue_aliases': [], 'year': 2025, 'track': track,
|
|
'status': 'accepted', 'observed_at': evidence['paper']['observed_at'],
|
|
'raw_decision': 'listed in official RSS 2025 proceedings and accepted program',
|
|
'tags': tags,
|
|
'metadata': {'publisher_paper_id': pid, 'citation_metadata': meta,
|
|
'publication_date': publication_date, 'original_bibtex': bibtex,
|
|
'index_entry': entry, 'accepted_program_entry': accepted,
|
|
'demo_program_entry': demo, 'source_evidence': evidence,
|
|
'missing_abstract_reason': None if abstract else 'empty_official_abstract_section',
|
|
'track_scope_note': ('Explicit official demo program ID; not inferred from title.' if demo else
|
|
'Proceedings and accepted list do not individually identify submission type; session is not a track.'),
|
|
'decision_date_precision': 'not_supplied',
|
|
'publication_evidence': 'Published proceedings and accepted program; not an original decision note.'}}
|
|
|
|
|
|
class Pages:
|
|
def __init__(self, *, cache=None, snapshots=None):
|
|
if bool(cache) == bool(snapshots):
|
|
raise ValueError('Choose exactly one cache or frozen snapshot directory')
|
|
self.cache = Path(cache) if cache else None
|
|
self.root = Path(snapshots).resolve() if snapshots else None
|
|
self.receipts = json.loads((self.root/'receipts.json').read_bytes())['receipts'] if self.root else []
|
|
self.evidence = {}
|
|
self.data = {}
|
|
|
|
def read(self, url):
|
|
if self.root:
|
|
receipt = one([r for r in self.receipts if r['url'] == url], 'exact frozen response receipt')
|
|
filename = Path(receipt['file'])
|
|
path = self.root/filename
|
|
if (filename.is_absolute() or '..' in filename.parts or not path.resolve().is_relative_to(self.root)
|
|
or receipt['http_status'] != 200 or receipt['content_type'] != 'text/html'
|
|
or receipt.get('final_url', url) != url):
|
|
raise ValueError('Unsafe or unsuccessful frozen response')
|
|
data = path.read_bytes()
|
|
observed = receipt['at']
|
|
else:
|
|
data = fetch(url, self.cache, 2*1024**2, delay=1)
|
|
receipt = json.loads((self.cache/hashlib.sha256(url.encode()).hexdigest()).with_suffix('.json').read_bytes())
|
|
observed = receipt['retrieved_at']
|
|
if (len(data) > 2*1024**2 or len(data) != receipt['bytes']
|
|
or hashlib.sha256(data).hexdigest() != receipt['sha256'] or receipt['url'] != url):
|
|
raise ValueError('Response checksum, URL or size differs')
|
|
evidence = {'url': url, 'sha256': hashlib.sha256(data).hexdigest(), 'bytes': len(data),
|
|
'file': 'evidence/'+hashlib.sha256(data).hexdigest()+'.html',
|
|
'observed_at': timestamp(observed), 'http_status': 200, 'response_receipt': receipt}
|
|
self.evidence[url] = evidence
|
|
self.data[url] = data
|
|
return data.decode('utf-8'), evidence
|
|
|
|
|
|
def collect(year, out, *, cache=None, snapshots=None):
|
|
if year != 2025:
|
|
raise ValueError('Only the independently reviewed RSS 2025 edition is supported')
|
|
out = Path(out)
|
|
out.mkdir(parents=True, exist_ok=True)
|
|
with (out/'rss.collect.lock').open('a') as lock:
|
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
report_path = out/'rss-2025.json'
|
|
if report_path.exists() or list(out.glob('rss-2025-*.jsonl')):
|
|
raise FileExistsError('Inspect existing output; never overwrite a completed or partial queue')
|
|
pages = Pages(cache=cache, snapshots=snapshots)
|
|
context = {key: pages.read(url) for key, url in PAGES.items()}
|
|
index = parse_index(context['index'][0])
|
|
accepted = parse_program(context['accepted'][0], 'Accepted Papers', ('ID', 'Session', 'Title', 'Authors'))
|
|
demos = parse_program(context['demos'][0], 'Demos', ('ID', 'Location', 'Title', 'Day', 'Time'))
|
|
validate_program(index, accepted, demos, context['authorinfo'][0])
|
|
records = []
|
|
for pid, entry in sorted(index.items()):
|
|
html, evidence = pages.read(entry['url'])
|
|
sources = {key: item[1] for key, item in context.items()}
|
|
sources['paper'] = evidence
|
|
records.append(parse_paper(html, pid, entry, accepted[pid], demos.get(pid), sources))
|
|
counts = Counter(r['track'] for r in records)
|
|
if counts != {'demo': len(DEMO_IDS), 'proceedings_unspecified': EXPECTED-len(DEMO_IDS)}:
|
|
raise ValueError('Track census differs; review official source changes')
|
|
files = []
|
|
with tempfile.TemporaryDirectory(prefix='.rss-', dir=out) as temporary:
|
|
stage = Path(temporary)
|
|
(stage/'evidence').mkdir()
|
|
for url, data in pages.data.items():
|
|
(stage/pages.evidence[url]['file']).write_bytes(data)
|
|
for track in sorted(counts):
|
|
group = [r for r in records if r['track'] == track]
|
|
path = stage/f'rss-2025-{track}.jsonl'
|
|
path.write_text(''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in group))
|
|
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'RSS', 'year': 2025,
|
|
'track': track, 'count': len(group), 'file': str(out/path.name),
|
|
'file_sha256': hashlib.sha256(path.read_bytes()).hexdigest(),
|
|
'list_url': ROOT, 'status': 'partial',
|
|
'missing_abstract_count': sum(r['abstract'] is None for r in group)}
|
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, ensure_ascii=False, indent=2)+'\n')
|
|
files.append(manifest)
|
|
result = {'source': SOURCE, 'year': 2025, 'records': len(records), 'files': files,
|
|
'source_evidence': list(pages.evidence.values()), 'database_written': False,
|
|
'pdfs_downloaded': False, 'status': 'collected_pending_identity_review',
|
|
'note': 'Only the published proceedings. Demo IDs have explicit program evidence; '
|
|
'other individual submission types remain unknown. No absent-based negative decisions. '
|
|
'Separate source identity review and independent production acceptance are required.'}
|
|
(stage/report_path.name).write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n')
|
|
(out/'evidence').mkdir(exist_ok=True)
|
|
for path in (stage/'evidence').iterdir():
|
|
path.replace(out/'evidence'/path.name)
|
|
for path in stage.iterdir():
|
|
if path.is_file() and path.name != report_path.name:
|
|
path.replace(out/path.name)
|
|
(stage/report_path.name).replace(report_path)
|
|
return result
|
|
|
|
|
|
def main(argv=None, *, prog=None):
|
|
parser = argparse.ArgumentParser(prog=prog)
|
|
parser.add_argument('--year', type=int, choices=[2025], required=True)
|
|
parser.add_argument('--out', required=True)
|
|
group = parser.add_mutually_exclusive_group(required=True)
|
|
group.add_argument('--cache')
|
|
group.add_argument('--snapshots')
|
|
args = parser.parse_args(argv)
|
|
print(json.dumps(collect(args.year, args.out, cache=args.cache, snapshots=args.snapshots), ensure_ascii=False))
|