Files
paper-atlas/paper_atlas/collectors/rss.py
T
codex-bot bf4d0cd38a
Metadata integrity checks / metadata (push) Failing after 1m58s
Collect RSS 2025 proceedings with explicit program evidence
2026-11-09 18:00:40 +08:00

280 lines
15 KiB
Python

"""Reviewed RSS 2025 proceedings HTML; no PDF or private review access."""
import argparse
from collections import Counter
from datetime import date
import fcntl
import hashlib
import json
from pathlib import Path
import re
import tempfile
import unicodedata
from paper_atlas.db.submissions import timestamp
from paper_atlas.io.html import Document, Node, clean
from paper_atlas.io.http import fetch
SOURCE = 'rss_proceedings'
ROOT = 'https://www.roboticsproceedings.org/rss21/'
HEADING = 'Robotics: Science and Systems XXI'
PAGES = {'index': ROOT,
'accepted': 'https://roboticsconference.org/2025/program/papers/',
'demos': 'https://roboticsconference.org/2025/program/demos/',
'authorinfo': 'https://roboticsconference.org/2025/information/authorinfo/'}
EXPECTED = 163
DEMO_IDS = {2, 20, 21, 33, 58, 59, 62, 84, 86, 92, 93, 107, 129, 132, 141}
def one(values, description):
if len(values) != 1:
raise ValueError('Expected one '+description)
return values[0]
def comparable(text):
return unicodedata.normalize('NFC', clean(text))
def parse_index(html):
document = Document(html)
nodes = list(document.root.walk())
if [clean(n.text()) for n in nodes if n.tag == 'h1'] != [HEADING]:
raise ValueError('Unexpected RSS proceedings edition')
result = {}
for row in (n for n in nodes if n.tag == 'tr'):
links = [n for n in row.walk() if n.tag == 'a'
and re.fullmatch(r'p[0-9]{3}\.html', n.attrs.get('href', ''))]
if not links:
continue
link = one(links, 'proceedings entry link')
pid = int(link.attrs['href'][1:4])
title = clean(link.text())
authors = clean(one([n for n in row.walk() if n.tag == 'i'], 'index authors').text())
if pid in result or pid < 1 or not title or not authors:
raise ValueError('Empty or repeated RSS index entry')
result[pid] = {'title': title, 'authors_display': authors,
'url': ROOT+link.attrs['href']}
occurrences = sum(n.tag == 'a' and bool(re.fullmatch(r'p[0-9]{3}\.html', n.attrs.get('href', '')))
for n in nodes)
if not result or len(result) != occurrences:
raise ValueError('Unparsed or duplicate proceedings links')
return result
def parse_program(html, heading, columns):
nodes = list(Document(html).root.walk())
if [clean(n.text()) for n in nodes if n.tag == 'h1'] != [heading]:
raise ValueError('Unexpected RSS program page')
headers = [[clean(n.text()) for n in row.children if isinstance(n, Node) and n.tag == 'th']
for row in nodes if row.tag == 'tr']
if list(columns) not in headers:
raise ValueError('Unexpected RSS program columns')
result = {}
for row in (n for n in nodes if n.tag == 'tr'):
cells = [n for n in row.children if isinstance(n, Node) and n.tag == 'td']
if not cells:
continue
if len(cells) != len(columns):
raise ValueError('Unexpected RSS program row width')
raw_id = clean(cells[0].text())
if not re.fullmatch('[1-9][0-9]*', raw_id):
raise ValueError('Invalid program ID')
pid = int(raw_id)
if pid in result:
raise ValueError('Duplicate program ID')
result[pid] = {key: clean(node.text()) for key, node in zip(columns, cells)}
if not result[pid]['Title']:
raise ValueError('Missing program title')
if not result:
raise ValueError('No RSS program rows')
return result
def validate_program(index, accepted, demos, authorinfo):
if set(index) != set(accepted) or len(index) != EXPECTED or set(demos) != DEMO_IDS:
raise ValueError('RSS program and reviewed proceedings census differ')
text = clean(Document(authorinfo).root.text())
if ('There are two types of submissions to RSS' not in text
or 'Both types of papers appear in the RSS proceedings.' not in text):
raise ValueError('Submission category evidence changed')
for pid, item in index.items():
variants = [accepted[pid]['Title']] + ([demos[pid]['Title']] if pid in demos else [])
if any(comparable(title) != comparable(item['title']) for title in variants):
raise ValueError('Conflicting official program title for '+str(pid))
def parse_paper(html, pid, entry, accepted, demo, evidence):
nodes = list(Document(html).root.walk())
meta = {}
for node in nodes:
if node.tag == 'meta' and node.attrs.get('name', '').startswith('citation_'):
meta.setdefault(node.attrs['name'], []).append(node.attrs.get('content', ''))
title = one(meta.get('citation_title', []), 'citation title')
authors = meta.get('citation_author', [])
publication_date = one(meta.get('citation_publication_date', []), 'publication date')
if not re.fullmatch(r'2025/[0-9]{2}/[0-9]{2}', publication_date):
raise ValueError('Unexpected publication year/date')
date.fromisoformat(publication_date.replace('/', '-'))
if (one(meta.get('citation_conference_title', []), 'conference') != HEADING
or one(meta.get('citation_volume', []), 'volume') != '21'
or [clean(n.text()) for n in nodes if n.tag == 'h1'] != [HEADING]
or [comparable(n.text()) for n in nodes if n.tag == 'h3'] != [comparable(title)]
or comparable(title) != comparable(entry['title']) or not authors
or any(not clean(author) for author in authors)
or comparable(', '.join(authors)) != comparable(entry['authors_display'])):
raise ValueError('Proceedings/citation title, authors or edition disagree')
pdf = one(meta.get('citation_pdf_url', []), 'citation PDF URL')
if pdf not in (ROOT+f'p{pid:03d}.pdf', ROOT.replace('https:', 'http:')+f'p{pid:03d}.pdf'):
raise ValueError('Citation PDF ID differs; URL is metadata only')
bibtex = one([n.text() for n in nodes if n.tag == 'pre'], 'original BibTeX')
doi = one(re.findall(r'^\s*DOI\s*=\s*\{([^{}]+)\}', bibtex, re.I | re.M), 'BibTeX DOI')
year = one(re.findall(r'^\s*YEAR\s*=\s*\{([^{}]+)\}', bibtex, re.I | re.M), 'BibTeX year')
if doi != f'10.15607/RSS.2025.XXI.{pid:03d}' or year != '2025':
raise ValueError('BibTeX DOI/year differs from volume and paper ID')
abstract_parts = []
inside = False
markers = []
for node in (n for n in nodes if n.tag == 'p'):
labels = [clean(n.text()) for n in node.walk() if n.tag == 'b']
if 'Abstract:' in labels:
markers.append('start')
inside = True
value = clean(node.text()).removeprefix('Abstract:').strip()
if value:
abstract_parts.append(value)
elif 'Download:' in labels:
markers.append('end')
inside = False
elif inside:
abstract_parts.append(clean(node.text()))
if markers != ['start', 'end']:
raise ValueError('Abstract boundaries missing or repeated')
abstract = clean(' '.join(abstract_parts)) or None
track = 'demo' if demo else 'proceedings_unspecified'
tags = [{'namespace': SOURCE+':session', 'value': accepted['Session']}]
if demo:
tags.append({'namespace': SOURCE+':program_role', 'value': 'demonstration'})
return {'source': SOURCE, 'external_id': f'2025/p{pid:03d}', 'url': entry['url'],
'title': title, 'abstract': abstract, 'authors': authors,
'identifiers': {'doi': doi.lower()},
'submission_identifiers': [{'scheme': 'publication_doi', 'value': doi.lower()}],
'venue': 'RSS', 'venue_aliases': [], 'year': 2025, 'track': track,
'status': 'accepted', 'observed_at': evidence['paper']['observed_at'],
'raw_decision': 'listed in official RSS 2025 proceedings and accepted program',
'tags': tags,
'metadata': {'publisher_paper_id': pid, 'citation_metadata': meta,
'publication_date': publication_date, 'original_bibtex': bibtex,
'index_entry': entry, 'accepted_program_entry': accepted,
'demo_program_entry': demo, 'source_evidence': evidence,
'missing_abstract_reason': None if abstract else 'empty_official_abstract_section',
'track_scope_note': ('Explicit official demo program ID; not inferred from title.' if demo else
'Proceedings and accepted list do not individually identify submission type; session is not a track.'),
'decision_date_precision': 'not_supplied',
'publication_evidence': 'Published proceedings and accepted program; not an original decision note.'}}
class Pages:
def __init__(self, *, cache=None, snapshots=None):
if bool(cache) == bool(snapshots):
raise ValueError('Choose exactly one cache or frozen snapshot directory')
self.cache = Path(cache) if cache else None
self.root = Path(snapshots).resolve() if snapshots else None
self.receipts = json.loads((self.root/'receipts.json').read_bytes())['receipts'] if self.root else []
self.evidence = {}
self.data = {}
def read(self, url):
if self.root:
receipt = one([r for r in self.receipts if r['url'] == url], 'exact frozen response receipt')
filename = Path(receipt['file'])
path = self.root/filename
if (filename.is_absolute() or '..' in filename.parts or not path.resolve().is_relative_to(self.root)
or receipt['http_status'] != 200 or receipt['content_type'] != 'text/html'
or receipt.get('final_url', url) != url):
raise ValueError('Unsafe or unsuccessful frozen response')
data = path.read_bytes()
observed = receipt['at']
else:
data = fetch(url, self.cache, 2*1024**2, delay=1)
receipt = json.loads((self.cache/hashlib.sha256(url.encode()).hexdigest()).with_suffix('.json').read_bytes())
observed = receipt['retrieved_at']
if (len(data) > 2*1024**2 or len(data) != receipt['bytes']
or hashlib.sha256(data).hexdigest() != receipt['sha256'] or receipt['url'] != url):
raise ValueError('Response checksum, URL or size differs')
evidence = {'url': url, 'sha256': hashlib.sha256(data).hexdigest(), 'bytes': len(data),
'file': 'evidence/'+hashlib.sha256(data).hexdigest()+'.html',
'observed_at': timestamp(observed), 'http_status': 200, 'response_receipt': receipt}
self.evidence[url] = evidence
self.data[url] = data
return data.decode('utf-8'), evidence
def collect(year, out, *, cache=None, snapshots=None):
if year != 2025:
raise ValueError('Only the independently reviewed RSS 2025 edition is supported')
out = Path(out)
out.mkdir(parents=True, exist_ok=True)
with (out/'rss.collect.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
report_path = out/'rss-2025.json'
if report_path.exists() or list(out.glob('rss-2025-*.jsonl')):
raise FileExistsError('Inspect existing output; never overwrite a completed or partial queue')
pages = Pages(cache=cache, snapshots=snapshots)
context = {key: pages.read(url) for key, url in PAGES.items()}
index = parse_index(context['index'][0])
accepted = parse_program(context['accepted'][0], 'Accepted Papers', ('ID', 'Session', 'Title', 'Authors'))
demos = parse_program(context['demos'][0], 'Demos', ('ID', 'Location', 'Title', 'Day', 'Time'))
validate_program(index, accepted, demos, context['authorinfo'][0])
records = []
for pid, entry in sorted(index.items()):
html, evidence = pages.read(entry['url'])
sources = {key: item[1] for key, item in context.items()}
sources['paper'] = evidence
records.append(parse_paper(html, pid, entry, accepted[pid], demos.get(pid), sources))
counts = Counter(r['track'] for r in records)
if counts != {'demo': len(DEMO_IDS), 'proceedings_unspecified': EXPECTED-len(DEMO_IDS)}:
raise ValueError('Track census differs; review official source changes')
files = []
with tempfile.TemporaryDirectory(prefix='.rss-', dir=out) as temporary:
stage = Path(temporary)
(stage/'evidence').mkdir()
for url, data in pages.data.items():
(stage/pages.evidence[url]['file']).write_bytes(data)
for track in sorted(counts):
group = [r for r in records if r['track'] == track]
path = stage/f'rss-2025-{track}.jsonl'
path.write_text(''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in group))
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'RSS', 'year': 2025,
'track': track, 'count': len(group), 'file': str(out/path.name),
'file_sha256': hashlib.sha256(path.read_bytes()).hexdigest(),
'list_url': ROOT, 'status': 'partial',
'missing_abstract_count': sum(r['abstract'] is None for r in group)}
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, ensure_ascii=False, indent=2)+'\n')
files.append(manifest)
result = {'source': SOURCE, 'year': 2025, 'records': len(records), 'files': files,
'source_evidence': list(pages.evidence.values()), 'database_written': False,
'pdfs_downloaded': False, 'status': 'collected_pending_identity_review',
'note': 'Only the published proceedings. Demo IDs have explicit program evidence; '
'other individual submission types remain unknown. No absent-based negative decisions. '
'Separate source identity review and independent production acceptance are required.'}
(stage/report_path.name).write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n')
(out/'evidence').mkdir(exist_ok=True)
for path in (stage/'evidence').iterdir():
path.replace(out/'evidence'/path.name)
for path in stage.iterdir():
if path.is_file() and path.name != report_path.name:
path.replace(out/path.name)
(stage/report_path.name).replace(report_path)
return result
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog)
parser.add_argument('--year', type=int, choices=[2025], required=True)
parser.add_argument('--out', required=True)
group = parser.add_mutually_exclusive_group(required=True)
group.add_argument('--cache')
group.add_argument('--snapshots')
args = parser.parse_args(argv)
print(json.dumps(collect(args.year, args.out, cache=args.cache, snapshots=args.snapshots), ensure_ascii=False))