Compare commits
7 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 19d03ced54 | |||
| c33f5422de | |||
| 1cfd44d8b6 | |||
| 5de7f6ff22 | |||
| 93f00d1a64 | |||
| 8b7bc6d871 | |||
| 7588a4e904 |
@@ -48,6 +48,28 @@ jobs:
|
||||
outside.mkdir()
|
||||
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--help'],cwd=outside,check=True)
|
||||
rss_fixture = json.loads(gzip.decompress((src/'tests/fixtures/rss2026-source.json.gz').read_bytes()))
|
||||
rss_snapshots = root/'rss2026-snapshots'
|
||||
rss_snapshots.mkdir()
|
||||
for name, html in rss_fixture['pages'].items():
|
||||
(rss_snapshots/name).write_bytes(html.encode('utf-8'))
|
||||
(rss_snapshots/'receipts.json').write_text(json.dumps(rss_fixture['receipts']))
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--snapshots',str(rss_snapshots),'--out',str(root/'installed-rss2026')],cwd=outside,check=True)
|
||||
assert json.loads((root/'installed-rss2026/collection-report.json').read_text())['records'] == 210
|
||||
corl_fixture = json.loads(gzip.decompress((src/'tests/fixtures/corl2026-source.json.gz').read_bytes()))
|
||||
corl_official = root/'corl2026-official'
|
||||
corl_archive = root/'corl2026-archive'
|
||||
corl_official.mkdir()
|
||||
corl_archive.mkdir()
|
||||
(corl_official/'accepted.html').write_text(corl_fixture['official_html'])
|
||||
(corl_official/'research.json').write_text(json.dumps(corl_fixture['research']))
|
||||
(corl_archive/'explorer-paper-metadata-filtered-v1.json').write_text(corl_fixture['filtered'])
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','corl2026-program','--official-snapshots',str(corl_official),'--archive-snapshots',str(corl_archive),'--out',str(root/'installed-corl2026')],cwd=outside,check=True)
|
||||
corl_report = json.loads((root/'installed-corl2026/collection-report.json').read_text())
|
||||
assert corl_report['records'] == 687 and corl_report['presentations'] == 780
|
||||
assert len(corl_report['title_variants']) == 34 and corl_report['public_rejected_availability'] == 'unknown'
|
||||
assert not corl_report['database_written'] and not corl_report['pdfs_downloaded']
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','www-archive','--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','neurips-program','--help'],cwd=outside,check=True)
|
||||
|
||||
@@ -118,3 +118,9 @@ CI 在已授权的远程 runner 上运行测试、构建 wheel,并从源码目
|
||||
已导入整卷的逐条轨道修正见 [edition-tracks 维护流程](docs/edition-tracks.md):按完整证据生成只读计划,保留原投稿和决定 ID,事务性修正 edition、标识与覆盖,并防止旧整卷刷新覆盖分类。
|
||||
|
||||
入库前可使用通用的[来源身份只读预检](docs/source-identity-audit.md),导出候选论文、稳定标识和既有投稿决定证据;候选结果需要审核,不会自动批准导入。
|
||||
|
||||
RSS 2026 已独立适配官方论文集与录用表;使用方法和与 2025 demo 分类的区别见 [RSS 2026 来源说明](docs/rss2026-proceedings.md)。
|
||||
|
||||
### CoRL 2026 官方程序快照
|
||||
|
||||
`collect corl2026-program` 将官方 687 个论坛与固定公开档案按 ID 对照,保留 34 条标题版本差异及分字段来源。不会自动接受 arXiv 匹配或推断拒稿;详情见 [CoRL 2026 使用与恢复说明](docs/corl2026-program.md)。
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
# CoRL 2026:官方程序与公开档案
|
||||
|
||||
这是长期维护、按需运行的来源适配器。命令只读已冻结的元数据快照,生成待核验文件;不连接数据库,不抓取 PDF、评审或作者个人档案,不执行网页脚本,也不尝试通过 OpenReview 验证挑战。
|
||||
|
||||
```sh
|
||||
paper-atlas collect corl2026-program \
|
||||
--official-snapshots /data/paper-index/collected/corl2026-official-listing-research-v1 \
|
||||
--archive-snapshots /data/paper-index/collected/corl2026-public-archive-inventory-v1 \
|
||||
--out /data/paper-index/collected/corl2026-normalized-v1
|
||||
```
|
||||
|
||||
官方快照包含 `accepted.html` 和 `research.json` 的 HTTP 回执。适配器解析 Google Sites 的 `data-code`,不把外层文本提取为空误作无论文。687 个论坛有 780 次展示:687 次海报、21 次口头、72 次 Spotlight。论坛 ID 确定投稿身份,展示形式不是新的投稿或 track。
|
||||
|
||||
作者及摘要来自 `gisbi-kim/conference-explorers` 的固定提交 `75b3f7a04b3a77c454540fe2506bfa615b8e038e`,经字段过滤后的 `explorer-paper-metadata-filtered-v1.json`。输入、上游原始数据 SHA-256 及 Git blob 均固定;改变来源需要另行研究、更新快照及检查,不能静默接受。
|
||||
|
||||
这是独立整理者的版本证据,并非本次重新核验的 OpenReview 原页。每条保留档案标题、作者、摘要、分字段来源 URL、官方标题及全部展示场次。34 个官方与档案标题差异会显式标记,不能视作勘误或自动作者别名。规范记录中的标题与作者摘要保持档案版本一致;官方标题单独保留,避免拼成没有来源的版本。
|
||||
|
||||
档案的 arXiv 链接和匹配标题只存为未批准候选,不进入作品标识;不沿用其推断机构、个人档案、作者 ID 或生成标签。只有 OpenReview 论坛 ID 进入作品和投稿标识。
|
||||
|
||||
所有输出均为 partial。官方政策只证实录用论文及评审公开,没有证实 rejected 集可见性。不能由此生成拒稿记录,也不能声称这 687 条覆盖了公开拒稿或全部论文关系。
|
||||
|
||||
输出目录必须全新;失败留下 `failed.json`,成功报告最后原子发布。恢复时先检查原目录,不覆盖失败现场或重跑已消费的导入队列。生成文件仍需元数据版本审阅、当前数据库身份候选检查、独立保护基线及导入后核验,最后另做全库去重。
|
||||
|
||||
测试和安装验证只在用户授权的 `ces-control:host` runner 上运行。
|
||||
@@ -0,0 +1,21 @@
|
||||
# RSS 2026 论文集来源
|
||||
|
||||
`collect rss2026-proceedings` 是独立的年度来源适配器,不依赖尚未合入主线的 RSS 2025 PR #43,也不改变其 demo 分类。使用包内 HTML、HTTP 工具和统一 CLI;不连接数据库,不下载 PDF。
|
||||
|
||||
```sh
|
||||
paper-atlas collect rss2026-proceedings --cache /data/paper-index/cache/rss2026 \
|
||||
--out /data/paper-index/collected/rss2026-native-v1
|
||||
# 固定证据复现:包含 receipts.json 和对应 HTML 文件
|
||||
paper-atlas collect rss2026-proceedings --snapshots /data/paper-index/collected/rss2026-primary-v1 \
|
||||
--out /data/paper-index/collected/rss2026-native-replay-v1
|
||||
```
|
||||
|
||||
核对的来源为 [第22卷论文集](https://www.roboticsproceedings.org/rss22/)、[录用表](https://roboticsconference.org/2026/program/papers/)、[CFP](https://roboticsconference.org/2026/information/cfp/) 和 [Sponsor Demos](https://roboticsconference.org/2026/program/demos/)。当前审查目录为210篇。CFP明确2026没有单独 demo paper 类别,Science/Systems论文放在 main;赞助商展示不作为论文,也不推断逐篇是否现场演示。投稿录用后才公开、评审不公开;此来源不能提供拒稿或评审全集。
|
||||
|
||||
每篇保留出版社原始标题、完整作者顺序、摘要、日期、BibTeX、原始 DOI,以及独立节目标题、完整作者和缩略显示、session、显示链接。节目显示编号不是原始 OpenReview 投稿编号,不写入 submission_identifiers。六个缩略作者预览按完整隐藏副本恢复核验,不假设固定字符长度;六个姓名差异和一处 RLinf/RLux 标题差异按已审核原文指纹识别并保留,不推断人名别名或官方勘误。未知差异会停止。
|
||||
|
||||
快照收据必须逐文件匹配固定URL、HTTP200、字节数、哈希和带时区的真实采集时间;拒绝路径穿越、重复收据和符号链接。缓存模式只请求已审核的固定公开HTML URL,访问限制由HTTP工具正常报错,不绕过。每份原始HTML按哈希保存到输出 evidence/。
|
||||
|
||||
输出目录必须不存在;失败目录留存供检查,换新目录重试。`collection-report.json` 最后原子写入,缺少此文件不能视为完成。清单始终 partial;已采集不代表主库完整、身份已核验或科学作品已去重。DOI也是待核对的候选标识,入库前仍需实时 PostgreSQL 身份审计、版本/作者关联复核和保护基线;入库后独立验收,所有导入结束后再做全库去重。
|
||||
|
||||
2026原始来源审计已冻结214份HTML且210篇均有摘要。回归 fixture 同时包含原始来源和独立生成的逐篇预期值,测试比较210篇完整标题/作者/摘要/DOI/日期;构建和测试仅在已授权 ces-control:host 远程 runner 进行。
|
||||
@@ -23,9 +23,11 @@ COMMANDS = {
|
||||
('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'),
|
||||
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
|
||||
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
|
||||
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
||||
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
||||
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
|
||||
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
|
||||
('collect','corl2026-program'): ('paper_atlas.collectors.corl2026', {}, '核验 CoRL 2026 官方论坛名单与分字段公开档案'),
|
||||
('collect','neurips-program'): ('paper_atlas.collectors.neurips_program', {}, '按公开节目来源区分 NeurIPS 2026 投稿与重复展示'),
|
||||
('collect','www-archive'): ('paper_atlas.collectors.www_archive', {}, '核验早期WWW官方档案并分开待核查条目'),
|
||||
('collect','dblp'): ('paper_atlas.collectors.dblp_records', {}, '核对并整理 DBLP 历史发表记录'),
|
||||
|
||||
@@ -0,0 +1,183 @@
|
||||
"""Frozen CoRL 2026 official program joined to attributed public archive metadata."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import unicodedata
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
from paper_atlas.io.html import Document, clean
|
||||
|
||||
SOURCE = 'corl_program_archive'
|
||||
OFFICIAL = 'https://2026.corl.org/program/accepted-papers'
|
||||
REVISION = '75b3f7a04b3a77c454540fe2506bfa615b8e038e'
|
||||
ARCHIVE = f'https://raw.githubusercontent.com/gisbi-kim/conference-explorers/{REVISION}/corl2026/data.json'
|
||||
OFFICIAL_SHA = '3f026639c1a777b0b5fbd132e1935a27cdd519ca0a04e60ca3b650a3bc7eaef4'
|
||||
FILTERED_SHA = '6e6d7ecd57ee139c61283188fec39ebe9dc7f5c30ec14c8c5839d1277b03a588'
|
||||
RAW_SHA = '1e96ed2dffcd96c2584a7ba020201c831e49ae1a69eae3ef1553fd5a34802810'
|
||||
BLOB = '322eeef71126b85e13a98fdce9c73fac81934d3f'
|
||||
FIELDS = {'id', 'title', 'openreview', 'poster_session', 'poster_day', 'poster_date',
|
||||
'poster_time', 'poster_venue', 'poster_number', 'index', 'presentation_type',
|
||||
'arxiv', 'matched_title', 'abstract', 'metadata_source', 'author_source',
|
||||
'abstract_source', 'identity_note', 'authors', 'openreview_updated', 'metadata_checked'}
|
||||
|
||||
|
||||
def digest(raw):
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def comparable(text):
|
||||
return ''.join(c for c in unicodedata.normalize('NFKD', text).casefold() if c.isalnum())
|
||||
|
||||
|
||||
def forum_id(url):
|
||||
if not isinstance(url, str):
|
||||
raise ValueError('Forum URL must be text')
|
||||
u = urlsplit(url); q = parse_qs(u.query, keep_blank_values=True)
|
||||
if (u.scheme != 'https' or u.netloc != 'openreview.net' or u.path != '/forum'
|
||||
or u.fragment or set(q) != {'id'} or len(q['id']) != 1
|
||||
or not re.fullmatch(r'[A-Za-z0-9_-]{8,32}', q['id'][0])):
|
||||
raise ValueError('Invalid original forum identity')
|
||||
return q['id'][0]
|
||||
|
||||
|
||||
def parse_program(raw):
|
||||
embeds = [n.attrs['data-code'] for n in Document(raw).root.walk() if 'data-code' in n.attrs]
|
||||
if len(embeds) != 1:
|
||||
raise ValueError('Expected one official embedded program')
|
||||
doc = Document(embeds[0]); nodes = list(doc.root.walk())
|
||||
if [clean(n.text()) for n in nodes if n.tag == 'h1'] != ['CoRL 2026 Accepted Papers']:
|
||||
raise ValueError('Unexpected program edition')
|
||||
summary = [clean(n.text()) for n in nodes if n.tag == 'p' and n.attrs.get('class') == 'sub']
|
||||
if len(summary) != 1 or not summary[0].startswith('687 accepted papers · 21 orals · 72 spotlights · 687 posters'):
|
||||
raise ValueError('Reviewed program census changed')
|
||||
papers = {}; sessions = {}; occurrences = []
|
||||
for section in (n for n in nodes if n.tag == 'section' and 'session' in n.attrs.get('class', '').split()):
|
||||
sid = section.attrs.get('data-sess', section.attrs.get('id'))
|
||||
if not isinstance(sid, str) or not re.fullmatch(r'(?:O[1-7]|S[1-8]|post-p[1-3])', sid) or sid in sessions:
|
||||
raise ValueError('Unexpected or repeated presentation session')
|
||||
kind = 'oral' if sid.startswith('O') else 'spotlight' if sid.startswith('S') else 'poster'
|
||||
headings = [clean(n.text()) for n in section.walk() if n.tag == 'h3']
|
||||
if len(headings) != 1:
|
||||
raise ValueError('Ambiguous session heading')
|
||||
sessions[sid] = {'kind': kind, 'heading': headings[0]}
|
||||
seen = set()
|
||||
for link in (n for n in section.walk() if n.tag == 'a' and 'openreview.net' in n.attrs.get('href', '')):
|
||||
fid = forum_id(link.attrs['href']); title = clean(link.text())
|
||||
if fid in seen or not title:
|
||||
raise ValueError('Repeated or empty paper in session')
|
||||
seen.add(fid)
|
||||
paper = papers.setdefault(fid, {'title': title, 'presentations': []})
|
||||
if paper['title'] != title:
|
||||
raise ValueError('Conflicting official titles share a forum')
|
||||
event = {'session_id': sid, **sessions[sid]}
|
||||
paper['presentations'].append(event); occurrences.append((fid, sid, kind))
|
||||
counts = Counter(kind for _, _, kind in occurrences)
|
||||
if len(papers) != 687 or len(sessions) != 18 or counts != {'poster': 687, 'oral': 21, 'spotlight': 72}:
|
||||
raise ValueError('Incomplete official program')
|
||||
for paper in papers.values():
|
||||
kinds = Counter(p['kind'] for p in paper['presentations'])
|
||||
if kinds['poster'] != 1 or len(paper['presentations']) not in (1, 2):
|
||||
raise ValueError('Invalid poster/talk multiplicity')
|
||||
return papers
|
||||
|
||||
|
||||
def join_metadata(program, archive):
|
||||
if (archive.get('repository') != 'gisbi-kim/conference-explorers'
|
||||
or archive.get('revision') != REVISION or archive.get('source_url') != ARCHIVE
|
||||
or archive.get('source_data_sha256') != RAW_SHA or archive.get('source_git_blob_sha1') != BLOB
|
||||
or not isinstance(archive.get('papers'), list)):
|
||||
raise ValueError('Unpinned public metadata provenance')
|
||||
indexed = {}
|
||||
for row in archive['papers']:
|
||||
if not isinstance(row, dict) or set(row) - FIELDS:
|
||||
raise ValueError('Unexpected metadata/profile fields')
|
||||
fid = forum_id(row.get('openreview'))
|
||||
if fid != row.get('id') or fid in indexed:
|
||||
raise ValueError('Repeated or mismatched archive forum')
|
||||
if any(forum_id(row.get(key)) != fid for key in ('metadata_source', 'author_source', 'abstract_source')):
|
||||
raise ValueError('Field source belongs to another submission')
|
||||
if (not isinstance(row.get('title'), str) or not row['title'].strip()
|
||||
or not isinstance(row.get('abstract'), str) or not row['abstract'].strip()
|
||||
or not isinstance(row.get('authors'), list) or not row['authors']
|
||||
or any(not isinstance(a, str) or not a.strip() or '@' in a or 'anonymous' in a.casefold() for a in row['authors'])):
|
||||
raise ValueError('Incomplete scholarly metadata')
|
||||
indexed[fid] = row
|
||||
if set(indexed) != set(program):
|
||||
raise ValueError('Official and archive forum sets differ')
|
||||
return indexed
|
||||
|
||||
|
||||
def collect(out, *, official_snapshots, archive_snapshots):
|
||||
out = Path(out)
|
||||
if out.exists():
|
||||
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||
out.mkdir(parents=True)
|
||||
try:
|
||||
official_root = Path(official_snapshots); archive_root = Path(archive_snapshots)
|
||||
official = official_root/'accepted.html'; filtered = archive_root/'explorer-paper-metadata-filtered-v1.json'
|
||||
if official.is_symlink() or filtered.is_symlink():
|
||||
raise ValueError('Snapshot symlinks rejected')
|
||||
raw = official.read_bytes(); data = filtered.read_bytes()
|
||||
if len(raw) > 8*1024**2 or len(data) > 8*1024**2 or digest(raw) != OFFICIAL_SHA or digest(data) != FILTERED_SHA:
|
||||
raise ValueError('Reviewed snapshot hash changed')
|
||||
receipts = json.loads((official_root/'research.json').read_bytes())['receipts']
|
||||
matches = [r for r in receipts if r.get('url') == OFFICIAL]
|
||||
if len(matches) != 1:
|
||||
raise ValueError('Missing official source receipt')
|
||||
receipt = matches[0]; observed = receipt['at']
|
||||
if (receipt.get('final_url') != OFFICIAL or receipt['http_status'] != 200
|
||||
or receipt['sha256'] != digest(raw) or receipt['bytes'] != len(raw)
|
||||
or datetime.fromisoformat(observed.replace('Z', '+00:00')).utcoffset() is None):
|
||||
raise ValueError('Invalid official receipt')
|
||||
program = parse_program(raw.decode('utf-8')); archive = json.loads(data); indexed = join_metadata(program, archive)
|
||||
evidence = out/'evidence'; evidence.mkdir()
|
||||
sources = {}
|
||||
for body, suffix, url, name in [(raw, 'html', OFFICIAL, 'program'), (data, 'json', ARCHIVE, 'archive')]:
|
||||
rel = f'evidence/{digest(body)}.{suffix}'; (out/rel).write_bytes(body)
|
||||
sources[name] = {'file': rel, 'sha256': digest(body), 'url': url}
|
||||
rows = []; variants = []
|
||||
for fid in sorted(program):
|
||||
p = program[fid]; a = indexed[fid]; variant = comparable(p['title']) != comparable(a['title'])
|
||||
if variant: variants.append({'forum': fid, 'official_title': p['title'], 'archive_title': a['title']})
|
||||
rows.append({'source': SOURCE, 'external_id': f'2026/{fid}', 'url': a['openreview'],
|
||||
'title': a['title'], 'authors': a['authors'], 'abstract': a['abstract'],
|
||||
'identifiers': {'openreview': fid}, 'submission_identifiers': [{'scheme': 'openreview', 'value': fid}],
|
||||
'venue': 'CoRL', 'year': 2026, 'track': 'main', 'status': 'accepted',
|
||||
'raw_decision': 'listed in official CoRL 2026 accepted program', 'evidence_url': OFFICIAL,
|
||||
'observed_at': observed, 'tags': [], 'metadata': {
|
||||
'official_program': p, 'archive_fields': a, 'title_variant_requires_review': variant,
|
||||
'source_evidence': sources, 'archive_revision': REVISION,
|
||||
'field_provenance': {k: a[k] for k in ('metadata_source', 'author_source', 'abstract_source')},
|
||||
'original_decision_signature_verified': False,
|
||||
'scope': 'Acceptance from official forum listing; title/authors/abstract are curator snapshot versions, not live original metadata. arXiv links remain unapproved candidates. No reject inference or person aliases.'}})
|
||||
if len(variants) != 34:
|
||||
raise ValueError('Reviewed title-variant census changed')
|
||||
path = out/'corl-program-archive-2026-main.jsonl'
|
||||
payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
|
||||
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'CoRL', 'year': 2026, 'track': 'main',
|
||||
'count': len(rows), 'file': str(path), 'file_sha256': digest(payload), 'list_url': OFFICIAL,
|
||||
'status': 'partial', 'missing_abstract_count': 0}
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
|
||||
report = {'status': 'collected_pending_identity_review', 'records': len(rows), 'presentations': 780,
|
||||
'files': [manifest], 'title_variants': variants, 'source_evidence': sources,
|
||||
'database_written': False, 'pdfs_downloaded': False, 'public_rejected_availability': 'unknown',
|
||||
'scope': 'Source collection only; requires identity and metadata-version review before import; no exhaustive deduplication or rejected-set claim.'}
|
||||
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||
return report
|
||||
except Exception as error:
|
||||
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||
raise
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||
parser.add_argument('--out', type=Path, required=True)
|
||||
parser.add_argument('--official-snapshots', type=Path, required=True)
|
||||
parser.add_argument('--archive-snapshots', type=Path, required=True)
|
||||
args = parser.parse_args(argv)
|
||||
report = collect(args.out, official_snapshots=args.official_snapshots, archive_snapshots=args.archive_snapshots)
|
||||
print(json.dumps({'status': report['status'], 'records': report['records'], 'database_written': False}))
|
||||
@@ -0,0 +1,222 @@
|
||||
"""Reviewed RSS 2026 publisher metadata, with separate program evidence."""
|
||||
REVIEWED_AUTHOR_VARIANTS = {16: '04de7e813b7c68b5c1764e90fe41bcd2e45259d46749bb2cc54bac5150c8f25b', 27: '975f6adc4d15045f9c8829689a759c5e3a44eecb3410a18ff99b37226d9c8b97', 78: '8208d0950468295c2b73dae28658566b1c726f65a4167f490039cf3449aa2db4', 132: '31eee6289de404f3d859dd0d9de4dd40e968f86b89312de41df87ae01c529711', 199: 'ced0bc9e38b28b4a490799fd94315f104ece0df28be428b3cc417db8ea797b9d', 204: '7c6959e6668bf44af09959388f6b9e2d883a76106bf7f191dc9949368c969ee5'}
|
||||
import argparse
|
||||
from datetime import datetime, date
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
from paper_atlas.io.html import Document, Node, clean
|
||||
from paper_atlas.io.http import fetch
|
||||
|
||||
SOURCE = 'rss_proceedings'
|
||||
ROOT = 'https://www.roboticsproceedings.org/rss22/'
|
||||
HEADING = 'Robotics: Science and Systems XXII'
|
||||
PAGES = {'index.html': ROOT,
|
||||
'accepted.html': 'https://roboticsconference.org/2026/program/papers/',
|
||||
'cfp.html': 'https://roboticsconference.org/2026/information/cfp/',
|
||||
'sponsor-demos.html': 'https://roboticsconference.org/2026/program/demos/'}
|
||||
|
||||
|
||||
def digest(raw):
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def one(values):
|
||||
if len(values) != 1:
|
||||
raise ValueError('Expected one reviewed HTML field')
|
||||
return values[0]
|
||||
|
||||
|
||||
def comparable(value):
|
||||
return unicodedata.normalize('NFC', clean(value))
|
||||
|
||||
|
||||
def parse_lists(index_html, program_html, policy_html, sponsors_html):
|
||||
policy = clean(Document(policy_html).root.text())
|
||||
for text in ('RSS 2026 will not have a separate category for demo papers.',
|
||||
'papers will only be public after acceptance', 'reviews will not be publicly visible'):
|
||||
if text not in policy:
|
||||
raise ValueError('Reviewed 2026 publication/demo/privacy policy changed')
|
||||
if [clean(n.text()) for n in Document(sponsors_html).root.walk() if n.tag == 'h1'] != ['Sponsor Demos']:
|
||||
raise ValueError('Sponsor activity page changed; never classify these as papers')
|
||||
nodes = list(Document(index_html).root.walk())
|
||||
if [clean(n.text()) for n in nodes if n.tag == 'h1'] != [HEADING]:
|
||||
raise ValueError('Unexpected publisher edition')
|
||||
index = {}
|
||||
for row in (n for n in nodes if n.tag == 'tr'):
|
||||
links = [n for n in row.walk() if n.tag == 'a' and re.fullmatch(r'p[0-9]{3}\.html', n.attrs.get('href', ''))]
|
||||
if not links:
|
||||
continue
|
||||
link = one(links); pid = int(link.attrs['href'][1:4])
|
||||
if pid in index:
|
||||
raise ValueError('Repeated publisher ID')
|
||||
index[pid] = {'title': clean(link.text()), 'authors': clean(one([n for n in row.walk() if n.tag == 'i']).text())}
|
||||
program = {}
|
||||
if [clean(n.text()) for n in Document(program_html).root.walk() if n.tag == 'h1'] != ['Accepted Papers']:
|
||||
raise ValueError('Unexpected accepted program')
|
||||
for row in (n for n in Document(program_html).root.walk() if n.tag == 'tr'):
|
||||
cells = [n for n in row.children if isinstance(n, Node) and n.tag == 'td']
|
||||
if not cells:
|
||||
continue
|
||||
if len(cells) != 4 or not re.fullmatch('[1-9][0-9]*', clean(cells[0].text())):
|
||||
raise ValueError('Unexpected program row')
|
||||
pid = int(clean(cells[0].text()))
|
||||
if pid in program:
|
||||
raise ValueError('Repeated program ID')
|
||||
preview = clean(' '.join(n for n in cells[3].children if isinstance(n, str)))
|
||||
full = one([clean(n.text()) for n in cells[3].walk() if n.tag == 'div'
|
||||
and 'display:none' in n.attrs.get('style', '').replace(' ', '')])
|
||||
if not preview or not full.startswith(preview):
|
||||
raise ValueError('Author preview is not a prefix of complete author evidence')
|
||||
links = [n.attrs.get('href') for n in cells[2].walk() if n.tag == 'a']
|
||||
if links != [f'/2026/program/papers/{pid}/']:
|
||||
raise ValueError('Program display link changed')
|
||||
program[pid] = {'display_id': pid, 'title': clean(cells[2].text()), 'authors_complete': full,
|
||||
'authors_preview': preview, 'session': clean(cells[1].text()),
|
||||
'program_url': 'https://roboticsconference.org'+links[0]}
|
||||
if set(index) != set(program) or set(index) != set(range(1, 211)):
|
||||
raise ValueError('Reviewed 210-entry source census changed')
|
||||
return index, program
|
||||
|
||||
|
||||
def parse_paper(html, pid, index, program):
|
||||
nodes = list(Document(html).root.walk()); meta = {}
|
||||
for n in nodes:
|
||||
if n.tag == 'meta' and n.attrs.get('name', '').startswith('citation_'):
|
||||
meta.setdefault(n.attrs['name'], []).append(n.attrs.get('content', ''))
|
||||
title = one(meta.get('citation_title', [])); authors = meta.get('citation_author', [])
|
||||
day = one(meta.get('citation_publication_date', []))
|
||||
if not re.fullmatch(r'2026/[0-9]{2}/[0-9]{2}', day):
|
||||
raise ValueError('Unexpected publication date')
|
||||
date.fromisoformat(day.replace('/', '-'))
|
||||
if (meta.get('citation_volume') != ['22'] or meta.get('citation_conference_title') != [HEADING]
|
||||
or [clean(n.text()) for n in nodes if n.tag == 'h1'] != [HEADING]
|
||||
or not authors or any(not clean(a) for a in authors)
|
||||
or comparable(title) != comparable(index['title'])
|
||||
or comparable(', '.join(authors)) != comparable(index['authors'])
|
||||
or [comparable(n.text()) for n in nodes if n.tag == 'h3'] != [comparable(title)]):
|
||||
raise ValueError('Publisher citation/index/visible metadata disagree')
|
||||
pdf = one(meta.get('citation_pdf_url', []))
|
||||
if pdf not in (ROOT+f'p{pid:03d}.pdf', ROOT.replace('https:', 'http:')+f'p{pid:03d}.pdf'):
|
||||
raise ValueError('Citation PDF ID mismatch; PDFs are never fetched')
|
||||
bib = one([n.text() for n in nodes if n.tag == 'pre'])
|
||||
doi = one(re.findall(r'^\s*DOI\s*=\s*\{([^{}]+)\}', bib, re.I | re.M))
|
||||
year = one(re.findall(r'^\s*YEAR\s*=\s*\{([^{}]+)\}', bib, re.I | re.M))
|
||||
if doi != f'10.15607/RSS.2026.XXII.{pid:03d}' or year != '2026':
|
||||
raise ValueError('BibTeX DOI/year mismatch')
|
||||
variants = []
|
||||
if comparable(', '.join(authors)) != comparable(program['authors_complete']):
|
||||
key = digest(json.dumps([authors, program['authors_complete']], ensure_ascii=False, separators=(',', ':')).encode())
|
||||
if REVIEWED_AUTHOR_VARIANTS.get(pid) != key:
|
||||
raise ValueError('Unreviewed complete author variant')
|
||||
variants.append('reviewed_program_author_name_variant_not_person_alias')
|
||||
if comparable(title) != comparable(program['title']):
|
||||
if (pid, title, program['title']) != (89,
|
||||
'RLinf-VLA: A Unified and Efficient Framework for Reinforcement Learning of Vision-Language-Action Models',
|
||||
'RLux-VLA: A Unified and Efficient Framework for Reinforcement Learning of Vision-Language-Action Models'):
|
||||
raise ValueError('Unreviewed program title variant')
|
||||
variants.append('reviewed_program_title_variant_not_erratum')
|
||||
inside = False; parts = []; markers = []
|
||||
for n in (n for n in nodes if n.tag == 'p'):
|
||||
labels = [clean(b.text()) for b in n.walk() if b.tag == 'b']
|
||||
if 'Abstract:' in labels:
|
||||
inside = True; markers.append('start'); value = clean(n.text()).removeprefix('Abstract:').strip()
|
||||
if value: parts.append(value)
|
||||
elif 'Download:' in labels:
|
||||
inside = False; markers.append('end')
|
||||
elif inside:
|
||||
parts.append(clean(n.text()))
|
||||
if markers != ['start', 'end'] or not any(parts):
|
||||
raise ValueError('Missing or changed publisher abstract layout')
|
||||
return {'title': title, 'authors': authors, 'abstract': '\n\n'.join(v for v in parts if v),
|
||||
'doi_raw': doi, 'publication_date': day, 'bibtex': bib, 'program_variants': variants}
|
||||
|
||||
|
||||
def collect(out, *, snapshots=None, cache=None):
|
||||
if (snapshots is None) == (cache is None):
|
||||
raise ValueError('Specify snapshots or cache')
|
||||
out = Path(out)
|
||||
if out.exists():
|
||||
raise ValueError('Output directory must not exist; inspect partial/completed runs first')
|
||||
out.parent.mkdir(parents=True, exist_ok=True)
|
||||
out.mkdir()
|
||||
evidence = {}; receipts = {}
|
||||
if snapshots is not None:
|
||||
root = Path(snapshots)
|
||||
try:
|
||||
for r in json.loads((root/'receipts.json').read_bytes()):
|
||||
name = r['file']
|
||||
if not isinstance(name, str) or Path(name).name != name or name in receipts:
|
||||
raise ValueError('Unsafe or repeated snapshot receipt')
|
||||
receipts[name] = r
|
||||
except Exception as error:
|
||||
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||
raise
|
||||
def page(name, url):
|
||||
if snapshots is not None:
|
||||
path = root/name
|
||||
if path.is_symlink(): raise ValueError('Snapshot symlink rejected')
|
||||
raw = path.read_bytes(); r = receipts[name]
|
||||
if (r['url'] != url or r.get('final_url', url) != url or r['http_status'] != 200
|
||||
or r['bytes'] != len(raw) or r['sha256'] != digest(raw)):
|
||||
raise ValueError('Snapshot hash/receipt mismatch')
|
||||
at = r['at']
|
||||
else:
|
||||
raw = fetch(url, cache, limit=8*1024*1024)
|
||||
sidecar = json.loads((Path(cache)/digest(url.encode())).with_suffix('.json').read_bytes())
|
||||
if sidecar['sha256'] != digest(raw): raise ValueError('Cache receipt mismatch')
|
||||
at = sidecar['retrieved_at']
|
||||
if len(raw) > 8*1024*1024 or datetime.fromisoformat(at.replace('Z', '+00:00')).utcoffset() is None:
|
||||
raise ValueError('Invalid source size/timezone')
|
||||
relative = 'evidence/'+digest(raw)+'.html'; target = out/relative
|
||||
target.parent.mkdir(exist_ok=True); target.write_bytes(raw)
|
||||
evidence[name] = {'file': relative, 'sha256': digest(raw), 'url': url, 'observed_at': at}
|
||||
return raw.decode('utf-8')
|
||||
try:
|
||||
pages = {name: page(name, url) for name, url in PAGES.items()}
|
||||
index, program = parse_lists(pages['index.html'], pages['accepted.html'], pages['cfp.html'], pages['sponsor-demos.html'])
|
||||
records = []
|
||||
for pid in sorted(index):
|
||||
name = f'p{pid:03d}.html'; fields = parse_paper(page(name, ROOT+name), pid, index[pid], program[pid])
|
||||
row = {'source': SOURCE, 'external_id': f'2026/p{pid:03d}', 'url': ROOT+name,
|
||||
'title': fields['title'], 'abstract': fields['abstract'], 'authors': fields['authors'],
|
||||
'identifiers': {'doi': fields['doi_raw'].lower()}, 'venue': 'RSS', 'venue_aliases': [],
|
||||
'year': 2026, 'track': 'main', 'status': 'accepted',
|
||||
'raw_decision': 'listed in official RSS 2026 proceedings and accepted papers',
|
||||
'observed_at': evidence[name]['observed_at'],
|
||||
'metadata': {'publisher_date': fields['publication_date'], 'publisher_doi_raw': fields['doi_raw'],
|
||||
'original_bibtex': fields['bibtex'], 'program': program[pid],
|
||||
'program_variants': fields['program_variants'], 'source_evidence': evidence[name],
|
||||
'index_evidence': evidence['index.html'], 'program_evidence': evidence['accepted.html'],
|
||||
'policy_evidence': evidence['cfp.html'],
|
||||
'scope': '2026 Science/Systems papers; no separate demo-paper category. Program display ID is not an original submission ID.'}}
|
||||
records.append(row)
|
||||
path = out/'rss-proceedings-2026-main.jsonl'
|
||||
raw = b''.join((json.dumps(r, ensure_ascii=False)+'\n').encode() for r in records)
|
||||
path.write_bytes(raw)
|
||||
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'RSS', 'year': 2026, 'track': 'main',
|
||||
'count': 210, 'file': str(path), 'file_sha256': digest(raw), 'list_url': ROOT,
|
||||
'status': 'partial', 'missing_abstract_count': 0}
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
|
||||
report = {'status': 'collected_pending_identity_review', 'source': SOURCE, 'records': 210,
|
||||
'files': [manifest], 'source_evidence': evidence, 'database_written': False, 'pdfs_downloaded': False,
|
||||
'limitations': 'Publisher metadata only; not database import, scientific-work deduplication, private reviews or a public rejection set.'}
|
||||
temporary = out/'collection-report.json.part'; temporary.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n')
|
||||
temporary.replace(out/'collection-report.json')
|
||||
return report
|
||||
except Exception as error:
|
||||
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||
raise
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
p = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||
p.add_argument('--out', required=True, type=Path)
|
||||
g = p.add_mutually_exclusive_group(required=True)
|
||||
g.add_argument('--snapshots', type=Path); g.add_argument('--cache', type=Path)
|
||||
args = p.parse_args(argv)
|
||||
report = collect(args.out, snapshots=args.snapshots, cache=args.cache)
|
||||
print(json.dumps({'status': report['status'], 'records': report['records'], 'database_written': False}))
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
@@ -0,0 +1,75 @@
|
||||
import gzip
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from paper_atlas.collectors import corl2026 as c
|
||||
|
||||
|
||||
class CoRL2026Test(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.bundle = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/corl2026-source.json.gz').read_bytes()))
|
||||
|
||||
def test_official_census_and_presentation_dedup(self):
|
||||
program = c.parse_program(self.bundle['official_html'])
|
||||
self.assertEqual(len(program), 687)
|
||||
self.assertEqual(sum(len(p['presentations']) for p in program.values()), 780)
|
||||
self.assertEqual(sum(len(p['presentations']) == 2 for p in program.values()), 93)
|
||||
|
||||
def test_conflicting_official_title_is_not_silently_folded(self):
|
||||
html = self.bundle['official_html']
|
||||
title = 'Diagnosing Compositional Generalization in Sequential Robot Tasks'
|
||||
self.assertIn(title, html)
|
||||
with self.assertRaises(ValueError):
|
||||
c.parse_program(html.replace(title, 'Different paper', 1))
|
||||
|
||||
def test_foreign_forum_source_is_rejected(self):
|
||||
program = c.parse_program(self.bundle['official_html']); archive = json.loads(self.bundle['filtered'])
|
||||
archive['papers'][0]['author_source'] = 'https://openreview.net/forum?id=WrongPaper'
|
||||
with self.assertRaisesRegex(ValueError, 'another submission'):
|
||||
c.join_metadata(program, archive)
|
||||
|
||||
def test_unknown_profile_fields_are_rejected(self):
|
||||
program = c.parse_program(self.bundle['official_html']); archive = json.loads(self.bundle['filtered'])
|
||||
archive['papers'][0]['authorids'] = ['profile']
|
||||
with self.assertRaisesRegex(ValueError, 'profile'):
|
||||
c.join_metadata(program, archive)
|
||||
|
||||
def test_source_sets_must_agree(self):
|
||||
program = c.parse_program(self.bundle['official_html']); archive = json.loads(self.bundle['filtered'])
|
||||
archive['papers'].pop()
|
||||
with self.assertRaisesRegex(ValueError, 'forum sets'):
|
||||
c.join_metadata(program, archive)
|
||||
|
||||
def test_complete_snapshot_and_explicit_version_provenance(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); official = root/'official'; archive = root/'archive'
|
||||
official.mkdir(); archive.mkdir()
|
||||
(official/'accepted.html').write_text(self.bundle['official_html'])
|
||||
(official/'research.json').write_text(json.dumps(self.bundle['research']))
|
||||
(archive/'explorer-paper-metadata-filtered-v1.json').write_text(self.bundle['filtered'])
|
||||
out = root/'out'; report = c.collect(out, official_snapshots=official, archive_snapshots=archive)
|
||||
self.assertEqual(report['records'], 687); self.assertEqual(len(report['title_variants']), 34)
|
||||
self.assertEqual(report['public_rejected_availability'], 'unknown')
|
||||
rows = [json.loads(l) for l in (out/'corl-program-archive-2026-main.jsonl').read_text().splitlines()]
|
||||
self.assertEqual(len({r['external_id'] for r in rows}), 687)
|
||||
self.assertTrue(all(set(r['identifiers']) == {'openreview'} for r in rows))
|
||||
self.assertTrue(all(r['submission_identifiers'] == [{'scheme': 'openreview', 'value': r['identifiers']['openreview']}] for r in rows))
|
||||
self.assertTrue(all(not r['tags'] and not r['metadata']['original_decision_signature_verified'] for r in rows))
|
||||
self.assertTrue(all(r['title'] == r['metadata']['archive_fields']['title'] for r in rows))
|
||||
self.assertTrue(all(r['authors'] and r['abstract'] for r in rows))
|
||||
self.assertFalse(report['database_written']); self.assertFalse(report['pdfs_downloaded'])
|
||||
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||
c.collect(out, official_snapshots=official, archive_snapshots=archive)
|
||||
(official/'accepted.html').write_text('tampered')
|
||||
with self.assertRaisesRegex(ValueError, 'hash changed'):
|
||||
c.collect(root/'failed', official_snapshots=official, archive_snapshots=archive)
|
||||
self.assertTrue((root/'failed/failed.json').exists())
|
||||
self.assertFalse((root/'failed/collection-report.json').exists())
|
||||
|
||||
def test_forum_identity_validation(self):
|
||||
for bad in ['https://openreview.net/challenge?redirect=forum', 'https://evil.test/forum?id=abcdefgh', 'https://openreview.net/forum?id=abcdefgh&other=1']:
|
||||
with self.assertRaises(ValueError):
|
||||
c.forum_id(bad)
|
||||
@@ -0,0 +1,100 @@
|
||||
import gzip
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
from paper_atlas.collectors import rss2026 as rss
|
||||
|
||||
FIXTURE = Path(__file__).parent/'fixtures/rss2026-source.json.gz'
|
||||
|
||||
class Rss2026Tests(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.fixture=json.loads(gzip.decompress(FIXTURE.read_bytes()))
|
||||
|
||||
def setUp(self):
|
||||
self.tmp=tempfile.TemporaryDirectory();self.addCleanup(self.tmp.cleanup)
|
||||
self.root=Path(self.tmp.name);self.snap=self.root/'snapshots';self.snap.mkdir()
|
||||
for name,text in self.fixture['pages'].items():(self.snap/name).write_bytes(text.encode())
|
||||
(self.snap/'receipts.json').write_text(json.dumps(self.fixture['receipts']))
|
||||
self.pages=self.fixture['pages'];self.index,self.program=self.lists()
|
||||
|
||||
def lists(self,**changes):
|
||||
pages={**self.pages,**changes}
|
||||
return rss.parse_lists(*(pages[n] for n in rss.PAGES))
|
||||
|
||||
def test_all210_records_match_independent_primary_inventory_and_evidence(self):
|
||||
report=rss.collect(self.root/'out',snapshots=self.snap)
|
||||
self.assertEqual(report['records'],210);self.assertFalse(report['database_written'])
|
||||
self.assertFalse(report['pdfs_downloaded']);self.assertEqual(len(report['source_evidence']),214)
|
||||
manifest,=report['files'];self.assertEqual(manifest['status'],'partial')
|
||||
rows=[json.loads(l) for l in Path(manifest['file']).read_text().splitlines()]
|
||||
for row,gold in zip(rows,self.fixture['independent_gold']):
|
||||
self.assertEqual(row['external_id'],f"2026/p{gold['paper_id']:03d}")
|
||||
for key in ('title','authors','abstract'):self.assertEqual(row[key],gold[key])
|
||||
self.assertEqual(row['identifiers'],{'doi':gold['doi_raw'].lower()})
|
||||
self.assertEqual(row['metadata']['publisher_date'],gold['publication_date'])
|
||||
self.assertNotIn('submission_identifiers',row);self.assertEqual(row['track'],'main')
|
||||
e=row['metadata']['source_evidence'];self.assertEqual(hashlib.sha256((self.root/'out'/e['file']).read_bytes()).hexdigest(),e['sha256'])
|
||||
self.assertEqual(manifest['file_sha256'],hashlib.sha256(Path(manifest['file']).read_bytes()).hexdigest())
|
||||
self.assertEqual(manifest['missing_abstract_count'],0)
|
||||
|
||||
def test_full_hidden_authors_are_separate_from_six_short_previews(self):
|
||||
shortened=[p for p in self.program.values() if p['authors_complete']!=p['authors_preview']]
|
||||
self.assertEqual(len(shortened),6)
|
||||
p=rss.parse_paper(self.pages['p087.html'],87,self.index[87],self.program[87])
|
||||
self.assertEqual(len(p['authors']),55)
|
||||
self.assertFalse(p['program_variants'])
|
||||
|
||||
def test_six_reviewed_name_variants_preserve_original_strings(self):
|
||||
for pid in rss.REVIEWED_AUTHOR_VARIANTS:
|
||||
p=rss.parse_paper(self.pages[f'p{pid:03d}.html'],pid,self.index[pid],self.program[pid])
|
||||
self.assertIn('reviewed_program_author_name_variant_not_person_alias',p['program_variants'])
|
||||
self.assertNotEqual(', '.join(p['authors']),self.program[pid]['authors_complete'])
|
||||
|
||||
def test_reviewed_title_variant_is_not_an_automatic_erratum(self):
|
||||
p=rss.parse_paper(self.pages['p089.html'],89,self.index[89],self.program[89])
|
||||
self.assertTrue(p['title'].startswith('RLinf-VLA'))
|
||||
self.assertTrue(self.program[89]['title'].startswith('RLux-VLA'))
|
||||
self.assertIn('reviewed_program_title_variant_not_erratum',p['program_variants'])
|
||||
|
||||
def test_unreviewed_author_or_title_variant_stops(self):
|
||||
for changed in [{'authors_complete':'An Unrelated Person'},{'title':'A different paper'}]:
|
||||
with self.assertRaisesRegex(ValueError,'Unreviewed'):
|
||||
rss.parse_paper(self.pages['p001.html'],1,self.index[1],{**self.program[1],**changed})
|
||||
|
||||
def test_demo_policy_cannot_be_reused_from2025(self):
|
||||
with self.assertRaisesRegex(ValueError,'policy'):
|
||||
self.lists(**{'cfp.html':self.pages['cfp.html'].replace('will not have a separate category','will have a separate category')})
|
||||
with self.assertRaises(ValueError):
|
||||
self.lists(**{'sponsor-demos.html':self.pages['sponsor-demos.html'].replace('Sponsor Demos','Demo Papers')})
|
||||
|
||||
def test_wrong_doi_year_pdf_or_missing_abstract_is_rejected(self):
|
||||
html=self.pages['p001.html']
|
||||
for old,new in [('10.15607/RSS.2026.XXII.001','10.15607/RSS.2025.XXI.001'),
|
||||
('2026/07/13','2025/07/13'),('rss22/p001.pdf','rss22/p002.pdf'),('Abstract:','Summary:')]:
|
||||
self.assertIn(old,html)
|
||||
with self.assertRaises(ValueError):rss.parse_paper(html.replace(old,new),1,self.index[1],self.program[1])
|
||||
|
||||
def test_index_census_and_preview_tampering_stop(self):
|
||||
with self.assertRaises(ValueError):self.lists(**{'index.html':self.pages['index.html'].replace('p001.html','p999.html')})
|
||||
bad=self.pages['accepted.html'].replace('Huayi Zhou, Kui Jia','Somebody Else',1)
|
||||
with self.assertRaisesRegex(ValueError,'prefix'):self.lists(**{'accepted.html':bad})
|
||||
|
||||
def test_snapshot_tamper_has_no_completion_marker_and_never_fetches(self):
|
||||
p=self.snap/'p001.html';p.write_bytes(p.read_bytes()+b'changed')
|
||||
with patch.object(rss,'fetch',side_effect=AssertionError('network must not be used')):
|
||||
with self.assertRaisesRegex(ValueError,'receipt'):rss.collect(self.root/'out',snapshots=self.snap)
|
||||
self.assertFalse((self.root/'out/collection-report.json').exists())
|
||||
self.assertTrue((self.root/'out/failed.json').exists())
|
||||
|
||||
def test_existing_output_and_unsafe_receipt_are_rejected(self):
|
||||
out=self.root/'existing';out.mkdir()
|
||||
with self.assertRaises(ValueError):rss.collect(out,snapshots=self.snap)
|
||||
receipts=[*self.fixture['receipts'],{**self.fixture['receipts'][0],'file':'../outside'}]
|
||||
(self.snap/'receipts.json').write_text(json.dumps(receipts))
|
||||
with self.assertRaisesRegex(ValueError,'Unsafe'):rss.collect(self.root/'unsafe',snapshots=self.snap)
|
||||
self.assertTrue((self.root/'unsafe/failed.json').exists())
|
||||
self.assertFalse((self.root/'unsafe/collection-report.json').exists())
|
||||
Reference in New Issue
Block a user