diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index cf2364c..9af0399 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -75,6 +75,19 @@ jobs: (rss_snapshots/'receipts.json').write_text(json.dumps(rss_fixture['receipts'])) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--snapshots',str(rss_snapshots),'--out',str(root/'installed-rss2026')],cwd=outside,check=True) assert json.loads((root/'installed-rss2026/collection-report.json').read_text())['records'] == 210 + corl_fixture = json.loads(gzip.decompress((src/'tests/fixtures/corl2026-source.json.gz').read_bytes())) + corl_official = root/'corl2026-official' + corl_archive = root/'corl2026-archive' + corl_official.mkdir() + corl_archive.mkdir() + (corl_official/'accepted.html').write_text(corl_fixture['official_html']) + (corl_official/'research.json').write_text(json.dumps(corl_fixture['research'])) + (corl_archive/'explorer-paper-metadata-filtered-v1.json').write_text(corl_fixture['filtered']) + subprocess.run([python,'-I','-B','-m','paper_atlas','collect','corl2026-program','--official-snapshots',str(corl_official),'--archive-snapshots',str(corl_archive),'--out',str(root/'installed-corl2026')],cwd=outside,check=True) + corl_report = json.loads((root/'installed-corl2026/collection-report.json').read_text()) + assert corl_report['records'] == 687 and corl_report['presentations'] == 780 + assert len(corl_report['title_variants']) == 34 and corl_report['public_rejected_availability'] == 'unknown' + assert not corl_report['database_written'] and not corl_report['pdfs_downloaded'] subprocess.run([python,'-I','-B','-m','paper_atlas','collect','www-archive','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','neurips-program','--help'],cwd=outside,check=True) diff --git a/README.md b/README.md index 5f58671..867ffef 100644 --- a/README.md +++ b/README.md @@ -124,3 +124,7 @@ RSS 2026 已独立适配官方论文集与录用表;使用方法和与 2025 de ICAPS 2026 的[官方名单采集器](docs/icaps2026-program.md)区分 90 篇主会轨道组录用与 10 篇既有论文展示;逐篇 track、摘要和稳定标识尚未解决,输出均待审核,不能直接视作已完成入库。 ICAPS 2026 也提供独立的 `collect icaps2026-registry`,保留 Crossref 卷 36 的 DOI、作者与摘要版本;不覆盖节目表文字,不把卷 35 或版本差异自动并入。配置与边界见同一份[ICAPS 来源说明](docs/icaps2026-program.md)。 + +### CoRL 2026 官方程序快照 + +`collect corl2026-program` 将官方 687 个论坛与固定公开档案按 ID 对照,保留 34 条标题版本差异及分字段来源。不会自动接受 arXiv 匹配或推断拒稿;详情见 [CoRL 2026 使用与恢复说明](docs/corl2026-program.md)。 diff --git a/docs/corl2026-program.md b/docs/corl2026-program.md new file mode 100644 index 0000000..0f54f7e --- /dev/null +++ b/docs/corl2026-program.md @@ -0,0 +1,24 @@ +# CoRL 2026:官方程序与公开档案 + +这是长期维护、按需运行的来源适配器。命令只读已冻结的元数据快照,生成待核验文件;不连接数据库,不抓取 PDF、评审或作者个人档案,不执行网页脚本,也不尝试通过 OpenReview 验证挑战。 + +```sh +paper-atlas collect corl2026-program \ + --official-snapshots /data/paper-index/collected/corl2026-official-listing-research-v1 \ + --archive-snapshots /data/paper-index/collected/corl2026-public-archive-inventory-v1 \ + --out /data/paper-index/collected/corl2026-normalized-v1 +``` + +官方快照包含 `accepted.html` 和 `research.json` 的 HTTP 回执。适配器解析 Google Sites 的 `data-code`,不把外层文本提取为空误作无论文。687 个论坛有 780 次展示:687 次海报、21 次口头、72 次 Spotlight。论坛 ID 确定投稿身份,展示形式不是新的投稿或 track。 + +作者及摘要来自 `gisbi-kim/conference-explorers` 的固定提交 `75b3f7a04b3a77c454540fe2506bfa615b8e038e`,经字段过滤后的 `explorer-paper-metadata-filtered-v1.json`。输入、上游原始数据 SHA-256 及 Git blob 均固定;改变来源需要另行研究、更新快照及检查,不能静默接受。 + +这是独立整理者的版本证据,并非本次重新核验的 OpenReview 原页。每条保留档案标题、作者、摘要、分字段来源 URL、官方标题及全部展示场次。34 个官方与档案标题差异会显式标记,不能视作勘误或自动作者别名。规范记录中的标题与作者摘要保持档案版本一致;官方标题单独保留,避免拼成没有来源的版本。 + +档案的 arXiv 链接和匹配标题只存为未批准候选,不进入作品标识;不沿用其推断机构、个人档案、作者 ID 或生成标签。只有 OpenReview 论坛 ID 进入作品和投稿标识。 + +所有输出均为 partial。官方政策只证实录用论文及评审公开,没有证实 rejected 集可见性。不能由此生成拒稿记录,也不能声称这 687 条覆盖了公开拒稿或全部论文关系。 + +输出目录必须全新;失败留下 `failed.json`,成功报告最后原子发布。恢复时先检查原目录,不覆盖失败现场或重跑已消费的导入队列。生成文件仍需元数据版本审阅、当前数据库身份候选检查、独立保护基线及导入后核验,最后另做全库去重。 + +测试和安装验证只在用户授权的 `ces-control:host` runner 上运行。 diff --git a/paper_atlas/cli.py b/paper_atlas/cli.py index 2db46f3..0d9e787 100644 --- a/paper_atlas/cli.py +++ b/paper_atlas/cli.py @@ -29,6 +29,7 @@ COMMANDS = { ('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'), ('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'), ('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'), + ('collect','corl2026-program'): ('paper_atlas.collectors.corl2026', {}, '核验 CoRL 2026 官方论坛名单与分字段公开档案'), ('collect','neurips-program'): ('paper_atlas.collectors.neurips_program', {}, '按公开节目来源区分 NeurIPS 2026 投稿与重复展示'), ('collect','www-archive'): ('paper_atlas.collectors.www_archive', {}, '核验早期WWW官方档案并分开待核查条目'), ('collect','dblp'): ('paper_atlas.collectors.dblp_records', {}, '核对并整理 DBLP 历史发表记录'), diff --git a/paper_atlas/collectors/corl2026.py b/paper_atlas/collectors/corl2026.py new file mode 100644 index 0000000..bd7799d --- /dev/null +++ b/paper_atlas/collectors/corl2026.py @@ -0,0 +1,183 @@ +"""Frozen CoRL 2026 official program joined to attributed public archive metadata.""" +import argparse +from collections import Counter +from datetime import datetime +import hashlib +import json +from pathlib import Path +import re +import unicodedata +from urllib.parse import parse_qs, urlsplit + +from paper_atlas.io.html import Document, clean + +SOURCE = 'corl_program_archive' +OFFICIAL = 'https://2026.corl.org/program/accepted-papers' +REVISION = '75b3f7a04b3a77c454540fe2506bfa615b8e038e' +ARCHIVE = f'https://raw.githubusercontent.com/gisbi-kim/conference-explorers/{REVISION}/corl2026/data.json' +OFFICIAL_SHA = '3f026639c1a777b0b5fbd132e1935a27cdd519ca0a04e60ca3b650a3bc7eaef4' +FILTERED_SHA = '6e6d7ecd57ee139c61283188fec39ebe9dc7f5c30ec14c8c5839d1277b03a588' +RAW_SHA = '1e96ed2dffcd96c2584a7ba020201c831e49ae1a69eae3ef1553fd5a34802810' +BLOB = '322eeef71126b85e13a98fdce9c73fac81934d3f' +FIELDS = {'id', 'title', 'openreview', 'poster_session', 'poster_day', 'poster_date', + 'poster_time', 'poster_venue', 'poster_number', 'index', 'presentation_type', + 'arxiv', 'matched_title', 'abstract', 'metadata_source', 'author_source', + 'abstract_source', 'identity_note', 'authors', 'openreview_updated', 'metadata_checked'} + + +def digest(raw): + return hashlib.sha256(raw).hexdigest() + + +def comparable(text): + return ''.join(c for c in unicodedata.normalize('NFKD', text).casefold() if c.isalnum()) + + +def forum_id(url): + if not isinstance(url, str): + raise ValueError('Forum URL must be text') + u = urlsplit(url); q = parse_qs(u.query, keep_blank_values=True) + if (u.scheme != 'https' or u.netloc != 'openreview.net' or u.path != '/forum' + or u.fragment or set(q) != {'id'} or len(q['id']) != 1 + or not re.fullmatch(r'[A-Za-z0-9_-]{8,32}', q['id'][0])): + raise ValueError('Invalid original forum identity') + return q['id'][0] + + +def parse_program(raw): + embeds = [n.attrs['data-code'] for n in Document(raw).root.walk() if 'data-code' in n.attrs] + if len(embeds) != 1: + raise ValueError('Expected one official embedded program') + doc = Document(embeds[0]); nodes = list(doc.root.walk()) + if [clean(n.text()) for n in nodes if n.tag == 'h1'] != ['CoRL 2026 Accepted Papers']: + raise ValueError('Unexpected program edition') + summary = [clean(n.text()) for n in nodes if n.tag == 'p' and n.attrs.get('class') == 'sub'] + if len(summary) != 1 or not summary[0].startswith('687 accepted papers · 21 orals · 72 spotlights · 687 posters'): + raise ValueError('Reviewed program census changed') + papers = {}; sessions = {}; occurrences = [] + for section in (n for n in nodes if n.tag == 'section' and 'session' in n.attrs.get('class', '').split()): + sid = section.attrs.get('data-sess', section.attrs.get('id')) + if not isinstance(sid, str) or not re.fullmatch(r'(?:O[1-7]|S[1-8]|post-p[1-3])', sid) or sid in sessions: + raise ValueError('Unexpected or repeated presentation session') + kind = 'oral' if sid.startswith('O') else 'spotlight' if sid.startswith('S') else 'poster' + headings = [clean(n.text()) for n in section.walk() if n.tag == 'h3'] + if len(headings) != 1: + raise ValueError('Ambiguous session heading') + sessions[sid] = {'kind': kind, 'heading': headings[0]} + seen = set() + for link in (n for n in section.walk() if n.tag == 'a' and 'openreview.net' in n.attrs.get('href', '')): + fid = forum_id(link.attrs['href']); title = clean(link.text()) + if fid in seen or not title: + raise ValueError('Repeated or empty paper in session') + seen.add(fid) + paper = papers.setdefault(fid, {'title': title, 'presentations': []}) + if paper['title'] != title: + raise ValueError('Conflicting official titles share a forum') + event = {'session_id': sid, **sessions[sid]} + paper['presentations'].append(event); occurrences.append((fid, sid, kind)) + counts = Counter(kind for _, _, kind in occurrences) + if len(papers) != 687 or len(sessions) != 18 or counts != {'poster': 687, 'oral': 21, 'spotlight': 72}: + raise ValueError('Incomplete official program') + for paper in papers.values(): + kinds = Counter(p['kind'] for p in paper['presentations']) + if kinds['poster'] != 1 or len(paper['presentations']) not in (1, 2): + raise ValueError('Invalid poster/talk multiplicity') + return papers + + +def join_metadata(program, archive): + if (archive.get('repository') != 'gisbi-kim/conference-explorers' + or archive.get('revision') != REVISION or archive.get('source_url') != ARCHIVE + or archive.get('source_data_sha256') != RAW_SHA or archive.get('source_git_blob_sha1') != BLOB + or not isinstance(archive.get('papers'), list)): + raise ValueError('Unpinned public metadata provenance') + indexed = {} + for row in archive['papers']: + if not isinstance(row, dict) or set(row) - FIELDS: + raise ValueError('Unexpected metadata/profile fields') + fid = forum_id(row.get('openreview')) + if fid != row.get('id') or fid in indexed: + raise ValueError('Repeated or mismatched archive forum') + if any(forum_id(row.get(key)) != fid for key in ('metadata_source', 'author_source', 'abstract_source')): + raise ValueError('Field source belongs to another submission') + if (not isinstance(row.get('title'), str) or not row['title'].strip() + or not isinstance(row.get('abstract'), str) or not row['abstract'].strip() + or not isinstance(row.get('authors'), list) or not row['authors'] + or any(not isinstance(a, str) or not a.strip() or '@' in a or 'anonymous' in a.casefold() for a in row['authors'])): + raise ValueError('Incomplete scholarly metadata') + indexed[fid] = row + if set(indexed) != set(program): + raise ValueError('Official and archive forum sets differ') + return indexed + + +def collect(out, *, official_snapshots, archive_snapshots): + out = Path(out) + if out.exists(): + raise ValueError('Output exists; inspect previous operation before retrying') + out.mkdir(parents=True) + try: + official_root = Path(official_snapshots); archive_root = Path(archive_snapshots) + official = official_root/'accepted.html'; filtered = archive_root/'explorer-paper-metadata-filtered-v1.json' + if official.is_symlink() or filtered.is_symlink(): + raise ValueError('Snapshot symlinks rejected') + raw = official.read_bytes(); data = filtered.read_bytes() + if len(raw) > 8*1024**2 or len(data) > 8*1024**2 or digest(raw) != OFFICIAL_SHA or digest(data) != FILTERED_SHA: + raise ValueError('Reviewed snapshot hash changed') + receipts = json.loads((official_root/'research.json').read_bytes())['receipts'] + matches = [r for r in receipts if r.get('url') == OFFICIAL] + if len(matches) != 1: + raise ValueError('Missing official source receipt') + receipt = matches[0]; observed = receipt['at'] + if (receipt.get('final_url') != OFFICIAL or receipt['http_status'] != 200 + or receipt['sha256'] != digest(raw) or receipt['bytes'] != len(raw) + or datetime.fromisoformat(observed.replace('Z', '+00:00')).utcoffset() is None): + raise ValueError('Invalid official receipt') + program = parse_program(raw.decode('utf-8')); archive = json.loads(data); indexed = join_metadata(program, archive) + evidence = out/'evidence'; evidence.mkdir() + sources = {} + for body, suffix, url, name in [(raw, 'html', OFFICIAL, 'program'), (data, 'json', ARCHIVE, 'archive')]: + rel = f'evidence/{digest(body)}.{suffix}'; (out/rel).write_bytes(body) + sources[name] = {'file': rel, 'sha256': digest(body), 'url': url} + rows = []; variants = [] + for fid in sorted(program): + p = program[fid]; a = indexed[fid]; variant = comparable(p['title']) != comparable(a['title']) + if variant: variants.append({'forum': fid, 'official_title': p['title'], 'archive_title': a['title']}) + rows.append({'source': SOURCE, 'external_id': f'2026/{fid}', 'url': a['openreview'], + 'title': a['title'], 'authors': a['authors'], 'abstract': a['abstract'], + 'identifiers': {'openreview': fid}, 'submission_identifiers': [{'scheme': 'openreview', 'value': fid}], + 'venue': 'CoRL', 'year': 2026, 'track': 'main', 'status': 'accepted', + 'raw_decision': 'listed in official CoRL 2026 accepted program', 'evidence_url': OFFICIAL, + 'observed_at': observed, 'tags': [], 'metadata': { + 'official_program': p, 'archive_fields': a, 'title_variant_requires_review': variant, + 'source_evidence': sources, 'archive_revision': REVISION, + 'field_provenance': {k: a[k] for k in ('metadata_source', 'author_source', 'abstract_source')}, + 'original_decision_signature_verified': False, + 'scope': 'Acceptance from official forum listing; title/authors/abstract are curator snapshot versions, not live original metadata. arXiv links remain unapproved candidates. No reject inference or person aliases.'}}) + if len(variants) != 34: + raise ValueError('Reviewed title-variant census changed') + path = out/'corl-program-archive-2026-main.jsonl' + payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload) + manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'CoRL', 'year': 2026, 'track': 'main', + 'count': len(rows), 'file': str(path), 'file_sha256': digest(payload), 'list_url': OFFICIAL, + 'status': 'partial', 'missing_abstract_count': 0} + path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n') + report = {'status': 'collected_pending_identity_review', 'records': len(rows), 'presentations': 780, + 'files': [manifest], 'title_variants': variants, 'source_evidence': sources, + 'database_written': False, 'pdfs_downloaded': False, 'public_rejected_availability': 'unknown', + 'scope': 'Source collection only; requires identity and metadata-version review before import; no exhaustive deduplication or rejected-set claim.'} + part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json') + return report + except Exception as error: + (out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n') + raise + + +def main(argv=None, *, prog=None): + parser = argparse.ArgumentParser(prog=prog, description=__doc__) + parser.add_argument('--out', type=Path, required=True) + parser.add_argument('--official-snapshots', type=Path, required=True) + parser.add_argument('--archive-snapshots', type=Path, required=True) + args = parser.parse_args(argv) + report = collect(args.out, official_snapshots=args.official_snapshots, archive_snapshots=args.archive_snapshots) + print(json.dumps({'status': report['status'], 'records': report['records'], 'database_written': False})) diff --git a/tests/fixtures/corl2026-source.json.gz b/tests/fixtures/corl2026-source.json.gz new file mode 100644 index 0000000..55134d0 Binary files /dev/null and b/tests/fixtures/corl2026-source.json.gz differ diff --git a/tests/test_corl2026.py b/tests/test_corl2026.py new file mode 100644 index 0000000..431dda9 --- /dev/null +++ b/tests/test_corl2026.py @@ -0,0 +1,75 @@ +import gzip +import json +from pathlib import Path +import tempfile +import unittest + +from paper_atlas.collectors import corl2026 as c + + +class CoRL2026Test(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.bundle = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/corl2026-source.json.gz').read_bytes())) + + def test_official_census_and_presentation_dedup(self): + program = c.parse_program(self.bundle['official_html']) + self.assertEqual(len(program), 687) + self.assertEqual(sum(len(p['presentations']) for p in program.values()), 780) + self.assertEqual(sum(len(p['presentations']) == 2 for p in program.values()), 93) + + def test_conflicting_official_title_is_not_silently_folded(self): + html = self.bundle['official_html'] + title = 'Diagnosing Compositional Generalization in Sequential Robot Tasks' + self.assertIn(title, html) + with self.assertRaises(ValueError): + c.parse_program(html.replace(title, 'Different paper', 1)) + + def test_foreign_forum_source_is_rejected(self): + program = c.parse_program(self.bundle['official_html']); archive = json.loads(self.bundle['filtered']) + archive['papers'][0]['author_source'] = 'https://openreview.net/forum?id=WrongPaper' + with self.assertRaisesRegex(ValueError, 'another submission'): + c.join_metadata(program, archive) + + def test_unknown_profile_fields_are_rejected(self): + program = c.parse_program(self.bundle['official_html']); archive = json.loads(self.bundle['filtered']) + archive['papers'][0]['authorids'] = ['profile'] + with self.assertRaisesRegex(ValueError, 'profile'): + c.join_metadata(program, archive) + + def test_source_sets_must_agree(self): + program = c.parse_program(self.bundle['official_html']); archive = json.loads(self.bundle['filtered']) + archive['papers'].pop() + with self.assertRaisesRegex(ValueError, 'forum sets'): + c.join_metadata(program, archive) + + def test_complete_snapshot_and_explicit_version_provenance(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); official = root/'official'; archive = root/'archive' + official.mkdir(); archive.mkdir() + (official/'accepted.html').write_text(self.bundle['official_html']) + (official/'research.json').write_text(json.dumps(self.bundle['research'])) + (archive/'explorer-paper-metadata-filtered-v1.json').write_text(self.bundle['filtered']) + out = root/'out'; report = c.collect(out, official_snapshots=official, archive_snapshots=archive) + self.assertEqual(report['records'], 687); self.assertEqual(len(report['title_variants']), 34) + self.assertEqual(report['public_rejected_availability'], 'unknown') + rows = [json.loads(l) for l in (out/'corl-program-archive-2026-main.jsonl').read_text().splitlines()] + self.assertEqual(len({r['external_id'] for r in rows}), 687) + self.assertTrue(all(set(r['identifiers']) == {'openreview'} for r in rows)) + self.assertTrue(all(r['submission_identifiers'] == [{'scheme': 'openreview', 'value': r['identifiers']['openreview']}] for r in rows)) + self.assertTrue(all(not r['tags'] and not r['metadata']['original_decision_signature_verified'] for r in rows)) + self.assertTrue(all(r['title'] == r['metadata']['archive_fields']['title'] for r in rows)) + self.assertTrue(all(r['authors'] and r['abstract'] for r in rows)) + self.assertFalse(report['database_written']); self.assertFalse(report['pdfs_downloaded']) + with self.assertRaisesRegex(ValueError, 'Output exists'): + c.collect(out, official_snapshots=official, archive_snapshots=archive) + (official/'accepted.html').write_text('tampered') + with self.assertRaisesRegex(ValueError, 'hash changed'): + c.collect(root/'failed', official_snapshots=official, archive_snapshots=archive) + self.assertTrue((root/'failed/failed.json').exists()) + self.assertFalse((root/'failed/collection-report.json').exists()) + + def test_forum_identity_validation(self): + for bad in ['https://openreview.net/challenge?redirect=forum', 'https://evil.test/forum?id=abcdefgh', 'https://openreview.net/forum?id=abcdefgh&other=1']: + with self.assertRaises(ValueError): + c.forum_id(bad)