Preserve ICAPS 2026 official acceptance and presentation boundaries
Metadata integrity checks / metadata (push) Failing after 2m30s
Metadata integrity checks / metadata (push) Failing after 2m30s
This commit is contained in:
@@ -49,6 +49,17 @@ jobs:
|
||||
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--help'],cwd=outside,check=True)
|
||||
icaps_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icaps2026-source.json.gz').read_bytes()))
|
||||
icaps_snapshots = root/'icaps2026-snapshots'
|
||||
icaps_snapshots.mkdir()
|
||||
for name, html in icaps_fixture['pages'].items():
|
||||
(icaps_snapshots/name).write_text(html)
|
||||
(icaps_snapshots/'manifest.json').write_text(json.dumps(icaps_fixture['manifest']))
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','icaps2026-program','--snapshots',str(icaps_snapshots),'--out',str(root/'installed-icaps2026')],cwd=outside,check=True)
|
||||
icaps_report = json.loads((root/'installed-icaps2026/collection-report.json').read_text())
|
||||
assert icaps_report['records'] == 90 and icaps_report['previously_published_presentations'] == 10
|
||||
assert icaps_report['missing_abstract_count'] == 90 and not icaps_report['database_written']
|
||||
assert icaps_report['files'][0]['track'] == 'main_tracks_unclassified'
|
||||
rss_fixture = json.loads(gzip.decompress((src/'tests/fixtures/rss2026-source.json.gz').read_bytes()))
|
||||
rss_snapshots = root/'rss2026-snapshots'
|
||||
rss_snapshots.mkdir()
|
||||
|
||||
@@ -120,3 +120,5 @@ CI 在已授权的远程 runner 上运行测试、构建 wheel,并从源码目
|
||||
入库前可使用通用的[来源身份只读预检](docs/source-identity-audit.md),导出候选论文、稳定标识和既有投稿决定证据;候选结果需要审核,不会自动批准导入。
|
||||
|
||||
RSS 2026 已独立适配官方论文集与录用表;使用方法和与 2025 demo 分类的区别见 [RSS 2026 来源说明](docs/rss2026-proceedings.md)。
|
||||
|
||||
ICAPS 2026 的[官方名单采集器](docs/icaps2026-program.md)区分 90 篇主会轨道组录用与 10 篇既有论文展示;逐篇 track、摘要和稳定标识尚未解决,输出均待审核,不能直接视作已完成入库。
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
# ICAPS 2026 官方名单
|
||||
|
||||
这是长期维护、按需运行的元数据采集器。它只读取已核验的官方页面快照,生成待身份审核的文件;不连接数据库,不下载 PDF 或评审,不执行网页脚本。
|
||||
|
||||
```sh
|
||||
paper-atlas collect icaps2026-program \
|
||||
--snapshots /data/paper-index/collected/icaps2026-official-research-v1 \
|
||||
--out /data/paper-index/collected/icaps2026-normalized-v1
|
||||
```
|
||||
|
||||
快照目录包含 `accepted.html`、`cfp.html`、`previously-published.html` 和 `manifest.json`。采集器固定这三份已审阅页面的 SHA-256,核对 URL、最终 URL、字节数及带时区的观测时间;来源变化需要重新研究、固定证据并更新检查,不能静默接受。输出目录必须全新,失败会保留 `failed.json`,成功报告最后发布。恢复前先检查旧目录,不能覆盖失败现场或重放已消费的导入队列。
|
||||
|
||||
[官方录用表](https://icaps26.icaps-conference.org/program/accepted/)含两组:
|
||||
|
||||
- **90 篇 Papers accepted to the Main Tracks**:生成 partial JSONL,状态为 accepted。[CFP](https://icaps26.icaps-conference.org/calls/cfp/)包含主会及四个特别轨道,但该表没有逐篇归属,因此保存为 `main_tracks_unclassified`,不能把它们全部重命名为 main,也不能按论文主题猜特别轨道。position 是研究类别,不能直接推成投稿 track。
|
||||
- **10 篇 Previously Published Papers**:保存在单独的 `previously-published-presentations.json`,不生成新的研究论文录用记录。[该展示轨道的官方说明](https://icaps26.icaps-conference.org/calls/previously_published_research/)须随原文保存;后续关联原发表记录时不能重复制造论文或替换原会议。
|
||||
|
||||
名单没有逐篇链接、摘要、DOI 或原始投稿号。90 条摘要明确留空,作品及投稿标识均为空;完整作者顺序与原始作者文本一并保留。`external_id` 里的摘要哈希仅定位这份名单的标题和作者版本,不是原始投稿号,也不是跨来源作品身份。作者文字变化不代表人物别名,标题相同不代表可以自动合并作品。
|
||||
|
||||
主会 rejected、desk rejected、withdrawn 的公开范围仍未知,不能从缺席录用表推断拒稿,也不能把某一 workshop 的私密评审政策扩展到整个会议。出版社 OJS 页面曾返回访问挑战;本采集器不访问或绕过该挑战。当前来源只证明官方名单,未证明完整出版物或公开负面决定覆盖。
|
||||
|
||||
生产导入之前仍需当前数据库候选核查、独立身份审核、来源及历史保护基线、导入后逐项验收。所有来源导入完成后还必须另做全库去重、覆盖、摘要和约束查询审计。正常使用不会自动执行这些后续写入。
|
||||
|
||||
测试及安装后 CLI 验证只在已授权的 `ces-control:host` runner 上运行。冻结 fixture 含完整 100 条官方条目与独立盘点结果,用于检查两组边界、作者顺序、缺页、重复、来源篡改和恢复行为。
|
||||
@@ -23,6 +23,7 @@ COMMANDS = {
|
||||
('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'),
|
||||
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
|
||||
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
|
||||
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
|
||||
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
||||
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
||||
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
|
||||
|
||||
@@ -0,0 +1,155 @@
|
||||
"""ICAPS 2026 frozen official listing, with unresolved track membership explicit."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from datetime import datetime
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from paper_atlas.io.html import Document, Node, clean
|
||||
|
||||
SOURCE = 'icaps_official_program'
|
||||
ROOT = 'https://icaps26.icaps-conference.org/'
|
||||
PAGES = {
|
||||
'accepted.html': (ROOT+'program/accepted/', '235543c3e5917ecb3e10a1429a4754f9cb7fae9844b54404076254584da86443'),
|
||||
'cfp.html': (ROOT+'calls/cfp/', 'bdcf5a5b3b4144d6c6ecf3e324c856363f4f4e992e592c1f63988980434ecd5d'),
|
||||
'previously-published.html': (ROOT+'calls/previously_published_research/', 'ecd60c8c056c9aa73bac16eb11f220c756ddff562511f0ec42ada41919fb1946'),
|
||||
}
|
||||
MAIN = 'Papers accepted to the Main Tracks'
|
||||
PREVIOUS = 'Previously Published Papers'
|
||||
TRACK = 'main_tracks_unclassified'
|
||||
|
||||
|
||||
def digest(raw):
|
||||
return hashlib.sha256(raw).hexdigest()
|
||||
|
||||
|
||||
def authors_from_literal(literal):
|
||||
# This frozen listing uses commas between authors and a final "and".
|
||||
# Preserve the complete literal too; never infer person-name aliases.
|
||||
if literal.count(' and ') != 1 or ', and ' in literal:
|
||||
raise ValueError('Unreviewed author-list punctuation')
|
||||
first, last = literal.rsplit(' and ', 1)
|
||||
authors = first.split(', ') + [last]
|
||||
if any(not a or a != clean(a) or ',' in a or '@' in a or 'anonymous' in a.casefold() for a in authors):
|
||||
raise ValueError('Incomplete or ambiguous author list')
|
||||
if ', '.join(authors[:-1])+' and '+authors[-1] != literal:
|
||||
raise ValueError('Author literal cannot be reconstructed')
|
||||
return authors
|
||||
|
||||
|
||||
def parse_listing(raw):
|
||||
nodes = list(Document(raw).root.walk())
|
||||
headings = [clean(n.text()) for n in nodes if n.tag == 'h1']
|
||||
if headings != ['List of Papers Accepted to ICAPS 2026']:
|
||||
raise ValueError('Unexpected official conference edition')
|
||||
section = None; sections = []; rows = []; seen = set()
|
||||
for n in nodes:
|
||||
if n.tag == 'h2':
|
||||
section = clean(n.text()); sections.append(section)
|
||||
if n.tag != 'div' or 'paper' not in n.attrs.get('class', '').split():
|
||||
continue
|
||||
if section not in (MAIN, PREVIOUS):
|
||||
raise ValueError('Paper belongs to an unreviewed section')
|
||||
# Require the reviewed title and complete-author blocks; fail on layout drift.
|
||||
blocks = [b for b in n.children if isinstance(b, Node)]
|
||||
if len(blocks) != 2 or any(b.tag != 'div' for b in blocks):
|
||||
raise ValueError('Changed paper metadata layout')
|
||||
titles = [b for b in blocks[0].walk() if b.tag == 'strong']
|
||||
if len(titles) != 1 or any(b.tag == 'a' for b in n.walk()):
|
||||
raise ValueError('Changed paper title/link layout')
|
||||
title = clean(titles[0].text()); literal = clean(blocks[1].text())
|
||||
if not title or clean(blocks[0].text()) != title or title.casefold() in seen:
|
||||
raise ValueError('Empty or repeated paper title')
|
||||
seen.add(title.casefold())
|
||||
rows.append({'section': section, 'title': title, 'authors': authors_from_literal(literal),
|
||||
'authors_literal': literal})
|
||||
if sections != [MAIN, PREVIOUS] or Counter(r['section'] for r in rows) != {MAIN: 90, PREVIOUS: 10}:
|
||||
raise ValueError('Incomplete reviewed official listing')
|
||||
return rows
|
||||
|
||||
|
||||
def collect(out, *, snapshots):
|
||||
out = Path(out)
|
||||
if out.exists():
|
||||
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||
out.mkdir(parents=True)
|
||||
try:
|
||||
root = Path(snapshots)
|
||||
manifest_path = root/'manifest.json'
|
||||
if manifest_path.is_symlink():
|
||||
raise ValueError('Snapshot symlinks rejected')
|
||||
receipt = json.loads(manifest_path.read_bytes())
|
||||
observed = receipt['at']
|
||||
if (receipt.get('status') != 'collected_not_parsed_not_imported'
|
||||
or receipt.get('database_written') is not False
|
||||
or datetime.fromisoformat(observed.replace('Z', '+00:00')).utcoffset() is None):
|
||||
raise ValueError('Invalid reviewed snapshot receipt')
|
||||
receipts = receipt['sources']
|
||||
if len(receipts) != 3 or {r['file'] for r in receipts} != set(PAGES):
|
||||
raise ValueError('Unexpected snapshot source set')
|
||||
bodies = {}; evidence = {}; directory = out/'evidence'; directory.mkdir()
|
||||
for name, (url, expected) in PAGES.items():
|
||||
matches = [r for r in receipts if r['file'] == name]
|
||||
if len(matches) != 1:
|
||||
raise ValueError('Repeated snapshot receipt')
|
||||
r = matches[0]; p = root/name
|
||||
if p.is_symlink() or p.stat().st_size > 2*1024**2:
|
||||
raise ValueError('Unsafe or oversized snapshot')
|
||||
raw = p.read_bytes()
|
||||
if digest(raw) != expected or r['sha256'] != expected or r['bytes'] != len(raw):
|
||||
raise ValueError('Reviewed snapshot hash changed')
|
||||
if r['url'] != url or r['final_url'] != url:
|
||||
raise ValueError('Unexpected snapshot URL or redirect')
|
||||
rel = f'evidence/{expected}.html'; (out/rel).write_bytes(raw)
|
||||
bodies[name] = raw.decode('utf-8')
|
||||
evidence[name] = {'url': url, 'file': rel, 'sha256': expected}
|
||||
entries = parse_listing(bodies['accepted.html']); rows = []; previous = []
|
||||
for entry in entries:
|
||||
entry_key = digest(json.dumps([entry['title'], entry['authors_literal']], ensure_ascii=False, separators=(',', ':')).encode())
|
||||
if entry['section'] == PREVIOUS:
|
||||
previous.append({**entry, 'entry_key': entry_key,
|
||||
'presentation_kind': 'previously_published_research',
|
||||
'evidence_url': PAGES['accepted.html'][0], 'observed_at': observed,
|
||||
'scope': 'Official presentation listing, not evidence of new archival publication or a new research-paper acceptance.'})
|
||||
continue
|
||||
rows.append({'source': SOURCE, 'external_id': '2026/main-tracks/'+entry_key,
|
||||
'url': PAGES['accepted.html'][0], 'title': entry['title'], 'authors': entry['authors'],
|
||||
'abstract': None, 'identifiers': {}, 'submission_identifiers': [],
|
||||
'venue': 'ICAPS', 'year': 2026, 'track': TRACK, 'status': 'accepted',
|
||||
'raw_decision': 'listed in official ICAPS 2026 Papers accepted to the Main Tracks',
|
||||
'evidence_url': PAGES['accepted.html'][0], 'observed_at': observed, 'tags': [],
|
||||
'metadata': {'official_listing': entry, 'source_evidence': evidence,
|
||||
'source_entry_key': {'algorithm': 'sha256-json-title-and-literal-authors', 'value': entry_key,
|
||||
'scope': 'Snapshot entry locator only; not an original submission ID, DOI, or cross-source work identity.'},
|
||||
'track_assignment': 'Unresolved among main and four special tracks; do not relabel all records as main.',
|
||||
'abstract_availability': 'not_on_official_listing',
|
||||
'original_decision_signature_verified': False,
|
||||
'scope': 'Official listing evidence only; requires current identity review before import. No absence-based rejection or new-paper inference.'}})
|
||||
path = out/'icaps-official-program-2026-main-tracks-unclassified.jsonl'
|
||||
payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
|
||||
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICAPS', 'year': 2026, 'track': TRACK,
|
||||
'count': 90, 'file': str(path), 'file_sha256': digest(payload), 'list_url': PAGES['accepted.html'][0],
|
||||
'status': 'partial', 'missing_abstract_count': 90}
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
|
||||
sidecar = out/'previously-published-presentations.json'
|
||||
sidecar.write_text(json.dumps({'records': previous, 'count': 10, 'ingestion_ready': False,
|
||||
'scope': 'Presentation evidence retained separately; not emitted as new accepted research-paper records.'}, ensure_ascii=False, indent=2)+'\n')
|
||||
report = {'status': 'collected_pending_identity_review', 'records': 90, 'previously_published_presentations': 10,
|
||||
'files': [manifest], 'previously_published_file': sidecar.name, 'previously_published_sha256': digest(sidecar.read_bytes()),
|
||||
'source_evidence': evidence, 'database_written': False, 'pdfs_downloaded': False,
|
||||
'public_rejected_availability': 'unknown', 'missing_abstract_count': 90,
|
||||
'scope': 'Partial official list; individual tracks and stable identities unresolved; no all-paper, public-negative or final-dedup claim.'}
|
||||
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||
return report
|
||||
except Exception as error:
|
||||
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||
raise
|
||||
|
||||
|
||||
def main(argv=None):
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--snapshots', required=True)
|
||||
parser.add_argument('--out', required=True)
|
||||
args = parser.parse_args(argv)
|
||||
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
|
||||
BIN
Binary file not shown.
@@ -0,0 +1,95 @@
|
||||
import gzip
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
from paper_atlas.collectors import icaps2026 as c
|
||||
|
||||
|
||||
class ICAPS2026Test(unittest.TestCase):
|
||||
@classmethod
|
||||
def setUpClass(cls):
|
||||
cls.bundle = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/icaps2026-source.json.gz').read_bytes()))
|
||||
|
||||
def snapshots(self, root):
|
||||
root.mkdir()
|
||||
for name, html in self.bundle['pages'].items():
|
||||
(root/name).write_text(html)
|
||||
(root/'manifest.json').write_text(json.dumps(self.bundle['manifest']))
|
||||
return root
|
||||
|
||||
def test_full_listing_matches_independently_recorded_literals(self):
|
||||
rows = c.parse_listing(self.bundle['pages']['accepted.html'])
|
||||
self.assertEqual([{k: r[k] for k in ('section', 'title', 'authors_literal')} for r in rows],
|
||||
[{k: r[k] for k in ('section', 'title', 'authors_literal')} for r in self.bundle['expected']])
|
||||
self.assertEqual(sum(r['section'] == c.MAIN for r in rows), 90)
|
||||
self.assertEqual(sum(r['section'] == c.PREVIOUS for r in rows), 10)
|
||||
self.assertTrue(all(len(r['authors']) >= 2 for r in rows))
|
||||
|
||||
def test_incomplete_and_duplicate_listing_rejected(self):
|
||||
html = self.bundle['pages']['accepted.html']
|
||||
with self.assertRaises(ValueError):
|
||||
c.parse_listing(html.replace('class=paper>', 'class=omitted>', 1))
|
||||
with self.assertRaisesRegex(ValueError, 'repeated paper title'):
|
||||
c.parse_listing(html.replace(self.bundle['expected'][1]['title'], self.bundle['expected'][0]['title']))
|
||||
|
||||
def test_changed_edition_or_section_rejected(self):
|
||||
html = self.bundle['pages']['accepted.html']
|
||||
for changed in [html.replace('ICAPS 2026</h1>', 'ICAPS 2027</h1>'),
|
||||
html.replace(c.MAIN, 'Workshop Papers')]:
|
||||
with self.assertRaises(ValueError):
|
||||
c.parse_listing(changed)
|
||||
|
||||
def test_ambiguous_author_lists_not_guessed(self):
|
||||
for literal in ['Anonymous', 'Alice, and Bob', 'Alice and Bob and Carol', 'Alice, Bob and name@example.org']:
|
||||
with self.assertRaises(ValueError):
|
||||
c.authors_from_literal(literal)
|
||||
self.assertEqual(c.authors_from_literal('Augusto B. Corrêa, Simon Dold and Malte Helmert'),
|
||||
['Augusto B. Corrêa', 'Simon Dold', 'Malte Helmert'])
|
||||
|
||||
def test_collection_separates_presentations_and_unresolved_tracks(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); src = self.snapshots(root/'src'); out = root/'out'
|
||||
report = c.collect(out, snapshots=src)
|
||||
self.assertEqual((report['records'], report['previously_published_presentations'], report['missing_abstract_count']), (90, 10, 90))
|
||||
self.assertEqual(report['public_rejected_availability'], 'unknown')
|
||||
rows = [json.loads(l) for l in Path(report['files'][0]['file']).read_text().splitlines()]
|
||||
previous = json.loads((out/report['previously_published_file']).read_text())
|
||||
self.assertFalse(previous['ingestion_ready']); self.assertEqual(previous['count'], 10)
|
||||
self.assertFalse({r['title'] for r in rows} & {r['title'] for r in previous['records']})
|
||||
self.assertTrue(all(r['track'] == 'main_tracks_unclassified' and r['status'] == 'accepted' for r in rows))
|
||||
self.assertTrue(all(not r['identifiers'] and not r['submission_identifiers'] and r['abstract'] is None and not r['tags'] for r in rows))
|
||||
self.assertEqual(len({r['external_id'] for r in rows}), 90)
|
||||
self.assertTrue(all(r['metadata']['source_entry_key']['value'] in r['external_id'] for r in rows))
|
||||
self.assertFalse(report['database_written']); self.assertFalse(report['pdfs_downloaded'])
|
||||
self.assertEqual(report['files'][0]['status'], 'partial')
|
||||
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||
c.collect(out, snapshots=src)
|
||||
|
||||
def test_tampering_retains_failure_without_success_receipt(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); src = self.snapshots(root/'src')
|
||||
(src/'accepted.html').write_text('tampered')
|
||||
with self.assertRaisesRegex(ValueError, 'hash changed'):
|
||||
c.collect(root/'out', snapshots=src)
|
||||
self.assertTrue((root/'out/failed.json').exists())
|
||||
self.assertFalse((root/'out/collection-report.json').exists())
|
||||
|
||||
def test_receipt_redirect_naive_time_and_duplicate_source_rejected(self):
|
||||
for kind in ('redirect', 'time', 'duplicate'):
|
||||
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); src = self.snapshots(root/'src'); p = src/'manifest.json'; r = json.loads(p.read_bytes())
|
||||
if kind == 'redirect': r['sources'][0]['final_url'] = 'https://example.org/challenge'
|
||||
if kind == 'time': r['at'] = '2026-11-16T12:00:00'
|
||||
if kind == 'duplicate': r['sources'][1] = r['sources'][0]
|
||||
p.write_text(json.dumps(r))
|
||||
with self.assertRaises(ValueError):
|
||||
c.collect(root/'out', snapshots=src)
|
||||
self.assertFalse((root/'out/collection-report.json').exists())
|
||||
|
||||
def test_snapshot_symlink_rejected(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp); src = self.snapshots(root/'src'); p = src/'accepted.html'; raw = p.read_bytes(); p.unlink(); target = root/'external.html'; target.write_bytes(raw); p.symlink_to(target)
|
||||
with self.assertRaisesRegex(ValueError, 'Unsafe'):
|
||||
c.collect(root/'out', snapshots=src)
|
||||
Reference in New Issue
Block a user