diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index 75a110a..cf2364c 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -60,6 +60,13 @@ jobs: assert icaps_report['records'] == 90 and icaps_report['previously_published_presentations'] == 10 assert icaps_report['missing_abstract_count'] == 90 and not icaps_report['database_written'] assert icaps_report['files'][0]['track'] == 'main_tracks_unclassified' + registry_snapshots = root/'icaps2026-registry-snapshots' + registry_snapshots.mkdir() + (registry_snapshots/'crossref-filtered.json').write_bytes(gzip.decompress((src/'tests/fixtures/icaps2026-crossref.json.gz').read_bytes())) + subprocess.run([python,'-I','-B','-m','paper_atlas','collect','icaps2026-registry','--snapshots',str(registry_snapshots),'--out',str(root/'installed-icaps2026-registry')],cwd=outside,check=True) + registry_report = json.loads((root/'installed-icaps2026-registry/collection-report.json').read_text()) + assert registry_report['records'] == 90 and registry_report['missing_abstract_count'] == 0 + assert len(registry_report['excluded_volume35_dois']) == 1 and not registry_report['database_written'] rss_fixture = json.loads(gzip.decompress((src/'tests/fixtures/rss2026-source.json.gz').read_bytes())) rss_snapshots = root/'rss2026-snapshots' rss_snapshots.mkdir() diff --git a/README.md b/README.md index 6810488..5f58671 100644 --- a/README.md +++ b/README.md @@ -122,3 +122,5 @@ CI 在已授权的远程 runner 上运行测试、构建 wheel,并从源码目 RSS 2026 已独立适配官方论文集与录用表;使用方法和与 2025 demo 分类的区别见 [RSS 2026 来源说明](docs/rss2026-proceedings.md)。 ICAPS 2026 的[官方名单采集器](docs/icaps2026-program.md)区分 90 篇主会轨道组录用与 10 篇既有论文展示;逐篇 track、摘要和稳定标识尚未解决,输出均待审核,不能直接视作已完成入库。 + +ICAPS 2026 也提供独立的 `collect icaps2026-registry`,保留 Crossref 卷 36 的 DOI、作者与摘要版本;不覆盖节目表文字,不把卷 35 或版本差异自动并入。配置与边界见同一份[ICAPS 来源说明](docs/icaps2026-program.md)。 diff --git a/docs/icaps2026-program.md b/docs/icaps2026-program.md index 1240572..18bf015 100644 --- a/docs/icaps2026-program.md +++ b/docs/icaps2026-program.md @@ -22,3 +22,19 @@ paper-atlas collect icaps2026-program \ 生产导入之前仍需当前数据库候选核查、独立身份审核、来源及历史保护基线、导入后逐项验收。所有来源导入完成后还必须另做全库去重、覆盖、摘要和约束查询审计。正常使用不会自动执行这些后续写入。 测试及安装后 CLI 验证只在已授权的 `ces-control:host` runner 上运行。冻结 fixture 含完整 100 条官方条目与独立盘点结果,用于检查两组边界、作者顺序、缺页、重复、来源篡改和恢复行为。 + +## 独立注册元数据版本 + +```sh +paper-atlas collect icaps2026-registry \ + --snapshots /data/paper-index/collected/icaps2026-crossref-research-v1 \ + --out /data/paper-index/collected/icaps2026-registry-normalized-v1 +``` + +该命令只读固定 `crossref-filtered.json`,与官方节目表输出分开。来源是 Crossref 的 ICAPS 期刊注册元数据(ISSN 2334-0843),查询出版日期为 2026 年的结果共 91 条,其中卷 36、期 1 有 90 条,另有卷 35 的一条必须排除。不能仅凭出版日期归属会议届次。来源 URL、抓取时间、原响应哈希和字段过滤快照均保留。 + +卷 36 的 90 条均带 DOI 和摘要,输出保持注册标题、作者顺序及摘要版本一致;保留原始 JATS 摘要,只移除已核验的单个段落包装并规范空白,遇到其他标记(尤其公式、上下标)拒绝无声扁平化。作者个人 ID、机构、引用与文件链接没有进入过滤快照。原始注册类型 `journal-article` 留在 metadata 中,不据此给论文添加研究类型标签。 + +这里的 accepted 表示注册源记录了正式发表,不代表采到原始签名录用决定;publication_doi 标识发表对应的投稿事件,不是原始投稿号。具体 track 保持待分类。与官方 90 篇名单的来源核查发现:88 个规范标题一致,其中 78 个完整作者名单一致;另有 2 个标题和 10 个作者文字差异待审核。两份输出不会自动建立作品同一性、复制人名别名或覆盖文字。它们也不应未经审核同时导入,避免重复作品或投稿。 + +此来源仍标记 partial,不包含公开拒稿。生产导入前需核对 DOI 归属、既有作品及跨会议历史;已知 DOI 也可能存在错误绑定,不能代替最终全库去重验收。 diff --git a/paper_atlas/cli.py b/paper_atlas/cli.py index 77ea633..2db46f3 100644 --- a/paper_atlas/cli.py +++ b/paper_atlas/cli.py @@ -23,6 +23,7 @@ COMMANDS = { ('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'), ('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'), ('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'), + ('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'), ('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'), ('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'), ('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'), diff --git a/paper_atlas/collectors/icaps_registry2026.py b/paper_atlas/collectors/icaps_registry2026.py new file mode 100644 index 0000000..975e39e --- /dev/null +++ b/paper_atlas/collectors/icaps_registry2026.py @@ -0,0 +1,131 @@ +"""Frozen Crossref ICAPS volume 36 metadata, kept separate from program versions.""" +import argparse +from datetime import datetime, date +import hashlib +from html import unescape +import json +from pathlib import Path +import re + +from paper_atlas.io.html import clean + +SOURCE = 'crossref_icaps_registry' +SNAPSHOT_SHA = 'c7480a09f71680d457a16cc1eb35a479742b7510eaad5f236ce54c96e8a29a82' +RAW_RESPONSE_SHA = 'ac5268333b1ecdfa2e385680e2e02b5f6639939c3193738d213e66bb53e42a54' +URL = 'https://api.crossref.org/journals/2334-0843/works?filter=from-pub-date%3A2026-01-01%2Cuntil-pub-date%3A2026-12-31&rows=200' +CONTAINER = 'Proceedings of the International Conference on Automated Planning and Scheduling' +TRACK = 'main_tracks_unclassified' + + +def digest(raw): + return hashlib.sha256(raw).hexdigest() + + +def abstract_text(raw): + if not isinstance(raw, str): + raise ValueError('Missing registry abstract') + match = re.fullmatch(r'(.*?)', raw, re.S) + if not match or re.search(r'<[^>]*>', match[1]): + raise ValueError('Unreviewed abstract markup; preserve rather than flatten math') + text = clean(unescape(match[1])) + if not text: + raise ValueError('Empty registry abstract') + return text + + +def registry_papers(snapshot): + observed = snapshot['at'] + if (snapshot.get('status') != 'registry_snapshot_not_identity_approved_not_imported' + or snapshot.get('url') != URL or snapshot.get('raw_response_sha256') != RAW_RESPONSE_SHA + or snapshot.get('database_written') is not False + or datetime.fromisoformat(observed.replace('Z', '+00:00')).utcoffset() is None + or snapshot.get('total_results') != 91 or len(snapshot.get('records', [])) != 91): + raise ValueError('Unreviewed registry snapshot receipt') + papers = []; excluded = []; seen = set() + for row in snapshot['records']: + doi = row.get('DOI') + if not isinstance(doi, str) or doi in seen: + raise ValueError('Missing or repeated registry DOI') + seen.add(doi) + if row.get('volume') == '35': + excluded.append(doi) + continue + if (row.get('volume') != '36' or row.get('issue') != '1' + or row.get('container-title') != [CONTAINER] or row.get('type') != 'journal-article' + or not re.fullmatch(r'10\.1609/icaps\.v36i1\.[0-9]+', doi) + or row.get('URL') != 'https://doi.org/'+doi + or not isinstance(row.get('title'), list) or len(row['title']) != 1 + or not isinstance(row['title'][0], str) or not clean(row['title'][0])): + raise ValueError('Unreviewed publication volume, DOI, type or title') + day, = row['published']['date-parts'] + if len(day) != 3 or day[0] != 2026 or any(type(x) is not int for x in day): + raise ValueError('Unreviewed publication date') + publication_date = date(*day).isoformat() + authors = row.get('authors') + if (not isinstance(authors, list) or not authors + or any(not isinstance(a, dict) or set(a)-{'given','family','name','sequence'} + or not a.get('given') or not a.get('family') for a in authors)): + raise ValueError('Incomplete registry authors') + names = [clean(a['given']+' '+a['family']) for a in authors] + if (len(set(n.casefold() for n in names)) != len(names) + or any('@' in n or 'anonymous' in n.casefold() for n in names) + or [a.get('sequence') for a in authors] != ['first']+['additional']*(len(authors)-1)): + raise ValueError('Unreviewed registry author order or identity') + papers.append({'original': row, 'title': row['title'][0], 'authors': names, + 'abstract': abstract_text(row.get('abstract')), 'publication_date': publication_date}) + if len(papers) != 90 or len(excluded) != 1: + raise ValueError('Incomplete reviewed volume census') + return sorted(papers, key=lambda p: p['original']['DOI']), excluded + + +def collect(out, *, snapshots): + out = Path(out) + if out.exists(): + raise ValueError('Output exists; inspect previous operation before retrying') + out.mkdir(parents=True) + try: + path = Path(snapshots)/'crossref-filtered.json' + if path.is_symlink() or path.stat().st_size > 4*1024**2: + raise ValueError('Unsafe or oversized registry snapshot') + raw = path.read_bytes() + if digest(raw) != SNAPSHOT_SHA: + raise ValueError('Reviewed registry snapshot hash changed') + snapshot = json.loads(raw); papers, excluded = registry_papers(snapshot) + directory = out/'evidence'; directory.mkdir(); rel = f'evidence/{digest(raw)}.json'; (out/rel).write_bytes(raw) + evidence = {'file': rel, 'sha256': digest(raw), 'url': URL}; rows = [] + for paper in papers: + original = paper['original']; doi = original['DOI'] + rows.append({'source': SOURCE, 'external_id': doi, 'url': original['URL'], + 'title': paper['title'], 'authors': paper['authors'], 'abstract': paper['abstract'], + 'identifiers': {'doi': doi}, 'submission_identifiers': [{'scheme': 'publication_doi', 'value': doi}], + 'venue': 'ICAPS', 'year': 2026, 'track': TRACK, 'status': 'accepted', + 'raw_decision': 'publication recorded in Crossref ICAPS volume 36 issue 1; not an original signed acceptance decision', + 'evidence_url': URL, 'observed_at': snapshot['at'], 'tags': [], + 'metadata': {'registry_fields': original, 'source_evidence': evidence, + 'abstract_transform': 'Remove one reviewed jats:p wrapper, decode character entities and normalize whitespace; original JATS retained.', + 'publication_date': paper['publication_date'], 'original_decision_signature_verified': False, + 'track_assignment': 'Unresolved main/special-track membership; no per-paper track inferred from topic or publication order.', + 'program_relationship': 'Separate registry version; not substituted into official-listing records. Title/author variants require review.', + 'scope': 'Registered publication metadata, not original review decision or current live publisher page. Requires current work-identity and history review before import.'}}) + path = out/'icaps-crossref-registry-2026-main-tracks-unclassified.jsonl' + payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload) + manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICAPS', 'year': 2026, 'track': TRACK, + 'count': 90, 'file': str(path), 'file_sha256': digest(payload), 'list_url': URL, 'status': 'partial', 'missing_abstract_count': 0} + path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n') + report = {'status': 'collected_pending_identity_review', 'records': 90, 'files': [manifest], + 'excluded_volume35_dois': excluded, 'missing_abstract_count': 0, 'source_evidence': evidence, + 'database_written': False, 'pdfs_downloaded': False, 'public_rejected_availability': 'unknown', + 'scope': 'Crossref publication versions only; no automatic official-program linkage, person aliases, work merges or complete public-negative/final-dedup claim.'} + part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json') + return report + except Exception as error: + (out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n') + raise + + +def main(argv=None, *, prog=None): + parser = argparse.ArgumentParser(prog=prog, description=__doc__) + parser.add_argument('--snapshots', required=True) + parser.add_argument('--out', required=True) + args = parser.parse_args(argv) + print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False)) diff --git a/tests/fixtures/icaps2026-crossref.json.gz b/tests/fixtures/icaps2026-crossref.json.gz new file mode 100644 index 0000000..f8c7d97 Binary files /dev/null and b/tests/fixtures/icaps2026-crossref.json.gz differ diff --git a/tests/test_icaps_registry2026.py b/tests/test_icaps_registry2026.py new file mode 100644 index 0000000..58e323f --- /dev/null +++ b/tests/test_icaps_registry2026.py @@ -0,0 +1,96 @@ +import copy +import gzip +import json +from pathlib import Path +import tempfile +import unittest + +from paper_atlas.collectors import icaps_registry2026 as c + + +class ICAPSRegistry2026Test(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.raw = gzip.decompress((Path(__file__).parent/'fixtures/icaps2026-crossref.json.gz').read_bytes()) + cls.snapshot = json.loads(cls.raw) + + def test_exact_edition_not_publication_year_alone(self): + papers, excluded = c.registry_papers(self.snapshot) + self.assertEqual(len(papers), 90); self.assertEqual(len(excluded), 1) + self.assertTrue(all(p['original']['volume'] == '36' for p in papers)) + self.assertEqual(excluded, [r['DOI'] for r in self.snapshot['records'] if r['volume'] == '35']) + self.assertTrue(all(p['abstract'] and p['authors'] for p in papers)) + + def test_full_collection_keeps_registered_versions_and_publication_identity(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); src = root/'src'; src.mkdir(); (src/'crossref-filtered.json').write_bytes(self.raw) + report = c.collect(root/'out', snapshots=src) + self.assertEqual((report['records'], report['missing_abstract_count']), (90, 0)) + self.assertFalse(report['database_written']); self.assertFalse(report['pdfs_downloaded']) + self.assertEqual(report['public_rejected_availability'], 'unknown') + manifest, = report['files']; self.assertEqual(manifest['status'], 'partial') + rows = [json.loads(l) for l in Path(manifest['file']).read_text().splitlines()] + original = {r['DOI']: r for r in self.snapshot['records'] if r['volume'] == '36'} + self.assertEqual({r['external_id'] for r in rows}, set(original)) + for r in rows: + doi = r['external_id']; o = original[doi] + self.assertEqual(r['metadata']['registry_fields'], o) + self.assertEqual(r['title'], o['title'][0]) + self.assertEqual(r['authors'], [a['given']+' '+a['family'] for a in o['authors']]) + self.assertEqual(r['identifiers'], {'doi': doi}) + self.assertEqual(r['submission_identifiers'], [{'scheme': 'publication_doi', 'value': doi}]) + self.assertFalse(r['tags']); self.assertFalse(r['metadata']['original_decision_signature_verified']) + self.assertEqual(r['track'], 'main_tracks_unclassified') + self.assertEqual(r['metadata']['publication_date'], '2026-06-08') + changed = next(r for r in rows if r['external_id'] == '10.1609/icaps.v36i1.42860') + self.assertEqual(changed['title'], 'Analyzing Planner Design Trade-Offs for MAPF Under ADG-Based Realistic Execution') + self.assertIn('Stephen F. Smith', changed['authors']) + with self.assertRaisesRegex(ValueError, 'Output exists'): + c.collect(root/'out', snapshots=src) + + def test_math_markup_is_not_flattened_as_plain_numbers(self): + self.assertEqual(c.abstract_text('A < B and C & D.\n Next line.'), 'A < B and C & D. Next line.') + for raw in ['106', 'xy', ' ', None]: + with self.assertRaises(ValueError): + c.abstract_text(raw) + + def test_invalid_or_repeated_doi_and_edition_rejected(self): + for kind in ('doi', 'repeat', 'volume', 'container', 'year'): + with self.subTest(kind=kind): + s = copy.deepcopy(self.snapshot); rows = [r for r in s['records'] if r['volume'] == '36']; r = rows[0] + if kind == 'doi': r['DOI'] = '10.9999/foreign' + if kind == 'repeat': rows[1]['DOI'] = r['DOI'] + if kind == 'volume': r['volume'] = '37' + if kind == 'container': r['container-title'] = ['Other Conference'] + if kind == 'year': r['published']['date-parts'] = [[2025, 6, 8]] + with self.assertRaises(ValueError): + c.registry_papers(s) + + def test_missing_authors_and_changed_order_evidence_rejected(self): + for kind in ('missing', 'sequence', 'profile'): + s = copy.deepcopy(self.snapshot); r = next(r for r in s['records'] if r['volume'] == '36') + if kind == 'missing': r['authors'][0].pop('family') + if kind == 'sequence': r['authors'][0]['sequence'] = 'additional' + if kind == 'profile': r['authors'][0]['ORCID'] = 'unreviewed-profile' + with self.assertRaises(ValueError): + c.registry_papers(s) + + def test_receipt_scope_and_timezone_rejected(self): + for key, value in [('at', '2026-11-17T00:00:00'), ('url', 'https://example.org/works'), ('raw_response_sha256', 'other')]: + s = copy.deepcopy(self.snapshot); s[key] = value + with self.assertRaises(ValueError): + c.registry_papers(s) + + def test_tamper_failure_retained_and_success_not_published(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); src = root/'src'; src.mkdir(); (src/'crossref-filtered.json').write_bytes(self.raw+b' ') + with self.assertRaisesRegex(ValueError, 'hash changed'): + c.collect(root/'out', snapshots=src) + self.assertTrue((root/'out/failed.json').exists()) + self.assertFalse((root/'out/collection-report.json').exists()) + + def test_symlink_rejected(self): + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp); src = root/'src'; src.mkdir(); (root/'data.json').write_bytes(self.raw); (src/'crossref-filtered.json').symlink_to(root/'data.json') + with self.assertRaisesRegex(ValueError, 'Unsafe'): + c.collect(root/'out', snapshots=src)