Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 28805462ee |
@@ -47,6 +47,26 @@ jobs:
|
||||
subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True)
|
||||
outside = root / 'outside-checkout'
|
||||
outside.mkdir()
|
||||
cikm_snapshots = pathlib.Path('/data/paper-index/validation/cikm2026-source-snapshots-v1')
|
||||
if cikm_snapshots.is_dir():
|
||||
cikm_output = root / 'installed-cikm2026'
|
||||
command = [python, '-I', '-B', '-m', 'paper_atlas', 'collect', 'cikm2026-sources']
|
||||
subprocess.run([*command, '--snapshots', str(cikm_snapshots), '--out', str(cikm_output)], cwd=outside, check=True)
|
||||
report = json.loads((cikm_output/'collection-report.json').read_text())
|
||||
assert {name: entry['records'] for name, entry in report['files'].items()} == {
|
||||
'programme.jsonl': 1008, 'alternate-programme.jsonl': 1005,
|
||||
'supplemental-programme.jsonl': 73, 'crossref.jsonl': 1081}
|
||||
assert len(report['registry_outside_doi_programme']) == 73 and not report['database_written']
|
||||
programme = [json.loads(line) for line in (cikm_output/'programme.jsonl').read_text().splitlines()]
|
||||
assert all(row['abstract'] and row['author_display_names'] for row in programme)
|
||||
assert len({row['doi'] for row in programme}) == 1008
|
||||
broken = root/'tampered-cikm2026'
|
||||
broken.mkdir()
|
||||
(broken/'program.html').write_text('tampered')
|
||||
rejected = subprocess.run([*command, '--snapshots', str(broken), '--out', str(root/'rejected-cikm2026')], cwd=outside, capture_output=True, text=True)
|
||||
assert rejected.returncode != 0 and 'Snapshot hash mismatch' in rejected.stderr
|
||||
assert not (root/'rejected-cikm2026').exists()
|
||||
print('CIKM 2026 full frozen-source CLI check passed', flush=True)
|
||||
mm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/mm2026-source.json.gz').read_bytes()))
|
||||
mm_snapshots = root/'mm2026-snapshots'
|
||||
for name, body in mm_fixture['files'].items():
|
||||
|
||||
@@ -137,6 +137,8 @@ ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目
|
||||
|
||||
`collect sdm2026-proceedings` 核验 53 个 Crossref 论文章节,并单独保留 53 项日程观察及 52 页摘要。作者差异、取消展示和摘要缺失分别保留,不自动拼接来源或绑定作品;见 [SDM 2026 使用与恢复说明](docs/sdm2026-proceedings.md)。
|
||||
|
||||
`collect cikm2026-sources` 核验并分开导出两份官方节目、附属议程和出版登记。见 [CIKM 2026 来源采集与恢复](docs/cikm2026-sources.md)。
|
||||
|
||||
`collect icdm2026-program` 离线核验 324 条 ICDM2026 官方线下日程,按明确的研究/应用标签输出 241/83 条;展示长短、缺失摘要及未核实出版身份分别保留,不自动入库。见 [ICDM2026 使用与恢复说明](docs/icdm2026-program.md)。
|
||||
|
||||
`collect recsys2026-program` 核验九个 track 的 263 项官方录用贡献观察(260 个不同 DOI),保留摘要、原始作者和重复 DOI 的来源版本。见 [RecSys 2026 使用与恢复说明](docs/recsys2026-program.md)。
|
||||
|
||||
@@ -0,0 +1,22 @@
|
||||
# CIKM 2026 来源采集与恢复
|
||||
|
||||
使用固定节目和登记快照复现 CIKM 2026 来源盘点。采集命令生成四份独立 JSONL,输入逐文件校验 SHA-256。
|
||||
|
||||
```sh
|
||||
paper-atlas collect cikm2026-sources \
|
||||
--snapshots /data/paper-index/validation/cikm2026-source-snapshots-v1 \
|
||||
--out /data/paper-index/collected/cikm2026-sources-v1
|
||||
```
|
||||
|
||||
输入包括两版官方节目 HTML、Crossref 卷登记、五页逐篇登记及分页回执。服务器验证目录保留输入原文。输出目录必须是新目录。先检查 `collection-report.json`,报告在四份 JSONL 写完后发布。
|
||||
|
||||
| 文件 | 内容 |
|
||||
| --- | --- |
|
||||
| `programme.jsonl` | 带 DOI 的 1,008 条节目记录,包括标题、作者显示名、摘要和五类节目标签 |
|
||||
| `alternate-programme.jsonl` | 另一版节目 1,005 条记录及各自的字段 |
|
||||
| `supplemental-programme.jsonl` | 73 条演讲、教程、工作坊及其他附属活动观察 |
|
||||
| `crossref.jsonl` | 1,081 条 ACM 卷内登记,每条保留登记字段及分页文件位置 |
|
||||
|
||||
节目 DOI 指向 Crossref 的出版登记。确认作者和标题差异以及历史投稿绑定后,再为来源观察准备入库队列。报告另列 73 个未出现在带 DOI 节目区的登记,附属活动仍保留各自的活动类型。公开拒稿与撤回决定仍需独立来源核查。
|
||||
|
||||
官方原始来源见[带 DOI 的节目](https://cikm2026.diag.uniroma1.it/cikm-program/program.html)、[另一版节目](https://cikm2026.diag.uniroma1.it/cikm-program/CIKM2026_program.html)和 [Crossref 卷登记](https://api.crossref.org/works/10.1145/3799682)。
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,16 +0,0 @@
|
||||
# CIKM 2026 节目与出版登记核对
|
||||
|
||||
官方节目列有 1,008 项带 DOI 的条目,逐篇均有摘要。五个标签的数量分别为 Full Research 596、Short Research 234、Applied Research 86、Resources 46、Demo 46。另一份官方节目快照少三个节目编号,其余 1,005 个编号重合。两份原始快照和响应哈希保存在本机的 `cikm2026-official-detailed-program-v1` 工作目录。[带 DOI 的节目](https://cikm2026.diag.uniroma1.it/cikm-program/program.html) · [另一份官方节目](https://cikm2026.diag.uniroma1.it/cikm-program/CIKM2026_program.html)
|
||||
|
||||
正式卷 DOI 为 `10.1145/3799682`,ISBN 为 `9798400725395`。ISBN 查询只返回卷级登记。按卷名及出版日期分页取得 1,081 条登记,逐条核对卷 DOI 前缀、完整容器标题及来源类型。1,008 个节目 DOI 全部匹配,另有 73 条登记位于此前带 DOI 节目分区之外。附属议程核对结果见下文。该次 Crossref 响应中摘要字段均缺失。[ACM 出版条目](https://doi.org/10.1145/3799682.3840615) · [Crossref 卷登记](https://api.crossref.org/works/10.1145/3799682)
|
||||
|
||||
按已知排版标记、Unicode 和标点变化比较,同 DOI 的条目有 4 处标题差异、11 处有序作者列表差异。差异条目保留两份原文,等后续核对后再决定如何关联。当前 PostgreSQL 快照中,97 个节目 DOI 已绑定论文,304 个节目标题有候选,其中 95 个 DOI 绑定与标题候选重合。仍需核查已有作品与历次投稿历史,当前尚未写入。
|
||||
|
||||
[逐条差集、字段差异及证据哈希](cikm2026-source-reconciliation-2026-11-18.json)。先将出版登记与节目摘要作为各自来源采集,再核查两者的作品关系和轨道。公开拒稿、撤回及桌拒的可得性继续单独核查。
|
||||
|
||||
|
||||
## 附属议程与已有 DOI 的进一步核查
|
||||
|
||||
在官方节目页面的附属议程及主题演讲中,73 条登记有 72 条找到唯一规范标题对应,70 条同时通过有序作者比较。对应项包括 Industry Day 20 条、博士论坛 19 条、教程 14 条、工作坊条目 15 条、主题演讲 3 条及 AnalytiCup 1 条。工作坊条目记录该场活动对应的出版材料,独立 workshop 论文仍需各自目录。两条工作坊条目有作者写法差异,另有一条 LASS 标题带不同前后缀,三条均保留待核对。[官方完整节目](https://cikm2026.diag.uniroma1.it/cikm-program/program.html)
|
||||
|
||||
全卷 1,081 个 DOI 的 PostgreSQL 只读复查找到 98 个已有绑定,其中 97 个来自带 DOI 节目分区。89 项在现有论文、Crossref 和节目中有一致的规范标题、有序作者及明确节目标签,可优先准备来源关联。其余 9 项保留待核查。98 项的现有历史中无拒稿、桌拒或撤回记录,这一观察仅覆盖已有绑定的作品。此次仍未写入,入库时须重新确认绑定及保护既有历史。
|
||||
@@ -28,6 +28,7 @@ COMMANDS = {
|
||||
('collect','eccv2026-program'): ('paper_atlas.collectors.eccv2026', {}, '核验ECCV2026暂定节目并断点采集公开元数据'),
|
||||
('collect','icassp2026-registry'): ('paper_atlas.collectors.icassp2026', {}, '核验 ICASSP 2026 冻结注册论文元数据'),
|
||||
('collect','icdm2026-program'): ('paper_atlas.collectors.icdm2026', {}, '核验 ICDM2026 冻结日程及明确的研究/应用轨道'),
|
||||
('collect','cikm2026-sources'): ('paper_atlas.collectors.cikm2026', {}, '核对 CIKM 2026 节目、附属议程与 Crossref 出版登记'),
|
||||
('collect','sdm2026-proceedings'): ('paper_atlas.collectors.sdm2026', {}, '核验 SDM 2026 注册论文并单独保留公开日程版本'),
|
||||
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
|
||||
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
||||
|
||||
@@ -0,0 +1,164 @@
|
||||
"""Collect CIKM 2026 programme and Crossref metadata from frozen snapshots."""
|
||||
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from hashlib import sha256
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from paper_atlas.io.html import Document, clean
|
||||
|
||||
PROGRAM = 'https://cikm2026.diag.uniroma1.it/cikm-program/program.html'
|
||||
VOLUME = '10.1145/3799682'
|
||||
CONTAINER = 'Proceedings of the 35th ACM International Conference on Information and Knowledge Management'
|
||||
TRACKS = {'FULL': 596, 'SHORT': 234, 'APPLIED': 86, 'RESOURCE': 46, 'DEMO': 46}
|
||||
SUPPLEMENTAL = {'KEYNOTE', 'TUTORIAL', 'INDUSTRY', 'POSTER', 'PHD', 'PHD · REMOTE', 'WORKSHOP', 'ANALYTICUP'}
|
||||
PAGE_COUNTS = (250, 250, 250, 250, 81)
|
||||
HASHES = {
|
||||
'program.html': '3eaa668b6aaf09a671d8a3e19a2889b9ab21afaa7ded013794afa94bf35281d6',
|
||||
'CIKM2026_program.html': 'e3630bd1247df713b8b618f1d3cd9709e7020b2524bb677f82a10f83badd1732',
|
||||
'crossref-parent.json': 'f3370418d86ccfcfaeb972073bc5fcd1c1c737caed96570fc83f99db0f7d204a',
|
||||
'crossref-page-1.json': '16bed29f8ee82d668b124c369b632c5c1de0753bdbec7eb72ecb22ffdda5c710',
|
||||
'crossref-page-2.json': '54424ca31d99462394c9fde5fecfd7dd16337351d21ede3169f19af54d4f49c5',
|
||||
'crossref-page-3.json': '2dc31d89f41966614ed66a11d1ebaa4b2a6904dcf92945b7185e72e7e65382dd',
|
||||
'crossref-page-4.json': 'fbbec2b52650dc99a0251c743f035a6fd95be263b04c55c12134cf8473bdde76',
|
||||
'crossref-page-5.json': '06151c5f770c1e37e8ea9dabe5210be9c969a14fa3bf9925a2d7730ced47d6e5',
|
||||
'crossref-pages.json': '8be1f8f05139f7a1231b73ad6430f28d2f625809f1530cd9de654f8b3c68c1ae',
|
||||
}
|
||||
|
||||
|
||||
def fields(node, css_class):
|
||||
return [child for child in node.walk() if css_class in child.attrs.get('class', '').split()]
|
||||
|
||||
|
||||
def parse_program(raw, *, require_doi):
|
||||
document = Document(raw.decode('utf-8'))
|
||||
papers = {}
|
||||
for card in fields(document.root, 'paper'):
|
||||
pid = card.attrs.get('data-pid')
|
||||
if not pid:
|
||||
continue
|
||||
titles = fields(card, 'ptitle')
|
||||
labels = fields(card, 'badge')
|
||||
abstracts = fields(card, 'abstxt')
|
||||
if pid in papers or len(titles) != 1 or len(labels) != 1 or len(abstracts) > 1:
|
||||
raise ValueError(f'Invalid or repeated programme card {pid}')
|
||||
title = clean(titles[0].text())
|
||||
label = clean(labels[0].text())
|
||||
authors = [clean(author.text()) for author in fields(card, 'au')]
|
||||
abstract = clean(abstracts[0].text()) if abstracts else None
|
||||
doi_url = titles[0].attrs.get('href')
|
||||
doi = doi_url.removeprefix('https://doi.org/') if doi_url else None
|
||||
if label not in TRACKS or not title or not authors or not all(authors):
|
||||
raise ValueError(f'Incomplete programme metadata {pid}')
|
||||
if require_doi and (not doi or not re.fullmatch(re.escape(VOLUME) + r'\.\d+', doi) or not abstract):
|
||||
raise ValueError(f'Missing programme DOI or abstract {pid}')
|
||||
papers[pid] = {'programme_id': pid, 'title': title, 'author_display_names': authors,
|
||||
'abstract': abstract, 'programme_label': label, 'programme_doi_link': doi_url,
|
||||
'doi': doi if require_doi else None}
|
||||
return papers
|
||||
|
||||
|
||||
def parse_supplemental(raw):
|
||||
document = Document(raw.decode('utf-8'))
|
||||
events = {}
|
||||
for session in fields(document.root, 'sessblock'):
|
||||
section = session.attrs.get('id', '')
|
||||
if not section.startswith(('pre-', 'kn-')):
|
||||
continue
|
||||
for card in fields(session, 'paper'):
|
||||
labels = fields(card, 'badge')
|
||||
if len(labels) != 1:
|
||||
raise ValueError(f'Ambiguous supplemental label in {section}')
|
||||
label = clean(labels[0].text())
|
||||
if label not in SUPPLEMENTAL:
|
||||
continue
|
||||
titles = fields(card, 'ptitle')
|
||||
abstracts = fields(card, 'abstxt')
|
||||
locator = card.attrs.get('id') or section
|
||||
if locator in events or len(titles) != 1 or len(abstracts) > 1:
|
||||
raise ValueError(f'Ambiguous supplemental entry {locator}')
|
||||
events[locator] = {'programme_locator': locator, 'programme_section': section,
|
||||
'programme_label': label, 'title': clean(titles[0].text()),
|
||||
'author_display_names': [clean(a.text()) for a in fields(card, 'au')],
|
||||
'abstract': clean(abstracts[0].text()) if abstracts else None}
|
||||
return events
|
||||
|
||||
|
||||
def parse_registry(snapshots):
|
||||
volume = json.loads(snapshots['crossref-parent.json'])['message']
|
||||
if (volume['DOI'], volume['title'], volume['ISBN']) != (VOLUME, [CONTAINER], ['9798400725395']):
|
||||
raise ValueError('Unexpected Crossref volume')
|
||||
records = {}
|
||||
for number, count in enumerate(PAGE_COUNTS, 1):
|
||||
name = f'crossref-page-{number}.json'
|
||||
message = json.loads(snapshots[name])['message']
|
||||
if message['total-results'] != 1081 or len(message['items']) != count:
|
||||
raise ValueError(f'Unexpected Crossref pagination in {name}')
|
||||
for item in message['items']:
|
||||
doi = item['DOI'].lower()
|
||||
if (doi in records or not re.fullmatch(re.escape(VOLUME) + r'\.\d+', doi)
|
||||
or item.get('container-title') != [CONTAINER] or item.get('type') != 'proceedings-article'
|
||||
or item.get('published', {}).get('date-parts') != [[2026, 11, 6]]):
|
||||
raise ValueError(f'Unexpected Crossref record {doi}')
|
||||
records[doi] = {'registry': item, 'source_file': name}
|
||||
return records
|
||||
|
||||
|
||||
def collect(snapshots, output):
|
||||
snapshots = Path(snapshots)
|
||||
output = Path(output)
|
||||
if output.exists() or output.is_symlink():
|
||||
raise FileExistsError(output)
|
||||
bodies = {}
|
||||
for name, expected in HASHES.items():
|
||||
path = snapshots / name
|
||||
if path.is_symlink() or not path.is_file() or path.stat().st_size > 8 * 1024**2:
|
||||
raise ValueError(f'Invalid snapshot file {name}')
|
||||
raw = path.read_bytes()
|
||||
if sha256(raw).hexdigest() != expected:
|
||||
raise ValueError(f'Snapshot hash mismatch for {name}')
|
||||
bodies[name] = raw
|
||||
|
||||
programme = parse_program(bodies['program.html'], require_doi=True)
|
||||
alternate = parse_program(bodies['CIKM2026_program.html'], require_doi=False)
|
||||
supplemental = parse_supplemental(bodies['program.html'])
|
||||
registry = parse_registry(bodies)
|
||||
programme_dois = {r['doi'] for r in programme.values()}
|
||||
tracks = Counter(r['programme_label'] for r in programme.values())
|
||||
if (tracks != TRACKS or len(programme_dois) != len(programme) or programme_dois - registry.keys()
|
||||
or len(alternate) != 1005 or alternate.keys() - programme.keys()
|
||||
or len(supplemental) != 73 or len(registry) - len(programme_dois) != 73):
|
||||
raise ValueError('Unexpected programme or registry coverage')
|
||||
groups = {
|
||||
'programme': programme,
|
||||
'alternate-programme': alternate,
|
||||
'supplemental-programme': supplemental,
|
||||
'crossref': registry,
|
||||
}
|
||||
output.mkdir(parents=True, exist_ok=False)
|
||||
files = {}
|
||||
for name, rows in groups.items():
|
||||
filename = name + '.jsonl'
|
||||
payload = ''.join(json.dumps({'source_key': key, **row}, ensure_ascii=False) + '\n'
|
||||
for key, row in sorted(rows.items())).encode()
|
||||
(output / filename).write_bytes(payload)
|
||||
files[filename] = {'records': len(rows), 'sha256': sha256(payload).hexdigest()}
|
||||
report = {'status': 'collected_pending_identity_review', 'files': files, 'input_sha256': HASHES,
|
||||
'programme_by_label': dict(tracks), 'programme_dois_in_registry': len(programme_dois),
|
||||
'registry_outside_doi_programme': sorted(registry.keys() - programme_dois),
|
||||
'programme_ids_missing_from_alternate': sorted(programme.keys() - alternate.keys()),
|
||||
'database_written': False, 'public_negative_decisions_verified': False}
|
||||
partial = output / 'collection-report.json.part'
|
||||
partial.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||
partial.replace(output / 'collection-report.json')
|
||||
return report
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||
parser.add_argument('--snapshots', type=Path, required=True)
|
||||
parser.add_argument('--out', type=Path, required=True)
|
||||
options = parser.parse_args(argv)
|
||||
print(json.dumps(collect(options.snapshots, options.out), ensure_ascii=False))
|
||||
Reference in New Issue
Block a user