Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 6e5202c81a |
@@ -47,6 +47,20 @@ jobs:
|
||||
subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True)
|
||||
outside = root / 'outside-checkout'
|
||||
outside.mkdir()
|
||||
kdd_snapshot = pathlib.Path('/data/paper-index/validation/kdd2026-official-papers-html-v1.html')
|
||||
if kdd_snapshot.is_file():
|
||||
kdd_output = root / 'installed-kdd2026.json'
|
||||
subprocess.run([python, '-I', '-B', '-m', 'paper_atlas', 'collect', 'kdd2026-program',
|
||||
'--snapshot', str(kdd_snapshot), '--snapshot-sha256',
|
||||
'47ee82841dedf034614348d7fdb29714e8ad62fca8c6811e1d856845588f145c',
|
||||
'--out', str(kdd_output)], cwd=outside, check=True)
|
||||
kdd = json.loads(kdd_output.read_bytes())
|
||||
assert kdd['unique_dois'] == 1408 and kdd['identical_repeat_rows'] == 7
|
||||
assert {key:sum(counts.values()) for key,counts in kdd['source_rows_by_cycle_and_track'].items()} == {
|
||||
'july':256, 'february':1159}
|
||||
held, = [entry for entry in kdd['records'] if entry['programme_status'] != 'listed']
|
||||
assert held['doi'] == '10.1145/3770855.3818478' and held['source_withdrawal_date_literal'] == '6/24/26'
|
||||
print('KDD 2026 official cycle and withdrawal fixture passed', flush=True)
|
||||
cikm_snapshots = pathlib.Path('/data/paper-index/validation/cikm2026-source-snapshots-v1')
|
||||
if cikm_snapshots.is_dir():
|
||||
cikm_output = root / 'installed-cikm2026'
|
||||
|
||||
@@ -147,6 +147,8 @@ ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目
|
||||
|
||||
`collect recsys2026-program` 核验九个 track 的 263 项官方录用贡献观察(260 个不同 DOI),保留摘要、原始作者和重复 DOI 的来源版本。见 [RecSys 2026 使用与恢复说明](docs/recsys2026-program.md)。
|
||||
|
||||
[KDD 2026 官方目录](docs/kdd2026-program.md)可以按 DOI 核对两个投稿周期、分轨与目录中的撤回更换标注。
|
||||
|
||||
`collect mm2026-program` 从已冻结官方快照恢复全部 28 个来源 track 的 2,143 个节目 ID,保留 4,326 次展示的来源位置与场次上下文,单列 216 个摘要缺失项;输出供贡献类型、出版及作品身份核查使用,不是可直接导入的论文/决定队列。见 [MM 2026 使用、来源和恢复说明](docs/mm2026-source-status.md)。
|
||||
|
||||
`collect mm2026-publications` 核验完整论文集的 1,918 个注册出版 DOI,并单独关联节目分轨、展示上下文和摘要。1,743 份节目摘要明确标注来源(其中 26 份来自已复核的标题排版对应),166 项标题/作者版本差异保留为待核;所有节目及未匹配项均保留。注册文章类型不等于研究类型,输出尚需数据库身份/历史审查。见 [MM 2026 出版条目采集与恢复](docs/mm2026-publications.md)。
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
# KDD 2026 官方论文目录
|
||||
|
||||
从 [KDD 2026 论文目录](https://kdd2026.kdd.org/papers/) 保存公开 HTML,记录其 SHA-256,在已有目录中指定新的输出文件。
|
||||
|
||||
```sh
|
||||
paper-atlas collect kdd2026-program \
|
||||
--snapshot kdd2026-papers.html \
|
||||
--snapshot-sha256 <snapshot-sha256> \
|
||||
--out kdd2026-programme.json
|
||||
```
|
||||
|
||||
采集器只读取已保存的页面数据,逐项输出 DOI、官网标题、投稿周期、轨道和行哈希。完全相同的重复条目保留出现次数,冲突记录中止解析并等待人工复核。输出用于与来源和库内 DOI 核对,不能直接交给记录导入命令。
|
||||
|
||||
已核对的官网快照有 1,415 条目录记录和 1,408 个唯一 DOI。7 条重复记录的周期、轨道、标题和作者文本一致。February Cycle 列出 Research、Applied Data Science、Datasets and Benchmarks、AI for Sciences、Blue Sky Ideas 五轨。July Cycle 列出前三轨。官网列出的演讲、海报或扩展摘要需要出版方记录确认归档状态。[AI for Sciences 投稿及出版规则](https://kdd2026.kdd.org/ai4sciences-track-call-for-papers/)
|
||||
|
||||
Applied Data Science 有一条官网标注撤回并更换投稿 ID 的记录,旧链接为 `10.1145/3770855.3818478`。保留官网原文和日期字面值。原始投稿 ID 仍待公开证据核实。arXiv 当前指向 `10.1145/3770855.3828560`,这个 DOI 已属于同一份已有作品。两份来源分别记录,旧链接不作为新增作品标识。[arXiv 当前记录](https://arxiv.org/abs/2606.08923)
|
||||
|
||||
补分轨时按 DOI 检查库内已有投稿,先处理重复或缺少作者的记录。官网对录用材料的公开承诺适用于各轨道规定的范围,逐篇拒稿、初筛拒稿和其他撤回仍需公开决定证据。[Research](https://kdd2026.kdd.org/research-track-call-for-papers/) · [Applied Data Science](https://kdd2026.kdd.org/applied-data-science-ads-track-call-for-papers/) · [Datasets and Benchmarks](https://kdd2026.kdd.org/datasets-and-benchmarks-track-call-for-papers/) · [Blue Sky Ideas](https://kdd2026.kdd.org/kdd-2026-blue-sky-ideas-track-call-for-papers/)
|
||||
@@ -35,6 +35,7 @@ COMMANDS = {
|
||||
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
|
||||
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
||||
('collect','recsys2026-program'): ('paper_atlas.collectors.recsys2026', {}, '核验 RecSys 2026 公开录用贡献并保留 DOI 来源版本'),
|
||||
('collect','kdd2026-program'): ('paper_atlas.collectors.kdd2026', {}, '核对 KDD 2026 两个投稿周期、分轨及撤回更换标注'),
|
||||
('collect','mm2026-program'): ('paper_atlas.collectors.mm2026', {}, '恢复 MM 2026 全部来源 track 与重复展示上下文,待贡献和身份核查'),
|
||||
('collect','mm2026-publications'): ('paper_atlas.collectors.mm_publications2026', {}, '核验 MM 2026 全部注册出版条目并分别保留节目分轨、摘要与版本差异'),
|
||||
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
||||
|
||||
@@ -0,0 +1,129 @@
|
||||
"""Inventory KDD 2026 programme DOI, cycle and track observations."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from hashlib import sha256
|
||||
from html.parser import HTMLParser
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
SOURCE_URL = 'https://kdd2026.kdd.org/papers/'
|
||||
CYCLES = {1: 'july', 2: 'february'}
|
||||
TRACKS = {
|
||||
'rtp': 'research', 'ads': 'applied_data_science', 'dtb': 'datasets_and_benchmarks',
|
||||
'ais': 'ai_for_sciences', 'bsi': 'blue_sky_ideas',
|
||||
}
|
||||
DOI_URL = re.compile(r'https://doi\.org/(10\.1145/\d+\.\d+)')
|
||||
WITHDRAWN = re.compile(
|
||||
r'^\*\*WITHDRAWN/New SUBMISSION ID for Authorship Order Change (\d{1,2}/\d{1,2}/26) -- (.+)$'
|
||||
)
|
||||
OTHER_STATUS = re.compile(r'^[*\[(!#\s]*(?:withdrawn|rejected|desk[\s_-]+rejected|retracted|cancell?ed)\b', re.I)
|
||||
|
||||
|
||||
class ProgrammeScripts(HTMLParser):
|
||||
def __init__(self):
|
||||
super().__init__()
|
||||
self.current = None
|
||||
self.scripts = []
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
if tag == 'script' and 'src' not in dict(attrs):
|
||||
self.current = []
|
||||
|
||||
def handle_data(self, data):
|
||||
if self.current is not None:
|
||||
self.current.append(data)
|
||||
|
||||
def handle_endtag(self, tag):
|
||||
if tag == 'script' and self.current is not None:
|
||||
self.scripts.append(''.join(self.current))
|
||||
self.current = None
|
||||
|
||||
|
||||
def cycle_rows(scripts, cycle):
|
||||
pattern = re.compile(r'\bconst\s+cycle' + str(cycle) + r'Papers\s*=\s*(\[)')
|
||||
matches = [(code, match) for code in scripts for match in pattern.finditer(code)]
|
||||
if len(matches) != 1:
|
||||
raise ValueError(f'Expected one KDD programme array for cycle {cycle}')
|
||||
code, match = matches[0]
|
||||
body = code[match.start(1):]
|
||||
rows, consumed = json.JSONDecoder().raw_decode(body)
|
||||
if not rows or not body[consumed:].lstrip().startswith('.map((paper, index)'):
|
||||
raise ValueError(f'Unexpected KDD programme array boundary for cycle {cycle}')
|
||||
return rows
|
||||
|
||||
|
||||
def inventory(raw):
|
||||
parser = ProgrammeScripts()
|
||||
parser.feed(raw.decode('utf-8'))
|
||||
parser.close()
|
||||
records = {}
|
||||
source_counts = {}
|
||||
for cycle, cycle_name in CYCLES.items():
|
||||
rows = cycle_rows(parser.scripts, cycle)
|
||||
source_counts[cycle_name] = Counter()
|
||||
for row in rows:
|
||||
if (not isinstance(row, dict) or row.keys() != {'track', 'title', 'url', 'authors'}
|
||||
or not all(isinstance(value, str) for value in row.values())
|
||||
or row['track'] not in TRACKS or not row['title'].strip()):
|
||||
raise ValueError('Unexpected KDD programme row')
|
||||
match = DOI_URL.fullmatch(row['url'])
|
||||
if not match:
|
||||
raise ValueError('Invalid DOI link in KDD programme')
|
||||
doi = match[1]
|
||||
track = TRACKS[row['track']]
|
||||
source_counts[cycle_name][track] += 1
|
||||
row_hash = sha256(json.dumps(row, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
|
||||
if doi in records:
|
||||
entry = records[doi]
|
||||
if (entry['cycle'], entry['source_row_sha256']) != (cycle_name, row_hash):
|
||||
raise ValueError('Conflicting duplicate DOI in KDD programme: ' + doi)
|
||||
entry['occurrences'] += 1
|
||||
continue
|
||||
status = 'listed'
|
||||
withdrawn = WITHDRAWN.fullmatch(row['title'])
|
||||
if withdrawn:
|
||||
status = 'withdrawn_with_replacement_mentioned'
|
||||
elif OTHER_STATUS.match(row['title']):
|
||||
raise ValueError('Unrecognized status annotation in KDD programme: ' + doi)
|
||||
records[doi] = {
|
||||
'doi': doi, 'doi_url': row['url'], 'source_title': row['title'],
|
||||
'title': withdrawn[2] if withdrawn else row['title'],
|
||||
'cycle': cycle_name, 'track': track, 'programme_status': status,
|
||||
'authors_listed': bool(row['authors'].strip()), 'occurrences': 1,
|
||||
'source_row_sha256': row_hash,
|
||||
}
|
||||
if withdrawn:
|
||||
records[doi]['source_withdrawal_date_literal'] = withdrawn[1]
|
||||
ordered = [records[doi] for doi in sorted(records)]
|
||||
return ordered, source_counts
|
||||
|
||||
|
||||
def collect(snapshot, expected_sha256, out):
|
||||
snapshot = Path(snapshot)
|
||||
with snapshot.open('rb') as stream:
|
||||
raw = stream.read(4_000_001)
|
||||
if len(raw) > 4_000_000 or sha256(raw).hexdigest() != expected_sha256:
|
||||
raise ValueError('KDD programme snapshot hash or size differs')
|
||||
rows, cycle_counts = inventory(raw)
|
||||
result = {
|
||||
'source': SOURCE_URL, 'snapshot_sha256': expected_sha256,
|
||||
'source_rows_by_cycle_and_track': cycle_counts, 'unique_dois': len(rows),
|
||||
'identical_repeat_rows': sum(row['occurrences'] - 1 for row in rows),
|
||||
'programme_status_counts': Counter(row['programme_status'] for row in rows),
|
||||
'records': rows, 'database_written': False,
|
||||
}
|
||||
with Path(out).open('x', encoding='utf-8') as stream:
|
||||
json.dump(result, stream, ensure_ascii=False, indent=2)
|
||||
stream.write('\n')
|
||||
return result
|
||||
|
||||
|
||||
def main(argv=None, *, prog=None):
|
||||
parser = argparse.ArgumentParser(prog=prog)
|
||||
parser.add_argument('--snapshot', required=True)
|
||||
parser.add_argument('--snapshot-sha256', required=True)
|
||||
parser.add_argument('--out', required=True)
|
||||
args = parser.parse_args(argv)
|
||||
result = collect(args.snapshot, args.snapshot_sha256, args.out)
|
||||
print(json.dumps({k: result[k] for k in ('unique_dois', 'identical_repeat_rows', 'programme_status_counts')}))
|
||||
@@ -0,0 +1,68 @@
|
||||
from hashlib import sha256
|
||||
import json
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
import unittest
|
||||
|
||||
from paper_atlas.cli import main
|
||||
from paper_atlas.collectors.kdd2026 import inventory
|
||||
|
||||
|
||||
def row(number, track, title='Paper title', authors='Name (Affiliation)'):
|
||||
return {'track': track, 'title': title, 'url': f'https://doi.org/10.1145/3770855.{number}', 'authors': authors}
|
||||
|
||||
|
||||
def snapshot(july, february):
|
||||
arrays = [
|
||||
f'<script>const cycle{n}Papers = {json.dumps(rows)}.map((paper, index) => paper);</script>'
|
||||
for n, rows in ((1, july), (2, february))
|
||||
]
|
||||
return '\n'.join(arrays).encode()
|
||||
|
||||
|
||||
class KDD2026Tests(unittest.TestCase):
|
||||
def test_cli_preserves_withdrawal_marker_and_identical_occurrences(self):
|
||||
retained = row(1, 'rtp')
|
||||
withdrawn = row(2, 'ads', '**WITHDRAWN/New SUBMISSION ID for Authorship Order Change 6/24/26 -- A revised work')
|
||||
incomplete = row(3, 'ais', authors='')
|
||||
raw = snapshot([retained, retained], [withdrawn, incomplete])
|
||||
with TemporaryDirectory() as directory:
|
||||
source = Path(directory) / 'snapshot.html'
|
||||
source.write_bytes(raw)
|
||||
out = source.with_suffix('.json')
|
||||
args = ['collect', 'kdd2026-program', '--snapshot', str(source), '--snapshot-sha256',
|
||||
sha256(raw).hexdigest(), '--out', str(out)]
|
||||
main(args)
|
||||
report = json.loads(out.read_bytes())
|
||||
with self.assertRaises(FileExistsError):
|
||||
main(args)
|
||||
self.assertEqual(report['source_rows_by_cycle_and_track'], {
|
||||
'july': {'research': 2}, 'february': {'applied_data_science': 1, 'ai_for_sciences': 1},
|
||||
})
|
||||
self.assertEqual(report['unique_dois'], 3)
|
||||
self.assertEqual(report['identical_repeat_rows'], 1)
|
||||
kept, replaced, sparse = report['records']
|
||||
self.assertEqual(kept['occurrences'], 2)
|
||||
self.assertEqual(replaced['programme_status'], 'withdrawn_with_replacement_mentioned')
|
||||
self.assertEqual(replaced['title'], 'A revised work')
|
||||
self.assertEqual(replaced['source_title'], withdrawn['title'])
|
||||
self.assertEqual(replaced['source_withdrawal_date_literal'], '6/24/26')
|
||||
self.assertNotIn('submission_id', replaced)
|
||||
self.assertFalse(sparse['authors_listed'])
|
||||
self.assertFalse(report['database_written'])
|
||||
|
||||
def test_conflicting_duplicate_and_unknown_status_fail(self):
|
||||
with self.assertRaisesRegex(ValueError, 'Conflicting duplicate DOI'):
|
||||
inventory(snapshot([row(1, 'rtp')], [row(1, 'ads')]))
|
||||
with self.assertRaisesRegex(ValueError, 'Unrecognized status'):
|
||||
inventory(snapshot([row(1, 'rtp')], [row(2, 'ads', '***WITHDRAWN -- Unknown format')]))
|
||||
|
||||
def test_hash_mismatch_leaves_no_output(self):
|
||||
with TemporaryDirectory() as directory:
|
||||
source = Path(directory) / 'snapshot.html'
|
||||
source.write_bytes(snapshot([row(1, 'rtp')], [row(2, 'ads')]))
|
||||
out = source.with_suffix('.json')
|
||||
with self.assertRaisesRegex(ValueError, 'snapshot hash or size'):
|
||||
main(['collect', 'kdd2026-program', '--snapshot', str(source), '--snapshot-sha256',
|
||||
'0' * 64, '--out', str(out)])
|
||||
self.assertFalse(out.exists())
|
||||
Reference in New Issue
Block a user