Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4d97822d9e |
@@ -47,20 +47,6 @@ jobs:
|
|||||||
subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True)
|
subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True)
|
||||||
outside = root / 'outside-checkout'
|
outside = root / 'outside-checkout'
|
||||||
outside.mkdir()
|
outside.mkdir()
|
||||||
kdd_snapshot = pathlib.Path('/data/paper-index/validation/kdd2026-official-papers-html-v1.html')
|
|
||||||
if kdd_snapshot.is_file():
|
|
||||||
kdd_output = root / 'installed-kdd2026.json'
|
|
||||||
subprocess.run([python, '-I', '-B', '-m', 'paper_atlas', 'collect', 'kdd2026-program',
|
|
||||||
'--snapshot', str(kdd_snapshot), '--snapshot-sha256',
|
|
||||||
'47ee82841dedf034614348d7fdb29714e8ad62fca8c6811e1d856845588f145c',
|
|
||||||
'--out', str(kdd_output)], cwd=outside, check=True)
|
|
||||||
kdd = json.loads(kdd_output.read_bytes())
|
|
||||||
assert kdd['unique_dois'] == 1408 and kdd['identical_repeat_rows'] == 7
|
|
||||||
assert {key:sum(counts.values()) for key,counts in kdd['source_rows_by_cycle_and_track'].items()} == {
|
|
||||||
'july':256, 'february':1159}
|
|
||||||
held, = [entry for entry in kdd['records'] if entry['programme_status'] != 'listed']
|
|
||||||
assert held['doi'] == '10.1145/3770855.3818478' and held['source_withdrawal_date_literal'] == '6/24/26'
|
|
||||||
print('KDD 2026 official cycle and withdrawal fixture passed', flush=True)
|
|
||||||
cikm_snapshots = pathlib.Path('/data/paper-index/validation/cikm2026-source-snapshots-v1')
|
cikm_snapshots = pathlib.Path('/data/paper-index/validation/cikm2026-source-snapshots-v1')
|
||||||
if cikm_snapshots.is_dir():
|
if cikm_snapshots.is_dir():
|
||||||
cikm_output = root / 'installed-cikm2026'
|
cikm_output = root / 'installed-cikm2026'
|
||||||
|
|||||||
@@ -147,8 +147,6 @@ ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目
|
|||||||
|
|
||||||
`collect recsys2026-program` 核验九个 track 的 263 项官方录用贡献观察(260 个不同 DOI),保留摘要、原始作者和重复 DOI 的来源版本。见 [RecSys 2026 使用与恢复说明](docs/recsys2026-program.md)。
|
`collect recsys2026-program` 核验九个 track 的 263 项官方录用贡献观察(260 个不同 DOI),保留摘要、原始作者和重复 DOI 的来源版本。见 [RecSys 2026 使用与恢复说明](docs/recsys2026-program.md)。
|
||||||
|
|
||||||
[KDD 2026 官方目录](docs/kdd2026-program.md)可以按 DOI 核对两个投稿周期、分轨与目录中的撤回更换标注。
|
|
||||||
|
|
||||||
`collect mm2026-program` 从已冻结官方快照恢复全部 28 个来源 track 的 2,143 个节目 ID,保留 4,326 次展示的来源位置与场次上下文,单列 216 个摘要缺失项;输出供贡献类型、出版及作品身份核查使用,不是可直接导入的论文/决定队列。见 [MM 2026 使用、来源和恢复说明](docs/mm2026-source-status.md)。
|
`collect mm2026-program` 从已冻结官方快照恢复全部 28 个来源 track 的 2,143 个节目 ID,保留 4,326 次展示的来源位置与场次上下文,单列 216 个摘要缺失项;输出供贡献类型、出版及作品身份核查使用,不是可直接导入的论文/决定队列。见 [MM 2026 使用、来源和恢复说明](docs/mm2026-source-status.md)。
|
||||||
|
|
||||||
`collect mm2026-publications` 核验完整论文集的 1,918 个注册出版 DOI,并单独关联节目分轨、展示上下文和摘要。1,743 份节目摘要明确标注来源(其中 26 份来自已复核的标题排版对应),166 项标题/作者版本差异保留为待核;所有节目及未匹配项均保留。注册文章类型不等于研究类型,输出尚需数据库身份/历史审查。见 [MM 2026 出版条目采集与恢复](docs/mm2026-publications.md)。
|
`collect mm2026-publications` 核验完整论文集的 1,918 个注册出版 DOI,并单独关联节目分轨、展示上下文和摘要。1,743 份节目摘要明确标注来源(其中 26 份来自已复核的标题排版对应),166 项标题/作者版本差异保留为待核;所有节目及未匹配项均保留。注册文章类型不等于研究类型,输出尚需数据库身份/历史审查。见 [MM 2026 出版条目采集与恢复](docs/mm2026-publications.md)。
|
||||||
|
|||||||
@@ -11,6 +11,6 @@ paper-atlas collect eccv2026-publications \
|
|||||||
|
|
||||||
目录归档含 `receipts.jsonl`、`volume-checkpoint.json`、`summary.json` 及 156 页 HTML。摘要在章节详情页读取,章节归档沿用 `springer-selected` 的 `receipts.json` 格式。两类归档都按响应大小和 SHA-256 校验。目录卷号、分页、章节 DOI 归属、标题及 2026 年出处需要一致。已有输出目录需另查回执或换新空目录后运行。
|
目录归档含 `receipts.jsonl`、`volume-checkpoint.json`、`summary.json` 及 156 页 HTML。摘要在章节详情页读取,章节归档沿用 `springer-selected` 的 `receipts.json` 格式。两类归档都按响应大小和 SHA-256 校验。目录卷号、分页、章节 DOI 归属、标题及 2026 年出处需要一致。已有输出目录需另查回执或换新空目录后运行。
|
||||||
|
|
||||||
输出含规范化 JSONL、manifest、采集回执以及所选章节的内容寻址存证。每条保留出版 DOI、卷号、页码、按顺序的完整作者及公开摘要。目录证据用 URL 和哈希连接到输入归档,保存归档以便复核。主会出版观察依据 Springer 正式论文集,原始投稿号、决定日期及签名仍未知。工具仅处理已归档的公开 HTML。
|
输出含规范化 JSONL、manifest、采集回执以及所选章节的内容寻址存证。每条保留出版 DOI、卷号、页码、按顺序的完整作者及公开摘要。单名署名须由同页的结构化作者列表按顺序确认,确认后按原名保存为字符串,引用元数据仍保留原文。目录证据用 URL 和哈希连接到输入归档,保存归档以便复核。主会出版观察依据 Springer 正式论文集,原始投稿号、决定日期及签名仍未知。工具仅处理已归档的公开 HTML。
|
||||||
|
|
||||||
生成的文件待做论文身份复核后方可导入。准备生产批次时核实每个 DOI 与现有 arXiv 标识的绑定,保持两者的来源证据。对已存在的同届节目投稿需先核对投稿身份及轨道,再附加出版来源,避免因直接导入创建重复投稿。具有 ICLR 等既往决定的论文须保留原记录。新的增量导入 manifest 需以当前实际来源覆盖为父版本,留存待核差集和独立验收回执。
|
生成的文件待做论文身份复核后方可导入。准备生产批次时核实每个 DOI 与现有 arXiv 标识的绑定,保持两者的来源证据。对已存在的同届节目投稿需先核对投稿身份及轨道,再附加出版来源,避免因直接导入创建重复投稿。具有 ICLR 等既往决定的论文须保留原记录。新的增量导入 manifest 需以当前实际来源覆盖为父版本,留存待核差集和独立验收回执。
|
||||||
|
|||||||
@@ -1,18 +0,0 @@
|
|||||||
# KDD 2026 官方论文目录
|
|
||||||
|
|
||||||
从 [KDD 2026 论文目录](https://kdd2026.kdd.org/papers/) 保存公开 HTML,记录其 SHA-256,在已有目录中指定新的输出文件。
|
|
||||||
|
|
||||||
```sh
|
|
||||||
paper-atlas collect kdd2026-program \
|
|
||||||
--snapshot kdd2026-papers.html \
|
|
||||||
--snapshot-sha256 <snapshot-sha256> \
|
|
||||||
--out kdd2026-programme.json
|
|
||||||
```
|
|
||||||
|
|
||||||
采集器只读取已保存的页面数据,逐项输出 DOI、官网标题、投稿周期、轨道和行哈希。完全相同的重复条目保留出现次数,冲突记录中止解析并等待人工复核。输出用于与来源和库内 DOI 核对,不能直接交给记录导入命令。
|
|
||||||
|
|
||||||
已核对的官网快照有 1,415 条目录记录和 1,408 个唯一 DOI。7 条重复记录的周期、轨道、标题和作者文本一致。February Cycle 列出 Research、Applied Data Science、Datasets and Benchmarks、AI for Sciences、Blue Sky Ideas 五轨。July Cycle 列出前三轨。官网列出的演讲、海报或扩展摘要需要出版方记录确认归档状态。[AI for Sciences 投稿及出版规则](https://kdd2026.kdd.org/ai4sciences-track-call-for-papers/)
|
|
||||||
|
|
||||||
Applied Data Science 有一条官网标注撤回并更换投稿 ID 的记录,旧链接为 `10.1145/3770855.3818478`。保留官网原文和日期字面值。原始投稿 ID 仍待公开证据核实。arXiv 当前指向 `10.1145/3770855.3828560`,这个 DOI 已属于同一份已有作品。两份来源分别记录,旧链接不作为新增作品标识。[arXiv 当前记录](https://arxiv.org/abs/2606.08923)
|
|
||||||
|
|
||||||
补分轨时按 DOI 检查库内已有投稿,先处理重复或缺少作者的记录。官网对录用材料的公开承诺适用于各轨道规定的范围,逐篇拒稿、初筛拒稿和其他撤回仍需公开决定证据。[Research](https://kdd2026.kdd.org/research-track-call-for-papers/) · [Applied Data Science](https://kdd2026.kdd.org/applied-data-science-ads-track-call-for-papers/) · [Datasets and Benchmarks](https://kdd2026.kdd.org/datasets-and-benchmarks-track-call-for-papers/) · [Blue Sky Ideas](https://kdd2026.kdd.org/kdd-2026-blue-sky-ideas-track-call-for-papers/)
|
|
||||||
@@ -35,7 +35,6 @@ COMMANDS = {
|
|||||||
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
|
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
|
||||||
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
||||||
('collect','recsys2026-program'): ('paper_atlas.collectors.recsys2026', {}, '核验 RecSys 2026 公开录用贡献并保留 DOI 来源版本'),
|
('collect','recsys2026-program'): ('paper_atlas.collectors.recsys2026', {}, '核验 RecSys 2026 公开录用贡献并保留 DOI 来源版本'),
|
||||||
('collect','kdd2026-program'): ('paper_atlas.collectors.kdd2026', {}, '核对 KDD 2026 两个投稿周期、分轨及撤回更换标注'),
|
|
||||||
('collect','mm2026-program'): ('paper_atlas.collectors.mm2026', {}, '恢复 MM 2026 全部来源 track 与重复展示上下文,待贡献和身份核查'),
|
('collect','mm2026-program'): ('paper_atlas.collectors.mm2026', {}, '恢复 MM 2026 全部来源 track 与重复展示上下文,待贡献和身份核查'),
|
||||||
('collect','mm2026-publications'): ('paper_atlas.collectors.mm_publications2026', {}, '核验 MM 2026 全部注册出版条目并分别保留节目分轨、摘要与版本差异'),
|
('collect','mm2026-publications'): ('paper_atlas.collectors.mm_publications2026', {}, '核验 MM 2026 全部注册出版条目并分别保留节目分轨、摘要与版本差异'),
|
||||||
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
||||||
|
|||||||
@@ -1,129 +0,0 @@
|
|||||||
"""Inventory KDD 2026 programme DOI, cycle and track observations."""
|
|
||||||
import argparse
|
|
||||||
from collections import Counter
|
|
||||||
from hashlib import sha256
|
|
||||||
from html.parser import HTMLParser
|
|
||||||
import json
|
|
||||||
from pathlib import Path
|
|
||||||
import re
|
|
||||||
|
|
||||||
SOURCE_URL = 'https://kdd2026.kdd.org/papers/'
|
|
||||||
CYCLES = {1: 'july', 2: 'february'}
|
|
||||||
TRACKS = {
|
|
||||||
'rtp': 'research', 'ads': 'applied_data_science', 'dtb': 'datasets_and_benchmarks',
|
|
||||||
'ais': 'ai_for_sciences', 'bsi': 'blue_sky_ideas',
|
|
||||||
}
|
|
||||||
DOI_URL = re.compile(r'https://doi\.org/(10\.1145/\d+\.\d+)')
|
|
||||||
WITHDRAWN = re.compile(
|
|
||||||
r'^\*\*WITHDRAWN/New SUBMISSION ID for Authorship Order Change (\d{1,2}/\d{1,2}/26) -- (.+)$'
|
|
||||||
)
|
|
||||||
OTHER_STATUS = re.compile(r'^[*\[(!#\s]*(?:withdrawn|rejected|desk[\s_-]+rejected|retracted|cancell?ed)\b', re.I)
|
|
||||||
|
|
||||||
|
|
||||||
class ProgrammeScripts(HTMLParser):
|
|
||||||
def __init__(self):
|
|
||||||
super().__init__()
|
|
||||||
self.current = None
|
|
||||||
self.scripts = []
|
|
||||||
|
|
||||||
def handle_starttag(self, tag, attrs):
|
|
||||||
if tag == 'script' and 'src' not in dict(attrs):
|
|
||||||
self.current = []
|
|
||||||
|
|
||||||
def handle_data(self, data):
|
|
||||||
if self.current is not None:
|
|
||||||
self.current.append(data)
|
|
||||||
|
|
||||||
def handle_endtag(self, tag):
|
|
||||||
if tag == 'script' and self.current is not None:
|
|
||||||
self.scripts.append(''.join(self.current))
|
|
||||||
self.current = None
|
|
||||||
|
|
||||||
|
|
||||||
def cycle_rows(scripts, cycle):
|
|
||||||
pattern = re.compile(r'\bconst\s+cycle' + str(cycle) + r'Papers\s*=\s*(\[)')
|
|
||||||
matches = [(code, match) for code in scripts for match in pattern.finditer(code)]
|
|
||||||
if len(matches) != 1:
|
|
||||||
raise ValueError(f'Expected one KDD programme array for cycle {cycle}')
|
|
||||||
code, match = matches[0]
|
|
||||||
body = code[match.start(1):]
|
|
||||||
rows, consumed = json.JSONDecoder().raw_decode(body)
|
|
||||||
if not rows or not body[consumed:].lstrip().startswith('.map((paper, index)'):
|
|
||||||
raise ValueError(f'Unexpected KDD programme array boundary for cycle {cycle}')
|
|
||||||
return rows
|
|
||||||
|
|
||||||
|
|
||||||
def inventory(raw):
|
|
||||||
parser = ProgrammeScripts()
|
|
||||||
parser.feed(raw.decode('utf-8'))
|
|
||||||
parser.close()
|
|
||||||
records = {}
|
|
||||||
source_counts = {}
|
|
||||||
for cycle, cycle_name in CYCLES.items():
|
|
||||||
rows = cycle_rows(parser.scripts, cycle)
|
|
||||||
source_counts[cycle_name] = Counter()
|
|
||||||
for row in rows:
|
|
||||||
if (not isinstance(row, dict) or row.keys() != {'track', 'title', 'url', 'authors'}
|
|
||||||
or not all(isinstance(value, str) for value in row.values())
|
|
||||||
or row['track'] not in TRACKS or not row['title'].strip()):
|
|
||||||
raise ValueError('Unexpected KDD programme row')
|
|
||||||
match = DOI_URL.fullmatch(row['url'])
|
|
||||||
if not match:
|
|
||||||
raise ValueError('Invalid DOI link in KDD programme')
|
|
||||||
doi = match[1]
|
|
||||||
track = TRACKS[row['track']]
|
|
||||||
source_counts[cycle_name][track] += 1
|
|
||||||
row_hash = sha256(json.dumps(row, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
|
|
||||||
if doi in records:
|
|
||||||
entry = records[doi]
|
|
||||||
if (entry['cycle'], entry['source_row_sha256']) != (cycle_name, row_hash):
|
|
||||||
raise ValueError('Conflicting duplicate DOI in KDD programme: ' + doi)
|
|
||||||
entry['occurrences'] += 1
|
|
||||||
continue
|
|
||||||
status = 'listed'
|
|
||||||
withdrawn = WITHDRAWN.fullmatch(row['title'])
|
|
||||||
if withdrawn:
|
|
||||||
status = 'withdrawn_with_replacement_mentioned'
|
|
||||||
elif OTHER_STATUS.match(row['title']):
|
|
||||||
raise ValueError('Unrecognized status annotation in KDD programme: ' + doi)
|
|
||||||
records[doi] = {
|
|
||||||
'doi': doi, 'doi_url': row['url'], 'source_title': row['title'],
|
|
||||||
'title': withdrawn[2] if withdrawn else row['title'],
|
|
||||||
'cycle': cycle_name, 'track': track, 'programme_status': status,
|
|
||||||
'authors_listed': bool(row['authors'].strip()), 'occurrences': 1,
|
|
||||||
'source_row_sha256': row_hash,
|
|
||||||
}
|
|
||||||
if withdrawn:
|
|
||||||
records[doi]['source_withdrawal_date_literal'] = withdrawn[1]
|
|
||||||
ordered = [records[doi] for doi in sorted(records)]
|
|
||||||
return ordered, source_counts
|
|
||||||
|
|
||||||
|
|
||||||
def collect(snapshot, expected_sha256, out):
|
|
||||||
snapshot = Path(snapshot)
|
|
||||||
with snapshot.open('rb') as stream:
|
|
||||||
raw = stream.read(4_000_001)
|
|
||||||
if len(raw) > 4_000_000 or sha256(raw).hexdigest() != expected_sha256:
|
|
||||||
raise ValueError('KDD programme snapshot hash or size differs')
|
|
||||||
rows, cycle_counts = inventory(raw)
|
|
||||||
result = {
|
|
||||||
'source': SOURCE_URL, 'snapshot_sha256': expected_sha256,
|
|
||||||
'source_rows_by_cycle_and_track': cycle_counts, 'unique_dois': len(rows),
|
|
||||||
'identical_repeat_rows': sum(row['occurrences'] - 1 for row in rows),
|
|
||||||
'programme_status_counts': Counter(row['programme_status'] for row in rows),
|
|
||||||
'records': rows, 'database_written': False,
|
|
||||||
}
|
|
||||||
with Path(out).open('x', encoding='utf-8') as stream:
|
|
||||||
json.dump(result, stream, ensure_ascii=False, indent=2)
|
|
||||||
stream.write('\n')
|
|
||||||
return result
|
|
||||||
|
|
||||||
|
|
||||||
def main(argv=None, *, prog=None):
|
|
||||||
parser = argparse.ArgumentParser(prog=prog)
|
|
||||||
parser.add_argument('--snapshot', required=True)
|
|
||||||
parser.add_argument('--snapshot-sha256', required=True)
|
|
||||||
parser.add_argument('--out', required=True)
|
|
||||||
args = parser.parse_args(argv)
|
|
||||||
result = collect(args.snapshot, args.snapshot_sha256, args.out)
|
|
||||||
print(json.dumps({k: result[k] for k in ('unique_dois', 'identical_repeat_rows', 'programme_status_counts')}))
|
|
||||||
@@ -54,6 +54,27 @@ def read_page(index, url):
|
|||||||
return raw, evidence
|
return raw, evidence
|
||||||
|
|
||||||
|
|
||||||
|
def require_display_authors(nodes, authors):
|
||||||
|
matching_article_lists = []
|
||||||
|
for node in nodes:
|
||||||
|
if node.tag != 'script' or node.attrs.get('type') != 'application/ld+json':
|
||||||
|
continue
|
||||||
|
try:
|
||||||
|
article = json.loads(''.join(node.children))
|
||||||
|
except json.JSONDecodeError:
|
||||||
|
continue
|
||||||
|
if not isinstance(article, dict) or article.get('@type') != 'ScholarlyArticle':
|
||||||
|
continue
|
||||||
|
entries = article.get('author')
|
||||||
|
if not isinstance(entries, list) or not all(
|
||||||
|
isinstance(entry, dict) and isinstance(entry.get('name'), str) for entry in entries
|
||||||
|
):
|
||||||
|
raise ValueError('Unreviewed Springer article author list')
|
||||||
|
matching_article_lists.append([clean(entry['name']) for entry in entries])
|
||||||
|
if matching_article_lists != [authors]:
|
||||||
|
raise ValueError('Single-name author requires matching Springer display author list')
|
||||||
|
|
||||||
|
|
||||||
def parse_chapter(html, doi, *, book_title, publication_year):
|
def parse_chapter(html, doi, *, book_title, publication_year):
|
||||||
nodes = list(Document(html).root.walk())
|
nodes = list(Document(html).root.walk())
|
||||||
meta = defaultdict(list)
|
meta = defaultdict(list)
|
||||||
@@ -73,12 +94,22 @@ def parse_chapter(html, doi, *, book_title, publication_year):
|
|||||||
raise ValueError('Missing named chapter authors')
|
raise ValueError('Missing named chapter authors')
|
||||||
authors = []
|
authors = []
|
||||||
for raw in raw_authors:
|
for raw in raw_authors:
|
||||||
|
if ',' not in raw:
|
||||||
|
name = clean(raw)
|
||||||
|
if len(name.split()) != 1 or not any(character.isalpha() for character in name):
|
||||||
|
raise ValueError('Unreviewed citation author format')
|
||||||
|
authors.append(name)
|
||||||
|
continue
|
||||||
if raw.count(',') != 1:
|
if raw.count(',') != 1:
|
||||||
raise ValueError('Unreviewed citation author format')
|
raise ValueError('Unreviewed citation author format')
|
||||||
family, given = [clean(s) for s in raw.split(',', 1)]
|
family, given = [clean(s) for s in raw.split(',', 1)]
|
||||||
if not family or not given:
|
if not family or not given:
|
||||||
raise ValueError('Incomplete citation author')
|
raise ValueError('Incomplete citation author')
|
||||||
authors.append({'given':given, 'family':family})
|
authors.append({'given':given, 'family':family})
|
||||||
|
if any(isinstance(author, str) for author in authors):
|
||||||
|
display_names = [author if isinstance(author, str) else author['given']+' '+author['family']
|
||||||
|
for author in authors]
|
||||||
|
require_display_authors(nodes, display_names)
|
||||||
abstract_nodes = [n for n in nodes if n.attrs.get('id') == 'Abs1-content']
|
abstract_nodes = [n for n in nodes if n.attrs.get('id') == 'Abs1-content']
|
||||||
if len(abstract_nodes) > 1:
|
if len(abstract_nodes) > 1:
|
||||||
raise ValueError('Ambiguous abstract section')
|
raise ValueError('Ambiguous abstract section')
|
||||||
|
|||||||
@@ -1,68 +0,0 @@
|
|||||||
from hashlib import sha256
|
|
||||||
import json
|
|
||||||
from pathlib import Path
|
|
||||||
from tempfile import TemporaryDirectory
|
|
||||||
import unittest
|
|
||||||
|
|
||||||
from paper_atlas.cli import main
|
|
||||||
from paper_atlas.collectors.kdd2026 import inventory
|
|
||||||
|
|
||||||
|
|
||||||
def row(number, track, title='Paper title', authors='Name (Affiliation)'):
|
|
||||||
return {'track': track, 'title': title, 'url': f'https://doi.org/10.1145/3770855.{number}', 'authors': authors}
|
|
||||||
|
|
||||||
|
|
||||||
def snapshot(july, february):
|
|
||||||
arrays = [
|
|
||||||
f'<script>const cycle{n}Papers = {json.dumps(rows)}.map((paper, index) => paper);</script>'
|
|
||||||
for n, rows in ((1, july), (2, february))
|
|
||||||
]
|
|
||||||
return '\n'.join(arrays).encode()
|
|
||||||
|
|
||||||
|
|
||||||
class KDD2026Tests(unittest.TestCase):
|
|
||||||
def test_cli_preserves_withdrawal_marker_and_identical_occurrences(self):
|
|
||||||
retained = row(1, 'rtp')
|
|
||||||
withdrawn = row(2, 'ads', '**WITHDRAWN/New SUBMISSION ID for Authorship Order Change 6/24/26 -- A revised work')
|
|
||||||
incomplete = row(3, 'ais', authors='')
|
|
||||||
raw = snapshot([retained, retained], [withdrawn, incomplete])
|
|
||||||
with TemporaryDirectory() as directory:
|
|
||||||
source = Path(directory) / 'snapshot.html'
|
|
||||||
source.write_bytes(raw)
|
|
||||||
out = source.with_suffix('.json')
|
|
||||||
args = ['collect', 'kdd2026-program', '--snapshot', str(source), '--snapshot-sha256',
|
|
||||||
sha256(raw).hexdigest(), '--out', str(out)]
|
|
||||||
main(args)
|
|
||||||
report = json.loads(out.read_bytes())
|
|
||||||
with self.assertRaises(FileExistsError):
|
|
||||||
main(args)
|
|
||||||
self.assertEqual(report['source_rows_by_cycle_and_track'], {
|
|
||||||
'july': {'research': 2}, 'february': {'applied_data_science': 1, 'ai_for_sciences': 1},
|
|
||||||
})
|
|
||||||
self.assertEqual(report['unique_dois'], 3)
|
|
||||||
self.assertEqual(report['identical_repeat_rows'], 1)
|
|
||||||
kept, replaced, sparse = report['records']
|
|
||||||
self.assertEqual(kept['occurrences'], 2)
|
|
||||||
self.assertEqual(replaced['programme_status'], 'withdrawn_with_replacement_mentioned')
|
|
||||||
self.assertEqual(replaced['title'], 'A revised work')
|
|
||||||
self.assertEqual(replaced['source_title'], withdrawn['title'])
|
|
||||||
self.assertEqual(replaced['source_withdrawal_date_literal'], '6/24/26')
|
|
||||||
self.assertNotIn('submission_id', replaced)
|
|
||||||
self.assertFalse(sparse['authors_listed'])
|
|
||||||
self.assertFalse(report['database_written'])
|
|
||||||
|
|
||||||
def test_conflicting_duplicate_and_unknown_status_fail(self):
|
|
||||||
with self.assertRaisesRegex(ValueError, 'Conflicting duplicate DOI'):
|
|
||||||
inventory(snapshot([row(1, 'rtp')], [row(1, 'ads')]))
|
|
||||||
with self.assertRaisesRegex(ValueError, 'Unrecognized status'):
|
|
||||||
inventory(snapshot([row(1, 'rtp')], [row(2, 'ads', '***WITHDRAWN -- Unknown format')]))
|
|
||||||
|
|
||||||
def test_hash_mismatch_leaves_no_output(self):
|
|
||||||
with TemporaryDirectory() as directory:
|
|
||||||
source = Path(directory) / 'snapshot.html'
|
|
||||||
source.write_bytes(snapshot([row(1, 'rtp')], [row(2, 'ads')]))
|
|
||||||
out = source.with_suffix('.json')
|
|
||||||
with self.assertRaisesRegex(ValueError, 'snapshot hash or size'):
|
|
||||||
main(['collect', 'kdd2026-program', '--snapshot', str(source), '--snapshot-sha256',
|
|
||||||
'0' * 64, '--out', str(out)])
|
|
||||||
self.assertFalse(out.exists())
|
|
||||||
@@ -9,7 +9,7 @@ import tempfile
|
|||||||
import unittest
|
import unittest
|
||||||
|
|
||||||
from paper_atlas.collectors.springer_selected import (
|
from paper_atlas.collectors.springer_selected import (
|
||||||
BOOK, SOURCE, collect, parse_chapter, parse_toc, read_page, snapshot_index,
|
BOOK, SCOPE, SOURCE, collect, parse_chapter, parse_toc, read_page, snapshot_index,
|
||||||
)
|
)
|
||||||
from paper_atlas.db.store import Store
|
from paper_atlas.db.store import Store
|
||||||
from paper_atlas.ingestion.records import ingest_file
|
from paper_atlas.ingestion.records import ingest_file
|
||||||
@@ -153,6 +153,28 @@ class SpringerSelectedTests(unittest.TestCase):
|
|||||||
html = self.fixture['chapters']['chapter-01.html'].replace('id="Abs1-content"','id="not-an-abstract"')
|
html = self.fixture['chapters']['chapter-01.html'].replace('id="Abs1-content"','id="not-an-abstract"')
|
||||||
self.assertIsNone(parse_chapter(html, BOOK+'_1')['abstract'])
|
self.assertIsNone(parse_chapter(html, BOOK+'_1')['abstract'])
|
||||||
|
|
||||||
|
def test_single_name_preserves_literal_and_requires_ordered_display_confirmation(self):
|
||||||
|
html = f"""
|
||||||
|
<meta name="citation_doi" content="{BOOK}_1">
|
||||||
|
<meta name="citation_inbook_title" content="{SCOPE['book_title']}">
|
||||||
|
<meta name="citation_publication_date" content="{SCOPE['publication_year']}">
|
||||||
|
<meta name="citation_title" content="Example">
|
||||||
|
<meta name="citation_author" content="Rajkumar">
|
||||||
|
<meta name="citation_author" content="Doe, Jane">
|
||||||
|
<meta name="citation_firstpage" content="1">
|
||||||
|
<meta name="citation_lastpage" content="5">
|
||||||
|
"""
|
||||||
|
article = {'@type':'ScholarlyArticle', 'author':[{'name':'Rajkumar'}, {'name':'Jane Doe'}]}
|
||||||
|
script = '<script type="application/ld+json">{}</script>'
|
||||||
|
chapter = parse_chapter(html+script.format(json.dumps(article)), BOOK+'_1')
|
||||||
|
self.assertEqual(chapter['authors'], ['Rajkumar', {'family':'Doe', 'given':'Jane'}])
|
||||||
|
self.assertEqual(chapter['authors_citation_raw'], ['Rajkumar','Doe, Jane'])
|
||||||
|
with self.assertRaisesRegex(ValueError, 'matching Springer display author list'):
|
||||||
|
parse_chapter(html, BOOK+'_1')
|
||||||
|
article['author'].reverse()
|
||||||
|
with self.assertRaisesRegex(ValueError, 'matching Springer display author list'):
|
||||||
|
parse_chapter(html+script.format(json.dumps(article)), BOOK+'_1')
|
||||||
|
|
||||||
def test_toc_ignores_frontmatter_but_rejects_duplicate_publications(self):
|
def test_toc_ignores_frontmatter_but_rejects_duplicate_publications(self):
|
||||||
html = self.fixture['toc']['springer-toc1.html']
|
html = self.fixture['toc']['springer-toc1.html']
|
||||||
self.assertEqual(len(parse_toc(html)), 17)
|
self.assertEqual(len(parse_toc(html)), 17)
|
||||||
|
|||||||
Reference in New Issue
Block a user