Compare commits

..

1 Commits

Author SHA1 Message Date
codex-bot 6e5202c81a Inventory KDD 2026 programme cycles and withdrawal evidence
Metadata integrity checks / metadata (push) Successful in 8m7s
2026-11-18 23:14:36 +08:00
11 changed files with 247 additions and 403 deletions
+14
View File
@@ -47,6 +47,20 @@ jobs:
subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True) subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True)
outside = root / 'outside-checkout' outside = root / 'outside-checkout'
outside.mkdir() outside.mkdir()
kdd_snapshot = pathlib.Path('/data/paper-index/validation/kdd2026-official-papers-html-v1.html')
if kdd_snapshot.is_file():
kdd_output = root / 'installed-kdd2026.json'
subprocess.run([python, '-I', '-B', '-m', 'paper_atlas', 'collect', 'kdd2026-program',
'--snapshot', str(kdd_snapshot), '--snapshot-sha256',
'47ee82841dedf034614348d7fdb29714e8ad62fca8c6811e1d856845588f145c',
'--out', str(kdd_output)], cwd=outside, check=True)
kdd = json.loads(kdd_output.read_bytes())
assert kdd['unique_dois'] == 1408 and kdd['identical_repeat_rows'] == 7
assert {key:sum(counts.values()) for key,counts in kdd['source_rows_by_cycle_and_track'].items()} == {
'july':256, 'february':1159}
held, = [entry for entry in kdd['records'] if entry['programme_status'] != 'listed']
assert held['doi'] == '10.1145/3770855.3818478' and held['source_withdrawal_date_literal'] == '6/24/26'
print('KDD 2026 official cycle and withdrawal fixture passed', flush=True)
cikm_snapshots = pathlib.Path('/data/paper-index/validation/cikm2026-source-snapshots-v1') cikm_snapshots = pathlib.Path('/data/paper-index/validation/cikm2026-source-snapshots-v1')
if cikm_snapshots.is_dir(): if cikm_snapshots.is_dir():
cikm_output = root / 'installed-cikm2026' cikm_output = root / 'installed-cikm2026'
+3 -1
View File
@@ -147,8 +147,10 @@ ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目
`collect recsys2026-program` 核验九个 track 的 263 项官方录用贡献观察(260 个不同 DOI),保留摘要、原始作者和重复 DOI 的来源版本。见 [RecSys 2026 使用与恢复说明](docs/recsys2026-program.md)。 `collect recsys2026-program` 核验九个 track 的 263 项官方录用贡献观察(260 个不同 DOI),保留摘要、原始作者和重复 DOI 的来源版本。见 [RecSys 2026 使用与恢复说明](docs/recsys2026-program.md)。
[KDD 2026 官方目录](docs/kdd2026-program.md)可以按 DOI 核对两个投稿周期、分轨与目录中的撤回更换标注。
`collect mm2026-program` 从已冻结官方快照恢复全部 28 个来源 track 的 2,143 个节目 ID,保留 4,326 次展示的来源位置与场次上下文,单列 216 个摘要缺失项;输出供贡献类型、出版及作品身份核查使用,不是可直接导入的论文/决定队列。见 [MM 2026 使用、来源和恢复说明](docs/mm2026-source-status.md)。 `collect mm2026-program` 从已冻结官方快照恢复全部 28 个来源 track 的 2,143 个节目 ID,保留 4,326 次展示的来源位置与场次上下文,单列 216 个摘要缺失项;输出供贡献类型、出版及作品身份核查使用,不是可直接导入的论文/决定队列。见 [MM 2026 使用、来源和恢复说明](docs/mm2026-source-status.md)。
`collect mm2026-publications` 核验完整论文集的 1,918 个注册出版 DOI,并单独关联节目分轨、展示上下文和摘要。1,743 份节目摘要明确标注来源(其中 26 份来自已复核的标题排版对应),166 项标题/作者版本差异保留为待核;所有节目及未匹配项均保留。注册文章类型不等于研究类型,输出尚需数据库身份/历史审查。见 [MM 2026 出版条目采集与恢复](docs/mm2026-publications.md)。 `collect mm2026-publications` 核验完整论文集的 1,918 个注册出版 DOI,并单独关联节目分轨、展示上下文和摘要。1,743 份节目摘要明确标注来源(其中 26 份来自已复核的标题排版对应),166 项标题/作者版本差异保留为待核;所有节目及未匹配项均保留。注册文章类型不等于研究类型,输出尚需数据库身份/历史审查。见 [MM 2026 出版条目采集与恢复](docs/mm2026-publications.md)。
ECCV 2026 已归档 Springer 目录和章节可通过 [正式出版采集流程](docs/eccv2026-publications.md)整理为带 DOI 和可复核证据的 JSONL。同届节目投稿可按独立审核计划附加出版证据,保留原投稿 ID 和既往决定。 ECCV 2026 已归档 Springer 目录和章节可通过 [正式出版采集流程](docs/eccv2026-publications.md)整理为带 DOI 和可复核证据的 JSONL。
-15
View File
@@ -14,18 +14,3 @@ paper-atlas collect eccv2026-publications \
输出含规范化 JSONL、manifest、采集回执以及所选章节的内容寻址存证。每条保留出版 DOI、卷号、页码、按顺序的完整作者及公开摘要。目录证据用 URL 和哈希连接到输入归档,保存归档以便复核。主会出版观察依据 Springer 正式论文集,原始投稿号、决定日期及签名仍未知。工具仅处理已归档的公开 HTML。 输出含规范化 JSONL、manifest、采集回执以及所选章节的内容寻址存证。每条保留出版 DOI、卷号、页码、按顺序的完整作者及公开摘要。目录证据用 URL 和哈希连接到输入归档,保存归档以便复核。主会出版观察依据 Springer 正式论文集,原始投稿号、决定日期及签名仍未知。工具仅处理已归档的公开 HTML。
生成的文件待做论文身份复核后方可导入。准备生产批次时核实每个 DOI 与现有 arXiv 标识的绑定,保持两者的来源证据。对已存在的同届节目投稿需先核对投稿身份及轨道,再附加出版来源,避免因直接导入创建重复投稿。具有 ICLR 等既往决定的论文须保留原记录。新的增量导入 manifest 需以当前实际来源覆盖为父版本,留存待核差集和独立验收回执。 生成的文件待做论文身份复核后方可导入。准备生产批次时核实每个 DOI 与现有 arXiv 标识的绑定,保持两者的来源证据。对已存在的同届节目投稿需先核对投稿身份及轨道,再附加出版来源,避免因直接导入创建重复投稿。具有 ICLR 等既往决定的论文须保留原记录。新的增量导入 manifest 需以当前实际来源覆盖为父版本,留存待核差集和独立验收回执。
## 已有节目投稿的出版证据
先在只读快照上复核节目 ID、章节 DOI、标题、完整有序作者、摘要版本与当前投稿历史。保存计划和原始证据,计划应列出每条出版观察、保留的论文与投稿 ID、节目来源和决定 ID、当前与目标届次及完整历史包的 SHA-256。作者或版本仍有疑点的条目继续留在待核清单。
```sh
paper-atlas maintain eccv-publication-reconciliation \
--pg-service <writer-service> --pg-schema research \
--plan <reviewed-plan.json> --plan-sha256 <reviewed-plan-sha256> \
--springer-previous-count <last-verified-count>
```
命令接收 `readonly_same_submission_reconciliation_plan` 格式的已复核计划。它会在同一事务内核对实时历史和来源覆盖,把原节目投稿归入主会,追加出版来源与观测决定,并分别调整节目与 Springer 的覆盖数。节目主会覆盖保持 partial,因为其目录的主会分母尚未独立核定。随后通过只读连接核验投稿、决定及覆盖,保存计划和回执。相同计划只允许成功提交一次。若执行中失败,先查同一计划的提交标记、覆盖和投稿来源,再决定是否重新执行。
旧节目快照的全量未分类 manifest 无法刷新已有逐条更正的投稿。继续刷新时需按已复核轨道分别整理输出并生成 manifest。
+18
View File
@@ -0,0 +1,18 @@
# KDD 2026 官方论文目录
从 [KDD 2026 论文目录](https://kdd2026.kdd.org/papers/) 保存公开 HTML,记录其 SHA-256,在已有目录中指定新的输出文件。
```sh
paper-atlas collect kdd2026-program \
--snapshot kdd2026-papers.html \
--snapshot-sha256 <snapshot-sha256> \
--out kdd2026-programme.json
```
采集器只读取已保存的页面数据,逐项输出 DOI、官网标题、投稿周期、轨道和行哈希。完全相同的重复条目保留出现次数,冲突记录中止解析并等待人工复核。输出用于与来源和库内 DOI 核对,不能直接交给记录导入命令。
已核对的官网快照有 1,415 条目录记录和 1,408 个唯一 DOI。7 条重复记录的周期、轨道、标题和作者文本一致。February Cycle 列出 Research、Applied Data Science、Datasets and Benchmarks、AI for Sciences、Blue Sky Ideas 五轨。July Cycle 列出前三轨。官网列出的演讲、海报或扩展摘要需要出版方记录确认归档状态。[AI for Sciences 投稿及出版规则](https://kdd2026.kdd.org/ai4sciences-track-call-for-papers/)
Applied Data Science 有一条官网标注撤回并更换投稿 ID 的记录,旧链接为 `10.1145/3770855.3818478`。保留官网原文和日期字面值。原始投稿 ID 仍待公开证据核实。arXiv 当前指向 `10.1145/3770855.3828560`,这个 DOI 已属于同一份已有作品。两份来源分别记录,旧链接不作为新增作品标识。[arXiv 当前记录](https://arxiv.org/abs/2606.08923)
补分轨时按 DOI 检查库内已有投稿,先处理重复或缺少作者的记录。官网对录用材料的公开承诺适用于各轨道规定的范围,逐篇拒稿、初筛拒稿和其他撤回仍需公开决定证据。[Research](https://kdd2026.kdd.org/research-track-call-for-papers/) · [Applied Data Science](https://kdd2026.kdd.org/applied-data-science-ads-track-call-for-papers/) · [Datasets and Benchmarks](https://kdd2026.kdd.org/datasets-and-benchmarks-track-call-for-papers/) · [Blue Sky Ideas](https://kdd2026.kdd.org/kdd-2026-blue-sky-ideas-track-call-for-papers/)
+14 -20
View File
@@ -173,40 +173,34 @@ def find_candidates(connection, records):
records[hit['n']]['title_candidate_ids'].append(hit['paper_id']) records[hit['n']]['title_candidate_ids'].append(hit['paper_id'])
def _named_rows(connection, sql, params):
cursor = connection.execute(sql, params)
columns = [field[0] for field in cursor.description]
return [{name: row[i] for i, name in enumerate(columns)} for row in cursor]
def entity_packet(connection, paper_id): def entity_packet(connection, paper_id):
papers = _named_rows(connection, 'SELECT * FROM papers WHERE id=?', (paper_id,)) paper = connection.execute('SELECT * FROM papers WHERE id=?', (paper_id,)).fetchone()
if not papers: if paper is None:
raise RuntimeError('Candidate disappeared from read-only snapshot') raise RuntimeError('Candidate disappeared from read-only snapshot')
sources = _named_rows(connection, '''SELECT r.*,s.name AS source sources = [dict(r) for r in connection.execute('''SELECT r.*,s.name AS source
FROM source_records r JOIN sources s ON s.id=r.source_id FROM source_records r JOIN sources s ON s.id=r.source_id
WHERE r.paper_id=? ORDER BY r.id LIMIT 101''', (paper_id,)) WHERE r.paper_id=? ORDER BY r.id LIMIT 101''', (paper_id,))]
attempts = _named_rows(connection, '''SELECT s.*,v.acronym AS venue,e.year,e.track,e.event_key attempts = [dict(r) for r in connection.execute('''SELECT s.*,v.acronym AS venue,e.year,e.track,e.event_key
FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id
WHERE s.paper_id=? ORDER BY s.id LIMIT 201''', (paper_id,)) WHERE s.paper_id=? ORDER BY s.id LIMIT 201''', (paper_id,))]
packet = {'paper': papers[0], 'identifiers': _named_rows(connection, packet = {'paper': dict(paper), 'identifiers': [dict(r) for r in connection.execute(
'SELECT * FROM identifiers WHERE paper_id=? ORDER BY scheme,value', (paper_id,)), 'SELECT * FROM identifiers WHERE paper_id=? ORDER BY scheme,value', (paper_id,))],
'source_records': sources[:100], 'source_records_truncated': len(sources) > 100, 'source_records': sources[:100], 'source_records_truncated': len(sources) > 100,
'attempts': attempts[:200], 'attempts_truncated': len(attempts) > 200} 'attempts': attempts[:200], 'attempts_truncated': len(attempts) > 200}
for attempt in packet['attempts']: for attempt in packet['attempts']:
observations = _named_rows(connection, observations = [dict(r) for r in connection.execute(
'SELECT * FROM decisions WHERE submission_id=? ORDER BY id LIMIT 201', (attempt['id'],)) 'SELECT * FROM decisions WHERE submission_id=? ORDER BY id LIMIT 201', (attempt['id'],))]
attempt['decisions'], attempt['decisions_truncated'] = observations[:200], len(observations) > 200 attempt['decisions'], attempt['decisions_truncated'] = observations[:200], len(observations) > 200
provenance = _named_rows(connection, '''SELECT ss.*,r.external_id,s.name AS source provenance = [dict(r) for r in connection.execute('''SELECT ss.*,r.external_id,s.name AS source
FROM submission_sources ss JOIN source_records r ON r.id=ss.source_record_id FROM submission_sources ss JOIN source_records r ON r.id=ss.source_record_id
JOIN sources s ON s.id=r.source_id WHERE ss.submission_id=? JOIN sources s ON s.id=r.source_id WHERE ss.submission_id=?
ORDER BY ss.source_record_id,ss.source_submission_id LIMIT 101''', (attempt['id'],)) ORDER BY ss.source_record_id,ss.source_submission_id LIMIT 101''', (attempt['id'],))]
attempt['sources'], attempt['sources_truncated'] = provenance[:100], len(provenance) > 100 attempt['sources'], attempt['sources_truncated'] = provenance[:100], len(provenance) > 100
for decision in attempt['decisions']: for decision in attempt['decisions']:
provenance = _named_rows(connection, '''SELECT ds.*,r.external_id,s.name AS source provenance = [dict(r) for r in connection.execute('''SELECT ds.*,r.external_id,s.name AS source
FROM decision_sources ds JOIN source_records r ON r.id=ds.source_record_id FROM decision_sources ds JOIN source_records r ON r.id=ds.source_record_id
JOIN sources s ON s.id=r.source_id WHERE ds.decision_id=? JOIN sources s ON s.id=r.source_id WHERE ds.decision_id=?
ORDER BY ds.source_record_id,ds.source_decision_id LIMIT 101''', (decision['id'],)) ORDER BY ds.source_record_id,ds.source_decision_id LIMIT 101''', (decision['id'],))]
decision['sources'], decision['sources_truncated'] = provenance[:100], len(provenance) > 100 decision['sources'], decision['sources_truncated'] = provenance[:100], len(provenance) > 100
return packet return packet
+1 -1
View File
@@ -35,6 +35,7 @@ COMMANDS = {
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'), ('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'), ('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
('collect','recsys2026-program'): ('paper_atlas.collectors.recsys2026', {}, '核验 RecSys 2026 公开录用贡献并保留 DOI 来源版本'), ('collect','recsys2026-program'): ('paper_atlas.collectors.recsys2026', {}, '核验 RecSys 2026 公开录用贡献并保留 DOI 来源版本'),
('collect','kdd2026-program'): ('paper_atlas.collectors.kdd2026', {}, '核对 KDD 2026 两个投稿周期、分轨及撤回更换标注'),
('collect','mm2026-program'): ('paper_atlas.collectors.mm2026', {}, '恢复 MM 2026 全部来源 track 与重复展示上下文,待贡献和身份核查'), ('collect','mm2026-program'): ('paper_atlas.collectors.mm2026', {}, '恢复 MM 2026 全部来源 track 与重复展示上下文,待贡献和身份核查'),
('collect','mm2026-publications'): ('paper_atlas.collectors.mm_publications2026', {}, '核验 MM 2026 全部注册出版条目并分别保留节目分轨、摘要与版本差异'), ('collect','mm2026-publications'): ('paper_atlas.collectors.mm_publications2026', {}, '核验 MM 2026 全部注册出版条目并分别保留节目分轨、摘要与版本差异'),
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'), ('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
@@ -49,7 +50,6 @@ COMMANDS = {
('import','arxiv'): ('paper_atlas.ingestion.arxiv', {}, '导入 arXiv 元数据 ZIP'), ('import','arxiv'): ('paper_atlas.ingestion.arxiv', {}, '导入 arXiv 元数据 ZIP'),
('import','records'): ('paper_atlas.ingestion.records', {}, '导入核验过的会议文件'), ('import','records'): ('paper_atlas.ingestion.records', {}, '导入核验过的会议文件'),
('maintain','edition-tracks'): ('paper_atlas.ingestion.edition_tracks', {}, '按逐条来源证据修正既有投稿轨道并保护刷新'), ('maintain','edition-tracks'): ('paper_atlas.ingestion.edition_tracks', {}, '按逐条来源证据修正既有投稿轨道并保护刷新'),
('maintain','eccv-publication-reconciliation'): ('paper_atlas.ingestion.publication_reconciliation', {}, '向既有 ECCV 2026 投稿附加已复核的正式出版证据'),
('maintain','source-revisions'): ('paper_atlas.ingestion.source_revisions', {}, '按完整来源版本核验并修正出版分轨与覆盖数,保留历史并阻止旧版刷新'), ('maintain','source-revisions'): ('paper_atlas.ingestion.source_revisions', {}, '按完整来源版本核验并修正出版分轨与覆盖数,保留历史并阻止旧版刷新'),
('maintain','joint-editions'): ('paper_atlas.ingestion.joint_editions', {}, '按已核验计划修正既有联合会议届次归属'), ('maintain','joint-editions'): ('paper_atlas.ingestion.joint_editions', {}, '按已核验计划修正既有联合会议届次归属'),
('maintain','event-identity'): ('paper_atlas.ingestion.event_identity', {}, '按核验计划增加同年会议事件身份'), ('maintain','event-identity'): ('paper_atlas.ingestion.event_identity', {}, '按核验计划增加同年会议事件身份'),
+129
View File
@@ -0,0 +1,129 @@
"""Inventory KDD 2026 programme DOI, cycle and track observations."""
import argparse
from collections import Counter
from hashlib import sha256
from html.parser import HTMLParser
import json
from pathlib import Path
import re
SOURCE_URL = 'https://kdd2026.kdd.org/papers/'
CYCLES = {1: 'july', 2: 'february'}
TRACKS = {
'rtp': 'research', 'ads': 'applied_data_science', 'dtb': 'datasets_and_benchmarks',
'ais': 'ai_for_sciences', 'bsi': 'blue_sky_ideas',
}
DOI_URL = re.compile(r'https://doi\.org/(10\.1145/\d+\.\d+)')
WITHDRAWN = re.compile(
r'^\*\*WITHDRAWN/New SUBMISSION ID for Authorship Order Change (\d{1,2}/\d{1,2}/26) -- (.+)$'
)
OTHER_STATUS = re.compile(r'^[*\[(!#\s]*(?:withdrawn|rejected|desk[\s_-]+rejected|retracted|cancell?ed)\b', re.I)
class ProgrammeScripts(HTMLParser):
def __init__(self):
super().__init__()
self.current = None
self.scripts = []
def handle_starttag(self, tag, attrs):
if tag == 'script' and 'src' not in dict(attrs):
self.current = []
def handle_data(self, data):
if self.current is not None:
self.current.append(data)
def handle_endtag(self, tag):
if tag == 'script' and self.current is not None:
self.scripts.append(''.join(self.current))
self.current = None
def cycle_rows(scripts, cycle):
pattern = re.compile(r'\bconst\s+cycle' + str(cycle) + r'Papers\s*=\s*(\[)')
matches = [(code, match) for code in scripts for match in pattern.finditer(code)]
if len(matches) != 1:
raise ValueError(f'Expected one KDD programme array for cycle {cycle}')
code, match = matches[0]
body = code[match.start(1):]
rows, consumed = json.JSONDecoder().raw_decode(body)
if not rows or not body[consumed:].lstrip().startswith('.map((paper, index)'):
raise ValueError(f'Unexpected KDD programme array boundary for cycle {cycle}')
return rows
def inventory(raw):
parser = ProgrammeScripts()
parser.feed(raw.decode('utf-8'))
parser.close()
records = {}
source_counts = {}
for cycle, cycle_name in CYCLES.items():
rows = cycle_rows(parser.scripts, cycle)
source_counts[cycle_name] = Counter()
for row in rows:
if (not isinstance(row, dict) or row.keys() != {'track', 'title', 'url', 'authors'}
or not all(isinstance(value, str) for value in row.values())
or row['track'] not in TRACKS or not row['title'].strip()):
raise ValueError('Unexpected KDD programme row')
match = DOI_URL.fullmatch(row['url'])
if not match:
raise ValueError('Invalid DOI link in KDD programme')
doi = match[1]
track = TRACKS[row['track']]
source_counts[cycle_name][track] += 1
row_hash = sha256(json.dumps(row, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
if doi in records:
entry = records[doi]
if (entry['cycle'], entry['source_row_sha256']) != (cycle_name, row_hash):
raise ValueError('Conflicting duplicate DOI in KDD programme: ' + doi)
entry['occurrences'] += 1
continue
status = 'listed'
withdrawn = WITHDRAWN.fullmatch(row['title'])
if withdrawn:
status = 'withdrawn_with_replacement_mentioned'
elif OTHER_STATUS.match(row['title']):
raise ValueError('Unrecognized status annotation in KDD programme: ' + doi)
records[doi] = {
'doi': doi, 'doi_url': row['url'], 'source_title': row['title'],
'title': withdrawn[2] if withdrawn else row['title'],
'cycle': cycle_name, 'track': track, 'programme_status': status,
'authors_listed': bool(row['authors'].strip()), 'occurrences': 1,
'source_row_sha256': row_hash,
}
if withdrawn:
records[doi]['source_withdrawal_date_literal'] = withdrawn[1]
ordered = [records[doi] for doi in sorted(records)]
return ordered, source_counts
def collect(snapshot, expected_sha256, out):
snapshot = Path(snapshot)
with snapshot.open('rb') as stream:
raw = stream.read(4_000_001)
if len(raw) > 4_000_000 or sha256(raw).hexdigest() != expected_sha256:
raise ValueError('KDD programme snapshot hash or size differs')
rows, cycle_counts = inventory(raw)
result = {
'source': SOURCE_URL, 'snapshot_sha256': expected_sha256,
'source_rows_by_cycle_and_track': cycle_counts, 'unique_dois': len(rows),
'identical_repeat_rows': sum(row['occurrences'] - 1 for row in rows),
'programme_status_counts': Counter(row['programme_status'] for row in rows),
'records': rows, 'database_written': False,
}
with Path(out).open('x', encoding='utf-8') as stream:
json.dump(result, stream, ensure_ascii=False, indent=2)
stream.write('\n')
return result
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog)
parser.add_argument('--snapshot', required=True)
parser.add_argument('--snapshot-sha256', required=True)
parser.add_argument('--out', required=True)
args = parser.parse_args(argv)
result = collect(args.snapshot, args.snapshot_sha256, args.out)
print(json.dumps({k: result[k] for k in ('unique_dois', 'identical_repeat_rows', 'programme_status_counts')}))
@@ -1,222 +0,0 @@
"""Attach reviewed ECCV proceedings evidence to an existing programme attempt."""
import argparse
from contextlib import closing
from hashlib import sha256
import json
from pathlib import Path
import re
from paper_atlas.audits.source_identity import entity_packet
from paper_atlas.collectors.eccv2026 import SOURCE as PROGRAMME
from paper_atlas.collectors.eccv2026_publications import SOURCE as PUBLISHER
from paper_atlas.db.backend import database_arguments, open_store
from paper_atlas.db.edition_scopes import KEY
from paper_atlas.db.submissions import identities_for_row, timestamp
from paper_atlas.ingestion.edition_tracks import curation
from paper_atlas.ingestion.records import source_record_count
MARKER = 'eccv_publication_reconciliation_v1:'
PUBLISHER_COUNT = 2835
def require(condition, message):
if not condition:
raise ValueError(message)
def packet_digest(packet):
return sha256(json.dumps(packet, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
def validate_plan(plan):
require(plan.get('status') == 'readonly_same_submission_reconciliation_plan'
and plan.get('database_written') is False, 'A reviewed read-only plan is required')
rows = plan['plans']
require(0 < len(rows) <= 1000 and len(rows) == plan['guarded_same_submission_plans'],
'Unexpected approved plan size')
for key in ('doi', 'retained_paper_id', 'retained_submission_id', 'retained_programme_record_id'):
require(len({entry[key] for entry in rows}) == len(rows), 'Repeated reviewed ' + key)
for entry in rows:
doi = entry['doi']
row = entry['source_row']
url = 'https://link.springer.com/chapter/' + doi
evidence = row['metadata']['publisher_chapter_evidence']
require(re.fullmatch(r'10\.1007/978-3-032-\d{5}-[0-9Xx]_[1-9]\d*', doi)
and (row['source'], row['external_id'], row['url'], row['evidence_url']) ==
(PUBLISHER, doi, url, url)
and (row['venue'], row['year'], row['track'], row['status']) ==
('ECCV', 2026, 'main', 'accepted')
and row['identifiers'] == {'doi': doi}
and identities_for_row(row) == [('publication_doi', doi)]
and isinstance(row['abstract'], str) and row['abstract'].strip()
and re.fullmatch(r'[0-9a-f]{64}', evidence['sha256'])
and re.fullmatch(r'[0-9a-f]{64}', entry['before_packet_sha256']),
'Invalid reviewed publisher observation')
return rows
def existing_coverage(conn, edition, source):
rows = conn.execute('''SELECT list_url,status,expected_count,imported_count,cursor,last_success_at
FROM edition_sources WHERE edition_id=? AND source_id=?''', (edition, source)).fetchall()
require(len(rows) <= 1, 'Source coverage is ambiguous')
actual = source_record_count(conn, source, edition)
if not rows:
require(actual == 0, 'Linked records have no coverage listing')
return None
row = rows[0]
require(row[1] == 'partial' and row[3] == actual, 'Coverage differs from linked records')
return tuple(row)
def reconcile_attempt(store, entry, original, target, venue_id, event_key, venue_names, publisher, signature):
conn = store.conn
doi = entry['doi']
canonical_doi = doi.lower()
paper = entry['retained_paper_id']
submission = entry['retained_submission_id']
programme_record = entry['retained_programme_record_id']
row = entry['source_row']
before = entity_packet(conn, paper)
require(packet_digest(before) == entry['before_packet_sha256'], 'Reviewed paper history changed: ' + doi)
require(not before['source_records_truncated'] and not before['attempts_truncated']
and all(not attempt['decisions_truncated'] and not attempt['sources_truncated']
and all(not decision['sources_truncated'] for decision in attempt['decisions'])
for attempt in before['attempts']), 'Reviewed paper history exceeds verification limits')
same_event = [a for a in before['attempts'] if a['venue'].casefold() == 'eccv' and a['year'] == 2026]
require(len(same_event) == 1, 'Paper has multiple or missing ECCV 2026 attempts')
attempt = same_event[0]
require((attempt['id'], attempt['edition_id'], attempt['source_record_id']) ==
(submission, original, programme_record)
and [(d['id'], d['status']) for d in attempt['decisions']] ==
[(entry['retained_programme_decision_id'], 'accepted')]
and [(s['source_record_id'], s['source']) for s in attempt['sources']] ==
[(programme_record, PROGRAMME)], 'Original programme attempt differs')
programme = next(record for record in before['source_records'] if record['id'] == programme_record)
metadata = json.loads(programme['metadata_json'])
require(KEY not in metadata and attempt['source_submission_id'] == programme['external_id'],
'Programme observation already curated or ambiguous')
scope = curation({
'external_id': programme['external_id'], 'venue_names': venue_names, 'from_edition_id': original,
'year': 2026, 'from_track': 'program_unclassified', 'event_key': event_key,
'venue_id': venue_id, 'track': 'main', 'section': 'published ECCV refereed proceedings',
'evidence_url': row['evidence_url'], 'evidence_sha256': row['metadata']['publisher_chapter_evidence']['sha256'],
}, target, signature)
require(not conn.execute('SELECT 1 FROM submission_identifiers WHERE submission_id=?', (submission,)).fetchone()
and not conn.execute("SELECT 1 FROM submission_identifiers WHERE edition_id=? AND scheme='publication_doi' AND value=?",
(target, doi)).fetchone()
and not conn.execute('SELECT 1 FROM source_records WHERE source_id=? AND external_id=?',
(publisher, doi)).fetchone(), 'Publication observation or submission identity already exists')
require(not conn.execute('SELECT 1 FROM identifiers WHERE scheme=? AND value=?',
(PUBLISHER, doi)).fetchone(), 'Publisher identity already exists')
owner = conn.execute("SELECT paper_id FROM identifiers WHERE scheme='doi' AND value=?", (canonical_doi,)).fetchone()
require(owner is None or owner[0] == paper, 'Publication DOI belongs to another paper')
if owner is None:
conn.execute("INSERT INTO identifiers(scheme,value,paper_id) VALUES('doi',?,?)", (canonical_doi, paper))
before['identifiers'].append({'scheme': 'doi', 'value': canonical_doi, 'paper_id': paper})
before['identifiers'].append({'scheme': PUBLISHER, 'value': doi, 'paper_id': paper})
before['identifiers'].sort(key=lambda identifier: (identifier['scheme'], identifier['value']))
require(conn.execute('UPDATE submissions SET edition_id=? WHERE id=? AND edition_id=?',
(target, submission, original)).rowcount == 1, 'Programme attempt changed')
conn.execute("INSERT INTO submission_identifiers VALUES(?,'publication_doi',?,?)", (target, doi, submission))
metadata[KEY] = [scope]
programme['metadata_json'] = json.dumps(metadata, ensure_ascii=False)
conn.execute('UPDATE source_records SET metadata_json=? WHERE id=?',
(programme['metadata_json'], programme_record))
actual_paper, record = store.record(PUBLISHER, doi, row['url'], row['title'], row['abstract'],
row['authors'], row['identifiers'], row['metadata'])
require(actual_paper == paper, 'Unexpected canonical paper')
store.record_tags(paper, record, PUBLISHER, row['tags'])
actual_edition = store.decision(paper, record, 'ECCV', 2026, 'main', doi, row['evidence_url'],
'accepted', row['raw_decision'], identities=identities_for_row(row),
observed_at=row['observed_at'], event_key=event_key)
require(actual_edition == target, 'Unexpected target edition')
after = entity_packet(conn, paper)
published = [r for r in after['source_records'] if r['id'] == record]
require(len(published) == 1 and published[0]['external_id'] == doi
and published[0]['abstract'] == row['abstract'], 'Publication source is missing or changed')
after['source_records'].remove(published[0])
moved = next(a for a in after['attempts'] if a['id'] == submission)
attached = [s for s in moved['sources'] if s['source_record_id'] == record]
observed = [d for d in moved['decisions'] if any(s['source_record_id'] == record for s in d['sources'])]
require(len(attached) == len(observed) == 1 and attached[0]['source_submission_id'] == doi
and observed[0]['status'] == 'accepted' and observed[0]['evidence_url'] == row['evidence_url']
and len(observed[0]['sources']) == 1, 'Publisher evidence did not attach to the existing submission')
moved['sources'].remove(attached[0])
moved['decisions'].remove(observed[0])
attempt.update(edition_id=target, track='main')
require(before == after, 'Existing paper or submission history changed')
def apply(store, plan, signature, previous_count):
rows = validate_plan(plan)
conn = store.conn
marker = MARKER + signature
require(not conn.execute('SELECT 1 FROM schema_migrations WHERE name=?', (marker,)).fetchone(),
'Reconciliation already applied')
edition_rows = conn.execute('''SELECT e.track,e.id,e.venue_id,e.event_key
FROM editions e JOIN venues v ON v.id=e.venue_id WHERE v.acronym='ECCV' AND e.year=2026
AND e.track IN ('main','program_unclassified')''').fetchall()
require(len(edition_rows) == 2 and {r[0] for r in edition_rows} == {'main', 'program_unclassified'},
'Expected two unambiguous reviewed editions')
editions = {r[0]: (r[1], r[2], r[3]) for r in edition_rows}
original, venue, event = editions['program_unclassified']
target, target_venue, target_event = editions['main']
require(venue == target_venue and event == target_event
and all((r['move_from_edition_id'], r['move_to_edition_id']) == (original, target) for r in rows),
'Reviewed edition identity differs')
names = ['ECCV', *(r[0] for r in conn.execute('SELECT alias FROM venue_aliases WHERE venue_id=?', (venue,)))]
sources = {r[0]: r[1] for r in conn.execute('SELECT name,id FROM sources WHERE name IN (?,?)', (PROGRAMME, PUBLISHER))}
programme, publisher = sources[PROGRAMME], sources[PUBLISHER]
old = existing_coverage(conn, original, programme)
main = existing_coverage(conn, target, programme)
published = existing_coverage(conn, target, publisher)
require(old is not None and old[3] >= len(rows) and (main is None or main[0] == old[0])
and published is not None and published[2:4] == (PUBLISHER_COUNT, previous_count)
and previous_count + len(rows) <= PUBLISHER_COUNT, 'Reviewed coverage parent differs')
for entry in rows:
reconcile_attempt(store, entry, original, target, venue, event, names, publisher, signature)
remaining = old[3] - len(rows)
imported = previous_count + len(rows)
main_count = (main[3] if main else 0) + len(rows)
for edition, source, expected in ((original, programme, remaining),
(target, programme, main_count), (target, publisher, imported)):
require(source_record_count(conn, source, edition) == expected, 'Reconciled coverage differs from linked records')
conn.execute('UPDATE edition_sources SET imported_count=? WHERE edition_id=? AND source_id=? AND list_url=?',
(remaining, original, programme, old[0]))
conn.execute('''INSERT INTO edition_sources
(edition_id,source_id,list_url,status,imported_count,cursor,last_success_at) VALUES(?,?,?,'partial',?,?,?)
ON CONFLICT(edition_id,source_id,list_url) DO UPDATE SET imported_count=excluded.imported_count''',
(target, programme, old[0], main_count, old[4], old[5]))
conn.execute('''UPDATE edition_sources SET imported_count=?,last_success_at=?
WHERE edition_id=? AND source_id=? AND list_url=?''', (imported, timestamp(), target, publisher, published[0]))
conn.execute('INSERT INTO schema_migrations(name) VALUES(?)', (marker,))
return {'status': 'applied', 'plan_sha256': signature, 'programme_main': main_count,
'programme_unclassified': remaining, 'springer_main': imported, 'new_papers': 0, 'new_submissions': 0}
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog)
database_arguments(parser)
parser.add_argument('--plan', type=Path, required=True)
parser.add_argument('--plan-sha256', required=True)
parser.add_argument('--springer-previous-count', type=int, required=True)
args = parser.parse_args(argv)
raw = args.plan.read_bytes()
signature = sha256(raw).hexdigest()
require(signature == args.plan_sha256, 'Plan hash differs')
plan = json.loads(raw)
validate_plan(plan)
store = open_store(args.db, pg_service=args.pg_service, pg_schema=args.pg_schema)
with closing(store.conn) as conn:
try:
if not getattr(conn, 'postgres', False):
conn.commit()
conn.execute('BEGIN IMMEDIATE')
result = apply(store, plan, signature, args.springer_previous_count)
conn.commit()
except BaseException:
conn.rollback()
raise
print(json.dumps(result), flush=True)
-4
View File
@@ -26,10 +26,6 @@ class Result:
def rowcount(self): def rowcount(self):
return self.cursor.rowcount return self.cursor.rowcount
@property
def description(self):
return self.cursor.description
class WriteConnection: class WriteConnection:
postgres = True postgres = True
+68
View File
@@ -0,0 +1,68 @@
from hashlib import sha256
import json
from pathlib import Path
from tempfile import TemporaryDirectory
import unittest
from paper_atlas.cli import main
from paper_atlas.collectors.kdd2026 import inventory
def row(number, track, title='Paper title', authors='Name (Affiliation)'):
return {'track': track, 'title': title, 'url': f'https://doi.org/10.1145/3770855.{number}', 'authors': authors}
def snapshot(july, february):
arrays = [
f'<script>const cycle{n}Papers = {json.dumps(rows)}.map((paper, index) => paper);</script>'
for n, rows in ((1, july), (2, february))
]
return '\n'.join(arrays).encode()
class KDD2026Tests(unittest.TestCase):
def test_cli_preserves_withdrawal_marker_and_identical_occurrences(self):
retained = row(1, 'rtp')
withdrawn = row(2, 'ads', '**WITHDRAWN/New SUBMISSION ID for Authorship Order Change 6/24/26 -- A revised work')
incomplete = row(3, 'ais', authors='')
raw = snapshot([retained, retained], [withdrawn, incomplete])
with TemporaryDirectory() as directory:
source = Path(directory) / 'snapshot.html'
source.write_bytes(raw)
out = source.with_suffix('.json')
args = ['collect', 'kdd2026-program', '--snapshot', str(source), '--snapshot-sha256',
sha256(raw).hexdigest(), '--out', str(out)]
main(args)
report = json.loads(out.read_bytes())
with self.assertRaises(FileExistsError):
main(args)
self.assertEqual(report['source_rows_by_cycle_and_track'], {
'july': {'research': 2}, 'february': {'applied_data_science': 1, 'ai_for_sciences': 1},
})
self.assertEqual(report['unique_dois'], 3)
self.assertEqual(report['identical_repeat_rows'], 1)
kept, replaced, sparse = report['records']
self.assertEqual(kept['occurrences'], 2)
self.assertEqual(replaced['programme_status'], 'withdrawn_with_replacement_mentioned')
self.assertEqual(replaced['title'], 'A revised work')
self.assertEqual(replaced['source_title'], withdrawn['title'])
self.assertEqual(replaced['source_withdrawal_date_literal'], '6/24/26')
self.assertNotIn('submission_id', replaced)
self.assertFalse(sparse['authors_listed'])
self.assertFalse(report['database_written'])
def test_conflicting_duplicate_and_unknown_status_fail(self):
with self.assertRaisesRegex(ValueError, 'Conflicting duplicate DOI'):
inventory(snapshot([row(1, 'rtp')], [row(1, 'ads')]))
with self.assertRaisesRegex(ValueError, 'Unrecognized status'):
inventory(snapshot([row(1, 'rtp')], [row(2, 'ads', '***WITHDRAWN -- Unknown format')]))
def test_hash_mismatch_leaves_no_output(self):
with TemporaryDirectory() as directory:
source = Path(directory) / 'snapshot.html'
source.write_bytes(snapshot([row(1, 'rtp')], [row(2, 'ads')]))
out = source.with_suffix('.json')
with self.assertRaisesRegex(ValueError, 'snapshot hash or size'):
main(['collect', 'kdd2026-program', '--snapshot', str(source), '--snapshot-sha256',
'0' * 64, '--out', str(out)])
self.assertFalse(out.exists())
-140
View File
@@ -1,140 +0,0 @@
from contextlib import closing, redirect_stdout
from hashlib import sha256
import io
import json
import os
from pathlib import Path
from tempfile import TemporaryDirectory
import unittest
import uuid
from paper_atlas.audits.source_identity import entity_packet
from paper_atlas.cli import main
from paper_atlas.db.edition_scopes import resolve
from paper_atlas.db.store import Store
from paper_atlas.ingestion.publication_reconciliation import PROGRAMME, PUBLISHER, packet_digest
from paper_atlas.postgres.client import ReadConnection, connect
from paper_atlas.postgres.migration import migrate
from paper_atlas.postgres.schema import quote
class PublicationReconciliationTests(unittest.TestCase):
def setUp(self):
directory = TemporaryDirectory()
self.addCleanup(directory.cleanup)
self.root = Path(directory.name)
self.db = self.root / 'db.sqlite'
self.store = Store(self.db)
self.conn = self.store.conn
self.addCleanup(self.conn.close)
self.programme_url = 'https://eccv.ecva.net/Conferences/2026/AcceptedPapers'
self.publisher_url = 'https://link.springer.com/conference/eccv'
programme = self.store.source(PROGRAMME, self.programme_url)
publisher = self.store.source(PUBLISHER, self.publisher_url)
self.entries = [self.programme_attempt(str(i)) for i in range(1, 4)]
title = 'A distinct chapter already integrated in the main track'
paper, record = self.store.record(PUBLISHER, 'prior', self.publisher_url, title, 'Text', [])
self.target = self.store.accepted(paper, record, 'ECCV', 2026, 'main', 'prior', self.publisher_url)
self.original = self.entries[0]['move_from_edition_id']
for entry in self.entries:
entry['move_to_edition_id'] = self.target
self.conn.executemany('''INSERT INTO edition_sources
(edition_id,source_id,list_url,status,expected_count,imported_count) VALUES(?,?,?,'partial',?,?)''',
[(self.original, programme, self.programme_url, 2834, 3),
(self.target, publisher, self.publisher_url, 2835, 1)])
self.conn.commit()
def programme_attempt(self, number):
title = 'A programme observation for continuing research ' + number
url = self.programme_url + '/' + number
authors = ['Ada Example', 'Sam Example']
paper, record = self.store.record(PROGRAMME, number, url, title, 'Programme text', authors)
original = self.store.accepted(paper, record, 'ECCV', 2026, 'program_unclassified', number, url)
other, old_record = self.store.record('openreview', 'test-' + number, url, title, 'Programme text', authors)
self.assertEqual(other, paper)
self.store.decision(paper, old_record, 'ICLR', 2025, 'main', 'test-' + number, url, 'rejected', 'Reject')
packet = entity_packet(self.conn, paper)
attempt = next(row for row in packet['attempts'] if row['source_record_id'] == record)
doi = '10.1007/978-3-032-00000-1_' + number
chapter = 'https://link.springer.com/chapter/' + doi
row = {'source': PUBLISHER, 'external_id': doi, 'url': chapter, 'evidence_url': chapter,
'venue': 'ECCV', 'year': 2026, 'track': 'main', 'status': 'accepted',
'title': title, 'authors': authors, 'abstract': 'Publisher text',
'observed_at': '2026-11-18T00:00:00Z', 'raw_decision': 'Publication observation',
'identifiers': {'doi': doi},
'submission_identifiers': [{'scheme': 'publication_doi', 'value': doi}],
'metadata': {'publisher_chapter_evidence': {'sha256': 'a' * 64}},
'tags': [{'namespace': PUBLISHER + ':format', 'value': 'chapter'}]}
return {'doi': doi, 'retained_paper_id': paper, 'retained_programme_record_id': record,
'retained_submission_id': attempt['id'], 'retained_programme_decision_id': attempt['decisions'][0]['id'],
'move_from_edition_id': original, 'source_row': row, 'before_packet_sha256': packet_digest(packet)}
def run_plan(self, entries, *database):
path = self.root / 'plan.json'
raw = json.dumps({'status': 'readonly_same_submission_reconciliation_plan', 'database_written': False,
'plans': entries, 'guarded_same_submission_plans': len(entries)}).encode()
path.write_bytes(raw)
main(['maintain', 'eccv-publication-reconciliation', *(database or ('--db', str(self.db))),
'--plan', str(path), '--plan-sha256', sha256(raw).hexdigest(), '--springer-previous-count', '1'])
def test_preserves_history_and_updates_all_three_coverage_scopes(self):
entry = self.entries[0]
before = entity_packet(self.conn, entry['retained_paper_id'])
self.run_plan([entry])
packet = entity_packet(self.conn, entry['retained_paper_id'])
self.assertEqual(packet['paper'], before['paper'])
programme, other = packet['attempts']
self.assertEqual((programme['id'], programme['edition_id']), (entry['retained_submission_id'], self.target))
self.assertEqual(other, before['attempts'][1])
self.assertEqual(programme['decisions'][0], before['attempts'][0]['decisions'][0])
self.assertEqual([row['status'] for row in programme['decisions']], ['accepted', 'accepted'])
self.assertEqual({r['source'] for r in programme['sources']}, {PROGRAMME, PUBLISHER})
self.assertEqual(len(packet['source_records']), 3)
self.assertEqual(resolve(self.conn, entry['retained_programme_record_id'], '1',
'ECCV', 2026, 'program_unclassified', None), self.target)
coverage = {(edition, source): (status, expected, imported)
for edition, source, status, expected, imported in self.conn.execute('''SELECT
e.track,s.name,c.status,c.expected_count,c.imported_count FROM edition_sources c
JOIN editions e ON e.id=c.edition_id JOIN sources s ON s.id=c.source_id''')}
self.assertEqual(coverage, {('program_unclassified', PROGRAMME): ('partial', 2834, 2),
('main', PROGRAMME): ('partial', None, 1),
('main', PUBLISHER): ('partial', 2835, 2)})
with self.assertRaisesRegex(ValueError, 'already applied'):
self.run_plan([entry])
def test_changed_history_rolls_back_earlier_observation(self):
first, second = self.entries[:2]
stale = {**second, 'before_packet_sha256': 'b' * 64}
packet = entity_packet(self.conn, first['retained_paper_id'])
with self.assertRaisesRegex(ValueError, 'Reviewed paper history changed'):
self.run_plan([first, stale])
self.assertEqual(entity_packet(self.conn, first['retained_paper_id']), packet)
self.assertIsNone(self.conn.execute(
"SELECT 1 FROM identifiers WHERE scheme='doi' AND value=?", (first['doi'],)).fetchone())
self.assertEqual(self.conn.execute('SELECT count(*) FROM edition_sources').fetchone()[0], 2)
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires the remote PostgreSQL test service')
def test_postgres_write_verifies_history_from_the_read_backend(self):
service = os.environ['PAPER_ATLAS_TEST_PG_SERVICE']
schema = 'pa_eccv_' + uuid.uuid4().hex[:12]
def cleanup():
with connect(service) as conn:
for name, in conn.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)', (schema,)):
conn.execute('DROP SCHEMA ' + quote(name) + ' CASCADE')
self.addCleanup(cleanup)
self.conn.close()
with redirect_stdout(io.StringIO()):
migrate(self.db, service, self.root / 'migration.json', schema=schema, min_free_gib=0)
entry = self.entries[0]
with closing(ReadConnection(service, schema)) as conn:
before = entity_packet(conn, entry['retained_paper_id'])
reviewed = {**entry, 'before_packet_sha256': packet_digest(before)}
self.run_plan([reviewed], '--pg-service', service, '--pg-schema', schema)
with closing(ReadConnection(service, schema)) as conn:
packet = entity_packet(conn, entry['retained_paper_id'])
self.assertEqual(packet['attempts'][1], before['attempts'][1])
self.assertEqual(packet['attempts'][0]['id'], entry['retained_submission_id'])
self.assertEqual(packet['attempts'][0]['edition_id'], self.target)
self.assertEqual(len(packet['attempts'][0]['decisions']), 2)