Compare commits

..

40 Commits

Author SHA1 Message Date
codex-bot 6e5202c81a Inventory KDD 2026 programme cycles and withdrawal evidence
Metadata integrity checks / metadata (push) Successful in 8m7s
2026-11-18 23:14:36 +08:00
Zayden-Jung 48f8304517 Merge pull request '封装 ECCV 2026 Springer 正式出版采集器' (#80) from codex/eccv2026-publications into main
Metadata integrity checks / metadata (push) Successful in 8m36s
Reviewed-on: #80
2026-11-18 10:58:39 +00:00
Zayden-Jung f2b9fe0149 Merge pull request '核对 ECCV 2026 Springer 78 卷与大会节目单' (#79) from codex/eccv2026-publisher-crosswalk into main
Metadata integrity checks / metadata (push) Successful in 13m29s
Reviewed-on: #79
2026-11-18 10:58:32 +00:00
Zayden-Jung 32d99858c1 Merge pull request '记录 IROS 2026 会议录用与期刊报告来源' (#78) from codex/iros2026-source-audit into main
Metadata integrity checks / metadata (push) Successful in 11m10s
Reviewed-on: #78
2026-11-18 10:58:25 +00:00
Zayden-Jung 060d67aba5 Merge pull request '增加 CIKM 2026 节目与出版登记采集命令' (#77) from codex/cikm2026-collector into main
Metadata integrity checks / metadata (push) Successful in 8m59s
Reviewed-on: #77
2026-11-18 10:58:19 +00:00
Zayden-Jung d0a466977c Merge pull request '核对 ICDM 2026 海报范围与跨会历史' (#75) from codex/icdm-presentation-scope into main
Metadata integrity checks / metadata (push) Successful in 8m30s
Reviewed-on: #75
2026-11-18 10:58:13 +00:00
codex-bot 6d35a2e567 Collect verified ECCV 2026 Springer proceedings metadata
Metadata integrity checks / metadata (push) Successful in 7m51s
2026-11-18 18:30:01 +08:00
codex-bot 52f9740d99 Record ECCV 2026 publisher and programme reconciliation
Metadata integrity checks / metadata (push) Successful in 7m41s
2026-11-18 18:12:49 +08:00
Zayden-Jung 4e071ca0f1 Merge pull request '修复 PostgreSQL 只读增量覆盖预检' (#74) from codex/pg-incremental-preflight into main
Metadata integrity checks / metadata (push) Successful in 8m15s
Reviewed-on: #74
2026-11-18 10:03:24 +00:00
Zayden-Jung f97852b630 Merge pull request '记录 ISMIR 31 条来源关联与增量验收' (#73) from codex/ismir-existing-links into main
Metadata integrity checks / metadata (push) Successful in 8m28s
Reviewed-on: #73
2026-11-18 10:03:18 +00:00
codex-bot bcb8b2ad7e Record verified IROS arXiv work bindings
Metadata integrity checks / metadata (push) Successful in 11m14s
2026-11-18 17:28:03 +08:00
codex-bot 95ff77d77f Document IROS 2026 publication and presentation sources
Metadata integrity checks / metadata (push) Successful in 7m56s
2026-11-18 17:24:06 +08:00
Zayden-Jung 9da6a45ec0 Merge pull request '记录公开拒稿镜像差集与五条待核验 ID' (#72) from codex/negative-source-audit into main
Metadata integrity checks / metadata (push) Successful in 10m59s
Reviewed-on: #72
2026-11-18 08:54:56 +00:00
Zayden-Jung 7407f97ef2 Merge pull request '仅投影公开评审镜像索引字段并保留决定来源' (#71) from codex/reviewarena-metadata into main
Metadata integrity checks / metadata (push) Successful in 10m0s
Reviewed-on: #71
2026-11-18 08:49:53 +00:00
Zayden-Jung 31ff7d1626 Merge pull request '修复MM标题排版导致的26条节目摘要与分轨缺口' (#69) from codex/mm-title-formatting into main
Metadata integrity checks / metadata (push) Successful in 12m47s
Reviewed-on: #69
2026-11-18 08:23:13 +00:00
Zayden-Jung 8a7896d352 Merge pull request '版本化修正出版来源分轨与覆盖并保护旧版刷新' (#70) from codex/source-scope-revisions into main
Metadata integrity checks / metadata (push) Successful in 10m54s
Reviewed-on: #70
2026-11-18 08:23:02 +00:00
Zayden-Jung 35ff09a463 Merge pull request '核对 CIKM 2026 出版、附属议程与既有投稿历史' (#76) from codex/cikm-source-reconciliation into main
Metadata integrity checks / metadata (push) Successful in 7m38s
Reviewed-on: #76
2026-11-18 08:22:49 +00:00
codex-bot 28805462ee Add reproducible CIKM 2026 source collection
Metadata integrity checks / metadata (push) Successful in 7m34s
2026-11-18 15:49:13 +08:00
codex-bot c469785a64 Reconcile CIKM supplemental sessions and existing DOI history
Metadata integrity checks / metadata (push) Successful in 7m35s
2026-11-18 15:41:50 +08:00
codex-bot 2f445eddc8 Record CIKM programme and publisher registry reconciliation
Metadata integrity checks / metadata (push) Successful in 7m6s
2026-11-18 15:28:08 +08:00
codex-bot 2907c3774f Record ICDM poster coverage and cross-venue evidence
Metadata integrity checks / metadata (push) Successful in 6m55s
2026-11-18 15:10:27 +08:00
codex-bot a980b1a1df Record verified ISMIR increment to31 existing-work source links
Metadata integrity checks / metadata (push) Successful in 6m45s
2026-11-18 14:51:27 +08:00
codex-bot 3754545003 Fix incremental coverage preflight on PostgreSQL readonly rows
Metadata integrity checks / metadata (push) Successful in 7m15s
2026-11-18 13:34:09 +08:00
codex-bot e38b2f1ca8 Record verified ISMIR source links and preserved rejection history
Metadata integrity checks / metadata (push) Successful in 7m36s
2026-11-18 13:25:06 +08:00
codex-bot 8e548287e9 Preserve existing ACL acceptance for EZ-STANCE title variant
Metadata integrity checks / metadata (push) Successful in 7m7s
2026-11-18 12:10:40 +08:00
codex-bot e85902bf18 Preserve EZ-STANCE later-publication lead without anonymous identity inference
Metadata integrity checks / metadata (push) Successful in 6m44s
2026-11-18 12:07:24 +08:00
codex-bot e82abec4c7 Record current negative-source overlap and unresolved note identities
Metadata integrity checks / metadata (push) Successful in 7m20s
2026-11-18 12:05:05 +08:00
codex-bot 7d9c693761 Review mirror publication literals and replay frozen metadata offline
Metadata integrity checks / metadata (push) Successful in 6m40s
2026-11-18 09:34:10 +08:00
codex-bot 98ef415912 Use Arrow binary wheels compatible with runner Python 3.14
Metadata integrity checks / metadata (push) Successful in 7m13s
2026-11-18 09:14:27 +08:00
codex-bot 90fd789a17 Project pinned ReviewArena metadata without full-text payload pages
Metadata integrity checks / metadata (push) Failing after 47s
2026-11-18 09:10:29 +08:00
codex-bot 2000d340fa Add snapshot-bound publication source scope revisions
Metadata integrity checks / metadata (push) Successful in 7m16s
2026-11-18 06:59:21 +08:00
codex-bot 12f95cff91 Reject incomplete MM title markup before HTML parsing
Metadata integrity checks / metadata (push) Successful in 6m59s
2026-11-18 04:21:31 +08:00
codex-bot 9864632774 Recover MM program evidence through reviewed title formatting
Metadata integrity checks / metadata (push) Failing after 5m46s
2026-11-18 04:14:41 +08:00
Zayden-Jung f982851560 Merge pull request '记录 MM 出版核对与 UAI 原始决定来源边界' (#67) from codex/mm-publication into main
Metadata integrity checks / metadata (push) Successful in 9m35s
Reviewed-on: #67
2026-11-17 18:44:00 +00:00
Zayden-Jung 88aeb4d2eb Merge pull request '封装 MM 2026 全量出版记录与独立节目证据' (#68) from codex/mm-publications into main
Metadata integrity checks / metadata (push) Successful in 10m5s
Reviewed-on: #68
2026-11-17 18:43:55 +00:00
codex-bot 83f3ebd26b Fix MM publication tag provenance and exercise full native ingestion
Metadata integrity checks / metadata (push) Successful in 6m46s
2026-11-18 01:31:59 +08:00
codex-bot 13536d6268 Normalize all MM 2026 publications with attributed program evidence
Metadata integrity checks / metadata (push) Successful in 5m4s
2026-11-18 00:43:45 +08:00
codex-bot 7e63035570 Document MM publication anchors and UAI raw decision-source limits
Metadata integrity checks / metadata (push) Successful in 4m8s
2026-11-17 23:43:21 +08:00
Zayden-Jung 44f4146f3e Merge pull request '直接审计 MM 节目身份与既往投稿历史' (#66) from codex/mm-identity into main
Metadata integrity checks / metadata (push) Successful in 4m16s
Reviewed-on: #66
2026-11-17 14:57:42 +00:00
Zayden-Jung b0bd9b69dc Merge pull request '封装 MM 2026 全轨节目与重复展示证据采集' (#65) from codex/mm2026 into main
Metadata integrity checks / metadata (push) Successful in 3m41s
Reviewed-on: #65
2026-11-17 14:57:34 +00:00
52 changed files with 8747 additions and 92 deletions
+79 -3
View File
@@ -19,8 +19,9 @@ jobs:
# Public repository archive: no owner credentials or runner SSH keys. # Public repository archive: no owner credentials or runner SSH keys.
url = 'https://gitea.entroia.com/Zayden-Jung/paper-atlas/archive/' + sha + '.tar.gz' url = 'https://gitea.entroia.com/Zayden-Jung/paper-atlas/archive/' + sha + '.tar.gz'
with urllib.request.urlopen(url, timeout=60) as response: with urllib.request.urlopen(url, timeout=60) as response:
archive = response.read(16*1024*1024+1) # Full publication-source fixtures retain original bibliographic evidence.
if len(archive)>16*1024*1024: archive = response.read(24*1024*1024+1)
if len(archive)>24*1024*1024:
raise SystemExit('Unexpectedly large source archive') raise SystemExit('Unexpectedly large source archive')
with tempfile.TemporaryDirectory(prefix='paper-atlas-ci-') as tmp: with tempfile.TemporaryDirectory(prefix='paper-atlas-ci-') as tmp:
root = pathlib.Path(tmp) root = pathlib.Path(tmp)
@@ -34,7 +35,7 @@ jobs:
venv = root / 'venv' venv = root / 'venv'
subprocess.run(['python3','-B','-m','venv','--system-site-packages',str(venv)],check=True) subprocess.run(['python3','-B','-m','venv','--system-site-packages',str(venv)],check=True)
python = str(venv / 'bin/python') python = str(venv / 'bin/python')
subprocess.run([python,'-B','-m','pip','install','--only-binary=:all:','--no-compile','psycopg[binary]==3.3.8'],check=True) subprocess.run([python,'-B','-m','pip','install','--only-binary=:all:','--no-compile','psycopg[binary]==3.3.8','pyarrow==26.0.1'],check=True)
testenv = os.environ.copy() testenv = os.environ.copy()
service_file = pathlib.Path('/etc/paper-atlas/pg_service.conf') service_file = pathlib.Path('/etc/paper-atlas/pg_service.conf')
if service_file.is_file(): if service_file.is_file():
@@ -46,6 +47,40 @@ jobs:
subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True) subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True)
outside = root / 'outside-checkout' outside = root / 'outside-checkout'
outside.mkdir() outside.mkdir()
kdd_snapshot = pathlib.Path('/data/paper-index/validation/kdd2026-official-papers-html-v1.html')
if kdd_snapshot.is_file():
kdd_output = root / 'installed-kdd2026.json'
subprocess.run([python, '-I', '-B', '-m', 'paper_atlas', 'collect', 'kdd2026-program',
'--snapshot', str(kdd_snapshot), '--snapshot-sha256',
'47ee82841dedf034614348d7fdb29714e8ad62fca8c6811e1d856845588f145c',
'--out', str(kdd_output)], cwd=outside, check=True)
kdd = json.loads(kdd_output.read_bytes())
assert kdd['unique_dois'] == 1408 and kdd['identical_repeat_rows'] == 7
assert {key:sum(counts.values()) for key,counts in kdd['source_rows_by_cycle_and_track'].items()} == {
'july':256, 'february':1159}
held, = [entry for entry in kdd['records'] if entry['programme_status'] != 'listed']
assert held['doi'] == '10.1145/3770855.3818478' and held['source_withdrawal_date_literal'] == '6/24/26'
print('KDD 2026 official cycle and withdrawal fixture passed', flush=True)
cikm_snapshots = pathlib.Path('/data/paper-index/validation/cikm2026-source-snapshots-v1')
if cikm_snapshots.is_dir():
cikm_output = root / 'installed-cikm2026'
command = [python, '-I', '-B', '-m', 'paper_atlas', 'collect', 'cikm2026-sources']
subprocess.run([*command, '--snapshots', str(cikm_snapshots), '--out', str(cikm_output)], cwd=outside, check=True)
report = json.loads((cikm_output/'collection-report.json').read_text())
assert {name: entry['records'] for name, entry in report['files'].items()} == {
'programme.jsonl': 1008, 'alternate-programme.jsonl': 1005,
'supplemental-programme.jsonl': 73, 'crossref.jsonl': 1081}
assert len(report['registry_outside_doi_programme']) == 73 and not report['database_written']
programme = [json.loads(line) for line in (cikm_output/'programme.jsonl').read_text().splitlines()]
assert all(row['abstract'] and row['author_display_names'] for row in programme)
assert len({row['doi'] for row in programme}) == 1008
broken = root/'tampered-cikm2026'
broken.mkdir()
(broken/'program.html').write_text('tampered')
rejected = subprocess.run([*command, '--snapshots', str(broken), '--out', str(root/'rejected-cikm2026')], cwd=outside, capture_output=True, text=True)
assert rejected.returncode != 0 and 'Snapshot hash mismatch' in rejected.stderr
assert not (root/'rejected-cikm2026').exists()
print('CIKM 2026 full frozen-source CLI check passed', flush=True)
mm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/mm2026-source.json.gz').read_bytes())) mm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/mm2026-source.json.gz').read_bytes()))
mm_snapshots = root/'mm2026-snapshots' mm_snapshots = root/'mm2026-snapshots'
for name, body in mm_fixture['files'].items(): for name, body in mm_fixture['files'].items():
@@ -66,7 +101,31 @@ jobs:
assert hashlib.sha256(row['abstract'].encode()).hexdigest() == gold['abstract_sha256'] assert hashlib.sha256(row['abstract'].encode()).hexdigest() == gold['abstract_sha256']
assert [c['source_pointer'] for c in row['contexts']] == gold['locations'] assert [c['source_pointer'] for c in row['contexts']] == gold['locations']
assert not row['approved_for_import'] and not row['final_decision_verified'] and 'emails' not in row assert not row['approved_for_import'] and not row['final_decision_verified'] and 'emails' not in row
mm_pub_fixture = json.loads(gzip.decompress((src/'tests/fixtures/mm2026-publications.json.gz').read_bytes()))
mm_pub_registry = root/'mm2026-publication-registry'
for name,body in mm_pub_fixture['files'].items():
path=mm_pub_registry/name
path.parent.mkdir(parents=True,exist_ok=True)
path.write_bytes(body.encode('utf-8'))
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','mm2026-publications','--registry-snapshots',str(mm_pub_registry),'--program-snapshots',str(mm_snapshots),'--parent-snapshot',str(mm_pub_registry/'parent.json'),'--out',str(root/'installed-mm2026-publications')],cwd=outside,check=True)
mm_pub_report=json.loads((root/'installed-mm2026-publications/collection-report.json').read_text())
assert (mm_pub_report['records'],mm_pub_report['unique_registered_dois'],mm_pub_report['abstracts_from_exact_program_correspondence'],mm_pub_report['output_missing_abstract_count'])==(1918,1918,1743,175)
assert mm_pub_report['unmatched_program_count']==391 and not mm_pub_report['database_written'] and mm_pub_report['approved_for_import']==0
mm_pub_rows={r['external_id']:r for f in mm_pub_report['files'] for r in map(json.loads,pathlib.Path(f['file']).read_text().splitlines())}
assert len(mm_pub_rows)==1918
mm_format_gold=json.loads((src/'tests/fixtures/mm2026-title-formatting-review.json').read_text())
mm_format_overrides={g['doi']:g for g in [*mm_format_gold['gold'],mm_format_gold['held_gold']]}
assert mm_pub_report['abstracts_from_reviewed_title_formatting']==26 and mm_pub_report['abstracts_from_literal_program_correspondence']==1717
for original_gold in mm_pub_fixture['independent_gold']:
gold=mm_format_overrides.get(original_gold['doi'],original_gold)
row=mm_pub_rows[gold['doi']]
assert (row['title'],row['authors'],row['track'])==(gold['title'],gold['authors'],gold['track'])
assert (hashlib.sha256(row['abstract'].encode()).hexdigest() if row['abstract'] is not None else None)==gold['abstract_sha256']
assert row['metadata']['program_match_kind']==gold['kind'] and not row['metadata']['scientific_work_identity_approved'] and not row['metadata']['research_type_classified']
if service_file.is_file(): if service_file.is_file():
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','source-identities','--files',*[f['file'] for f in mm_pub_report['files']],'--pg-service','foundry-research','--pg-schema','research','--out',str(root/'installed-mm2026-publication-preflight')],cwd=outside,env=testenv,check=True)
mm_pub_preflight=json.loads((root/'installed-mm2026-publication-preflight/postgresql-preflight.json').read_text())
assert mm_pub_preflight['records']==1918 and mm_pub_preflight['transaction_read_only'] and not mm_pub_preflight['database_written'] and mm_pub_preflight['approved_for_import']==0
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','source-identities','--mm-program-report',str(root/'installed-mm2026/collection-report.json'),'--pg-service','foundry-research','--pg-schema','research','--out',str(root/'installed-mm2026-preflight')],cwd=outside,env=testenv,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','audit','source-identities','--mm-program-report',str(root/'installed-mm2026/collection-report.json'),'--pg-service','foundry-research','--pg-schema','research','--out',str(root/'installed-mm2026-preflight')],cwd=outside,env=testenv,check=True)
mm_preflight = json.loads((root/'installed-mm2026-preflight/postgresql-preflight.json').read_text()) mm_preflight = json.loads((root/'installed-mm2026-preflight/postgresql-preflight.json').read_text())
assert mm_preflight['records'] == 2143 and len(mm_preflight['inputs']) == 28 assert mm_preflight['records'] == 2143 and len(mm_preflight['inputs']) == 28
@@ -248,6 +307,23 @@ jobs:
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','iclr-archive','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','iclr-archive','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','acl','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','acl','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','springer-selected','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','springer-selected','--help'],cwd=outside,check=True)
publisher_fixture = "\n".join((
"import pathlib, runpy, sys",
"fixture = runpy.run_path(sys.argv[1])",
"root = pathlib.Path(sys.argv[2])",
"root.mkdir()",
"fixture['snapshots'](root)",
))
publisher_input = root/'eccv-publisher-snapshots'
subprocess.run([python,'-I','-B','-c',publisher_fixture,
str(src/'tests/test_eccv2026_publications.py'),str(publisher_input)],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','eccv2026-publications',
'--toc-snapshots',str(publisher_input/'toc'),
'--chapter-snapshots',str(publisher_input/'chapters'),
'--out',str(root/'installed-eccv-publisher')],cwd=outside,check=True)
publisher_report = json.loads((root/'installed-eccv-publisher/collection-report.json').read_bytes())
assert publisher_report['records'] == 1 and publisher_report['toc_chapters'] == 2835
assert not publisher_report['database_written']
springer = json.loads(gzip.decompress((src/'tests/fixtures/springer-aistats1993-fixture.json.gz').read_bytes())) springer = json.loads(gzip.decompress((src/'tests/fixtures/springer-aistats1993-fixture.json.gz').read_bytes()))
for group in ('toc','chapters'): for group in ('toc','chapters'):
folder = root/'springer-snapshots'/group folder = root/'springer-snapshots'/group
+88
View File
@@ -0,0 +1,88 @@
name: ReviewArena metadata projection
on:
workflow_dispatch:
inputs:
split:
description: 'Pinned public mirror family; metadata columns only'
required: true
default: emnlp
projection_name:
description: 'Optional existing metadata output directory name for offline replay'
required: false
default: ''
projection_sha256:
description: 'Exact cached collection-report.json SHA256, required for offline replay'
required: false
default: ''
jobs:
project:
runs-on: ces-control
env:
ARCHIVE_SHA: ${{ github.sha }}
MIRROR_SPLIT: ${{ inputs.split }}
MIRROR_PROJECTION: ${{ inputs.projection_name }}
MIRROR_PROJECTION_SHA: ${{ inputs.projection_sha256 }}
PYTHONDONTWRITEBYTECODE: '1'
steps:
- name: Project permitted metadata on the approved remote runner
run: |
python3 -B - <<'PY'
import hashlib, io, json, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request
sha = os.environ['ARCHIVE_SHA']
split = os.environ['MIRROR_SPLIT']
cached = os.environ.get('MIRROR_PROJECTION','')
cached_sha = os.environ.get('MIRROR_PROJECTION_SHA','')
if not re.fullmatch(r'[0-9a-f]{40}', sha) or split not in ('colm','corl','emnlp','iclr','icml','neurips','tmlr'):
raise SystemExit('Explicit approved source commit/split required')
base = pathlib.Path('/data/paper-index')
if not (base/'collected').is_dir() or not (base/'validation').is_dir():
raise SystemExit('Confirmed ces-control host dataset paths unavailable; do not use another host')
out = base/'collected'/('reviewarena-c2978add-'+split+'-'+sha[:8])
if out.exists():
raise SystemExit('Inspect existing output; no overwrite or automatic retry')
extra=[]
if cached:
if not re.fullmatch('reviewarena-c2978add-'+split+r'-[0-9a-f]{8}',cached) or not re.fullmatch('[0-9a-f]{64}',cached_sha):
raise SystemExit('Explicit permitted cached projection name and report hash required')
extra=['--from-projection',str(base/'collected'/cached/'collection-report.json'),'--projection-sha256',cached_sha]
elif cached_sha:
raise SystemExit('Cached projection name required')
url = 'https://gitea.entroia.com/Zayden-Jung/paper-atlas/archive/'+sha+'.tar.gz'
with urllib.request.urlopen(url,timeout=60) as response:
archive = response.read(24*1024*1024+1)
if len(archive)>24*1024*1024:
raise SystemExit('Oversized source archive')
with tempfile.TemporaryDirectory(prefix='paper-atlas-mirror-') as tmp:
root = pathlib.Path(tmp)
with tarfile.open(fileobj=io.BytesIO(archive),mode='r:gz') as tar:
tar.extractall(root,filter='data')
candidates=list(root.glob('*/pyproject.toml'))
if len(candidates)!=1:
raise SystemExit('Unexpected repository archive')
src=candidates[0].parent
venv=root/'venv'
subprocess.run(['python3','-B','-m','venv','--system-site-packages',str(venv)],check=True)
python=str(venv/'bin/python')
subprocess.run([python,'-B','-m','pip','install','--only-binary=:all:','--no-compile','pyarrow==26.0.1'],check=True)
wheels=root/'wheels'
subprocess.run([python,'-B','-m','pip','wheel','--no-deps','--wheel-dir',str(wheels),str(src)],check=True)
wheel,=wheels.glob('paper_atlas-*.whl')
subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True)
outside=root/'outside';outside.mkdir()
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','reviewarena-metadata','--split',split,'--out',str(out),*extra],cwd=outside,check=True)
report=json.loads((out/'collection-report.json').read_bytes())
assert report['database_written'] is False and report['approved_for_import']==0
receipt={'source_commit':sha,'source_archive_sha256':hashlib.sha256(archive).hexdigest(),'wheel_sha256':hashlib.sha256(wheel.read_bytes()).hexdigest(),'dataset_commit':report['commit'],'split':split,'records':report['records'],'metadata_bytes_read':report['metadata_bytes_read'],'cached_projection_report_sha256':report.get('cached_projection_report_sha256'),'raw_parquet_saved':False,'database_written':False}
(out/'runner-receipt.json').write_text(json.dumps(receipt,indent=2)+'\n')
backup=base/'validation'/(out.name+'-recovery.tar.gz')
if backup.exists():
raise SystemExit('Recovery archive already exists; inspect')
with tarfile.open(backup,'w:gz') as tar:
tar.add(out,arcname=str(out.relative_to(base)))
backup.chmod(0o444)
for path in out.iterdir():
if path.is_file():path.chmod(0o444)
result={'status':'metadata_projection_backed_up_not_imported','archive':str(backup),'bytes':backup.stat().st_size,'sha256':hashlib.sha256(backup.read_bytes()).hexdigest(),**receipt}
(base/'validation'/(out.name+'-recovery.json')).write_text(json.dumps(result,indent=2)+'\n')
print(json.dumps(result),flush=True)
PY
+12
View File
@@ -2,6 +2,8 @@
**长期维护、按需运行的论文元数据与公开投稿历史工具。** **长期维护、按需运行的论文元数据与公开投稿历史工具。**
其他会议的公开评审镜像可按 [ReviewArena 元数据投影流程](docs/reviewarena-metadata.md) 核查:仅取固定版本的标题、摘要、作者及决定等索引字段,保留镜像声明和稳定行定位,不下载正文/评审数据页,也不将镜像 ID 自动提升为官方论坛身份。
Paper Atlas 将 arXiv、AI 会议论文集及公开投稿决定整理到可追溯的数据库中,用于查询和后续研究分析。生产数据使用 PostgreSQL,支持从既有 SQLite 索引完整迁移;SQLite 保留离线核验和小型样本用途。它按批次运行,不需要常驻 Web 服务;采集标题、摘要、标识、标签及决定证据,不下载论文全文。 Paper Atlas 将 arXiv、AI 会议论文集及公开投稿决定整理到可追溯的数据库中,用于查询和后续研究分析。生产数据使用 PostgreSQL,支持从既有 SQLite 索引完整迁移;SQLite 保留离线核验和小型样本用途。它按批次运行,不需要常驻 Web 服务;采集标题、摘要、标识、标签及决定证据,不下载论文全文。
项目仍在建设中。候选范围包含 43 个会议(含 ACM MM、MLSys),尚未完成全部历史数据和最终去重复核。实际进度以数据库、来源清单和覆盖报告为准,而不是仓库文件数量。 项目仍在建设中。候选范围包含 43 个会议(含 ACM MM、MLSys),尚未完成全部历史数据和最终去重复核。实际进度以数据库、来源清单和覆盖报告为准,而不是仓库文件数量。
@@ -117,6 +119,8 @@ CI 在已授权的远程 runner 上运行测试、构建 wheel,并从源码目
已导入整卷的逐条轨道修正见 [edition-tracks 维护流程](docs/edition-tracks.md):按完整证据生成只读计划,保留原投稿和决定 ID,事务性修正 edition、标识与覆盖,并防止旧整卷刷新覆盖分类。 已导入整卷的逐条轨道修正见 [edition-tracks 维护流程](docs/edition-tracks.md):按完整证据生成只读计划,保留原投稿和决定 ID,事务性修正 edition、标识与覆盖,并防止旧整卷刷新覆盖分类。
完整出版来源的新旧版本导致分轨或摘要变化时,使用 [source-revisions 流程](docs/source-revisions.md):核对同一 DOI 全集,保存完整修正前数据与版本链,支持已有记录的目标 track,按新版本重算分区覆盖数,并阻止旧版元数据刷新。普通增量导入仍禁止缩小预期数量。
入库前可使用通用的[来源身份只读预检](docs/source-identity-audit.md),导出候选论文、稳定标识和既有投稿决定证据;候选结果需要审核,不会自动批准导入。 入库前可使用通用的[来源身份只读预检](docs/source-identity-audit.md),导出候选论文、稳定标识和既有投稿决定证据;候选结果需要审核,不会自动批准导入。
RSS 2026 已独立适配官方论文集与录用表;使用方法和与 2025 demo 分类的区别见 [RSS 2026 来源说明](docs/rss2026-proceedings.md)。 RSS 2026 已独立适配官方论文集与录用表;使用方法和与 2025 demo 分类的区别见 [RSS 2026 来源说明](docs/rss2026-proceedings.md)。
@@ -137,8 +141,16 @@ ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目
`collect sdm2026-proceedings` 核验 53 个 Crossref 论文章节,并单独保留 53 项日程观察及 52 页摘要。作者差异、取消展示和摘要缺失分别保留,不自动拼接来源或绑定作品;见 [SDM 2026 使用与恢复说明](docs/sdm2026-proceedings.md)。 `collect sdm2026-proceedings` 核验 53 个 Crossref 论文章节,并单独保留 53 项日程观察及 52 页摘要。作者差异、取消展示和摘要缺失分别保留,不自动拼接来源或绑定作品;见 [SDM 2026 使用与恢复说明](docs/sdm2026-proceedings.md)。
`collect cikm2026-sources` 核验并分开导出两份官方节目、附属议程和出版登记。见 [CIKM 2026 来源采集与恢复](docs/cikm2026-sources.md)。
`collect icdm2026-program` 离线核验 324 条 ICDM2026 官方线下日程,按明确的研究/应用标签输出 241/83 条;展示长短、缺失摘要及未核实出版身份分别保留,不自动入库。见 [ICDM2026 使用与恢复说明](docs/icdm2026-program.md)。 `collect icdm2026-program` 离线核验 324 条 ICDM2026 官方线下日程,按明确的研究/应用标签输出 241/83 条;展示长短、缺失摘要及未核实出版身份分别保留,不自动入库。见 [ICDM2026 使用与恢复说明](docs/icdm2026-program.md)。
`collect recsys2026-program` 核验九个 track 的 263 项官方录用贡献观察(260 个不同 DOI),保留摘要、原始作者和重复 DOI 的来源版本。见 [RecSys 2026 使用与恢复说明](docs/recsys2026-program.md)。 `collect recsys2026-program` 核验九个 track 的 263 项官方录用贡献观察(260 个不同 DOI),保留摘要、原始作者和重复 DOI 的来源版本。见 [RecSys 2026 使用与恢复说明](docs/recsys2026-program.md)。
[KDD 2026 官方目录](docs/kdd2026-program.md)可以按 DOI 核对两个投稿周期、分轨与目录中的撤回更换标注。
`collect mm2026-program` 从已冻结官方快照恢复全部 28 个来源 track 的 2,143 个节目 ID,保留 4,326 次展示的来源位置与场次上下文,单列 216 个摘要缺失项;输出供贡献类型、出版及作品身份核查使用,不是可直接导入的论文/决定队列。见 [MM 2026 使用、来源和恢复说明](docs/mm2026-source-status.md)。 `collect mm2026-program` 从已冻结官方快照恢复全部 28 个来源 track 的 2,143 个节目 ID,保留 4,326 次展示的来源位置与场次上下文,单列 216 个摘要缺失项;输出供贡献类型、出版及作品身份核查使用,不是可直接导入的论文/决定队列。见 [MM 2026 使用、来源和恢复说明](docs/mm2026-source-status.md)。
`collect mm2026-publications` 核验完整论文集的 1,918 个注册出版 DOI,并单独关联节目分轨、展示上下文和摘要。1,743 份节目摘要明确标注来源(其中 26 份来自已复核的标题排版对应),166 项标题/作者版本差异保留为待核;所有节目及未匹配项均保留。注册文章类型不等于研究类型,输出尚需数据库身份/历史审查。见 [MM 2026 出版条目采集与恢复](docs/mm2026-publications.md)。
ECCV 2026 已归档 Springer 目录和章节可通过 [正式出版采集流程](docs/eccv2026-publications.md)整理为带 DOI 和可复核证据的 JSONL。
+22
View File
@@ -0,0 +1,22 @@
# CIKM 2026 来源采集与恢复
使用固定节目和登记快照复现 CIKM 2026 来源盘点。采集命令生成四份独立 JSONL,输入逐文件校验 SHA-256。
```sh
paper-atlas collect cikm2026-sources \
--snapshots /data/paper-index/validation/cikm2026-source-snapshots-v1 \
--out /data/paper-index/collected/cikm2026-sources-v1
```
输入包括两版官方节目 HTML、Crossref 卷登记、五页逐篇登记及分页回执。服务器验证目录保留输入原文。输出目录必须是新目录。先检查 `collection-report.json`,报告在四份 JSONL 写完后发布。
| 文件 | 内容 |
| --- | --- |
| `programme.jsonl` | 带 DOI 的 1,008 条节目记录,包括标题、作者显示名、摘要和五类节目标签 |
| `alternate-programme.jsonl` | 另一版节目 1,005 条记录及各自的字段 |
| `supplemental-programme.jsonl` | 73 条演讲、教程、工作坊及其他附属活动观察 |
| `crossref.jsonl` | 1,081 条 ACM 卷内登记,每条保留登记字段及分页文件位置 |
节目 DOI 指向 Crossref 的出版登记。确认作者和标题差异以及历史投稿绑定后,再为来源观察准备入库队列。报告另列 73 个未出现在带 DOI 节目区的登记,附属活动仍保留各自的活动类型。公开拒稿与撤回决定仍需独立来源核查。
官方原始来源见[带 DOI 的节目](https://cikm2026.diag.uniroma1.it/cikm-program/program.html)、[另一版节目](https://cikm2026.diag.uniroma1.it/cikm-program/CIKM2026_program.html)和 [Crossref 卷登记](https://api.crossref.org/works/10.1145/3799682)。
+6 -2
View File
@@ -1,6 +1,6 @@
# 会议覆盖与公开决定核查 # 会议覆盖与公开决定核查
最新生产进展和剩余工作见 [2026-11-17 补漏状态](backfill-status-2026-11-17.md):固定 278 个会议年份槽位中,270 个有记录、8 个仍为空;这不代表完整覆盖,第 14 步仍未完成。 最新生产验收与剩余工作见 [2026-11-17 补漏状态](backfill-status-2026-11-17.md):当日 `14:06:28 UTC` 固定范围快照中,278 个会议年份槽位有 272 个含记录、6 个仍为空;这不代表完整覆盖,第 14 步仍未完成。后续 MM 规范化、只读候选审计与出版核查不等于生产导入。
**当前工作范围:2020 年起主动补漏。** 2000 年起已发现资料保留,恢复工作从 [补漏范围与恢复清单(2026-11-10)](backfill-handoff-2026-11-10.md) 开始;下文旧日期快照不改写,较早年份排除不算补齐。 **当前工作范围:2020 年起主动补漏。** 2000 年起已发现资料保留,恢复工作从 [补漏范围与恢复清单(2026-11-10)](backfill-handoff-2026-11-10.md) 开始;下文旧日期快照不改写,较早年份排除不算补齐。
@@ -72,7 +72,7 @@ PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas coverage \
## 优先缺口 ## 优先缺口
- **UAI 2024–2026**:官方均允许作者选择公开拒稿,当前主库拒稿仍为零。原 notes 接口要求 Challenge 验证,公开镜像尚无已核验来源,必须保留这个真实缺口;不能根据 PMLR 录用名单推断拒稿。参见 [2025 投稿指南](https://www.auai.org/uai2025/submission_instructions)、JSON 内其他年份官方证据。 - **UAI 2024–2026**:官方均允许作者选择公开拒稿,首份生产快照拒稿为零。原 notes 接口要求 Challenge 验证;新核验镜像的 2022–2025 年 902 个论坛来自录用子集,不能填补公开拒稿缺口。2024 年的元评审也不能直接转成最终决定,见 [2026-11-17 原始来源与回复结构核查](uai-public-negative-source-status-2026-11-17.md)。不能根据 PMLR 或镜像录用名单反推拒稿;政策参见 [2025 投稿指南](https://www.auai.org/uai2025/submission_instructions)、JSON 内其他年份官方证据。
- **EMNLP 2023**:官方允许公开 opt-in 拒稿,目前只导入 9 条。需继续核验实际公开全集。ARR 评审、conference commitment 与最终会议决定分开;[ARR 作者说明](https://aclrollingreview.org/authors) 明确其自愿捐赠数据与 EMNLP 2023 直接投稿发布的范围不同。 - **EMNLP 2023**:官方允许公开 opt-in 拒稿,目前只导入 9 条。需继续核验实际公开全集。ARR 评审、conference commitment 与最终会议决定分开;[ARR 作者说明](https://aclrollingreview.org/authors) 明确其自愿捐赠数据与 EMNLP 2023 直接投稿发布的范围不同。
- **ICLR**:固定快照已观察 2017 与 2020–2026;快照之后,2013/2018/2019 的 1,654 条来源记录已完成独立 PostgreSQL 验收(报告时间 `2026-11-07T19:52:28.378Z`),不回写旧快照计数,也不据此宣称逐届全集已齐。快照中的 10,208 次 unknown 不等于 rejected,近期补采与冲突核对仍在进行;评审推荐不能当最终决定。 - **ICLR**:固定快照已观察 2017 与 2020–2026;快照之后,2013/2018/2019 的 1,654 条来源记录已完成独立 PostgreSQL 验收(报告时间 `2026-11-07T19:52:28.378Z`),不回写旧快照计数,也不据此宣称逐届全集已齐。快照中的 10,208 次 unknown 不等于 rejected,近期补采与冲突核对仍在进行;评审推荐不能当最终决定。
- **MM / MLSys**:MM 11,200 条书目记录保留 `proceedings_unspecified`;MLSys 的 448 次官方投稿与 313 条 DBLP 记录重叠。2026 MLSys 的研究/工业轨道在 [官方 CFP](https://mlsys.org/Conferences/2026/CallForResearchPapers) 中分开,仍须核对逐篇轨道证据,不能仅凭会议名全部归 main。 - **MM / MLSys**:MM 11,200 条书目记录保留 `proceedings_unspecified`;MLSys 的 448 次官方投稿与 313 条 DBLP 记录重叠。2026 MLSys 的研究/工业轨道在 [官方 CFP](https://mlsys.org/Conferences/2026/CallForResearchPapers) 中分开,仍须核对逐篇轨道证据,不能仅凭会议名全部归 main。
@@ -202,3 +202,7 @@ IJCAI 联办届次的逐条后续核查见 [2018/2022 分区与目录差集](ijc
[早期NLP来源范围复核](historical-source-scope-2026-11-09.md)保留3份官方网页的响应哈希,排除4个COLING卷对ACL缺口的误匹配,并区分会议举办记录、目录可获取性与论文级覆盖;尚未导入或关闭这些年度缺口。 [早期NLP来源范围复核](historical-source-scope-2026-11-09.md)保留3份官方网页的响应哈希,排除4个COLING卷对ACL缺口的误匹配,并区分会议举办记录、目录可获取性与论文级覆盖;尚未导入或关闭这些年度缺口。
- [MM 2026 官方节目来源状态](../mm2026-source-status.md):完整官方日程 JSON、重复展示盘点、28 个来源 track、缺失作者/摘要和四条匿名决定路径的 403 证据;这些不是已导入或公开拒稿全集。 - [MM 2026 官方节目来源状态](../mm2026-source-status.md):完整官方日程 JSON、重复展示盘点、28 个来源 track、缺失作者/摘要和四条匿名决定路径的 403 证据;这些不是已导入或公开拒稿全集。
- [2026-11-18 公开负面来源核查](public-negative-source-audit-2026-11-18.md):ResearchArcade 930 条非 ICLR 声明的当前库比对、5 条未绑定声明及原始接口访问证据;不把镜像差集算作新增拒稿。
- [ECCV 2026 出版目录与节目单对账](eccv2026-publisher-crosswalk-2026-11-18.md)记录 Springer 78 卷的 DOI、节目单差异及 PostgreSQL 只读绑定核查。
@@ -1493,7 +1493,7 @@
}, },
"TMLR_current20261108": { "TMLR_current20261108": {
"url": "https://www.jmlr.org/tmlr/faq.html", "url": "https://www.jmlr.org/tmlr/faq.html",
"finding": "Additionalnonconferencevenuealreadyinregistry:accepted/rejected/withdrawn/retractedallpublic; deskrejectedneverpublic. Authorshiddenonnotyetacceptedpapers; donotinferanonymousidentities. Continuousjournal; eventcertification/journal-to-conferencepresentationnotanotherindependentreviewacceptance." "finding": "Additionalnonconferencepolicyreference; TMLR is not in the current 43-conference registry:accepted/rejected/withdrawn/retractedallpublic; deskrejectedneverpublic. Authorshiddenonnotyetacceptedpapers; donotinferanonymousidentities. Continuousjournal; eventcertification/journal-to-conferencepresentationnotanotherindependentreviewacceptance."
} }
}, },
"public_negative_recovery_notes": { "public_negative_recovery_notes": {
@@ -1507,6 +1507,12 @@
"original_rejection_notes_obtained": false, "original_rejection_notes_obtained": false,
"prior_decisions_inserted": 0, "prior_decisions_inserted": 0,
"requires": "Per-paper primary original identifiers/decisions or an explicitly distinguished policy-supported relation; never manufacture an original note." "requires": "Per-paper primary original identifiers/decisions or an explicitly distinguished policy-supported relation; never manufacture an original note."
},
"researcharcade_20261118": {
"audit": "public-negative-source-audit-2026-11-18.json",
"summary": "930 non-ICLR 2020+ mirror claims:925 scoped ID matches all have rejected status;5 held,3 with equal title+source abstract but distinct unverified note IDs. No writes or new rejections approved.",
"access": "NeurIPS2021 and EMNLP2023 original note queries403; stopped, no bypass.",
"next": "Seek independent primary evidence for note.id/forum relationships and final decisions; no title-only merge or duplicate-rejection import. No full-public-set coverage claim."
} }
}, },
"restore_rules": [ "restore_rules": [
+3 -1
View File
@@ -33,7 +33,7 @@ SDM 新增事件使 SeBA 和 AdamG 同时保有较早 ICLR 拒稿与后来 SDM
| ECCV | 2,834 个不同初步节目 ID 及公开摘要已独立采集;1,997 个只读候选历史证据包已保存,尚未入库 | 初步日程不等于 Springer 最终论文集;继续作品/版本和出版核查。见 [来源状态](../eccv2026-source-status.md) | | ECCV | 2,834 个不同初步节目 ID 及公开摘要已独立采集;1,997 个只读候选历史证据包已保存,尚未入库 | 初步日程不等于 Springer 最终论文集;继续作品/版本和出版核查。见 [来源状态](../eccv2026-source-status.md) |
| ICDM | 324 项线下节目观察,明确研究241/应用83;演讲长168/短156,摘要均缺失;规范化和80个候选历史包已核验 | 展示长短不能当成论文类型;需审查作品与正式出版版本,尚未导入主库。见 [采集器说明](../icdm2026-program.md) | | ICDM | 324 项线下节目观察,明确研究241/应用83;演讲长168/短156,摘要均缺失;规范化和80个候选历史包已核验 | 展示长短不能当成论文类型;需审查作品与正式出版版本,尚未导入主库。见 [采集器说明](../icdm2026-program.md) |
| ISMIR | 140 项节目和正式名单均已核验,140份摘要,101组自愿公开评审;两处标题、三处作者名单差异保留,78个候选历史包已核验 | 数字 miniconf ID 不是原始 OpenReview ID;评审意见不转成最终拒稿。继续作品身份/历史关联,尚未入库。采集器见 [PR #63](https://gitea.entroia.com/Zayden-Jung/paper-atlas/pulls/63) | | ISMIR | 140 项节目和正式名单均已核验,140份摘要,101组自愿公开评审;两处标题、三处作者名单差异保留,78个候选历史包已核验 | 数字 miniconf ID 不是原始 OpenReview ID;评审意见不转成最终拒稿。继续作品身份/历史关联,尚未入库。采集器见 [PR #63](https://gitea.entroia.com/Zayden-Jung/paper-atlas/pulls/63) |
| MM | 官方10.8MB日程JSON已冻结,4,326次观察对应2,143个节目ID、28个来源track;重复展示核心字段一致,216个ID缺摘要,其中两项主轨同时缺作者 | 不是2,143篇已核验科研论文。先区分各轨贡献与活动、保留全部上下文并单列缺失;尚未规范化/入库。见 [来源与访问证据](../mm2026-source-status.md) | | MM | 全28轨2143个节目ID/4326展示位置已规范化;814个只读候选历史证据包已核验,24篇候选涉及24次负面投稿/37条来源观察;67项原始arXiv DOI中66项注册元数据一致、1项404暂缓 | 不是2143篇已核验科研论文。继续逐轨贡献/出版依据与作品审查,0身份批准、0入库。ISBN查询与父XML均只返回父记录,不作逐篇清单。见 [来源、审计与恢复证据](../mm2026-source-status.md) |
| CIKM | 官方录用/日程入口与五个分轨页面已保存;五个分轨链接实际是CFP/政策,不是逐篇录用名单 | 查找实际节目或出版社完整目录;TBD和没有静态表不能当作没有论文 | | CIKM | 官方录用/日程入口与五个分轨页面已保存;五个分轨链接实际是CFP/政策,不是逐篇录用名单 | 查找实际节目或出版社完整目录;TBD和没有静态表不能当作没有论文 |
| IROS | 官方论文索引入口的原始HTTP403已保存 | 保留访问限制,不重试绕过;寻找可合法读取、能独立核验的出版元数据 | | IROS | 官方论文索引入口的原始HTTP403已保存 | 保留访问限制,不重试绕过;寻找可合法读取、能独立核验的出版元数据 |
@@ -43,6 +43,8 @@ SDM 新增事件使 SeBA 和 AdamG 同时保有较早 ICLR 拒稿与后来 SDM
MM 官方群组元数据可匿名读取,但根据其给出的 submission invitation 和 rejected/withdrawn/desk-rejected venue ID,四次限1条的匿名 notes 核查均为403。查询、时间、原文和哈希已保留,没有认证重试或绕过权限。访问限制不等于零条负面记录;详见 [MM来源证据](../mm2026-source-status.md)。ISMIR公开的101组评审来自录用节目,不能记成101条拒稿。 MM 官方群组元数据可匿名读取,但根据其给出的 submission invitation 和 rejected/withdrawn/desk-rejected venue ID,四次限1条的匿名 notes 核查均为403。查询、时间、原文和哈希已保留,没有认证重试或绕过权限。访问限制不等于零条负面记录;详见 [MM来源证据](../mm2026-source-status.md)。ISMIR公开的101组评审来自录用节目,不能记成101条拒稿。
UAI新增固定镜像已核对2022–2025的902个原始论坛及3053个回复的元数据/字段结构。整理版均标录用,2022保存230条明确Accept决定;2024仅有评审、作者回复、元评审,2023/2025未保存回复,不能据此补拒稿。来源限制和恢复见 [逐年原始核查](uai-public-negative-source-status-2026-11-17.md)。这不是UAI没有公开拒稿的结论,缺口保持未补齐。
第14步仍需逐会议、年份、track完成公开拒稿/desk-reject/withdrawn来源核查和可得数据采集;继续核查UAI近年自愿公开范围、EMNLP实际覆盖范围及其余已列明缺口。不能从未见录用、评分、评审推荐、缺少摘要或接口不可访问制造负面决定。 第14步仍需逐会议、年份、track完成公开拒稿/desk-reject/withdrawn来源核查和可得数据采集;继续核查UAI近年自愿公开范围、EMNLP实际覆盖范围及其余已列明缺口。不能从未见录用、评分、评审推荐、缺少摘要或接口不可访问制造负面决定。
之后仍需完成跨会作品/历次投稿与类型标签关联(第15步),所有导入结束后的独立全库去重、标识冲突、近似标题作者、摘要/约束/查询及覆盖审计(第16步),以及长期运行与恢复文档最终整理(第17步)。阶段性来源入库、测试通过、某年度有记录均不能替代这些验收。 之后仍需完成跨会作品/历次投稿与类型标签关联(第15步),所有导入结束后的独立全库去重、标识冲突、近似标题作者、摘要/约束/查询及覆盖审计(第16步),以及长期运行与恢复文档最终整理(第17步)。阶段性来源入库、测试通过、某年度有记录均不能替代这些验收。
@@ -0,0 +1,35 @@
# 2020 起补漏状态:2026-11-18
总体仍为 **13/17,第 14 步未完成**。主动补漏从 2020 年起,2000–2019 年已发现线索仍保留,43 个早期历史缺口仍为延期,没有据此算作补齐。更早状态页保留为历史快照。
## 当前生产结果
ISMIR 两批累计31项已关联到现有论文,新增论文实体为0。第二批新增来源记录、独立投稿、录用观察各21条及173条来源标签证据;两批累计267条标签证据。覆盖 `partial31/140`,109项仍待核验。详见[来源说明](../ismir2026-program.md)、[首批验收](ismir2026-import10-2026-11-18.json)和[第二批逐条验收](ismir2026-import21-2026-11-18.json)。
最新精确计数:论文实体 **3,568,022**,来源记录 **3,815,534**,投稿 **532,223**,决定来源观察 **571,775**。实体数量不是已经完成最终科学工作去重的数量;同一次投稿可能有多条决定来源观察。
paper `2757497` 保留 ICLR2026 一次拒稿投稿下的两条来源观察,并新增 ISMIR2026 录用投稿;实际 CLI 查询显示已知录用,旧拒稿历史仍可查询。不将两条来源观察计作两次拒稿,也不将来源采集时间当作决定时间。
## 固定范围存在性盘点
只读快照 `2026-11-18 06:46:44.099675+00`,范围仍为 43 会、278 个目标年度槽位:**274 个至少有一条投稿,4 个完全为空**,均为 2026 年的 **CIKM、ECCV、ICDM、IROS**。首批ISMIR使空槽从5降至4;第二批只增加同届来源覆盖,空槽数不变。
这只统计“该年度至少有数据”,不是逐篇、逐 track、摘要或公开拒稿覆盖完整。2020–2025 没有整年空槽,仍有来源完整性和公开负面历史缺口。
|证据(相对服务器 `/data/paper-index/`)|SHA256|
|---|---|
|`collected/active-scope-census-20261118-v22/summary.json`|`ee4ec2e6d6eaed55ce3475e735bfba9bd94e512893cd3c3024035b48b744f475`|
|`collected/active-scope-census-20261118-v22/snapshot.json`|`5aabc91217ad787946ba50d6a670296b1e3152a54f5ecaa0e3a9b3a952b63fa7`|
|`validation/active-scope-fixed278-v1.json`|`a835bef5b9c3a0f76069d840cecdb9a6bcf65d3a38796d70dbfdfd4b49242264`|
本机使用固定分母、会议别名和逐 edition 快照独立重算 274/278;没有使用 `max(id)` 代替计数。
## 尚待完成
- ISMIR 仍有109项;62项无精确标题候选不等于新作品,6项作者部分匹配仍待审查。31项摘要逐字一致候选已分两批审查并消费;其余候选不能仅凭标题、作者或摘要检索结果算作已批准。
- MM 已验收572项来源观察,注册清单1918项中仍有1346项待处理。已有134个暂建索引实体仍须后续身份审计;H2AL 来源及轨道纠正、版本化来源覆盖方案尚未用于生产,不能把不同版本轨道预期数相加当完整分母。
- ECCV 的初步节目、ICDM 的研究/应用来源和缺失摘要仍需作品及出版核查;CIKM 缺实际逐篇目录;IROS 官方访问403仍保留限制,不重试绕过。各来源的采集、批准和导入状态必须分开。
- 公开 rejected、desk-rejected、withdrawn 继续按会议、年份、track核查。ISMIR 的101组自愿公开评审来自录用节目,不是101条拒稿;UAI accepted-only 镜像与 EMNLP/NeurIPS 原始接口403均不能关闭拒稿缺口。近期镜像差集核查见 [PR #72](https://gitea.entroia.com/Zayden-Jung/paper-atlas/pulls/72)。
- 后续仍需完成跨会作品与历史、独立类型和标签关系(第15步);全部导入结束后的全库去重、标识冲突、近似标题作者、摘要/约束/查询及覆盖审计(第16步);长期运行与恢复文档最终整理(第17步)。
已消费输入不得重放。下次从[ISMIR 当前增量父状态和恢复说明](../ismir2026-program.md)及 `work-state.json` 的本批已接受回执继续;保留旧 SQLite 只读备份,不因查询超时重新执行导入。
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,16 @@
# CIKM 2026 节目与出版登记核对
官方节目列有 1,008 项带 DOI 的条目,逐篇均有摘要。五个标签的数量分别为 Full Research 596、Short Research 234、Applied Research 86、Resources 46、Demo 46。另一份官方节目快照少三个节目编号,其余 1,005 个编号重合。两份原始快照和响应哈希保存在本机的 `cikm2026-official-detailed-program-v1` 工作目录。[带 DOI 的节目](https://cikm2026.diag.uniroma1.it/cikm-program/program.html) · [另一份官方节目](https://cikm2026.diag.uniroma1.it/cikm-program/CIKM2026_program.html)
正式卷 DOI 为 `10.1145/3799682`,ISBN 为 `9798400725395`。ISBN 查询只返回卷级登记。按卷名及出版日期分页取得 1,081 条登记,逐条核对卷 DOI 前缀、完整容器标题及来源类型。1,008 个节目 DOI 全部匹配,另有 73 条登记位于此前带 DOI 节目分区之外。附属议程核对结果见下文。该次 Crossref 响应中摘要字段均缺失。[ACM 出版条目](https://doi.org/10.1145/3799682.3840615) · [Crossref 卷登记](https://api.crossref.org/works/10.1145/3799682)
按已知排版标记、Unicode 和标点变化比较,同 DOI 的条目有 4 处标题差异、11 处有序作者列表差异。差异条目保留两份原文,等后续核对后再决定如何关联。当前 PostgreSQL 快照中,97 个节目 DOI 已绑定论文,304 个节目标题有候选,其中 95 个 DOI 绑定与标题候选重合。仍需核查已有作品与历次投稿历史,当前尚未写入。
[逐条差集、字段差异及证据哈希](cikm2026-source-reconciliation-2026-11-18.json)。先将出版登记与节目摘要作为各自来源采集,再核查两者的作品关系和轨道。公开拒稿、撤回及桌拒的可得性继续单独核查。
## 附属议程与已有 DOI 的进一步核查
在官方节目页面的附属议程及主题演讲中,73 条登记有 72 条找到唯一规范标题对应,70 条同时通过有序作者比较。对应项包括 Industry Day 20 条、博士论坛 19 条、教程 14 条、工作坊条目 15 条、主题演讲 3 条及 AnalytiCup 1 条。工作坊条目记录该场活动对应的出版材料,独立 workshop 论文仍需各自目录。两条工作坊条目有作者写法差异,另有一条 LASS 标题带不同前后缀,三条均保留待核对。[官方完整节目](https://cikm2026.diag.uniroma1.it/cikm-program/program.html)
全卷 1,081 个 DOI 的 PostgreSQL 只读复查找到 98 个已有绑定,其中 97 个来自带 DOI 节目分区。89 项在现有论文、Crossref 和节目中有一致的规范标题、有序作者及明确节目标签,可优先准备来源关联。其余 9 项保留待核查。98 项的现有历史中无拒稿、桌拒或撤回记录,这一观察仅覆盖已有绑定的作品。此次仍未写入,入库时须重新确认绑定及保护既有历史。
File diff suppressed because it is too large Load Diff
@@ -0,0 +1,20 @@
# ECCV 2026 出版目录与节目单对账
2026-11-18 核验了 Springer LNCS 17001–17078 的全部 78 卷。会议索引显示 **2,835 条**,逐卷目录合计也有 **2,835 条带 DOI 的章节**,另有 156 项卷首或卷末资料。出版方卷册简介及大会节目单均为 **2,834 条**。两个来源仍有逐条差异,汇总数量相差一条的原因待核。逐卷数字、目录页哈希和只读数据库回执见[机器可读清单](eccv2026-publisher-crosswalk-2026-11-18.json)。[Springer 会议索引](https://link.springer.com/conference/eccv) · [第一卷](https://link.springer.com/book/10.1007/978-3-032-36984-0) · [大会节目单](https://eccv.ecva.net/Conferences/2026/AcceptedPapers)
## 逐篇对账
|核对对象|唯一标题且完整作者顺序一致|唯一标题且目录所显示作者顺序一致|标题对得上、作者有差异|标题待核|
|---|---:|---:|---:|---:|
|全部 2,834 条节目记录|1,332|709|535|258|
|首批已入库的 30 条节目记录|23|3|4|0|
|本次只读核验通过的 248 条既有论文候选|173|72|2|1|
|20 篇有既往拒稿、桌拒或撤回的优先论文|12|7|1|0|
目录作者栏有时只显示前六人。作者差异包括缩写、姓名文字变化与排列变化。原始名单、顺序和来源保留,前缀匹配与作者差异需查章节作者页。节目单与出版目录的摘要、分轨及公开负面决定覆盖仍待分别核查。
## 当前库核对
只读核对了全部 2,835 个出版 DOI,68 个已有论文绑定。对账后另检查 271 个已知论文关联,涵盖 245 个本次候选和 26 个已入库节目记录。其中 14 个 DOI 已指向同一论文,257 个 DOI 尚未绑定。245 个候选内有 10 篇带既往负面决定。
下一批准备以出版 DOI、正式分轨和完整作者证据复核关联。首批 30 条节目记录保留原投稿、决定和来源。确认同届同一投稿后,再补充出版证据,避免重复创建 ECCV 投稿。本次核查未写入数据库。
@@ -0,0 +1,339 @@
{
"sources": {
"poster.html": {
"url": "https://www3.cs.stonybrook.edu/~icdm2026/PosterwDemo/list.htm",
"final_url": "https://www3.cs.stonybrook.edu/~icdm2026/PosterwDemo/list.htm",
"http_status": 200,
"bytes": 132977,
"sha256": "b02083ca7574a188fe876851a700a12c131c262353db557bd43a8320fad5f747",
"at": "2026-11-18T07:06:11.984184+00:00"
},
"schedule.html": {
"url": "https://icdm2026.neu.edu.cn/DetailedSessions/list.htm",
"final_url": "https://icdm2026.neu.edu.cn/DetailedSessions/list.htm",
"http_status": 200,
"bytes": 213830,
"sha256": "c1294b89bae8c2dfcf652d8a25556e733ef42dc3ac3387070abe08419b961b0f",
"at": "2026-11-18T07:06:12.349258+00:00"
},
"deepsync.html": {
"url": "https://dm.cs.univie.ac.at/news/story/deepsync-paper-accepted-at-icdm-2026",
"final_url": "https://dm.cs.univie.ac.at/news/story/deepsync-paper-accepted-at-icdm-2026",
"http_status": 200,
"bytes": 90333,
"sha256": "e243196943434225acec2841b53c77a2b1459184e7c185e1e1cbb055118fe282",
"at": "2026-11-18T07:06:20.561308+00:00"
}
},
"poster_count": 343,
"poster_groups": {
"long-posters": 177,
"short-posters": 166
},
"oral_count": 324,
"shared_count": 324,
"poster_only": {
"DM1022": {
"group": "short-posters",
"floor": "3F",
"area": "Area C",
"slot": "P2-3C-085",
"session": "Applied Track: Real-world AI in finance, healthcare and industry II",
"title": "Beyond Bidding Agents: A Dual-Role Multi-Agent Framework with Dynamic Reserve Pricing for Auto-Bidding Auctions",
"line": 683
},
"DM1189": {
"group": "long-posters",
"floor": "3F",
"area": "Area A",
"slot": "P1-3A-032",
"session": "Heterogeneous data mining: spatiotemporal and representation learning",
"title": "Traj-Transformer: Diffusion Models with Transformer for GPS Trajectory Generation",
"line": 456
},
"DM1234": {
"group": "long-posters",
"floor": "2F",
"area": "Area A",
"slot": "P1-2A-007",
"session": "Foundations, causality and learning theory",
"title": "Prediction-Powered Corpus Mining: Coverage-Valid Statistics from Scarce Labels",
"line": 354
},
"DM1246": {
"group": "short-posters",
"floor": "3F",
"area": "Area B",
"slot": "P2-3B-074",
"session": "LLM-driven data mining and novel applications II",
"title": "Feature Selection for Trajectory Inference",
"line": 672
},
"DM1345": {
"group": "short-posters",
"floor": "3F",
"area": "Area B",
"slot": "P2-3B-060",
"session": "LLM-driven data mining and heterogeneous learning",
"title": "TRACER: An LLM-based Coarse-to-Fine Framework for Adaptive Network Traffic Classification",
"line": 658
},
"DM1408": {
"group": "long-posters",
"floor": "2F",
"area": "Area B",
"slot": "P1-2B-043",
"session": "Applied Track: Industrial, financial and decision intelligence",
"title": "A Generation-to-Application Framework for Semantic ID Collision in Industrial Generative Recommendation",
"line": 390
},
"DM1644": {
"group": "long-posters",
"floor": "2F",
"area": "Area B",
"slot": "P1-2B-072",
"session": "Recommendation, adaptation and spatiotemporal modeling II",
"title": "LQ-Rec: Late Quantization for Sequential Recommendation",
"line": 419
},
"DM1751": {
"group": "short-posters",
"floor": "3F",
"area": "Area A",
"slot": "P2-3A-014",
"session": "ML/DL and statistical methods",
"title": "MissGNN: Missingness-Guided Graph Neural Network for Feature- and Sample-Aware Imputation",
"line": 612
},
"DM1995": {
"group": "long-posters",
"floor": "2F",
"area": "Area B",
"slot": "P1-2B-053",
"session": "Applied Track: Recommendation, security and operational intelligence II",
"title": "Multi-Modal Causal Disentanglement for Multi-Domain Sequential Audio Recommendation",
"line": 400
},
"DM2002": {
"group": "long-posters",
"floor": "3F",
"area": "Area B",
"slot": "P1-3B-048",
"session": "Heterogeneous data mining and systems",
"title": "HiMamba: A Hierarchical State Space Framework for Asymmetric Multimodal Data Mining in Clinical Assessment",
"line": 472
},
"DM2055": {
"group": "short-posters",
"floor": "3F",
"area": "Area A",
"slot": "P2-3A-038",
"session": "Applied Track: Real-world AI in finance, healthcare and industry I",
"title": "CGSBR: Adaptive Cross-Source Gating for Cold-Start Sequential Bundle Recommendation",
"line": 636
},
"DM2140": {
"group": "long-posters",
"floor": "3F",
"area": "Area A",
"slot": "P1-3A-008",
"session": "Machine learning and statistical methods",
"title": "Reliability-Guided Pseudo-Label Learning and Prototype Alignment for Domain-Adaptive Colorectal Segmentation",
"line": 432
},
"DM252": {
"group": "short-posters",
"floor": "2F",
"area": "Area B",
"slot": "P2-2B-046",
"session": "LLM-driven data mining and novel applications I",
"title": "Trust-But-Verify: Poisoning-Resilient Locally Private Graph Learning Protocols",
"line": 572
},
"DM383": {
"group": "long-posters",
"floor": "2F",
"area": "Area A",
"slot": "P1-2A-022",
"session": "Heterogeneous graphs and multimodal intelligence",
"title": "Subgraph Meta-Learning for Direction and Time Aware Transaction Anomaly Detection",
"line": 369
},
"DM463": {
"group": "short-posters",
"floor": "2F",
"area": "Area B",
"slot": "P2-2B-050",
"session": "Modeling, recommendation and LLM methods",
"title": "CaST-POI: Candidate-Conditioned Spatiotemporal Ranking for Next POI Recommendation",
"line": 576
},
"DM483": {
"group": "short-posters",
"floor": "2F",
"area": "Area B",
"slot": "P2-2B-051",
"session": "Modeling, recommendation and LLM methods",
"title": "Causal Masking Lowers Intrinsic Dimension in Full-Vocabulary Sequential Recommendation: Evidence from a Single-Variable Attention Ablation",
"line": 577
},
"DM713": {
"group": "short-posters",
"floor": "2F",
"area": "Area B",
"slot": "P2-2B-060",
"session": "Modeling, recommendation and LLM methods",
"title": "Hint-Conditioned Knowledge Tracing: A Causal-Inspired View of Context-Dependent Hint Usage",
"line": 586
},
"DM736": {
"group": "long-posters",
"floor": "3F",
"area": "Area B",
"slot": "P1-3B-050",
"session": "LLM-driven data mining and retrieval I",
"title": "From Natural-Language Hypotheses to Verified Data-Mining Discoveries",
"line": 474
},
"DM803": {
"group": "short-posters",
"floor": "2F",
"area": "Area B",
"slot": "P2-2B-032",
"session": "Heterogeneous data mining and multimodal learning I",
"title": "Graph-Guided LLM Reasoning for Incident Prediction Over Multi-Source Data Streams",
"line": 558
}
},
"oral_only": [],
"title_differences": [
{
"paper_id": "DM1319",
"poster_title": "FFE-FRCN: A Frequency Feature Enhanced Few-shot Learning Approach for Object Detection in Microscopic Images",
"oral_title": "An FFE-FRCN-Based Data Platform for Environmental Microorganism Image Retrieval and Knowledge Platform"
}
],
"nonpaper_table_rows": [],
"database_written": false,
"scope": "Official presentation metadata. Shared programme IDs identify repeated appearances within this event. Poster-only IDs are scheduling observations pending identity and publication review.",
"pg_readonly": {
"snapshot": {
"at": "2026-11-18 07:08:21.138511+00",
"snapshot": "1095530:1095530:"
},
"report_sha256": "04ad5547342d7e3c2939046f290ecff647878c202c867e47acbdc438d4e10ad4",
"existing_icdm2026_editions": [],
"existing_icdm_program_source": false,
"reviewed_links": [
{
"external_id": "2026/DM1125",
"paper_id": 2783475,
"track": "research",
"source_row_sha256": "4e981605cf471a5ea29bb3ba3a676784cea55f7777a681eb5fe267ff9c83fa4a",
"source_abstract": null,
"work_anchor": {
"scheme": "arxiv",
"value": "2608.29635"
},
"independent_source": "https://arxiv.org/abs/2608.29635",
"existing_history": [
{
"id": 132984,
"venue": "ICLR",
"year": 2026,
"observations": [
{
"id": 134279,
"status": "rejected"
},
{
"id": 539402,
"status": "rejected"
}
]
}
],
"original_icdm_decision_verified": false,
"imported": false
},
{
"external_id": "2026/DM790",
"paper_id": 2485633,
"track": "applied",
"source_row_sha256": "14e03ca3833b267d6f677325f4e4b9295d1fa42cc77be6707b9bca745444888b",
"source_abstract": null,
"work_anchor": {
"scheme": "arxiv",
"value": "2509.19774"
},
"independent_source": "https://arxiv.org/abs/2509.19774",
"existing_history": [
{
"id": 138980,
"venue": "ICLR",
"year": 2026,
"observations": [
{
"id": 545219,
"status": "withdrawn"
}
]
}
],
"original_icdm_decision_verified": false,
"imported": false
},
{
"external_id": "2026/DM1157",
"paper_id": 3286364,
"track": "research",
"source_row_sha256": "22ed027f29a161bc978b3c33e921b42c67d999d520eed49ec46ce5935a6f2f2a",
"source_abstract": null,
"work_anchor": {
"scheme": "openreview_snapshot",
"value": "2026/1dGzRTSLqL"
},
"independent_source": "https://dm.cs.univie.ac.at/news/story/deepsync-paper-accepted-at-icdm-2026",
"existing_history": [
{
"id": 127709,
"venue": "ICLR",
"year": 2026,
"observations": [
{
"id": 129094,
"status": "rejected"
},
{
"id": 534293,
"status": "rejected"
}
]
}
],
"original_icdm_decision_verified": false,
"imported": false
}
],
"held": [
{
"external_id": "2026/DM1791",
"paper_id": 3288747,
"reason": "DomainBridging has name-order/format differences and no arxiv self-accepted link; token-bag retrieval is not author or work identity approval."
},
{
"external_id": "2026/DM1801",
"paper_id": 3288866,
"reason": "LightGFC author lists substantially differ; hold version/work review, no automatic author aliases or merge."
},
{
"external_id": "2026/DM1413",
"paper_id": 3292171,
"reason": "OFS author lists differ including JianchengLv vs JindiLv; not treated as aliases; hold independent version/work verification."
}
]
},
"publisher_catalog_status": "No 2026 IEEE publisher catalogue verified in this check",
"public_negative_availability": "unverified",
"next": "Use the existing oral source denominator by track. Compare the three reviewed candidates with the fresh import baseline and preserve their earlier attempts. Keep the poster-only rows as a separate source inventory, and resolve the title disagreement before linking that poster."
}
@@ -0,0 +1,204 @@
{
"checked_on": "2026-11-18",
"venue": "IROS",
"year": 2026,
"conference_program": {
"url": "https://2026.ieee-iros.org/program/paper_index/",
"status": "unavailable_in_checked_environment",
"paper_level_full_index_verified": false
},
"publisher_registration_check": {
"status": "no_target_container_in_first_25_relevance_results",
"rows_checked": 25,
"observed_container": "2026 IEEE 13th International Conference on Intelligent Systems (IS)",
"complete_registry_search": false
},
"journal_presentations": {
"journal": "IEEE Transactions on Robotics",
"scope": "IROS 2026 conference presentation of journal papers",
"rendered_index_and_embedded_entries_match": true,
"count": 30,
"xplore_document_ids": [
"10989573",
"11251175",
"11488837",
"11268969",
"11098164",
"11219220",
"11304178",
"11267071",
"11304172",
"11455384",
"11397863",
"11288096",
"11544114",
"11534108",
"11488912",
"11535315",
"11342351",
"11353077",
"11429535",
"11364144",
"11427341",
"11427344",
"11455389",
"11488904",
"11535317",
"11554436",
"11495270",
"11575122",
"11488843",
"11581355"
],
"conference_submission_decision_verified": false,
"publication_dois_verified": false
},
"author_reported_conference_acceptance_candidates": [
{
"arxiv": "2607.06499",
"acceptance_evidence": [
"uw-roundup.html",
"uw-faculty.html"
],
"author_count_current": 3,
"work_identity_status": "existing_work_binding_verified_read_only",
"database_paper_id": 2731624,
"ordered_authors_match": true,
"arxiv_versions_stored": [
"v1"
]
},
{
"arxiv": "2505.04999",
"acceptance_evidence": [
"usc-acceptance.html"
],
"author_list_evidence": "usc-clam-project.html",
"author_count_current": 7,
"work_identity_status": "existing_work_binding_verified_read_only",
"database_paper_id": 2381843,
"ordered_authors_match": true,
"arxiv_versions_stored": [
"v1",
"v2"
]
},
{
"acceptance_evidence": [
"ntt-contributed.html"
],
"author_count_current": 6,
"work_identity_status": "stable_identifier_and_track_pending"
}
],
"purdue_scope_note": {
"evidence": "purdue-mars.html",
"reported_combined_count": 10,
"reported_tro_subset": 2
},
"public_rejected_desk_rejected_withdrawn": "not_located_in_checked_public_sources",
"coverage_status": "partial_source_audit",
"database_written": false,
"receipts": [
{
"name": "ieee-tro-list.html",
"url": "https://www.ieee-ras.org/publications/t-ro/t-ro-list-of-conference-papers/",
"final_url": "https://www.ieee-ras.org/publications/t-ro/t-ro-list-of-conference-papers/",
"status": 200,
"bytes": 610713,
"sha256": "2f2bb466446c62c37dc0927f8df45b161a096cbf8ab1aeffc4f43fa5d27e6358"
},
{
"name": "ieee-tro-policy.html",
"url": "https://www.ieee-ras.org/publications/t-ro/",
"final_url": "https://www.ieee-ras.org/publications/t-ro/",
"status": 200,
"bytes": 566001,
"sha256": "37b8877babb62e69d61e8b32193378afad4372117025a139cf70ffc26fcada5c"
},
{
"name": "ntt-contributed.html",
"url": "https://group.ntt/en/topics/2026/09/28/iros2026.html",
"final_url": "https://group.ntt/en/topics/2026/09/28/iros2026.html",
"status": 200,
"bytes": 221293,
"sha256": "ade0a0320a83e1d4504ff5da24c9f495aed581cffa01aa38b1e4cbec331f6dbc"
},
{
"name": "crossref-proceedings-query.json",
"url": "https://api.crossref.org/works?query.container-title=2026+IEEE%2FRSJ+International+Conference+on+Intelligent+Robots+and+Systems+%28IROS%29&filter=prefix%3A10.1109%2Ctype%3Aproceedings-article%2Cfrom-pub-date%3A2026-01-01%2Cuntil-pub-date%3A2026-12-31&rows=25&select=DOI%2Ctitle%2Cevent%2Ccontainer-title%2Cpublisher%2Cpublished%2CURL",
"final_url": "https://api.crossref.org/works?query.container-title=2026+IEEE%2FRSJ+International+Conference+on+Intelligent+Robots+and+Systems+%28IROS%29&filter=prefix%3A10.1109%2Ctype%3Aproceedings-article%2Cfrom-pub-date%3A2026-01-01%2Cuntil-pub-date%3A2026-12-31&rows=25&select=DOI%2Ctitle%2Cevent%2Ccontainer-title%2Cpublisher%2Cpublished%2CURL",
"status": 200,
"bytes": 13634,
"sha256": "415bf105de8006a42d3f2f8e6eb0078eb54c51bd65315296c0735e853431b228"
},
{
"name": "purdue-mars.html",
"url": "https://engineering.purdue.edu/IE/news/2026/she-iros-mars-lab",
"final_url": "https://engineering.purdue.edu/IE/news/2026/she-iros-mars-lab",
"status": 200,
"bytes": 60386,
"sha256": "fae6ba9637ea141ff3fda93643b97ddfb65d6ea3689f1bca3580956272b2d373"
},
{
"name": "uw-acceptance.html",
"url": "https://faculty.washington.edu/chx/post/2026-06-17-iros-mppi/",
"status": 404,
"error": "HTTP Error 404: "
},
{
"name": "usc-acceptance.html",
"url": "https://liralab.usc.edu/news/",
"final_url": "https://liralab.usc.edu/news/",
"status": 200,
"bytes": 25381,
"sha256": "9fa6b272b7995448a4318dd91cc94b93eeb514ddaa92d7df6880b3d93d91fceb"
},
{
"name": "arxiv-2607.06499.html",
"url": "https://arxiv.org/abs/2607.06499",
"final_url": "https://arxiv.org/abs/2607.06499",
"status": 200,
"bytes": 43362,
"sha256": "98fcbcaf39832f3002fa2910f69ca650481bb473cdb450290e5f9ca9245646a2"
},
{
"name": "arxiv-2505.04999.html",
"url": "https://arxiv.org/abs/2505.04999",
"final_url": "https://arxiv.org/abs/2505.04999",
"status": 200,
"bytes": 42506,
"sha256": "26ea0fd9ac8753b911ff97f1aaf29ea495f50b5c1e106b33c1336ea87c03b0df"
},
{
"name": "uw-faculty.html",
"url": "https://www.me.washington.edu/facultyfinder/xu-chen",
"final_url": "https://www.me.washington.edu/facultyfinder/xu-chen",
"status": 200,
"bytes": 99274,
"sha256": "877af368b564ba26a62b1ce4f947b439cceada2490dfb446b8730ba132c72c78"
},
{
"name": "uw-roundup.html",
"url": "https://faculty.washington.edu/chx/post/2026-09-07-new-papers-2026/",
"final_url": "https://faculty.washington.edu/chx/post/2026-09-07-new-papers-2026/",
"status": 200,
"bytes": 180954,
"sha256": "bc9ef19a07ea2875f8cfffe9afd62bcc2998fbe7c4bc8a4440c1bc87bae3856b"
},
{
"name": "usc-clam-project.html",
"url": "https://clamrobot.github.io/",
"final_url": "https://clamrobot.github.io/",
"status": 200,
"bytes": 20098,
"sha256": "3b3ea3f817ce6483807ace9b5a42ccf9835b811837346b63411190316fe2819c"
}
],
"work_identity_review": {
"snapshot_at": "2026-11-18 09:26:04.582744+00",
"evidence_sha256": "5e5cfb57525a369e8c81e1a9ea6edeb4f5e83a33e1961eeead8a014e3d59ad6d",
"existing_work_matches": 2,
"database_written": false
}
}
@@ -0,0 +1,21 @@
# IROS 2026 来源核查
2026-11-18 核查了会议节目、期刊报告名单、作者机构公布的录用信息和 DOI 登记线索。逐项来源、响应哈希与文献定位符见[机器可读清单](iros2026-source-audit-2026-11-18.json)。本次来源核查未写入数据库。
## 期刊论文在会上报告
IEEE RAS 的 T-RO 名单在 IROS 2026 一节列出 30 个不同的 IEEE Xplore 文献编号。页面渲染内容和内嵌数据的编号及顺序完全一致。T-RO 政策允许已录用的期刊文章到 IROS 报告。后续应保留期刊出版与会议报告两种关系,并在确认期刊 DOI 后核对现有论文身份。本清单没有逐条验证 DOI 或会议投稿决定。[T-RO 报告名单](https://www.ieee-ras.org/publications/t-ro/t-ro-list-of-conference-papers/) · [T-RO 报告政策](https://www.ieee-ras.org/publications/t-ro/)
普渡大学的通告将团队的 10 项成果一同介绍,其中 2 项属于 T-RO。这个团队样本可以帮助核查来源口径,暂不用于生成逐篇录用记录。[普渡大学通告](https://engineering.purdue.edu/IE/news/2026/she-iros-mars-lab)
## 作者机构公布的录用线索
- 华盛顿大学的作者通告列出 IROS 2026 待出版论文,标题与 arXiv 2607.06499 对应,三位作者一致。只读查询确认对应论文 2731624,库内标题与三位作者的顺序一致。[作者通告](https://faculty.washington.edu/chx/post/2026-09-07-new-papers-2026/) · [arXiv](https://arxiv.org/abs/2607.06499)
- USC 实验室确认 CLAM 在 IROS 2026 录用。项目页指向 arXiv 2505.04999,当前作者名单为七人。库内 arXiv 保留 v1、v2,现有论文 2381843 的标题和七位作者顺序与当前详情页一致。[实验室通告](https://liralab.usc.edu/news/) · [作者项目页](https://clamrobot.github.io/) · [arXiv](https://arxiv.org/abs/2505.04999)
- NTT 公布一条题名及六位作者,待核实稳定标识和会议 track。[NTT 通告](https://group.ntt/en/topics/2026/09/28/iros2026.html)
这两条工作身份已核实,尚未录入 IROS 录用证据。
## 待补证据
[会议论文索引](https://2026.ieee-iros.org/program/paper_index/)在受检环境未能读取。一次限定 IEEE、2026 年及会议论文类型的 Crossref 相关性查询返回的前 25 条属于另一会议。实际会议论文集 DOI、完整出版清单、正式分轨和逐篇覆盖仍待核查。已检查的公开来源中未找到逐篇拒稿、桌拒或撤回清单,继续保持这些决定的公开可得性待查。
@@ -0,0 +1,247 @@
{
"as_of": "2026-11-18",
"goal_progress": "13/17; stage14 incomplete",
"source": "ismir_program",
"scope": "First10 reviewed existing-work source links, partial10/140, not public-rejection completeness or final deduplication",
"database_acceptance": {
"status": "passed",
"at": "2026-11-18 05:15:27.565655+00:00",
"writer_commit": "12f95cff91813a2143bc8fab5bc3e89df1bd0fdb",
"before_counts": {
"papers": {
"rows": 3568022,
"max_id": 3568595
},
"source_records": {
"rows": 3815503,
"max_id": 3815504
},
"submissions": {
"rows": 532192,
"max_id": 532193
},
"decisions": {
"rows": 571744,
"max_id": 571744
}
},
"after_counts": {
"papers": {
"rows": 3568022,
"max_id": 3568595
},
"source_records": {
"rows": 3815513,
"max_id": 3815514
},
"submissions": {
"rows": 532202,
"max_id": 532203
},
"decisions": {
"rows": 571754,
"max_id": 571754
}
},
"new_papers": 0,
"new_sources": 1,
"new_editions": 1,
"new_source_records": 10,
"new_attempts": 10,
"new_decisions": 10,
"source_id": 123160,
"source_tag_evidence_links": 94,
"protected_papers": 6394,
"all_protected_old_profiles_sources_tags_identifiers_preserved": true,
"source_observations": 10,
"source_expected": 140,
"source_remaining": 130,
"coverage": "partial",
"foundry_readyz": [
200
],
"sqlite_frozen": true,
"imported": [
{
"external_id": "2026/3",
"paper_id": 2813483,
"record_id": 3815505,
"submission_id": 532194,
"decision_id": 571745,
"edition_id": 104044
},
{
"external_id": "2026/12",
"paper_id": 2785459,
"record_id": 3815506,
"submission_id": 532195,
"decision_id": 571746,
"edition_id": 104044
},
{
"external_id": "2026/35",
"paper_id": 2757859,
"record_id": 3815507,
"submission_id": 532196,
"decision_id": 571747,
"edition_id": 104044
},
{
"external_id": "2026/51",
"paper_id": 2752643,
"record_id": 3815508,
"submission_id": 532197,
"decision_id": 571748,
"edition_id": 104044
},
{
"external_id": "2026/74",
"paper_id": 2762340,
"record_id": 3815509,
"submission_id": 532198,
"decision_id": 571749,
"edition_id": 104044
},
{
"external_id": "2026/75",
"paper_id": 2757480,
"record_id": 3815510,
"submission_id": 532199,
"decision_id": 571750,
"edition_id": 104044
},
{
"external_id": "2026/78",
"paper_id": 2768260,
"record_id": 3815511,
"submission_id": 532200,
"decision_id": 571751,
"edition_id": 104044
},
{
"external_id": "2026/84",
"paper_id": 2740614,
"record_id": 3815512,
"submission_id": 532201,
"decision_id": 571752,
"edition_id": 104044
},
{
"external_id": "2026/87",
"paper_id": 2752271,
"record_id": 3815513,
"submission_id": 532202,
"decision_id": 571753,
"edition_id": 104044
},
{
"external_id": "2026/363",
"paper_id": 2757497,
"record_id": 3815514,
"submission_id": 532203,
"decision_id": 571754,
"edition_id": 104044
}
]
},
"history_protection": {
"submissions": {
"rows": 532192,
"sha256": "34e2fe27cccf89e6e80228318212a846369fdd315c4cfe67d7d535013eac8298"
},
"decisions": {
"rows": 571744,
"sha256": "efbda4e91950c5c7298f536bc5967f7fee57d8b20a7843e00dc69871b1553bd1"
},
"decision_sources": {
"rows": 571744,
"sha256": "1d8899b16b40e9f37ac84dc601efc3e7906af5c06bca49946100697f22c96174"
},
"submission_sources": {
"rows": 581976,
"sha256": "ab3ab200c4abbfef39bc5a2a757550191441c703ae505372d38d9afa9a4daf65"
},
"submission_identifiers": {
"rows": 383956,
"sha256": "0e8f514e186e08bd00a65f8e14fc7f63db6add76722271e0111e14f5d27b9dbe"
}
},
"evidence": {
"accepted_report": "/data/paper-index/collected/ismir2026-import10-v1/postgresql-postcheck-v1.json",
"accepted_report_sha256": "8a08015bbd725b400de53477837e850ca92a50479d7f86f2cfed56d50bb22577",
"baseline": "/data/paper-index/collected/ismir2026-baseline10-v1/baseline.json",
"baseline_sha256": "a0b5b8c2c21ee567cf80152b16f66643309bbe33ee9cd67d19da33d6b6337656",
"baseline_archive_sha256": "eb94f99cb53c9a30f63d34ea11b23c11759e45b571b7efce2dcbe8cb4fe52fb8",
"preparation_sha256": "0d9fad74640ab57fc8bea8aaf72ba07202a40b174d05253db7348b14455a0462",
"acceptance_archive": "/data/paper-index/validation/ismir2026-import10-acceptance-recovery-v1.tar.gz",
"acceptance_archive_sha256": "138e1fb986f5aa6a42794eed982fdc91ae994b8be82f8bf25360db2f7ba6c70c",
"fresh_packets_sha256": "88e423aede0ea83b9f736891c09b12d0cb24f2f8b70508e4863cd07760de5e0d",
"local_acceptance_check_sha256": "42d31337c76b96f93505011db7c797123b67ce2eec7defd0175ea0e9c1df2089",
"census_summary_sha256": "2157035b72d351e2fe91f2bc0c6ebb305f91d95f8772ab9a088076ce4b86e1c2"
},
"native_query_example": {
"paper_id": 2757497,
"publication_outcome": "accepted",
"prior_venue": "ICLR",
"prior_year": 2026,
"prior_submission_id": 131721,
"prior_negative_attempts": 1,
"prior_negative_observations": 2,
"prior_decision_ids": [
133086,
538172
],
"later_observed_venue": "ISMIR",
"later_observed_year": 2026,
"original_ismir_signed_decision_verified": false,
"ismir_decision_date": null,
"observation_time_is_not_decision_time": true
},
"presence_census": {
"snapshot_at": "2026-11-18 05:20:48.641315+00",
"scope": "Fixed278 venue-year presence only, not paper/track/abstract/public-negative completeness or final deduplication. Legacy43 gaps deferred, not filled.",
"target_venues": 43,
"expected_slots": 278,
"populated_slots": 274,
"missing_slots": 4,
"missing": [
{
"venue": "CIKM",
"year": 2026
},
{
"venue": "ECCV",
"year": 2026
},
{
"venue": "ICDM",
"year": 2026
},
{
"venue": "IROS",
"year": 2026
}
],
"delta_from_v20": {
"newly_present": [
{
"venue": "ISMIR",
"year": 2026
}
],
"previous_missing": 5,
"current_missing": 4
},
"snapshot_sha256": "c14bab883c5fcab33b450fcf240dec7225f2f17bc04c3e728b5583657b4ad23b",
"fixed_scope_sha256": "a835bef5b9c3a0f76069d840cecdb9a6bcf65d3a38796d70dbfdfd4b49242264",
"accepted_import_sha256": "8a08015bbd725b400de53477837e850ca92a50479d7f86f2cfed56d50bb22577",
"database_written": false
},
"consumed_queue": "/data/paper-index/collected/ismir2026-prepared10-v1: NEVER REPLAY",
"next": [
"Review130 remaining source observations, including31 total verbatim-abstract candidates of which10 consumed; do not count a title/author screen as identity approval.",
"Fresh incremental parent must be partial10/140; old preparation0/140 is consumed.",
"Continue all venue/year/track public negative availability and formal source coverage.",
"Finish cross-venue identities/types, then final whole-DB dedup/identifier/abstract/constraints/query audit after imports."
]
}
@@ -0,0 +1,330 @@
{
"as_of": "2026-11-18",
"goal_progress": "13/17; stage14 incomplete",
"source": "ismir_program",
"scope": "21 additional reviewed existing-work source links; cumulativepartial31/140, not public-rejection completeness or final deduplication",
"database_acceptance": {
"status": "passed",
"at": "2026-11-18 06:38:55.148539+00:00",
"writer_commit": "3754545003e1f96b88c6533cc3a69dc557f07305",
"before_counts": {
"papers": {
"rows": 3568022,
"max_id": 3568595
},
"source_records": {
"rows": 3815513,
"max_id": 3815514
},
"submissions": {
"rows": 532202,
"max_id": 532203
},
"decisions": {
"rows": 571754,
"max_id": 571754
}
},
"after_counts": {
"papers": {
"rows": 3568022,
"max_id": 3568595
},
"source_records": {
"rows": 3815534,
"max_id": 3815535
},
"submissions": {
"rows": 532223,
"max_id": 532224
},
"decisions": {
"rows": 571775,
"max_id": 571775
}
},
"new_papers": 0,
"new_sources": 0,
"new_editions": 0,
"new_source_records": 21,
"new_attempts": 21,
"new_decisions": 21,
"source_id": 123160,
"source_tag_evidence_links": 173,
"protected_papers": 6415,
"all_protected_old_profiles_sources_tags_identifiers_preserved": true,
"source_observations": 31,
"source_expected": 140,
"source_remaining": 109,
"coverage": "partial",
"foundry_readyz": [
200
],
"sqlite_frozen": true,
"imported": [
{
"external_id": "2026/148",
"paper_id": 2755057,
"record_id": 3815515,
"submission_id": 532204,
"decision_id": 571755,
"edition_id": 104044
},
{
"external_id": "2026/184",
"paper_id": 2752723,
"record_id": 3815516,
"submission_id": 532205,
"decision_id": 571756,
"edition_id": 104044
},
{
"external_id": "2026/196",
"paper_id": 2784571,
"record_id": 3815517,
"submission_id": 532206,
"decision_id": 571757,
"edition_id": 104044
},
{
"external_id": "2026/240",
"paper_id": 2738896,
"record_id": 3815518,
"submission_id": 532207,
"decision_id": 571758,
"edition_id": 104044
},
{
"external_id": "2026/251",
"paper_id": 2753889,
"record_id": 3815519,
"submission_id": 532208,
"decision_id": 571759,
"edition_id": 104044
},
{
"external_id": "2026/258",
"paper_id": 2662030,
"record_id": 3815520,
"submission_id": 532209,
"decision_id": 571760,
"edition_id": 104044
},
{
"external_id": "2026/270",
"paper_id": 2769109,
"record_id": 3815521,
"submission_id": 532210,
"decision_id": 571761,
"edition_id": 104044
},
{
"external_id": "2026/298",
"paper_id": 2752114,
"record_id": 3815522,
"submission_id": 532211,
"decision_id": 571762,
"edition_id": 104044
},
{
"external_id": "2026/317",
"paper_id": 2765378,
"record_id": 3815523,
"submission_id": 532212,
"decision_id": 571763,
"edition_id": 104044
},
{
"external_id": "2026/318",
"paper_id": 2764935,
"record_id": 3815524,
"submission_id": 532213,
"decision_id": 571764,
"edition_id": 104044
},
{
"external_id": "2026/342",
"paper_id": 2738585,
"record_id": 3815525,
"submission_id": 532214,
"decision_id": 571765,
"edition_id": 104044
},
{
"external_id": "2026/370",
"paper_id": 2830871,
"record_id": 3815526,
"submission_id": 532215,
"decision_id": 571766,
"edition_id": 104044
},
{
"external_id": "2026/371",
"paper_id": 2858445,
"record_id": 3815527,
"submission_id": 532216,
"decision_id": 571767,
"edition_id": 104044
},
{
"external_id": "2026/373",
"paper_id": 2820562,
"record_id": 3815528,
"submission_id": 532217,
"decision_id": 571768,
"edition_id": 104044
},
{
"external_id": "2026/409",
"paper_id": 2522863,
"record_id": 3815529,
"submission_id": 532218,
"decision_id": 571769,
"edition_id": 104044
},
{
"external_id": "2026/418",
"paper_id": 2750749,
"record_id": 3815530,
"submission_id": 532219,
"decision_id": 571770,
"edition_id": 104044
},
{
"external_id": "2026/468",
"paper_id": 2846824,
"record_id": 3815531,
"submission_id": 532220,
"decision_id": 571771,
"edition_id": 104044
},
{
"external_id": "2026/518",
"paper_id": 2761686,
"record_id": 3815532,
"submission_id": 532221,
"decision_id": 571772,
"edition_id": 104044
},
{
"external_id": "2026/524",
"paper_id": 2784543,
"record_id": 3815533,
"submission_id": 532222,
"decision_id": 571773,
"edition_id": 104044
},
{
"external_id": "2026/539",
"paper_id": 2761760,
"record_id": 3815534,
"submission_id": 532223,
"decision_id": 571774,
"edition_id": 104044
},
{
"external_id": "2026/552",
"paper_id": 2758575,
"record_id": 3815535,
"submission_id": 532224,
"decision_id": 571775,
"edition_id": 104044
}
]
},
"history_protection": {
"submissions": {
"rows": 532202,
"sha256": "9d41955c80fc91dae4f65d65e0f69198ce2d79508ceebfe943bdc19825983df8"
},
"decisions": {
"rows": 571754,
"sha256": "c8dc762d089d1656ee1308dacec9109605e1a4be7e5fcfa8f95013e77a0cb55c"
},
"decision_sources": {
"rows": 571754,
"sha256": "231f15b4c70435977d6ab784112ff5c748541eeb4d54c28bb861f8835f1eba10"
},
"submission_sources": {
"rows": 581986,
"sha256": "b9d8c39e945df340daf1c695a6359cfb549b8bd2f391cb139dabba2c16afa242"
},
"submission_identifiers": {
"rows": 383956,
"sha256": "0e8f514e186e08bd00a65f8e14fc7f63db6add76722271e0111e14f5d27b9dbe"
}
},
"evidence": {
"parent_acceptance_sha256": "8a08015bbd725b400de53477837e850ca92a50479d7f86f2cfed56d50bb22577",
"accepted_report": "/data/paper-index/collected/ismir2026-import21-v1/postgresql-postcheck-v1.json",
"accepted_report_sha256": "153ef13dbac9f4243c81bdd2a16d3ade74372d3e45ca5e84acc2b7258bca7bfc",
"baseline": "/data/paper-index/collected/ismir2026-baseline21-v1/baseline.json",
"baseline_sha256": "204b571b47e164d68930158c2578f609814969940628d421a2b64c3a6ce1b018",
"baseline_archive_sha256": "ecac32e1d3890786a5670ff06f88fa7bd92155306cf8133f3164de0cb9f48e6b",
"preparation_sha256": "6781edfaa67798d76329fb5c7948d83693607c96352a38eccce0067faa95deaa",
"acceptance_archive": "/data/paper-index/validation/ismir2026-import21-acceptance-recovery-v1.tar.gz",
"acceptance_archive_sha256": "55e1cc8301548ff0abc195add5f40d7082a06e4f087f6da5838ffd2f6b33339c",
"fresh_packets_sha256": "79e0e8187a522478c443d036fdbf2f8d5e6a22c949d46fb0592a664ec1bc22fe",
"local_acceptance_check_sha256": "04f4ec6969a71655d70f446347646f63efc56e5a5a8e974b931703482e5e7ceb",
"census_summary_sha256": "ee4ec2e6d6eaed55ce3475e735bfba9bd94e512893cd3c3024035b48b744f475",
"ci_run": 1257,
"ci_tests": 556,
"ci_proof_sha256": "5ad08aa907caec5a1ca63df5e7345db4802dab4e3a91293a0303bf481a903ff8"
},
"consumed_queue": "/data/paper-index/collected/ismir2026-prepared21-v2: NEVER REPLAY",
"previous_consumed_queue": "/data/paper-index/collected/ismir2026-prepared10-v1: NEVER REPLAY",
"query_example_rechecked": {
"paper_id": 2757497,
"publication_outcome": "accepted",
"rejected_attempts": 1,
"rejection_source_observations": 2,
"distinct_ismir_accepted_attempts": 1,
"original_ismir_signed_decision_verified": false,
"ismir_decision_date": null
},
"new_batch_prior_negative_attempts": 0,
"new_batch_prior_negative_observations": 0,
"source_tag_links_cumulative": 267,
"presence_census": {
"snapshot_at": "2026-11-18 06:46:44.099675+00",
"scope": "Fixed278 venue-year presence only, not paper/track/abstract/public-negative completeness or final deduplication. Legacy43 gaps deferred, not filled.",
"target_venues": 43,
"expected_slots": 278,
"populated_slots": 274,
"missing_slots": 4,
"missing": [
{
"venue": "CIKM",
"year": 2026
},
{
"venue": "ECCV",
"year": 2026
},
{
"venue": "ICDM",
"year": 2026
},
{
"venue": "IROS",
"year": 2026
}
],
"delta_from_v21": {
"newly_present": [],
"previous_missing": 4,
"current_missing": 4,
"ISMIR_source_observations": 31
},
"snapshot_sha256": "5aabc91217ad787946ba50d6a670296b1e3152a54f5ecaa0e3a9b3a952b63fa7",
"fixed_scope_sha256": "a835bef5b9c3a0f76069d840cecdb9a6bcf65d3a38796d70dbfdfd4b49242264",
"accepted_import_sha256": "153ef13dbac9f4243c81bdd2a16d3ade74372d3e45ca5e84acc2b7258bca7bfc",
"database_written": false
},
"next": [
"Remaining109 observations require review;31 exact-source-abstract candidates are now consumed, not reusable queues.",
"Future increments require a fresh partial31/140 parent and new preservation baseline.",
"Keep raw authors/title/abstract/keyword versions; no numeric miniconf-to-OpenReview-ID assumption, global person aliases or inferred research types.",
"Continue all43 venue/year/track formal-source and public-negative availability audits, then final whole-DB dedup after imports."
]
}
@@ -0,0 +1,320 @@
{
"version": 1,
"as_of": "2026-11-18",
"scope": "930 cached ResearchArcade non-ICLR negative claims, NeurIPS 2021–2025 and EMNLP 2023; not a conference-completeness audit",
"database_snapshot": {
"database": "foundry_data",
"schema": "research",
"at": "2026-11-18 03:48:34.65659+00",
"snapshot": "1048885:1048885:"
},
"database_written": false,
"approved_for_import": 0,
"claims": 930,
"known_same_scope_attempt_claims": 925,
"all_925_have_rejected_status": true,
"without_same_scope_attempt_claims": 5,
"by_venue_year": [
{
"venue": "EMNLP",
"year": 2023,
"claims": 10
},
{
"venue": "NeurIPS",
"year": 2021,
"claims": 136
},
{
"venue": "NeurIPS",
"year": 2022,
"claims": 153
},
{
"venue": "NeurIPS",
"year": 2023,
"claims": 176
},
{
"venue": "NeurIPS",
"year": 2024,
"claims": 201
},
{
"venue": "NeurIPS",
"year": 2025,
"claims": 254
}
],
"evidence": {
"diff_report_sha256": "d49361a8622f3b3f59e824690ceb8bf1c878be35c3e3758366d3b46a407673f5",
"candidate_packets_sha256": "3b21df27e0f30e56b81086d16d3943c4563bbe382200e866b15de14510e07472",
"dataset_revision": "c61bb99b0452429c32f85b1e392f3afdb65731c4",
"dataset_parquet_sha256": "bb6e165ee152a95d91a7ffa16abc9d4a6effbdfcf59183b3bb819d09171689d1",
"candidate_jsonl_sha256": "848e646596e8c747e0c636c2381a139978aff8f2c19f01ad49c98e7f4270c5e6",
"collector_repository": "https://github.com/ulab-uiuc/research-arcade",
"collector_revision": "ecf9c7c2490840b029bd2add4f02d04aca1d014e",
"collector_openreview_crawler_sha256": "4539d951ca05b20785169638eab3884148aa6c354a34c5ecc04e15c84a5910fe",
"collector_revision_proven_to_generate_dataset": false
},
"collector_semantics": {
"legacy_2020_2022": "Reads Blind_Submission note.id; first reply with content.decision, no invitation/signature verification. Not a verified final-decision note.",
"legacy_2019": "Reads first recommendation, which must not be promoted to a final decision. Outside active 2020+ backfill.",
"modern_branch": "Reads Submission note.id and last venueid component; Conference replaced by venue literal. Withdrawn_Submission skipped.",
"identity": "The exported field paper_openreview_id is note.id; forum equality or equivalence to another mirror ID is not independently proved.",
"coverage": "Six-field table has no authors or decision signatures; no completeness inference for public withdrawn/rejected records."
},
"held_claims": [
{
"source_row": 33219,
"note_id_claim": "FSf9h6t152",
"venue": "NeurIPS",
"year": 2025,
"raw_decision": "Rejected_Submission",
"title": "Hierarchical Restructuring of Graph Neural Networks for Robust Learning under Edge Perturbations",
"same_scope_identifier_match": false,
"exact_title_candidates": [],
"approved_for_import": false,
"disposition": "hold_no_exact_title_candidate_not_proof_of_novelty"
},
{
"source_row": 44669,
"note_id_claim": "Pk972F7J8Um",
"venue": "NeurIPS",
"year": 2021,
"raw_decision": "Reject",
"title": "Differential Privacy of Dirichlet Posterior Sampling",
"same_scope_identifier_match": false,
"exact_title_candidates": [
{
"paper_id": 3527429,
"existing_identifiers": [
{
"scheme": "openreview",
"value": "fgBxcKcz9ea",
"paper_id": 3527429
},
{
"scheme": "papercopilot_openreview",
"value": "nips/2021/fgBxcKcz9ea",
"paper_id": 3527429
}
],
"source_abstract_exact_equal": true,
"existing_attempts": [
{
"id": 444666,
"venue": "NeurIPS",
"year": 2021,
"source_submission_id": "nips/2021/fgBxcKcz9ea",
"statuses": [
"rejected"
]
}
],
"attempts_truncated": false,
"identity_approved": false,
"id_alias_approved": false
}
],
"approved_for_import": false,
"disposition": "hold_distinct_note_ids_even_with_equal_title_and_abstract"
},
{
"source_row": 44696,
"note_id_claim": "ki4eJ1fSJNq",
"venue": "NeurIPS",
"year": 2021,
"raw_decision": "Reject",
"title": "Fitting large mixture models using stochastic component selection",
"same_scope_identifier_match": false,
"exact_title_candidates": [
{
"paper_id": 1545058,
"existing_identifiers": [
{
"scheme": "arxiv",
"value": "2110.04776",
"paper_id": 1545058
},
{
"scheme": "openreview",
"value": "ZS394D3djsg",
"paper_id": 1545058
},
{
"scheme": "papercopilot_openreview",
"value": "nips/2021/ZS394D3djsg",
"paper_id": 1545058
}
],
"source_abstract_exact_equal": true,
"existing_attempts": [
{
"id": 444652,
"venue": "NeurIPS",
"year": 2021,
"source_submission_id": "nips/2021/ZS394D3djsg",
"statuses": [
"rejected"
]
}
],
"attempts_truncated": false,
"identity_approved": false,
"id_alias_approved": false
}
],
"approved_for_import": false,
"disposition": "hold_distinct_note_ids_even_with_equal_title_and_abstract"
},
{
"source_row": 45757,
"note_id_claim": "bDP7I2fYwLj",
"venue": "NeurIPS",
"year": 2021,
"raw_decision": "Reject",
"title": "WeaveNet for Approximating Assignment Problems",
"same_scope_identifier_match": false,
"exact_title_candidates": [
{
"paper_id": 3527390,
"existing_identifiers": [
{
"scheme": "openreview",
"value": "2rAiDBJgR_",
"paper_id": 3527390
},
{
"scheme": "papercopilot_openreview",
"value": "nips/2021/2rAiDBJgR_",
"paper_id": 3527390
}
],
"source_abstract_exact_equal": true,
"existing_attempts": [
{
"id": 444580,
"venue": "NeurIPS",
"year": 2021,
"source_submission_id": "nips/2021/2rAiDBJgR_",
"statuses": [
"rejected"
]
}
],
"attempts_truncated": false,
"identity_approved": false,
"id_alias_approved": false
}
],
"approved_for_import": false,
"disposition": "hold_distinct_note_ids_even_with_equal_title_and_abstract"
},
{
"source_row": 49327,
"note_id_claim": "yB8cQIICqe",
"venue": "EMNLP",
"year": 2023,
"raw_decision": "Desk_Rejected_Submission",
"title": "EZ-STANCE: A Large Dataset for Zero-Shot Stance Detection",
"same_scope_identifier_match": false,
"exact_title_candidates": [],
"approved_for_import": false,
"disposition": "hold_possible_later_publication_with_changed_title_and_dataset_size_no_identity_or_author_transfer",
"later_publication_retrieval_lead": {
"url": "https://aclanthology.org/2024.acl-long.838/",
"http_status": 200,
"bytes": 40557,
"response_sha256": "b50dd55c48e31e4c10e97927ce01679369735834aa2c0f8d9f41f8e8e02fc776",
"checked_at": "2026-11-18T04:07:03.699767+00:00",
"anthology_id": "2024.acl-long.838",
"doi": "10.18653/v1/2024.acl-long.838",
"title": "EZ-STANCE: A Large Dataset for English Zero-Shot Stance Detection",
"venue": "ACL",
"year": 2024,
"publication_verified": true,
"link_to_anonymous_rejection_verified": false,
"authors_transferred_to_anonymous_claim": false,
"dataset_pairs_published_version": 47316,
"dataset_pairs_rejection_mirror_version": 30606,
"database_written": false,
"current_database_evidence": {
"snapshot": {
"at": "2026-11-18 04:09:23.720956+00",
"snapshot": "1051887:1051887:"
},
"paper_id": 2131805,
"verified_identifiers": [
{
"scheme": "acl",
"value": "2024.acl-long.838",
"paper_id": 2131805
},
{
"scheme": "acl_anthology",
"value": "2024.acl-long.838",
"paper_id": 2131805
},
{
"scheme": "doi",
"value": "10.18653/v1/2024.acl-long.838",
"paper_id": 2131805
}
],
"existing_attempts": [
{
"venue": "ACL",
"year": 2024,
"track": "main",
"statuses": [
"accepted"
]
}
],
"packet_file_sha256": "de5dfa658177adef87134cf7723afc086bd7395405a31166ff9597b03b3c8af3",
"original_rejection_link_verified": false
}
}
}
],
"access_probes": [
{
"local_evidence": "researcharcade-neurips2021-original-note-check-v1/report.json",
"sha256": "746d752535b516abba43b0e8e6ad901bf39c07da532188dfa97ff911869084ae",
"receipt_summary": {},
"results": [
{
"requested_note_id": "Pk972F7J8Um",
"status": 403,
"url": "https://api.openreview.net/notes?id=Pk972F7J8Um&select=id%2Cforum%2Cinvitation%2Creaders%2Csignatures%2Ccontent.title%2Ccontent.abstract%2Ccontent.venue%2Ccontent.venueid",
"sha256": "b0113384cebc9d724284062f776b45044d22450219112342721d8f9683571c08",
"checked_at": "2026-11-18T04:01:58.643552+00:00"
}
]
},
{
"local_evidence": "emnlp2023-primary-public-metadata-probe-v1/notes-probe.json",
"sha256": "47799dc4520e113d6a2e7c9ea2f362577bb21e3ad583ee02149a36c21cf6ea9d",
"receipt_summary": {
"url": "https://api.openreview.net/notes?invitation=EMNLP%2F2023%2FConference%2F-%2FSubmission&limit=1&select=id%2Cnumber%2Cforum%2Cinvitation%2Creaders%2Csignatures%2Ccdate%2Ctcdate%2Cmdate%2Ctmdate%2Ccontent.title%2Ccontent.abstract%2Ccontent.authors%2Ccontent.venue%2Ccontent.venueid%2Ccontent.keywords",
"http_status": 403,
"response_sha256": "798e964735eb1eff1e4954d5c9e79aad2afd40c3f88dcb82f8ef5bbc55cc0a61",
"response_bytes": 546,
"at": "2026-11-18T03:38:50.112624+00:00"
}
},
{
"local_evidence": "emnlp2023-primary-public-metadata-probe-v1/group-access.json",
"sha256": "8fcb9ccd5615a737c17ef0eeedc9121e85a84ebd63667e83df0bda9ffb83d792",
"receipt_summary": {
"url": "https://api.openreview.net/groups?id=EMNLP%2F2023%2FConference",
"http_status": 200,
"response_sha256": "7c0cc75e2e8be1c18dffc96a9ceabea34f0f7810139c0a990a6bb117d66f6456",
"response_bytes": 13143,
"at": "2026-11-18T03:38:49.929085+00:00"
}
}
],
"access_limit": "NeurIPS 2021 first original note request and EMNLP 2023 source-declared Submission query returned 403. Requests stopped. No alternate API version, endpoint, authentication, or challenge bypass. 403 does not prove zero public records."
}
@@ -0,0 +1,50 @@
# 公开负面来源核查:2026-11-18
本次检查 ResearchArcade 缓存中 NeurIPS 2021–2025、EMNLP 2023 的 930 条负面声明,不包括 ICLR 或 2020 年以前的补漏。原始候选文件未重新下载,PostgreSQL 查询为只读。逐条待办、来源哈希与数据库快照见[机器可读记录](public-negative-source-audit-2026-11-18.json)。没有新增论文、决定、ID 别名或合并。
## 当前库比对
925 条声明通过同会议、同年份的 OpenReview 标识匹配到投稿,且每条都有 rejected 决定。这里只证明对应拒稿已记录,未核验这些论文的所有后续录用历史,也未证明会议公开拒稿全集完整。
另 5 条没有同届标识匹配:
|来源 note.id 声明|会议|标题或识别信息|处置|
|---|---|---|---|
|FSf9h6t152|NeurIPS 2025|Hierarchical Restructuring of Graph Neural Networks for Robust Learning under Edge Perturbations|无精确标题候选;不是已证明的新论文|
|Pk972F7J8Um|NeurIPS 2021|Differential Privacy of Dirichlet Posterior Sampling|候选 paper 3527429,已有同届 rejected,原 ID fgBxcKcz9ea|
|ki4eJ1fSJNq|NeurIPS 2021|Fitting large mixture models using stochastic component selection|候选 paper 1545058,已有同届 rejected,原 ID ZS394D3djsg;另保留 arXiv 2110.04776|
|bDP7I2fYwLj|NeurIPS 2021|WeaveNet for Approximating Assignment Problems|候选 paper 3527390,已有同届 rejected,原 ID 2rAiDBJgR_|
|yB8cQIICqe|EMNLP 2023|EZ-STANCE: A Large Dataset for Zero-Shot Stance Detection|镜像称 Desk_Rejected_Submission;原始公开范围及身份未核验|
三个标题候选的镜像摘要都与已有 PaperCopilot 来源摘要逐字相等,但来源 ID 不同。相等文本不能独立证明两个 note.id 指向同一个 forum,也不能据此创建第二次投稿或批准 ID 别名。现有实体包没有截断投稿列表,均只有已记录的 NeurIPS 2021 拒稿;这不代表从未在其他地方录用。
## 采集器证据与使用限制
作者仓库 [research-arcade](https://github.com/ulab-uiuc/research-arcade) 固定版本 `ecf9c7c2490840b029bd2add4f02d04aca1d014e` 的 `openreview_crawler.py` 显示:
- 2020–2022 的旧接口路径读取 Blind_Submission 的 `note.id`,从 replies 中取第一个带 `decision` 字段的值,未验证决定 invitation 或签名。
- 2019 路径取第一个 `recommendation`,不能将其升级为正式决定。该年度属于保留线索的范围,本轮不补导。
- 新接口路径读取 Submission 的 `note.id`,用 `venueid` 尾段作为标签;遇 Conference 再取 `venue`,并跳过 Withdrawn_Submission。
三个源码文件均核对 Git blob SHA-1 和完整文件 SHA-256;没有执行第三方采集器。源码版本早于数据集版本,但未证明该提交实际生成了此数据集。数据集仅有六个元数据字段,缺少作者和决定签名,因此既不能证明完整 withdrawn 覆盖,也不能将导出的 ID 无条件认作已验证 forum。
## 原始来源访问结果
EMNLP 2023 官方分组元数据返回 200,其声明的 Submission invitation 与既有配置一致;对该 invitation 的匿名元数据查询返回 403。NeurIPS 2021 对第一条候选 note `Pk972F7J8Um` 的匿名元数据查询同样返回 403,余下五个 ID 未发请求。均已停止对应来源查询,没有换接口版本、登录或绕过访问限制。403 不能说明没有公开投稿;未保留完整错误正文,也不将具体拒绝原因猜测为已知。
[EMNLP 2023 官方政策](https://2023.emnlp.org/blog/EMNLP-2023-openreview/)说明公开录用稿件与选择公开的拒稿,现有 9 条 rejected 或本镜像的 1 条 desk-rejected 均不能证明完整公开集合。
## EZ-STANCE 后续发表线索
独立的 [ACL Anthology 官方条目](https://aclanthology.org/2024.acl-long.838/)确认相近标题的 EZ-STANCE 在 ACL 2024 Long Papers 发表,DOI `10.18653/v1/2024.acl-long.838`。正式标题多了 English,数据集由拒稿镜像中的 30,606 对变为 47,316 对。已保存官方 HTML 的完整哈希,登记为版本关系核查线索;尚未证明与匿名 EMNLP 2023 声明属于同一投稿或同一科学工作,没有把已发表作者转填到匿名记录,也没有写入关联或第二份论文。后续必须核验版本关系并保留各自的投稿决定证据。
当前 PostgreSQL 的正式标题变体查询命中已有 paper `2131805`,其 ACL/DOI 标识与官方条目一致,已有 `ACL / 2024 / main / accepted` 投稿历史。该实体无需重复新建。匿名 EMNLP 声明与该实体的关系仍未批准,不能把这条线索表述为已经确认的“先拒后录”。快照、标识及完整实体包哈希已记入机器记录。
## 下次从哪里继续
1. 保留五条候选及旧 ID,等待可验证的原始公开元数据或独立可靠来源;不要反复重放已拒绝的原始接口请求。
2. 对三对 ID 分歧先取得 note/forum 关系证据,再独立审核科学工作身份与逐次投稿关系;保留已有拒稿及以后发现的录用。
3. 两条没有精确标题候选的记录仍需标题变体、作者及摘要交叉核查;匿名元数据不能用于推断作者身份。
4. 继续其余会议与年度的公开负面可用性核查,不将这 930 条当作全会、全年份覆盖,也不将 accepted-only 镜像算作拒稿补全。
另修正交接记录:TMLR 是额外的期刊政策线索,当前 43 个会议的 registry 中没有 TMLR;没有因此更改会议分母。2000–2019 已发现线索仍保留。
@@ -0,0 +1,51 @@
# UAI 公开负面决定来源:2026-11-17 核查
**公开拒稿缺口仍未补齐,总体仍为 13/17。** 本页保存新来源的实际范围与恢复线索,避免下次把录用子集误当作全部投稿。主动补漏范围仍从2020年开始;本次镜像只含2022–2025,不能替代其他年份或其他会议的核查。
## 政策与访问边界
已有逐年核查记录保存在 [覆盖清单](inventory-2026-11-08.json) 的 UAI `policy_checks`:2023–2026 官方说明允许决定后公开录用及作者选择公开的拒稿,默认私密拒稿不属于公开可采集范围。政策并不证明任何镜像完整。
- [2023 投稿说明](https://www.auai.org/uai2023/submission_instructions)
- [2024 投稿说明](https://auai.org/uai2024/submission_instructions)
- [2025 投稿说明](https://www.auai.org/uai2025/submission_instructions)
- [2026 投稿说明](https://www.auai.org/uai2026/submission_instructions)
先前匿名 notes 请求遇到 ChallengeRequiredError/403,仍作为访问限制保留。本轮没有认证、重试或绕过这些入口;读取的是整理者公开发布的固定版本镜像。复制的 `readers`、`signatures` 和邀请字段仍只是镜像内的来源断言,不是对当前原始接口可见性或签名真实性的验证。
## 实际镜像范围
来源:[latkes/ai-research-index-iclr-openreview](https://huggingface.co/datasets/latkes/ai-research-index-iclr-openreview/tree/4c9e09931a7814d7500630ffd064b74f1c01bcf3),固定提交 `4c9e09931a7814d7500630ffd064b74f1c01bcf3`。目录中存在 `papers/uai-YYYY/papers.jsonl` 与四个 `raw/uaiYYYY.jsonl`,不能仅凭数据集说明宣称拒稿覆盖。
| 年份 | 整理版及原始论坛数 | 原始回复对象 | 实际回复角色 | 标准结构化决定字段 |
| --- | ---: | ---: | --- | --- |
| 2022 | 230 | 1059 | Official_Review 829,Decision 230 | Accept (Poster) 194,Accept (Oral) 36 |
| 2023 | 243 | 0 | 没有保存回复 | 没有保存决定记录 |
| 2024 | 200 | 1994 | Official_Review 905,Rebuttal 889,Meta_Review 200 | 没有单独 Decision 记录或结构化 decision 字段 |
| 2025 | 229 | 0 | 没有保存回复 | 没有保存决定记录 |
整理版902条均被整理者标为 `accepted=true`;原始根记录的 `venue` 为相应会议名称或 oral/poster/spotlight 文本,不能替代逐次决定证据,更不能算作902条已认证的原始录用决定。这不是四届全部投稿或全部录用的完整性证明。2024年还有 `Edit` 邀请,属于同一批回复上的另一个邀请,不能再累加成1994个新回复。
为排除字段漏读,对含回复的2022、2024快照另做了全部3053个回复的字段名、包装结构、邀请角色和短决定类字段核查;重新读取的正文哈希与首次捕获完全一致。2022的230条明确字段是录用标签。2024仅保存评审、作者回复和元评审,没有可据此新增的结构化拒稿/撤稿决定。没有从元评审文本、评分、推荐或缺少录用推断负面决定,也没有保存评审正文、作者回复正文或论文全文。
**这只描述上述固定镜像子集,不证明UAI没有公开拒稿。** 本批没有生成负面决定导入队列,也未写数据库。2020、2021、2026及其他可用来源仍需分别核查;某文件未出现在这个镜像目录中不代表全网不存在。
## 产物与恢复
服务器目录均相对 `/data/paper-index/collected/`:
- `uai2022-2025-mirror-metadata-v1`:902条整理版元数据投影、字段盘点与标签计数。
- `uai2022-2025-raw-decisions-v1`:902个原始论坛的元数据、明确决定字段及来源正文/逐行哈希。
- `uai2022-2024-reply-schema-v2`:3053个回复的元数据、字段名与短标签;评审正文不保存。角色核查包含直接记录和 `note` 包装,未发生静默遗漏。
源响应正文仅在内存中读取、校验,保存元数据投影与固定来源URL/字节哈希;不是保留了含评审文本的完整原始快照。复核原始源字节时应使用固定提交,并区分投影文件哈希与源正文哈希。恢复包已在本机独立核验全部902个论坛、3053个回复的元数据结构、逐文件哈希及来源关联。
| 恢复证据 | SHA256 |
| --- | --- |
| 整理版元数据恢复包 | 55889fb2b9005260cef1001b2cf36e203f3b7e1deeec0291f058b52eeae880e7 |
| 原始元数据/回复结构恢复包 | caade7df899dd3e65572d06e2bf2326fad692cc5e1c1e4ab002fdc6d7823743f |
| 原始结构独立核验报告 | 5f0ef2948d0db68fe0cd89eb4c8b57a1088b94b53bed98f180d2b96629746e72 |
服务器恢复包分别为 `/data/paper-index/validation/uai2022-2025-mirror-metadata-recovery-v1.tar.gz` 与 `uai2022-2025-raw-reply-recovery-v1.tar.gz`。本地同名备份位于 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`。先检查实际单元、完整manifest和哈希,再恢复;不要因进度文件旧状态或日志安静重跑。
后续继续寻找能明确覆盖作者公开拒稿选择的来源,保留每次投稿及决定原文标签、原始ID与镜像关系,核对跨来源重叠和后续录用。UAI缺口不关闭,全部43个目标的逐年公开负面决定核查也不缩减;最终全库去重仍为独立必做阶段。
+16
View File
@@ -0,0 +1,16 @@
# ECCV 2026 正式出版章节采集
`collect eccv2026-publications` 读取已归档的 Springer LNCS 17001–17078 目录及所选章节的公开 HTML,输出可复核的 Paper Atlas 记录。目录覆盖 2,835 个章节 DOI。会议节目单与出版方卷册简介均为 2,834 条,逐条差异仍待核。Workshop 论文集另行处理。
```sh
paper-atlas collect eccv2026-publications \
--toc-snapshots /data/paper-index/collected/eccv2026-springer-toc-v1 \
--chapter-snapshots /data/paper-index/validation/eccv-verified-chapter-pages \
--out /data/paper-index/collected/eccv-publisher-metadata
```
目录归档含 `receipts.jsonl`、`volume-checkpoint.json`、`summary.json` 及 156 页 HTML。摘要在章节详情页读取,章节归档沿用 `springer-selected` 的 `receipts.json` 格式。两类归档都按响应大小和 SHA-256 校验。目录卷号、分页、章节 DOI 归属、标题及 2026 年出处需要一致。已有输出目录需另查回执或换新空目录后运行。
输出含规范化 JSONL、manifest、采集回执以及所选章节的内容寻址存证。每条保留出版 DOI、卷号、页码、按顺序的完整作者及公开摘要。目录证据用 URL 和哈希连接到输入归档,保存归档以便复核。主会出版观察依据 Springer 正式论文集,原始投稿号、决定日期及签名仍未知。工具仅处理已归档的公开 HTML。
生成的文件待做论文身份复核后方可导入。准备生产批次时核实每个 DOI 与现有 arXiv 标识的绑定,保持两者的来源证据。对已存在的同届节目投稿需先核对投稿身份及轨道,再附加出版来源,避免因直接导入创建重复投稿。具有 ICLR 等既往决定的论文须保留原记录。新的增量导入 manifest 需以当前实际来源覆盖为父版本,留存待核差集和独立验收回执。
+10
View File
@@ -44,3 +44,13 @@ paper-atlas collect icdm2026-program \
原始来源恢复包为 `/data/paper-index/validation/remaining2026-official-leads-v1-recovery.tar.gz`,SHA-256 为 `0be8c84bb5c591e5afaaa5f95dd5334bf13a9be1671cb942e46e998f502495bb`。独立盘点为 `/data/paper-index/validation/icdm2026-independent-program-inventory-v1.json`,SHA-256 为 `e13eaea0a158dcaafdf5dfb4af7948e4793b0e98041ec87d42b52513acfdc5f6`;本机工作记录目录也保留同名备份。 原始来源恢复包为 `/data/paper-index/validation/remaining2026-official-leads-v1-recovery.tar.gz`,SHA-256 为 `0be8c84bb5c591e5afaaa5f95dd5334bf13a9be1671cb942e46e998f502495bb`。独立盘点为 `/data/paper-index/validation/icdm2026-independent-program-inventory-v1.json`,SHA-256 为 `e13eaea0a158dcaafdf5dfb4af7948e4793b0e98041ec87d42b52513acfdc5f6`;本机工作记录目录也保留同名备份。
下次继续先验证源码版本、原始证据和实际输出,再寻找可公开取得的摘要与正式出版记录,核查逐篇版本、稳定标识及旧投稿历史,准备新的只读预检和可审阅入库队列。完成日程规范化不意味着 ICDM2026 整届出版覆盖、公开负面决定普查或最终全库去重完成。 下次继续先验证源码版本、原始证据和实际输出,再寻找可公开取得的摘要与正式出版记录,核查逐篇版本、稳定标识及旧投稿历史,准备新的只读预检和可审阅入库队列。完成日程规范化不意味着 ICDM2026 整届出版覆盖、公开负面决定普查或最终全库去重完成。
## 2026-11-18 官方展示来源核查
口头日程的 324 个节目编号全部出现在官方海报排期。海报表有 343 个不同编号,其中长演讲组安排 177 项海报,短演讲组安排 166 项。19 个只见于海报表的编号作为独立来源线索登记。两份排期的覆盖范围各自保留,正式出版目录仍待核验。[官方详细日程](https://icdm2026.neu.edu.cn/DetailedSessions/list.htm) · [官方海报表](https://www3.cs.stonybrook.edu/~icdm2026/PosterwDemo/list.htm)
共同编号 DM1319 的标题在两个来源中实质不同。详细日程描述基于 FFE-FRCN 的数据平台,海报表描述 FFE-FRCN 的少样本检测方法。两份标题及来源位置均已留存,该编号的跨来源作品关联待核验。
生产只读快照确认 ICDM 2026 仍无入库届次。节目、作者和独立录用证据支持继续处理 DM1125、DM790、DM1157 三项既有作品关联。DM1125 和 DM1157 各有一次 ICLR 拒稿投稿及两条来源观察,DM790 有一次 ICLR 撤回投稿。三项均尚未导入,节目原始摘要字段缺失。另有三个候选存在作者名称顺序或名单差异,继续暂缓作品关联。[FALCON 作者记录](https://arxiv.org/abs/2608.29635) · [PPGFlowECG 作者记录](https://arxiv.org/abs/2509.19774) · [DeepSynC 大学录用说明](https://dm.cs.univie.ac.at/news/story/deepsync-paper-accepted-at-icdm-2026)
本轮仍未核实公开的 ICDM 2026 拒稿或撤回决定导出。[逐条差集、标题冲突、快照和后续入口](coverage/icdm2026-presentation-scope-2026-11-18.json)
+33
View File
@@ -48,3 +48,36 @@ paper-atlas collect ismir2026-program \
正式名单与政策恢复包:同目录 `remaining2026-listings-research-v2-recovery.tar.gz`,SHA256 为 `f5ef7c2e73d46712146e4d6711ad5ad562e16b8b55c0cd75d1041cd0b81142cb`。安全恢复三个源目录、核验文件后可重跑上述命令。 正式名单与政策恢复包:同目录 `remaining2026-listings-research-v2-recovery.tar.gz`,SHA256 为 `f5ef7c2e73d46712146e4d6711ad5ad562e16b8b55c0cd75d1041cd0b81142cb`。安全恢复三个源目录、核验文件后可重跑上述命令。
完整来源和独立 140 项清单在测试夹具中冻结。测试、构建、打包、安装及实际隔离目录 CLI 验证只在用户指定 `ces-control:host` runner 执行。生产规范化后还需独立核验全部记录,并用 PostgreSQL 只读预检检查候选作品和既往投稿;不自动合并来源差异或人物身份。 完整来源和独立 140 项清单在测试夹具中冻结。测试、构建、打包、安装及实际隔离目录 CLI 验证只在用户指定 `ces-control:host` runner 执行。生产规范化后还需独立核验全部记录,并用 PostgreSQL 只读预检检查候选作品和既往投稿;不自动合并来源差异或人物身份。
## 2026-11-18:首批 10 条验收(历史阶段)
本段保留首批历史状态;当前累计结果和增量父状态见下方最新验收段。
首批 10 项已逐篇核查研究内容,并通过当前数据库实体、作者、摘要及完整历史的只读复核,随后使用已通过远程 CI 的包内 `import records` 导入。新增论文实体 **0**,新增来源记录、投稿、录用观察各 **10**,来源标签证据 **94** 条;规范标题、作者、摘要及旧历史保留。原始来源摘要另存,不覆盖规范摘要。覆盖为 `partial10/140`,还有 **130** 项待核验。数值节目 ID 仍只作为节目定位符,未冒充 OpenReview 投稿 ID,未新增全局人名别名或推断研究类型。
[逐条验收与恢复记录](coverage/ismir2026-import10-2026-11-18.json)包含生产行 ID、保护基线、归档及证据哈希。服务器基线保护此前 6,384 篇与本批 10 篇的并集 **6,394** 篇,全库旧投稿、决定及来源关系哈希未变;独立新快照的 10 个完整实体包、来源字段、标签和标识也逐项核验。Foundry `/readyz` 返回200,17,987,751,936字节的 SQLite 备份保持只读。该验收不是最终全库科学工作去重。
例如 paper `2757497`(Learning Music Style for Piano Arrangement Through Cross-Modal Bootstrapping)保留一次 ICLR2026 拒稿投稿和两条拒稿来源观察,并新增独立 ISMIR2026 录用投稿。实际执行以下查询,返回 `publication_outcome=accepted`,同时保留原拒稿历史:
```sh
paper-atlas query --pg-service foundry-research --pg-schema research \
--paper 2757497 --outcome accepted
```
ISMIR 的 accepted 来自正式名单和节目,尚无原始签名决定或具体决定日期;不能把采集时间当作决定时间。决定的规范时间使用包内既有毫秒格式,来源回执的原始微秒时间仍保留于元数据。
**禁止重放** `/data/paper-index/collected/ismir2026-prepared10-v1` 的文件。导入单元 `paper-atlas-ismir2026-import10-v1.service`、invocation `538ce9c9324a4f59b5bb49d3094d5265` 已成功结束;该阶段建立了 `partial10/140` 父状态;后续累计结果见下方最新段,不得沿用已消费的 `0/140` 父状态。保护基线恢复包为 `/data/paper-index/validation/ismir2026-baseline10-recovery-v1.tar.gz`,验收恢复包为 `/data/paper-index/validation/ismir2026-import10-acceptance-recovery-v1.tar.gz`;本机 paper-index 工作目录有同名副本及独立检查记录。
## 2026-11-18:新增 21 条,累计 31/140
第二批21项已通过逐篇研究内容审查、新鲜只读父状态检查、保护基线及独立入库验收。新增论文实体、来源和edition均为0;新增来源记录、投稿、录用观察各21条,新增173条来源标签证据。累计31条录用来源观察、267条标签证据,覆盖仍为 `partial31/140`,**109项待核验**。三十一项摘要都保留在原始来源记录中,既有规范资料没有被覆盖。
本批明确保留不同研究工作,例如UOT-IR音乐压缩与SymphonyGen交响生成,以及AI混合音轨占比回归与BAMM真实广播检测;不因作者或主题相近合并。源关键词和展示方式继续使用各自namespace,没有推断全局人名别名、原始OpenReview投稿ID或研究类型。
[第二批逐条验收及恢复记录](coverage/ismir2026-import21-2026-11-18.json)包含21组生产行ID、源码提交、全部证据哈希和新计数。保护并核验了6,415篇作品的规范资料、旧来源与标签,以及所有旧全局投稿/决定/来源关系指纹;新快照另导出21个完整实体包逐字段核验。首批10项保留;paper2757497的实际查询仍显示accepted及原ICLR一次拒稿下的两条来源观察。
本批使用固定源码 `3754545003e1f96b88c6533cc3a69dc557f07305`,已在远程 `ces-control` CI1257通过556项测试,含真实PostgreSQL只读增量门禁回归。该修复解决预检对Mapping行对象切片的错误,未弱化覆盖校验;旧的只读准备v1失败时没有写库,修复后v2重新检查通过。默认运行时未因本批操作切换。
**禁止重放** `/data/paper-index/collected/ismir2026-prepared21-v2`。服务 `paper-atlas-ismir2026-import21-v1.service`、invocation `78077b901a274426bb161ef8d69963ad` 已成功结束。新批次必须从当前 `partial31/140` 准备,不能再用 `10/140` 或 `0/140`。保护基线恢复包为 `/data/paper-index/validation/ismir2026-baseline21-recovery-v1.tar.gz`,验收恢复包为 `/data/paper-index/validation/ismir2026-import21-acceptance-recovery-v1.tar.gz`;本机paper-index目录有同名副本与独立检查结果。
新的固定范围盘点仍为274/278个年度有数据;本批提高ISMIR来源覆盖,没有填补其他空年度,也没有关闭公开拒稿或最终全库去重任务。Foundry就绪检查200,SQLite备份继续只读。
+18
View File
@@ -0,0 +1,18 @@
# KDD 2026 官方论文目录
从 [KDD 2026 论文目录](https://kdd2026.kdd.org/papers/) 保存公开 HTML,记录其 SHA-256,在已有目录中指定新的输出文件。
```sh
paper-atlas collect kdd2026-program \
--snapshot kdd2026-papers.html \
--snapshot-sha256 <snapshot-sha256> \
--out kdd2026-programme.json
```
采集器只读取已保存的页面数据,逐项输出 DOI、官网标题、投稿周期、轨道和行哈希。完全相同的重复条目保留出现次数,冲突记录中止解析并等待人工复核。输出用于与来源和库内 DOI 核对,不能直接交给记录导入命令。
已核对的官网快照有 1,415 条目录记录和 1,408 个唯一 DOI。7 条重复记录的周期、轨道、标题和作者文本一致。February Cycle 列出 Research、Applied Data Science、Datasets and Benchmarks、AI for Sciences、Blue Sky Ideas 五轨。July Cycle 列出前三轨。官网列出的演讲、海报或扩展摘要需要出版方记录确认归档状态。[AI for Sciences 投稿及出版规则](https://kdd2026.kdd.org/ai4sciences-track-call-for-papers/)
Applied Data Science 有一条官网标注撤回并更换投稿 ID 的记录,旧链接为 `10.1145/3770855.3818478`。保留官网原文和日期字面值。原始投稿 ID 仍待公开证据核实。arXiv 当前指向 `10.1145/3770855.3828560`,这个 DOI 已属于同一份已有作品。两份来源分别记录,旧链接不作为新增作品标识。[arXiv 当前记录](https://arxiv.org/abs/2606.08923)
补分轨时按 DOI 检查库内已有投稿,先处理重复或缺少作者的记录。官网对录用材料的公开承诺适用于各轨道规定的范围,逐篇拒稿、初筛拒稿和其他撤回仍需公开决定证据。[Research](https://kdd2026.kdd.org/research-track-call-for-papers/) · [Applied Data Science](https://kdd2026.kdd.org/applied-data-science-ads-track-call-for-papers/) · [Datasets and Benchmarks](https://kdd2026.kdd.org/datasets-and-benchmarks-track-call-for-papers/) · [Blue Sky Ideas](https://kdd2026.kdd.org/kdd-2026-blue-sky-ideas-track-call-for-papers/)
+73
View File
@@ -0,0 +1,73 @@
# MM 2026 出版条目采集与恢复
`collect mm2026-publications` 是包内可复用的冻结快照规范化命令,读取完整注册目录和官方节目,不连接数据库、不请求网络、不下载PDF。总体仍为13/17,来源规范化、测试通过或节目匹配都不能替代身份批准、生产入库或最终去重。
## 使用
在已配置的服务器环境中,`--registry-snapshots` 是包含 `mm2026-container-registry-v2/` 的父目录;`--program-snapshots` 是包含 `mm2026-program-and-public-probes-v1/` 的父目录。父论文集正文使用独立固定快照,不能从不存在的文章ISBN字段猜测:
```sh
paper-atlas collect mm2026-publications \
--registry-snapshots /data/paper-index/collected \
--program-snapshots /data/paper-index/collected \
--parent-snapshot /data/paper-index/collected/mm2026-publication-metadata-v1/parent.json \
--out /data/paper-index/collected/mm2026-publications-native-v3
```
输出目录必须不存在。失败输出保留 `failed.json`,完整 `collection-report.json` 只在全部校验成功后原子生成;不要覆盖输出或因旧进度文件、日志安静重跑。先观察实际单元/报告,再选择新的恢复目录。
## 注册目录与节目分别保留
源目录基于 Crossref [官方精确过滤器](https://www.crossref.org/documentation/retrieve-metadata/rest-api/rest-api-filters/) 的 `container-title` 和出版者member320,实际捕获8页、1918个不同DOI。每页请求、时间、原始正文、字节哈希、总量与唯一DOI均核验。父DOI为 `10.1145/3767308`,题名为 *Proceedings of the 34th ACM International Conference on Multimedia*,明确事件为MM2026;文章均注册为 `proceedings-article`。
ISBN查询和父Unixref XML只返回父项,不能代表逐篇目录。宽泛 `query.container-title` 的检索总量也不能当会议篇数。当前1918是这个明确论文集过滤范围内的注册条目数,不是已经去重的科研作品数、所有MM workshop或全部28个来源track的出版全集。
每个出版条目都输出,稳定外部标识为真实DOI。身份标识仅为 `doi`,投稿侧标识仅为 `publication_doi`;不推断原始OpenReview/forum/投稿号或arXiv标识。来源名 `mm_registry_program` 明确表示派生的注册/节目材料,不能把其中所有字段都说成Crossref原文。
原始注册响应及父记录复制到 `evidence/registry/`,完整节目及四个既有403回执复制到 `evidence/program/`,不重新请求受限入口。每项元数据保留注册正文的文件/哈希、数组位置和规范JSON对象哈希;常用书目字段另作投影,完整引用、资助等原始字段仍在原始响应中,未丢弃也不在每条JSONL重复膨胀。
## 分轨、摘要与版本差异
按原始题名的 Unicode/字母数字规范化与完整作者多重集合比对,1726 项有唯一节目对应。原始题名没有候选时,再比较经过受限排版投影的题名:只移除平衡、无属性的 `i`、`scp`、`sup`、`sub` 标签,保留上下标内容,HTML 实体只解码一次。未知或不平衡标签、属性、注释及声明不参与这种投影;不使用模糊题名或人物别名。
26 项新增对应均保留完整作者多重集合一致、唯一节目 ID 以及逐项复核名单,复制节目原文的摘要、track 和展示上下文。原始注册题名、作者和节目字段继续逐字保留,比较文本单独存入 `program_title_formatting_comparison`;这只证明节目材料对应,不批准科学作品合并。原始题名已有作者分歧时,不通过排版回退绕过。其余 122 项原始作者分歧、1 项排版题名下的作者分歧、43 项题名差异,共 166 项仍在 `proceedings_unclassified`。
| 输出分区 | 注册条目 |
| --- | ---: |
| main | 1555 |
| grand_challenge | 77 |
| interactive_art | 17 |
| datasets | 41 |
| reproducibility | 1 |
| bni | 22 |
| open_source | 11 |
| doctoral_symposium | 2 |
| demo_and_video | 23 |
| industry_talk | 3 |
| proceedings_unclassified | 166 |
注册摘要 1918 项仍全缺失。1743 项有明确对应的节目摘要:1717 项使用 `abstract_source=official_program_exact_metadata_correspondence`;新增 26 项使用 `official_program_reviewed_title_formatting_correspondence`,两者都逐字复制节目原文。报告分别列出 `abstracts_from_literal_program_correspondence` 和 `abstracts_from_reviewed_title_formatting`;保留的汇总字段 `abstracts_from_exact_program_correspondence` 为 1743,表示两类非模糊对应之和。
其余 175 项保持摘要缺失,包括 9 项节目本身没有摘要及 166 个版本待核条目。注册原始摘要仍标为缺失,不能把节目内容说成出版者摘要。
8 个作者仅有单一姓名字段,保留登记值并标记,不猜测 given/family;37 个含排版标记或实体的题名仍保留字面原文和原始标记。排版比较保留上下标字符,不改写原始字段。
完整2143个节目ID和4326个展示位置仍在匹配证据或391个未匹配节目中,28个原始track均保留。391个节目与166个出版版本待核来自两种观察口径,不能直接相加当成不同论文。Workshop、艺术、行业等贡献不能因为不在这个论文集范围就被当成拒稿、撤稿或不值得保留。
注册出版支持来源侧 `status=accepted` 的发表观察,`raw_decision` 明确不是原始签署决定;出版日期仅放在注册元数据中,不伪造决定日期。`mm_registry_program:crossref:type` 与 `mm_registry_program:mm_program:track` 分别保存注册书目类型和节目 track;共同的派生来源前缀满足写入器的来源校验,两个标签命名空间仍相互独立,原始平台字段与证据继续保留在各自元数据中,`research_type_classified=false`:书目类型不是研究类型,行业、艺术、demo条目不能默认变成主轨研究论文。
## 只读数据库预检与导入边界
每个分区都有带count/SHA256的manifest,coverage始终为partial。规范化完成后,对实际报告中的全部输出文件运行现有 `audit source-identities --files ... --pg-service foundry-research --pg-schema research --out <新目录>`,不能靠已有66个arXiv/DOI候选代表全目录。预检使用只读一致性快照,并保存来源/标识/历次投稿/逐次决定的证据包;缺失或截断必须单列。
`approved_for_import=0`,每项 `scientific_work_identity_approved=false`。DOI绑定可能有误,精确题名/作者匹配不证明科学作品身份,没有匹配也不证明新作品。进入增量导入前仍须审查已有绑定、科学版本与候选历史,做新鲜保护检查、完整基线/备份,再使用原生写入者。不得覆盖旧拒稿、后续录用、标签、原始元数据或人物姓名版本;既有已消费队列不可重放。
## 恢复
完整注册目录及独立节目比对恢复包:`/data/paper-index/validation/mm2026-container-registry-recovery-v2.tar.gz`,SHA256 `ddb05a8134f6437b3fb40250e45ed6350d55431d953812abda3300871b6dd0ac`。其中含注册源目录、父记录 `validation/mm2026-parent.json`、逐项比对和请求方法证据。安全解包后可把该父记录路径传给 `--parent-snapshot`。
节目原始来源恢复包见 [MM来源说明](mm2026-source-status.md)。本地独立核验备份位于 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`。固定回执/正文哈希变化、总量变化、未知注册上下文或节目对应改变都须重新核查,不能放宽为无条件接受。最终全库去重、近似题名作者、标识冲突、摘要、约束、覆盖与查询审计仍是独立必做阶段。
实际导入回归在隔离的 SQLite 和 PostgreSQL 测试库中运行全部1918条、全部11个输出分区,核验原始字段、1918个书目类型标签、1752个节目标签、publication_doi 身份、partial覆盖,以及已有作品的规范信息和旧拒稿决定不被覆盖。这些测试不会写入生产 research schema;测试通过仍不替代逐条生产身份审阅。
旧的 `native-v1`、`native-v2` 来源目录及已消费的导入队列不可覆盖或重放。新版本写入新的输出目录,先核对已导入记录的源字段和分轨是否变化;新增来源关联仍须新鲜身份预检、保护基线和独立验收。程序输出的分区预期数量已改变,未来增量 manifest 必须明确保留旧覆盖父值并声明新预期值,不能重置已导入计数。
+29
View File
@@ -108,3 +108,32 @@ paper-atlas audit source-identities \
候选审计中的 `status=unknown` 仅为内部查询占位符,不代表原始提交的最终状态。节目ID以 `mm_program` 来源键查询,不转成全局 DOI、arXiv 或 OpenReview 标识;保留原始 `listed_in_official_program` 观察与全部展示位置。所有贡献类型、出版身份、最终决定核验标志均为false;缺作者的两项也保留并标记。 候选审计中的 `status=unknown` 仅为内部查询占位符,不代表原始提交的最终状态。节目ID以 `mm_program` 来源键查询,不转成全局 DOI、arXiv 或 OpenReview 标识;保留原始 `listed_in_official_program` 观察与全部展示位置。所有贡献类型、出版身份、最终决定核验标志均为false;缺作者的两项也保留并标记。
查询使用 PostgreSQL 只读一致性快照,候选作品的原始来源、标识、历次投稿和逐次决定压缩保存并记录哈希;截断会显式标记。标题/作者候选匹配不是作品身份批准,没有匹配也不证明是新论文。既往拒稿和后续录用不能互相覆盖,评分或评审推荐不能充当最终决定。输出目录必须不存在;遇到已有输出或静默运行,先检查具体单元和报告,不重启重放。 查询使用 PostgreSQL 只读一致性快照,候选作品的原始来源、标识、历次投稿和逐次决定压缩保存并记录哈希;截断会显式标记。标题/作者候选匹配不是作品身份批准,没有匹配也不证明是新论文。既往拒稿和后续录用不能互相覆盖,评分或评审推荐不能充当最终决定。输出目录必须不存在;遇到已有输出或静默运行,先检查具体单元和报告,不重启重放。
## 已完成的规范化、只读审计与出版核对
采集器提交 `c8db611` 和直接节目审计提交 `09a2d4e` 已通过PR #65、#66由用户合并。授权远程runner的535/543项测试、wheel构建安装和安装后全量命令检查通过;没有本机测试或构建。服务器规范化和只读候选报告也已独立验收,并在本机备份、复核全部2143项与814个候选证据包。
| 只读候选分类 | 来源节目项数 |
| --- | ---: |
| 标题与完整作者多重集合一致,仍需作品核查 | 660 |
| 仅部分作者重叠 | 153 |
| 无精确标题候选,不证明是新作品 | 1329 |
| 有标题候选但无作者一致性 | 1 |
全部4326个展示上下文一致,814个证据包均无截断。24篇候选作品涉及24次已有负面投稿、37条负面来源观察;37不能直接算作37次拒稿。所有候选均尚未批准身份或入库,旧负面决定必须在之后关联时保留。
对已有作品的原始arXiv记录、标题/完整作者多重集合和MM2026语境进一步筛查,得到67个 Main Track 出版DOI核对候选。独立查询其明确给出的DOI及父论文集,共68次单次请求,67次HTTP200、1次404,没有认证或重试。父注册记录为[第34届ACM Multimedia论文集](https://doi.org/10.1145/3767308),ISBN `9798400722134`,出版者ACM,出版年2026。
其中66项的原始arXiv引用DOI与注册DOI一致,按既有审计规则规范化的标题、完整作者多重集合、出版年、出版者、明确会议名和论文集标题也一致。注册文章未提供ISBN,不能伪称逐篇ISBN已核验;归属依据是文章明确给出的会议和论文集字段,以及独立的原始arXiv语境和DOI。仍需逐项作品/版本审查及新鲜数据库保护检查,**不是66项已入库或最终身份批准**。
`mfp7067` 对应既有候选作品2749947,原始arXiv提供的 `10.1145/3767308.3836169` 返回404,暂缓。404不是撤稿/拒稿证据,也不据此删除或修正既有标识;需独立核对后再处理。
完整会议出版名单仍需继续核对。实际ISBN查询只返回父论文集一个条目,父DOI的公开Unixref XML也只含该父DOI、0个 `conference_paper` 节点;都不能代表逐篇名单。按会议名检索得到的 `total-results` 是出版者全年范围内的宽泛检索量,不能当作MM篇数;必须逐项核对明确会议/论文集字段、节目与分轨归属,并保留未匹配项。全部28个来源track继续保留,不能把本批Main Track候选当成会议完整覆盖。
| 已独立验证的恢复证据 | SHA256 |
| --- | --- |
| 全轨规范化恢复包 | b119081618b8ce98037dc139be159512f9fdb86a0ccfb127c8b8a1c5722f3d87 |
| 只读候选/814证据包恢复包 | 42d0d0404bba04e64e638344b050ad4e7287c3e4c75a777b7069ecc7d7a6ed60 |
| 67项DOI及父论文集原始注册回执恢复包 | dd7a0ff76f12070f403cc7352ac54269e454758d807896a086c58c1c1ac7b798 |
服务器恢复包位于 `/data/paper-index/validation/`,分别名为 `mm2026-native-recovery-v1.tar.gz`、`mm2026-preflight-recovery-v1.tar.gz`、`mm2026-publication-metadata-recovery-v1.tar.gz`;本地同名备份在 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`。来源目录为 `collected/mm2026-native-v1`、`mm2026-preflight-v1`、`mm2026-publication-metadata-v1`。先核对完整manifest及实际单元结果,不能用旧进度文件的running字样重启。其他会议已消费的导入队列仍禁止重放;本轮均为来源核查、只读候选审计,没有生产写入。
+52
View File
@@ -0,0 +1,52 @@
# ReviewArena 镜像的索引字段投影
ReviewArena 是公开评审镜像,固定版本为 `c2978add17c2099219eaddbc2599974d69d4d09b`。上游数据卡描述七类会议、2020–2026 年的部分来源;这不是逐会议、逐年度的公开决定全集。文件清单有 15 个 Parquet 分片,覆盖 COLM、CoRL、EMNLP、ICLR、ICML、NeurIPS、TMLR。这里不包括 UAI,不能用它填补 UAI 缺口。
- [固定版本上游数据卡](https://huggingface.co/datasets/anonymousNeurIPS2026submission4281/reviewarena/blob/c2978add17c2099219eaddbc2599974d69d4d09b/README.md)
- [固定版本文件元数据 API](https://huggingface.co/api/datasets/anonymousNeurIPS2026submission4281/reviewarena/tree/c2978add17c2099219eaddbc2599974d69d4d09b/data?recursive=false&expand=false)
- [Arrow ParquetFile 投影接口](https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetFile.html)
## 运行
仅在已授权远程 runner 安装 `paper-atlas[mirrors]`,使用固定 PyArrow 26.0.1 二进制 wheel。本机不执行安装、构建或项目测试。常规 CI 包含投影、安全边界和镜像决定语义测试;真实网络采集由独立的 `ReviewArena metadata projection` workflow 手动触发,指定已通过 CI 的代码 ref 和 split,例如 `emnlp`。该 workflow 仍使用 `ces-control` / `aliyun-ces-control`,在服务器固定数据目录生成输出和恢复归档,不更改默认 runtime 或数据库。
包内命令如下,`--out` 必须是既有父目录内的新目录:
```sh
paper-atlas collect reviewarena-metadata \
--split emnlp \
--out /data/paper-index/collected/reviewarena-emnlp-reviewed-version
```
输出目录已存在时拒绝执行。遇到禁止访问、来源变化或读取失败,保留失败报告,不自动重试或切换到整文件下载。不能把原始访问拒绝解释为没有拒稿。
已保存的完整元数据投影可以离线重新解释决定字面值。提供原 `collection-report.json` 及其已独立验证的精确文件 SHA-256,程序核对全部元数据文件哈希、逐行来源身份和原始字段,生成新目录;不会再次访问上游 Parquet:
```sh
paper-atlas collect reviewarena-metadata --split emnlp \
--from-projection /data/paper-index/collected/<既有投影>/collection-report.json \
--projection-sha256 <已验证的原报告SHA256> \
--out /data/paper-index/collected/<新解释版本>
```
手动 workflow 对应输入为 `projection_name` 与 `projection_sha256`。离线报告的 `metadata_bytes_read` 为本次请求字节数 `0`,`source_projection_bytes` 保留原始列块投影字节数;原报告哈希建立恢复链,原始记录与来源定位不变。
## 实际读取和保存范围
仅投影以下字段:`forum_id`、`conference`、`year`、`track`、`venue_id`、`paper_number`、`title`、`abstract`、`authors`、`keywords`、`venue`、`decision`。
程序先读取文件魔数、尾部长度和有界 Parquet 页脚,在内存中解析 schema 和列块偏移,然后只请求这些列的字典/数据页。Arrow 可能请求较大的页脚窗口,未获准区域由本地稀疏文件合成零字节,不会为该窗口补下载相邻正文页。逐个选中列块还要检查不与未选列块重叠。单文件上限为 64 MiB 所需字节、512 个范围请求和 100,000 行;响应必须是准确的 HTTP 206 / Content-Range,ETag 变化时停止。该边界适用于每个分片,不能据此启动其他分片的未授权全文件下载。
不读取 `markdown`、`reviews_json`、`decision_comment`、`author_rebuttal` 等正文或评审内容的数据页;不保存原 Parquet、页脚原文或未选字段统计值。保存的 JSONL 只有上述索引字段、来源定位和审计标记。报告记录每个请求范围及其 SHA-256、选中列名、页脚哈希、投影数量和保存文件哈希,便于独立复核。
上游 LFS SHA-256 与大小是固定版本 API 提供的对象声明。由于不下载整个 Parquet,本流程**不声称验证了完整 Parquet 内容哈希**。范围对象大小/ETag 和各段哈希用于检查本次读取的一致性。
## 决定和身份边界
`mirror_decision_category` 只解释明确的镜像字面值;未知、缺失或未审核的决定保持 `unknown`,不会根据 venue、是否录用、摘要缺失或目录缺席推断拒稿。类别数量是镜像行数,不能解释为独立作品数量或原始官方决定数量。
真实 EMNLP 2023 固定分片包含 2,009 行:原始决定为 `Accept-Main` 994 行、`Accept-Findings` 1,015 行,原始 `track` 却全部为 `main`。两种已核对字面值属于镜像录用声明;`mirror_decision_track_claim` 分开保留其明确的 main/findings 声明,原 `track` 和所有 `raw_fields` 不被重写。`Accept (Poster)` 等展示形式不能自行推出主会归属。此分片没有明确负面决定,不能用于补齐拒稿缺口,更不能由此断言 EMNLP 原始平台没有公开拒稿。首次实际投影仅请求 1,720,498 字节;原文件 90,794,691 字节,未下载完整文件。
上游 `forum_id`、`venue_id`、投稿号、作者和决定均作为原始声明保留。`forum_id` 不直接成为全局 OpenReview 标识;稳定行定位由数据集、固定 commit、文件和行号组成。输出 `identifiers` 为空,原始论坛身份和决定签名标记为未验证,保留匿名和缺失作者。源数据中的字符串命名不能被当成真实人员别名。
输出故意不是可直接 `import records` 的规范化队列,没有授予导入或合并权限。下一步必须检查原始来源、跨来源重合和作品身份,再决定哪些来源观察能进入生产。即使明确镜像拒稿,也要保留该投稿历史并继续查找后续其他会议的录用;不能把“目前没有关联录用”表述为从未录用。本采集不代替最终全库去重、决定冲突、来源覆盖和查询验收。
+57
View File
@@ -0,0 +1,57 @@
# 出版来源版本与覆盖修正
`paper-atlas maintain source-revisions` 用于同一出版清单的新版本:稳定 DOI 全集、标题、作者和决定不变,但官方分轨、摘要或对应来源证据已经过重新核对。它不导入新论文、不合并实体、不推断拒稿,不把出版分类当成研究类型。
例如,同一清单的“未分类”由 192 条减少到 166 条,而主会等分区增加。直接放松增量导入的计数保护会隐藏清单丢失。本命令先核验完整的新旧清单及数据库快照,再单独修正版本;普通 `import records` 的单调计数规则保持原样。
## 准备完整证据
在工作区外保存 inventory JSON,包含以下字段:
| 字段 | 内容 |
| --- | --- |
| `format` | `1` |
| `source`, `venue`, `year`, `event_key`, `list_url` | 明确的已有来源、会议缩写、年份、事件键和 HTTPS 出版清单地址 |
| `previous_rows`, `rows` | 完整旧版、新版规范化来源记录数组,每版最多 10,000 条;每条包含稳定 DOI、来源字段、作者、摘要、track、决定、标签、原始来源 metadata |
| `evidence` | 一个或多个 `{ "url": "https://…", "sha256": "…" }`,绑定已独立核验的原始证据文件 |
两版必须有完全相同的 DOI 集合;仅允许 `abstract`、`track`、`tags`、`metadata` 的有据变化。必须保留 collector 原始字段,不能将工作身份核查产生的额外锚点混入原始来源版本。只有明确 `accepted` 的出版观察可进入本流程。证据 URL/hash 是审核输入,本命令不联网确认它们;执行者必须另行验证文件哈希、完整性及对应关系。
数据库可以部分导入,甚至各分区的预期数暂时来自不同版本。现有覆盖必须为 `partial`、无错误,导入数与实际来源记录一致;预期数必须恰好属于旧版或新版的同一分区。已有目标 edition 和对应覆盖记录必须存在,目标中可以有其他来源及既有投稿。本版本不会新建 edition,也不会处理共享来源投稿、多次既有 scope curation 的再迁移,或成员集合变化;遇到这些情况会拒绝并要求单独核查。
## 先生成只读计划
```sh
paper-atlas maintain source-revisions \
--pg-service foundry-research --pg-schema research \
--inventory /data/paper-index/validation/reviewed-source-version.json \
--out /data/paper-index/validation/source-revision-plan.json
```
计划包含明确数据库身份、两个完整来源版本、当前版本父节点、覆盖表原值和修正值、逐条变更,以及受影响会议 scope 内所有论文的规范记录、稳定标识、全部跨会议投稿和决定、来源关系、标签与已存在的 annotation。拒稿后录用的历史也在保护范围内,不能只校验当前会议的录用观察。
计划和原始证据应保存到服务器与独立恢复位置,核对完整差异及计划文件 SHA-256。执行前重新核对生产保护基线与单写入安排。输出文件必须是不存在的新路径,禁止覆盖旧报告。
## 按文件哈希应用并验收
```sh
paper-atlas maintain source-revisions \
--pg-service foundry-research --pg-schema research \
--plan /data/paper-index/validation/source-revision-plan.json \
--plan-sha256 <已审核的计划文件SHA256> \
--out /data/paper-index/validation/source-revision-result.json
```
应用时重新生成计划并要求完全一致;PostgreSQL 使用已有 writer 锁及相关表写锁,SQLite 使用立即事务。任何保护数据、当前版本或目标标识冲突都使操作失败。操作保留投稿和决定 ID、原始决定证据、规范论文文本、全部稳定标识、来源关系和现有标签;只修改计划列出的投稿/投稿标识 edition,以及来源记录的摘要与 metadata 当前投影。旧投影完整保存在版本历史中,来源原始获取时间不会伪造为修正时间。
覆盖只调整明确分区的预期数和来源观察数,保留 `partial`、cursor、成功时间和错误字段原值。尚未导入的分区不产生虚构投稿或来源观察。覆盖数不能被解读为科学作品数量,或所有 track、摘要、负面决定已完整。
同一事务显式增加可共享的可选表 `source_scope_revisions`,保持原 21 张业务表的身份。每个版本存储新来源全集、父版本、完整计划及修正前数据;应用接口只追加版本,不更新或删除旧版本。没有新建 PostgreSQL 实例。该表纳入 SQLite→PostgreSQL 的可选表迁移、全量内容哈希、约束与计数验收。`version_sha256` 是规范化计划内容哈希;CLI 的 `--plan-sha256` 是计划文件字节哈希,二者用途不同。
提交前在同一事务比较全部保护数据与明确预期的变更;失败则全部回滚,包括新增表和版本记录。相同计划的重放还会重新验收当前状态;如后续数据已变化则拒绝,不会重复迁移或恢复旧状态。完整计划与 before-image 用于审查和恢复准备,本命令没有自动回滚历史版本的入口;不得手工删除版本来绕开刷新保护。
## 后续刷新及限制
`import records` 检查当前版本的逐条完整来源字段;`Store.record` 也检查原始文本、作者、来源 metadata 和基础标识,防止直接调用绕回旧版摘要。来源字段必须与当前版本相符,可以额外携带已经核验的工作身份/curation 证据,不能替换来源自身字段。相同来源、会议事件和清单地址下未登记的成员必须另行核验新版本。
版本表通过包内接口追加,未声明数据库管理员的任意 SQL 为安全接口。新来源或新版本在导入前仍需要独立工作身份核查;姓名、标题或无候选都不证明作品身份。实现测试和版本修正验收也不替代所有导入结束后的全库去重、标识冲突、近似标题/作者、摘要、来源覆盖与查询审计。
+6
View File
@@ -20,19 +20,24 @@ COMMANDS = {
('collect','public-openreview'): ('paper_atlas.collectors.public_openreview', {}, '核验其他会议的公开拒稿及撤稿标签'), ('collect','public-openreview'): ('paper_atlas.collectors.public_openreview', {}, '核验其他会议的公开拒稿及撤稿标签'),
('collect','icml-public-archive'): ('paper_atlas.collectors.icml_public_archive', {}, 'ICML 2026 公开拒稿原始投稿快照'), ('collect','icml-public-archive'): ('paper_atlas.collectors.icml_public_archive', {}, 'ICML 2026 公开拒稿原始投稿快照'),
('collect','peerread'): ('paper_atlas.collectors.peerread', {}, 'PeerRead 历史决定补充'), ('collect','peerread'): ('paper_atlas.collectors.peerread', {}, 'PeerRead 历史决定补充'),
('collect','reviewarena-metadata'): ('paper_atlas.collectors.reviewarena', {}, '仅投影固定版本评审镜像的索引字段,保留决定声明而不下载正文或评审页'),
('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'), ('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'),
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'), ('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'), ('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
('collect','ismir2026-program'): ('paper_atlas.collectors.ismir2026', {}, '核验 ISMIR 2026 公开节目与录用名单并保留原始摘要、来源差异和自愿公开评审'), ('collect','ismir2026-program'): ('paper_atlas.collectors.ismir2026', {}, '核验 ISMIR 2026 公开节目与录用名单并保留原始摘要、来源差异和自愿公开评审'),
('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'), ('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'),
('collect','eccv2026-program'): ('paper_atlas.collectors.eccv2026', {}, '核验ECCV2026暂定节目并断点采集公开元数据'), ('collect','eccv2026-program'): ('paper_atlas.collectors.eccv2026', {}, '核验ECCV2026暂定节目并断点采集公开元数据'),
('collect','eccv2026-publications'): ('paper_atlas.collectors.eccv2026_publications', {}, '核验ECCV2026 Springer 正式出版章节'),
('collect','icassp2026-registry'): ('paper_atlas.collectors.icassp2026', {}, '核验 ICASSP 2026 冻结注册论文元数据'), ('collect','icassp2026-registry'): ('paper_atlas.collectors.icassp2026', {}, '核验 ICASSP 2026 冻结注册论文元数据'),
('collect','icdm2026-program'): ('paper_atlas.collectors.icdm2026', {}, '核验 ICDM2026 冻结日程及明确的研究/应用轨道'), ('collect','icdm2026-program'): ('paper_atlas.collectors.icdm2026', {}, '核验 ICDM2026 冻结日程及明确的研究/应用轨道'),
('collect','cikm2026-sources'): ('paper_atlas.collectors.cikm2026', {}, '核对 CIKM 2026 节目、附属议程与 Crossref 出版登记'),
('collect','sdm2026-proceedings'): ('paper_atlas.collectors.sdm2026', {}, '核验 SDM 2026 注册论文并单独保留公开日程版本'), ('collect','sdm2026-proceedings'): ('paper_atlas.collectors.sdm2026', {}, '核验 SDM 2026 注册论文并单独保留公开日程版本'),
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'), ('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'), ('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
('collect','recsys2026-program'): ('paper_atlas.collectors.recsys2026', {}, '核验 RecSys 2026 公开录用贡献并保留 DOI 来源版本'), ('collect','recsys2026-program'): ('paper_atlas.collectors.recsys2026', {}, '核验 RecSys 2026 公开录用贡献并保留 DOI 来源版本'),
('collect','kdd2026-program'): ('paper_atlas.collectors.kdd2026', {}, '核对 KDD 2026 两个投稿周期、分轨及撤回更换标注'),
('collect','mm2026-program'): ('paper_atlas.collectors.mm2026', {}, '恢复 MM 2026 全部来源 track 与重复展示上下文,待贡献和身份核查'), ('collect','mm2026-program'): ('paper_atlas.collectors.mm2026', {}, '恢复 MM 2026 全部来源 track 与重复展示上下文,待贡献和身份核查'),
('collect','mm2026-publications'): ('paper_atlas.collectors.mm_publications2026', {}, '核验 MM 2026 全部注册出版条目并分别保留节目分轨、摘要与版本差异'),
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'), ('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'), ('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'), ('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
@@ -45,6 +50,7 @@ COMMANDS = {
('import','arxiv'): ('paper_atlas.ingestion.arxiv', {}, '导入 arXiv 元数据 ZIP'), ('import','arxiv'): ('paper_atlas.ingestion.arxiv', {}, '导入 arXiv 元数据 ZIP'),
('import','records'): ('paper_atlas.ingestion.records', {}, '导入核验过的会议文件'), ('import','records'): ('paper_atlas.ingestion.records', {}, '导入核验过的会议文件'),
('maintain','edition-tracks'): ('paper_atlas.ingestion.edition_tracks', {}, '按逐条来源证据修正既有投稿轨道并保护刷新'), ('maintain','edition-tracks'): ('paper_atlas.ingestion.edition_tracks', {}, '按逐条来源证据修正既有投稿轨道并保护刷新'),
('maintain','source-revisions'): ('paper_atlas.ingestion.source_revisions', {}, '按完整来源版本核验并修正出版分轨与覆盖数,保留历史并阻止旧版刷新'),
('maintain','joint-editions'): ('paper_atlas.ingestion.joint_editions', {}, '按已核验计划修正既有联合会议届次归属'), ('maintain','joint-editions'): ('paper_atlas.ingestion.joint_editions', {}, '按已核验计划修正既有联合会议届次归属'),
('maintain','event-identity'): ('paper_atlas.ingestion.event_identity', {}, '按核验计划增加同年会议事件身份'), ('maintain','event-identity'): ('paper_atlas.ingestion.event_identity', {}, '按核验计划增加同年会议事件身份'),
('maintain','merge'): ('paper_atlas.ingestion.maintenance', {}, '协调单写入合并与恢复'), ('maintain','merge'): ('paper_atlas.ingestion.maintenance', {}, '协调单写入合并与恢复'),
+164
View File
@@ -0,0 +1,164 @@
"""Collect CIKM 2026 programme and Crossref metadata from frozen snapshots."""
import argparse
from collections import Counter
from hashlib import sha256
import json
from pathlib import Path
import re
from paper_atlas.io.html import Document, clean
PROGRAM = 'https://cikm2026.diag.uniroma1.it/cikm-program/program.html'
VOLUME = '10.1145/3799682'
CONTAINER = 'Proceedings of the 35th ACM International Conference on Information and Knowledge Management'
TRACKS = {'FULL': 596, 'SHORT': 234, 'APPLIED': 86, 'RESOURCE': 46, 'DEMO': 46}
SUPPLEMENTAL = {'KEYNOTE', 'TUTORIAL', 'INDUSTRY', 'POSTER', 'PHD', 'PHD · REMOTE', 'WORKSHOP', 'ANALYTICUP'}
PAGE_COUNTS = (250, 250, 250, 250, 81)
HASHES = {
'program.html': '3eaa668b6aaf09a671d8a3e19a2889b9ab21afaa7ded013794afa94bf35281d6',
'CIKM2026_program.html': 'e3630bd1247df713b8b618f1d3cd9709e7020b2524bb677f82a10f83badd1732',
'crossref-parent.json': 'f3370418d86ccfcfaeb972073bc5fcd1c1c737caed96570fc83f99db0f7d204a',
'crossref-page-1.json': '16bed29f8ee82d668b124c369b632c5c1de0753bdbec7eb72ecb22ffdda5c710',
'crossref-page-2.json': '54424ca31d99462394c9fde5fecfd7dd16337351d21ede3169f19af54d4f49c5',
'crossref-page-3.json': '2dc31d89f41966614ed66a11d1ebaa4b2a6904dcf92945b7185e72e7e65382dd',
'crossref-page-4.json': 'fbbec2b52650dc99a0251c743f035a6fd95be263b04c55c12134cf8473bdde76',
'crossref-page-5.json': '06151c5f770c1e37e8ea9dabe5210be9c969a14fa3bf9925a2d7730ced47d6e5',
'crossref-pages.json': '8be1f8f05139f7a1231b73ad6430f28d2f625809f1530cd9de654f8b3c68c1ae',
}
def fields(node, css_class):
return [child for child in node.walk() if css_class in child.attrs.get('class', '').split()]
def parse_program(raw, *, require_doi):
document = Document(raw.decode('utf-8'))
papers = {}
for card in fields(document.root, 'paper'):
pid = card.attrs.get('data-pid')
if not pid:
continue
titles = fields(card, 'ptitle')
labels = fields(card, 'badge')
abstracts = fields(card, 'abstxt')
if pid in papers or len(titles) != 1 or len(labels) != 1 or len(abstracts) > 1:
raise ValueError(f'Invalid or repeated programme card {pid}')
title = clean(titles[0].text())
label = clean(labels[0].text())
authors = [clean(author.text()) for author in fields(card, 'au')]
abstract = clean(abstracts[0].text()) if abstracts else None
doi_url = titles[0].attrs.get('href')
doi = doi_url.removeprefix('https://doi.org/') if doi_url else None
if label not in TRACKS or not title or not authors or not all(authors):
raise ValueError(f'Incomplete programme metadata {pid}')
if require_doi and (not doi or not re.fullmatch(re.escape(VOLUME) + r'\.\d+', doi) or not abstract):
raise ValueError(f'Missing programme DOI or abstract {pid}')
papers[pid] = {'programme_id': pid, 'title': title, 'author_display_names': authors,
'abstract': abstract, 'programme_label': label, 'programme_doi_link': doi_url,
'doi': doi if require_doi else None}
return papers
def parse_supplemental(raw):
document = Document(raw.decode('utf-8'))
events = {}
for session in fields(document.root, 'sessblock'):
section = session.attrs.get('id', '')
if not section.startswith(('pre-', 'kn-')):
continue
for card in fields(session, 'paper'):
labels = fields(card, 'badge')
if len(labels) != 1:
raise ValueError(f'Ambiguous supplemental label in {section}')
label = clean(labels[0].text())
if label not in SUPPLEMENTAL:
continue
titles = fields(card, 'ptitle')
abstracts = fields(card, 'abstxt')
locator = card.attrs.get('id') or section
if locator in events or len(titles) != 1 or len(abstracts) > 1:
raise ValueError(f'Ambiguous supplemental entry {locator}')
events[locator] = {'programme_locator': locator, 'programme_section': section,
'programme_label': label, 'title': clean(titles[0].text()),
'author_display_names': [clean(a.text()) for a in fields(card, 'au')],
'abstract': clean(abstracts[0].text()) if abstracts else None}
return events
def parse_registry(snapshots):
volume = json.loads(snapshots['crossref-parent.json'])['message']
if (volume['DOI'], volume['title'], volume['ISBN']) != (VOLUME, [CONTAINER], ['9798400725395']):
raise ValueError('Unexpected Crossref volume')
records = {}
for number, count in enumerate(PAGE_COUNTS, 1):
name = f'crossref-page-{number}.json'
message = json.loads(snapshots[name])['message']
if message['total-results'] != 1081 or len(message['items']) != count:
raise ValueError(f'Unexpected Crossref pagination in {name}')
for item in message['items']:
doi = item['DOI'].lower()
if (doi in records or not re.fullmatch(re.escape(VOLUME) + r'\.\d+', doi)
or item.get('container-title') != [CONTAINER] or item.get('type') != 'proceedings-article'
or item.get('published', {}).get('date-parts') != [[2026, 11, 6]]):
raise ValueError(f'Unexpected Crossref record {doi}')
records[doi] = {'registry': item, 'source_file': name}
return records
def collect(snapshots, output):
snapshots = Path(snapshots)
output = Path(output)
if output.exists() or output.is_symlink():
raise FileExistsError(output)
bodies = {}
for name, expected in HASHES.items():
path = snapshots / name
if path.is_symlink() or not path.is_file() or path.stat().st_size > 8 * 1024**2:
raise ValueError(f'Invalid snapshot file {name}')
raw = path.read_bytes()
if sha256(raw).hexdigest() != expected:
raise ValueError(f'Snapshot hash mismatch for {name}')
bodies[name] = raw
programme = parse_program(bodies['program.html'], require_doi=True)
alternate = parse_program(bodies['CIKM2026_program.html'], require_doi=False)
supplemental = parse_supplemental(bodies['program.html'])
registry = parse_registry(bodies)
programme_dois = {r['doi'] for r in programme.values()}
tracks = Counter(r['programme_label'] for r in programme.values())
if (tracks != TRACKS or len(programme_dois) != len(programme) or programme_dois - registry.keys()
or len(alternate) != 1005 or alternate.keys() - programme.keys()
or len(supplemental) != 73 or len(registry) - len(programme_dois) != 73):
raise ValueError('Unexpected programme or registry coverage')
groups = {
'programme': programme,
'alternate-programme': alternate,
'supplemental-programme': supplemental,
'crossref': registry,
}
output.mkdir(parents=True, exist_ok=False)
files = {}
for name, rows in groups.items():
filename = name + '.jsonl'
payload = ''.join(json.dumps({'source_key': key, **row}, ensure_ascii=False) + '\n'
for key, row in sorted(rows.items())).encode()
(output / filename).write_bytes(payload)
files[filename] = {'records': len(rows), 'sha256': sha256(payload).hexdigest()}
report = {'status': 'collected_pending_identity_review', 'files': files, 'input_sha256': HASHES,
'programme_by_label': dict(tracks), 'programme_dois_in_registry': len(programme_dois),
'registry_outside_doi_programme': sorted(registry.keys() - programme_dois),
'programme_ids_missing_from_alternate': sorted(programme.keys() - alternate.keys()),
'database_written': False, 'public_negative_decisions_verified': False}
partial = output / 'collection-report.json.part'
partial.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
partial.replace(output / 'collection-report.json')
return report
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
parser.add_argument('--snapshots', type=Path, required=True)
parser.add_argument('--out', type=Path, required=True)
options = parser.parse_args(argv)
print(json.dumps(collect(options.snapshots, options.out), ensure_ascii=False))
@@ -0,0 +1,184 @@
"""Normalize frozen ECCV 2026 Springer proceedings and preserve publisher evidence."""
import argparse
import fcntl
import hashlib
import json
from pathlib import Path
import re
import tempfile
from paper_atlas.io.html import Document, clean
from paper_atlas.io.springer import parse_chapter, read_page, snapshot_index
SOURCE = 'eccv2026_springer'
ORIGIN = 'https://link.springer.com'
BOOK_TITLE = 'Computer Vision – ECCV 2026'
LIST_URL = ORIGIN + '/conference/eccv'
BOOK = re.compile(r'10\.1007/978-3-032-\d{5}-[\dXx]')
CHAPTER = re.compile(BOOK.pattern + r'_[1-9]\d*')
VOLUMES = set(range(17001, 17079))
PUBLICATIONS = 2835
def fingerprint(raw):
return hashlib.sha256(raw).hexdigest()
def toc_page(html, book_doi, volume):
tree = Document(html).root
text = clean(tree.text())
if BOOK_TITLE not in text or 'Malmö' not in text or f'(LNCS, volume {volume})' not in text:
raise ValueError('Unexpected Springer ECCV volume scope')
entries = {}
for node in tree.walk():
if node.tag != 'li' or node.attrs.get('data-test') != 'chapter':
continue
headings = [n for n in node.walk() if n.tag == 'h3']
if len(headings) != 1:
raise ValueError('Ambiguous Springer contents heading')
heading, = headings
title = clean(heading.text())
links = [n.attrs.get('href', '') for n in heading.walk() if n.tag == 'a']
if title in ('Front Matter', 'Back Matter') and not links:
continue
if len(links) != 1 or not title:
raise ValueError('Authored contents entry lacks one chapter link')
link, = links
doi = link.removeprefix('/chapter/')
if (not link.startswith('/chapter/') or not CHAPTER.fullmatch(doi)
or not doi.startswith(book_doi + '_') or doi in entries):
raise ValueError('Unexpected or duplicate Springer chapter DOI')
entries[doi] = title
if not entries:
raise ValueError('Empty Springer contents page')
return entries
def publisher_inventory(root):
root = Path(root).resolve()
receipt_bytes = (root / 'receipts.jsonl').read_bytes()
inventory_bytes = (root / 'volume-checkpoint.json').read_bytes()
summary = json.loads((root / 'summary.json').read_bytes())
volumes = json.loads(inventory_bytes)
receipt_rows = [json.loads(line) for line in receipt_bytes.splitlines()]
receipts = {row['file']: row for row in receipt_rows}
if (summary['raw_receipts_sha256'] != fingerprint(receipt_bytes)
or summary['volume_checkpoint_sha256'] != fingerprint(inventory_bytes)
or set(volumes) != {str(n) for n in VOLUMES}
or len(receipts) != len(receipt_rows)):
raise ValueError('Incomplete or inconsistent Springer contents snapshot')
entries, used = {}, set()
for number, volume in sorted(volumes.items()):
book_url = volume['book']
book_doi = book_url.removeprefix(ORIGIN + '/book/')
if not BOOK.fullmatch(book_doi) or book_url != ORIGIN + '/book/' + book_doi:
raise ValueError('Unexpected Springer volume URL')
if set(volume['files']) != {'1', '2'}:
raise ValueError('Missing reviewed Springer contents page')
volume_dois = set()
for index, file in volume['files'].items():
receipt = receipts[file]
path = root / file
page_url = book_url + ('?page=2' if index == '2' else '')
if (file in used or Path(file).name != file or not path.resolve().is_relative_to(root)
or receipt['url'] != page_url or receipt['status'] != 200
or not 0 < receipt['bytes'] <= 3_000_000):
raise ValueError('Unsafe, reused or mismatched Springer contents receipt')
raw = path.read_bytes()
if len(raw) != receipt['bytes'] or fingerprint(raw) != receipt['sha256']:
raise ValueError('Springer contents bytes disagree with receipt')
used.add(file)
page_entries = toc_page(raw.decode(), book_doi, int(number))
if entries.keys() & page_entries.keys():
raise ValueError('Springer chapter repeats across contents pages')
volume_dois.update(page_entries)
entries.update({doi: {'title': title, 'lncs_volume': int(number), 'book_doi': book_doi,
'evidence_url': page_url, 'evidence_sha256': receipt['sha256']}
for doi, title in page_entries.items()})
if volume_dois != set(volume['chapter_dois']) or len(volume_dois) != len(volume['chapter_dois']):
raise ValueError('Springer volume chapter set differs from checkpoint')
if used != receipts.keys() or len(entries) != PUBLICATIONS or summary['chapter_link_count'] != PUBLICATIONS:
raise ValueError('Springer contents chapter total differs from the reviewed archive')
return entries
def publisher_row(chapter, toc, evidence):
doi = chapter['doi']
if clean(chapter['title']) != toc['title']:
raise ValueError('Chapter title differs from Springer contents, review the version')
url = ORIGIN + '/chapter/' + doi
return {
'source': SOURCE, 'external_id': doi, 'url': url,
'venue': 'ECCV', 'year': 2026, 'track': 'main',
'title': chapter['title'], 'authors': chapter['authors'], 'abstract': chapter['abstract'],
'status': 'accepted', 'identifiers': {'doi': doi},
'submission_identifiers': [{'scheme': 'publication_doi', 'value': doi}],
'evidence_url': url, 'observed_at': evidence['observed_at'],
'raw_decision': 'Chapter published in the refereed ECCV 2026 conference proceedings.',
'tags': [{'namespace': SOURCE + ':publication_format', 'value': 'proceedings_chapter'}],
'metadata': {
'book_title': BOOK_TITLE, 'book_doi': toc['book_doi'], 'lncs_volume': toc['lncs_volume'],
'authors_citation_raw': chapter['authors_citation_raw'], 'first_page': chapter['first_page'],
'last_page': chapter['last_page'], 'publisher_chapter_evidence': evidence,
'publisher_contents_evidence': {'url': toc['evidence_url'], 'sha256': toc['evidence_sha256']},
'scope': 'Refereed conference proceedings, LNCS 17001–17078.',
'original_decision_signature_verified': False,
},
}
def collect(out, *, toc_snapshots, chapter_snapshots):
contents = publisher_inventory(toc_snapshots)
chapters = snapshot_index(chapter_snapshots)
urls = {ORIGIN + '/chapter/' + doi: doi for doi in contents}
if not chapters or not chapters.keys() <= urls.keys():
raise ValueError('No chapter snapshots or chapter outside reviewed ECCV proceedings')
rows, raw = [], {}
for url in sorted(chapters):
doi = urls[url]
body, evidence = read_page(chapters, url)
chapter = parse_chapter(body.decode(), doi, book_title=BOOK_TITLE, publication_year=2026)
rows.append(publisher_row(chapter, contents[doi], evidence))
raw[evidence['file']] = body
out = Path(out)
out.mkdir(parents=True, exist_ok=True)
lock_path = out / 'eccv2026-publications.collect.lock'
filename = 'eccv2026-springer-main.jsonl'
with lock_path.open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if any(p != lock_path for p in out.iterdir()):
raise FileExistsError('Existing output must be inspected, not overwritten')
with tempfile.TemporaryDirectory(prefix='.eccv2026-publications-', dir=out) as temporary:
stage = Path(temporary)
(stage / 'evidence').mkdir()
for path, body in raw.items():
(stage / path).write_bytes(body)
data = ''.join(json.dumps(row, ensure_ascii=False) + '\n' for row in rows).encode()
(stage / filename).write_bytes(data)
manifest = {
'source': SOURCE, 'venue': 'ECCV', 'year': 2026, 'track': 'main', 'parser_version': 1,
'file': str(out / filename), 'file_sha256': fingerprint(data), 'count': len(rows),
'list_url': LIST_URL, 'source_expected_count': PUBLICATIONS, 'status': 'partial',
'missing_abstract_count': sum(not r['abstract'] for r in rows),
}
report = {
'status': 'collected_pending_identity_review', 'records': len(rows), 'files': [manifest],
'toc_chapters': len(contents), 'chapter_details_missing': len(contents) - len(rows),
'database_written': False,
}
metadata = Path(filename).with_suffix('.manifest.json')
(stage / metadata).write_text(json.dumps(manifest, indent=2) + '\n')
(stage / 'collection-report.json').write_text(json.dumps(report, indent=2) + '\n')
for item in ('evidence', filename, metadata, 'collection-report.json'):
(stage / item).replace(out / item)
return report
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog)
parser.add_argument('--toc-snapshots', required=True)
parser.add_argument('--chapter-snapshots', required=True)
parser.add_argument('--out', required=True)
args = parser.parse_args(argv)
print(json.dumps(collect(args.out, toc_snapshots=args.toc_snapshots,
chapter_snapshots=args.chapter_snapshots), ensure_ascii=False))
+129
View File
@@ -0,0 +1,129 @@
"""Inventory KDD 2026 programme DOI, cycle and track observations."""
import argparse
from collections import Counter
from hashlib import sha256
from html.parser import HTMLParser
import json
from pathlib import Path
import re
SOURCE_URL = 'https://kdd2026.kdd.org/papers/'
CYCLES = {1: 'july', 2: 'february'}
TRACKS = {
'rtp': 'research', 'ads': 'applied_data_science', 'dtb': 'datasets_and_benchmarks',
'ais': 'ai_for_sciences', 'bsi': 'blue_sky_ideas',
}
DOI_URL = re.compile(r'https://doi\.org/(10\.1145/\d+\.\d+)')
WITHDRAWN = re.compile(
r'^\*\*WITHDRAWN/New SUBMISSION ID for Authorship Order Change (\d{1,2}/\d{1,2}/26) -- (.+)$'
)
OTHER_STATUS = re.compile(r'^[*\[(!#\s]*(?:withdrawn|rejected|desk[\s_-]+rejected|retracted|cancell?ed)\b', re.I)
class ProgrammeScripts(HTMLParser):
def __init__(self):
super().__init__()
self.current = None
self.scripts = []
def handle_starttag(self, tag, attrs):
if tag == 'script' and 'src' not in dict(attrs):
self.current = []
def handle_data(self, data):
if self.current is not None:
self.current.append(data)
def handle_endtag(self, tag):
if tag == 'script' and self.current is not None:
self.scripts.append(''.join(self.current))
self.current = None
def cycle_rows(scripts, cycle):
pattern = re.compile(r'\bconst\s+cycle' + str(cycle) + r'Papers\s*=\s*(\[)')
matches = [(code, match) for code in scripts for match in pattern.finditer(code)]
if len(matches) != 1:
raise ValueError(f'Expected one KDD programme array for cycle {cycle}')
code, match = matches[0]
body = code[match.start(1):]
rows, consumed = json.JSONDecoder().raw_decode(body)
if not rows or not body[consumed:].lstrip().startswith('.map((paper, index)'):
raise ValueError(f'Unexpected KDD programme array boundary for cycle {cycle}')
return rows
def inventory(raw):
parser = ProgrammeScripts()
parser.feed(raw.decode('utf-8'))
parser.close()
records = {}
source_counts = {}
for cycle, cycle_name in CYCLES.items():
rows = cycle_rows(parser.scripts, cycle)
source_counts[cycle_name] = Counter()
for row in rows:
if (not isinstance(row, dict) or row.keys() != {'track', 'title', 'url', 'authors'}
or not all(isinstance(value, str) for value in row.values())
or row['track'] not in TRACKS or not row['title'].strip()):
raise ValueError('Unexpected KDD programme row')
match = DOI_URL.fullmatch(row['url'])
if not match:
raise ValueError('Invalid DOI link in KDD programme')
doi = match[1]
track = TRACKS[row['track']]
source_counts[cycle_name][track] += 1
row_hash = sha256(json.dumps(row, sort_keys=True, ensure_ascii=False).encode()).hexdigest()
if doi in records:
entry = records[doi]
if (entry['cycle'], entry['source_row_sha256']) != (cycle_name, row_hash):
raise ValueError('Conflicting duplicate DOI in KDD programme: ' + doi)
entry['occurrences'] += 1
continue
status = 'listed'
withdrawn = WITHDRAWN.fullmatch(row['title'])
if withdrawn:
status = 'withdrawn_with_replacement_mentioned'
elif OTHER_STATUS.match(row['title']):
raise ValueError('Unrecognized status annotation in KDD programme: ' + doi)
records[doi] = {
'doi': doi, 'doi_url': row['url'], 'source_title': row['title'],
'title': withdrawn[2] if withdrawn else row['title'],
'cycle': cycle_name, 'track': track, 'programme_status': status,
'authors_listed': bool(row['authors'].strip()), 'occurrences': 1,
'source_row_sha256': row_hash,
}
if withdrawn:
records[doi]['source_withdrawal_date_literal'] = withdrawn[1]
ordered = [records[doi] for doi in sorted(records)]
return ordered, source_counts
def collect(snapshot, expected_sha256, out):
snapshot = Path(snapshot)
with snapshot.open('rb') as stream:
raw = stream.read(4_000_001)
if len(raw) > 4_000_000 or sha256(raw).hexdigest() != expected_sha256:
raise ValueError('KDD programme snapshot hash or size differs')
rows, cycle_counts = inventory(raw)
result = {
'source': SOURCE_URL, 'snapshot_sha256': expected_sha256,
'source_rows_by_cycle_and_track': cycle_counts, 'unique_dois': len(rows),
'identical_repeat_rows': sum(row['occurrences'] - 1 for row in rows),
'programme_status_counts': Counter(row['programme_status'] for row in rows),
'records': rows, 'database_written': False,
}
with Path(out).open('x', encoding='utf-8') as stream:
json.dump(result, stream, ensure_ascii=False, indent=2)
stream.write('\n')
return result
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog)
parser.add_argument('--snapshot', required=True)
parser.add_argument('--snapshot-sha256', required=True)
parser.add_argument('--out', required=True)
args = parser.parse_args(argv)
result = collect(args.snapshot, args.snapshot_sha256, args.out)
print(json.dumps({k: result[k] for k in ('unique_dois', 'identical_repeat_rows', 'programme_status_counts')}))
@@ -0,0 +1,336 @@
"""Frozen MM 2026 registered publications with separately attributed program evidence.
All publications survive normalization. A program metadata match is not a
scientific-work identity decision; publisher article type is not research type.
"""
import argparse
from collections import Counter, defaultdict
from datetime import datetime
import hashlib
from html.parser import HTMLParser
import json
from pathlib import Path
import re
import unicodedata
from urllib.parse import parse_qs, urlsplit
from paper_atlas.collectors import mm2026 as program
SOURCE = 'mm_registry_program'
SNAPSHOT = 'mm2026-container-registry-v2'
MANIFEST_SHA = '128b9d02c72261a66edb9a4537d118bdccab3ed80d49a1e971fefdac399f9855'
PARENT_SHA = 'bc3ce7da2914f9ddd198c928b3acb2734a8370add3348c23c5f9f12135381628'
PARENT_DOI = '10.1145/3767308'
CONTAINER = 'Proceedings of the 34th ACM International Conference on Multimedia'
EVENT = "MM '26: The 34th ACM International Conference on Multimedia"
TRACKS = {'Main Track':'main','Grand Challenge':'grand_challenge',
'Interactive Art':'interactive_art','Datasets':'datasets','Reproducibility':'reproducibility',
'BNI':'bni','Open Source':'open_source','Doctoral Symposium':'doctoral_symposium',
'Demo and Video':'demo_and_video','Industry Talk':'industry_talk'}
BIBLIOGRAPHY = {'DOI','URL','title','original-title','subtitle','short-title','author',
'publisher','member','type','container-title','short-container-title','event','abstract',
'page','ISBN','isbn-type','published','published-print','published-online','issued',
'license','resource','relation','alternative-id'}
def digest(raw):
return hashlib.sha256(raw).hexdigest()
def key(text):
return ''.join(c for c in unicodedata.normalize('NFKD',text).casefold() if c.isalnum())
def author_names(values):
return Counter(key(v) for v in values)
def load_registry(root, parent_path):
directory = Path(root)/SNAPSHOT
raw_manifest = program.safe_read(directory/'manifest.json', 128*1024)
raw_parent = program.safe_read(Path(parent_path), 128*1024)
if digest(raw_manifest) != MANIFEST_SHA or digest(raw_parent) != PARENT_SHA:
raise ValueError('Reviewed registry or parent hash changed')
manifest = json.loads(raw_manifest);parent = json.loads(raw_parent)['message']
if (manifest['status'] != 'complete_exact_container_registry_capture_not_work_or_track_approval'
or manifest['unique_registry_DOIs'] != 1918 or manifest['expected_registry_items'] != 1918
or len(manifest['pages']) != 8 or manifest['database_written'] is not False
or manifest['approved_for_import'] != 0 or manifest['parent_body_sha256'] != PARENT_SHA
or parent['DOI'] != PARENT_DOI or parent['type'] != 'proceedings'
or parent['title'] != [CONTAINER] or parent['event']['name'] != EVENT
or parent['member'] != '320' or parent['ISBN'] != ['9798400722134']):
raise ValueError('Incomplete reviewed registry or wrong proceedings')
evidence = {'manifest.json':raw_manifest,'parent.json':raw_parent};rows=[];seen=set()
for number,receipt in enumerate(manifest['pages'],1):
name=receipt['file']
if name != f'page-{number:03}.json':
raise ValueError('Unexpected registry page filename')
body=program.safe_read(directory/name,8*1024**2)
url=urlsplit(receipt['url']);query=parse_qs(url.query)
expected_filter='container-title:'+CONTAINER+',member:320'
if (receipt['http_status'] != 200 or receipt['url'] != receipt['final_url']
or url.scheme != 'https' or url.netloc != 'api.crossref.org' or url.path != '/works'
or set(query) != {'filter','rows','cursor'} or query['filter'] != [expected_filter]
or query['rows'] != ['250'] or not query['cursor'][0]
or datetime.fromisoformat(receipt['at']).utcoffset() is None
or receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)):
raise ValueError('Registry receipt and source body disagree')
obj=json.loads(body);message=obj['message'];expected=250 if number<8 else 168
if (obj['status'] != 'ok' or message['total-results'] != 1918
or len(message['items']) != expected or receipt['items'] != expected):
raise ValueError('Incomplete registry pagination')
evidence[name]=body
for position,original in enumerate(message['items']):
doi=original.get('DOI')
if not isinstance(doi,str) or not re.fullmatch(r'10\.1145/3767308\.[0-9]+',doi) or doi in seen:
raise ValueError('Missing, foreign or repeated publication DOI')
seen.add(doi)
if (original.get('URL') != 'https://doi.org/'+doi or original.get('type') != 'proceedings-article'
or original.get('container-title') != [CONTAINER] or original.get('event',{}).get('name') != EVENT
or original.get('member') != '320' or original.get('publisher') != 'ACM'
or original.get('published') != {'date-parts':[[2026,11,9]]} or original.get('abstract')):
raise ValueError('Wrong registered publication context or changed abstract availability')
titles=original.get('title');authors=original.get('author')
if (not isinstance(titles,list) or len(titles)!=1 or not isinstance(titles[0],str) or not titles[0].strip()
or not isinstance(authors,list) or not authors):
raise ValueError('Missing publication title or authors')
names=[];flags=[]
for i,author in enumerate(authors,1):
family=author.get('family');given=author.get('given','')
if not isinstance(family,str) or not family.strip() or not isinstance(given,str):
raise ValueError('Invalid registered author fields')
name=' '.join(x.strip() for x in (given,family) if x.strip())
if '@' in name or name.casefold()=='anonymous':
raise ValueError('Unreviewed author identity')
names.append(name)
if not given.strip():flags.append({'kind':'single_registered_name_field_not_split','author_position':i})
if re.search(r'<[^>]+>|&[A-Za-z#0-9]+;',titles[0]):
flags.append({'kind':'literal_title_markup_or_entity_retained_requires_review'})
canonical=json.dumps(original,ensure_ascii=False,sort_keys=True,separators=(',',':')).encode()
rows.append({'doi':doi,'title':titles[0],'authors':names,'flags':flags,
'fields':{k:v for k,v in original.items() if k in BIBLIOGRAPHY},
'original_field_names':sorted(original),'original_canonical_json_sha256':digest(canonical),
'receipt':receipt,'source_position':position})
if len(rows)!=1918:
raise ValueError('Incomplete registered publication census')
return evidence,rows
class _TitleFormatting(HTMLParser):
"""Decode text once, removing only balanced, attribute-free reviewed tags."""
def __init__(self):
super().__init__(convert_charrefs=True)
self.parts=[];self.stack=[];self.valid=True
def handle_data(self, text):
self.parts.append(text)
def handle_starttag(self, tag, attrs):
if tag not in {'i','scp','sup','sub'} or attrs:
self.valid=False
else:
self.stack.append(tag)
def handle_endtag(self, tag):
if not self.stack or self.stack[-1]!=tag:
self.valid=False
else:
self.stack.pop()
def handle_startendtag(self, tag, attrs):
self.valid=False
def handle_comment(self, text):
self.valid=False
def handle_decl(self, text):
self.valid=False
def handle_pi(self, text):
self.valid=False
def unknown_decl(self, text):
self.valid=False
def formatting_text(title):
# HTMLParser may silently discard a trailing incomplete start tag.
# Check lexical coverage before parsing so no title content can disappear.
literal=re.sub(r'</?(?:i|scp|sup|sub)\s*>','',title,flags=re.I)
if '<' in literal or '>' in literal:
return None
parser=_TitleFormatting()
parser.feed(title);parser.close()
text=''.join(parser.parts)
return text if parser.valid and not parser.stack and text.strip() and '<' not in text and '>' not in text else None
FORMATTING_MATCHES = {'10.1145/3767308.3834907': 'mfp0053',
'10.1145/3767308.3834936': 'mfp0229',
'10.1145/3767308.3834946': 'mfp0297',
'10.1145/3767308.3834956': 'mfp0339',
'10.1145/3767308.3835035': 'mfp0798',
'10.1145/3767308.3835055': 'mfp0903',
'10.1145/3767308.3835101': 'mfp1175',
'10.1145/3767308.3835374': 'mfp2617',
'10.1145/3767308.3835424': 'mfp2909',
'10.1145/3767308.3835452': 'mfp3048',
'10.1145/3767308.3835460': 'mfp3109',
'10.1145/3767308.3835544': 'mfp3554',
'10.1145/3767308.3835591': 'mfp3783',
'10.1145/3767308.3835667': 'mfp4173',
'10.1145/3767308.3835763': 'mfp4778',
'10.1145/3767308.3835895': 'mfp5537',
'10.1145/3767308.3835959': 'mfp5823',
'10.1145/3767308.3836215': 'mfp7332',
'10.1145/3767308.3836272': 'mfp7635',
'10.1145/3767308.3836293': 'mfp7732',
'10.1145/3767308.3836447': 'mfp8648',
'10.1145/3767308.3836600': 'mfp9815',
'10.1145/3767308.3838656': 'mds177',
'10.1145/3767308.3839012': 'mde41',
'10.1145/3767308.3839024': 'mde15',
'10.1145/3767308.3839028': 'mde02'}
def match_program(rows, observations):
titles=defaultdict(list);formatted=defaultdict(list)
for p in observations:
titles[key(p['title'])].append(p)
text=formatting_text(p['title'])
if text is not None:formatted[key(text)].append(p)
matched=set();kinds=Counter()
for row in rows:
row.pop('program',None);row.pop('formatting_comparison',None)
candidates=titles[key(row['title'])]
method='literal_title'
# A literal-title author disagreement must not be bypassed by fallback.
if not candidates:
text=formatting_text(row['title'])
if text is not None:
candidates=formatted[key(text)]
if candidates:method='reviewed_title_formatting'
exact=[p for p in candidates if author_names(p['authors'])==author_names(row['authors'])]
if len(exact)==1:
row['kind']=('unique_program_format_title_fullauthor_agreement_not_identity_approval'
if method=='reviewed_title_formatting' else 'unique_program_title_fullauthor_agreement_not_identity_approval')
row['program']=exact[0];matched.add(exact[0]['paper_id'])
if exact[0]['track'] not in TRACKS:
raise ValueError('New registered program track requires review')
elif len(exact)>1:row['kind']='multiple_program_ids_same_title_authors_hold'
elif candidates:
row['kind']=('program_format_author_version_requires_review'
if method=='reviewed_title_formatting' else 'program_author_version_requires_review')
else:row['kind']='no_exact_program_title_requires_review'
row['program_title_candidates']=[p['paper_id'] for p in candidates]
row['program_exact_candidates']=[p['paper_id'] for p in exact]
if method=='reviewed_title_formatting':
row['formatting_comparison']={'method':'balanced_attribute_free_i_scp_sup_sub_and_single_entity_decode_then_nfkd_alnum',
'registry_comparison_text':formatting_text(row['title']),
'program_comparison_texts':{p['paper_id']:formatting_text(p['title']) for p in candidates},
'raw_titles_preserved':True,'work_identity_approved':False}
kinds[row['kind']]+=1
return dict(kinds),[p for p in observations if p['paper_id'] not in matched]
def correlate(rows, observations):
kinds,unmatched=match_program(rows,observations)
expected={'unique_program_title_fullauthor_agreement_not_identity_approval':1726,
'unique_program_format_title_fullauthor_agreement_not_identity_approval':26,
'program_author_version_requires_review':122,
'program_format_author_version_requires_review':1,
'no_exact_program_title_requires_review':43}
actual={r['doi']:r['program']['paper_id'] for r in rows
if r['kind']=='unique_program_format_title_fullauthor_agreement_not_identity_approval'}
if kinds != expected or actual != FORMATTING_MATCHES:
raise ValueError('Reviewed program/publication correspondence changed')
return kinds,unmatched
def collect(out, *, registry_snapshots, program_snapshots, parent_snapshot):
out=Path(out)
if out.exists() or out.is_symlink():
raise ValueError('Output exists; inspect prior operation, never overwrite')
if any(p.is_symlink() for p in out.parents):
raise ValueError('Unsafe output parent')
out.mkdir(parents=True)
try:
registry_evidence,publications=load_registry(registry_snapshots,parent_snapshot)
program_evidence=program.load_snapshot(program_snapshots)
observations,_=program.parse_program(program_evidence['program-data.json'])
kinds,unmatched=correlate(publications,observations)
for group,files in [('registry',registry_evidence),('program',program_evidence)]:
directory=out/'evidence'/group;directory.mkdir(parents=True)
for name,body in files.items():(directory/name).write_bytes(body)
buckets=defaultdict(list);missing=0;copied=0;known_tracks=Counter()
for item in sorted(publications,key=lambda r:r['doi']):
p=item.get('program');doi=item['doi'];receipt=item['receipt']
track=TRACKS[p['track']] if p else 'proceedings_unclassified'
abstract=p['abstract'] if p and p['abstract'] else None
missing+=abstract is None;copied+=abstract is not None
if p:known_tracks[p['track']]+=1
tags=[{'namespace':SOURCE+':crossref:type','value':'proceedings-article'}]
if p:tags.append({'namespace':SOURCE+':mm_program:track','value':p['track']})
buckets[track].append({'source':SOURCE,'external_id':doi,'url':'https://doi.org/'+doi,
'title':item['title'],'authors':item['authors'],'abstract':abstract,
'identifiers':{'doi':doi},'submission_identifiers':[{'scheme':'publication_doi','value':doi}],
'venue':'MM','year':2026,'track':track,'status':'accepted',
'raw_decision':'Registered publication in MM2026 proceedings; not an original signed decision.',
'evidence_url':receipt['url'],'observed_at':receipt['at'],'tags':tags,
'metadata':{**({'program_title_formatting_comparison':item['formatting_comparison']} if 'formatting_comparison' in item else {}),'registry_fields':item['fields'],'registry_original_field_names':item['original_field_names'],
'registry_source_evidence':{'file':'evidence/registry/'+receipt['file'],'sha256':receipt['sha256'],
'url':receipt['url'],'json_pointer':'/message/items/'+str(item['source_position']),
'canonical_object_json_sha256':item['original_canonical_json_sha256']},
'registry_manifest':{'file':'evidence/registry/manifest.json','sha256':MANIFEST_SHA},
'parent_evidence':{'file':'evidence/registry/parent.json','sha256':PARENT_SHA},
'program_source_evidence':{'file':'evidence/program/program-data.json','sha256':program.BODY_SHA},
'program_fields':p,'program_match_kind':item['kind'],
'program_title_candidates':item['program_title_candidates'],
'program_exact_candidates':item['program_exact_candidates'],
'abstract_source':('official_program_reviewed_title_formatting_correspondence' if item.get('formatting_comparison') else 'official_program_exact_metadata_correspondence') if abstract is not None else 'not_available_from_reviewed_sources',
'registry_abstract_available':False,'original_decision_signature_verified':False,
'scientific_work_identity_approved':False,'research_type_classified':False,
'review_flags':item['flags'],
'scope':'Derived publication/program metadata with separate original evidence. All registered articles retained, including art/industry/demo; bibliographic type is not research type. Program correlation does not approve work identity or person aliases. Publication dates are not decision dates. DOI is publication evidence, not original OpenReview ID.'}})
outputs=[]
for track,rows in sorted(buckets.items()):
path=out/f'mm-publications-2026-{track}.jsonl'
raw=''.join(json.dumps(row,ensure_ascii=False)+'\n' for row in rows).encode();path.write_bytes(raw)
manifest={'source':SOURCE,'parser_version':2,'venue':'MM','year':2026,'track':track,'count':len(rows),
'file':str(path),'file_sha256':digest(raw),'list_url':'https://doi.org/'+PARENT_DOI,
'status':'partial','missing_abstract_count':sum(row['abstract'] is None for row in rows),
'scope':'Registered container partition only; not allMMprogram/workshops, independent works or public-negative coverage.'}
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,ensure_ascii=False,indent=2)+'\n');outputs.append(manifest)
report={'status':'collected_publications_pending_work_identity_review','records':1918,'unique_registered_dois':1918,
'files':outputs,'program_ids':2143,'program_observations':4326,'program_match_counts':kinds,
'matched_program_track_counts':dict(known_tracks),'unmatched_program':unmatched,
'unmatched_program_count':len(unmatched),'registry_missing_abstract_count':1918,
'abstracts_from_exact_program_correspondence':copied,'output_missing_abstract_count':missing,
'abstracts_from_literal_program_correspondence':copied-26,'abstracts_from_reviewed_title_formatting':26,
'registered_single_name_fields':sum(f['kind']=='single_registered_name_field_not_split' for r in publications for f in r['flags']),
'registered_title_markup_flags':sum(any(f['kind']=='literal_title_markup_or_entity_retained_requires_review' for f in r['flags']) for r in publications),
'database_written':False,'approved_for_import':0,'network_requests':0,'pdfs_downloaded':False,
'public_negative_availability':'unknown_no_negative_decisions_inferred',
'scope':'1918registeredpublication DOIs, not1918proven distinct scientificworks.166title/authorvariants remainunclassified,391unmatchedprogram items retained.26formattingcorrespondences retainedseparately;rawtitlesunchanged. All28programtracks remain independent census; article type never implies research. Per-work/source/DOI/history review andfinalwholeDBaudit remain.'}
if (sum(f['count'] for f in outputs)!=1918 or copied!=1743 or missing!=175 or len(unmatched)!=391):
raise ValueError('Reviewed output partition or abstract census changed')
part=out/'collection-report.json.part';part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n');part.replace(out/'collection-report.json')
return report
except Exception as error:
(out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n')
raise
def main(argv=None, *, prog=None):
parser=argparse.ArgumentParser(prog=prog,description=__doc__)
parser.add_argument('--registry-snapshots',required=True)
parser.add_argument('--program-snapshots',required=True)
parser.add_argument('--parent-snapshot',required=True)
parser.add_argument('--out',required=True)
args=parser.parse_args(argv)
report=collect(args.out,registry_snapshots=args.registry_snapshots,
program_snapshots=args.program_snapshots,parent_snapshot=args.parent_snapshot)
print(json.dumps({k:report[k] for k in ('status','records','files','program_match_counts',
'abstracts_from_exact_program_correspondence','output_missing_abstract_count',
'unmatched_program_count','database_written','approved_for_import')},ensure_ascii=False))
+216
View File
@@ -0,0 +1,216 @@
"""Source-attributed mirror metadata; no paper/review text or identity promotion."""
import argparse
from collections import Counter
from datetime import datetime, timezone
import hashlib
from importlib.resources import files
import json
from pathlib import Path
import re
from paper_atlas.io.parquet_metadata import RangeSource, projection
DATASET = 'anonymousNeurIPS2026submission4281/reviewarena'
COMMIT = 'c2978add17c2099219eaddbc2599974d69d4d09b'
TREE_SHA256 = 'bbc75f5774b019c4a9ee3efe5c0f37b14761bc3c5a054570db1df8f5603eac20'
SOURCE = 'reviewarena_mirror'
SPLITS = ('colm', 'corl', 'emnlp', 'iclr', 'icml', 'neurips', 'tmlr')
COLUMNS = {'forum_id': 'string', 'conference': 'string', 'year': 'integer',
'track': 'string', 'venue_id': 'string', 'paper_number': 'integer',
'title': 'string', 'abstract': 'string', 'authors': 'strings',
'keywords': 'strings', 'venue': 'string', 'decision': 'string'}
def inventory(split):
if split not in SPLITS:
raise ValueError('Explicit supported mirror split required')
raw = files('paper_atlas.resources').joinpath('reviewarena-files.json').read_bytes()
if hashlib.sha256(raw).hexdigest() != TREE_SHA256:
raise ValueError('Pinned mirror file inventory changed')
items = json.loads(raw)
if len(items) != 15:
raise ValueError('Pinned mirror shard inventory changed')
selected = []
for item in items:
path = item['path']
if not re.fullmatch(r'data/(colm|corl|emnlp|iclr|icml|neurips|tmlr)-\d{5}-of-\d{5}\.parquet', path):
raise ValueError('Unexpected mirror shard path')
if (item['type'] != 'file' or type(item['size']) is not int or item['size'] < 12
or item['lfs']['size'] != item['size'] or not re.fullmatch('[0-9a-f]{64}', item['lfs']['oid'])):
raise ValueError('Missing source object size/checksum claim')
if path.startswith('data/'+split+'-'):
selected.append(item)
parts = sorted(selected, key=lambda x: x['path'])
if not parts or any(int(x['path'][-13:-8]) != len(parts) for x in parts):
raise ValueError('Incomplete selected mirror split inventory')
if [int(x['path'].split('-')[-3]) for x in parts] != list(range(len(parts))):
raise ValueError('Mirror shard sequence changed')
return parts
def decision_claim(raw):
value = raw.strip().casefold() if isinstance(raw, str) else ''
if value in {'reject', 'rejected'}:
return 'rejected'
if value in {'desk reject', 'desk rejected', 'desk_rejected'}:
return 'desk_rejected'
if value in {'withdraw', 'withdrawn'}:
return 'withdrawn'
if value in {'accept', 'accepted', 'accept-main', 'accept-findings', 'accept (main)', 'accept (findings)',
'accept (poster)', 'accept (oral)', 'accept (spotlight)', 'accept (talk)'}:
return 'accepted'
return 'unknown'
def decision_track_claim(raw):
value = raw.strip().casefold() if isinstance(raw, str) else ''
if value in {'accept-findings', 'accept (findings)'}:
return 'findings'
if value in {'accept-main', 'accept (main)'}:
return 'main'
return None
def record(row, split, item, row_index):
if set(row) != set(COLUMNS):
raise ValueError('Unexpected projected metadata fields')
if (not isinstance(row['conference'], str) or row['conference'].casefold() != split
or type(row['year']) is not int or not 2020 <= row['year'] <= 2026):
raise ValueError('Mirror conference/year outside pinned source scope')
locator = {'dataset': DATASET, 'commit': COMMIT, 'file': item['path'], 'row_index': row_index,
'lfs_sha256_claim_not_full_file_verification': item['lfs']['oid']}
return {'source': SOURCE, 'external_id': COMMIT+':'+item['path']+':'+str(row_index),
'url': 'https://huggingface.co/datasets/'+DATASET+'/blob/'+COMMIT+'/'+item['path'],
'title': row['title'], 'abstract': row['abstract'], 'authors': row['authors'],
'conference': row['conference'], 'year': row['year'], 'track': row['track'],
'mirror_decision_category': decision_claim(row['decision']),
'mirror_decision_track_claim': decision_track_claim(row['decision']), 'raw_fields': row,
'source_locator': locator, 'identifiers': {},
'metadata': {'forum_id_claim': row['forum_id'], 'original_forum_id_verified': False,
'original_decision_signature_verified': False, 'source_visibility': 'public_mirror',
'scientific_work_identity_approved': False, 'research_type_classified': False,
'approved_for_import': False,
'scope': 'Mirror row only. No original forum/person/decision verification; absence is not rejection or proof of no later acceptance.'}}
def cached_projections(path, expected_sha256, split, items):
path = Path(path)
raw = path.read_bytes()
if (not isinstance(expected_sha256, str) or not re.fullmatch('[0-9a-f]{64}', expected_sha256)
or hashlib.sha256(raw).hexdigest() != expected_sha256):
raise ValueError('Cached projection report hash differs')
report = json.loads(raw)
if (report.get('dataset') != DATASET or report.get('commit') != COMMIT
or report.get('split') != split or report.get('inventory_sha256') != TREE_SHA256
or report.get('raw_parquet_saved') is not False or report.get('database_written') is not False
or report.get('paper_body_saved') is not False or report.get('review_text_saved') is not False
or report.get('approved_for_import') != 0):
raise ValueError('Cached report scope differs from permitted mirror projection')
declared = report.get('files', [])
if len(declared) != len(items) or len({f['source_file'] for f in declared}) != len(items):
raise ValueError('Incomplete cached mirror shard set')
result = {}
for item in items:
fm = next((f for f in declared if f['source_file'] == item['path']), None)
filename = Path(item['path']).stem+'.metadata.jsonl'
if not fm or Path(fm['file']).name != filename or fm['source_lfs_sha256_claim'] != item['lfs']['oid']:
raise ValueError('Cached mirror source differs')
file = path.parent/filename
if file.is_symlink():
raise ValueError('Symlink metadata cache forbidden')
content = file.read_bytes()
if hashlib.sha256(content).hexdigest() != fm['file_sha256']:
raise ValueError('Cached metadata file hash differs')
rows = []
for index, line in enumerate(content.splitlines()):
previous = json.loads(line); row = previous['raw_fields']; expected = record(row, split, item, index)
for key in ('source', 'external_id', 'url', 'title', 'abstract', 'authors', 'conference', 'year', 'track', 'raw_fields', 'source_locator', 'identifiers', 'metadata'):
if previous.get(key) != expected[key]:
raise ValueError('Cached raw metadata/identity differs: '+key)
rows.append(row)
proof = fm['projection']
if (len(rows) != fm['records'] or len(rows) != proof['rows'] or proof['columns'] != COLUMNS
or proof['source_bytes'] != item['size'] or proof['raw_parquet_saved'] is not False
or proof['footer_saved'] is not False or proof['excluded_payload_pages_downloaded'] is not False
or proof['full_file_checksum_verified'] is not False):
raise ValueError('Cached projection proof differs')
result[item['path']] = (rows, proof)
if sum(len(rows) for rows, _ in result.values()) != report['records']:
raise ValueError('Cached projection row count differs')
return result
def collect(split, out, *, source_factory=RangeSource, progress=None, from_projection=None, projection_sha256=None):
items = inventory(split); out = Path(out)
if bool(from_projection) != bool(projection_sha256):
raise ValueError('Cached projection and exact report hash required together')
if out.exists() or not out.parent.is_dir() or out.is_symlink():
raise ValueError('New output directory in existing parent required')
out.mkdir()
counts = Counter(); track_claims = Counter(); raw_counts = Counter(); years = Counter(); records = 0; reports = []
observed = datetime.now(timezone.utc).isoformat()
try:
cached = cached_projections(from_projection, projection_sha256, split, items) if from_projection else None
for item in items:
if cached is None:
source_url = 'https://huggingface.co/datasets/'+DATASET+'/resolve/'+COMMIT+'/'+item['path']
source = source_factory(source_url, item['size'])
rows, proof = projection(source, COLUMNS, progress=progress)
else:
rows, proof = cached[item['path']]
if progress:
progress({'phase': 'cached_metadata_projection', 'file': item['path'], 'rows': len(rows)})
path = out/(Path(item['path']).stem+'.metadata.jsonl')
part = path.with_suffix(path.suffix+'.part'); h = hashlib.sha256()
with part.open('wb') as stream:
for index, row in enumerate(rows):
item_record = record(row, split, item, index)
raw = (json.dumps(item_record, ensure_ascii=False)+'\n').encode()
stream.write(raw); h.update(raw); records += 1
counts[item_record['mirror_decision_category']] += 1
track_claims[item_record['mirror_decision_track_claim'] or 'unspecified'] += 1
raw_counts[json.dumps(row['decision'], ensure_ascii=False)] += 1
years[(row['year'], row['track'])] += 1
part.rename(path)
reports.append({'source_file': item['path'], 'file': str(path), 'records': len(rows),
'file_sha256': h.hexdigest(), 'source_lfs_sha256_claim': item['lfs']['oid'],
'projection': proof})
report = {'status': 'collected_mirror_metadata_pending_original_provenance_and_work_identity',
'dataset': DATASET, 'commit': COMMIT, 'inventory_sha256': TREE_SHA256,
'split': split, 'observed_at': observed, 'records': records, 'files': reports,
'mirror_decision_counts': dict(counts), 'raw_mirror_decision_counts': dict(raw_counts),
'mirror_decision_track_claim_counts': dict(track_claims),
'year_track_counts': [{'year': y, 'track': t, 'rows': n} for (y, t), n in sorted(years.items(), key=lambda x: (x[0][0], str(x[0][1])))],
'metadata_bytes_read': sum(x['projection']['metadata_bytes_read'] for x in reports) if cached is None else 0,
'source_projection_bytes': sum(x['projection']['metadata_bytes_read'] for x in reports),
'cached_projection_report_sha256': projection_sha256,
'excluded_payload_pages_downloaded': False, 'raw_parquet_saved': False,
'paper_body_saved': False, 'review_text_saved': False, 'database_written': False,
'approved_for_import': 0,
'limitations': ['Public mirror copies do not verify original forum IDs, visibility or signed decisions.',
'Original source LFS SHA256 is a pinned claim, not a computed full-Parquet checksum.',
'Every selected row and row locator is retained, including missing or anonymous author data.',
'Decision categories describe only explicit mirror literals; venue labels and missing decisions do not imply rejection.',
'Rows are not independent scientific works or a complete venue/year public negative census.',
'Cross-source identity, chronological rejection/acceptance history and final dedup remain.']}
(out/'collection-report.json').write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n')
return report
except BaseException as error:
# No source-specific body or signed redirect URLs in failure reports.
failure = {'status': 'incomplete_do_not_import', 'split': split, 'commit': COMMIT,
'completed_files': reports, 'error_type': type(error).__name__,
'database_written': False, 'approved_for_import': 0, 'no_automatic_retry': True}
(out/'failure.json').write_text(json.dumps(failure, ensure_ascii=False, indent=2)+'\n')
raise
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog)
parser.add_argument('--split', required=True, choices=SPLITS)
parser.add_argument('--out', required=True)
parser.add_argument('--from-projection', help='Offline replay of a complete permitted metadata projection')
parser.add_argument('--projection-sha256', help='Exact cached collection-report.json SHA256')
args = parser.parse_args(argv)
report = collect(args.split, args.out, from_projection=args.from_projection, projection_sha256=args.projection_sha256,
progress=lambda row: print(json.dumps(row), flush=True))
print(json.dumps({'status': report['status'], 'records': report['records'], 'out': args.out}), flush=True)
+3 -81
View File
@@ -1,16 +1,14 @@
"""Reviewed postconference selected volumes from frozen public HTML metadata.""" """Reviewed postconference selected volumes from frozen public HTML metadata."""
import argparse import argparse
from collections import defaultdict
import fcntl import fcntl
import hashlib import hashlib
import json import json
from pathlib import Path from pathlib import Path
import re import re
import tempfile import tempfile
from urllib.parse import parse_qs, urlsplit
from paper_atlas.db.submissions import timestamp
from paper_atlas.io.html import Document, clean from paper_atlas.io.html import Document, clean
from paper_atlas.io.springer import parse_chapter as parse_citation_chapter, read_page, snapshot_index
SOURCE = 'springer_selected_volume' SOURCE = 'springer_selected_volume'
BOOK = '10.1007/978-1-4612-2660-4' BOOK = '10.1007/978-1-4612-2660-4'
@@ -26,50 +24,6 @@ SCOPE = {
} }
def snapshot_index(root):
root = Path(root).resolve()
receipts = json.loads((root/'receipts.json').read_bytes())['responses']
index = {}
for receipt in receipts:
url = receipt['url']
if url in index:
raise ValueError('Duplicate snapshot URL')
index[url] = (root, receipt)
return index
def read_page(index, url):
if url not in index:
raise ValueError('Missing required public HTML snapshot: '+url)
root, receipt = index[url]
filename = Path(receipt['file'])
path = root/filename
content_type = {k.lower():v for k,v in receipt.get('headers', {}).items()}.get('content-type', '')
requested, final = urlsplit(url), urlsplit(receipt.get('final_url', ''))
query = parse_qs(final.query, keep_blank_values=True)
original_query = parse_qs(requested.query, keep_blank_values=True)
# Springer returns the public metadata with an explicit cookie diagnostic.
# Retain that final URL verbatim; accept no other redirect or changed page.
if query != original_query and requested.netloc == 'link.springer.com':
if (query.pop('error', None) != ['cookies_not_supported']
or not re.fullmatch(r'[0-9a-f-]{36}', ''.join(query.pop('code', [])))):
raise ValueError('Unreviewed Springer redirect query')
if (filename.is_absolute() or '..' in filename.parts or not path.resolve().is_relative_to(root)
or receipt.get('http_status') != 200
or (final.scheme, final.netloc, final.path) != (requested.scheme, requested.netloc, requested.path)
or final.fragment or query != original_query
or 'text/html' not in content_type.lower()):
raise ValueError('Unsafe, redirected or unsuccessful public HTML snapshot')
raw = path.read_bytes()
digest = hashlib.sha256(raw).hexdigest()
if len(raw) > 2_000_000 or len(raw) != receipt['bytes'] or digest != receipt['sha256']:
raise ValueError('Snapshot checksum or size differs')
evidence = {'url':url, 'sha256':digest, 'bytes':len(raw),
'file':'evidence/'+digest+'.html', 'http_status':200,
'observed_at':timestamp(receipt['at']), 'response_receipt':receipt}
return raw, evidence
def parse_toc(html, *, excluded_materials=None): def parse_toc(html, *, excluded_materials=None):
entries = {} entries = {}
for node in Document(html).root.walk(): for node in Document(html).root.walk():
@@ -97,40 +51,8 @@ def parse_toc(html, *, excluded_materials=None):
def parse_chapter(html, doi): def parse_chapter(html, doi):
nodes = list(Document(html).root.walk()) return parse_citation_chapter(html, doi, book_title=SCOPE['book_title'],
meta = defaultdict(list) publication_year=SCOPE['publication_year'])
for node in nodes:
if node.tag == 'meta' and node.attrs.get('name', '').startswith('citation_'):
meta[node.attrs['name']].append(node.attrs.get('content', ''))
def one(key):
values = meta[key]
if len(values) != 1 or not values[0]:
raise ValueError('Missing or ambiguous citation field: '+key)
return values[0]
if (one('citation_doi') != doi or one('citation_inbook_title') != SCOPE['book_title']
or one('citation_publication_date') != str(SCOPE['publication_year'])):
raise ValueError('Chapter DOI, book or publication year differs from reviewed scope')
raw_authors = meta['citation_author']
if not raw_authors:
raise ValueError('Missing named chapter authors')
authors = []
for raw in raw_authors:
if raw.count(',') != 1:
raise ValueError('Unreviewed citation author format')
family, given = [clean(s) for s in raw.split(',', 1)]
if not family or not given:
raise ValueError('Incomplete citation author')
authors.append({'given':given, 'family':family})
abstract_nodes = [n for n in nodes if n.attrs.get('id') == 'Abs1-content']
if len(abstract_nodes) > 1:
raise ValueError('Ambiguous abstract section')
abstract = clean(abstract_nodes[0].text()) if abstract_nodes else None
first, last = one('citation_firstpage'), one('citation_lastpage')
if not first.isdigit() or not last.isdigit() or int(first) > int(last):
raise ValueError('Invalid chapter pages')
return {'doi':doi, 'title':one('citation_title'), 'authors':authors,
'authors_citation_raw':list(raw_authors), 'abstract':abstract or None,
'first_page':first, 'last_page':last}
def record_for(chapter, evidence, scope_evidence): def record_for(chapter, evidence, scope_evidence):
+128
View File
@@ -0,0 +1,128 @@
"""Versioned publication-source scopes and guards against stale source refreshes."""
import json
import re
TABLE = 'source_scope_revisions'
MARKER = 'source_scope_revisions_v1'
COLUMNS = ('version_sha256', 'source_id', 'venue_id', 'year', 'event_key', 'list_url',
'parent_sha256', 'records_json', 'plan_json', 'applied_at')
def available(c):
if getattr(c, 'postgres', False):
return bool(c.execute("SELECT 1 FROM information_schema.tables WHERE table_schema=current_schema() AND table_name=?", (TABLE,)).fetchone())
return bool(c.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,)).fetchone())
def initialize(c):
exists = available(c)
marker = bool(c.execute('SELECT 1 FROM schema_migrations WHERE name=?', (MARKER,)).fetchone())
if exists != marker:
raise ValueError('Source revision table/marker mismatch')
if exists:
result = c.execute('SELECT * FROM source_scope_revisions LIMIT 0')
description = result.cursor.description if getattr(c, 'postgres', False) else result.description
if tuple(x[0] for x in description) != COLUMNS:
raise ValueError('Unexpected source revision schema')
return
ddl = '''CREATE TABLE source_scope_revisions (
version_sha256 TEXT PRIMARY KEY,
source_id INTEGER NOT NULL REFERENCES sources(id),
venue_id INTEGER NOT NULL REFERENCES venues(id),
year INTEGER NOT NULL, event_key TEXT NOT NULL, list_url TEXT NOT NULL,
parent_sha256 TEXT UNIQUE REFERENCES source_scope_revisions(version_sha256),
records_json TEXT NOT NULL, plan_json TEXT NOT NULL,
applied_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP)'''
if getattr(c, 'postgres', False):
ddl = re.sub(r'\bINTEGER\b', 'BIGINT', ddl)
ddl = re.sub(r'\bTEXT\b', 'TEXT COLLATE "C"', ddl)
c.execute(ddl)
c.execute('CREATE INDEX source_scope_revisions_scope ON source_scope_revisions(source_id,venue_id,year,event_key,list_url)')
c.execute('INSERT INTO schema_migrations(name) VALUES(?)', (MARKER,))
def heads(c, source_id):
if not available(c):
return []
if not c.execute('SELECT 1 FROM schema_migrations WHERE name=?', (MARKER,)).fetchone():
raise ValueError('Source revision table/marker mismatch')
return c.execute('''SELECT v.version_sha256,v.venue_id,v.year,v.event_key,v.list_url
FROM source_scope_revisions v WHERE v.source_id=? AND NOT EXISTS
(SELECT 1 FROM source_scope_revisions child WHERE child.parent_sha256=v.version_sha256)
ORDER BY v.version_sha256''', (source_id,)).fetchall()
def current_rows(c, source_id, cache):
active = heads(c, source_id)
signatures = tuple(r[0] for r in active)
# A new head invalidates the cache, even when maintenance uses the same Store.
if cache.get('signatures') != (source_id, signatures):
records = {}; scopes = {}
for signature in signatures:
raw = c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone()[0]
scopes[signature] = json.loads(raw)
for row in scopes[signature]:
ext = row['external_id']
if ext in records and records[ext] != row:
raise ValueError('Conflicting active source snapshots')
records[ext] = row
cache.clear()
cache.update(signatures=(source_id, signatures), records=records, scopes=scopes)
return cache['records']
def subset(expected, actual):
"""Reviewed source fields must survive additional work-identity/curation evidence."""
return isinstance(actual, dict) and all(k in actual and actual[k] == v for k, v in expected.items())
def require_record(c, source_id, external, url, title, abstract, authors, identifiers, metadata, cache):
expected = current_rows(c, source_id, cache).get(external)
if expected is None:
return
actual = {'url': url, 'title': title, 'abstract': abstract, 'authors': authors}
if any(actual[k] != expected.get(k) for k in actual):
raise ValueError('Stale source snapshot: reconcile the source version before refresh')
if not subset(expected.get('metadata', {}), metadata or {}) or not subset(expected.get('identifiers', {}), identifiers or {}):
raise ValueError('Stale source snapshot metadata or identifiers')
def scope_head(c, source_id, manifest):
venue = c.execute('SELECT id FROM venues WHERE acronym=?', (manifest.get('venue'),)).fetchone()
if not venue:
return None
matches = [r for r in heads(c, source_id)
if [r[i] for i in range(1, 5)] == [venue[0], manifest.get('year'), manifest.get('event_key', ''), manifest.get('list_url')]]
if len(matches) > 1:
raise ValueError('Forked source revision history')
return matches[0][0] if matches else None
def require_manifest(c, source_id, manifest, cache):
current_rows(c, source_id, cache)
signature = scope_head(c, source_id, manifest)
if signature is None:
return
expected = sum(r['track'] == manifest.get('track') for r in cache['scopes'][signature])
bound = manifest.get('coverage_expected_count') if manifest.get('coverage_mode') == 'incremental' else manifest.get('count')
if expected < 1 or bound != expected or manifest.get('status') != 'partial':
raise ValueError('Source coverage manifest differs from current source snapshot')
def require_row(c, source_id, row, manifest, cache):
expected = current_rows(c, source_id, cache).get(row['external_id'])
scoped_manifest = {**manifest, 'venue': row['venue'], 'year': row['year'], 'event_key': row.get('event_key', '')}
signature = scope_head(c, source_id, scoped_manifest)
if expected is None:
if signature is not None:
raise ValueError('Publication absent from current source snapshot; reconcile source membership first')
return
if signature is None or not any(r['external_id'] == row['external_id'] for r in cache['scopes'][signature]):
raise ValueError('Source listing differs from current source snapshot')
for key, value in expected.items():
if key in ('metadata', 'identifiers'):
okay = subset(value, row.get(key))
else:
okay = row.get(key) == value
if not okay:
raise ValueError('Stale source snapshot field: ' + key)
+4
View File
@@ -96,6 +96,10 @@ class Store:
if found: if found:
matches.add(found[0]) matches.add(found[0])
old = c.execute('SELECT paper_id,title,abstract,retrieved_at,metadata_json FROM source_records WHERE source_id=? AND external_id=?',(source_id:=self.source(source,url),external_id)).fetchone() old = c.execute('SELECT paper_id,title,abstract,retrieved_at,metadata_json FROM source_records WHERE source_id=? AND external_id=?',(source_id:=self.source(source,url),external_id)).fetchone()
from paper_atlas.db.source_revisions import require_record
if not hasattr(self, '_source_revision_cache'):
self._source_revision_cache = {}
require_record(c, source_id, external_id, url, title, abstract, authors, identifiers, metadata, self._source_revision_cache)
from paper_atlas.db.edition_scopes import preserve from paper_atlas.db.edition_scopes import preserve
metadata = preserve(dict(metadata or {}), old[4] if old else None) metadata = preserve(dict(metadata or {}), old[4] if old else None)
if old: if old:
+8 -1
View File
@@ -66,7 +66,7 @@ def require_incremental_parent(c, source, manifest, options):
if entries: if entries:
if len(entries) != 1 or entries[0][0] != manifest['list_url']: if len(entries) != 1 or entries[0][0] != manifest['list_url']:
raise ValueError('Incremental coverage requires one unambiguous source listing') raise ValueError('Incremental coverage requires one unambiguous source listing')
if tuple(entries[0][1:]) != ('partial', prior_expected, previous): if tuple(entries[0][i] for i in (1, 2, 3)) != ('partial', prior_expected, previous):
raise ValueError('Incremental coverage parent changed or is not partial') raise ValueError('Incremental coverage parent changed or is not partial')
elif previous != 0 or prior_expected is not None: elif previous != 0 or prior_expected is not None:
raise ValueError('Incremental coverage parent is absent') raise ValueError('Incremental coverage parent is absent')
@@ -101,6 +101,9 @@ def ingest_file(db,path, *, pg_service=None, pg_schema=None):
c.commit() c.commit()
if incremental is not None: if incremental is not None:
require_incremental_parent(c, source_id, manifest, incremental) require_incremental_parent(c, source_id, manifest, incremental)
from paper_atlas.db.source_revisions import require_manifest
store._source_revision_cache = {}
require_manifest(c, source_id, manifest, store._source_revision_cache)
digest = hashlib.sha256() digest = hashlib.sha256()
with path.open('rb') as stream: with path.open('rb') as stream:
for line in stream: for line in stream:
@@ -113,6 +116,10 @@ def ingest_file(db,path, *, pg_service=None, pg_schema=None):
require_track_record(row, manifest) require_track_record(row, manifest)
from paper_atlas.db.events import validate_event_key from paper_atlas.db.events import validate_event_key
validate_event_key(row.get('event_key')) validate_event_key(row.get('event_key'))
from paper_atlas.db.source_revisions import require_row
if not hasattr(store, '_source_revision_cache'):
store._source_revision_cache = {}
require_row(c, source_id, row, manifest, store._source_revision_cache)
external = row['external_id'] external = row['external_id']
if external in seen: if external in seen:
raise ValueError('Duplicate external ID in one collected file: '+external) raise ValueError('Duplicate external ID in one collected file: '+external)
+327
View File
@@ -0,0 +1,327 @@
"""Plan-bound publication scope revisions with complete before-images, never reimport."""
import argparse
from collections import Counter
from contextlib import closing
import copy
import hashlib
import json
from pathlib import Path
import re
from paper_atlas.db import source_revisions as versions
from paper_atlas.db.backend import database_arguments, open_read_connection, open_store
from paper_atlas.db.edition_scopes import KEY
from paper_atlas.ingestion.edition_tracks import curation
from paper_atlas.ingestion.event_identity import database_identity, digest, values
MAX_ROWS = 10000
def validate(inventory):
required = {'format', 'source', 'venue', 'year', 'event_key', 'list_url',
'previous_rows', 'rows', 'evidence'}
if not isinstance(inventory, dict) or set(inventory) != required or inventory['format'] != 1:
raise ValueError('Unsupported source revision inventory')
if (type(inventory['year']) is not int or inventory['year'] < 2000
or not all(isinstance(inventory[k], str) and inventory[k] for k in ('source', 'venue', 'list_url'))
or not isinstance(inventory['event_key'], str)):
raise ValueError('Explicit publication source scope required')
if not inventory['list_url'].startswith('https://'):
raise ValueError('HTTPS source listing required')
evidence = inventory['evidence']
if not isinstance(evidence, list) or not 1 <= len(evidence) <= 100:
raise ValueError('Bounded source evidence required')
for item in evidence:
if (not isinstance(item, dict) or set(item) != {'url', 'sha256'}
or not isinstance(item['url'], str) or not item['url'].startswith('https://')
or not isinstance(item['sha256'], str) or not re.fullmatch('[0-9a-f]{64}', item['sha256'])):
raise ValueError('Source evidence URL/hash required')
snapshots = []
for key in ('previous_rows', 'rows'):
items = inventory[key]
if not isinstance(items, list) or not 1 <= len(items) <= MAX_ROWS:
raise ValueError('Complete bounded source snapshots required')
records = {}
for row in items:
if not isinstance(row, dict) or any(row.get(k) != inventory[k] for k in ('source', 'venue', 'year')):
raise ValueError('Source snapshot scope mismatch')
ext = row.get('external_id')
if (not isinstance(ext, str) or ext in records or row.get('status') != 'accepted'
or not re.fullmatch(r'10\.\d{4,9}/\S+', ext)
or row.get('identifiers', {}).get('doi') != ext
or not isinstance(row.get('track'), str) or not row['track']
or row.get('event_key', '') != inventory['event_key']
or not isinstance(row.get('title'), str) or not row['title']
or not isinstance(row.get('authors'), list)
or not isinstance(row.get('metadata'), dict)):
raise ValueError('Unique DOI-bearing accepted publication records required')
records[ext] = row
snapshots.append(records)
previous, current = snapshots
if set(previous) != set(current):
raise ValueError('Source membership changed; this operation cannot add or remove publications')
for ext, row in current.items():
before = previous[ext]
stable = (set(before) | set(row)) - {'abstract', 'track', 'tags', 'metadata'}
if any(before.get(k) != row.get(k) for k in stable):
raise ValueError('Stable publication identity or decision changed')
return previous, current
def bounded(c, sql, params):
result = values(c.execute(sql + ' LIMIT 100001', params))
if len(result) > 100000:
raise ValueError('Source revision protection exceeds bounded scope')
return result
def capture(c, inventory):
previous, current = validate(inventory)
source = c.execute('SELECT id FROM sources WHERE name=?', (inventory['source'],)).fetchone()
venue = c.execute('SELECT id FROM venues WHERE acronym=?', (inventory['venue'],)).fetchone()
if not source or not venue:
raise ValueError('Existing source and venue required')
sid, vid = source[0], venue[0]
editions = bounded(c, 'SELECT * FROM editions WHERE venue_id=? AND year=? AND event_key=? ORDER BY id',
(vid, inventory['year'], inventory['event_key']))
by_track = {r[3]: r[0] for r in editions}
# The reviewed scope includes all its existing target attempts, even when
# they originate from another source. No empty-target shortcut is used.
scope_subs = 'SELECT s.id FROM submissions s JOIN editions e ON e.id=s.edition_id WHERE e.venue_id=? AND e.year=? AND e.event_key=?'
scope_params = (vid, inventory['year'], inventory['event_key'])
scoped = bounded(c, 'SELECT * FROM submissions WHERE id IN (' + scope_subs + ') ORDER BY id', scope_params)
own_records = bounded(c, '''SELECT DISTINCT r.* FROM source_records r JOIN submission_sources ss ON ss.source_record_id=r.id
WHERE r.source_id=? AND ss.submission_id IN (''' + scope_subs + ') ORDER BY r.id', (sid, *scope_params))
if not own_records:
raise ValueError('Existing source observations required')
if any(r[2] not in current for r in own_records):
raise ValueError('Snapshot omits an existing source observation')
pids = sorted({r[1] for r in scoped})
if not pids or len(pids) > MAX_ROWS:
raise ValueError('Bounded nonempty protected paper scope required')
marks = ','.join('?' for _ in pids)
papers = 'SELECT id FROM papers WHERE id IN (' + marks + ')'
subs = 'SELECT id FROM submissions WHERE paper_id IN (' + papers + ')'
decisions = 'SELECT id FROM decisions WHERE submission_id IN (' + subs + ')'
queries = {
'papers': 'SELECT * FROM papers WHERE id IN (' + papers + ') ORDER BY id',
'identifiers': 'SELECT * FROM identifiers WHERE paper_id IN (' + papers + ') ORDER BY scheme,value',
'source_records': 'SELECT * FROM source_records WHERE paper_id IN (' + papers + ') ORDER BY id',
'submissions': 'SELECT * FROM submissions WHERE paper_id IN (' + papers + ') ORDER BY id',
'decisions': 'SELECT * FROM decisions WHERE submission_id IN (' + subs + ') ORDER BY id',
'submission_sources': 'SELECT * FROM submission_sources WHERE submission_id IN (' + subs + ') ORDER BY submission_id,source_record_id,source_submission_id',
'submission_identifiers': 'SELECT * FROM submission_identifiers WHERE submission_id IN (' + subs + ') ORDER BY edition_id,scheme,value',
'decision_sources': 'SELECT * FROM decision_sources WHERE decision_id IN (' + decisions + ') ORDER BY decision_id,source_record_id,source_decision_id',
'paper_tags': 'SELECT * FROM paper_tags WHERE paper_id IN (' + papers + ') ORDER BY paper_id,tag_id,evidence_key',
}
from paper_atlas.curation.annotations import available
if available(c):
queries['annotation_events'] = 'SELECT * FROM annotation_events WHERE paper_id IN (' + papers + ') ORDER BY id'
packet = {k: bounded(c, q, pids) for k, q in queries.items()}
coverage = bounded(c, '''SELECT es.* FROM edition_sources es JOIN editions e ON e.id=es.edition_id
WHERE es.source_id=? AND e.venue_id=? AND e.year=? AND e.event_key=? ORDER BY es.edition_id,es.list_url''', (sid, *scope_params))
if not coverage or any(r[2] != inventory['list_url'] for r in coverage):
raise ValueError('Unambiguous source listing required')
if len({r[0] for r in coverage}) != len(coverage):
raise ValueError('One source coverage entry per edition required')
heads = [[r[i] for i in range(len(r))] for r in versions.heads(c, sid)]
own_heads = [r for r in heads if r[1:] == [vid, inventory['year'], inventory['event_key'], inventory['list_url']]]
if len(own_heads) > 1:
raise ValueError('Forked source revision history')
parent = own_heads[0][0] if own_heads else None
return {'source_id': sid, 'venue_id': vid, 'editions': editions, 'coverage': coverage,
'packet': packet, 'own_records': own_records, 'revision_heads': heads,
'parent_sha256': parent, 'by_track': by_track,
'venue_names': sorted({inventory['venue']} | {r[0] for r in c.execute('SELECT alias FROM venue_aliases WHERE venue_id=?', (vid,))})}
def changes(c, inventory, before):
previous, current = validate(inventory)
packet = before['packet']; result = []; sid = before['source_id']
editions = {r[0]: r for r in before['editions']}
for raw in before['own_records']:
ext = raw[2]; old = previous[ext]; new = current[ext]
metadata = json.loads(raw[8])
candidates = [old, new]
if not any(raw[4] == r['url'] and raw[5] == r['title'] and raw[6] == r.get('abstract')
and json.loads(raw[7]) == r['authors'] and versions.subset(r['metadata'], metadata) for r in candidates):
raise ValueError('Current raw source differs from both reviewed snapshots')
if metadata.get('record_identifiers', {}).get('doi') != ext:
raise ValueError('Current stable DOI differs from source evidence')
attempts = [s for s in packet['submissions'] if s[3] == raw[0] and s[2] in editions]
if len(attempts) != 1:
raise ValueError('Exactly one publication attempt per source record required')
sub = attempts[0]; origin = editions[sub[2]]
if origin[3] not in {old['track'], new['track']} or sub[4] != ext or sub[1] != raw[3]:
raise ValueError('Current source attempt identity/scope differs')
if [x for x in packet['submission_sources'] if x[0] == sub[0]] != [[sub[0], raw[0], ext]]:
raise ValueError('Shared-source attempt requires separate reconciliation')
decisions = [d for d in packet['decisions'] if d[1] == sub[0]]
if not decisions or any(d[2] != 'accepted' for d in decisions):
raise ValueError('Only accepted publication observations can be revised')
target = before['by_track'].get(new['track'])
if target is None:
raise ValueError('Reviewed target edition must already exist')
identities = [x for x in packet['submission_identifiers'] if x[3] == sub[0]]
if not identities or any(x[0] != sub[2] for x in identities):
raise ValueError('Inconsistent scoped submission identifiers')
for _, scheme, value, _ in identities:
conflict = c.execute('SELECT submission_id FROM submission_identifiers WHERE edition_id=? AND scheme=? AND value=?', (target, scheme, value)).fetchone()
if conflict and conflict[0] != sub[0]:
raise ValueError('Target submission identifier collision')
replacement = copy.deepcopy(metadata); replacement.update(new['metadata'])
if sub[2] != target and KEY in metadata:
raise ValueError('An already curated attempt needs explicit scope-chain reconciliation')
if sub[2] != target or raw[6] != new.get('abstract') or replacement != metadata:
result.append({'source_record_id': raw[0], 'paper_id': raw[3], 'submission_id': sub[0],
'external_id': ext, 'from_edition_id': sub[2], 'target_edition_id': target,
'from_track': origin[3], 'track': new['track'],
'new_abstract': new.get('abstract'), 'metadata': replacement})
old_expected = Counter(r['track'] for r in previous.values()); new_expected = Counter(r['track'] for r in current.values())
observed = Counter()
own_ids = {r[0] for r in before['own_records']}
for s in packet['submissions']:
if s[3] in own_ids and s[2] in editions:
observed[s[2]] += 1
coverage = []
for raw in before['coverage']:
track = editions[raw[0]][3]
if raw[3] != 'partial' or raw[8] is not None or raw[5] != observed[raw[0]]:
raise ValueError('Coverage parent is not the verified partial source observation count')
if raw[4] not in {old_expected[track], new_expected[track]}:
raise ValueError('Coverage expected count differs from both source versions')
count = raw[5] + sum((r['target_edition_id'] == raw[0]) - (r['from_edition_id'] == raw[0]) for r in result)
if count < 0 or count > new_expected[track]:
raise ValueError('Corrected coverage exceeds current source membership')
coverage.append({'before': raw, 'expected_count': new_expected[track], 'imported_count': count})
covered = {r['before'][0] for r in coverage}
if any(r['target_edition_id'] not in covered for r in result):
raise ValueError('Target source coverage must already exist')
return result, coverage
def proposal(c, inventory, identity):
before = capture(c, inventory)
if before['parent_sha256']:
saved = json.loads(c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (before['parent_sha256'],)).fetchone()[0])
if sorted(saved, key=lambda r: r['external_id']) != sorted(inventory['previous_rows'], key=lambda r: r['external_id']):
raise ValueError('Previous snapshot differs from active source version')
records, coverage = changes(c, inventory, before)
return {'format': 1, 'database': identity, 'inventory': inventory, 'before': before,
'changes': records, 'coverage': coverage}
def verify(c, plan, signature):
after = capture(c, plan['inventory']); expected = copy.deepcopy(plan['before']['packet'])
for change in plan['changes']:
for row in expected['submissions']:
if row[0] == change['submission_id']:
row[2] = change['target_edition_id']
for row in expected['submission_identifiers']:
if row[3] == change['submission_id']:
row[0] = change['target_edition_id']
for row in expected['source_records']:
if row[0] == change['source_record_id']:
row[6] = change['new_abstract']
row[8] = encoded_metadata(plan, change, signature)
expected['submission_identifiers'].sort(key=lambda r: (r[0], r[1], r[2]))
if after['packet'] != expected:
raise ValueError('Protected profiles/history/tags/identifiers differ; roll back')
if after['editions'] != plan['before']['editions']:
raise ValueError('Edition identities changed')
expected_heads = [r for r in plan['before']['revision_heads'] if r[0] != plan['before']['parent_sha256']]
inventory = plan['inventory']
expected_heads.append([signature, plan['before']['venue_id'], inventory['year'], inventory['event_key'], inventory['list_url']])
if after['revision_heads'] != sorted(expected_heads):
raise ValueError('Active source version differs')
expected_coverage = []
for item in plan['coverage']:
row = item['before'][:]; row[4] = item['expected_count']; row[5] = item['imported_count']
expected_coverage.append(row)
if after['coverage'] != expected_coverage:
raise ValueError('Reconciled coverage differs; roll back')
stored = c.execute('SELECT records_json,plan_json,parent_sha256 FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone()
if (not stored or json.loads(stored[0]) != plan['inventory']['rows']
or json.loads(stored[1]) != plan or stored[2] != plan['before']['parent_sha256']):
raise ValueError('Versioned source or complete before-images changed')
return {'records_changed': len(plan['changes']), 'events_moved': sum(r['from_edition_id'] != r['target_edition_id'] for r in plan['changes']),
'protected_papers': len(expected['papers']), 'protected_decisions': len(expected['decisions'])}
def encoded_metadata(plan, change, signature):
metadata = copy.deepcopy(change['metadata'])
if change['from_edition_id'] != change['target_edition_id']:
inventory = plan['inventory']
evidence = inventory['evidence'][0]
row = {**change, 'source': inventory['source'], 'doi': change['external_id'],
'venue_id': plan['before']['venue_id'], 'year': inventory['year'],
'event_key': inventory['event_key'], 'venue_names': plan['before']['venue_names'],
'section': change['track'], 'evidence_url': evidence['url'], 'evidence_sha256': evidence['sha256']}
metadata[KEY] = [curation(row, change['target_edition_id'], signature)]
return json.dumps(metadata, ensure_ascii=False)
def apply_plan(c, plan):
if plan.get('format') != 1:
raise ValueError('Unsupported source revision plan')
if not getattr(c, 'postgres', False) and not c.in_transaction:
c.execute('BEGIN IMMEDIATE')
signature = digest(plan)
if versions.available(c) and c.execute('SELECT 1 FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone():
return {'status': 'already_applied', 'version_sha256': signature, **verify(c, plan, signature)}
if getattr(c, 'postgres', False):
c.execute('LOCK TABLE sources,papers,identifiers,source_records,venues,venue_aliases,editions,edition_sources,submissions,decisions,submission_sources,submission_identifiers,decision_sources,tags,paper_tags IN SHARE ROW EXCLUSIVE MODE')
if proposal(c, plan['inventory'], plan['database']) != plan:
raise ValueError('Stale source revision plan; create and review a fresh plan')
versions.initialize(c)
for row in plan['changes']:
if row['from_edition_id'] != row['target_edition_id']:
c.execute('UPDATE submission_identifiers SET edition_id=? WHERE submission_id=?', (row['target_edition_id'], row['submission_id']))
c.execute('UPDATE submissions SET edition_id=? WHERE id=? AND edition_id=?', (row['target_edition_id'], row['submission_id'], row['from_edition_id']))
c.execute('UPDATE source_records SET abstract=?,metadata_json=? WHERE id=?',
(row['new_abstract'], encoded_metadata(plan, row, signature), row['source_record_id']))
for row in plan['coverage']:
before = row['before']
c.execute('UPDATE edition_sources SET expected_count=?,imported_count=? WHERE edition_id=? AND source_id=? AND list_url=?',
(row['expected_count'], row['imported_count'], before[0], before[1], before[2]))
inventory = plan['inventory']
c.execute('''INSERT INTO source_scope_revisions(version_sha256,source_id,venue_id,year,event_key,list_url,parent_sha256,records_json,plan_json)
VALUES(?,?,?,?,?,?,?,?,?)''', (signature, plan['before']['source_id'], plan['before']['venue_id'], inventory['year'], inventory['event_key'],
inventory['list_url'], plan['before']['parent_sha256'], json.dumps(inventory['rows'], ensure_ascii=False), json.dumps(plan, ensure_ascii=False)))
return {'status': 'applied', 'version_sha256': signature, **verify(c, plan, signature)}
def run(db, out, *, pg_service=None, pg_schema=None, inventory=None, plan=None, plan_sha256=None):
out = Path(out)
if out.exists() or not out.parent.is_dir():
raise ValueError('New output in existing directory required')
if bool(inventory) == bool(plan) or bool(plan) != bool(plan_sha256):
raise ValueError('Choose inventory preflight or plan plus exact file hash')
if inventory:
with closing(open_read_connection(db, pg_service=pg_service, pg_schema=pg_schema)) as c:
result = proposal(c, json.loads(Path(inventory).read_bytes()), database_identity(c, db))
else:
raw = Path(plan).read_bytes()
if hashlib.sha256(raw).hexdigest() != plan_sha256:
raise ValueError('Plan file hash differs')
proposed = json.loads(raw)
with closing(open_store(db, pg_service=pg_service, pg_schema=pg_schema).conn) as c:
try:
if proposed['database'] != database_identity(c, db):
raise ValueError('Plan belongs to another database')
if not getattr(c, 'postgres', False):
c.commit(); c.execute('BEGIN IMMEDIATE')
result = apply_plan(c, proposed); c.commit()
except BaseException:
c.rollback(); raise
out.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n')
return result
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog); database_arguments(parser)
parser.add_argument('--inventory'); parser.add_argument('--plan'); parser.add_argument('--plan-sha256'); parser.add_argument('--out', required=True)
args = parser.parse_args(argv)
report = run(args.db, args.out, pg_service=args.pg_service, pg_schema=args.pg_schema,
inventory=args.inventory, plan=args.plan, plan_sha256=args.plan_sha256)
print(json.dumps({'status': report.get('status', 'planned'), 'out': args.out}), flush=True)
+195
View File
@@ -0,0 +1,195 @@
"""Read only selected Parquet column chunks; never fetch other payload pages."""
import hashlib
import io
import re
import struct
import urllib.error
import urllib.request
MAX_FOOTER = 4 * 1024**2
MAX_SELECTED_BYTES = 64 * 1024**2
MAX_REQUESTS = 512
class RangeSource:
"""Strict bounded HTTP ranges, no full-file fallback, redirects kept private."""
def __init__(self, url, size, *, opener=None):
self.url, self.size = url, size
self.opener = opener or urllib.request.build_opener()
self.etag = None
self.receipts = []
self.bytes_read = 0
def read(self, start, length):
if (type(start) is not int or type(length) is not int or length <= 0
or start < 0 or start + length > self.size
or self.bytes_read + length > MAX_SELECTED_BYTES
or len(self.receipts) >= MAX_REQUESTS):
raise ValueError('Bounded metadata range budget exceeded')
headers = {'Range': f'bytes={start}-{start + length - 1}', 'Accept-Encoding': 'identity'}
if self.etag and not self.etag.startswith('W/'):
headers['If-Match'] = self.etag
request = urllib.request.Request(self.url, headers=headers)
try:
with self.opener.open(request, timeout=45) as response:
if response.status != 206:
raise ValueError('Server did not honor metadata range; full download forbidden')
if response.headers.get('Content-Range') != f'bytes {start}-{start + length - 1}/{self.size}':
raise ValueError('Object size or range differs from pinned inventory')
encoding = response.headers.get('Content-Encoding', 'identity')
if encoding != 'identity':
raise ValueError('Encoded HTTP ranges are unsupported')
etag = response.headers.get('ETag')
if self.receipts and etag != self.etag:
raise ValueError('Object ETag changed between metadata ranges')
raw = response.read(length + 1)
if len(raw) != length:
raise ValueError('Truncated or oversized metadata range')
if not self.receipts:
self.etag = etag
except urllib.error.HTTPError as error:
# Do not expose signed CDN redirect URLs or retry denied resources.
raise RuntimeError('Metadata source HTTP ' + str(error.code) + '; no retry or full-file fallback') from None
self.receipts.append({'start': start, 'length': length, 'sha256': hashlib.sha256(raw).hexdigest()})
self.bytes_read += length
return raw
class SparseFile(io.RawIOBase):
"""A logical file backed only by approved bytes; all other positions are zeros.
Arrow may read a 64 KiB footer window. Synthesizing its unused prefix avoids
fetching the preceding OCR/review page. Data pages are explicitly selected
from footer offsets before they can be downloaded.
"""
def __init__(self, size, segments):
self.size, self.segments, self.position = size, segments, 0
def readable(self):
return True
def seekable(self):
return True
def tell(self):
return self.position
def seek(self, offset, whence=0):
base = {0: 0, 1: self.position, 2: self.size}.get(whence)
if base is None or base + offset < 0:
raise ValueError('Invalid sparse metadata seek')
self.position = base + offset
return self.position
def read(self, length=-1):
if length < 0:
length = self.size - self.position
length = min(length, max(0, self.size - self.position))
if length > MAX_SELECTED_BYTES:
raise ValueError('Unbounded sparse read forbidden')
start, end = self.position, self.position + length
raw = bytearray(length)
for offset, data in self.segments:
left, right = max(offset, start), min(offset + len(data), end)
if left < right:
raw[left-start:right-start] = data[left-offset:right-offset]
self.position = end
return bytes(raw)
def readinto(self, buffer):
raw = self.read(len(buffer)); buffer[:len(raw)] = raw
return len(raw)
def arrow():
try:
import pyarrow as pa
import pyarrow.parquet as pq
except ImportError:
raise RuntimeError('Install paper-atlas[mirrors] only on the approved remote runner') from None
return pa, pq
def projection(source, columns, *, progress=None):
"""Return complete metadata rows and range receipts, no full Parquet snapshot."""
pa, pq = arrow()
if type(source.size) is not int or not 12 <= source.size <= 4 * 1024**3:
raise ValueError('Bounded pinned Parquet size required')
if (not isinstance(columns, dict) or not columns
or any(not re.fullmatch('[a-z][a-z0-9_]*', key) for key in columns)):
raise ValueError('Explicit top-level metadata columns required')
header = source.read(0, 4); trailer = source.read(source.size - 8, 8)
if header != b'PAR1' or trailer[4:] != b'PAR1':
raise ValueError('Expected unencrypted Parquet framing')
length = struct.unpack('<I', trailer[:4])[0]
if not 0 < length <= min(MAX_FOOTER, source.size - 12):
raise ValueError('Invalid or oversized Parquet footer')
footer_start = source.size - 8 - length
footer = source.read(footer_start, length)
segments = [(0, header), (footer_start, footer), (source.size - 8, trailer)]
sparse = SparseFile(source.size, segments)
parquet = pq.ParquetFile(sparse, pre_buffer=False, buffer_size=0)
try:
schema = parquet.schema_arrow
if len(set(schema.names)) != len(schema.names) or not set(columns) <= set(schema.names):
raise ValueError('Missing or repeated metadata columns')
for key, kind in columns.items():
value = schema.field(key).type
if kind == 'string':
okay = pa.types.is_string(value) or pa.types.is_large_string(value)
elif kind == 'integer':
okay = pa.types.is_integer(value)
elif kind == 'strings':
okay = (pa.types.is_list(value) or pa.types.is_large_list(value)) and (
pa.types.is_string(value.value_type) or pa.types.is_large_string(value.value_type))
else:
raise ValueError('Unsupported safe metadata type')
if not okay:
raise ValueError('Metadata column type differs: ' + key)
metadata = parquet.metadata
if not 0 < metadata.num_rows <= 100000 or metadata.num_row_groups > 1000:
raise ValueError('Bounded metadata row count required')
wanted, excluded = [], []
for group in range(metadata.num_row_groups):
rg = metadata.row_group(group)
for i in range(rg.num_columns):
col = rg.column(i)
if col.file_path:
raise ValueError('External Parquet column files forbidden')
offsets = [x for x in (col.dictionary_page_offset, col.data_page_offset) if x is not None and x > 0]
if not offsets:
raise ValueError('Missing column chunk offset')
start = min(offsets); end = start + col.total_compressed_size
if not 4 <= start < end <= footer_start:
raise ValueError('Column chunk lies outside the pinned payload')
item = (start, end, group, col.path_in_schema)
(wanted if col.path_in_schema.split('.')[0] in columns else excluded).append(item)
if any(max(a, c) < min(b, d) for a, b, _, _ in wanted for c, d, _, _ in excluded):
raise ValueError('Selected range overlaps an excluded payload column')
if sum(end-start for start, end, _, _ in wanted) + length + 12 > MAX_SELECTED_BYTES:
raise ValueError('Metadata projection exceeds byte budget')
if len(wanted) + 3 > MAX_REQUESTS:
raise ValueError('Metadata projection exceeds request budget')
for start, end, group, path in sorted(wanted):
segments.append((start, source.read(start, end-start)))
if progress:
progress({'phase': 'metadata_column_range', 'row_group': group,
'column': path, 'metadata_bytes_read': source.bytes_read})
rows = []
for group in range(metadata.num_row_groups):
table = parquet.read_row_group(group, columns=list(columns), use_threads=False)
if set(table.column_names) != set(columns) or len(table.column_names) != len(columns):
raise ValueError('Unexpected projected metadata fields')
rows.extend(table.select(list(columns)).to_pylist())
if len(rows) != metadata.num_rows:
raise ValueError('Incomplete metadata projection')
report = {'rows': len(rows), 'row_groups': metadata.num_row_groups,
'columns': dict(columns), 'footer_sha256': hashlib.sha256(footer).hexdigest(),
'column_ranges': [{'start': a, 'length': b-a, 'row_group': g, 'column': p} for a, b, g, p in wanted],
'range_receipts': source.receipts, 'metadata_bytes_read': source.bytes_read,
'source_bytes': source.size, 'full_file_checksum_verified': False,
'excluded_payload_pages_downloaded': False,
'footer_saved': False, 'raw_parquet_saved': False}
return rows, report
finally:
parquet.close()
+91
View File
@@ -0,0 +1,91 @@
"""Frozen Springer public HTML receipts and chapter citation metadata."""
from collections import defaultdict
import hashlib
from pathlib import Path
import json
import re
from urllib.parse import parse_qs, urlsplit
from paper_atlas.db.submissions import timestamp
from paper_atlas.io.html import Document, clean
def snapshot_index(root):
root = Path(root).resolve()
receipts = json.loads((root/'receipts.json').read_bytes())['responses']
index = {}
for receipt in receipts:
url = receipt['url']
if url in index:
raise ValueError('Duplicate snapshot URL')
index[url] = (root, receipt)
return index
def read_page(index, url):
if url not in index:
raise ValueError('Missing required public HTML snapshot: '+url)
root, receipt = index[url]
filename = Path(receipt['file'])
path = root/filename
content_type = {k.lower():v for k,v in receipt.get('headers', {}).items()}.get('content-type', '')
requested, final = urlsplit(url), urlsplit(receipt.get('final_url', ''))
query = parse_qs(final.query, keep_blank_values=True)
original_query = parse_qs(requested.query, keep_blank_values=True)
# Springer returns the public metadata with an explicit cookie diagnostic.
# Retain that final URL verbatim; accept no other redirect or changed page.
if query != original_query and requested.netloc == 'link.springer.com':
if (query.pop('error', None) != ['cookies_not_supported']
or not re.fullmatch(r'[0-9a-f-]{36}', ''.join(query.pop('code', [])))):
raise ValueError('Unreviewed Springer redirect query')
if (filename.is_absolute() or '..' in filename.parts or not path.resolve().is_relative_to(root)
or receipt.get('http_status') != 200
or (final.scheme, final.netloc, final.path) != (requested.scheme, requested.netloc, requested.path)
or final.fragment or query != original_query
or 'text/html' not in content_type.lower()):
raise ValueError('Unsafe, redirected or unsuccessful public HTML snapshot')
raw = path.read_bytes()
digest = hashlib.sha256(raw).hexdigest()
if len(raw) > 2_000_000 or len(raw) != receipt['bytes'] or digest != receipt['sha256']:
raise ValueError('Snapshot checksum or size differs')
evidence = {'url':url, 'sha256':digest, 'bytes':len(raw),
'file':'evidence/'+digest+'.html', 'http_status':200,
'observed_at':timestamp(receipt['at']), 'response_receipt':receipt}
return raw, evidence
def parse_chapter(html, doi, *, book_title, publication_year):
nodes = list(Document(html).root.walk())
meta = defaultdict(list)
for node in nodes:
if node.tag == 'meta' and node.attrs.get('name', '').startswith('citation_'):
meta[node.attrs['name']].append(node.attrs.get('content', ''))
def one(key):
values = meta[key]
if len(values) != 1 or not values[0]:
raise ValueError('Missing or ambiguous citation field: '+key)
return values[0]
if (one('citation_doi') != doi or one('citation_inbook_title') != book_title
or one('citation_publication_date') != str(publication_year)):
raise ValueError('Chapter DOI, book or publication year differs from reviewed scope')
raw_authors = meta['citation_author']
if not raw_authors:
raise ValueError('Missing named chapter authors')
authors = []
for raw in raw_authors:
if raw.count(',') != 1:
raise ValueError('Unreviewed citation author format')
family, given = [clean(s) for s in raw.split(',', 1)]
if not family or not given:
raise ValueError('Incomplete citation author')
authors.append({'given':given, 'family':family})
abstract_nodes = [n for n in nodes if n.attrs.get('id') == 'Abs1-content']
if len(abstract_nodes) > 1:
raise ValueError('Ambiguous abstract section')
abstract = clean(abstract_nodes[0].text()) if abstract_nodes else None
first, last = one('citation_firstpage'), one('citation_lastpage')
if not first.isdigit() or not last.isdigit() or int(first) > int(last):
raise ValueError('Invalid chapter pages')
return {'doi':doi, 'title':one('citation_title'), 'authors':authors,
'authors_citation_raw':list(raw_authors), 'abstract':abstract or None,
'first_page':first, 'last_page':last}
+1 -1
View File
@@ -8,7 +8,7 @@ TABLES = ('sources', 'papers', 'identifiers', 'source_records', 'venues', 'editi
'submission_sources', 'submission_identifiers', 'decision_sources', 'submission_sources', 'submission_identifiers', 'decision_sources',
'submission_merges', 'decision_merges', 'schema_migrations', 'tags', 'submission_merges', 'decision_merges', 'schema_migrations', 'tags',
'paper_tags', 'match_candidates', 'paper_merges', 'ingest_runs') 'paper_tags', 'match_candidates', 'paper_merges', 'ingest_runs')
OPTIONAL_TABLES = ('annotation_events',) OPTIONAL_TABLES = ('annotation_events', 'source_scope_revisions')
CHECKS = { CHECKS = {
'annotation_events': ["kind IN ('tag','research_type')", 'retracts_id IS NULL OR retracts_id<>id'], 'annotation_events': ["kind IN ('tag','research_type')", 'retracts_id IS NULL OR retracts_id<>id'],
'venues': ["venue_type IN ('conference','journal','review_platform')"], 'venues': ["venue_type IN ('conference','journal','review_platform')"],
@@ -0,0 +1 @@
[{"type":"file","oid":"f87639c8a74ab77de91ea6bb4c88718a52e4250f","size":37291354,"lfs":{"oid":"bcb446a0398dfe62593e1db617f82f43bc792fc413d6d4658b0e0d32c358b888","size":37291354,"pointerSize":133},"xetHash":"c9a6d5d70b6f3a9ee09b4e085327e4f61cad91b9b61244e6bf99031fef9dd85b","path":"data/colm-00000-of-00001.parquet"},{"type":"file","oid":"27ef0f337b76d92ac1aaa4e1dc41a1253768545b","size":28627916,"lfs":{"oid":"930dd4ef90fa352fe6bccca7b73eb36eeabfb99f1e77e783572e2db83e15d779","size":28627916,"pointerSize":133},"xetHash":"468b49fe8b9a921e852c1995c4902e175ad7b54fcefb048a485e3fbad8b62bfd","path":"data/corl-00000-of-00001.parquet"},{"type":"file","oid":"c8de3418de080cbd24667e28c55abf684a8a3d9f","size":90794691,"lfs":{"oid":"e9e3aab42d28ca32a02287ed0bd0f1f9e1f3af5cb8ed8d6a9bf28d4f630773da","size":90794691,"pointerSize":133},"xetHash":"c21b83e5820cea8b7a6bcac34bbd334a0a6ac5b7acb1c4c681715a52ec2cceda","path":"data/emnlp-00000-of-00001.parquet"},{"type":"file","oid":"0a226079bb3fe5256a51750aa874c0e7325f60fa","size":167662887,"lfs":{"oid":"3a65801f95fe490b30282df2cb668fa33fe4182e9d80544148ea66da35215ad8","size":167662887,"pointerSize":134},"xetHash":"2225125f6d18015bb3c44037c82a7f53363eeb636d309de3445e1f1b49f51a03","path":"data/iclr-00000-of-00005.parquet"},{"type":"file","oid":"7b53c49c39f2c287a778e64b18969690926710a8","size":196573710,"lfs":{"oid":"736c05be45eed9db00848471d88dc9cd5a1b1a7d5a0db6432639a21dfb8b4562","size":196573710,"pointerSize":134},"xetHash":"48da5a74c33ca8b04411f4d6f41e34a4ba57d8c4aecb1a72be5dcad89a72bd65","path":"data/iclr-00001-of-00005.parquet"},{"type":"file","oid":"cbf09e34f7f00e33f5bb501461a915ab4bda2ec2","size":210807163,"lfs":{"oid":"1c0b77c370b8735e3d4ebd578706ec8dfbb5e53eb6a71e8cbfeaec36c78ee13f","size":210807163,"pointerSize":134},"xetHash":"fa8bf4848b55d274464b6d06413e1183321e6ffd467e1efa369ee32e0749e8af","path":"data/iclr-00002-of-00005.parquet"},{"type":"file","oid":"4082a290b2e2785413340920e3475c4b5a58b05f","size":250525528,"lfs":{"oid":"078d441ef3854be30f8fd73d34e8442a2a119cd26dbaddf9ab8aba2008df507f","size":250525528,"pointerSize":134},"xetHash":"63ba7c9757b53b02b1793f2186ddc55b79149dbda9a37bacedaf2b0466604d61","path":"data/iclr-00003-of-00005.parquet"},{"type":"file","oid":"d4528dcfcd6d7f56619d2bed75d4705c11811bfb","size":263725824,"lfs":{"oid":"de5efe8578b744f59c8ff9d7c1171cdaa760008920dde190c04b73e5c45901a5","size":263725824,"pointerSize":134},"xetHash":"5934eb773ec49ebf9f48c89d9d7e88ae0d2eb5f5e7f83bfd13f51080aa6d4090","path":"data/iclr-00004-of-00005.parquet"},{"type":"file","oid":"68cd41fa1983b2ba666deb74d1e6da2f9fa9003d","size":204752048,"lfs":{"oid":"4f50e2a0fd7e40a0fdb239595b636c753ca89af1ad9692b7d63f366e7a25eda0","size":204752048,"pointerSize":134},"xetHash":"de8ac963c1738ffe68728255bf7f32fbfff3863b3b07ca1f7026f4c76f123ec0","path":"data/icml-00000-of-00001.parquet"},{"type":"file","oid":"580a98d667a2fc7252fd2bcc2eeeaff116f7030d","size":126503813,"lfs":{"oid":"d2c4359745d30900bd19a44762c9fb6b8415554e2aead6f22f902362aca3e61a","size":126503813,"pointerSize":134},"xetHash":"27cd8f2318a0d096cf3a6aa7178fc502ce52c5d026887f64661b46af74b4b9d5","path":"data/neurips-00000-of-00005.parquet"},{"type":"file","oid":"e90e69bdd1985a36f9a72b4b3b3a9b54d90b7b9d","size":166685127,"lfs":{"oid":"32be2572bc42d33ff227ceadbdcc64e27c996b31e82195d25eced11ec4617963","size":166685127,"pointerSize":134},"xetHash":"2e3528c026d585513df96b23efe39d174b368abd3088e494d85fcf806504a241","path":"data/neurips-00001-of-00005.parquet"},{"type":"file","oid":"b7be2e96d3c3726b87c4a997fcb2c144288b70b4","size":229115194,"lfs":{"oid":"7f9dc8025aebe1d78f74bfccc68e1792f6a076142a920305065d805ff04ab248","size":229115194,"pointerSize":134},"xetHash":"77a4b411199fee5bd1b3fb8752ad4bf1ac090d560a4bfde54b75cf19d48c545b","path":"data/neurips-00002-of-00005.parquet"},{"type":"file","oid":"127763711c2fe76631137432291fa839746536a3","size":264212756,"lfs":{"oid":"32d86e74f00d3fce8eef1cf1db1dd78eed341ddda0e09a924ba17d08c3d566b2","size":264212756,"pointerSize":134},"xetHash":"abe6a12cf6f6f2db52ba28b84a0b91fcfab1411dfa0eb0ab7cdae540b86c5c9d","path":"data/neurips-00003-of-00005.parquet"},{"type":"file","oid":"89eb22ff72a8f44d461e601176239d4f997bbec7","size":279556753,"lfs":{"oid":"b4a536cdb6d306049a32121272b8224ef0e37360423ca04c274f0da74d4dc64c","size":279556753,"pointerSize":134},"xetHash":"d2b68be869a02a2cbe81c74e50fcac3c35585422dbaffcb19985e2376a90e0a7","path":"data/neurips-00004-of-00005.parquet"},{"type":"file","oid":"2b6048e08f894bb47b874a485d3bc5655a43a449","size":211604858,"lfs":{"oid":"df21b892ec531bc223b114d03cd51be6f4f2c48563edde0bdd1b153c36a74a98","size":211604858,"pointerSize":134},"xetHash":"1e81fc6612959ef821e97b7b5ed06a9ebbb3748f0ecc2fe2027b41f7626ef863","path":"data/tmlr-00000-of-00001.parquet"}]
+2 -1
View File
@@ -12,6 +12,7 @@ dependencies = ["PyYAML==6.0.3"]
[project.optional-dependencies] [project.optional-dependencies]
postgres = ["psycopg[binary]==3.3.8"] postgres = ["psycopg[binary]==3.3.8"]
mirrors = ["pyarrow==26.0.1"]
[project.scripts] [project.scripts]
paper-atlas = "paper_atlas.cli:main" paper-atlas = "paper_atlas.cli:main"
@@ -22,4 +23,4 @@ exclude = ["tests*"]
[tool.setuptools.package-data] [tool.setuptools.package-data]
"paper_atlas.db" = ["schema.sql", "annotations.sql"] "paper_atlas.db" = ["schema.sql", "annotations.sql"]
"paper_atlas.resources" = ["venues.json"] "paper_atlas.resources" = ["venues.json", "reviewarena-files.json"]
Binary file not shown.
+706
View File
@@ -0,0 +1,706 @@
{
"status": "reviewed26_presentation_only_title_correspondences_not_work_identity_approval",
"gold": [
{
"doi": "10.1145/3767308.3834907",
"title": "H\n <sup>2</sup>\n AL: Hyperbolic Hierarchy-aware Aggregative Learning for Registration-based Few-shot Medical Image Segmentation",
"authors": [
"Jia Wang",
"Jiaming Cai",
"Zunying Hu",
"Zhanjie Wu",
"Jinyuan Liu",
"Hua Cheng",
"Yun Peng",
"Xin Fan"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-004.json",
"source_position": 246,
"registry_object_sha256": "dd4a627c13ca806e15e59d3ae894c9216d14d33c85bc5ce7362b51fad8c8a692",
"program_id": "mfp0053",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "516fac88f1b7a2821c1e2b97cc41530f32abbe2514b65b5823427ff3ecdc9a23",
"program_context_pointers": [
"/entries/129/papers/19",
"/entries/129/subsessions/1/items/7"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3834936",
"title": "M\n <sup>3</sup>\n TR: Temporal Retrieval Enhanced Multi-Modal Micro-video Popularity Prediction",
"authors": [
"Jiacheng Lu",
"Weijian Wang",
"Mingyuan Xiao",
"Yang Hua",
"Tao Song",
"Bo Peng",
"Cheng Hua",
"Haibing Guan"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-005.json",
"source_position": 25,
"registry_object_sha256": "93340f8b923d7cc0d848941107351fb661d95effa35b1869921e7163da36bae4",
"program_id": "mfp0229",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "6968e6a2d364075020e0b6c95c9076742531681232c40df511a2e8992c6b25f9",
"program_context_pointers": [
"/entries/109/papers/27",
"/entries/109/subsessions/2/items/3"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3834946",
"title": "<i>PAST:</i>\n Prompt-Adaptive Sampling Termination for Efficient Diffusion Model",
"authors": [
"Renye Yan",
"Jikang Cheng",
"You Wu",
"Wei Peng",
"Zongwei Wang",
"Ling Liang",
"Yimao Cai"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-005.json",
"source_position": 35,
"registry_object_sha256": "4210191c2871090711820aef9aff06289694380c9d0194ec9b8b1e5e40fad00d",
"program_id": "mfp0297",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "b47bd7502bf3a8e9e9dc87e3d9552efe928cc455f843d527a4860c430c5e9490",
"program_context_pointers": [
"/entries/61/papers/40",
"/entries/61/subsessions/3/items/4"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3834956",
"title": "<i>SwiftExplorer:</i>\n Training-free diffusion model alignment with Swift Diversity Exploration",
"authors": [
"Renye Yan",
"Jikang Cheng",
"You Wu",
"Baojin Huang",
"Wei Peng",
"Zongwei Wang",
"Ling Liang",
"Yimao Cai"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-006.json",
"source_position": 235,
"registry_object_sha256": "7f2812ca4072437ecaaa585e85139e7cba0e0d721f9a2c0ac4e9897739faed47",
"program_id": "mfp0339",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "f7809f57290e44fe291ffdc07ea956e5484aa491390cd385db75d056f95b840a",
"program_context_pointers": [
"/entries/88/papers/48",
"/entries/88/subsessions/4/items/0"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835035",
"title": "<i>Balancing Efficiency and Efficacy:</i>\n Training-Free Attention-Guided Switching Between Explicit and Latent Thoughts for MLLMs",
"authors": [
"Haoqian Kang",
"Liupeng Li",
"Kuofeng Gao",
"Jinpeng Wang",
"Zhenyu Lu",
"Bin Chen",
"Ke Chen",
"Yaowei Wang"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-004.json",
"source_position": 8,
"registry_object_sha256": "3d76f692754dc159642eeb1706491631267d437a95c4459674361ebf35f416b9",
"program_id": "mfp0798",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "477e42fa3d5c8236d6415ff18ba26d799d659b9e2f3f09c1d80bc0971dbca125",
"program_context_pointers": [
"/entries/88/papers/39",
"/entries/88/subsessions/3/items/3"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835055",
"title": "TM\n <sup>2</sup>\n Q-Det: Boosting Video Object Detection via Text-Guided Spatiotemporal Mamba and Multimodal Query Learning",
"authors": [
"Qiang Qi",
"Wenqi Shang",
"Xiao Wang",
"Shuyuan Lin"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-004.json",
"source_position": 28,
"registry_object_sha256": "7e3c3e6acdad588970c681bdb92969f5134086baa2a5a7b9414871135d5ea137",
"program_id": "mfp0903",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "8549d38e781896079ab04ef6a771a4061a0fe42df2fa5c5c03a037d59cc9f548",
"program_context_pointers": [
"/entries/129/papers/55",
"/entries/129/subsessions/4/items/7"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835101",
"title": "<scp>ConfAvatar:</scp>\n Confidence-Aware Self-Bootstrapping for Monocular 4D Head Avatar Reconstruction",
"authors": [
"Zifan Zheng",
"Yu Luo",
"Yuxin Huang",
"Chuancheng Shi",
"Mohammad Masbaul Alam Polash",
"Zhiyong Wang",
"Kun Hu"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-008.json",
"source_position": 82,
"registry_object_sha256": "b838756bd0b6af036e408936f8f13a6398d83e524ce0282167199eca092bc7d8",
"program_id": "mfp1175",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "cb8dbe67e16e163e5b136ad3401fc57c827905e9704d7c0a84961bc8f12565f4",
"program_context_pointers": [
"/entries/109/papers/86",
"/entries/109/subsessions/7/items/2"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835374",
"title": "<scp>DateGBC:</scp>\n Deep Fuzzy Granular-Ball Clustering via Adaptive Topology Evolution for Unsupervised Image Segmentation",
"authors": [
"Qianpeng Chong",
"Jiakun Wen",
"Guangyi Wei",
"Wenyi Zeng"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-006.json",
"source_position": 59,
"registry_object_sha256": "a7fe694a04449e841be5181888ae35786d0a4938d87eca1fffb0776d4d350e30",
"program_id": "mfp2617",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "b57b7026c750aeebb47598860b82980215c9e4e4935f6e8611a31b2fe64f73b9",
"program_context_pointers": [
"/entries/61/papers/80",
"/entries/61/subsessions/6/items/8"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835424",
"title": "<i>RTCBench</i>\n : A Scalable Benchmarking Framework for Multi-Party Video Conferencing",
"authors": [
"Evan Ram",
"Tuan Tran",
"Youngwook Son",
"Sangtae Ha"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-005.json",
"source_position": 108,
"registry_object_sha256": "b21198e0eba35d175bc83292be8c87bca2b80a9c8e01f40b640a4e9efc7ecde5",
"program_id": "mfp2909",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "2398ec6539c50b14c7ed4fa773277cc860151ab77b5430b49d935f88e90a487e",
"program_context_pointers": [
"/entries/129/papers/130",
"/entries/129/subsessions/8/items/22"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835452",
"title": "<scp>HeadRouter:</scp>\n Dynamic Head-Weight Routing for Task-Adaptive Audio Token Pruning in Large Audio Language Models",
"authors": [
"He Peize",
"Yaodi Luo",
"Xiaoqian Liu",
"Xuyang Liu",
"Jiahang Deng",
"Yaosong Du",
"Li Bangyu",
"Xiyan Gui",
"Yuxuan Chen",
"Linfeng Zhang"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-004.json",
"source_position": 167,
"registry_object_sha256": "ee31b12972ea5844a50c6c63a6fc3430875c3388a4bf261f8e76cab31112d351",
"program_id": "mfp3048",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "c455de7c27f43d0fb49990ead5fca795334c0b613eafa2b3c6150eb147d9c3fa",
"program_context_pointers": [
"/entries/46/papers/78",
"/entries/46/subsessions/6/items/6"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835460",
"title": "<i>CoAnchor:</i>\n Robust Collaborative Perception under Spatio-Temporal Misalignment via Object-Level Anchors",
"authors": [
"Chi Li",
"Rui Lin",
"Aobo Ji",
"Dongzhu Xu"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-004.json",
"source_position": 175,
"registry_object_sha256": "7c41b0dcb65ad27a1f0961f095889ccc73044776250ec816cf95fc25c30da7f1",
"program_id": "mfp3109",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "5f3d97775d83528d6f41fb2fc481d1848cac7c3f9de1d469039450f9b2ecbff1",
"program_context_pointers": [
"/entries/109/papers/121",
"/entries/109/subsessions/8/items/13"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835544",
"title": "<i>OC-GS:</i>\n Object-Centric Representation and Demand Alignment for Immersive 3D Gaussian Streaming",
"authors": [
"Boyan Li",
"Ruijun Mao",
"Zhicheng Liang",
"Dayou Zhang",
"Fangxin Wang"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-006.json",
"source_position": 129,
"registry_object_sha256": "afc0c7db304251511eef5e223f027dc6314dd70895955ff1a1cb0a461dbfdbe9",
"program_id": "mfp3554",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "00b2ffe1073feede8447bac439ac7842c1ba0841cb6ee38ec1b5e06c3a3dc22e",
"program_context_pointers": [
"/entries/46/papers/153",
"/entries/46/subsessions/9/items/21"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835591",
"title": "DP\n <sup>2</sup>\n -NeRF: Dual-Prior Probabilistic Neural Radiance Fields for Event-Guided Motion Deblurring",
"authors": [
"Jiafeng Huang",
"Wenzhen Jia",
"Shilong Dong",
"Shengjie Zhao"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-007.json",
"source_position": 97,
"registry_object_sha256": "145025e784ec251247f8cd86d0dbee1749666ec71f35566c20e41edd0a2347ce",
"program_id": "mfp3783",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "0b2d267736cfe506de666618820e2934c23395bd25dab63c5a347710445a361c",
"program_context_pointers": [
"/entries/33/papers/158",
"/entries/33/subsessions/10/items/2"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835667",
"title": "MMCoT\n <sup>3</sup>\n : Empowering 3D Scene Understanding with 3D-Text Interleaved Multi-Modal Chain-of-Thought Reasoning",
"authors": [
"Yongqi Shan",
"Yifan Wang",
"Yunzhi Zhuge",
"Lijun Wang",
"Huchuan Lu"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-005.json",
"source_position": 209,
"registry_object_sha256": "8b5ad8d891692e441b6e210304dcd35103835fd35d086c860c7611bf5385a6d9",
"program_id": "mfp4173",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "4924721a2c3328e6bca661654569bca86d3fa69aa5f6c9e3719f827c5f0d7639",
"program_context_pointers": [
"/entries/88/papers/114",
"/entries/88/subsessions/8/items/6"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835763",
"title": "<scp>AudioKV:</scp>\n KV Cache Eviction in Efficient Large Audio Language Models",
"authors": [
"Yuxuan Wang",
"He Peize",
"Xiyan Gui",
"Xiaoqian Liu",
"Junhao He",
"Xuyang Liu",
"Xuming Hu",
"Linfeng Zhang"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-006.json",
"source_position": 192,
"registry_object_sha256": "70054b0a5a2586fd08e75494eee08a26498730554601a5fe2e4efddcaad309de",
"program_id": "mfp4778",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "274b360da553dfc7afdb9072505c2957cec4213e096373e2edbf58236a3fc3a2",
"program_context_pointers": [
"/entries/129/papers/100",
"/entries/129/subsessions/7/items/16"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835895",
"title": "F\n <sup>2</sup>\n -Adapter: A Focus-and-Forget Framework for Generalizable CLIP-based Face Forgery Detection",
"authors": [
"Zan Gao",
"Xin Duan",
"Nian Hu",
"Hao Jia",
"Yibo Zhao",
"Cong Liu"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-001.json",
"source_position": 39,
"registry_object_sha256": "16bdbcc4398df4d9556c14d4d4203127349070669f5d0e1c4bf87756e241b698",
"program_id": "mfp5537",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "b3ef626b66d93ac37a73caf15a4cf9ff92241209c80ee66be5575e6b56f9c899",
"program_context_pointers": [
"/entries/88/papers/184",
"/entries/88/subsessions/11/items/6"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3835959",
"title": "UPR-VAD:\n <i>U</i>\n ncertainty-guided Signal\n <i>P</i>\n urification and\n <i>R</i>\n egularization for Weakly-Supervised\n <i>V</i>\n ideo\n <i>A</i>\n nomaly\n <i>D</i>\n etection",
"authors": [
"Weiliang Huang",
"Weichao Cai",
"Bob Zhang",
"Chao Huang",
"Biao Xue",
"Rongxin Zhang"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-007.json",
"source_position": 21,
"registry_object_sha256": "e96f9e9d67cce31a9235bdc7dc5dba551676f3d36ba72654ae3d5a98ae8d5174",
"program_id": "mfp5823",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "d8469bcb68f8cc19070f3bf9a866c3377ba27fe6445b5bf9337b7599a6881592",
"program_context_pointers": [
"/entries/88/papers/66",
"/entries/88/subsessions/5/items/6"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3836215",
"title": "BLM\n <sub>1</sub>\n : A Boundless Large Model for Cross-Space, Cross-Task, and Cross-Embodiment Learning",
"authors": [
"Wentao Tan",
"Bowen Wang",
"Heng Zhi",
"Chenyv Liu",
"Zhe Li",
"Jian Liu",
"Zengrong Lin",
"Yukun Dai",
"Yipeng Chen",
"Wenjie Yang",
"Enci Xie",
"Hao Xue",
"Baixu Ji",
"Chen Xu",
"Zhibin Wang",
"Tianshi Wang",
"Lei Zhu",
"Heng Tao Shen"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-003.json",
"source_position": 46,
"registry_object_sha256": "b4b51e5d0a4c0a330d01e7638c0bcc73871d750312081bafbd3b8a10529153f2",
"program_id": "mfp7332",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "aa458905209ad562117d72dd33d4d61e000b0d9efaa55dd9672919f40e6cc2fd",
"program_context_pointers": [
"/entries/33/papers/13",
"/entries/33/subsessions/1/items/1",
"/entries/47/papers/0"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3836272",
"title": "SEE: Structure-aware Exploring &amp; Exploiting for Long-horizon GUI Agent Trajectory Synthesis",
"authors": [
"Zhuohang Fan",
"Beichen Zhang",
"Yuanfa Li",
"Changqiao Wu",
"Wei Liu",
"Jian Luan",
"Weigang Zhang"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-002.json",
"source_position": 127,
"registry_object_sha256": "66eebbdef9e2d07fe08f3cb41cd577095f282f1256bdb73fe1e0d4c5d6e55d51",
"program_id": "mfp7635",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "df11d727337f397bab72977bcdce4246b388ef0c8e1198ac47fdc32e68a3aff4",
"program_context_pointers": [
"/entries/88/papers/131",
"/entries/88/subsessions/8/items/23"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3836293",
"title": "Video\n <sup>*</sup>\n Bench: Benchmarking Fine-grained Video Perception with \"Think with Videos\"",
"authors": [
"Ruizhe Chen",
"Zhiting Fan",
"Yang Shi",
"Enxin Song",
"Wenhao Chai",
"Tongkun Guan",
"Songtao Jiang",
"Ruilin Luo",
"Yuanxing Zhang",
"Shuai Bai",
"Sibo Song",
"Zhibo Yang",
"Zuozhu Liu"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-002.json",
"source_position": 148,
"registry_object_sha256": "7332426edb3d15499f175fb98c093b78df21ea95fdd9cfea33da5e925cbec171",
"program_id": "mfp7732",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "f6e4df110fe85d7f6d34eb712cd338443c2e34e924393a4e65e07a0c7b152c9e",
"program_context_pointers": [
"/entries/46/papers/198",
"/entries/46/subsessions/11/items/20"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3836447",
"title": "<i>Insert In Style</i>\n : A Zero-Shot Generative Framework for Harmonious Cross-Domain Object Composition",
"authors": [
"Raghu Vamsi Chittersu",
"Yuvraj Singh Rathore",
"Pranav Adlinge",
"Eswar Sai Pashaula",
"Kunal Swami"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-002.json",
"source_position": 183,
"registry_object_sha256": "92b2c094b96cebbace0150805b65b0a1bc3ca99ed42124bcb8de442a81094e1b",
"program_id": "mfp8648",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "39d3f21f774010980f8b30f2fd800b5243ea7ddc6e091b4d8769de3dee622273",
"program_context_pointers": [
"/entries/107/papers/4",
"/entries/117/papers/30",
"/entries/117/subsessions/2/items/6"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3836600",
"title": "<scp>EEGaze</scp>\n : Multimodal Attention Prediction in Visual Search via EEG-Based Saliency Refinement",
"authors": [
"Mansi Sharma",
"Dinar Orazgaliyev",
"Antonio Krüger",
"Philipp Müller"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-003.json",
"source_position": 140,
"registry_object_sha256": "30b99b234aa2b944d6890119b2524d3b3c3c425bde22f20e813b9517b8f8536b",
"program_id": "mfp9815",
"track_literal": "Main Track",
"track": "main",
"abstract_sha256": "47dcb456aee600663d0df59429c1745b335dbae9bdef2dd1063265429fff49fb",
"program_context_pointers": [
"/entries/46/papers/219",
"/entries/46/subsessions/12/items/17"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3838656",
"title": "<scp>MedMKG:</scp>\n Benchmarking Medical Knowledge Exploitation with Multimodal Knowledge Graph",
"authors": [
"Xiaochen Wang",
"Yuan Zhong",
"Lingwei Zhang",
"Lisong Dai",
"Ting Wang",
"Fenglong Ma"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-003.json",
"source_position": 225,
"registry_object_sha256": "b910b66541ca65127646d35d0cd5737a56ae2e0f311a79526005dec75fca739f",
"program_id": "mds177",
"track_literal": "Datasets",
"track": "datasets",
"abstract_sha256": "5bee3b5e9ff60690509d1f34953f3dcc185c50861e112d2c4a3baf5673f6b945",
"program_context_pointers": [
"/entries/46/papers/23",
"/entries/46/subsessions/1/items/11"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3839012",
"title": "<i>EULALens</i>\n : SEE What YOU Agree!",
"authors": [
"Abhijeet Narang",
"Hagar Elbatanouny",
"Vanishri Shekhawat",
"Hasan Al-Nashash",
"Abhinav Dhall",
"Usman Tariq"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-007.json",
"source_position": 238,
"registry_object_sha256": "81dd29dafd6228f51ae2f32e2bb713272a88df78c78347b9170b54f83db00c84",
"program_id": "mde41",
"track_literal": "Demo and Video",
"track": "demo_and_video",
"abstract_sha256": "9acbb264b8019916aad0845aeae4eb5a9c94903ac5bca3cfb612c857306f6d55",
"program_context_pointers": [
"/entries/94/papers/7"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3839024",
"title": "<scp>ThreatVerse:</scp>\n A Closed-Loop World Model Simulator for Autonomous Driving Safety Evaluation",
"authors": [
"Jiangfan Liu",
"Zexuan Cui",
"Tianyuan Zhang",
"Zonglei Jing",
"Yaoyuan Zhang",
"Jiakai Wang",
"Jinyang Guo",
"Yuqing Ma",
"Lu Sheng",
"Aishan Liu",
"Xianglong Liu"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-008.json",
"source_position": 0,
"registry_object_sha256": "d0571f4562b30e4e7fb2e3aa826068ac14a540d1ae70e8cf0b11fd73c67f26ee",
"program_id": "mde15",
"track_literal": "Demo and Video",
"track": "demo_and_video",
"abstract_sha256": "02fc9b31d2b7667ea13f00c74f923a375373042171aaa1ff1f2010ee9697211a",
"program_context_pointers": [
"/entries/49/papers/4"
],
"single_name_positions": []
},
{
"doi": "10.1145/3767308.3839028",
"title": "<scp>FinAlogy:</scp>\n A Visual Analogy Retrieval System for Financial K-Line Analysis",
"authors": [
"Xiaoyan Yu",
"Ziya Zhao",
"Yunshan Ma",
"Yifan Wei",
"Dianbo Sui",
"Tat-Seng Chua"
],
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
"source_page": "page-008.json",
"source_position": 4,
"registry_object_sha256": "b18c8e2821873a59318f6f09d1f453dd02195ccedad5c8e0ce1da60ac07466fb",
"program_id": "mde02",
"track_literal": "Demo and Video",
"track": "demo_and_video",
"abstract_sha256": "22f978dd02c05439663c5724f25bb997098fd85f5558d01d836711252d119931",
"program_context_pointers": [
"/entries/49/papers/0"
],
"single_name_positions": []
}
],
"author_disagreement_hold": {
"doi": "10.1145/3767308.3836308",
"program_id": "mfp7801"
},
"scope": "Onlytitlecomparisonchanges;originalregistryandprogramfixturesremainunchanged;sourceABandtrackcomefromtheexplicitprogramID.",
"held_gold": {
"doi": "10.1145/3767308.3836308",
"title": "Dynamic Late Fusion Multiple Kernel\n <i>k</i>\n -means Clustering based on Adaptive Sample Selection and Alignment",
"authors": [
"Zhirong Tian",
"Yi Zhang",
"Weixuan Liang",
"Miaomiao Li",
"Xinwang Liu"
],
"kind": "program_format_author_version_requires_review",
"source_page": "page-001.json",
"source_position": 82,
"registry_object_sha256": "26a3c1715d276b5fbe988596e3115091086158f6ef12600a36e5206785fc9dfb",
"program_id": null,
"track_literal": null,
"track": "proceedings_unclassified",
"abstract_sha256": null,
"program_context_pointers": [],
"single_name_positions": []
}
}
+124
View File
@@ -0,0 +1,124 @@
"""Checks for published ECCV metadata and preserved submission histories."""
from hashlib import sha256
import json
from pathlib import Path
import sqlite3
import tempfile
import unittest
from paper_atlas.collectors.eccv2026_publications import BOOK_TITLE, SOURCE, collect
from paper_atlas.db.store import Store
from paper_atlas.ingestion.records import ingest_file
FIRST_BOOK = '10.1007/978-3-032-10000-0'
FIRST_DOI = FIRST_BOOK + '_1'
def snapshots(root):
toc, chapters = root / 'toc', root / 'chapters'
toc.mkdir()
chapters.mkdir()
receipts, volumes = [], {}
for offset in range(78):
volume = 17001 + offset
book = f'10.1007/978-3-032-{10000 + offset}-0'
book_url = 'https://link.springer.com/book/' + book
count = 37 if offset < 27 else 36
files, chapter_dois = {}, []
for page, indices in ((1, range(1, 20)), (2, range(20, count + 1))):
filename = f'{volume}-{page}.html'
html = f'<h1>{BOOK_TITLE}</h1><p>Malmö (LNCS, volume {volume})</p>'
html += '<li data-test="chapter"><h3>Front Matter</h3></li>'
for i in indices:
doi = f'{book}_{i}'
html += f'<li data-test="chapter"><h3><a href="/chapter/{doi}">Title {volume}-{i}</a></h3></li>'
chapter_dois.append(doi)
raw = html.encode()
(toc / filename).write_bytes(raw)
files[str(page)] = filename
receipts.append({'file': filename, 'status': 200, 'bytes': len(raw), 'sha256': sha256(raw).hexdigest(),
'url': book_url + ('?page=2' if page == 2 else '')})
volumes[str(volume)] = {'book': book_url, 'files': files, 'chapter_dois': chapter_dois}
data = ''.join(json.dumps(row) + '\n' for row in receipts).encode()
(toc / 'receipts.jsonl').write_bytes(data)
checkpoint = json.dumps(volumes).encode()
(toc / 'volume-checkpoint.json').write_bytes(checkpoint)
(toc / 'summary.json').write_text(json.dumps({
'raw_receipts_sha256': sha256(data).hexdigest(),
'volume_checkpoint_sha256': sha256(checkpoint).hexdigest(),
'chapter_link_count': 2835,
}))
url = 'https://link.springer.com/chapter/' + FIRST_DOI
citation = {
'citation_doi': FIRST_DOI, 'citation_title': 'Title 17001-1',
'citation_inbook_title': BOOK_TITLE, 'citation_publication_date': '2026',
'citation_firstpage': '1', 'citation_lastpage': '10',
'citation_author': 'Chen, Lei',
}
body = ''.join(f'<meta name="{k}" content="{v}">' for k, v in citation.items())
body += '<div id="Abs1-content"><p>Published abstract.</p></div>'
raw = body.encode()
(chapters / 'first.html').write_bytes(raw)
(chapters / 'receipts.json').write_text(json.dumps({'responses': [{
'url': url, 'final_url': url, 'file': 'first.html', 'at': '2026-11-18T00:00:00Z',
'http_status': 200, 'bytes': len(raw), 'sha256': sha256(raw).hexdigest(),
'headers': {'Content-Type': 'text/html'},
}]}))
return {'toc_snapshots': toc, 'chapter_snapshots': chapters}
class ECCV2026PublicationsTests(unittest.TestCase):
def test_frozen_snapshot_and_existing_history(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
source = snapshots(root)
report = collect(root / 'out', **source)
self.assertEqual((report['toc_chapters'], report['chapter_details_missing']), (2835, 2834))
manifest, = report['files']
self.assertEqual((manifest['track'], manifest['status']), ('main', 'partial'))
row = json.loads(Path(manifest['file']).read_text())
self.assertEqual(row['identifiers'], {'doi': FIRST_DOI})
self.assertEqual(row['submission_identifiers'], [{'scheme': 'publication_doi', 'value': FIRST_DOI}])
self.assertEqual(row['authors'], [{'family': 'Chen', 'given': 'Lei'}])
self.assertEqual(row['abstract'], 'Published abstract.')
self.assertEqual(row['metadata']['lncs_volume'], 17001)
self.assertFalse(row['metadata']['original_decision_signature_verified'])
detail = row['metadata']['publisher_chapter_evidence']
self.assertEqual(sha256((root / 'out' / detail['file']).read_bytes()).hexdigest(), detail['sha256'])
db = root / 'example.sqlite'
store = Store(db)
pid, rid = store.record('prior', 'work', 'https://example.org/work',
row['title'], 'Original preprint abstract.', row['authors'],
{'arxiv': '2601.00001'})
store.decision(pid, rid, 'ICLR', 2026, 'main', 'work', 'https://example.org/decision',
'rejected', 'Original ICLR result.')
store.conn.commit()
store.conn.close()
row['identifiers']['arxiv'] = '2601.00001'
path = Path(manifest['file'])
raw = (json.dumps(row) + '\n').encode()
path.write_bytes(raw)
manifest['file_sha256'] = sha256(raw).hexdigest()
path.with_suffix('.manifest.json').write_text(json.dumps(manifest))
ingest_file(db, path)
ingest_file(db, path)
with sqlite3.connect(db) as conn:
self.assertEqual(conn.execute('SELECT id,abstract FROM papers').fetchall(),
[(pid, 'Original preprint abstract.')])
self.assertEqual(conn.execute('SELECT status FROM decisions ORDER BY id').fetchall(),
[('rejected',), ('accepted',)])
self.assertEqual(conn.execute('SELECT scheme,value FROM submission_identifiers').fetchall(),
[('publication_doi', FIRST_DOI)])
self.assertEqual(conn.execute('SELECT name FROM sources ORDER BY id').fetchall(), [('prior',), (SOURCE,)])
with self.assertRaises(FileExistsError):
collect(root / 'out', **source)
def test_changed_contents_receipt_stops_before_output(self):
with tempfile.TemporaryDirectory() as tmp:
root = Path(tmp)
source = snapshots(root)
(source['toc_snapshots'] / '17001-1.html').write_text('unverified bytes')
with self.assertRaisesRegex(ValueError, 'contents bytes disagree'):
collect(root / 'out', **source)
self.assertFalse((root / 'out').exists())
+68
View File
@@ -0,0 +1,68 @@
from hashlib import sha256
import json
from pathlib import Path
from tempfile import TemporaryDirectory
import unittest
from paper_atlas.cli import main
from paper_atlas.collectors.kdd2026 import inventory
def row(number, track, title='Paper title', authors='Name (Affiliation)'):
return {'track': track, 'title': title, 'url': f'https://doi.org/10.1145/3770855.{number}', 'authors': authors}
def snapshot(july, february):
arrays = [
f'<script>const cycle{n}Papers = {json.dumps(rows)}.map((paper, index) => paper);</script>'
for n, rows in ((1, july), (2, february))
]
return '\n'.join(arrays).encode()
class KDD2026Tests(unittest.TestCase):
def test_cli_preserves_withdrawal_marker_and_identical_occurrences(self):
retained = row(1, 'rtp')
withdrawn = row(2, 'ads', '**WITHDRAWN/New SUBMISSION ID for Authorship Order Change 6/24/26 -- A revised work')
incomplete = row(3, 'ais', authors='')
raw = snapshot([retained, retained], [withdrawn, incomplete])
with TemporaryDirectory() as directory:
source = Path(directory) / 'snapshot.html'
source.write_bytes(raw)
out = source.with_suffix('.json')
args = ['collect', 'kdd2026-program', '--snapshot', str(source), '--snapshot-sha256',
sha256(raw).hexdigest(), '--out', str(out)]
main(args)
report = json.loads(out.read_bytes())
with self.assertRaises(FileExistsError):
main(args)
self.assertEqual(report['source_rows_by_cycle_and_track'], {
'july': {'research': 2}, 'february': {'applied_data_science': 1, 'ai_for_sciences': 1},
})
self.assertEqual(report['unique_dois'], 3)
self.assertEqual(report['identical_repeat_rows'], 1)
kept, replaced, sparse = report['records']
self.assertEqual(kept['occurrences'], 2)
self.assertEqual(replaced['programme_status'], 'withdrawn_with_replacement_mentioned')
self.assertEqual(replaced['title'], 'A revised work')
self.assertEqual(replaced['source_title'], withdrawn['title'])
self.assertEqual(replaced['source_withdrawal_date_literal'], '6/24/26')
self.assertNotIn('submission_id', replaced)
self.assertFalse(sparse['authors_listed'])
self.assertFalse(report['database_written'])
def test_conflicting_duplicate_and_unknown_status_fail(self):
with self.assertRaisesRegex(ValueError, 'Conflicting duplicate DOI'):
inventory(snapshot([row(1, 'rtp')], [row(1, 'ads')]))
with self.assertRaisesRegex(ValueError, 'Unrecognized status'):
inventory(snapshot([row(1, 'rtp')], [row(2, 'ads', '***WITHDRAWN -- Unknown format')]))
def test_hash_mismatch_leaves_no_output(self):
with TemporaryDirectory() as directory:
source = Path(directory) / 'snapshot.html'
source.write_bytes(snapshot([row(1, 'rtp')], [row(2, 'ads')]))
out = source.with_suffix('.json')
with self.assertRaisesRegex(ValueError, 'snapshot hash or size'):
main(['collect', 'kdd2026-program', '--snapshot', str(source), '--snapshot-sha256',
'0' * 64, '--out', str(out)])
self.assertFalse(out.exists())
+273
View File
@@ -0,0 +1,273 @@
from contextlib import redirect_stdout
import io
import os
import uuid
import gzip
import hashlib
import json
from pathlib import Path
import re
import tempfile
import unittest
from paper_atlas.collectors import mm_publications2026 as c
class MMPublications2026Tests(unittest.TestCase):
@classmethod
def setUpClass(cls):
fixtures=Path(__file__).parent/'fixtures'
cls.fixture=json.loads(gzip.decompress((fixtures/'mm2026-publications.json.gz').read_bytes()))
cls.program_fixture=json.loads(gzip.decompress((fixtures/'mm2026-source.json.gz').read_bytes()))
cls.format_gold=json.loads((fixtures/'mm2026-title-formatting-review.json').read_text())
overrides={g['doi']:g for g in [*cls.format_gold['gold'],cls.format_gold['held_gold']]}
cls.gold=[overrides.get(g['doi'],g) for g in cls.fixture['independent_gold']]
cls.temp=tempfile.TemporaryDirectory();cls.root=Path(cls.temp.name)
cls.materialize(cls.root/'registry',cls.fixture['files'])
cls.materialize(cls.root/'program',cls.program_fixture['files'])
cls.out=cls.root/'out'
cls.report=c.collect(cls.out,registry_snapshots=cls.root/'registry',program_snapshots=cls.root/'program',parent_snapshot=cls.root/'registry/parent.json')
cls.rows=[json.loads(line) for f in cls.report['files'] for line in Path(f['file']).read_bytes().splitlines()]
cls.by_doi={r['external_id']:r for r in cls.rows}
@classmethod
def tearDownClass(cls):
cls.temp.cleanup()
@staticmethod
def materialize(root,files):
for name,body in files.items():
path=root/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(body.encode('utf-8'))
def test_all1918_publications_against_independent_registry_program_gold(self):
self.assertEqual(len(self.rows),1918);self.assertEqual(len(self.by_doi),1918)
self.assertEqual(set(self.by_doi),{g['doi'] for g in self.gold})
for g in self.gold:
r=self.by_doi[g['doi']];m=r['metadata'];p=m['program_fields']
self.assertEqual((r['title'],r['authors'],r['track']),(g['title'],g['authors'],g['track']))
self.assertEqual(m['program_match_kind'],g['kind'])
self.assertEqual(m['registry_source_evidence']['json_pointer'],'/message/items/'+str(g['source_position']))
self.assertEqual(m['registry_source_evidence']['file'],'evidence/registry/'+g['source_page'])
self.assertEqual(m['registry_source_evidence']['canonical_object_json_sha256'],g['registry_object_sha256'])
self.assertEqual(p['paper_id'] if p else None,g['program_id'])
self.assertEqual([x['source_pointer'] for x in p['contexts']] if p else [],g['program_context_pointers'])
def test_abstract_source_is_separate_and_verbatim_not_fabricated_registry_abstract(self):
count=0
for g in self.gold:
r=self.by_doi[g['doi']];m=r['metadata']
self.assertFalse(m['registry_abstract_available']);self.assertFalse(m['registry_fields'].get('abstract'))
if g['abstract_sha256'] is not None:
count+=1;self.assertEqual(hashlib.sha256(r['abstract'].encode()).hexdigest(),g['abstract_sha256'])
self.assertEqual(r['abstract'],m['program_fields']['abstract'])
self.assertEqual(m['abstract_source'],'official_program_reviewed_title_formatting_correspondence' if g['kind'].startswith('unique_program_format_') else 'official_program_exact_metadata_correspondence')
else:
self.assertIsNone(r['abstract']);self.assertEqual(m['abstract_source'],'not_available_from_reviewed_sources')
self.assertEqual(count,1743);self.assertEqual(self.report['output_missing_abstract_count'],175)
def test_all_program_tracks_and_4326_contexts_survive_in_matches_or_unmatched(self):
programs={r['metadata']['program_fields']['paper_id']:r['metadata']['program_fields'] for r in self.rows if r['metadata']['program_fields']}
for p in self.report['unmatched_program']:
self.assertNotIn(p['paper_id'],programs);programs[p['paper_id']]=p
self.assertEqual(len(programs),2143);self.assertEqual(self.report['unmatched_program_count'],391)
gold=self.program_fixture['independent_inventory']['inventory']
self.assertEqual(set(programs),{g['paper_id'] for g in gold})
for g in gold:
p=programs[g['paper_id']]
self.assertEqual((p['title'],p['authors'],p['track']),(g['title'],g['authors'],g['track_literal']))
self.assertEqual([x['source_pointer'] for x in p['contexts']],g['locations'])
self.assertEqual(sum(len(p['contexts']) for p in programs.values()),4326)
self.assertEqual(len({p['track'] for p in programs.values()}),28)
def test_no_person_alias_original_submission_decision_or_research_type_inference(self):
for r in self.rows:
doi=r['external_id'];m=r['metadata']
self.assertEqual(r['identifiers'],{'doi':doi})
self.assertEqual(r['submission_identifiers'],[{'scheme':'publication_doi','value':doi}])
self.assertEqual((r['source'],r['venue'],r['year']),(c.SOURCE,'MM',2026))
self.assertEqual(r['status'],'accepted');self.assertIn('not an original signed decision',r['raw_decision'])
self.assertFalse(m['original_decision_signature_verified']);self.assertFalse(m['research_type_classified'])
self.assertFalse(m['scientific_work_identity_approved'])
self.assertTrue({'decided_at','decision_date','source_decision_id','paper_type'}.isdisjoint(r))
self.assertIn({'namespace':c.SOURCE+':crossref:type','value':'proceedings-article'},r['tags'])
self.assertTrue({x['namespace'] for x in r['tags']} <= {c.SOURCE+':crossref:type',c.SOURCE+':mm_program:track'})
for track in ('interactive_art','industry_talk','demo_and_video'):
self.assertTrue(any(r['track']==track for r in self.rows))
variants=[r for r in self.rows if r['track']=='proceedings_unclassified']
self.assertEqual(len(variants),166)
self.assertTrue(all(r['metadata']['program_fields'] is None and r['abstract'] is None for r in variants))
def test_original_bibliography_evidence_kept_without_duplicating_reference_payload(self):
authors_by_page={}
for name,body in self.fixture['files'].items():
actual=self.out/'evidence/registry'/('parent.json' if name=='parent.json' else Path(name).name)
self.assertEqual(actual.read_bytes(),body.encode())
if re.fullmatch(r'page-[0-9]+\.json',Path(name).name):
authors_by_page[str(actual.relative_to(self.out))]=[row['author'] for row in json.loads(body)['message']['items']]
for name,body in self.program_fixture['files'].items():
self.assertEqual((self.out/'evidence/program'/Path(name).name).read_bytes(),body.encode())
for r in self.rows:
m=r['metadata'];self.assertNotIn('reference',m['registry_fields'])
self.assertEqual(m['registry_fields']['title'],[r['title']])
self.assertEqual(m['registry_fields']['author'],authors_by_page[m['registry_source_evidence']['file']][int(m['registry_source_evidence']['json_pointer'].rsplit('/',1)[-1])])
for f in self.report['files']:
raw=Path(f['file']).read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),f['file_sha256'])
self.assertEqual(len(raw.splitlines()),f['count']);self.assertEqual(f['status'],'partial')
def test_eight_single_name_fields_preserved_without_splitting(self):
n=0
for g in self.gold:
r=self.by_doi[g['doi']];flags=r['metadata']['review_flags']
actual=[x['author_position'] for x in flags if x['kind']=='single_registered_name_field_not_split']
self.assertEqual(actual,g['single_name_positions']);n+=len(actual)
for i in actual:
self.assertEqual(r['authors'][i-1],r['metadata']['registry_fields']['author'][i-1]['family'].strip())
self.assertEqual(n,8)
def test_structured_title_markup_retained_and_flagged_without_flattening(self):
count=0
for g in self.gold:
if re.search(r'<[^>]+>|&[A-Za-z#0-9]+;',g['title']):
count+=1;r=self.by_doi[g['doi']]
self.assertEqual(r['title'],g['title'])
self.assertIn({'kind':'literal_title_markup_or_entity_retained_requires_review'},r['metadata']['review_flags'])
self.assertGreater(count,0);self.assertEqual(count,self.report['registered_title_markup_flags'])
def test_formatting_gold_preserves_raw_fields_and_copies_exact_program_evidence(self):
self.assertEqual(len(self.format_gold['gold']),26)
for gold in self.format_gold['gold']:
row=self.by_doi[gold['doi']];m=row['metadata'];p=m['program_fields']
self.assertEqual((row['title'],row['authors']),(gold['title'],gold['authors']))
self.assertEqual((p['paper_id'],row['track']),(gold['program_id'],gold['track']))
self.assertEqual(hashlib.sha256(row['abstract'].encode()).hexdigest(),gold['abstract_sha256'])
comparison=m['program_title_formatting_comparison']
self.assertTrue(comparison['raw_titles_preserved']);self.assertFalse(comparison['work_identity_approved'])
self.assertEqual(set(comparison['program_comparison_texts']),{p['paper_id']})
self.assertFalse(m['scientific_work_identity_approved'])
held=self.by_doi[self.format_gold['author_disagreement_hold']['doi']]
self.assertEqual(held['metadata']['program_title_candidates'],['mfp7801'])
self.assertEqual(held['metadata']['program_exact_candidates'],[])
self.assertEqual(held['metadata']['program_match_kind'],'program_format_author_version_requires_review')
self.assertIsNone(held['abstract']);self.assertEqual(held['track'],'proceedings_unclassified')
def test_reviewed_formatting_keeps_superscript_subscript_and_decodes_entities_once(self):
self.assertEqual(c.formatting_text('<i>A</i> H<sup>2</sup> B<sub>1</sub> &amp; C'),'A H2 B1 & C')
self.assertEqual(c.formatting_text('A &amp;amp; B'),'A &amp; B')
def test_unknown_attributes_unbalanced_or_encoded_markup_not_discarded(self):
for title in ['A <model>B</model>','A <i class="x">B</i>','A <i>B','A </i>B',
'A <sup/>B','A <!--hidden-->B','<!DOCTYPE html>A','A &lt;i&gt;B&lt;/i&gt;',
'A <sup','A <i>B</sup>','<![CDATA[A]]>']:
with self.subTest(title=title):self.assertIsNone(c.formatting_text(title))
def test_ambiguous_program_and_author_multiset_differences_remain_held(self):
def observation(pid,authors):
return {'paper_id':pid,'title':'M3 Model','authors':authors,'track':'Main Track'}
rows=[{'title':'M<sup>3</sup> Model','authors':['A','B']}]
kinds,_=c.match_program(rows,[observation('p1',['A','B']),observation('p2',['B','A'])])
self.assertEqual(kinds,{'multiple_program_ids_same_title_authors_hold':1})
self.assertNotIn('program',rows[0])
rows=[{'title':'M<sup>3</sup> Model','authors':['A','A','B']}]
c.match_program(rows,[observation('p1',['A','B'])])
self.assertEqual(rows[0]['kind'],'program_format_author_version_requires_review')
self.assertNotIn('program',rows[0])
def test_literal_title_author_disagreement_does_not_use_formatting_bypass(self):
rows=[{'title':'M<sup>3</sup> Model','authors':['A','B']}]
observations=[{'paper_id':'literal','title':'M<sup>3</sup> Model','authors':['Other'],'track':'Main Track'},
{'paper_id':'formatted','title':'M3 Model','authors':['A','B'],'track':'Main Track'}]
c.match_program(rows,observations)
self.assertEqual(rows[0]['kind'],'program_author_version_requires_review')
self.assertEqual(rows[0]['program_title_candidates'],['literal']);self.assertNotIn('program',rows[0])
def test_corrupt_registry_parent_or_program_keeps_failure_not_complete_output(self):
for kind in ('registry_manifest','registry_page','parent','program'):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);self.materialize(root/'registry',self.fixture['files']);self.materialize(root/'program',self.program_fixture['files'])
path={'registry_manifest':root/'registry'/c.SNAPSHOT/'manifest.json','registry_page':root/'registry'/c.SNAPSHOT/'page-001.json','parent':root/'registry/parent.json','program':root/'program'/c.program.SNAPSHOT/'program-data.json'}[kind]
path.write_bytes(path.read_bytes()+b' ')
with self.subTest(kind=kind),self.assertRaises(ValueError):
c.collect(root/'out',registry_snapshots=root/'registry',program_snapshots=root/'program',parent_snapshot=root/'registry/parent.json')
self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists())
def test_symlink_source_rejected_and_no_previous_output_overwrite(self):
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);self.materialize(root/'registry',self.fixture['files'])
path=root/'registry/parent.json';original=root/'registry/actual.json';path.rename(original);path.symlink_to(original)
with self.assertRaisesRegex(ValueError,'Unsafe'):
c.collect(root/'out',registry_snapshots=root/'registry',program_snapshots=self.root/'program',parent_snapshot=path)
with self.assertRaisesRegex(ValueError,'Output exists'):
c.collect(self.out,registry_snapshots=self.root/'registry',program_snapshots=self.root/'program',parent_snapshot=self.root/'registry/parent.json')
def test_complete_report_retains_source_partitions_not_import_or_final_dedup_claim(self):
self.assertEqual(self.report['program_match_counts'],{'unique_program_title_fullauthor_agreement_not_identity_approval':1726,'unique_program_format_title_fullauthor_agreement_not_identity_approval':26,'program_author_version_requires_review':122,'program_format_author_version_requires_review':1,'no_exact_program_title_requires_review':43})
self.assertEqual(self.report['registry_missing_abstract_count'],1918)
self.assertEqual(self.report['abstracts_from_exact_program_correspondence'],1743)
self.assertEqual(self.report['abstracts_from_literal_program_correspondence'],1717)
self.assertEqual(self.report['abstracts_from_reviewed_title_formatting'],26)
self.assertEqual(self.report['unmatched_program_count'],391)
self.assertEqual(sum(f['count'] for f in self.report['files']),1918)
self.assertEqual(self.report['approved_for_import'],0);self.assertFalse(self.report['database_written'])
self.assertFalse(self.report['pdfs_downloaded']);self.assertEqual(self.report['network_requests'],0)
self.assertIn('unknown',self.report['public_negative_availability'])
def assert_native_ingestion(self, pg_service=None):
# An isolated synthetic database exercises the writer contract, not a
# production identity approval. All source rows/track partitions run.
from paper_atlas.db.store import Store
from paper_atlas.db.backend import open_store
from paper_atlas.ingestion.records import ingest_file
from paper_atlas.postgres.client import connect
from paper_atlas.postgres.migration import migrate
from paper_atlas.postgres.schema import quote
with tempfile.TemporaryDirectory() as tmp:
root=Path(tmp);db=root/'seed.sqlite';seed=self.rows[0]
store=Store(db)
paper,record=store.record('arxiv','synthetic-mm-anchor','https://example.org/arxiv',
seed['title'],'Existing canonical abstract',seed['authors'],seed['identifiers'])
store.decision(paper,record,'ICLR',2025,'main','synthetic-old-forum',
'https://example.org/decision','rejected','Reject')
before=tuple(store.conn.execute('SELECT * FROM papers WHERE id=?',(paper,)).fetchone())
store.conn.commit();store.conn.close()
schema='pa_mm_pub_'+uuid.uuid4().hex[:12] if pg_service else None
try:
with redirect_stdout(io.StringIO()):
if pg_service:migrate(db,pg_service,root/'migration.json',schema=schema,min_free_gib=0)
for f in self.report['files']:
ingest_file(None if pg_service else db,f['file'],pg_service=pg_service,pg_schema=schema)
store=open_store(None if pg_service else db,pg_service=pg_service,pg_schema=schema)
try:
conn=store.conn
self.assertEqual(tuple(conn.execute('SELECT * FROM papers WHERE id=?',(paper,)).fetchone()),before)
self.assertEqual(conn.execute('SELECT status FROM decisions WHERE submission_id=1').fetchone()[0],'rejected')
actual={r[0]:r for r in conn.execute("SELECT sr.external_id,sr.paper_id,sr.title,sr.abstract,sr.authors_json,sr.metadata_json,sr.id FROM source_records sr JOIN sources s ON s.id=sr.source_id WHERE s.name=?",(c.SOURCE,))}
self.assertEqual(set(actual),set(self.by_doi))
self.assertEqual(actual[seed['external_id']][1],paper)
for doi,row in self.by_doi.items():
got=actual[doi];self.assertEqual((got[2],got[3],json.loads(got[4])),(row['title'],row['abstract'],row['authors']))
meta=json.loads(got[5]);self.assertEqual(meta.pop('record_identifiers'),{'doi':doi,c.SOURCE:doi});self.assertEqual(meta,row['metadata'])
tags=conn.execute("SELECT t.namespace,t.value,count(*) FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id GROUP BY t.namespace,t.value").fetchall()
self.assertEqual(sum(r[2] for r in tags if r[0]==c.SOURCE+':crossref:type'),1918)
self.assertEqual(sum(r[2] for r in tags if r[0]==c.SOURCE+':mm_program:track'),1752)
self.assertEqual({r[0] for r in tags},{c.SOURCE+':crossref:type',c.SOURCE+':mm_program:track'})
decisions=conn.execute("SELECT d.status,d.decision_date,count(*) FROM decisions d JOIN submissions s ON s.id=d.submission_id JOIN source_records r ON r.id=s.source_record_id JOIN sources src ON src.id=r.source_id WHERE src.name=? GROUP BY d.status,d.decision_date",(c.SOURCE,)).fetchall()
self.assertEqual([tuple(r) for r in decisions],[('accepted',None,1918)])
self.assertEqual(conn.execute("SELECT count(*) FROM submission_identifiers WHERE scheme='publication_doi'").fetchone()[0],1918)
coverage=conn.execute("SELECT es.status,sum(es.imported_count) FROM edition_sources es JOIN sources s ON s.id=es.source_id WHERE s.name=? GROUP BY es.status",(c.SOURCE,)).fetchall()
self.assertEqual([tuple(r) for r in coverage],[('partial',1918)])
finally:store.conn.close()
finally:
if pg_service:
with connect(pg_service) as pg:
for name, in pg.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)',(schema,)).fetchall():
pg.execute('DROP SCHEMA '+quote(name)+' CASCADE')
def test_all1918_native_sqlite_source_tags_and_prior_rejection_survive(self):
self.assert_native_ingestion()
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'),'Requires approved remote PostgreSQL test service')
def test_all1918_native_postgres_source_tags_and_prior_rejection_survive(self):
self.assert_native_ingestion(os.environ['PAPER_ATLAS_TEST_PG_SERVICE'])
+31 -1
View File
@@ -15,7 +15,7 @@ from paper_atlas.db.backend import open_store, open_read_connection
from paper_atlas.db.store import Store from paper_atlas.db.store import Store
from paper_atlas.ingestion import arxiv from paper_atlas.ingestion import arxiv
from paper_atlas.ingestion.backfill import backfill from paper_atlas.ingestion.backfill import backfill
from paper_atlas.ingestion.records import ingest_file from paper_atlas.ingestion.records import coverage_options, ingest_file, require_incremental_parent
from paper_atlas.postgres.client import connect, ReadConnection from paper_atlas.postgres.client import connect, ReadConnection
from paper_atlas.postgres.migration import migrate from paper_atlas.postgres.migration import migrate
from paper_atlas.postgres.schema import quote from paper_atlas.postgres.schema import quote
@@ -114,6 +114,36 @@ class PostgreSQLIngestionTests(unittest.TestCase):
self.assertEqual(run[0],1) self.assertEqual(run[0],1)
self.assertEqual(json.loads(run[1])['coverage']['imported_count'],3) self.assertEqual(json.loads(run[1])['coverage']['imported_count'],3)
def test_incremental_parent_preflight_uses_readonly_postgres_rows_and_keeps_guards(self):
rows = [self.row(source='dblp', external=key, status='accepted', year=2001, venue='UAI')
for key in ('preflight1', 'preflight2')]
path = self.file('preflight-parent', rows)
target = path.with_suffix('.manifest.json')
manifest = json.loads(target.read_text())
manifest['status'] = 'partial'
target.write_text(json.dumps(manifest))
self.ingest(path)
proposed = dict(manifest, coverage_mode='incremental', count=1,
coverage_previous_count=2, coverage_previous_expected_count=2,
coverage_expected_count=4)
with closing(ReadConnection(self.service, self.schema)) as c:
self.assertEqual(c.execute('SHOW transaction_read_only').fetchone()[0], 'on')
source = c.execute("SELECT id FROM sources WHERE name='dblp'").fetchone()[0]
before = {table: c.execute('SELECT count(*) FROM '+table).fetchone()[0]
for table in ('papers', 'source_records', 'submissions', 'decisions', 'ingest_runs')}
coverage = [dict(row) for row in c.execute('SELECT * FROM edition_sources ORDER BY edition_id,source_id,list_url')]
require_incremental_parent(c, source, proposed, coverage_options(proposed))
for changes, error in (({'coverage_previous_count': 1}, 'parent changed'),
({'coverage_previous_expected_count': 3}, 'parent changed'),
({'list_url': 'https://example.org/other-list'}, 'unambiguous source listing')):
with self.subTest(changes=changes):
stale = dict(proposed, **changes)
with self.assertRaisesRegex(ValueError, error):
require_incremental_parent(c, source, stale, coverage_options(stale))
self.assertEqual({table: c.execute('SELECT count(*) FROM '+table).fetchone()[0]
for table in before}, before)
self.assertEqual([dict(row) for row in c.execute('SELECT * FROM edition_sources ORDER BY edition_id,source_id,list_url')], coverage)
def test_incremental_postgres_bound_failure_rolls_back_rows_and_summary(self): def test_incremental_postgres_bound_failure_rolls_back_rows_and_summary(self):
listing='https://example.org/uai/2001' listing='https://example.org/uai/2001'
rows=[self.row(source='dblp',external=key,status='accepted',year=2001,venue='UAI') rows=[self.row(source='dblp',external=key,status='accepted',year=2001,venue='UAI')
+203
View File
@@ -0,0 +1,203 @@
"""Metadata projection never requests OCR/review column pages or promotes mirror IDs."""
from collections import Counter
import hashlib
import importlib.util
import io
import json
from pathlib import Path
import re
import struct
import tempfile
import unittest
from unittest.mock import patch
import urllib.error
from paper_atlas.collectors.reviewarena import COLUMNS, COMMIT, collect, decision_claim, decision_track_claim, inventory, record
from paper_atlas.io.parquet_metadata import RangeSource, SparseFile, projection
class MemorySource:
def __init__(self, raw):
self.raw, self.size, self.bytes_read, self.receipts = raw, len(raw), 0, []
def read(self, start, length):
raw = self.raw[start:start+length]
self.bytes_read += len(raw)
self.receipts.append({'start': start, 'length': length, 'sha256': hashlib.sha256(raw).hexdigest()})
return raw
def fixture():
import pyarrow as pa
import pyarrow.parquet as pq
rows = []
for i, decision in enumerate(['Accept-Findings', 'Accept-Main', 'Reject', 'Withdrawn', None]):
row = {'forum_id': 'unverified-source-'+str(i)+'.xml', 'conference': 'emnlp', 'year': 2023,
'track': 'main', 'venue_id': 'source-venue-claim',
'paper_number': i, 'title': 'Metadata title '+str(i), 'abstract': 'Metadata abstract\x00 '+str(i),
'authors': ['Anonymous'] if i == 2 else ['Ada Author', 'Yi Researcher'],
'keywords': ['metadata'], 'venue': 'Source venue label', 'decision': decision,
'markdown': 'OCR_FULLTEXT_SENTINEL '+('private-ish fixture body '*10000),
'reviews_json': 'REVIEW_FULLTEXT_SENTINEL '+('fixture review '*10000),
'decision_comment': 'META_REVIEW_SENTINEL '+('fixture meta-review '*10000)}
rows.append(row)
sink = io.BytesIO(); pq.write_table(pa.Table.from_pylist(rows), sink, row_group_size=2, compression='snappy')
return sink.getvalue(), [{k: v for k, v in r.items() if k in COLUMNS} for r in rows]
class RangeHTTPTests(unittest.TestCase):
def response(self, data, status=206, start=2, size=10, etag='"stable"'):
response = io.BytesIO(data); response.status = status
response.headers = {'Content-Range': f'bytes {start}-{start+len(data)-1}/{size}', 'ETag': etag}
return response
def test_strict_range_and_changed_object_detection(self):
opener = unittest.mock.Mock(); opener.open.side_effect = [self.response(b'ab'), self.response(b'cd', start=4, etag='"changed"')]
source = RangeSource('https://example.org/public', 10, opener=opener)
self.assertEqual(source.read(2, 2), b'ab')
self.assertEqual(opener.open.call_args.args[0].get_header('Range'), 'bytes=2-3')
with self.assertRaisesRegex(ValueError, 'ETag'):
source.read(4, 2)
self.assertEqual(source.bytes_read, 2)
def test_no_full_file_fallback_or_denied_source_retry(self):
opener = unittest.mock.Mock(); response = self.response(b'ab', status=200)
response.read = unittest.mock.Mock(side_effect=AssertionError('Must not consume full body'))
opener.open.return_value = response
with self.assertRaisesRegex(ValueError, 'full download forbidden'):
RangeSource('https://example.org/public', 10, opener=opener).read(2, 2)
response.read.assert_not_called()
opener = unittest.mock.Mock(); opener.open.side_effect = urllib.error.HTTPError('https://cdn.example/signed?SECRET=hidden', 403, 'Forbidden', {}, None)
with self.assertRaisesRegex(RuntimeError, '^Metadata source HTTP 403; no retry or full-file fallback$'):
RangeSource('https://example.org/public', 10, opener=opener).read(2, 2)
self.assertEqual(opener.open.call_count, 1)
def test_sparse_footer_windows_are_synthetic_not_network_reads(self):
source = SparseFile(100, [(0, b'PAR1'), (95, b'footer')])
source.seek(90); self.assertEqual(source.read(10), b'\x00'*5+b'foote')
source.seek(-4, 2); self.assertEqual(source.tell(), 96)
with self.assertRaises(ValueError):
source.seek(-101, 2)
@unittest.skipUnless(importlib.util.find_spec('pyarrow'), 'Install mirrors extra on the approved remote runner')
class ParquetProjectionTests(unittest.TestCase):
def test_complete_selected_rows_and_no_excluded_page_requests(self):
import pyarrow.parquet as pq
raw, expected = fixture(); source = MemorySource(raw); rows, proof = projection(source, COLUMNS)
self.assertEqual(rows, expected)
metadata = pq.ParquetFile(io.BytesIO(raw)).metadata
excluded = []
for i in range(metadata.num_row_groups):
group = metadata.row_group(i)
for j in range(group.num_columns):
col = group.column(j)
if col.path_in_schema.split('.')[0] not in COLUMNS:
start = min(x for x in [col.data_page_offset, col.dictionary_page_offset] if x is not None and x > 0)
excluded.append((start, start+col.total_compressed_size))
for request in source.receipts:
a, b = request['start'], request['start']+request['length']
self.assertFalse(any(max(a, c) < min(b, d) for c, d in excluded))
self.assertFalse(proof['raw_parquet_saved']); self.assertFalse(proof['full_file_checksum_verified'])
self.assertLess(source.bytes_read, len(raw))
encoded = json.dumps(rows)+json.dumps(proof)
self.assertNotIn('OCR_FULLTEXT_SENTINEL', encoded); self.assertNotIn('REVIEW_FULLTEXT_SENTINEL', encoded)
self.assertNotIn('META_REVIEW_SENTINEL', encoded)
def test_wrong_schema_and_footer_fail_before_any_payload_download(self):
raw, _ = fixture(); source = MemorySource(raw)
with self.assertRaisesRegex(ValueError, 'Missing'):
projection(source, {'missing_column': 'string'})
self.assertEqual(len(source.receipts), 3)
source = MemorySource(raw)
with self.assertRaisesRegex(ValueError, 'type differs'):
projection(source, {**COLUMNS, 'year': 'string'})
self.assertEqual(len(source.receipts), 3)
bad = bytearray(raw); bad[-8:-4] = struct.pack('<I', len(raw))
source = MemorySource(bytes(bad))
with self.assertRaisesRegex(ValueError, 'footer'):
projection(source, COLUMNS)
self.assertEqual(len(source.receipts), 2)
def test_collector_output_preserves_claims_without_original_identity(self):
raw, _ = fixture(); item = {'path': 'data/emnlp-00000-of-00001.parquet', 'size': len(raw), 'lfs': {'oid': 'a'*64}}
with tempfile.TemporaryDirectory() as temp:
out = Path(temp)/'projected'
with patch('paper_atlas.collectors.reviewarena.inventory', return_value=[item]):
report = collect('emnlp', out, source_factory=lambda url, size: MemorySource(raw))
self.assertEqual(report['mirror_decision_counts'], {'accepted': 2, 'rejected': 1, 'withdrawn': 1, 'unknown': 1})
self.assertEqual(report['records'], 5); self.assertFalse(report['database_written'])
projected = [json.loads(line) for line in next(out.glob('*.jsonl')).read_text().splitlines()]
for row in projected:
self.assertEqual(row['identifiers'], {})
self.assertFalse(row['metadata']['original_forum_id_verified'])
self.assertFalse(row['metadata']['original_decision_signature_verified'])
self.assertNotIn('status', row)
self.assertIn(COMMIT, row['external_id'])
self.assertEqual(projected[2]['authors'], ['Anonymous'])
self.assertEqual(projected[2]['metadata']['forum_id_claim'], 'unverified-source-2.xml')
self.assertEqual(projected[0]['track'], 'main')
self.assertEqual(projected[0]['mirror_decision_track_claim'], 'findings')
all_saved=''.join(p.read_text() for p in out.iterdir())
self.assertNotIn('FULLTEXT_SENTINEL', all_saved); self.assertNotIn('META_REVIEW_SENTINEL', all_saved)
with self.assertRaisesRegex(ValueError, 'New output'):
collect('emnlp', out)
def test_offline_replay_reclassifies_literals_preserving_all_raw_fields_without_network(self):
raw, _ = fixture(); item = {'path': 'data/emnlp-00000-of-00001.parquet', 'size': len(raw), 'lfs': {'oid': 'a'*64}}
with tempfile.TemporaryDirectory() as temp, patch('paper_atlas.collectors.reviewarena.inventory', return_value=[item]):
out = Path(temp)/'v1'; collect('emnlp', out, source_factory=lambda url, size: MemorySource(raw))
file, = out.glob('*.jsonl'); before = [json.loads(x) for x in file.read_text().splitlines()]
# Mimic the first reviewed source version: new literal spellings were unknown.
for row in before[:2]:
row['mirror_decision_category'] = 'unknown'; row.pop('mirror_decision_track_claim')
file.write_text(''.join(json.dumps(x)+'\n' for x in before))
report_path = out/'collection-report.json'; report = json.loads(report_path.read_text())
report['files'][0]['file_sha256'] = hashlib.sha256(file.read_bytes()).hexdigest()
report_path.write_text(json.dumps(report)); report_hash = hashlib.sha256(report_path.read_bytes()).hexdigest()
no_network = unittest.mock.Mock(side_effect=AssertionError('Offline replay must not contact source'))
result = collect('emnlp', Path(temp)/'v2', from_projection=report_path, projection_sha256=report_hash, source_factory=no_network)
no_network.assert_not_called(); self.assertEqual(result['metadata_bytes_read'], 0)
self.assertEqual(result['source_projection_bytes'], report['metadata_bytes_read'])
self.assertEqual(result['mirror_decision_counts']['accepted'], 2)
self.assertEqual(result['mirror_decision_track_claim_counts'], {'findings': 1, 'main': 1, 'unspecified': 3})
new = [json.loads(x) for x in next((Path(temp)/'v2').glob('*.jsonl')).read_text().splitlines()]
self.assertEqual([r['raw_fields'] for r in new], [r['raw_fields'] for r in before])
self.assertEqual([r['source_locator'] for r in new], [r['source_locator'] for r in before])
with self.assertRaisesRegex(ValueError, 'hash differs'):
collect('emnlp', Path(temp)/'bad', from_projection=report_path, projection_sha256='0'*64, source_factory=no_network)
no_network.assert_not_called()
def test_failure_is_retained_without_source_payload_or_retries(self):
item = {'path': 'data/emnlp-00000-of-00001.parquet', 'size': 50, 'lfs': {'oid': 'a'*64}}
with tempfile.TemporaryDirectory() as temp:
out = Path(temp)/'failed'
with patch('paper_atlas.collectors.reviewarena.inventory', return_value=[item]), patch('paper_atlas.collectors.reviewarena.projection', side_effect=RuntimeError('private body example')):
with self.assertRaises(RuntimeError):
collect('emnlp', out, source_factory=lambda url, size: None)
report = json.loads((out/'failure.json').read_text())
self.assertEqual(report['status'], 'incomplete_do_not_import')
self.assertTrue(report['no_automatic_retry']); self.assertNotIn('private body', (out/'failure.json').read_text())
class MirrorSemanticsTests(unittest.TestCase):
def test_pinned_inventory_covers_all_seven_families_and_fifteen_shards(self):
splits = ['colm', 'corl', 'emnlp', 'iclr', 'icml', 'neurips', 'tmlr']
result = {s: inventory(s) for s in splits}
self.assertEqual(sum(len(v) for v in result.values()), 15)
self.assertEqual([len(result[s]) for s in splits], [1, 1, 1, 5, 1, 5, 1])
self.assertEqual(result['emnlp'][0]['size'], 90794691)
self.assertEqual(result['emnlp'][0]['lfs']['oid'], 'e9e3aab42d28ca32a02287ed0bd0f1f9e1f3af5cb8ed8d6a9bf28d4f630773da')
with self.assertRaises(ValueError):
inventory('uai')
def test_missing_or_ambiguous_decision_is_never_inferred_from_venue(self):
for literal in ['', None, 'Under review', 'Final venue pending', 'Accept maybe', 'Reject?']:
self.assertEqual(decision_claim(literal), 'unknown')
self.assertEqual(decision_claim('Desk Rejected'), 'desk_rejected')
self.assertEqual(decision_claim('Withdrawn'), 'withdrawn')
self.assertEqual(decision_claim('Accept (Findings)'), 'accepted')
self.assertEqual(decision_claim('Accept-Main'), 'accepted')
self.assertEqual(decision_claim('Accept-Findings'), 'accepted')
self.assertEqual(decision_track_claim('Accept-Findings'), 'findings')
self.assertIsNone(decision_track_claim('Accept (Poster)'))
+209
View File
@@ -0,0 +1,209 @@
"""Whole-source scope correction, stale refresh, full history and atomicity on both engines."""
from contextlib import closing, redirect_stdout
import copy
import hashlib
import io
import json
import os
from pathlib import Path
import tempfile
import unittest
from unittest.mock import patch
import uuid
from paper_atlas.db.backend import open_store
from paper_atlas.db.edition_scopes import KEY
from paper_atlas.db.source_revisions import available, require_row, require_manifest
from paper_atlas.db.store import Store
from paper_atlas.ingestion.source_revisions import proposal, apply_plan, run, validate
from paper_atlas.ingestion.event_identity import database_identity, digest, values
from paper_atlas.ingestion.records import coverage_options, ingest_file
from paper_atlas.postgres.client import connect
from paper_atlas.postgres.migration import migrate
from paper_atlas.postgres.schema import quote
URL = 'https://example.org/volume'
def seed(path):
old = []
for i in range(4):
doi = '10.1234/publication'+str(i)
old.append({'source': 'fixture', 'venue': 'MM', 'year': 2026, 'track': 'main' if i == 0 else 'unclassified',
'external_id': doi, 'url': 'https://doi.org/'+doi, 'title': 'Distinct publication '+str(i),
'authors': ['Alice Author', 'Bob Writer'], 'abstract': None if i == 1 else 'Source abstract '+str(i),
'identifiers': {'doi': doi}, 'submission_identifiers': {'publication_doi': doi},
'status': 'accepted', 'raw_decision': 'Publication observation', 'evidence_url': URL,
'observed_at': '2026-06-01T00:00:00Z', 'tags': [{'namespace': 'fixture:track', 'value': 'main' if i == 0 else 'unclassified'}],
'metadata': {'original': i, 'program': None, 'scientific_work_identity_approved': False}})
current = copy.deepcopy(old)
current[1].update(track='main', abstract='Reviewed program abstract with NUL\x00 and SOH\x01')
current[1]['metadata']['program'] = {'id': 'p1', 'evidence_url': URL}
current[1]['tags'][0]['value'] = 'main'
store = Store(path)
for row in old[:3]:
p, r = store.record(row['source'], row['external_id'], row['url'], row['title'], row['abstract'], row['authors'], row['identifiers'], row['metadata'])
store.decision(p, r, 'MM', 2026, row['track'], row['external_id'], URL, 'accepted', row['raw_decision'],
identities=[('publication_doi', row['external_id'])], observed_at=row['observed_at'])
store.record_tags(p, r, 'fixture', row['tags'])
if row == old[1]:
store.conn.execute('UPDATE papers SET abstract=? WHERE id=?', ('Canonical preprint abstract', p))
p, other = store.record('prior', 'old-forum', 'https://example.org/prior', row['title'], 'Canonical preprint abstract', row['authors'], row['identifiers'])
store.decision(p, other, 'PRIOR', 2025, 'main', 'old-forum', 'https://example.org/prior', 'rejected', 'Reject', observed_at='2025-01-01T00:00:00Z')
# An unrelated source already has an attempt in the populated target edition.
p, r = store.record('other', 'other', 'https://example.org/other', 'Unrelated target paper', 'Other abstract', ['Someone Else'])
store.decision(p, r, 'MM', 2026, 'main', 'other', 'https://example.org/other', 'accepted', 'Published')
sid = store.conn.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0]
for track, expected, imported in [('main', 1, 1), ('unclassified', 3, 2)]:
eid = store.conn.execute('SELECT id FROM editions WHERE year=2026 AND track=?', (track,)).fetchone()[0]
store.conn.execute("INSERT INTO edition_sources VALUES(?,?,?,'partial',?,?,?,'2026-06-01',NULL)", (eid, sid, URL, expected, imported, 'checkpoint'))
store.conn.commit(); store.conn.close()
return {'format': 1, 'source': 'fixture', 'venue': 'MM', 'year': 2026, 'event_key': '', 'list_url': URL,
'previous_rows': old, 'rows': current, 'evidence': [{'url': URL, 'sha256': 'a'*64}]}
class RevisionCases:
def store(self):
return open_store(**self.target)
def plan(self, c):
return proposal(c, self.inventory, database_identity(c, self.target.get('db')))
def test_move_to_populated_target_preserves_full_history_and_before_images(self):
with closing(self.store().conn) as c:
plan = self.plan(c); self.assertEqual(len(plan['changes']), 1)
self.assertEqual(len(plan['before']['packet']['submissions']), 5)
before_papers = plan['before']['packet']['papers']
before_tags = plan['before']['packet']['paper_tags']
result = apply_plan(c, plan); c.commit()
self.assertEqual((result['events_moved'], result['records_changed']), (1, 1))
self.assertEqual(result['protected_decisions'], 5)
self.assertEqual(values(c.execute('SELECT * FROM papers ORDER BY id')), before_papers)
self.assertEqual(values(c.execute('SELECT * FROM paper_tags ORDER BY paper_id,tag_id,evidence_key')), before_tags)
self.assertEqual(c.execute("SELECT count(*) FROM decisions WHERE status='rejected'").fetchone()[0], 1)
self.assertEqual(values(c.execute('SELECT status,expected_count,imported_count,cursor,last_success_at,last_error FROM edition_sources ORDER BY edition_id')),
[['partial', 2, 2, 'checkpoint', '2026-06-01', None], ['partial', 2, 1, 'checkpoint', '2026-06-01', None]])
saved = json.loads(c.execute('SELECT plan_json FROM source_scope_revisions').fetchone()[0])
self.assertEqual(saved, plan)
self.assertEqual(apply_plan(c, plan)['status'], 'already_applied')
self.assertEqual(c.execute('SELECT count(*) FROM source_scope_revisions').fetchone()[0], 1)
def test_stale_direct_store_and_jsonl_refresh_fail_but_current_refresh_succeeds(self):
with closing(self.store().conn) as c:
plan = self.plan(c); apply_plan(c, plan); c.commit()
with closing(self.store().conn) as c:
store = object.__new__(Store); store.conn = c; store.event_identity = True
old = self.inventory['previous_rows'][1]
with self.assertRaisesRegex(ValueError, 'Stale source'):
store.record('fixture', old['external_id'], old['url'], old['title'], old['abstract'], old['authors'], old['identifiers'], old['metadata'])
c.rollback()
row = self.inventory['rows'][1]
p, r = store.record('fixture', row['external_id'], row['url'], row['title'], row['abstract'], row['authors'], row['identifiers'], row['metadata'])
self.assertEqual(c.execute('SELECT abstract FROM papers WHERE id=?', (p,)).fetchone()[0], 'Canonical preprint abstract')
self.assertEqual(json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?', (r,)).fetchone()[0])[KEY][0]['target']['track'], 'main')
c.rollback()
path = self.root/'old.jsonl'; raw = (json.dumps(old)+'\n').encode(); path.write_bytes(raw)
path.with_suffix('.manifest.json').write_text(json.dumps({'source': 'fixture', 'venue': 'MM', 'year': 2026,
'track': 'unclassified', 'list_url': URL, 'status': 'partial', 'count': 1, 'file_sha256': hashlib.sha256(raw).hexdigest()}))
with self.assertRaisesRegex(ValueError, 'source snapshot'):
ingest_file(self.target.get('db'), path, **{k: v for k, v in self.target.items() if k != 'db'})
def test_unregistered_member_and_changed_program_metadata_are_rejected(self):
with closing(self.store().conn) as c:
apply_plan(c, self.plan(c)); c.commit()
sid = c.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0]
cache = {}; unknown = copy.deepcopy(self.inventory['rows'][0]); unknown['external_id'] = '10.1234/unlisted'
with self.assertRaisesRegex(ValueError, 'absent from current'):
require_row(c, sid, unknown, {'list_url': URL}, cache)
bad = copy.deepcopy(self.inventory['rows'][1]); bad['metadata']['program'] = None
with self.assertRaisesRegex(ValueError, 'Stale source'):
require_row(c, sid, bad, {'list_url': URL}, cache)
good = copy.deepcopy(self.inventory['rows'][1]); good['metadata']['verified_work_anchor'] = {'reason': 'extra evidence'}
require_row(c, sid, good, {'list_url': URL}, cache)
with self.assertRaisesRegex(ValueError, 'coverage manifest'):
require_manifest(c, sid, {'venue': 'MM', 'year': 2026, 'track': 'main', 'list_url': URL, 'count': 1, 'status': 'partial'}, cache)
require_manifest(c, sid, {'venue': 'MM', 'year': 2026, 'track': 'main', 'list_url': URL, 'count': 2, 'status': 'partial'}, cache)
def test_stale_plan_and_target_identifier_collision_prevent_writes(self):
with closing(self.store().conn) as c:
plan = self.plan(c)
c.execute("UPDATE decisions SET raw_value='Additional evidence' WHERE status='rejected'")
with self.assertRaisesRegex(ValueError, 'Stale'):
apply_plan(c, plan)
c.rollback()
move = plan['changes'][0]
target_sub = c.execute('SELECT id FROM submissions WHERE edition_id=? LIMIT 1', (move['target_edition_id'],)).fetchone()[0]
c.execute('INSERT INTO submission_identifiers VALUES(?,?,?,?)', (move['target_edition_id'], 'publication_doi', move['external_id'], target_sub))
with self.assertRaisesRegex(ValueError, 'collision'):
self.plan(c)
c.rollback()
self.assertFalse(available(c))
def test_failed_postcheck_rolls_back_rows_extension_and_coverage(self):
with closing(self.store().conn) as c:
plan = self.plan(c)
with patch('paper_atlas.ingestion.source_revisions.verify', side_effect=ValueError('Injected verification failure')):
with self.assertRaisesRegex(ValueError, 'Injected'):
apply_plan(c, plan)
c.rollback()
self.assertFalse(available(c))
self.assertEqual(self.plan(c), plan)
def test_cli_binds_database_and_file_hash_without_writing_during_preflight(self):
inventory = self.root/'inventory.json'; inventory.write_text(json.dumps(self.inventory))
path = self.root/'plan.json'
args = {k: v for k, v in self.target.items() if k != 'db'}
run(self.target.get('db'), path, inventory=inventory, **args)
with closing(self.store().conn) as c:
self.assertFalse(available(c))
with self.assertRaisesRegex(ValueError, 'hash'):
run(self.target.get('db'), self.root/'bad.json', plan=path, plan_sha256='0'*64, **args)
report = run(self.target.get('db'), self.root/'applied.json', plan=path, plan_sha256=hashlib.sha256(path.read_bytes()).hexdigest(), **args)
self.assertEqual(report['events_moved'], 1)
def test_membership_identity_changes_and_incremental_shrink_remain_forbidden(self):
for mutate in [lambda x: x['rows'].pop(), lambda x: x['rows'][0].update(title='Different work'),
lambda x: x['rows'][0].update(status='rejected')]:
bad = copy.deepcopy(self.inventory); mutate(bad)
with self.assertRaises(ValueError):
validate(bad)
with self.assertRaisesRegex(ValueError, 'incremental'):
coverage_options({'coverage_mode': 'incremental', 'venue': 'MM', 'year': 2026, 'track': 'unclassified',
'status': 'partial', 'count': 1, 'coverage_previous_count': 2,
'coverage_previous_expected_count': 3, 'coverage_expected_count': 2})
class SQLiteSourceRevisionTests(RevisionCases, unittest.TestCase):
def setUp(self):
temp = tempfile.TemporaryDirectory(); self.addCleanup(temp.cleanup); self.root = Path(temp.name)
db = self.root/'source.sqlite'; self.inventory = seed(db); self.target = {'db': db}
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires approved remote PostgreSQL service')
def test_optional_revision_table_migrates_with_full_history(self):
with closing(self.store().conn) as c:
plan = self.plan(c); apply_plan(c, plan); c.commit()
service = os.environ['PAPER_ATLAS_TEST_PG_SERVICE']; schema = 'pa_rev_copy_'+uuid.uuid4().hex[:10]
self.addCleanup(drop_schema, service, schema)
with redirect_stdout(io.StringIO()):
migrate(self.target['db'], service, self.root/'copied.json', schema=schema, min_free_gib=0)
with closing(open_store(pg_service=service, pg_schema=schema).conn) as c:
self.assertEqual(json.loads(c.execute('SELECT plan_json FROM source_scope_revisions').fetchone()[0]), plan)
self.assertEqual(c.execute('SELECT count(*) FROM source_scope_revisions').fetchone()[0], 1)
def drop_schema(service, schema):
with connect(service) as c:
for name, in c.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)', (schema,)):
c.execute('DROP SCHEMA '+quote(name)+' CASCADE')
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires approved remote PostgreSQL service')
class PostgreSQLSourceRevisionTests(RevisionCases, unittest.TestCase):
def setUp(self):
temp = tempfile.TemporaryDirectory(); self.addCleanup(temp.cleanup); self.root = Path(temp.name)
db = self.root/'source.sqlite'; self.inventory = seed(db)
service = os.environ['PAPER_ATLAS_TEST_PG_SERVICE']; schema = 'pa_rev_'+uuid.uuid4().hex[:12]
self.addCleanup(drop_schema, service, schema)
with redirect_stdout(io.StringIO()):
migrate(db, service, self.root/'migration.json', schema=schema, min_free_gib=0)
self.target = {'pg_service': service, 'pg_schema': schema}