Compare commits
59 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| bc01f90e70 | |||
| 48f8304517 | |||
| f2b9fe0149 | |||
| 32d99858c1 | |||
| 060d67aba5 | |||
| d0a466977c | |||
| 6d35a2e567 | |||
| 52f9740d99 | |||
| 4e071ca0f1 | |||
| f97852b630 | |||
| bcb8b2ad7e | |||
| 95ff77d77f | |||
| 9da6a45ec0 | |||
| 7407f97ef2 | |||
| 31ff7d1626 | |||
| 8a7896d352 | |||
| 35ff09a463 | |||
| 28805462ee | |||
| c469785a64 | |||
| 2f445eddc8 | |||
| 2907c3774f | |||
| a980b1a1df | |||
| 3754545003 | |||
| e38b2f1ca8 | |||
| 8e548287e9 | |||
| e85902bf18 | |||
| e82abec4c7 | |||
| 7d9c693761 | |||
| 98ef415912 | |||
| 90fd789a17 | |||
| 2000d340fa | |||
| 12f95cff91 | |||
| 9864632774 | |||
| f982851560 | |||
| 88aeb4d2eb | |||
| 83f3ebd26b | |||
| 13536d6268 | |||
| 7e63035570 | |||
| 44f4146f3e | |||
| b0bd9b69dc | |||
| 09a2d4ed20 | |||
| c8db6118d4 | |||
| 1c64971a52 | |||
| ac14df0625 | |||
| afb3b939ca | |||
| 8c147f5e6e | |||
| 6961710e96 | |||
| f4be9de547 | |||
| 33bb378369 | |||
| 5bdf9f0fe5 | |||
| 7ed5966d3a | |||
| 871b2aa81e | |||
| b14193063a | |||
| 564821d883 | |||
| 2f1b5c959a | |||
| f139c7e49d | |||
| 6a89f5f6c2 | |||
| cb3d97ed92 | |||
| 06467835c7 |
+154
-4
@@ -12,15 +12,16 @@ jobs:
|
|||||||
- name: Verify metadata storage and query commands
|
- name: Verify metadata storage and query commands
|
||||||
run: |
|
run: |
|
||||||
python3 -B - <<'PY'
|
python3 -B - <<'PY'
|
||||||
import gzip, io, json, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request
|
import base64, gzip, io, json, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request
|
||||||
sha = os.environ['ARCHIVE_SHA']
|
sha = os.environ['ARCHIVE_SHA']
|
||||||
if not re.fullmatch(r'[0-9a-fA-F]{40,64}', sha):
|
if not re.fullmatch(r'[0-9a-fA-F]{40,64}', sha):
|
||||||
raise SystemExit('Invalid source commit')
|
raise SystemExit('Invalid source commit')
|
||||||
# Public repository archive: no owner credentials or runner SSH keys.
|
# Public repository archive: no owner credentials or runner SSH keys.
|
||||||
url = 'https://gitea.entroia.com/Zayden-Jung/paper-atlas/archive/' + sha + '.tar.gz'
|
url = 'https://gitea.entroia.com/Zayden-Jung/paper-atlas/archive/' + sha + '.tar.gz'
|
||||||
with urllib.request.urlopen(url, timeout=60) as response:
|
with urllib.request.urlopen(url, timeout=60) as response:
|
||||||
archive = response.read(16*1024*1024+1)
|
# Full publication-source fixtures retain original bibliographic evidence.
|
||||||
if len(archive)>16*1024*1024:
|
archive = response.read(24*1024*1024+1)
|
||||||
|
if len(archive)>24*1024*1024:
|
||||||
raise SystemExit('Unexpectedly large source archive')
|
raise SystemExit('Unexpectedly large source archive')
|
||||||
with tempfile.TemporaryDirectory(prefix='paper-atlas-ci-') as tmp:
|
with tempfile.TemporaryDirectory(prefix='paper-atlas-ci-') as tmp:
|
||||||
root = pathlib.Path(tmp)
|
root = pathlib.Path(tmp)
|
||||||
@@ -34,7 +35,7 @@ jobs:
|
|||||||
venv = root / 'venv'
|
venv = root / 'venv'
|
||||||
subprocess.run(['python3','-B','-m','venv','--system-site-packages',str(venv)],check=True)
|
subprocess.run(['python3','-B','-m','venv','--system-site-packages',str(venv)],check=True)
|
||||||
python = str(venv / 'bin/python')
|
python = str(venv / 'bin/python')
|
||||||
subprocess.run([python,'-B','-m','pip','install','--only-binary=:all:','--no-compile','psycopg[binary]==3.3.8'],check=True)
|
subprocess.run([python,'-B','-m','pip','install','--only-binary=:all:','--no-compile','psycopg[binary]==3.3.8','pyarrow==26.0.1'],check=True)
|
||||||
testenv = os.environ.copy()
|
testenv = os.environ.copy()
|
||||||
service_file = pathlib.Path('/etc/paper-atlas/pg_service.conf')
|
service_file = pathlib.Path('/etc/paper-atlas/pg_service.conf')
|
||||||
if service_file.is_file():
|
if service_file.is_file():
|
||||||
@@ -46,6 +47,77 @@ jobs:
|
|||||||
subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True)
|
subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True)
|
||||||
outside = root / 'outside-checkout'
|
outside = root / 'outside-checkout'
|
||||||
outside.mkdir()
|
outside.mkdir()
|
||||||
|
cikm_snapshots = pathlib.Path('/data/paper-index/validation/cikm2026-source-snapshots-v1')
|
||||||
|
if cikm_snapshots.is_dir():
|
||||||
|
cikm_output = root / 'installed-cikm2026'
|
||||||
|
command = [python, '-I', '-B', '-m', 'paper_atlas', 'collect', 'cikm2026-sources']
|
||||||
|
subprocess.run([*command, '--snapshots', str(cikm_snapshots), '--out', str(cikm_output)], cwd=outside, check=True)
|
||||||
|
report = json.loads((cikm_output/'collection-report.json').read_text())
|
||||||
|
assert {name: entry['records'] for name, entry in report['files'].items()} == {
|
||||||
|
'programme.jsonl': 1008, 'alternate-programme.jsonl': 1005,
|
||||||
|
'supplemental-programme.jsonl': 73, 'crossref.jsonl': 1081}
|
||||||
|
assert len(report['registry_outside_doi_programme']) == 73 and not report['database_written']
|
||||||
|
programme = [json.loads(line) for line in (cikm_output/'programme.jsonl').read_text().splitlines()]
|
||||||
|
assert all(row['abstract'] and row['author_display_names'] for row in programme)
|
||||||
|
assert len({row['doi'] for row in programme}) == 1008
|
||||||
|
broken = root/'tampered-cikm2026'
|
||||||
|
broken.mkdir()
|
||||||
|
(broken/'program.html').write_text('tampered')
|
||||||
|
rejected = subprocess.run([*command, '--snapshots', str(broken), '--out', str(root/'rejected-cikm2026')], cwd=outside, capture_output=True, text=True)
|
||||||
|
assert rejected.returncode != 0 and 'Snapshot hash mismatch' in rejected.stderr
|
||||||
|
assert not (root/'rejected-cikm2026').exists()
|
||||||
|
print('CIKM 2026 full frozen-source CLI check passed', flush=True)
|
||||||
|
mm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/mm2026-source.json.gz').read_bytes()))
|
||||||
|
mm_snapshots = root/'mm2026-snapshots'
|
||||||
|
for name, body in mm_fixture['files'].items():
|
||||||
|
path = mm_snapshots/name
|
||||||
|
path.parent.mkdir(parents=True,exist_ok=True)
|
||||||
|
path.write_bytes(body.encode('utf-8'))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','mm2026-program','--snapshots',str(mm_snapshots),'--out',str(root/'installed-mm2026')],cwd=outside,check=True)
|
||||||
|
mm_report = json.loads((root/'installed-mm2026/collection-report.json').read_text())
|
||||||
|
assert (mm_report['records'],mm_report['program_observations'],mm_report['extra_presentation_observations']) == (2143,4326,2183)
|
||||||
|
assert len(mm_report['files']) == 28 and mm_report['missing_abstract_count'] == 216
|
||||||
|
assert mm_report['session_count'] == 157 and not mm_report['database_written'] and mm_report['network_requests'] == 0 and mm_report['approved_for_import'] == 0
|
||||||
|
mm_rows = {r['paper_id']:r for f in mm_report['files'] for r in map(json.loads,pathlib.Path(f['file']).read_text().splitlines())}
|
||||||
|
import hashlib
|
||||||
|
assert set(mm_rows) == {g['paper_id'] for g in mm_fixture['independent_inventory']['inventory']}
|
||||||
|
for gold in mm_fixture['independent_inventory']['inventory']:
|
||||||
|
row = mm_rows[gold['paper_id']]
|
||||||
|
assert (row['title'],row['authors'],row['affiliations'],row['track']) == (gold['title'],gold['authors'],gold['affiliations'],gold['track_literal'])
|
||||||
|
assert hashlib.sha256(row['abstract'].encode()).hexdigest() == gold['abstract_sha256']
|
||||||
|
assert [c['source_pointer'] for c in row['contexts']] == gold['locations']
|
||||||
|
assert not row['approved_for_import'] and not row['final_decision_verified'] and 'emails' not in row
|
||||||
|
mm_pub_fixture = json.loads(gzip.decompress((src/'tests/fixtures/mm2026-publications.json.gz').read_bytes()))
|
||||||
|
mm_pub_registry = root/'mm2026-publication-registry'
|
||||||
|
for name,body in mm_pub_fixture['files'].items():
|
||||||
|
path=mm_pub_registry/name
|
||||||
|
path.parent.mkdir(parents=True,exist_ok=True)
|
||||||
|
path.write_bytes(body.encode('utf-8'))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','mm2026-publications','--registry-snapshots',str(mm_pub_registry),'--program-snapshots',str(mm_snapshots),'--parent-snapshot',str(mm_pub_registry/'parent.json'),'--out',str(root/'installed-mm2026-publications')],cwd=outside,check=True)
|
||||||
|
mm_pub_report=json.loads((root/'installed-mm2026-publications/collection-report.json').read_text())
|
||||||
|
assert (mm_pub_report['records'],mm_pub_report['unique_registered_dois'],mm_pub_report['abstracts_from_exact_program_correspondence'],mm_pub_report['output_missing_abstract_count'])==(1918,1918,1743,175)
|
||||||
|
assert mm_pub_report['unmatched_program_count']==391 and not mm_pub_report['database_written'] and mm_pub_report['approved_for_import']==0
|
||||||
|
mm_pub_rows={r['external_id']:r for f in mm_pub_report['files'] for r in map(json.loads,pathlib.Path(f['file']).read_text().splitlines())}
|
||||||
|
assert len(mm_pub_rows)==1918
|
||||||
|
mm_format_gold=json.loads((src/'tests/fixtures/mm2026-title-formatting-review.json').read_text())
|
||||||
|
mm_format_overrides={g['doi']:g for g in [*mm_format_gold['gold'],mm_format_gold['held_gold']]}
|
||||||
|
assert mm_pub_report['abstracts_from_reviewed_title_formatting']==26 and mm_pub_report['abstracts_from_literal_program_correspondence']==1717
|
||||||
|
for original_gold in mm_pub_fixture['independent_gold']:
|
||||||
|
gold=mm_format_overrides.get(original_gold['doi'],original_gold)
|
||||||
|
row=mm_pub_rows[gold['doi']]
|
||||||
|
assert (row['title'],row['authors'],row['track'])==(gold['title'],gold['authors'],gold['track'])
|
||||||
|
assert (hashlib.sha256(row['abstract'].encode()).hexdigest() if row['abstract'] is not None else None)==gold['abstract_sha256']
|
||||||
|
assert row['metadata']['program_match_kind']==gold['kind'] and not row['metadata']['scientific_work_identity_approved'] and not row['metadata']['research_type_classified']
|
||||||
|
if service_file.is_file():
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','source-identities','--files',*[f['file'] for f in mm_pub_report['files']],'--pg-service','foundry-research','--pg-schema','research','--out',str(root/'installed-mm2026-publication-preflight')],cwd=outside,env=testenv,check=True)
|
||||||
|
mm_pub_preflight=json.loads((root/'installed-mm2026-publication-preflight/postgresql-preflight.json').read_text())
|
||||||
|
assert mm_pub_preflight['records']==1918 and mm_pub_preflight['transaction_read_only'] and not mm_pub_preflight['database_written'] and mm_pub_preflight['approved_for_import']==0
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','source-identities','--mm-program-report',str(root/'installed-mm2026/collection-report.json'),'--pg-service','foundry-research','--pg-schema','research','--out',str(root/'installed-mm2026-preflight')],cwd=outside,env=testenv,check=True)
|
||||||
|
mm_preflight = json.loads((root/'installed-mm2026-preflight/postgresql-preflight.json').read_text())
|
||||||
|
assert mm_preflight['records'] == 2143 and len(mm_preflight['inputs']) == 28
|
||||||
|
assert mm_preflight['transaction_read_only'] and not mm_preflight['database_written'] and mm_preflight['approved_for_import'] == 0
|
||||||
|
assert sum(len(r['program_contexts']) for r in mm_preflight['entries']) == 4326
|
||||||
|
assert all(r['status'] == 'unknown' and not r['identifiers'] and not r['contribution_type_verified'] and not r['publication_identity_verified'] and not r['final_decision_verified'] for r in mm_preflight['entries'])
|
||||||
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
|
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--help'],cwd=outside,check=True)
|
||||||
@@ -77,6 +149,22 @@ jobs:
|
|||||||
eccv_literal_row = next(row for row in eccv_rows if row['external_id'] == '2026/3844')
|
eccv_literal_row = next(row for row in eccv_rows if row['external_id'] == '2026/3844')
|
||||||
assert eccv_literal_row['title'] == eccv_literal['gold']['visible_title']
|
assert eccv_literal_row['title'] == eccv_literal['gold']['visible_title']
|
||||||
assert '&' in eccv_literal_row['title']
|
assert '&' in eccv_literal_row['title']
|
||||||
|
icassp_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icassp2026-source.json.gz').read_bytes()))
|
||||||
|
icassp_snapshots = root/'icassp2026-snapshots'
|
||||||
|
icassp_snapshots.mkdir()
|
||||||
|
for name, encoded in icassp_fixture['files'].items():
|
||||||
|
target = icassp_snapshots/name
|
||||||
|
target.parent.mkdir(parents=True,exist_ok=True)
|
||||||
|
target.write_bytes(base64.b64decode(encoded))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','icassp2026-registry','--snapshots',str(icassp_snapshots),'--out',str(root/'installed-icassp2026')],cwd=outside,check=True)
|
||||||
|
icassp_report = json.loads((root/'installed-icassp2026/collection-report.json').read_text())
|
||||||
|
assert icassp_report['records'] == 4582 and icassp_report['missing_abstract_count'] == 4582
|
||||||
|
assert icassp_report['single_name_author_fields'] == 23
|
||||||
|
assert len(icassp_report['excluded_nonpaper_items']) == 7 and len(icassp_report['excluded_other_conference_dois']) == 56
|
||||||
|
assert not icassp_report['database_written'] and icassp_report['network_requests'] == 0 and icassp_report['approved_for_import'] == 0
|
||||||
|
icassp_rows = [json.loads(line) for line in pathlib.Path(icassp_report['files'][0]['file']).read_text().splitlines()]
|
||||||
|
assert [{'doi':r['external_id'],'title':r['title'],'authors':r['authors']} for r in icassp_rows] == sorted(icassp_fixture['independent_gold'],key=lambda r:r['doi'])
|
||||||
|
assert all(r['abstract'] is None and r['track'] == 'proceedings_unclassified' for r in icassp_rows)
|
||||||
sdm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/sdm2026-source.json.gz').read_bytes()))
|
sdm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/sdm2026-source.json.gz').read_bytes()))
|
||||||
sdm_snapshots = root/'sdm2026-snapshots'
|
sdm_snapshots = root/'sdm2026-snapshots'
|
||||||
for name, text in sdm_fixture['files'].items():
|
for name, text in sdm_fixture['files'].items():
|
||||||
@@ -94,6 +182,21 @@ jobs:
|
|||||||
sdm_observations = [json.loads(line) for line in pathlib.Path(sdm_report['program_observations']['file']).read_text().splitlines()]
|
sdm_observations = [json.loads(line) for line in pathlib.Path(sdm_report['program_observations']['file']).read_text().splitlines()]
|
||||||
assert sum(row['cancelled_presentation'] for row in sdm_observations) == 1
|
assert sum(row['cancelled_presentation'] for row in sdm_observations) == 1
|
||||||
assert all('status' not in row and 'identifiers' not in row for row in sdm_observations)
|
assert all('status' not in row and 'identifiers' not in row for row in sdm_observations)
|
||||||
|
icdm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icdm2026-source.json.gz').read_bytes()))
|
||||||
|
icdm_snapshots = root/'icdm2026-snapshots'
|
||||||
|
for name, text in icdm_fixture['files'].items():
|
||||||
|
target = icdm_snapshots/name
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
target.write_bytes(text.encode('utf-8'))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','icdm2026-program','--snapshots',str(icdm_snapshots),'--out',str(root/'installed-icdm2026')],cwd=outside,check=True)
|
||||||
|
icdm_report = json.loads((root/'installed-icdm2026/collection-report.json').read_text())
|
||||||
|
assert icdm_report['records'] == 324 and icdm_report['missing_abstract_count'] == 324
|
||||||
|
assert icdm_report['track_counts'] == {'research':241,'applied':83}
|
||||||
|
assert icdm_report['presentation_counts'] == {'long':168,'short':156}
|
||||||
|
assert not icdm_report['database_written'] and icdm_report['network_requests'] == 0
|
||||||
|
icdm_rows = [json.loads(line) for f in icdm_report['files'] for line in pathlib.Path(f['file']).read_text().splitlines()]
|
||||||
|
assert len(icdm_rows) == 324 and all(row['abstract'] is None and not row['identifiers'] and not row['submission_identifiers'] for row in icdm_rows)
|
||||||
|
assert all(row['metadata']['paper_type'] is None and not row['metadata']['original_decision_signature_verified'] for row in icdm_rows)
|
||||||
icaps_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icaps2026-source.json.gz').read_bytes()))
|
icaps_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icaps2026-source.json.gz').read_bytes()))
|
||||||
icaps_snapshots = root/'icaps2026-snapshots'
|
icaps_snapshots = root/'icaps2026-snapshots'
|
||||||
icaps_snapshots.mkdir()
|
icaps_snapshots.mkdir()
|
||||||
@@ -120,6 +223,36 @@ jobs:
|
|||||||
(rss_snapshots/'receipts.json').write_text(json.dumps(rss_fixture['receipts']))
|
(rss_snapshots/'receipts.json').write_text(json.dumps(rss_fixture['receipts']))
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--snapshots',str(rss_snapshots),'--out',str(root/'installed-rss2026')],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--snapshots',str(rss_snapshots),'--out',str(root/'installed-rss2026')],cwd=outside,check=True)
|
||||||
assert json.loads((root/'installed-rss2026/collection-report.json').read_text())['records'] == 210
|
assert json.loads((root/'installed-rss2026/collection-report.json').read_text())['records'] == 210
|
||||||
|
recsys_fixture = json.loads(gzip.decompress((src/'tests/fixtures/recsys2026-source.json.gz').read_bytes()))
|
||||||
|
recsys_snapshots = root/'recsys2026-snapshots'
|
||||||
|
recsys_snapshots.mkdir()
|
||||||
|
for name,encoded in recsys_fixture['files'].items():
|
||||||
|
path = recsys_snapshots/name
|
||||||
|
path.parent.mkdir(parents=True,exist_ok=True)
|
||||||
|
path.write_bytes(base64.b64decode(encoded))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','recsys2026-program','--snapshots',str(recsys_snapshots),'--out',str(root/'installed-recsys2026')],cwd=outside,check=True)
|
||||||
|
recsys_report = json.loads((root/'installed-recsys2026/collection-report.json').read_text())
|
||||||
|
assert (recsys_report['records'],recsys_report['unique_dois'],recsys_report['missing_abstract_count']) == (263,260,0)
|
||||||
|
assert recsys_report['duplicate_doi_observations'] == 6 and len(recsys_report['repeated_dois']) == 3
|
||||||
|
assert not recsys_report['database_written'] and recsys_report['network_requests'] == 0 and recsys_report['approved_for_import'] == 0
|
||||||
|
recsys_rows = [json.loads(line) for manifest in recsys_report['files'] for line in pathlib.Path(manifest['file']).read_text().splitlines()]
|
||||||
|
gold_keys = recsys_fixture['independent_gold'][0].keys()
|
||||||
|
assert [{k:r['metadata']['program_fields'][k] for k in gold_keys} for r in recsys_rows] == recsys_fixture['independent_gold']
|
||||||
|
ismir_fixture = json.loads(gzip.decompress((src/'tests/fixtures/ismir2026-source.json.gz').read_bytes()))
|
||||||
|
ismir_snapshots = root/'ismir2026-snapshots'
|
||||||
|
ismir_snapshots.mkdir()
|
||||||
|
for name,encoded in ismir_fixture['files'].items():
|
||||||
|
target = ismir_snapshots/name
|
||||||
|
target.parent.mkdir(parents=True,exist_ok=True)
|
||||||
|
target.write_bytes(base64.b64decode(encoded))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','ismir2026-program','--snapshots',str(ismir_snapshots),'--out',str(root/'installed-ismir2026')],cwd=outside,check=True)
|
||||||
|
ismir_report = json.loads((root/'installed-ismir2026/collection-report.json').read_text())
|
||||||
|
assert (ismir_report['records'],ismir_report['missing_abstract_count'],ismir_report['review_optin_yes'],ismir_report['review_optin_no']) == (140,0,101,39)
|
||||||
|
assert ismir_report['literal_title_version_differences'] == 2 and ismir_report['author_sequence_version_differences'] == 3
|
||||||
|
assert not ismir_report['database_written'] and ismir_report['network_requests'] == 0 and ismir_report['approved_for_import'] == 0
|
||||||
|
ismir_rows = [json.loads(line) for line in pathlib.Path(ismir_report['files'][0]['file']).read_text().splitlines()]
|
||||||
|
assert [(r['external_id'],r['title'],r['authors'],r['abstract']) for r in ismir_rows] == [('2026/'+g['program_id'],g['title'],g['authors'],g['abstract']) for g in ismir_fixture['independent_gold']]
|
||||||
|
assert all(not r['identifiers'] and not r['submission_identifiers'] and r['metadata']['paper_type'] is None for r in ismir_rows)
|
||||||
corl_fixture = json.loads(gzip.decompress((src/'tests/fixtures/corl2026-source.json.gz').read_bytes()))
|
corl_fixture = json.loads(gzip.decompress((src/'tests/fixtures/corl2026-source.json.gz').read_bytes()))
|
||||||
corl_official = root/'corl2026-official'
|
corl_official = root/'corl2026-official'
|
||||||
corl_archive = root/'corl2026-archive'
|
corl_archive = root/'corl2026-archive'
|
||||||
@@ -160,6 +293,23 @@ jobs:
|
|||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','iclr-archive','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','iclr-archive','--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','acl','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','acl','--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','springer-selected','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','springer-selected','--help'],cwd=outside,check=True)
|
||||||
|
publisher_fixture = "\n".join((
|
||||||
|
"import pathlib, runpy, sys",
|
||||||
|
"fixture = runpy.run_path(sys.argv[1])",
|
||||||
|
"root = pathlib.Path(sys.argv[2])",
|
||||||
|
"root.mkdir()",
|
||||||
|
"fixture['snapshots'](root)",
|
||||||
|
))
|
||||||
|
publisher_input = root/'eccv-publisher-snapshots'
|
||||||
|
subprocess.run([python,'-I','-B','-c',publisher_fixture,
|
||||||
|
str(src/'tests/test_eccv2026_publications.py'),str(publisher_input)],cwd=outside,check=True)
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','eccv2026-publications',
|
||||||
|
'--toc-snapshots',str(publisher_input/'toc'),
|
||||||
|
'--chapter-snapshots',str(publisher_input/'chapters'),
|
||||||
|
'--out',str(root/'installed-eccv-publisher')],cwd=outside,check=True)
|
||||||
|
publisher_report = json.loads((root/'installed-eccv-publisher/collection-report.json').read_bytes())
|
||||||
|
assert publisher_report['records'] == 1 and publisher_report['toc_chapters'] == 2835
|
||||||
|
assert not publisher_report['database_written']
|
||||||
springer = json.loads(gzip.decompress((src/'tests/fixtures/springer-aistats1993-fixture.json.gz').read_bytes()))
|
springer = json.loads(gzip.decompress((src/'tests/fixtures/springer-aistats1993-fixture.json.gz').read_bytes()))
|
||||||
for group in ('toc','chapters'):
|
for group in ('toc','chapters'):
|
||||||
folder = root/'springer-snapshots'/group
|
folder = root/'springer-snapshots'/group
|
||||||
|
|||||||
@@ -0,0 +1,88 @@
|
|||||||
|
name: ReviewArena metadata projection
|
||||||
|
on:
|
||||||
|
workflow_dispatch:
|
||||||
|
inputs:
|
||||||
|
split:
|
||||||
|
description: 'Pinned public mirror family; metadata columns only'
|
||||||
|
required: true
|
||||||
|
default: emnlp
|
||||||
|
projection_name:
|
||||||
|
description: 'Optional existing metadata output directory name for offline replay'
|
||||||
|
required: false
|
||||||
|
default: ''
|
||||||
|
projection_sha256:
|
||||||
|
description: 'Exact cached collection-report.json SHA256, required for offline replay'
|
||||||
|
required: false
|
||||||
|
default: ''
|
||||||
|
jobs:
|
||||||
|
project:
|
||||||
|
runs-on: ces-control
|
||||||
|
env:
|
||||||
|
ARCHIVE_SHA: ${{ github.sha }}
|
||||||
|
MIRROR_SPLIT: ${{ inputs.split }}
|
||||||
|
MIRROR_PROJECTION: ${{ inputs.projection_name }}
|
||||||
|
MIRROR_PROJECTION_SHA: ${{ inputs.projection_sha256 }}
|
||||||
|
PYTHONDONTWRITEBYTECODE: '1'
|
||||||
|
steps:
|
||||||
|
- name: Project permitted metadata on the approved remote runner
|
||||||
|
run: |
|
||||||
|
python3 -B - <<'PY'
|
||||||
|
import hashlib, io, json, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request
|
||||||
|
sha = os.environ['ARCHIVE_SHA']
|
||||||
|
split = os.environ['MIRROR_SPLIT']
|
||||||
|
cached = os.environ.get('MIRROR_PROJECTION','')
|
||||||
|
cached_sha = os.environ.get('MIRROR_PROJECTION_SHA','')
|
||||||
|
if not re.fullmatch(r'[0-9a-f]{40}', sha) or split not in ('colm','corl','emnlp','iclr','icml','neurips','tmlr'):
|
||||||
|
raise SystemExit('Explicit approved source commit/split required')
|
||||||
|
base = pathlib.Path('/data/paper-index')
|
||||||
|
if not (base/'collected').is_dir() or not (base/'validation').is_dir():
|
||||||
|
raise SystemExit('Confirmed ces-control host dataset paths unavailable; do not use another host')
|
||||||
|
out = base/'collected'/('reviewarena-c2978add-'+split+'-'+sha[:8])
|
||||||
|
if out.exists():
|
||||||
|
raise SystemExit('Inspect existing output; no overwrite or automatic retry')
|
||||||
|
extra=[]
|
||||||
|
if cached:
|
||||||
|
if not re.fullmatch('reviewarena-c2978add-'+split+r'-[0-9a-f]{8}',cached) or not re.fullmatch('[0-9a-f]{64}',cached_sha):
|
||||||
|
raise SystemExit('Explicit permitted cached projection name and report hash required')
|
||||||
|
extra=['--from-projection',str(base/'collected'/cached/'collection-report.json'),'--projection-sha256',cached_sha]
|
||||||
|
elif cached_sha:
|
||||||
|
raise SystemExit('Cached projection name required')
|
||||||
|
url = 'https://gitea.entroia.com/Zayden-Jung/paper-atlas/archive/'+sha+'.tar.gz'
|
||||||
|
with urllib.request.urlopen(url,timeout=60) as response:
|
||||||
|
archive = response.read(24*1024*1024+1)
|
||||||
|
if len(archive)>24*1024*1024:
|
||||||
|
raise SystemExit('Oversized source archive')
|
||||||
|
with tempfile.TemporaryDirectory(prefix='paper-atlas-mirror-') as tmp:
|
||||||
|
root = pathlib.Path(tmp)
|
||||||
|
with tarfile.open(fileobj=io.BytesIO(archive),mode='r:gz') as tar:
|
||||||
|
tar.extractall(root,filter='data')
|
||||||
|
candidates=list(root.glob('*/pyproject.toml'))
|
||||||
|
if len(candidates)!=1:
|
||||||
|
raise SystemExit('Unexpected repository archive')
|
||||||
|
src=candidates[0].parent
|
||||||
|
venv=root/'venv'
|
||||||
|
subprocess.run(['python3','-B','-m','venv','--system-site-packages',str(venv)],check=True)
|
||||||
|
python=str(venv/'bin/python')
|
||||||
|
subprocess.run([python,'-B','-m','pip','install','--only-binary=:all:','--no-compile','pyarrow==26.0.1'],check=True)
|
||||||
|
wheels=root/'wheels'
|
||||||
|
subprocess.run([python,'-B','-m','pip','wheel','--no-deps','--wheel-dir',str(wheels),str(src)],check=True)
|
||||||
|
wheel,=wheels.glob('paper_atlas-*.whl')
|
||||||
|
subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True)
|
||||||
|
outside=root/'outside';outside.mkdir()
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','reviewarena-metadata','--split',split,'--out',str(out),*extra],cwd=outside,check=True)
|
||||||
|
report=json.loads((out/'collection-report.json').read_bytes())
|
||||||
|
assert report['database_written'] is False and report['approved_for_import']==0
|
||||||
|
receipt={'source_commit':sha,'source_archive_sha256':hashlib.sha256(archive).hexdigest(),'wheel_sha256':hashlib.sha256(wheel.read_bytes()).hexdigest(),'dataset_commit':report['commit'],'split':split,'records':report['records'],'metadata_bytes_read':report['metadata_bytes_read'],'cached_projection_report_sha256':report.get('cached_projection_report_sha256'),'raw_parquet_saved':False,'database_written':False}
|
||||||
|
(out/'runner-receipt.json').write_text(json.dumps(receipt,indent=2)+'\n')
|
||||||
|
backup=base/'validation'/(out.name+'-recovery.tar.gz')
|
||||||
|
if backup.exists():
|
||||||
|
raise SystemExit('Recovery archive already exists; inspect')
|
||||||
|
with tarfile.open(backup,'w:gz') as tar:
|
||||||
|
tar.add(out,arcname=str(out.relative_to(base)))
|
||||||
|
backup.chmod(0o444)
|
||||||
|
for path in out.iterdir():
|
||||||
|
if path.is_file():path.chmod(0o444)
|
||||||
|
result={'status':'metadata_projection_backed_up_not_imported','archive':str(backup),'bytes':backup.stat().st_size,'sha256':hashlib.sha256(backup.read_bytes()).hexdigest(),**receipt}
|
||||||
|
(base/'validation'/(out.name+'-recovery.json')).write_text(json.dumps(result,indent=2)+'\n')
|
||||||
|
print(json.dumps(result),flush=True)
|
||||||
|
PY
|
||||||
@@ -2,6 +2,8 @@
|
|||||||
|
|
||||||
**长期维护、按需运行的论文元数据与公开投稿历史工具。**
|
**长期维护、按需运行的论文元数据与公开投稿历史工具。**
|
||||||
|
|
||||||
|
其他会议的公开评审镜像可按 [ReviewArena 元数据投影流程](docs/reviewarena-metadata.md) 核查:仅取固定版本的标题、摘要、作者及决定等索引字段,保留镜像声明和稳定行定位,不下载正文/评审数据页,也不将镜像 ID 自动提升为官方论坛身份。
|
||||||
|
|
||||||
Paper Atlas 将 arXiv、AI 会议论文集及公开投稿决定整理到可追溯的数据库中,用于查询和后续研究分析。生产数据使用 PostgreSQL,支持从既有 SQLite 索引完整迁移;SQLite 保留离线核验和小型样本用途。它按批次运行,不需要常驻 Web 服务;采集标题、摘要、标识、标签及决定证据,不下载论文全文。
|
Paper Atlas 将 arXiv、AI 会议论文集及公开投稿决定整理到可追溯的数据库中,用于查询和后续研究分析。生产数据使用 PostgreSQL,支持从既有 SQLite 索引完整迁移;SQLite 保留离线核验和小型样本用途。它按批次运行,不需要常驻 Web 服务;采集标题、摘要、标识、标签及决定证据,不下载论文全文。
|
||||||
|
|
||||||
项目仍在建设中。候选范围包含 43 个会议(含 ACM MM、MLSys),尚未完成全部历史数据和最终去重复核。实际进度以数据库、来源清单和覆盖报告为准,而不是仓库文件数量。
|
项目仍在建设中。候选范围包含 43 个会议(含 ACM MM、MLSys),尚未完成全部历史数据和最终去重复核。实际进度以数据库、来源清单和覆盖报告为准,而不是仓库文件数量。
|
||||||
@@ -117,6 +119,8 @@ CI 在已授权的远程 runner 上运行测试、构建 wheel,并从源码目
|
|||||||
|
|
||||||
已导入整卷的逐条轨道修正见 [edition-tracks 维护流程](docs/edition-tracks.md):按完整证据生成只读计划,保留原投稿和决定 ID,事务性修正 edition、标识与覆盖,并防止旧整卷刷新覆盖分类。
|
已导入整卷的逐条轨道修正见 [edition-tracks 维护流程](docs/edition-tracks.md):按完整证据生成只读计划,保留原投稿和决定 ID,事务性修正 edition、标识与覆盖,并防止旧整卷刷新覆盖分类。
|
||||||
|
|
||||||
|
完整出版来源的新旧版本导致分轨或摘要变化时,使用 [source-revisions 流程](docs/source-revisions.md):核对同一 DOI 全集,保存完整修正前数据与版本链,支持已有记录的目标 track,按新版本重算分区覆盖数,并阻止旧版元数据刷新。普通增量导入仍禁止缩小预期数量。
|
||||||
|
|
||||||
入库前可使用通用的[来源身份只读预检](docs/source-identity-audit.md),导出候选论文、稳定标识和既有投稿决定证据;候选结果需要审核,不会自动批准导入。
|
入库前可使用通用的[来源身份只读预检](docs/source-identity-audit.md),导出候选论文、稳定标识和既有投稿决定证据;候选结果需要审核,不会自动批准导入。
|
||||||
|
|
||||||
RSS 2026 已独立适配官方论文集与录用表;使用方法和与 2025 demo 分类的区别见 [RSS 2026 来源说明](docs/rss2026-proceedings.md)。
|
RSS 2026 已独立适配官方论文集与录用表;使用方法和与 2025 demo 分类的区别见 [RSS 2026 来源说明](docs/rss2026-proceedings.md)。
|
||||||
@@ -131,4 +135,20 @@ ICAPS 2026 也提供独立的 `collect icaps2026-registry`,保留 Crossref 卷
|
|||||||
|
|
||||||
ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目 ID,并以不可变缓存断点采集公开元数据;保留摘要原文、来源关键词及未分类 track,始终待身份审核,不自动入库。
|
ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目 ID,并以不可变缓存断点采集公开元数据;保留摘要原文、来源关键词及未分类 track,始终待身份审核,不自动入库。
|
||||||
|
|
||||||
|
`collect ismir2026-program` 核验 140 项 ISMIR 官方录用节目及原始摘要,保留两处标题、三处作者名单差异和 101 组自愿公开评审文本;不推断原始 OpenReview ID 或拒稿。见 [ISMIR 2026 使用与恢复说明](docs/ismir2026-program.md)。
|
||||||
|
|
||||||
|
`collect icassp2026-registry` 从冻结的 10 页 Crossref 元数据核验 4,582 条论文,明确排除 7 条非论文及 56 条其他会议记录;保留摘要缺失和单字段作者姓名,不自动融合节目表。见 [ICASSP 2026 采集与恢复说明](docs/icassp2026-registry.md)。
|
||||||
|
|
||||||
`collect sdm2026-proceedings` 核验 53 个 Crossref 论文章节,并单独保留 53 项日程观察及 52 页摘要。作者差异、取消展示和摘要缺失分别保留,不自动拼接来源或绑定作品;见 [SDM 2026 使用与恢复说明](docs/sdm2026-proceedings.md)。
|
`collect sdm2026-proceedings` 核验 53 个 Crossref 论文章节,并单独保留 53 项日程观察及 52 页摘要。作者差异、取消展示和摘要缺失分别保留,不自动拼接来源或绑定作品;见 [SDM 2026 使用与恢复说明](docs/sdm2026-proceedings.md)。
|
||||||
|
|
||||||
|
`collect cikm2026-sources` 核验并分开导出两份官方节目、附属议程和出版登记。见 [CIKM 2026 来源采集与恢复](docs/cikm2026-sources.md)。
|
||||||
|
|
||||||
|
`collect icdm2026-program` 离线核验 324 条 ICDM2026 官方线下日程,按明确的研究/应用标签输出 241/83 条;展示长短、缺失摘要及未核实出版身份分别保留,不自动入库。见 [ICDM2026 使用与恢复说明](docs/icdm2026-program.md)。
|
||||||
|
|
||||||
|
`collect recsys2026-program` 核验九个 track 的 263 项官方录用贡献观察(260 个不同 DOI),保留摘要、原始作者和重复 DOI 的来源版本。见 [RecSys 2026 使用与恢复说明](docs/recsys2026-program.md)。
|
||||||
|
|
||||||
|
`collect mm2026-program` 从已冻结官方快照恢复全部 28 个来源 track 的 2,143 个节目 ID,保留 4,326 次展示的来源位置与场次上下文,单列 216 个摘要缺失项;输出供贡献类型、出版及作品身份核查使用,不是可直接导入的论文/决定队列。见 [MM 2026 使用、来源和恢复说明](docs/mm2026-source-status.md)。
|
||||||
|
|
||||||
|
`collect mm2026-publications` 核验完整论文集的 1,918 个注册出版 DOI,并单独关联节目分轨、展示上下文和摘要。1,743 份节目摘要明确标注来源(其中 26 份来自已复核的标题排版对应),166 项标题/作者版本差异保留为待核;所有节目及未匹配项均保留。注册文章类型不等于研究类型,输出尚需数据库身份/历史审查。见 [MM 2026 出版条目采集与恢复](docs/mm2026-publications.md)。
|
||||||
|
|
||||||
|
ECCV 2026 已归档 Springer 目录和章节可通过 [正式出版采集流程](docs/eccv2026-publications.md)整理为带 DOI 和可复核证据的 JSONL。
|
||||||
|
|||||||
@@ -0,0 +1,22 @@
|
|||||||
|
# CIKM 2026 来源采集与恢复
|
||||||
|
|
||||||
|
使用固定节目和登记快照复现 CIKM 2026 来源盘点。采集命令生成四份独立 JSONL,输入逐文件校验 SHA-256。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect cikm2026-sources \
|
||||||
|
--snapshots /data/paper-index/validation/cikm2026-source-snapshots-v1 \
|
||||||
|
--out /data/paper-index/collected/cikm2026-sources-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
输入包括两版官方节目 HTML、Crossref 卷登记、五页逐篇登记及分页回执。服务器验证目录保留输入原文。输出目录必须是新目录。先检查 `collection-report.json`,报告在四份 JSONL 写完后发布。
|
||||||
|
|
||||||
|
| 文件 | 内容 |
|
||||||
|
| --- | --- |
|
||||||
|
| `programme.jsonl` | 带 DOI 的 1,008 条节目记录,包括标题、作者显示名、摘要和五类节目标签 |
|
||||||
|
| `alternate-programme.jsonl` | 另一版节目 1,005 条记录及各自的字段 |
|
||||||
|
| `supplemental-programme.jsonl` | 73 条演讲、教程、工作坊及其他附属活动观察 |
|
||||||
|
| `crossref.jsonl` | 1,081 条 ACM 卷内登记,每条保留登记字段及分页文件位置 |
|
||||||
|
|
||||||
|
节目 DOI 指向 Crossref 的出版登记。确认作者和标题差异以及历史投稿绑定后,再为来源观察准备入库队列。报告另列 73 个未出现在带 DOI 节目区的登记,附属活动仍保留各自的活动类型。公开拒稿与撤回决定仍需独立来源核查。
|
||||||
|
|
||||||
|
官方原始来源见[带 DOI 的节目](https://cikm2026.diag.uniroma1.it/cikm-program/program.html)、[另一版节目](https://cikm2026.diag.uniroma1.it/cikm-program/CIKM2026_program.html)和 [Crossref 卷登记](https://api.crossref.org/works/10.1145/3799682)。
|
||||||
+11
-1
@@ -1,5 +1,7 @@
|
|||||||
# 会议覆盖与公开决定核查
|
# 会议覆盖与公开决定核查
|
||||||
|
|
||||||
|
最新生产验收与剩余工作见 [2026-11-17 补漏状态](backfill-status-2026-11-17.md):当日 `14:06:28 UTC` 固定范围快照中,278 个会议年份槽位有 272 个含记录、6 个仍为空;这不代表完整覆盖,第 14 步仍未完成。后续 MM 规范化、只读候选审计与出版核查不等于生产导入。
|
||||||
|
|
||||||
**当前工作范围:2020 年起主动补漏。** 2000 年起已发现资料保留,恢复工作从 [补漏范围与恢复清单(2026-11-10)](backfill-handoff-2026-11-10.md) 开始;下文旧日期快照不改写,较早年份排除不算补齐。
|
**当前工作范围:2020 年起主动补漏。** 2000 年起已发现资料保留,恢复工作从 [补漏范围与恢复清单(2026-11-10)](backfill-handoff-2026-11-10.md) 开始;下文旧日期快照不改写,较早年份排除不算补齐。
|
||||||
|
|
||||||
这是 2026-11-08 的首份核查基线,**完整逐年政策普查和最终去重仍未完成**。收录 43 个目标,不能把“有记录”当作“完整覆盖”。结构化清单见 [inventory-2026-11-08.json](inventory-2026-11-08.json)。清单中的 `policy_checks` 均限定所列年份和轨道;空数组表示尚未核实,绝不表示没有公开拒稿。
|
这是 2026-11-08 的首份核查基线,**完整逐年政策普查和最终去重仍未完成**。收录 43 个目标,不能把“有记录”当作“完整覆盖”。结构化清单见 [inventory-2026-11-08.json](inventory-2026-11-08.json)。清单中的 `policy_checks` 均限定所列年份和轨道;空数组表示尚未核实,绝不表示没有公开拒稿。
|
||||||
@@ -70,7 +72,7 @@ PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas coverage \
|
|||||||
|
|
||||||
## 优先缺口
|
## 优先缺口
|
||||||
|
|
||||||
- **UAI 2024–2026**:官方均允许作者选择公开拒稿,当前主库拒稿仍为零。原 notes 接口要求 Challenge 验证,公开镜像尚无已核验来源,必须保留这个真实缺口;不能根据 PMLR 录用名单推断拒稿。参见 [2025 投稿指南](https://www.auai.org/uai2025/submission_instructions)、JSON 内其他年份官方证据。
|
- **UAI 2024–2026**:官方均允许作者选择公开拒稿,首份生产快照拒稿为零。原 notes 接口要求 Challenge 验证;新核验镜像的 2022–2025 年 902 个论坛来自录用子集,不能填补公开拒稿缺口。2024 年的元评审也不能直接转成最终决定,见 [2026-11-17 原始来源与回复结构核查](uai-public-negative-source-status-2026-11-17.md)。不能根据 PMLR 或镜像录用名单反推拒稿;政策参见 [2025 投稿指南](https://www.auai.org/uai2025/submission_instructions)、JSON 内其他年份官方证据。
|
||||||
- **EMNLP 2023**:官方允许公开 opt-in 拒稿,目前只导入 9 条。需继续核验实际公开全集。ARR 评审、conference commitment 与最终会议决定分开;[ARR 作者说明](https://aclrollingreview.org/authors) 明确其自愿捐赠数据与 EMNLP 2023 直接投稿发布的范围不同。
|
- **EMNLP 2023**:官方允许公开 opt-in 拒稿,目前只导入 9 条。需继续核验实际公开全集。ARR 评审、conference commitment 与最终会议决定分开;[ARR 作者说明](https://aclrollingreview.org/authors) 明确其自愿捐赠数据与 EMNLP 2023 直接投稿发布的范围不同。
|
||||||
- **ICLR**:固定快照已观察 2017 与 2020–2026;快照之后,2013/2018/2019 的 1,654 条来源记录已完成独立 PostgreSQL 验收(报告时间 `2026-11-07T19:52:28.378Z`),不回写旧快照计数,也不据此宣称逐届全集已齐。快照中的 10,208 次 unknown 不等于 rejected,近期补采与冲突核对仍在进行;评审推荐不能当最终决定。
|
- **ICLR**:固定快照已观察 2017 与 2020–2026;快照之后,2013/2018/2019 的 1,654 条来源记录已完成独立 PostgreSQL 验收(报告时间 `2026-11-07T19:52:28.378Z`),不回写旧快照计数,也不据此宣称逐届全集已齐。快照中的 10,208 次 unknown 不等于 rejected,近期补采与冲突核对仍在进行;评审推荐不能当最终决定。
|
||||||
- **MM / MLSys**:MM 11,200 条书目记录保留 `proceedings_unspecified`;MLSys 的 448 次官方投稿与 313 条 DBLP 记录重叠。2026 MLSys 的研究/工业轨道在 [官方 CFP](https://mlsys.org/Conferences/2026/CallForResearchPapers) 中分开,仍须核对逐篇轨道证据,不能仅凭会议名全部归 main。
|
- **MM / MLSys**:MM 11,200 条书目记录保留 `proceedings_unspecified`;MLSys 的 448 次官方投稿与 313 条 DBLP 记录重叠。2026 MLSys 的研究/工业轨道在 [官方 CFP](https://mlsys.org/Conferences/2026/CallForResearchPapers) 中分开,仍须核对逐篇轨道证据,不能仅凭会议名全部归 main。
|
||||||
@@ -187,6 +189,8 @@ ICAPS官方共同历史明确列出两个前身系列,现有DBLP映射只覆
|
|||||||
|
|
||||||
MM 2026 的[官方 CFP](https://2026.acmmm.org/site/cfp-guidelines.html)说明会后公开最终评审及适用的作者回复,但未明确包含全部被拒投稿。2026-11-08 对官方组的匿名 API 查询返回200,对该组 `venueid` 的投稿查询返回403;[证据](mm-editions-2026-11-08.json)保存查询、时间和响应哈希。公开拒稿可用性仍为 unknown,403不能作为零记录或不存在公开拒稿的证据。
|
MM 2026 的[官方 CFP](https://2026.acmmm.org/site/cfp-guidelines.html)说明会后公开最终评审及适用的作者回复,但未明确包含全部被拒投稿。2026-11-08 对官方组的匿名 API 查询返回200,对该组 `venueid` 的投稿查询返回403;[证据](mm-editions-2026-11-08.json)保存查询、时间和响应哈希。公开拒稿可用性仍为 unknown,403不能作为零记录或不存在公开拒稿的证据。
|
||||||
|
|
||||||
|
2026-11-17后续:完整节目JSON与按声明的负面venue ID进行的四条匿名403核查已冻结,见 [MM最新来源状态](../mm2026-source-status.md);没有据此关闭公开拒稿缺口。
|
||||||
|
|
||||||
WWW早期逐来源核查见 `www-early-proceedings-2026-11-08.json`:1995春季论文三目录均为38个相同条目ID,海报摘要53个条目与作者链接47个条目不齐;1994春季50个Title字段与主页49场报告说法待逐项核对。上述为源目录观察,不是已导入或全部录用数量;主会、海报、教程等继续独立核验。
|
WWW早期逐来源核查见 `www-early-proceedings-2026-11-08.json`:1995春季论文三目录均为38个相同条目ID,海报摘要53个条目与作者链接47个条目不齐;1994春季50个Title字段与主页49场报告说法待逐项核对。上述为源目录观察,不是已导入或全部录用数量;主会、海报、教程等继续独立核验。
|
||||||
|
|
||||||
WWW事件标识生产迁移已独立验收(报告SHA `f5dbb8a8`):保留1399个原届次字段及IDs、59个Fall投稿和完整全库历史哈希。服务器已缓存46个官方HTML来源(约223KB),只作为后续维护式采集器的输入;尚未补导早期WWW论文,也未启用可选注解表。
|
WWW事件标识生产迁移已独立验收(报告SHA `f5dbb8a8`):保留1399个原届次字段及IDs、59个Fall投稿和完整全库历史哈希。服务器已缓存46个官方HTML来源(约223KB),只作为后续维护式采集器的输入;尚未补导早期WWW论文,也未启用可选注解表。
|
||||||
@@ -196,3 +200,9 @@ IJCAI 联办届次的逐条后续核查见 [2018/2022 分区与目录差集](ijc
|
|||||||
2026年11月9日的[当前历史年度缺口核对](historical-gaps-2026-11-09.md)单列135个历史会议—年份、14个2026年度及85个经范围修正的初步候选来源年度;它使用当前生产edition/别名快照,保留原固定快照不变。年度有记录不等于逐篇、逐track或公开拒稿历史完整。
|
2026年11月9日的[当前历史年度缺口核对](historical-gaps-2026-11-09.md)单列135个历史会议—年份、14个2026年度及85个经范围修正的初步候选来源年度;它使用当前生产edition/别名快照,保留原固定快照不变。年度有记录不等于逐篇、逐track或公开拒稿历史完整。
|
||||||
|
|
||||||
[早期NLP来源范围复核](historical-source-scope-2026-11-09.md)保留3份官方网页的响应哈希,排除4个COLING卷对ACL缺口的误匹配,并区分会议举办记录、目录可获取性与论文级覆盖;尚未导入或关闭这些年度缺口。
|
[早期NLP来源范围复核](historical-source-scope-2026-11-09.md)保留3份官方网页的响应哈希,排除4个COLING卷对ACL缺口的误匹配,并区分会议举办记录、目录可获取性与论文级覆盖;尚未导入或关闭这些年度缺口。
|
||||||
|
|
||||||
|
- [MM 2026 官方节目来源状态](../mm2026-source-status.md):完整官方日程 JSON、重复展示盘点、28 个来源 track、缺失作者/摘要和四条匿名决定路径的 403 证据;这些不是已导入或公开拒稿全集。
|
||||||
|
|
||||||
|
- [2026-11-18 公开负面来源核查](public-negative-source-audit-2026-11-18.md):ResearchArcade 930 条非 ICLR 声明的当前库比对、5 条未绑定声明及原始接口访问证据;不把镜像差集算作新增拒稿。
|
||||||
|
|
||||||
|
- [ECCV 2026 出版目录与节目单对账](eccv2026-publisher-crosswalk-2026-11-18.md)记录 Springer 78 卷的 DOI、节目单差异及 PostgreSQL 只读绑定核查。
|
||||||
|
|||||||
@@ -1493,7 +1493,7 @@
|
|||||||
},
|
},
|
||||||
"TMLR_current20261108": {
|
"TMLR_current20261108": {
|
||||||
"url": "https://www.jmlr.org/tmlr/faq.html",
|
"url": "https://www.jmlr.org/tmlr/faq.html",
|
||||||
"finding": "Additionalnonconferencevenuealreadyinregistry:accepted/rejected/withdrawn/retractedallpublic; deskrejectedneverpublic. Authorshiddenonnotyetacceptedpapers; donotinferanonymousidentities. Continuousjournal; eventcertification/journal-to-conferencepresentationnotanotherindependentreviewacceptance."
|
"finding": "Additionalnonconferencepolicyreference; TMLR is not in the current 43-conference registry:accepted/rejected/withdrawn/retractedallpublic; deskrejectedneverpublic. Authorshiddenonnotyetacceptedpapers; donotinferanonymousidentities. Continuousjournal; eventcertification/journal-to-conferencepresentationnotanotherindependentreviewacceptance."
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"public_negative_recovery_notes": {
|
"public_negative_recovery_notes": {
|
||||||
@@ -1507,6 +1507,12 @@
|
|||||||
"original_rejection_notes_obtained": false,
|
"original_rejection_notes_obtained": false,
|
||||||
"prior_decisions_inserted": 0,
|
"prior_decisions_inserted": 0,
|
||||||
"requires": "Per-paper primary original identifiers/decisions or an explicitly distinguished policy-supported relation; never manufacture an original note."
|
"requires": "Per-paper primary original identifiers/decisions or an explicitly distinguished policy-supported relation; never manufacture an original note."
|
||||||
|
},
|
||||||
|
"researcharcade_20261118": {
|
||||||
|
"audit": "public-negative-source-audit-2026-11-18.json",
|
||||||
|
"summary": "930 non-ICLR 2020+ mirror claims:925 scoped ID matches all have rejected status;5 held,3 with equal title+source abstract but distinct unverified note IDs. No writes or new rejections approved.",
|
||||||
|
"access": "NeurIPS2021 and EMNLP2023 original note queries403; stopped, no bypass.",
|
||||||
|
"next": "Seek independent primary evidence for note.id/forum relationships and final decisions; no title-only merge or duplicate-rejection import. No full-public-set coverage claim."
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"restore_rules": [
|
"restore_rules": [
|
||||||
|
|||||||
@@ -0,0 +1,50 @@
|
|||||||
|
# 2020 起补漏状态:2026-11-17
|
||||||
|
|
||||||
|
**总体仍为 13/17,第 14 步未完成。** 本页更新当日最新生产结果;[2026-11-10 恢复清单](backfill-handoff-2026-11-10.md)和更早快照继续保留。用户确定的范围不变:2020 年起主动补漏;43 个更早历史缺口延期而非补齐,2000–2019 已发现的名单和身份线索继续保留。
|
||||||
|
|
||||||
|
## 最新数据库盘点
|
||||||
|
|
||||||
|
只读快照 `2026-11-17 14:06:28.803415+00`,固定 43 个目标会议、278 个应有会议年份槽位,其中 **272 个至少有一条投稿记录,6 个仍完全为空**,均为 2026 年:**CIKM、ECCV、ICDM、IROS、ISMIR、MM**。SDM25、ICASSP112、RecSys47 三批分别使空槽从 9 降至 8、7、6。
|
||||||
|
|
||||||
|
这些是会议—年份的存在性统计,**不表示 272 个槽位已逐篇、逐 track、摘要或公开拒稿覆盖完整**。2020–2025 没有整年空槽,仍有来源完整性和公开负面决定缺口。RecSys 九轨、ICASSP 和 SDM 的本批来源均仍为 partial;不能据此宣布第 14 步完成。
|
||||||
|
|
||||||
|
| 生产文件(相对 `/data/paper-index/`) | SHA256 |
|
||||||
|
| --- | --- |
|
||||||
|
| collected/active-scope-census-20261117-v19/summary.json | cf0c5a013b6efe7d232187c9c76d9820bc8b33e2dd5674cab53d78cd3369ab4c |
|
||||||
|
| collected/active-scope-census-20261117-v19/snapshot.json | 38d09e3782372edbdd2840f5922d65fb497c425b1eaf2d1fb10342956a7a82f2 |
|
||||||
|
| validation/active-scope-fixed278-v1.json | a835bef5b9c3a0f76069d840cecdb9a6bcf65d3a38796d70dbfdfd4b49242264 |
|
||||||
|
|
||||||
|
原始快照、范围定义与生成脚本已备份,本机已按固定范围和别名逐条重算 272/278,未用 `max(id)` 推算数量。此前 v17/v18 快照保留,不能将其旧数字当作当前状态。
|
||||||
|
|
||||||
|
## 已独立接受的近期补录
|
||||||
|
|
||||||
|
- **RecSys2026**:官方九轨 263 项观察、260 个不同 DOI 均已冻结并规范化。47 条现有作品关联通过独立验收,新增论文实体为 0;新增一个来源、九个 edition、47 次投稿/47 条录用观察、47 个发表 DOI 投稿标识及47条来源类型标签证据。原始摘要单独保存,规范摘要不覆盖。5,812 篇受保护旧作品及全部旧投稿/决定历史保持不变,foundry 就绪检查200、SQLite备份只读。216项待审查,包括三组重复 DOI 的六项观察。见 [九轨验收与恢复](../recsys2026-program.md)。
|
||||||
|
- **ICASSP2026**:Crossref 冻结的 4,645 个条目中排除 56 个其他会议条目和 7 个目录/封面等非论文条目,剩余 4,582 条注册论文;全部注册摘要缺失。112 条已有 DOI 的来源/录用历史已独立验收,新增论文实体为 0,coverage 为 partial112/4582;4,470 条仍待身份核查。正式节目标题表与注册元数据保持独立,不自动融合作者或摘要。见 [来源边界与恢复](../icassp2026-registry.md)。
|
||||||
|
- **SDM2026**:53 个注册章节、53 项独立日程观察及 52 页摘要已冻结。25 条来源关联已独立接受,新增论文实体为 0,28 条待审查;见 [生产验收与恢复](../sdm2026-identity-review.md)。
|
||||||
|
|
||||||
|
三批已经消费的输入队列均**禁止重放**。最新已接受生产结果为 RecSys47:论文实体仍为 **3,567,888**,来源记录 **3,814,931**,投稿 **531,620**,决定来源观察 **571,172**。观察数不等于不同论文或不同投稿的数量;后续须使用当前基线和新的增量队列。
|
||||||
|
|
||||||
|
SDM 新增事件使 SeBA 和 AdamG 同时保有较早 ICLR 拒稿与后来 SDM 发表事实。原来的两次 ICLR 投稿及四条决定来源观察均保留,没有被 accepted 覆盖;四条观察不能直接累加成四次拒稿。本批 RecSys47 本身没有既往负面观察,但全部旧全库历史同样通过保护验收。
|
||||||
|
|
||||||
|
## 已采集、尚待关联或规范化的来源
|
||||||
|
|
||||||
|
| 会议 | 已取得的来源与状态 | 下一步 |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| ECCV | 2,834 个不同初步节目 ID 及公开摘要已独立采集;1,997 个只读候选历史证据包已保存,尚未入库 | 初步日程不等于 Springer 最终论文集;继续作品/版本和出版核查。见 [来源状态](../eccv2026-source-status.md) |
|
||||||
|
| ICDM | 324 项线下节目观察,明确研究241/应用83;演讲长168/短156,摘要均缺失;规范化和80个候选历史包已核验 | 展示长短不能当成论文类型;需审查作品与正式出版版本,尚未导入主库。见 [采集器说明](../icdm2026-program.md) |
|
||||||
|
| ISMIR | 140 项节目和正式名单均已核验,140份摘要,101组自愿公开评审;两处标题、三处作者名单差异保留,78个候选历史包已核验 | 数字 miniconf ID 不是原始 OpenReview ID;评审意见不转成最终拒稿。继续作品身份/历史关联,尚未入库。采集器见 [PR #63](https://gitea.entroia.com/Zayden-Jung/paper-atlas/pulls/63) |
|
||||||
|
| MM | 全28轨2143个节目ID/4326展示位置已规范化;814个只读候选历史证据包已核验,24篇候选涉及24次负面投稿/37条来源观察;67项原始arXiv DOI中66项注册元数据一致、1项404暂缓 | 不是2143篇已核验科研论文。继续逐轨贡献/出版依据与作品审查,0身份批准、0入库。ISBN查询与父XML均只返回父记录,不作逐篇清单。见 [来源、审计与恢复证据](../mm2026-source-status.md) |
|
||||||
|
| CIKM | 官方录用/日程入口与五个分轨页面已保存;五个分轨链接实际是CFP/政策,不是逐篇录用名单 | 查找实际节目或出版社完整目录;TBD和没有静态表不能当作没有论文 |
|
||||||
|
| IROS | 官方论文索引入口的原始HTTP403已保存 | 保留访问限制,不重试绕过;寻找可合法读取、能独立核验的出版元数据 |
|
||||||
|
|
||||||
|
早期 14 个入口快照保持在 `/data/paper-index/collected/remaining2026-official-leads-v1`,后续新增原文另建冻结目录,不覆盖早期回执。HTTP200只证明入口可读,不直接作为可导入记录或全部论文数量。
|
||||||
|
|
||||||
|
## 公开拒稿与后续工作
|
||||||
|
|
||||||
|
MM 官方群组元数据可匿名读取,但根据其给出的 submission invitation 和 rejected/withdrawn/desk-rejected venue ID,四次限1条的匿名 notes 核查均为403。查询、时间、原文和哈希已保留,没有认证重试或绕过权限。访问限制不等于零条负面记录;详见 [MM来源证据](../mm2026-source-status.md)。ISMIR公开的101组评审来自录用节目,不能记成101条拒稿。
|
||||||
|
|
||||||
|
UAI新增固定镜像已核对2022–2025的902个原始论坛及3053个回复的元数据/字段结构。整理版均标录用,2022保存230条明确Accept决定;2024仅有评审、作者回复、元评审,2023/2025未保存回复,不能据此补拒稿。来源限制和恢复见 [逐年原始核查](uai-public-negative-source-status-2026-11-17.md)。这不是UAI没有公开拒稿的结论,缺口保持未补齐。
|
||||||
|
|
||||||
|
第14步仍需逐会议、年份、track完成公开拒稿/desk-reject/withdrawn来源核查和可得数据采集;继续核查UAI近年自愿公开范围、EMNLP实际覆盖范围及其余已列明缺口。不能从未见录用、评分、评审推荐、缺少摘要或接口不可访问制造负面决定。
|
||||||
|
|
||||||
|
之后仍需完成跨会作品/历次投稿与类型标签关联(第15步),所有导入结束后的独立全库去重、标识冲突、近似标题作者、摘要/约束/查询及覆盖审计(第16步),以及长期运行与恢复文档最终整理(第17步)。阶段性来源入库、测试通过、某年度有记录均不能替代这些验收。
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
# 2020 起补漏状态:2026-11-18
|
||||||
|
|
||||||
|
总体仍为 **13/17,第 14 步未完成**。主动补漏从 2020 年起,2000–2019 年已发现线索仍保留,43 个早期历史缺口仍为延期,没有据此算作补齐。更早状态页保留为历史快照。
|
||||||
|
|
||||||
|
## 当前生产结果
|
||||||
|
|
||||||
|
ISMIR 两批累计31项已关联到现有论文,新增论文实体为0。第二批新增来源记录、独立投稿、录用观察各21条及173条来源标签证据;两批累计267条标签证据。覆盖 `partial31/140`,109项仍待核验。详见[来源说明](../ismir2026-program.md)、[首批验收](ismir2026-import10-2026-11-18.json)和[第二批逐条验收](ismir2026-import21-2026-11-18.json)。
|
||||||
|
|
||||||
|
最新精确计数:论文实体 **3,568,022**,来源记录 **3,815,534**,投稿 **532,223**,决定来源观察 **571,775**。实体数量不是已经完成最终科学工作去重的数量;同一次投稿可能有多条决定来源观察。
|
||||||
|
|
||||||
|
paper `2757497` 保留 ICLR2026 一次拒稿投稿下的两条来源观察,并新增 ISMIR2026 录用投稿;实际 CLI 查询显示已知录用,旧拒稿历史仍可查询。不将两条来源观察计作两次拒稿,也不将来源采集时间当作决定时间。
|
||||||
|
|
||||||
|
## 固定范围存在性盘点
|
||||||
|
|
||||||
|
只读快照 `2026-11-18 06:46:44.099675+00`,范围仍为 43 会、278 个目标年度槽位:**274 个至少有一条投稿,4 个完全为空**,均为 2026 年的 **CIKM、ECCV、ICDM、IROS**。首批ISMIR使空槽从5降至4;第二批只增加同届来源覆盖,空槽数不变。
|
||||||
|
|
||||||
|
这只统计“该年度至少有数据”,不是逐篇、逐 track、摘要或公开拒稿覆盖完整。2020–2025 没有整年空槽,仍有来源完整性和公开负面历史缺口。
|
||||||
|
|
||||||
|
|证据(相对服务器 `/data/paper-index/`)|SHA256|
|
||||||
|
|---|---|
|
||||||
|
|`collected/active-scope-census-20261118-v22/summary.json`|`ee4ec2e6d6eaed55ce3475e735bfba9bd94e512893cd3c3024035b48b744f475`|
|
||||||
|
|`collected/active-scope-census-20261118-v22/snapshot.json`|`5aabc91217ad787946ba50d6a670296b1e3152a54f5ecaa0e3a9b3a952b63fa7`|
|
||||||
|
|`validation/active-scope-fixed278-v1.json`|`a835bef5b9c3a0f76069d840cecdb9a6bcf65d3a38796d70dbfdfd4b49242264`|
|
||||||
|
|
||||||
|
本机使用固定分母、会议别名和逐 edition 快照独立重算 274/278;没有使用 `max(id)` 代替计数。
|
||||||
|
|
||||||
|
## 尚待完成
|
||||||
|
|
||||||
|
- ISMIR 仍有109项;62项无精确标题候选不等于新作品,6项作者部分匹配仍待审查。31项摘要逐字一致候选已分两批审查并消费;其余候选不能仅凭标题、作者或摘要检索结果算作已批准。
|
||||||
|
- MM 已验收572项来源观察,注册清单1918项中仍有1346项待处理。已有134个暂建索引实体仍须后续身份审计;H2AL 来源及轨道纠正、版本化来源覆盖方案尚未用于生产,不能把不同版本轨道预期数相加当完整分母。
|
||||||
|
- ECCV 的初步节目、ICDM 的研究/应用来源和缺失摘要仍需作品及出版核查;CIKM 缺实际逐篇目录;IROS 官方访问403仍保留限制,不重试绕过。各来源的采集、批准和导入状态必须分开。
|
||||||
|
- 公开 rejected、desk-rejected、withdrawn 继续按会议、年份、track核查。ISMIR 的101组自愿公开评审来自录用节目,不是101条拒稿;UAI accepted-only 镜像与 EMNLP/NeurIPS 原始接口403均不能关闭拒稿缺口。近期镜像差集核查见 [PR #72](https://gitea.entroia.com/Zayden-Jung/paper-atlas/pulls/72)。
|
||||||
|
- 后续仍需完成跨会作品与历史、独立类型和标签关系(第15步);全部导入结束后的全库去重、标识冲突、近似标题作者、摘要/约束/查询及覆盖审计(第16步);长期运行与恢复文档最终整理(第17步)。
|
||||||
|
|
||||||
|
已消费输入不得重放。下次从[ISMIR 当前增量父状态和恢复说明](../ismir2026-program.md)及 `work-state.json` 的本批已接受回执继续;保留旧 SQLite 只读备份,不因查询超时重新执行导入。
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,16 @@
|
|||||||
|
# CIKM 2026 节目与出版登记核对
|
||||||
|
|
||||||
|
官方节目列有 1,008 项带 DOI 的条目,逐篇均有摘要。五个标签的数量分别为 Full Research 596、Short Research 234、Applied Research 86、Resources 46、Demo 46。另一份官方节目快照少三个节目编号,其余 1,005 个编号重合。两份原始快照和响应哈希保存在本机的 `cikm2026-official-detailed-program-v1` 工作目录。[带 DOI 的节目](https://cikm2026.diag.uniroma1.it/cikm-program/program.html) · [另一份官方节目](https://cikm2026.diag.uniroma1.it/cikm-program/CIKM2026_program.html)
|
||||||
|
|
||||||
|
正式卷 DOI 为 `10.1145/3799682`,ISBN 为 `9798400725395`。ISBN 查询只返回卷级登记。按卷名及出版日期分页取得 1,081 条登记,逐条核对卷 DOI 前缀、完整容器标题及来源类型。1,008 个节目 DOI 全部匹配,另有 73 条登记位于此前带 DOI 节目分区之外。附属议程核对结果见下文。该次 Crossref 响应中摘要字段均缺失。[ACM 出版条目](https://doi.org/10.1145/3799682.3840615) · [Crossref 卷登记](https://api.crossref.org/works/10.1145/3799682)
|
||||||
|
|
||||||
|
按已知排版标记、Unicode 和标点变化比较,同 DOI 的条目有 4 处标题差异、11 处有序作者列表差异。差异条目保留两份原文,等后续核对后再决定如何关联。当前 PostgreSQL 快照中,97 个节目 DOI 已绑定论文,304 个节目标题有候选,其中 95 个 DOI 绑定与标题候选重合。仍需核查已有作品与历次投稿历史,当前尚未写入。
|
||||||
|
|
||||||
|
[逐条差集、字段差异及证据哈希](cikm2026-source-reconciliation-2026-11-18.json)。先将出版登记与节目摘要作为各自来源采集,再核查两者的作品关系和轨道。公开拒稿、撤回及桌拒的可得性继续单独核查。
|
||||||
|
|
||||||
|
|
||||||
|
## 附属议程与已有 DOI 的进一步核查
|
||||||
|
|
||||||
|
在官方节目页面的附属议程及主题演讲中,73 条登记有 72 条找到唯一规范标题对应,70 条同时通过有序作者比较。对应项包括 Industry Day 20 条、博士论坛 19 条、教程 14 条、工作坊条目 15 条、主题演讲 3 条及 AnalytiCup 1 条。工作坊条目记录该场活动对应的出版材料,独立 workshop 论文仍需各自目录。两条工作坊条目有作者写法差异,另有一条 LASS 标题带不同前后缀,三条均保留待核对。[官方完整节目](https://cikm2026.diag.uniroma1.it/cikm-program/program.html)
|
||||||
|
|
||||||
|
全卷 1,081 个 DOI 的 PostgreSQL 只读复查找到 98 个已有绑定,其中 97 个来自带 DOI 节目分区。89 项在现有论文、Crossref 和节目中有一致的规范标题、有序作者及明确节目标签,可优先准备来源关联。其余 9 项保留待核查。98 项的现有历史中无拒稿、桌拒或撤回记录,这一观察仅覆盖已有绑定的作品。此次仍未写入,入库时须重新确认绑定及保护既有历史。
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,20 @@
|
|||||||
|
# ECCV 2026 出版目录与节目单对账
|
||||||
|
|
||||||
|
2026-11-18 核验了 Springer LNCS 17001–17078 的全部 78 卷。会议索引显示 **2,835 条**,逐卷目录合计也有 **2,835 条带 DOI 的章节**,另有 156 项卷首或卷末资料。出版方卷册简介及大会节目单均为 **2,834 条**。两个来源仍有逐条差异,汇总数量相差一条的原因待核。逐卷数字、目录页哈希和只读数据库回执见[机器可读清单](eccv2026-publisher-crosswalk-2026-11-18.json)。[Springer 会议索引](https://link.springer.com/conference/eccv) · [第一卷](https://link.springer.com/book/10.1007/978-3-032-36984-0) · [大会节目单](https://eccv.ecva.net/Conferences/2026/AcceptedPapers)
|
||||||
|
|
||||||
|
## 逐篇对账
|
||||||
|
|
||||||
|
|核对对象|唯一标题且完整作者顺序一致|唯一标题且目录所显示作者顺序一致|标题对得上、作者有差异|标题待核|
|
||||||
|
|---|---:|---:|---:|---:|
|
||||||
|
|全部 2,834 条节目记录|1,332|709|535|258|
|
||||||
|
|首批已入库的 30 条节目记录|23|3|4|0|
|
||||||
|
|本次只读核验通过的 248 条既有论文候选|173|72|2|1|
|
||||||
|
|20 篇有既往拒稿、桌拒或撤回的优先论文|12|7|1|0|
|
||||||
|
|
||||||
|
目录作者栏有时只显示前六人。作者差异包括缩写、姓名文字变化与排列变化。原始名单、顺序和来源保留,前缀匹配与作者差异需查章节作者页。节目单与出版目录的摘要、分轨及公开负面决定覆盖仍待分别核查。
|
||||||
|
|
||||||
|
## 当前库核对
|
||||||
|
|
||||||
|
只读核对了全部 2,835 个出版 DOI,68 个已有论文绑定。对账后另检查 271 个已知论文关联,涵盖 245 个本次候选和 26 个已入库节目记录。其中 14 个 DOI 已指向同一论文,257 个 DOI 尚未绑定。245 个候选内有 10 篇带既往负面决定。
|
||||||
|
|
||||||
|
下一批准备以出版 DOI、正式分轨和完整作者证据复核关联。首批 30 条节目记录保留原投稿、决定和来源。确认同届同一投稿后,再补充出版证据,避免重复创建 ECCV 投稿。本次核查未写入数据库。
|
||||||
@@ -0,0 +1,339 @@
|
|||||||
|
{
|
||||||
|
"sources": {
|
||||||
|
"poster.html": {
|
||||||
|
"url": "https://www3.cs.stonybrook.edu/~icdm2026/PosterwDemo/list.htm",
|
||||||
|
"final_url": "https://www3.cs.stonybrook.edu/~icdm2026/PosterwDemo/list.htm",
|
||||||
|
"http_status": 200,
|
||||||
|
"bytes": 132977,
|
||||||
|
"sha256": "b02083ca7574a188fe876851a700a12c131c262353db557bd43a8320fad5f747",
|
||||||
|
"at": "2026-11-18T07:06:11.984184+00:00"
|
||||||
|
},
|
||||||
|
"schedule.html": {
|
||||||
|
"url": "https://icdm2026.neu.edu.cn/DetailedSessions/list.htm",
|
||||||
|
"final_url": "https://icdm2026.neu.edu.cn/DetailedSessions/list.htm",
|
||||||
|
"http_status": 200,
|
||||||
|
"bytes": 213830,
|
||||||
|
"sha256": "c1294b89bae8c2dfcf652d8a25556e733ef42dc3ac3387070abe08419b961b0f",
|
||||||
|
"at": "2026-11-18T07:06:12.349258+00:00"
|
||||||
|
},
|
||||||
|
"deepsync.html": {
|
||||||
|
"url": "https://dm.cs.univie.ac.at/news/story/deepsync-paper-accepted-at-icdm-2026",
|
||||||
|
"final_url": "https://dm.cs.univie.ac.at/news/story/deepsync-paper-accepted-at-icdm-2026",
|
||||||
|
"http_status": 200,
|
||||||
|
"bytes": 90333,
|
||||||
|
"sha256": "e243196943434225acec2841b53c77a2b1459184e7c185e1e1cbb055118fe282",
|
||||||
|
"at": "2026-11-18T07:06:20.561308+00:00"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"poster_count": 343,
|
||||||
|
"poster_groups": {
|
||||||
|
"long-posters": 177,
|
||||||
|
"short-posters": 166
|
||||||
|
},
|
||||||
|
"oral_count": 324,
|
||||||
|
"shared_count": 324,
|
||||||
|
"poster_only": {
|
||||||
|
"DM1022": {
|
||||||
|
"group": "short-posters",
|
||||||
|
"floor": "3F",
|
||||||
|
"area": "Area C",
|
||||||
|
"slot": "P2-3C-085",
|
||||||
|
"session": "Applied Track: Real-world AI in finance, healthcare and industry II",
|
||||||
|
"title": "Beyond Bidding Agents: A Dual-Role Multi-Agent Framework with Dynamic Reserve Pricing for Auto-Bidding Auctions",
|
||||||
|
"line": 683
|
||||||
|
},
|
||||||
|
"DM1189": {
|
||||||
|
"group": "long-posters",
|
||||||
|
"floor": "3F",
|
||||||
|
"area": "Area A",
|
||||||
|
"slot": "P1-3A-032",
|
||||||
|
"session": "Heterogeneous data mining: spatiotemporal and representation learning",
|
||||||
|
"title": "Traj-Transformer: Diffusion Models with Transformer for GPS Trajectory Generation",
|
||||||
|
"line": 456
|
||||||
|
},
|
||||||
|
"DM1234": {
|
||||||
|
"group": "long-posters",
|
||||||
|
"floor": "2F",
|
||||||
|
"area": "Area A",
|
||||||
|
"slot": "P1-2A-007",
|
||||||
|
"session": "Foundations, causality and learning theory",
|
||||||
|
"title": "Prediction-Powered Corpus Mining: Coverage-Valid Statistics from Scarce Labels",
|
||||||
|
"line": 354
|
||||||
|
},
|
||||||
|
"DM1246": {
|
||||||
|
"group": "short-posters",
|
||||||
|
"floor": "3F",
|
||||||
|
"area": "Area B",
|
||||||
|
"slot": "P2-3B-074",
|
||||||
|
"session": "LLM-driven data mining and novel applications II",
|
||||||
|
"title": "Feature Selection for Trajectory Inference",
|
||||||
|
"line": 672
|
||||||
|
},
|
||||||
|
"DM1345": {
|
||||||
|
"group": "short-posters",
|
||||||
|
"floor": "3F",
|
||||||
|
"area": "Area B",
|
||||||
|
"slot": "P2-3B-060",
|
||||||
|
"session": "LLM-driven data mining and heterogeneous learning",
|
||||||
|
"title": "TRACER: An LLM-based Coarse-to-Fine Framework for Adaptive Network Traffic Classification",
|
||||||
|
"line": 658
|
||||||
|
},
|
||||||
|
"DM1408": {
|
||||||
|
"group": "long-posters",
|
||||||
|
"floor": "2F",
|
||||||
|
"area": "Area B",
|
||||||
|
"slot": "P1-2B-043",
|
||||||
|
"session": "Applied Track: Industrial, financial and decision intelligence",
|
||||||
|
"title": "A Generation-to-Application Framework for Semantic ID Collision in Industrial Generative Recommendation",
|
||||||
|
"line": 390
|
||||||
|
},
|
||||||
|
"DM1644": {
|
||||||
|
"group": "long-posters",
|
||||||
|
"floor": "2F",
|
||||||
|
"area": "Area B",
|
||||||
|
"slot": "P1-2B-072",
|
||||||
|
"session": "Recommendation, adaptation and spatiotemporal modeling II",
|
||||||
|
"title": "LQ-Rec: Late Quantization for Sequential Recommendation",
|
||||||
|
"line": 419
|
||||||
|
},
|
||||||
|
"DM1751": {
|
||||||
|
"group": "short-posters",
|
||||||
|
"floor": "3F",
|
||||||
|
"area": "Area A",
|
||||||
|
"slot": "P2-3A-014",
|
||||||
|
"session": "ML/DL and statistical methods",
|
||||||
|
"title": "MissGNN: Missingness-Guided Graph Neural Network for Feature- and Sample-Aware Imputation",
|
||||||
|
"line": 612
|
||||||
|
},
|
||||||
|
"DM1995": {
|
||||||
|
"group": "long-posters",
|
||||||
|
"floor": "2F",
|
||||||
|
"area": "Area B",
|
||||||
|
"slot": "P1-2B-053",
|
||||||
|
"session": "Applied Track: Recommendation, security and operational intelligence II",
|
||||||
|
"title": "Multi-Modal Causal Disentanglement for Multi-Domain Sequential Audio Recommendation",
|
||||||
|
"line": 400
|
||||||
|
},
|
||||||
|
"DM2002": {
|
||||||
|
"group": "long-posters",
|
||||||
|
"floor": "3F",
|
||||||
|
"area": "Area B",
|
||||||
|
"slot": "P1-3B-048",
|
||||||
|
"session": "Heterogeneous data mining and systems",
|
||||||
|
"title": "HiMamba: A Hierarchical State Space Framework for Asymmetric Multimodal Data Mining in Clinical Assessment",
|
||||||
|
"line": 472
|
||||||
|
},
|
||||||
|
"DM2055": {
|
||||||
|
"group": "short-posters",
|
||||||
|
"floor": "3F",
|
||||||
|
"area": "Area A",
|
||||||
|
"slot": "P2-3A-038",
|
||||||
|
"session": "Applied Track: Real-world AI in finance, healthcare and industry I",
|
||||||
|
"title": "CGSBR: Adaptive Cross-Source Gating for Cold-Start Sequential Bundle Recommendation",
|
||||||
|
"line": 636
|
||||||
|
},
|
||||||
|
"DM2140": {
|
||||||
|
"group": "long-posters",
|
||||||
|
"floor": "3F",
|
||||||
|
"area": "Area A",
|
||||||
|
"slot": "P1-3A-008",
|
||||||
|
"session": "Machine learning and statistical methods",
|
||||||
|
"title": "Reliability-Guided Pseudo-Label Learning and Prototype Alignment for Domain-Adaptive Colorectal Segmentation",
|
||||||
|
"line": 432
|
||||||
|
},
|
||||||
|
"DM252": {
|
||||||
|
"group": "short-posters",
|
||||||
|
"floor": "2F",
|
||||||
|
"area": "Area B",
|
||||||
|
"slot": "P2-2B-046",
|
||||||
|
"session": "LLM-driven data mining and novel applications I",
|
||||||
|
"title": "Trust-But-Verify: Poisoning-Resilient Locally Private Graph Learning Protocols",
|
||||||
|
"line": 572
|
||||||
|
},
|
||||||
|
"DM383": {
|
||||||
|
"group": "long-posters",
|
||||||
|
"floor": "2F",
|
||||||
|
"area": "Area A",
|
||||||
|
"slot": "P1-2A-022",
|
||||||
|
"session": "Heterogeneous graphs and multimodal intelligence",
|
||||||
|
"title": "Subgraph Meta-Learning for Direction and Time Aware Transaction Anomaly Detection",
|
||||||
|
"line": 369
|
||||||
|
},
|
||||||
|
"DM463": {
|
||||||
|
"group": "short-posters",
|
||||||
|
"floor": "2F",
|
||||||
|
"area": "Area B",
|
||||||
|
"slot": "P2-2B-050",
|
||||||
|
"session": "Modeling, recommendation and LLM methods",
|
||||||
|
"title": "CaST-POI: Candidate-Conditioned Spatiotemporal Ranking for Next POI Recommendation",
|
||||||
|
"line": 576
|
||||||
|
},
|
||||||
|
"DM483": {
|
||||||
|
"group": "short-posters",
|
||||||
|
"floor": "2F",
|
||||||
|
"area": "Area B",
|
||||||
|
"slot": "P2-2B-051",
|
||||||
|
"session": "Modeling, recommendation and LLM methods",
|
||||||
|
"title": "Causal Masking Lowers Intrinsic Dimension in Full-Vocabulary Sequential Recommendation: Evidence from a Single-Variable Attention Ablation",
|
||||||
|
"line": 577
|
||||||
|
},
|
||||||
|
"DM713": {
|
||||||
|
"group": "short-posters",
|
||||||
|
"floor": "2F",
|
||||||
|
"area": "Area B",
|
||||||
|
"slot": "P2-2B-060",
|
||||||
|
"session": "Modeling, recommendation and LLM methods",
|
||||||
|
"title": "Hint-Conditioned Knowledge Tracing: A Causal-Inspired View of Context-Dependent Hint Usage",
|
||||||
|
"line": 586
|
||||||
|
},
|
||||||
|
"DM736": {
|
||||||
|
"group": "long-posters",
|
||||||
|
"floor": "3F",
|
||||||
|
"area": "Area B",
|
||||||
|
"slot": "P1-3B-050",
|
||||||
|
"session": "LLM-driven data mining and retrieval I",
|
||||||
|
"title": "From Natural-Language Hypotheses to Verified Data-Mining Discoveries",
|
||||||
|
"line": 474
|
||||||
|
},
|
||||||
|
"DM803": {
|
||||||
|
"group": "short-posters",
|
||||||
|
"floor": "2F",
|
||||||
|
"area": "Area B",
|
||||||
|
"slot": "P2-2B-032",
|
||||||
|
"session": "Heterogeneous data mining and multimodal learning I",
|
||||||
|
"title": "Graph-Guided LLM Reasoning for Incident Prediction Over Multi-Source Data Streams",
|
||||||
|
"line": 558
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"oral_only": [],
|
||||||
|
"title_differences": [
|
||||||
|
{
|
||||||
|
"paper_id": "DM1319",
|
||||||
|
"poster_title": "FFE-FRCN: A Frequency Feature Enhanced Few-shot Learning Approach for Object Detection in Microscopic Images",
|
||||||
|
"oral_title": "An FFE-FRCN-Based Data Platform for Environmental Microorganism Image Retrieval and Knowledge Platform"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"nonpaper_table_rows": [],
|
||||||
|
"database_written": false,
|
||||||
|
"scope": "Official presentation metadata. Shared programme IDs identify repeated appearances within this event. Poster-only IDs are scheduling observations pending identity and publication review.",
|
||||||
|
"pg_readonly": {
|
||||||
|
"snapshot": {
|
||||||
|
"at": "2026-11-18 07:08:21.138511+00",
|
||||||
|
"snapshot": "1095530:1095530:"
|
||||||
|
},
|
||||||
|
"report_sha256": "04ad5547342d7e3c2939046f290ecff647878c202c867e47acbdc438d4e10ad4",
|
||||||
|
"existing_icdm2026_editions": [],
|
||||||
|
"existing_icdm_program_source": false,
|
||||||
|
"reviewed_links": [
|
||||||
|
{
|
||||||
|
"external_id": "2026/DM1125",
|
||||||
|
"paper_id": 2783475,
|
||||||
|
"track": "research",
|
||||||
|
"source_row_sha256": "4e981605cf471a5ea29bb3ba3a676784cea55f7777a681eb5fe267ff9c83fa4a",
|
||||||
|
"source_abstract": null,
|
||||||
|
"work_anchor": {
|
||||||
|
"scheme": "arxiv",
|
||||||
|
"value": "2608.29635"
|
||||||
|
},
|
||||||
|
"independent_source": "https://arxiv.org/abs/2608.29635",
|
||||||
|
"existing_history": [
|
||||||
|
{
|
||||||
|
"id": 132984,
|
||||||
|
"venue": "ICLR",
|
||||||
|
"year": 2026,
|
||||||
|
"observations": [
|
||||||
|
{
|
||||||
|
"id": 134279,
|
||||||
|
"status": "rejected"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 539402,
|
||||||
|
"status": "rejected"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"original_icdm_decision_verified": false,
|
||||||
|
"imported": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/DM790",
|
||||||
|
"paper_id": 2485633,
|
||||||
|
"track": "applied",
|
||||||
|
"source_row_sha256": "14e03ca3833b267d6f677325f4e4b9295d1fa42cc77be6707b9bca745444888b",
|
||||||
|
"source_abstract": null,
|
||||||
|
"work_anchor": {
|
||||||
|
"scheme": "arxiv",
|
||||||
|
"value": "2509.19774"
|
||||||
|
},
|
||||||
|
"independent_source": "https://arxiv.org/abs/2509.19774",
|
||||||
|
"existing_history": [
|
||||||
|
{
|
||||||
|
"id": 138980,
|
||||||
|
"venue": "ICLR",
|
||||||
|
"year": 2026,
|
||||||
|
"observations": [
|
||||||
|
{
|
||||||
|
"id": 545219,
|
||||||
|
"status": "withdrawn"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"original_icdm_decision_verified": false,
|
||||||
|
"imported": false
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/DM1157",
|
||||||
|
"paper_id": 3286364,
|
||||||
|
"track": "research",
|
||||||
|
"source_row_sha256": "22ed027f29a161bc978b3c33e921b42c67d999d520eed49ec46ce5935a6f2f2a",
|
||||||
|
"source_abstract": null,
|
||||||
|
"work_anchor": {
|
||||||
|
"scheme": "openreview_snapshot",
|
||||||
|
"value": "2026/1dGzRTSLqL"
|
||||||
|
},
|
||||||
|
"independent_source": "https://dm.cs.univie.ac.at/news/story/deepsync-paper-accepted-at-icdm-2026",
|
||||||
|
"existing_history": [
|
||||||
|
{
|
||||||
|
"id": 127709,
|
||||||
|
"venue": "ICLR",
|
||||||
|
"year": 2026,
|
||||||
|
"observations": [
|
||||||
|
{
|
||||||
|
"id": 129094,
|
||||||
|
"status": "rejected"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"id": 534293,
|
||||||
|
"status": "rejected"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"original_icdm_decision_verified": false,
|
||||||
|
"imported": false
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"held": [
|
||||||
|
{
|
||||||
|
"external_id": "2026/DM1791",
|
||||||
|
"paper_id": 3288747,
|
||||||
|
"reason": "DomainBridging has name-order/format differences and no arxiv self-accepted link; token-bag retrieval is not author or work identity approval."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/DM1801",
|
||||||
|
"paper_id": 3288866,
|
||||||
|
"reason": "LightGFC author lists substantially differ; hold version/work review, no automatic author aliases or merge."
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/DM1413",
|
||||||
|
"paper_id": 3292171,
|
||||||
|
"reason": "OFS author lists differ including JianchengLv vs JindiLv; not treated as aliases; hold independent version/work verification."
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"publisher_catalog_status": "No 2026 IEEE publisher catalogue verified in this check",
|
||||||
|
"public_negative_availability": "unverified",
|
||||||
|
"next": "Use the existing oral source denominator by track. Compare the three reviewed candidates with the fresh import baseline and preserve their earlier attempts. Keep the poster-only rows as a separate source inventory, and resolve the title disagreement before linking that poster."
|
||||||
|
}
|
||||||
@@ -0,0 +1,204 @@
|
|||||||
|
{
|
||||||
|
"checked_on": "2026-11-18",
|
||||||
|
"venue": "IROS",
|
||||||
|
"year": 2026,
|
||||||
|
"conference_program": {
|
||||||
|
"url": "https://2026.ieee-iros.org/program/paper_index/",
|
||||||
|
"status": "unavailable_in_checked_environment",
|
||||||
|
"paper_level_full_index_verified": false
|
||||||
|
},
|
||||||
|
"publisher_registration_check": {
|
||||||
|
"status": "no_target_container_in_first_25_relevance_results",
|
||||||
|
"rows_checked": 25,
|
||||||
|
"observed_container": "2026 IEEE 13th International Conference on Intelligent Systems (IS)",
|
||||||
|
"complete_registry_search": false
|
||||||
|
},
|
||||||
|
"journal_presentations": {
|
||||||
|
"journal": "IEEE Transactions on Robotics",
|
||||||
|
"scope": "IROS 2026 conference presentation of journal papers",
|
||||||
|
"rendered_index_and_embedded_entries_match": true,
|
||||||
|
"count": 30,
|
||||||
|
"xplore_document_ids": [
|
||||||
|
"10989573",
|
||||||
|
"11251175",
|
||||||
|
"11488837",
|
||||||
|
"11268969",
|
||||||
|
"11098164",
|
||||||
|
"11219220",
|
||||||
|
"11304178",
|
||||||
|
"11267071",
|
||||||
|
"11304172",
|
||||||
|
"11455384",
|
||||||
|
"11397863",
|
||||||
|
"11288096",
|
||||||
|
"11544114",
|
||||||
|
"11534108",
|
||||||
|
"11488912",
|
||||||
|
"11535315",
|
||||||
|
"11342351",
|
||||||
|
"11353077",
|
||||||
|
"11429535",
|
||||||
|
"11364144",
|
||||||
|
"11427341",
|
||||||
|
"11427344",
|
||||||
|
"11455389",
|
||||||
|
"11488904",
|
||||||
|
"11535317",
|
||||||
|
"11554436",
|
||||||
|
"11495270",
|
||||||
|
"11575122",
|
||||||
|
"11488843",
|
||||||
|
"11581355"
|
||||||
|
],
|
||||||
|
"conference_submission_decision_verified": false,
|
||||||
|
"publication_dois_verified": false
|
||||||
|
},
|
||||||
|
"author_reported_conference_acceptance_candidates": [
|
||||||
|
{
|
||||||
|
"arxiv": "2607.06499",
|
||||||
|
"acceptance_evidence": [
|
||||||
|
"uw-roundup.html",
|
||||||
|
"uw-faculty.html"
|
||||||
|
],
|
||||||
|
"author_count_current": 3,
|
||||||
|
"work_identity_status": "existing_work_binding_verified_read_only",
|
||||||
|
"database_paper_id": 2731624,
|
||||||
|
"ordered_authors_match": true,
|
||||||
|
"arxiv_versions_stored": [
|
||||||
|
"v1"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"arxiv": "2505.04999",
|
||||||
|
"acceptance_evidence": [
|
||||||
|
"usc-acceptance.html"
|
||||||
|
],
|
||||||
|
"author_list_evidence": "usc-clam-project.html",
|
||||||
|
"author_count_current": 7,
|
||||||
|
"work_identity_status": "existing_work_binding_verified_read_only",
|
||||||
|
"database_paper_id": 2381843,
|
||||||
|
"ordered_authors_match": true,
|
||||||
|
"arxiv_versions_stored": [
|
||||||
|
"v1",
|
||||||
|
"v2"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"acceptance_evidence": [
|
||||||
|
"ntt-contributed.html"
|
||||||
|
],
|
||||||
|
"author_count_current": 6,
|
||||||
|
"work_identity_status": "stable_identifier_and_track_pending"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"purdue_scope_note": {
|
||||||
|
"evidence": "purdue-mars.html",
|
||||||
|
"reported_combined_count": 10,
|
||||||
|
"reported_tro_subset": 2
|
||||||
|
},
|
||||||
|
"public_rejected_desk_rejected_withdrawn": "not_located_in_checked_public_sources",
|
||||||
|
"coverage_status": "partial_source_audit",
|
||||||
|
"database_written": false,
|
||||||
|
"receipts": [
|
||||||
|
{
|
||||||
|
"name": "ieee-tro-list.html",
|
||||||
|
"url": "https://www.ieee-ras.org/publications/t-ro/t-ro-list-of-conference-papers/",
|
||||||
|
"final_url": "https://www.ieee-ras.org/publications/t-ro/t-ro-list-of-conference-papers/",
|
||||||
|
"status": 200,
|
||||||
|
"bytes": 610713,
|
||||||
|
"sha256": "2f2bb466446c62c37dc0927f8df45b161a096cbf8ab1aeffc4f43fa5d27e6358"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "ieee-tro-policy.html",
|
||||||
|
"url": "https://www.ieee-ras.org/publications/t-ro/",
|
||||||
|
"final_url": "https://www.ieee-ras.org/publications/t-ro/",
|
||||||
|
"status": 200,
|
||||||
|
"bytes": 566001,
|
||||||
|
"sha256": "37b8877babb62e69d61e8b32193378afad4372117025a139cf70ffc26fcada5c"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "ntt-contributed.html",
|
||||||
|
"url": "https://group.ntt/en/topics/2026/09/28/iros2026.html",
|
||||||
|
"final_url": "https://group.ntt/en/topics/2026/09/28/iros2026.html",
|
||||||
|
"status": 200,
|
||||||
|
"bytes": 221293,
|
||||||
|
"sha256": "ade0a0320a83e1d4504ff5da24c9f495aed581cffa01aa38b1e4cbec331f6dbc"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "crossref-proceedings-query.json",
|
||||||
|
"url": "https://api.crossref.org/works?query.container-title=2026+IEEE%2FRSJ+International+Conference+on+Intelligent+Robots+and+Systems+%28IROS%29&filter=prefix%3A10.1109%2Ctype%3Aproceedings-article%2Cfrom-pub-date%3A2026-01-01%2Cuntil-pub-date%3A2026-12-31&rows=25&select=DOI%2Ctitle%2Cevent%2Ccontainer-title%2Cpublisher%2Cpublished%2CURL",
|
||||||
|
"final_url": "https://api.crossref.org/works?query.container-title=2026+IEEE%2FRSJ+International+Conference+on+Intelligent+Robots+and+Systems+%28IROS%29&filter=prefix%3A10.1109%2Ctype%3Aproceedings-article%2Cfrom-pub-date%3A2026-01-01%2Cuntil-pub-date%3A2026-12-31&rows=25&select=DOI%2Ctitle%2Cevent%2Ccontainer-title%2Cpublisher%2Cpublished%2CURL",
|
||||||
|
"status": 200,
|
||||||
|
"bytes": 13634,
|
||||||
|
"sha256": "415bf105de8006a42d3f2f8e6eb0078eb54c51bd65315296c0735e853431b228"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "purdue-mars.html",
|
||||||
|
"url": "https://engineering.purdue.edu/IE/news/2026/she-iros-mars-lab",
|
||||||
|
"final_url": "https://engineering.purdue.edu/IE/news/2026/she-iros-mars-lab",
|
||||||
|
"status": 200,
|
||||||
|
"bytes": 60386,
|
||||||
|
"sha256": "fae6ba9637ea141ff3fda93643b97ddfb65d6ea3689f1bca3580956272b2d373"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "uw-acceptance.html",
|
||||||
|
"url": "https://faculty.washington.edu/chx/post/2026-06-17-iros-mppi/",
|
||||||
|
"status": 404,
|
||||||
|
"error": "HTTP Error 404: "
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "usc-acceptance.html",
|
||||||
|
"url": "https://liralab.usc.edu/news/",
|
||||||
|
"final_url": "https://liralab.usc.edu/news/",
|
||||||
|
"status": 200,
|
||||||
|
"bytes": 25381,
|
||||||
|
"sha256": "9fa6b272b7995448a4318dd91cc94b93eeb514ddaa92d7df6880b3d93d91fceb"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "arxiv-2607.06499.html",
|
||||||
|
"url": "https://arxiv.org/abs/2607.06499",
|
||||||
|
"final_url": "https://arxiv.org/abs/2607.06499",
|
||||||
|
"status": 200,
|
||||||
|
"bytes": 43362,
|
||||||
|
"sha256": "98fcbcaf39832f3002fa2910f69ca650481bb473cdb450290e5f9ca9245646a2"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "arxiv-2505.04999.html",
|
||||||
|
"url": "https://arxiv.org/abs/2505.04999",
|
||||||
|
"final_url": "https://arxiv.org/abs/2505.04999",
|
||||||
|
"status": 200,
|
||||||
|
"bytes": 42506,
|
||||||
|
"sha256": "26ea0fd9ac8753b911ff97f1aaf29ea495f50b5c1e106b33c1336ea87c03b0df"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "uw-faculty.html",
|
||||||
|
"url": "https://www.me.washington.edu/facultyfinder/xu-chen",
|
||||||
|
"final_url": "https://www.me.washington.edu/facultyfinder/xu-chen",
|
||||||
|
"status": 200,
|
||||||
|
"bytes": 99274,
|
||||||
|
"sha256": "877af368b564ba26a62b1ce4f947b439cceada2490dfb446b8730ba132c72c78"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "uw-roundup.html",
|
||||||
|
"url": "https://faculty.washington.edu/chx/post/2026-09-07-new-papers-2026/",
|
||||||
|
"final_url": "https://faculty.washington.edu/chx/post/2026-09-07-new-papers-2026/",
|
||||||
|
"status": 200,
|
||||||
|
"bytes": 180954,
|
||||||
|
"sha256": "bc9ef19a07ea2875f8cfffe9afd62bcc2998fbe7c4bc8a4440c1bc87bae3856b"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "usc-clam-project.html",
|
||||||
|
"url": "https://clamrobot.github.io/",
|
||||||
|
"final_url": "https://clamrobot.github.io/",
|
||||||
|
"status": 200,
|
||||||
|
"bytes": 20098,
|
||||||
|
"sha256": "3b3ea3f817ce6483807ace9b5a42ccf9835b811837346b63411190316fe2819c"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"work_identity_review": {
|
||||||
|
"snapshot_at": "2026-11-18 09:26:04.582744+00",
|
||||||
|
"evidence_sha256": "5e5cfb57525a369e8c81e1a9ea6edeb4f5e83a33e1961eeead8a014e3d59ad6d",
|
||||||
|
"existing_work_matches": 2,
|
||||||
|
"database_written": false
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
# IROS 2026 来源核查
|
||||||
|
|
||||||
|
2026-11-18 核查了会议节目、期刊报告名单、作者机构公布的录用信息和 DOI 登记线索。逐项来源、响应哈希与文献定位符见[机器可读清单](iros2026-source-audit-2026-11-18.json)。本次来源核查未写入数据库。
|
||||||
|
|
||||||
|
## 期刊论文在会上报告
|
||||||
|
|
||||||
|
IEEE RAS 的 T-RO 名单在 IROS 2026 一节列出 30 个不同的 IEEE Xplore 文献编号。页面渲染内容和内嵌数据的编号及顺序完全一致。T-RO 政策允许已录用的期刊文章到 IROS 报告。后续应保留期刊出版与会议报告两种关系,并在确认期刊 DOI 后核对现有论文身份。本清单没有逐条验证 DOI 或会议投稿决定。[T-RO 报告名单](https://www.ieee-ras.org/publications/t-ro/t-ro-list-of-conference-papers/) · [T-RO 报告政策](https://www.ieee-ras.org/publications/t-ro/)
|
||||||
|
|
||||||
|
普渡大学的通告将团队的 10 项成果一同介绍,其中 2 项属于 T-RO。这个团队样本可以帮助核查来源口径,暂不用于生成逐篇录用记录。[普渡大学通告](https://engineering.purdue.edu/IE/news/2026/she-iros-mars-lab)
|
||||||
|
|
||||||
|
## 作者机构公布的录用线索
|
||||||
|
|
||||||
|
- 华盛顿大学的作者通告列出 IROS 2026 待出版论文,标题与 arXiv 2607.06499 对应,三位作者一致。只读查询确认对应论文 2731624,库内标题与三位作者的顺序一致。[作者通告](https://faculty.washington.edu/chx/post/2026-09-07-new-papers-2026/) · [arXiv](https://arxiv.org/abs/2607.06499)
|
||||||
|
- USC 实验室确认 CLAM 在 IROS 2026 录用。项目页指向 arXiv 2505.04999,当前作者名单为七人。库内 arXiv 保留 v1、v2,现有论文 2381843 的标题和七位作者顺序与当前详情页一致。[实验室通告](https://liralab.usc.edu/news/) · [作者项目页](https://clamrobot.github.io/) · [arXiv](https://arxiv.org/abs/2505.04999)
|
||||||
|
- NTT 公布一条题名及六位作者,待核实稳定标识和会议 track。[NTT 通告](https://group.ntt/en/topics/2026/09/28/iros2026.html)
|
||||||
|
|
||||||
|
这两条工作身份已核实,尚未录入 IROS 录用证据。
|
||||||
|
|
||||||
|
## 待补证据
|
||||||
|
|
||||||
|
[会议论文索引](https://2026.ieee-iros.org/program/paper_index/)在受检环境未能读取。一次限定 IEEE、2026 年及会议论文类型的 Crossref 相关性查询返回的前 25 条属于另一会议。实际会议论文集 DOI、完整出版清单、正式分轨和逐篇覆盖仍待核查。已检查的公开来源中未找到逐篇拒稿、桌拒或撤回清单,继续保持这些决定的公开可得性待查。
|
||||||
@@ -0,0 +1,247 @@
|
|||||||
|
{
|
||||||
|
"as_of": "2026-11-18",
|
||||||
|
"goal_progress": "13/17; stage14 incomplete",
|
||||||
|
"source": "ismir_program",
|
||||||
|
"scope": "First10 reviewed existing-work source links, partial10/140, not public-rejection completeness or final deduplication",
|
||||||
|
"database_acceptance": {
|
||||||
|
"status": "passed",
|
||||||
|
"at": "2026-11-18 05:15:27.565655+00:00",
|
||||||
|
"writer_commit": "12f95cff91813a2143bc8fab5bc3e89df1bd0fdb",
|
||||||
|
"before_counts": {
|
||||||
|
"papers": {
|
||||||
|
"rows": 3568022,
|
||||||
|
"max_id": 3568595
|
||||||
|
},
|
||||||
|
"source_records": {
|
||||||
|
"rows": 3815503,
|
||||||
|
"max_id": 3815504
|
||||||
|
},
|
||||||
|
"submissions": {
|
||||||
|
"rows": 532192,
|
||||||
|
"max_id": 532193
|
||||||
|
},
|
||||||
|
"decisions": {
|
||||||
|
"rows": 571744,
|
||||||
|
"max_id": 571744
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"after_counts": {
|
||||||
|
"papers": {
|
||||||
|
"rows": 3568022,
|
||||||
|
"max_id": 3568595
|
||||||
|
},
|
||||||
|
"source_records": {
|
||||||
|
"rows": 3815513,
|
||||||
|
"max_id": 3815514
|
||||||
|
},
|
||||||
|
"submissions": {
|
||||||
|
"rows": 532202,
|
||||||
|
"max_id": 532203
|
||||||
|
},
|
||||||
|
"decisions": {
|
||||||
|
"rows": 571754,
|
||||||
|
"max_id": 571754
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"new_papers": 0,
|
||||||
|
"new_sources": 1,
|
||||||
|
"new_editions": 1,
|
||||||
|
"new_source_records": 10,
|
||||||
|
"new_attempts": 10,
|
||||||
|
"new_decisions": 10,
|
||||||
|
"source_id": 123160,
|
||||||
|
"source_tag_evidence_links": 94,
|
||||||
|
"protected_papers": 6394,
|
||||||
|
"all_protected_old_profiles_sources_tags_identifiers_preserved": true,
|
||||||
|
"source_observations": 10,
|
||||||
|
"source_expected": 140,
|
||||||
|
"source_remaining": 130,
|
||||||
|
"coverage": "partial",
|
||||||
|
"foundry_readyz": [
|
||||||
|
200
|
||||||
|
],
|
||||||
|
"sqlite_frozen": true,
|
||||||
|
"imported": [
|
||||||
|
{
|
||||||
|
"external_id": "2026/3",
|
||||||
|
"paper_id": 2813483,
|
||||||
|
"record_id": 3815505,
|
||||||
|
"submission_id": 532194,
|
||||||
|
"decision_id": 571745,
|
||||||
|
"edition_id": 104044
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/12",
|
||||||
|
"paper_id": 2785459,
|
||||||
|
"record_id": 3815506,
|
||||||
|
"submission_id": 532195,
|
||||||
|
"decision_id": 571746,
|
||||||
|
"edition_id": 104044
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/35",
|
||||||
|
"paper_id": 2757859,
|
||||||
|
"record_id": 3815507,
|
||||||
|
"submission_id": 532196,
|
||||||
|
"decision_id": 571747,
|
||||||
|
"edition_id": 104044
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/51",
|
||||||
|
"paper_id": 2752643,
|
||||||
|
"record_id": 3815508,
|
||||||
|
"submission_id": 532197,
|
||||||
|
"decision_id": 571748,
|
||||||
|
"edition_id": 104044
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/74",
|
||||||
|
"paper_id": 2762340,
|
||||||
|
"record_id": 3815509,
|
||||||
|
"submission_id": 532198,
|
||||||
|
"decision_id": 571749,
|
||||||
|
"edition_id": 104044
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/75",
|
||||||
|
"paper_id": 2757480,
|
||||||
|
"record_id": 3815510,
|
||||||
|
"submission_id": 532199,
|
||||||
|
"decision_id": 571750,
|
||||||
|
"edition_id": 104044
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/78",
|
||||||
|
"paper_id": 2768260,
|
||||||
|
"record_id": 3815511,
|
||||||
|
"submission_id": 532200,
|
||||||
|
"decision_id": 571751,
|
||||||
|
"edition_id": 104044
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/84",
|
||||||
|
"paper_id": 2740614,
|
||||||
|
"record_id": 3815512,
|
||||||
|
"submission_id": 532201,
|
||||||
|
"decision_id": 571752,
|
||||||
|
"edition_id": 104044
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/87",
|
||||||
|
"paper_id": 2752271,
|
||||||
|
"record_id": 3815513,
|
||||||
|
"submission_id": 532202,
|
||||||
|
"decision_id": 571753,
|
||||||
|
"edition_id": 104044
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/363",
|
||||||
|
"paper_id": 2757497,
|
||||||
|
"record_id": 3815514,
|
||||||
|
"submission_id": 532203,
|
||||||
|
"decision_id": 571754,
|
||||||
|
"edition_id": 104044
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"history_protection": {
|
||||||
|
"submissions": {
|
||||||
|
"rows": 532192,
|
||||||
|
"sha256": "34e2fe27cccf89e6e80228318212a846369fdd315c4cfe67d7d535013eac8298"
|
||||||
|
},
|
||||||
|
"decisions": {
|
||||||
|
"rows": 571744,
|
||||||
|
"sha256": "efbda4e91950c5c7298f536bc5967f7fee57d8b20a7843e00dc69871b1553bd1"
|
||||||
|
},
|
||||||
|
"decision_sources": {
|
||||||
|
"rows": 571744,
|
||||||
|
"sha256": "1d8899b16b40e9f37ac84dc601efc3e7906af5c06bca49946100697f22c96174"
|
||||||
|
},
|
||||||
|
"submission_sources": {
|
||||||
|
"rows": 581976,
|
||||||
|
"sha256": "ab3ab200c4abbfef39bc5a2a757550191441c703ae505372d38d9afa9a4daf65"
|
||||||
|
},
|
||||||
|
"submission_identifiers": {
|
||||||
|
"rows": 383956,
|
||||||
|
"sha256": "0e8f514e186e08bd00a65f8e14fc7f63db6add76722271e0111e14f5d27b9dbe"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"evidence": {
|
||||||
|
"accepted_report": "/data/paper-index/collected/ismir2026-import10-v1/postgresql-postcheck-v1.json",
|
||||||
|
"accepted_report_sha256": "8a08015bbd725b400de53477837e850ca92a50479d7f86f2cfed56d50bb22577",
|
||||||
|
"baseline": "/data/paper-index/collected/ismir2026-baseline10-v1/baseline.json",
|
||||||
|
"baseline_sha256": "a0b5b8c2c21ee567cf80152b16f66643309bbe33ee9cd67d19da33d6b6337656",
|
||||||
|
"baseline_archive_sha256": "eb94f99cb53c9a30f63d34ea11b23c11759e45b571b7efce2dcbe8cb4fe52fb8",
|
||||||
|
"preparation_sha256": "0d9fad74640ab57fc8bea8aaf72ba07202a40b174d05253db7348b14455a0462",
|
||||||
|
"acceptance_archive": "/data/paper-index/validation/ismir2026-import10-acceptance-recovery-v1.tar.gz",
|
||||||
|
"acceptance_archive_sha256": "138e1fb986f5aa6a42794eed982fdc91ae994b8be82f8bf25360db2f7ba6c70c",
|
||||||
|
"fresh_packets_sha256": "88e423aede0ea83b9f736891c09b12d0cb24f2f8b70508e4863cd07760de5e0d",
|
||||||
|
"local_acceptance_check_sha256": "42d31337c76b96f93505011db7c797123b67ce2eec7defd0175ea0e9c1df2089",
|
||||||
|
"census_summary_sha256": "2157035b72d351e2fe91f2bc0c6ebb305f91d95f8772ab9a088076ce4b86e1c2"
|
||||||
|
},
|
||||||
|
"native_query_example": {
|
||||||
|
"paper_id": 2757497,
|
||||||
|
"publication_outcome": "accepted",
|
||||||
|
"prior_venue": "ICLR",
|
||||||
|
"prior_year": 2026,
|
||||||
|
"prior_submission_id": 131721,
|
||||||
|
"prior_negative_attempts": 1,
|
||||||
|
"prior_negative_observations": 2,
|
||||||
|
"prior_decision_ids": [
|
||||||
|
133086,
|
||||||
|
538172
|
||||||
|
],
|
||||||
|
"later_observed_venue": "ISMIR",
|
||||||
|
"later_observed_year": 2026,
|
||||||
|
"original_ismir_signed_decision_verified": false,
|
||||||
|
"ismir_decision_date": null,
|
||||||
|
"observation_time_is_not_decision_time": true
|
||||||
|
},
|
||||||
|
"presence_census": {
|
||||||
|
"snapshot_at": "2026-11-18 05:20:48.641315+00",
|
||||||
|
"scope": "Fixed278 venue-year presence only, not paper/track/abstract/public-negative completeness or final deduplication. Legacy43 gaps deferred, not filled.",
|
||||||
|
"target_venues": 43,
|
||||||
|
"expected_slots": 278,
|
||||||
|
"populated_slots": 274,
|
||||||
|
"missing_slots": 4,
|
||||||
|
"missing": [
|
||||||
|
{
|
||||||
|
"venue": "CIKM",
|
||||||
|
"year": 2026
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"venue": "ECCV",
|
||||||
|
"year": 2026
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"venue": "ICDM",
|
||||||
|
"year": 2026
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"venue": "IROS",
|
||||||
|
"year": 2026
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"delta_from_v20": {
|
||||||
|
"newly_present": [
|
||||||
|
{
|
||||||
|
"venue": "ISMIR",
|
||||||
|
"year": 2026
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"previous_missing": 5,
|
||||||
|
"current_missing": 4
|
||||||
|
},
|
||||||
|
"snapshot_sha256": "c14bab883c5fcab33b450fcf240dec7225f2f17bc04c3e728b5583657b4ad23b",
|
||||||
|
"fixed_scope_sha256": "a835bef5b9c3a0f76069d840cecdb9a6bcf65d3a38796d70dbfdfd4b49242264",
|
||||||
|
"accepted_import_sha256": "8a08015bbd725b400de53477837e850ca92a50479d7f86f2cfed56d50bb22577",
|
||||||
|
"database_written": false
|
||||||
|
},
|
||||||
|
"consumed_queue": "/data/paper-index/collected/ismir2026-prepared10-v1: NEVER REPLAY",
|
||||||
|
"next": [
|
||||||
|
"Review130 remaining source observations, including31 total verbatim-abstract candidates of which10 consumed; do not count a title/author screen as identity approval.",
|
||||||
|
"Fresh incremental parent must be partial10/140; old preparation0/140 is consumed.",
|
||||||
|
"Continue all venue/year/track public negative availability and formal source coverage.",
|
||||||
|
"Finish cross-venue identities/types, then final whole-DB dedup/identifier/abstract/constraints/query audit after imports."
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,330 @@
|
|||||||
|
{
|
||||||
|
"as_of": "2026-11-18",
|
||||||
|
"goal_progress": "13/17; stage14 incomplete",
|
||||||
|
"source": "ismir_program",
|
||||||
|
"scope": "21 additional reviewed existing-work source links; cumulativepartial31/140, not public-rejection completeness or final deduplication",
|
||||||
|
"database_acceptance": {
|
||||||
|
"status": "passed",
|
||||||
|
"at": "2026-11-18 06:38:55.148539+00:00",
|
||||||
|
"writer_commit": "3754545003e1f96b88c6533cc3a69dc557f07305",
|
||||||
|
"before_counts": {
|
||||||
|
"papers": {
|
||||||
|
"rows": 3568022,
|
||||||
|
"max_id": 3568595
|
||||||
|
},
|
||||||
|
"source_records": {
|
||||||
|
"rows": 3815513,
|
||||||
|
"max_id": 3815514
|
||||||
|
},
|
||||||
|
"submissions": {
|
||||||
|
"rows": 532202,
|
||||||
|
"max_id": 532203
|
||||||
|
},
|
||||||
|
"decisions": {
|
||||||
|
"rows": 571754,
|
||||||
|
"max_id": 571754
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"after_counts": {
|
||||||
|
"papers": {
|
||||||
|
"rows": 3568022,
|
||||||
|
"max_id": 3568595
|
||||||
|
},
|
||||||
|
"source_records": {
|
||||||
|
"rows": 3815534,
|
||||||
|
"max_id": 3815535
|
||||||
|
},
|
||||||
|
"submissions": {
|
||||||
|
"rows": 532223,
|
||||||
|
"max_id": 532224
|
||||||
|
},
|
||||||
|
"decisions": {
|
||||||
|
"rows": 571775,
|
||||||
|
"max_id": 571775
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"new_papers": 0,
|
||||||
|
"new_sources": 0,
|
||||||
|
"new_editions": 0,
|
||||||
|
"new_source_records": 21,
|
||||||
|
"new_attempts": 21,
|
||||||
|
"new_decisions": 21,
|
||||||
|
"source_id": 123160,
|
||||||
|
"source_tag_evidence_links": 173,
|
||||||
|
"protected_papers": 6415,
|
||||||
|
"all_protected_old_profiles_sources_tags_identifiers_preserved": true,
|
||||||
|
"source_observations": 31,
|
||||||
|
"source_expected": 140,
|
||||||
|
"source_remaining": 109,
|
||||||
|
"coverage": "partial",
|
||||||
|
"foundry_readyz": [
|
||||||
|
200
|
||||||
|
],
|
||||||
|
"sqlite_frozen": true,
|
||||||
|
"imported": [
|
||||||
|
{
|
||||||
|
"external_id": "2026/148",
|
||||||
|
"paper_id": 2755057,
|
||||||
|
"record_id": 3815515,
|
||||||
|
"submission_id": 532204,
|
||||||
|
"decision_id": 571755,
|
||||||
|
"edition_id": 104044
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/184",
|
||||||
|
"paper_id": 2752723,
|
||||||
|
"record_id": 3815516,
|
||||||
|
"submission_id": 532205,
|
||||||
|
"decision_id": 571756,
|
||||||
|
"edition_id": 104044
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/196",
|
||||||
|
"paper_id": 2784571,
|
||||||
|
"record_id": 3815517,
|
||||||
|
"submission_id": 532206,
|
||||||
|
"decision_id": 571757,
|
||||||
|
"edition_id": 104044
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/240",
|
||||||
|
"paper_id": 2738896,
|
||||||
|
"record_id": 3815518,
|
||||||
|
"submission_id": 532207,
|
||||||
|
"decision_id": 571758,
|
||||||
|
"edition_id": 104044
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/251",
|
||||||
|
"paper_id": 2753889,
|
||||||
|
"record_id": 3815519,
|
||||||
|
"submission_id": 532208,
|
||||||
|
"decision_id": 571759,
|
||||||
|
"edition_id": 104044
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/258",
|
||||||
|
"paper_id": 2662030,
|
||||||
|
"record_id": 3815520,
|
||||||
|
"submission_id": 532209,
|
||||||
|
"decision_id": 571760,
|
||||||
|
"edition_id": 104044
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/270",
|
||||||
|
"paper_id": 2769109,
|
||||||
|
"record_id": 3815521,
|
||||||
|
"submission_id": 532210,
|
||||||
|
"decision_id": 571761,
|
||||||
|
"edition_id": 104044
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/298",
|
||||||
|
"paper_id": 2752114,
|
||||||
|
"record_id": 3815522,
|
||||||
|
"submission_id": 532211,
|
||||||
|
"decision_id": 571762,
|
||||||
|
"edition_id": 104044
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/317",
|
||||||
|
"paper_id": 2765378,
|
||||||
|
"record_id": 3815523,
|
||||||
|
"submission_id": 532212,
|
||||||
|
"decision_id": 571763,
|
||||||
|
"edition_id": 104044
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/318",
|
||||||
|
"paper_id": 2764935,
|
||||||
|
"record_id": 3815524,
|
||||||
|
"submission_id": 532213,
|
||||||
|
"decision_id": 571764,
|
||||||
|
"edition_id": 104044
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/342",
|
||||||
|
"paper_id": 2738585,
|
||||||
|
"record_id": 3815525,
|
||||||
|
"submission_id": 532214,
|
||||||
|
"decision_id": 571765,
|
||||||
|
"edition_id": 104044
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/370",
|
||||||
|
"paper_id": 2830871,
|
||||||
|
"record_id": 3815526,
|
||||||
|
"submission_id": 532215,
|
||||||
|
"decision_id": 571766,
|
||||||
|
"edition_id": 104044
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/371",
|
||||||
|
"paper_id": 2858445,
|
||||||
|
"record_id": 3815527,
|
||||||
|
"submission_id": 532216,
|
||||||
|
"decision_id": 571767,
|
||||||
|
"edition_id": 104044
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/373",
|
||||||
|
"paper_id": 2820562,
|
||||||
|
"record_id": 3815528,
|
||||||
|
"submission_id": 532217,
|
||||||
|
"decision_id": 571768,
|
||||||
|
"edition_id": 104044
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/409",
|
||||||
|
"paper_id": 2522863,
|
||||||
|
"record_id": 3815529,
|
||||||
|
"submission_id": 532218,
|
||||||
|
"decision_id": 571769,
|
||||||
|
"edition_id": 104044
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/418",
|
||||||
|
"paper_id": 2750749,
|
||||||
|
"record_id": 3815530,
|
||||||
|
"submission_id": 532219,
|
||||||
|
"decision_id": 571770,
|
||||||
|
"edition_id": 104044
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/468",
|
||||||
|
"paper_id": 2846824,
|
||||||
|
"record_id": 3815531,
|
||||||
|
"submission_id": 532220,
|
||||||
|
"decision_id": 571771,
|
||||||
|
"edition_id": 104044
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/518",
|
||||||
|
"paper_id": 2761686,
|
||||||
|
"record_id": 3815532,
|
||||||
|
"submission_id": 532221,
|
||||||
|
"decision_id": 571772,
|
||||||
|
"edition_id": 104044
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/524",
|
||||||
|
"paper_id": 2784543,
|
||||||
|
"record_id": 3815533,
|
||||||
|
"submission_id": 532222,
|
||||||
|
"decision_id": 571773,
|
||||||
|
"edition_id": 104044
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/539",
|
||||||
|
"paper_id": 2761760,
|
||||||
|
"record_id": 3815534,
|
||||||
|
"submission_id": 532223,
|
||||||
|
"decision_id": 571774,
|
||||||
|
"edition_id": 104044
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"external_id": "2026/552",
|
||||||
|
"paper_id": 2758575,
|
||||||
|
"record_id": 3815535,
|
||||||
|
"submission_id": 532224,
|
||||||
|
"decision_id": 571775,
|
||||||
|
"edition_id": 104044
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"history_protection": {
|
||||||
|
"submissions": {
|
||||||
|
"rows": 532202,
|
||||||
|
"sha256": "9d41955c80fc91dae4f65d65e0f69198ce2d79508ceebfe943bdc19825983df8"
|
||||||
|
},
|
||||||
|
"decisions": {
|
||||||
|
"rows": 571754,
|
||||||
|
"sha256": "c8dc762d089d1656ee1308dacec9109605e1a4be7e5fcfa8f95013e77a0cb55c"
|
||||||
|
},
|
||||||
|
"decision_sources": {
|
||||||
|
"rows": 571754,
|
||||||
|
"sha256": "231f15b4c70435977d6ab784112ff5c748541eeb4d54c28bb861f8835f1eba10"
|
||||||
|
},
|
||||||
|
"submission_sources": {
|
||||||
|
"rows": 581986,
|
||||||
|
"sha256": "b9d8c39e945df340daf1c695a6359cfb549b8bd2f391cb139dabba2c16afa242"
|
||||||
|
},
|
||||||
|
"submission_identifiers": {
|
||||||
|
"rows": 383956,
|
||||||
|
"sha256": "0e8f514e186e08bd00a65f8e14fc7f63db6add76722271e0111e14f5d27b9dbe"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"evidence": {
|
||||||
|
"parent_acceptance_sha256": "8a08015bbd725b400de53477837e850ca92a50479d7f86f2cfed56d50bb22577",
|
||||||
|
"accepted_report": "/data/paper-index/collected/ismir2026-import21-v1/postgresql-postcheck-v1.json",
|
||||||
|
"accepted_report_sha256": "153ef13dbac9f4243c81bdd2a16d3ade74372d3e45ca5e84acc2b7258bca7bfc",
|
||||||
|
"baseline": "/data/paper-index/collected/ismir2026-baseline21-v1/baseline.json",
|
||||||
|
"baseline_sha256": "204b571b47e164d68930158c2578f609814969940628d421a2b64c3a6ce1b018",
|
||||||
|
"baseline_archive_sha256": "ecac32e1d3890786a5670ff06f88fa7bd92155306cf8133f3164de0cb9f48e6b",
|
||||||
|
"preparation_sha256": "6781edfaa67798d76329fb5c7948d83693607c96352a38eccce0067faa95deaa",
|
||||||
|
"acceptance_archive": "/data/paper-index/validation/ismir2026-import21-acceptance-recovery-v1.tar.gz",
|
||||||
|
"acceptance_archive_sha256": "55e1cc8301548ff0abc195add5f40d7082a06e4f087f6da5838ffd2f6b33339c",
|
||||||
|
"fresh_packets_sha256": "79e0e8187a522478c443d036fdbf2f8d5e6a22c949d46fb0592a664ec1bc22fe",
|
||||||
|
"local_acceptance_check_sha256": "04f4ec6969a71655d70f446347646f63efc56e5a5a8e974b931703482e5e7ceb",
|
||||||
|
"census_summary_sha256": "ee4ec2e6d6eaed55ce3475e735bfba9bd94e512893cd3c3024035b48b744f475",
|
||||||
|
"ci_run": 1257,
|
||||||
|
"ci_tests": 556,
|
||||||
|
"ci_proof_sha256": "5ad08aa907caec5a1ca63df5e7345db4802dab4e3a91293a0303bf481a903ff8"
|
||||||
|
},
|
||||||
|
"consumed_queue": "/data/paper-index/collected/ismir2026-prepared21-v2: NEVER REPLAY",
|
||||||
|
"previous_consumed_queue": "/data/paper-index/collected/ismir2026-prepared10-v1: NEVER REPLAY",
|
||||||
|
"query_example_rechecked": {
|
||||||
|
"paper_id": 2757497,
|
||||||
|
"publication_outcome": "accepted",
|
||||||
|
"rejected_attempts": 1,
|
||||||
|
"rejection_source_observations": 2,
|
||||||
|
"distinct_ismir_accepted_attempts": 1,
|
||||||
|
"original_ismir_signed_decision_verified": false,
|
||||||
|
"ismir_decision_date": null
|
||||||
|
},
|
||||||
|
"new_batch_prior_negative_attempts": 0,
|
||||||
|
"new_batch_prior_negative_observations": 0,
|
||||||
|
"source_tag_links_cumulative": 267,
|
||||||
|
"presence_census": {
|
||||||
|
"snapshot_at": "2026-11-18 06:46:44.099675+00",
|
||||||
|
"scope": "Fixed278 venue-year presence only, not paper/track/abstract/public-negative completeness or final deduplication. Legacy43 gaps deferred, not filled.",
|
||||||
|
"target_venues": 43,
|
||||||
|
"expected_slots": 278,
|
||||||
|
"populated_slots": 274,
|
||||||
|
"missing_slots": 4,
|
||||||
|
"missing": [
|
||||||
|
{
|
||||||
|
"venue": "CIKM",
|
||||||
|
"year": 2026
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"venue": "ECCV",
|
||||||
|
"year": 2026
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"venue": "ICDM",
|
||||||
|
"year": 2026
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"venue": "IROS",
|
||||||
|
"year": 2026
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"delta_from_v21": {
|
||||||
|
"newly_present": [],
|
||||||
|
"previous_missing": 4,
|
||||||
|
"current_missing": 4,
|
||||||
|
"ISMIR_source_observations": 31
|
||||||
|
},
|
||||||
|
"snapshot_sha256": "5aabc91217ad787946ba50d6a670296b1e3152a54f5ecaa0e3a9b3a952b63fa7",
|
||||||
|
"fixed_scope_sha256": "a835bef5b9c3a0f76069d840cecdb9a6bcf65d3a38796d70dbfdfd4b49242264",
|
||||||
|
"accepted_import_sha256": "153ef13dbac9f4243c81bdd2a16d3ade74372d3e45ca5e84acc2b7258bca7bfc",
|
||||||
|
"database_written": false
|
||||||
|
},
|
||||||
|
"next": [
|
||||||
|
"Remaining109 observations require review;31 exact-source-abstract candidates are now consumed, not reusable queues.",
|
||||||
|
"Future increments require a fresh partial31/140 parent and new preservation baseline.",
|
||||||
|
"Keep raw authors/title/abstract/keyword versions; no numeric miniconf-to-OpenReview-ID assumption, global person aliases or inferred research types.",
|
||||||
|
"Continue all43 venue/year/track formal-source and public-negative availability audits, then final whole-DB dedup after imports."
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,320 @@
|
|||||||
|
{
|
||||||
|
"version": 1,
|
||||||
|
"as_of": "2026-11-18",
|
||||||
|
"scope": "930 cached ResearchArcade non-ICLR negative claims, NeurIPS 2021–2025 and EMNLP 2023; not a conference-completeness audit",
|
||||||
|
"database_snapshot": {
|
||||||
|
"database": "foundry_data",
|
||||||
|
"schema": "research",
|
||||||
|
"at": "2026-11-18 03:48:34.65659+00",
|
||||||
|
"snapshot": "1048885:1048885:"
|
||||||
|
},
|
||||||
|
"database_written": false,
|
||||||
|
"approved_for_import": 0,
|
||||||
|
"claims": 930,
|
||||||
|
"known_same_scope_attempt_claims": 925,
|
||||||
|
"all_925_have_rejected_status": true,
|
||||||
|
"without_same_scope_attempt_claims": 5,
|
||||||
|
"by_venue_year": [
|
||||||
|
{
|
||||||
|
"venue": "EMNLP",
|
||||||
|
"year": 2023,
|
||||||
|
"claims": 10
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"venue": "NeurIPS",
|
||||||
|
"year": 2021,
|
||||||
|
"claims": 136
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"venue": "NeurIPS",
|
||||||
|
"year": 2022,
|
||||||
|
"claims": 153
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"venue": "NeurIPS",
|
||||||
|
"year": 2023,
|
||||||
|
"claims": 176
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"venue": "NeurIPS",
|
||||||
|
"year": 2024,
|
||||||
|
"claims": 201
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"venue": "NeurIPS",
|
||||||
|
"year": 2025,
|
||||||
|
"claims": 254
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"evidence": {
|
||||||
|
"diff_report_sha256": "d49361a8622f3b3f59e824690ceb8bf1c878be35c3e3758366d3b46a407673f5",
|
||||||
|
"candidate_packets_sha256": "3b21df27e0f30e56b81086d16d3943c4563bbe382200e866b15de14510e07472",
|
||||||
|
"dataset_revision": "c61bb99b0452429c32f85b1e392f3afdb65731c4",
|
||||||
|
"dataset_parquet_sha256": "bb6e165ee152a95d91a7ffa16abc9d4a6effbdfcf59183b3bb819d09171689d1",
|
||||||
|
"candidate_jsonl_sha256": "848e646596e8c747e0c636c2381a139978aff8f2c19f01ad49c98e7f4270c5e6",
|
||||||
|
"collector_repository": "https://github.com/ulab-uiuc/research-arcade",
|
||||||
|
"collector_revision": "ecf9c7c2490840b029bd2add4f02d04aca1d014e",
|
||||||
|
"collector_openreview_crawler_sha256": "4539d951ca05b20785169638eab3884148aa6c354a34c5ecc04e15c84a5910fe",
|
||||||
|
"collector_revision_proven_to_generate_dataset": false
|
||||||
|
},
|
||||||
|
"collector_semantics": {
|
||||||
|
"legacy_2020_2022": "Reads Blind_Submission note.id; first reply with content.decision, no invitation/signature verification. Not a verified final-decision note.",
|
||||||
|
"legacy_2019": "Reads first recommendation, which must not be promoted to a final decision. Outside active 2020+ backfill.",
|
||||||
|
"modern_branch": "Reads Submission note.id and last venueid component; Conference replaced by venue literal. Withdrawn_Submission skipped.",
|
||||||
|
"identity": "The exported field paper_openreview_id is note.id; forum equality or equivalence to another mirror ID is not independently proved.",
|
||||||
|
"coverage": "Six-field table has no authors or decision signatures; no completeness inference for public withdrawn/rejected records."
|
||||||
|
},
|
||||||
|
"held_claims": [
|
||||||
|
{
|
||||||
|
"source_row": 33219,
|
||||||
|
"note_id_claim": "FSf9h6t152",
|
||||||
|
"venue": "NeurIPS",
|
||||||
|
"year": 2025,
|
||||||
|
"raw_decision": "Rejected_Submission",
|
||||||
|
"title": "Hierarchical Restructuring of Graph Neural Networks for Robust Learning under Edge Perturbations",
|
||||||
|
"same_scope_identifier_match": false,
|
||||||
|
"exact_title_candidates": [],
|
||||||
|
"approved_for_import": false,
|
||||||
|
"disposition": "hold_no_exact_title_candidate_not_proof_of_novelty"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source_row": 44669,
|
||||||
|
"note_id_claim": "Pk972F7J8Um",
|
||||||
|
"venue": "NeurIPS",
|
||||||
|
"year": 2021,
|
||||||
|
"raw_decision": "Reject",
|
||||||
|
"title": "Differential Privacy of Dirichlet Posterior Sampling",
|
||||||
|
"same_scope_identifier_match": false,
|
||||||
|
"exact_title_candidates": [
|
||||||
|
{
|
||||||
|
"paper_id": 3527429,
|
||||||
|
"existing_identifiers": [
|
||||||
|
{
|
||||||
|
"scheme": "openreview",
|
||||||
|
"value": "fgBxcKcz9ea",
|
||||||
|
"paper_id": 3527429
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"scheme": "papercopilot_openreview",
|
||||||
|
"value": "nips/2021/fgBxcKcz9ea",
|
||||||
|
"paper_id": 3527429
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source_abstract_exact_equal": true,
|
||||||
|
"existing_attempts": [
|
||||||
|
{
|
||||||
|
"id": 444666,
|
||||||
|
"venue": "NeurIPS",
|
||||||
|
"year": 2021,
|
||||||
|
"source_submission_id": "nips/2021/fgBxcKcz9ea",
|
||||||
|
"statuses": [
|
||||||
|
"rejected"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"attempts_truncated": false,
|
||||||
|
"identity_approved": false,
|
||||||
|
"id_alias_approved": false
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"approved_for_import": false,
|
||||||
|
"disposition": "hold_distinct_note_ids_even_with_equal_title_and_abstract"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source_row": 44696,
|
||||||
|
"note_id_claim": "ki4eJ1fSJNq",
|
||||||
|
"venue": "NeurIPS",
|
||||||
|
"year": 2021,
|
||||||
|
"raw_decision": "Reject",
|
||||||
|
"title": "Fitting large mixture models using stochastic component selection",
|
||||||
|
"same_scope_identifier_match": false,
|
||||||
|
"exact_title_candidates": [
|
||||||
|
{
|
||||||
|
"paper_id": 1545058,
|
||||||
|
"existing_identifiers": [
|
||||||
|
{
|
||||||
|
"scheme": "arxiv",
|
||||||
|
"value": "2110.04776",
|
||||||
|
"paper_id": 1545058
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"scheme": "openreview",
|
||||||
|
"value": "ZS394D3djsg",
|
||||||
|
"paper_id": 1545058
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"scheme": "papercopilot_openreview",
|
||||||
|
"value": "nips/2021/ZS394D3djsg",
|
||||||
|
"paper_id": 1545058
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source_abstract_exact_equal": true,
|
||||||
|
"existing_attempts": [
|
||||||
|
{
|
||||||
|
"id": 444652,
|
||||||
|
"venue": "NeurIPS",
|
||||||
|
"year": 2021,
|
||||||
|
"source_submission_id": "nips/2021/ZS394D3djsg",
|
||||||
|
"statuses": [
|
||||||
|
"rejected"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"attempts_truncated": false,
|
||||||
|
"identity_approved": false,
|
||||||
|
"id_alias_approved": false
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"approved_for_import": false,
|
||||||
|
"disposition": "hold_distinct_note_ids_even_with_equal_title_and_abstract"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source_row": 45757,
|
||||||
|
"note_id_claim": "bDP7I2fYwLj",
|
||||||
|
"venue": "NeurIPS",
|
||||||
|
"year": 2021,
|
||||||
|
"raw_decision": "Reject",
|
||||||
|
"title": "WeaveNet for Approximating Assignment Problems",
|
||||||
|
"same_scope_identifier_match": false,
|
||||||
|
"exact_title_candidates": [
|
||||||
|
{
|
||||||
|
"paper_id": 3527390,
|
||||||
|
"existing_identifiers": [
|
||||||
|
{
|
||||||
|
"scheme": "openreview",
|
||||||
|
"value": "2rAiDBJgR_",
|
||||||
|
"paper_id": 3527390
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"scheme": "papercopilot_openreview",
|
||||||
|
"value": "nips/2021/2rAiDBJgR_",
|
||||||
|
"paper_id": 3527390
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"source_abstract_exact_equal": true,
|
||||||
|
"existing_attempts": [
|
||||||
|
{
|
||||||
|
"id": 444580,
|
||||||
|
"venue": "NeurIPS",
|
||||||
|
"year": 2021,
|
||||||
|
"source_submission_id": "nips/2021/2rAiDBJgR_",
|
||||||
|
"statuses": [
|
||||||
|
"rejected"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"attempts_truncated": false,
|
||||||
|
"identity_approved": false,
|
||||||
|
"id_alias_approved": false
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"approved_for_import": false,
|
||||||
|
"disposition": "hold_distinct_note_ids_even_with_equal_title_and_abstract"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source_row": 49327,
|
||||||
|
"note_id_claim": "yB8cQIICqe",
|
||||||
|
"venue": "EMNLP",
|
||||||
|
"year": 2023,
|
||||||
|
"raw_decision": "Desk_Rejected_Submission",
|
||||||
|
"title": "EZ-STANCE: A Large Dataset for Zero-Shot Stance Detection",
|
||||||
|
"same_scope_identifier_match": false,
|
||||||
|
"exact_title_candidates": [],
|
||||||
|
"approved_for_import": false,
|
||||||
|
"disposition": "hold_possible_later_publication_with_changed_title_and_dataset_size_no_identity_or_author_transfer",
|
||||||
|
"later_publication_retrieval_lead": {
|
||||||
|
"url": "https://aclanthology.org/2024.acl-long.838/",
|
||||||
|
"http_status": 200,
|
||||||
|
"bytes": 40557,
|
||||||
|
"response_sha256": "b50dd55c48e31e4c10e97927ce01679369735834aa2c0f8d9f41f8e8e02fc776",
|
||||||
|
"checked_at": "2026-11-18T04:07:03.699767+00:00",
|
||||||
|
"anthology_id": "2024.acl-long.838",
|
||||||
|
"doi": "10.18653/v1/2024.acl-long.838",
|
||||||
|
"title": "EZ-STANCE: A Large Dataset for English Zero-Shot Stance Detection",
|
||||||
|
"venue": "ACL",
|
||||||
|
"year": 2024,
|
||||||
|
"publication_verified": true,
|
||||||
|
"link_to_anonymous_rejection_verified": false,
|
||||||
|
"authors_transferred_to_anonymous_claim": false,
|
||||||
|
"dataset_pairs_published_version": 47316,
|
||||||
|
"dataset_pairs_rejection_mirror_version": 30606,
|
||||||
|
"database_written": false,
|
||||||
|
"current_database_evidence": {
|
||||||
|
"snapshot": {
|
||||||
|
"at": "2026-11-18 04:09:23.720956+00",
|
||||||
|
"snapshot": "1051887:1051887:"
|
||||||
|
},
|
||||||
|
"paper_id": 2131805,
|
||||||
|
"verified_identifiers": [
|
||||||
|
{
|
||||||
|
"scheme": "acl",
|
||||||
|
"value": "2024.acl-long.838",
|
||||||
|
"paper_id": 2131805
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"scheme": "acl_anthology",
|
||||||
|
"value": "2024.acl-long.838",
|
||||||
|
"paper_id": 2131805
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"scheme": "doi",
|
||||||
|
"value": "10.18653/v1/2024.acl-long.838",
|
||||||
|
"paper_id": 2131805
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"existing_attempts": [
|
||||||
|
{
|
||||||
|
"venue": "ACL",
|
||||||
|
"year": 2024,
|
||||||
|
"track": "main",
|
||||||
|
"statuses": [
|
||||||
|
"accepted"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"packet_file_sha256": "de5dfa658177adef87134cf7723afc086bd7395405a31166ff9597b03b3c8af3",
|
||||||
|
"original_rejection_link_verified": false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"access_probes": [
|
||||||
|
{
|
||||||
|
"local_evidence": "researcharcade-neurips2021-original-note-check-v1/report.json",
|
||||||
|
"sha256": "746d752535b516abba43b0e8e6ad901bf39c07da532188dfa97ff911869084ae",
|
||||||
|
"receipt_summary": {},
|
||||||
|
"results": [
|
||||||
|
{
|
||||||
|
"requested_note_id": "Pk972F7J8Um",
|
||||||
|
"status": 403,
|
||||||
|
"url": "https://api.openreview.net/notes?id=Pk972F7J8Um&select=id%2Cforum%2Cinvitation%2Creaders%2Csignatures%2Ccontent.title%2Ccontent.abstract%2Ccontent.venue%2Ccontent.venueid",
|
||||||
|
"sha256": "b0113384cebc9d724284062f776b45044d22450219112342721d8f9683571c08",
|
||||||
|
"checked_at": "2026-11-18T04:01:58.643552+00:00"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"local_evidence": "emnlp2023-primary-public-metadata-probe-v1/notes-probe.json",
|
||||||
|
"sha256": "47799dc4520e113d6a2e7c9ea2f362577bb21e3ad583ee02149a36c21cf6ea9d",
|
||||||
|
"receipt_summary": {
|
||||||
|
"url": "https://api.openreview.net/notes?invitation=EMNLP%2F2023%2FConference%2F-%2FSubmission&limit=1&select=id%2Cnumber%2Cforum%2Cinvitation%2Creaders%2Csignatures%2Ccdate%2Ctcdate%2Cmdate%2Ctmdate%2Ccontent.title%2Ccontent.abstract%2Ccontent.authors%2Ccontent.venue%2Ccontent.venueid%2Ccontent.keywords",
|
||||||
|
"http_status": 403,
|
||||||
|
"response_sha256": "798e964735eb1eff1e4954d5c9e79aad2afd40c3f88dcb82f8ef5bbc55cc0a61",
|
||||||
|
"response_bytes": 546,
|
||||||
|
"at": "2026-11-18T03:38:50.112624+00:00"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"local_evidence": "emnlp2023-primary-public-metadata-probe-v1/group-access.json",
|
||||||
|
"sha256": "8fcb9ccd5615a737c17ef0eeedc9121e85a84ebd63667e83df0bda9ffb83d792",
|
||||||
|
"receipt_summary": {
|
||||||
|
"url": "https://api.openreview.net/groups?id=EMNLP%2F2023%2FConference",
|
||||||
|
"http_status": 200,
|
||||||
|
"response_sha256": "7c0cc75e2e8be1c18dffc96a9ceabea34f0f7810139c0a990a6bb117d66f6456",
|
||||||
|
"response_bytes": 13143,
|
||||||
|
"at": "2026-11-18T03:38:49.929085+00:00"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"access_limit": "NeurIPS 2021 first original note request and EMNLP 2023 source-declared Submission query returned 403. Requests stopped. No alternate API version, endpoint, authentication, or challenge bypass. 403 does not prove zero public records."
|
||||||
|
}
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
# 公开负面来源核查:2026-11-18
|
||||||
|
|
||||||
|
本次检查 ResearchArcade 缓存中 NeurIPS 2021–2025、EMNLP 2023 的 930 条负面声明,不包括 ICLR 或 2020 年以前的补漏。原始候选文件未重新下载,PostgreSQL 查询为只读。逐条待办、来源哈希与数据库快照见[机器可读记录](public-negative-source-audit-2026-11-18.json)。没有新增论文、决定、ID 别名或合并。
|
||||||
|
|
||||||
|
## 当前库比对
|
||||||
|
|
||||||
|
925 条声明通过同会议、同年份的 OpenReview 标识匹配到投稿,且每条都有 rejected 决定。这里只证明对应拒稿已记录,未核验这些论文的所有后续录用历史,也未证明会议公开拒稿全集完整。
|
||||||
|
|
||||||
|
另 5 条没有同届标识匹配:
|
||||||
|
|
||||||
|
|来源 note.id 声明|会议|标题或识别信息|处置|
|
||||||
|
|---|---|---|---|
|
||||||
|
|FSf9h6t152|NeurIPS 2025|Hierarchical Restructuring of Graph Neural Networks for Robust Learning under Edge Perturbations|无精确标题候选;不是已证明的新论文|
|
||||||
|
|Pk972F7J8Um|NeurIPS 2021|Differential Privacy of Dirichlet Posterior Sampling|候选 paper 3527429,已有同届 rejected,原 ID fgBxcKcz9ea|
|
||||||
|
|ki4eJ1fSJNq|NeurIPS 2021|Fitting large mixture models using stochastic component selection|候选 paper 1545058,已有同届 rejected,原 ID ZS394D3djsg;另保留 arXiv 2110.04776|
|
||||||
|
|bDP7I2fYwLj|NeurIPS 2021|WeaveNet for Approximating Assignment Problems|候选 paper 3527390,已有同届 rejected,原 ID 2rAiDBJgR_|
|
||||||
|
|yB8cQIICqe|EMNLP 2023|EZ-STANCE: A Large Dataset for Zero-Shot Stance Detection|镜像称 Desk_Rejected_Submission;原始公开范围及身份未核验|
|
||||||
|
|
||||||
|
三个标题候选的镜像摘要都与已有 PaperCopilot 来源摘要逐字相等,但来源 ID 不同。相等文本不能独立证明两个 note.id 指向同一个 forum,也不能据此创建第二次投稿或批准 ID 别名。现有实体包没有截断投稿列表,均只有已记录的 NeurIPS 2021 拒稿;这不代表从未在其他地方录用。
|
||||||
|
|
||||||
|
## 采集器证据与使用限制
|
||||||
|
|
||||||
|
作者仓库 [research-arcade](https://github.com/ulab-uiuc/research-arcade) 固定版本 `ecf9c7c2490840b029bd2add4f02d04aca1d014e` 的 `openreview_crawler.py` 显示:
|
||||||
|
|
||||||
|
- 2020–2022 的旧接口路径读取 Blind_Submission 的 `note.id`,从 replies 中取第一个带 `decision` 字段的值,未验证决定 invitation 或签名。
|
||||||
|
- 2019 路径取第一个 `recommendation`,不能将其升级为正式决定。该年度属于保留线索的范围,本轮不补导。
|
||||||
|
- 新接口路径读取 Submission 的 `note.id`,用 `venueid` 尾段作为标签;遇 Conference 再取 `venue`,并跳过 Withdrawn_Submission。
|
||||||
|
|
||||||
|
三个源码文件均核对 Git blob SHA-1 和完整文件 SHA-256;没有执行第三方采集器。源码版本早于数据集版本,但未证明该提交实际生成了此数据集。数据集仅有六个元数据字段,缺少作者和决定签名,因此既不能证明完整 withdrawn 覆盖,也不能将导出的 ID 无条件认作已验证 forum。
|
||||||
|
|
||||||
|
## 原始来源访问结果
|
||||||
|
|
||||||
|
EMNLP 2023 官方分组元数据返回 200,其声明的 Submission invitation 与既有配置一致;对该 invitation 的匿名元数据查询返回 403。NeurIPS 2021 对第一条候选 note `Pk972F7J8Um` 的匿名元数据查询同样返回 403,余下五个 ID 未发请求。均已停止对应来源查询,没有换接口版本、登录或绕过访问限制。403 不能说明没有公开投稿;未保留完整错误正文,也不将具体拒绝原因猜测为已知。
|
||||||
|
|
||||||
|
[EMNLP 2023 官方政策](https://2023.emnlp.org/blog/EMNLP-2023-openreview/)说明公开录用稿件与选择公开的拒稿,现有 9 条 rejected 或本镜像的 1 条 desk-rejected 均不能证明完整公开集合。
|
||||||
|
|
||||||
|
## EZ-STANCE 后续发表线索
|
||||||
|
|
||||||
|
独立的 [ACL Anthology 官方条目](https://aclanthology.org/2024.acl-long.838/)确认相近标题的 EZ-STANCE 在 ACL 2024 Long Papers 发表,DOI `10.18653/v1/2024.acl-long.838`。正式标题多了 English,数据集由拒稿镜像中的 30,606 对变为 47,316 对。已保存官方 HTML 的完整哈希,登记为版本关系核查线索;尚未证明与匿名 EMNLP 2023 声明属于同一投稿或同一科学工作,没有把已发表作者转填到匿名记录,也没有写入关联或第二份论文。后续必须核验版本关系并保留各自的投稿决定证据。
|
||||||
|
|
||||||
|
当前 PostgreSQL 的正式标题变体查询命中已有 paper `2131805`,其 ACL/DOI 标识与官方条目一致,已有 `ACL / 2024 / main / accepted` 投稿历史。该实体无需重复新建。匿名 EMNLP 声明与该实体的关系仍未批准,不能把这条线索表述为已经确认的“先拒后录”。快照、标识及完整实体包哈希已记入机器记录。
|
||||||
|
|
||||||
|
## 下次从哪里继续
|
||||||
|
|
||||||
|
1. 保留五条候选及旧 ID,等待可验证的原始公开元数据或独立可靠来源;不要反复重放已拒绝的原始接口请求。
|
||||||
|
2. 对三对 ID 分歧先取得 note/forum 关系证据,再独立审核科学工作身份与逐次投稿关系;保留已有拒稿及以后发现的录用。
|
||||||
|
3. 两条没有精确标题候选的记录仍需标题变体、作者及摘要交叉核查;匿名元数据不能用于推断作者身份。
|
||||||
|
4. 继续其余会议与年度的公开负面可用性核查,不将这 930 条当作全会、全年份覆盖,也不将 accepted-only 镜像算作拒稿补全。
|
||||||
|
|
||||||
|
另修正交接记录:TMLR 是额外的期刊政策线索,当前 43 个会议的 registry 中没有 TMLR;没有因此更改会议分母。2000–2019 已发现线索仍保留。
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
# UAI 公开负面决定来源:2026-11-17 核查
|
||||||
|
|
||||||
|
**公开拒稿缺口仍未补齐,总体仍为 13/17。** 本页保存新来源的实际范围与恢复线索,避免下次把录用子集误当作全部投稿。主动补漏范围仍从2020年开始;本次镜像只含2022–2025,不能替代其他年份或其他会议的核查。
|
||||||
|
|
||||||
|
## 政策与访问边界
|
||||||
|
|
||||||
|
已有逐年核查记录保存在 [覆盖清单](inventory-2026-11-08.json) 的 UAI `policy_checks`:2023–2026 官方说明允许决定后公开录用及作者选择公开的拒稿,默认私密拒稿不属于公开可采集范围。政策并不证明任何镜像完整。
|
||||||
|
|
||||||
|
- [2023 投稿说明](https://www.auai.org/uai2023/submission_instructions)
|
||||||
|
- [2024 投稿说明](https://auai.org/uai2024/submission_instructions)
|
||||||
|
- [2025 投稿说明](https://www.auai.org/uai2025/submission_instructions)
|
||||||
|
- [2026 投稿说明](https://www.auai.org/uai2026/submission_instructions)
|
||||||
|
|
||||||
|
先前匿名 notes 请求遇到 ChallengeRequiredError/403,仍作为访问限制保留。本轮没有认证、重试或绕过这些入口;读取的是整理者公开发布的固定版本镜像。复制的 `readers`、`signatures` 和邀请字段仍只是镜像内的来源断言,不是对当前原始接口可见性或签名真实性的验证。
|
||||||
|
|
||||||
|
## 实际镜像范围
|
||||||
|
|
||||||
|
来源:[latkes/ai-research-index-iclr-openreview](https://huggingface.co/datasets/latkes/ai-research-index-iclr-openreview/tree/4c9e09931a7814d7500630ffd064b74f1c01bcf3),固定提交 `4c9e09931a7814d7500630ffd064b74f1c01bcf3`。目录中存在 `papers/uai-YYYY/papers.jsonl` 与四个 `raw/uaiYYYY.jsonl`,不能仅凭数据集说明宣称拒稿覆盖。
|
||||||
|
|
||||||
|
| 年份 | 整理版及原始论坛数 | 原始回复对象 | 实际回复角色 | 标准结构化决定字段 |
|
||||||
|
| --- | ---: | ---: | --- | --- |
|
||||||
|
| 2022 | 230 | 1059 | Official_Review 829,Decision 230 | Accept (Poster) 194,Accept (Oral) 36 |
|
||||||
|
| 2023 | 243 | 0 | 没有保存回复 | 没有保存决定记录 |
|
||||||
|
| 2024 | 200 | 1994 | Official_Review 905,Rebuttal 889,Meta_Review 200 | 没有单独 Decision 记录或结构化 decision 字段 |
|
||||||
|
| 2025 | 229 | 0 | 没有保存回复 | 没有保存决定记录 |
|
||||||
|
|
||||||
|
整理版902条均被整理者标为 `accepted=true`;原始根记录的 `venue` 为相应会议名称或 oral/poster/spotlight 文本,不能替代逐次决定证据,更不能算作902条已认证的原始录用决定。这不是四届全部投稿或全部录用的完整性证明。2024年还有 `Edit` 邀请,属于同一批回复上的另一个邀请,不能再累加成1994个新回复。
|
||||||
|
|
||||||
|
为排除字段漏读,对含回复的2022、2024快照另做了全部3053个回复的字段名、包装结构、邀请角色和短决定类字段核查;重新读取的正文哈希与首次捕获完全一致。2022的230条明确字段是录用标签。2024仅保存评审、作者回复和元评审,没有可据此新增的结构化拒稿/撤稿决定。没有从元评审文本、评分、推荐或缺少录用推断负面决定,也没有保存评审正文、作者回复正文或论文全文。
|
||||||
|
|
||||||
|
**这只描述上述固定镜像子集,不证明UAI没有公开拒稿。** 本批没有生成负面决定导入队列,也未写数据库。2020、2021、2026及其他可用来源仍需分别核查;某文件未出现在这个镜像目录中不代表全网不存在。
|
||||||
|
|
||||||
|
## 产物与恢复
|
||||||
|
|
||||||
|
服务器目录均相对 `/data/paper-index/collected/`:
|
||||||
|
|
||||||
|
- `uai2022-2025-mirror-metadata-v1`:902条整理版元数据投影、字段盘点与标签计数。
|
||||||
|
- `uai2022-2025-raw-decisions-v1`:902个原始论坛的元数据、明确决定字段及来源正文/逐行哈希。
|
||||||
|
- `uai2022-2024-reply-schema-v2`:3053个回复的元数据、字段名与短标签;评审正文不保存。角色核查包含直接记录和 `note` 包装,未发生静默遗漏。
|
||||||
|
|
||||||
|
源响应正文仅在内存中读取、校验,保存元数据投影与固定来源URL/字节哈希;不是保留了含评审文本的完整原始快照。复核原始源字节时应使用固定提交,并区分投影文件哈希与源正文哈希。恢复包已在本机独立核验全部902个论坛、3053个回复的元数据结构、逐文件哈希及来源关联。
|
||||||
|
|
||||||
|
| 恢复证据 | SHA256 |
|
||||||
|
| --- | --- |
|
||||||
|
| 整理版元数据恢复包 | 55889fb2b9005260cef1001b2cf36e203f3b7e1deeec0291f058b52eeae880e7 |
|
||||||
|
| 原始元数据/回复结构恢复包 | caade7df899dd3e65572d06e2bf2326fad692cc5e1c1e4ab002fdc6d7823743f |
|
||||||
|
| 原始结构独立核验报告 | 5f0ef2948d0db68fe0cd89eb4c8b57a1088b94b53bed98f180d2b96629746e72 |
|
||||||
|
|
||||||
|
服务器恢复包分别为 `/data/paper-index/validation/uai2022-2025-mirror-metadata-recovery-v1.tar.gz` 与 `uai2022-2025-raw-reply-recovery-v1.tar.gz`。本地同名备份位于 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`。先检查实际单元、完整manifest和哈希,再恢复;不要因进度文件旧状态或日志安静重跑。
|
||||||
|
|
||||||
|
后续继续寻找能明确覆盖作者公开拒稿选择的来源,保留每次投稿及决定原文标签、原始ID与镜像关系,核对跨来源重叠和后续录用。UAI缺口不关闭,全部43个目标的逐年公开负面决定核查也不缩减;最终全库去重仍为独立必做阶段。
|
||||||
@@ -0,0 +1,16 @@
|
|||||||
|
# ECCV 2026 正式出版章节采集
|
||||||
|
|
||||||
|
`collect eccv2026-publications` 读取已归档的 Springer LNCS 17001–17078 目录及所选章节的公开 HTML,输出可复核的 Paper Atlas 记录。目录覆盖 2,835 个章节 DOI。会议节目单与出版方卷册简介均为 2,834 条,逐条差异仍待核。Workshop 论文集另行处理。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect eccv2026-publications \
|
||||||
|
--toc-snapshots /data/paper-index/collected/eccv2026-springer-toc-v1 \
|
||||||
|
--chapter-snapshots /data/paper-index/validation/eccv-verified-chapter-pages \
|
||||||
|
--out /data/paper-index/collected/eccv-publisher-metadata
|
||||||
|
```
|
||||||
|
|
||||||
|
目录归档含 `receipts.jsonl`、`volume-checkpoint.json`、`summary.json` 及 156 页 HTML。摘要在章节详情页读取,章节归档沿用 `springer-selected` 的 `receipts.json` 格式。两类归档都按响应大小和 SHA-256 校验。目录卷号、分页、章节 DOI 归属、标题及 2026 年出处需要一致。数字上标的两种数学定界符可按同一标题核对,章节标题和目录原文分别保存。已有输出目录需另查回执或换新空目录后运行。
|
||||||
|
|
||||||
|
输出含规范化 JSONL、manifest、采集回执以及所选章节的内容寻址存证。每条保留出版 DOI、卷号、页码、按顺序的完整作者及公开摘要。目录证据用 URL 和哈希连接到输入归档,保存归档以便复核。主会出版观察依据 Springer 正式论文集,原始投稿号、决定日期及签名仍未知。工具仅处理已归档的公开 HTML。
|
||||||
|
|
||||||
|
生成的文件待做论文身份复核后方可导入。准备生产批次时核实每个 DOI 与现有 arXiv 标识的绑定,保持两者的来源证据。对已存在的同届节目投稿需先核对投稿身份及轨道,再附加出版来源,避免因直接导入创建重复投稿。具有 ICLR 等既往决定的论文须保留原记录。新的增量导入 manifest 需以当前实际来源覆盖为父版本,留存待核差集和独立验收回执。
|
||||||
@@ -37,3 +37,23 @@ group 的 `rejected_venue_id` 实际配置为 `thecvf.com/ECCV/2026/Conference/R
|
|||||||
| anonymous-decision-probes/manifest.json | a522756ed3ec2880e5d942a211d7ad13819b18c8acada551dd3a8718b5e4d63b |
|
| anonymous-decision-probes/manifest.json | a522756ed3ec2880e5d942a211d7ad13819b18c8acada551dd3a8718b5e4d63b |
|
||||||
|
|
||||||
这里的补充证据与采集器的三份冻结节目/政策输入分开保存,不能替换采集器固定版本的 manifest。下次补漏先核查官方是否提供公开决定导出或明确的公开元数据说明;访问仍受限时保留缺口。若发现允许公开访问的新入口,另存带时间与哈希的快照,先审阅读者权限、决定语义和逐篇身份,再准备入库队列。
|
这里的补充证据与采集器的三份冻结节目/政策输入分开保存,不能替换采集器固定版本的 manifest。下次补漏先核查官方是否提供公开决定导出或明确的公开元数据说明;访问仍受限时保留缺口。若发现允许公开访问的新入口,另存带时间与哈希的快照,先审阅读者权限、决定语义和逐篇身份,再准备入库队列。
|
||||||
|
|
||||||
|
|
||||||
|
## 2026-11-17 全量采集与只读预检
|
||||||
|
|
||||||
|
第三次采集 `paper-atlas-eccv2026-collect-pass03.service`(invocation `147c8779a3af4223a4d5d60dbdc88264`)已终态成功:2,834 个唯一日程条目全部取得公开摘要,新取 1,775 页,复用已验收缓存 1,059 页;2,851 次日程出现中的 17 处重复保留原始位置。独立原文/回执/字段审计通过,缺少条目和摘要均为 0,5 处 JSON-LD 显示文字规范化仍留待逐篇审阅。这里仍是官方初步日程版本,不改称出版社最终论文集。
|
||||||
|
|
||||||
|
| 服务器证据 | SHA-256 |
|
||||||
|
| --- | --- |
|
||||||
|
| validation/eccv2026-pass03-audit-v1.json | ad90199e08a234c0b2a01f44e1edfbecb2a9101b45de87e00a64eadfd5c92848 |
|
||||||
|
| collected/eccv2026-pass03/eccv-program-2026-unclassified.jsonl | d0a80745fc6f58163f8722af351bbba86803af97857ea30554a0f3062d8ab2ef |
|
||||||
|
| validation/eccv2026-pass03-recovery-v1.tar.gz | df55493ad5f8d6413dca86e9b9aca1d8b544ac1dc222b3849f95b04f94accba9 |
|
||||||
|
| collected/eccv2026-pass03-preflight-v1/postgresql-preflight.json | 97bf57828cc67d05de0221188a9d988eef35631c845b18372186a6249f64e5da |
|
||||||
|
| validation/eccv2026-pass03-preflight-packets-check-v1.json | 8a4f44f5d2c3a15d05b5a1df96fe465a96a2fe8914d7ed68958e6978c30985d4 |
|
||||||
|
| validation/eccv2026-pass03-preflight-recovery-v1.tar.gz | 69c84fd0f24024ca56bfa4f848c977c578267bfc53c713105987e030df3f92a2 |
|
||||||
|
|
||||||
|
全量原文恢复包为 85,430,941 字节,保留输出、全部缓存原文与回执、官方输入和固定源码/CI 凭证。失败的第一次打包任务在创建文件前因命令引用错误退出;改用文件脚本的新单元 `paper-atlas-eccv2026-pass03-recovery-v2.service` 成功,未重启或重跑采集器。
|
||||||
|
|
||||||
|
新的 PostgreSQL 只读预检覆盖全部 2,834 条:1,519 条精确标题和作者集合一致、468 条仅部分作者重合、840 条无精确匹配、4 条没有作者重合、3 条命中多个同标题实体。1,997 个候选实体包的哈希独立核对通过,来源、投稿、决定与来源关系未触发截断。该结果只生成候选,`database_written=false`、`approved_for_import=0`;没有批准合并、创建新论文或导入 ECCV 事件。无精确匹配不等于新作品,相同作者也不证明人物或作品身份。
|
||||||
|
|
||||||
|
后续先保留这批完整来源并推进其他会议来源补漏,再做逐篇内容与版本审阅、当前快照守护和保护基线。当前全局最新基线来自 SDM25 验收,保护 5,653 篇论文;不能回退到 ICAPS34 之前的计数或重放已消费批次。最终全库去重仍须另行执行。
|
||||||
|
|||||||
@@ -0,0 +1,40 @@
|
|||||||
|
# ICASSP 2026 注册元数据采集
|
||||||
|
|
||||||
|
这是长期维护项目中的可重复运行采集器,按需离线处理冻结的公开元数据;不会启动数据库导入、联网或下载论文 PDF。命令仅输出待身份审查的来源版本。
|
||||||
|
|
||||||
|
## 输入与使用
|
||||||
|
|
||||||
|
2026-11-17 在数据服务器采集 Crossref 公开 REST API。先从正式节目表抽查的两篇标题确定注册会议名称和 DOI 前缀,再依据 IEEE DOI 前缀、proceedings-article 类型及注册的活动开始日期 `2026-05-03` 分页获取;日期过滤并不意味着所有结果都属于 ICASSP。
|
||||||
|
|
||||||
|
需要服务器 `/data/paper-index/collected/icassp2026-registry-capture-v1` 下的 `manifest.json` 与 `page-001.json` 至 `page-010.json`。将其父目录传给:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect icassp2026-registry \
|
||||||
|
--snapshots /data/paper-index/collected \
|
||||||
|
--out /data/paper-index/collected/icassp2026-normalized-<new-run>
|
||||||
|
```
|
||||||
|
|
||||||
|
输出目录必须不存在。复跑前先检查原任务和产物,不能覆盖成功或失败输出。采集器验证固定清单 SHA256、全部页面原始字节、HTTP 回执、时区、页面数量、总数及唯一 DOI,再生成 JSONL、导入格式的部分覆盖清单、全部原始来源证据和采集报告。导入格式不代表身份审查通过;本命令批准导入数始终为零。
|
||||||
|
|
||||||
|
## 数据范围
|
||||||
|
|
||||||
|
- 10 页共有 4,645 个唯一 DOI;56 个属于 ICPEA,明确排除。
|
||||||
|
- ICASSP 的 4,589 个注册条目中,7 个是封面、目录、作者索引、委员会和审稿人名单;依据已审查的准确 DOI、标题和页码排除,不以“没有作者”泛化排除论文。
|
||||||
|
- 剩余 **4,582 条论文**全部缺少注册摘要,输出 `null`。保留完整注册字段、作者顺序、机构、原始标题、日期及原始来源哈希。
|
||||||
|
- 23 个作者条目只提供单个姓名字段,原样保留并标记;不猜测名字拆分、拼音边界或人物别名。
|
||||||
|
- `accepted` 表示已登记在该会议论文集中的发表观察,并非原始签名录用决定。DOI 用 `publication_doi`,不复制先前投稿系统的 ID。
|
||||||
|
- track 为 `proceedings_unclassified`,不从技术主题推断 main、special session、workshop 或研究类型;没有自造标签。
|
||||||
|
|
||||||
|
正式节目表有 4,589 个编号和标题,与注册条目不是已证明的一一对应关系。独立标题规范化比较得到 4,415 个来源候选和 174 个未匹配节目标题;标题匹配不是作品身份批准,未匹配不等于拒稿、撤稿或未发表。节目表作者与注册作者的版本也不应自动拼接。
|
||||||
|
|
||||||
|
注册活动日期为 5 月 3–8 日,会议网站为 5 月 4–8 日,两者原始值均保留。Crossref 记录中的链接只是原始字段,不会访问 PDF。IEEE Open Preview 入口读取遇到 HTTP 418,未绕过访问限制。公开拒稿元数据是否可得仍为 unknown,本批不是拒稿覆盖证明,也不是第 14 项整体完成证明。
|
||||||
|
|
||||||
|
## 证据与恢复
|
||||||
|
|
||||||
|
注册来源清单 SHA256:`1f8ffa040b1f682a4b65f476f4115cb95629056d8de5c324c8417d823846ae93`。
|
||||||
|
|
||||||
|
原始证据恢复包在服务器 `/data/paper-index/validation/icassp2026-registry-capture-v1-recovery.tar.gz`,本地备份位于 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/` 同名文件,SHA256 为 `cdc506e95800fb8d37ed87f85740e96f56150f2d96daa971de21a5a7a9ab8388`。安全解包并核验后使用上述命令;测试夹具也冻结了全部 10 页及独立论文清单,供远程 CI 验证。
|
||||||
|
|
||||||
|
本批还需生产规范化验收、数据库当前 DOI/作品关联及历次投稿审查。所有导入结束后仍须执行全库去重、覆盖、摘要和历史完整性审计。编译、打包及测试均交给用户指定的 `ces-control:host` runner。
|
||||||
|
|
||||||
|
参考:[Crossref 筛选器文档](https://www.crossref.org/documentation/retrieve-metadata/rest-api/rest-api-filters/)、[正式录用标题表](https://cmsworkshops.com/ICASSP2026/papers/accepted_papers.php)、[会议日程](https://2026.ieeeicassp.org/technical-program/)、[发表和展示类型说明](https://2026.ieeeicassp.org/publishing-and-paper-presentation-options/)。
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
# ICDM2026 冻结日程适配器
|
||||||
|
|
||||||
|
这是长期维护、按需运行的来源模块。它读取已经冻结的官方页面,离线生成可审阅的标准记录,不连接数据库、不请求网络、不下载论文 PDF;后续入库仍需作品身份、发表版本和旧投稿历史核查。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect icdm2026-program \
|
||||||
|
--snapshots /data/paper-index/collected \
|
||||||
|
--out /data/paper-index/collected/icdm2026-normalized-pass01
|
||||||
|
```
|
||||||
|
|
||||||
|
输入根目录下须有 `remaining2026-official-leads-v1/manifest.json`,及 manifest 所列的三个 ICDM HTML:`icdm-detailed-sessions.html`、`icdm-research-policy.html`、`icdm-applied-policy.html`。共享研究 manifest 也保留其他会议的入口回执,但本适配器只读取并验证 ICDM 的三个明确 URL,不要求复制其他会议文件。固定 manifest SHA-256 为 `0e13313d94e3bff17f071d3a1384f6ed83ece156239fe5bc4633cc7a34fb8e04`。
|
||||||
|
|
||||||
|
适配器核对每个回执的 URL、HTTP 200、HTML 内容类型、时区时间戳、原始字节数和哈希;保留 CRLF。来源更新必须另存版本并重新审阅,不能修改旧 manifest 来冒充旧快照。源码测试 fixture 同时保存原始页面、回执和使用另一解析方法得到的独立逐行盘点。
|
||||||
|
|
||||||
|
## 两个明确来源轨道
|
||||||
|
|
||||||
|
[官方详细日程](https://icdm2026.neu.edu.cn/DetailedSessions/list.htm)自称为线下展示清单,日期为 2026 年 11 月 13–15 日,34 个分会场、324 个不同节目编号。适配器分别输出:
|
||||||
|
|
||||||
|
| 输出文件 | 官方明确标签 | 记录数 |
|
||||||
|
| --- | --- | ---: |
|
||||||
|
| icdm-program-2026-research.jsonl | research / Research Track | 241 |
|
||||||
|
| icdm-program-2026-applied.jsonl | applied / Applied Track | 83 |
|
||||||
|
|
||||||
|
`data-track` 属性必须与可见标签一致;轨道来自这两份明确证据,不根据标题、会场名称或研究主题推测。`data-format` 和可见 Long/Short 标签、展示说明必须一致;168 个 long 和 156 个 short 仅表示演讲形式,**不自动转成论文长短、position、findings 或研究类型**。日程主题也不生成自定义人物或研究分类。
|
||||||
|
|
||||||
|
每条保留日程显示标题、原始作者文字、分号分隔的来源姓名顺序、节目编号、分会场、时间、地点、全局来源位置、来源回执和对应原文文件。作者仅标为日程报告的姓名,不声称为最终出版社完整作者版本,也不生成姓名别名。
|
||||||
|
|
||||||
|
`2026/DM...` 是 `icdm_program` 来源下的节目定位键;页面锚点只是来源定位,不能当 DOI、arXiv 或原始签名投稿/决定标识。`identifiers` 和 `submission_identifiers` 都为空。`status=accepted` 表示列入官方研究/应用日程的来源观察,`raw_decision` 明确其并非原始签名决定或已核验出版社记录;不根据该状态生成原始决定日期。每条 `original_decision_signature_verified=false`。
|
||||||
|
|
||||||
|
## 始终保留范围限制
|
||||||
|
|
||||||
|
324 条日程记录都没有摘要,输出 `abstract=null`,每个文件的 `missing_abstract_count` 如实填写。manifest 始终为 partial;`source_expected_count` 仅对应这个冻结线下日程分区,不能当作所有录用、正式出版或线上展示的应有总数。其他页面的 workshop、博士论坛、BlueSky、demo 等不在本次输入范围内,也不能假定不存在。
|
||||||
|
|
||||||
|
标签使用 `icdm_program:session_track` 与 `icdm_program:presentation` namespace;它们是平台来源证据,不与其他来源标签或用户自定义标签混为一谈。`paper_type` 保留为空。
|
||||||
|
|
||||||
|
[研究轨政策](https://icdm2026.neu.edu.cn/CallforResearchTrack/list.htm)描述 triple-blind 审稿,[应用轨政策](https://icdm2026.neu.edu.cn/CallforAppliedTrack/list.htm)描述 single-blind 审稿,并指向 CyberChair 投稿系统。这些是审稿期间政策,**不能据此断言最终拒稿元数据不公开**。报告保留 `public_rejected_metadata=unknown_no_public_export_verified`;未访问登录资料、未绕过限制,也未从日程缺席推断 rejected、desk_rejected 或 withdrawn。
|
||||||
|
|
||||||
|
## 验证与恢复
|
||||||
|
|
||||||
|
完整测试在已授权 `ces-control:host` 远程 runner 执行,包含全部 324 条与独立盘点逐字段对照、轨道/展示属性与可见文字冲突、重复或变化的节目编号、会场计数、空作者、原始字节/回执校验和失败输出保护。CI 另在安装 wheel 后、工作目录外运行真实 CLI,检查 241/83 两个输出及缺失摘要、未生成稳定标识等边界。构建和测试不在用户本机运行;实际结果见对应提交的远程 CI。
|
||||||
|
|
||||||
|
输出目录必须不存在。输入校验失败时不会创建输出;输出生成失败会保存 `failed.json`,成功报告最后原子发布为 `collection-report.json`。遇到已有目录,先核对原操作,不覆盖、不盲目重放。
|
||||||
|
|
||||||
|
原始来源恢复包为 `/data/paper-index/validation/remaining2026-official-leads-v1-recovery.tar.gz`,SHA-256 为 `0be8c84bb5c591e5afaaa5f95dd5334bf13a9be1671cb942e46e998f502495bb`。独立盘点为 `/data/paper-index/validation/icdm2026-independent-program-inventory-v1.json`,SHA-256 为 `e13eaea0a158dcaafdf5dfb4af7948e4793b0e98041ec87d42b52513acfdc5f6`;本机工作记录目录也保留同名备份。
|
||||||
|
|
||||||
|
下次继续先验证源码版本、原始证据和实际输出,再寻找可公开取得的摘要与正式出版记录,核查逐篇版本、稳定标识及旧投稿历史,准备新的只读预检和可审阅入库队列。完成日程规范化不意味着 ICDM2026 整届出版覆盖、公开负面决定普查或最终全库去重完成。
|
||||||
|
|
||||||
|
## 2026-11-18 官方展示来源核查
|
||||||
|
|
||||||
|
口头日程的 324 个节目编号全部出现在官方海报排期。海报表有 343 个不同编号,其中长演讲组安排 177 项海报,短演讲组安排 166 项。19 个只见于海报表的编号作为独立来源线索登记。两份排期的覆盖范围各自保留,正式出版目录仍待核验。[官方详细日程](https://icdm2026.neu.edu.cn/DetailedSessions/list.htm) · [官方海报表](https://www3.cs.stonybrook.edu/~icdm2026/PosterwDemo/list.htm)
|
||||||
|
|
||||||
|
共同编号 DM1319 的标题在两个来源中实质不同。详细日程描述基于 FFE-FRCN 的数据平台,海报表描述 FFE-FRCN 的少样本检测方法。两份标题及来源位置均已留存,该编号的跨来源作品关联待核验。
|
||||||
|
|
||||||
|
生产只读快照确认 ICDM 2026 仍无入库届次。节目、作者和独立录用证据支持继续处理 DM1125、DM790、DM1157 三项既有作品关联。DM1125 和 DM1157 各有一次 ICLR 拒稿投稿及两条来源观察,DM790 有一次 ICLR 撤回投稿。三项均尚未导入,节目原始摘要字段缺失。另有三个候选存在作者名称顺序或名单差异,继续暂缓作品关联。[FALCON 作者记录](https://arxiv.org/abs/2608.29635) · [PPGFlowECG 作者记录](https://arxiv.org/abs/2509.19774) · [DeepSynC 大学录用说明](https://dm.cs.univie.ac.at/news/story/deepsync-paper-accepted-at-icdm-2026)
|
||||||
|
|
||||||
|
本轮仍未核实公开的 ICDM 2026 拒稿或撤回决定导出。[逐条差集、标题冲突、快照和后续入口](coverage/icdm2026-presentation-scope-2026-11-18.json)
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
# ISMIR 2026 官方节目与摘要采集
|
||||||
|
|
||||||
|
这是长期维护项目中的离线采集器,按需处理冻结的官方元数据,不联网、不执行网页脚本、不下载论文 PDF、不连接数据库。输出仍须经过作品身份与投稿历史审查。
|
||||||
|
|
||||||
|
## 使用
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect ismir2026-program \
|
||||||
|
--snapshots /data/paper-index/collected \
|
||||||
|
--out /data/paper-index/collected/ismir2026-normalized-<new-run>
|
||||||
|
```
|
||||||
|
|
||||||
|
输入父目录需要三个冻结目录:
|
||||||
|
|
||||||
|
- `remaining2026-listings-research-v2`:清单、`ismir-accepted.html`、`ismir-call.html`。
|
||||||
|
- `ismir2026-script-research-v1`:清单及 `papers.js`,用于证明官方程序明确加载的数据文件和 poster 链接模板;只读取脚本原文。
|
||||||
|
- `ismir2026-json-research-v1`:清单及 `papers.json`,提供 140 项完整元数据和公开摘要。
|
||||||
|
|
||||||
|
固定清单和全部输入原始字节均校验 SHA256、HTTP 回执、官方 URL 及带时区的采集时间。已有输出目录不能覆盖;先检查原任务是否已结束,再选择新目录。失败保留 `failed.json`,只有全批成功才发布 `collection-report.json`。
|
||||||
|
|
||||||
|
## 来源与版本
|
||||||
|
|
||||||
|
[正式录用名单](https://ismir2026.ismir.net/accepted-papers)和[官方虚拟平台](https://ismir2026program.ismir.net/papers.html)各有 140 项,经来源内标题对应核查一致;其中 2 处标题有直引号/弯引号文字差异,3 处作者名单有差异,两个来源的原始写法分别保留并标记,不静默修正或拼接作者。
|
||||||
|
|
||||||
|
作者差异涉及节目 ID `60`、`126`、`433`,包含大小写和姓名完整度问题。正式名单的作者机构可能含括号和分号,采集器只在括号外拆分作者;作者原始字符串也保留。来源内名单对应不代表数据库作品身份已经获准关联。
|
||||||
|
|
||||||
|
- 原始 JSON 摘要全文保留,包括数学符号、换行和 Unicode,不作扁平化提取。
|
||||||
|
- 数字 `id`、`forum` 是 miniconf 节目定位符,不视为原始 OpenReview 投稿 ID;不推断 DOI 或 arXiv ID,投稿标识列表保持空。
|
||||||
|
- `accepted` 来自正式录用名单与节目平台,不是原始签名决定。`track=program_unclassified`,不从主题、session 或远程展示方式推断研究方法、workshop 或论文类型。
|
||||||
|
- `ismir_program:keyword` 保存来源非空关键词,规范空白并去除同条记录内完全重复值;原始关键词列表(含空项)保留在元数据。
|
||||||
|
- `ismir_program:presentation` 保存来源展示方式。Remote/In-person 是展示方式,不是研究类型。
|
||||||
|
- poster URL 严格依据冻结的官方 JS 模板生成;本命令不请求 poster 页面。
|
||||||
|
|
||||||
|
## 公开评审与拒稿边界
|
||||||
|
|
||||||
|
原始来源中有 **101 组明确选择公开的评审材料**;其余 39 项 `publish_reviews=No`,评审正文为空。采集器验证此偏好和正文的一致性,原文保存在来源元数据,不把评审意见或 meta-review 中的推荐转换成额外的最终决定。
|
||||||
|
|
||||||
|
这些记录来自录用节目,不能据此当作 101 条公开拒稿记录。公开拒稿导出是否可得仍记为 unknown,需要单独核查。[征稿与评审流程](https://ismir2026.ismir.net/authors/call-for-papers)中的评审规则也不等于具体投稿决定。
|
||||||
|
|
||||||
|
输出包含一份 140 条 JSONL、部分覆盖清单、七个原始来源文件及报告,批准导入数为零。采集成功不等于会议论文/track/公开负面决定覆盖完整,也不替代最后的全库去重审计。
|
||||||
|
|
||||||
|
## 恢复与验证
|
||||||
|
|
||||||
|
`papers.json` SHA256:`864d384cbc928554504e1b39849137bb380143e247a48f2d71563a698cb8e62d`。
|
||||||
|
|
||||||
|
原始 JSON 和 JS 恢复包:服务器 `/data/paper-index/validation/ismir2026-json-research-recovery-v1.tar.gz`,本地 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/` 同名文件;SHA256 为 `bba7b236cfe4d8b259d6c1230b3fe09a3e9071f5203884c92b016ca0718fb095`。
|
||||||
|
|
||||||
|
正式名单与政策恢复包:同目录 `remaining2026-listings-research-v2-recovery.tar.gz`,SHA256 为 `f5ef7c2e73d46712146e4d6711ad5ad562e16b8b55c0cd75d1041cd0b81142cb`。安全恢复三个源目录、核验文件后可重跑上述命令。
|
||||||
|
|
||||||
|
完整来源和独立 140 项清单在测试夹具中冻结。测试、构建、打包、安装及实际隔离目录 CLI 验证只在用户指定 `ces-control:host` runner 执行。生产规范化后还需独立核验全部记录,并用 PostgreSQL 只读预检检查候选作品和既往投稿;不自动合并来源差异或人物身份。
|
||||||
|
|
||||||
|
## 2026-11-18:首批 10 条验收(历史阶段)
|
||||||
|
|
||||||
|
本段保留首批历史状态;当前累计结果和增量父状态见下方最新验收段。
|
||||||
|
|
||||||
|
首批 10 项已逐篇核查研究内容,并通过当前数据库实体、作者、摘要及完整历史的只读复核,随后使用已通过远程 CI 的包内 `import records` 导入。新增论文实体 **0**,新增来源记录、投稿、录用观察各 **10**,来源标签证据 **94** 条;规范标题、作者、摘要及旧历史保留。原始来源摘要另存,不覆盖规范摘要。覆盖为 `partial10/140`,还有 **130** 项待核验。数值节目 ID 仍只作为节目定位符,未冒充 OpenReview 投稿 ID,未新增全局人名别名或推断研究类型。
|
||||||
|
|
||||||
|
[逐条验收与恢复记录](coverage/ismir2026-import10-2026-11-18.json)包含生产行 ID、保护基线、归档及证据哈希。服务器基线保护此前 6,384 篇与本批 10 篇的并集 **6,394** 篇,全库旧投稿、决定及来源关系哈希未变;独立新快照的 10 个完整实体包、来源字段、标签和标识也逐项核验。Foundry `/readyz` 返回200,17,987,751,936字节的 SQLite 备份保持只读。该验收不是最终全库科学工作去重。
|
||||||
|
|
||||||
|
例如 paper `2757497`(Learning Music Style for Piano Arrangement Through Cross-Modal Bootstrapping)保留一次 ICLR2026 拒稿投稿和两条拒稿来源观察,并新增独立 ISMIR2026 录用投稿。实际执行以下查询,返回 `publication_outcome=accepted`,同时保留原拒稿历史:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas query --pg-service foundry-research --pg-schema research \
|
||||||
|
--paper 2757497 --outcome accepted
|
||||||
|
```
|
||||||
|
|
||||||
|
ISMIR 的 accepted 来自正式名单和节目,尚无原始签名决定或具体决定日期;不能把采集时间当作决定时间。决定的规范时间使用包内既有毫秒格式,来源回执的原始微秒时间仍保留于元数据。
|
||||||
|
|
||||||
|
**禁止重放** `/data/paper-index/collected/ismir2026-prepared10-v1` 的文件。导入单元 `paper-atlas-ismir2026-import10-v1.service`、invocation `538ce9c9324a4f59b5bb49d3094d5265` 已成功结束;该阶段建立了 `partial10/140` 父状态;后续累计结果见下方最新段,不得沿用已消费的 `0/140` 父状态。保护基线恢复包为 `/data/paper-index/validation/ismir2026-baseline10-recovery-v1.tar.gz`,验收恢复包为 `/data/paper-index/validation/ismir2026-import10-acceptance-recovery-v1.tar.gz`;本机 paper-index 工作目录有同名副本及独立检查记录。
|
||||||
|
|
||||||
|
## 2026-11-18:新增 21 条,累计 31/140
|
||||||
|
|
||||||
|
第二批21项已通过逐篇研究内容审查、新鲜只读父状态检查、保护基线及独立入库验收。新增论文实体、来源和edition均为0;新增来源记录、投稿、录用观察各21条,新增173条来源标签证据。累计31条录用来源观察、267条标签证据,覆盖仍为 `partial31/140`,**109项待核验**。三十一项摘要都保留在原始来源记录中,既有规范资料没有被覆盖。
|
||||||
|
|
||||||
|
本批明确保留不同研究工作,例如UOT-IR音乐压缩与SymphonyGen交响生成,以及AI混合音轨占比回归与BAMM真实广播检测;不因作者或主题相近合并。源关键词和展示方式继续使用各自namespace,没有推断全局人名别名、原始OpenReview投稿ID或研究类型。
|
||||||
|
|
||||||
|
[第二批逐条验收及恢复记录](coverage/ismir2026-import21-2026-11-18.json)包含21组生产行ID、源码提交、全部证据哈希和新计数。保护并核验了6,415篇作品的规范资料、旧来源与标签,以及所有旧全局投稿/决定/来源关系指纹;新快照另导出21个完整实体包逐字段核验。首批10项保留;paper2757497的实际查询仍显示accepted及原ICLR一次拒稿下的两条来源观察。
|
||||||
|
|
||||||
|
本批使用固定源码 `3754545003e1f96b88c6533cc3a69dc557f07305`,已在远程 `ces-control` CI1257通过556项测试,含真实PostgreSQL只读增量门禁回归。该修复解决预检对Mapping行对象切片的错误,未弱化覆盖校验;旧的只读准备v1失败时没有写库,修复后v2重新检查通过。默认运行时未因本批操作切换。
|
||||||
|
|
||||||
|
**禁止重放** `/data/paper-index/collected/ismir2026-prepared21-v2`。服务 `paper-atlas-ismir2026-import21-v1.service`、invocation `78077b901a274426bb161ef8d69963ad` 已成功结束。新批次必须从当前 `partial31/140` 准备,不能再用 `10/140` 或 `0/140`。保护基线恢复包为 `/data/paper-index/validation/ismir2026-baseline21-recovery-v1.tar.gz`,验收恢复包为 `/data/paper-index/validation/ismir2026-import21-acceptance-recovery-v1.tar.gz`;本机paper-index目录有同名副本与独立检查结果。
|
||||||
|
|
||||||
|
新的固定范围盘点仍为274/278个年度有数据;本批提高ISMIR来源覆盖,没有填补其他空年度,也没有关闭公开拒稿或最终全库去重任务。Foundry就绪检查200,SQLite备份继续只读。
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
# MM 2026 出版条目采集与恢复
|
||||||
|
|
||||||
|
`collect mm2026-publications` 是包内可复用的冻结快照规范化命令,读取完整注册目录和官方节目,不连接数据库、不请求网络、不下载PDF。总体仍为13/17,来源规范化、测试通过或节目匹配都不能替代身份批准、生产入库或最终去重。
|
||||||
|
|
||||||
|
## 使用
|
||||||
|
|
||||||
|
在已配置的服务器环境中,`--registry-snapshots` 是包含 `mm2026-container-registry-v2/` 的父目录;`--program-snapshots` 是包含 `mm2026-program-and-public-probes-v1/` 的父目录。父论文集正文使用独立固定快照,不能从不存在的文章ISBN字段猜测:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect mm2026-publications \
|
||||||
|
--registry-snapshots /data/paper-index/collected \
|
||||||
|
--program-snapshots /data/paper-index/collected \
|
||||||
|
--parent-snapshot /data/paper-index/collected/mm2026-publication-metadata-v1/parent.json \
|
||||||
|
--out /data/paper-index/collected/mm2026-publications-native-v3
|
||||||
|
```
|
||||||
|
|
||||||
|
输出目录必须不存在。失败输出保留 `failed.json`,完整 `collection-report.json` 只在全部校验成功后原子生成;不要覆盖输出或因旧进度文件、日志安静重跑。先观察实际单元/报告,再选择新的恢复目录。
|
||||||
|
|
||||||
|
## 注册目录与节目分别保留
|
||||||
|
|
||||||
|
源目录基于 Crossref [官方精确过滤器](https://www.crossref.org/documentation/retrieve-metadata/rest-api/rest-api-filters/) 的 `container-title` 和出版者member320,实际捕获8页、1918个不同DOI。每页请求、时间、原始正文、字节哈希、总量与唯一DOI均核验。父DOI为 `10.1145/3767308`,题名为 *Proceedings of the 34th ACM International Conference on Multimedia*,明确事件为MM2026;文章均注册为 `proceedings-article`。
|
||||||
|
|
||||||
|
ISBN查询和父Unixref XML只返回父项,不能代表逐篇目录。宽泛 `query.container-title` 的检索总量也不能当会议篇数。当前1918是这个明确论文集过滤范围内的注册条目数,不是已经去重的科研作品数、所有MM workshop或全部28个来源track的出版全集。
|
||||||
|
|
||||||
|
每个出版条目都输出,稳定外部标识为真实DOI。身份标识仅为 `doi`,投稿侧标识仅为 `publication_doi`;不推断原始OpenReview/forum/投稿号或arXiv标识。来源名 `mm_registry_program` 明确表示派生的注册/节目材料,不能把其中所有字段都说成Crossref原文。
|
||||||
|
|
||||||
|
原始注册响应及父记录复制到 `evidence/registry/`,完整节目及四个既有403回执复制到 `evidence/program/`,不重新请求受限入口。每项元数据保留注册正文的文件/哈希、数组位置和规范JSON对象哈希;常用书目字段另作投影,完整引用、资助等原始字段仍在原始响应中,未丢弃也不在每条JSONL重复膨胀。
|
||||||
|
|
||||||
|
## 分轨、摘要与版本差异
|
||||||
|
|
||||||
|
按原始题名的 Unicode/字母数字规范化与完整作者多重集合比对,1726 项有唯一节目对应。原始题名没有候选时,再比较经过受限排版投影的题名:只移除平衡、无属性的 `i`、`scp`、`sup`、`sub` 标签,保留上下标内容,HTML 实体只解码一次。未知或不平衡标签、属性、注释及声明不参与这种投影;不使用模糊题名或人物别名。
|
||||||
|
|
||||||
|
26 项新增对应均保留完整作者多重集合一致、唯一节目 ID 以及逐项复核名单,复制节目原文的摘要、track 和展示上下文。原始注册题名、作者和节目字段继续逐字保留,比较文本单独存入 `program_title_formatting_comparison`;这只证明节目材料对应,不批准科学作品合并。原始题名已有作者分歧时,不通过排版回退绕过。其余 122 项原始作者分歧、1 项排版题名下的作者分歧、43 项题名差异,共 166 项仍在 `proceedings_unclassified`。
|
||||||
|
|
||||||
|
| 输出分区 | 注册条目 |
|
||||||
|
| --- | ---: |
|
||||||
|
| main | 1555 |
|
||||||
|
| grand_challenge | 77 |
|
||||||
|
| interactive_art | 17 |
|
||||||
|
| datasets | 41 |
|
||||||
|
| reproducibility | 1 |
|
||||||
|
| bni | 22 |
|
||||||
|
| open_source | 11 |
|
||||||
|
| doctoral_symposium | 2 |
|
||||||
|
| demo_and_video | 23 |
|
||||||
|
| industry_talk | 3 |
|
||||||
|
| proceedings_unclassified | 166 |
|
||||||
|
|
||||||
|
注册摘要 1918 项仍全缺失。1743 项有明确对应的节目摘要:1717 项使用 `abstract_source=official_program_exact_metadata_correspondence`;新增 26 项使用 `official_program_reviewed_title_formatting_correspondence`,两者都逐字复制节目原文。报告分别列出 `abstracts_from_literal_program_correspondence` 和 `abstracts_from_reviewed_title_formatting`;保留的汇总字段 `abstracts_from_exact_program_correspondence` 为 1743,表示两类非模糊对应之和。
|
||||||
|
|
||||||
|
其余 175 项保持摘要缺失,包括 9 项节目本身没有摘要及 166 个版本待核条目。注册原始摘要仍标为缺失,不能把节目内容说成出版者摘要。
|
||||||
|
|
||||||
|
8 个作者仅有单一姓名字段,保留登记值并标记,不猜测 given/family;37 个含排版标记或实体的题名仍保留字面原文和原始标记。排版比较保留上下标字符,不改写原始字段。
|
||||||
|
|
||||||
|
完整2143个节目ID和4326个展示位置仍在匹配证据或391个未匹配节目中,28个原始track均保留。391个节目与166个出版版本待核来自两种观察口径,不能直接相加当成不同论文。Workshop、艺术、行业等贡献不能因为不在这个论文集范围就被当成拒稿、撤稿或不值得保留。
|
||||||
|
|
||||||
|
注册出版支持来源侧 `status=accepted` 的发表观察,`raw_decision` 明确不是原始签署决定;出版日期仅放在注册元数据中,不伪造决定日期。`mm_registry_program:crossref:type` 与 `mm_registry_program:mm_program:track` 分别保存注册书目类型和节目 track;共同的派生来源前缀满足写入器的来源校验,两个标签命名空间仍相互独立,原始平台字段与证据继续保留在各自元数据中,`research_type_classified=false`:书目类型不是研究类型,行业、艺术、demo条目不能默认变成主轨研究论文。
|
||||||
|
|
||||||
|
## 只读数据库预检与导入边界
|
||||||
|
|
||||||
|
每个分区都有带count/SHA256的manifest,coverage始终为partial。规范化完成后,对实际报告中的全部输出文件运行现有 `audit source-identities --files ... --pg-service foundry-research --pg-schema research --out <新目录>`,不能靠已有66个arXiv/DOI候选代表全目录。预检使用只读一致性快照,并保存来源/标识/历次投稿/逐次决定的证据包;缺失或截断必须单列。
|
||||||
|
|
||||||
|
`approved_for_import=0`,每项 `scientific_work_identity_approved=false`。DOI绑定可能有误,精确题名/作者匹配不证明科学作品身份,没有匹配也不证明新作品。进入增量导入前仍须审查已有绑定、科学版本与候选历史,做新鲜保护检查、完整基线/备份,再使用原生写入者。不得覆盖旧拒稿、后续录用、标签、原始元数据或人物姓名版本;既有已消费队列不可重放。
|
||||||
|
|
||||||
|
## 恢复
|
||||||
|
|
||||||
|
完整注册目录及独立节目比对恢复包:`/data/paper-index/validation/mm2026-container-registry-recovery-v2.tar.gz`,SHA256 `ddb05a8134f6437b3fb40250e45ed6350d55431d953812abda3300871b6dd0ac`。其中含注册源目录、父记录 `validation/mm2026-parent.json`、逐项比对和请求方法证据。安全解包后可把该父记录路径传给 `--parent-snapshot`。
|
||||||
|
|
||||||
|
节目原始来源恢复包见 [MM来源说明](mm2026-source-status.md)。本地独立核验备份位于 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`。固定回执/正文哈希变化、总量变化、未知注册上下文或节目对应改变都须重新核查,不能放宽为无条件接受。最终全库去重、近似题名作者、标识冲突、摘要、约束、覆盖与查询审计仍是独立必做阶段。
|
||||||
|
|
||||||
|
实际导入回归在隔离的 SQLite 和 PostgreSQL 测试库中运行全部1918条、全部11个输出分区,核验原始字段、1918个书目类型标签、1752个节目标签、publication_doi 身份、partial覆盖,以及已有作品的规范信息和旧拒稿决定不被覆盖。这些测试不会写入生产 research schema;测试通过仍不替代逐条生产身份审阅。
|
||||||
|
|
||||||
|
旧的 `native-v1`、`native-v2` 来源目录及已消费的导入队列不可覆盖或重放。新版本写入新的输出目录,先核对已导入记录的源字段和分轨是否变化;新增来源关联仍须新鲜身份预检、保护基线和独立验收。程序输出的分区预期数量已改变,未来增量 manifest 必须明确保留旧覆盖父值并声明新预期值,不能重置已导入计数。
|
||||||
@@ -0,0 +1,139 @@
|
|||||||
|
# ACM MM 2026 官方节目来源状态
|
||||||
|
|
||||||
|
2026-11-17 已冻结官网正式日程、技术日程、征稿/评审政策、两个日程脚本和脚本明确引用的 `program-data.json`。本页记录可恢复的来源证据,不宣称这些节目贡献全部都是已发表科研论文;总体仍为 13/17,第 14 步未完成。
|
||||||
|
|
||||||
|
## 已取得的数据
|
||||||
|
|
||||||
|
[官方技术日程](https://2026.acmmm.org/site/technical-programme.html)明确加载 `technical-programme.js`,该脚本读取 [program-data.json](https://2026.acmmm.org/site/program-data.json)。原始 JSON 为 10,808,209 字节,生成时间为 `2026-11-14T18:36:26.860677+00:00`,保存了 5 天、20 个房间、157 个日程场次。
|
||||||
|
|
||||||
|
递归盘点得到 **4,326 次论文样式的节目观察,涉及 2,143 个不同节目 ID**。额外 2,183 次是直接列表和子场次/多次展示中重复的观察。相同 ID 的标题、作者、摘要、track 和机构字段一致;所有来源位置均保留,不能把重复展示累加成新论文。
|
||||||
|
|
||||||
|
独立盘点的逐 ID 来源 track 数量如下。它们来自 `paper.track`;157 个外层日程条目的 `entry.track` 均为空,不能用外层空字段覆盖真实来源分类。展示形式和学术 track 也不是同一概念。
|
||||||
|
|
||||||
|
| 来源 track 原文 | 不同节目 ID 数 |
|
||||||
|
| --- | ---: |
|
||||||
|
| MRAC | 35 |
|
||||||
|
| SUMAC | 9 |
|
||||||
|
| NeuroMM | 10 |
|
||||||
|
| IMPACT | 8 |
|
||||||
|
| RichMediaGAI | 10 |
|
||||||
|
| MMSports | 16 |
|
||||||
|
| MUWS | 5 |
|
||||||
|
| EReL-MIR | 7 |
|
||||||
|
| GreenMM | 10 |
|
||||||
|
| DFF | 12 |
|
||||||
|
| Main Track | 1703 |
|
||||||
|
| Grand Challenge | 97 |
|
||||||
|
| Doctoral Symposium | 4 |
|
||||||
|
| Datasets | 50 |
|
||||||
|
| Interactive Art | 23 |
|
||||||
|
| Reproducibility | 1 |
|
||||||
|
| Demo and Video | 27 |
|
||||||
|
| Open Source | 14 |
|
||||||
|
| Industry Talk | 7 |
|
||||||
|
| BNI | 25 |
|
||||||
|
| QoEVMA | 4 |
|
||||||
|
| HUMA | 14 |
|
||||||
|
| WIPAI | 8 |
|
||||||
|
| AMI | 11 |
|
||||||
|
| Mulsemedia | 9 |
|
||||||
|
| UAVM | 11 |
|
||||||
|
| MCHM | 7 |
|
||||||
|
| JAV-CG | 6 |
|
||||||
|
|
||||||
|
共有 1,927 个节目 ID 带摘要,216 个缺少摘要。Main Track 的 1,703 个 ID 中,有两项同时缺少作者与摘要:
|
||||||
|
|
||||||
|
- `mfp4971`:ASDG: MLLM-based Audio Semantics for Dance Generation。
|
||||||
|
- `mfp7801`:Dynamic Late Fusion Multiple Kernel k-means Clustering based on Adaptive Sample Selection and Alignment。
|
||||||
|
|
||||||
|
两项仍有科学标题,原文没有明确撤稿或拒稿标签;因此只能标记字段缺失、暂缓关联,不能推断负面决定。非主轨的 workshop、grand challenge、行业报告、艺术、博士生等项目还需逐轨核对存档与贡献类型,不能把全部 2,143 个节目 ID 都计作已发表科研论文。原始公开联系邮箱不进入独立盘点、人物别名或论文标识。
|
||||||
|
|
||||||
|
## 公开负面决定的访问边界
|
||||||
|
|
||||||
|
[官方 CFP](https://2026.acmmm.org/site/cfp-guidelines.html)链接至 [OpenReview 群组](https://openreview.net/group?id=acmmm.org/ACMMM/2026/Conference)。匿名群组元数据可读,但根据群组给出的原始 submission invitation、rejected/withdrawn/desk-rejected venue ID,进行的四次 `limit=1` 匿名 notes 核查全部返回 **HTTP 403**。
|
||||||
|
|
||||||
|
保存了准确查询、时间、状态、响应原文和哈希;没有认证、重试或绕过权限。403 是已观察接口的访问限制,不代表零条拒稿、从未拒稿,或排除了其他可能公开的来源。评审公开规则也不能替代具体最终决定记录。当前不从评分、评审推荐、节目缺席或空字段制造 rejected/withdrawn 事件。
|
||||||
|
|
||||||
|
## 产物与恢复
|
||||||
|
|
||||||
|
原始目录(均相对 `/data/paper-index/collected/`):
|
||||||
|
|
||||||
|
- `mm2026-program-policy-research-v1`:六页官方日程和分轨/评审政策。
|
||||||
|
- `mm2026-navigation-research-v1`:官网明确加载的导航原文。
|
||||||
|
- `mm2026-program-data-discovery-v1`:两份脚本及公开群组元数据。
|
||||||
|
- `mm2026-program-and-public-probes-v1`:完整节目 JSON 和四个匿名访问限制回执。
|
||||||
|
|
||||||
|
独立盘点:`/data/paper-index/validation/mm2026-independent-program-inventory-v1.json`,包含每个节目 ID 的所有来源位置、track、标题/作者、摘要可用性及核心版本数量。全部响应原文、回执及盘点已在本机备份,并独立核验了原始字节哈希。
|
||||||
|
|
||||||
|
| 证据 | SHA256 |
|
||||||
|
| --- | --- |
|
||||||
|
| program-data.json | 08e2976ecba2d0d535000b0935f41c85076ec674e518a1df2c5843d9bde4d083 |
|
||||||
|
| 独立逐 ID 盘点 | 375dd975fdff023f2ff591d47066ec616c2bfc3b717b6016ab162d74d587f3e7 |
|
||||||
|
| 完整来源恢复包 | 8247149ed7437a36c13ca2b6128431e2b0f3f8a388a5ddefb62427b0c1ce6f8d |
|
||||||
|
|
||||||
|
服务器恢复包:`/data/paper-index/validation/mm2026-source-discovery-and-inventory-recovery-v1.tar.gz`;本地备份:`/Users/zzyang/Documents/Codex/2026-11-06/paper-index/` 同名文件。先安全解包并核验回执,再做规范化,不能重新抓取 403 入口作为恢复方式。
|
||||||
|
|
||||||
|
## 可复用采集命令
|
||||||
|
|
||||||
|
在已配置的服务器环境中运行,`--snapshots` 是包含 `mm2026-program-and-public-probes-v1/` 的父目录;恢复时先安全解包上述已核验恢复包。命令只读快照,不请求网络、不连接数据库,不重新访问403入口,也不下载PDF:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect mm2026-program \
|
||||||
|
--snapshots /data/paper-index/collected \
|
||||||
|
--out /data/paper-index/collected/mm2026-native-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
输出目录必须不存在;已存在、失败或部分输出均不能覆盖重跑,应先查看报告,另选新的目录。`collection-report.json` 仅在全量验证成功后生成;失败保留 `failed.json`,不冒充完成。
|
||||||
|
|
||||||
|
采集器核验固定回执、正文哈希、URL、抓取时间、完整逐轨数量和字段缺失盘点。28个逐轨 JSONL 保留原始标题、作者、机构、逐字摘要以及同一节目ID的全部展示位置、时间、poster/oral号和展示kind;157个场次及子场次/非论文活动另存 `sessions.jsonl`。外层空track不覆盖 `paper.track`,不同核心版本、未知字段或未处理嵌套需要重新审查,不能静默丢弃或取第一条。公开联系邮箱数组只留在原始证据中,不进入规范记录。`evidence/` 保留原始节目JSON、完整回执及四个既有403响应。
|
||||||
|
|
||||||
|
这些文件的格式为 `mm_program_observation_v1_not_ingestion_record`,所有项 `approved_for_import=false`。节目ID仅为 `mm_program:program_id` 来源定位符,track标签使用 `mm_program:track`;不建立科学作品、原始投稿ID或最终决定,不推断 DOI/arXiv/OpenReview 标识、论文类型、拒稿或撤稿。行业报告、艺术、workshop及其他来源贡献均保留,尚需逐轨核对贡献类型与正式出版依据。
|
||||||
|
|
||||||
|
本批尚未形成生产导入队列。后续做贡献类型、作品/出版身份及既往投稿审查,再生成经批准的增量队列。已消费的其他会议导入队列不可重放;最终全库去重和完整审计仍为独立必做阶段。
|
||||||
|
|
||||||
|
## 直接审计节目身份与既往历史
|
||||||
|
|
||||||
|
全量节目观察可以直接进入只读候选审计,无需伪造成录用论文队列:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
PGSERVICEFILE=/etc/paper-atlas/pg_service.conf \
|
||||||
|
paper-atlas audit source-identities \
|
||||||
|
--mm-program-report /data/paper-index/collected/mm2026-native-v1/collection-report.json \
|
||||||
|
--pg-service foundry-research --pg-schema research \
|
||||||
|
--out /data/paper-index/collected/mm2026-preflight-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
`--mm-program-report` 与常规 `--files` 互斥。审计先核对冻结原文、全部28轨的逐文件哈希/数量,以及每个节目ID的完整字段和展示上下文;缺轨、重复ID、篡改后的标识/决定或上下文均在数据库连接前拒绝。报告中记录的绝对文件路径仅作来源证据,实际读取报告同目录中的对应文件。
|
||||||
|
|
||||||
|
候选审计中的 `status=unknown` 仅为内部查询占位符,不代表原始提交的最终状态。节目ID以 `mm_program` 来源键查询,不转成全局 DOI、arXiv 或 OpenReview 标识;保留原始 `listed_in_official_program` 观察与全部展示位置。所有贡献类型、出版身份、最终决定核验标志均为false;缺作者的两项也保留并标记。
|
||||||
|
|
||||||
|
查询使用 PostgreSQL 只读一致性快照,候选作品的原始来源、标识、历次投稿和逐次决定压缩保存并记录哈希;截断会显式标记。标题/作者候选匹配不是作品身份批准,没有匹配也不证明是新论文。既往拒稿和后续录用不能互相覆盖,评分或评审推荐不能充当最终决定。输出目录必须不存在;遇到已有输出或静默运行,先检查具体单元和报告,不重启重放。
|
||||||
|
|
||||||
|
## 已完成的规范化、只读审计与出版核对
|
||||||
|
|
||||||
|
采集器提交 `c8db611` 和直接节目审计提交 `09a2d4e` 已通过PR #65、#66由用户合并。授权远程runner的535/543项测试、wheel构建安装和安装后全量命令检查通过;没有本机测试或构建。服务器规范化和只读候选报告也已独立验收,并在本机备份、复核全部2143项与814个候选证据包。
|
||||||
|
|
||||||
|
| 只读候选分类 | 来源节目项数 |
|
||||||
|
| --- | ---: |
|
||||||
|
| 标题与完整作者多重集合一致,仍需作品核查 | 660 |
|
||||||
|
| 仅部分作者重叠 | 153 |
|
||||||
|
| 无精确标题候选,不证明是新作品 | 1329 |
|
||||||
|
| 有标题候选但无作者一致性 | 1 |
|
||||||
|
|
||||||
|
全部4326个展示上下文一致,814个证据包均无截断。24篇候选作品涉及24次已有负面投稿、37条负面来源观察;37不能直接算作37次拒稿。所有候选均尚未批准身份或入库,旧负面决定必须在之后关联时保留。
|
||||||
|
|
||||||
|
对已有作品的原始arXiv记录、标题/完整作者多重集合和MM2026语境进一步筛查,得到67个 Main Track 出版DOI核对候选。独立查询其明确给出的DOI及父论文集,共68次单次请求,67次HTTP200、1次404,没有认证或重试。父注册记录为[第34届ACM Multimedia论文集](https://doi.org/10.1145/3767308),ISBN `9798400722134`,出版者ACM,出版年2026。
|
||||||
|
|
||||||
|
其中66项的原始arXiv引用DOI与注册DOI一致,按既有审计规则规范化的标题、完整作者多重集合、出版年、出版者、明确会议名和论文集标题也一致。注册文章未提供ISBN,不能伪称逐篇ISBN已核验;归属依据是文章明确给出的会议和论文集字段,以及独立的原始arXiv语境和DOI。仍需逐项作品/版本审查及新鲜数据库保护检查,**不是66项已入库或最终身份批准**。
|
||||||
|
|
||||||
|
`mfp7067` 对应既有候选作品2749947,原始arXiv提供的 `10.1145/3767308.3836169` 返回404,暂缓。404不是撤稿/拒稿证据,也不据此删除或修正既有标识;需独立核对后再处理。
|
||||||
|
|
||||||
|
完整会议出版名单仍需继续核对。实际ISBN查询只返回父论文集一个条目,父DOI的公开Unixref XML也只含该父DOI、0个 `conference_paper` 节点;都不能代表逐篇名单。按会议名检索得到的 `total-results` 是出版者全年范围内的宽泛检索量,不能当作MM篇数;必须逐项核对明确会议/论文集字段、节目与分轨归属,并保留未匹配项。全部28个来源track继续保留,不能把本批Main Track候选当成会议完整覆盖。
|
||||||
|
|
||||||
|
| 已独立验证的恢复证据 | SHA256 |
|
||||||
|
| --- | --- |
|
||||||
|
| 全轨规范化恢复包 | b119081618b8ce98037dc139be159512f9fdb86a0ccfb127c8b8a1c5722f3d87 |
|
||||||
|
| 只读候选/814证据包恢复包 | 42d0d0404bba04e64e638344b050ad4e7287c3e4c75a777b7069ecc7d7a6ed60 |
|
||||||
|
| 67项DOI及父论文集原始注册回执恢复包 | dd7a0ff76f12070f403cc7352ac54269e454758d807896a086c58c1c1ac7b798 |
|
||||||
|
|
||||||
|
服务器恢复包位于 `/data/paper-index/validation/`,分别名为 `mm2026-native-recovery-v1.tar.gz`、`mm2026-preflight-recovery-v1.tar.gz`、`mm2026-publication-metadata-recovery-v1.tar.gz`;本地同名备份在 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`。来源目录为 `collected/mm2026-native-v1`、`mm2026-preflight-v1`、`mm2026-publication-metadata-v1`。先核对完整manifest及实际单元结果,不能用旧进度文件的running字样重启。其他会议已消费的导入队列仍禁止重放;本轮均为来源核查、只读候选审计,没有生产写入。
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
# RecSys 2026 公开录用贡献采集
|
||||||
|
|
||||||
|
这是长期维护项目中的离线采集器,按需处理冻结的官方元数据,不连接数据库、不访问论文链接或下载 PDF。输出须经过作品、DOI 和投稿历史审查才能导入。
|
||||||
|
|
||||||
|
## 使用
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect recsys2026-program \
|
||||||
|
--snapshots /data/paper-index/collected \
|
||||||
|
--out /data/paper-index/collected/recsys2026-normalized-<new-run>
|
||||||
|
```
|
||||||
|
|
||||||
|
输入父目录应包含 `remaining2026-listings-research-v2/manifest.json` 与 `recsys-contributions.html`。采集器检查固定清单 SHA256、官方 URL、HTTP 200、采集时区、原始字节哈希、九个 track 的菜单及完整条目数。现有输出目录不可覆盖;失败保留 `failed.json`,成功才发布 `collection-report.json`。再次运行前先检查原任务是否结束。
|
||||||
|
|
||||||
|
## 数据范围与重复项
|
||||||
|
|
||||||
|
来源是 [RecSys 官方录用贡献页面](https://recsys.acm.org/recsys26/contributions/),冻结时间为 2026-11-17。九个来源 track:
|
||||||
|
|
||||||
|
| track | 观察数 |
|
||||||
|
| --- | ---: |
|
||||||
|
| long | 65 |
|
||||||
|
| short | 29 |
|
||||||
|
| past_present_future | 13 |
|
||||||
|
| reproducibility | 4 |
|
||||||
|
| resource | 7 |
|
||||||
|
| industry | 83 |
|
||||||
|
| research_practice_notes | 34 |
|
||||||
|
| demo | 18 |
|
||||||
|
| doctoral_symposium | 10 |
|
||||||
|
|
||||||
|
**263 项来源观察对应 260 个不同 DOI,并非 263 篇不同论文。** 每条记录用官方页面中的 accordion locator 标识,分别保留来源标题、作者原文、摘要和位置,不假设 locator 是原始投稿 ID。
|
||||||
|
|
||||||
|
以下 DOI 各出现两次,六个观察均显式标记待审查:
|
||||||
|
|
||||||
|
- `10.1145/3773078.3831839`:Dart 两条来源记录的标题、作者和摘要都有差异;不得静默选一个覆盖另一个。
|
||||||
|
- `10.1145/3773078.3831819`:标题与作者相同、摘要版本不同。
|
||||||
|
- `10.1145/3773078.3831861`:重复展示的文字相同,保留两个来源位置。
|
||||||
|
|
||||||
|
所有观察都有公开摘要。正文只转换为 HTML 展示文字并规范空白,原始页面完整保留;遇到未审查的数学或图片结构会停止,不把上标扁平化成数字。作者按来源的逗号和最后一个 and 分隔,原始名单同时保留;这不是已验证的作者人物身份或别名。
|
||||||
|
|
||||||
|
`accepted` 仅指官方页面明示录用,不是原始签名评审决定。DOI 来自官方超链接,尚未独立核查实时出版商版本;归入 `publication_doi`,不复制其他会议的投稿 ID。track 明确来自页面区块;`recsys_program:contribution_type` 仅保存页面原始类型标签(Research、PastPresentFuture、Doctoral Symposium),各 track 的 RES/IND/RPN 等缩写同时保留在来源字段,不统一推断研究方法类型。
|
||||||
|
|
||||||
|
本页只含录用贡献,不能据此推断未出现的稿件被拒。公开拒稿导出仍为 unknown,征稿页的 desk rejection 政策不等于具体拒稿记录。不得将本批采集成功当成会议全部来源、负面决定或全库去重完成。
|
||||||
|
|
||||||
|
## 输出与恢复
|
||||||
|
|
||||||
|
输出九份 JSONL、对应部分覆盖清单、原始 HTML 和清单证据,以及记录观察数和不同 DOI 数的报告;批准导入数为零。
|
||||||
|
|
||||||
|
清单 SHA256:`fea7b710587636db06794ee195cd083634c41e9054a217859d9dd0ccdd1afab9`。
|
||||||
|
|
||||||
|
服务器恢复包 `/data/paper-index/validation/remaining2026-listings-research-v2-recovery.tar.gz`,本地备份 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/remaining2026-listings-research-v2-recovery.tar.gz`,SHA256 为 `f5ef7c2e73d46712146e4d6711ad5ad562e16b8b55c0cd75d1041cd0b81142cb`。安全恢复原始目录、核对哈希后重跑上述命令。
|
||||||
|
|
||||||
|
完整来源与独立 263 项基准已保存在测试夹具,所有测试、构建和安装验证交给指定的 `ces-control:host` runner。生产导入前还需独立核验规范化结果,并检查当前数据库已有 DOI 绑定及历次投稿。最终全库去重审计仍是独立必做阶段。
|
||||||
|
|
||||||
|
## 2026-11-17 PostgreSQL 补录验收
|
||||||
|
|
||||||
|
采集器仍只生成待审查文件;本节记录随后单独完成的生产补录。47 条来源记录已关联到 47 篇既有作品,依据是已绑定的同一发表 DOI、规范化标题、完整作者集合,以及原始 arXiv 来源明确引用的同一 DOI。逐条当前资料和历史包已与只读预检核对;没有新增论文实体、人物别名、arXiv 标识或原始 OpenReview ID。
|
||||||
|
|
||||||
|
| track | 已导入观察 / 来源应有观察 |
|
||||||
|
| --- | ---: |
|
||||||
|
| demo | 1 / 18 |
|
||||||
|
| doctoral_symposium | 2 / 10 |
|
||||||
|
| industry | 18 / 83 |
|
||||||
|
| long | 13 / 65 |
|
||||||
|
| past_present_future | 2 / 13 |
|
||||||
|
| reproducibility | 1 / 4 |
|
||||||
|
| research_practice_notes | 4 / 34 |
|
||||||
|
| resource | 2 / 7 |
|
||||||
|
| short | 4 / 29 |
|
||||||
|
|
||||||
|
九个 track 全部保持 **partial**。总计 47/263 个来源观察已导入,216 个待审查,其中包括三组重复 DOI 的六项观察。来源的 263 项观察对应 260 个不同 DOI,不能把观察数量当成不同科学论文数量。
|
||||||
|
|
||||||
|
本批新增一个来源、九个 edition、47 条来源记录、47 次投稿/47 条录用观察、47 个 `publication_doi` 投稿标识及47条来源类型标签证据关联。源摘要分别保存,47 篇既有作品的规范摘要保持不变;原始程序字段、来源链接、时间和类型标签也逐条核验。Research/PastPresentFuture/Doctoral Symposium 仍属于来源 namespace,不强制映射成统一研究方法类别。
|
||||||
|
|
||||||
|
独立验收确认:5,812 篇受保护旧作品及其旧资料、来源、标签、标识不变,全部旧投稿/决定原文与关联历史不变;foundry-core `/readyz` 为 200,受保护 SQLite 备份大小/只读权限不变。论文实体仍为 3,567,888;不能用 `max(id)` 代替行数。
|
||||||
|
|
||||||
|
- 完整验收:`/data/paper-index/collected/recsys2026-import47-v1/postgresql-postcheck-v1.json`,SHA256 `7c7e2dcd06bdbc839666740a989c1f187fa47290800a9055cf4540be64336cf4`。
|
||||||
|
- 保护基线:`/data/paper-index/collected/recsys2026-import47-baseline-v1/baseline.json`,SHA256 `aa86d51ff18774a505b1c59510ad867c6221c152334dc4983d31c5290cb8ac0f`。
|
||||||
|
- 导入前完整队列/基线恢复包:`/data/paper-index/validation/recsys2026-reviewed47-recovery-v1.tar.gz`,SHA256 `ae054a6447e8c2864fd8e365867cd3308dee974f375fafff5653fff2a51044e1`;本地同名备份位于 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`,已核验全部 47 个原始身份包、九份输入和基线。
|
||||||
|
|
||||||
|
执行代码固定在提交 `8c147f5e6edda900e2eed19372ab031f661b31c5` 的独立验证目录,85 个包文件校验通过;24 个写入关键文件与此前已验收版本一致。授权 `ces-control:host` runner 的 CI1230/job1327通过525项测试及安装后实际RecSys263项/260DOI验证。没有改变服务器默认运行代码目录或共享数据库配置。
|
||||||
|
|
||||||
|
**九份输入均已消费,禁止重放。** 恢复时先检查实际 PostgreSQL、执行回执和终态验收;不能因日志安静、观测超时或重新启动 Goal 就复跑。后续216条须另建审查与增量队列,并重新取当前基线。本批通过不等于整届 RecSys、公开拒稿覆盖或最终全库去重已完成。
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
# ReviewArena 镜像的索引字段投影
|
||||||
|
|
||||||
|
ReviewArena 是公开评审镜像,固定版本为 `c2978add17c2099219eaddbc2599974d69d4d09b`。上游数据卡描述七类会议、2020–2026 年的部分来源;这不是逐会议、逐年度的公开决定全集。文件清单有 15 个 Parquet 分片,覆盖 COLM、CoRL、EMNLP、ICLR、ICML、NeurIPS、TMLR。这里不包括 UAI,不能用它填补 UAI 缺口。
|
||||||
|
|
||||||
|
- [固定版本上游数据卡](https://huggingface.co/datasets/anonymousNeurIPS2026submission4281/reviewarena/blob/c2978add17c2099219eaddbc2599974d69d4d09b/README.md)
|
||||||
|
- [固定版本文件元数据 API](https://huggingface.co/api/datasets/anonymousNeurIPS2026submission4281/reviewarena/tree/c2978add17c2099219eaddbc2599974d69d4d09b/data?recursive=false&expand=false)
|
||||||
|
- [Arrow ParquetFile 投影接口](https://arrow.apache.org/docs/python/generated/pyarrow.parquet.ParquetFile.html)
|
||||||
|
|
||||||
|
## 运行
|
||||||
|
|
||||||
|
仅在已授权远程 runner 安装 `paper-atlas[mirrors]`,使用固定 PyArrow 26.0.1 二进制 wheel。本机不执行安装、构建或项目测试。常规 CI 包含投影、安全边界和镜像决定语义测试;真实网络采集由独立的 `ReviewArena metadata projection` workflow 手动触发,指定已通过 CI 的代码 ref 和 split,例如 `emnlp`。该 workflow 仍使用 `ces-control` / `aliyun-ces-control`,在服务器固定数据目录生成输出和恢复归档,不更改默认 runtime 或数据库。
|
||||||
|
|
||||||
|
包内命令如下,`--out` 必须是既有父目录内的新目录:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect reviewarena-metadata \
|
||||||
|
--split emnlp \
|
||||||
|
--out /data/paper-index/collected/reviewarena-emnlp-reviewed-version
|
||||||
|
```
|
||||||
|
|
||||||
|
输出目录已存在时拒绝执行。遇到禁止访问、来源变化或读取失败,保留失败报告,不自动重试或切换到整文件下载。不能把原始访问拒绝解释为没有拒稿。
|
||||||
|
|
||||||
|
已保存的完整元数据投影可以离线重新解释决定字面值。提供原 `collection-report.json` 及其已独立验证的精确文件 SHA-256,程序核对全部元数据文件哈希、逐行来源身份和原始字段,生成新目录;不会再次访问上游 Parquet:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect reviewarena-metadata --split emnlp \
|
||||||
|
--from-projection /data/paper-index/collected/<既有投影>/collection-report.json \
|
||||||
|
--projection-sha256 <已验证的原报告SHA256> \
|
||||||
|
--out /data/paper-index/collected/<新解释版本>
|
||||||
|
```
|
||||||
|
|
||||||
|
手动 workflow 对应输入为 `projection_name` 与 `projection_sha256`。离线报告的 `metadata_bytes_read` 为本次请求字节数 `0`,`source_projection_bytes` 保留原始列块投影字节数;原报告哈希建立恢复链,原始记录与来源定位不变。
|
||||||
|
|
||||||
|
## 实际读取和保存范围
|
||||||
|
|
||||||
|
仅投影以下字段:`forum_id`、`conference`、`year`、`track`、`venue_id`、`paper_number`、`title`、`abstract`、`authors`、`keywords`、`venue`、`decision`。
|
||||||
|
|
||||||
|
程序先读取文件魔数、尾部长度和有界 Parquet 页脚,在内存中解析 schema 和列块偏移,然后只请求这些列的字典/数据页。Arrow 可能请求较大的页脚窗口,未获准区域由本地稀疏文件合成零字节,不会为该窗口补下载相邻正文页。逐个选中列块还要检查不与未选列块重叠。单文件上限为 64 MiB 所需字节、512 个范围请求和 100,000 行;响应必须是准确的 HTTP 206 / Content-Range,ETag 变化时停止。该边界适用于每个分片,不能据此启动其他分片的未授权全文件下载。
|
||||||
|
|
||||||
|
不读取 `markdown`、`reviews_json`、`decision_comment`、`author_rebuttal` 等正文或评审内容的数据页;不保存原 Parquet、页脚原文或未选字段统计值。保存的 JSONL 只有上述索引字段、来源定位和审计标记。报告记录每个请求范围及其 SHA-256、选中列名、页脚哈希、投影数量和保存文件哈希,便于独立复核。
|
||||||
|
|
||||||
|
上游 LFS SHA-256 与大小是固定版本 API 提供的对象声明。由于不下载整个 Parquet,本流程**不声称验证了完整 Parquet 内容哈希**。范围对象大小/ETag 和各段哈希用于检查本次读取的一致性。
|
||||||
|
|
||||||
|
## 决定和身份边界
|
||||||
|
|
||||||
|
`mirror_decision_category` 只解释明确的镜像字面值;未知、缺失或未审核的决定保持 `unknown`,不会根据 venue、是否录用、摘要缺失或目录缺席推断拒稿。类别数量是镜像行数,不能解释为独立作品数量或原始官方决定数量。
|
||||||
|
|
||||||
|
真实 EMNLP 2023 固定分片包含 2,009 行:原始决定为 `Accept-Main` 994 行、`Accept-Findings` 1,015 行,原始 `track` 却全部为 `main`。两种已核对字面值属于镜像录用声明;`mirror_decision_track_claim` 分开保留其明确的 main/findings 声明,原 `track` 和所有 `raw_fields` 不被重写。`Accept (Poster)` 等展示形式不能自行推出主会归属。此分片没有明确负面决定,不能用于补齐拒稿缺口,更不能由此断言 EMNLP 原始平台没有公开拒稿。首次实际投影仅请求 1,720,498 字节;原文件 90,794,691 字节,未下载完整文件。
|
||||||
|
|
||||||
|
上游 `forum_id`、`venue_id`、投稿号、作者和决定均作为原始声明保留。`forum_id` 不直接成为全局 OpenReview 标识;稳定行定位由数据集、固定 commit、文件和行号组成。输出 `identifiers` 为空,原始论坛身份和决定签名标记为未验证,保留匿名和缺失作者。源数据中的字符串命名不能被当成真实人员别名。
|
||||||
|
|
||||||
|
输出故意不是可直接 `import records` 的规范化队列,没有授予导入或合并权限。下一步必须检查原始来源、跨来源重合和作品身份,再决定哪些来源观察能进入生产。即使明确镜像拒稿,也要保留该投稿历史并继续查找后续其他会议的录用;不能把“目前没有关联录用”表述为从未录用。本采集不代替最终全库去重、决定冲突、来源覆盖和查询验收。
|
||||||
@@ -0,0 +1,97 @@
|
|||||||
|
# SDM 2026:生产规范化与身份核查记录
|
||||||
|
|
||||||
|
本页记录 2026-11-17 的来源验收、早期只读核查和随后的生产导入。**当前 53 条注册记录中,25 条已入库并独立验收,28 条仍待处理;没有新增论文实体。** 下文早期报告保持原始状态,不将它们改写成事后批准;最终结果见“25 条生产导入验收”。
|
||||||
|
|
||||||
|
## 已核实的生产输出
|
||||||
|
|
||||||
|
生产规范化使用通过远程 CI 1216 的源码版本 `f2753bc741d0fff06134cf64019dac38002fae37`,对应 494 项测试、wheel 构建及安装后 CLI 检查。该版本包含已合并的 SDM 适配器;这里没有替换服务器默认代码或运行环境。本页是文档更新,不能把此前 CI 结果当作本次提交的 CI 结果。
|
||||||
|
|
||||||
|
`/data/paper-index/collected/sdm2026-normalized-v1` 保留:
|
||||||
|
|
||||||
|
- 53 条 Crossref 论文章节记录,全部没有注册摘要;排除论文集和前言。
|
||||||
|
- 53 条独立日程观察,其中 52 条有公开摘要;20 处作者序列差异仍保留。
|
||||||
|
- 取消展示的条目与已注册发表记录分别保留,不生成撤回决定。
|
||||||
|
|
||||||
|
规范化服务 `paper-atlas-sdm2026-normalize-v1.service` 的 invocation 为 `3021248cae4044ac88eb80fb8b964a26`,终态成功。独立审计逐条核对冻结注册字段、完整作者顺序、原始日程和摘要证据,未以适配器自身统计替代验收。
|
||||||
|
|
||||||
|
| 证据文件 | SHA-256 |
|
||||||
|
| --- | --- |
|
||||||
|
| validation/sdm2026-normalized-v1-audit.json | 66d4e698ff8828731ccb657e273fb2f0d19cbce8fd91ade2c81ed68124c6d3a3 |
|
||||||
|
| collected/sdm2026-normalized-v1/sdm-crossref-registry-2026-unclassified.jsonl | 7edd78170ced9b79e4405eb16d63f8a9a38d38dfc906e892580f0ac4be165b2c |
|
||||||
|
| collected/sdm2026-normalized-v1/program-observations.jsonl | 421a7187369817fa90aa3484575e8a8a2f3e008b9d152bed6beca678b1e12204 |
|
||||||
|
|
||||||
|
表中服务器相对路径均位于 `/data/paper-index/`。本机工作记录根目录为 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`,保存同名输出及审计备份;数据证据不作为项目源码提交。
|
||||||
|
|
||||||
|
## 当前数据库只读预检
|
||||||
|
|
||||||
|
`paper-atlas-sdm2026-preflight-v1.service`,invocation `9be7227930354633abf8f41cdbd0dd73`,终态成功。数据库为 `foundry_data.research`,快照时间 `2026-11-17 08:34:34.270409+00`。报告核实事务为 READ ONLY,`database_written=false`、`approved_for_import=0`。
|
||||||
|
|
||||||
|
| 候选类别 | 记录数 | 含义 |
|
||||||
|
| --- | ---: | --- |
|
||||||
|
| 精确标题与作者集合一致 | 26 | 需要进一步核对作品、版本和已有标识 |
|
||||||
|
| 精确标题但仅部分作者重合 | 2 | 作者版本冲突尚未解决 |
|
||||||
|
| 没有精确匹配 | 25 | 不是新作品证明,不能直接创建论文 |
|
||||||
|
|
||||||
|
报告位于 `collected/sdm2026-preflight-v1/postgresql-preflight.json`,SHA-256 为 `eb0cc45fa5bf01159133af0536ba6621a2403914ccd9591a5d1a6a230bc3791e`。28 个候选论文压缩证据包的哈希均核对通过;来源记录、投稿、决定及其来源关系均未触发报告的截断标志。这里的完整性仅限这些候选包,不代表全库审计完成。
|
||||||
|
|
||||||
|
随后逐项阅读来源摘要、既有摘要、注册作者和已有标识,为 25 个候选记录了支持同一作品的具体方法、数据集、实验结果或摘要一致性证据,仍需最新数据库保护基线和导入前检查。相似度只用于安排审阅顺序,不作为自动批准阈值。以下三条保留作者版本问题:
|
||||||
|
|
||||||
|
| 章节 DOI 尾号 | 既有 paper_id | 尚存问题 |
|
||||||
|
| --- | ---: | --- |
|
||||||
|
| .10 | 2656770 | 注册姓名 Tiejun Chen 与既有及日程姓名 Tiejin Chen 不同;不生成别名 |
|
||||||
|
| .15 | 2745754 | 注册 Sarang Patil 与既有 Sarang Rajendra Patil 不同;日程另有作者顺序与姓名缩写变化 |
|
||||||
|
| .49 | 2506693 | 注册和既有完整作者集合相同,但日程列出不同名单;ScaffAug 内容一致性仍不能消除来源矛盾 |
|
||||||
|
|
||||||
|
这里的尾号均属于 `10.1137/1.9781611979329`。逐条内容审阅文件为本机工作记录根目录的 `sdm2026-candidate-evidence-review-v1.json`,SHA-256 为 `6ff791f87eafeb50fa853766c632e91e3f0377ab3f619365117d557f6d56100d`。其中 25 条为 `same_work_supported_pending_fresh_import_guards`,3 条为 `hold_author_version_conflict`;所有条目的 `approved_for_import` 仍为 false。不能据此直接重放来源、绑定 DOI 或拼接日程摘要。
|
||||||
|
|
||||||
|
## 需要保护的较早拒稿历史
|
||||||
|
|
||||||
|
| SDM 候选作品 | paper_id | 已有较早决定 |
|
||||||
|
| --- | ---: | --- |
|
||||||
|
| SeBA | 2672776 | ICLR 2026 rejected |
|
||||||
|
| Towards Reliability of Parameter-Free Optimization | 3282961 | ICLR 2025 rejected |
|
||||||
|
| Scaffold-Aware Generative Augmentation and Reranking for Enhanced Virtual Screening | 2506693 | ICLR 2026 desk_rejected |
|
||||||
|
|
||||||
|
前两条在后续生产验收中各自新增独立 SDM 发表事件;第三条仍保留作者版本问题,未导入。旧投稿、旧决定及各自来源均完整保留。两个来源对同一拒稿的观察不能直接统计成两次独立投稿或拒稿。不能覆盖旧状态,也不能因这批录用候选就宣称完整投稿历史已补齐。
|
||||||
|
|
||||||
|
## 未精确匹配的补充检索
|
||||||
|
|
||||||
|
对另外 25 条运行了有时限、数量上限和只读快照的补充检索,使用现有索引,不新增数据库索引或改变全局配置。
|
||||||
|
|
||||||
|
- 全文检索 `paper-atlas-sdm2026-title-variants-v1.service`,invocation `29c5d2a288804ce7a9c3e2f7db4dca6f`,终态成功。每项使用完整标题及明确记录的锚点词,8 条至少有一次查询达到候选数量上限,不能据此排除重复。
|
||||||
|
- 标题前缀范围检索 `paper-atlas-sdm2026-title-prefix-v2.service`,invocation `3087f1c0f252427485a661a63c50b27f`,终态成功。25 条均使用既有标题索引,没有查询错误或数量截断。18 字符前缀只是补充方法,不是全库模糊匹配验收,也不会消除前一轮查询的限制。
|
||||||
|
|
||||||
|
全文候选中的既有论文 `2025464` 与 SDM .9 共享作者,但前者讨论 CLOUD、离散/混合/连续变量,后者讨论 SLICE、类别数据与 HCR 模型。标题前缀候选 `938886` 与 SDM .28 只共享一名作者,既有版本讨论随机游走相似度、有效电阻和社交网络推断。不能凭研究主题或作者重合将这些论文合并。25 条仍保留为待进一步查证,没有获准创建新作品。
|
||||||
|
|
||||||
|
| 服务器恢复文件 | SHA-256 |
|
||||||
|
| --- | --- |
|
||||||
|
| validation/sdm2026-title-variants-v1-recovery.tar.gz | 1f387f2ff4d5684aa72abd419bddeb7e972624f6a56f6e9a943168aa944b9137 |
|
||||||
|
| validation/sdm2026-title-prefix-v2-recovery.tar.gz | 4941343bd6152c5046778f7c82588e3c6ad41c9bb938e0107f8b2b426ac10772 |
|
||||||
|
|
||||||
|
两个原报告的 SHA-256 分别为 `b6ab2e9228b8e8e9548b27df70834faf25bfda551844979f8410a74b6e6aadd0` 和 `c1f902b4e4f33997eb182b10dc78332e182ceb6051f1a7c14b21465c8878263d`。前者保留 1 个进一步审阅的完整候选包,后者保留全部 113 个前缀命中的候选包;这些数量不是论文批准数。
|
||||||
|
|
||||||
|
## 25 条生产导入验收
|
||||||
|
|
||||||
|
`paper-atlas-sdm2026-import25-v1.service`,invocation `003b490a2f2c449da078a2f8625b31b8`,终态成功。执行前核验 81 个源码文件,并确认 24 个核心写入文件与此前已验收的 canonical writer 相同;复用原生 `paper-atlas import records`,未修改默认运行目录。
|
||||||
|
|
||||||
|
- 新增 25 条来源、25 次会议事件、25 条发表决定观察及 25 个 `publication_doi` 投稿标识,新增论文实体为 0。
|
||||||
|
- 24 篇使用已核实的 arXiv 标识补注;AdamG 没有 arXiv 标识,采用单独审阅并实时守护的唯一精确标题、四名完整作者和具体方法证据。旧 OpenReview ID 未复制为 SDM 投稿标识。
|
||||||
|
- 28 条待处理记录全部排除;25 个注册来源摘要仍为空,52 份日程摘要保持独立来源版本。覆盖状态为 `partial`,应有 53 条、当前导入 25 条,track 保留 `proceedings_unclassified`。
|
||||||
|
- 全局旧原始投稿与决定历史、5,653 篇受保护论文的原资料/来源/标签/既有标识均通过导入后比较。SeBA、AdamG 的两次较早 ICLR 投稿及四条 rejected 来源观察保留,没有将四条观察说成四次投稿。
|
||||||
|
- 约束有效、实际全文查询与 GIN 计划通过;foundry-core `/readyz` 返回 200,受保护 SQLite 快照保持只读。
|
||||||
|
|
||||||
|
| 生产证据 | SHA-256 |
|
||||||
|
| --- | --- |
|
||||||
|
| collected/sdm2026-import25-v1/postgresql-postcheck-v1.json | 0c861722f0d455462a2686a42d634e0b963d075cf58f90fa9f48e1ae1fc82c27 |
|
||||||
|
| collected/sdm2026-import25-baseline-v1/baseline.json | 42ffbaf6f843a91ebf64f0bfefa4af36f6ac72f044b0e8587e65c5e6648c0931 |
|
||||||
|
| collected/sdm2026-prepared25-v1/independent-preparation-check.json | 8ea8eb59f4b1601a484d1bc0535b289cb950b83747ed5f82b0eac55e527c4a7c |
|
||||||
|
| validation/sdm2026-reviewed25-recovery-v1.tar.gz | 13447944ec34e0e2a6eee07a1d9820bfb8066756aa11016673e416c0d6f8b2b2 |
|
||||||
|
|
||||||
|
恢复包包含 263 个文件;本机已独立核对队列、70 份内容寻址证据和 28 个早期候选证据包。**25 条队列已消费,禁止重放。** 28 条剩余记录需要新的导入后快照;先前候选包不能当作当前数据库状态。后续批次须沿用最新 5,653 篇保护基线,并加上各自新目标。
|
||||||
|
|
||||||
|
## 下次恢复
|
||||||
|
|
||||||
|
1. 先核对实际服务终态、来源及报告哈希,再读取上述证据;终态成功的研究任务无需重启。
|
||||||
|
2. 保留已验收的 25 条,不重放;继续解决 3 条作者版本问题和 25 条未精确匹配记录;不把缺少候选当作新作品证明,不把源端 403 当作零条公开拒稿。
|
||||||
|
3. 对后续新获内容支持的候选另做新快照检查:作品与 DOI 绑定、来源是否已导入、已有会议事件及全部历史保护。建立新的明确入库队列和可恢复备份后再执行写入,不能直接使用本次候选报告当导入计划。
|
||||||
|
4. 导入后逐项核验新事件、旧历史、来源关系和精确数量变化;注册摘要缺失仍如实保留。完整来源范围结束后另做全库去重、标识冲突、近似标题/作者、覆盖、摘要、约束和查询验收。
|
||||||
@@ -42,3 +42,5 @@ paper-atlas collect sdm2026-proceedings \
|
|||||||
服务器恢复包:`/data/paper-index/validation/sdm2026-sources-recovery-v1.tar.gz`,SHA-256 为 `d6d14c0e4bb2c26e16a4888e3b150e44f982d689351638b1b9a28644736c4387`;本机工作记录目录有同名备份。包内保留原始来源、独立盘点及逐页版本比较记录,含义是可复现来源证据,不是已完成导入或身份批准。
|
服务器恢复包:`/data/paper-index/validation/sdm2026-sources-recovery-v1.tar.gz`,SHA-256 为 `d6d14c0e4bb2c26e16a4888e3b150e44f982d689351638b1b9a28644736c4387`;本机工作记录目录有同名备份。包内保留原始来源、独立盘点及逐页版本比较记录,含义是可复现来源证据,不是已完成导入或身份批准。
|
||||||
|
|
||||||
下次继续时,先核对来源版本和访问状态,再做当前数据库的 DOI 绑定、标题/作者近似候选与完整投稿历史核查,逐条解决作者及摘要版本问题。批准后另建入库队列、保护基线和导入后验收;所有来源结束后仍需全库去重、覆盖、摘要、约束和实际查询验收。测试、构建和安装验证只在已授权 `ces-control:host` 远程 runner 执行。
|
下次继续时,先核对来源版本和访问状态,再做当前数据库的 DOI 绑定、标题/作者近似候选与完整投稿历史核查,逐条解决作者及摘要版本问题。批准后另建入库队列、保护基线和导入后验收;所有来源结束后仍需全库去重、覆盖、摘要、约束和实际查询验收。测试、构建和安装验证只在已授权 `ces-control:host` 远程 runner 执行。
|
||||||
|
|
||||||
|
生产规范化验收、当前只读候选数量、已有拒稿历史保护及下次恢复入口见 [SDM2026 身份核查记录](sdm2026-identity-review.md)。该记录不代表已批准或完成导入。
|
||||||
|
|||||||
@@ -0,0 +1,57 @@
|
|||||||
|
# 出版来源版本与覆盖修正
|
||||||
|
|
||||||
|
`paper-atlas maintain source-revisions` 用于同一出版清单的新版本:稳定 DOI 全集、标题、作者和决定不变,但官方分轨、摘要或对应来源证据已经过重新核对。它不导入新论文、不合并实体、不推断拒稿,不把出版分类当成研究类型。
|
||||||
|
|
||||||
|
例如,同一清单的“未分类”由 192 条减少到 166 条,而主会等分区增加。直接放松增量导入的计数保护会隐藏清单丢失。本命令先核验完整的新旧清单及数据库快照,再单独修正版本;普通 `import records` 的单调计数规则保持原样。
|
||||||
|
|
||||||
|
## 准备完整证据
|
||||||
|
|
||||||
|
在工作区外保存 inventory JSON,包含以下字段:
|
||||||
|
|
||||||
|
| 字段 | 内容 |
|
||||||
|
| --- | --- |
|
||||||
|
| `format` | `1` |
|
||||||
|
| `source`, `venue`, `year`, `event_key`, `list_url` | 明确的已有来源、会议缩写、年份、事件键和 HTTPS 出版清单地址 |
|
||||||
|
| `previous_rows`, `rows` | 完整旧版、新版规范化来源记录数组,每版最多 10,000 条;每条包含稳定 DOI、来源字段、作者、摘要、track、决定、标签、原始来源 metadata |
|
||||||
|
| `evidence` | 一个或多个 `{ "url": "https://…", "sha256": "…" }`,绑定已独立核验的原始证据文件 |
|
||||||
|
|
||||||
|
两版必须有完全相同的 DOI 集合;仅允许 `abstract`、`track`、`tags`、`metadata` 的有据变化。必须保留 collector 原始字段,不能将工作身份核查产生的额外锚点混入原始来源版本。只有明确 `accepted` 的出版观察可进入本流程。证据 URL/hash 是审核输入,本命令不联网确认它们;执行者必须另行验证文件哈希、完整性及对应关系。
|
||||||
|
|
||||||
|
数据库可以部分导入,甚至各分区的预期数暂时来自不同版本。现有覆盖必须为 `partial`、无错误,导入数与实际来源记录一致;预期数必须恰好属于旧版或新版的同一分区。已有目标 edition 和对应覆盖记录必须存在,目标中可以有其他来源及既有投稿。本版本不会新建 edition,也不会处理共享来源投稿、多次既有 scope curation 的再迁移,或成员集合变化;遇到这些情况会拒绝并要求单独核查。
|
||||||
|
|
||||||
|
## 先生成只读计划
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas maintain source-revisions \
|
||||||
|
--pg-service foundry-research --pg-schema research \
|
||||||
|
--inventory /data/paper-index/validation/reviewed-source-version.json \
|
||||||
|
--out /data/paper-index/validation/source-revision-plan.json
|
||||||
|
```
|
||||||
|
|
||||||
|
计划包含明确数据库身份、两个完整来源版本、当前版本父节点、覆盖表原值和修正值、逐条变更,以及受影响会议 scope 内所有论文的规范记录、稳定标识、全部跨会议投稿和决定、来源关系、标签与已存在的 annotation。拒稿后录用的历史也在保护范围内,不能只校验当前会议的录用观察。
|
||||||
|
|
||||||
|
计划和原始证据应保存到服务器与独立恢复位置,核对完整差异及计划文件 SHA-256。执行前重新核对生产保护基线与单写入安排。输出文件必须是不存在的新路径,禁止覆盖旧报告。
|
||||||
|
|
||||||
|
## 按文件哈希应用并验收
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas maintain source-revisions \
|
||||||
|
--pg-service foundry-research --pg-schema research \
|
||||||
|
--plan /data/paper-index/validation/source-revision-plan.json \
|
||||||
|
--plan-sha256 <已审核的计划文件SHA256> \
|
||||||
|
--out /data/paper-index/validation/source-revision-result.json
|
||||||
|
```
|
||||||
|
|
||||||
|
应用时重新生成计划并要求完全一致;PostgreSQL 使用已有 writer 锁及相关表写锁,SQLite 使用立即事务。任何保护数据、当前版本或目标标识冲突都使操作失败。操作保留投稿和决定 ID、原始决定证据、规范论文文本、全部稳定标识、来源关系和现有标签;只修改计划列出的投稿/投稿标识 edition,以及来源记录的摘要与 metadata 当前投影。旧投影完整保存在版本历史中,来源原始获取时间不会伪造为修正时间。
|
||||||
|
|
||||||
|
覆盖只调整明确分区的预期数和来源观察数,保留 `partial`、cursor、成功时间和错误字段原值。尚未导入的分区不产生虚构投稿或来源观察。覆盖数不能被解读为科学作品数量,或所有 track、摘要、负面决定已完整。
|
||||||
|
|
||||||
|
同一事务显式增加可共享的可选表 `source_scope_revisions`,保持原 21 张业务表的身份。每个版本存储新来源全集、父版本、完整计划及修正前数据;应用接口只追加版本,不更新或删除旧版本。没有新建 PostgreSQL 实例。该表纳入 SQLite→PostgreSQL 的可选表迁移、全量内容哈希、约束与计数验收。`version_sha256` 是规范化计划内容哈希;CLI 的 `--plan-sha256` 是计划文件字节哈希,二者用途不同。
|
||||||
|
|
||||||
|
提交前在同一事务比较全部保护数据与明确预期的变更;失败则全部回滚,包括新增表和版本记录。相同计划的重放还会重新验收当前状态;如后续数据已变化则拒绝,不会重复迁移或恢复旧状态。完整计划与 before-image 用于审查和恢复准备,本命令没有自动回滚历史版本的入口;不得手工删除版本来绕开刷新保护。
|
||||||
|
|
||||||
|
## 后续刷新及限制
|
||||||
|
|
||||||
|
`import records` 检查当前版本的逐条完整来源字段;`Store.record` 也检查原始文本、作者、来源 metadata 和基础标识,防止直接调用绕回旧版摘要。来源字段必须与当前版本相符,可以额外携带已经核验的工作身份/curation 证据,不能替换来源自身字段。相同来源、会议事件和清单地址下未登记的成员必须另行核验新版本。
|
||||||
|
|
||||||
|
版本表通过包内接口追加,未声明数据库管理员的任意 SQL 为安全接口。新来源或新版本在导入前仍需要独立工作身份核查;姓名、标题或无候选都不证明作品身份。实现测试和版本修正验收也不替代所有导入结束后的全库去重、标识冲突、近似标题/作者、摘要、来源覆盖与查询审计。
|
||||||
@@ -74,6 +74,74 @@ def load_inputs(files):
|
|||||||
return records, inputs
|
return records, inputs
|
||||||
|
|
||||||
|
|
||||||
|
def load_mm_program_inputs(report_path):
|
||||||
|
"""Read every MM observation without turning a program into import records.
|
||||||
|
|
||||||
|
Reconcile the complete output against its frozen raw source before any DB
|
||||||
|
connection. The synthetic unknown status is an audit placeholder only.
|
||||||
|
"""
|
||||||
|
from paper_atlas.collectors import mm2026
|
||||||
|
|
||||||
|
report_path = Path(report_path).absolute()
|
||||||
|
raw_report = mm2026.safe_read(report_path, 256*1024)
|
||||||
|
report = json.loads(raw_report)
|
||||||
|
if (report.get('source') != 'mm_program' or report.get('records') != 2143
|
||||||
|
or report.get('program_observations') != 4326 or len(report.get('files', [])) != 28
|
||||||
|
or report.get('source_body_sha256') != mm2026.BODY_SHA
|
||||||
|
or report.get('source_manifest_sha256') != mm2026.MANIFEST_SHA
|
||||||
|
or report.get('database_written') is not False or report.get('approved_for_import') != 0):
|
||||||
|
raise ValueError('Unreviewed MM observation report')
|
||||||
|
root = report_path.parent
|
||||||
|
raw_source = mm2026.safe_read(root/'evidence/program-data.json', 12*1024**2)
|
||||||
|
raw_receipts = mm2026.safe_read(root/'evidence/manifest.json', 128*1024)
|
||||||
|
if digest(raw_source) != mm2026.BODY_SHA or digest(raw_receipts) != mm2026.MANIFEST_SHA:
|
||||||
|
raise ValueError('MM raw evidence changed')
|
||||||
|
expected_rows, _ = mm2026.parse_program(raw_source)
|
||||||
|
expected = {r['paper_id']:r for r in expected_rows}
|
||||||
|
records, inputs, seen, tracks = [], [], set(), set()
|
||||||
|
for item in report['files']:
|
||||||
|
track = item.get('track_literal')
|
||||||
|
if (track not in mm2026.TRACK_COUNTS or track in tracks
|
||||||
|
or item.get('format') != 'mm_program_observation_v1_not_ingestion_record'
|
||||||
|
or item.get('approved_for_import') != 0):
|
||||||
|
raise ValueError('Duplicate or unreviewed MM track format')
|
||||||
|
tracks.add(track)
|
||||||
|
# Stored absolute paths are provenance, not instructions to read elsewhere.
|
||||||
|
path = root/Path(item['file']).name
|
||||||
|
if path.suffix != '.jsonl':
|
||||||
|
raise ValueError('Invalid MM observation filename')
|
||||||
|
raw = mm2026.safe_read(path, 16*1024**2)
|
||||||
|
lines = raw.splitlines(keepends=True)
|
||||||
|
if digest(raw) != item.get('file_sha256') or len(lines) != item.get('count') or len(lines) != mm2026.TRACK_COUNTS[track]:
|
||||||
|
raise ValueError('MM observation file count/hash mismatch')
|
||||||
|
for number,line in enumerate(lines, 1):
|
||||||
|
row = json.loads(line)
|
||||||
|
pid = row.get('paper_id')
|
||||||
|
if not isinstance(pid,str) or pid in seen or row != expected.get(pid) or row['track'] != track:
|
||||||
|
raise ValueError('MM observation differs from frozen source or repeats an ID')
|
||||||
|
seen.add(pid)
|
||||||
|
records.append({'source':'mm_program','external_id':'2026/'+pid,
|
||||||
|
'title':row['title'],'authors':row['authors'],'venue':'MM','year':2026,'track':track,
|
||||||
|
'status':'unknown','identifiers':{},'file':str(path),'line':number,
|
||||||
|
'raw_line_sha256':digest(line),'source_record_hits':[],
|
||||||
|
'identifier_hits':[],'title_candidate_ids':[],
|
||||||
|
'input_format':'mm_program_observation_v1_not_ingestion_record',
|
||||||
|
'program_id':pid,'program_contexts':row['contexts'],
|
||||||
|
'source_observed_status':row['observed_status'],
|
||||||
|
'missing_source_fields':row['missing_fields'],
|
||||||
|
'publication_identity_verified':False,'final_decision_verified':False,
|
||||||
|
'contribution_type_verified':False})
|
||||||
|
inputs.append({'file':str(path),'file_sha256':digest(raw),'count':len(lines),
|
||||||
|
'manifest_sha256':digest(raw_report),'manifest':{
|
||||||
|
'source':'mm_program','venue':'MM','year':2026,'track':track,'status':'unknown',
|
||||||
|
'format':'mm_program_observation_v1_not_ingestion_record',
|
||||||
|
'scope':'Read-only lookup input only; unknown is not a source decision.',
|
||||||
|
'source_report':str(report_path),'source_body_sha256':digest(raw_source)}})
|
||||||
|
if seen != expected.keys() or tracks != mm2026.TRACK_COUNTS.keys():
|
||||||
|
raise ValueError('Incomplete MM observation input; all28tracks required')
|
||||||
|
return records, inputs
|
||||||
|
|
||||||
|
|
||||||
def _lookup(connection, values, columns, lateral):
|
def _lookup(connection, values, columns, lateral):
|
||||||
"""Bound scalar VALUES allow the existing text codec to encode each key."""
|
"""Bound scalar VALUES allow the existing text codec to encode each key."""
|
||||||
width = len(columns)
|
width = len(columns)
|
||||||
@@ -187,9 +255,11 @@ def _atomic_json(path, value):
|
|||||||
temporary.replace(path)
|
temporary.replace(path)
|
||||||
|
|
||||||
|
|
||||||
def audit(files, out, *, pg_service, pg_schema):
|
def audit(files, out, *, pg_service, pg_schema, mm_program_report=None):
|
||||||
# Reject input corruption before opening a database connection or reserving output.
|
# Reject input corruption before opening a database connection or reserving output.
|
||||||
records, inputs = load_inputs(files)
|
if bool(files) == bool(mm_program_report):
|
||||||
|
raise ValueError('Choose source files or one MM program report')
|
||||||
|
records, inputs = load_mm_program_inputs(mm_program_report) if mm_program_report else load_inputs(files)
|
||||||
out = Path(out)
|
out = Path(out)
|
||||||
out.mkdir(parents=True, exist_ok=False)
|
out.mkdir(parents=True, exist_ok=False)
|
||||||
(out / 'audit.lock').write_text(timestamp() + '\n')
|
(out / 'audit.lock').write_text(timestamp() + '\n')
|
||||||
@@ -229,6 +299,7 @@ def audit(files, out, *, pg_service, pg_schema):
|
|||||||
'Name agreement does not prove person or scientific-work identity.',
|
'Name agreement does not prove person or scientific-work identity.',
|
||||||
'Bound identifiers can be incorrect; review titles, authors and original evidence.',
|
'Bound identifiers can be incorrect; review titles, authors and original evidence.',
|
||||||
'Source/history limits are explicit in compressed entity packets.',
|
'Source/history limits are explicit in compressed entity packets.',
|
||||||
|
'Program observations and unknown audit status do not establish contribution type, publication or a final decision.',
|
||||||
'Not a final whole-database deduplication or coverage audit.']}
|
'Not a final whole-database deduplication or coverage audit.']}
|
||||||
_atomic_json(out / 'postgresql-preflight.json', report)
|
_atomic_json(out / 'postgresql-preflight.json', report)
|
||||||
return report
|
return report
|
||||||
@@ -243,10 +314,14 @@ def audit(files, out, *, pg_service, pg_schema):
|
|||||||
|
|
||||||
def main(argv=None, *, prog=None):
|
def main(argv=None, *, prog=None):
|
||||||
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
parser.add_argument('--files', nargs='+', required=True, type=Path)
|
inputs = parser.add_mutually_exclusive_group(required=True)
|
||||||
|
inputs.add_argument('--files', nargs='+', type=Path)
|
||||||
|
inputs.add_argument('--mm-program-report', type=Path,
|
||||||
|
help='Read all frozen MM program observations directly; never generate an import queue')
|
||||||
parser.add_argument('--pg-service', required=True)
|
parser.add_argument('--pg-service', required=True)
|
||||||
parser.add_argument('--pg-schema', required=True)
|
parser.add_argument('--pg-schema', required=True)
|
||||||
parser.add_argument('--out', required=True, type=Path)
|
parser.add_argument('--out', required=True, type=Path)
|
||||||
args = parser.parse_args(argv)
|
args = parser.parse_args(argv)
|
||||||
report = audit(args.files, args.out, pg_service=args.pg_service, pg_schema=args.pg_schema)
|
report = audit(args.files, args.out, pg_service=args.pg_service, pg_schema=args.pg_schema,
|
||||||
|
mm_program_report=args.mm_program_report)
|
||||||
print(json.dumps({k: report[k] for k in ('status', 'records', 'kind_counts', 'approved_for_import')}, ensure_ascii=False))
|
print(json.dumps({k: report[k] for k in ('status', 'records', 'kind_counts', 'approved_for_import')}, ensure_ascii=False))
|
||||||
|
|||||||
@@ -20,14 +20,23 @@ COMMANDS = {
|
|||||||
('collect','public-openreview'): ('paper_atlas.collectors.public_openreview', {}, '核验其他会议的公开拒稿及撤稿标签'),
|
('collect','public-openreview'): ('paper_atlas.collectors.public_openreview', {}, '核验其他会议的公开拒稿及撤稿标签'),
|
||||||
('collect','icml-public-archive'): ('paper_atlas.collectors.icml_public_archive', {}, 'ICML 2026 公开拒稿原始投稿快照'),
|
('collect','icml-public-archive'): ('paper_atlas.collectors.icml_public_archive', {}, 'ICML 2026 公开拒稿原始投稿快照'),
|
||||||
('collect','peerread'): ('paper_atlas.collectors.peerread', {}, 'PeerRead 历史决定补充'),
|
('collect','peerread'): ('paper_atlas.collectors.peerread', {}, 'PeerRead 历史决定补充'),
|
||||||
|
('collect','reviewarena-metadata'): ('paper_atlas.collectors.reviewarena', {}, '仅投影固定版本评审镜像的索引字段,保留决定声明而不下载正文或评审页'),
|
||||||
('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'),
|
('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'),
|
||||||
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
|
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
|
||||||
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
|
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
|
||||||
|
('collect','ismir2026-program'): ('paper_atlas.collectors.ismir2026', {}, '核验 ISMIR 2026 公开节目与录用名单并保留原始摘要、来源差异和自愿公开评审'),
|
||||||
('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'),
|
('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'),
|
||||||
('collect','eccv2026-program'): ('paper_atlas.collectors.eccv2026', {}, '核验ECCV2026暂定节目并断点采集公开元数据'),
|
('collect','eccv2026-program'): ('paper_atlas.collectors.eccv2026', {}, '核验ECCV2026暂定节目并断点采集公开元数据'),
|
||||||
|
('collect','eccv2026-publications'): ('paper_atlas.collectors.eccv2026_publications', {}, '核验ECCV2026 Springer 正式出版章节'),
|
||||||
|
('collect','icassp2026-registry'): ('paper_atlas.collectors.icassp2026', {}, '核验 ICASSP 2026 冻结注册论文元数据'),
|
||||||
|
('collect','icdm2026-program'): ('paper_atlas.collectors.icdm2026', {}, '核验 ICDM2026 冻结日程及明确的研究/应用轨道'),
|
||||||
|
('collect','cikm2026-sources'): ('paper_atlas.collectors.cikm2026', {}, '核对 CIKM 2026 节目、附属议程与 Crossref 出版登记'),
|
||||||
('collect','sdm2026-proceedings'): ('paper_atlas.collectors.sdm2026', {}, '核验 SDM 2026 注册论文并单独保留公开日程版本'),
|
('collect','sdm2026-proceedings'): ('paper_atlas.collectors.sdm2026', {}, '核验 SDM 2026 注册论文并单独保留公开日程版本'),
|
||||||
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
|
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
|
||||||
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
||||||
|
('collect','recsys2026-program'): ('paper_atlas.collectors.recsys2026', {}, '核验 RecSys 2026 公开录用贡献并保留 DOI 来源版本'),
|
||||||
|
('collect','mm2026-program'): ('paper_atlas.collectors.mm2026', {}, '恢复 MM 2026 全部来源 track 与重复展示上下文,待贡献和身份核查'),
|
||||||
|
('collect','mm2026-publications'): ('paper_atlas.collectors.mm_publications2026', {}, '核验 MM 2026 全部注册出版条目并分别保留节目分轨、摘要与版本差异'),
|
||||||
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
||||||
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
|
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
|
||||||
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
|
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
|
||||||
@@ -40,6 +49,7 @@ COMMANDS = {
|
|||||||
('import','arxiv'): ('paper_atlas.ingestion.arxiv', {}, '导入 arXiv 元数据 ZIP'),
|
('import','arxiv'): ('paper_atlas.ingestion.arxiv', {}, '导入 arXiv 元数据 ZIP'),
|
||||||
('import','records'): ('paper_atlas.ingestion.records', {}, '导入核验过的会议文件'),
|
('import','records'): ('paper_atlas.ingestion.records', {}, '导入核验过的会议文件'),
|
||||||
('maintain','edition-tracks'): ('paper_atlas.ingestion.edition_tracks', {}, '按逐条来源证据修正既有投稿轨道并保护刷新'),
|
('maintain','edition-tracks'): ('paper_atlas.ingestion.edition_tracks', {}, '按逐条来源证据修正既有投稿轨道并保护刷新'),
|
||||||
|
('maintain','source-revisions'): ('paper_atlas.ingestion.source_revisions', {}, '按完整来源版本核验并修正出版分轨与覆盖数,保留历史并阻止旧版刷新'),
|
||||||
('maintain','joint-editions'): ('paper_atlas.ingestion.joint_editions', {}, '按已核验计划修正既有联合会议届次归属'),
|
('maintain','joint-editions'): ('paper_atlas.ingestion.joint_editions', {}, '按已核验计划修正既有联合会议届次归属'),
|
||||||
('maintain','event-identity'): ('paper_atlas.ingestion.event_identity', {}, '按核验计划增加同年会议事件身份'),
|
('maintain','event-identity'): ('paper_atlas.ingestion.event_identity', {}, '按核验计划增加同年会议事件身份'),
|
||||||
('maintain','merge'): ('paper_atlas.ingestion.maintenance', {}, '协调单写入合并与恢复'),
|
('maintain','merge'): ('paper_atlas.ingestion.maintenance', {}, '协调单写入合并与恢复'),
|
||||||
|
|||||||
@@ -0,0 +1,164 @@
|
|||||||
|
"""Collect CIKM 2026 programme and Crossref metadata from frozen snapshots."""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from hashlib import sha256
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, clean
|
||||||
|
|
||||||
|
PROGRAM = 'https://cikm2026.diag.uniroma1.it/cikm-program/program.html'
|
||||||
|
VOLUME = '10.1145/3799682'
|
||||||
|
CONTAINER = 'Proceedings of the 35th ACM International Conference on Information and Knowledge Management'
|
||||||
|
TRACKS = {'FULL': 596, 'SHORT': 234, 'APPLIED': 86, 'RESOURCE': 46, 'DEMO': 46}
|
||||||
|
SUPPLEMENTAL = {'KEYNOTE', 'TUTORIAL', 'INDUSTRY', 'POSTER', 'PHD', 'PHD · REMOTE', 'WORKSHOP', 'ANALYTICUP'}
|
||||||
|
PAGE_COUNTS = (250, 250, 250, 250, 81)
|
||||||
|
HASHES = {
|
||||||
|
'program.html': '3eaa668b6aaf09a671d8a3e19a2889b9ab21afaa7ded013794afa94bf35281d6',
|
||||||
|
'CIKM2026_program.html': 'e3630bd1247df713b8b618f1d3cd9709e7020b2524bb677f82a10f83badd1732',
|
||||||
|
'crossref-parent.json': 'f3370418d86ccfcfaeb972073bc5fcd1c1c737caed96570fc83f99db0f7d204a',
|
||||||
|
'crossref-page-1.json': '16bed29f8ee82d668b124c369b632c5c1de0753bdbec7eb72ecb22ffdda5c710',
|
||||||
|
'crossref-page-2.json': '54424ca31d99462394c9fde5fecfd7dd16337351d21ede3169f19af54d4f49c5',
|
||||||
|
'crossref-page-3.json': '2dc31d89f41966614ed66a11d1ebaa4b2a6904dcf92945b7185e72e7e65382dd',
|
||||||
|
'crossref-page-4.json': 'fbbec2b52650dc99a0251c743f035a6fd95be263b04c55c12134cf8473bdde76',
|
||||||
|
'crossref-page-5.json': '06151c5f770c1e37e8ea9dabe5210be9c969a14fa3bf9925a2d7730ced47d6e5',
|
||||||
|
'crossref-pages.json': '8be1f8f05139f7a1231b73ad6430f28d2f625809f1530cd9de654f8b3c68c1ae',
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def fields(node, css_class):
|
||||||
|
return [child for child in node.walk() if css_class in child.attrs.get('class', '').split()]
|
||||||
|
|
||||||
|
|
||||||
|
def parse_program(raw, *, require_doi):
|
||||||
|
document = Document(raw.decode('utf-8'))
|
||||||
|
papers = {}
|
||||||
|
for card in fields(document.root, 'paper'):
|
||||||
|
pid = card.attrs.get('data-pid')
|
||||||
|
if not pid:
|
||||||
|
continue
|
||||||
|
titles = fields(card, 'ptitle')
|
||||||
|
labels = fields(card, 'badge')
|
||||||
|
abstracts = fields(card, 'abstxt')
|
||||||
|
if pid in papers or len(titles) != 1 or len(labels) != 1 or len(abstracts) > 1:
|
||||||
|
raise ValueError(f'Invalid or repeated programme card {pid}')
|
||||||
|
title = clean(titles[0].text())
|
||||||
|
label = clean(labels[0].text())
|
||||||
|
authors = [clean(author.text()) for author in fields(card, 'au')]
|
||||||
|
abstract = clean(abstracts[0].text()) if abstracts else None
|
||||||
|
doi_url = titles[0].attrs.get('href')
|
||||||
|
doi = doi_url.removeprefix('https://doi.org/') if doi_url else None
|
||||||
|
if label not in TRACKS or not title or not authors or not all(authors):
|
||||||
|
raise ValueError(f'Incomplete programme metadata {pid}')
|
||||||
|
if require_doi and (not doi or not re.fullmatch(re.escape(VOLUME) + r'\.\d+', doi) or not abstract):
|
||||||
|
raise ValueError(f'Missing programme DOI or abstract {pid}')
|
||||||
|
papers[pid] = {'programme_id': pid, 'title': title, 'author_display_names': authors,
|
||||||
|
'abstract': abstract, 'programme_label': label, 'programme_doi_link': doi_url,
|
||||||
|
'doi': doi if require_doi else None}
|
||||||
|
return papers
|
||||||
|
|
||||||
|
|
||||||
|
def parse_supplemental(raw):
|
||||||
|
document = Document(raw.decode('utf-8'))
|
||||||
|
events = {}
|
||||||
|
for session in fields(document.root, 'sessblock'):
|
||||||
|
section = session.attrs.get('id', '')
|
||||||
|
if not section.startswith(('pre-', 'kn-')):
|
||||||
|
continue
|
||||||
|
for card in fields(session, 'paper'):
|
||||||
|
labels = fields(card, 'badge')
|
||||||
|
if len(labels) != 1:
|
||||||
|
raise ValueError(f'Ambiguous supplemental label in {section}')
|
||||||
|
label = clean(labels[0].text())
|
||||||
|
if label not in SUPPLEMENTAL:
|
||||||
|
continue
|
||||||
|
titles = fields(card, 'ptitle')
|
||||||
|
abstracts = fields(card, 'abstxt')
|
||||||
|
locator = card.attrs.get('id') or section
|
||||||
|
if locator in events or len(titles) != 1 or len(abstracts) > 1:
|
||||||
|
raise ValueError(f'Ambiguous supplemental entry {locator}')
|
||||||
|
events[locator] = {'programme_locator': locator, 'programme_section': section,
|
||||||
|
'programme_label': label, 'title': clean(titles[0].text()),
|
||||||
|
'author_display_names': [clean(a.text()) for a in fields(card, 'au')],
|
||||||
|
'abstract': clean(abstracts[0].text()) if abstracts else None}
|
||||||
|
return events
|
||||||
|
|
||||||
|
|
||||||
|
def parse_registry(snapshots):
|
||||||
|
volume = json.loads(snapshots['crossref-parent.json'])['message']
|
||||||
|
if (volume['DOI'], volume['title'], volume['ISBN']) != (VOLUME, [CONTAINER], ['9798400725395']):
|
||||||
|
raise ValueError('Unexpected Crossref volume')
|
||||||
|
records = {}
|
||||||
|
for number, count in enumerate(PAGE_COUNTS, 1):
|
||||||
|
name = f'crossref-page-{number}.json'
|
||||||
|
message = json.loads(snapshots[name])['message']
|
||||||
|
if message['total-results'] != 1081 or len(message['items']) != count:
|
||||||
|
raise ValueError(f'Unexpected Crossref pagination in {name}')
|
||||||
|
for item in message['items']:
|
||||||
|
doi = item['DOI'].lower()
|
||||||
|
if (doi in records or not re.fullmatch(re.escape(VOLUME) + r'\.\d+', doi)
|
||||||
|
or item.get('container-title') != [CONTAINER] or item.get('type') != 'proceedings-article'
|
||||||
|
or item.get('published', {}).get('date-parts') != [[2026, 11, 6]]):
|
||||||
|
raise ValueError(f'Unexpected Crossref record {doi}')
|
||||||
|
records[doi] = {'registry': item, 'source_file': name}
|
||||||
|
return records
|
||||||
|
|
||||||
|
|
||||||
|
def collect(snapshots, output):
|
||||||
|
snapshots = Path(snapshots)
|
||||||
|
output = Path(output)
|
||||||
|
if output.exists() or output.is_symlink():
|
||||||
|
raise FileExistsError(output)
|
||||||
|
bodies = {}
|
||||||
|
for name, expected in HASHES.items():
|
||||||
|
path = snapshots / name
|
||||||
|
if path.is_symlink() or not path.is_file() or path.stat().st_size > 8 * 1024**2:
|
||||||
|
raise ValueError(f'Invalid snapshot file {name}')
|
||||||
|
raw = path.read_bytes()
|
||||||
|
if sha256(raw).hexdigest() != expected:
|
||||||
|
raise ValueError(f'Snapshot hash mismatch for {name}')
|
||||||
|
bodies[name] = raw
|
||||||
|
|
||||||
|
programme = parse_program(bodies['program.html'], require_doi=True)
|
||||||
|
alternate = parse_program(bodies['CIKM2026_program.html'], require_doi=False)
|
||||||
|
supplemental = parse_supplemental(bodies['program.html'])
|
||||||
|
registry = parse_registry(bodies)
|
||||||
|
programme_dois = {r['doi'] for r in programme.values()}
|
||||||
|
tracks = Counter(r['programme_label'] for r in programme.values())
|
||||||
|
if (tracks != TRACKS or len(programme_dois) != len(programme) or programme_dois - registry.keys()
|
||||||
|
or len(alternate) != 1005 or alternate.keys() - programme.keys()
|
||||||
|
or len(supplemental) != 73 or len(registry) - len(programme_dois) != 73):
|
||||||
|
raise ValueError('Unexpected programme or registry coverage')
|
||||||
|
groups = {
|
||||||
|
'programme': programme,
|
||||||
|
'alternate-programme': alternate,
|
||||||
|
'supplemental-programme': supplemental,
|
||||||
|
'crossref': registry,
|
||||||
|
}
|
||||||
|
output.mkdir(parents=True, exist_ok=False)
|
||||||
|
files = {}
|
||||||
|
for name, rows in groups.items():
|
||||||
|
filename = name + '.jsonl'
|
||||||
|
payload = ''.join(json.dumps({'source_key': key, **row}, ensure_ascii=False) + '\n'
|
||||||
|
for key, row in sorted(rows.items())).encode()
|
||||||
|
(output / filename).write_bytes(payload)
|
||||||
|
files[filename] = {'records': len(rows), 'sha256': sha256(payload).hexdigest()}
|
||||||
|
report = {'status': 'collected_pending_identity_review', 'files': files, 'input_sha256': HASHES,
|
||||||
|
'programme_by_label': dict(tracks), 'programme_dois_in_registry': len(programme_dois),
|
||||||
|
'registry_outside_doi_programme': sorted(registry.keys() - programme_dois),
|
||||||
|
'programme_ids_missing_from_alternate': sorted(programme.keys() - alternate.keys()),
|
||||||
|
'database_written': False, 'public_negative_decisions_verified': False}
|
||||||
|
partial = output / 'collection-report.json.part'
|
||||||
|
partial.write_text(json.dumps(report, ensure_ascii=False, indent=2) + '\n', encoding='utf-8')
|
||||||
|
partial.replace(output / 'collection-report.json')
|
||||||
|
return report
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
|
parser.add_argument('--snapshots', type=Path, required=True)
|
||||||
|
parser.add_argument('--out', type=Path, required=True)
|
||||||
|
options = parser.parse_args(argv)
|
||||||
|
print(json.dumps(collect(options.snapshots, options.out), ensure_ascii=False))
|
||||||
@@ -0,0 +1,194 @@
|
|||||||
|
"""Normalize frozen ECCV 2026 Springer proceedings and preserve publisher evidence."""
|
||||||
|
import argparse
|
||||||
|
import fcntl
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, clean
|
||||||
|
from paper_atlas.io.springer import parse_chapter, read_page, snapshot_index
|
||||||
|
|
||||||
|
SOURCE = 'eccv2026_springer'
|
||||||
|
ORIGIN = 'https://link.springer.com'
|
||||||
|
BOOK_TITLE = 'Computer Vision – ECCV 2026'
|
||||||
|
LIST_URL = ORIGIN + '/conference/eccv'
|
||||||
|
BOOK = re.compile(r'10\.1007/978-3-032-\d{5}-[\dXx]')
|
||||||
|
CHAPTER = re.compile(BOOK.pattern + r'_[1-9]\d*')
|
||||||
|
VOLUMES = set(range(17001, 17079))
|
||||||
|
PUBLICATIONS = 2835
|
||||||
|
|
||||||
|
|
||||||
|
def fingerprint(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def toc_page(html, book_doi, volume):
|
||||||
|
tree = Document(html).root
|
||||||
|
text = clean(tree.text())
|
||||||
|
if BOOK_TITLE not in text or 'Malmö' not in text or f'(LNCS, volume {volume})' not in text:
|
||||||
|
raise ValueError('Unexpected Springer ECCV volume scope')
|
||||||
|
entries = {}
|
||||||
|
for node in tree.walk():
|
||||||
|
if node.tag != 'li' or node.attrs.get('data-test') != 'chapter':
|
||||||
|
continue
|
||||||
|
headings = [n for n in node.walk() if n.tag == 'h3']
|
||||||
|
if len(headings) != 1:
|
||||||
|
raise ValueError('Ambiguous Springer contents heading')
|
||||||
|
heading, = headings
|
||||||
|
title = clean(heading.text())
|
||||||
|
links = [n.attrs.get('href', '') for n in heading.walk() if n.tag == 'a']
|
||||||
|
if title in ('Front Matter', 'Back Matter') and not links:
|
||||||
|
continue
|
||||||
|
if len(links) != 1 or not title:
|
||||||
|
raise ValueError('Authored contents entry lacks one chapter link')
|
||||||
|
link, = links
|
||||||
|
doi = link.removeprefix('/chapter/')
|
||||||
|
if (not link.startswith('/chapter/') or not CHAPTER.fullmatch(doi)
|
||||||
|
or not doi.startswith(book_doi + '_') or doi in entries):
|
||||||
|
raise ValueError('Unexpected or duplicate Springer chapter DOI')
|
||||||
|
entries[doi] = title
|
||||||
|
if not entries:
|
||||||
|
raise ValueError('Empty Springer contents page')
|
||||||
|
return entries
|
||||||
|
|
||||||
|
|
||||||
|
def publisher_inventory(root):
|
||||||
|
root = Path(root).resolve()
|
||||||
|
receipt_bytes = (root / 'receipts.jsonl').read_bytes()
|
||||||
|
inventory_bytes = (root / 'volume-checkpoint.json').read_bytes()
|
||||||
|
summary = json.loads((root / 'summary.json').read_bytes())
|
||||||
|
volumes = json.loads(inventory_bytes)
|
||||||
|
receipt_rows = [json.loads(line) for line in receipt_bytes.splitlines()]
|
||||||
|
receipts = {row['file']: row for row in receipt_rows}
|
||||||
|
if (summary['raw_receipts_sha256'] != fingerprint(receipt_bytes)
|
||||||
|
or summary['volume_checkpoint_sha256'] != fingerprint(inventory_bytes)
|
||||||
|
or set(volumes) != {str(n) for n in VOLUMES}
|
||||||
|
or len(receipts) != len(receipt_rows)):
|
||||||
|
raise ValueError('Incomplete or inconsistent Springer contents snapshot')
|
||||||
|
entries, used = {}, set()
|
||||||
|
for number, volume in sorted(volumes.items()):
|
||||||
|
book_url = volume['book']
|
||||||
|
book_doi = book_url.removeprefix(ORIGIN + '/book/')
|
||||||
|
if not BOOK.fullmatch(book_doi) or book_url != ORIGIN + '/book/' + book_doi:
|
||||||
|
raise ValueError('Unexpected Springer volume URL')
|
||||||
|
if set(volume['files']) != {'1', '2'}:
|
||||||
|
raise ValueError('Missing reviewed Springer contents page')
|
||||||
|
volume_dois = set()
|
||||||
|
for index, file in volume['files'].items():
|
||||||
|
receipt = receipts[file]
|
||||||
|
path = root / file
|
||||||
|
page_url = book_url + ('?page=2' if index == '2' else '')
|
||||||
|
if (file in used or Path(file).name != file or not path.resolve().is_relative_to(root)
|
||||||
|
or receipt['url'] != page_url or receipt['status'] != 200
|
||||||
|
or not 0 < receipt['bytes'] <= 3_000_000):
|
||||||
|
raise ValueError('Unsafe, reused or mismatched Springer contents receipt')
|
||||||
|
raw = path.read_bytes()
|
||||||
|
if len(raw) != receipt['bytes'] or fingerprint(raw) != receipt['sha256']:
|
||||||
|
raise ValueError('Springer contents bytes disagree with receipt')
|
||||||
|
used.add(file)
|
||||||
|
page_entries = toc_page(raw.decode(), book_doi, int(number))
|
||||||
|
if entries.keys() & page_entries.keys():
|
||||||
|
raise ValueError('Springer chapter repeats across contents pages')
|
||||||
|
volume_dois.update(page_entries)
|
||||||
|
entries.update({doi: {'title': title, 'lncs_volume': int(number), 'book_doi': book_doi,
|
||||||
|
'evidence_url': page_url, 'evidence_sha256': receipt['sha256']}
|
||||||
|
for doi, title in page_entries.items()})
|
||||||
|
if volume_dois != set(volume['chapter_dois']) or len(volume_dois) != len(volume['chapter_dois']):
|
||||||
|
raise ValueError('Springer volume chapter set differs from checkpoint')
|
||||||
|
if used != receipts.keys() or len(entries) != PUBLICATIONS or summary['chapter_link_count'] != PUBLICATIONS:
|
||||||
|
raise ValueError('Springer contents chapter total differs from the reviewed archive')
|
||||||
|
return entries
|
||||||
|
|
||||||
|
|
||||||
|
def title_for_comparison(title):
|
||||||
|
# Springer may render a numeric superscript with inline or display math delimiters.
|
||||||
|
return clean(re.sub(r'\\\((\^[0-9]+)\\\)|\$\$(\^[0-9]+)\$\$',
|
||||||
|
lambda match: match[1] or match[2], title))
|
||||||
|
|
||||||
|
|
||||||
|
def publisher_row(chapter, toc, evidence):
|
||||||
|
doi = chapter['doi']
|
||||||
|
literal_title_differs = clean(chapter['title']) != toc['title']
|
||||||
|
if literal_title_differs and title_for_comparison(chapter['title']) != title_for_comparison(toc['title']):
|
||||||
|
raise ValueError('Chapter title differs from Springer contents, review the version')
|
||||||
|
contents_evidence = {'url': toc['evidence_url'], 'sha256': toc['evidence_sha256']}
|
||||||
|
if literal_title_differs:
|
||||||
|
contents_evidence['title_literal'] = toc['title']
|
||||||
|
url = ORIGIN + '/chapter/' + doi
|
||||||
|
return {
|
||||||
|
'source': SOURCE, 'external_id': doi, 'url': url,
|
||||||
|
'venue': 'ECCV', 'year': 2026, 'track': 'main',
|
||||||
|
'title': chapter['title'], 'authors': chapter['authors'], 'abstract': chapter['abstract'],
|
||||||
|
'status': 'accepted', 'identifiers': {'doi': doi},
|
||||||
|
'submission_identifiers': [{'scheme': 'publication_doi', 'value': doi}],
|
||||||
|
'evidence_url': url, 'observed_at': evidence['observed_at'],
|
||||||
|
'raw_decision': 'Chapter published in the refereed ECCV 2026 conference proceedings.',
|
||||||
|
'tags': [{'namespace': SOURCE + ':publication_format', 'value': 'proceedings_chapter'}],
|
||||||
|
'metadata': {
|
||||||
|
'book_title': BOOK_TITLE, 'book_doi': toc['book_doi'], 'lncs_volume': toc['lncs_volume'],
|
||||||
|
'authors_citation_raw': chapter['authors_citation_raw'], 'first_page': chapter['first_page'],
|
||||||
|
'last_page': chapter['last_page'], 'publisher_chapter_evidence': evidence,
|
||||||
|
'publisher_contents_evidence': contents_evidence,
|
||||||
|
'scope': 'Refereed conference proceedings, LNCS 17001–17078.',
|
||||||
|
'original_decision_signature_verified': False,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, toc_snapshots, chapter_snapshots):
|
||||||
|
contents = publisher_inventory(toc_snapshots)
|
||||||
|
chapters = snapshot_index(chapter_snapshots)
|
||||||
|
urls = {ORIGIN + '/chapter/' + doi: doi for doi in contents}
|
||||||
|
if not chapters or not chapters.keys() <= urls.keys():
|
||||||
|
raise ValueError('No chapter snapshots or chapter outside reviewed ECCV proceedings')
|
||||||
|
rows, raw = [], {}
|
||||||
|
for url in sorted(chapters):
|
||||||
|
doi = urls[url]
|
||||||
|
body, evidence = read_page(chapters, url)
|
||||||
|
chapter = parse_chapter(body.decode(), doi, book_title=BOOK_TITLE, publication_year=2026)
|
||||||
|
rows.append(publisher_row(chapter, contents[doi], evidence))
|
||||||
|
raw[evidence['file']] = body
|
||||||
|
out = Path(out)
|
||||||
|
out.mkdir(parents=True, exist_ok=True)
|
||||||
|
lock_path = out / 'eccv2026-publications.collect.lock'
|
||||||
|
filename = 'eccv2026-springer-main.jsonl'
|
||||||
|
with lock_path.open('a') as lock:
|
||||||
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||||
|
if any(p != lock_path for p in out.iterdir()):
|
||||||
|
raise FileExistsError('Existing output must be inspected, not overwritten')
|
||||||
|
with tempfile.TemporaryDirectory(prefix='.eccv2026-publications-', dir=out) as temporary:
|
||||||
|
stage = Path(temporary)
|
||||||
|
(stage / 'evidence').mkdir()
|
||||||
|
for path, body in raw.items():
|
||||||
|
(stage / path).write_bytes(body)
|
||||||
|
data = ''.join(json.dumps(row, ensure_ascii=False) + '\n' for row in rows).encode()
|
||||||
|
(stage / filename).write_bytes(data)
|
||||||
|
manifest = {
|
||||||
|
'source': SOURCE, 'venue': 'ECCV', 'year': 2026, 'track': 'main', 'parser_version': 1,
|
||||||
|
'file': str(out / filename), 'file_sha256': fingerprint(data), 'count': len(rows),
|
||||||
|
'list_url': LIST_URL, 'source_expected_count': PUBLICATIONS, 'status': 'partial',
|
||||||
|
'missing_abstract_count': sum(not r['abstract'] for r in rows),
|
||||||
|
}
|
||||||
|
report = {
|
||||||
|
'status': 'collected_pending_identity_review', 'records': len(rows), 'files': [manifest],
|
||||||
|
'toc_chapters': len(contents), 'chapter_details_missing': len(contents) - len(rows),
|
||||||
|
'database_written': False,
|
||||||
|
}
|
||||||
|
metadata = Path(filename).with_suffix('.manifest.json')
|
||||||
|
(stage / metadata).write_text(json.dumps(manifest, indent=2) + '\n')
|
||||||
|
(stage / 'collection-report.json').write_text(json.dumps(report, indent=2) + '\n')
|
||||||
|
for item in ('evidence', filename, metadata, 'collection-report.json'):
|
||||||
|
(stage / item).replace(out / item)
|
||||||
|
return report
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog)
|
||||||
|
parser.add_argument('--toc-snapshots', required=True)
|
||||||
|
parser.add_argument('--chapter-snapshots', required=True)
|
||||||
|
parser.add_argument('--out', required=True)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
print(json.dumps(collect(args.out, toc_snapshots=args.toc_snapshots,
|
||||||
|
chapter_snapshots=args.chapter_snapshots), ensure_ascii=False))
|
||||||
@@ -0,0 +1,172 @@
|
|||||||
|
"""Normalize the frozen ICASSP 2026 Crossref census without network or database writes."""
|
||||||
|
import argparse
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
from paper_atlas.io.html import clean
|
||||||
|
|
||||||
|
SOURCE = 'crossref_icassp_registry'
|
||||||
|
TRACK = 'proceedings_unclassified'
|
||||||
|
SNAPSHOT = 'icassp2026-registry-capture-v1'
|
||||||
|
MANIFEST_SHA = '1f8ffa040b1f682a4b65f476f4115cb95629056d8de5c324c8417d823846ae93'
|
||||||
|
CONTAINER = 'ICASSP 2026 - 2026 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)'
|
||||||
|
OTHER_CONTAINER = '2026 IEEE 6th International Conference in Power Engineering Applications (ICPEA)'
|
||||||
|
PREFIX = '10.1109/icassp55912.2026.'
|
||||||
|
NONPAPERS = {
|
||||||
|
PREFIX+'11463724': ('ICASSP 2026 Reviewers', 'xii-lxxxvi'),
|
||||||
|
PREFIX+'11464828': ('Author Index', 'A-1-A-152'),
|
||||||
|
PREFIX+'11464776': ('Title Page', 'i-ii'),
|
||||||
|
PREFIX+'11462944': ('ICASSP 2026 Committees', 'vi-xii'),
|
||||||
|
PREFIX+'11464852': ('ICASSP 2026 Organizing Committee', 'iii-iii'),
|
||||||
|
PREFIX+'11460797': ('ICASSP 2026 Cover Page', 'c1-c1'),
|
||||||
|
PREFIX+'11463523': ('Table of Contents', 'xlix-cdlxii'),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def load_snapshot(root):
|
||||||
|
root = Path(root); directory = root/SNAPSHOT; path = directory/'manifest.json'
|
||||||
|
if root.is_symlink() or directory.is_symlink() or path.is_symlink() or path.stat().st_size > 128*1024:
|
||||||
|
raise ValueError('Unsafe snapshot manifest')
|
||||||
|
raw = path.read_bytes()
|
||||||
|
if digest(raw) != MANIFEST_SHA:
|
||||||
|
raise ValueError('Reviewed manifest hash changed')
|
||||||
|
manifest = json.loads(raw)
|
||||||
|
if (manifest['status'] != 'registry_census_completed_not_import_approval'
|
||||||
|
or manifest['records_captured'] != 4645 or manifest['unique_dois'] != 4645
|
||||||
|
or manifest['database_written'] is not False or manifest['approved_for_import'] != 0
|
||||||
|
or len(manifest['pages']) != 10):
|
||||||
|
raise ValueError('Incomplete frozen census receipt')
|
||||||
|
files = {'manifest.json': raw}; rows = []
|
||||||
|
for index, receipt in enumerate(manifest['pages'], 1):
|
||||||
|
name = receipt['file']; path = directory/name
|
||||||
|
if (name != f'page-{index:03}.json' or path.is_symlink()
|
||||||
|
or path.stat().st_size > 4*1024**2):
|
||||||
|
raise ValueError('Unsafe snapshot page')
|
||||||
|
body = path.read_bytes()
|
||||||
|
if (receipt['http_status'] != 200 or receipt['url'] != receipt['final_url']
|
||||||
|
or not receipt['url'].startswith('https://api.crossref.org/works?')
|
||||||
|
or datetime.fromisoformat(receipt['at']).utcoffset() is None
|
||||||
|
or receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)):
|
||||||
|
raise ValueError('Snapshot receipt/body disagree')
|
||||||
|
data = json.loads(body); message = data['message']
|
||||||
|
expected = 500 if index < 10 else 145
|
||||||
|
if data['status'] != 'ok' or message['total-results'] != 4645 or len(message['items']) != expected:
|
||||||
|
raise ValueError('Incomplete registry page')
|
||||||
|
files[name] = body
|
||||||
|
for original in message['items']:
|
||||||
|
rows.append((original, receipt))
|
||||||
|
return manifest, files, rows
|
||||||
|
|
||||||
|
|
||||||
|
def registry_papers(rows):
|
||||||
|
seen = set(); papers = []; nonpapers = []; other = []
|
||||||
|
for original, receipt in rows:
|
||||||
|
doi = original.get('DOI')
|
||||||
|
if not isinstance(doi, str) or doi in seen:
|
||||||
|
raise ValueError('Missing or repeated registry DOI')
|
||||||
|
seen.add(doi)
|
||||||
|
if original.get('container-title') == [OTHER_CONTAINER]:
|
||||||
|
if doi.startswith(PREFIX):
|
||||||
|
raise ValueError('Conference DOI/container conflict')
|
||||||
|
other.append(doi)
|
||||||
|
continue
|
||||||
|
if (original.get('container-title') != [CONTAINER]
|
||||||
|
or original.get('event', {}).get('name') != CONTAINER
|
||||||
|
or not re.fullmatch(re.escape(PREFIX)+r'[0-9]+', doi)
|
||||||
|
or original.get('publisher') != 'IEEE' or original.get('type') != 'proceedings-article'
|
||||||
|
or original.get('URL') != 'https://doi.org/'+doi
|
||||||
|
or original.get('published') != {'date-parts': [[2026, 5, 3]]}
|
||||||
|
or original.get('event', {}).get('start') != {'date-parts': [[2026, 5, 3]]}
|
||||||
|
or original.get('event', {}).get('end') != {'date-parts': [[2026, 5, 8]]}):
|
||||||
|
raise ValueError('Wrong registered conference, DOI, publisher or dates')
|
||||||
|
if doi in NONPAPERS:
|
||||||
|
title, page = NONPAPERS[doi]
|
||||||
|
if original.get('title') != [title] or original.get('page') != page or original.get('author'):
|
||||||
|
raise ValueError('Changed reviewed nonpaper evidence')
|
||||||
|
nonpapers.append({'doi': doi, 'title': title, 'page': page})
|
||||||
|
continue
|
||||||
|
titles = original.get('title')
|
||||||
|
if not isinstance(titles, list) or len(titles) != 1 or not isinstance(titles[0], str) or not clean(titles[0]):
|
||||||
|
raise ValueError('Missing registry title')
|
||||||
|
authors = original.get('author')
|
||||||
|
if not isinstance(authors, list) or not authors or original.get('abstract'):
|
||||||
|
raise ValueError('Missing authors or changed abstract availability')
|
||||||
|
names = []; flags = []
|
||||||
|
for position, author in enumerate(authors):
|
||||||
|
if (not isinstance(author, dict) or not isinstance(author.get('family'), str)
|
||||||
|
or not clean(author['family']) or ('given' in author and not isinstance(author['given'], str))
|
||||||
|
or author.get('sequence') != ('first' if position == 0 else 'additional')):
|
||||||
|
raise ValueError('Incomplete registry author or sequence')
|
||||||
|
name = clean(' '.join(x for x in (author.get('given'), author['family']) if x))
|
||||||
|
if not name or '@' in name or name.casefold() == 'anonymous':
|
||||||
|
raise ValueError('Unreviewed author identity')
|
||||||
|
if not author.get('given'):
|
||||||
|
flags.append({'kind': 'registry_single_name_field_not_split', 'author_position': position+1})
|
||||||
|
names.append(name)
|
||||||
|
papers.append({'original': original, 'receipt': receipt, 'title': clean(titles[0]), 'authors': names, 'review_flags': flags})
|
||||||
|
if len(seen) != 4645 or len(papers) != 4582 or len(other) != 56 or {r['doi'] for r in nonpapers} != set(NONPAPERS):
|
||||||
|
raise ValueError('Incomplete reviewed registry partition')
|
||||||
|
return sorted(papers, key=lambda p: p['original']['DOI']), nonpapers, other
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
manifest, files, items = load_snapshot(snapshots)
|
||||||
|
papers, nonpapers, other = registry_papers(items)
|
||||||
|
evidence_dir = out/'evidence'; evidence_dir.mkdir()
|
||||||
|
for name, body in files.items():
|
||||||
|
(evidence_dir/name).write_bytes(body)
|
||||||
|
rows = []
|
||||||
|
for paper in papers:
|
||||||
|
original = paper['original']; receipt = paper['receipt']; doi = original['DOI']
|
||||||
|
rows.append({'source': SOURCE, 'external_id': doi, 'url': original['URL'],
|
||||||
|
'title': paper['title'], 'authors': paper['authors'], 'abstract': None,
|
||||||
|
'identifiers': {'doi': doi}, 'submission_identifiers': [{'scheme': 'publication_doi', 'value': doi}],
|
||||||
|
'venue': 'ICASSP', 'year': 2026, 'track': TRACK, 'status': 'accepted',
|
||||||
|
'raw_decision': 'Publication registered in Crossref ICASSP 2026 proceedings; not an original signed decision.',
|
||||||
|
'evidence_url': receipt['url'], 'observed_at': receipt['at'], 'tags': [],
|
||||||
|
'metadata': {'registry_fields': original, 'review_flags': paper['review_flags'],
|
||||||
|
'source_evidence': {'file': 'evidence/'+receipt['file'], 'sha256': receipt['sha256'], 'url': receipt['url']},
|
||||||
|
'source_manifest': {'file': 'evidence/manifest.json', 'sha256': MANIFEST_SHA},
|
||||||
|
'original_decision_signature_verified': False,
|
||||||
|
'abstract_availability': 'Absent in frozen registry deposit; no abstract inferred or fused from program versions.',
|
||||||
|
'author_transform': 'Whitespace normalization of given + family in registered order; single fields retained, not guessed or split; all original author fields retained.',
|
||||||
|
'publication_dates': {k: original[k] for k in ('published', 'published-print', 'published-online') if k in original},
|
||||||
|
'date_scope': 'Registered event starts May 3; conference website says May 4–8. Source values preserved without substitution.',
|
||||||
|
'scope': 'Registered publication version only. Requires work/DOI/history review before import. No automatic program match, aliases, track/type or public-negative inference.'}})
|
||||||
|
path = out/'icassp-crossref-registry-2026-unclassified.jsonl'
|
||||||
|
raw = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(raw)
|
||||||
|
file_manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICASSP', 'year': 2026, 'track': TRACK,
|
||||||
|
'count': len(rows), 'file': str(path), 'file_sha256': digest(raw), 'list_url': manifest['pages'][0]['url'],
|
||||||
|
'status': 'partial', 'missing_abstract_count': len(rows)}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(file_manifest, indent=2)+'\n')
|
||||||
|
report = {'status': 'collected_pending_identity_review', 'records': len(rows), 'files': [file_manifest],
|
||||||
|
'filtered_registry_records': 4645, 'excluded_nonpaper_items': nonpapers, 'excluded_other_conference_dois': other,
|
||||||
|
'missing_abstract_count': len(rows), 'single_name_author_fields': sum(len(p['review_flags']) for p in papers),
|
||||||
|
'database_written': False, 'network_requests': 0, 'pdfs_downloaded': False, 'approved_for_import': 0,
|
||||||
|
'public_rejected_availability': 'unknown', 'source_manifest_sha256': MANIFEST_SHA,
|
||||||
|
'scope': 'Registry census, not completeness of program, abstracts, tracks, negative decisions or work identities. Official program has4589 titles;174 unmatched normalized title candidates remain separate source discrepancies, not rejection or withdrawal evidence.'}
|
||||||
|
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
|
parser.add_argument('--snapshots', required=True)
|
||||||
|
parser.add_argument('--out', required=True)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
|
||||||
@@ -0,0 +1,191 @@
|
|||||||
|
"""Frozen ICDM2026 in-person program, with explicit source track evidence."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, Node, clean
|
||||||
|
|
||||||
|
SOURCE = 'icdm_program'
|
||||||
|
SNAPSHOT = 'remaining2026-official-leads-v1'
|
||||||
|
MANIFEST_SHA = '0e13313d94e3bff17f071d3a1384f6ed83ece156239fe5bc4633cc7a34fb8e04'
|
||||||
|
PROGRAM = 'https://icdm2026.neu.edu.cn/DetailedSessions/list.htm'
|
||||||
|
PAGES = {
|
||||||
|
'detailed-sessions': PROGRAM,
|
||||||
|
'research-policy': 'https://icdm2026.neu.edu.cn/CallforResearchTrack/list.htm',
|
||||||
|
'applied-policy': 'https://icdm2026.neu.edu.cn/CallforAppliedTrack/list.htm',
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def one(items):
|
||||||
|
if len(items) != 1:
|
||||||
|
raise ValueError('Expected exactly one reviewed field')
|
||||||
|
return items[0]
|
||||||
|
|
||||||
|
|
||||||
|
def has_class(node, name):
|
||||||
|
return name in node.attrs.get('class', '').split()
|
||||||
|
|
||||||
|
|
||||||
|
def field(node, name):
|
||||||
|
return clean(one([n for n in node.walk() if has_class(n, name)]).text())
|
||||||
|
|
||||||
|
|
||||||
|
def load_snapshot(root):
|
||||||
|
root = Path(root); folder = root/SNAPSHOT; manifest = folder/'manifest.json'
|
||||||
|
if root.is_symlink() or folder.is_symlink() or manifest.is_symlink() or manifest.stat().st_size > 256*1024:
|
||||||
|
raise ValueError('Unsafe source manifest')
|
||||||
|
raw = manifest.read_bytes()
|
||||||
|
if digest(raw) != MANIFEST_SHA:
|
||||||
|
raise ValueError('Frozen manifest hash changed')
|
||||||
|
data = json.loads(raw)
|
||||||
|
if data['status'] != 'research_completed_not_coverage_approval' or data['year'] != 2026:
|
||||||
|
raise ValueError('Wrong source edition or incomplete research run')
|
||||||
|
receipts = [r for r in data['entries'] if r['venue'] == 'ICDM']
|
||||||
|
if len(receipts) != 3 or {r['label'] for r in receipts} != PAGES.keys():
|
||||||
|
raise ValueError('Incomplete ICDM source set')
|
||||||
|
files = {'manifest.json': raw}; indexed = {}
|
||||||
|
for receipt in receipts:
|
||||||
|
filename = receipt['file']; path = folder/filename
|
||||||
|
if Path(filename).name != filename or path.is_symlink() or path.stat().st_size > 8*1024**2:
|
||||||
|
raise ValueError('Unsafe source file')
|
||||||
|
body = path.read_bytes()
|
||||||
|
if (receipt['url'] != PAGES[receipt['label']] or receipt['final_url'] != receipt['url']
|
||||||
|
or receipt['http_status'] != 200 or receipt['bytes'] != len(body)
|
||||||
|
or receipt['sha256'] != digest(body)
|
||||||
|
or not receipt['content_type'].lower().startswith('text/html')
|
||||||
|
or datetime.fromisoformat(receipt['at']).utcoffset() is None):
|
||||||
|
raise ValueError('Source receipt/body mismatch')
|
||||||
|
if filename in files:
|
||||||
|
raise ValueError('Repeated source filename')
|
||||||
|
files[filename] = body; indexed[receipt['label']] = receipt
|
||||||
|
return indexed, files
|
||||||
|
|
||||||
|
|
||||||
|
def parse_program(raw):
|
||||||
|
doc = Document(raw.decode('utf-8'))
|
||||||
|
main = one([n for n in doc.root.walk() if n.tag == 'main' and n.attrs.get('id') == 'session-details'])
|
||||||
|
text = clean(main.text())
|
||||||
|
if ('IEEE ICDM 2026' not in text or 'November 13–15, 2026' not in text
|
||||||
|
or 'In-person session presentation list.' not in text or '34 sessions · 324 papers' not in text):
|
||||||
|
raise ValueError('Wrong edition or unreviewed program scope')
|
||||||
|
sessions = [n for n in main.walk() if n.tag == 'details' and has_class(n, 'sd-session')]
|
||||||
|
expected = {f'L{i}' for i in range(1, 21)} | {f'S{i}' for i in range(1, 15)}
|
||||||
|
if len(sessions) != 34 or {n.attrs.get('data-session-id') for n in sessions} != expected:
|
||||||
|
raise ValueError('Incomplete or repeated session census')
|
||||||
|
records = []; seen = set()
|
||||||
|
for session in sessions:
|
||||||
|
sid = session.attrs['data-session-id']; track = session.attrs.get('data-track')
|
||||||
|
form = session.attrs.get('data-format')
|
||||||
|
if (session.attrs.get('id') != sid or track not in ('research', 'applied')
|
||||||
|
or form not in ('long', 'short')):
|
||||||
|
raise ValueError('Invalid session ID, track or presentation label')
|
||||||
|
label = 'Research Track' if track == 'research' else 'Applied Track'
|
||||||
|
if field(session, 'sd-track') != label:
|
||||||
|
raise ValueError('Visible track and source attribute disagree')
|
||||||
|
count = re.fullmatch(r'(Long|Short) · ([1-9][0-9]*) papers', field(session, 'sd-format'))
|
||||||
|
if count is None or count[1].lower() != form:
|
||||||
|
raise ValueError('Visible presentation label disagrees')
|
||||||
|
rows = [n for n in session.walk() if n.tag == 'tr' and 'data-paper-id' in n.attrs]
|
||||||
|
if len(rows) != int(count[2]):
|
||||||
|
raise ValueError('Session paper count changed')
|
||||||
|
note = field(session, 'sd-table-note')
|
||||||
|
if not note.startswith(f'{form.title()} Presentation ({len(rows)} papers) · '):
|
||||||
|
raise ValueError('Unreviewed presentation-note semantics')
|
||||||
|
context = {'id': sid, 'track_literal': track, 'presentation_literal': form,
|
||||||
|
'title': field(session, 'sd-session-title'),
|
||||||
|
'time_literal': field(session, 'sd-session-time'), 'room_literal': field(session, 'sd-room')}
|
||||||
|
for row in rows:
|
||||||
|
pid = row.attrs['data-paper-id']
|
||||||
|
cells = [n for n in row.children if isinstance(n, Node) and n.tag in ('th', 'td')]
|
||||||
|
if (not re.fullmatch(r'DM[1-9][0-9]*', pid) or pid in seen
|
||||||
|
or row.attrs.get('id') != 'paper-'+pid or len(cells) != 4
|
||||||
|
or [clean(n.text()) for n in cells] != [field(row, k) for k in
|
||||||
|
('sd-time', 'sd-paper-id', 'sd-paper-title', 'sd-authors')]
|
||||||
|
or field(row, 'sd-paper-id') != pid):
|
||||||
|
raise ValueError('Repeated/malformed program paper identity or cells')
|
||||||
|
seen.add(pid)
|
||||||
|
title = field(row, 'sd-paper-title'); authors_literal = field(row, 'sd-authors')
|
||||||
|
authors = [name.strip() for name in authors_literal.split(';')]
|
||||||
|
if not title or any(not name for name in authors):
|
||||||
|
raise ValueError('Missing program title or author')
|
||||||
|
records.append({'paper_id_literal': pid, 'title': title, 'authors_literal': authors_literal,
|
||||||
|
'source_reported_authors': authors, 'time_literal': field(row, 'sd-time'),
|
||||||
|
'session': context.copy(), 'source_position': len(records)+1})
|
||||||
|
if (len(records) != 324 or len([n for n in main.walk() if 'data-paper-id' in n.attrs]) != 324
|
||||||
|
or Counter(r['session']['track_literal'] for r in records) != {'research': 241, 'applied': 83}
|
||||||
|
or Counter(r['session']['presentation_literal'] for r in records) != {'long': 168, 'short': 156}):
|
||||||
|
raise ValueError('Incomplete reviewed paper/track/presentation census')
|
||||||
|
return records
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
receipts, files = load_snapshot(snapshots)
|
||||||
|
program = receipts['detailed-sessions']; observations = parse_program(files[program['file']])
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
evidence = []; directory = out/'evidence'; directory.mkdir()
|
||||||
|
for name, raw in files.items():
|
||||||
|
rel = 'evidence/'+digest(raw)+Path(name).suffix; (out/rel).write_bytes(raw)
|
||||||
|
evidence.append({'file': name, 'output_file': rel, 'sha256': digest(raw)})
|
||||||
|
outputs = []
|
||||||
|
for track in ('research', 'applied'):
|
||||||
|
rows = []
|
||||||
|
for item in observations:
|
||||||
|
if item['session']['track_literal'] != track:
|
||||||
|
continue
|
||||||
|
pid = item['paper_id_literal']; url = PROGRAM+'#paper-'+pid
|
||||||
|
rows.append({'source': SOURCE, 'external_id': '2026/'+pid, 'url': url,
|
||||||
|
'title': item['title'], 'authors': item['source_reported_authors'], 'abstract': None,
|
||||||
|
'venue': 'ICDM', 'year': 2026, 'track': track, 'status': 'accepted',
|
||||||
|
'identifiers': {}, 'submission_identifiers': [], 'observed_at': program['at'],
|
||||||
|
'evidence_url': url,
|
||||||
|
'raw_decision': 'Listed in official ICDM2026 in-person '+track+' program; not an original signed decision or verified publisher record.',
|
||||||
|
'tags': [{'namespace': SOURCE+':session_track', 'value': track},
|
||||||
|
{'namespace': SOURCE+':presentation', 'value': item['session']['presentation_literal']}],
|
||||||
|
'metadata': {'program_fields': item, 'program_receipt': program,
|
||||||
|
'original_decision_signature_verified': False,
|
||||||
|
'program_id_scope': 'Source program locator only; not asserted original submission ID or publication DOI.',
|
||||||
|
'author_scope': 'Source-reported program names; no publisher completeness or person alias assertion.',
|
||||||
|
'abstract_availability': 'No abstracts in this frozen program table.',
|
||||||
|
'track_evidence': 'Explicit data-track attribute agrees with visible session label; not inferred from topic.',
|
||||||
|
'paper_type': None,
|
||||||
|
'scope': 'In-person program only; excludes other program sections. Long/short describe presentation format. Requires work/version and publication review before import.'}})
|
||||||
|
path = out/f'icdm-program-2026-{track}.jsonl'
|
||||||
|
payload = ''.join(json.dumps(row, ensure_ascii=False)+'\n' for row in rows).encode(); path.write_bytes(payload)
|
||||||
|
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICDM', 'year': 2026,
|
||||||
|
'track': track, 'count': len(rows), 'file': str(path), 'file_sha256': digest(payload),
|
||||||
|
'list_url': PROGRAM, 'status': 'partial', 'missing_abstract_count': len(rows),
|
||||||
|
'source_expected_count': len(rows), 'scope': 'Frozen in-person program section, not all accepted/published papers.'}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n'); outputs.append(manifest)
|
||||||
|
report = {'status': 'collected_pending_identity_review', 'records': 324, 'sessions': 34,
|
||||||
|
'files': outputs, 'track_counts': {'research': 241, 'applied': 83},
|
||||||
|
'presentation_counts': {'long': 168, 'short': 156}, 'missing_abstract_count': 324,
|
||||||
|
'evidence': evidence, 'public_rejected_metadata': 'unknown_no_public_export_verified',
|
||||||
|
'negative_policy_scope': 'Research triple-blind and applied single-blind concern review-time visibility; neither proves final public decision coverage.',
|
||||||
|
'database_written': False, 'network_requests': 0, 'pdfs_downloaded': False,
|
||||||
|
'scope': 'In-person program observations only. No work merge, inferred author alias, publication identity, original signed decision or final whole-database audit.'}
|
||||||
|
temporary = out/'collection-report.json.part'; temporary.write_text(json.dumps(report, indent=2)+'\n')
|
||||||
|
temporary.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status': 'failed', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
|
parser.add_argument('--snapshots', required=True, type=Path)
|
||||||
|
parser.add_argument('--out', required=True, type=Path)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
|
||||||
@@ -0,0 +1,208 @@
|
|||||||
|
"""Frozen ISMIR 2026 accepted program, abstracts and opt-in review provenance."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import unicodedata
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, Node, clean
|
||||||
|
|
||||||
|
SOURCE = 'ismir_program'
|
||||||
|
TRACK = 'program_unclassified'
|
||||||
|
PAPERS = 'https://ismir2026program.ismir.net/papers.html'
|
||||||
|
ACCEPTED = 'https://ismir2026.ismir.net/accepted-papers'
|
||||||
|
SNAPSHOTS = {
|
||||||
|
'remaining2026-listings-research-v2': ('fea7b710587636db06794ee195cd083634c41e9054a217859d9dd0ccdd1afab9',
|
||||||
|
'research_completed_not_coverage_or_import_approval', {
|
||||||
|
'ismir-accepted.html': ACCEPTED,
|
||||||
|
'ismir-call.html': 'https://ismir2026.ismir.net/authors/call-for-papers'}),
|
||||||
|
'ismir2026-script-research-v1': ('5885b4bcfc2fc65374e5dc7a0d74f4eb86a0a36b8689f9fe8edbd10d49a08794',
|
||||||
|
'public_script_research_only', {'papers.js': 'https://ismir2026program.ismir.net/static/js/papers.js'}),
|
||||||
|
'ismir2026-json-research-v1': ('8be3e23277c199720264d940917118572655d18c0f9ac6db1b9cd30a6561b34d',
|
||||||
|
'public_metadata_json_research_only', {'papers.json': 'https://ismir2026program.ismir.net/papers.json'}),
|
||||||
|
}
|
||||||
|
REVIEW_KEYS = {'meta_review','publish_reviews','review1','review2','review3','review4'}
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def one(items):
|
||||||
|
if len(items) != 1:
|
||||||
|
raise ValueError('Expected exactly one reviewed source field')
|
||||||
|
return items[0]
|
||||||
|
|
||||||
|
|
||||||
|
def title_key(text):
|
||||||
|
return ''.join(c for c in unicodedata.normalize('NFKD', text).casefold() if c.isalnum())
|
||||||
|
|
||||||
|
|
||||||
|
def load_snapshots(root):
|
||||||
|
root = Path(root)
|
||||||
|
if root.is_symlink():
|
||||||
|
raise ValueError('Unsafe snapshot root')
|
||||||
|
result = {}
|
||||||
|
for name, (expected, status, urls) in SNAPSHOTS.items():
|
||||||
|
directory = root/name; path = directory/'manifest.json'
|
||||||
|
if directory.is_symlink() or path.is_symlink() or path.stat().st_size > 128*1024:
|
||||||
|
raise ValueError('Unsafe snapshot manifest')
|
||||||
|
raw = path.read_bytes()
|
||||||
|
if digest(raw) != expected:
|
||||||
|
raise ValueError('Reviewed manifest hash changed')
|
||||||
|
manifest = json.loads(raw)
|
||||||
|
if manifest['status'] != status or manifest['database_written'] is not False:
|
||||||
|
raise ValueError('Unreviewed snapshot scope')
|
||||||
|
files = {'manifest.json': raw}; receipts = {}
|
||||||
|
for filename, url in urls.items():
|
||||||
|
receipt = one([r for r in manifest['entries'] if r.get('file') == filename])
|
||||||
|
path = directory/filename
|
||||||
|
if path.is_symlink() or path.stat().st_size > 8*1024**2:
|
||||||
|
raise ValueError('Unsafe snapshot source file')
|
||||||
|
body = path.read_bytes()
|
||||||
|
if (receipt['url'] != url or receipt['final_url'] != url or receipt['http_status'] != 200
|
||||||
|
or receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)
|
||||||
|
or datetime.fromisoformat(receipt['at']).utcoffset() is None):
|
||||||
|
raise ValueError('Source receipt/body disagree')
|
||||||
|
files[filename] = body; receipts[filename] = receipt
|
||||||
|
result[name] = (files, receipts)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def author_names(literal):
|
||||||
|
# Institution names can contain semicolons; split only outside parentheses.
|
||||||
|
parts = []; depth = 0; start = 0
|
||||||
|
for index, char in enumerate(literal):
|
||||||
|
if char == '(': depth += 1
|
||||||
|
elif char == ')':
|
||||||
|
depth -= 1
|
||||||
|
if depth < 0: raise ValueError('Unbalanced author affiliation')
|
||||||
|
elif char == ';' and depth == 0:
|
||||||
|
parts.append(literal[start:index]); start = index+1
|
||||||
|
if depth:
|
||||||
|
raise ValueError('Unbalanced author affiliation')
|
||||||
|
parts.append(literal[start:])
|
||||||
|
names = [clean(p.split('(', 1)[0]) for p in parts]
|
||||||
|
if any(not name for name in names):
|
||||||
|
raise ValueError('Missing accepted-list author')
|
||||||
|
return names
|
||||||
|
|
||||||
|
|
||||||
|
def accepted_listing(raw):
|
||||||
|
doc = Document(raw.decode('utf-8')); tables = []
|
||||||
|
for table in doc.root.walk():
|
||||||
|
if table.tag != 'table': continue
|
||||||
|
rows = []
|
||||||
|
for row in table.walk():
|
||||||
|
if row.tag == 'tr':
|
||||||
|
rows.append([clean(n.text()) for n in row.children if isinstance(n, Node) and n.tag in ('th','td')])
|
||||||
|
if rows and rows[0] == ['Paper Title', 'Authors']:
|
||||||
|
tables.append(rows)
|
||||||
|
rows = one(tables)[1:]; result = {}
|
||||||
|
if len(rows) != 140:
|
||||||
|
raise ValueError('Incomplete official accepted-list census')
|
||||||
|
for row in rows:
|
||||||
|
if len(row) != 2 or not row[0] or not row[1] or title_key(row[0]) in result:
|
||||||
|
raise ValueError('Missing or ambiguous accepted-list title')
|
||||||
|
result[title_key(row[0])] = {'title': row[0], 'authors_literal': row[1], 'author_names': author_names(row[1])}
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def program_observations(rows, listing):
|
||||||
|
if not isinstance(rows, list) or len(rows) != 140:
|
||||||
|
raise ValueError('Incomplete public program census')
|
||||||
|
observations = []; seen = set(); used = set()
|
||||||
|
for original in rows:
|
||||||
|
if not isinstance(original, dict) or not isinstance(original.get('content'), dict):
|
||||||
|
raise ValueError('Invalid public program record')
|
||||||
|
pid = original.get('id'); content = original.get('content', {})
|
||||||
|
if not isinstance(pid, str) or not re.fullmatch(r'[1-9][0-9]*', pid) or pid in seen or original.get('forum') != pid:
|
||||||
|
raise ValueError('Missing/repeated numeric program locator, not an OpenReview ID')
|
||||||
|
seen.add(pid); title = content.get('title'); authors = content.get('authors'); abstract = content.get('abstract')
|
||||||
|
if (not isinstance(title, str) or not clean(title) or not isinstance(abstract, str) or not abstract.strip()
|
||||||
|
or not isinstance(authors, list) or not authors or any(not isinstance(a, str) or not clean(a) for a in authors)):
|
||||||
|
raise ValueError('Missing program title, author sequence or abstract')
|
||||||
|
key = title_key(title)
|
||||||
|
if key not in listing or key in used:
|
||||||
|
raise ValueError('Program/accepted-list title membership mismatch')
|
||||||
|
used.add(key); accepted = listing[key]
|
||||||
|
keywords = content.get('keywords'); presentation = content.get('paper_presentation')
|
||||||
|
session = content.get('session'); reviews = original.get('reviews')
|
||||||
|
if (not isinstance(keywords, list) or any(not isinstance(k, str) for k in keywords)
|
||||||
|
or presentation not in ('Remote','In-person','Remote during oral, In-person during nominee')
|
||||||
|
or not isinstance(session, list) or len(session) != 1 or not isinstance(session[0], str) or session[0] != original.get('session')
|
||||||
|
or not re.fullmatch(r'(?:[1-9]|1[0-4])', session[0])):
|
||||||
|
raise ValueError('Unreviewed source keywords, presentation or session')
|
||||||
|
if (not isinstance(reviews, dict) or set(reviews) != REVIEW_KEYS
|
||||||
|
or any(not isinstance(v, str) for v in reviews.values())
|
||||||
|
or reviews['publish_reviews'] not in ('Yes','No')):
|
||||||
|
raise ValueError('Unreviewed opt-in review structure')
|
||||||
|
review_text = [reviews[k] for k in REVIEW_KEYS-{'publish_reviews'}]
|
||||||
|
if (reviews['publish_reviews'] == 'No' and any(review_text)) or (reviews['publish_reviews'] == 'Yes' and not any(review_text)):
|
||||||
|
raise ValueError('Review publication preference/body disagree')
|
||||||
|
flags = []
|
||||||
|
if title != accepted['title']: flags.append('program_and_accepted_title_literal_versions_differ')
|
||||||
|
if authors != accepted['author_names']: flags.append('program_and_accepted_author_sequence_versions_differ')
|
||||||
|
observations.append({'original': original, 'accepted_listing': accepted, 'review_flags': flags})
|
||||||
|
if (used != listing.keys() or sum('program_and_accepted_title_literal_versions_differ' in o['review_flags'] for o in observations) != 2
|
||||||
|
or {o['original']['id'] for o in observations if 'program_and_accepted_author_sequence_versions_differ' in o['review_flags']} != {'60','126','433'}
|
||||||
|
or Counter(o['original']['reviews']['publish_reviews'] for o in observations) != {'Yes':101,'No':39}):
|
||||||
|
raise ValueError('Unreviewed source-version or opt-in census')
|
||||||
|
return observations
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists(): raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
sources = load_snapshots(snapshots)
|
||||||
|
official, _ = sources['remaining2026-listings-research-v2']; script, _ = sources['ismir2026-script-research-v1']
|
||||||
|
program, receipts = sources['ismir2026-json-research-v1']; receipt = receipts['papers.json']
|
||||||
|
if b"d3.json('papers.json')" not in script['papers.js'] or b'href="poster_${openreview.id}.html"' not in script['papers.js']:
|
||||||
|
raise ValueError('Unreviewed public program data path or poster link template')
|
||||||
|
observations = program_observations(json.loads(program['papers.json']), accepted_listing(official['ismir-accepted.html']))
|
||||||
|
evidence = out/'evidence'; evidence.mkdir()
|
||||||
|
for name, (files, _) in sources.items():
|
||||||
|
directory = evidence/name; directory.mkdir()
|
||||||
|
for filename, body in files.items(): (directory/filename).write_bytes(body)
|
||||||
|
rows = []
|
||||||
|
for observation in observations:
|
||||||
|
original = observation['original']; content = original['content']; pid = original['id']
|
||||||
|
tags = [{'namespace':SOURCE+':presentation','value':content['paper_presentation']}]
|
||||||
|
tags += [{'namespace':SOURCE+':keyword','value':value} for value in dict.fromkeys(clean(k) for k in content['keywords'] if clean(k))]
|
||||||
|
rows.append({'source':SOURCE,'external_id':'2026/'+pid,'url':'https://ismir2026program.ismir.net/poster_'+pid+'.html',
|
||||||
|
'title':content['title'],'authors':content['authors'],'abstract':content['abstract'],
|
||||||
|
'identifiers':{},'submission_identifiers':[],'venue':'ISMIR','year':2026,'track':TRACK,'status':'accepted',
|
||||||
|
'raw_decision':'Present in official ISMIR2026 accepted list and public virtual program; not an original signed decision.',
|
||||||
|
'evidence_url':ACCEPTED,'observed_at':receipt['at'],'tags':tags,
|
||||||
|
'metadata':{'program_entry':original,'accepted_listing':observation['accepted_listing'],'review_flags':observation['review_flags'],
|
||||||
|
'program_receipt':receipt,'source_evidence':{'file':'evidence/ismir2026-json-research-v1/papers.json','sha256':receipt['sha256'],'url':receipt['url']},
|
||||||
|
'original_decision_signature_verified':False,'paper_type':None,
|
||||||
|
'id_scope':'Numeric miniconf id/forum fields are program locators, not original OpenReview submission IDs; no inferred DOI or arxiv ID.',
|
||||||
|
'track_scope':'Unclassified regular accepted-list program; no workshop or research-method class inferred from topic/session/remote presentation.',
|
||||||
|
'review_scope':'101 source-opt-in review sets retained as raw public text;39No contain no review text. Reviewer recommendations/meta-review prose are not separately inferred final negative decisions.',
|
||||||
|
'abstract_transform':'Original JSON abstract string retained verbatim, including math, newlines and Unicode; no markup flattening or PDF extraction.',
|
||||||
|
'program_listing_scope':'Source versions remain separate.2title literal differences and3author-list differences flagged, not silently fused or resolved.',
|
||||||
|
'scope':'Pending work/history review. Poster URL follows frozen official JS template; poster pages not downloaded. No work merge, aliases, negative decision or final coverage/dedup claim.'}})
|
||||||
|
path = out/'ismir-program-2026-unclassified.jsonl';raw=''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in rows).encode();path.write_bytes(raw)
|
||||||
|
manifest={'source':SOURCE,'parser_version':1,'venue':'ISMIR','year':2026,'track':TRACK,'count':140,'file':str(path),
|
||||||
|
'file_sha256':digest(raw),'list_url':ACCEPTED,'status':'partial','missing_abstract_count':0,'source_expected_count':140}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)+'\n')
|
||||||
|
report={'status':'collected_pending_identity_review','records':140,'files':[manifest],'missing_abstract_count':0,
|
||||||
|
'literal_title_version_differences':2,'author_sequence_version_differences':3,'review_optin_yes':101,'review_optin_no':39,
|
||||||
|
'database_written':False,'network_requests':0,'pdfs_downloaded':False,'approved_for_import':0,
|
||||||
|
'public_rejected_availability':'unknown_accepted_program_reviews_are_not_a_rejected_export',
|
||||||
|
'scope':'140accepted observations;101opt-in review sets, not101rejected records. Source keyword/presentation tags are namespace-scoped; no original OpenReview ID assumption or source-version fusion.'}
|
||||||
|
part=out/'collection-report.json.part';part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n');part.replace(out/'collection-report.json');return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser=argparse.ArgumentParser(prog=prog,description=__doc__);parser.add_argument('--snapshots',required=True);parser.add_argument('--out',required=True)
|
||||||
|
args=parser.parse_args(argv);print(json.dumps(collect(args.out,snapshots=args.snapshots),ensure_ascii=False))
|
||||||
@@ -0,0 +1,202 @@
|
|||||||
|
"""Restore frozen ACM MM 2026 program observations, not publication decisions."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
SOURCE = 'mm_program'
|
||||||
|
SNAPSHOT = 'mm2026-program-and-public-probes-v1'
|
||||||
|
URL = 'https://2026.acmmm.org/site/program-data.json'
|
||||||
|
BODY_SHA = '08e2976ecba2d0d535000b0935f41c85076ec674e518a1df2c5843d9bde4d083'
|
||||||
|
MANIFEST_SHA = 'e43ac6cdfc86bac61305553a044dd6148800b5aa41a5d1bf06c486e90d0ed7b8'
|
||||||
|
TRACK_COUNTS = {'MRAC':35,'SUMAC':9,'NeuroMM':10,'IMPACT':8,'RichMediaGAI':10,
|
||||||
|
'MMSports':16,'MUWS':5,'EReL-MIR':7,'GreenMM':10,'DFF':12,'Main Track':1703,
|
||||||
|
'Grand Challenge':97,'Doctoral Symposium':4,'Datasets':50,'Interactive Art':23,
|
||||||
|
'Reproducibility':1,'Demo and Video':27,'Open Source':14,'Industry Talk':7,
|
||||||
|
'BNI':25,'QoEVMA':4,'HUMA':14,'WIPAI':8,'AMI':11,'Mulsemedia':9,
|
||||||
|
'UAVM':11,'MCHM':7,'JAV-CG':6}
|
||||||
|
CORE = ('paper_id','title','authors','affiliations','abstract','track')
|
||||||
|
PRESENTATION = ('kind','start_time','end_time','poster_id','oral_id')
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def safe_read(path, limit):
|
||||||
|
if any(p.is_symlink() for p in (path, *path.parents)) or not path.is_file() or path.stat().st_size > limit:
|
||||||
|
raise ValueError('Unsafe snapshot path or size')
|
||||||
|
return path.read_bytes()
|
||||||
|
|
||||||
|
|
||||||
|
def load_snapshot(root):
|
||||||
|
directory = Path(root)/SNAPSHOT
|
||||||
|
raw = safe_read(directory/'manifest.json', 128*1024)
|
||||||
|
if digest(raw) != MANIFEST_SHA:
|
||||||
|
raise ValueError('Reviewed manifest hash changed')
|
||||||
|
manifest = json.loads(raw)
|
||||||
|
if manifest.get('database_written') is not False or len(manifest['entries']) != 5:
|
||||||
|
raise ValueError('Unreviewed manifest scope')
|
||||||
|
evidence = {'manifest.json':raw}
|
||||||
|
for receipt in manifest['entries']:
|
||||||
|
name = receipt['file']
|
||||||
|
if Path(name).name != name or name in evidence:
|
||||||
|
raise ValueError('Unsafe or duplicate receipt name')
|
||||||
|
body = safe_read(directory/name, 12*1024**2)
|
||||||
|
if (receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)
|
||||||
|
or receipt['url'] != receipt['final_url']
|
||||||
|
or not receipt['content_type'].startswith('application/json')
|
||||||
|
or datetime.fromisoformat(receipt['at']).utcoffset() is None):
|
||||||
|
raise ValueError('Snapshot receipt/body disagree')
|
||||||
|
if name == 'program-data.json':
|
||||||
|
if receipt['url'] != URL or receipt['http_status'] != 200 or digest(body) != BODY_SHA:
|
||||||
|
raise ValueError('Wrong program source')
|
||||||
|
elif receipt['http_status'] != 403:
|
||||||
|
raise ValueError('Unreviewed public-access result')
|
||||||
|
evidence[name] = body
|
||||||
|
if 'program-data.json' not in evidence:
|
||||||
|
raise ValueError('Missing program source')
|
||||||
|
return evidence
|
||||||
|
|
||||||
|
|
||||||
|
def without_emails(value):
|
||||||
|
"""Keep public session structure; contact arrays remain in raw evidence only."""
|
||||||
|
if isinstance(value, dict):
|
||||||
|
return {k:without_emails(v) for k,v in value.items() if k != 'emails'}
|
||||||
|
if isinstance(value, list):
|
||||||
|
return [without_emails(v) for v in value]
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def program_locations(node, pointer=''):
|
||||||
|
"""Independent structural guard against silently skipping a new nesting form."""
|
||||||
|
if isinstance(node, dict):
|
||||||
|
if 'paper_id' in node:
|
||||||
|
yield pointer
|
||||||
|
for key,value in node.items():
|
||||||
|
escaped = key.replace('~','~0').replace('/','~1')
|
||||||
|
yield from program_locations(value, pointer+'/'+escaped)
|
||||||
|
elif isinstance(node, list):
|
||||||
|
for i,value in enumerate(node):
|
||||||
|
yield from program_locations(value, pointer+'/'+str(i))
|
||||||
|
|
||||||
|
|
||||||
|
def parse_program(raw):
|
||||||
|
data = json.loads(raw)
|
||||||
|
if (data.get('generated_at') != '2026-11-14T18:36:26.860677+00:00'
|
||||||
|
or data.get('days') != [1,2,3,4,5] or len(data.get('rooms',[])) != 20
|
||||||
|
or len(data.get('entries',[])) != 157):
|
||||||
|
raise ValueError('Unreviewed program edition or census')
|
||||||
|
records = {}; sessions = []; seen = []
|
||||||
|
|
||||||
|
def observe(paper, pointer, session_pointer, subsession_pointer=None):
|
||||||
|
if not set(CORE) <= paper.keys() or set(paper)-set(CORE)-set(PRESENTATION)-{'emails'}:
|
||||||
|
raise ValueError('Unreviewed program fields')
|
||||||
|
for key in ('paper_id','title','abstract','track'):
|
||||||
|
if not isinstance(paper[key],str):
|
||||||
|
raise ValueError('Wrong program field type')
|
||||||
|
for key in ('authors','affiliations'):
|
||||||
|
if not isinstance(paper[key],list) or any(not isinstance(v,str) for v in paper[key]):
|
||||||
|
raise ValueError('Wrong author or affiliation list')
|
||||||
|
if not paper['paper_id'].strip() or not paper['title'].strip() or paper['track'] not in TRACK_COUNTS:
|
||||||
|
raise ValueError('Missing identity/title or unreviewed source track')
|
||||||
|
core = {k:paper[k] for k in CORE}
|
||||||
|
pid = paper['paper_id']; seen.append(pointer)
|
||||||
|
if pid not in records:
|
||||||
|
missing = [k for k in ('authors','abstract') if not paper[k]]
|
||||||
|
records[pid] = {**core,'source':SOURCE,'year':2026,'venue':'MM',
|
||||||
|
'source_locator':{'namespace':'mm_program:program_id','value':pid},
|
||||||
|
'observed_status':'listed_in_official_program',
|
||||||
|
'publication_identity_verified':False,'final_decision_verified':False,
|
||||||
|
'approved_for_import':False,'missing_fields':missing,
|
||||||
|
'tags':[{'namespace':'mm_program:track','value':paper['track']}],
|
||||||
|
'contexts':[]}
|
||||||
|
elif {k:records[pid][k] for k in CORE} != core:
|
||||||
|
raise ValueError('Conflicting core versions for one program ID; review required')
|
||||||
|
records[pid]['contexts'].append({'source_pointer':pointer,'session_pointer':session_pointer,
|
||||||
|
'subsession_pointer':subsession_pointer,
|
||||||
|
'presentation':{k:paper[k] for k in PRESENTATION if k in paper}})
|
||||||
|
|
||||||
|
for i,entry in enumerate(data['entries']):
|
||||||
|
pointer = '/entries/'+str(i)
|
||||||
|
if entry['track'] != '':
|
||||||
|
raise ValueError('Unreviewed outer track; do not override paper.track')
|
||||||
|
session = without_emails({k:v for k,v in entry.items() if k not in ('papers','subsessions')})
|
||||||
|
session.update(source_pointer=pointer,subsessions=[])
|
||||||
|
for j,paper in enumerate(entry.get('papers') or []):
|
||||||
|
observe(paper,pointer+'/papers/'+str(j),pointer)
|
||||||
|
for j,sub in enumerate(entry.get('subsessions') or []):
|
||||||
|
sp = pointer+'/subsessions/'+str(j)
|
||||||
|
saved = without_emails({k:v for k,v in sub.items() if k != 'items'})
|
||||||
|
saved.update(source_pointer=sp,nonpaper_items=[])
|
||||||
|
for k,item in enumerate(sub.get('items') or []):
|
||||||
|
ip = sp+'/items/'+str(k)
|
||||||
|
if 'paper_id' in item:
|
||||||
|
observe(item,ip,pointer,sp)
|
||||||
|
else:
|
||||||
|
saved['nonpaper_items'].append({'source_pointer':ip,'fields':without_emails(item)})
|
||||||
|
session['subsessions'].append(saved)
|
||||||
|
sessions.append(session)
|
||||||
|
if set(seen) != set(program_locations(data)) or len(set(seen)) != len(seen):
|
||||||
|
raise ValueError('Unhandled or duplicated program location')
|
||||||
|
items = list(records.values())
|
||||||
|
if (len(seen) != 4326 or len(items) != 2143
|
||||||
|
or Counter(r['track'] for r in items) != TRACK_COUNTS
|
||||||
|
or sum(not r['abstract'] for r in items) != 216
|
||||||
|
or {r['paper_id'] for r in items if not r['authors']} != {'mfp4971','mfp7801'}):
|
||||||
|
raise ValueError('Program count or missing-field census changed')
|
||||||
|
return items,sessions
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists() or out.is_symlink():
|
||||||
|
raise ValueError('Output exists; use a fresh directory')
|
||||||
|
if any(p.is_symlink() for p in out.parents):
|
||||||
|
raise ValueError('Unsafe output parent')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
evidence = load_snapshot(snapshots)
|
||||||
|
rows,sessions = parse_program(evidence['program-data.json'])
|
||||||
|
(out/'evidence').mkdir()
|
||||||
|
for name,body in evidence.items():
|
||||||
|
(out/'evidence'/name).write_bytes(body)
|
||||||
|
outputs = []
|
||||||
|
for track,count in TRACK_COUNTS.items():
|
||||||
|
slug = re.sub('[^a-z0-9]+','_',track.lower()).strip('_')
|
||||||
|
path = out/('mm-program-2026-'+slug+'.jsonl')
|
||||||
|
raw = ''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in rows if r['track']==track).encode()
|
||||||
|
path.write_bytes(raw)
|
||||||
|
outputs.append({'file':str(path),'file_sha256':digest(raw),'count':count,
|
||||||
|
'track_literal':track,'format':'mm_program_observation_v1_not_ingestion_record',
|
||||||
|
'status':'pending_contribution_and_identity_review','approved_for_import':0})
|
||||||
|
session_raw = ''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in sessions).encode()
|
||||||
|
(out/'sessions.jsonl').write_bytes(session_raw)
|
||||||
|
report = {'status':'collected_pending_contribution_and_identity_review','source':SOURCE,
|
||||||
|
'records':2143,'program_observations':4326,'extra_presentation_observations':2183,
|
||||||
|
'track_counts':TRACK_COUNTS,'files':outputs,'session_count':157,
|
||||||
|
'sessions_file_sha256':digest(session_raw),'missing_abstract_count':216,
|
||||||
|
'missing_author_program_ids':['mfp4971','mfp7801'],
|
||||||
|
'source_body_sha256':BODY_SHA,'source_manifest_sha256':MANIFEST_SHA,
|
||||||
|
'database_written':False,'pdfs_downloaded':False,'network_requests':0,'approved_for_import':0,
|
||||||
|
'public_negative_access':{'anonymous_notes_probes':4,'http_statuses':[403]*4,
|
||||||
|
'negative_decisions_inferred':0,'scope':'Observed endpoints only; not proof of zero public negative decisions.'},
|
||||||
|
'scope':'Program IDs are source locators, not scientific-work or original submission IDs. All28source tracks retained; presentation times/kinds/locators and session context kept separately. No publication status, paper type, DOI, arxiv/OpenReview ID, rejection or withdrawal inferred. Blank fields are not negative decisions. Contribution/publication/work/history review and final whole-database audit remain.'}
|
||||||
|
part = out/'collection-report.json.part'
|
||||||
|
part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n')
|
||||||
|
part.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog,description=__doc__)
|
||||||
|
parser.add_argument('--snapshots',required=True)
|
||||||
|
parser.add_argument('--out',required=True)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
print(json.dumps(collect(args.out,snapshots=args.snapshots),ensure_ascii=False))
|
||||||
@@ -0,0 +1,336 @@
|
|||||||
|
"""Frozen MM 2026 registered publications with separately attributed program evidence.
|
||||||
|
|
||||||
|
All publications survive normalization. A program metadata match is not a
|
||||||
|
scientific-work identity decision; publisher article type is not research type.
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter, defaultdict
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
from html.parser import HTMLParser
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import unicodedata
|
||||||
|
from urllib.parse import parse_qs, urlsplit
|
||||||
|
|
||||||
|
from paper_atlas.collectors import mm2026 as program
|
||||||
|
|
||||||
|
SOURCE = 'mm_registry_program'
|
||||||
|
SNAPSHOT = 'mm2026-container-registry-v2'
|
||||||
|
MANIFEST_SHA = '128b9d02c72261a66edb9a4537d118bdccab3ed80d49a1e971fefdac399f9855'
|
||||||
|
PARENT_SHA = 'bc3ce7da2914f9ddd198c928b3acb2734a8370add3348c23c5f9f12135381628'
|
||||||
|
PARENT_DOI = '10.1145/3767308'
|
||||||
|
CONTAINER = 'Proceedings of the 34th ACM International Conference on Multimedia'
|
||||||
|
EVENT = "MM '26: The 34th ACM International Conference on Multimedia"
|
||||||
|
TRACKS = {'Main Track':'main','Grand Challenge':'grand_challenge',
|
||||||
|
'Interactive Art':'interactive_art','Datasets':'datasets','Reproducibility':'reproducibility',
|
||||||
|
'BNI':'bni','Open Source':'open_source','Doctoral Symposium':'doctoral_symposium',
|
||||||
|
'Demo and Video':'demo_and_video','Industry Talk':'industry_talk'}
|
||||||
|
BIBLIOGRAPHY = {'DOI','URL','title','original-title','subtitle','short-title','author',
|
||||||
|
'publisher','member','type','container-title','short-container-title','event','abstract',
|
||||||
|
'page','ISBN','isbn-type','published','published-print','published-online','issued',
|
||||||
|
'license','resource','relation','alternative-id'}
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def key(text):
|
||||||
|
return ''.join(c for c in unicodedata.normalize('NFKD',text).casefold() if c.isalnum())
|
||||||
|
|
||||||
|
|
||||||
|
def author_names(values):
|
||||||
|
return Counter(key(v) for v in values)
|
||||||
|
|
||||||
|
|
||||||
|
def load_registry(root, parent_path):
|
||||||
|
directory = Path(root)/SNAPSHOT
|
||||||
|
raw_manifest = program.safe_read(directory/'manifest.json', 128*1024)
|
||||||
|
raw_parent = program.safe_read(Path(parent_path), 128*1024)
|
||||||
|
if digest(raw_manifest) != MANIFEST_SHA or digest(raw_parent) != PARENT_SHA:
|
||||||
|
raise ValueError('Reviewed registry or parent hash changed')
|
||||||
|
manifest = json.loads(raw_manifest);parent = json.loads(raw_parent)['message']
|
||||||
|
if (manifest['status'] != 'complete_exact_container_registry_capture_not_work_or_track_approval'
|
||||||
|
or manifest['unique_registry_DOIs'] != 1918 or manifest['expected_registry_items'] != 1918
|
||||||
|
or len(manifest['pages']) != 8 or manifest['database_written'] is not False
|
||||||
|
or manifest['approved_for_import'] != 0 or manifest['parent_body_sha256'] != PARENT_SHA
|
||||||
|
or parent['DOI'] != PARENT_DOI or parent['type'] != 'proceedings'
|
||||||
|
or parent['title'] != [CONTAINER] or parent['event']['name'] != EVENT
|
||||||
|
or parent['member'] != '320' or parent['ISBN'] != ['9798400722134']):
|
||||||
|
raise ValueError('Incomplete reviewed registry or wrong proceedings')
|
||||||
|
evidence = {'manifest.json':raw_manifest,'parent.json':raw_parent};rows=[];seen=set()
|
||||||
|
for number,receipt in enumerate(manifest['pages'],1):
|
||||||
|
name=receipt['file']
|
||||||
|
if name != f'page-{number:03}.json':
|
||||||
|
raise ValueError('Unexpected registry page filename')
|
||||||
|
body=program.safe_read(directory/name,8*1024**2)
|
||||||
|
url=urlsplit(receipt['url']);query=parse_qs(url.query)
|
||||||
|
expected_filter='container-title:'+CONTAINER+',member:320'
|
||||||
|
if (receipt['http_status'] != 200 or receipt['url'] != receipt['final_url']
|
||||||
|
or url.scheme != 'https' or url.netloc != 'api.crossref.org' or url.path != '/works'
|
||||||
|
or set(query) != {'filter','rows','cursor'} or query['filter'] != [expected_filter]
|
||||||
|
or query['rows'] != ['250'] or not query['cursor'][0]
|
||||||
|
or datetime.fromisoformat(receipt['at']).utcoffset() is None
|
||||||
|
or receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)):
|
||||||
|
raise ValueError('Registry receipt and source body disagree')
|
||||||
|
obj=json.loads(body);message=obj['message'];expected=250 if number<8 else 168
|
||||||
|
if (obj['status'] != 'ok' or message['total-results'] != 1918
|
||||||
|
or len(message['items']) != expected or receipt['items'] != expected):
|
||||||
|
raise ValueError('Incomplete registry pagination')
|
||||||
|
evidence[name]=body
|
||||||
|
for position,original in enumerate(message['items']):
|
||||||
|
doi=original.get('DOI')
|
||||||
|
if not isinstance(doi,str) or not re.fullmatch(r'10\.1145/3767308\.[0-9]+',doi) or doi in seen:
|
||||||
|
raise ValueError('Missing, foreign or repeated publication DOI')
|
||||||
|
seen.add(doi)
|
||||||
|
if (original.get('URL') != 'https://doi.org/'+doi or original.get('type') != 'proceedings-article'
|
||||||
|
or original.get('container-title') != [CONTAINER] or original.get('event',{}).get('name') != EVENT
|
||||||
|
or original.get('member') != '320' or original.get('publisher') != 'ACM'
|
||||||
|
or original.get('published') != {'date-parts':[[2026,11,9]]} or original.get('abstract')):
|
||||||
|
raise ValueError('Wrong registered publication context or changed abstract availability')
|
||||||
|
titles=original.get('title');authors=original.get('author')
|
||||||
|
if (not isinstance(titles,list) or len(titles)!=1 or not isinstance(titles[0],str) or not titles[0].strip()
|
||||||
|
or not isinstance(authors,list) or not authors):
|
||||||
|
raise ValueError('Missing publication title or authors')
|
||||||
|
names=[];flags=[]
|
||||||
|
for i,author in enumerate(authors,1):
|
||||||
|
family=author.get('family');given=author.get('given','')
|
||||||
|
if not isinstance(family,str) or not family.strip() or not isinstance(given,str):
|
||||||
|
raise ValueError('Invalid registered author fields')
|
||||||
|
name=' '.join(x.strip() for x in (given,family) if x.strip())
|
||||||
|
if '@' in name or name.casefold()=='anonymous':
|
||||||
|
raise ValueError('Unreviewed author identity')
|
||||||
|
names.append(name)
|
||||||
|
if not given.strip():flags.append({'kind':'single_registered_name_field_not_split','author_position':i})
|
||||||
|
if re.search(r'<[^>]+>|&[A-Za-z#0-9]+;',titles[0]):
|
||||||
|
flags.append({'kind':'literal_title_markup_or_entity_retained_requires_review'})
|
||||||
|
canonical=json.dumps(original,ensure_ascii=False,sort_keys=True,separators=(',',':')).encode()
|
||||||
|
rows.append({'doi':doi,'title':titles[0],'authors':names,'flags':flags,
|
||||||
|
'fields':{k:v for k,v in original.items() if k in BIBLIOGRAPHY},
|
||||||
|
'original_field_names':sorted(original),'original_canonical_json_sha256':digest(canonical),
|
||||||
|
'receipt':receipt,'source_position':position})
|
||||||
|
if len(rows)!=1918:
|
||||||
|
raise ValueError('Incomplete registered publication census')
|
||||||
|
return evidence,rows
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
class _TitleFormatting(HTMLParser):
|
||||||
|
"""Decode text once, removing only balanced, attribute-free reviewed tags."""
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__(convert_charrefs=True)
|
||||||
|
self.parts=[];self.stack=[];self.valid=True
|
||||||
|
|
||||||
|
def handle_data(self, text):
|
||||||
|
self.parts.append(text)
|
||||||
|
|
||||||
|
def handle_starttag(self, tag, attrs):
|
||||||
|
if tag not in {'i','scp','sup','sub'} or attrs:
|
||||||
|
self.valid=False
|
||||||
|
else:
|
||||||
|
self.stack.append(tag)
|
||||||
|
|
||||||
|
def handle_endtag(self, tag):
|
||||||
|
if not self.stack or self.stack[-1]!=tag:
|
||||||
|
self.valid=False
|
||||||
|
else:
|
||||||
|
self.stack.pop()
|
||||||
|
|
||||||
|
def handle_startendtag(self, tag, attrs):
|
||||||
|
self.valid=False
|
||||||
|
|
||||||
|
def handle_comment(self, text):
|
||||||
|
self.valid=False
|
||||||
|
|
||||||
|
def handle_decl(self, text):
|
||||||
|
self.valid=False
|
||||||
|
|
||||||
|
def handle_pi(self, text):
|
||||||
|
self.valid=False
|
||||||
|
|
||||||
|
def unknown_decl(self, text):
|
||||||
|
self.valid=False
|
||||||
|
|
||||||
|
|
||||||
|
def formatting_text(title):
|
||||||
|
# HTMLParser may silently discard a trailing incomplete start tag.
|
||||||
|
# Check lexical coverage before parsing so no title content can disappear.
|
||||||
|
literal=re.sub(r'</?(?:i|scp|sup|sub)\s*>','',title,flags=re.I)
|
||||||
|
if '<' in literal or '>' in literal:
|
||||||
|
return None
|
||||||
|
parser=_TitleFormatting()
|
||||||
|
parser.feed(title);parser.close()
|
||||||
|
text=''.join(parser.parts)
|
||||||
|
return text if parser.valid and not parser.stack and text.strip() and '<' not in text and '>' not in text else None
|
||||||
|
|
||||||
|
|
||||||
|
FORMATTING_MATCHES = {'10.1145/3767308.3834907': 'mfp0053',
|
||||||
|
'10.1145/3767308.3834936': 'mfp0229',
|
||||||
|
'10.1145/3767308.3834946': 'mfp0297',
|
||||||
|
'10.1145/3767308.3834956': 'mfp0339',
|
||||||
|
'10.1145/3767308.3835035': 'mfp0798',
|
||||||
|
'10.1145/3767308.3835055': 'mfp0903',
|
||||||
|
'10.1145/3767308.3835101': 'mfp1175',
|
||||||
|
'10.1145/3767308.3835374': 'mfp2617',
|
||||||
|
'10.1145/3767308.3835424': 'mfp2909',
|
||||||
|
'10.1145/3767308.3835452': 'mfp3048',
|
||||||
|
'10.1145/3767308.3835460': 'mfp3109',
|
||||||
|
'10.1145/3767308.3835544': 'mfp3554',
|
||||||
|
'10.1145/3767308.3835591': 'mfp3783',
|
||||||
|
'10.1145/3767308.3835667': 'mfp4173',
|
||||||
|
'10.1145/3767308.3835763': 'mfp4778',
|
||||||
|
'10.1145/3767308.3835895': 'mfp5537',
|
||||||
|
'10.1145/3767308.3835959': 'mfp5823',
|
||||||
|
'10.1145/3767308.3836215': 'mfp7332',
|
||||||
|
'10.1145/3767308.3836272': 'mfp7635',
|
||||||
|
'10.1145/3767308.3836293': 'mfp7732',
|
||||||
|
'10.1145/3767308.3836447': 'mfp8648',
|
||||||
|
'10.1145/3767308.3836600': 'mfp9815',
|
||||||
|
'10.1145/3767308.3838656': 'mds177',
|
||||||
|
'10.1145/3767308.3839012': 'mde41',
|
||||||
|
'10.1145/3767308.3839024': 'mde15',
|
||||||
|
'10.1145/3767308.3839028': 'mde02'}
|
||||||
|
|
||||||
|
|
||||||
|
def match_program(rows, observations):
|
||||||
|
titles=defaultdict(list);formatted=defaultdict(list)
|
||||||
|
for p in observations:
|
||||||
|
titles[key(p['title'])].append(p)
|
||||||
|
text=formatting_text(p['title'])
|
||||||
|
if text is not None:formatted[key(text)].append(p)
|
||||||
|
matched=set();kinds=Counter()
|
||||||
|
for row in rows:
|
||||||
|
row.pop('program',None);row.pop('formatting_comparison',None)
|
||||||
|
candidates=titles[key(row['title'])]
|
||||||
|
method='literal_title'
|
||||||
|
# A literal-title author disagreement must not be bypassed by fallback.
|
||||||
|
if not candidates:
|
||||||
|
text=formatting_text(row['title'])
|
||||||
|
if text is not None:
|
||||||
|
candidates=formatted[key(text)]
|
||||||
|
if candidates:method='reviewed_title_formatting'
|
||||||
|
exact=[p for p in candidates if author_names(p['authors'])==author_names(row['authors'])]
|
||||||
|
if len(exact)==1:
|
||||||
|
row['kind']=('unique_program_format_title_fullauthor_agreement_not_identity_approval'
|
||||||
|
if method=='reviewed_title_formatting' else 'unique_program_title_fullauthor_agreement_not_identity_approval')
|
||||||
|
row['program']=exact[0];matched.add(exact[0]['paper_id'])
|
||||||
|
if exact[0]['track'] not in TRACKS:
|
||||||
|
raise ValueError('New registered program track requires review')
|
||||||
|
elif len(exact)>1:row['kind']='multiple_program_ids_same_title_authors_hold'
|
||||||
|
elif candidates:
|
||||||
|
row['kind']=('program_format_author_version_requires_review'
|
||||||
|
if method=='reviewed_title_formatting' else 'program_author_version_requires_review')
|
||||||
|
else:row['kind']='no_exact_program_title_requires_review'
|
||||||
|
row['program_title_candidates']=[p['paper_id'] for p in candidates]
|
||||||
|
row['program_exact_candidates']=[p['paper_id'] for p in exact]
|
||||||
|
if method=='reviewed_title_formatting':
|
||||||
|
row['formatting_comparison']={'method':'balanced_attribute_free_i_scp_sup_sub_and_single_entity_decode_then_nfkd_alnum',
|
||||||
|
'registry_comparison_text':formatting_text(row['title']),
|
||||||
|
'program_comparison_texts':{p['paper_id']:formatting_text(p['title']) for p in candidates},
|
||||||
|
'raw_titles_preserved':True,'work_identity_approved':False}
|
||||||
|
kinds[row['kind']]+=1
|
||||||
|
return dict(kinds),[p for p in observations if p['paper_id'] not in matched]
|
||||||
|
|
||||||
|
|
||||||
|
def correlate(rows, observations):
|
||||||
|
kinds,unmatched=match_program(rows,observations)
|
||||||
|
expected={'unique_program_title_fullauthor_agreement_not_identity_approval':1726,
|
||||||
|
'unique_program_format_title_fullauthor_agreement_not_identity_approval':26,
|
||||||
|
'program_author_version_requires_review':122,
|
||||||
|
'program_format_author_version_requires_review':1,
|
||||||
|
'no_exact_program_title_requires_review':43}
|
||||||
|
actual={r['doi']:r['program']['paper_id'] for r in rows
|
||||||
|
if r['kind']=='unique_program_format_title_fullauthor_agreement_not_identity_approval'}
|
||||||
|
if kinds != expected or actual != FORMATTING_MATCHES:
|
||||||
|
raise ValueError('Reviewed program/publication correspondence changed')
|
||||||
|
return kinds,unmatched
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, registry_snapshots, program_snapshots, parent_snapshot):
|
||||||
|
out=Path(out)
|
||||||
|
if out.exists() or out.is_symlink():
|
||||||
|
raise ValueError('Output exists; inspect prior operation, never overwrite')
|
||||||
|
if any(p.is_symlink() for p in out.parents):
|
||||||
|
raise ValueError('Unsafe output parent')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
registry_evidence,publications=load_registry(registry_snapshots,parent_snapshot)
|
||||||
|
program_evidence=program.load_snapshot(program_snapshots)
|
||||||
|
observations,_=program.parse_program(program_evidence['program-data.json'])
|
||||||
|
kinds,unmatched=correlate(publications,observations)
|
||||||
|
for group,files in [('registry',registry_evidence),('program',program_evidence)]:
|
||||||
|
directory=out/'evidence'/group;directory.mkdir(parents=True)
|
||||||
|
for name,body in files.items():(directory/name).write_bytes(body)
|
||||||
|
buckets=defaultdict(list);missing=0;copied=0;known_tracks=Counter()
|
||||||
|
for item in sorted(publications,key=lambda r:r['doi']):
|
||||||
|
p=item.get('program');doi=item['doi'];receipt=item['receipt']
|
||||||
|
track=TRACKS[p['track']] if p else 'proceedings_unclassified'
|
||||||
|
abstract=p['abstract'] if p and p['abstract'] else None
|
||||||
|
missing+=abstract is None;copied+=abstract is not None
|
||||||
|
if p:known_tracks[p['track']]+=1
|
||||||
|
tags=[{'namespace':SOURCE+':crossref:type','value':'proceedings-article'}]
|
||||||
|
if p:tags.append({'namespace':SOURCE+':mm_program:track','value':p['track']})
|
||||||
|
buckets[track].append({'source':SOURCE,'external_id':doi,'url':'https://doi.org/'+doi,
|
||||||
|
'title':item['title'],'authors':item['authors'],'abstract':abstract,
|
||||||
|
'identifiers':{'doi':doi},'submission_identifiers':[{'scheme':'publication_doi','value':doi}],
|
||||||
|
'venue':'MM','year':2026,'track':track,'status':'accepted',
|
||||||
|
'raw_decision':'Registered publication in MM2026 proceedings; not an original signed decision.',
|
||||||
|
'evidence_url':receipt['url'],'observed_at':receipt['at'],'tags':tags,
|
||||||
|
'metadata':{**({'program_title_formatting_comparison':item['formatting_comparison']} if 'formatting_comparison' in item else {}),'registry_fields':item['fields'],'registry_original_field_names':item['original_field_names'],
|
||||||
|
'registry_source_evidence':{'file':'evidence/registry/'+receipt['file'],'sha256':receipt['sha256'],
|
||||||
|
'url':receipt['url'],'json_pointer':'/message/items/'+str(item['source_position']),
|
||||||
|
'canonical_object_json_sha256':item['original_canonical_json_sha256']},
|
||||||
|
'registry_manifest':{'file':'evidence/registry/manifest.json','sha256':MANIFEST_SHA},
|
||||||
|
'parent_evidence':{'file':'evidence/registry/parent.json','sha256':PARENT_SHA},
|
||||||
|
'program_source_evidence':{'file':'evidence/program/program-data.json','sha256':program.BODY_SHA},
|
||||||
|
'program_fields':p,'program_match_kind':item['kind'],
|
||||||
|
'program_title_candidates':item['program_title_candidates'],
|
||||||
|
'program_exact_candidates':item['program_exact_candidates'],
|
||||||
|
'abstract_source':('official_program_reviewed_title_formatting_correspondence' if item.get('formatting_comparison') else 'official_program_exact_metadata_correspondence') if abstract is not None else 'not_available_from_reviewed_sources',
|
||||||
|
'registry_abstract_available':False,'original_decision_signature_verified':False,
|
||||||
|
'scientific_work_identity_approved':False,'research_type_classified':False,
|
||||||
|
'review_flags':item['flags'],
|
||||||
|
'scope':'Derived publication/program metadata with separate original evidence. All registered articles retained, including art/industry/demo; bibliographic type is not research type. Program correlation does not approve work identity or person aliases. Publication dates are not decision dates. DOI is publication evidence, not original OpenReview ID.'}})
|
||||||
|
outputs=[]
|
||||||
|
for track,rows in sorted(buckets.items()):
|
||||||
|
path=out/f'mm-publications-2026-{track}.jsonl'
|
||||||
|
raw=''.join(json.dumps(row,ensure_ascii=False)+'\n' for row in rows).encode();path.write_bytes(raw)
|
||||||
|
manifest={'source':SOURCE,'parser_version':2,'venue':'MM','year':2026,'track':track,'count':len(rows),
|
||||||
|
'file':str(path),'file_sha256':digest(raw),'list_url':'https://doi.org/'+PARENT_DOI,
|
||||||
|
'status':'partial','missing_abstract_count':sum(row['abstract'] is None for row in rows),
|
||||||
|
'scope':'Registered container partition only; not allMMprogram/workshops, independent works or public-negative coverage.'}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,ensure_ascii=False,indent=2)+'\n');outputs.append(manifest)
|
||||||
|
report={'status':'collected_publications_pending_work_identity_review','records':1918,'unique_registered_dois':1918,
|
||||||
|
'files':outputs,'program_ids':2143,'program_observations':4326,'program_match_counts':kinds,
|
||||||
|
'matched_program_track_counts':dict(known_tracks),'unmatched_program':unmatched,
|
||||||
|
'unmatched_program_count':len(unmatched),'registry_missing_abstract_count':1918,
|
||||||
|
'abstracts_from_exact_program_correspondence':copied,'output_missing_abstract_count':missing,
|
||||||
|
'abstracts_from_literal_program_correspondence':copied-26,'abstracts_from_reviewed_title_formatting':26,
|
||||||
|
'registered_single_name_fields':sum(f['kind']=='single_registered_name_field_not_split' for r in publications for f in r['flags']),
|
||||||
|
'registered_title_markup_flags':sum(any(f['kind']=='literal_title_markup_or_entity_retained_requires_review' for f in r['flags']) for r in publications),
|
||||||
|
'database_written':False,'approved_for_import':0,'network_requests':0,'pdfs_downloaded':False,
|
||||||
|
'public_negative_availability':'unknown_no_negative_decisions_inferred',
|
||||||
|
'scope':'1918registeredpublication DOIs, not1918proven distinct scientificworks.166title/authorvariants remainunclassified,391unmatchedprogram items retained.26formattingcorrespondences retainedseparately;rawtitlesunchanged. All28programtracks remain independent census; article type never implies research. Per-work/source/DOI/history review andfinalwholeDBaudit remain.'}
|
||||||
|
if (sum(f['count'] for f in outputs)!=1918 or copied!=1743 or missing!=175 or len(unmatched)!=391):
|
||||||
|
raise ValueError('Reviewed output partition or abstract census changed')
|
||||||
|
part=out/'collection-report.json.part';part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n');part.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser=argparse.ArgumentParser(prog=prog,description=__doc__)
|
||||||
|
parser.add_argument('--registry-snapshots',required=True)
|
||||||
|
parser.add_argument('--program-snapshots',required=True)
|
||||||
|
parser.add_argument('--parent-snapshot',required=True)
|
||||||
|
parser.add_argument('--out',required=True)
|
||||||
|
args=parser.parse_args(argv)
|
||||||
|
report=collect(args.out,registry_snapshots=args.registry_snapshots,
|
||||||
|
program_snapshots=args.program_snapshots,parent_snapshot=args.parent_snapshot)
|
||||||
|
print(json.dumps({k:report[k] for k in ('status','records','files','program_match_counts',
|
||||||
|
'abstracts_from_exact_program_correspondence','output_missing_abstract_count',
|
||||||
|
'unmatched_program_count','database_written','approved_for_import')},ensure_ascii=False))
|
||||||
@@ -0,0 +1,172 @@
|
|||||||
|
"""Frozen RecSys 2026 accepted-contribution observations, preserving source variants."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, Node, clean
|
||||||
|
|
||||||
|
SOURCE = 'recsys_program'
|
||||||
|
SNAPSHOT = 'remaining2026-listings-research-v2'
|
||||||
|
MANIFEST_SHA = 'fea7b710587636db06794ee195cd083634c41e9054a217859d9dd0ccdd1afab9'
|
||||||
|
URL = 'https://recsys.acm.org/recsys26/contributions/'
|
||||||
|
TRACKS = [('long', 'Long Papers', 65), ('short', 'Short Papers', 29),
|
||||||
|
('past_present_future', 'PPF Papers', 13), ('reproducibility', 'Reproducibility', 4),
|
||||||
|
('resource', 'Resource', 7), ('industry', 'Industry', 83),
|
||||||
|
('research_practice_notes', 'R&P Notes', 34), ('demo', 'Demos', 18),
|
||||||
|
('doctoral_symposium', 'Doctoral', 10)]
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def one(items):
|
||||||
|
if len(items) != 1:
|
||||||
|
raise ValueError('Expected exactly one reviewed source field')
|
||||||
|
return items[0]
|
||||||
|
|
||||||
|
|
||||||
|
def load_snapshot(root):
|
||||||
|
root = Path(root); directory = root/SNAPSHOT; path = directory/'manifest.json'
|
||||||
|
if root.is_symlink() or directory.is_symlink() or path.is_symlink() or path.stat().st_size > 128*1024:
|
||||||
|
raise ValueError('Unsafe snapshot manifest')
|
||||||
|
raw = path.read_bytes()
|
||||||
|
if digest(raw) != MANIFEST_SHA:
|
||||||
|
raise ValueError('Reviewed manifest hash changed')
|
||||||
|
manifest = json.loads(raw)
|
||||||
|
if manifest['status'] != 'research_completed_not_coverage_or_import_approval' or manifest['database_written'] is not False:
|
||||||
|
raise ValueError('Unreviewed snapshot scope')
|
||||||
|
receipt = one([e for e in manifest['entries'] if e['venue'] == 'RecSys' and e['label'] == 'contributions'])
|
||||||
|
name = receipt['file']; path = directory/name
|
||||||
|
if Path(name).name != name or path.is_symlink() or path.stat().st_size > 4*1024**2:
|
||||||
|
raise ValueError('Unsafe snapshot body')
|
||||||
|
body = path.read_bytes()
|
||||||
|
if (receipt['url'] != URL or receipt['final_url'] != URL or receipt['http_status'] != 200
|
||||||
|
or not receipt['content_type'].startswith('text/html') or receipt['bytes'] != len(body)
|
||||||
|
or receipt['sha256'] != digest(body) or datetime.fromisoformat(receipt['at']).utcoffset() is None):
|
||||||
|
raise ValueError('Snapshot receipt/body disagree')
|
||||||
|
return receipt, {'manifest.json': raw, name: body}, body
|
||||||
|
|
||||||
|
|
||||||
|
def body_text(node, doi_url):
|
||||||
|
if not isinstance(node, Node):
|
||||||
|
return node
|
||||||
|
if node.tag in ('script', 'style') or (node.tag == 'a' and node.attrs.get('href') == doi_url):
|
||||||
|
return ''
|
||||||
|
text = ''.join(body_text(c, doi_url) for c in node.children)
|
||||||
|
return ' '+text+' ' if node.tag in ('p','div','li','br','section') else text
|
||||||
|
|
||||||
|
|
||||||
|
def parse_program(raw):
|
||||||
|
doc = Document(raw.decode('utf-8')); nodes = list(doc.root.walk())
|
||||||
|
if not any(n.tag == 'title' and 'RecSys 26 - Accepted Contributions' in n.text() for n in nodes):
|
||||||
|
raise ValueError('Wrong conference edition')
|
||||||
|
menu = one([n for n in nodes if n.tag == 'ul' and n.attrs.get('id') == 'tab-1'])
|
||||||
|
labels = [(n.attrs.get('href'), clean(n.text())) for n in menu.walk() if n.tag == 'a']
|
||||||
|
if labels != [(f'#content-tab-1-{i}', label) for i, (_,label,_) in enumerate(TRACKS)]:
|
||||||
|
raise ValueError('Unreviewed track menu')
|
||||||
|
observations = []; seen = set()
|
||||||
|
for tab, (track, label, count) in enumerate(TRACKS):
|
||||||
|
section = one([n for n in nodes if n.tag == 'div' and n.attrs.get('id') == f'content-tab-1-{tab}'])
|
||||||
|
listing = one([n for n in section.walk() if n.tag == 'ul' and n.attrs.get('id') == f'accordion-{tab+1}'])
|
||||||
|
rows = [n for n in listing.children if isinstance(n, Node) and n.tag == 'li']
|
||||||
|
if len(rows) != count:
|
||||||
|
raise ValueError('Incomplete reviewed track census')
|
||||||
|
for position, row in enumerate(rows, 1):
|
||||||
|
anchor = one([n for n in row.children if isinstance(n, Node) and n.tag == 'a'])
|
||||||
|
locator = anchor.attrs.get('rel', '').removeprefix('#')
|
||||||
|
if not re.fullmatch(f'accordion-{tab+1}-slide-[0-9]+', locator) or locator in seen:
|
||||||
|
raise ValueError('Missing or repeated program locator')
|
||||||
|
seen.add(locator)
|
||||||
|
types = [n for n in anchor.walk() if n.tag == 'span' and n.attrs.get('class') == 'paper-type']
|
||||||
|
type_node = one(types); type_literal = type_node.attrs.get('title'); abbreviation = clean(type_node.text())
|
||||||
|
expected_type = [('Research','RES'),('Research','RES'),('PastPresentFuture','PPF'),
|
||||||
|
('Research','REPR'),('Research','RSC'),('Research','IND'),('Research','RPN'),('Research','DEMO'),('Doctoral Symposium','DS')][tab]
|
||||||
|
if (type_literal,abbreviation) != expected_type:
|
||||||
|
raise ValueError('Unreviewed source contribution type')
|
||||||
|
title = clean(''.join(c.text() if isinstance(c, Node) else c for c in anchor.children
|
||||||
|
if not isinstance(c, Node) or c.tag not in ('span','em','br')))
|
||||||
|
authors_literal = clean(one([n for n in anchor.walk() if n.tag == 'em']).text())
|
||||||
|
if not authors_literal.startswith('by '):
|
||||||
|
raise ValueError('Missing program author prefix')
|
||||||
|
authors_literal = authors_literal[3:]
|
||||||
|
authors = [n.strip() for n in re.split(r',\s+|\s+and\s+', authors_literal)]
|
||||||
|
if not title or any(not n or '@' in n for n in authors):
|
||||||
|
raise ValueError('Missing title or program author')
|
||||||
|
body = one([n for n in row.children if isinstance(n, Node) and n.tag == 'div'])
|
||||||
|
doi_url = one([n.attrs.get('href') for n in body.walk() if n.tag == 'a'
|
||||||
|
and n.attrs.get('href', '').startswith('https://doi.org/')])
|
||||||
|
if not re.fullmatch(r'https://doi.org/10\.1145/3773078\.[0-9]+', doi_url):
|
||||||
|
raise ValueError('Wrong publication DOI family')
|
||||||
|
abstract = clean(body_text(body, doi_url))
|
||||||
|
if len(abstract) < 100 or any(n.tag in ('math','sup','sub','img','svg') for n in body.walk()):
|
||||||
|
raise ValueError('Missing abstract or unreviewed structured markup')
|
||||||
|
observations.append({'doi': doi_url.removeprefix('https://doi.org/'), 'tab': tab, 'position': position,
|
||||||
|
'locator': locator, 'title': title, 'authors': authors, 'authors_literal': authors_literal,
|
||||||
|
'abstract': abstract, 'type_labels': [type_literal], 'source_type_abbreviation': abbreviation,
|
||||||
|
'track': track, 'track_label': label})
|
||||||
|
counts = Counter(r['doi'] for r in observations)
|
||||||
|
if len(observations) != 263 or len(counts) != 260 or {d:n for d,n in counts.items() if n>1} != {
|
||||||
|
'10.1145/3773078.3831839':2,'10.1145/3773078.3831819':2,'10.1145/3773078.3831861':2}:
|
||||||
|
raise ValueError('Unreviewed DOI observation census')
|
||||||
|
for item in observations:
|
||||||
|
item['review_flags'] = ['repeated_program_doi_preserve_versions_requires_review'] if counts[item['doi']] > 1 else []
|
||||||
|
return observations
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
receipt, files, body = load_snapshot(snapshots); observations = parse_program(body)
|
||||||
|
evidence = out/'evidence'; evidence.mkdir()
|
||||||
|
for name, raw in files.items():
|
||||||
|
(evidence/name).write_bytes(raw)
|
||||||
|
outputs = []
|
||||||
|
for track,label,count in TRACKS:
|
||||||
|
rows = []
|
||||||
|
for item in observations:
|
||||||
|
if item['track'] != track:
|
||||||
|
continue
|
||||||
|
doi = item['doi']; url = URL+'#'+item['locator']
|
||||||
|
rows.append({'source': SOURCE, 'external_id': '2026/'+item['locator'], 'url': url,
|
||||||
|
'title': item['title'], 'authors': item['authors'], 'abstract': item['abstract'],
|
||||||
|
'identifiers': {'doi': doi}, 'submission_identifiers': [{'scheme':'publication_doi','value':doi}],
|
||||||
|
'venue':'RecSys', 'year':2026, 'track':track, 'status':'accepted', 'observed_at':receipt['at'],
|
||||||
|
'evidence_url':URL, 'raw_decision':'Listed as an accepted contribution in the official RecSys2026 '+label+' section; not a signed review decision.',
|
||||||
|
'tags':[{'namespace':SOURCE+':contribution_type','value':item['type_labels'][0]}],
|
||||||
|
'metadata':{'program_fields':item, 'program_receipt':receipt,
|
||||||
|
'source_evidence':{'file':'evidence/'+receipt['file'],'sha256':receipt['sha256'],'url':URL},
|
||||||
|
'source_manifest':{'file':'evidence/manifest.json','sha256':MANIFEST_SHA},
|
||||||
|
'original_decision_signature_verified':False,
|
||||||
|
'author_scope':'Source-reported comma/and-delimited names; original literal retained. Not verified publisher person identities or aliases.',
|
||||||
|
'doi_scope':'DOI asserted by official program hyperlink, not independently verified live publisher metadata.',
|
||||||
|
'abstract_transform':'HTML display text and whitespace normalization; source retained. No PDF or cross-source fusion.',
|
||||||
|
'scope':'Program observation keyed by source locator. Repeated DOI variants retained separately and flagged, not counted as distinct scientific works. Requires DOI/work/history review before import.'}})
|
||||||
|
path = out/f'recsys-program-2026-{track}.jsonl';raw=''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in rows).encode();path.write_bytes(raw)
|
||||||
|
manifest={'source':SOURCE,'parser_version':1,'venue':'RecSys','year':2026,'track':track,'count':len(rows),
|
||||||
|
'file':str(path),'file_sha256':digest(raw),'list_url':URL,'status':'partial','missing_abstract_count':0,
|
||||||
|
'scope':'Official accepted-contribution observations; unique DOI counts differ from observation counts.'}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)+'\n');outputs.append(manifest)
|
||||||
|
report={'status':'collected_pending_identity_review','records':263,'unique_dois':260,'files':outputs,
|
||||||
|
'track_counts':{track:count for track,_,count in TRACKS},'missing_abstract_count':0,
|
||||||
|
'duplicate_doi_observations':6,'repeated_dois':{d:n for d,n in Counter(r['doi'] for r in observations).items() if n>1},
|
||||||
|
'source_manifest_sha256':MANIFEST_SHA,'database_written':False,'network_requests':0,'pdfs_downloaded':False,
|
||||||
|
'approved_for_import':0,'public_rejected_availability':'unknown_no_public_export_verified',
|
||||||
|
'scope':'263 program observations for260 DOIs, not263 unique papers or final work identities. Track/type labels are source scoped. Not complete public-negative coverage or a final whole-database audit.'}
|
||||||
|
part=out/'collection-report.json.part';part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n');part.replace(out/'collection-report.json');return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser=argparse.ArgumentParser(prog=prog,description=__doc__)
|
||||||
|
parser.add_argument('--snapshots',required=True);parser.add_argument('--out',required=True)
|
||||||
|
args=parser.parse_args(argv);print(json.dumps(collect(args.out,snapshots=args.snapshots),ensure_ascii=False))
|
||||||
@@ -0,0 +1,216 @@
|
|||||||
|
"""Source-attributed mirror metadata; no paper/review text or identity promotion."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import hashlib
|
||||||
|
from importlib.resources import files
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
from paper_atlas.io.parquet_metadata import RangeSource, projection
|
||||||
|
|
||||||
|
DATASET = 'anonymousNeurIPS2026submission4281/reviewarena'
|
||||||
|
COMMIT = 'c2978add17c2099219eaddbc2599974d69d4d09b'
|
||||||
|
TREE_SHA256 = 'bbc75f5774b019c4a9ee3efe5c0f37b14761bc3c5a054570db1df8f5603eac20'
|
||||||
|
SOURCE = 'reviewarena_mirror'
|
||||||
|
SPLITS = ('colm', 'corl', 'emnlp', 'iclr', 'icml', 'neurips', 'tmlr')
|
||||||
|
COLUMNS = {'forum_id': 'string', 'conference': 'string', 'year': 'integer',
|
||||||
|
'track': 'string', 'venue_id': 'string', 'paper_number': 'integer',
|
||||||
|
'title': 'string', 'abstract': 'string', 'authors': 'strings',
|
||||||
|
'keywords': 'strings', 'venue': 'string', 'decision': 'string'}
|
||||||
|
|
||||||
|
|
||||||
|
def inventory(split):
|
||||||
|
if split not in SPLITS:
|
||||||
|
raise ValueError('Explicit supported mirror split required')
|
||||||
|
raw = files('paper_atlas.resources').joinpath('reviewarena-files.json').read_bytes()
|
||||||
|
if hashlib.sha256(raw).hexdigest() != TREE_SHA256:
|
||||||
|
raise ValueError('Pinned mirror file inventory changed')
|
||||||
|
items = json.loads(raw)
|
||||||
|
if len(items) != 15:
|
||||||
|
raise ValueError('Pinned mirror shard inventory changed')
|
||||||
|
selected = []
|
||||||
|
for item in items:
|
||||||
|
path = item['path']
|
||||||
|
if not re.fullmatch(r'data/(colm|corl|emnlp|iclr|icml|neurips|tmlr)-\d{5}-of-\d{5}\.parquet', path):
|
||||||
|
raise ValueError('Unexpected mirror shard path')
|
||||||
|
if (item['type'] != 'file' or type(item['size']) is not int or item['size'] < 12
|
||||||
|
or item['lfs']['size'] != item['size'] or not re.fullmatch('[0-9a-f]{64}', item['lfs']['oid'])):
|
||||||
|
raise ValueError('Missing source object size/checksum claim')
|
||||||
|
if path.startswith('data/'+split+'-'):
|
||||||
|
selected.append(item)
|
||||||
|
parts = sorted(selected, key=lambda x: x['path'])
|
||||||
|
if not parts or any(int(x['path'][-13:-8]) != len(parts) for x in parts):
|
||||||
|
raise ValueError('Incomplete selected mirror split inventory')
|
||||||
|
if [int(x['path'].split('-')[-3]) for x in parts] != list(range(len(parts))):
|
||||||
|
raise ValueError('Mirror shard sequence changed')
|
||||||
|
return parts
|
||||||
|
|
||||||
|
|
||||||
|
def decision_claim(raw):
|
||||||
|
value = raw.strip().casefold() if isinstance(raw, str) else ''
|
||||||
|
if value in {'reject', 'rejected'}:
|
||||||
|
return 'rejected'
|
||||||
|
if value in {'desk reject', 'desk rejected', 'desk_rejected'}:
|
||||||
|
return 'desk_rejected'
|
||||||
|
if value in {'withdraw', 'withdrawn'}:
|
||||||
|
return 'withdrawn'
|
||||||
|
if value in {'accept', 'accepted', 'accept-main', 'accept-findings', 'accept (main)', 'accept (findings)',
|
||||||
|
'accept (poster)', 'accept (oral)', 'accept (spotlight)', 'accept (talk)'}:
|
||||||
|
return 'accepted'
|
||||||
|
return 'unknown'
|
||||||
|
|
||||||
|
|
||||||
|
def decision_track_claim(raw):
|
||||||
|
value = raw.strip().casefold() if isinstance(raw, str) else ''
|
||||||
|
if value in {'accept-findings', 'accept (findings)'}:
|
||||||
|
return 'findings'
|
||||||
|
if value in {'accept-main', 'accept (main)'}:
|
||||||
|
return 'main'
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def record(row, split, item, row_index):
|
||||||
|
if set(row) != set(COLUMNS):
|
||||||
|
raise ValueError('Unexpected projected metadata fields')
|
||||||
|
if (not isinstance(row['conference'], str) or row['conference'].casefold() != split
|
||||||
|
or type(row['year']) is not int or not 2020 <= row['year'] <= 2026):
|
||||||
|
raise ValueError('Mirror conference/year outside pinned source scope')
|
||||||
|
locator = {'dataset': DATASET, 'commit': COMMIT, 'file': item['path'], 'row_index': row_index,
|
||||||
|
'lfs_sha256_claim_not_full_file_verification': item['lfs']['oid']}
|
||||||
|
return {'source': SOURCE, 'external_id': COMMIT+':'+item['path']+':'+str(row_index),
|
||||||
|
'url': 'https://huggingface.co/datasets/'+DATASET+'/blob/'+COMMIT+'/'+item['path'],
|
||||||
|
'title': row['title'], 'abstract': row['abstract'], 'authors': row['authors'],
|
||||||
|
'conference': row['conference'], 'year': row['year'], 'track': row['track'],
|
||||||
|
'mirror_decision_category': decision_claim(row['decision']),
|
||||||
|
'mirror_decision_track_claim': decision_track_claim(row['decision']), 'raw_fields': row,
|
||||||
|
'source_locator': locator, 'identifiers': {},
|
||||||
|
'metadata': {'forum_id_claim': row['forum_id'], 'original_forum_id_verified': False,
|
||||||
|
'original_decision_signature_verified': False, 'source_visibility': 'public_mirror',
|
||||||
|
'scientific_work_identity_approved': False, 'research_type_classified': False,
|
||||||
|
'approved_for_import': False,
|
||||||
|
'scope': 'Mirror row only. No original forum/person/decision verification; absence is not rejection or proof of no later acceptance.'}}
|
||||||
|
|
||||||
|
|
||||||
|
def cached_projections(path, expected_sha256, split, items):
|
||||||
|
path = Path(path)
|
||||||
|
raw = path.read_bytes()
|
||||||
|
if (not isinstance(expected_sha256, str) or not re.fullmatch('[0-9a-f]{64}', expected_sha256)
|
||||||
|
or hashlib.sha256(raw).hexdigest() != expected_sha256):
|
||||||
|
raise ValueError('Cached projection report hash differs')
|
||||||
|
report = json.loads(raw)
|
||||||
|
if (report.get('dataset') != DATASET or report.get('commit') != COMMIT
|
||||||
|
or report.get('split') != split or report.get('inventory_sha256') != TREE_SHA256
|
||||||
|
or report.get('raw_parquet_saved') is not False or report.get('database_written') is not False
|
||||||
|
or report.get('paper_body_saved') is not False or report.get('review_text_saved') is not False
|
||||||
|
or report.get('approved_for_import') != 0):
|
||||||
|
raise ValueError('Cached report scope differs from permitted mirror projection')
|
||||||
|
declared = report.get('files', [])
|
||||||
|
if len(declared) != len(items) or len({f['source_file'] for f in declared}) != len(items):
|
||||||
|
raise ValueError('Incomplete cached mirror shard set')
|
||||||
|
result = {}
|
||||||
|
for item in items:
|
||||||
|
fm = next((f for f in declared if f['source_file'] == item['path']), None)
|
||||||
|
filename = Path(item['path']).stem+'.metadata.jsonl'
|
||||||
|
if not fm or Path(fm['file']).name != filename or fm['source_lfs_sha256_claim'] != item['lfs']['oid']:
|
||||||
|
raise ValueError('Cached mirror source differs')
|
||||||
|
file = path.parent/filename
|
||||||
|
if file.is_symlink():
|
||||||
|
raise ValueError('Symlink metadata cache forbidden')
|
||||||
|
content = file.read_bytes()
|
||||||
|
if hashlib.sha256(content).hexdigest() != fm['file_sha256']:
|
||||||
|
raise ValueError('Cached metadata file hash differs')
|
||||||
|
rows = []
|
||||||
|
for index, line in enumerate(content.splitlines()):
|
||||||
|
previous = json.loads(line); row = previous['raw_fields']; expected = record(row, split, item, index)
|
||||||
|
for key in ('source', 'external_id', 'url', 'title', 'abstract', 'authors', 'conference', 'year', 'track', 'raw_fields', 'source_locator', 'identifiers', 'metadata'):
|
||||||
|
if previous.get(key) != expected[key]:
|
||||||
|
raise ValueError('Cached raw metadata/identity differs: '+key)
|
||||||
|
rows.append(row)
|
||||||
|
proof = fm['projection']
|
||||||
|
if (len(rows) != fm['records'] or len(rows) != proof['rows'] or proof['columns'] != COLUMNS
|
||||||
|
or proof['source_bytes'] != item['size'] or proof['raw_parquet_saved'] is not False
|
||||||
|
or proof['footer_saved'] is not False or proof['excluded_payload_pages_downloaded'] is not False
|
||||||
|
or proof['full_file_checksum_verified'] is not False):
|
||||||
|
raise ValueError('Cached projection proof differs')
|
||||||
|
result[item['path']] = (rows, proof)
|
||||||
|
if sum(len(rows) for rows, _ in result.values()) != report['records']:
|
||||||
|
raise ValueError('Cached projection row count differs')
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def collect(split, out, *, source_factory=RangeSource, progress=None, from_projection=None, projection_sha256=None):
|
||||||
|
items = inventory(split); out = Path(out)
|
||||||
|
if bool(from_projection) != bool(projection_sha256):
|
||||||
|
raise ValueError('Cached projection and exact report hash required together')
|
||||||
|
if out.exists() or not out.parent.is_dir() or out.is_symlink():
|
||||||
|
raise ValueError('New output directory in existing parent required')
|
||||||
|
out.mkdir()
|
||||||
|
counts = Counter(); track_claims = Counter(); raw_counts = Counter(); years = Counter(); records = 0; reports = []
|
||||||
|
observed = datetime.now(timezone.utc).isoformat()
|
||||||
|
try:
|
||||||
|
cached = cached_projections(from_projection, projection_sha256, split, items) if from_projection else None
|
||||||
|
for item in items:
|
||||||
|
if cached is None:
|
||||||
|
source_url = 'https://huggingface.co/datasets/'+DATASET+'/resolve/'+COMMIT+'/'+item['path']
|
||||||
|
source = source_factory(source_url, item['size'])
|
||||||
|
rows, proof = projection(source, COLUMNS, progress=progress)
|
||||||
|
else:
|
||||||
|
rows, proof = cached[item['path']]
|
||||||
|
if progress:
|
||||||
|
progress({'phase': 'cached_metadata_projection', 'file': item['path'], 'rows': len(rows)})
|
||||||
|
path = out/(Path(item['path']).stem+'.metadata.jsonl')
|
||||||
|
part = path.with_suffix(path.suffix+'.part'); h = hashlib.sha256()
|
||||||
|
with part.open('wb') as stream:
|
||||||
|
for index, row in enumerate(rows):
|
||||||
|
item_record = record(row, split, item, index)
|
||||||
|
raw = (json.dumps(item_record, ensure_ascii=False)+'\n').encode()
|
||||||
|
stream.write(raw); h.update(raw); records += 1
|
||||||
|
counts[item_record['mirror_decision_category']] += 1
|
||||||
|
track_claims[item_record['mirror_decision_track_claim'] or 'unspecified'] += 1
|
||||||
|
raw_counts[json.dumps(row['decision'], ensure_ascii=False)] += 1
|
||||||
|
years[(row['year'], row['track'])] += 1
|
||||||
|
part.rename(path)
|
||||||
|
reports.append({'source_file': item['path'], 'file': str(path), 'records': len(rows),
|
||||||
|
'file_sha256': h.hexdigest(), 'source_lfs_sha256_claim': item['lfs']['oid'],
|
||||||
|
'projection': proof})
|
||||||
|
report = {'status': 'collected_mirror_metadata_pending_original_provenance_and_work_identity',
|
||||||
|
'dataset': DATASET, 'commit': COMMIT, 'inventory_sha256': TREE_SHA256,
|
||||||
|
'split': split, 'observed_at': observed, 'records': records, 'files': reports,
|
||||||
|
'mirror_decision_counts': dict(counts), 'raw_mirror_decision_counts': dict(raw_counts),
|
||||||
|
'mirror_decision_track_claim_counts': dict(track_claims),
|
||||||
|
'year_track_counts': [{'year': y, 'track': t, 'rows': n} for (y, t), n in sorted(years.items(), key=lambda x: (x[0][0], str(x[0][1])))],
|
||||||
|
'metadata_bytes_read': sum(x['projection']['metadata_bytes_read'] for x in reports) if cached is None else 0,
|
||||||
|
'source_projection_bytes': sum(x['projection']['metadata_bytes_read'] for x in reports),
|
||||||
|
'cached_projection_report_sha256': projection_sha256,
|
||||||
|
'excluded_payload_pages_downloaded': False, 'raw_parquet_saved': False,
|
||||||
|
'paper_body_saved': False, 'review_text_saved': False, 'database_written': False,
|
||||||
|
'approved_for_import': 0,
|
||||||
|
'limitations': ['Public mirror copies do not verify original forum IDs, visibility or signed decisions.',
|
||||||
|
'Original source LFS SHA256 is a pinned claim, not a computed full-Parquet checksum.',
|
||||||
|
'Every selected row and row locator is retained, including missing or anonymous author data.',
|
||||||
|
'Decision categories describe only explicit mirror literals; venue labels and missing decisions do not imply rejection.',
|
||||||
|
'Rows are not independent scientific works or a complete venue/year public negative census.',
|
||||||
|
'Cross-source identity, chronological rejection/acceptance history and final dedup remain.']}
|
||||||
|
(out/'collection-report.json').write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n')
|
||||||
|
return report
|
||||||
|
except BaseException as error:
|
||||||
|
# No source-specific body or signed redirect URLs in failure reports.
|
||||||
|
failure = {'status': 'incomplete_do_not_import', 'split': split, 'commit': COMMIT,
|
||||||
|
'completed_files': reports, 'error_type': type(error).__name__,
|
||||||
|
'database_written': False, 'approved_for_import': 0, 'no_automatic_retry': True}
|
||||||
|
(out/'failure.json').write_text(json.dumps(failure, ensure_ascii=False, indent=2)+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog)
|
||||||
|
parser.add_argument('--split', required=True, choices=SPLITS)
|
||||||
|
parser.add_argument('--out', required=True)
|
||||||
|
parser.add_argument('--from-projection', help='Offline replay of a complete permitted metadata projection')
|
||||||
|
parser.add_argument('--projection-sha256', help='Exact cached collection-report.json SHA256')
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
report = collect(args.split, args.out, from_projection=args.from_projection, projection_sha256=args.projection_sha256,
|
||||||
|
progress=lambda row: print(json.dumps(row), flush=True))
|
||||||
|
print(json.dumps({'status': report['status'], 'records': report['records'], 'out': args.out}), flush=True)
|
||||||
@@ -1,16 +1,14 @@
|
|||||||
"""Reviewed postconference selected volumes from frozen public HTML metadata."""
|
"""Reviewed postconference selected volumes from frozen public HTML metadata."""
|
||||||
import argparse
|
import argparse
|
||||||
from collections import defaultdict
|
|
||||||
import fcntl
|
import fcntl
|
||||||
import hashlib
|
import hashlib
|
||||||
import json
|
import json
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
import re
|
import re
|
||||||
import tempfile
|
import tempfile
|
||||||
from urllib.parse import parse_qs, urlsplit
|
|
||||||
|
|
||||||
from paper_atlas.db.submissions import timestamp
|
|
||||||
from paper_atlas.io.html import Document, clean
|
from paper_atlas.io.html import Document, clean
|
||||||
|
from paper_atlas.io.springer import parse_chapter as parse_citation_chapter, read_page, snapshot_index
|
||||||
|
|
||||||
SOURCE = 'springer_selected_volume'
|
SOURCE = 'springer_selected_volume'
|
||||||
BOOK = '10.1007/978-1-4612-2660-4'
|
BOOK = '10.1007/978-1-4612-2660-4'
|
||||||
@@ -26,50 +24,6 @@ SCOPE = {
|
|||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
def snapshot_index(root):
|
|
||||||
root = Path(root).resolve()
|
|
||||||
receipts = json.loads((root/'receipts.json').read_bytes())['responses']
|
|
||||||
index = {}
|
|
||||||
for receipt in receipts:
|
|
||||||
url = receipt['url']
|
|
||||||
if url in index:
|
|
||||||
raise ValueError('Duplicate snapshot URL')
|
|
||||||
index[url] = (root, receipt)
|
|
||||||
return index
|
|
||||||
|
|
||||||
|
|
||||||
def read_page(index, url):
|
|
||||||
if url not in index:
|
|
||||||
raise ValueError('Missing required public HTML snapshot: '+url)
|
|
||||||
root, receipt = index[url]
|
|
||||||
filename = Path(receipt['file'])
|
|
||||||
path = root/filename
|
|
||||||
content_type = {k.lower():v for k,v in receipt.get('headers', {}).items()}.get('content-type', '')
|
|
||||||
requested, final = urlsplit(url), urlsplit(receipt.get('final_url', ''))
|
|
||||||
query = parse_qs(final.query, keep_blank_values=True)
|
|
||||||
original_query = parse_qs(requested.query, keep_blank_values=True)
|
|
||||||
# Springer returns the public metadata with an explicit cookie diagnostic.
|
|
||||||
# Retain that final URL verbatim; accept no other redirect or changed page.
|
|
||||||
if query != original_query and requested.netloc == 'link.springer.com':
|
|
||||||
if (query.pop('error', None) != ['cookies_not_supported']
|
|
||||||
or not re.fullmatch(r'[0-9a-f-]{36}', ''.join(query.pop('code', [])))):
|
|
||||||
raise ValueError('Unreviewed Springer redirect query')
|
|
||||||
if (filename.is_absolute() or '..' in filename.parts or not path.resolve().is_relative_to(root)
|
|
||||||
or receipt.get('http_status') != 200
|
|
||||||
or (final.scheme, final.netloc, final.path) != (requested.scheme, requested.netloc, requested.path)
|
|
||||||
or final.fragment or query != original_query
|
|
||||||
or 'text/html' not in content_type.lower()):
|
|
||||||
raise ValueError('Unsafe, redirected or unsuccessful public HTML snapshot')
|
|
||||||
raw = path.read_bytes()
|
|
||||||
digest = hashlib.sha256(raw).hexdigest()
|
|
||||||
if len(raw) > 2_000_000 or len(raw) != receipt['bytes'] or digest != receipt['sha256']:
|
|
||||||
raise ValueError('Snapshot checksum or size differs')
|
|
||||||
evidence = {'url':url, 'sha256':digest, 'bytes':len(raw),
|
|
||||||
'file':'evidence/'+digest+'.html', 'http_status':200,
|
|
||||||
'observed_at':timestamp(receipt['at']), 'response_receipt':receipt}
|
|
||||||
return raw, evidence
|
|
||||||
|
|
||||||
|
|
||||||
def parse_toc(html, *, excluded_materials=None):
|
def parse_toc(html, *, excluded_materials=None):
|
||||||
entries = {}
|
entries = {}
|
||||||
for node in Document(html).root.walk():
|
for node in Document(html).root.walk():
|
||||||
@@ -97,40 +51,8 @@ def parse_toc(html, *, excluded_materials=None):
|
|||||||
|
|
||||||
|
|
||||||
def parse_chapter(html, doi):
|
def parse_chapter(html, doi):
|
||||||
nodes = list(Document(html).root.walk())
|
return parse_citation_chapter(html, doi, book_title=SCOPE['book_title'],
|
||||||
meta = defaultdict(list)
|
publication_year=SCOPE['publication_year'])
|
||||||
for node in nodes:
|
|
||||||
if node.tag == 'meta' and node.attrs.get('name', '').startswith('citation_'):
|
|
||||||
meta[node.attrs['name']].append(node.attrs.get('content', ''))
|
|
||||||
def one(key):
|
|
||||||
values = meta[key]
|
|
||||||
if len(values) != 1 or not values[0]:
|
|
||||||
raise ValueError('Missing or ambiguous citation field: '+key)
|
|
||||||
return values[0]
|
|
||||||
if (one('citation_doi') != doi or one('citation_inbook_title') != SCOPE['book_title']
|
|
||||||
or one('citation_publication_date') != str(SCOPE['publication_year'])):
|
|
||||||
raise ValueError('Chapter DOI, book or publication year differs from reviewed scope')
|
|
||||||
raw_authors = meta['citation_author']
|
|
||||||
if not raw_authors:
|
|
||||||
raise ValueError('Missing named chapter authors')
|
|
||||||
authors = []
|
|
||||||
for raw in raw_authors:
|
|
||||||
if raw.count(',') != 1:
|
|
||||||
raise ValueError('Unreviewed citation author format')
|
|
||||||
family, given = [clean(s) for s in raw.split(',', 1)]
|
|
||||||
if not family or not given:
|
|
||||||
raise ValueError('Incomplete citation author')
|
|
||||||
authors.append({'given':given, 'family':family})
|
|
||||||
abstract_nodes = [n for n in nodes if n.attrs.get('id') == 'Abs1-content']
|
|
||||||
if len(abstract_nodes) > 1:
|
|
||||||
raise ValueError('Ambiguous abstract section')
|
|
||||||
abstract = clean(abstract_nodes[0].text()) if abstract_nodes else None
|
|
||||||
first, last = one('citation_firstpage'), one('citation_lastpage')
|
|
||||||
if not first.isdigit() or not last.isdigit() or int(first) > int(last):
|
|
||||||
raise ValueError('Invalid chapter pages')
|
|
||||||
return {'doi':doi, 'title':one('citation_title'), 'authors':authors,
|
|
||||||
'authors_citation_raw':list(raw_authors), 'abstract':abstract or None,
|
|
||||||
'first_page':first, 'last_page':last}
|
|
||||||
|
|
||||||
|
|
||||||
def record_for(chapter, evidence, scope_evidence):
|
def record_for(chapter, evidence, scope_evidence):
|
||||||
|
|||||||
@@ -0,0 +1,128 @@
|
|||||||
|
"""Versioned publication-source scopes and guards against stale source refreshes."""
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
|
||||||
|
TABLE = 'source_scope_revisions'
|
||||||
|
MARKER = 'source_scope_revisions_v1'
|
||||||
|
COLUMNS = ('version_sha256', 'source_id', 'venue_id', 'year', 'event_key', 'list_url',
|
||||||
|
'parent_sha256', 'records_json', 'plan_json', 'applied_at')
|
||||||
|
|
||||||
|
|
||||||
|
def available(c):
|
||||||
|
if getattr(c, 'postgres', False):
|
||||||
|
return bool(c.execute("SELECT 1 FROM information_schema.tables WHERE table_schema=current_schema() AND table_name=?", (TABLE,)).fetchone())
|
||||||
|
return bool(c.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,)).fetchone())
|
||||||
|
|
||||||
|
|
||||||
|
def initialize(c):
|
||||||
|
exists = available(c)
|
||||||
|
marker = bool(c.execute('SELECT 1 FROM schema_migrations WHERE name=?', (MARKER,)).fetchone())
|
||||||
|
if exists != marker:
|
||||||
|
raise ValueError('Source revision table/marker mismatch')
|
||||||
|
if exists:
|
||||||
|
result = c.execute('SELECT * FROM source_scope_revisions LIMIT 0')
|
||||||
|
description = result.cursor.description if getattr(c, 'postgres', False) else result.description
|
||||||
|
if tuple(x[0] for x in description) != COLUMNS:
|
||||||
|
raise ValueError('Unexpected source revision schema')
|
||||||
|
return
|
||||||
|
ddl = '''CREATE TABLE source_scope_revisions (
|
||||||
|
version_sha256 TEXT PRIMARY KEY,
|
||||||
|
source_id INTEGER NOT NULL REFERENCES sources(id),
|
||||||
|
venue_id INTEGER NOT NULL REFERENCES venues(id),
|
||||||
|
year INTEGER NOT NULL, event_key TEXT NOT NULL, list_url TEXT NOT NULL,
|
||||||
|
parent_sha256 TEXT UNIQUE REFERENCES source_scope_revisions(version_sha256),
|
||||||
|
records_json TEXT NOT NULL, plan_json TEXT NOT NULL,
|
||||||
|
applied_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP)'''
|
||||||
|
if getattr(c, 'postgres', False):
|
||||||
|
ddl = re.sub(r'\bINTEGER\b', 'BIGINT', ddl)
|
||||||
|
ddl = re.sub(r'\bTEXT\b', 'TEXT COLLATE "C"', ddl)
|
||||||
|
c.execute(ddl)
|
||||||
|
c.execute('CREATE INDEX source_scope_revisions_scope ON source_scope_revisions(source_id,venue_id,year,event_key,list_url)')
|
||||||
|
c.execute('INSERT INTO schema_migrations(name) VALUES(?)', (MARKER,))
|
||||||
|
|
||||||
|
|
||||||
|
def heads(c, source_id):
|
||||||
|
if not available(c):
|
||||||
|
return []
|
||||||
|
if not c.execute('SELECT 1 FROM schema_migrations WHERE name=?', (MARKER,)).fetchone():
|
||||||
|
raise ValueError('Source revision table/marker mismatch')
|
||||||
|
return c.execute('''SELECT v.version_sha256,v.venue_id,v.year,v.event_key,v.list_url
|
||||||
|
FROM source_scope_revisions v WHERE v.source_id=? AND NOT EXISTS
|
||||||
|
(SELECT 1 FROM source_scope_revisions child WHERE child.parent_sha256=v.version_sha256)
|
||||||
|
ORDER BY v.version_sha256''', (source_id,)).fetchall()
|
||||||
|
|
||||||
|
|
||||||
|
def current_rows(c, source_id, cache):
|
||||||
|
active = heads(c, source_id)
|
||||||
|
signatures = tuple(r[0] for r in active)
|
||||||
|
# A new head invalidates the cache, even when maintenance uses the same Store.
|
||||||
|
if cache.get('signatures') != (source_id, signatures):
|
||||||
|
records = {}; scopes = {}
|
||||||
|
for signature in signatures:
|
||||||
|
raw = c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone()[0]
|
||||||
|
scopes[signature] = json.loads(raw)
|
||||||
|
for row in scopes[signature]:
|
||||||
|
ext = row['external_id']
|
||||||
|
if ext in records and records[ext] != row:
|
||||||
|
raise ValueError('Conflicting active source snapshots')
|
||||||
|
records[ext] = row
|
||||||
|
cache.clear()
|
||||||
|
cache.update(signatures=(source_id, signatures), records=records, scopes=scopes)
|
||||||
|
return cache['records']
|
||||||
|
|
||||||
|
|
||||||
|
def subset(expected, actual):
|
||||||
|
"""Reviewed source fields must survive additional work-identity/curation evidence."""
|
||||||
|
return isinstance(actual, dict) and all(k in actual and actual[k] == v for k, v in expected.items())
|
||||||
|
|
||||||
|
|
||||||
|
def require_record(c, source_id, external, url, title, abstract, authors, identifiers, metadata, cache):
|
||||||
|
expected = current_rows(c, source_id, cache).get(external)
|
||||||
|
if expected is None:
|
||||||
|
return
|
||||||
|
actual = {'url': url, 'title': title, 'abstract': abstract, 'authors': authors}
|
||||||
|
if any(actual[k] != expected.get(k) for k in actual):
|
||||||
|
raise ValueError('Stale source snapshot: reconcile the source version before refresh')
|
||||||
|
if not subset(expected.get('metadata', {}), metadata or {}) or not subset(expected.get('identifiers', {}), identifiers or {}):
|
||||||
|
raise ValueError('Stale source snapshot metadata or identifiers')
|
||||||
|
|
||||||
|
|
||||||
|
def scope_head(c, source_id, manifest):
|
||||||
|
venue = c.execute('SELECT id FROM venues WHERE acronym=?', (manifest.get('venue'),)).fetchone()
|
||||||
|
if not venue:
|
||||||
|
return None
|
||||||
|
matches = [r for r in heads(c, source_id)
|
||||||
|
if [r[i] for i in range(1, 5)] == [venue[0], manifest.get('year'), manifest.get('event_key', ''), manifest.get('list_url')]]
|
||||||
|
if len(matches) > 1:
|
||||||
|
raise ValueError('Forked source revision history')
|
||||||
|
return matches[0][0] if matches else None
|
||||||
|
|
||||||
|
|
||||||
|
def require_manifest(c, source_id, manifest, cache):
|
||||||
|
current_rows(c, source_id, cache)
|
||||||
|
signature = scope_head(c, source_id, manifest)
|
||||||
|
if signature is None:
|
||||||
|
return
|
||||||
|
expected = sum(r['track'] == manifest.get('track') for r in cache['scopes'][signature])
|
||||||
|
bound = manifest.get('coverage_expected_count') if manifest.get('coverage_mode') == 'incremental' else manifest.get('count')
|
||||||
|
if expected < 1 or bound != expected or manifest.get('status') != 'partial':
|
||||||
|
raise ValueError('Source coverage manifest differs from current source snapshot')
|
||||||
|
|
||||||
|
|
||||||
|
def require_row(c, source_id, row, manifest, cache):
|
||||||
|
expected = current_rows(c, source_id, cache).get(row['external_id'])
|
||||||
|
scoped_manifest = {**manifest, 'venue': row['venue'], 'year': row['year'], 'event_key': row.get('event_key', '')}
|
||||||
|
signature = scope_head(c, source_id, scoped_manifest)
|
||||||
|
if expected is None:
|
||||||
|
if signature is not None:
|
||||||
|
raise ValueError('Publication absent from current source snapshot; reconcile source membership first')
|
||||||
|
return
|
||||||
|
if signature is None or not any(r['external_id'] == row['external_id'] for r in cache['scopes'][signature]):
|
||||||
|
raise ValueError('Source listing differs from current source snapshot')
|
||||||
|
for key, value in expected.items():
|
||||||
|
if key in ('metadata', 'identifiers'):
|
||||||
|
okay = subset(value, row.get(key))
|
||||||
|
else:
|
||||||
|
okay = row.get(key) == value
|
||||||
|
if not okay:
|
||||||
|
raise ValueError('Stale source snapshot field: ' + key)
|
||||||
@@ -96,6 +96,10 @@ class Store:
|
|||||||
if found:
|
if found:
|
||||||
matches.add(found[0])
|
matches.add(found[0])
|
||||||
old = c.execute('SELECT paper_id,title,abstract,retrieved_at,metadata_json FROM source_records WHERE source_id=? AND external_id=?',(source_id:=self.source(source,url),external_id)).fetchone()
|
old = c.execute('SELECT paper_id,title,abstract,retrieved_at,metadata_json FROM source_records WHERE source_id=? AND external_id=?',(source_id:=self.source(source,url),external_id)).fetchone()
|
||||||
|
from paper_atlas.db.source_revisions import require_record
|
||||||
|
if not hasattr(self, '_source_revision_cache'):
|
||||||
|
self._source_revision_cache = {}
|
||||||
|
require_record(c, source_id, external_id, url, title, abstract, authors, identifiers, metadata, self._source_revision_cache)
|
||||||
from paper_atlas.db.edition_scopes import preserve
|
from paper_atlas.db.edition_scopes import preserve
|
||||||
metadata = preserve(dict(metadata or {}), old[4] if old else None)
|
metadata = preserve(dict(metadata or {}), old[4] if old else None)
|
||||||
if old:
|
if old:
|
||||||
|
|||||||
@@ -66,7 +66,7 @@ def require_incremental_parent(c, source, manifest, options):
|
|||||||
if entries:
|
if entries:
|
||||||
if len(entries) != 1 or entries[0][0] != manifest['list_url']:
|
if len(entries) != 1 or entries[0][0] != manifest['list_url']:
|
||||||
raise ValueError('Incremental coverage requires one unambiguous source listing')
|
raise ValueError('Incremental coverage requires one unambiguous source listing')
|
||||||
if tuple(entries[0][1:]) != ('partial', prior_expected, previous):
|
if tuple(entries[0][i] for i in (1, 2, 3)) != ('partial', prior_expected, previous):
|
||||||
raise ValueError('Incremental coverage parent changed or is not partial')
|
raise ValueError('Incremental coverage parent changed or is not partial')
|
||||||
elif previous != 0 or prior_expected is not None:
|
elif previous != 0 or prior_expected is not None:
|
||||||
raise ValueError('Incremental coverage parent is absent')
|
raise ValueError('Incremental coverage parent is absent')
|
||||||
@@ -101,6 +101,9 @@ def ingest_file(db,path, *, pg_service=None, pg_schema=None):
|
|||||||
c.commit()
|
c.commit()
|
||||||
if incremental is not None:
|
if incremental is not None:
|
||||||
require_incremental_parent(c, source_id, manifest, incremental)
|
require_incremental_parent(c, source_id, manifest, incremental)
|
||||||
|
from paper_atlas.db.source_revisions import require_manifest
|
||||||
|
store._source_revision_cache = {}
|
||||||
|
require_manifest(c, source_id, manifest, store._source_revision_cache)
|
||||||
digest = hashlib.sha256()
|
digest = hashlib.sha256()
|
||||||
with path.open('rb') as stream:
|
with path.open('rb') as stream:
|
||||||
for line in stream:
|
for line in stream:
|
||||||
@@ -113,6 +116,10 @@ def ingest_file(db,path, *, pg_service=None, pg_schema=None):
|
|||||||
require_track_record(row, manifest)
|
require_track_record(row, manifest)
|
||||||
from paper_atlas.db.events import validate_event_key
|
from paper_atlas.db.events import validate_event_key
|
||||||
validate_event_key(row.get('event_key'))
|
validate_event_key(row.get('event_key'))
|
||||||
|
from paper_atlas.db.source_revisions import require_row
|
||||||
|
if not hasattr(store, '_source_revision_cache'):
|
||||||
|
store._source_revision_cache = {}
|
||||||
|
require_row(c, source_id, row, manifest, store._source_revision_cache)
|
||||||
external = row['external_id']
|
external = row['external_id']
|
||||||
if external in seen:
|
if external in seen:
|
||||||
raise ValueError('Duplicate external ID in one collected file: '+external)
|
raise ValueError('Duplicate external ID in one collected file: '+external)
|
||||||
|
|||||||
@@ -0,0 +1,327 @@
|
|||||||
|
"""Plan-bound publication scope revisions with complete before-images, never reimport."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from contextlib import closing
|
||||||
|
import copy
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
from paper_atlas.db import source_revisions as versions
|
||||||
|
from paper_atlas.db.backend import database_arguments, open_read_connection, open_store
|
||||||
|
from paper_atlas.db.edition_scopes import KEY
|
||||||
|
from paper_atlas.ingestion.edition_tracks import curation
|
||||||
|
from paper_atlas.ingestion.event_identity import database_identity, digest, values
|
||||||
|
|
||||||
|
MAX_ROWS = 10000
|
||||||
|
|
||||||
|
|
||||||
|
def validate(inventory):
|
||||||
|
required = {'format', 'source', 'venue', 'year', 'event_key', 'list_url',
|
||||||
|
'previous_rows', 'rows', 'evidence'}
|
||||||
|
if not isinstance(inventory, dict) or set(inventory) != required or inventory['format'] != 1:
|
||||||
|
raise ValueError('Unsupported source revision inventory')
|
||||||
|
if (type(inventory['year']) is not int or inventory['year'] < 2000
|
||||||
|
or not all(isinstance(inventory[k], str) and inventory[k] for k in ('source', 'venue', 'list_url'))
|
||||||
|
or not isinstance(inventory['event_key'], str)):
|
||||||
|
raise ValueError('Explicit publication source scope required')
|
||||||
|
if not inventory['list_url'].startswith('https://'):
|
||||||
|
raise ValueError('HTTPS source listing required')
|
||||||
|
evidence = inventory['evidence']
|
||||||
|
if not isinstance(evidence, list) or not 1 <= len(evidence) <= 100:
|
||||||
|
raise ValueError('Bounded source evidence required')
|
||||||
|
for item in evidence:
|
||||||
|
if (not isinstance(item, dict) or set(item) != {'url', 'sha256'}
|
||||||
|
or not isinstance(item['url'], str) or not item['url'].startswith('https://')
|
||||||
|
or not isinstance(item['sha256'], str) or not re.fullmatch('[0-9a-f]{64}', item['sha256'])):
|
||||||
|
raise ValueError('Source evidence URL/hash required')
|
||||||
|
snapshots = []
|
||||||
|
for key in ('previous_rows', 'rows'):
|
||||||
|
items = inventory[key]
|
||||||
|
if not isinstance(items, list) or not 1 <= len(items) <= MAX_ROWS:
|
||||||
|
raise ValueError('Complete bounded source snapshots required')
|
||||||
|
records = {}
|
||||||
|
for row in items:
|
||||||
|
if not isinstance(row, dict) or any(row.get(k) != inventory[k] for k in ('source', 'venue', 'year')):
|
||||||
|
raise ValueError('Source snapshot scope mismatch')
|
||||||
|
ext = row.get('external_id')
|
||||||
|
if (not isinstance(ext, str) or ext in records or row.get('status') != 'accepted'
|
||||||
|
or not re.fullmatch(r'10\.\d{4,9}/\S+', ext)
|
||||||
|
or row.get('identifiers', {}).get('doi') != ext
|
||||||
|
or not isinstance(row.get('track'), str) or not row['track']
|
||||||
|
or row.get('event_key', '') != inventory['event_key']
|
||||||
|
or not isinstance(row.get('title'), str) or not row['title']
|
||||||
|
or not isinstance(row.get('authors'), list)
|
||||||
|
or not isinstance(row.get('metadata'), dict)):
|
||||||
|
raise ValueError('Unique DOI-bearing accepted publication records required')
|
||||||
|
records[ext] = row
|
||||||
|
snapshots.append(records)
|
||||||
|
previous, current = snapshots
|
||||||
|
if set(previous) != set(current):
|
||||||
|
raise ValueError('Source membership changed; this operation cannot add or remove publications')
|
||||||
|
for ext, row in current.items():
|
||||||
|
before = previous[ext]
|
||||||
|
stable = (set(before) | set(row)) - {'abstract', 'track', 'tags', 'metadata'}
|
||||||
|
if any(before.get(k) != row.get(k) for k in stable):
|
||||||
|
raise ValueError('Stable publication identity or decision changed')
|
||||||
|
return previous, current
|
||||||
|
|
||||||
|
|
||||||
|
def bounded(c, sql, params):
|
||||||
|
result = values(c.execute(sql + ' LIMIT 100001', params))
|
||||||
|
if len(result) > 100000:
|
||||||
|
raise ValueError('Source revision protection exceeds bounded scope')
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def capture(c, inventory):
|
||||||
|
previous, current = validate(inventory)
|
||||||
|
source = c.execute('SELECT id FROM sources WHERE name=?', (inventory['source'],)).fetchone()
|
||||||
|
venue = c.execute('SELECT id FROM venues WHERE acronym=?', (inventory['venue'],)).fetchone()
|
||||||
|
if not source or not venue:
|
||||||
|
raise ValueError('Existing source and venue required')
|
||||||
|
sid, vid = source[0], venue[0]
|
||||||
|
editions = bounded(c, 'SELECT * FROM editions WHERE venue_id=? AND year=? AND event_key=? ORDER BY id',
|
||||||
|
(vid, inventory['year'], inventory['event_key']))
|
||||||
|
by_track = {r[3]: r[0] for r in editions}
|
||||||
|
# The reviewed scope includes all its existing target attempts, even when
|
||||||
|
# they originate from another source. No empty-target shortcut is used.
|
||||||
|
scope_subs = 'SELECT s.id FROM submissions s JOIN editions e ON e.id=s.edition_id WHERE e.venue_id=? AND e.year=? AND e.event_key=?'
|
||||||
|
scope_params = (vid, inventory['year'], inventory['event_key'])
|
||||||
|
scoped = bounded(c, 'SELECT * FROM submissions WHERE id IN (' + scope_subs + ') ORDER BY id', scope_params)
|
||||||
|
own_records = bounded(c, '''SELECT DISTINCT r.* FROM source_records r JOIN submission_sources ss ON ss.source_record_id=r.id
|
||||||
|
WHERE r.source_id=? AND ss.submission_id IN (''' + scope_subs + ') ORDER BY r.id', (sid, *scope_params))
|
||||||
|
if not own_records:
|
||||||
|
raise ValueError('Existing source observations required')
|
||||||
|
if any(r[2] not in current for r in own_records):
|
||||||
|
raise ValueError('Snapshot omits an existing source observation')
|
||||||
|
pids = sorted({r[1] for r in scoped})
|
||||||
|
if not pids or len(pids) > MAX_ROWS:
|
||||||
|
raise ValueError('Bounded nonempty protected paper scope required')
|
||||||
|
marks = ','.join('?' for _ in pids)
|
||||||
|
papers = 'SELECT id FROM papers WHERE id IN (' + marks + ')'
|
||||||
|
subs = 'SELECT id FROM submissions WHERE paper_id IN (' + papers + ')'
|
||||||
|
decisions = 'SELECT id FROM decisions WHERE submission_id IN (' + subs + ')'
|
||||||
|
queries = {
|
||||||
|
'papers': 'SELECT * FROM papers WHERE id IN (' + papers + ') ORDER BY id',
|
||||||
|
'identifiers': 'SELECT * FROM identifiers WHERE paper_id IN (' + papers + ') ORDER BY scheme,value',
|
||||||
|
'source_records': 'SELECT * FROM source_records WHERE paper_id IN (' + papers + ') ORDER BY id',
|
||||||
|
'submissions': 'SELECT * FROM submissions WHERE paper_id IN (' + papers + ') ORDER BY id',
|
||||||
|
'decisions': 'SELECT * FROM decisions WHERE submission_id IN (' + subs + ') ORDER BY id',
|
||||||
|
'submission_sources': 'SELECT * FROM submission_sources WHERE submission_id IN (' + subs + ') ORDER BY submission_id,source_record_id,source_submission_id',
|
||||||
|
'submission_identifiers': 'SELECT * FROM submission_identifiers WHERE submission_id IN (' + subs + ') ORDER BY edition_id,scheme,value',
|
||||||
|
'decision_sources': 'SELECT * FROM decision_sources WHERE decision_id IN (' + decisions + ') ORDER BY decision_id,source_record_id,source_decision_id',
|
||||||
|
'paper_tags': 'SELECT * FROM paper_tags WHERE paper_id IN (' + papers + ') ORDER BY paper_id,tag_id,evidence_key',
|
||||||
|
}
|
||||||
|
from paper_atlas.curation.annotations import available
|
||||||
|
if available(c):
|
||||||
|
queries['annotation_events'] = 'SELECT * FROM annotation_events WHERE paper_id IN (' + papers + ') ORDER BY id'
|
||||||
|
packet = {k: bounded(c, q, pids) for k, q in queries.items()}
|
||||||
|
coverage = bounded(c, '''SELECT es.* FROM edition_sources es JOIN editions e ON e.id=es.edition_id
|
||||||
|
WHERE es.source_id=? AND e.venue_id=? AND e.year=? AND e.event_key=? ORDER BY es.edition_id,es.list_url''', (sid, *scope_params))
|
||||||
|
if not coverage or any(r[2] != inventory['list_url'] for r in coverage):
|
||||||
|
raise ValueError('Unambiguous source listing required')
|
||||||
|
if len({r[0] for r in coverage}) != len(coverage):
|
||||||
|
raise ValueError('One source coverage entry per edition required')
|
||||||
|
heads = [[r[i] for i in range(len(r))] for r in versions.heads(c, sid)]
|
||||||
|
own_heads = [r for r in heads if r[1:] == [vid, inventory['year'], inventory['event_key'], inventory['list_url']]]
|
||||||
|
if len(own_heads) > 1:
|
||||||
|
raise ValueError('Forked source revision history')
|
||||||
|
parent = own_heads[0][0] if own_heads else None
|
||||||
|
return {'source_id': sid, 'venue_id': vid, 'editions': editions, 'coverage': coverage,
|
||||||
|
'packet': packet, 'own_records': own_records, 'revision_heads': heads,
|
||||||
|
'parent_sha256': parent, 'by_track': by_track,
|
||||||
|
'venue_names': sorted({inventory['venue']} | {r[0] for r in c.execute('SELECT alias FROM venue_aliases WHERE venue_id=?', (vid,))})}
|
||||||
|
|
||||||
|
|
||||||
|
def changes(c, inventory, before):
|
||||||
|
previous, current = validate(inventory)
|
||||||
|
packet = before['packet']; result = []; sid = before['source_id']
|
||||||
|
editions = {r[0]: r for r in before['editions']}
|
||||||
|
for raw in before['own_records']:
|
||||||
|
ext = raw[2]; old = previous[ext]; new = current[ext]
|
||||||
|
metadata = json.loads(raw[8])
|
||||||
|
candidates = [old, new]
|
||||||
|
if not any(raw[4] == r['url'] and raw[5] == r['title'] and raw[6] == r.get('abstract')
|
||||||
|
and json.loads(raw[7]) == r['authors'] and versions.subset(r['metadata'], metadata) for r in candidates):
|
||||||
|
raise ValueError('Current raw source differs from both reviewed snapshots')
|
||||||
|
if metadata.get('record_identifiers', {}).get('doi') != ext:
|
||||||
|
raise ValueError('Current stable DOI differs from source evidence')
|
||||||
|
attempts = [s for s in packet['submissions'] if s[3] == raw[0] and s[2] in editions]
|
||||||
|
if len(attempts) != 1:
|
||||||
|
raise ValueError('Exactly one publication attempt per source record required')
|
||||||
|
sub = attempts[0]; origin = editions[sub[2]]
|
||||||
|
if origin[3] not in {old['track'], new['track']} or sub[4] != ext or sub[1] != raw[3]:
|
||||||
|
raise ValueError('Current source attempt identity/scope differs')
|
||||||
|
if [x for x in packet['submission_sources'] if x[0] == sub[0]] != [[sub[0], raw[0], ext]]:
|
||||||
|
raise ValueError('Shared-source attempt requires separate reconciliation')
|
||||||
|
decisions = [d for d in packet['decisions'] if d[1] == sub[0]]
|
||||||
|
if not decisions or any(d[2] != 'accepted' for d in decisions):
|
||||||
|
raise ValueError('Only accepted publication observations can be revised')
|
||||||
|
target = before['by_track'].get(new['track'])
|
||||||
|
if target is None:
|
||||||
|
raise ValueError('Reviewed target edition must already exist')
|
||||||
|
identities = [x for x in packet['submission_identifiers'] if x[3] == sub[0]]
|
||||||
|
if not identities or any(x[0] != sub[2] for x in identities):
|
||||||
|
raise ValueError('Inconsistent scoped submission identifiers')
|
||||||
|
for _, scheme, value, _ in identities:
|
||||||
|
conflict = c.execute('SELECT submission_id FROM submission_identifiers WHERE edition_id=? AND scheme=? AND value=?', (target, scheme, value)).fetchone()
|
||||||
|
if conflict and conflict[0] != sub[0]:
|
||||||
|
raise ValueError('Target submission identifier collision')
|
||||||
|
replacement = copy.deepcopy(metadata); replacement.update(new['metadata'])
|
||||||
|
if sub[2] != target and KEY in metadata:
|
||||||
|
raise ValueError('An already curated attempt needs explicit scope-chain reconciliation')
|
||||||
|
if sub[2] != target or raw[6] != new.get('abstract') or replacement != metadata:
|
||||||
|
result.append({'source_record_id': raw[0], 'paper_id': raw[3], 'submission_id': sub[0],
|
||||||
|
'external_id': ext, 'from_edition_id': sub[2], 'target_edition_id': target,
|
||||||
|
'from_track': origin[3], 'track': new['track'],
|
||||||
|
'new_abstract': new.get('abstract'), 'metadata': replacement})
|
||||||
|
old_expected = Counter(r['track'] for r in previous.values()); new_expected = Counter(r['track'] for r in current.values())
|
||||||
|
observed = Counter()
|
||||||
|
own_ids = {r[0] for r in before['own_records']}
|
||||||
|
for s in packet['submissions']:
|
||||||
|
if s[3] in own_ids and s[2] in editions:
|
||||||
|
observed[s[2]] += 1
|
||||||
|
coverage = []
|
||||||
|
for raw in before['coverage']:
|
||||||
|
track = editions[raw[0]][3]
|
||||||
|
if raw[3] != 'partial' or raw[8] is not None or raw[5] != observed[raw[0]]:
|
||||||
|
raise ValueError('Coverage parent is not the verified partial source observation count')
|
||||||
|
if raw[4] not in {old_expected[track], new_expected[track]}:
|
||||||
|
raise ValueError('Coverage expected count differs from both source versions')
|
||||||
|
count = raw[5] + sum((r['target_edition_id'] == raw[0]) - (r['from_edition_id'] == raw[0]) for r in result)
|
||||||
|
if count < 0 or count > new_expected[track]:
|
||||||
|
raise ValueError('Corrected coverage exceeds current source membership')
|
||||||
|
coverage.append({'before': raw, 'expected_count': new_expected[track], 'imported_count': count})
|
||||||
|
covered = {r['before'][0] for r in coverage}
|
||||||
|
if any(r['target_edition_id'] not in covered for r in result):
|
||||||
|
raise ValueError('Target source coverage must already exist')
|
||||||
|
return result, coverage
|
||||||
|
|
||||||
|
|
||||||
|
def proposal(c, inventory, identity):
|
||||||
|
before = capture(c, inventory)
|
||||||
|
if before['parent_sha256']:
|
||||||
|
saved = json.loads(c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (before['parent_sha256'],)).fetchone()[0])
|
||||||
|
if sorted(saved, key=lambda r: r['external_id']) != sorted(inventory['previous_rows'], key=lambda r: r['external_id']):
|
||||||
|
raise ValueError('Previous snapshot differs from active source version')
|
||||||
|
records, coverage = changes(c, inventory, before)
|
||||||
|
return {'format': 1, 'database': identity, 'inventory': inventory, 'before': before,
|
||||||
|
'changes': records, 'coverage': coverage}
|
||||||
|
|
||||||
|
|
||||||
|
def verify(c, plan, signature):
|
||||||
|
after = capture(c, plan['inventory']); expected = copy.deepcopy(plan['before']['packet'])
|
||||||
|
for change in plan['changes']:
|
||||||
|
for row in expected['submissions']:
|
||||||
|
if row[0] == change['submission_id']:
|
||||||
|
row[2] = change['target_edition_id']
|
||||||
|
for row in expected['submission_identifiers']:
|
||||||
|
if row[3] == change['submission_id']:
|
||||||
|
row[0] = change['target_edition_id']
|
||||||
|
for row in expected['source_records']:
|
||||||
|
if row[0] == change['source_record_id']:
|
||||||
|
row[6] = change['new_abstract']
|
||||||
|
row[8] = encoded_metadata(plan, change, signature)
|
||||||
|
expected['submission_identifiers'].sort(key=lambda r: (r[0], r[1], r[2]))
|
||||||
|
if after['packet'] != expected:
|
||||||
|
raise ValueError('Protected profiles/history/tags/identifiers differ; roll back')
|
||||||
|
if after['editions'] != plan['before']['editions']:
|
||||||
|
raise ValueError('Edition identities changed')
|
||||||
|
expected_heads = [r for r in plan['before']['revision_heads'] if r[0] != plan['before']['parent_sha256']]
|
||||||
|
inventory = plan['inventory']
|
||||||
|
expected_heads.append([signature, plan['before']['venue_id'], inventory['year'], inventory['event_key'], inventory['list_url']])
|
||||||
|
if after['revision_heads'] != sorted(expected_heads):
|
||||||
|
raise ValueError('Active source version differs')
|
||||||
|
expected_coverage = []
|
||||||
|
for item in plan['coverage']:
|
||||||
|
row = item['before'][:]; row[4] = item['expected_count']; row[5] = item['imported_count']
|
||||||
|
expected_coverage.append(row)
|
||||||
|
if after['coverage'] != expected_coverage:
|
||||||
|
raise ValueError('Reconciled coverage differs; roll back')
|
||||||
|
stored = c.execute('SELECT records_json,plan_json,parent_sha256 FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone()
|
||||||
|
if (not stored or json.loads(stored[0]) != plan['inventory']['rows']
|
||||||
|
or json.loads(stored[1]) != plan or stored[2] != plan['before']['parent_sha256']):
|
||||||
|
raise ValueError('Versioned source or complete before-images changed')
|
||||||
|
return {'records_changed': len(plan['changes']), 'events_moved': sum(r['from_edition_id'] != r['target_edition_id'] for r in plan['changes']),
|
||||||
|
'protected_papers': len(expected['papers']), 'protected_decisions': len(expected['decisions'])}
|
||||||
|
|
||||||
|
|
||||||
|
def encoded_metadata(plan, change, signature):
|
||||||
|
metadata = copy.deepcopy(change['metadata'])
|
||||||
|
if change['from_edition_id'] != change['target_edition_id']:
|
||||||
|
inventory = plan['inventory']
|
||||||
|
evidence = inventory['evidence'][0]
|
||||||
|
row = {**change, 'source': inventory['source'], 'doi': change['external_id'],
|
||||||
|
'venue_id': plan['before']['venue_id'], 'year': inventory['year'],
|
||||||
|
'event_key': inventory['event_key'], 'venue_names': plan['before']['venue_names'],
|
||||||
|
'section': change['track'], 'evidence_url': evidence['url'], 'evidence_sha256': evidence['sha256']}
|
||||||
|
metadata[KEY] = [curation(row, change['target_edition_id'], signature)]
|
||||||
|
return json.dumps(metadata, ensure_ascii=False)
|
||||||
|
|
||||||
|
|
||||||
|
def apply_plan(c, plan):
|
||||||
|
if plan.get('format') != 1:
|
||||||
|
raise ValueError('Unsupported source revision plan')
|
||||||
|
if not getattr(c, 'postgres', False) and not c.in_transaction:
|
||||||
|
c.execute('BEGIN IMMEDIATE')
|
||||||
|
signature = digest(plan)
|
||||||
|
if versions.available(c) and c.execute('SELECT 1 FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone():
|
||||||
|
return {'status': 'already_applied', 'version_sha256': signature, **verify(c, plan, signature)}
|
||||||
|
if getattr(c, 'postgres', False):
|
||||||
|
c.execute('LOCK TABLE sources,papers,identifiers,source_records,venues,venue_aliases,editions,edition_sources,submissions,decisions,submission_sources,submission_identifiers,decision_sources,tags,paper_tags IN SHARE ROW EXCLUSIVE MODE')
|
||||||
|
if proposal(c, plan['inventory'], plan['database']) != plan:
|
||||||
|
raise ValueError('Stale source revision plan; create and review a fresh plan')
|
||||||
|
versions.initialize(c)
|
||||||
|
for row in plan['changes']:
|
||||||
|
if row['from_edition_id'] != row['target_edition_id']:
|
||||||
|
c.execute('UPDATE submission_identifiers SET edition_id=? WHERE submission_id=?', (row['target_edition_id'], row['submission_id']))
|
||||||
|
c.execute('UPDATE submissions SET edition_id=? WHERE id=? AND edition_id=?', (row['target_edition_id'], row['submission_id'], row['from_edition_id']))
|
||||||
|
c.execute('UPDATE source_records SET abstract=?,metadata_json=? WHERE id=?',
|
||||||
|
(row['new_abstract'], encoded_metadata(plan, row, signature), row['source_record_id']))
|
||||||
|
for row in plan['coverage']:
|
||||||
|
before = row['before']
|
||||||
|
c.execute('UPDATE edition_sources SET expected_count=?,imported_count=? WHERE edition_id=? AND source_id=? AND list_url=?',
|
||||||
|
(row['expected_count'], row['imported_count'], before[0], before[1], before[2]))
|
||||||
|
inventory = plan['inventory']
|
||||||
|
c.execute('''INSERT INTO source_scope_revisions(version_sha256,source_id,venue_id,year,event_key,list_url,parent_sha256,records_json,plan_json)
|
||||||
|
VALUES(?,?,?,?,?,?,?,?,?)''', (signature, plan['before']['source_id'], plan['before']['venue_id'], inventory['year'], inventory['event_key'],
|
||||||
|
inventory['list_url'], plan['before']['parent_sha256'], json.dumps(inventory['rows'], ensure_ascii=False), json.dumps(plan, ensure_ascii=False)))
|
||||||
|
return {'status': 'applied', 'version_sha256': signature, **verify(c, plan, signature)}
|
||||||
|
|
||||||
|
|
||||||
|
def run(db, out, *, pg_service=None, pg_schema=None, inventory=None, plan=None, plan_sha256=None):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists() or not out.parent.is_dir():
|
||||||
|
raise ValueError('New output in existing directory required')
|
||||||
|
if bool(inventory) == bool(plan) or bool(plan) != bool(plan_sha256):
|
||||||
|
raise ValueError('Choose inventory preflight or plan plus exact file hash')
|
||||||
|
if inventory:
|
||||||
|
with closing(open_read_connection(db, pg_service=pg_service, pg_schema=pg_schema)) as c:
|
||||||
|
result = proposal(c, json.loads(Path(inventory).read_bytes()), database_identity(c, db))
|
||||||
|
else:
|
||||||
|
raw = Path(plan).read_bytes()
|
||||||
|
if hashlib.sha256(raw).hexdigest() != plan_sha256:
|
||||||
|
raise ValueError('Plan file hash differs')
|
||||||
|
proposed = json.loads(raw)
|
||||||
|
with closing(open_store(db, pg_service=pg_service, pg_schema=pg_schema).conn) as c:
|
||||||
|
try:
|
||||||
|
if proposed['database'] != database_identity(c, db):
|
||||||
|
raise ValueError('Plan belongs to another database')
|
||||||
|
if not getattr(c, 'postgres', False):
|
||||||
|
c.commit(); c.execute('BEGIN IMMEDIATE')
|
||||||
|
result = apply_plan(c, proposed); c.commit()
|
||||||
|
except BaseException:
|
||||||
|
c.rollback(); raise
|
||||||
|
out.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n')
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog); database_arguments(parser)
|
||||||
|
parser.add_argument('--inventory'); parser.add_argument('--plan'); parser.add_argument('--plan-sha256'); parser.add_argument('--out', required=True)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
report = run(args.db, args.out, pg_service=args.pg_service, pg_schema=args.pg_schema,
|
||||||
|
inventory=args.inventory, plan=args.plan, plan_sha256=args.plan_sha256)
|
||||||
|
print(json.dumps({'status': report.get('status', 'planned'), 'out': args.out}), flush=True)
|
||||||
@@ -0,0 +1,195 @@
|
|||||||
|
"""Read only selected Parquet column chunks; never fetch other payload pages."""
|
||||||
|
import hashlib
|
||||||
|
import io
|
||||||
|
import re
|
||||||
|
import struct
|
||||||
|
import urllib.error
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
MAX_FOOTER = 4 * 1024**2
|
||||||
|
MAX_SELECTED_BYTES = 64 * 1024**2
|
||||||
|
MAX_REQUESTS = 512
|
||||||
|
|
||||||
|
|
||||||
|
class RangeSource:
|
||||||
|
"""Strict bounded HTTP ranges, no full-file fallback, redirects kept private."""
|
||||||
|
def __init__(self, url, size, *, opener=None):
|
||||||
|
self.url, self.size = url, size
|
||||||
|
self.opener = opener or urllib.request.build_opener()
|
||||||
|
self.etag = None
|
||||||
|
self.receipts = []
|
||||||
|
self.bytes_read = 0
|
||||||
|
|
||||||
|
def read(self, start, length):
|
||||||
|
if (type(start) is not int or type(length) is not int or length <= 0
|
||||||
|
or start < 0 or start + length > self.size
|
||||||
|
or self.bytes_read + length > MAX_SELECTED_BYTES
|
||||||
|
or len(self.receipts) >= MAX_REQUESTS):
|
||||||
|
raise ValueError('Bounded metadata range budget exceeded')
|
||||||
|
headers = {'Range': f'bytes={start}-{start + length - 1}', 'Accept-Encoding': 'identity'}
|
||||||
|
if self.etag and not self.etag.startswith('W/'):
|
||||||
|
headers['If-Match'] = self.etag
|
||||||
|
request = urllib.request.Request(self.url, headers=headers)
|
||||||
|
try:
|
||||||
|
with self.opener.open(request, timeout=45) as response:
|
||||||
|
if response.status != 206:
|
||||||
|
raise ValueError('Server did not honor metadata range; full download forbidden')
|
||||||
|
if response.headers.get('Content-Range') != f'bytes {start}-{start + length - 1}/{self.size}':
|
||||||
|
raise ValueError('Object size or range differs from pinned inventory')
|
||||||
|
encoding = response.headers.get('Content-Encoding', 'identity')
|
||||||
|
if encoding != 'identity':
|
||||||
|
raise ValueError('Encoded HTTP ranges are unsupported')
|
||||||
|
etag = response.headers.get('ETag')
|
||||||
|
if self.receipts and etag != self.etag:
|
||||||
|
raise ValueError('Object ETag changed between metadata ranges')
|
||||||
|
raw = response.read(length + 1)
|
||||||
|
if len(raw) != length:
|
||||||
|
raise ValueError('Truncated or oversized metadata range')
|
||||||
|
if not self.receipts:
|
||||||
|
self.etag = etag
|
||||||
|
except urllib.error.HTTPError as error:
|
||||||
|
# Do not expose signed CDN redirect URLs or retry denied resources.
|
||||||
|
raise RuntimeError('Metadata source HTTP ' + str(error.code) + '; no retry or full-file fallback') from None
|
||||||
|
self.receipts.append({'start': start, 'length': length, 'sha256': hashlib.sha256(raw).hexdigest()})
|
||||||
|
self.bytes_read += length
|
||||||
|
return raw
|
||||||
|
|
||||||
|
|
||||||
|
class SparseFile(io.RawIOBase):
|
||||||
|
"""A logical file backed only by approved bytes; all other positions are zeros.
|
||||||
|
|
||||||
|
Arrow may read a 64 KiB footer window. Synthesizing its unused prefix avoids
|
||||||
|
fetching the preceding OCR/review page. Data pages are explicitly selected
|
||||||
|
from footer offsets before they can be downloaded.
|
||||||
|
"""
|
||||||
|
def __init__(self, size, segments):
|
||||||
|
self.size, self.segments, self.position = size, segments, 0
|
||||||
|
|
||||||
|
def readable(self):
|
||||||
|
return True
|
||||||
|
|
||||||
|
def seekable(self):
|
||||||
|
return True
|
||||||
|
|
||||||
|
def tell(self):
|
||||||
|
return self.position
|
||||||
|
|
||||||
|
def seek(self, offset, whence=0):
|
||||||
|
base = {0: 0, 1: self.position, 2: self.size}.get(whence)
|
||||||
|
if base is None or base + offset < 0:
|
||||||
|
raise ValueError('Invalid sparse metadata seek')
|
||||||
|
self.position = base + offset
|
||||||
|
return self.position
|
||||||
|
|
||||||
|
def read(self, length=-1):
|
||||||
|
if length < 0:
|
||||||
|
length = self.size - self.position
|
||||||
|
length = min(length, max(0, self.size - self.position))
|
||||||
|
if length > MAX_SELECTED_BYTES:
|
||||||
|
raise ValueError('Unbounded sparse read forbidden')
|
||||||
|
start, end = self.position, self.position + length
|
||||||
|
raw = bytearray(length)
|
||||||
|
for offset, data in self.segments:
|
||||||
|
left, right = max(offset, start), min(offset + len(data), end)
|
||||||
|
if left < right:
|
||||||
|
raw[left-start:right-start] = data[left-offset:right-offset]
|
||||||
|
self.position = end
|
||||||
|
return bytes(raw)
|
||||||
|
|
||||||
|
def readinto(self, buffer):
|
||||||
|
raw = self.read(len(buffer)); buffer[:len(raw)] = raw
|
||||||
|
return len(raw)
|
||||||
|
|
||||||
|
|
||||||
|
def arrow():
|
||||||
|
try:
|
||||||
|
import pyarrow as pa
|
||||||
|
import pyarrow.parquet as pq
|
||||||
|
except ImportError:
|
||||||
|
raise RuntimeError('Install paper-atlas[mirrors] only on the approved remote runner') from None
|
||||||
|
return pa, pq
|
||||||
|
|
||||||
|
|
||||||
|
def projection(source, columns, *, progress=None):
|
||||||
|
"""Return complete metadata rows and range receipts, no full Parquet snapshot."""
|
||||||
|
pa, pq = arrow()
|
||||||
|
if type(source.size) is not int or not 12 <= source.size <= 4 * 1024**3:
|
||||||
|
raise ValueError('Bounded pinned Parquet size required')
|
||||||
|
if (not isinstance(columns, dict) or not columns
|
||||||
|
or any(not re.fullmatch('[a-z][a-z0-9_]*', key) for key in columns)):
|
||||||
|
raise ValueError('Explicit top-level metadata columns required')
|
||||||
|
header = source.read(0, 4); trailer = source.read(source.size - 8, 8)
|
||||||
|
if header != b'PAR1' or trailer[4:] != b'PAR1':
|
||||||
|
raise ValueError('Expected unencrypted Parquet framing')
|
||||||
|
length = struct.unpack('<I', trailer[:4])[0]
|
||||||
|
if not 0 < length <= min(MAX_FOOTER, source.size - 12):
|
||||||
|
raise ValueError('Invalid or oversized Parquet footer')
|
||||||
|
footer_start = source.size - 8 - length
|
||||||
|
footer = source.read(footer_start, length)
|
||||||
|
segments = [(0, header), (footer_start, footer), (source.size - 8, trailer)]
|
||||||
|
sparse = SparseFile(source.size, segments)
|
||||||
|
parquet = pq.ParquetFile(sparse, pre_buffer=False, buffer_size=0)
|
||||||
|
try:
|
||||||
|
schema = parquet.schema_arrow
|
||||||
|
if len(set(schema.names)) != len(schema.names) or not set(columns) <= set(schema.names):
|
||||||
|
raise ValueError('Missing or repeated metadata columns')
|
||||||
|
for key, kind in columns.items():
|
||||||
|
value = schema.field(key).type
|
||||||
|
if kind == 'string':
|
||||||
|
okay = pa.types.is_string(value) or pa.types.is_large_string(value)
|
||||||
|
elif kind == 'integer':
|
||||||
|
okay = pa.types.is_integer(value)
|
||||||
|
elif kind == 'strings':
|
||||||
|
okay = (pa.types.is_list(value) or pa.types.is_large_list(value)) and (
|
||||||
|
pa.types.is_string(value.value_type) or pa.types.is_large_string(value.value_type))
|
||||||
|
else:
|
||||||
|
raise ValueError('Unsupported safe metadata type')
|
||||||
|
if not okay:
|
||||||
|
raise ValueError('Metadata column type differs: ' + key)
|
||||||
|
metadata = parquet.metadata
|
||||||
|
if not 0 < metadata.num_rows <= 100000 or metadata.num_row_groups > 1000:
|
||||||
|
raise ValueError('Bounded metadata row count required')
|
||||||
|
wanted, excluded = [], []
|
||||||
|
for group in range(metadata.num_row_groups):
|
||||||
|
rg = metadata.row_group(group)
|
||||||
|
for i in range(rg.num_columns):
|
||||||
|
col = rg.column(i)
|
||||||
|
if col.file_path:
|
||||||
|
raise ValueError('External Parquet column files forbidden')
|
||||||
|
offsets = [x for x in (col.dictionary_page_offset, col.data_page_offset) if x is not None and x > 0]
|
||||||
|
if not offsets:
|
||||||
|
raise ValueError('Missing column chunk offset')
|
||||||
|
start = min(offsets); end = start + col.total_compressed_size
|
||||||
|
if not 4 <= start < end <= footer_start:
|
||||||
|
raise ValueError('Column chunk lies outside the pinned payload')
|
||||||
|
item = (start, end, group, col.path_in_schema)
|
||||||
|
(wanted if col.path_in_schema.split('.')[0] in columns else excluded).append(item)
|
||||||
|
if any(max(a, c) < min(b, d) for a, b, _, _ in wanted for c, d, _, _ in excluded):
|
||||||
|
raise ValueError('Selected range overlaps an excluded payload column')
|
||||||
|
if sum(end-start for start, end, _, _ in wanted) + length + 12 > MAX_SELECTED_BYTES:
|
||||||
|
raise ValueError('Metadata projection exceeds byte budget')
|
||||||
|
if len(wanted) + 3 > MAX_REQUESTS:
|
||||||
|
raise ValueError('Metadata projection exceeds request budget')
|
||||||
|
for start, end, group, path in sorted(wanted):
|
||||||
|
segments.append((start, source.read(start, end-start)))
|
||||||
|
if progress:
|
||||||
|
progress({'phase': 'metadata_column_range', 'row_group': group,
|
||||||
|
'column': path, 'metadata_bytes_read': source.bytes_read})
|
||||||
|
rows = []
|
||||||
|
for group in range(metadata.num_row_groups):
|
||||||
|
table = parquet.read_row_group(group, columns=list(columns), use_threads=False)
|
||||||
|
if set(table.column_names) != set(columns) or len(table.column_names) != len(columns):
|
||||||
|
raise ValueError('Unexpected projected metadata fields')
|
||||||
|
rows.extend(table.select(list(columns)).to_pylist())
|
||||||
|
if len(rows) != metadata.num_rows:
|
||||||
|
raise ValueError('Incomplete metadata projection')
|
||||||
|
report = {'rows': len(rows), 'row_groups': metadata.num_row_groups,
|
||||||
|
'columns': dict(columns), 'footer_sha256': hashlib.sha256(footer).hexdigest(),
|
||||||
|
'column_ranges': [{'start': a, 'length': b-a, 'row_group': g, 'column': p} for a, b, g, p in wanted],
|
||||||
|
'range_receipts': source.receipts, 'metadata_bytes_read': source.bytes_read,
|
||||||
|
'source_bytes': source.size, 'full_file_checksum_verified': False,
|
||||||
|
'excluded_payload_pages_downloaded': False,
|
||||||
|
'footer_saved': False, 'raw_parquet_saved': False}
|
||||||
|
return rows, report
|
||||||
|
finally:
|
||||||
|
parquet.close()
|
||||||
@@ -0,0 +1,91 @@
|
|||||||
|
"""Frozen Springer public HTML receipts and chapter citation metadata."""
|
||||||
|
from collections import defaultdict
|
||||||
|
import hashlib
|
||||||
|
from pathlib import Path
|
||||||
|
import json
|
||||||
|
import re
|
||||||
|
from urllib.parse import parse_qs, urlsplit
|
||||||
|
|
||||||
|
from paper_atlas.db.submissions import timestamp
|
||||||
|
from paper_atlas.io.html import Document, clean
|
||||||
|
|
||||||
|
|
||||||
|
def snapshot_index(root):
|
||||||
|
root = Path(root).resolve()
|
||||||
|
receipts = json.loads((root/'receipts.json').read_bytes())['responses']
|
||||||
|
index = {}
|
||||||
|
for receipt in receipts:
|
||||||
|
url = receipt['url']
|
||||||
|
if url in index:
|
||||||
|
raise ValueError('Duplicate snapshot URL')
|
||||||
|
index[url] = (root, receipt)
|
||||||
|
return index
|
||||||
|
|
||||||
|
|
||||||
|
def read_page(index, url):
|
||||||
|
if url not in index:
|
||||||
|
raise ValueError('Missing required public HTML snapshot: '+url)
|
||||||
|
root, receipt = index[url]
|
||||||
|
filename = Path(receipt['file'])
|
||||||
|
path = root/filename
|
||||||
|
content_type = {k.lower():v for k,v in receipt.get('headers', {}).items()}.get('content-type', '')
|
||||||
|
requested, final = urlsplit(url), urlsplit(receipt.get('final_url', ''))
|
||||||
|
query = parse_qs(final.query, keep_blank_values=True)
|
||||||
|
original_query = parse_qs(requested.query, keep_blank_values=True)
|
||||||
|
# Springer returns the public metadata with an explicit cookie diagnostic.
|
||||||
|
# Retain that final URL verbatim; accept no other redirect or changed page.
|
||||||
|
if query != original_query and requested.netloc == 'link.springer.com':
|
||||||
|
if (query.pop('error', None) != ['cookies_not_supported']
|
||||||
|
or not re.fullmatch(r'[0-9a-f-]{36}', ''.join(query.pop('code', [])))):
|
||||||
|
raise ValueError('Unreviewed Springer redirect query')
|
||||||
|
if (filename.is_absolute() or '..' in filename.parts or not path.resolve().is_relative_to(root)
|
||||||
|
or receipt.get('http_status') != 200
|
||||||
|
or (final.scheme, final.netloc, final.path) != (requested.scheme, requested.netloc, requested.path)
|
||||||
|
or final.fragment or query != original_query
|
||||||
|
or 'text/html' not in content_type.lower()):
|
||||||
|
raise ValueError('Unsafe, redirected or unsuccessful public HTML snapshot')
|
||||||
|
raw = path.read_bytes()
|
||||||
|
digest = hashlib.sha256(raw).hexdigest()
|
||||||
|
if len(raw) > 2_000_000 or len(raw) != receipt['bytes'] or digest != receipt['sha256']:
|
||||||
|
raise ValueError('Snapshot checksum or size differs')
|
||||||
|
evidence = {'url':url, 'sha256':digest, 'bytes':len(raw),
|
||||||
|
'file':'evidence/'+digest+'.html', 'http_status':200,
|
||||||
|
'observed_at':timestamp(receipt['at']), 'response_receipt':receipt}
|
||||||
|
return raw, evidence
|
||||||
|
|
||||||
|
|
||||||
|
def parse_chapter(html, doi, *, book_title, publication_year):
|
||||||
|
nodes = list(Document(html).root.walk())
|
||||||
|
meta = defaultdict(list)
|
||||||
|
for node in nodes:
|
||||||
|
if node.tag == 'meta' and node.attrs.get('name', '').startswith('citation_'):
|
||||||
|
meta[node.attrs['name']].append(node.attrs.get('content', ''))
|
||||||
|
def one(key):
|
||||||
|
values = meta[key]
|
||||||
|
if len(values) != 1 or not values[0]:
|
||||||
|
raise ValueError('Missing or ambiguous citation field: '+key)
|
||||||
|
return values[0]
|
||||||
|
if (one('citation_doi') != doi or one('citation_inbook_title') != book_title
|
||||||
|
or one('citation_publication_date') != str(publication_year)):
|
||||||
|
raise ValueError('Chapter DOI, book or publication year differs from reviewed scope')
|
||||||
|
raw_authors = meta['citation_author']
|
||||||
|
if not raw_authors:
|
||||||
|
raise ValueError('Missing named chapter authors')
|
||||||
|
authors = []
|
||||||
|
for raw in raw_authors:
|
||||||
|
if raw.count(',') != 1:
|
||||||
|
raise ValueError('Unreviewed citation author format')
|
||||||
|
family, given = [clean(s) for s in raw.split(',', 1)]
|
||||||
|
if not family or not given:
|
||||||
|
raise ValueError('Incomplete citation author')
|
||||||
|
authors.append({'given':given, 'family':family})
|
||||||
|
abstract_nodes = [n for n in nodes if n.attrs.get('id') == 'Abs1-content']
|
||||||
|
if len(abstract_nodes) > 1:
|
||||||
|
raise ValueError('Ambiguous abstract section')
|
||||||
|
abstract = clean(abstract_nodes[0].text()) if abstract_nodes else None
|
||||||
|
first, last = one('citation_firstpage'), one('citation_lastpage')
|
||||||
|
if not first.isdigit() or not last.isdigit() or int(first) > int(last):
|
||||||
|
raise ValueError('Invalid chapter pages')
|
||||||
|
return {'doi':doi, 'title':one('citation_title'), 'authors':authors,
|
||||||
|
'authors_citation_raw':list(raw_authors), 'abstract':abstract or None,
|
||||||
|
'first_page':first, 'last_page':last}
|
||||||
@@ -8,7 +8,7 @@ TABLES = ('sources', 'papers', 'identifiers', 'source_records', 'venues', 'editi
|
|||||||
'submission_sources', 'submission_identifiers', 'decision_sources',
|
'submission_sources', 'submission_identifiers', 'decision_sources',
|
||||||
'submission_merges', 'decision_merges', 'schema_migrations', 'tags',
|
'submission_merges', 'decision_merges', 'schema_migrations', 'tags',
|
||||||
'paper_tags', 'match_candidates', 'paper_merges', 'ingest_runs')
|
'paper_tags', 'match_candidates', 'paper_merges', 'ingest_runs')
|
||||||
OPTIONAL_TABLES = ('annotation_events',)
|
OPTIONAL_TABLES = ('annotation_events', 'source_scope_revisions')
|
||||||
CHECKS = {
|
CHECKS = {
|
||||||
'annotation_events': ["kind IN ('tag','research_type')", 'retracts_id IS NULL OR retracts_id<>id'],
|
'annotation_events': ["kind IN ('tag','research_type')", 'retracts_id IS NULL OR retracts_id<>id'],
|
||||||
'venues': ["venue_type IN ('conference','journal','review_platform')"],
|
'venues': ["venue_type IN ('conference','journal','review_platform')"],
|
||||||
|
|||||||
@@ -0,0 +1 @@
|
|||||||
|
[{"type":"file","oid":"f87639c8a74ab77de91ea6bb4c88718a52e4250f","size":37291354,"lfs":{"oid":"bcb446a0398dfe62593e1db617f82f43bc792fc413d6d4658b0e0d32c358b888","size":37291354,"pointerSize":133},"xetHash":"c9a6d5d70b6f3a9ee09b4e085327e4f61cad91b9b61244e6bf99031fef9dd85b","path":"data/colm-00000-of-00001.parquet"},{"type":"file","oid":"27ef0f337b76d92ac1aaa4e1dc41a1253768545b","size":28627916,"lfs":{"oid":"930dd4ef90fa352fe6bccca7b73eb36eeabfb99f1e77e783572e2db83e15d779","size":28627916,"pointerSize":133},"xetHash":"468b49fe8b9a921e852c1995c4902e175ad7b54fcefb048a485e3fbad8b62bfd","path":"data/corl-00000-of-00001.parquet"},{"type":"file","oid":"c8de3418de080cbd24667e28c55abf684a8a3d9f","size":90794691,"lfs":{"oid":"e9e3aab42d28ca32a02287ed0bd0f1f9e1f3af5cb8ed8d6a9bf28d4f630773da","size":90794691,"pointerSize":133},"xetHash":"c21b83e5820cea8b7a6bcac34bbd334a0a6ac5b7acb1c4c681715a52ec2cceda","path":"data/emnlp-00000-of-00001.parquet"},{"type":"file","oid":"0a226079bb3fe5256a51750aa874c0e7325f60fa","size":167662887,"lfs":{"oid":"3a65801f95fe490b30282df2cb668fa33fe4182e9d80544148ea66da35215ad8","size":167662887,"pointerSize":134},"xetHash":"2225125f6d18015bb3c44037c82a7f53363eeb636d309de3445e1f1b49f51a03","path":"data/iclr-00000-of-00005.parquet"},{"type":"file","oid":"7b53c49c39f2c287a778e64b18969690926710a8","size":196573710,"lfs":{"oid":"736c05be45eed9db00848471d88dc9cd5a1b1a7d5a0db6432639a21dfb8b4562","size":196573710,"pointerSize":134},"xetHash":"48da5a74c33ca8b04411f4d6f41e34a4ba57d8c4aecb1a72be5dcad89a72bd65","path":"data/iclr-00001-of-00005.parquet"},{"type":"file","oid":"cbf09e34f7f00e33f5bb501461a915ab4bda2ec2","size":210807163,"lfs":{"oid":"1c0b77c370b8735e3d4ebd578706ec8dfbb5e53eb6a71e8cbfeaec36c78ee13f","size":210807163,"pointerSize":134},"xetHash":"fa8bf4848b55d274464b6d06413e1183321e6ffd467e1efa369ee32e0749e8af","path":"data/iclr-00002-of-00005.parquet"},{"type":"file","oid":"4082a290b2e2785413340920e3475c4b5a58b05f","size":250525528,"lfs":{"oid":"078d441ef3854be30f8fd73d34e8442a2a119cd26dbaddf9ab8aba2008df507f","size":250525528,"pointerSize":134},"xetHash":"63ba7c9757b53b02b1793f2186ddc55b79149dbda9a37bacedaf2b0466604d61","path":"data/iclr-00003-of-00005.parquet"},{"type":"file","oid":"d4528dcfcd6d7f56619d2bed75d4705c11811bfb","size":263725824,"lfs":{"oid":"de5efe8578b744f59c8ff9d7c1171cdaa760008920dde190c04b73e5c45901a5","size":263725824,"pointerSize":134},"xetHash":"5934eb773ec49ebf9f48c89d9d7e88ae0d2eb5f5e7f83bfd13f51080aa6d4090","path":"data/iclr-00004-of-00005.parquet"},{"type":"file","oid":"68cd41fa1983b2ba666deb74d1e6da2f9fa9003d","size":204752048,"lfs":{"oid":"4f50e2a0fd7e40a0fdb239595b636c753ca89af1ad9692b7d63f366e7a25eda0","size":204752048,"pointerSize":134},"xetHash":"de8ac963c1738ffe68728255bf7f32fbfff3863b3b07ca1f7026f4c76f123ec0","path":"data/icml-00000-of-00001.parquet"},{"type":"file","oid":"580a98d667a2fc7252fd2bcc2eeeaff116f7030d","size":126503813,"lfs":{"oid":"d2c4359745d30900bd19a44762c9fb6b8415554e2aead6f22f902362aca3e61a","size":126503813,"pointerSize":134},"xetHash":"27cd8f2318a0d096cf3a6aa7178fc502ce52c5d026887f64661b46af74b4b9d5","path":"data/neurips-00000-of-00005.parquet"},{"type":"file","oid":"e90e69bdd1985a36f9a72b4b3b3a9b54d90b7b9d","size":166685127,"lfs":{"oid":"32be2572bc42d33ff227ceadbdcc64e27c996b31e82195d25eced11ec4617963","size":166685127,"pointerSize":134},"xetHash":"2e3528c026d585513df96b23efe39d174b368abd3088e494d85fcf806504a241","path":"data/neurips-00001-of-00005.parquet"},{"type":"file","oid":"b7be2e96d3c3726b87c4a997fcb2c144288b70b4","size":229115194,"lfs":{"oid":"7f9dc8025aebe1d78f74bfccc68e1792f6a076142a920305065d805ff04ab248","size":229115194,"pointerSize":134},"xetHash":"77a4b411199fee5bd1b3fb8752ad4bf1ac090d560a4bfde54b75cf19d48c545b","path":"data/neurips-00002-of-00005.parquet"},{"type":"file","oid":"127763711c2fe76631137432291fa839746536a3","size":264212756,"lfs":{"oid":"32d86e74f00d3fce8eef1cf1db1dd78eed341ddda0e09a924ba17d08c3d566b2","size":264212756,"pointerSize":134},"xetHash":"abe6a12cf6f6f2db52ba28b84a0b91fcfab1411dfa0eb0ab7cdae540b86c5c9d","path":"data/neurips-00003-of-00005.parquet"},{"type":"file","oid":"89eb22ff72a8f44d461e601176239d4f997bbec7","size":279556753,"lfs":{"oid":"b4a536cdb6d306049a32121272b8224ef0e37360423ca04c274f0da74d4dc64c","size":279556753,"pointerSize":134},"xetHash":"d2b68be869a02a2cbe81c74e50fcac3c35585422dbaffcb19985e2376a90e0a7","path":"data/neurips-00004-of-00005.parquet"},{"type":"file","oid":"2b6048e08f894bb47b874a485d3bc5655a43a449","size":211604858,"lfs":{"oid":"df21b892ec531bc223b114d03cd51be6f4f2c48563edde0bdd1b153c36a74a98","size":211604858,"pointerSize":134},"xetHash":"1e81fc6612959ef821e97b7b5ed06a9ebbb3748f0ecc2fe2027b41f7626ef863","path":"data/tmlr-00000-of-00001.parquet"}]
|
||||||
+2
-1
@@ -12,6 +12,7 @@ dependencies = ["PyYAML==6.0.3"]
|
|||||||
|
|
||||||
[project.optional-dependencies]
|
[project.optional-dependencies]
|
||||||
postgres = ["psycopg[binary]==3.3.8"]
|
postgres = ["psycopg[binary]==3.3.8"]
|
||||||
|
mirrors = ["pyarrow==26.0.1"]
|
||||||
|
|
||||||
[project.scripts]
|
[project.scripts]
|
||||||
paper-atlas = "paper_atlas.cli:main"
|
paper-atlas = "paper_atlas.cli:main"
|
||||||
@@ -22,4 +23,4 @@ exclude = ["tests*"]
|
|||||||
|
|
||||||
[tool.setuptools.package-data]
|
[tool.setuptools.package-data]
|
||||||
"paper_atlas.db" = ["schema.sql", "annotations.sql"]
|
"paper_atlas.db" = ["schema.sql", "annotations.sql"]
|
||||||
"paper_atlas.resources" = ["venues.json"]
|
"paper_atlas.resources" = ["venues.json", "reviewarena-files.json"]
|
||||||
|
|||||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
Vendored
BIN
Binary file not shown.
+706
@@ -0,0 +1,706 @@
|
|||||||
|
{
|
||||||
|
"status": "reviewed26_presentation_only_title_correspondences_not_work_identity_approval",
|
||||||
|
"gold": [
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3834907",
|
||||||
|
"title": "H\n <sup>2</sup>\n AL: Hyperbolic Hierarchy-aware Aggregative Learning for Registration-based Few-shot Medical Image Segmentation",
|
||||||
|
"authors": [
|
||||||
|
"Jia Wang",
|
||||||
|
"Jiaming Cai",
|
||||||
|
"Zunying Hu",
|
||||||
|
"Zhanjie Wu",
|
||||||
|
"Jinyuan Liu",
|
||||||
|
"Hua Cheng",
|
||||||
|
"Yun Peng",
|
||||||
|
"Xin Fan"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-004.json",
|
||||||
|
"source_position": 246,
|
||||||
|
"registry_object_sha256": "dd4a627c13ca806e15e59d3ae894c9216d14d33c85bc5ce7362b51fad8c8a692",
|
||||||
|
"program_id": "mfp0053",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "516fac88f1b7a2821c1e2b97cc41530f32abbe2514b65b5823427ff3ecdc9a23",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/129/papers/19",
|
||||||
|
"/entries/129/subsessions/1/items/7"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3834936",
|
||||||
|
"title": "M\n <sup>3</sup>\n TR: Temporal Retrieval Enhanced Multi-Modal Micro-video Popularity Prediction",
|
||||||
|
"authors": [
|
||||||
|
"Jiacheng Lu",
|
||||||
|
"Weijian Wang",
|
||||||
|
"Mingyuan Xiao",
|
||||||
|
"Yang Hua",
|
||||||
|
"Tao Song",
|
||||||
|
"Bo Peng",
|
||||||
|
"Cheng Hua",
|
||||||
|
"Haibing Guan"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-005.json",
|
||||||
|
"source_position": 25,
|
||||||
|
"registry_object_sha256": "93340f8b923d7cc0d848941107351fb661d95effa35b1869921e7163da36bae4",
|
||||||
|
"program_id": "mfp0229",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "6968e6a2d364075020e0b6c95c9076742531681232c40df511a2e8992c6b25f9",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/109/papers/27",
|
||||||
|
"/entries/109/subsessions/2/items/3"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3834946",
|
||||||
|
"title": "<i>PAST:</i>\n Prompt-Adaptive Sampling Termination for Efficient Diffusion Model",
|
||||||
|
"authors": [
|
||||||
|
"Renye Yan",
|
||||||
|
"Jikang Cheng",
|
||||||
|
"You Wu",
|
||||||
|
"Wei Peng",
|
||||||
|
"Zongwei Wang",
|
||||||
|
"Ling Liang",
|
||||||
|
"Yimao Cai"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-005.json",
|
||||||
|
"source_position": 35,
|
||||||
|
"registry_object_sha256": "4210191c2871090711820aef9aff06289694380c9d0194ec9b8b1e5e40fad00d",
|
||||||
|
"program_id": "mfp0297",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "b47bd7502bf3a8e9e9dc87e3d9552efe928cc455f843d527a4860c430c5e9490",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/61/papers/40",
|
||||||
|
"/entries/61/subsessions/3/items/4"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3834956",
|
||||||
|
"title": "<i>SwiftExplorer:</i>\n Training-free diffusion model alignment with Swift Diversity Exploration",
|
||||||
|
"authors": [
|
||||||
|
"Renye Yan",
|
||||||
|
"Jikang Cheng",
|
||||||
|
"You Wu",
|
||||||
|
"Baojin Huang",
|
||||||
|
"Wei Peng",
|
||||||
|
"Zongwei Wang",
|
||||||
|
"Ling Liang",
|
||||||
|
"Yimao Cai"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-006.json",
|
||||||
|
"source_position": 235,
|
||||||
|
"registry_object_sha256": "7f2812ca4072437ecaaa585e85139e7cba0e0d721f9a2c0ac4e9897739faed47",
|
||||||
|
"program_id": "mfp0339",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "f7809f57290e44fe291ffdc07ea956e5484aa491390cd385db75d056f95b840a",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/88/papers/48",
|
||||||
|
"/entries/88/subsessions/4/items/0"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3835035",
|
||||||
|
"title": "<i>Balancing Efficiency and Efficacy:</i>\n Training-Free Attention-Guided Switching Between Explicit and Latent Thoughts for MLLMs",
|
||||||
|
"authors": [
|
||||||
|
"Haoqian Kang",
|
||||||
|
"Liupeng Li",
|
||||||
|
"Kuofeng Gao",
|
||||||
|
"Jinpeng Wang",
|
||||||
|
"Zhenyu Lu",
|
||||||
|
"Bin Chen",
|
||||||
|
"Ke Chen",
|
||||||
|
"Yaowei Wang"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-004.json",
|
||||||
|
"source_position": 8,
|
||||||
|
"registry_object_sha256": "3d76f692754dc159642eeb1706491631267d437a95c4459674361ebf35f416b9",
|
||||||
|
"program_id": "mfp0798",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "477e42fa3d5c8236d6415ff18ba26d799d659b9e2f3f09c1d80bc0971dbca125",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/88/papers/39",
|
||||||
|
"/entries/88/subsessions/3/items/3"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3835055",
|
||||||
|
"title": "TM\n <sup>2</sup>\n Q-Det: Boosting Video Object Detection via Text-Guided Spatiotemporal Mamba and Multimodal Query Learning",
|
||||||
|
"authors": [
|
||||||
|
"Qiang Qi",
|
||||||
|
"Wenqi Shang",
|
||||||
|
"Xiao Wang",
|
||||||
|
"Shuyuan Lin"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-004.json",
|
||||||
|
"source_position": 28,
|
||||||
|
"registry_object_sha256": "7e3c3e6acdad588970c681bdb92969f5134086baa2a5a7b9414871135d5ea137",
|
||||||
|
"program_id": "mfp0903",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "8549d38e781896079ab04ef6a771a4061a0fe42df2fa5c5c03a037d59cc9f548",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/129/papers/55",
|
||||||
|
"/entries/129/subsessions/4/items/7"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3835101",
|
||||||
|
"title": "<scp>ConfAvatar:</scp>\n Confidence-Aware Self-Bootstrapping for Monocular 4D Head Avatar Reconstruction",
|
||||||
|
"authors": [
|
||||||
|
"Zifan Zheng",
|
||||||
|
"Yu Luo",
|
||||||
|
"Yuxin Huang",
|
||||||
|
"Chuancheng Shi",
|
||||||
|
"Mohammad Masbaul Alam Polash",
|
||||||
|
"Zhiyong Wang",
|
||||||
|
"Kun Hu"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-008.json",
|
||||||
|
"source_position": 82,
|
||||||
|
"registry_object_sha256": "b838756bd0b6af036e408936f8f13a6398d83e524ce0282167199eca092bc7d8",
|
||||||
|
"program_id": "mfp1175",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "cb8dbe67e16e163e5b136ad3401fc57c827905e9704d7c0a84961bc8f12565f4",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/109/papers/86",
|
||||||
|
"/entries/109/subsessions/7/items/2"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3835374",
|
||||||
|
"title": "<scp>DateGBC:</scp>\n Deep Fuzzy Granular-Ball Clustering via Adaptive Topology Evolution for Unsupervised Image Segmentation",
|
||||||
|
"authors": [
|
||||||
|
"Qianpeng Chong",
|
||||||
|
"Jiakun Wen",
|
||||||
|
"Guangyi Wei",
|
||||||
|
"Wenyi Zeng"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-006.json",
|
||||||
|
"source_position": 59,
|
||||||
|
"registry_object_sha256": "a7fe694a04449e841be5181888ae35786d0a4938d87eca1fffb0776d4d350e30",
|
||||||
|
"program_id": "mfp2617",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "b57b7026c750aeebb47598860b82980215c9e4e4935f6e8611a31b2fe64f73b9",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/61/papers/80",
|
||||||
|
"/entries/61/subsessions/6/items/8"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3835424",
|
||||||
|
"title": "<i>RTCBench</i>\n : A Scalable Benchmarking Framework for Multi-Party Video Conferencing",
|
||||||
|
"authors": [
|
||||||
|
"Evan Ram",
|
||||||
|
"Tuan Tran",
|
||||||
|
"Youngwook Son",
|
||||||
|
"Sangtae Ha"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-005.json",
|
||||||
|
"source_position": 108,
|
||||||
|
"registry_object_sha256": "b21198e0eba35d175bc83292be8c87bca2b80a9c8e01f40b640a4e9efc7ecde5",
|
||||||
|
"program_id": "mfp2909",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "2398ec6539c50b14c7ed4fa773277cc860151ab77b5430b49d935f88e90a487e",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/129/papers/130",
|
||||||
|
"/entries/129/subsessions/8/items/22"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3835452",
|
||||||
|
"title": "<scp>HeadRouter:</scp>\n Dynamic Head-Weight Routing for Task-Adaptive Audio Token Pruning in Large Audio Language Models",
|
||||||
|
"authors": [
|
||||||
|
"He Peize",
|
||||||
|
"Yaodi Luo",
|
||||||
|
"Xiaoqian Liu",
|
||||||
|
"Xuyang Liu",
|
||||||
|
"Jiahang Deng",
|
||||||
|
"Yaosong Du",
|
||||||
|
"Li Bangyu",
|
||||||
|
"Xiyan Gui",
|
||||||
|
"Yuxuan Chen",
|
||||||
|
"Linfeng Zhang"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-004.json",
|
||||||
|
"source_position": 167,
|
||||||
|
"registry_object_sha256": "ee31b12972ea5844a50c6c63a6fc3430875c3388a4bf261f8e76cab31112d351",
|
||||||
|
"program_id": "mfp3048",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "c455de7c27f43d0fb49990ead5fca795334c0b613eafa2b3c6150eb147d9c3fa",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/46/papers/78",
|
||||||
|
"/entries/46/subsessions/6/items/6"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3835460",
|
||||||
|
"title": "<i>CoAnchor:</i>\n Robust Collaborative Perception under Spatio-Temporal Misalignment via Object-Level Anchors",
|
||||||
|
"authors": [
|
||||||
|
"Chi Li",
|
||||||
|
"Rui Lin",
|
||||||
|
"Aobo Ji",
|
||||||
|
"Dongzhu Xu"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-004.json",
|
||||||
|
"source_position": 175,
|
||||||
|
"registry_object_sha256": "7c41b0dcb65ad27a1f0961f095889ccc73044776250ec816cf95fc25c30da7f1",
|
||||||
|
"program_id": "mfp3109",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "5f3d97775d83528d6f41fb2fc481d1848cac7c3f9de1d469039450f9b2ecbff1",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/109/papers/121",
|
||||||
|
"/entries/109/subsessions/8/items/13"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3835544",
|
||||||
|
"title": "<i>OC-GS:</i>\n Object-Centric Representation and Demand Alignment for Immersive 3D Gaussian Streaming",
|
||||||
|
"authors": [
|
||||||
|
"Boyan Li",
|
||||||
|
"Ruijun Mao",
|
||||||
|
"Zhicheng Liang",
|
||||||
|
"Dayou Zhang",
|
||||||
|
"Fangxin Wang"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-006.json",
|
||||||
|
"source_position": 129,
|
||||||
|
"registry_object_sha256": "afc0c7db304251511eef5e223f027dc6314dd70895955ff1a1cb0a461dbfdbe9",
|
||||||
|
"program_id": "mfp3554",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "00b2ffe1073feede8447bac439ac7842c1ba0841cb6ee38ec1b5e06c3a3dc22e",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/46/papers/153",
|
||||||
|
"/entries/46/subsessions/9/items/21"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3835591",
|
||||||
|
"title": "DP\n <sup>2</sup>\n -NeRF: Dual-Prior Probabilistic Neural Radiance Fields for Event-Guided Motion Deblurring",
|
||||||
|
"authors": [
|
||||||
|
"Jiafeng Huang",
|
||||||
|
"Wenzhen Jia",
|
||||||
|
"Shilong Dong",
|
||||||
|
"Shengjie Zhao"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-007.json",
|
||||||
|
"source_position": 97,
|
||||||
|
"registry_object_sha256": "145025e784ec251247f8cd86d0dbee1749666ec71f35566c20e41edd0a2347ce",
|
||||||
|
"program_id": "mfp3783",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "0b2d267736cfe506de666618820e2934c23395bd25dab63c5a347710445a361c",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/33/papers/158",
|
||||||
|
"/entries/33/subsessions/10/items/2"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3835667",
|
||||||
|
"title": "MMCoT\n <sup>3</sup>\n : Empowering 3D Scene Understanding with 3D-Text Interleaved Multi-Modal Chain-of-Thought Reasoning",
|
||||||
|
"authors": [
|
||||||
|
"Yongqi Shan",
|
||||||
|
"Yifan Wang",
|
||||||
|
"Yunzhi Zhuge",
|
||||||
|
"Lijun Wang",
|
||||||
|
"Huchuan Lu"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-005.json",
|
||||||
|
"source_position": 209,
|
||||||
|
"registry_object_sha256": "8b5ad8d891692e441b6e210304dcd35103835fd35d086c860c7611bf5385a6d9",
|
||||||
|
"program_id": "mfp4173",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "4924721a2c3328e6bca661654569bca86d3fa69aa5f6c9e3719f827c5f0d7639",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/88/papers/114",
|
||||||
|
"/entries/88/subsessions/8/items/6"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3835763",
|
||||||
|
"title": "<scp>AudioKV:</scp>\n KV Cache Eviction in Efficient Large Audio Language Models",
|
||||||
|
"authors": [
|
||||||
|
"Yuxuan Wang",
|
||||||
|
"He Peize",
|
||||||
|
"Xiyan Gui",
|
||||||
|
"Xiaoqian Liu",
|
||||||
|
"Junhao He",
|
||||||
|
"Xuyang Liu",
|
||||||
|
"Xuming Hu",
|
||||||
|
"Linfeng Zhang"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-006.json",
|
||||||
|
"source_position": 192,
|
||||||
|
"registry_object_sha256": "70054b0a5a2586fd08e75494eee08a26498730554601a5fe2e4efddcaad309de",
|
||||||
|
"program_id": "mfp4778",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "274b360da553dfc7afdb9072505c2957cec4213e096373e2edbf58236a3fc3a2",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/129/papers/100",
|
||||||
|
"/entries/129/subsessions/7/items/16"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3835895",
|
||||||
|
"title": "F\n <sup>2</sup>\n -Adapter: A Focus-and-Forget Framework for Generalizable CLIP-based Face Forgery Detection",
|
||||||
|
"authors": [
|
||||||
|
"Zan Gao",
|
||||||
|
"Xin Duan",
|
||||||
|
"Nian Hu",
|
||||||
|
"Hao Jia",
|
||||||
|
"Yibo Zhao",
|
||||||
|
"Cong Liu"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-001.json",
|
||||||
|
"source_position": 39,
|
||||||
|
"registry_object_sha256": "16bdbcc4398df4d9556c14d4d4203127349070669f5d0e1c4bf87756e241b698",
|
||||||
|
"program_id": "mfp5537",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "b3ef626b66d93ac37a73caf15a4cf9ff92241209c80ee66be5575e6b56f9c899",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/88/papers/184",
|
||||||
|
"/entries/88/subsessions/11/items/6"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3835959",
|
||||||
|
"title": "UPR-VAD:\n <i>U</i>\n ncertainty-guided Signal\n <i>P</i>\n urification and\n <i>R</i>\n egularization for Weakly-Supervised\n <i>V</i>\n ideo\n <i>A</i>\n nomaly\n <i>D</i>\n etection",
|
||||||
|
"authors": [
|
||||||
|
"Weiliang Huang",
|
||||||
|
"Weichao Cai",
|
||||||
|
"Bob Zhang",
|
||||||
|
"Chao Huang",
|
||||||
|
"Biao Xue",
|
||||||
|
"Rongxin Zhang"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-007.json",
|
||||||
|
"source_position": 21,
|
||||||
|
"registry_object_sha256": "e96f9e9d67cce31a9235bdc7dc5dba551676f3d36ba72654ae3d5a98ae8d5174",
|
||||||
|
"program_id": "mfp5823",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "d8469bcb68f8cc19070f3bf9a866c3377ba27fe6445b5bf9337b7599a6881592",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/88/papers/66",
|
||||||
|
"/entries/88/subsessions/5/items/6"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3836215",
|
||||||
|
"title": "BLM\n <sub>1</sub>\n : A Boundless Large Model for Cross-Space, Cross-Task, and Cross-Embodiment Learning",
|
||||||
|
"authors": [
|
||||||
|
"Wentao Tan",
|
||||||
|
"Bowen Wang",
|
||||||
|
"Heng Zhi",
|
||||||
|
"Chenyv Liu",
|
||||||
|
"Zhe Li",
|
||||||
|
"Jian Liu",
|
||||||
|
"Zengrong Lin",
|
||||||
|
"Yukun Dai",
|
||||||
|
"Yipeng Chen",
|
||||||
|
"Wenjie Yang",
|
||||||
|
"Enci Xie",
|
||||||
|
"Hao Xue",
|
||||||
|
"Baixu Ji",
|
||||||
|
"Chen Xu",
|
||||||
|
"Zhibin Wang",
|
||||||
|
"Tianshi Wang",
|
||||||
|
"Lei Zhu",
|
||||||
|
"Heng Tao Shen"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-003.json",
|
||||||
|
"source_position": 46,
|
||||||
|
"registry_object_sha256": "b4b51e5d0a4c0a330d01e7638c0bcc73871d750312081bafbd3b8a10529153f2",
|
||||||
|
"program_id": "mfp7332",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "aa458905209ad562117d72dd33d4d61e000b0d9efaa55dd9672919f40e6cc2fd",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/33/papers/13",
|
||||||
|
"/entries/33/subsessions/1/items/1",
|
||||||
|
"/entries/47/papers/0"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3836272",
|
||||||
|
"title": "SEE: Structure-aware Exploring & Exploiting for Long-horizon GUI Agent Trajectory Synthesis",
|
||||||
|
"authors": [
|
||||||
|
"Zhuohang Fan",
|
||||||
|
"Beichen Zhang",
|
||||||
|
"Yuanfa Li",
|
||||||
|
"Changqiao Wu",
|
||||||
|
"Wei Liu",
|
||||||
|
"Jian Luan",
|
||||||
|
"Weigang Zhang"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-002.json",
|
||||||
|
"source_position": 127,
|
||||||
|
"registry_object_sha256": "66eebbdef9e2d07fe08f3cb41cd577095f282f1256bdb73fe1e0d4c5d6e55d51",
|
||||||
|
"program_id": "mfp7635",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "df11d727337f397bab72977bcdce4246b388ef0c8e1198ac47fdc32e68a3aff4",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/88/papers/131",
|
||||||
|
"/entries/88/subsessions/8/items/23"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3836293",
|
||||||
|
"title": "Video\n <sup>*</sup>\n Bench: Benchmarking Fine-grained Video Perception with \"Think with Videos\"",
|
||||||
|
"authors": [
|
||||||
|
"Ruizhe Chen",
|
||||||
|
"Zhiting Fan",
|
||||||
|
"Yang Shi",
|
||||||
|
"Enxin Song",
|
||||||
|
"Wenhao Chai",
|
||||||
|
"Tongkun Guan",
|
||||||
|
"Songtao Jiang",
|
||||||
|
"Ruilin Luo",
|
||||||
|
"Yuanxing Zhang",
|
||||||
|
"Shuai Bai",
|
||||||
|
"Sibo Song",
|
||||||
|
"Zhibo Yang",
|
||||||
|
"Zuozhu Liu"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-002.json",
|
||||||
|
"source_position": 148,
|
||||||
|
"registry_object_sha256": "7332426edb3d15499f175fb98c093b78df21ea95fdd9cfea33da5e925cbec171",
|
||||||
|
"program_id": "mfp7732",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "f6e4df110fe85d7f6d34eb712cd338443c2e34e924393a4e65e07a0c7b152c9e",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/46/papers/198",
|
||||||
|
"/entries/46/subsessions/11/items/20"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3836447",
|
||||||
|
"title": "<i>Insert In Style</i>\n : A Zero-Shot Generative Framework for Harmonious Cross-Domain Object Composition",
|
||||||
|
"authors": [
|
||||||
|
"Raghu Vamsi Chittersu",
|
||||||
|
"Yuvraj Singh Rathore",
|
||||||
|
"Pranav Adlinge",
|
||||||
|
"Eswar Sai Pashaula",
|
||||||
|
"Kunal Swami"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-002.json",
|
||||||
|
"source_position": 183,
|
||||||
|
"registry_object_sha256": "92b2c094b96cebbace0150805b65b0a1bc3ca99ed42124bcb8de442a81094e1b",
|
||||||
|
"program_id": "mfp8648",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "39d3f21f774010980f8b30f2fd800b5243ea7ddc6e091b4d8769de3dee622273",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/107/papers/4",
|
||||||
|
"/entries/117/papers/30",
|
||||||
|
"/entries/117/subsessions/2/items/6"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3836600",
|
||||||
|
"title": "<scp>EEGaze</scp>\n : Multimodal Attention Prediction in Visual Search via EEG-Based Saliency Refinement",
|
||||||
|
"authors": [
|
||||||
|
"Mansi Sharma",
|
||||||
|
"Dinar Orazgaliyev",
|
||||||
|
"Antonio Krüger",
|
||||||
|
"Philipp Müller"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-003.json",
|
||||||
|
"source_position": 140,
|
||||||
|
"registry_object_sha256": "30b99b234aa2b944d6890119b2524d3b3c3c425bde22f20e813b9517b8f8536b",
|
||||||
|
"program_id": "mfp9815",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "47dcb456aee600663d0df59429c1745b335dbae9bdef2dd1063265429fff49fb",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/46/papers/219",
|
||||||
|
"/entries/46/subsessions/12/items/17"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3838656",
|
||||||
|
"title": "<scp>MedMKG:</scp>\n Benchmarking Medical Knowledge Exploitation with Multimodal Knowledge Graph",
|
||||||
|
"authors": [
|
||||||
|
"Xiaochen Wang",
|
||||||
|
"Yuan Zhong",
|
||||||
|
"Lingwei Zhang",
|
||||||
|
"Lisong Dai",
|
||||||
|
"Ting Wang",
|
||||||
|
"Fenglong Ma"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-003.json",
|
||||||
|
"source_position": 225,
|
||||||
|
"registry_object_sha256": "b910b66541ca65127646d35d0cd5737a56ae2e0f311a79526005dec75fca739f",
|
||||||
|
"program_id": "mds177",
|
||||||
|
"track_literal": "Datasets",
|
||||||
|
"track": "datasets",
|
||||||
|
"abstract_sha256": "5bee3b5e9ff60690509d1f34953f3dcc185c50861e112d2c4a3baf5673f6b945",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/46/papers/23",
|
||||||
|
"/entries/46/subsessions/1/items/11"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3839012",
|
||||||
|
"title": "<i>EULALens</i>\n : SEE What YOU Agree!",
|
||||||
|
"authors": [
|
||||||
|
"Abhijeet Narang",
|
||||||
|
"Hagar Elbatanouny",
|
||||||
|
"Vanishri Shekhawat",
|
||||||
|
"Hasan Al-Nashash",
|
||||||
|
"Abhinav Dhall",
|
||||||
|
"Usman Tariq"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-007.json",
|
||||||
|
"source_position": 238,
|
||||||
|
"registry_object_sha256": "81dd29dafd6228f51ae2f32e2bb713272a88df78c78347b9170b54f83db00c84",
|
||||||
|
"program_id": "mde41",
|
||||||
|
"track_literal": "Demo and Video",
|
||||||
|
"track": "demo_and_video",
|
||||||
|
"abstract_sha256": "9acbb264b8019916aad0845aeae4eb5a9c94903ac5bca3cfb612c857306f6d55",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/94/papers/7"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3839024",
|
||||||
|
"title": "<scp>ThreatVerse:</scp>\n A Closed-Loop World Model Simulator for Autonomous Driving Safety Evaluation",
|
||||||
|
"authors": [
|
||||||
|
"Jiangfan Liu",
|
||||||
|
"Zexuan Cui",
|
||||||
|
"Tianyuan Zhang",
|
||||||
|
"Zonglei Jing",
|
||||||
|
"Yaoyuan Zhang",
|
||||||
|
"Jiakai Wang",
|
||||||
|
"Jinyang Guo",
|
||||||
|
"Yuqing Ma",
|
||||||
|
"Lu Sheng",
|
||||||
|
"Aishan Liu",
|
||||||
|
"Xianglong Liu"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-008.json",
|
||||||
|
"source_position": 0,
|
||||||
|
"registry_object_sha256": "d0571f4562b30e4e7fb2e3aa826068ac14a540d1ae70e8cf0b11fd73c67f26ee",
|
||||||
|
"program_id": "mde15",
|
||||||
|
"track_literal": "Demo and Video",
|
||||||
|
"track": "demo_and_video",
|
||||||
|
"abstract_sha256": "02fc9b31d2b7667ea13f00c74f923a375373042171aaa1ff1f2010ee9697211a",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/49/papers/4"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3839028",
|
||||||
|
"title": "<scp>FinAlogy:</scp>\n A Visual Analogy Retrieval System for Financial K-Line Analysis",
|
||||||
|
"authors": [
|
||||||
|
"Xiaoyan Yu",
|
||||||
|
"Ziya Zhao",
|
||||||
|
"Yunshan Ma",
|
||||||
|
"Yifan Wei",
|
||||||
|
"Dianbo Sui",
|
||||||
|
"Tat-Seng Chua"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-008.json",
|
||||||
|
"source_position": 4,
|
||||||
|
"registry_object_sha256": "b18c8e2821873a59318f6f09d1f453dd02195ccedad5c8e0ce1da60ac07466fb",
|
||||||
|
"program_id": "mde02",
|
||||||
|
"track_literal": "Demo and Video",
|
||||||
|
"track": "demo_and_video",
|
||||||
|
"abstract_sha256": "22f978dd02c05439663c5724f25bb997098fd85f5558d01d836711252d119931",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/49/papers/0"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"author_disagreement_hold": {
|
||||||
|
"doi": "10.1145/3767308.3836308",
|
||||||
|
"program_id": "mfp7801"
|
||||||
|
},
|
||||||
|
"scope": "Onlytitlecomparisonchanges;originalregistryandprogramfixturesremainunchanged;sourceABandtrackcomefromtheexplicitprogramID.",
|
||||||
|
"held_gold": {
|
||||||
|
"doi": "10.1145/3767308.3836308",
|
||||||
|
"title": "Dynamic Late Fusion Multiple Kernel\n <i>k</i>\n -means Clustering based on Adaptive Sample Selection and Alignment",
|
||||||
|
"authors": [
|
||||||
|
"Zhirong Tian",
|
||||||
|
"Yi Zhang",
|
||||||
|
"Weixuan Liang",
|
||||||
|
"Miaomiao Li",
|
||||||
|
"Xinwang Liu"
|
||||||
|
],
|
||||||
|
"kind": "program_format_author_version_requires_review",
|
||||||
|
"source_page": "page-001.json",
|
||||||
|
"source_position": 82,
|
||||||
|
"registry_object_sha256": "26a3c1715d276b5fbe988596e3115091086158f6ef12600a36e5206785fc9dfb",
|
||||||
|
"program_id": null,
|
||||||
|
"track_literal": null,
|
||||||
|
"track": "proceedings_unclassified",
|
||||||
|
"abstract_sha256": null,
|
||||||
|
"program_context_pointers": [],
|
||||||
|
"single_name_positions": []
|
||||||
|
}
|
||||||
|
}
|
||||||
BIN
Binary file not shown.
@@ -0,0 +1,137 @@
|
|||||||
|
"""Checks for published ECCV metadata and preserved submission histories."""
|
||||||
|
from hashlib import sha256
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sqlite3
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors.eccv2026_publications import BOOK_TITLE, SOURCE, collect, publisher_row
|
||||||
|
from paper_atlas.db.store import Store
|
||||||
|
from paper_atlas.ingestion.records import ingest_file
|
||||||
|
|
||||||
|
|
||||||
|
FIRST_BOOK = '10.1007/978-3-032-10000-0'
|
||||||
|
FIRST_DOI = FIRST_BOOK + '_1'
|
||||||
|
|
||||||
|
|
||||||
|
def snapshots(root):
|
||||||
|
toc, chapters = root / 'toc', root / 'chapters'
|
||||||
|
toc.mkdir()
|
||||||
|
chapters.mkdir()
|
||||||
|
receipts, volumes = [], {}
|
||||||
|
for offset in range(78):
|
||||||
|
volume = 17001 + offset
|
||||||
|
book = f'10.1007/978-3-032-{10000 + offset}-0'
|
||||||
|
book_url = 'https://link.springer.com/book/' + book
|
||||||
|
count = 37 if offset < 27 else 36
|
||||||
|
files, chapter_dois = {}, []
|
||||||
|
for page, indices in ((1, range(1, 20)), (2, range(20, count + 1))):
|
||||||
|
filename = f'{volume}-{page}.html'
|
||||||
|
html = f'<h1>{BOOK_TITLE}</h1><p>Malmö (LNCS, volume {volume})</p>'
|
||||||
|
html += '<li data-test="chapter"><h3>Front Matter</h3></li>'
|
||||||
|
for i in indices:
|
||||||
|
doi = f'{book}_{i}'
|
||||||
|
html += f'<li data-test="chapter"><h3><a href="/chapter/{doi}">Title {volume}-{i}</a></h3></li>'
|
||||||
|
chapter_dois.append(doi)
|
||||||
|
raw = html.encode()
|
||||||
|
(toc / filename).write_bytes(raw)
|
||||||
|
files[str(page)] = filename
|
||||||
|
receipts.append({'file': filename, 'status': 200, 'bytes': len(raw), 'sha256': sha256(raw).hexdigest(),
|
||||||
|
'url': book_url + ('?page=2' if page == 2 else '')})
|
||||||
|
volumes[str(volume)] = {'book': book_url, 'files': files, 'chapter_dois': chapter_dois}
|
||||||
|
data = ''.join(json.dumps(row) + '\n' for row in receipts).encode()
|
||||||
|
(toc / 'receipts.jsonl').write_bytes(data)
|
||||||
|
checkpoint = json.dumps(volumes).encode()
|
||||||
|
(toc / 'volume-checkpoint.json').write_bytes(checkpoint)
|
||||||
|
(toc / 'summary.json').write_text(json.dumps({
|
||||||
|
'raw_receipts_sha256': sha256(data).hexdigest(),
|
||||||
|
'volume_checkpoint_sha256': sha256(checkpoint).hexdigest(),
|
||||||
|
'chapter_link_count': 2835,
|
||||||
|
}))
|
||||||
|
url = 'https://link.springer.com/chapter/' + FIRST_DOI
|
||||||
|
citation = {
|
||||||
|
'citation_doi': FIRST_DOI, 'citation_title': 'Title 17001-1',
|
||||||
|
'citation_inbook_title': BOOK_TITLE, 'citation_publication_date': '2026',
|
||||||
|
'citation_firstpage': '1', 'citation_lastpage': '10',
|
||||||
|
'citation_author': 'Chen, Lei',
|
||||||
|
}
|
||||||
|
body = ''.join(f'<meta name="{k}" content="{v}">' for k, v in citation.items())
|
||||||
|
body += '<div id="Abs1-content"><p>Published abstract.</p></div>'
|
||||||
|
raw = body.encode()
|
||||||
|
(chapters / 'first.html').write_bytes(raw)
|
||||||
|
(chapters / 'receipts.json').write_text(json.dumps({'responses': [{
|
||||||
|
'url': url, 'final_url': url, 'file': 'first.html', 'at': '2026-11-18T00:00:00Z',
|
||||||
|
'http_status': 200, 'bytes': len(raw), 'sha256': sha256(raw).hexdigest(),
|
||||||
|
'headers': {'Content-Type': 'text/html'},
|
||||||
|
}]}))
|
||||||
|
return {'toc_snapshots': toc, 'chapter_snapshots': chapters}
|
||||||
|
|
||||||
|
|
||||||
|
class ECCV2026PublicationsTests(unittest.TestCase):
|
||||||
|
def test_frozen_snapshot_and_existing_history(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp)
|
||||||
|
source = snapshots(root)
|
||||||
|
report = collect(root / 'out', **source)
|
||||||
|
self.assertEqual((report['toc_chapters'], report['chapter_details_missing']), (2835, 2834))
|
||||||
|
manifest, = report['files']
|
||||||
|
self.assertEqual((manifest['track'], manifest['status']), ('main', 'partial'))
|
||||||
|
row = json.loads(Path(manifest['file']).read_text())
|
||||||
|
self.assertEqual(row['identifiers'], {'doi': FIRST_DOI})
|
||||||
|
self.assertEqual(row['submission_identifiers'], [{'scheme': 'publication_doi', 'value': FIRST_DOI}])
|
||||||
|
self.assertEqual(row['authors'], [{'family': 'Chen', 'given': 'Lei'}])
|
||||||
|
self.assertEqual(row['abstract'], 'Published abstract.')
|
||||||
|
self.assertEqual(row['metadata']['lncs_volume'], 17001)
|
||||||
|
self.assertFalse(row['metadata']['original_decision_signature_verified'])
|
||||||
|
detail = row['metadata']['publisher_chapter_evidence']
|
||||||
|
self.assertEqual(sha256((root / 'out' / detail['file']).read_bytes()).hexdigest(), detail['sha256'])
|
||||||
|
db = root / 'example.sqlite'
|
||||||
|
store = Store(db)
|
||||||
|
pid, rid = store.record('prior', 'work', 'https://example.org/work',
|
||||||
|
row['title'], 'Original preprint abstract.', row['authors'],
|
||||||
|
{'arxiv': '2601.00001'})
|
||||||
|
store.decision(pid, rid, 'ICLR', 2026, 'main', 'work', 'https://example.org/decision',
|
||||||
|
'rejected', 'Original ICLR result.')
|
||||||
|
store.conn.commit()
|
||||||
|
store.conn.close()
|
||||||
|
row['identifiers']['arxiv'] = '2601.00001'
|
||||||
|
path = Path(manifest['file'])
|
||||||
|
raw = (json.dumps(row) + '\n').encode()
|
||||||
|
path.write_bytes(raw)
|
||||||
|
manifest['file_sha256'] = sha256(raw).hexdigest()
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest))
|
||||||
|
ingest_file(db, path)
|
||||||
|
ingest_file(db, path)
|
||||||
|
with sqlite3.connect(db) as conn:
|
||||||
|
self.assertEqual(conn.execute('SELECT id,abstract FROM papers').fetchall(),
|
||||||
|
[(pid, 'Original preprint abstract.')])
|
||||||
|
self.assertEqual(conn.execute('SELECT status FROM decisions ORDER BY id').fetchall(),
|
||||||
|
[('rejected',), ('accepted',)])
|
||||||
|
self.assertEqual(conn.execute('SELECT scheme,value FROM submission_identifiers').fetchall(),
|
||||||
|
[('publication_doi', FIRST_DOI)])
|
||||||
|
self.assertEqual(conn.execute('SELECT name FROM sources ORDER BY id').fetchall(), [('prior',), (SOURCE,)])
|
||||||
|
with self.assertRaises(FileExistsError):
|
||||||
|
collect(root / 'out', **source)
|
||||||
|
|
||||||
|
def test_numeric_superscript_wrappers_preserve_both_title_literals(self):
|
||||||
|
chapter = {'doi': FIRST_DOI, 'title': 'Example $$^2$$ : Formal title',
|
||||||
|
'authors': [], 'abstract': 'Published abstract.', 'authors_citation_raw': [],
|
||||||
|
'first_page': '1', 'last_page': '10'}
|
||||||
|
toc = {'title': r'Example \(^2\) : Formal title', 'book_doi': FIRST_BOOK, 'lncs_volume': 17001,
|
||||||
|
'evidence_url': 'https://link.springer.com/book/' + FIRST_BOOK, 'evidence_sha256': 'a' * 64}
|
||||||
|
evidence = {'observed_at': '2026-11-18T00:00:00Z'}
|
||||||
|
row = publisher_row(chapter, toc, evidence)
|
||||||
|
self.assertEqual(row['title'], chapter['title'])
|
||||||
|
self.assertEqual(row['metadata']['publisher_contents_evidence']['title_literal'], toc['title'])
|
||||||
|
with self.assertRaisesRegex(ValueError, 'title differs'):
|
||||||
|
publisher_row(chapter, {**toc, 'title': r'Example \(^3\) : Formal title'}, evidence)
|
||||||
|
|
||||||
|
def test_changed_contents_receipt_stops_before_output(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp)
|
||||||
|
source = snapshots(root)
|
||||||
|
(source['toc_snapshots'] / '17001-1.html').write_text('unverified bytes')
|
||||||
|
with self.assertRaisesRegex(ValueError, 'contents bytes disagree'):
|
||||||
|
collect(root / 'out', **source)
|
||||||
|
self.assertFalse((root / 'out').exists())
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
import base64
|
||||||
|
import copy
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import icassp2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class ICASSP2026Test(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/icassp2026-source.json.gz').read_bytes()))
|
||||||
|
cls.files = {name: base64.b64decode(body) for name, body in cls.fixture['files'].items()}
|
||||||
|
cls.items = []
|
||||||
|
cls.manifest = json.loads(cls.files[c.SNAPSHOT+'/manifest.json'])
|
||||||
|
for receipt in cls.manifest['pages']:
|
||||||
|
cls.items.extend((r, receipt) for r in json.loads(cls.files[c.SNAPSHOT+'/'+receipt['file']])['message']['items'])
|
||||||
|
|
||||||
|
def materialize(self, root):
|
||||||
|
for name, body in self.files.items():
|
||||||
|
path = root/name; path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(body)
|
||||||
|
|
||||||
|
def test_full_census_matches_independent_gold_and_preserves_source(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); src = root/'src'; self.materialize(src)
|
||||||
|
report = c.collect(root/'out', snapshots=src)
|
||||||
|
self.assertEqual((report['records'],report['missing_abstract_count'],report['single_name_author_fields']), (4582,4582,23))
|
||||||
|
self.assertEqual(len(report['excluded_nonpaper_items']),7); self.assertEqual(len(report['excluded_other_conference_dois']),56)
|
||||||
|
self.assertFalse(report['database_written']); self.assertFalse(report['pdfs_downloaded']); self.assertEqual(report['network_requests'],0)
|
||||||
|
self.assertEqual(report['approved_for_import'],0); self.assertEqual(report['public_rejected_availability'],'unknown')
|
||||||
|
manifest, = report['files']; path = Path(manifest['file']); raw = path.read_bytes()
|
||||||
|
self.assertEqual(hashlib.sha256(raw).hexdigest(),manifest['file_sha256']); self.assertEqual(manifest['status'],'partial')
|
||||||
|
rows = [json.loads(l) for l in raw.splitlines()]
|
||||||
|
actual = [{'doi':r['external_id'],'title':r['title'],'authors':r['authors']} for r in rows]
|
||||||
|
gold = sorted(self.fixture['independent_gold'],key=lambda r:r['doi'])
|
||||||
|
self.assertEqual(actual,gold)
|
||||||
|
original = {r['DOI']:r for r,_ in self.items}
|
||||||
|
evidence_hashes = {str(p.relative_to(root/'out')):hashlib.sha256(p.read_bytes()).hexdigest() for p in (root/'out/evidence').iterdir()}
|
||||||
|
for r in rows:
|
||||||
|
doi = r['external_id']; self.assertEqual(r['metadata']['registry_fields'],original[doi])
|
||||||
|
self.assertEqual(r['identifiers'],{'doi':doi}); self.assertEqual(r['submission_identifiers'],[{'scheme':'publication_doi','value':doi}])
|
||||||
|
self.assertEqual(r['track'],'proceedings_unclassified'); self.assertFalse(r['tags']); self.assertFalse(r['abstract'])
|
||||||
|
self.assertFalse(r['metadata']['original_decision_signature_verified'])
|
||||||
|
evidence = r['metadata']['source_evidence']
|
||||||
|
self.assertEqual(evidence_hashes[evidence['file']],evidence['sha256'])
|
||||||
|
self.assertIn('Fareedha',next(r for r in rows if r['external_id'].endswith('.11461474'))['authors'])
|
||||||
|
with self.assertRaisesRegex(ValueError,'Output exists'):
|
||||||
|
c.collect(root/'out',snapshots=src)
|
||||||
|
|
||||||
|
def test_wrong_conference_doi_date_and_unexpected_abstract_rejected(self):
|
||||||
|
for kind in ('container','event','doi','date','author','sequence','abstract'):
|
||||||
|
with self.subTest(kind=kind):
|
||||||
|
items = copy.deepcopy(self.items); r = next(r for r,_ in items if r.get('author') and r['DOI'].startswith(c.PREFIX))
|
||||||
|
if kind=='container': r['container-title']=['Foreign']
|
||||||
|
if kind=='event': r['event']['name']='Other'
|
||||||
|
if kind=='doi': r['DOI']='10.1109/foreign.2026.1'
|
||||||
|
if kind=='date': r['published']['date-parts']=[[2025,5,3]]
|
||||||
|
if kind=='author': r['author']=[]
|
||||||
|
if kind=='sequence': r['author'][0]['sequence']='additional'
|
||||||
|
if kind=='abstract': r['abstract']='new source version'
|
||||||
|
with self.assertRaises(ValueError): c.registry_papers(items)
|
||||||
|
|
||||||
|
def test_nonpaper_exclusion_requires_exact_reviewed_evidence(self):
|
||||||
|
for kind in ('title','page','author'):
|
||||||
|
items = copy.deepcopy(self.items); r = next(r for r,_ in items if r['DOI'] in c.NONPAPERS)
|
||||||
|
if kind=='title': r['title']=['A research paper']
|
||||||
|
if kind=='page': r['page']='1-5'
|
||||||
|
if kind=='author': r['author']=[{'family':'Smith'}]
|
||||||
|
with self.assertRaisesRegex(ValueError,'nonpaper'): c.registry_papers(items)
|
||||||
|
|
||||||
|
def test_duplicate_and_incomplete_census_rejected(self):
|
||||||
|
with self.assertRaisesRegex(ValueError,'repeated'): c.registry_papers(self.items+[self.items[0]])
|
||||||
|
with self.assertRaisesRegex(ValueError,'partition'): c.registry_papers(self.items[:-1])
|
||||||
|
|
||||||
|
def test_corruption_and_partial_input_preserve_failure_without_success(self):
|
||||||
|
for kind in ('manifest','body','missing'):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp); src=root/'src'; self.materialize(src)
|
||||||
|
path=src/c.SNAPSHOT/('manifest.json' if kind=='manifest' else 'page-002.json')
|
||||||
|
if kind=='missing': path.unlink()
|
||||||
|
else: path.write_bytes(path.read_bytes()+b' ')
|
||||||
|
with self.assertRaises((ValueError,FileNotFoundError)): c.collect(root/'out',snapshots=src)
|
||||||
|
self.assertTrue((root/'out/failed.json').exists()); self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_symlink_page_and_snapshot_rejected(self):
|
||||||
|
for kind in ('page','directory'):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp); src=root/'src'; self.materialize(src)
|
||||||
|
path=src/c.SNAPSHOT/('page-001.json' if kind=='page' else '')
|
||||||
|
renamed=path.with_name(path.name+'-original'); path.rename(renamed); path.symlink_to(renamed,target_is_directory=kind=='directory')
|
||||||
|
with self.assertRaisesRegex(ValueError,'Unsafe'): c.collect(root/'out',snapshots=src)
|
||||||
@@ -0,0 +1,113 @@
|
|||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import icdm2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class ICDM2026Tests(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/icdm2026-source.json.gz').read_bytes()))
|
||||||
|
cls.raw = cls.fixture['files'][c.SNAPSHOT+'/icdm-detailed-sessions.html'].encode()
|
||||||
|
cls.gold = cls.fixture['independent_inventory']
|
||||||
|
|
||||||
|
def snapshots(self, root):
|
||||||
|
for name, text in self.fixture['files'].items():
|
||||||
|
path = root/name; path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(text.encode())
|
||||||
|
|
||||||
|
def test_all_rows_match_separately_parsed_frozen_inventory(self):
|
||||||
|
result = c.parse_program(self.raw)
|
||||||
|
self.assertEqual(len(result), len(self.gold['entries']))
|
||||||
|
for position, (row, gold) in enumerate(zip(result, self.gold['entries']), 1):
|
||||||
|
for field in ('paper_id_literal', 'title', 'authors_literal', 'source_reported_authors', 'time_literal'):
|
||||||
|
self.assertEqual(row[field], gold[field])
|
||||||
|
self.assertEqual(row['source_position'], position)
|
||||||
|
for field, value in gold['session'].items():
|
||||||
|
self.assertEqual(row['session'][field], value)
|
||||||
|
|
||||||
|
def test_instantiable_collection_retains_scope_fields_and_source_bytes(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); self.snapshots(root/'src')
|
||||||
|
report = c.collect(root/'out', snapshots=root/'src')
|
||||||
|
self.assertEqual(report['track_counts'], self.gold['track_counts_literal'])
|
||||||
|
self.assertEqual(report['presentation_counts'], self.gold['presentation_counts_literal'])
|
||||||
|
self.assertEqual((report['records'], report['missing_abstract_count']), (324, 324))
|
||||||
|
self.assertFalse(report['database_written']); self.assertEqual(report['network_requests'], 0)
|
||||||
|
self.assertEqual(report['public_rejected_metadata'], 'unknown_no_public_export_verified')
|
||||||
|
all_rows = []
|
||||||
|
for manifest in report['files']:
|
||||||
|
path = Path(manifest['file']); raw = path.read_bytes()
|
||||||
|
self.assertEqual(hashlib.sha256(raw).hexdigest(), manifest['file_sha256'])
|
||||||
|
self.assertEqual(manifest['status'], 'partial')
|
||||||
|
rows = [json.loads(line) for line in raw.splitlines()]
|
||||||
|
self.assertEqual(len(rows), manifest['missing_abstract_count'])
|
||||||
|
for row in rows:
|
||||||
|
self.assertEqual((row['venue'], row['year'], row['track']), ('ICDM', 2026, manifest['track']))
|
||||||
|
self.assertIsNone(row['abstract']); self.assertIsNone(row['metadata']['paper_type'])
|
||||||
|
self.assertEqual(row['identifiers'], {}); self.assertEqual(row['submission_identifiers'], [])
|
||||||
|
self.assertFalse(row['metadata']['original_decision_signature_verified'])
|
||||||
|
self.assertEqual(row['external_id'], '2026/'+row['metadata']['program_fields']['paper_id_literal'])
|
||||||
|
self.assertTrue(row['url'].startswith(c.PROGRAM+'#paper-DM'))
|
||||||
|
self.assertTrue(all(t['namespace'].startswith(c.SOURCE+':') for t in row['tags']))
|
||||||
|
self.assertNotIn('doi', row['metadata']); self.assertNotIn('decision_date', row)
|
||||||
|
all_rows.extend(rows)
|
||||||
|
self.assertEqual(len({r['external_id'] for r in all_rows}), 324)
|
||||||
|
for item in report['evidence']:
|
||||||
|
raw = (root/'out'/item['output_file']).read_bytes()
|
||||||
|
self.assertEqual(raw, self.fixture['files'][c.SNAPSHOT+'/'+item['file']].encode())
|
||||||
|
self.assertTrue((root/'out/collection-report.json').is_file())
|
||||||
|
self.assertFalse((root/'out/collection-report.json.part').exists())
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||||
|
c.collect(root/'out', snapshots=root/'src')
|
||||||
|
|
||||||
|
def test_stale_or_different_conference_rejected(self):
|
||||||
|
for old, new in [(b'IEEE ICDM 2026', b'IEEE ICDM 2025'),
|
||||||
|
(b'In-person session presentation list.', b'Workshop presentation list.')]:
|
||||||
|
with self.subTest(old=old), self.assertRaises(ValueError):
|
||||||
|
c.parse_program(self.raw.replace(old, new))
|
||||||
|
|
||||||
|
def test_track_attribute_must_agree_with_visible_label(self):
|
||||||
|
changed = self.raw.replace(b'data-track="research"', b'data-track="applied"', 1)
|
||||||
|
with self.assertRaisesRegex(ValueError, 'track'):
|
||||||
|
c.parse_program(changed)
|
||||||
|
|
||||||
|
def test_presentation_attribute_is_not_inferred_from_session_id(self):
|
||||||
|
changed = self.raw.replace(b'data-format="long"', b'data-format="short"', 1)
|
||||||
|
with self.assertRaisesRegex(ValueError, 'presentation'):
|
||||||
|
c.parse_program(changed)
|
||||||
|
|
||||||
|
def test_repeated_paper_and_source_locator_disagreement_rejected(self):
|
||||||
|
for old, new in [(b'DM1278', b'DM827'), (b'id="paper-DM827"', b'id="paper-DM99999"')]:
|
||||||
|
with self.subTest(old=old), self.assertRaises(ValueError):
|
||||||
|
c.parse_program(self.raw.replace(old, new))
|
||||||
|
|
||||||
|
def test_lost_session_or_changed_session_counts_rejected(self):
|
||||||
|
for old, new in [(b'data-session-id="L1"', b'data-session-id="L2"'),
|
||||||
|
(b'Long \xc2\xb7 7 papers', b'Long \xc2\xb7 6 papers')]:
|
||||||
|
with self.subTest(old=old), self.assertRaises(ValueError):
|
||||||
|
c.parse_program(self.raw.replace(old, new, 1))
|
||||||
|
|
||||||
|
def test_empty_author_is_not_silently_dropped(self):
|
||||||
|
changed = self.raw.replace(b'Qishuang Fu; Andreas Deppeler;', b'Qishuang Fu;;', 1)
|
||||||
|
with self.assertRaisesRegex(ValueError, 'author'):
|
||||||
|
c.parse_program(changed)
|
||||||
|
|
||||||
|
def test_snapshot_manifest_and_body_corruption_rejected_before_output(self):
|
||||||
|
for name in ('manifest.json', 'icdm-detailed-sessions.html', 'icdm-research-policy.html', 'icdm-applied-policy.html'):
|
||||||
|
with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); self.snapshots(root/'src'); path = root/'src'/c.SNAPSHOT/name
|
||||||
|
path.write_bytes(path.read_bytes()+b' ')
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.collect(root/'out', snapshots=root/'src')
|
||||||
|
self.assertFalse((root/'out').exists())
|
||||||
|
|
||||||
|
def test_snapshot_source_symlink_rejected(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); self.snapshots(root/'src'); path = root/'src'/c.SNAPSHOT/'icdm-detailed-sessions.html'
|
||||||
|
real = root/'real.html'; path.rename(real); path.symlink_to(real)
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Unsafe source'):
|
||||||
|
c.collect(root/'out', snapshots=root/'src')
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
import base64
|
||||||
|
import copy
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import ismir2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class ISMIR2026Test(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture=json.loads(gzip.decompress((Path(__file__).parent/'fixtures/ismir2026-source.json.gz').read_bytes()))
|
||||||
|
cls.files={name:base64.b64decode(raw) for name,raw in cls.fixture['files'].items()}
|
||||||
|
cls.program=json.loads(cls.files['ismir2026-json-research-v1/papers.json'])
|
||||||
|
cls.accepted=cls.files['remaining2026-listings-research-v2/ismir-accepted.html']
|
||||||
|
|
||||||
|
def materialize(self,root):
|
||||||
|
for name,raw in self.files.items():
|
||||||
|
p=root/name;p.parent.mkdir(parents=True,exist_ok=True);p.write_bytes(raw)
|
||||||
|
|
||||||
|
def test140_source_versions_and_review_optins_against_independent_gold(self):
|
||||||
|
listing=c.accepted_listing(self.accepted);observations=c.program_observations(self.program,listing)
|
||||||
|
self.assertEqual(len(observations),140)
|
||||||
|
for observation,gold in zip(observations,self.fixture['independent_gold']):
|
||||||
|
r=observation['original'];v=r['content'];official=observation['accepted_listing']
|
||||||
|
self.assertEqual((r['id'],r['forum'],v['title'],v['authors'],v['abstract']),(gold['program_id'],gold['forum_literal'],gold['title'],gold['authors'],gold['abstract']))
|
||||||
|
self.assertEqual(official,{'title':gold['official_title'],'authors_literal':gold['official_authors_literal'],'author_names':gold['official_author_names']})
|
||||||
|
self.assertEqual(v['session'],gold['session_literal']);self.assertEqual(v['paper_presentation'],gold['presentation_literal']);self.assertEqual(v['keywords'],gold['keywords'])
|
||||||
|
self.assertEqual(sum('program_and_accepted_title_literal_versions_differ' in r['review_flags'] for r in observations),2)
|
||||||
|
self.assertEqual({r['original']['id'] for r in observations if 'program_and_accepted_author_sequence_versions_differ' in r['review_flags']},{'60','126','433'})
|
||||||
|
self.assertEqual(sum(r['original']['reviews']['publish_reviews']=='Yes' for r in observations),101)
|
||||||
|
|
||||||
|
def test_full_collection_keeps_abstract_bytes_review_text_tags_and_no_inferred_ids(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);src=root/'src';self.materialize(src);out=root/'out';report=c.collect(out,snapshots=src)
|
||||||
|
self.assertEqual((report['records'],report['missing_abstract_count'],report['review_optin_yes'],report['review_optin_no']),(140,0,101,39))
|
||||||
|
self.assertFalse(report['database_written']);self.assertFalse(report['pdfs_downloaded']);self.assertEqual(report['network_requests'],0);self.assertEqual(report['approved_for_import'],0)
|
||||||
|
self.assertEqual(report['public_rejected_availability'],'unknown_accepted_program_reviews_are_not_a_rejected_export')
|
||||||
|
manifest,=report['files'];raw=Path(manifest['file']).read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),manifest['file_sha256']);self.assertEqual(manifest['status'],'partial')
|
||||||
|
rows=[json.loads(l) for l in raw.splitlines()];self.assertEqual(len(rows),140)
|
||||||
|
for row,original,gold in zip(rows,self.program,self.fixture['independent_gold']):
|
||||||
|
self.assertEqual(row['metadata']['program_entry'],original);self.assertEqual(row['abstract'],gold['abstract']);self.assertEqual(row['title'],gold['title']);self.assertEqual(row['authors'],gold['authors'])
|
||||||
|
self.assertEqual(row['external_id'],'2026/'+gold['program_id']);self.assertEqual(row['identifiers'],{});self.assertEqual(row['submission_identifiers'],[])
|
||||||
|
self.assertEqual(row['status'],'accepted');self.assertEqual(row['track'],'program_unclassified');self.assertIsNone(row['metadata']['paper_type']);self.assertFalse(row['metadata']['original_decision_signature_verified'])
|
||||||
|
tags=[{'namespace':'ismir_program:presentation','value':gold['presentation_literal']}]+[{'namespace':'ismir_program:keyword','value':v} for v in dict.fromkeys(' '.join(k.split()) for k in gold['keywords'] if k.strip())]
|
||||||
|
self.assertEqual(row['tags'],tags);self.assertTrue(all(t['value'] for t in row['tags']))
|
||||||
|
if original['reviews']['publish_reviews']=='No':self.assertFalse(any(v for k,v in row['metadata']['program_entry']['reviews'].items() if k!='publish_reviews'))
|
||||||
|
for name,body in self.files.items():self.assertEqual((out/'evidence'/name).read_bytes(),body)
|
||||||
|
with self.assertRaisesRegex(ValueError,'Output exists'):c.collect(out,snapshots=src)
|
||||||
|
|
||||||
|
def test_duplicate_and_malformed_locator_or_incomplete_census_rejected(self):
|
||||||
|
listing=c.accepted_listing(self.accepted)
|
||||||
|
for kind in ('repeat','forum','non_numeric','missing_row'):
|
||||||
|
rows=copy.deepcopy(self.program)
|
||||||
|
if kind=='repeat':rows[1]['id']=rows[0]['id'];rows[1]['forum']=rows[0]['forum']
|
||||||
|
if kind=='forum':rows[0]['forum']='OriginalOpenReviewID'
|
||||||
|
if kind=='non_numeric':rows[0]['id']='abc';rows[0]['forum']='abc'
|
||||||
|
if kind=='missing_row':rows.pop()
|
||||||
|
with self.subTest(kind=kind),self.assertRaises(ValueError):c.program_observations(rows,listing)
|
||||||
|
|
||||||
|
def test_empty_fields_foreign_title_and_session_disagree_rejected(self):
|
||||||
|
listing=c.accepted_listing(self.accepted)
|
||||||
|
for field,value in [('authors',[]),('abstract',''),('title','A foreign paper'),('session',['99']),('keywords',[42])]:
|
||||||
|
rows=copy.deepcopy(self.program);rows[0]['content'][field]=value
|
||||||
|
with self.subTest(field=field),self.assertRaises(ValueError):c.program_observations(rows,listing)
|
||||||
|
|
||||||
|
def test_source_optin_no_cannot_publish_review_text(self):
|
||||||
|
rows=copy.deepcopy(self.program);r=next(r for r in rows if r['reviews']['publish_reviews']=='No');r['reviews']['meta_review']='Do not infer a negative decision'
|
||||||
|
with self.assertRaisesRegex(ValueError,'preference'):c.program_observations(rows,c.accepted_listing(self.accepted))
|
||||||
|
|
||||||
|
def test_parenthesized_affiliation_semicolons_do_not_split_people(self):
|
||||||
|
self.assertEqual(c.author_names('A Person (University A; Lab (X)); B Person (Institute B)'),['A Person','B Person'])
|
||||||
|
for raw in ['A Person (Broken','; A Person (Lab)','A Person )']:
|
||||||
|
with self.assertRaises(ValueError):c.author_names(raw)
|
||||||
|
|
||||||
|
def test_ambiguous_official_title_is_not_fused(self):
|
||||||
|
source=self.accepted.replace(b'FusID: Modality-Fused Semantic IDs for Generative Music Recommendation',b'Off-Manifold Robustness in Synthesizer Inversion with Joint Distribution Flow Matching',1)
|
||||||
|
with self.assertRaisesRegex(ValueError,'ambiguous'):c.accepted_listing(source)
|
||||||
|
|
||||||
|
def test_tamper_missing_file_and_symlink_fail_without_success_report(self):
|
||||||
|
for kind in ('manifest','body','missing','symlink'):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);src=root/'src';self.materialize(src);path=src/'ismir2026-json-research-v1'/('manifest.json' if kind=='manifest' else 'papers.json')
|
||||||
|
if kind=='missing':path.unlink()
|
||||||
|
elif kind=='symlink':other=root/'actual.json';path.rename(other);path.symlink_to(other)
|
||||||
|
else:path.write_bytes(path.read_bytes()+b' ')
|
||||||
|
with self.subTest(kind=kind),self.assertRaises((ValueError,FileNotFoundError)):c.collect(root/'out',snapshots=src)
|
||||||
|
self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
@@ -0,0 +1,146 @@
|
|||||||
|
import copy
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import mm2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class MM2026Test(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/mm2026-source.json.gz').read_bytes()))
|
||||||
|
cls.files = {name:body.encode('utf-8') for name,body in cls.fixture['files'].items()}
|
||||||
|
cls.raw = cls.files[c.SNAPSHOT+'/program-data.json']
|
||||||
|
cls.data = json.loads(cls.raw)
|
||||||
|
cls.gold = cls.fixture['independent_inventory']
|
||||||
|
|
||||||
|
def materialize(self, root):
|
||||||
|
for name,body in self.files.items():
|
||||||
|
path = root/name; path.parent.mkdir(parents=True,exist_ok=True); path.write_bytes(body)
|
||||||
|
|
||||||
|
def parse_changed(self, data):
|
||||||
|
return c.parse_program(json.dumps(data,ensure_ascii=False).encode())
|
||||||
|
|
||||||
|
def test_all2143_ids_and4326_locations_against_independent_inventory(self):
|
||||||
|
rows,sessions = c.parse_program(self.raw)
|
||||||
|
self.assertEqual(len(rows),2143); self.assertEqual(len(sessions),157)
|
||||||
|
self.assertEqual(sum(len(r['contexts']) for r in rows),4326)
|
||||||
|
actual = {r['paper_id']:r for r in rows}
|
||||||
|
self.assertEqual(set(actual),{g['paper_id'] for g in self.gold['inventory']})
|
||||||
|
for gold in self.gold['inventory']:
|
||||||
|
r = actual[gold['paper_id']]
|
||||||
|
self.assertEqual((r['title'],r['authors'],r['track'],r['affiliations']),
|
||||||
|
(gold['title'],gold['authors'],gold['track_literal'],gold['affiliations']))
|
||||||
|
self.assertEqual(hashlib.sha256(r['abstract'].encode()).hexdigest(),gold['abstract_sha256'])
|
||||||
|
self.assertEqual([x['source_pointer'] for x in r['contexts']],gold['locations'])
|
||||||
|
|
||||||
|
def test_every_presentation_and_session_context_is_preserved(self):
|
||||||
|
rows,sessions = c.parse_program(self.raw)
|
||||||
|
for r in rows:
|
||||||
|
for context in r['contexts']:
|
||||||
|
node = self.data
|
||||||
|
for key in context['source_pointer'].split('/')[1:]:
|
||||||
|
node = node[int(key)] if isinstance(node,list) else node[key]
|
||||||
|
self.assertEqual(context['presentation'],{k:node[k] for k in c.PRESENTATION if k in node})
|
||||||
|
self.assertEqual(r['paper_id'],node['paper_id'])
|
||||||
|
for i,session in enumerate(sessions):
|
||||||
|
original = self.data['entries'][i]
|
||||||
|
self.assertEqual(session['source_pointer'],'/entries/'+str(i))
|
||||||
|
self.assertEqual(session['track'],'')
|
||||||
|
for key in original.keys()-{'papers','subsessions'}:
|
||||||
|
self.assertEqual(session[key],original[key])
|
||||||
|
for j,sub in enumerate(original.get('subsessions') or []):
|
||||||
|
saved = session['subsessions'][j]
|
||||||
|
self.assertEqual(saved['title'],sub['title'])
|
||||||
|
expected = [x for x in sub.get('items') or [] if 'paper_id' not in x]
|
||||||
|
self.assertEqual([x['fields'] for x in saved['nonpaper_items']],expected)
|
||||||
|
|
||||||
|
def test_all_tracks_retained_without_publication_or_negative_inference(self):
|
||||||
|
rows,_ = c.parse_program(self.raw)
|
||||||
|
self.assertEqual({r['track'] for r in rows},set(self.gold['unique_id_track_counts']))
|
||||||
|
for r in rows:
|
||||||
|
self.assertFalse(r['approved_for_import']); self.assertFalse(r['final_decision_verified'])
|
||||||
|
self.assertFalse(r['publication_identity_verified'])
|
||||||
|
self.assertEqual(r['source_locator'],{'namespace':'mm_program:program_id','value':r['paper_id']})
|
||||||
|
self.assertEqual(r['tags'],[{'namespace':'mm_program:track','value':r['track']}])
|
||||||
|
self.assertEqual(r['observed_status'],'listed_in_official_program')
|
||||||
|
self.assertTrue({'status','identifiers','submission_identifiers','paper_type','emails'}.isdisjoint(r))
|
||||||
|
missing = [r for r in rows if not r['authors']]
|
||||||
|
self.assertEqual({r['paper_id'] for r in missing},{'mfp4971','mfp7801'})
|
||||||
|
self.assertTrue(all(r['missing_fields']==['authors','abstract'] for r in missing))
|
||||||
|
self.assertEqual(sum(not r['abstract'] for r in rows),216)
|
||||||
|
|
||||||
|
def test_collect_evidence_hashes_and_no_contact_arrays_in_normalized_output(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);self.materialize(root/'src')
|
||||||
|
report=c.collect(root/'out',snapshots=root/'src')
|
||||||
|
self.assertEqual((report['records'],report['program_observations'],report['extra_presentation_observations']),(2143,4326,2183))
|
||||||
|
self.assertFalse(report['database_written']);self.assertFalse(report['pdfs_downloaded'])
|
||||||
|
self.assertEqual(report['network_requests'],0);self.assertEqual(report['approved_for_import'],0)
|
||||||
|
self.assertEqual(report['public_negative_access']['http_statuses'],[403]*4)
|
||||||
|
self.assertEqual(report['track_counts'],self.gold['unique_id_track_counts'])
|
||||||
|
for name,body in self.files.items():
|
||||||
|
self.assertEqual((root/'out/evidence'/Path(name).name).read_bytes(),body)
|
||||||
|
n=0
|
||||||
|
for item in report['files']:
|
||||||
|
raw=Path(item['file']).read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),item['file_sha256'])
|
||||||
|
self.assertNotIn(b'"emails"',raw);self.assertEqual(len(raw.splitlines()),item['count']);n+=item['count']
|
||||||
|
self.assertEqual(item['format'],'mm_program_observation_v1_not_ingestion_record')
|
||||||
|
self.assertEqual(n,2143)
|
||||||
|
raw=(root/'out/sessions.jsonl').read_bytes()
|
||||||
|
self.assertEqual(hashlib.sha256(raw).hexdigest(),report['sessions_file_sha256']);self.assertNotIn(b'"emails"',raw)
|
||||||
|
with self.assertRaisesRegex(ValueError,'Output exists'):c.collect(root/'out',snapshots=root/'src')
|
||||||
|
|
||||||
|
def test_conflicting_core_version_is_not_silently_merged(self):
|
||||||
|
data=copy.deepcopy(self.data)
|
||||||
|
first=data['entries'][0]['papers'][0]
|
||||||
|
data['entries'][0]['papers'].append({**first,'abstract':'conflicting abstract'})
|
||||||
|
with self.assertRaisesRegex(ValueError,'Conflicting core'):self.parse_changed(data)
|
||||||
|
|
||||||
|
def test_unknown_nesting_cannot_silently_lose_observations(self):
|
||||||
|
data=copy.deepcopy(self.data)
|
||||||
|
moved=data['entries'][0]['papers'].pop()
|
||||||
|
data['entries'][0]['unreviewed_bucket']=[moved]
|
||||||
|
with self.assertRaisesRegex(ValueError,'Unhandled'):self.parse_changed(data)
|
||||||
|
|
||||||
|
def test_track_drift_unknown_fields_or_missing_title_require_review(self):
|
||||||
|
for kind in ('outer_track','paper_track','field','title','authors'):
|
||||||
|
data=copy.deepcopy(self.data);paper=data['entries'][0]['papers'][0]
|
||||||
|
if kind=='outer_track':data['entries'][0]['track']='Main Track'
|
||||||
|
elif kind=='paper_track':paper['track']='New Track'
|
||||||
|
elif kind=='field':paper['doi']='unreviewed'
|
||||||
|
elif kind=='title':paper['title']=''
|
||||||
|
else:paper['authors']='Unparsed author string'
|
||||||
|
with self.subTest(kind=kind),self.assertRaises(ValueError):self.parse_changed(data)
|
||||||
|
|
||||||
|
def test_edition_or_census_change_requires_review(self):
|
||||||
|
for kind in ('edition','count'):
|
||||||
|
data=copy.deepcopy(self.data)
|
||||||
|
if kind=='edition':data['generated_at']='2025-11-14T18:36:26.860677+00:00'
|
||||||
|
else:data['entries'][0]['papers'].pop()
|
||||||
|
with self.subTest(kind=kind),self.assertRaises(ValueError):self.parse_changed(data)
|
||||||
|
|
||||||
|
def test_corruption_missing_evidence_and_symlink_fail_without_completed_report(self):
|
||||||
|
for kind in ('manifest','program','probe','missing','symlink'):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);self.materialize(root/'src');directory=root/'src'/c.SNAPSHOT
|
||||||
|
name='manifest.json' if kind=='manifest' else 'openreview-rejected-probe.json' if kind=='probe' else 'program-data.json'
|
||||||
|
path=directory/name
|
||||||
|
if kind=='missing':path.unlink()
|
||||||
|
elif kind=='symlink':
|
||||||
|
original=directory/'original.json';path.rename(original);path.symlink_to(original)
|
||||||
|
else:path.write_bytes(path.read_bytes()+b' ')
|
||||||
|
with self.subTest(kind=kind),self.assertRaises((ValueError,FileNotFoundError)):c.collect(root/'out',snapshots=root/'src')
|
||||||
|
self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_literal_abstract_math_and_nonpaper_session_events_retained(self):
|
||||||
|
rows,sessions=c.parse_program(self.raw)
|
||||||
|
self.assertTrue(any('\\' in r['abstract'] for r in rows))
|
||||||
|
self.assertTrue(any('\n' in r['abstract'] for r in rows))
|
||||||
|
self.assertTrue(any(s['type']=='coffee_break' for s in sessions))
|
||||||
|
self.assertTrue(any(s['type']=='industry_talk' for s in sessions))
|
||||||
|
self.assertEqual(c.without_emails({'emails':['contact'],'items':[{'emails':['contact'],'title':'talk'}]}),{'items':[{'title':'talk'}]})
|
||||||
@@ -0,0 +1,273 @@
|
|||||||
|
from contextlib import redirect_stdout
|
||||||
|
import io
|
||||||
|
import os
|
||||||
|
import uuid
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import mm_publications2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class MMPublications2026Tests(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
fixtures=Path(__file__).parent/'fixtures'
|
||||||
|
cls.fixture=json.loads(gzip.decompress((fixtures/'mm2026-publications.json.gz').read_bytes()))
|
||||||
|
cls.program_fixture=json.loads(gzip.decompress((fixtures/'mm2026-source.json.gz').read_bytes()))
|
||||||
|
cls.format_gold=json.loads((fixtures/'mm2026-title-formatting-review.json').read_text())
|
||||||
|
overrides={g['doi']:g for g in [*cls.format_gold['gold'],cls.format_gold['held_gold']]}
|
||||||
|
cls.gold=[overrides.get(g['doi'],g) for g in cls.fixture['independent_gold']]
|
||||||
|
cls.temp=tempfile.TemporaryDirectory();cls.root=Path(cls.temp.name)
|
||||||
|
cls.materialize(cls.root/'registry',cls.fixture['files'])
|
||||||
|
cls.materialize(cls.root/'program',cls.program_fixture['files'])
|
||||||
|
cls.out=cls.root/'out'
|
||||||
|
cls.report=c.collect(cls.out,registry_snapshots=cls.root/'registry',program_snapshots=cls.root/'program',parent_snapshot=cls.root/'registry/parent.json')
|
||||||
|
cls.rows=[json.loads(line) for f in cls.report['files'] for line in Path(f['file']).read_bytes().splitlines()]
|
||||||
|
cls.by_doi={r['external_id']:r for r in cls.rows}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls):
|
||||||
|
cls.temp.cleanup()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def materialize(root,files):
|
||||||
|
for name,body in files.items():
|
||||||
|
path=root/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(body.encode('utf-8'))
|
||||||
|
|
||||||
|
def test_all1918_publications_against_independent_registry_program_gold(self):
|
||||||
|
self.assertEqual(len(self.rows),1918);self.assertEqual(len(self.by_doi),1918)
|
||||||
|
self.assertEqual(set(self.by_doi),{g['doi'] for g in self.gold})
|
||||||
|
for g in self.gold:
|
||||||
|
r=self.by_doi[g['doi']];m=r['metadata'];p=m['program_fields']
|
||||||
|
self.assertEqual((r['title'],r['authors'],r['track']),(g['title'],g['authors'],g['track']))
|
||||||
|
self.assertEqual(m['program_match_kind'],g['kind'])
|
||||||
|
self.assertEqual(m['registry_source_evidence']['json_pointer'],'/message/items/'+str(g['source_position']))
|
||||||
|
self.assertEqual(m['registry_source_evidence']['file'],'evidence/registry/'+g['source_page'])
|
||||||
|
self.assertEqual(m['registry_source_evidence']['canonical_object_json_sha256'],g['registry_object_sha256'])
|
||||||
|
self.assertEqual(p['paper_id'] if p else None,g['program_id'])
|
||||||
|
self.assertEqual([x['source_pointer'] for x in p['contexts']] if p else [],g['program_context_pointers'])
|
||||||
|
|
||||||
|
def test_abstract_source_is_separate_and_verbatim_not_fabricated_registry_abstract(self):
|
||||||
|
count=0
|
||||||
|
for g in self.gold:
|
||||||
|
r=self.by_doi[g['doi']];m=r['metadata']
|
||||||
|
self.assertFalse(m['registry_abstract_available']);self.assertFalse(m['registry_fields'].get('abstract'))
|
||||||
|
if g['abstract_sha256'] is not None:
|
||||||
|
count+=1;self.assertEqual(hashlib.sha256(r['abstract'].encode()).hexdigest(),g['abstract_sha256'])
|
||||||
|
self.assertEqual(r['abstract'],m['program_fields']['abstract'])
|
||||||
|
self.assertEqual(m['abstract_source'],'official_program_reviewed_title_formatting_correspondence' if g['kind'].startswith('unique_program_format_') else 'official_program_exact_metadata_correspondence')
|
||||||
|
else:
|
||||||
|
self.assertIsNone(r['abstract']);self.assertEqual(m['abstract_source'],'not_available_from_reviewed_sources')
|
||||||
|
self.assertEqual(count,1743);self.assertEqual(self.report['output_missing_abstract_count'],175)
|
||||||
|
|
||||||
|
def test_all_program_tracks_and_4326_contexts_survive_in_matches_or_unmatched(self):
|
||||||
|
programs={r['metadata']['program_fields']['paper_id']:r['metadata']['program_fields'] for r in self.rows if r['metadata']['program_fields']}
|
||||||
|
for p in self.report['unmatched_program']:
|
||||||
|
self.assertNotIn(p['paper_id'],programs);programs[p['paper_id']]=p
|
||||||
|
self.assertEqual(len(programs),2143);self.assertEqual(self.report['unmatched_program_count'],391)
|
||||||
|
gold=self.program_fixture['independent_inventory']['inventory']
|
||||||
|
self.assertEqual(set(programs),{g['paper_id'] for g in gold})
|
||||||
|
for g in gold:
|
||||||
|
p=programs[g['paper_id']]
|
||||||
|
self.assertEqual((p['title'],p['authors'],p['track']),(g['title'],g['authors'],g['track_literal']))
|
||||||
|
self.assertEqual([x['source_pointer'] for x in p['contexts']],g['locations'])
|
||||||
|
self.assertEqual(sum(len(p['contexts']) for p in programs.values()),4326)
|
||||||
|
self.assertEqual(len({p['track'] for p in programs.values()}),28)
|
||||||
|
|
||||||
|
def test_no_person_alias_original_submission_decision_or_research_type_inference(self):
|
||||||
|
for r in self.rows:
|
||||||
|
doi=r['external_id'];m=r['metadata']
|
||||||
|
self.assertEqual(r['identifiers'],{'doi':doi})
|
||||||
|
self.assertEqual(r['submission_identifiers'],[{'scheme':'publication_doi','value':doi}])
|
||||||
|
self.assertEqual((r['source'],r['venue'],r['year']),(c.SOURCE,'MM',2026))
|
||||||
|
self.assertEqual(r['status'],'accepted');self.assertIn('not an original signed decision',r['raw_decision'])
|
||||||
|
self.assertFalse(m['original_decision_signature_verified']);self.assertFalse(m['research_type_classified'])
|
||||||
|
self.assertFalse(m['scientific_work_identity_approved'])
|
||||||
|
self.assertTrue({'decided_at','decision_date','source_decision_id','paper_type'}.isdisjoint(r))
|
||||||
|
self.assertIn({'namespace':c.SOURCE+':crossref:type','value':'proceedings-article'},r['tags'])
|
||||||
|
self.assertTrue({x['namespace'] for x in r['tags']} <= {c.SOURCE+':crossref:type',c.SOURCE+':mm_program:track'})
|
||||||
|
for track in ('interactive_art','industry_talk','demo_and_video'):
|
||||||
|
self.assertTrue(any(r['track']==track for r in self.rows))
|
||||||
|
variants=[r for r in self.rows if r['track']=='proceedings_unclassified']
|
||||||
|
self.assertEqual(len(variants),166)
|
||||||
|
self.assertTrue(all(r['metadata']['program_fields'] is None and r['abstract'] is None for r in variants))
|
||||||
|
|
||||||
|
def test_original_bibliography_evidence_kept_without_duplicating_reference_payload(self):
|
||||||
|
authors_by_page={}
|
||||||
|
for name,body in self.fixture['files'].items():
|
||||||
|
actual=self.out/'evidence/registry'/('parent.json' if name=='parent.json' else Path(name).name)
|
||||||
|
self.assertEqual(actual.read_bytes(),body.encode())
|
||||||
|
if re.fullmatch(r'page-[0-9]+\.json',Path(name).name):
|
||||||
|
authors_by_page[str(actual.relative_to(self.out))]=[row['author'] for row in json.loads(body)['message']['items']]
|
||||||
|
for name,body in self.program_fixture['files'].items():
|
||||||
|
self.assertEqual((self.out/'evidence/program'/Path(name).name).read_bytes(),body.encode())
|
||||||
|
for r in self.rows:
|
||||||
|
m=r['metadata'];self.assertNotIn('reference',m['registry_fields'])
|
||||||
|
self.assertEqual(m['registry_fields']['title'],[r['title']])
|
||||||
|
self.assertEqual(m['registry_fields']['author'],authors_by_page[m['registry_source_evidence']['file']][int(m['registry_source_evidence']['json_pointer'].rsplit('/',1)[-1])])
|
||||||
|
for f in self.report['files']:
|
||||||
|
raw=Path(f['file']).read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),f['file_sha256'])
|
||||||
|
self.assertEqual(len(raw.splitlines()),f['count']);self.assertEqual(f['status'],'partial')
|
||||||
|
|
||||||
|
def test_eight_single_name_fields_preserved_without_splitting(self):
|
||||||
|
n=0
|
||||||
|
for g in self.gold:
|
||||||
|
r=self.by_doi[g['doi']];flags=r['metadata']['review_flags']
|
||||||
|
actual=[x['author_position'] for x in flags if x['kind']=='single_registered_name_field_not_split']
|
||||||
|
self.assertEqual(actual,g['single_name_positions']);n+=len(actual)
|
||||||
|
for i in actual:
|
||||||
|
self.assertEqual(r['authors'][i-1],r['metadata']['registry_fields']['author'][i-1]['family'].strip())
|
||||||
|
self.assertEqual(n,8)
|
||||||
|
|
||||||
|
def test_structured_title_markup_retained_and_flagged_without_flattening(self):
|
||||||
|
count=0
|
||||||
|
for g in self.gold:
|
||||||
|
if re.search(r'<[^>]+>|&[A-Za-z#0-9]+;',g['title']):
|
||||||
|
count+=1;r=self.by_doi[g['doi']]
|
||||||
|
self.assertEqual(r['title'],g['title'])
|
||||||
|
self.assertIn({'kind':'literal_title_markup_or_entity_retained_requires_review'},r['metadata']['review_flags'])
|
||||||
|
self.assertGreater(count,0);self.assertEqual(count,self.report['registered_title_markup_flags'])
|
||||||
|
|
||||||
|
|
||||||
|
def test_formatting_gold_preserves_raw_fields_and_copies_exact_program_evidence(self):
|
||||||
|
self.assertEqual(len(self.format_gold['gold']),26)
|
||||||
|
for gold in self.format_gold['gold']:
|
||||||
|
row=self.by_doi[gold['doi']];m=row['metadata'];p=m['program_fields']
|
||||||
|
self.assertEqual((row['title'],row['authors']),(gold['title'],gold['authors']))
|
||||||
|
self.assertEqual((p['paper_id'],row['track']),(gold['program_id'],gold['track']))
|
||||||
|
self.assertEqual(hashlib.sha256(row['abstract'].encode()).hexdigest(),gold['abstract_sha256'])
|
||||||
|
comparison=m['program_title_formatting_comparison']
|
||||||
|
self.assertTrue(comparison['raw_titles_preserved']);self.assertFalse(comparison['work_identity_approved'])
|
||||||
|
self.assertEqual(set(comparison['program_comparison_texts']),{p['paper_id']})
|
||||||
|
self.assertFalse(m['scientific_work_identity_approved'])
|
||||||
|
held=self.by_doi[self.format_gold['author_disagreement_hold']['doi']]
|
||||||
|
self.assertEqual(held['metadata']['program_title_candidates'],['mfp7801'])
|
||||||
|
self.assertEqual(held['metadata']['program_exact_candidates'],[])
|
||||||
|
self.assertEqual(held['metadata']['program_match_kind'],'program_format_author_version_requires_review')
|
||||||
|
self.assertIsNone(held['abstract']);self.assertEqual(held['track'],'proceedings_unclassified')
|
||||||
|
|
||||||
|
def test_reviewed_formatting_keeps_superscript_subscript_and_decodes_entities_once(self):
|
||||||
|
self.assertEqual(c.formatting_text('<i>A</i> H<sup>2</sup> B<sub>1</sub> & C'),'A H2 B1 & C')
|
||||||
|
self.assertEqual(c.formatting_text('A &amp; B'),'A & B')
|
||||||
|
|
||||||
|
def test_unknown_attributes_unbalanced_or_encoded_markup_not_discarded(self):
|
||||||
|
for title in ['A <model>B</model>','A <i class="x">B</i>','A <i>B','A </i>B',
|
||||||
|
'A <sup/>B','A <!--hidden-->B','<!DOCTYPE html>A','A <i>B</i>',
|
||||||
|
'A <sup','A <i>B</sup>','<![CDATA[A]]>']:
|
||||||
|
with self.subTest(title=title):self.assertIsNone(c.formatting_text(title))
|
||||||
|
|
||||||
|
def test_ambiguous_program_and_author_multiset_differences_remain_held(self):
|
||||||
|
def observation(pid,authors):
|
||||||
|
return {'paper_id':pid,'title':'M3 Model','authors':authors,'track':'Main Track'}
|
||||||
|
rows=[{'title':'M<sup>3</sup> Model','authors':['A','B']}]
|
||||||
|
kinds,_=c.match_program(rows,[observation('p1',['A','B']),observation('p2',['B','A'])])
|
||||||
|
self.assertEqual(kinds,{'multiple_program_ids_same_title_authors_hold':1})
|
||||||
|
self.assertNotIn('program',rows[0])
|
||||||
|
rows=[{'title':'M<sup>3</sup> Model','authors':['A','A','B']}]
|
||||||
|
c.match_program(rows,[observation('p1',['A','B'])])
|
||||||
|
self.assertEqual(rows[0]['kind'],'program_format_author_version_requires_review')
|
||||||
|
self.assertNotIn('program',rows[0])
|
||||||
|
|
||||||
|
def test_literal_title_author_disagreement_does_not_use_formatting_bypass(self):
|
||||||
|
rows=[{'title':'M<sup>3</sup> Model','authors':['A','B']}]
|
||||||
|
observations=[{'paper_id':'literal','title':'M<sup>3</sup> Model','authors':['Other'],'track':'Main Track'},
|
||||||
|
{'paper_id':'formatted','title':'M3 Model','authors':['A','B'],'track':'Main Track'}]
|
||||||
|
c.match_program(rows,observations)
|
||||||
|
self.assertEqual(rows[0]['kind'],'program_author_version_requires_review')
|
||||||
|
self.assertEqual(rows[0]['program_title_candidates'],['literal']);self.assertNotIn('program',rows[0])
|
||||||
|
|
||||||
|
|
||||||
|
def test_corrupt_registry_parent_or_program_keeps_failure_not_complete_output(self):
|
||||||
|
for kind in ('registry_manifest','registry_page','parent','program'):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);self.materialize(root/'registry',self.fixture['files']);self.materialize(root/'program',self.program_fixture['files'])
|
||||||
|
path={'registry_manifest':root/'registry'/c.SNAPSHOT/'manifest.json','registry_page':root/'registry'/c.SNAPSHOT/'page-001.json','parent':root/'registry/parent.json','program':root/'program'/c.program.SNAPSHOT/'program-data.json'}[kind]
|
||||||
|
path.write_bytes(path.read_bytes()+b' ')
|
||||||
|
with self.subTest(kind=kind),self.assertRaises(ValueError):
|
||||||
|
c.collect(root/'out',registry_snapshots=root/'registry',program_snapshots=root/'program',parent_snapshot=root/'registry/parent.json')
|
||||||
|
self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_symlink_source_rejected_and_no_previous_output_overwrite(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);self.materialize(root/'registry',self.fixture['files'])
|
||||||
|
path=root/'registry/parent.json';original=root/'registry/actual.json';path.rename(original);path.symlink_to(original)
|
||||||
|
with self.assertRaisesRegex(ValueError,'Unsafe'):
|
||||||
|
c.collect(root/'out',registry_snapshots=root/'registry',program_snapshots=self.root/'program',parent_snapshot=path)
|
||||||
|
with self.assertRaisesRegex(ValueError,'Output exists'):
|
||||||
|
c.collect(self.out,registry_snapshots=self.root/'registry',program_snapshots=self.root/'program',parent_snapshot=self.root/'registry/parent.json')
|
||||||
|
|
||||||
|
def test_complete_report_retains_source_partitions_not_import_or_final_dedup_claim(self):
|
||||||
|
self.assertEqual(self.report['program_match_counts'],{'unique_program_title_fullauthor_agreement_not_identity_approval':1726,'unique_program_format_title_fullauthor_agreement_not_identity_approval':26,'program_author_version_requires_review':122,'program_format_author_version_requires_review':1,'no_exact_program_title_requires_review':43})
|
||||||
|
self.assertEqual(self.report['registry_missing_abstract_count'],1918)
|
||||||
|
self.assertEqual(self.report['abstracts_from_exact_program_correspondence'],1743)
|
||||||
|
self.assertEqual(self.report['abstracts_from_literal_program_correspondence'],1717)
|
||||||
|
self.assertEqual(self.report['abstracts_from_reviewed_title_formatting'],26)
|
||||||
|
self.assertEqual(self.report['unmatched_program_count'],391)
|
||||||
|
self.assertEqual(sum(f['count'] for f in self.report['files']),1918)
|
||||||
|
self.assertEqual(self.report['approved_for_import'],0);self.assertFalse(self.report['database_written'])
|
||||||
|
self.assertFalse(self.report['pdfs_downloaded']);self.assertEqual(self.report['network_requests'],0)
|
||||||
|
self.assertIn('unknown',self.report['public_negative_availability'])
|
||||||
|
|
||||||
|
|
||||||
|
def assert_native_ingestion(self, pg_service=None):
|
||||||
|
# An isolated synthetic database exercises the writer contract, not a
|
||||||
|
# production identity approval. All source rows/track partitions run.
|
||||||
|
from paper_atlas.db.store import Store
|
||||||
|
from paper_atlas.db.backend import open_store
|
||||||
|
from paper_atlas.ingestion.records import ingest_file
|
||||||
|
from paper_atlas.postgres.client import connect
|
||||||
|
from paper_atlas.postgres.migration import migrate
|
||||||
|
from paper_atlas.postgres.schema import quote
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);db=root/'seed.sqlite';seed=self.rows[0]
|
||||||
|
store=Store(db)
|
||||||
|
paper,record=store.record('arxiv','synthetic-mm-anchor','https://example.org/arxiv',
|
||||||
|
seed['title'],'Existing canonical abstract',seed['authors'],seed['identifiers'])
|
||||||
|
store.decision(paper,record,'ICLR',2025,'main','synthetic-old-forum',
|
||||||
|
'https://example.org/decision','rejected','Reject')
|
||||||
|
before=tuple(store.conn.execute('SELECT * FROM papers WHERE id=?',(paper,)).fetchone())
|
||||||
|
store.conn.commit();store.conn.close()
|
||||||
|
schema='pa_mm_pub_'+uuid.uuid4().hex[:12] if pg_service else None
|
||||||
|
try:
|
||||||
|
with redirect_stdout(io.StringIO()):
|
||||||
|
if pg_service:migrate(db,pg_service,root/'migration.json',schema=schema,min_free_gib=0)
|
||||||
|
for f in self.report['files']:
|
||||||
|
ingest_file(None if pg_service else db,f['file'],pg_service=pg_service,pg_schema=schema)
|
||||||
|
store=open_store(None if pg_service else db,pg_service=pg_service,pg_schema=schema)
|
||||||
|
try:
|
||||||
|
conn=store.conn
|
||||||
|
self.assertEqual(tuple(conn.execute('SELECT * FROM papers WHERE id=?',(paper,)).fetchone()),before)
|
||||||
|
self.assertEqual(conn.execute('SELECT status FROM decisions WHERE submission_id=1').fetchone()[0],'rejected')
|
||||||
|
actual={r[0]:r for r in conn.execute("SELECT sr.external_id,sr.paper_id,sr.title,sr.abstract,sr.authors_json,sr.metadata_json,sr.id FROM source_records sr JOIN sources s ON s.id=sr.source_id WHERE s.name=?",(c.SOURCE,))}
|
||||||
|
self.assertEqual(set(actual),set(self.by_doi))
|
||||||
|
self.assertEqual(actual[seed['external_id']][1],paper)
|
||||||
|
for doi,row in self.by_doi.items():
|
||||||
|
got=actual[doi];self.assertEqual((got[2],got[3],json.loads(got[4])),(row['title'],row['abstract'],row['authors']))
|
||||||
|
meta=json.loads(got[5]);self.assertEqual(meta.pop('record_identifiers'),{'doi':doi,c.SOURCE:doi});self.assertEqual(meta,row['metadata'])
|
||||||
|
tags=conn.execute("SELECT t.namespace,t.value,count(*) FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id GROUP BY t.namespace,t.value").fetchall()
|
||||||
|
self.assertEqual(sum(r[2] for r in tags if r[0]==c.SOURCE+':crossref:type'),1918)
|
||||||
|
self.assertEqual(sum(r[2] for r in tags if r[0]==c.SOURCE+':mm_program:track'),1752)
|
||||||
|
self.assertEqual({r[0] for r in tags},{c.SOURCE+':crossref:type',c.SOURCE+':mm_program:track'})
|
||||||
|
decisions=conn.execute("SELECT d.status,d.decision_date,count(*) FROM decisions d JOIN submissions s ON s.id=d.submission_id JOIN source_records r ON r.id=s.source_record_id JOIN sources src ON src.id=r.source_id WHERE src.name=? GROUP BY d.status,d.decision_date",(c.SOURCE,)).fetchall()
|
||||||
|
self.assertEqual([tuple(r) for r in decisions],[('accepted',None,1918)])
|
||||||
|
self.assertEqual(conn.execute("SELECT count(*) FROM submission_identifiers WHERE scheme='publication_doi'").fetchone()[0],1918)
|
||||||
|
coverage=conn.execute("SELECT es.status,sum(es.imported_count) FROM edition_sources es JOIN sources s ON s.id=es.source_id WHERE s.name=? GROUP BY es.status",(c.SOURCE,)).fetchall()
|
||||||
|
self.assertEqual([tuple(r) for r in coverage],[('partial',1918)])
|
||||||
|
finally:store.conn.close()
|
||||||
|
finally:
|
||||||
|
if pg_service:
|
||||||
|
with connect(pg_service) as pg:
|
||||||
|
for name, in pg.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)',(schema,)).fetchall():
|
||||||
|
pg.execute('DROP SCHEMA '+quote(name)+' CASCADE')
|
||||||
|
|
||||||
|
def test_all1918_native_sqlite_source_tags_and_prior_rejection_survive(self):
|
||||||
|
self.assert_native_ingestion()
|
||||||
|
|
||||||
|
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'),'Requires approved remote PostgreSQL test service')
|
||||||
|
def test_all1918_native_postgres_source_tags_and_prior_rejection_survive(self):
|
||||||
|
self.assert_native_ingestion(os.environ['PAPER_ATLAS_TEST_PG_SERVICE'])
|
||||||
@@ -0,0 +1,101 @@
|
|||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from paper_atlas.collectors import mm2026
|
||||||
|
from paper_atlas.audits.source_identity import audit, load_inputs, load_mm_program_inputs
|
||||||
|
|
||||||
|
|
||||||
|
class MMProgramIdentityInputTests(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/mm2026-source.json.gz').read_bytes()))
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.addCleanup(self.tmp.cleanup)
|
||||||
|
self.root = Path(self.tmp.name)
|
||||||
|
src = self.root/'source'
|
||||||
|
for name,body in self.fixture['files'].items():
|
||||||
|
path=src/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(body.encode())
|
||||||
|
mm2026.collect(self.root/'program', snapshots=src)
|
||||||
|
self.report_path = self.root/'program/collection-report.json'
|
||||||
|
|
||||||
|
def change_report(self, change):
|
||||||
|
report=json.loads(self.report_path.read_text());change(report)
|
||||||
|
self.report_path.write_text(json.dumps(report))
|
||||||
|
|
||||||
|
def assert_rejected_before_database(self):
|
||||||
|
with patch('paper_atlas.audits.source_identity.ReadConnection') as connection:
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
audit(None,self.root/'audit',pg_service='unused',pg_schema='research',mm_program_report=self.report_path)
|
||||||
|
connection.assert_not_called()
|
||||||
|
self.assertFalse((self.root/'audit').exists())
|
||||||
|
|
||||||
|
def test_all2143_programs_are_unknown_readonly_inputs_not_import_records(self):
|
||||||
|
rows,inputs=load_mm_program_inputs(self.report_path)
|
||||||
|
self.assertEqual(len(rows),2143);self.assertEqual(len(inputs),28)
|
||||||
|
self.assertEqual(sum(len(r['program_contexts']) for r in rows),4326)
|
||||||
|
actual={r['program_id']:r for r in rows}
|
||||||
|
for gold in self.fixture['independent_inventory']['inventory']:
|
||||||
|
r=actual[gold['paper_id']]
|
||||||
|
self.assertEqual((r['title'],r['authors'],r['track']),
|
||||||
|
(gold['title'],gold['authors'],gold['track_literal']))
|
||||||
|
self.assertEqual(r['external_id'],'2026/'+gold['paper_id'])
|
||||||
|
self.assertEqual(r['status'],'unknown');self.assertEqual(r['identifiers'],{})
|
||||||
|
self.assertEqual(r['source_observed_status'],'listed_in_official_program')
|
||||||
|
self.assertFalse(r['publication_identity_verified']);self.assertFalse(r['final_decision_verified'])
|
||||||
|
self.assertFalse(r['contribution_type_verified'])
|
||||||
|
self.assertEqual([c['source_pointer'] for c in r['program_contexts']],gold['locations'])
|
||||||
|
self.assertEqual({r['program_id'] for r in rows if not r['authors']},{'mfp4971','mfp7801'})
|
||||||
|
self.assertTrue(all('unknown is not a source decision' in i['manifest']['scope'] for i in inputs))
|
||||||
|
self.assertFalse(any(self.root.rglob('*.manifest.json')))
|
||||||
|
|
||||||
|
def test_untrusted_absolute_file_paths_are_not_followed(self):
|
||||||
|
self.change_report(lambda r:[f.update(file='/untrusted/'+Path(f['file']).name) for f in r['files']])
|
||||||
|
rows,inputs=load_mm_program_inputs(self.report_path)
|
||||||
|
self.assertEqual(len(rows),2143)
|
||||||
|
self.assertTrue(all(Path(i['file']).parent==self.report_path.parent for i in inputs))
|
||||||
|
|
||||||
|
def test_program_files_are_not_silently_accepted_by_regular_ingestion_input_loader(self):
|
||||||
|
report=json.loads(self.report_path.read_text())
|
||||||
|
with self.assertRaises(FileNotFoundError):load_inputs([report['files'][0]['file']])
|
||||||
|
|
||||||
|
def test_corrupt_jsonl_fails_before_connection_or_output(self):
|
||||||
|
report=json.loads(self.report_path.read_text());path=Path(report['files'][0]['file'])
|
||||||
|
path.write_bytes(path.read_bytes()+b' ')
|
||||||
|
self.assert_rejected_before_database()
|
||||||
|
|
||||||
|
def test_rehashed_forged_identifiers_decisions_and_contexts_rejected(self):
|
||||||
|
original=self.report_path.read_bytes();report=json.loads(original);path=Path(report['files'][0]['file']);raw=path.read_bytes()
|
||||||
|
for field,value in [('identifiers',{'openreview':'invented'}),('status','accepted'),('contexts',[])]:
|
||||||
|
path.write_bytes(raw);self.report_path.write_bytes(original)
|
||||||
|
lines=raw.splitlines();row=json.loads(lines[0]);row[field]=value;lines[0]=json.dumps(row).encode()
|
||||||
|
changed=b'\n'.join(lines)+b'\n';path.write_bytes(changed)
|
||||||
|
self.change_report(lambda r:r['files'][0].update(file_sha256=hashlib.sha256(changed).hexdigest()))
|
||||||
|
with self.subTest(field=field):self.assert_rejected_before_database()
|
||||||
|
|
||||||
|
def test_missing_or_repeated_track_rejected(self):
|
||||||
|
original=self.report_path.read_bytes()
|
||||||
|
for kind in ('missing','repeated'):
|
||||||
|
self.report_path.write_bytes(original)
|
||||||
|
if kind=='missing':self.change_report(lambda r:r['files'].pop())
|
||||||
|
else:self.change_report(lambda r:r['files'].__setitem__(1,r['files'][0]))
|
||||||
|
with self.subTest(kind=kind):self.assert_rejected_before_database()
|
||||||
|
|
||||||
|
def test_raw_source_corruption_or_symlink_rejected(self):
|
||||||
|
path=self.report_path.parent/'evidence/program-data.json';raw=path.read_bytes()
|
||||||
|
path.write_bytes(raw+b' ');self.assert_rejected_before_database()
|
||||||
|
path.write_bytes(raw);saved=path.with_name('original.json');path.rename(saved);path.symlink_to(saved)
|
||||||
|
self.assert_rejected_before_database()
|
||||||
|
|
||||||
|
def test_mixed_input_modes_rejected_before_connection(self):
|
||||||
|
with patch('paper_atlas.audits.source_identity.ReadConnection') as connection:
|
||||||
|
for files,program in [([],None),([Path('unused')],self.report_path)]:
|
||||||
|
with self.assertRaisesRegex(ValueError,'Choose source files'):
|
||||||
|
audit(files,self.root/'out',pg_service='unused',pg_schema='research',mm_program_report=program)
|
||||||
|
connection.assert_not_called()
|
||||||
@@ -15,7 +15,7 @@ from paper_atlas.db.backend import open_store, open_read_connection
|
|||||||
from paper_atlas.db.store import Store
|
from paper_atlas.db.store import Store
|
||||||
from paper_atlas.ingestion import arxiv
|
from paper_atlas.ingestion import arxiv
|
||||||
from paper_atlas.ingestion.backfill import backfill
|
from paper_atlas.ingestion.backfill import backfill
|
||||||
from paper_atlas.ingestion.records import ingest_file
|
from paper_atlas.ingestion.records import coverage_options, ingest_file, require_incremental_parent
|
||||||
from paper_atlas.postgres.client import connect, ReadConnection
|
from paper_atlas.postgres.client import connect, ReadConnection
|
||||||
from paper_atlas.postgres.migration import migrate
|
from paper_atlas.postgres.migration import migrate
|
||||||
from paper_atlas.postgres.schema import quote
|
from paper_atlas.postgres.schema import quote
|
||||||
@@ -114,6 +114,36 @@ class PostgreSQLIngestionTests(unittest.TestCase):
|
|||||||
self.assertEqual(run[0],1)
|
self.assertEqual(run[0],1)
|
||||||
self.assertEqual(json.loads(run[1])['coverage']['imported_count'],3)
|
self.assertEqual(json.loads(run[1])['coverage']['imported_count'],3)
|
||||||
|
|
||||||
|
def test_incremental_parent_preflight_uses_readonly_postgres_rows_and_keeps_guards(self):
|
||||||
|
rows = [self.row(source='dblp', external=key, status='accepted', year=2001, venue='UAI')
|
||||||
|
for key in ('preflight1', 'preflight2')]
|
||||||
|
path = self.file('preflight-parent', rows)
|
||||||
|
target = path.with_suffix('.manifest.json')
|
||||||
|
manifest = json.loads(target.read_text())
|
||||||
|
manifest['status'] = 'partial'
|
||||||
|
target.write_text(json.dumps(manifest))
|
||||||
|
self.ingest(path)
|
||||||
|
proposed = dict(manifest, coverage_mode='incremental', count=1,
|
||||||
|
coverage_previous_count=2, coverage_previous_expected_count=2,
|
||||||
|
coverage_expected_count=4)
|
||||||
|
with closing(ReadConnection(self.service, self.schema)) as c:
|
||||||
|
self.assertEqual(c.execute('SHOW transaction_read_only').fetchone()[0], 'on')
|
||||||
|
source = c.execute("SELECT id FROM sources WHERE name='dblp'").fetchone()[0]
|
||||||
|
before = {table: c.execute('SELECT count(*) FROM '+table).fetchone()[0]
|
||||||
|
for table in ('papers', 'source_records', 'submissions', 'decisions', 'ingest_runs')}
|
||||||
|
coverage = [dict(row) for row in c.execute('SELECT * FROM edition_sources ORDER BY edition_id,source_id,list_url')]
|
||||||
|
require_incremental_parent(c, source, proposed, coverage_options(proposed))
|
||||||
|
for changes, error in (({'coverage_previous_count': 1}, 'parent changed'),
|
||||||
|
({'coverage_previous_expected_count': 3}, 'parent changed'),
|
||||||
|
({'list_url': 'https://example.org/other-list'}, 'unambiguous source listing')):
|
||||||
|
with self.subTest(changes=changes):
|
||||||
|
stale = dict(proposed, **changes)
|
||||||
|
with self.assertRaisesRegex(ValueError, error):
|
||||||
|
require_incremental_parent(c, source, stale, coverage_options(stale))
|
||||||
|
self.assertEqual({table: c.execute('SELECT count(*) FROM '+table).fetchone()[0]
|
||||||
|
for table in before}, before)
|
||||||
|
self.assertEqual([dict(row) for row in c.execute('SELECT * FROM edition_sources ORDER BY edition_id,source_id,list_url')], coverage)
|
||||||
|
|
||||||
def test_incremental_postgres_bound_failure_rolls_back_rows_and_summary(self):
|
def test_incremental_postgres_bound_failure_rolls_back_rows_and_summary(self):
|
||||||
listing='https://example.org/uai/2001'
|
listing='https://example.org/uai/2001'
|
||||||
rows=[self.row(source='dblp',external=key,status='accepted',year=2001,venue='UAI')
|
rows=[self.row(source='dblp',external=key,status='accepted',year=2001,venue='UAI')
|
||||||
|
|||||||
@@ -0,0 +1,79 @@
|
|||||||
|
import base64
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import recsys2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class RecSys2026Test(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture=json.loads(gzip.decompress((Path(__file__).parent/'fixtures/recsys2026-source.json.gz').read_bytes()))
|
||||||
|
cls.files={n:base64.b64decode(raw) for n,raw in cls.fixture['files'].items()}
|
||||||
|
cls.body=cls.files[c.SNAPSHOT+'/recsys-contributions.html']
|
||||||
|
|
||||||
|
def materialize(self,root):
|
||||||
|
for name,raw in self.files.items():
|
||||||
|
path=root/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(raw)
|
||||||
|
|
||||||
|
def test_all263_observations_against_independent_gold(self):
|
||||||
|
items=c.parse_program(self.body)
|
||||||
|
keys=self.fixture['independent_gold'][0].keys()
|
||||||
|
self.assertEqual([{k:r[k] for k in keys} for r in items],self.fixture['independent_gold'])
|
||||||
|
self.assertEqual(len(items),263);self.assertEqual(len({r['doi'] for r in items}),260)
|
||||||
|
self.assertEqual(sum(bool(r['review_flags']) for r in items),6)
|
||||||
|
variants=[r for r in items if r['doi']=='10.1145/3773078.3831839']
|
||||||
|
self.assertNotEqual(variants[0]['title'],variants[1]['title']);self.assertNotEqual(variants[0]['authors'],variants[1]['authors'])
|
||||||
|
self.assertNotEqual(variants[0]['abstract'],variants[1]['abstract'])
|
||||||
|
|
||||||
|
def test_installed_format_tracks_evidence_and_original_fields(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);src=root/'src';self.materialize(src);out=root/'out'
|
||||||
|
report=c.collect(out,snapshots=src)
|
||||||
|
self.assertEqual((report['records'],report['unique_dois'],report['missing_abstract_count']),(263,260,0))
|
||||||
|
self.assertFalse(report['database_written']);self.assertFalse(report['pdfs_downloaded']);self.assertEqual(report['network_requests'],0);self.assertEqual(report['approved_for_import'],0)
|
||||||
|
self.assertEqual(report['track_counts'],{k:n for k,_,n in c.TRACKS});self.assertEqual(report['duplicate_doi_observations'],6)
|
||||||
|
gold={r['locator']:r for r in self.fixture['independent_gold']};seen=set()
|
||||||
|
for manifest in report['files']:
|
||||||
|
path=Path(manifest['file']);raw=path.read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),manifest['file_sha256'])
|
||||||
|
rows=[json.loads(l) for l in raw.splitlines()];self.assertEqual(len(rows),manifest['count']);self.assertEqual(manifest['status'],'partial')
|
||||||
|
for r in rows:
|
||||||
|
item=r['metadata']['program_fields'];locator=item['locator'];seen.add(locator);expected=gold[locator]
|
||||||
|
for key in expected:self.assertEqual(item[key],expected[key])
|
||||||
|
self.assertEqual(r['external_id'],'2026/'+locator);self.assertEqual(r['title'],expected['title']);self.assertEqual(r['authors'],expected['authors']);self.assertEqual(r['abstract'],expected['abstract'])
|
||||||
|
self.assertEqual(r['track'],manifest['track']);self.assertEqual(r['identifiers'],{'doi':expected['doi']});self.assertEqual(r['submission_identifiers'],[{'scheme':'publication_doi','value':expected['doi']}])
|
||||||
|
self.assertEqual(r['tags'],[{'namespace':'recsys_program:contribution_type','value':expected['type_labels'][0]}]);self.assertFalse(r['metadata']['original_decision_signature_verified'])
|
||||||
|
self.assertEqual(seen,set(gold));self.assertEqual((out/'evidence/recsys-contributions.html').read_bytes(),self.body)
|
||||||
|
with self.assertRaisesRegex(ValueError,'Output exists'):c.collect(out,snapshots=src)
|
||||||
|
|
||||||
|
def test_edition_and_track_label_drift_rejected(self):
|
||||||
|
for old,new in [(b'RecSys 26 - Accepted Contributions',b'RecSys 25 - Accepted Contributions'),(b'>Long Papers</a>',b'>Workshops</a>'),(b"title='Research'",b"title='Unreviewed'")]:
|
||||||
|
with self.subTest(old=old),self.assertRaises(ValueError):c.parse_program(self.body.replace(old,new,1))
|
||||||
|
|
||||||
|
def test_duplicate_locator_missing_author_and_foreign_doi_rejected(self):
|
||||||
|
for old,new in [(b'accordion-1-slide-2"',b'accordion-1-slide-1"'),(b'by Md Aminul Islam, Kathryn Vasilaky and Elena Zheleva',b'by '),(b'https://doi.org/10.1145/3773078.3831824',b'https://doi.org/10.1145/9999999.3831824')]:
|
||||||
|
with self.subTest(old=old),self.assertRaises(ValueError):c.parse_program(self.body.replace(old,new,1))
|
||||||
|
|
||||||
|
def test_math_markup_requires_review_and_is_not_flattened(self):
|
||||||
|
changed=self.body.replace(b'Learning-to-rank (LTR)',b'Learning-to-rank <sup>2</sup> (LTR)',1)
|
||||||
|
with self.assertRaisesRegex(ValueError,'markup'):c.parse_program(changed)
|
||||||
|
|
||||||
|
def test_corruption_and_missing_receipt_body_preserve_failure(self):
|
||||||
|
for kind in ('manifest','body','missing'):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);src=root/'src';self.materialize(src);path=src/c.SNAPSHOT/('manifest.json' if kind=='manifest' else 'recsys-contributions.html')
|
||||||
|
if kind=='missing':path.unlink()
|
||||||
|
else:path.write_bytes(path.read_bytes()+b' ')
|
||||||
|
with self.assertRaises((ValueError,FileNotFoundError)):c.collect(root/'out',snapshots=src)
|
||||||
|
self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_symlink_source_and_root_rejected(self):
|
||||||
|
for kind in ('source','root'):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);src=root/'src';self.materialize(src);path=src/c.SNAPSHOT/'recsys-contributions.html' if kind=='source' else src
|
||||||
|
original=path.with_name(path.name+'-original');path.rename(original);path.symlink_to(original,target_is_directory=kind=='root')
|
||||||
|
with self.assertRaisesRegex(ValueError,'Unsafe'):c.collect(root/'out',snapshots=src)
|
||||||
@@ -0,0 +1,203 @@
|
|||||||
|
"""Metadata projection never requests OCR/review column pages or promotes mirror IDs."""
|
||||||
|
from collections import Counter
|
||||||
|
import hashlib
|
||||||
|
import importlib.util
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import struct
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
import urllib.error
|
||||||
|
|
||||||
|
from paper_atlas.collectors.reviewarena import COLUMNS, COMMIT, collect, decision_claim, decision_track_claim, inventory, record
|
||||||
|
from paper_atlas.io.parquet_metadata import RangeSource, SparseFile, projection
|
||||||
|
|
||||||
|
|
||||||
|
class MemorySource:
|
||||||
|
def __init__(self, raw):
|
||||||
|
self.raw, self.size, self.bytes_read, self.receipts = raw, len(raw), 0, []
|
||||||
|
|
||||||
|
def read(self, start, length):
|
||||||
|
raw = self.raw[start:start+length]
|
||||||
|
self.bytes_read += len(raw)
|
||||||
|
self.receipts.append({'start': start, 'length': length, 'sha256': hashlib.sha256(raw).hexdigest()})
|
||||||
|
return raw
|
||||||
|
|
||||||
|
|
||||||
|
def fixture():
|
||||||
|
import pyarrow as pa
|
||||||
|
import pyarrow.parquet as pq
|
||||||
|
rows = []
|
||||||
|
for i, decision in enumerate(['Accept-Findings', 'Accept-Main', 'Reject', 'Withdrawn', None]):
|
||||||
|
row = {'forum_id': 'unverified-source-'+str(i)+'.xml', 'conference': 'emnlp', 'year': 2023,
|
||||||
|
'track': 'main', 'venue_id': 'source-venue-claim',
|
||||||
|
'paper_number': i, 'title': 'Metadata title '+str(i), 'abstract': 'Metadata abstract\x00 '+str(i),
|
||||||
|
'authors': ['Anonymous'] if i == 2 else ['Ada Author', 'Yi Researcher'],
|
||||||
|
'keywords': ['metadata'], 'venue': 'Source venue label', 'decision': decision,
|
||||||
|
'markdown': 'OCR_FULLTEXT_SENTINEL '+('private-ish fixture body '*10000),
|
||||||
|
'reviews_json': 'REVIEW_FULLTEXT_SENTINEL '+('fixture review '*10000),
|
||||||
|
'decision_comment': 'META_REVIEW_SENTINEL '+('fixture meta-review '*10000)}
|
||||||
|
rows.append(row)
|
||||||
|
sink = io.BytesIO(); pq.write_table(pa.Table.from_pylist(rows), sink, row_group_size=2, compression='snappy')
|
||||||
|
return sink.getvalue(), [{k: v for k, v in r.items() if k in COLUMNS} for r in rows]
|
||||||
|
|
||||||
|
|
||||||
|
class RangeHTTPTests(unittest.TestCase):
|
||||||
|
def response(self, data, status=206, start=2, size=10, etag='"stable"'):
|
||||||
|
response = io.BytesIO(data); response.status = status
|
||||||
|
response.headers = {'Content-Range': f'bytes {start}-{start+len(data)-1}/{size}', 'ETag': etag}
|
||||||
|
return response
|
||||||
|
|
||||||
|
def test_strict_range_and_changed_object_detection(self):
|
||||||
|
opener = unittest.mock.Mock(); opener.open.side_effect = [self.response(b'ab'), self.response(b'cd', start=4, etag='"changed"')]
|
||||||
|
source = RangeSource('https://example.org/public', 10, opener=opener)
|
||||||
|
self.assertEqual(source.read(2, 2), b'ab')
|
||||||
|
self.assertEqual(opener.open.call_args.args[0].get_header('Range'), 'bytes=2-3')
|
||||||
|
with self.assertRaisesRegex(ValueError, 'ETag'):
|
||||||
|
source.read(4, 2)
|
||||||
|
self.assertEqual(source.bytes_read, 2)
|
||||||
|
|
||||||
|
def test_no_full_file_fallback_or_denied_source_retry(self):
|
||||||
|
opener = unittest.mock.Mock(); response = self.response(b'ab', status=200)
|
||||||
|
response.read = unittest.mock.Mock(side_effect=AssertionError('Must not consume full body'))
|
||||||
|
opener.open.return_value = response
|
||||||
|
with self.assertRaisesRegex(ValueError, 'full download forbidden'):
|
||||||
|
RangeSource('https://example.org/public', 10, opener=opener).read(2, 2)
|
||||||
|
response.read.assert_not_called()
|
||||||
|
opener = unittest.mock.Mock(); opener.open.side_effect = urllib.error.HTTPError('https://cdn.example/signed?SECRET=hidden', 403, 'Forbidden', {}, None)
|
||||||
|
with self.assertRaisesRegex(RuntimeError, '^Metadata source HTTP 403; no retry or full-file fallback$'):
|
||||||
|
RangeSource('https://example.org/public', 10, opener=opener).read(2, 2)
|
||||||
|
self.assertEqual(opener.open.call_count, 1)
|
||||||
|
|
||||||
|
def test_sparse_footer_windows_are_synthetic_not_network_reads(self):
|
||||||
|
source = SparseFile(100, [(0, b'PAR1'), (95, b'footer')])
|
||||||
|
source.seek(90); self.assertEqual(source.read(10), b'\x00'*5+b'foote')
|
||||||
|
source.seek(-4, 2); self.assertEqual(source.tell(), 96)
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
source.seek(-101, 2)
|
||||||
|
|
||||||
|
|
||||||
|
@unittest.skipUnless(importlib.util.find_spec('pyarrow'), 'Install mirrors extra on the approved remote runner')
|
||||||
|
class ParquetProjectionTests(unittest.TestCase):
|
||||||
|
def test_complete_selected_rows_and_no_excluded_page_requests(self):
|
||||||
|
import pyarrow.parquet as pq
|
||||||
|
raw, expected = fixture(); source = MemorySource(raw); rows, proof = projection(source, COLUMNS)
|
||||||
|
self.assertEqual(rows, expected)
|
||||||
|
metadata = pq.ParquetFile(io.BytesIO(raw)).metadata
|
||||||
|
excluded = []
|
||||||
|
for i in range(metadata.num_row_groups):
|
||||||
|
group = metadata.row_group(i)
|
||||||
|
for j in range(group.num_columns):
|
||||||
|
col = group.column(j)
|
||||||
|
if col.path_in_schema.split('.')[0] not in COLUMNS:
|
||||||
|
start = min(x for x in [col.data_page_offset, col.dictionary_page_offset] if x is not None and x > 0)
|
||||||
|
excluded.append((start, start+col.total_compressed_size))
|
||||||
|
for request in source.receipts:
|
||||||
|
a, b = request['start'], request['start']+request['length']
|
||||||
|
self.assertFalse(any(max(a, c) < min(b, d) for c, d in excluded))
|
||||||
|
self.assertFalse(proof['raw_parquet_saved']); self.assertFalse(proof['full_file_checksum_verified'])
|
||||||
|
self.assertLess(source.bytes_read, len(raw))
|
||||||
|
encoded = json.dumps(rows)+json.dumps(proof)
|
||||||
|
self.assertNotIn('OCR_FULLTEXT_SENTINEL', encoded); self.assertNotIn('REVIEW_FULLTEXT_SENTINEL', encoded)
|
||||||
|
self.assertNotIn('META_REVIEW_SENTINEL', encoded)
|
||||||
|
|
||||||
|
def test_wrong_schema_and_footer_fail_before_any_payload_download(self):
|
||||||
|
raw, _ = fixture(); source = MemorySource(raw)
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Missing'):
|
||||||
|
projection(source, {'missing_column': 'string'})
|
||||||
|
self.assertEqual(len(source.receipts), 3)
|
||||||
|
source = MemorySource(raw)
|
||||||
|
with self.assertRaisesRegex(ValueError, 'type differs'):
|
||||||
|
projection(source, {**COLUMNS, 'year': 'string'})
|
||||||
|
self.assertEqual(len(source.receipts), 3)
|
||||||
|
bad = bytearray(raw); bad[-8:-4] = struct.pack('<I', len(raw))
|
||||||
|
source = MemorySource(bytes(bad))
|
||||||
|
with self.assertRaisesRegex(ValueError, 'footer'):
|
||||||
|
projection(source, COLUMNS)
|
||||||
|
self.assertEqual(len(source.receipts), 2)
|
||||||
|
|
||||||
|
def test_collector_output_preserves_claims_without_original_identity(self):
|
||||||
|
raw, _ = fixture(); item = {'path': 'data/emnlp-00000-of-00001.parquet', 'size': len(raw), 'lfs': {'oid': 'a'*64}}
|
||||||
|
with tempfile.TemporaryDirectory() as temp:
|
||||||
|
out = Path(temp)/'projected'
|
||||||
|
with patch('paper_atlas.collectors.reviewarena.inventory', return_value=[item]):
|
||||||
|
report = collect('emnlp', out, source_factory=lambda url, size: MemorySource(raw))
|
||||||
|
self.assertEqual(report['mirror_decision_counts'], {'accepted': 2, 'rejected': 1, 'withdrawn': 1, 'unknown': 1})
|
||||||
|
self.assertEqual(report['records'], 5); self.assertFalse(report['database_written'])
|
||||||
|
projected = [json.loads(line) for line in next(out.glob('*.jsonl')).read_text().splitlines()]
|
||||||
|
for row in projected:
|
||||||
|
self.assertEqual(row['identifiers'], {})
|
||||||
|
self.assertFalse(row['metadata']['original_forum_id_verified'])
|
||||||
|
self.assertFalse(row['metadata']['original_decision_signature_verified'])
|
||||||
|
self.assertNotIn('status', row)
|
||||||
|
self.assertIn(COMMIT, row['external_id'])
|
||||||
|
self.assertEqual(projected[2]['authors'], ['Anonymous'])
|
||||||
|
self.assertEqual(projected[2]['metadata']['forum_id_claim'], 'unverified-source-2.xml')
|
||||||
|
self.assertEqual(projected[0]['track'], 'main')
|
||||||
|
self.assertEqual(projected[0]['mirror_decision_track_claim'], 'findings')
|
||||||
|
all_saved=''.join(p.read_text() for p in out.iterdir())
|
||||||
|
self.assertNotIn('FULLTEXT_SENTINEL', all_saved); self.assertNotIn('META_REVIEW_SENTINEL', all_saved)
|
||||||
|
with self.assertRaisesRegex(ValueError, 'New output'):
|
||||||
|
collect('emnlp', out)
|
||||||
|
|
||||||
|
def test_offline_replay_reclassifies_literals_preserving_all_raw_fields_without_network(self):
|
||||||
|
raw, _ = fixture(); item = {'path': 'data/emnlp-00000-of-00001.parquet', 'size': len(raw), 'lfs': {'oid': 'a'*64}}
|
||||||
|
with tempfile.TemporaryDirectory() as temp, patch('paper_atlas.collectors.reviewarena.inventory', return_value=[item]):
|
||||||
|
out = Path(temp)/'v1'; collect('emnlp', out, source_factory=lambda url, size: MemorySource(raw))
|
||||||
|
file, = out.glob('*.jsonl'); before = [json.loads(x) for x in file.read_text().splitlines()]
|
||||||
|
# Mimic the first reviewed source version: new literal spellings were unknown.
|
||||||
|
for row in before[:2]:
|
||||||
|
row['mirror_decision_category'] = 'unknown'; row.pop('mirror_decision_track_claim')
|
||||||
|
file.write_text(''.join(json.dumps(x)+'\n' for x in before))
|
||||||
|
report_path = out/'collection-report.json'; report = json.loads(report_path.read_text())
|
||||||
|
report['files'][0]['file_sha256'] = hashlib.sha256(file.read_bytes()).hexdigest()
|
||||||
|
report_path.write_text(json.dumps(report)); report_hash = hashlib.sha256(report_path.read_bytes()).hexdigest()
|
||||||
|
no_network = unittest.mock.Mock(side_effect=AssertionError('Offline replay must not contact source'))
|
||||||
|
result = collect('emnlp', Path(temp)/'v2', from_projection=report_path, projection_sha256=report_hash, source_factory=no_network)
|
||||||
|
no_network.assert_not_called(); self.assertEqual(result['metadata_bytes_read'], 0)
|
||||||
|
self.assertEqual(result['source_projection_bytes'], report['metadata_bytes_read'])
|
||||||
|
self.assertEqual(result['mirror_decision_counts']['accepted'], 2)
|
||||||
|
self.assertEqual(result['mirror_decision_track_claim_counts'], {'findings': 1, 'main': 1, 'unspecified': 3})
|
||||||
|
new = [json.loads(x) for x in next((Path(temp)/'v2').glob('*.jsonl')).read_text().splitlines()]
|
||||||
|
self.assertEqual([r['raw_fields'] for r in new], [r['raw_fields'] for r in before])
|
||||||
|
self.assertEqual([r['source_locator'] for r in new], [r['source_locator'] for r in before])
|
||||||
|
with self.assertRaisesRegex(ValueError, 'hash differs'):
|
||||||
|
collect('emnlp', Path(temp)/'bad', from_projection=report_path, projection_sha256='0'*64, source_factory=no_network)
|
||||||
|
no_network.assert_not_called()
|
||||||
|
|
||||||
|
def test_failure_is_retained_without_source_payload_or_retries(self):
|
||||||
|
item = {'path': 'data/emnlp-00000-of-00001.parquet', 'size': 50, 'lfs': {'oid': 'a'*64}}
|
||||||
|
with tempfile.TemporaryDirectory() as temp:
|
||||||
|
out = Path(temp)/'failed'
|
||||||
|
with patch('paper_atlas.collectors.reviewarena.inventory', return_value=[item]), patch('paper_atlas.collectors.reviewarena.projection', side_effect=RuntimeError('private body example')):
|
||||||
|
with self.assertRaises(RuntimeError):
|
||||||
|
collect('emnlp', out, source_factory=lambda url, size: None)
|
||||||
|
report = json.loads((out/'failure.json').read_text())
|
||||||
|
self.assertEqual(report['status'], 'incomplete_do_not_import')
|
||||||
|
self.assertTrue(report['no_automatic_retry']); self.assertNotIn('private body', (out/'failure.json').read_text())
|
||||||
|
|
||||||
|
|
||||||
|
class MirrorSemanticsTests(unittest.TestCase):
|
||||||
|
def test_pinned_inventory_covers_all_seven_families_and_fifteen_shards(self):
|
||||||
|
splits = ['colm', 'corl', 'emnlp', 'iclr', 'icml', 'neurips', 'tmlr']
|
||||||
|
result = {s: inventory(s) for s in splits}
|
||||||
|
self.assertEqual(sum(len(v) for v in result.values()), 15)
|
||||||
|
self.assertEqual([len(result[s]) for s in splits], [1, 1, 1, 5, 1, 5, 1])
|
||||||
|
self.assertEqual(result['emnlp'][0]['size'], 90794691)
|
||||||
|
self.assertEqual(result['emnlp'][0]['lfs']['oid'], 'e9e3aab42d28ca32a02287ed0bd0f1f9e1f3af5cb8ed8d6a9bf28d4f630773da')
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
inventory('uai')
|
||||||
|
|
||||||
|
def test_missing_or_ambiguous_decision_is_never_inferred_from_venue(self):
|
||||||
|
for literal in ['', None, 'Under review', 'Final venue pending', 'Accept maybe', 'Reject?']:
|
||||||
|
self.assertEqual(decision_claim(literal), 'unknown')
|
||||||
|
self.assertEqual(decision_claim('Desk Rejected'), 'desk_rejected')
|
||||||
|
self.assertEqual(decision_claim('Withdrawn'), 'withdrawn')
|
||||||
|
self.assertEqual(decision_claim('Accept (Findings)'), 'accepted')
|
||||||
|
self.assertEqual(decision_claim('Accept-Main'), 'accepted')
|
||||||
|
self.assertEqual(decision_claim('Accept-Findings'), 'accepted')
|
||||||
|
self.assertEqual(decision_track_claim('Accept-Findings'), 'findings')
|
||||||
|
self.assertIsNone(decision_track_claim('Accept (Poster)'))
|
||||||
@@ -0,0 +1,209 @@
|
|||||||
|
"""Whole-source scope correction, stale refresh, full history and atomicity on both engines."""
|
||||||
|
from contextlib import closing, redirect_stdout
|
||||||
|
import copy
|
||||||
|
import hashlib
|
||||||
|
import io
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
from paper_atlas.db.backend import open_store
|
||||||
|
from paper_atlas.db.edition_scopes import KEY
|
||||||
|
from paper_atlas.db.source_revisions import available, require_row, require_manifest
|
||||||
|
from paper_atlas.db.store import Store
|
||||||
|
from paper_atlas.ingestion.source_revisions import proposal, apply_plan, run, validate
|
||||||
|
from paper_atlas.ingestion.event_identity import database_identity, digest, values
|
||||||
|
from paper_atlas.ingestion.records import coverage_options, ingest_file
|
||||||
|
from paper_atlas.postgres.client import connect
|
||||||
|
from paper_atlas.postgres.migration import migrate
|
||||||
|
from paper_atlas.postgres.schema import quote
|
||||||
|
|
||||||
|
URL = 'https://example.org/volume'
|
||||||
|
|
||||||
|
|
||||||
|
def seed(path):
|
||||||
|
old = []
|
||||||
|
for i in range(4):
|
||||||
|
doi = '10.1234/publication'+str(i)
|
||||||
|
old.append({'source': 'fixture', 'venue': 'MM', 'year': 2026, 'track': 'main' if i == 0 else 'unclassified',
|
||||||
|
'external_id': doi, 'url': 'https://doi.org/'+doi, 'title': 'Distinct publication '+str(i),
|
||||||
|
'authors': ['Alice Author', 'Bob Writer'], 'abstract': None if i == 1 else 'Source abstract '+str(i),
|
||||||
|
'identifiers': {'doi': doi}, 'submission_identifiers': {'publication_doi': doi},
|
||||||
|
'status': 'accepted', 'raw_decision': 'Publication observation', 'evidence_url': URL,
|
||||||
|
'observed_at': '2026-06-01T00:00:00Z', 'tags': [{'namespace': 'fixture:track', 'value': 'main' if i == 0 else 'unclassified'}],
|
||||||
|
'metadata': {'original': i, 'program': None, 'scientific_work_identity_approved': False}})
|
||||||
|
current = copy.deepcopy(old)
|
||||||
|
current[1].update(track='main', abstract='Reviewed program abstract with NUL\x00 and SOH\x01')
|
||||||
|
current[1]['metadata']['program'] = {'id': 'p1', 'evidence_url': URL}
|
||||||
|
current[1]['tags'][0]['value'] = 'main'
|
||||||
|
store = Store(path)
|
||||||
|
for row in old[:3]:
|
||||||
|
p, r = store.record(row['source'], row['external_id'], row['url'], row['title'], row['abstract'], row['authors'], row['identifiers'], row['metadata'])
|
||||||
|
store.decision(p, r, 'MM', 2026, row['track'], row['external_id'], URL, 'accepted', row['raw_decision'],
|
||||||
|
identities=[('publication_doi', row['external_id'])], observed_at=row['observed_at'])
|
||||||
|
store.record_tags(p, r, 'fixture', row['tags'])
|
||||||
|
if row == old[1]:
|
||||||
|
store.conn.execute('UPDATE papers SET abstract=? WHERE id=?', ('Canonical preprint abstract', p))
|
||||||
|
p, other = store.record('prior', 'old-forum', 'https://example.org/prior', row['title'], 'Canonical preprint abstract', row['authors'], row['identifiers'])
|
||||||
|
store.decision(p, other, 'PRIOR', 2025, 'main', 'old-forum', 'https://example.org/prior', 'rejected', 'Reject', observed_at='2025-01-01T00:00:00Z')
|
||||||
|
# An unrelated source already has an attempt in the populated target edition.
|
||||||
|
p, r = store.record('other', 'other', 'https://example.org/other', 'Unrelated target paper', 'Other abstract', ['Someone Else'])
|
||||||
|
store.decision(p, r, 'MM', 2026, 'main', 'other', 'https://example.org/other', 'accepted', 'Published')
|
||||||
|
sid = store.conn.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0]
|
||||||
|
for track, expected, imported in [('main', 1, 1), ('unclassified', 3, 2)]:
|
||||||
|
eid = store.conn.execute('SELECT id FROM editions WHERE year=2026 AND track=?', (track,)).fetchone()[0]
|
||||||
|
store.conn.execute("INSERT INTO edition_sources VALUES(?,?,?,'partial',?,?,?,'2026-06-01',NULL)", (eid, sid, URL, expected, imported, 'checkpoint'))
|
||||||
|
store.conn.commit(); store.conn.close()
|
||||||
|
return {'format': 1, 'source': 'fixture', 'venue': 'MM', 'year': 2026, 'event_key': '', 'list_url': URL,
|
||||||
|
'previous_rows': old, 'rows': current, 'evidence': [{'url': URL, 'sha256': 'a'*64}]}
|
||||||
|
|
||||||
|
|
||||||
|
class RevisionCases:
|
||||||
|
def store(self):
|
||||||
|
return open_store(**self.target)
|
||||||
|
|
||||||
|
def plan(self, c):
|
||||||
|
return proposal(c, self.inventory, database_identity(c, self.target.get('db')))
|
||||||
|
|
||||||
|
def test_move_to_populated_target_preserves_full_history_and_before_images(self):
|
||||||
|
with closing(self.store().conn) as c:
|
||||||
|
plan = self.plan(c); self.assertEqual(len(plan['changes']), 1)
|
||||||
|
self.assertEqual(len(plan['before']['packet']['submissions']), 5)
|
||||||
|
before_papers = plan['before']['packet']['papers']
|
||||||
|
before_tags = plan['before']['packet']['paper_tags']
|
||||||
|
result = apply_plan(c, plan); c.commit()
|
||||||
|
self.assertEqual((result['events_moved'], result['records_changed']), (1, 1))
|
||||||
|
self.assertEqual(result['protected_decisions'], 5)
|
||||||
|
self.assertEqual(values(c.execute('SELECT * FROM papers ORDER BY id')), before_papers)
|
||||||
|
self.assertEqual(values(c.execute('SELECT * FROM paper_tags ORDER BY paper_id,tag_id,evidence_key')), before_tags)
|
||||||
|
self.assertEqual(c.execute("SELECT count(*) FROM decisions WHERE status='rejected'").fetchone()[0], 1)
|
||||||
|
self.assertEqual(values(c.execute('SELECT status,expected_count,imported_count,cursor,last_success_at,last_error FROM edition_sources ORDER BY edition_id')),
|
||||||
|
[['partial', 2, 2, 'checkpoint', '2026-06-01', None], ['partial', 2, 1, 'checkpoint', '2026-06-01', None]])
|
||||||
|
saved = json.loads(c.execute('SELECT plan_json FROM source_scope_revisions').fetchone()[0])
|
||||||
|
self.assertEqual(saved, plan)
|
||||||
|
self.assertEqual(apply_plan(c, plan)['status'], 'already_applied')
|
||||||
|
self.assertEqual(c.execute('SELECT count(*) FROM source_scope_revisions').fetchone()[0], 1)
|
||||||
|
|
||||||
|
def test_stale_direct_store_and_jsonl_refresh_fail_but_current_refresh_succeeds(self):
|
||||||
|
with closing(self.store().conn) as c:
|
||||||
|
plan = self.plan(c); apply_plan(c, plan); c.commit()
|
||||||
|
with closing(self.store().conn) as c:
|
||||||
|
store = object.__new__(Store); store.conn = c; store.event_identity = True
|
||||||
|
old = self.inventory['previous_rows'][1]
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Stale source'):
|
||||||
|
store.record('fixture', old['external_id'], old['url'], old['title'], old['abstract'], old['authors'], old['identifiers'], old['metadata'])
|
||||||
|
c.rollback()
|
||||||
|
row = self.inventory['rows'][1]
|
||||||
|
p, r = store.record('fixture', row['external_id'], row['url'], row['title'], row['abstract'], row['authors'], row['identifiers'], row['metadata'])
|
||||||
|
self.assertEqual(c.execute('SELECT abstract FROM papers WHERE id=?', (p,)).fetchone()[0], 'Canonical preprint abstract')
|
||||||
|
self.assertEqual(json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?', (r,)).fetchone()[0])[KEY][0]['target']['track'], 'main')
|
||||||
|
c.rollback()
|
||||||
|
path = self.root/'old.jsonl'; raw = (json.dumps(old)+'\n').encode(); path.write_bytes(raw)
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps({'source': 'fixture', 'venue': 'MM', 'year': 2026,
|
||||||
|
'track': 'unclassified', 'list_url': URL, 'status': 'partial', 'count': 1, 'file_sha256': hashlib.sha256(raw).hexdigest()}))
|
||||||
|
with self.assertRaisesRegex(ValueError, 'source snapshot'):
|
||||||
|
ingest_file(self.target.get('db'), path, **{k: v for k, v in self.target.items() if k != 'db'})
|
||||||
|
|
||||||
|
def test_unregistered_member_and_changed_program_metadata_are_rejected(self):
|
||||||
|
with closing(self.store().conn) as c:
|
||||||
|
apply_plan(c, self.plan(c)); c.commit()
|
||||||
|
sid = c.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0]
|
||||||
|
cache = {}; unknown = copy.deepcopy(self.inventory['rows'][0]); unknown['external_id'] = '10.1234/unlisted'
|
||||||
|
with self.assertRaisesRegex(ValueError, 'absent from current'):
|
||||||
|
require_row(c, sid, unknown, {'list_url': URL}, cache)
|
||||||
|
bad = copy.deepcopy(self.inventory['rows'][1]); bad['metadata']['program'] = None
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Stale source'):
|
||||||
|
require_row(c, sid, bad, {'list_url': URL}, cache)
|
||||||
|
good = copy.deepcopy(self.inventory['rows'][1]); good['metadata']['verified_work_anchor'] = {'reason': 'extra evidence'}
|
||||||
|
require_row(c, sid, good, {'list_url': URL}, cache)
|
||||||
|
with self.assertRaisesRegex(ValueError, 'coverage manifest'):
|
||||||
|
require_manifest(c, sid, {'venue': 'MM', 'year': 2026, 'track': 'main', 'list_url': URL, 'count': 1, 'status': 'partial'}, cache)
|
||||||
|
require_manifest(c, sid, {'venue': 'MM', 'year': 2026, 'track': 'main', 'list_url': URL, 'count': 2, 'status': 'partial'}, cache)
|
||||||
|
|
||||||
|
def test_stale_plan_and_target_identifier_collision_prevent_writes(self):
|
||||||
|
with closing(self.store().conn) as c:
|
||||||
|
plan = self.plan(c)
|
||||||
|
c.execute("UPDATE decisions SET raw_value='Additional evidence' WHERE status='rejected'")
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Stale'):
|
||||||
|
apply_plan(c, plan)
|
||||||
|
c.rollback()
|
||||||
|
move = plan['changes'][0]
|
||||||
|
target_sub = c.execute('SELECT id FROM submissions WHERE edition_id=? LIMIT 1', (move['target_edition_id'],)).fetchone()[0]
|
||||||
|
c.execute('INSERT INTO submission_identifiers VALUES(?,?,?,?)', (move['target_edition_id'], 'publication_doi', move['external_id'], target_sub))
|
||||||
|
with self.assertRaisesRegex(ValueError, 'collision'):
|
||||||
|
self.plan(c)
|
||||||
|
c.rollback()
|
||||||
|
self.assertFalse(available(c))
|
||||||
|
|
||||||
|
def test_failed_postcheck_rolls_back_rows_extension_and_coverage(self):
|
||||||
|
with closing(self.store().conn) as c:
|
||||||
|
plan = self.plan(c)
|
||||||
|
with patch('paper_atlas.ingestion.source_revisions.verify', side_effect=ValueError('Injected verification failure')):
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Injected'):
|
||||||
|
apply_plan(c, plan)
|
||||||
|
c.rollback()
|
||||||
|
self.assertFalse(available(c))
|
||||||
|
self.assertEqual(self.plan(c), plan)
|
||||||
|
|
||||||
|
def test_cli_binds_database_and_file_hash_without_writing_during_preflight(self):
|
||||||
|
inventory = self.root/'inventory.json'; inventory.write_text(json.dumps(self.inventory))
|
||||||
|
path = self.root/'plan.json'
|
||||||
|
args = {k: v for k, v in self.target.items() if k != 'db'}
|
||||||
|
run(self.target.get('db'), path, inventory=inventory, **args)
|
||||||
|
with closing(self.store().conn) as c:
|
||||||
|
self.assertFalse(available(c))
|
||||||
|
with self.assertRaisesRegex(ValueError, 'hash'):
|
||||||
|
run(self.target.get('db'), self.root/'bad.json', plan=path, plan_sha256='0'*64, **args)
|
||||||
|
report = run(self.target.get('db'), self.root/'applied.json', plan=path, plan_sha256=hashlib.sha256(path.read_bytes()).hexdigest(), **args)
|
||||||
|
self.assertEqual(report['events_moved'], 1)
|
||||||
|
|
||||||
|
def test_membership_identity_changes_and_incremental_shrink_remain_forbidden(self):
|
||||||
|
for mutate in [lambda x: x['rows'].pop(), lambda x: x['rows'][0].update(title='Different work'),
|
||||||
|
lambda x: x['rows'][0].update(status='rejected')]:
|
||||||
|
bad = copy.deepcopy(self.inventory); mutate(bad)
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
validate(bad)
|
||||||
|
with self.assertRaisesRegex(ValueError, 'incremental'):
|
||||||
|
coverage_options({'coverage_mode': 'incremental', 'venue': 'MM', 'year': 2026, 'track': 'unclassified',
|
||||||
|
'status': 'partial', 'count': 1, 'coverage_previous_count': 2,
|
||||||
|
'coverage_previous_expected_count': 3, 'coverage_expected_count': 2})
|
||||||
|
|
||||||
|
|
||||||
|
class SQLiteSourceRevisionTests(RevisionCases, unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
temp = tempfile.TemporaryDirectory(); self.addCleanup(temp.cleanup); self.root = Path(temp.name)
|
||||||
|
db = self.root/'source.sqlite'; self.inventory = seed(db); self.target = {'db': db}
|
||||||
|
|
||||||
|
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires approved remote PostgreSQL service')
|
||||||
|
def test_optional_revision_table_migrates_with_full_history(self):
|
||||||
|
with closing(self.store().conn) as c:
|
||||||
|
plan = self.plan(c); apply_plan(c, plan); c.commit()
|
||||||
|
service = os.environ['PAPER_ATLAS_TEST_PG_SERVICE']; schema = 'pa_rev_copy_'+uuid.uuid4().hex[:10]
|
||||||
|
self.addCleanup(drop_schema, service, schema)
|
||||||
|
with redirect_stdout(io.StringIO()):
|
||||||
|
migrate(self.target['db'], service, self.root/'copied.json', schema=schema, min_free_gib=0)
|
||||||
|
with closing(open_store(pg_service=service, pg_schema=schema).conn) as c:
|
||||||
|
self.assertEqual(json.loads(c.execute('SELECT plan_json FROM source_scope_revisions').fetchone()[0]), plan)
|
||||||
|
self.assertEqual(c.execute('SELECT count(*) FROM source_scope_revisions').fetchone()[0], 1)
|
||||||
|
|
||||||
|
|
||||||
|
def drop_schema(service, schema):
|
||||||
|
with connect(service) as c:
|
||||||
|
for name, in c.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)', (schema,)):
|
||||||
|
c.execute('DROP SCHEMA '+quote(name)+' CASCADE')
|
||||||
|
|
||||||
|
|
||||||
|
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires approved remote PostgreSQL service')
|
||||||
|
class PostgreSQLSourceRevisionTests(RevisionCases, unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
temp = tempfile.TemporaryDirectory(); self.addCleanup(temp.cleanup); self.root = Path(temp.name)
|
||||||
|
db = self.root/'source.sqlite'; self.inventory = seed(db)
|
||||||
|
service = os.environ['PAPER_ATLAS_TEST_PG_SERVICE']; schema = 'pa_rev_'+uuid.uuid4().hex[:12]
|
||||||
|
self.addCleanup(drop_schema, service, schema)
|
||||||
|
with redirect_stdout(io.StringIO()):
|
||||||
|
migrate(db, service, self.root/'migration.json', schema=schema, min_free_gib=0)
|
||||||
|
self.target = {'pg_service': service, 'pg_schema': schema}
|
||||||
Reference in New Issue
Block a user