Compare commits
74 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 12f95cff91 | |||
| 9864632774 | |||
| f982851560 | |||
| 88aeb4d2eb | |||
| 83f3ebd26b | |||
| 13536d6268 | |||
| 7e63035570 | |||
| 44f4146f3e | |||
| b0bd9b69dc | |||
| 09a2d4ed20 | |||
| c8db6118d4 | |||
| 1c64971a52 | |||
| ac14df0625 | |||
| afb3b939ca | |||
| 8c147f5e6e | |||
| 6961710e96 | |||
| f4be9de547 | |||
| 33bb378369 | |||
| 5bdf9f0fe5 | |||
| 7ed5966d3a | |||
| 871b2aa81e | |||
| b14193063a | |||
| 564821d883 | |||
| 2f1b5c959a | |||
| f139c7e49d | |||
| 6a89f5f6c2 | |||
| cb3d97ed92 | |||
| 06467835c7 | |||
| f2753bc741 | |||
| 1ea17c810d | |||
| f800c6ecb4 | |||
| 28255743ef | |||
| 279d6a2735 | |||
| 8996a21e09 | |||
| e9bcbc3aef | |||
| c97ea94b49 | |||
| 59053fd4fb | |||
| b50136d5bc | |||
| 87a0689fcc | |||
| 9093a8ce25 | |||
| e19cb83cd6 | |||
| 4d2df2a826 | |||
| 19d03ced54 | |||
| c33f5422de | |||
| 1cfd44d8b6 | |||
| 5de7f6ff22 | |||
| b4e249e675 | |||
| 93f00d1a64 | |||
| 8b7bc6d871 | |||
| 7588a4e904 | |||
| 891a63d982 | |||
| fe3e2502a5 | |||
| b675a38b97 | |||
| cb875b8eef | |||
| 6df904e6d8 | |||
| 57a557c66b | |||
| 5b5e606f0c | |||
| 3ff9ee9a94 | |||
| 9f84fd7caf | |||
| bf05d3cdc8 | |||
| 930df2a4d5 | |||
| 37380e1d3b | |||
| 25ed4559ca | |||
| e0ac62464a | |||
| e37fa46e42 | |||
| 70463c0250 | |||
| 5c78a36fd5 | |||
| 7facde6c94 | |||
| c06981b244 | |||
| 8b1e983e7d | |||
| fe0f08a550 | |||
| b226b3b98b | |||
| 05ada8daf8 | |||
| 443a79dce2 |
+236
-3
@@ -12,15 +12,16 @@ jobs:
|
|||||||
- name: Verify metadata storage and query commands
|
- name: Verify metadata storage and query commands
|
||||||
run: |
|
run: |
|
||||||
python3 -B - <<'PY'
|
python3 -B - <<'PY'
|
||||||
import io, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request
|
import base64, gzip, io, json, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request
|
||||||
sha = os.environ['ARCHIVE_SHA']
|
sha = os.environ['ARCHIVE_SHA']
|
||||||
if not re.fullmatch(r'[0-9a-fA-F]{40,64}', sha):
|
if not re.fullmatch(r'[0-9a-fA-F]{40,64}', sha):
|
||||||
raise SystemExit('Invalid source commit')
|
raise SystemExit('Invalid source commit')
|
||||||
# Public repository archive: no owner credentials or runner SSH keys.
|
# Public repository archive: no owner credentials or runner SSH keys.
|
||||||
url = 'https://gitea.entroia.com/Zayden-Jung/paper-atlas/archive/' + sha + '.tar.gz'
|
url = 'https://gitea.entroia.com/Zayden-Jung/paper-atlas/archive/' + sha + '.tar.gz'
|
||||||
with urllib.request.urlopen(url, timeout=60) as response:
|
with urllib.request.urlopen(url, timeout=60) as response:
|
||||||
archive = response.read(16*1024*1024+1)
|
# Full publication-source fixtures retain original bibliographic evidence.
|
||||||
if len(archive)>16*1024*1024:
|
archive = response.read(24*1024*1024+1)
|
||||||
|
if len(archive)>24*1024*1024:
|
||||||
raise SystemExit('Unexpectedly large source archive')
|
raise SystemExit('Unexpectedly large source archive')
|
||||||
with tempfile.TemporaryDirectory(prefix='paper-atlas-ci-') as tmp:
|
with tempfile.TemporaryDirectory(prefix='paper-atlas-ci-') as tmp:
|
||||||
root = pathlib.Path(tmp)
|
root = pathlib.Path(tmp)
|
||||||
@@ -46,12 +47,243 @@ jobs:
|
|||||||
subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True)
|
subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True)
|
||||||
outside = root / 'outside-checkout'
|
outside = root / 'outside-checkout'
|
||||||
outside.mkdir()
|
outside.mkdir()
|
||||||
|
mm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/mm2026-source.json.gz').read_bytes()))
|
||||||
|
mm_snapshots = root/'mm2026-snapshots'
|
||||||
|
for name, body in mm_fixture['files'].items():
|
||||||
|
path = mm_snapshots/name
|
||||||
|
path.parent.mkdir(parents=True,exist_ok=True)
|
||||||
|
path.write_bytes(body.encode('utf-8'))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','mm2026-program','--snapshots',str(mm_snapshots),'--out',str(root/'installed-mm2026')],cwd=outside,check=True)
|
||||||
|
mm_report = json.loads((root/'installed-mm2026/collection-report.json').read_text())
|
||||||
|
assert (mm_report['records'],mm_report['program_observations'],mm_report['extra_presentation_observations']) == (2143,4326,2183)
|
||||||
|
assert len(mm_report['files']) == 28 and mm_report['missing_abstract_count'] == 216
|
||||||
|
assert mm_report['session_count'] == 157 and not mm_report['database_written'] and mm_report['network_requests'] == 0 and mm_report['approved_for_import'] == 0
|
||||||
|
mm_rows = {r['paper_id']:r for f in mm_report['files'] for r in map(json.loads,pathlib.Path(f['file']).read_text().splitlines())}
|
||||||
|
import hashlib
|
||||||
|
assert set(mm_rows) == {g['paper_id'] for g in mm_fixture['independent_inventory']['inventory']}
|
||||||
|
for gold in mm_fixture['independent_inventory']['inventory']:
|
||||||
|
row = mm_rows[gold['paper_id']]
|
||||||
|
assert (row['title'],row['authors'],row['affiliations'],row['track']) == (gold['title'],gold['authors'],gold['affiliations'],gold['track_literal'])
|
||||||
|
assert hashlib.sha256(row['abstract'].encode()).hexdigest() == gold['abstract_sha256']
|
||||||
|
assert [c['source_pointer'] for c in row['contexts']] == gold['locations']
|
||||||
|
assert not row['approved_for_import'] and not row['final_decision_verified'] and 'emails' not in row
|
||||||
|
mm_pub_fixture = json.loads(gzip.decompress((src/'tests/fixtures/mm2026-publications.json.gz').read_bytes()))
|
||||||
|
mm_pub_registry = root/'mm2026-publication-registry'
|
||||||
|
for name,body in mm_pub_fixture['files'].items():
|
||||||
|
path=mm_pub_registry/name
|
||||||
|
path.parent.mkdir(parents=True,exist_ok=True)
|
||||||
|
path.write_bytes(body.encode('utf-8'))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','mm2026-publications','--registry-snapshots',str(mm_pub_registry),'--program-snapshots',str(mm_snapshots),'--parent-snapshot',str(mm_pub_registry/'parent.json'),'--out',str(root/'installed-mm2026-publications')],cwd=outside,check=True)
|
||||||
|
mm_pub_report=json.loads((root/'installed-mm2026-publications/collection-report.json').read_text())
|
||||||
|
assert (mm_pub_report['records'],mm_pub_report['unique_registered_dois'],mm_pub_report['abstracts_from_exact_program_correspondence'],mm_pub_report['output_missing_abstract_count'])==(1918,1918,1743,175)
|
||||||
|
assert mm_pub_report['unmatched_program_count']==391 and not mm_pub_report['database_written'] and mm_pub_report['approved_for_import']==0
|
||||||
|
mm_pub_rows={r['external_id']:r for f in mm_pub_report['files'] for r in map(json.loads,pathlib.Path(f['file']).read_text().splitlines())}
|
||||||
|
assert len(mm_pub_rows)==1918
|
||||||
|
mm_format_gold=json.loads((src/'tests/fixtures/mm2026-title-formatting-review.json').read_text())
|
||||||
|
mm_format_overrides={g['doi']:g for g in [*mm_format_gold['gold'],mm_format_gold['held_gold']]}
|
||||||
|
assert mm_pub_report['abstracts_from_reviewed_title_formatting']==26 and mm_pub_report['abstracts_from_literal_program_correspondence']==1717
|
||||||
|
for original_gold in mm_pub_fixture['independent_gold']:
|
||||||
|
gold=mm_format_overrides.get(original_gold['doi'],original_gold)
|
||||||
|
row=mm_pub_rows[gold['doi']]
|
||||||
|
assert (row['title'],row['authors'],row['track'])==(gold['title'],gold['authors'],gold['track'])
|
||||||
|
assert (hashlib.sha256(row['abstract'].encode()).hexdigest() if row['abstract'] is not None else None)==gold['abstract_sha256']
|
||||||
|
assert row['metadata']['program_match_kind']==gold['kind'] and not row['metadata']['scientific_work_identity_approved'] and not row['metadata']['research_type_classified']
|
||||||
|
if service_file.is_file():
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','source-identities','--files',*[f['file'] for f in mm_pub_report['files']],'--pg-service','foundry-research','--pg-schema','research','--out',str(root/'installed-mm2026-publication-preflight')],cwd=outside,env=testenv,check=True)
|
||||||
|
mm_pub_preflight=json.loads((root/'installed-mm2026-publication-preflight/postgresql-preflight.json').read_text())
|
||||||
|
assert mm_pub_preflight['records']==1918 and mm_pub_preflight['transaction_read_only'] and not mm_pub_preflight['database_written'] and mm_pub_preflight['approved_for_import']==0
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','source-identities','--mm-program-report',str(root/'installed-mm2026/collection-report.json'),'--pg-service','foundry-research','--pg-schema','research','--out',str(root/'installed-mm2026-preflight')],cwd=outside,env=testenv,check=True)
|
||||||
|
mm_preflight = json.loads((root/'installed-mm2026-preflight/postgresql-preflight.json').read_text())
|
||||||
|
assert mm_preflight['records'] == 2143 and len(mm_preflight['inputs']) == 28
|
||||||
|
assert mm_preflight['transaction_read_only'] and not mm_preflight['database_written'] and mm_preflight['approved_for_import'] == 0
|
||||||
|
assert sum(len(r['program_contexts']) for r in mm_preflight['entries']) == 4326
|
||||||
|
assert all(r['status'] == 'unknown' and not r['identifiers'] and not r['contribution_type_verified'] and not r['publication_identity_verified'] and not r['final_decision_verified'] for r in mm_preflight['entries'])
|
||||||
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
|
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--help'],cwd=outside,check=True)
|
||||||
|
eccv_fixture = json.loads(gzip.decompress((src/'tests/fixtures/eccv2026-source.json.gz').read_bytes()))
|
||||||
|
eccv_official = root/'eccv2026-official'
|
||||||
|
eccv_papers = root/'eccv2026-paper-samples'
|
||||||
|
eccv_official.mkdir(); eccv_papers.mkdir()
|
||||||
|
for name, html in eccv_fixture['official_pages'].items():
|
||||||
|
(eccv_official/name).write_bytes(html.encode('utf-8'))
|
||||||
|
(eccv_official/'manifest.json').write_bytes(eccv_fixture['official_manifest'].encode('utf-8'))
|
||||||
|
for pid, html in eccv_fixture['paper_pages'].items():
|
||||||
|
(eccv_papers/(pid+'.html')).write_bytes(html.encode('utf-8'))
|
||||||
|
eccv_encoding = eccv_fixture['encoding_sample']
|
||||||
|
(eccv_papers/(eccv_encoding['paper_id']+'.html')).write_bytes(eccv_encoding['html'].encode('utf-8'))
|
||||||
|
eccv_fixture['paper_manifest']['records'].append({**eccv_encoding['receipt'], 'paper_id': eccv_encoding['paper_id'], 'file': eccv_encoding['paper_id']+'.html'})
|
||||||
|
eccv_literal = eccv_fixture['literal_title_sample']
|
||||||
|
(eccv_papers/(eccv_literal['paper_id']+'.html')).write_bytes(eccv_literal['html'].encode('utf-8'))
|
||||||
|
eccv_fixture['paper_manifest']['records'].append({**eccv_literal['receipt'], 'paper_id': eccv_literal['paper_id'], 'file': eccv_literal['paper_id']+'.html'})
|
||||||
|
(eccv_papers/'manifest.json').write_text(json.dumps(eccv_fixture['paper_manifest']))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','eccv2026-program','--official-snapshots',str(eccv_official),'--paper-snapshots',str(eccv_papers),'--out',str(root/'installed-eccv2026')],cwd=outside,check=True)
|
||||||
|
eccv_report = json.loads((root/'installed-eccv2026/collection-report.json').read_text())
|
||||||
|
assert eccv_report['records'] == 5 and eccv_report['expected_program_ids'] == 2834
|
||||||
|
assert eccv_report['duplicate_program_occurrences'] == 17 and len(eccv_report['uncollected_program_ids']) == 2829
|
||||||
|
assert not eccv_report['database_written'] and eccv_report['files'][0]['track'] == 'program_unclassified'
|
||||||
|
eccv_rows = [json.loads(line) for line in pathlib.Path(eccv_report['files'][0]['file']).read_text().splitlines()]
|
||||||
|
eccv_encoded_row = next(row for row in eccv_rows if row['external_id'] == '2026/4199')
|
||||||
|
assert eccv_encoded_row['authors'] == eccv_encoding['gold']['authors']
|
||||||
|
assert 'jsonld_display_text_normalized_requires_review' in eccv_encoded_row['metadata']['public_page_fields']['review_flags']
|
||||||
|
eccv_literal_row = next(row for row in eccv_rows if row['external_id'] == '2026/3844')
|
||||||
|
assert eccv_literal_row['title'] == eccv_literal['gold']['visible_title']
|
||||||
|
assert '&' in eccv_literal_row['title']
|
||||||
|
icassp_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icassp2026-source.json.gz').read_bytes()))
|
||||||
|
icassp_snapshots = root/'icassp2026-snapshots'
|
||||||
|
icassp_snapshots.mkdir()
|
||||||
|
for name, encoded in icassp_fixture['files'].items():
|
||||||
|
target = icassp_snapshots/name
|
||||||
|
target.parent.mkdir(parents=True,exist_ok=True)
|
||||||
|
target.write_bytes(base64.b64decode(encoded))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','icassp2026-registry','--snapshots',str(icassp_snapshots),'--out',str(root/'installed-icassp2026')],cwd=outside,check=True)
|
||||||
|
icassp_report = json.loads((root/'installed-icassp2026/collection-report.json').read_text())
|
||||||
|
assert icassp_report['records'] == 4582 and icassp_report['missing_abstract_count'] == 4582
|
||||||
|
assert icassp_report['single_name_author_fields'] == 23
|
||||||
|
assert len(icassp_report['excluded_nonpaper_items']) == 7 and len(icassp_report['excluded_other_conference_dois']) == 56
|
||||||
|
assert not icassp_report['database_written'] and icassp_report['network_requests'] == 0 and icassp_report['approved_for_import'] == 0
|
||||||
|
icassp_rows = [json.loads(line) for line in pathlib.Path(icassp_report['files'][0]['file']).read_text().splitlines()]
|
||||||
|
assert [{'doi':r['external_id'],'title':r['title'],'authors':r['authors']} for r in icassp_rows] == sorted(icassp_fixture['independent_gold'],key=lambda r:r['doi'])
|
||||||
|
assert all(r['abstract'] is None and r['track'] == 'proceedings_unclassified' for r in icassp_rows)
|
||||||
|
sdm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/sdm2026-source.json.gz').read_bytes()))
|
||||||
|
sdm_snapshots = root/'sdm2026-snapshots'
|
||||||
|
for name, text in sdm_fixture['files'].items():
|
||||||
|
target = sdm_snapshots/name
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
target.write_bytes(text.encode('utf-8'))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','sdm2026-proceedings','--snapshots',str(sdm_snapshots),'--out',str(root/'installed-sdm2026')],cwd=outside,check=True)
|
||||||
|
sdm_report = json.loads((root/'installed-sdm2026/collection-report.json').read_text())
|
||||||
|
assert sdm_report['records'] == 53 and sdm_report['missing_abstract_count'] == 53
|
||||||
|
assert not sdm_report['database_written'] and sdm_report['network_requests'] == 0
|
||||||
|
assert sdm_report['program_observations']['count'] == 53 and sdm_report['program_observations']['abstract_pages'] == 52
|
||||||
|
assert sdm_report['program_registry_author_sequence_differences'] == 20
|
||||||
|
sdm_rows = [json.loads(line) for line in pathlib.Path(sdm_report['files'][0]['file']).read_text().splitlines()]
|
||||||
|
assert len(sdm_rows) == 53 and all(row['abstract'] is None for row in sdm_rows)
|
||||||
|
sdm_observations = [json.loads(line) for line in pathlib.Path(sdm_report['program_observations']['file']).read_text().splitlines()]
|
||||||
|
assert sum(row['cancelled_presentation'] for row in sdm_observations) == 1
|
||||||
|
assert all('status' not in row and 'identifiers' not in row for row in sdm_observations)
|
||||||
|
icdm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icdm2026-source.json.gz').read_bytes()))
|
||||||
|
icdm_snapshots = root/'icdm2026-snapshots'
|
||||||
|
for name, text in icdm_fixture['files'].items():
|
||||||
|
target = icdm_snapshots/name
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
target.write_bytes(text.encode('utf-8'))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','icdm2026-program','--snapshots',str(icdm_snapshots),'--out',str(root/'installed-icdm2026')],cwd=outside,check=True)
|
||||||
|
icdm_report = json.loads((root/'installed-icdm2026/collection-report.json').read_text())
|
||||||
|
assert icdm_report['records'] == 324 and icdm_report['missing_abstract_count'] == 324
|
||||||
|
assert icdm_report['track_counts'] == {'research':241,'applied':83}
|
||||||
|
assert icdm_report['presentation_counts'] == {'long':168,'short':156}
|
||||||
|
assert not icdm_report['database_written'] and icdm_report['network_requests'] == 0
|
||||||
|
icdm_rows = [json.loads(line) for f in icdm_report['files'] for line in pathlib.Path(f['file']).read_text().splitlines()]
|
||||||
|
assert len(icdm_rows) == 324 and all(row['abstract'] is None and not row['identifiers'] and not row['submission_identifiers'] for row in icdm_rows)
|
||||||
|
assert all(row['metadata']['paper_type'] is None and not row['metadata']['original_decision_signature_verified'] for row in icdm_rows)
|
||||||
|
icaps_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icaps2026-source.json.gz').read_bytes()))
|
||||||
|
icaps_snapshots = root/'icaps2026-snapshots'
|
||||||
|
icaps_snapshots.mkdir()
|
||||||
|
for name, html in icaps_fixture['pages'].items():
|
||||||
|
(icaps_snapshots/name).write_text(html)
|
||||||
|
(icaps_snapshots/'manifest.json').write_text(json.dumps(icaps_fixture['manifest']))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','icaps2026-program','--snapshots',str(icaps_snapshots),'--out',str(root/'installed-icaps2026')],cwd=outside,check=True)
|
||||||
|
icaps_report = json.loads((root/'installed-icaps2026/collection-report.json').read_text())
|
||||||
|
assert icaps_report['records'] == 90 and icaps_report['previously_published_presentations'] == 10
|
||||||
|
assert icaps_report['missing_abstract_count'] == 90 and not icaps_report['database_written']
|
||||||
|
assert icaps_report['files'][0]['track'] == 'main_tracks_unclassified'
|
||||||
|
registry_snapshots = root/'icaps2026-registry-snapshots'
|
||||||
|
registry_snapshots.mkdir()
|
||||||
|
(registry_snapshots/'crossref-filtered.json').write_bytes(gzip.decompress((src/'tests/fixtures/icaps2026-crossref.json.gz').read_bytes()))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','icaps2026-registry','--snapshots',str(registry_snapshots),'--out',str(root/'installed-icaps2026-registry')],cwd=outside,check=True)
|
||||||
|
registry_report = json.loads((root/'installed-icaps2026-registry/collection-report.json').read_text())
|
||||||
|
assert registry_report['records'] == 90 and registry_report['missing_abstract_count'] == 0
|
||||||
|
assert len(registry_report['excluded_volume35_dois']) == 1 and not registry_report['database_written']
|
||||||
|
rss_fixture = json.loads(gzip.decompress((src/'tests/fixtures/rss2026-source.json.gz').read_bytes()))
|
||||||
|
rss_snapshots = root/'rss2026-snapshots'
|
||||||
|
rss_snapshots.mkdir()
|
||||||
|
for name, html in rss_fixture['pages'].items():
|
||||||
|
(rss_snapshots/name).write_bytes(html.encode('utf-8'))
|
||||||
|
(rss_snapshots/'receipts.json').write_text(json.dumps(rss_fixture['receipts']))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--snapshots',str(rss_snapshots),'--out',str(root/'installed-rss2026')],cwd=outside,check=True)
|
||||||
|
assert json.loads((root/'installed-rss2026/collection-report.json').read_text())['records'] == 210
|
||||||
|
recsys_fixture = json.loads(gzip.decompress((src/'tests/fixtures/recsys2026-source.json.gz').read_bytes()))
|
||||||
|
recsys_snapshots = root/'recsys2026-snapshots'
|
||||||
|
recsys_snapshots.mkdir()
|
||||||
|
for name,encoded in recsys_fixture['files'].items():
|
||||||
|
path = recsys_snapshots/name
|
||||||
|
path.parent.mkdir(parents=True,exist_ok=True)
|
||||||
|
path.write_bytes(base64.b64decode(encoded))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','recsys2026-program','--snapshots',str(recsys_snapshots),'--out',str(root/'installed-recsys2026')],cwd=outside,check=True)
|
||||||
|
recsys_report = json.loads((root/'installed-recsys2026/collection-report.json').read_text())
|
||||||
|
assert (recsys_report['records'],recsys_report['unique_dois'],recsys_report['missing_abstract_count']) == (263,260,0)
|
||||||
|
assert recsys_report['duplicate_doi_observations'] == 6 and len(recsys_report['repeated_dois']) == 3
|
||||||
|
assert not recsys_report['database_written'] and recsys_report['network_requests'] == 0 and recsys_report['approved_for_import'] == 0
|
||||||
|
recsys_rows = [json.loads(line) for manifest in recsys_report['files'] for line in pathlib.Path(manifest['file']).read_text().splitlines()]
|
||||||
|
gold_keys = recsys_fixture['independent_gold'][0].keys()
|
||||||
|
assert [{k:r['metadata']['program_fields'][k] for k in gold_keys} for r in recsys_rows] == recsys_fixture['independent_gold']
|
||||||
|
ismir_fixture = json.loads(gzip.decompress((src/'tests/fixtures/ismir2026-source.json.gz').read_bytes()))
|
||||||
|
ismir_snapshots = root/'ismir2026-snapshots'
|
||||||
|
ismir_snapshots.mkdir()
|
||||||
|
for name,encoded in ismir_fixture['files'].items():
|
||||||
|
target = ismir_snapshots/name
|
||||||
|
target.parent.mkdir(parents=True,exist_ok=True)
|
||||||
|
target.write_bytes(base64.b64decode(encoded))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','ismir2026-program','--snapshots',str(ismir_snapshots),'--out',str(root/'installed-ismir2026')],cwd=outside,check=True)
|
||||||
|
ismir_report = json.loads((root/'installed-ismir2026/collection-report.json').read_text())
|
||||||
|
assert (ismir_report['records'],ismir_report['missing_abstract_count'],ismir_report['review_optin_yes'],ismir_report['review_optin_no']) == (140,0,101,39)
|
||||||
|
assert ismir_report['literal_title_version_differences'] == 2 and ismir_report['author_sequence_version_differences'] == 3
|
||||||
|
assert not ismir_report['database_written'] and ismir_report['network_requests'] == 0 and ismir_report['approved_for_import'] == 0
|
||||||
|
ismir_rows = [json.loads(line) for line in pathlib.Path(ismir_report['files'][0]['file']).read_text().splitlines()]
|
||||||
|
assert [(r['external_id'],r['title'],r['authors'],r['abstract']) for r in ismir_rows] == [('2026/'+g['program_id'],g['title'],g['authors'],g['abstract']) for g in ismir_fixture['independent_gold']]
|
||||||
|
assert all(not r['identifiers'] and not r['submission_identifiers'] and r['metadata']['paper_type'] is None for r in ismir_rows)
|
||||||
|
corl_fixture = json.loads(gzip.decompress((src/'tests/fixtures/corl2026-source.json.gz').read_bytes()))
|
||||||
|
corl_official = root/'corl2026-official'
|
||||||
|
corl_archive = root/'corl2026-archive'
|
||||||
|
corl_official.mkdir()
|
||||||
|
corl_archive.mkdir()
|
||||||
|
(corl_official/'accepted.html').write_text(corl_fixture['official_html'])
|
||||||
|
(corl_official/'research.json').write_text(json.dumps(corl_fixture['research']))
|
||||||
|
(corl_archive/'explorer-paper-metadata-filtered-v1.json').write_text(corl_fixture['filtered'])
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','corl2026-program','--official-snapshots',str(corl_official),'--archive-snapshots',str(corl_archive),'--out',str(root/'installed-corl2026')],cwd=outside,check=True)
|
||||||
|
corl_report = json.loads((root/'installed-corl2026/collection-report.json').read_text())
|
||||||
|
assert corl_report['records'] == 687 and corl_report['presentations'] == 780
|
||||||
|
assert len(corl_report['title_variants']) == 34 and corl_report['public_rejected_availability'] == 'unknown'
|
||||||
|
assert not corl_report['database_written'] and not corl_report['pdfs_downloaded']
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','www-archive','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','www-archive','--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--help'],cwd=outside,check=True)
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','neurips-program','--help'],cwd=outside,check=True)
|
||||||
|
neurips_snapshots = root/'neurips2026-program-snapshots'
|
||||||
|
neurips_snapshots.mkdir()
|
||||||
|
(neurips_snapshots/'program-json.response').write_bytes(gzip.decompress((src/'tests/fixtures/neurips2026-program.json.gz').read_bytes()))
|
||||||
|
(neurips_snapshots/'receipts.json').write_bytes((src/'tests/fixtures/neurips2026-receipts.json').read_bytes())
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','neurips-program','--snapshots',str(neurips_snapshots),'--out',str(root/'installed-neurips2026-program')],cwd=outside,check=True)
|
||||||
|
neurips_report = json.loads((root/'installed-neurips2026-program/neurips-program-2026.json').read_text())
|
||||||
|
assert neurips_report['records'] == 9004 and len(neurips_report['held_presentations']) == 139
|
||||||
|
assert len(neurips_report['exact_duplicate_event_rows']) == 1 and neurips_report['folded_extra_presentation_rows'] == 132
|
||||||
|
assert not neurips_report['database_written'] and not neurips_report['pdfs_downloaded']
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','dblp-family','--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--year','2025','--snapshots',str(src/'tests/fixtures/aamas2025'),'--out',str(root/'installed-aamas2025')],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--year','2025','--snapshots',str(src/'tests/fixtures/aamas2025'),'--out',str(root/'installed-aamas2025')],cwd=outside,check=True)
|
||||||
|
aamas2026 = json.loads(gzip.decompress((src/'tests/fixtures/aamas2026-fixture.json.gz').read_bytes()))
|
||||||
|
aamas2026_snapshots = root/'aamas2026-snapshots'
|
||||||
|
aamas2026_snapshots.mkdir()
|
||||||
|
(aamas2026_snapshots/'contents.html').write_text(aamas2026['contents'])
|
||||||
|
(aamas2026_snapshots/'receipts.json').write_text(json.dumps(aamas2026['receipts']))
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--year','2026','--snapshots',str(aamas2026_snapshots),'--out',str(root/'installed-aamas2026')],cwd=outside,check=True)
|
||||||
|
aamas2026_report = json.loads((root/'installed-aamas2026/aamas-proceedings-2026.json').read_text())
|
||||||
|
assert aamas2026_report['records'] == 639 and len(aamas2026_report['held_publications']) == 1
|
||||||
|
assert aamas2026_report['status_counts'] == {'accepted': 638, 'withdrawn': 1}
|
||||||
|
assert not aamas2026_report['database_written'] and not aamas2026_report['pdfs_downloaded']
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','iclr-archive','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','iclr-archive','--help'],cwd=outside,check=True)
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','acl','--help'],cwd=outside,check=True)
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','springer-selected','--help'],cwd=outside,check=True)
|
||||||
|
springer = json.loads(gzip.decompress((src/'tests/fixtures/springer-aistats1993-fixture.json.gz').read_bytes()))
|
||||||
|
for group in ('toc','chapters'):
|
||||||
|
folder = root/'springer-snapshots'/group
|
||||||
|
folder.mkdir(parents=True)
|
||||||
|
for name, content in springer[group].items():
|
||||||
|
(folder/name).write_text(content)
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','springer-selected','--toc-snapshots',str(root/'springer-snapshots/toc'),'--chapter-snapshots',str(root/'springer-snapshots/chapters'),'--out',str(root/'installed-springer')],cwd=outside,check=True)
|
||||||
|
springer_report = json.loads((root/'installed-springer/collection-report.json').read_bytes())
|
||||||
|
assert springer_report['records'] == 49 and not springer_report['database_written']
|
||||||
|
springer_manifest, = springer_report['files']
|
||||||
|
assert springer_manifest['status'] == 'partial' and springer_manifest['missing_abstract_count'] == 0
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','postgres','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','postgres','--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','edition-tracks','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','edition-tracks','--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','joint-editions','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','joint-editions','--help'],cwd=outside,check=True)
|
||||||
@@ -62,6 +294,7 @@ jobs:
|
|||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','public-openreview','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','public-openreview','--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','coverage','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','coverage','--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','histories','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','histories','--help'],cwd=outside,check=True)
|
||||||
|
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','source-identities','--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-m','paper_atlas','curate','identifiers','--help'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-m','paper_atlas','curate','identifiers','--help'],cwd=outside,check=True)
|
||||||
subprocess.run([python,'-I','-B','-c','from paper_atlas.postgres.writer import PostgresStore, WriteConnection'],cwd=outside,check=True)
|
subprocess.run([python,'-I','-B','-c','from paper_atlas.postgres.writer import PostgresStore, WriteConnection'],cwd=outside,check=True)
|
||||||
smoke = "from paper_atlas.db.store import Store; from paper_atlas.config import DEFAULT_REGISTRY; import json; assert {'COLM','MM','MLSys'} <= {v['acronym'] for v in json.loads(DEFAULT_REGISTRY.read_text())['venues']}; s=Store('installed.sqlite'); s.conn.close()"
|
smoke = "from paper_atlas.db.store import Store; from paper_atlas.config import DEFAULT_REGISTRY; import json; assert {'COLM','MM','MLSys'} <= {v['acronym'] for v in json.loads(DEFAULT_REGISTRY.read_text())['venues']}; s=Store('installed.sqlite'); s.conn.close()"
|
||||||
|
|||||||
@@ -6,10 +6,18 @@ Paper Atlas 将 arXiv、AI 会议论文集及公开投稿决定整理到可追
|
|||||||
|
|
||||||
项目仍在建设中。候选范围包含 43 个会议(含 ACM MM、MLSys),尚未完成全部历史数据和最终去重复核。实际进度以数据库、来源清单和覆盖报告为准,而不是仓库文件数量。
|
项目仍在建设中。候选范围包含 43 个会议(含 ACM MM、MLSys),尚未完成全部历史数据和最终去重复核。实际进度以数据库、来源清单和覆盖报告为准,而不是仓库文件数量。
|
||||||
|
|
||||||
|
当前主动补漏从 **2020 年**起;2000 年起已发现的名单、缺口与待办保存在[补漏恢复清单](docs/coverage/backfill-handoff-2026-11-10.md),下次从这里接续。已有旧数据保留,范围排除不算补齐。
|
||||||
|
|
||||||
|
AAMAS 2026 的官方 HTML 目录由 `collect aamas-proceedings --year 2026` 适配,区分 track 和发表形式,keynote 排除、撤稿证据单独保留;[采集与恢复说明](docs/aamas-proceedings.md)。采集不写数据库,仍需身份核验和增量导入。
|
||||||
|
|
||||||
历史 ICLR 2014–2016 官方录用页由 `collect iclr-archive` 适配,保留主会/workshop及重复展示证据;运行和身份核验边界见 [历史档案说明](docs/coverage/iclr-archive-2014-2016.md)。
|
历史 ICLR 2014–2016 官方录用页由 `collect iclr-archive` 适配,保留主会/workshop及重复展示证据;运行和身份核验边界见 [历史档案说明](docs/coverage/iclr-archive-2014-2016.md)。
|
||||||
|
|
||||||
|
`collect springer-selected` 核验限定的会后选集,分开保存会议年份、出版年份及选集事件身份。目前支持 AISTATS 1993 对应的 1994 年出版选集,原始会议决定仍须单独核查;配置、证据和恢复见 [会后选集说明](docs/springer-selected-volumes.md)。
|
||||||
|
|
||||||
## 项目结构
|
## 项目结构
|
||||||
|
|
||||||
|
ACL 1963 的 AMTCL 年会摘要与 1978 的 TINLAP-2 来源关系均按精确卷适配,保留原系列、出版形式和不明逐篇角色;不将 EarlyMT 全系列映射到 ACL,1965 的替代关系也不转成全局别名。来源、排除条目和恢复边界见 [ACL 早期来源核查](docs/coverage/acl-early-sources-2026-11-09.md)。
|
||||||
|
|
||||||
```text
|
```text
|
||||||
paper_atlas/
|
paper_atlas/
|
||||||
cli.py 统一命令入口
|
cli.py 统一命令入口
|
||||||
@@ -62,12 +70,16 @@ paper-atlas collect colm --cache /data/paper-index/raw/http-cache \
|
|||||||
--out /data/paper-index/collected
|
--out /data/paper-index/collected
|
||||||
```
|
```
|
||||||
|
|
||||||
|
NeurIPS 2026 的[官方节目采集器](docs/neurips-program.md)按 main、evaluations/datasets、position 分开保存,合并同一 forum 的口头与海报展示,保留期刊等未决关系;运行 `paper-atlas collect neurips-program`。它只生成元数据文件,身份核验和生产导入另行执行。
|
||||||
|
|
||||||
早期主系列的Workshop名称按[ICML/IROS精确届次证据](docs/coverage/historical-primary-series-2026-11-09.md)识别,保留原名与范围证据;代码支持不代表已补齐生产数据。
|
早期主系列的Workshop名称按[ICML/IROS精确届次证据](docs/coverage/historical-primary-series-2026-11-09.md)识别,保留原名与范围证据;代码支持不代表已补齐生产数据。
|
||||||
|
|
||||||
EWSL、EPS、ECP、EWSP 的[具体前身卷与完整快照要求](docs/coverage/historical-predecessors-2026-11-09.md)单独核验,保留原始系列及事件/出版年份;旧候选文件缺少 ECP/EWSP 时会明确提示,避免静默遗漏。
|
EWSL、EPS、ECP、EWSP 的[具体前身卷与完整快照要求](docs/coverage/historical-predecessors-2026-11-09.md)单独核验,保留原始系列及事件/出版年份;旧候选文件缺少 ECP/EWSP 时会明确提示,避免静默遗漏。
|
||||||
|
|
||||||
AAMAS 的[官方论文集采集器](docs/aamas-proceedings.md)支持已核验的 2012、2025 版式,可从缓存或已校验快照采集元数据。2025 年目录含 476 条非 keynote 出版物,保留完整作者和机构,HTML 未提供摘要时留空;发表形式与投稿 track 分开,不根据主题推定主会轨道或研究类型。
|
AAMAS 的[官方论文集采集器](docs/aamas-proceedings.md)支持已核验的 2012、2025 版式,可从缓存或已校验快照采集元数据。2025 年目录含 476 条非 keynote 出版物,保留完整作者和机构,HTML 未提供摘要时留空;发表形式与投稿 track 分开,不根据主题推定主会轨道或研究类型。
|
||||||
|
|
||||||
|
旧 DBLP 候选文件可能漏掉新命名空间。`collect dblp-family` 可从已校验的固定压缩快照和完整目录中[恢复指定 family 的原始候选](docs/dblp-family-recovery.md),例如 `ifaamas`;恢复文件保留卷、keynote 和 workshop 等原始条目,需另做来源范围与身份核验。
|
||||||
|
|
||||||
`paper-atlas collect --help` 查看来源;`paper-atlas import --help` 查看导入方式。完整批次步骤见[使用指南](docs/usage.md)。
|
`paper-atlas collect --help` 查看来源;`paper-atlas import --help` 查看导入方式。完整批次步骤见[使用指南](docs/usage.md)。
|
||||||
|
|
||||||
共享实例名为 `foundry-postgres`,数据位于 `foundry_data` 库的 `research` schema;其他项目可按权限复用同一组表。库名通过 libpq service 配置、schema 通过 search_path 或 `--pg-schema` 指定,不写死在查询中。PostgreSQL 凭据放在服务器权限为 `0600` 的 libpq service 文件中;命令行只使用服务名称。`paper-atlas maintain postgres --help` 提供迁移入口:先完整核验 21 张业务表,再发布目标 schema。SQLite FTS5 内部表由 PostgreSQL GIN 索引替代。切换后 SQLite 为只读备份;`import records`、`import arxiv` 和 `maintain backfill` 使用 `--pg-service foundry-research` 写入主库,整个写入会话只允许一个协调写入者,会议文件失败时整体回滚并保留失败批次记录。
|
共享实例名为 `foundry-postgres`,数据位于 `foundry_data` 库的 `research` schema;其他项目可按权限复用同一组表。库名通过 libpq service 配置、schema 通过 search_path 或 `--pg-schema` 指定,不写死在查询中。PostgreSQL 凭据放在服务器权限为 `0600` 的 libpq service 文件中;命令行只使用服务名称。`paper-atlas maintain postgres --help` 提供迁移入口:先完整核验 21 张业务表,再发布目标 schema。SQLite FTS5 内部表由 PostgreSQL GIN 索引替代。切换后 SQLite 为只读备份;`import records`、`import arxiv` 和 `maintain backfill` 使用 `--pg-service foundry-research` 写入主库,整个写入会话只允许一个协调写入者,会议文件失败时整体回滚并保留失败批次记录。
|
||||||
@@ -104,3 +116,31 @@ CI 在已授权的远程 runner 上运行测试、构建 wheel,并从源码目
|
|||||||
整合后的运行版本依赖、交互约束和生产切换顺序见 [运行版本整合](docs/research-runtime.md)。
|
整合后的运行版本依赖、交互约束和生产切换顺序见 [运行版本整合](docs/research-runtime.md)。
|
||||||
|
|
||||||
已导入整卷的逐条轨道修正见 [edition-tracks 维护流程](docs/edition-tracks.md):按完整证据生成只读计划,保留原投稿和决定 ID,事务性修正 edition、标识与覆盖,并防止旧整卷刷新覆盖分类。
|
已导入整卷的逐条轨道修正见 [edition-tracks 维护流程](docs/edition-tracks.md):按完整证据生成只读计划,保留原投稿和决定 ID,事务性修正 edition、标识与覆盖,并防止旧整卷刷新覆盖分类。
|
||||||
|
|
||||||
|
入库前可使用通用的[来源身份只读预检](docs/source-identity-audit.md),导出候选论文、稳定标识和既有投稿决定证据;候选结果需要审核,不会自动批准导入。
|
||||||
|
|
||||||
|
RSS 2026 已独立适配官方论文集与录用表;使用方法和与 2025 demo 分类的区别见 [RSS 2026 来源说明](docs/rss2026-proceedings.md)。
|
||||||
|
|
||||||
|
ICAPS 2026 的[官方名单采集器](docs/icaps2026-program.md)区分 90 篇主会轨道组录用与 10 篇既有论文展示;逐篇 track、摘要和稳定标识尚未解决,输出均待审核,不能直接视作已完成入库。
|
||||||
|
|
||||||
|
ICAPS 2026 也提供独立的 `collect icaps2026-registry`,保留 Crossref 卷 36 的 DOI、作者与摘要版本;不覆盖节目表文字,不把卷 35 或版本差异自动并入。配置与边界见同一份[ICAPS 来源说明](docs/icaps2026-program.md)。
|
||||||
|
|
||||||
|
### CoRL 2026 官方程序快照
|
||||||
|
|
||||||
|
`collect corl2026-program` 将官方 687 个论坛与固定公开档案按 ID 对照,保留 34 条标题版本差异及分字段来源。不会自动接受 arXiv 匹配或推断拒稿;详情见 [CoRL 2026 使用与恢复说明](docs/corl2026-program.md)。
|
||||||
|
|
||||||
|
ECCV 2026 的[暂定节目采集器](docs/eccv2026-program.md)合并重复节目 ID,并以不可变缓存断点采集公开元数据;保留摘要原文、来源关键词及未分类 track,始终待身份审核,不自动入库。
|
||||||
|
|
||||||
|
`collect ismir2026-program` 核验 140 项 ISMIR 官方录用节目及原始摘要,保留两处标题、三处作者名单差异和 101 组自愿公开评审文本;不推断原始 OpenReview ID 或拒稿。见 [ISMIR 2026 使用与恢复说明](docs/ismir2026-program.md)。
|
||||||
|
|
||||||
|
`collect icassp2026-registry` 从冻结的 10 页 Crossref 元数据核验 4,582 条论文,明确排除 7 条非论文及 56 条其他会议记录;保留摘要缺失和单字段作者姓名,不自动融合节目表。见 [ICASSP 2026 采集与恢复说明](docs/icassp2026-registry.md)。
|
||||||
|
|
||||||
|
`collect sdm2026-proceedings` 核验 53 个 Crossref 论文章节,并单独保留 53 项日程观察及 52 页摘要。作者差异、取消展示和摘要缺失分别保留,不自动拼接来源或绑定作品;见 [SDM 2026 使用与恢复说明](docs/sdm2026-proceedings.md)。
|
||||||
|
|
||||||
|
`collect icdm2026-program` 离线核验 324 条 ICDM2026 官方线下日程,按明确的研究/应用标签输出 241/83 条;展示长短、缺失摘要及未核实出版身份分别保留,不自动入库。见 [ICDM2026 使用与恢复说明](docs/icdm2026-program.md)。
|
||||||
|
|
||||||
|
`collect recsys2026-program` 核验九个 track 的 263 项官方录用贡献观察(260 个不同 DOI),保留摘要、原始作者和重复 DOI 的来源版本。见 [RecSys 2026 使用与恢复说明](docs/recsys2026-program.md)。
|
||||||
|
|
||||||
|
`collect mm2026-program` 从已冻结官方快照恢复全部 28 个来源 track 的 2,143 个节目 ID,保留 4,326 次展示的来源位置与场次上下文,单列 216 个摘要缺失项;输出供贡献类型、出版及作品身份核查使用,不是可直接导入的论文/决定队列。见 [MM 2026 使用、来源和恢复说明](docs/mm2026-source-status.md)。
|
||||||
|
|
||||||
|
`collect mm2026-publications` 核验完整论文集的 1,918 个注册出版 DOI,并单独关联节目分轨、展示上下文和摘要。1,743 份节目摘要明确标注来源(其中 26 份来自已复核的标题排版对应),166 项标题/作者版本差异保留为待核;所有节目及未匹配项均保留。注册文章类型不等于研究类型,输出尚需数据库身份/历史审查。见 [MM 2026 出版条目采集与恢复](docs/mm2026-publications.md)。
|
||||||
|
|||||||
@@ -1,6 +1,31 @@
|
|||||||
# AAMAS 官方论文集元数据
|
# AAMAS 官方论文集元数据
|
||||||
|
|
||||||
`paper_atlas.collectors.aamas` 是按需运行的采集适配器,通过统一入口 `collect aamas-proceedings` 使用。支持已独立核验的 **AAMAS 2012、2025** HTML 版式;其他年份需先核对官方目录和逐篇结构,再增加适配,不能仅替换年份。2025 解析单独封装在 `paper_atlas.collectors.aamas2025`。
|
`paper_atlas.collectors.aamas` 是按需运行的采集适配器,通过统一入口 `collect aamas-proceedings` 使用。支持已独立核验的 **AAMAS 2012、2025、2026** HTML 版式;其他年份需先核对官方目录和逐篇结构,再增加适配,不能仅替换年份。2025、2026 解析分别封装在 `paper_atlas.collectors.aamas2025`、`paper_atlas.collectors.aamas2026`。
|
||||||
|
|
||||||
|
|
||||||
|
## AAMAS 2026
|
||||||
|
|
||||||
|
2026 只读取[官方 HTML 目录](https://www.ifaamas.org/Proceedings/aamas2026/forms/contents.htm),不下载 PDF。冻结目录 SHA256 `ce2bc7fc5da340930343901506a9ce37c0334d4068f07546431c9deee3f13de9`,643 个条目:keynote 3、research 338、extended abstract 185、AAAI 19、Blue Sky 17、doctoral 34、demo 44、JAAMAS 3。
|
||||||
|
|
||||||
|
排除 3 个 keynote;research 的 p238 明确标注 `Retracted June 8, 2026`,完整题名、作者、栏目、日期、原文和来源回执保存在采集报告 `held_publications` 中,**不会输出 accepted 或 withdrawn 投稿记录**。当前决定模型没有独立 retracted 状态,后续须单独建模和核验;原始证据不能删除。任何新增或改变的撤稿标记都会停止采集,等待人工核查。
|
||||||
|
|
||||||
|
另一个 extended abstract p3178 明确标注 `Withdrawn June 8, 2026`,输出 `withdrawn`,在 metadata 保留日期 `2026-06-08` 和日精度;原网页没有时区/时刻,不能伪造决定 timestamp,不补造其此前的接受决定或 PDF 链接。因此 639 个输出条目是 **638 accepted、1 withdrawn**,按六个 track 分组:`proceedings_unspecified` 522、`aaai_transfer` 19、`blue_sky` 17、`doctoral` 34、`demo` 44、`jaamas` 3。全文/扩展摘要是发表形式,不是研究类型;research 栏目不能证明每篇原始 main/special 投稿轨道。JAAMAS 保留期刊展示语义,不制造另一次独立审稿录用。
|
||||||
|
|
||||||
|
[AAAI 转投详细政策](https://cyprusconferences.org/aamas2026/call-for-papers-aaai-track/) 要求基于 AAAI-26 rejected 稿件;[总览](https://cyprusconferences.org/aamas2026/calls/) 却写 AAAI-25,须保留冲突。采集器保留 AAAI 栏目归属,不据此生成原 AAAI 投稿 ID、年份或逐篇拒稿决定。目录不是原始决定 note,未来关联仍需逐篇证据。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect aamas-proceedings --year 2026 \
|
||||||
|
--snapshots /data/paper-index/collected/aamas2026-primary-evidence-v1 \
|
||||||
|
--out /data/paper-index/collected/aamas2026-normalized-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
在线模式将 `--snapshots` 换成 `--cache /data/paper-index/cache/aamas2026`。冻结模式 `receipts.json` 的 `receipts` 至少包含官方目录 URL、相对 `file`、`bytes`、`sha256`、`status=200` 和带时区 `finished_at`。拒绝越界路径、越界符号链接、失败状态、哈希/大小不符的页面。这里只读取目录对应回执,其他已冻结政策网页继续保留。
|
||||||
|
|
||||||
|
输出六组 JSONL/manifest、原始 HTML 与最后发布的 `aamas-proceedings-2026.json`;统一入口沿用输出目录锁和暂存目录。manifest 全部为 `partial`,639 条 HTML 摘要缺失如实记录。发布编号使用年份和起始页,随机 PDF 文档 ID 单独保留,不能沿用 2025 的 pN.pdf 规则。
|
||||||
|
|
||||||
|
首次部署先核对来源哈希和生产状态,再在新输出目录运行采集、独立比对 643 条标题/作者/栏目分区,随后做 PostgreSQL 身份预检与增量导入。采集、远程 CI 通过或目录含某个年份均不代表生产已导入、摘要齐全或最终去重完成。较早年份不自动启动补采,当前主动补漏范围按用户要求从 2020 年起。
|
||||||
|
|
||||||
|
## AAMAS 2025
|
||||||
|
|
||||||
2025 版式使用[官方首页](https://www.ifaamas.org/Proceedings/aamas2025/)和[出版目录](https://www.ifaamas.org/Proceedings/aamas2025/forms/contents.htm)。原始目录共 479 条:3 keynote、260 research paper、148 extended abstract、12 Blue Sky、25 doctoral、22 Demo 栏目、9 JAAMAS。排除 keynote,输出 476 条;Demo 栏目中的 p3000 是明确标题的 COMP24 竞赛报告,发表年份仍为 2025,格式记 `competition_report`、投稿 track 留作 `proceedings_unspecified`,保留原 Demo 栏目证据。其余输出按 track 分组为 409 unspecified、12 blue_sky、25 doctoral、21 demo、9 jaamas。
|
2025 版式使用[官方首页](https://www.ifaamas.org/Proceedings/aamas2025/)和[出版目录](https://www.ifaamas.org/Proceedings/aamas2025/forms/contents.htm)。原始目录共 479 条:3 keynote、260 research paper、148 extended abstract、12 Blue Sky、25 doctoral、22 Demo 栏目、9 JAAMAS。排除 keynote,输出 476 条;Demo 栏目中的 p3000 是明确标题的 COMP24 竞赛报告,发表年份仍为 2025,格式记 `competition_report`、投稿 track 留作 `proceedings_unspecified`,保留原 Demo 栏目证据。其余输出按 track 分组为 409 unspecified、12 blue_sky、25 doctoral、21 demo、9 jaamas。
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
# CoRL 2026:官方程序与公开档案
|
||||||
|
|
||||||
|
这是长期维护、按需运行的来源适配器。命令只读已冻结的元数据快照,生成待核验文件;不连接数据库,不抓取 PDF、评审或作者个人档案,不执行网页脚本,也不尝试通过 OpenReview 验证挑战。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect corl2026-program \
|
||||||
|
--official-snapshots /data/paper-index/collected/corl2026-official-listing-research-v1 \
|
||||||
|
--archive-snapshots /data/paper-index/collected/corl2026-public-archive-inventory-v1 \
|
||||||
|
--out /data/paper-index/collected/corl2026-normalized-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
官方快照包含 `accepted.html` 和 `research.json` 的 HTTP 回执。适配器解析 Google Sites 的 `data-code`,不把外层文本提取为空误作无论文。687 个论坛有 780 次展示:687 次海报、21 次口头、72 次 Spotlight。论坛 ID 确定投稿身份,展示形式不是新的投稿或 track。
|
||||||
|
|
||||||
|
作者及摘要来自 `gisbi-kim/conference-explorers` 的固定提交 `75b3f7a04b3a77c454540fe2506bfa615b8e038e`,经字段过滤后的 `explorer-paper-metadata-filtered-v1.json`。输入、上游原始数据 SHA-256 及 Git blob 均固定;改变来源需要另行研究、更新快照及检查,不能静默接受。
|
||||||
|
|
||||||
|
这是独立整理者的版本证据,并非本次重新核验的 OpenReview 原页。每条保留档案标题、作者、摘要、分字段来源 URL、官方标题及全部展示场次。34 个官方与档案标题差异会显式标记,不能视作勘误或自动作者别名。规范记录中的标题与作者摘要保持档案版本一致;官方标题单独保留,避免拼成没有来源的版本。
|
||||||
|
|
||||||
|
档案的 arXiv 链接和匹配标题只存为未批准候选,不进入作品标识;不沿用其推断机构、个人档案、作者 ID 或生成标签。只有 OpenReview 论坛 ID 进入作品和投稿标识。
|
||||||
|
|
||||||
|
所有输出均为 partial。官方政策只证实录用论文及评审公开,没有证实 rejected 集可见性。不能由此生成拒稿记录,也不能声称这 687 条覆盖了公开拒稿或全部论文关系。
|
||||||
|
|
||||||
|
输出目录必须全新;失败留下 `failed.json`,成功报告最后原子发布。恢复时先检查原目录,不覆盖失败现场或重跑已消费的导入队列。生成文件仍需元数据版本审阅、当前数据库身份候选检查、独立保护基线及导入后核验,最后另做全库去重。
|
||||||
|
|
||||||
|
测试和安装验证只在用户授权的 `ces-control:host` runner 上运行。
|
||||||
@@ -1,5 +1,9 @@
|
|||||||
# 会议覆盖与公开决定核查
|
# 会议覆盖与公开决定核查
|
||||||
|
|
||||||
|
最新生产验收与剩余工作见 [2026-11-17 补漏状态](backfill-status-2026-11-17.md):当日 `14:06:28 UTC` 固定范围快照中,278 个会议年份槽位有 272 个含记录、6 个仍为空;这不代表完整覆盖,第 14 步仍未完成。后续 MM 规范化、只读候选审计与出版核查不等于生产导入。
|
||||||
|
|
||||||
|
**当前工作范围:2020 年起主动补漏。** 2000 年起已发现资料保留,恢复工作从 [补漏范围与恢复清单(2026-11-10)](backfill-handoff-2026-11-10.md) 开始;下文旧日期快照不改写,较早年份排除不算补齐。
|
||||||
|
|
||||||
这是 2026-11-08 的首份核查基线,**完整逐年政策普查和最终去重仍未完成**。收录 43 个目标,不能把“有记录”当作“完整覆盖”。结构化清单见 [inventory-2026-11-08.json](inventory-2026-11-08.json)。清单中的 `policy_checks` 均限定所列年份和轨道;空数组表示尚未核实,绝不表示没有公开拒稿。
|
这是 2026-11-08 的首份核查基线,**完整逐年政策普查和最终去重仍未完成**。收录 43 个目标,不能把“有记录”当作“完整覆盖”。结构化清单见 [inventory-2026-11-08.json](inventory-2026-11-08.json)。清单中的 `policy_checks` 均限定所列年份和轨道;空数组表示尚未核实,绝不表示没有公开拒稿。
|
||||||
|
|
||||||
## 快照与复核
|
## 快照与复核
|
||||||
@@ -68,7 +72,7 @@ PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas coverage \
|
|||||||
|
|
||||||
## 优先缺口
|
## 优先缺口
|
||||||
|
|
||||||
- **UAI 2024–2026**:官方均允许作者选择公开拒稿,当前主库拒稿仍为零。原 notes 接口要求 Challenge 验证,公开镜像尚无已核验来源,必须保留这个真实缺口;不能根据 PMLR 录用名单推断拒稿。参见 [2025 投稿指南](https://www.auai.org/uai2025/submission_instructions)、JSON 内其他年份官方证据。
|
- **UAI 2024–2026**:官方均允许作者选择公开拒稿,首份生产快照拒稿为零。原 notes 接口要求 Challenge 验证;新核验镜像的 2022–2025 年 902 个论坛来自录用子集,不能填补公开拒稿缺口。2024 年的元评审也不能直接转成最终决定,见 [2026-11-17 原始来源与回复结构核查](uai-public-negative-source-status-2026-11-17.md)。不能根据 PMLR 或镜像录用名单反推拒稿;政策参见 [2025 投稿指南](https://www.auai.org/uai2025/submission_instructions)、JSON 内其他年份官方证据。
|
||||||
- **EMNLP 2023**:官方允许公开 opt-in 拒稿,目前只导入 9 条。需继续核验实际公开全集。ARR 评审、conference commitment 与最终会议决定分开;[ARR 作者说明](https://aclrollingreview.org/authors) 明确其自愿捐赠数据与 EMNLP 2023 直接投稿发布的范围不同。
|
- **EMNLP 2023**:官方允许公开 opt-in 拒稿,目前只导入 9 条。需继续核验实际公开全集。ARR 评审、conference commitment 与最终会议决定分开;[ARR 作者说明](https://aclrollingreview.org/authors) 明确其自愿捐赠数据与 EMNLP 2023 直接投稿发布的范围不同。
|
||||||
- **ICLR**:固定快照已观察 2017 与 2020–2026;快照之后,2013/2018/2019 的 1,654 条来源记录已完成独立 PostgreSQL 验收(报告时间 `2026-11-07T19:52:28.378Z`),不回写旧快照计数,也不据此宣称逐届全集已齐。快照中的 10,208 次 unknown 不等于 rejected,近期补采与冲突核对仍在进行;评审推荐不能当最终决定。
|
- **ICLR**:固定快照已观察 2017 与 2020–2026;快照之后,2013/2018/2019 的 1,654 条来源记录已完成独立 PostgreSQL 验收(报告时间 `2026-11-07T19:52:28.378Z`),不回写旧快照计数,也不据此宣称逐届全集已齐。快照中的 10,208 次 unknown 不等于 rejected,近期补采与冲突核对仍在进行;评审推荐不能当最终决定。
|
||||||
- **MM / MLSys**:MM 11,200 条书目记录保留 `proceedings_unspecified`;MLSys 的 448 次官方投稿与 313 条 DBLP 记录重叠。2026 MLSys 的研究/工业轨道在 [官方 CFP](https://mlsys.org/Conferences/2026/CallForResearchPapers) 中分开,仍须核对逐篇轨道证据,不能仅凭会议名全部归 main。
|
- **MM / MLSys**:MM 11,200 条书目记录保留 `proceedings_unspecified`;MLSys 的 448 次官方投稿与 313 条 DBLP 记录重叠。2026 MLSys 的研究/工业轨道在 [官方 CFP](https://mlsys.org/Conferences/2026/CallForResearchPapers) 中分开,仍须核对逐篇轨道证据,不能仅凭会议名全部归 main。
|
||||||
@@ -185,6 +189,8 @@ ICAPS官方共同历史明确列出两个前身系列,现有DBLP映射只覆
|
|||||||
|
|
||||||
MM 2026 的[官方 CFP](https://2026.acmmm.org/site/cfp-guidelines.html)说明会后公开最终评审及适用的作者回复,但未明确包含全部被拒投稿。2026-11-08 对官方组的匿名 API 查询返回200,对该组 `venueid` 的投稿查询返回403;[证据](mm-editions-2026-11-08.json)保存查询、时间和响应哈希。公开拒稿可用性仍为 unknown,403不能作为零记录或不存在公开拒稿的证据。
|
MM 2026 的[官方 CFP](https://2026.acmmm.org/site/cfp-guidelines.html)说明会后公开最终评审及适用的作者回复,但未明确包含全部被拒投稿。2026-11-08 对官方组的匿名 API 查询返回200,对该组 `venueid` 的投稿查询返回403;[证据](mm-editions-2026-11-08.json)保存查询、时间和响应哈希。公开拒稿可用性仍为 unknown,403不能作为零记录或不存在公开拒稿的证据。
|
||||||
|
|
||||||
|
2026-11-17后续:完整节目JSON与按声明的负面venue ID进行的四条匿名403核查已冻结,见 [MM最新来源状态](../mm2026-source-status.md);没有据此关闭公开拒稿缺口。
|
||||||
|
|
||||||
WWW早期逐来源核查见 `www-early-proceedings-2026-11-08.json`:1995春季论文三目录均为38个相同条目ID,海报摘要53个条目与作者链接47个条目不齐;1994春季50个Title字段与主页49场报告说法待逐项核对。上述为源目录观察,不是已导入或全部录用数量;主会、海报、教程等继续独立核验。
|
WWW早期逐来源核查见 `www-early-proceedings-2026-11-08.json`:1995春季论文三目录均为38个相同条目ID,海报摘要53个条目与作者链接47个条目不齐;1994春季50个Title字段与主页49场报告说法待逐项核对。上述为源目录观察,不是已导入或全部录用数量;主会、海报、教程等继续独立核验。
|
||||||
|
|
||||||
WWW事件标识生产迁移已独立验收(报告SHA `f5dbb8a8`):保留1399个原届次字段及IDs、59个Fall投稿和完整全库历史哈希。服务器已缓存46个官方HTML来源(约223KB),只作为后续维护式采集器的输入;尚未补导早期WWW论文,也未启用可选注解表。
|
WWW事件标识生产迁移已独立验收(报告SHA `f5dbb8a8`):保留1399个原届次字段及IDs、59个Fall投稿和完整全库历史哈希。服务器已缓存46个官方HTML来源(约223KB),只作为后续维护式采集器的输入;尚未补导早期WWW论文,也未启用可选注解表。
|
||||||
@@ -194,3 +200,5 @@ IJCAI 联办届次的逐条后续核查见 [2018/2022 分区与目录差集](ijc
|
|||||||
2026年11月9日的[当前历史年度缺口核对](historical-gaps-2026-11-09.md)单列135个历史会议—年份、14个2026年度及85个经范围修正的初步候选来源年度;它使用当前生产edition/别名快照,保留原固定快照不变。年度有记录不等于逐篇、逐track或公开拒稿历史完整。
|
2026年11月9日的[当前历史年度缺口核对](historical-gaps-2026-11-09.md)单列135个历史会议—年份、14个2026年度及85个经范围修正的初步候选来源年度;它使用当前生产edition/别名快照,保留原固定快照不变。年度有记录不等于逐篇、逐track或公开拒稿历史完整。
|
||||||
|
|
||||||
[早期NLP来源范围复核](historical-source-scope-2026-11-09.md)保留3份官方网页的响应哈希,排除4个COLING卷对ACL缺口的误匹配,并区分会议举办记录、目录可获取性与论文级覆盖;尚未导入或关闭这些年度缺口。
|
[早期NLP来源范围复核](historical-source-scope-2026-11-09.md)保留3份官方网页的响应哈希,排除4个COLING卷对ACL缺口的误匹配,并区分会议举办记录、目录可获取性与论文级覆盖;尚未导入或关闭这些年度缺口。
|
||||||
|
|
||||||
|
- [MM 2026 官方节目来源状态](../mm2026-source-status.md):完整官方日程 JSON、重复展示盘点、28 个来源 track、缺失作者/摘要和四条匿名决定路径的 403 证据;这些不是已导入或公开拒稿全集。
|
||||||
|
|||||||
@@ -0,0 +1,64 @@
|
|||||||
|
# ACL 早期年份:会议存在、出版目录与逐篇资料分开核验
|
||||||
|
|
||||||
|
固定普查中 ACL 1963–1978 仍有年份缺口。不能因为会议年表连续,就假设每年都有独立的正式论文集;也不能因为当前目录始于 1979,就断言此前没有摘要、通知或其他公开资料。本次不修改固定的 1,266 个目标年份基准。
|
||||||
|
|
||||||
|
## 1978 年:限定 T78-1 的来源关系
|
||||||
|
|
||||||
|
[同期年会通知](https://aclanthology.org/J77-4001.pdf)的 PDF 第 9–10 页(印刷页码 9–10)明确说明,1978 年 ACL 年会采用第二届 Theoretical Issues in Natural Language Processing 的形式,地点为 Urbana/Champaign,日期为 7 月 25–27 日。通知中的 ACM 联合赞助确认仍处于待确认状态,不能将该预告改写为已完成的行政决定。
|
||||||
|
|
||||||
|
[1982 年 Donald E. Walker 的历史回顾](https://aclanthology.org/www.mt-archive.info/70/ACL-1982-Walker-1.pdf)第 3 页(印刷页码 91)的年会表及缩写说明,独立对应了 1978 年的时间、地点与 TNLP/TINLAP-2 关系。两份扫描件相关页已渲染并逐页目视核对,不仅依靠 OCR。
|
||||||
|
|
||||||
|
[官方卷目录](https://aclanthology.org/volumes/T78-1/)与固定 XML `T78.xml` 保留原系列名 TINLAP、完整卷名、1978 年及编辑 David L. Waltz。适配仅接受精确的 collection `T78`、volume `1` 和上述字段组合;关键元数据变化会拒绝处理,等待重新核查。
|
||||||
|
|
||||||
|
- 规范事件名使用 `ACL+TINLAP`,原系列名 `TINLAP`、原始 XML 的 `venue_codes: [tinlap]` 及证据均保留。
|
||||||
|
- ACL/TINLAP 别名只挂在这一规范事件系列上,不把所有 TINLAP 年份都算成 ACL。T75、T87 及 T78 其他卷没有此例外。
|
||||||
|
- 通知描述了邀请与提交论文两种参与方式,XML 没有逐篇角色。统一暂存为 `proceedings_unspecified`,不猜测每篇属于 invited、main 或 workshop。
|
||||||
|
- XML 共 36 条条目,其中 T78-1036 是无作者的场次讨论问题列表。精确识别并在 manifest 的 `excluded_materials` 保留其标识、标题与原因;其余 35 条作为具名贡献整理。该排除不推广到所有无作者条目,已识别条目发生变化也须重新核查。
|
||||||
|
- 全部 35 条 XML 均无摘要。出版目录中的 accepted 是来源观察;通知提及录用/不录用通知时间,不提供任何可识别的拒稿决定,不能制造 rejected 或 withdrawn。
|
||||||
|
|
||||||
|
采集器仅生成元数据文件,不写业务数据库。真实来源逐条验收、PostgreSQL 身份预检、串行导入与独立验收仍是后续门槛。35 条来源不等于 35 个新增论文实体,更不等于完成全库去重。
|
||||||
|
|
||||||
|
## 1963 年:AMTCL 旧名称与 EarlyMT 分类分开保留
|
||||||
|
|
||||||
|
[ACL 官方历史介绍](https://2023.aclweb.org/participants/about_acl/)确认 AMTCL 是协会旧名称,1968 年改称 ACL。[1963 官方卷目录](https://aclanthology.org/volumes/1963.earlymt-1/)和固定 XML `1963.earlymt.xml` 明确记录 AMTCL 年会、Denver、1963 年 8 月 25–26 日。源目录将该卷归在 EarlyMT;这是资料分类,不能把其他 EarlyMT 会议全部变成 ACL。
|
||||||
|
|
||||||
|
适配仅接受 collection `1963.earlymt`、volume `1`,并核对旧名称全称、年份、地点、日期和原始 venue code。规范会议为 ACL,保留 AMTCL 别名及原始 `earlymt` 分类。关键字段变化或重复时拒绝处理,等待重新审核。
|
||||||
|
|
||||||
|
34 条均有作者和摘要,来源是出版摘要清单,记录 `published_abstract` 出版形式及 `acl_anthology:publication_format` 标签;逐篇 track 未提供,使用 `proceedings_unspecified`。`accepted` 只表示被列入该出版清单,不推断现代 main track、原始评审决定或任何拒稿记录。
|
||||||
|
|
||||||
|
XML commit 与下文相同,SHA256 为 `4a4626d792ac507de0281cb358769f7246f4a0d6b6a9176ac02e61545c1635fc`。原始 fixture 为 `tests/fixtures/acl-1963-earlymt.xml`。官方历史 HTML SHA256 为 `0f6bc260c2b34c9e2a51d971aeb43c3270a5b15b6db1d6d41e50669f316345f3`,响应回执与固定 XML 保存在服务器 `/data/paper-index/collected/acl1963-scope-evidence-v1`。另有卷目录 HTML 回执保存在 `/data/paper-index/collected/historical-primary-candidates-20261109-v1`。未下载论文 PDF。
|
||||||
|
|
||||||
|
使用下文相同采集命令,将 `--collection` 改为 `1963.earlymt`,输出到新的独立目录。采集结果需逐条验收和 PostgreSQL 身份预检,再进入增量导入;34 条来源不等于 34 个新增论文实体,代码测试通过也不代表生产覆盖已补齐。
|
||||||
|
|
||||||
|
## 1965 年:替代关系不改写为别名
|
||||||
|
|
||||||
|
Walker 回顾第 1 页(印刷页码 89)说明,1965 年 ICCL(后来称 COLING)替代了当年的年会。[官方 C65-1 目录](https://aclanthology.org/volumes/C65-1/)及 XML 仍以 COLING 标记该卷。这是限定年份的历史关系,不证明 COLING 与 ACL 是两个可全局互换的名字。
|
||||||
|
|
||||||
|
本次保留 C65 原有 `COLING` 归属,不添加 ACL 全局别名、不重放已导入记录、不将某一年的日历关系推广到整个系列,也不据此悄悄减少固定普查缺口。其他 1963–1977 年资料仍须逐年核查;本次不能证明这些年份整体属于非归档例外。
|
||||||
|
|
||||||
|
## 固定来源与恢复
|
||||||
|
|
||||||
|
服务器证据目录:`/data/paper-index/collected/acl-early-source-evidence-v1`。五份官方响应均 HTTP 200,另有两份来自既有官方 XML checkout 的文件,已与 Git 对象逐字节核对。未下载论文全文集合;两份 PDF 是本次历史/行政取证资料,不进入论文全文库。
|
||||||
|
|
||||||
|
| 文件 | SHA256 |
|
||||||
|
|---|---|
|
||||||
|
| `acl-venue.html` | `c60f64cb4d72e677aeb84a5844cde28a8d50579e1995f08e8c7801eed1e3fac5` |
|
||||||
|
| `acl-history-1982.pdf` | `35c21bd4983975bcaccbb94bd8ab91c9019fc190feec1791255407cf18a7e49a` |
|
||||||
|
| `acl-1978-notice.pdf` | `14b5deb471c55afb07d789529b95d0462a2a1a77df3dfba7fa1f5d3ab75eb411` |
|
||||||
|
| `tinlap1978-volume.html` | `7e20d709f72f415819e4d17e3674b19dc7d871eabcf0cf8b4935f0392d775729` |
|
||||||
|
| `coling1965-volume.html` | `0d2f2f49f2a311f9c0b80a8832f6476eb32f94f35abf36f6d2f2e10063ac880f` |
|
||||||
|
| `T78.xml` | `f712f94cff0cba05f786b7ac31dad2ea9408fde2a17a04531130df623f57dd84` |
|
||||||
|
| `C65.xml` | `572791610e20b7c6045d984d968e73a18a144bcb5c3da64112001c781826808e` |
|
||||||
|
|
||||||
|
XML commit:`a7be623d01c48c61405dec31c871121cdbed5a01`。测试 fixture `tests/fixtures/acl-t78.xml` 是该原始文件,不是手工模拟论文;不包含 PDF。PDF 哈希建立完整性,不单独证明语义,语义依据是上述具体页面。
|
||||||
|
|
||||||
|
在服务器使用既有命令,固定 checkout 并输出到新目录:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect acl \
|
||||||
|
--checkout /data/paper-index/raw/metadata-checkouts/acl-anthology \
|
||||||
|
--collection T78 \
|
||||||
|
--out /data/paper-index/collected/acl-t78-source-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
该命令会校验/更新官方 metadata checkout,实际 commit 必须以生成清单为准;严格重现本次取证时,使用已冻结的 XML 和上述 commit,不把更新后的内容冒充旧快照。不要将无 collection 限制的全部历史输出直接作为增量队列。采集完成不代表导入完成,恢复前先检查当前单元和报告,不因日志安静重启已执行的写入。
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,52 @@
|
|||||||
|
# 补漏范围与恢复清单(2026-11-10)
|
||||||
|
|
||||||
|
用户确定:**后续主动补漏从 2020 年起**。2000 年起已经发现的名单、缺口、资料组合和身份核查记录保留在项目中,供下次接着做;已有旧数据继续保留。停止补采的年份标为范围外或延期,不算作已补齐。
|
||||||
|
|
||||||
|
本页是下一次工作的入口,结构化记录见 [backfill-handoff-2026-11-10.json](backfill-handoff-2026-11-10.json)。它保存 43 个会议的 2000–2026 已观察年份、缺年、官方目录文件入口、近期待办与恢复证据。旧的日期快照不改写,也不能当实时数据库。
|
||||||
|
|
||||||
|
## 当前范围和年份差集
|
||||||
|
|
||||||
|
依据生产只读快照 `2026-11-09 16:26:41.688509+00:00`,原全历史口径为 1266 个目标年份槽位:1208 已观察、43 个历史缺年、14 个 2026 缺年,以及 1 个 SysML 2018 非存档例外。原 135 个历史缺口中有 92 个通过数据补入解决;其余 43 个现在停止补采,不能因此宣称全部解决。
|
||||||
|
|
||||||
|
- **2020–2025**:没有整年完全缺记录的目标槽位,但逐篇、track、摘要、公开负面决定仍不完整。
|
||||||
|
- **2026**:NeurIPS、ECCV、RecSys、CIKM、ICDM、SDM、IROS、RSS、CoRL、ICAPS、AAMAS、ICASSP、ISMIR、ACM MM,共 14 个槽位。须逐个核查是否已经公开出版;不能将尚未发布的资料说成漏采。
|
||||||
|
- **2000–2019 延期线索**:WWW 2000、ICAPS 2001 两个整年缺口;官方历届记录分别见 [WWW](www-editions-2026-11-08.json)、[ICAPS](icaps-editions-2026-11-08.json)。SysML 2018 是非存档首届,见 [MLSys](mlsys-editions-2026-11-08.json),不能制造正式出版记录。
|
||||||
|
- **2000 年前**:剩余 41 个历史缺年停止补采。旧来源文件继续保留作证据,不再作为当前工作队列。
|
||||||
|
|
||||||
|
以上只判断目标会议年份是否至少有一条投稿记录,绝不等于目录完整、独立论文数量或最终去重验收。
|
||||||
|
|
||||||
|
## 近期资料与恢复位置
|
||||||
|
|
||||||
|
服务器采集根目录为 `/data/paper-index/collected`,本地证据备份为 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index`。这些目录提供大体积原始文件,仓库 JSON 提供路径、状态和可用哈希;恢复前先验证文件存在、哈希和生产实际状态。
|
||||||
|
|
||||||
|
| 资料 | 当前状态 | 下一步与证据入口 |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| AAMAS 2026 | 官方目录 643 条,3 个 keynote 排除;640 个非 keynote 中含 1 个明确撤稿。尚未导入 | `aamas2026-primary-evidence-v1/` 保存官方 HTML、回执、逐项目录、审计和校验清单。八个标题差异/撤稿前缀条目需核查。先扩展正式 collector 并远程验证,再做新 PostgreSQL 身份预检。 |
|
||||||
|
| CVPR 2026 Findings | 941 条摘要已核验,尚未导入 | `cvf-2026-verified/recovery-audit.json`;原 404 与 RVLF/FVLF 路径更正证据保留。先核对 manifests、track 和实体关系。 |
|
||||||
|
| AAMAS 2025 | 476 个非 keynote 来源中,240 已验收、236 待身份核查 | `aamas2025-import225-v1/` 与 `aamas2025-import15-v1/` 是已消费批次,禁止重放。15 条最近验收报告哈希见 JSON。余下资料从当前身份候选与新只读基线继续。 |
|
||||||
|
| RSS 2025 | 163 条目录中 162 已验收,`2025/p117` 待核查 | 保留 spinal/RMCC 关系问题;查看工作状态键 `rss2025_held7_primary_evidence_v1`,不能重跑已消费的 6/88/54/14 批次。 |
|
||||||
|
| 近期公开负面决定首批 | 1625 条已验收:1624 rejected、1 withdrawn | `public-negative-postgresql-import-postcheck-v1.json`;仅该批次验收,不是公开拒稿全集,不要重放。 |
|
||||||
|
|
||||||
|
AAMAS 2026 官方目录分区:research 338、extended abstracts 185、AAAI 19、Blue Sky 17、doctoral 34、demo 44、JAAMAS 3,另有 keynote 3。JAAMAS 是期刊展示关系,不能凭目录再造一次独立审稿录用。官方撤稿条目在 research 分区,不能把全目录都标为当前 accepted。
|
||||||
|
|
||||||
|
AAAI 转投政策有年份冲突:[详细征稿页](https://cyprusconferences.org/aamas2026/call-for-papers-aaai-track/) 指 AAAI-26 rejected,[总览](https://cyprusconferences.org/aamas2026/calls/) 写 AAAI-25。保留两份证据;19 个 AAAI track 条目不能直接转成 19 条有原始 note 的拒稿决定。原投稿 ID、逐篇决定和版本关系仍需核实。
|
||||||
|
|
||||||
|
## 2000–2019 已发现但延期的身份与名单工作
|
||||||
|
|
||||||
|
- **AAMAS 2012**:官方 334 条,319 已独立验收、15 个身份条目保留待查。验收报告 `aamas2012-import-v1/postgresql-postcheck-v1.json`,哈希在 JSON;已验收 319 条及被替代的 316 条计划均不得重放。来源布局与用法见 [AAMAS 适配说明](../aamas-proceedings.md)。
|
||||||
|
- **UAI 2008**:官方 programme 与归档资料确认了四篇独立论文,但部分原始 DBLP 键的标题、作者、页码、出版链接对应仍有冲突。JSON 保留四组候选、arXiv ID、官方题名及归档报告哈希。之前仅根据页码推断题名/作者错误的结论已撤回,未应用纠正;恢复时不得据此覆盖或合并。
|
||||||
|
- **其他名单**:本目录全部 `*-editions-2026-11-08.json` 保留各会议官方年份及来源;[历史缺口](historical-gaps-2026-11-09.md)、[来源候选](historical-source-candidates-2026-11-09.json)、[来源范围](historical-source-scope-2026-11-09.md)、[主系列](historical-primary-series-2026-11-09.md)、[前身会议](historical-predecessors-2026-11-09.md)、[ICLR 档案](iclr-archive-2014-2016.md)继续留档。这些旧报告的“缺失”可能已被后续导入解决,恢复时必须对照当前状态,不能直接当可执行队列。
|
||||||
|
|
||||||
|
## 公开拒稿资料不能遗漏
|
||||||
|
|
||||||
|
结构化清单保留最近已查的政策 URL、限定年份和可用性结论。优先继续 UAI 2024–2026 的 opt-in 公开拒稿来源、EMNLP 2023 的公开集合范围及其他目标的逐年核查;平台使用 OpenReview 不能证明 rejected 公开。访问受限如实记录,不绕过 Challenge。
|
||||||
|
|
||||||
|
保留 rejected、desk rejected、withdrawn、retracted、resubmit 的区别;评审建议不等于最终决定。COLM 2026 的不公开政策只适用于已核实年份,不能推广到其他会议。PaperCopilot 与原 OpenReview、镜像间有重叠,候选计数不能相加。论文先被 A 拒绝再被 B 录用时,两次投稿、决定、来源与证据全部保留。
|
||||||
|
|
||||||
|
## 下次启动顺序
|
||||||
|
|
||||||
|
1. 读取本页和 JSON,再核实服务器工作状态、明确的 systemd invocation 与终态报告。权威工作状态 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/work-state.json` 中部分旧字段会过期;以最新验收和实际查询为准。
|
||||||
|
2. 当前主动范围为 2020 起,2000–2019 只恢复线索,不自动启动其采集。先检查新来源是否已导入,验收过的队列禁止重放。
|
||||||
|
3. 使用现有 `foundry-postgres` 的 `foundry_data.research`。当前已验证 writer 为提交 `714f745bd5c3fbc742be1efae0b31ab37a55e21f` 的固定 release;恢复时重新确认,不使用旧 `/data/paper-index/code` 替代。服务配置仅由服务器受保护文件读取,不复制密码。
|
||||||
|
4. 对待导入资料做新只读身份预检,处理稳定标识、短标题、作者/版本差异、track 与跨会历史,再生成小范围增量队列。每批独立验收,SQLite 只读备份保持保护。
|
||||||
|
5. 编译、测试或 wheel 构建仅在已确认 `ces-control:host` runner 执行。代码按工作分支交付 PR,等待用户手动合并。范围内导入结束后仍须全库去重、近似标题/作者冲突、来源-年份-track、摘要、约束、实际全文查询验收;不能用单批导入或 CI 成功替代。
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
# 2020 起补漏状态:2026-11-17
|
||||||
|
|
||||||
|
**总体仍为 13/17,第 14 步未完成。** 本页更新当日最新生产结果;[2026-11-10 恢复清单](backfill-handoff-2026-11-10.md)和更早快照继续保留。用户确定的范围不变:2020 年起主动补漏;43 个更早历史缺口延期而非补齐,2000–2019 已发现的名单和身份线索继续保留。
|
||||||
|
|
||||||
|
## 最新数据库盘点
|
||||||
|
|
||||||
|
只读快照 `2026-11-17 14:06:28.803415+00`,固定 43 个目标会议、278 个应有会议年份槽位,其中 **272 个至少有一条投稿记录,6 个仍完全为空**,均为 2026 年:**CIKM、ECCV、ICDM、IROS、ISMIR、MM**。SDM25、ICASSP112、RecSys47 三批分别使空槽从 9 降至 8、7、6。
|
||||||
|
|
||||||
|
这些是会议—年份的存在性统计,**不表示 272 个槽位已逐篇、逐 track、摘要或公开拒稿覆盖完整**。2020–2025 没有整年空槽,仍有来源完整性和公开负面决定缺口。RecSys 九轨、ICASSP 和 SDM 的本批来源均仍为 partial;不能据此宣布第 14 步完成。
|
||||||
|
|
||||||
|
| 生产文件(相对 `/data/paper-index/`) | SHA256 |
|
||||||
|
| --- | --- |
|
||||||
|
| collected/active-scope-census-20261117-v19/summary.json | cf0c5a013b6efe7d232187c9c76d9820bc8b33e2dd5674cab53d78cd3369ab4c |
|
||||||
|
| collected/active-scope-census-20261117-v19/snapshot.json | 38d09e3782372edbdd2840f5922d65fb497c425b1eaf2d1fb10342956a7a82f2 |
|
||||||
|
| validation/active-scope-fixed278-v1.json | a835bef5b9c3a0f76069d840cecdb9a6bcf65d3a38796d70dbfdfd4b49242264 |
|
||||||
|
|
||||||
|
原始快照、范围定义与生成脚本已备份,本机已按固定范围和别名逐条重算 272/278,未用 `max(id)` 推算数量。此前 v17/v18 快照保留,不能将其旧数字当作当前状态。
|
||||||
|
|
||||||
|
## 已独立接受的近期补录
|
||||||
|
|
||||||
|
- **RecSys2026**:官方九轨 263 项观察、260 个不同 DOI 均已冻结并规范化。47 条现有作品关联通过独立验收,新增论文实体为 0;新增一个来源、九个 edition、47 次投稿/47 条录用观察、47 个发表 DOI 投稿标识及47条来源类型标签证据。原始摘要单独保存,规范摘要不覆盖。5,812 篇受保护旧作品及全部旧投稿/决定历史保持不变,foundry 就绪检查200、SQLite备份只读。216项待审查,包括三组重复 DOI 的六项观察。见 [九轨验收与恢复](../recsys2026-program.md)。
|
||||||
|
- **ICASSP2026**:Crossref 冻结的 4,645 个条目中排除 56 个其他会议条目和 7 个目录/封面等非论文条目,剩余 4,582 条注册论文;全部注册摘要缺失。112 条已有 DOI 的来源/录用历史已独立验收,新增论文实体为 0,coverage 为 partial112/4582;4,470 条仍待身份核查。正式节目标题表与注册元数据保持独立,不自动融合作者或摘要。见 [来源边界与恢复](../icassp2026-registry.md)。
|
||||||
|
- **SDM2026**:53 个注册章节、53 项独立日程观察及 52 页摘要已冻结。25 条来源关联已独立接受,新增论文实体为 0,28 条待审查;见 [生产验收与恢复](../sdm2026-identity-review.md)。
|
||||||
|
|
||||||
|
三批已经消费的输入队列均**禁止重放**。最新已接受生产结果为 RecSys47:论文实体仍为 **3,567,888**,来源记录 **3,814,931**,投稿 **531,620**,决定来源观察 **571,172**。观察数不等于不同论文或不同投稿的数量;后续须使用当前基线和新的增量队列。
|
||||||
|
|
||||||
|
SDM 新增事件使 SeBA 和 AdamG 同时保有较早 ICLR 拒稿与后来 SDM 发表事实。原来的两次 ICLR 投稿及四条决定来源观察均保留,没有被 accepted 覆盖;四条观察不能直接累加成四次拒稿。本批 RecSys47 本身没有既往负面观察,但全部旧全库历史同样通过保护验收。
|
||||||
|
|
||||||
|
## 已采集、尚待关联或规范化的来源
|
||||||
|
|
||||||
|
| 会议 | 已取得的来源与状态 | 下一步 |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| ECCV | 2,834 个不同初步节目 ID 及公开摘要已独立采集;1,997 个只读候选历史证据包已保存,尚未入库 | 初步日程不等于 Springer 最终论文集;继续作品/版本和出版核查。见 [来源状态](../eccv2026-source-status.md) |
|
||||||
|
| ICDM | 324 项线下节目观察,明确研究241/应用83;演讲长168/短156,摘要均缺失;规范化和80个候选历史包已核验 | 展示长短不能当成论文类型;需审查作品与正式出版版本,尚未导入主库。见 [采集器说明](../icdm2026-program.md) |
|
||||||
|
| ISMIR | 140 项节目和正式名单均已核验,140份摘要,101组自愿公开评审;两处标题、三处作者名单差异保留,78个候选历史包已核验 | 数字 miniconf ID 不是原始 OpenReview ID;评审意见不转成最终拒稿。继续作品身份/历史关联,尚未入库。采集器见 [PR #63](https://gitea.entroia.com/Zayden-Jung/paper-atlas/pulls/63) |
|
||||||
|
| MM | 全28轨2143个节目ID/4326展示位置已规范化;814个只读候选历史证据包已核验,24篇候选涉及24次负面投稿/37条来源观察;67项原始arXiv DOI中66项注册元数据一致、1项404暂缓 | 不是2143篇已核验科研论文。继续逐轨贡献/出版依据与作品审查,0身份批准、0入库。ISBN查询与父XML均只返回父记录,不作逐篇清单。见 [来源、审计与恢复证据](../mm2026-source-status.md) |
|
||||||
|
| CIKM | 官方录用/日程入口与五个分轨页面已保存;五个分轨链接实际是CFP/政策,不是逐篇录用名单 | 查找实际节目或出版社完整目录;TBD和没有静态表不能当作没有论文 |
|
||||||
|
| IROS | 官方论文索引入口的原始HTTP403已保存 | 保留访问限制,不重试绕过;寻找可合法读取、能独立核验的出版元数据 |
|
||||||
|
|
||||||
|
早期 14 个入口快照保持在 `/data/paper-index/collected/remaining2026-official-leads-v1`,后续新增原文另建冻结目录,不覆盖早期回执。HTTP200只证明入口可读,不直接作为可导入记录或全部论文数量。
|
||||||
|
|
||||||
|
## 公开拒稿与后续工作
|
||||||
|
|
||||||
|
MM 官方群组元数据可匿名读取,但根据其给出的 submission invitation 和 rejected/withdrawn/desk-rejected venue ID,四次限1条的匿名 notes 核查均为403。查询、时间、原文和哈希已保留,没有认证重试或绕过权限。访问限制不等于零条负面记录;详见 [MM来源证据](../mm2026-source-status.md)。ISMIR公开的101组评审来自录用节目,不能记成101条拒稿。
|
||||||
|
|
||||||
|
UAI新增固定镜像已核对2022–2025的902个原始论坛及3053个回复的元数据/字段结构。整理版均标录用,2022保存230条明确Accept决定;2024仅有评审、作者回复、元评审,2023/2025未保存回复,不能据此补拒稿。来源限制和恢复见 [逐年原始核查](uai-public-negative-source-status-2026-11-17.md)。这不是UAI没有公开拒稿的结论,缺口保持未补齐。
|
||||||
|
|
||||||
|
第14步仍需逐会议、年份、track完成公开拒稿/desk-reject/withdrawn来源核查和可得数据采集;继续核查UAI近年自愿公开范围、EMNLP实际覆盖范围及其余已列明缺口。不能从未见录用、评分、评审推荐、缺少摘要或接口不可访问制造负面决定。
|
||||||
|
|
||||||
|
之后仍需完成跨会作品/历次投稿与类型标签关联(第15步),所有导入结束后的独立全库去重、标识冲突、近似标题作者、摘要/约束/查询及覆盖审计(第16步),以及长期运行与恢复文档最终整理(第17步)。阶段性来源入库、测试通过、某年度有记录均不能替代这些验收。
|
||||||
@@ -0,0 +1,51 @@
|
|||||||
|
# UAI 公开负面决定来源:2026-11-17 核查
|
||||||
|
|
||||||
|
**公开拒稿缺口仍未补齐,总体仍为 13/17。** 本页保存新来源的实际范围与恢复线索,避免下次把录用子集误当作全部投稿。主动补漏范围仍从2020年开始;本次镜像只含2022–2025,不能替代其他年份或其他会议的核查。
|
||||||
|
|
||||||
|
## 政策与访问边界
|
||||||
|
|
||||||
|
已有逐年核查记录保存在 [覆盖清单](inventory-2026-11-08.json) 的 UAI `policy_checks`:2023–2026 官方说明允许决定后公开录用及作者选择公开的拒稿,默认私密拒稿不属于公开可采集范围。政策并不证明任何镜像完整。
|
||||||
|
|
||||||
|
- [2023 投稿说明](https://www.auai.org/uai2023/submission_instructions)
|
||||||
|
- [2024 投稿说明](https://auai.org/uai2024/submission_instructions)
|
||||||
|
- [2025 投稿说明](https://www.auai.org/uai2025/submission_instructions)
|
||||||
|
- [2026 投稿说明](https://www.auai.org/uai2026/submission_instructions)
|
||||||
|
|
||||||
|
先前匿名 notes 请求遇到 ChallengeRequiredError/403,仍作为访问限制保留。本轮没有认证、重试或绕过这些入口;读取的是整理者公开发布的固定版本镜像。复制的 `readers`、`signatures` 和邀请字段仍只是镜像内的来源断言,不是对当前原始接口可见性或签名真实性的验证。
|
||||||
|
|
||||||
|
## 实际镜像范围
|
||||||
|
|
||||||
|
来源:[latkes/ai-research-index-iclr-openreview](https://huggingface.co/datasets/latkes/ai-research-index-iclr-openreview/tree/4c9e09931a7814d7500630ffd064b74f1c01bcf3),固定提交 `4c9e09931a7814d7500630ffd064b74f1c01bcf3`。目录中存在 `papers/uai-YYYY/papers.jsonl` 与四个 `raw/uaiYYYY.jsonl`,不能仅凭数据集说明宣称拒稿覆盖。
|
||||||
|
|
||||||
|
| 年份 | 整理版及原始论坛数 | 原始回复对象 | 实际回复角色 | 标准结构化决定字段 |
|
||||||
|
| --- | ---: | ---: | --- | --- |
|
||||||
|
| 2022 | 230 | 1059 | Official_Review 829,Decision 230 | Accept (Poster) 194,Accept (Oral) 36 |
|
||||||
|
| 2023 | 243 | 0 | 没有保存回复 | 没有保存决定记录 |
|
||||||
|
| 2024 | 200 | 1994 | Official_Review 905,Rebuttal 889,Meta_Review 200 | 没有单独 Decision 记录或结构化 decision 字段 |
|
||||||
|
| 2025 | 229 | 0 | 没有保存回复 | 没有保存决定记录 |
|
||||||
|
|
||||||
|
整理版902条均被整理者标为 `accepted=true`;原始根记录的 `venue` 为相应会议名称或 oral/poster/spotlight 文本,不能替代逐次决定证据,更不能算作902条已认证的原始录用决定。这不是四届全部投稿或全部录用的完整性证明。2024年还有 `Edit` 邀请,属于同一批回复上的另一个邀请,不能再累加成1994个新回复。
|
||||||
|
|
||||||
|
为排除字段漏读,对含回复的2022、2024快照另做了全部3053个回复的字段名、包装结构、邀请角色和短决定类字段核查;重新读取的正文哈希与首次捕获完全一致。2022的230条明确字段是录用标签。2024仅保存评审、作者回复和元评审,没有可据此新增的结构化拒稿/撤稿决定。没有从元评审文本、评分、推荐或缺少录用推断负面决定,也没有保存评审正文、作者回复正文或论文全文。
|
||||||
|
|
||||||
|
**这只描述上述固定镜像子集,不证明UAI没有公开拒稿。** 本批没有生成负面决定导入队列,也未写数据库。2020、2021、2026及其他可用来源仍需分别核查;某文件未出现在这个镜像目录中不代表全网不存在。
|
||||||
|
|
||||||
|
## 产物与恢复
|
||||||
|
|
||||||
|
服务器目录均相对 `/data/paper-index/collected/`:
|
||||||
|
|
||||||
|
- `uai2022-2025-mirror-metadata-v1`:902条整理版元数据投影、字段盘点与标签计数。
|
||||||
|
- `uai2022-2025-raw-decisions-v1`:902个原始论坛的元数据、明确决定字段及来源正文/逐行哈希。
|
||||||
|
- `uai2022-2024-reply-schema-v2`:3053个回复的元数据、字段名与短标签;评审正文不保存。角色核查包含直接记录和 `note` 包装,未发生静默遗漏。
|
||||||
|
|
||||||
|
源响应正文仅在内存中读取、校验,保存元数据投影与固定来源URL/字节哈希;不是保留了含评审文本的完整原始快照。复核原始源字节时应使用固定提交,并区分投影文件哈希与源正文哈希。恢复包已在本机独立核验全部902个论坛、3053个回复的元数据结构、逐文件哈希及来源关联。
|
||||||
|
|
||||||
|
| 恢复证据 | SHA256 |
|
||||||
|
| --- | --- |
|
||||||
|
| 整理版元数据恢复包 | 55889fb2b9005260cef1001b2cf36e203f3b7e1deeec0291f058b52eeae880e7 |
|
||||||
|
| 原始元数据/回复结构恢复包 | caade7df899dd3e65572d06e2bf2326fad692cc5e1c1e4ab002fdc6d7823743f |
|
||||||
|
| 原始结构独立核验报告 | 5f0ef2948d0db68fe0cd89eb4c8b57a1088b94b53bed98f180d2b96629746e72 |
|
||||||
|
|
||||||
|
服务器恢复包分别为 `/data/paper-index/validation/uai2022-2025-mirror-metadata-recovery-v1.tar.gz` 与 `uai2022-2025-raw-reply-recovery-v1.tar.gz`。本地同名备份位于 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`。先检查实际单元、完整manifest和哈希,再恢复;不要因进度文件旧状态或日志安静重跑。
|
||||||
|
|
||||||
|
后续继续寻找能明确覆盖作者公开拒稿选择的来源,保留每次投稿及决定原文标签、原始ID与镜像关系,核对跨来源重叠和后续录用。UAI缺口不关闭,全部43个目标的逐年公开负面决定核查也不缩减;最终全库去重仍为独立必做阶段。
|
||||||
@@ -342,3 +342,9 @@ ICML 2024 起的一个 PMLR 卷可以同时包含研究主轨道与 Position Pap
|
|||||||
2024 年的官方节目单把两轨都放在 `Conference`,但提供 PMLR 论文自链接。该年仅采用组织方在 [2025 Position CFP](https://icml.cc/Conferences/2025/CallForPositionPapers) 明确链接的 [2024 Position 论文标题筛选](https://icml.cc/virtual/2024/papers.html?filter=titles&search=position%3A) 规则;同时保存规则页面的哈希。这个规则不用于其他会议或年份。官方节目单将 `guyard24a` 链接误写为 `theo24a`:只对这条已核查的映射额外要求标题精确一致、至少两名公开作者重合,不进行全库模糊匹配。
|
2024 年的官方节目单把两轨都放在 `Conference`,但提供 PMLR 论文自链接。该年仅采用组织方在 [2025 Position CFP](https://icml.cc/Conferences/2025/CallForPositionPapers) 明确链接的 [2024 Position 论文标题筛选](https://icml.cc/virtual/2024/papers.html?filter=titles&search=position%3A) 规则;同时保存规则页面的哈希。这个规则不用于其他会议或年份。官方节目单将 `guyard24a` 链接误写为 `theo24a`:只对这条已核查的映射额外要求标题精确一致、至少两名公开作者重合,不进行全库模糊匹配。
|
||||||
|
|
||||||
导入与身份回填拒绝 ICML 2024 起 parser_version<3 或没有官方轨道证据的旧文件。已有主库中的错误轨道需要在独占维护锁内迁移原投稿事件、保留决定与来源关联;仅重新导入拆分文件会留下旧的主轨道事件,不能作为修复方式。完成迁移前,不把新文件直接导入现有库。
|
导入与身份回填拒绝 ICML 2024 起 parser_version<3 或没有官方轨道证据的旧文件。已有主库中的错误轨道需要在独占维护锁内迁移原投稿事件、保留决定与来源关联;仅重新导入拆分文件会留下旧的主轨道事件,不能作为修复方式。完成迁移前,不把新文件直接导入现有库。
|
||||||
|
|
||||||
|
### RSS 发表身份
|
||||||
|
|
||||||
|
已核验的 `rss_proceedings` 录用记录与其他出版社来源一样,将 DOI 作为届次范围内的 `publication_doi` 投稿身份。这样同一 RSS 出版物先由官方论文集、后由 DBLP 导入(或反向)时可以共享一次投稿,并保留各自来源和决定观察。此规则不把节目显示编号当原始 OpenReview 投稿号,不把拒稿/撤稿升级为录用,也不跨届次合并历史。
|
||||||
|
|
||||||
|
这不证明任意 DOI 的科学作品归属:导入前仍需核查原始标题、完整作者、版本和已有标识冲突。变更只作用于后续导入;不会自动回填、合并或改写旧 RSS 记录,旧记录需要独立审查计划。
|
||||||
|
|||||||
@@ -0,0 +1,37 @@
|
|||||||
|
# DBLP family 候选恢复
|
||||||
|
|
||||||
|
DBLP 的会议命名空间会变化。旧候选过滤器包含 `atal`、`aamas`,遗漏了 `ifaamas`;2026-11-01 固定快照的完整目录实际已有 `conf/ifaamas/2025` 和 `conf/ifaamas/2026`。新目录盘点已把 `ifaamas` 纳入广泛候选范围,已有快照也可按需恢复,无需重新下载或重建完整目录。
|
||||||
|
|
||||||
|
## 运行
|
||||||
|
|
||||||
|
在已有元数据的服务器运行:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect dblp-family \
|
||||||
|
--archive /data/paper-index/raw/dblp-2026-11-01.xml.gz \
|
||||||
|
--dtd /data/paper-index/raw/dblp-2023-06-28.dtd \
|
||||||
|
--catalogue /data/paper-index/collected/dblp/dblp-catalogue.sqlite \
|
||||||
|
--family ifaamas \
|
||||||
|
--out /data/paper-index/collected/ifaamas-recovered
|
||||||
|
```
|
||||||
|
|
||||||
|
`--family` 可以重复,但仅接受项目明确列出的候选 family。`--out` 必须尚不存在。同名输出由文件锁防止并发恢复;任务运行中应观测原任务,不能因为日志间隔长就重复启动。完整扫描会定期打印进度。
|
||||||
|
|
||||||
|
命令重新验证固定压缩包 SHA256、DTD MD5 与大小,并只读打开目录。目录旁的 `dblp-ai-candidates.manifest.json` 必须属于同一个快照和 DTD,包含完整记录类型计数。旧候选文件不作为完整性依据,也不会被改写。
|
||||||
|
|
||||||
|
恢复扫描使用项目原有 XML 流式解析器,仅允许已校验的本地 DTD,不联网读取 XML 实体。最终会比对完整快照的记录总数、逐 family 数量和全部选中卷的原始元数据。全部一致后才原子发布输出目录;失败时不发布半成品,也不覆盖既有文件。排查明确终止的失败任务后,可用尚不存在的输出目录重新运行。
|
||||||
|
|
||||||
|
## 输出与后续流程
|
||||||
|
|
||||||
|
- `dblp-family-candidates.jsonl`:原始 `inproceedings` 和 `proceedings`,保留书目 key、原始作者、标题、年份、卷引用、出版链接等字段。
|
||||||
|
- `recovery-manifest.json`:快照、DTD、目录凭据哈希,完整扫描数量、逐 family 数量、选中卷、旧候选文件的已知数量及输出哈希。状态为 `recovered_raw_candidates_pending_scope_review`。
|
||||||
|
|
||||||
|
恢复结果没有新增规范化的 `venue`、`track` 或决定标签,也不是 `import records` 的输入队列。`ifaamas` 的整卷可能混有研究论文、扩展摘要、keynote、博士生、演示和 JAAMAS 展示。目录过滤修复不会把这些条目统一标成 main;规范化的范围映射仍需第一方逐栏目证据。
|
||||||
|
|
||||||
|
后续先用官方目录核对标题、页码、作者及发表形式,再查数据库稳定标识和候选实体,保留每次投稿、决定和标签来源。只有审核后的新增分区才能原生导入并独立验收;已验收的来源记录不能重放。单个年份有记录不代表逐篇、track、摘要或公开拒稿覆盖完整。
|
||||||
|
|
||||||
|
## 2026-11-09 核查实例
|
||||||
|
|
||||||
|
对既有固定快照的独立原始元数据恢复发现 `ifaamas` 有 1,122 条出版物候选及 2 卷:2025 年 479 条、2026 年 643 条。2025 年的 479 条标题与起始页均对应官方目录,其中 3 条 keynote 应排除,余下 476 条出版物需逐项处理。该数字包含已经验收的 225 条来源,不是 1,122 篇可新增论文。
|
||||||
|
|
||||||
|
此实例说明过滤遗漏的影响,不代表恢复命令已完成生产导入。2026 年仍需核验官方出版清单、栏目、身份及逐次决定;缺少录用记录不能推断为拒稿。最终全库去重、覆盖和查询审计继续单独进行。
|
||||||
@@ -0,0 +1,35 @@
|
|||||||
|
# ECCV 2026:暂定节目与公开元数据
|
||||||
|
|
||||||
|
这是长期维护、按需运行的来源适配器。它读取已核验的官方节目快照,采集公开论文页的标题、完整作者和摘要,输出待身份审核文件;不连接数据库,不下载 PDF、评审、个人档案或受保护的直播内容。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect eccv2026-program \
|
||||||
|
--official-snapshots /data/paper-index/collected/eccv2026-official-research-v1 \
|
||||||
|
--cache /data/paper-index/cache/eccv2026-public-pages \
|
||||||
|
--max-new-pages 200 \
|
||||||
|
--out /data/paper-index/collected/eccv2026-pass01
|
||||||
|
```
|
||||||
|
|
||||||
|
官方目录包含 `accepted.html`、`policies.html`、`faq.html` 和冻结的 `manifest.json`。当前适配器核验固定版本:节目 2,851 行、2,834 个节目 ID;17 个 ID 各重复一次,标题、作者、关键词、展示信息均一致。输出按节目 ID 合并这些重复行,并保留全部行序号。它不是跨来源作品去重。
|
||||||
|
|
||||||
|
`--max-new-pages` 限制本轮新请求数;不指定时遍历整个冻结名单。再次运行时使用同一 `--cache` 和**全新** `--out`,复用已验证缓存并继续读取缺失页。缓存不按 24 小时过期,不自动刷新或覆盖已成功的原始响应。每次运行输出包含本轮能核验的全部缓存条目,并明确列出尚未采集的 ID;不能未经审核把多轮重叠输出全部重复导入。
|
||||||
|
|
||||||
|
公开请求按固定来源和路径限制,只接受 HTTP 200、原 URL、HTML 和限定大小的完整响应。请求失败、重定向、访问挑战、非 HTML 或异常响应留下阻断标记,不自动重试或更换账号、域名、代理绕过限制。记录的 `Retry-After` 供人工核查使用。网络超时等未产生成功缓存的情况可在确认进程终止后用新输出目录继续。缓存数据与回执须成对且哈希一致;遇到孤立文件或 `.part` 时保留现场,先核查再把本任务的残留文件移至恢复目录,不静默丢弃。写同一缓存时持有非阻塞单写锁。
|
||||||
|
|
||||||
|
离线重放可改用 `--paper-snapshots /path/to/papers`,该目录包含 `<节目ID>.html` 和 `manifest.json`(带时区的 `at`、`records`,各条含文件、URL、最终 URL、HTTP 状态、类型、字节数、SHA-256)。缺少的公开页会明确列为未采集;三篇样本只会生成三条 partial 记录,不会冒充全量成功。
|
||||||
|
|
||||||
|
## 来源边界
|
||||||
|
|
||||||
|
[官方名单](https://eccv.ecva.net/Conferences/2026/AcceptedPapers)明确注明 **preliminary pending publisher checks**。这里的 accepted 表示冻结时列入暂定官方名单,不代表已核验 Springer 最终发表或原始签名决定。所有输出始终 partial。节目 ID 是官方节目定位符,不冒充原始 OpenReview 投稿号;尚未核实跨平台标识时作品标识和投稿标识均为空。
|
||||||
|
|
||||||
|
公开页的 canonical URL、可见标题/作者和 JSON-LD 相互核对;页面版本与冻结节目文字有差异时分别保留并标记待审阅,不制造人名别名。JSON-LD 日期只是网站条目时间,不当作论文发表日期或原始决定日期。摘要保留公开页 DOM 文本(规范空白),包括源头的断词、缺空格;不自动修补。原始 HTML 单独留存;遇到公式、上下标或图片结构会标记待审阅,不能把扁平化文本当作无损数学表达。页面上的登录提示对应直播内容,不用于访问受保护内容。
|
||||||
|
|
||||||
|
第 4199 页的 JSON-LD 作者姓名存在双重 HTML 实体编码;第 3844 页的可见标题和 JSON-LD 则已经一致地包含字面 `&`。解析器优先保留已经一致的文字,仅在字段不一致时尝试最多两层实体解码来核对可见文本,不把已经一致的标题再次解码。完整作者序列仍须精确一致,真实差异或更多编码层次仍停止。`jsonld_raw_text` 保留原始结构化标题/姓名,发生文本规范化时增加 `jsonld_display_text_normalized_requires_review`,不生成姓名别名。恢复必须使用校验过的修复版本、新输出目录和原缓存;保留旧失败目录,不重新请求已有成功页面。
|
||||||
|
|
||||||
|
官方关键词保存在独立 `eccv_program:keyword` 命名空间,并保留原始单元格和证据。关键词、展示形式、贡献/研究类型、自定义标签相互独立,投稿 track 暂用 `program_unclassified`,不按主题推断 main、findings、workshop 或 position。
|
||||||
|
|
||||||
|
[2026 投稿政策](https://eccv.ecva.net/Conferences/2026/SubmissionPolicies)明确评审和作者回复不公开;这不能代替对公开 rejected、desk rejected、withdrawn **决定元数据**的单独核查。当前拒稿决定可见性仍记为 unknown,不能从录用表缺席推断拒稿。
|
||||||
|
|
||||||
|
贡献类型的官方定义、OpenReview 状态入口与匿名访问核查见[来源核查记录](eccv2026-source-status.md)。这是独立于冻结采集输入的证据,不会改变已核验节目版本,也不能用类型定义替论文推断标签。
|
||||||
|
|
||||||
|
失败输出保留 `failed.json`,成功报告最后发布;恢复前核查既有进程、输出及缓存,不因日志安静重启。身份、来源版本和完整投稿历史审阅后,另行准备专用入库队列、保护基线和导入后验收;全部来源导入结束后仍须全库去重、覆盖、摘要、约束及查询审计。测试、构建及安装验证只在已授权 `ces-control:host` runner 执行。
|
||||||
@@ -0,0 +1,59 @@
|
|||||||
|
# ECCV 2026 来源核查记录
|
||||||
|
|
||||||
|
核查时间:2026-11-17,以下状态是当时的公开可见性。长期补漏时先核实来源是否变化,保留旧快照,不覆盖旧结论或重复导入。
|
||||||
|
|
||||||
|
## 贡献类型与 track
|
||||||
|
|
||||||
|
[作者贡献类型说明](https://eccv.ecva.net/Conferences/2026/AuthorContributionTypes)通过官方导航确认。它要求作者选择一项主要贡献类型;官方 FAQ 明确贡献类型不是独立 track。公开 OpenReview group 的 `source_submissions_query_mapping` 也包含五项对应值:
|
||||||
|
|
||||||
|
| 官方 group 字段值 | 含义摘要 |
|
||||||
|
| --- | --- |
|
||||||
|
| Algorithms/General | 方法、模型或算法;也是边界情形的默认选择 |
|
||||||
|
| Theory/Foundational | 理论、形式分析或基础理解 |
|
||||||
|
| Applied/Systems | 实际系统、部署与工程约束 |
|
||||||
|
| Datasets/Benchmarks | 数据集、基准或评估协议 |
|
||||||
|
| Concept & Feasibility | 新概念及可行性证据 |
|
||||||
|
|
||||||
|
这些是类型定义和配置值,**尚未核验逐篇论文的作者所选类型**。不能根据标题、摘要、主题关键词或展示形式补写类型;公开节目采集器继续输出 `program_unclassified`,主题关键词继续使用 `eccv_program:keyword`。以后获得逐篇类型证据时,保留独立命名空间、原始值、来源和版本,不把它写成投稿 track。
|
||||||
|
|
||||||
|
## 公开负面决定元数据
|
||||||
|
|
||||||
|
[投稿指南](https://eccv.ecva.net/Conferences/2026/AuthorGuide)链接的会议 group 是 [`thecvf.com/ECCV/2026/Conference`](https://openreview.net/group?id=thecvf.com/ECCV/2026/Conference)。匿名 API 读取 group 成功(HTTP 200),返回 `readers: ["everyone"]`。这只说明 group 配置公开,不证明论文或决定公开。
|
||||||
|
|
||||||
|
group 的 `rejected_venue_id` 实际配置为 `thecvf.com/ECCV/2026/Conference/Rejected_Submission`。使用该值查询公开 notes(`content.venueid`,`limit=1`)返回 HTTP 403,随后停止;没有自动重试、切换账号、代理或其他绕过操作。其余 desk-rejected、withdrawn 和 submission 状态查询没有执行,不能报告为已核查或零条。
|
||||||
|
|
||||||
|
结论仍为 **拒稿决定元数据覆盖未核实,当前匿名查询受限**。403 的具体原因未确定;不能由此推断真实投稿数量、全局不存在公开拒稿数据,或某篇论文遭拒。[投稿政策](https://eccv.ecva.net/Conferences/2026/SubmissionPolicies)关于评审与作者回复不公开的规定也不能代替决定元数据核查。
|
||||||
|
|
||||||
|
## 可恢复证据与后续入口
|
||||||
|
|
||||||
|
服务器证据目录:`/data/paper-index/collected/eccv2026-policy-research-v2`。本机工作记录目录保留同名备份;三份成功响应均已逐文件校验。证据不含登录凭据。
|
||||||
|
|
||||||
|
| 文件 | SHA-256 |
|
||||||
|
| --- | --- |
|
||||||
|
| manifest.json | 5a04e52bf6ea34ab25fe1d91fa9a1df18c9be5abb4f554586ee28fe9aa566e6e |
|
||||||
|
| contribution-types.html | 6a5128b1f47a275411acacbb08ab91320a767bdcc48aaad09fe1be9e31dff00f |
|
||||||
|
| author-guide.html | 779196716f5b37fcd10ddc191e9cd0530d7e9d43e5bc66b16a5546cd152959c4 |
|
||||||
|
| openreview-group.json | 055bcf99d412d77f79a760890739d4de13a20ea099857154ee3e714c0647e763 |
|
||||||
|
| anonymous-decision-probes/manifest.json | a522756ed3ec2880e5d942a211d7ad13819b18c8acada551dd3a8718b5e4d63b |
|
||||||
|
|
||||||
|
这里的补充证据与采集器的三份冻结节目/政策输入分开保存,不能替换采集器固定版本的 manifest。下次补漏先核查官方是否提供公开决定导出或明确的公开元数据说明;访问仍受限时保留缺口。若发现允许公开访问的新入口,另存带时间与哈希的快照,先审阅读者权限、决定语义和逐篇身份,再准备入库队列。
|
||||||
|
|
||||||
|
|
||||||
|
## 2026-11-17 全量采集与只读预检
|
||||||
|
|
||||||
|
第三次采集 `paper-atlas-eccv2026-collect-pass03.service`(invocation `147c8779a3af4223a4d5d60dbdc88264`)已终态成功:2,834 个唯一日程条目全部取得公开摘要,新取 1,775 页,复用已验收缓存 1,059 页;2,851 次日程出现中的 17 处重复保留原始位置。独立原文/回执/字段审计通过,缺少条目和摘要均为 0,5 处 JSON-LD 显示文字规范化仍留待逐篇审阅。这里仍是官方初步日程版本,不改称出版社最终论文集。
|
||||||
|
|
||||||
|
| 服务器证据 | SHA-256 |
|
||||||
|
| --- | --- |
|
||||||
|
| validation/eccv2026-pass03-audit-v1.json | ad90199e08a234c0b2a01f44e1edfbecb2a9101b45de87e00a64eadfd5c92848 |
|
||||||
|
| collected/eccv2026-pass03/eccv-program-2026-unclassified.jsonl | d0a80745fc6f58163f8722af351bbba86803af97857ea30554a0f3062d8ab2ef |
|
||||||
|
| validation/eccv2026-pass03-recovery-v1.tar.gz | df55493ad5f8d6413dca86e9b9aca1d8b544ac1dc222b3849f95b04f94accba9 |
|
||||||
|
| collected/eccv2026-pass03-preflight-v1/postgresql-preflight.json | 97bf57828cc67d05de0221188a9d988eef35631c845b18372186a6249f64e5da |
|
||||||
|
| validation/eccv2026-pass03-preflight-packets-check-v1.json | 8a4f44f5d2c3a15d05b5a1df96fe465a96a2fe8914d7ed68958e6978c30985d4 |
|
||||||
|
| validation/eccv2026-pass03-preflight-recovery-v1.tar.gz | 69c84fd0f24024ca56bfa4f848c977c578267bfc53c713105987e030df3f92a2 |
|
||||||
|
|
||||||
|
全量原文恢复包为 85,430,941 字节,保留输出、全部缓存原文与回执、官方输入和固定源码/CI 凭证。失败的第一次打包任务在创建文件前因命令引用错误退出;改用文件脚本的新单元 `paper-atlas-eccv2026-pass03-recovery-v2.service` 成功,未重启或重跑采集器。
|
||||||
|
|
||||||
|
新的 PostgreSQL 只读预检覆盖全部 2,834 条:1,519 条精确标题和作者集合一致、468 条仅部分作者重合、840 条无精确匹配、4 条没有作者重合、3 条命中多个同标题实体。1,997 个候选实体包的哈希独立核对通过,来源、投稿、决定与来源关系未触发截断。该结果只生成候选,`database_written=false`、`approved_for_import=0`;没有批准合并、创建新论文或导入 ECCV 事件。无精确匹配不等于新作品,相同作者也不证明人物或作品身份。
|
||||||
|
|
||||||
|
后续先保留这批完整来源并推进其他会议来源补漏,再做逐篇内容与版本审阅、当前快照守护和保护基线。当前全局最新基线来自 SDM25 验收,保护 5,653 篇论文;不能回退到 ICAPS34 之前的计数或重放已消费批次。最终全库去重仍须另行执行。
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
# ICAPS 2026 官方名单
|
||||||
|
|
||||||
|
这是长期维护、按需运行的元数据采集器。它只读取已核验的官方页面快照,生成待身份审核的文件;不连接数据库,不下载 PDF 或评审,不执行网页脚本。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect icaps2026-program \
|
||||||
|
--snapshots /data/paper-index/collected/icaps2026-official-research-v1 \
|
||||||
|
--out /data/paper-index/collected/icaps2026-normalized-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
快照目录包含 `accepted.html`、`cfp.html`、`previously-published.html` 和 `manifest.json`。采集器固定这三份已审阅页面的 SHA-256,核对 URL、最终 URL、字节数及带时区的观测时间;来源变化需要重新研究、固定证据并更新检查,不能静默接受。输出目录必须全新,失败会保留 `failed.json`,成功报告最后发布。恢复前先检查旧目录,不能覆盖失败现场或重放已消费的导入队列。
|
||||||
|
|
||||||
|
[官方录用表](https://icaps26.icaps-conference.org/program/accepted/)含两组:
|
||||||
|
|
||||||
|
- **90 篇 Papers accepted to the Main Tracks**:生成 partial JSONL,状态为 accepted。[CFP](https://icaps26.icaps-conference.org/calls/cfp/)包含主会及四个特别轨道,但该表没有逐篇归属,因此保存为 `main_tracks_unclassified`,不能把它们全部重命名为 main,也不能按论文主题猜特别轨道。position 是研究类别,不能直接推成投稿 track。
|
||||||
|
- **10 篇 Previously Published Papers**:保存在单独的 `previously-published-presentations.json`,不生成新的研究论文录用记录。[该展示轨道的官方说明](https://icaps26.icaps-conference.org/calls/previously_published_research/)须随原文保存;后续关联原发表记录时不能重复制造论文或替换原会议。
|
||||||
|
|
||||||
|
名单没有逐篇链接、摘要、DOI 或原始投稿号。90 条摘要明确留空,作品及投稿标识均为空;完整作者顺序与原始作者文本一并保留。`external_id` 里的哈希仅定位这份名单的标题和作者版本,不是原始投稿号,也不是跨来源作品身份。作者文字变化不代表人物别名,标题相同不代表可以自动合并作品。
|
||||||
|
|
||||||
|
主会 rejected、desk rejected、withdrawn 的公开范围仍未知,不能从缺席录用表推断拒稿,也不能把某一 workshop 的私密评审政策扩展到整个会议。出版社 OJS 页面曾返回访问挑战;本采集器不访问或绕过该挑战。当前来源只证明官方名单,未证明完整出版物或公开负面决定覆盖。
|
||||||
|
|
||||||
|
生产导入之前仍需当前数据库候选核查、独立身份审核、来源及历史保护基线、导入后逐项验收。所有来源导入完成后还必须另做全库去重、覆盖、摘要和约束查询审计。正常使用不会自动执行这些后续写入。
|
||||||
|
|
||||||
|
测试及安装后 CLI 验证只在已授权的 `ces-control:host` runner 上运行。冻结 fixture 含完整 100 条官方条目与独立盘点结果,用于检查两组边界、作者顺序、缺页、重复、来源篡改和恢复行为。
|
||||||
|
|
||||||
|
## 独立注册元数据版本
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect icaps2026-registry \
|
||||||
|
--snapshots /data/paper-index/collected/icaps2026-crossref-research-v1 \
|
||||||
|
--out /data/paper-index/collected/icaps2026-registry-normalized-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
该命令只读固定 `crossref-filtered.json`,与官方节目表输出分开。来源是 Crossref 的 ICAPS 期刊注册元数据(ISSN 2334-0843),查询出版日期为 2026 年的结果共 91 条,其中卷 36、期 1 有 90 条,另有卷 35 的一条必须排除。不能仅凭出版日期归属会议届次。来源 URL、抓取时间、原响应哈希和字段过滤快照均保留。
|
||||||
|
|
||||||
|
卷 36 的 90 条均带 DOI 和摘要,输出保持注册标题、作者顺序及摘要版本一致;保留原始 JATS 摘要,只移除已核验的单个段落包装并规范空白,遇到其他标记(尤其公式、上下标)拒绝无声扁平化。作者个人 ID、机构、引用与文件链接没有进入过滤快照。原始注册类型 `journal-article` 留在 metadata 中,不据此给论文添加研究类型标签。
|
||||||
|
|
||||||
|
这里的 accepted 表示注册源记录了正式发表,不代表采到原始签名录用决定;publication_doi 标识发表对应的投稿事件,不是原始投稿号。具体 track 保持待分类。与官方 90 篇名单的来源核查发现:88 个规范标题一致,其中 78 个完整作者名单一致;另有 2 个标题和 10 个作者文字差异待审核。两份输出不会自动建立作品同一性、复制人名别名或覆盖文字。它们也不应未经审核同时导入,避免重复作品或投稿。
|
||||||
|
|
||||||
|
此来源仍标记 partial,不包含公开拒稿。生产导入前需核对 DOI 归属、既有作品及跨会议历史;已知 DOI 也可能存在错误绑定,不能代替最终全库去重验收。
|
||||||
@@ -0,0 +1,40 @@
|
|||||||
|
# ICASSP 2026 注册元数据采集
|
||||||
|
|
||||||
|
这是长期维护项目中的可重复运行采集器,按需离线处理冻结的公开元数据;不会启动数据库导入、联网或下载论文 PDF。命令仅输出待身份审查的来源版本。
|
||||||
|
|
||||||
|
## 输入与使用
|
||||||
|
|
||||||
|
2026-11-17 在数据服务器采集 Crossref 公开 REST API。先从正式节目表抽查的两篇标题确定注册会议名称和 DOI 前缀,再依据 IEEE DOI 前缀、proceedings-article 类型及注册的活动开始日期 `2026-05-03` 分页获取;日期过滤并不意味着所有结果都属于 ICASSP。
|
||||||
|
|
||||||
|
需要服务器 `/data/paper-index/collected/icassp2026-registry-capture-v1` 下的 `manifest.json` 与 `page-001.json` 至 `page-010.json`。将其父目录传给:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect icassp2026-registry \
|
||||||
|
--snapshots /data/paper-index/collected \
|
||||||
|
--out /data/paper-index/collected/icassp2026-normalized-<new-run>
|
||||||
|
```
|
||||||
|
|
||||||
|
输出目录必须不存在。复跑前先检查原任务和产物,不能覆盖成功或失败输出。采集器验证固定清单 SHA256、全部页面原始字节、HTTP 回执、时区、页面数量、总数及唯一 DOI,再生成 JSONL、导入格式的部分覆盖清单、全部原始来源证据和采集报告。导入格式不代表身份审查通过;本命令批准导入数始终为零。
|
||||||
|
|
||||||
|
## 数据范围
|
||||||
|
|
||||||
|
- 10 页共有 4,645 个唯一 DOI;56 个属于 ICPEA,明确排除。
|
||||||
|
- ICASSP 的 4,589 个注册条目中,7 个是封面、目录、作者索引、委员会和审稿人名单;依据已审查的准确 DOI、标题和页码排除,不以“没有作者”泛化排除论文。
|
||||||
|
- 剩余 **4,582 条论文**全部缺少注册摘要,输出 `null`。保留完整注册字段、作者顺序、机构、原始标题、日期及原始来源哈希。
|
||||||
|
- 23 个作者条目只提供单个姓名字段,原样保留并标记;不猜测名字拆分、拼音边界或人物别名。
|
||||||
|
- `accepted` 表示已登记在该会议论文集中的发表观察,并非原始签名录用决定。DOI 用 `publication_doi`,不复制先前投稿系统的 ID。
|
||||||
|
- track 为 `proceedings_unclassified`,不从技术主题推断 main、special session、workshop 或研究类型;没有自造标签。
|
||||||
|
|
||||||
|
正式节目表有 4,589 个编号和标题,与注册条目不是已证明的一一对应关系。独立标题规范化比较得到 4,415 个来源候选和 174 个未匹配节目标题;标题匹配不是作品身份批准,未匹配不等于拒稿、撤稿或未发表。节目表作者与注册作者的版本也不应自动拼接。
|
||||||
|
|
||||||
|
注册活动日期为 5 月 3–8 日,会议网站为 5 月 4–8 日,两者原始值均保留。Crossref 记录中的链接只是原始字段,不会访问 PDF。IEEE Open Preview 入口读取遇到 HTTP 418,未绕过访问限制。公开拒稿元数据是否可得仍为 unknown,本批不是拒稿覆盖证明,也不是第 14 项整体完成证明。
|
||||||
|
|
||||||
|
## 证据与恢复
|
||||||
|
|
||||||
|
注册来源清单 SHA256:`1f8ffa040b1f682a4b65f476f4115cb95629056d8de5c324c8417d823846ae93`。
|
||||||
|
|
||||||
|
原始证据恢复包在服务器 `/data/paper-index/validation/icassp2026-registry-capture-v1-recovery.tar.gz`,本地备份位于 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/` 同名文件,SHA256 为 `cdc506e95800fb8d37ed87f85740e96f56150f2d96daa971de21a5a7a9ab8388`。安全解包并核验后使用上述命令;测试夹具也冻结了全部 10 页及独立论文清单,供远程 CI 验证。
|
||||||
|
|
||||||
|
本批还需生产规范化验收、数据库当前 DOI/作品关联及历次投稿审查。所有导入结束后仍须执行全库去重、覆盖、摘要和历史完整性审计。编译、打包及测试均交给用户指定的 `ces-control:host` runner。
|
||||||
|
|
||||||
|
参考:[Crossref 筛选器文档](https://www.crossref.org/documentation/retrieve-metadata/rest-api/rest-api-filters/)、[正式录用标题表](https://cmsworkshops.com/ICASSP2026/papers/accepted_papers.php)、[会议日程](https://2026.ieeeicassp.org/technical-program/)、[发表和展示类型说明](https://2026.ieeeicassp.org/publishing-and-paper-presentation-options/)。
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
# ICDM2026 冻结日程适配器
|
||||||
|
|
||||||
|
这是长期维护、按需运行的来源模块。它读取已经冻结的官方页面,离线生成可审阅的标准记录,不连接数据库、不请求网络、不下载论文 PDF;后续入库仍需作品身份、发表版本和旧投稿历史核查。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect icdm2026-program \
|
||||||
|
--snapshots /data/paper-index/collected \
|
||||||
|
--out /data/paper-index/collected/icdm2026-normalized-pass01
|
||||||
|
```
|
||||||
|
|
||||||
|
输入根目录下须有 `remaining2026-official-leads-v1/manifest.json`,及 manifest 所列的三个 ICDM HTML:`icdm-detailed-sessions.html`、`icdm-research-policy.html`、`icdm-applied-policy.html`。共享研究 manifest 也保留其他会议的入口回执,但本适配器只读取并验证 ICDM 的三个明确 URL,不要求复制其他会议文件。固定 manifest SHA-256 为 `0e13313d94e3bff17f071d3a1384f6ed83ece156239fe5bc4633cc7a34fb8e04`。
|
||||||
|
|
||||||
|
适配器核对每个回执的 URL、HTTP 200、HTML 内容类型、时区时间戳、原始字节数和哈希;保留 CRLF。来源更新必须另存版本并重新审阅,不能修改旧 manifest 来冒充旧快照。源码测试 fixture 同时保存原始页面、回执和使用另一解析方法得到的独立逐行盘点。
|
||||||
|
|
||||||
|
## 两个明确来源轨道
|
||||||
|
|
||||||
|
[官方详细日程](https://icdm2026.neu.edu.cn/DetailedSessions/list.htm)自称为线下展示清单,日期为 2026 年 11 月 13–15 日,34 个分会场、324 个不同节目编号。适配器分别输出:
|
||||||
|
|
||||||
|
| 输出文件 | 官方明确标签 | 记录数 |
|
||||||
|
| --- | --- | ---: |
|
||||||
|
| icdm-program-2026-research.jsonl | research / Research Track | 241 |
|
||||||
|
| icdm-program-2026-applied.jsonl | applied / Applied Track | 83 |
|
||||||
|
|
||||||
|
`data-track` 属性必须与可见标签一致;轨道来自这两份明确证据,不根据标题、会场名称或研究主题推测。`data-format` 和可见 Long/Short 标签、展示说明必须一致;168 个 long 和 156 个 short 仅表示演讲形式,**不自动转成论文长短、position、findings 或研究类型**。日程主题也不生成自定义人物或研究分类。
|
||||||
|
|
||||||
|
每条保留日程显示标题、原始作者文字、分号分隔的来源姓名顺序、节目编号、分会场、时间、地点、全局来源位置、来源回执和对应原文文件。作者仅标为日程报告的姓名,不声称为最终出版社完整作者版本,也不生成姓名别名。
|
||||||
|
|
||||||
|
`2026/DM...` 是 `icdm_program` 来源下的节目定位键;页面锚点只是来源定位,不能当 DOI、arXiv 或原始签名投稿/决定标识。`identifiers` 和 `submission_identifiers` 都为空。`status=accepted` 表示列入官方研究/应用日程的来源观察,`raw_decision` 明确其并非原始签名决定或已核验出版社记录;不根据该状态生成原始决定日期。每条 `original_decision_signature_verified=false`。
|
||||||
|
|
||||||
|
## 始终保留范围限制
|
||||||
|
|
||||||
|
324 条日程记录都没有摘要,输出 `abstract=null`,每个文件的 `missing_abstract_count` 如实填写。manifest 始终为 partial;`source_expected_count` 仅对应这个冻结线下日程分区,不能当作所有录用、正式出版或线上展示的应有总数。其他页面的 workshop、博士论坛、BlueSky、demo 等不在本次输入范围内,也不能假定不存在。
|
||||||
|
|
||||||
|
标签使用 `icdm_program:session_track` 与 `icdm_program:presentation` namespace;它们是平台来源证据,不与其他来源标签或用户自定义标签混为一谈。`paper_type` 保留为空。
|
||||||
|
|
||||||
|
[研究轨政策](https://icdm2026.neu.edu.cn/CallforResearchTrack/list.htm)描述 triple-blind 审稿,[应用轨政策](https://icdm2026.neu.edu.cn/CallforAppliedTrack/list.htm)描述 single-blind 审稿,并指向 CyberChair 投稿系统。这些是审稿期间政策,**不能据此断言最终拒稿元数据不公开**。报告保留 `public_rejected_metadata=unknown_no_public_export_verified`;未访问登录资料、未绕过限制,也未从日程缺席推断 rejected、desk_rejected 或 withdrawn。
|
||||||
|
|
||||||
|
## 验证与恢复
|
||||||
|
|
||||||
|
完整测试在已授权 `ces-control:host` 远程 runner 执行,包含全部 324 条与独立盘点逐字段对照、轨道/展示属性与可见文字冲突、重复或变化的节目编号、会场计数、空作者、原始字节/回执校验和失败输出保护。CI 另在安装 wheel 后、工作目录外运行真实 CLI,检查 241/83 两个输出及缺失摘要、未生成稳定标识等边界。构建和测试不在用户本机运行;实际结果见对应提交的远程 CI。
|
||||||
|
|
||||||
|
输出目录必须不存在。输入校验失败时不会创建输出;输出生成失败会保存 `failed.json`,成功报告最后原子发布为 `collection-report.json`。遇到已有目录,先核对原操作,不覆盖、不盲目重放。
|
||||||
|
|
||||||
|
原始来源恢复包为 `/data/paper-index/validation/remaining2026-official-leads-v1-recovery.tar.gz`,SHA-256 为 `0be8c84bb5c591e5afaaa5f95dd5334bf13a9be1671cb942e46e998f502495bb`。独立盘点为 `/data/paper-index/validation/icdm2026-independent-program-inventory-v1.json`,SHA-256 为 `e13eaea0a158dcaafdf5dfb4af7948e4793b0e98041ec87d42b52513acfdc5f6`;本机工作记录目录也保留同名备份。
|
||||||
|
|
||||||
|
下次继续先验证源码版本、原始证据和实际输出,再寻找可公开取得的摘要与正式出版记录,核查逐篇版本、稳定标识及旧投稿历史,准备新的只读预检和可审阅入库队列。完成日程规范化不意味着 ICDM2026 整届出版覆盖、公开负面决定普查或最终全库去重完成。
|
||||||
@@ -0,0 +1,50 @@
|
|||||||
|
# ISMIR 2026 官方节目与摘要采集
|
||||||
|
|
||||||
|
这是长期维护项目中的离线采集器,按需处理冻结的官方元数据,不联网、不执行网页脚本、不下载论文 PDF、不连接数据库。输出仍须经过作品身份与投稿历史审查。
|
||||||
|
|
||||||
|
## 使用
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect ismir2026-program \
|
||||||
|
--snapshots /data/paper-index/collected \
|
||||||
|
--out /data/paper-index/collected/ismir2026-normalized-<new-run>
|
||||||
|
```
|
||||||
|
|
||||||
|
输入父目录需要三个冻结目录:
|
||||||
|
|
||||||
|
- `remaining2026-listings-research-v2`:清单、`ismir-accepted.html`、`ismir-call.html`。
|
||||||
|
- `ismir2026-script-research-v1`:清单及 `papers.js`,用于证明官方程序明确加载的数据文件和 poster 链接模板;只读取脚本原文。
|
||||||
|
- `ismir2026-json-research-v1`:清单及 `papers.json`,提供 140 项完整元数据和公开摘要。
|
||||||
|
|
||||||
|
固定清单和全部输入原始字节均校验 SHA256、HTTP 回执、官方 URL 及带时区的采集时间。已有输出目录不能覆盖;先检查原任务是否已结束,再选择新目录。失败保留 `failed.json`,只有全批成功才发布 `collection-report.json`。
|
||||||
|
|
||||||
|
## 来源与版本
|
||||||
|
|
||||||
|
[正式录用名单](https://ismir2026.ismir.net/accepted-papers)和[官方虚拟平台](https://ismir2026program.ismir.net/papers.html)各有 140 项,经来源内标题对应核查一致;其中 2 处标题有直引号/弯引号文字差异,3 处作者名单有差异,两个来源的原始写法分别保留并标记,不静默修正或拼接作者。
|
||||||
|
|
||||||
|
作者差异涉及节目 ID `60`、`126`、`433`,包含大小写和姓名完整度问题。正式名单的作者机构可能含括号和分号,采集器只在括号外拆分作者;作者原始字符串也保留。来源内名单对应不代表数据库作品身份已经获准关联。
|
||||||
|
|
||||||
|
- 原始 JSON 摘要全文保留,包括数学符号、换行和 Unicode,不作扁平化提取。
|
||||||
|
- 数字 `id`、`forum` 是 miniconf 节目定位符,不视为原始 OpenReview 投稿 ID;不推断 DOI 或 arXiv ID,投稿标识列表保持空。
|
||||||
|
- `accepted` 来自正式录用名单与节目平台,不是原始签名决定。`track=program_unclassified`,不从主题、session 或远程展示方式推断研究方法、workshop 或论文类型。
|
||||||
|
- `ismir_program:keyword` 保存来源非空关键词,规范空白并去除同条记录内完全重复值;原始关键词列表(含空项)保留在元数据。
|
||||||
|
- `ismir_program:presentation` 保存来源展示方式。Remote/In-person 是展示方式,不是研究类型。
|
||||||
|
- poster URL 严格依据冻结的官方 JS 模板生成;本命令不请求 poster 页面。
|
||||||
|
|
||||||
|
## 公开评审与拒稿边界
|
||||||
|
|
||||||
|
原始来源中有 **101 组明确选择公开的评审材料**;其余 39 项 `publish_reviews=No`,评审正文为空。采集器验证此偏好和正文的一致性,原文保存在来源元数据,不把评审意见或 meta-review 中的推荐转换成额外的最终决定。
|
||||||
|
|
||||||
|
这些记录来自录用节目,不能据此当作 101 条公开拒稿记录。公开拒稿导出是否可得仍记为 unknown,需要单独核查。[征稿与评审流程](https://ismir2026.ismir.net/authors/call-for-papers)中的评审规则也不等于具体投稿决定。
|
||||||
|
|
||||||
|
输出包含一份 140 条 JSONL、部分覆盖清单、七个原始来源文件及报告,批准导入数为零。采集成功不等于会议论文/track/公开负面决定覆盖完整,也不替代最后的全库去重审计。
|
||||||
|
|
||||||
|
## 恢复与验证
|
||||||
|
|
||||||
|
`papers.json` SHA256:`864d384cbc928554504e1b39849137bb380143e247a48f2d71563a698cb8e62d`。
|
||||||
|
|
||||||
|
原始 JSON 和 JS 恢复包:服务器 `/data/paper-index/validation/ismir2026-json-research-recovery-v1.tar.gz`,本地 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/` 同名文件;SHA256 为 `bba7b236cfe4d8b259d6c1230b3fe09a3e9071f5203884c92b016ca0718fb095`。
|
||||||
|
|
||||||
|
正式名单与政策恢复包:同目录 `remaining2026-listings-research-v2-recovery.tar.gz`,SHA256 为 `f5ef7c2e73d46712146e4d6711ad5ad562e16b8b55c0cd75d1041cd0b81142cb`。安全恢复三个源目录、核验文件后可重跑上述命令。
|
||||||
|
|
||||||
|
完整来源和独立 140 项清单在测试夹具中冻结。测试、构建、打包、安装及实际隔离目录 CLI 验证只在用户指定 `ces-control:host` runner 执行。生产规范化后还需独立核验全部记录,并用 PostgreSQL 只读预检检查候选作品和既往投稿;不自动合并来源差异或人物身份。
|
||||||
@@ -0,0 +1,73 @@
|
|||||||
|
# MM 2026 出版条目采集与恢复
|
||||||
|
|
||||||
|
`collect mm2026-publications` 是包内可复用的冻结快照规范化命令,读取完整注册目录和官方节目,不连接数据库、不请求网络、不下载PDF。总体仍为13/17,来源规范化、测试通过或节目匹配都不能替代身份批准、生产入库或最终去重。
|
||||||
|
|
||||||
|
## 使用
|
||||||
|
|
||||||
|
在已配置的服务器环境中,`--registry-snapshots` 是包含 `mm2026-container-registry-v2/` 的父目录;`--program-snapshots` 是包含 `mm2026-program-and-public-probes-v1/` 的父目录。父论文集正文使用独立固定快照,不能从不存在的文章ISBN字段猜测:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect mm2026-publications \
|
||||||
|
--registry-snapshots /data/paper-index/collected \
|
||||||
|
--program-snapshots /data/paper-index/collected \
|
||||||
|
--parent-snapshot /data/paper-index/collected/mm2026-publication-metadata-v1/parent.json \
|
||||||
|
--out /data/paper-index/collected/mm2026-publications-native-v3
|
||||||
|
```
|
||||||
|
|
||||||
|
输出目录必须不存在。失败输出保留 `failed.json`,完整 `collection-report.json` 只在全部校验成功后原子生成;不要覆盖输出或因旧进度文件、日志安静重跑。先观察实际单元/报告,再选择新的恢复目录。
|
||||||
|
|
||||||
|
## 注册目录与节目分别保留
|
||||||
|
|
||||||
|
源目录基于 Crossref [官方精确过滤器](https://www.crossref.org/documentation/retrieve-metadata/rest-api/rest-api-filters/) 的 `container-title` 和出版者member320,实际捕获8页、1918个不同DOI。每页请求、时间、原始正文、字节哈希、总量与唯一DOI均核验。父DOI为 `10.1145/3767308`,题名为 *Proceedings of the 34th ACM International Conference on Multimedia*,明确事件为MM2026;文章均注册为 `proceedings-article`。
|
||||||
|
|
||||||
|
ISBN查询和父Unixref XML只返回父项,不能代表逐篇目录。宽泛 `query.container-title` 的检索总量也不能当会议篇数。当前1918是这个明确论文集过滤范围内的注册条目数,不是已经去重的科研作品数、所有MM workshop或全部28个来源track的出版全集。
|
||||||
|
|
||||||
|
每个出版条目都输出,稳定外部标识为真实DOI。身份标识仅为 `doi`,投稿侧标识仅为 `publication_doi`;不推断原始OpenReview/forum/投稿号或arXiv标识。来源名 `mm_registry_program` 明确表示派生的注册/节目材料,不能把其中所有字段都说成Crossref原文。
|
||||||
|
|
||||||
|
原始注册响应及父记录复制到 `evidence/registry/`,完整节目及四个既有403回执复制到 `evidence/program/`,不重新请求受限入口。每项元数据保留注册正文的文件/哈希、数组位置和规范JSON对象哈希;常用书目字段另作投影,完整引用、资助等原始字段仍在原始响应中,未丢弃也不在每条JSONL重复膨胀。
|
||||||
|
|
||||||
|
## 分轨、摘要与版本差异
|
||||||
|
|
||||||
|
按原始题名的 Unicode/字母数字规范化与完整作者多重集合比对,1726 项有唯一节目对应。原始题名没有候选时,再比较经过受限排版投影的题名:只移除平衡、无属性的 `i`、`scp`、`sup`、`sub` 标签,保留上下标内容,HTML 实体只解码一次。未知或不平衡标签、属性、注释及声明不参与这种投影;不使用模糊题名或人物别名。
|
||||||
|
|
||||||
|
26 项新增对应均保留完整作者多重集合一致、唯一节目 ID 以及逐项复核名单,复制节目原文的摘要、track 和展示上下文。原始注册题名、作者和节目字段继续逐字保留,比较文本单独存入 `program_title_formatting_comparison`;这只证明节目材料对应,不批准科学作品合并。原始题名已有作者分歧时,不通过排版回退绕过。其余 122 项原始作者分歧、1 项排版题名下的作者分歧、43 项题名差异,共 166 项仍在 `proceedings_unclassified`。
|
||||||
|
|
||||||
|
| 输出分区 | 注册条目 |
|
||||||
|
| --- | ---: |
|
||||||
|
| main | 1555 |
|
||||||
|
| grand_challenge | 77 |
|
||||||
|
| interactive_art | 17 |
|
||||||
|
| datasets | 41 |
|
||||||
|
| reproducibility | 1 |
|
||||||
|
| bni | 22 |
|
||||||
|
| open_source | 11 |
|
||||||
|
| doctoral_symposium | 2 |
|
||||||
|
| demo_and_video | 23 |
|
||||||
|
| industry_talk | 3 |
|
||||||
|
| proceedings_unclassified | 166 |
|
||||||
|
|
||||||
|
注册摘要 1918 项仍全缺失。1743 项有明确对应的节目摘要:1717 项使用 `abstract_source=official_program_exact_metadata_correspondence`;新增 26 项使用 `official_program_reviewed_title_formatting_correspondence`,两者都逐字复制节目原文。报告分别列出 `abstracts_from_literal_program_correspondence` 和 `abstracts_from_reviewed_title_formatting`;保留的汇总字段 `abstracts_from_exact_program_correspondence` 为 1743,表示两类非模糊对应之和。
|
||||||
|
|
||||||
|
其余 175 项保持摘要缺失,包括 9 项节目本身没有摘要及 166 个版本待核条目。注册原始摘要仍标为缺失,不能把节目内容说成出版者摘要。
|
||||||
|
|
||||||
|
8 个作者仅有单一姓名字段,保留登记值并标记,不猜测 given/family;37 个含排版标记或实体的题名仍保留字面原文和原始标记。排版比较保留上下标字符,不改写原始字段。
|
||||||
|
|
||||||
|
完整2143个节目ID和4326个展示位置仍在匹配证据或391个未匹配节目中,28个原始track均保留。391个节目与166个出版版本待核来自两种观察口径,不能直接相加当成不同论文。Workshop、艺术、行业等贡献不能因为不在这个论文集范围就被当成拒稿、撤稿或不值得保留。
|
||||||
|
|
||||||
|
注册出版支持来源侧 `status=accepted` 的发表观察,`raw_decision` 明确不是原始签署决定;出版日期仅放在注册元数据中,不伪造决定日期。`mm_registry_program:crossref:type` 与 `mm_registry_program:mm_program:track` 分别保存注册书目类型和节目 track;共同的派生来源前缀满足写入器的来源校验,两个标签命名空间仍相互独立,原始平台字段与证据继续保留在各自元数据中,`research_type_classified=false`:书目类型不是研究类型,行业、艺术、demo条目不能默认变成主轨研究论文。
|
||||||
|
|
||||||
|
## 只读数据库预检与导入边界
|
||||||
|
|
||||||
|
每个分区都有带count/SHA256的manifest,coverage始终为partial。规范化完成后,对实际报告中的全部输出文件运行现有 `audit source-identities --files ... --pg-service foundry-research --pg-schema research --out <新目录>`,不能靠已有66个arXiv/DOI候选代表全目录。预检使用只读一致性快照,并保存来源/标识/历次投稿/逐次决定的证据包;缺失或截断必须单列。
|
||||||
|
|
||||||
|
`approved_for_import=0`,每项 `scientific_work_identity_approved=false`。DOI绑定可能有误,精确题名/作者匹配不证明科学作品身份,没有匹配也不证明新作品。进入增量导入前仍须审查已有绑定、科学版本与候选历史,做新鲜保护检查、完整基线/备份,再使用原生写入者。不得覆盖旧拒稿、后续录用、标签、原始元数据或人物姓名版本;既有已消费队列不可重放。
|
||||||
|
|
||||||
|
## 恢复
|
||||||
|
|
||||||
|
完整注册目录及独立节目比对恢复包:`/data/paper-index/validation/mm2026-container-registry-recovery-v2.tar.gz`,SHA256 `ddb05a8134f6437b3fb40250e45ed6350d55431d953812abda3300871b6dd0ac`。其中含注册源目录、父记录 `validation/mm2026-parent.json`、逐项比对和请求方法证据。安全解包后可把该父记录路径传给 `--parent-snapshot`。
|
||||||
|
|
||||||
|
节目原始来源恢复包见 [MM来源说明](mm2026-source-status.md)。本地独立核验备份位于 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`。固定回执/正文哈希变化、总量变化、未知注册上下文或节目对应改变都须重新核查,不能放宽为无条件接受。最终全库去重、近似题名作者、标识冲突、摘要、约束、覆盖与查询审计仍是独立必做阶段。
|
||||||
|
|
||||||
|
实际导入回归在隔离的 SQLite 和 PostgreSQL 测试库中运行全部1918条、全部11个输出分区,核验原始字段、1918个书目类型标签、1752个节目标签、publication_doi 身份、partial覆盖,以及已有作品的规范信息和旧拒稿决定不被覆盖。这些测试不会写入生产 research schema;测试通过仍不替代逐条生产身份审阅。
|
||||||
|
|
||||||
|
旧的 `native-v1`、`native-v2` 来源目录及已消费的导入队列不可覆盖或重放。新版本写入新的输出目录,先核对已导入记录的源字段和分轨是否变化;新增来源关联仍须新鲜身份预检、保护基线和独立验收。程序输出的分区预期数量已改变,未来增量 manifest 必须明确保留旧覆盖父值并声明新预期值,不能重置已导入计数。
|
||||||
@@ -0,0 +1,139 @@
|
|||||||
|
# ACM MM 2026 官方节目来源状态
|
||||||
|
|
||||||
|
2026-11-17 已冻结官网正式日程、技术日程、征稿/评审政策、两个日程脚本和脚本明确引用的 `program-data.json`。本页记录可恢复的来源证据,不宣称这些节目贡献全部都是已发表科研论文;总体仍为 13/17,第 14 步未完成。
|
||||||
|
|
||||||
|
## 已取得的数据
|
||||||
|
|
||||||
|
[官方技术日程](https://2026.acmmm.org/site/technical-programme.html)明确加载 `technical-programme.js`,该脚本读取 [program-data.json](https://2026.acmmm.org/site/program-data.json)。原始 JSON 为 10,808,209 字节,生成时间为 `2026-11-14T18:36:26.860677+00:00`,保存了 5 天、20 个房间、157 个日程场次。
|
||||||
|
|
||||||
|
递归盘点得到 **4,326 次论文样式的节目观察,涉及 2,143 个不同节目 ID**。额外 2,183 次是直接列表和子场次/多次展示中重复的观察。相同 ID 的标题、作者、摘要、track 和机构字段一致;所有来源位置均保留,不能把重复展示累加成新论文。
|
||||||
|
|
||||||
|
独立盘点的逐 ID 来源 track 数量如下。它们来自 `paper.track`;157 个外层日程条目的 `entry.track` 均为空,不能用外层空字段覆盖真实来源分类。展示形式和学术 track 也不是同一概念。
|
||||||
|
|
||||||
|
| 来源 track 原文 | 不同节目 ID 数 |
|
||||||
|
| --- | ---: |
|
||||||
|
| MRAC | 35 |
|
||||||
|
| SUMAC | 9 |
|
||||||
|
| NeuroMM | 10 |
|
||||||
|
| IMPACT | 8 |
|
||||||
|
| RichMediaGAI | 10 |
|
||||||
|
| MMSports | 16 |
|
||||||
|
| MUWS | 5 |
|
||||||
|
| EReL-MIR | 7 |
|
||||||
|
| GreenMM | 10 |
|
||||||
|
| DFF | 12 |
|
||||||
|
| Main Track | 1703 |
|
||||||
|
| Grand Challenge | 97 |
|
||||||
|
| Doctoral Symposium | 4 |
|
||||||
|
| Datasets | 50 |
|
||||||
|
| Interactive Art | 23 |
|
||||||
|
| Reproducibility | 1 |
|
||||||
|
| Demo and Video | 27 |
|
||||||
|
| Open Source | 14 |
|
||||||
|
| Industry Talk | 7 |
|
||||||
|
| BNI | 25 |
|
||||||
|
| QoEVMA | 4 |
|
||||||
|
| HUMA | 14 |
|
||||||
|
| WIPAI | 8 |
|
||||||
|
| AMI | 11 |
|
||||||
|
| Mulsemedia | 9 |
|
||||||
|
| UAVM | 11 |
|
||||||
|
| MCHM | 7 |
|
||||||
|
| JAV-CG | 6 |
|
||||||
|
|
||||||
|
共有 1,927 个节目 ID 带摘要,216 个缺少摘要。Main Track 的 1,703 个 ID 中,有两项同时缺少作者与摘要:
|
||||||
|
|
||||||
|
- `mfp4971`:ASDG: MLLM-based Audio Semantics for Dance Generation。
|
||||||
|
- `mfp7801`:Dynamic Late Fusion Multiple Kernel k-means Clustering based on Adaptive Sample Selection and Alignment。
|
||||||
|
|
||||||
|
两项仍有科学标题,原文没有明确撤稿或拒稿标签;因此只能标记字段缺失、暂缓关联,不能推断负面决定。非主轨的 workshop、grand challenge、行业报告、艺术、博士生等项目还需逐轨核对存档与贡献类型,不能把全部 2,143 个节目 ID 都计作已发表科研论文。原始公开联系邮箱不进入独立盘点、人物别名或论文标识。
|
||||||
|
|
||||||
|
## 公开负面决定的访问边界
|
||||||
|
|
||||||
|
[官方 CFP](https://2026.acmmm.org/site/cfp-guidelines.html)链接至 [OpenReview 群组](https://openreview.net/group?id=acmmm.org/ACMMM/2026/Conference)。匿名群组元数据可读,但根据群组给出的原始 submission invitation、rejected/withdrawn/desk-rejected venue ID,进行的四次 `limit=1` 匿名 notes 核查全部返回 **HTTP 403**。
|
||||||
|
|
||||||
|
保存了准确查询、时间、状态、响应原文和哈希;没有认证、重试或绕过权限。403 是已观察接口的访问限制,不代表零条拒稿、从未拒稿,或排除了其他可能公开的来源。评审公开规则也不能替代具体最终决定记录。当前不从评分、评审推荐、节目缺席或空字段制造 rejected/withdrawn 事件。
|
||||||
|
|
||||||
|
## 产物与恢复
|
||||||
|
|
||||||
|
原始目录(均相对 `/data/paper-index/collected/`):
|
||||||
|
|
||||||
|
- `mm2026-program-policy-research-v1`:六页官方日程和分轨/评审政策。
|
||||||
|
- `mm2026-navigation-research-v1`:官网明确加载的导航原文。
|
||||||
|
- `mm2026-program-data-discovery-v1`:两份脚本及公开群组元数据。
|
||||||
|
- `mm2026-program-and-public-probes-v1`:完整节目 JSON 和四个匿名访问限制回执。
|
||||||
|
|
||||||
|
独立盘点:`/data/paper-index/validation/mm2026-independent-program-inventory-v1.json`,包含每个节目 ID 的所有来源位置、track、标题/作者、摘要可用性及核心版本数量。全部响应原文、回执及盘点已在本机备份,并独立核验了原始字节哈希。
|
||||||
|
|
||||||
|
| 证据 | SHA256 |
|
||||||
|
| --- | --- |
|
||||||
|
| program-data.json | 08e2976ecba2d0d535000b0935f41c85076ec674e518a1df2c5843d9bde4d083 |
|
||||||
|
| 独立逐 ID 盘点 | 375dd975fdff023f2ff591d47066ec616c2bfc3b717b6016ab162d74d587f3e7 |
|
||||||
|
| 完整来源恢复包 | 8247149ed7437a36c13ca2b6128431e2b0f3f8a388a5ddefb62427b0c1ce6f8d |
|
||||||
|
|
||||||
|
服务器恢复包:`/data/paper-index/validation/mm2026-source-discovery-and-inventory-recovery-v1.tar.gz`;本地备份:`/Users/zzyang/Documents/Codex/2026-11-06/paper-index/` 同名文件。先安全解包并核验回执,再做规范化,不能重新抓取 403 入口作为恢复方式。
|
||||||
|
|
||||||
|
## 可复用采集命令
|
||||||
|
|
||||||
|
在已配置的服务器环境中运行,`--snapshots` 是包含 `mm2026-program-and-public-probes-v1/` 的父目录;恢复时先安全解包上述已核验恢复包。命令只读快照,不请求网络、不连接数据库,不重新访问403入口,也不下载PDF:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect mm2026-program \
|
||||||
|
--snapshots /data/paper-index/collected \
|
||||||
|
--out /data/paper-index/collected/mm2026-native-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
输出目录必须不存在;已存在、失败或部分输出均不能覆盖重跑,应先查看报告,另选新的目录。`collection-report.json` 仅在全量验证成功后生成;失败保留 `failed.json`,不冒充完成。
|
||||||
|
|
||||||
|
采集器核验固定回执、正文哈希、URL、抓取时间、完整逐轨数量和字段缺失盘点。28个逐轨 JSONL 保留原始标题、作者、机构、逐字摘要以及同一节目ID的全部展示位置、时间、poster/oral号和展示kind;157个场次及子场次/非论文活动另存 `sessions.jsonl`。外层空track不覆盖 `paper.track`,不同核心版本、未知字段或未处理嵌套需要重新审查,不能静默丢弃或取第一条。公开联系邮箱数组只留在原始证据中,不进入规范记录。`evidence/` 保留原始节目JSON、完整回执及四个既有403响应。
|
||||||
|
|
||||||
|
这些文件的格式为 `mm_program_observation_v1_not_ingestion_record`,所有项 `approved_for_import=false`。节目ID仅为 `mm_program:program_id` 来源定位符,track标签使用 `mm_program:track`;不建立科学作品、原始投稿ID或最终决定,不推断 DOI/arXiv/OpenReview 标识、论文类型、拒稿或撤稿。行业报告、艺术、workshop及其他来源贡献均保留,尚需逐轨核对贡献类型与正式出版依据。
|
||||||
|
|
||||||
|
本批尚未形成生产导入队列。后续做贡献类型、作品/出版身份及既往投稿审查,再生成经批准的增量队列。已消费的其他会议导入队列不可重放;最终全库去重和完整审计仍为独立必做阶段。
|
||||||
|
|
||||||
|
## 直接审计节目身份与既往历史
|
||||||
|
|
||||||
|
全量节目观察可以直接进入只读候选审计,无需伪造成录用论文队列:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
PGSERVICEFILE=/etc/paper-atlas/pg_service.conf \
|
||||||
|
paper-atlas audit source-identities \
|
||||||
|
--mm-program-report /data/paper-index/collected/mm2026-native-v1/collection-report.json \
|
||||||
|
--pg-service foundry-research --pg-schema research \
|
||||||
|
--out /data/paper-index/collected/mm2026-preflight-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
`--mm-program-report` 与常规 `--files` 互斥。审计先核对冻结原文、全部28轨的逐文件哈希/数量,以及每个节目ID的完整字段和展示上下文;缺轨、重复ID、篡改后的标识/决定或上下文均在数据库连接前拒绝。报告中记录的绝对文件路径仅作来源证据,实际读取报告同目录中的对应文件。
|
||||||
|
|
||||||
|
候选审计中的 `status=unknown` 仅为内部查询占位符,不代表原始提交的最终状态。节目ID以 `mm_program` 来源键查询,不转成全局 DOI、arXiv 或 OpenReview 标识;保留原始 `listed_in_official_program` 观察与全部展示位置。所有贡献类型、出版身份、最终决定核验标志均为false;缺作者的两项也保留并标记。
|
||||||
|
|
||||||
|
查询使用 PostgreSQL 只读一致性快照,候选作品的原始来源、标识、历次投稿和逐次决定压缩保存并记录哈希;截断会显式标记。标题/作者候选匹配不是作品身份批准,没有匹配也不证明是新论文。既往拒稿和后续录用不能互相覆盖,评分或评审推荐不能充当最终决定。输出目录必须不存在;遇到已有输出或静默运行,先检查具体单元和报告,不重启重放。
|
||||||
|
|
||||||
|
## 已完成的规范化、只读审计与出版核对
|
||||||
|
|
||||||
|
采集器提交 `c8db611` 和直接节目审计提交 `09a2d4e` 已通过PR #65、#66由用户合并。授权远程runner的535/543项测试、wheel构建安装和安装后全量命令检查通过;没有本机测试或构建。服务器规范化和只读候选报告也已独立验收,并在本机备份、复核全部2143项与814个候选证据包。
|
||||||
|
|
||||||
|
| 只读候选分类 | 来源节目项数 |
|
||||||
|
| --- | ---: |
|
||||||
|
| 标题与完整作者多重集合一致,仍需作品核查 | 660 |
|
||||||
|
| 仅部分作者重叠 | 153 |
|
||||||
|
| 无精确标题候选,不证明是新作品 | 1329 |
|
||||||
|
| 有标题候选但无作者一致性 | 1 |
|
||||||
|
|
||||||
|
全部4326个展示上下文一致,814个证据包均无截断。24篇候选作品涉及24次已有负面投稿、37条负面来源观察;37不能直接算作37次拒稿。所有候选均尚未批准身份或入库,旧负面决定必须在之后关联时保留。
|
||||||
|
|
||||||
|
对已有作品的原始arXiv记录、标题/完整作者多重集合和MM2026语境进一步筛查,得到67个 Main Track 出版DOI核对候选。独立查询其明确给出的DOI及父论文集,共68次单次请求,67次HTTP200、1次404,没有认证或重试。父注册记录为[第34届ACM Multimedia论文集](https://doi.org/10.1145/3767308),ISBN `9798400722134`,出版者ACM,出版年2026。
|
||||||
|
|
||||||
|
其中66项的原始arXiv引用DOI与注册DOI一致,按既有审计规则规范化的标题、完整作者多重集合、出版年、出版者、明确会议名和论文集标题也一致。注册文章未提供ISBN,不能伪称逐篇ISBN已核验;归属依据是文章明确给出的会议和论文集字段,以及独立的原始arXiv语境和DOI。仍需逐项作品/版本审查及新鲜数据库保护检查,**不是66项已入库或最终身份批准**。
|
||||||
|
|
||||||
|
`mfp7067` 对应既有候选作品2749947,原始arXiv提供的 `10.1145/3767308.3836169` 返回404,暂缓。404不是撤稿/拒稿证据,也不据此删除或修正既有标识;需独立核对后再处理。
|
||||||
|
|
||||||
|
完整会议出版名单仍需继续核对。实际ISBN查询只返回父论文集一个条目,父DOI的公开Unixref XML也只含该父DOI、0个 `conference_paper` 节点;都不能代表逐篇名单。按会议名检索得到的 `total-results` 是出版者全年范围内的宽泛检索量,不能当作MM篇数;必须逐项核对明确会议/论文集字段、节目与分轨归属,并保留未匹配项。全部28个来源track继续保留,不能把本批Main Track候选当成会议完整覆盖。
|
||||||
|
|
||||||
|
| 已独立验证的恢复证据 | SHA256 |
|
||||||
|
| --- | --- |
|
||||||
|
| 全轨规范化恢复包 | b119081618b8ce98037dc139be159512f9fdb86a0ccfb127c8b8a1c5722f3d87 |
|
||||||
|
| 只读候选/814证据包恢复包 | 42d0d0404bba04e64e638344b050ad4e7287c3e4c75a777b7069ecc7d7a6ed60 |
|
||||||
|
| 67项DOI及父论文集原始注册回执恢复包 | dd7a0ff76f12070f403cc7352ac54269e454758d807896a086c58c1c1ac7b798 |
|
||||||
|
|
||||||
|
服务器恢复包位于 `/data/paper-index/validation/`,分别名为 `mm2026-native-recovery-v1.tar.gz`、`mm2026-preflight-recovery-v1.tar.gz`、`mm2026-publication-metadata-recovery-v1.tar.gz`;本地同名备份在 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`。来源目录为 `collected/mm2026-native-v1`、`mm2026-preflight-v1`、`mm2026-publication-metadata-v1`。先核对完整manifest及实际单元结果,不能用旧进度文件的running字样重启。其他会议已消费的导入队列仍禁止重放;本轮均为来源核查、只读候选审计,没有生产写入。
|
||||||
@@ -0,0 +1,56 @@
|
|||||||
|
# NeurIPS 官方节目元数据
|
||||||
|
|
||||||
|
`paper-atlas collect neurips-program` 是按需运行的采集器,当前核验范围是 2026 年的[官方 orals/posters JSON](https://neurips.cc/static/virtual/data/neurips-2026-orals-posters.json)。它只请求这个固定公开元数据文件,不连接数据库,不请求 OpenReview API,不下载 PDF、视频或个人资料页。模块位于 `paper_atlas.collectors.neurips_program`,通过统一 CLI 使用。
|
||||||
|
|
||||||
|
## 已核验快照
|
||||||
|
|
||||||
|
2026-11-09T17:57:40Z 冻结的原始响应共 20,447,021 字节,SHA256 `879293a863057fa05370443b3b5ddee12a10eeb37c810abc6c1d53f71f545997`。原始 `count=9276` 与结果数组长度一致,`next`、`previous` 均为空;这只证明此份公开文件已完整取得,不是全部会议出版物或公开拒稿全集。
|
||||||
|
|
||||||
|
按官方逐条 `sourceurl` 中的明确投稿来源分开处理:
|
||||||
|
|
||||||
|
| 原始 OpenReview group | 数据库 track | 不同 forum 数 |
|
||||||
|
| --- | --- | ---: |
|
||||||
|
| `NeurIPS.cc/2026/Conference` | `main` | 7896 |
|
||||||
|
| `NeurIPS.cc/2026/Evaluations_and_Datasets_Track` | `evaluations_datasets` | 971 |
|
||||||
|
| `NeurIPS.cc/2026/Position_Paper_Track` | `position` | 137 |
|
||||||
|
| 合计 | | 9004 |
|
||||||
|
|
||||||
|
只接受已核验的显式 `Accept`、`Accept (poster/spotlight/oral)` 标签,保留原始大小写和文本。未知来源、决定、展示类型、可见性或无法验证的 forum 链接进入 `held_presentations`,不推成 accepted 或 rejected。空标题、缺少作者、相同 event ID 不同内容、同一 forum 的题名/作者/来源/决定冲突会停止采集。
|
||||||
|
|
||||||
|
原 9276 行中,event 154866 有一条内容完全相同的重复行,单独保留其原始行号;另外 132 个口头展示与同 forum 的海报合并为一条投稿候选,完整保留两种展示及时间、场次、原始带符号 `sourceid`。负数 `sourceid` 不能被解释为拒稿或另一次投稿,也不转换为全局论文标识。
|
||||||
|
|
||||||
|
其余 139 行单独留档:87 个期刊展示、36 个 Bridging the Future 展示、16 个未提供明确投稿来源的条目。这些不因缺少链接而被推为拒稿,也不能被冒充为独立主会审稿录用。后续须核实原期刊发表、认证、复现或其他展示关系。
|
||||||
|
|
||||||
|
9004 条是不同 forum 的来源记录,不是已经新增 9004 篇独立科学论文。不同 forum 即使题名和作者相同也保留各自投稿;跨会及版本关系另行核验。题名、完整作者顺序和机构取自原始 JSON,口头与海报不是研究类型;展示标签独立使用 `neurips_program:presentation_format` namespace。节目中的未来展示时间不冒充决定时间。
|
||||||
|
|
||||||
|
## 运行与恢复
|
||||||
|
|
||||||
|
在线模式只请求一个固定元数据 URL,并复用受限大小的响应缓存:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect neurips-program \
|
||||||
|
--cache /data/paper-index/cache/neurips2026-program \
|
||||||
|
--out /data/paper-index/collected/neurips2026-program-YYYYMMDD
|
||||||
|
```
|
||||||
|
|
||||||
|
冻结模式只读 `receipts.json` 对应的原始文件:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect neurips-program \
|
||||||
|
--snapshots /data/paper-index/collected/neurips2026-program-probe-v1 \
|
||||||
|
--out /data/paper-index/collected/neurips2026-program-normalized-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
快照回执包含精确 `url`、相对 `file`、`bytes`、`sha256`、`status=200` 和带时区的 `finished_at`。拒绝越界路径/符号链接、重复回执、失败响应、大小或哈希不符。在线模式沿用项目 HTTP 缓存和限速,验证回执时间后保存原始 JSON。输出目录有互斥锁,完成报告最后发布;已有完成报告时要求新建输出目录,避免覆盖后误重放。
|
||||||
|
|
||||||
|
输出三个 track 的 JSONL/manifest、原始 JSON 证据和 `neurips-program-2026.json`。manifest 始终为 `partial`;原始节目 JSON 没有摘要,9004 条 `abstract=null` 和缺失原因如实保存,不能用别篇摘要填充。命令终端只输出数量与路径,完整待处理展示与原始行号在报告中。
|
||||||
|
|
||||||
|
恢复时先校验文件和回执,使用同一完整快照生成新输出;若已有完成报告或导入回执,先核实实际来源绑定和 checkpoint,不重放已消费队列。生成文件后还需 PostgreSQL 身份预检、关联审核、增量队列、原生导入和独立验收;源文件成功、CI 通过均不能替代生产导入或最终全库去重。
|
||||||
|
|
||||||
|
## 目录差异与公开拒稿限制
|
||||||
|
|
||||||
|
同期[官网 Downloads 页面](https://neurips.cc/Downloads/2026)的活动总数不是本 JSON 的论文总数。逐个 poster ID/题名对照后,网页另有 event 155165(Lemon)未出现在本 JSON 中,须单独取得并审核元数据,不能补造其 track 或 forum。16 个未提供来源的条目在 JSON 中使用通用 session URL,而网页有各自 poster URL;原值保留,不能因 URL 映射差异制造新论文。
|
||||||
|
|
||||||
|
[2026 Main Track Handbook](https://neurips.cc/Conferences/2026/MainTrackHandbook)明确允许 rejected 作者选择公开。当前无账号公开 notes API 请求返回 `ChallengeRequiredError`、HTTP 403,失败原文和回执保留,不绕过验证;因此不能把本 accepted 节目文件当作拒稿数据源或据此推断拒稿是否存在。官网下载表单的 JSON 导出也曾超时,未取得导出内容,已改用独立公开静态节目文件并保留失败证据,不重试超时导出。
|
||||||
|
|
||||||
|
当前主动补漏范围从 2020 年起。期刊/展示关系、网页独有条目、摘要和公开负面决定仍有待办;冻结资料和代码适配不代表完整会议覆盖。
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
# RecSys 2026 公开录用贡献采集
|
||||||
|
|
||||||
|
这是长期维护项目中的离线采集器,按需处理冻结的官方元数据,不连接数据库、不访问论文链接或下载 PDF。输出须经过作品、DOI 和投稿历史审查才能导入。
|
||||||
|
|
||||||
|
## 使用
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect recsys2026-program \
|
||||||
|
--snapshots /data/paper-index/collected \
|
||||||
|
--out /data/paper-index/collected/recsys2026-normalized-<new-run>
|
||||||
|
```
|
||||||
|
|
||||||
|
输入父目录应包含 `remaining2026-listings-research-v2/manifest.json` 与 `recsys-contributions.html`。采集器检查固定清单 SHA256、官方 URL、HTTP 200、采集时区、原始字节哈希、九个 track 的菜单及完整条目数。现有输出目录不可覆盖;失败保留 `failed.json`,成功才发布 `collection-report.json`。再次运行前先检查原任务是否结束。
|
||||||
|
|
||||||
|
## 数据范围与重复项
|
||||||
|
|
||||||
|
来源是 [RecSys 官方录用贡献页面](https://recsys.acm.org/recsys26/contributions/),冻结时间为 2026-11-17。九个来源 track:
|
||||||
|
|
||||||
|
| track | 观察数 |
|
||||||
|
| --- | ---: |
|
||||||
|
| long | 65 |
|
||||||
|
| short | 29 |
|
||||||
|
| past_present_future | 13 |
|
||||||
|
| reproducibility | 4 |
|
||||||
|
| resource | 7 |
|
||||||
|
| industry | 83 |
|
||||||
|
| research_practice_notes | 34 |
|
||||||
|
| demo | 18 |
|
||||||
|
| doctoral_symposium | 10 |
|
||||||
|
|
||||||
|
**263 项来源观察对应 260 个不同 DOI,并非 263 篇不同论文。** 每条记录用官方页面中的 accordion locator 标识,分别保留来源标题、作者原文、摘要和位置,不假设 locator 是原始投稿 ID。
|
||||||
|
|
||||||
|
以下 DOI 各出现两次,六个观察均显式标记待审查:
|
||||||
|
|
||||||
|
- `10.1145/3773078.3831839`:Dart 两条来源记录的标题、作者和摘要都有差异;不得静默选一个覆盖另一个。
|
||||||
|
- `10.1145/3773078.3831819`:标题与作者相同、摘要版本不同。
|
||||||
|
- `10.1145/3773078.3831861`:重复展示的文字相同,保留两个来源位置。
|
||||||
|
|
||||||
|
所有观察都有公开摘要。正文只转换为 HTML 展示文字并规范空白,原始页面完整保留;遇到未审查的数学或图片结构会停止,不把上标扁平化成数字。作者按来源的逗号和最后一个 and 分隔,原始名单同时保留;这不是已验证的作者人物身份或别名。
|
||||||
|
|
||||||
|
`accepted` 仅指官方页面明示录用,不是原始签名评审决定。DOI 来自官方超链接,尚未独立核查实时出版商版本;归入 `publication_doi`,不复制其他会议的投稿 ID。track 明确来自页面区块;`recsys_program:contribution_type` 仅保存页面原始类型标签(Research、PastPresentFuture、Doctoral Symposium),各 track 的 RES/IND/RPN 等缩写同时保留在来源字段,不统一推断研究方法类型。
|
||||||
|
|
||||||
|
本页只含录用贡献,不能据此推断未出现的稿件被拒。公开拒稿导出仍为 unknown,征稿页的 desk rejection 政策不等于具体拒稿记录。不得将本批采集成功当成会议全部来源、负面决定或全库去重完成。
|
||||||
|
|
||||||
|
## 输出与恢复
|
||||||
|
|
||||||
|
输出九份 JSONL、对应部分覆盖清单、原始 HTML 和清单证据,以及记录观察数和不同 DOI 数的报告;批准导入数为零。
|
||||||
|
|
||||||
|
清单 SHA256:`fea7b710587636db06794ee195cd083634c41e9054a217859d9dd0ccdd1afab9`。
|
||||||
|
|
||||||
|
服务器恢复包 `/data/paper-index/validation/remaining2026-listings-research-v2-recovery.tar.gz`,本地备份 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/remaining2026-listings-research-v2-recovery.tar.gz`,SHA256 为 `f5ef7c2e73d46712146e4d6711ad5ad562e16b8b55c0cd75d1041cd0b81142cb`。安全恢复原始目录、核对哈希后重跑上述命令。
|
||||||
|
|
||||||
|
完整来源与独立 263 项基准已保存在测试夹具,所有测试、构建和安装验证交给指定的 `ces-control:host` runner。生产导入前还需独立核验规范化结果,并检查当前数据库已有 DOI 绑定及历次投稿。最终全库去重审计仍是独立必做阶段。
|
||||||
|
|
||||||
|
## 2026-11-17 PostgreSQL 补录验收
|
||||||
|
|
||||||
|
采集器仍只生成待审查文件;本节记录随后单独完成的生产补录。47 条来源记录已关联到 47 篇既有作品,依据是已绑定的同一发表 DOI、规范化标题、完整作者集合,以及原始 arXiv 来源明确引用的同一 DOI。逐条当前资料和历史包已与只读预检核对;没有新增论文实体、人物别名、arXiv 标识或原始 OpenReview ID。
|
||||||
|
|
||||||
|
| track | 已导入观察 / 来源应有观察 |
|
||||||
|
| --- | ---: |
|
||||||
|
| demo | 1 / 18 |
|
||||||
|
| doctoral_symposium | 2 / 10 |
|
||||||
|
| industry | 18 / 83 |
|
||||||
|
| long | 13 / 65 |
|
||||||
|
| past_present_future | 2 / 13 |
|
||||||
|
| reproducibility | 1 / 4 |
|
||||||
|
| research_practice_notes | 4 / 34 |
|
||||||
|
| resource | 2 / 7 |
|
||||||
|
| short | 4 / 29 |
|
||||||
|
|
||||||
|
九个 track 全部保持 **partial**。总计 47/263 个来源观察已导入,216 个待审查,其中包括三组重复 DOI 的六项观察。来源的 263 项观察对应 260 个不同 DOI,不能把观察数量当成不同科学论文数量。
|
||||||
|
|
||||||
|
本批新增一个来源、九个 edition、47 条来源记录、47 次投稿/47 条录用观察、47 个 `publication_doi` 投稿标识及47条来源类型标签证据关联。源摘要分别保存,47 篇既有作品的规范摘要保持不变;原始程序字段、来源链接、时间和类型标签也逐条核验。Research/PastPresentFuture/Doctoral Symposium 仍属于来源 namespace,不强制映射成统一研究方法类别。
|
||||||
|
|
||||||
|
独立验收确认:5,812 篇受保护旧作品及其旧资料、来源、标签、标识不变,全部旧投稿/决定原文与关联历史不变;foundry-core `/readyz` 为 200,受保护 SQLite 备份大小/只读权限不变。论文实体仍为 3,567,888;不能用 `max(id)` 代替行数。
|
||||||
|
|
||||||
|
- 完整验收:`/data/paper-index/collected/recsys2026-import47-v1/postgresql-postcheck-v1.json`,SHA256 `7c7e2dcd06bdbc839666740a989c1f187fa47290800a9055cf4540be64336cf4`。
|
||||||
|
- 保护基线:`/data/paper-index/collected/recsys2026-import47-baseline-v1/baseline.json`,SHA256 `aa86d51ff18774a505b1c59510ad867c6221c152334dc4983d31c5290cb8ac0f`。
|
||||||
|
- 导入前完整队列/基线恢复包:`/data/paper-index/validation/recsys2026-reviewed47-recovery-v1.tar.gz`,SHA256 `ae054a6447e8c2864fd8e365867cd3308dee974f375fafff5653fff2a51044e1`;本地同名备份位于 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`,已核验全部 47 个原始身份包、九份输入和基线。
|
||||||
|
|
||||||
|
执行代码固定在提交 `8c147f5e6edda900e2eed19372ab031f661b31c5` 的独立验证目录,85 个包文件校验通过;24 个写入关键文件与此前已验收版本一致。授权 `ces-control:host` runner 的 CI1230/job1327通过525项测试及安装后实际RecSys263项/260DOI验证。没有改变服务器默认运行代码目录或共享数据库配置。
|
||||||
|
|
||||||
|
**九份输入均已消费,禁止重放。** 恢复时先检查实际 PostgreSQL、执行回执和终态验收;不能因日志安静、观测超时或重新启动 Goal 就复跑。后续216条须另建审查与增量队列,并重新取当前基线。本批通过不等于整届 RecSys、公开拒稿覆盖或最终全库去重已完成。
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
# RSS 2026 论文集来源
|
||||||
|
|
||||||
|
`collect rss2026-proceedings` 是独立的年度来源适配器,不依赖尚未合入主线的 RSS 2025 PR #43,也不改变其 demo 分类。使用包内 HTML、HTTP 工具和统一 CLI;不连接数据库,不下载 PDF。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect rss2026-proceedings --cache /data/paper-index/cache/rss2026 \
|
||||||
|
--out /data/paper-index/collected/rss2026-native-v1
|
||||||
|
# 固定证据复现:包含 receipts.json 和对应 HTML 文件
|
||||||
|
paper-atlas collect rss2026-proceedings --snapshots /data/paper-index/collected/rss2026-primary-v1 \
|
||||||
|
--out /data/paper-index/collected/rss2026-native-replay-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
核对的来源为 [第22卷论文集](https://www.roboticsproceedings.org/rss22/)、[录用表](https://roboticsconference.org/2026/program/papers/)、[CFP](https://roboticsconference.org/2026/information/cfp/) 和 [Sponsor Demos](https://roboticsconference.org/2026/program/demos/)。当前审查目录为210篇。CFP明确2026没有单独 demo paper 类别,Science/Systems论文放在 main;赞助商展示不作为论文,也不推断逐篇是否现场演示。投稿录用后才公开、评审不公开;此来源不能提供拒稿或评审全集。
|
||||||
|
|
||||||
|
每篇保留出版社原始标题、完整作者顺序、摘要、日期、BibTeX、原始 DOI,以及独立节目标题、完整作者和缩略显示、session、显示链接。节目显示编号不是原始 OpenReview 投稿编号,不写入 submission_identifiers。六个缩略作者预览按完整隐藏副本恢复核验,不假设固定字符长度;六个姓名差异和一处 RLinf/RLux 标题差异按已审核原文指纹识别并保留,不推断人名别名或官方勘误。未知差异会停止。
|
||||||
|
|
||||||
|
快照收据必须逐文件匹配固定URL、HTTP200、字节数、哈希和带时区的真实采集时间;拒绝路径穿越、重复收据和符号链接。缓存模式只请求已审核的固定公开HTML URL,访问限制由HTTP工具正常报错,不绕过。每份原始HTML按哈希保存到输出 evidence/。
|
||||||
|
|
||||||
|
输出目录必须不存在;失败目录留存供检查,换新目录重试。`collection-report.json` 最后原子写入,缺少此文件不能视为完成。清单始终 partial;已采集不代表主库完整、身份已核验或科学作品已去重。DOI也是待核对的候选标识,入库前仍需实时 PostgreSQL 身份审计、版本/作者关联复核和保护基线;入库后独立验收,所有导入结束后再做全库去重。
|
||||||
|
|
||||||
|
2026原始来源审计已冻结214份HTML且210篇均有摘要。回归 fixture 同时包含原始来源和独立生成的逐篇预期值,测试比较210篇完整标题/作者/摘要/DOI/日期;构建和测试仅在已授权 ces-control:host 远程 runner 进行。
|
||||||
@@ -0,0 +1,97 @@
|
|||||||
|
# SDM 2026:生产规范化与身份核查记录
|
||||||
|
|
||||||
|
本页记录 2026-11-17 的来源验收、早期只读核查和随后的生产导入。**当前 53 条注册记录中,25 条已入库并独立验收,28 条仍待处理;没有新增论文实体。** 下文早期报告保持原始状态,不将它们改写成事后批准;最终结果见“25 条生产导入验收”。
|
||||||
|
|
||||||
|
## 已核实的生产输出
|
||||||
|
|
||||||
|
生产规范化使用通过远程 CI 1216 的源码版本 `f2753bc741d0fff06134cf64019dac38002fae37`,对应 494 项测试、wheel 构建及安装后 CLI 检查。该版本包含已合并的 SDM 适配器;这里没有替换服务器默认代码或运行环境。本页是文档更新,不能把此前 CI 结果当作本次提交的 CI 结果。
|
||||||
|
|
||||||
|
`/data/paper-index/collected/sdm2026-normalized-v1` 保留:
|
||||||
|
|
||||||
|
- 53 条 Crossref 论文章节记录,全部没有注册摘要;排除论文集和前言。
|
||||||
|
- 53 条独立日程观察,其中 52 条有公开摘要;20 处作者序列差异仍保留。
|
||||||
|
- 取消展示的条目与已注册发表记录分别保留,不生成撤回决定。
|
||||||
|
|
||||||
|
规范化服务 `paper-atlas-sdm2026-normalize-v1.service` 的 invocation 为 `3021248cae4044ac88eb80fb8b964a26`,终态成功。独立审计逐条核对冻结注册字段、完整作者顺序、原始日程和摘要证据,未以适配器自身统计替代验收。
|
||||||
|
|
||||||
|
| 证据文件 | SHA-256 |
|
||||||
|
| --- | --- |
|
||||||
|
| validation/sdm2026-normalized-v1-audit.json | 66d4e698ff8828731ccb657e273fb2f0d19cbce8fd91ade2c81ed68124c6d3a3 |
|
||||||
|
| collected/sdm2026-normalized-v1/sdm-crossref-registry-2026-unclassified.jsonl | 7edd78170ced9b79e4405eb16d63f8a9a38d38dfc906e892580f0ac4be165b2c |
|
||||||
|
| collected/sdm2026-normalized-v1/program-observations.jsonl | 421a7187369817fa90aa3484575e8a8a2f3e008b9d152bed6beca678b1e12204 |
|
||||||
|
|
||||||
|
表中服务器相对路径均位于 `/data/paper-index/`。本机工作记录根目录为 `/Users/zzyang/Documents/Codex/2026-11-06/paper-index/`,保存同名输出及审计备份;数据证据不作为项目源码提交。
|
||||||
|
|
||||||
|
## 当前数据库只读预检
|
||||||
|
|
||||||
|
`paper-atlas-sdm2026-preflight-v1.service`,invocation `9be7227930354633abf8f41cdbd0dd73`,终态成功。数据库为 `foundry_data.research`,快照时间 `2026-11-17 08:34:34.270409+00`。报告核实事务为 READ ONLY,`database_written=false`、`approved_for_import=0`。
|
||||||
|
|
||||||
|
| 候选类别 | 记录数 | 含义 |
|
||||||
|
| --- | ---: | --- |
|
||||||
|
| 精确标题与作者集合一致 | 26 | 需要进一步核对作品、版本和已有标识 |
|
||||||
|
| 精确标题但仅部分作者重合 | 2 | 作者版本冲突尚未解决 |
|
||||||
|
| 没有精确匹配 | 25 | 不是新作品证明,不能直接创建论文 |
|
||||||
|
|
||||||
|
报告位于 `collected/sdm2026-preflight-v1/postgresql-preflight.json`,SHA-256 为 `eb0cc45fa5bf01159133af0536ba6621a2403914ccd9591a5d1a6a230bc3791e`。28 个候选论文压缩证据包的哈希均核对通过;来源记录、投稿、决定及其来源关系均未触发报告的截断标志。这里的完整性仅限这些候选包,不代表全库审计完成。
|
||||||
|
|
||||||
|
随后逐项阅读来源摘要、既有摘要、注册作者和已有标识,为 25 个候选记录了支持同一作品的具体方法、数据集、实验结果或摘要一致性证据,仍需最新数据库保护基线和导入前检查。相似度只用于安排审阅顺序,不作为自动批准阈值。以下三条保留作者版本问题:
|
||||||
|
|
||||||
|
| 章节 DOI 尾号 | 既有 paper_id | 尚存问题 |
|
||||||
|
| --- | ---: | --- |
|
||||||
|
| .10 | 2656770 | 注册姓名 Tiejun Chen 与既有及日程姓名 Tiejin Chen 不同;不生成别名 |
|
||||||
|
| .15 | 2745754 | 注册 Sarang Patil 与既有 Sarang Rajendra Patil 不同;日程另有作者顺序与姓名缩写变化 |
|
||||||
|
| .49 | 2506693 | 注册和既有完整作者集合相同,但日程列出不同名单;ScaffAug 内容一致性仍不能消除来源矛盾 |
|
||||||
|
|
||||||
|
这里的尾号均属于 `10.1137/1.9781611979329`。逐条内容审阅文件为本机工作记录根目录的 `sdm2026-candidate-evidence-review-v1.json`,SHA-256 为 `6ff791f87eafeb50fa853766c632e91e3f0377ab3f619365117d557f6d56100d`。其中 25 条为 `same_work_supported_pending_fresh_import_guards`,3 条为 `hold_author_version_conflict`;所有条目的 `approved_for_import` 仍为 false。不能据此直接重放来源、绑定 DOI 或拼接日程摘要。
|
||||||
|
|
||||||
|
## 需要保护的较早拒稿历史
|
||||||
|
|
||||||
|
| SDM 候选作品 | paper_id | 已有较早决定 |
|
||||||
|
| --- | ---: | --- |
|
||||||
|
| SeBA | 2672776 | ICLR 2026 rejected |
|
||||||
|
| Towards Reliability of Parameter-Free Optimization | 3282961 | ICLR 2025 rejected |
|
||||||
|
| Scaffold-Aware Generative Augmentation and Reranking for Enhanced Virtual Screening | 2506693 | ICLR 2026 desk_rejected |
|
||||||
|
|
||||||
|
前两条在后续生产验收中各自新增独立 SDM 发表事件;第三条仍保留作者版本问题,未导入。旧投稿、旧决定及各自来源均完整保留。两个来源对同一拒稿的观察不能直接统计成两次独立投稿或拒稿。不能覆盖旧状态,也不能因这批录用候选就宣称完整投稿历史已补齐。
|
||||||
|
|
||||||
|
## 未精确匹配的补充检索
|
||||||
|
|
||||||
|
对另外 25 条运行了有时限、数量上限和只读快照的补充检索,使用现有索引,不新增数据库索引或改变全局配置。
|
||||||
|
|
||||||
|
- 全文检索 `paper-atlas-sdm2026-title-variants-v1.service`,invocation `29c5d2a288804ce7a9c3e2f7db4dca6f`,终态成功。每项使用完整标题及明确记录的锚点词,8 条至少有一次查询达到候选数量上限,不能据此排除重复。
|
||||||
|
- 标题前缀范围检索 `paper-atlas-sdm2026-title-prefix-v2.service`,invocation `3087f1c0f252427485a661a63c50b27f`,终态成功。25 条均使用既有标题索引,没有查询错误或数量截断。18 字符前缀只是补充方法,不是全库模糊匹配验收,也不会消除前一轮查询的限制。
|
||||||
|
|
||||||
|
全文候选中的既有论文 `2025464` 与 SDM .9 共享作者,但前者讨论 CLOUD、离散/混合/连续变量,后者讨论 SLICE、类别数据与 HCR 模型。标题前缀候选 `938886` 与 SDM .28 只共享一名作者,既有版本讨论随机游走相似度、有效电阻和社交网络推断。不能凭研究主题或作者重合将这些论文合并。25 条仍保留为待进一步查证,没有获准创建新作品。
|
||||||
|
|
||||||
|
| 服务器恢复文件 | SHA-256 |
|
||||||
|
| --- | --- |
|
||||||
|
| validation/sdm2026-title-variants-v1-recovery.tar.gz | 1f387f2ff4d5684aa72abd419bddeb7e972624f6a56f6e9a943168aa944b9137 |
|
||||||
|
| validation/sdm2026-title-prefix-v2-recovery.tar.gz | 4941343bd6152c5046778f7c82588e3c6ad41c9bb938e0107f8b2b426ac10772 |
|
||||||
|
|
||||||
|
两个原报告的 SHA-256 分别为 `b6ab2e9228b8e8e9548b27df70834faf25bfda551844979f8410a74b6e6aadd0` 和 `c1f902b4e4f33997eb182b10dc78332e182ceb6051f1a7c14b21465c8878263d`。前者保留 1 个进一步审阅的完整候选包,后者保留全部 113 个前缀命中的候选包;这些数量不是论文批准数。
|
||||||
|
|
||||||
|
## 25 条生产导入验收
|
||||||
|
|
||||||
|
`paper-atlas-sdm2026-import25-v1.service`,invocation `003b490a2f2c449da078a2f8625b31b8`,终态成功。执行前核验 81 个源码文件,并确认 24 个核心写入文件与此前已验收的 canonical writer 相同;复用原生 `paper-atlas import records`,未修改默认运行目录。
|
||||||
|
|
||||||
|
- 新增 25 条来源、25 次会议事件、25 条发表决定观察及 25 个 `publication_doi` 投稿标识,新增论文实体为 0。
|
||||||
|
- 24 篇使用已核实的 arXiv 标识补注;AdamG 没有 arXiv 标识,采用单独审阅并实时守护的唯一精确标题、四名完整作者和具体方法证据。旧 OpenReview ID 未复制为 SDM 投稿标识。
|
||||||
|
- 28 条待处理记录全部排除;25 个注册来源摘要仍为空,52 份日程摘要保持独立来源版本。覆盖状态为 `partial`,应有 53 条、当前导入 25 条,track 保留 `proceedings_unclassified`。
|
||||||
|
- 全局旧原始投稿与决定历史、5,653 篇受保护论文的原资料/来源/标签/既有标识均通过导入后比较。SeBA、AdamG 的两次较早 ICLR 投稿及四条 rejected 来源观察保留,没有将四条观察说成四次投稿。
|
||||||
|
- 约束有效、实际全文查询与 GIN 计划通过;foundry-core `/readyz` 返回 200,受保护 SQLite 快照保持只读。
|
||||||
|
|
||||||
|
| 生产证据 | SHA-256 |
|
||||||
|
| --- | --- |
|
||||||
|
| collected/sdm2026-import25-v1/postgresql-postcheck-v1.json | 0c861722f0d455462a2686a42d634e0b963d075cf58f90fa9f48e1ae1fc82c27 |
|
||||||
|
| collected/sdm2026-import25-baseline-v1/baseline.json | 42ffbaf6f843a91ebf64f0bfefa4af36f6ac72f044b0e8587e65c5e6648c0931 |
|
||||||
|
| collected/sdm2026-prepared25-v1/independent-preparation-check.json | 8ea8eb59f4b1601a484d1bc0535b289cb950b83747ed5f82b0eac55e527c4a7c |
|
||||||
|
| validation/sdm2026-reviewed25-recovery-v1.tar.gz | 13447944ec34e0e2a6eee07a1d9820bfb8066756aa11016673e416c0d6f8b2b2 |
|
||||||
|
|
||||||
|
恢复包包含 263 个文件;本机已独立核对队列、70 份内容寻址证据和 28 个早期候选证据包。**25 条队列已消费,禁止重放。** 28 条剩余记录需要新的导入后快照;先前候选包不能当作当前数据库状态。后续批次须沿用最新 5,653 篇保护基线,并加上各自新目标。
|
||||||
|
|
||||||
|
## 下次恢复
|
||||||
|
|
||||||
|
1. 先核对实际服务终态、来源及报告哈希,再读取上述证据;终态成功的研究任务无需重启。
|
||||||
|
2. 保留已验收的 25 条,不重放;继续解决 3 条作者版本问题和 25 条未精确匹配记录;不把缺少候选当作新作品证明,不把源端 403 当作零条公开拒稿。
|
||||||
|
3. 对后续新获内容支持的候选另做新快照检查:作品与 DOI 绑定、来源是否已导入、已有会议事件及全部历史保护。建立新的明确入库队列和可恢复备份后再执行写入,不能直接使用本次候选报告当导入计划。
|
||||||
|
4. 导入后逐项核验新事件、旧历史、来源关系和精确数量变化;注册摘要缺失仍如实保留。完整来源范围结束后另做全库去重、标识冲突、近似标题/作者、覆盖、摘要、约束和查询验收。
|
||||||
@@ -0,0 +1,46 @@
|
|||||||
|
# SDM 2026:论文集与日程版本
|
||||||
|
|
||||||
|
这是长期维护、按需运行的离线来源适配器。它复用固定的公开元数据快照,不请求网络、不下载论文 PDF、不连接数据库,也不会把候选版本关联当成已通过的作品去重。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect sdm2026-proceedings \
|
||||||
|
--snapshots /data/paper-index/collected \
|
||||||
|
--out /data/paper-index/collected/sdm2026-normalized-pass01
|
||||||
|
```
|
||||||
|
|
||||||
|
`--snapshots` 下需保留四个目录:`sdm2026-official-research-v1`、`sdm2026-crossref-research-v1`、`sdm2026-program-research-v1`、`sdm2026-public-abstract-research-v1`。适配器固定核验各自 manifest 和全部成功响应的字节数、哈希、URL 与状态。测试 fixture 保留原始响应字节及独立盘点结果。使用全新输出目录恢复,保留失败目录中的 `failed.json`;成功报告最后以原子替换发布。来源更新时另存版本并重新审阅,不能修改旧 manifest 后冒充旧快照。
|
||||||
|
|
||||||
|
## 两种输出
|
||||||
|
|
||||||
|
[论文集](https://epubs.siam.org/doi/book/10.1137/1.9781611979329)的 ISBN 为 `9781611979329`。[Crossref 精确 ISBN 查询](https://api.crossref.org/works?filter=isbn%3A9781611979329&rows=1000)的冻结响应有 55 项:53 个论文章节、1 本论文集、1 份前言。适配器只将 53 个章节放入 `report.files` 的待身份审阅 JSONL,排除论文集和前言;核对完整章节 DOI 集合、父 DOI、容器标题、ISBN、作者顺序证据及出版年份。
|
||||||
|
|
||||||
|
正式记录保留 Crossref 作者版本与原始字段,使用 `proceedings_unclassified`,不按主题或分会场推断 track。注册发表记录表达已收入论文集的事实,不冒充原始签名录用决定。印刷与在线出版日期分别保留,不能用它们替代会议举办日期或决定日期。两个标题含 `<sup>*</sup>`:显示标题保留脚注星号并标记待审阅,原始标题完整保留;去掉这一特定脚注的文字仅用于列出待审阅候选,不用于自动改名或绑定作品。
|
||||||
|
|
||||||
|
**53 个 Crossref 章节都没有摘要字段。**主记录的 `abstract` 因此为空,报告如实记为 53 个摘要缺失,不能因另一来源有同标题摘要而直接填入。
|
||||||
|
|
||||||
|
[官方在线日程](https://meetings.siam.org/program.cfm?CONFCODE=sdm26)有 9 个研究论文分会场、53 项日程条目和 52 个公开摘要页。`program-observations.jsonl` 单独保留日程版本、公开页标题、来源报告的作者姓名、摘要、会场引用、回执与哈希;它不是入库 manifest,也不在 `report.files` 中。页面中作者行的单位、国家及联系邮箱不规范化成人物字段,原始页面留在证据文件中。
|
||||||
|
|
||||||
|
52 个公开页中,32 个作者序列与 Crossref 完全一致,20 个不同,包括姓名拼写、顺序和作者缺失。例如 *Scaffold-Aware Generative Augmentation and Reranking for Enhanced Virtual Screening* 的日程页列出 Xin Wang、Qinwen Ge,Crossref 列出 Xin Wang、Yu Wang、Yunchao Liu、Jens Meiler、Tyler Derr。它们必须保留为独立来源版本,不能仅凭标题相同拼接摘要与作者,或生成姓名别名。候选 DOI 字段明确表示待审阅关系,不是逐篇身份批准;作者列表相同也不能代替全库作品身份核查。
|
||||||
|
|
||||||
|
*A Trajectory View of Statistical Policy Comparison* 的日程展示标为取消,且没有公开摘要链接;它仍出现在冻结的 Crossref 论文章节集合中。适配器保留这两项事实,**不把取消展示转换成论文撤回、拒稿或删除发表记录**。邀请报告、博士论坛和其他非研究论文活动也不计入这里的 53 个章节。
|
||||||
|
|
||||||
|
## 公开拒稿与访问限制
|
||||||
|
|
||||||
|
[官方投稿说明](https://www.siam.org/conferences-events/siam-conferences/sdm26/submissions/)链接 OpenReview group `SIAM.org/2026/Conference_on_Data_Mining`。匿名读取 group 配置成功;按实际 `rejected_venue_id` 查询 notes 返回 HTTP 403,随后停止,没有绕过或继续查询其他负面状态。因此公开拒稿、desk-rejected 和 withdrawn 决定覆盖仍未核实,不能报告为零条或从录用名单缺席推断拒稿。
|
||||||
|
|
||||||
|
来源冻结时,服务器访问 SIAM 主站的两份说明页及出版社目录也返回 403;回执如实保留。可读取的是公开在线日程及独立公开的 Crossref 注册元数据。Crossref 记录不是受限出版社页面的重建,也不被称作已经核验的出版社摘要版本。
|
||||||
|
|
||||||
|
## 来源恢复与后续核查
|
||||||
|
|
||||||
|
| 冻结目录 | manifest SHA-256 |
|
||||||
|
| --- | --- |
|
||||||
|
| sdm2026-official-research-v1 | a39a6ee6a12b69051ba2dad7c9cbc5b0fe215af85100c7aa2f5166d44c22fe61 |
|
||||||
|
| sdm2026-crossref-research-v1 | 51d337a153a1046e4a00ebc031707faf72e733577e4fc243a31672ee93277da5 |
|
||||||
|
| sdm2026-program-research-v1 | 689a2aaaa15be532957c4c88e03e1e2e87909334b6f529be16905d8b9cc1fb77 |
|
||||||
|
| sdm2026-public-abstract-research-v1 | 8e1c29611e145cbfa54ec4afaa2efe152557b4e04e339901af5cd33abefc29a5 |
|
||||||
|
|
||||||
|
服务器恢复包:`/data/paper-index/validation/sdm2026-sources-recovery-v1.tar.gz`,SHA-256 为 `d6d14c0e4bb2c26e16a4888e3b150e44f982d689351638b1b9a28644736c4387`;本机工作记录目录有同名备份。包内保留原始来源、独立盘点及逐页版本比较记录,含义是可复现来源证据,不是已完成导入或身份批准。
|
||||||
|
|
||||||
|
下次继续时,先核对来源版本和访问状态,再做当前数据库的 DOI 绑定、标题/作者近似候选与完整投稿历史核查,逐条解决作者及摘要版本问题。批准后另建入库队列、保护基线和导入后验收;所有来源结束后仍需全库去重、覆盖、摘要、约束和实际查询验收。测试、构建和安装验证只在已授权 `ces-control:host` 远程 runner 执行。
|
||||||
|
|
||||||
|
生产规范化验收、当前只读候选数量、已有拒稿历史保护及下次恢复入口见 [SDM2026 身份核查记录](sdm2026-identity-review.md)。该记录不代表已批准或完成导入。
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
# 来源身份只读预检
|
||||||
|
|
||||||
|
`audit source-identities` 在 PostgreSQL 的同一 `REPEATABLE READ READ ONLY` 快照中,检查规范化 JSONL 的来源绑定、稳定标识及精确标题候选,并导出候选论文的原始来源和投稿决定历史。用于入库前评审,不执行导入、合并或标识修改。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas audit source-identities \
|
||||||
|
--pg-service foundry-research --pg-schema research \
|
||||||
|
--files /data/paper-index/collected/example/main.jsonl \
|
||||||
|
/data/paper-index/collected/example/position.jsonl \
|
||||||
|
--out /data/paper-index/collected/example-identity-preflight-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
每个 JSONL 必须有同名 `.manifest.json`,包含 `source`、`venue`、`year`、`track`、`count` 和 `file_sha256`。逐行校验届次、显式决定状态、标识类型和跨文件来源主键唯一性;文件哈希和数量必须相符。只读取调用者指定的文件,不跟随 manifest 中的绝对 `file` 路径;原始行哈希包含换行符。原始 JSONL 和 manifest 应与报告一起保留。
|
||||||
|
|
||||||
|
输出目录必须不存在,防止复用旧报告或覆盖证据。失败会保留 `failed.json` 及已完成分片,不能把部分输出当成功;排查后使用新的输出目录重新核查。成功报告 `postgresql-preflight.json` 最后原子写入,记录输入哈希、数据库快照和每条候选。实体详情存为 `entities/<分片>/<paper_id>.json.gz`,报告中列出压缩文件的 SHA-256;包含完整 paper、标识和限量原始来源、投稿、决定及出处。每篇最多保留 100 条来源、200 次投稿,每次投稿最多 200 条决定,每组决定或投稿出处最多 100 条,所有截断均显式标记。精确标题最多取 21 个候选;达到上限即标记,不能据此推算总数。这里没有计算全表精确数量。
|
||||||
|
|
||||||
|
候选分类用于安排审核:既有来源记录需要核对并避免重放,多个绑定冲突必须暂缓;单个 DOI、arXiv 或 OpenReview 绑定也可能错误,必须核对原文、标题、作者和历史。作者比较保留完整规范化姓名的重复次数,但姓名相同不能证明是同一个人。未找到精确标题或标识不能证明是新作品,仍需近似标题、版本和跨会历史核查。
|
||||||
|
|
||||||
|
所有条目固定为 `not_approved_for_import`,批准数量为 0。不要直接把此报告当成 curation 计划或导入队列。后续获批导入前应重新检查实时来源绑定;来源摘要缺失不得覆盖已有摘要,旧拒稿与新录用应作为不同投稿经历保留。本命令不能代替所有导入结束后的全库去重、覆盖和约束验收。
|
||||||
|
|
||||||
|
构建与测试仅在已确认的 `ces-control:host` 远程 runner 执行。集成测试使用已有 PostgreSQL 服务做真实只读候选查询,不往生产库添加测试记录。
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
# 会后选集的元数据采集
|
||||||
|
|
||||||
|
`collect springer-selected` 读取已冻结、带 HTTP 回执的公开 HTML,不访问 PDF、不写业务库。目前仅支持核验过的 *Selecting Models from Data: Artificial Intelligence and Statistics IV*,书籍 DOI 为 `10.1007/978-1-4612-2660-4`。增加其他卷前必须另行核对会议关系、目录、年份和字段格式。
|
||||||
|
|
||||||
|
[会议官方旧站](https://www.gatsby.ucl.ac.uk/aistats/workshops.html)区分现场分发的 Preliminary Papers 与之后修订、再次评审的选集。[出版社卷说明](https://link.springer.com/book/10.1007/978-1-4612-2660-4)将本书关联到 1993 年 1 月的第四届 workshop,书籍出版年份为 1994。三页目录共有 49 个章节 DOI;分区 Front Matter 不作为章节论文导入。
|
||||||
|
|
||||||
|
规范记录保留:
|
||||||
|
|
||||||
|
- 会议年份 1993、独立 track `selected_volume` 与事件标识 `aistats-1993-selected-volume-1994`;出版年份单独记为 1994。
|
||||||
|
- 章节本身的 DOI、作者原始引用写法及顺序、姓名首字母、标题、摘要、页码和逐页原始证据。不扩写作者首字母。
|
||||||
|
- 明确提供本章 `publication_doi` 投稿出版标识,不把它转移到旧的原始会议评审记录,也不因短标题相同而强制合并实体。
|
||||||
|
- `springer_selected_volume:publication_format` 标签与独立来源 namespace。
|
||||||
|
- `accepted` 仅表示章节已列入这本出版选集,原始 1993 投稿评审决定、决定日期和全部录用清单均未知。后续原会议 main、拒稿或撤稿证据可以独立保留。
|
||||||
|
|
||||||
|
不能据此宣称会议全部录用、全部投稿或公开拒稿覆盖完整,也不能把出版社机器添加的关键词视为作者标签。未命中选集不等于被拒。
|
||||||
|
|
||||||
|
## 按需运行与恢复
|
||||||
|
|
||||||
|
输入分别是卷目录/旧站说明和章节页的快照目录。每个目录的 `receipts.json` 包含 `responses` 列表,每条保留 `url`、`final_url`、`file`、`http_status`、`headers`、`bytes`、`sha256`、`at`。需要三页目录、官方旧站说明及全部 49 个章节页面。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
paper-atlas collect springer-selected \
|
||||||
|
--toc-snapshots /data/paper-index/collected/aistats1993-book-evidence-v1 \
|
||||||
|
--chapter-snapshots /data/paper-index/collected/aistats1993-chapter-evidence-v1 \
|
||||||
|
--out /data/paper-index/collected/aistats1993-source-v1
|
||||||
|
```
|
||||||
|
|
||||||
|
采集器验证回执和内容哈希、文件路径、HTTP 类型、目录/章节 DOI 和标题、作者格式、出版年份及页码。Springer 对公开元数据返回的同路径 `cookies_not_supported` 诊断参数会原样保留;不接受其他域、路径、分页变化或未知查询参数。官方旧站说明固定哈希为 `6db205b9dbc82b4a096556efd2aa67e065cad8e0bd6351176b1962ffd1fca179`,变化时须重新审核。
|
||||||
|
|
||||||
|
全部文件先核验,输出使用排他锁和暂存目录,完成报告最后发布。已有输出拒绝覆盖;发现不完整或已消费文件,应先查运行状态与报告,另建明确版本目录,不能重放生产导入。
|
||||||
|
|
||||||
|
输出 JSONL 配套 manifest 始终为 `partial`。收集完成后还必须做 PostgreSQL 稳定标识/标题/作者身份预检、排除已导入记录、准备限定增量文件、保存旧数据保护基线、原生导入和独立验收。49 条来源不代表 49 个新增论文实体,全部来源处理后仍须全库去重审计。
|
||||||
|
|
||||||
|
## 固定测试资料
|
||||||
|
|
||||||
|
`tests/fixtures/springer-aistats1993-fixture.json.gz` 保存 4 份范围/目录 HTML、49 份章节 HTML、原始请求回执及独立原始字段审计的预期元数据,SHA256 为 `94028fd37080f9d8027e5790e379cc151ee6eab17ed0d1dcc0f59e6e75f9d4d0`。摘要按 HTML 段落边界分隔;未修改原始 HTML 或回执。它是压缩的公开元数据测试资料,不包含论文 PDF。测试覆盖全部真实章节字段、缺失和漂移输入、来源安全边界、幂等导入及旧拒稿历史保留。构建和测试只在已授权远程 runner 执行。
|
||||||
@@ -0,0 +1,327 @@
|
|||||||
|
"""Read-only, evidence-preserving identity candidates for normalized source files.
|
||||||
|
|
||||||
|
A candidate is never permission to import or merge. Absence of an exact match is
|
||||||
|
never proof of a new work. Run again against a fresh snapshot before curation.
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from paper_atlas.db.store import author_keys, normalized
|
||||||
|
from paper_atlas.db.submissions import timestamp
|
||||||
|
from paper_atlas.postgres.client import ReadConnection
|
||||||
|
|
||||||
|
STATUSES = {'accepted', 'rejected', 'desk_rejected', 'withdrawn', 'under_review', 'unknown'}
|
||||||
|
|
||||||
|
|
||||||
|
def digest(data):
|
||||||
|
return hashlib.sha256(data).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def authors_multiset(authors):
|
||||||
|
# author_keys intentionally removes duplicates; identity review must retain them.
|
||||||
|
return Counter(key for author in authors for key in author_keys([author]))
|
||||||
|
|
||||||
|
|
||||||
|
def load_inputs(files):
|
||||||
|
records, inputs, seen = [], [], set()
|
||||||
|
for filename in files:
|
||||||
|
path = Path(filename).resolve()
|
||||||
|
manifest_path = path.with_suffix('.manifest.json')
|
||||||
|
raw_manifest = manifest_path.read_bytes()
|
||||||
|
manifest = json.loads(raw_manifest)
|
||||||
|
hasher, count = hashlib.sha256(), 0
|
||||||
|
with path.open('rb') as stream:
|
||||||
|
for line_number, line in enumerate(stream, 1):
|
||||||
|
hasher.update(line)
|
||||||
|
row = json.loads(line)
|
||||||
|
for key in ('source', 'external_id', 'title', 'venue', 'track'):
|
||||||
|
if not isinstance(row.get(key), str) or not row[key].strip():
|
||||||
|
raise ValueError(f'Invalid {key}: {path.name}:{line_number}')
|
||||||
|
if type(row.get('year')) is not int or row.get('status') not in STATUSES:
|
||||||
|
raise ValueError('Invalid year or explicit source status')
|
||||||
|
for key in ('source', 'venue', 'year', 'track'):
|
||||||
|
if row[key] != manifest.get(key):
|
||||||
|
raise ValueError(f'Manifest scope mismatch: {key}')
|
||||||
|
authors = row.get('authors')
|
||||||
|
if not isinstance(authors, list) or any(not isinstance(a, (str, list, dict)) for a in authors):
|
||||||
|
raise ValueError('Invalid authors')
|
||||||
|
identifiers = row.get('identifiers', {})
|
||||||
|
if not isinstance(identifiers, dict) or any(
|
||||||
|
not isinstance(k, str) or not k.strip() or not isinstance(v, str) or not v.strip()
|
||||||
|
for k, v in identifiers.items()):
|
||||||
|
raise ValueError('Identifiers must contain singular nonempty strings')
|
||||||
|
key = (row['source'], row['external_id'])
|
||||||
|
if key in seen:
|
||||||
|
raise ValueError('Duplicate source/external_id across inputs')
|
||||||
|
seen.add(key)
|
||||||
|
record = {k: row[k] for k in ('source', 'external_id', 'title', 'authors',
|
||||||
|
'venue', 'year', 'track', 'status')}
|
||||||
|
record.update(identifiers=identifiers, file=str(path), line=line_number,
|
||||||
|
raw_line_sha256=digest(line), source_record_hits=[],
|
||||||
|
identifier_hits=[], title_candidate_ids=[])
|
||||||
|
records.append(record)
|
||||||
|
count += 1
|
||||||
|
if not count or count != manifest.get('count') or hasher.hexdigest() != manifest.get('file_sha256'):
|
||||||
|
raise ValueError('Input count/hash mismatch or empty input')
|
||||||
|
inputs.append({'file': str(path), 'file_sha256': hasher.hexdigest(), 'count': count,
|
||||||
|
'manifest_sha256': digest(raw_manifest), 'manifest': manifest})
|
||||||
|
if not records:
|
||||||
|
raise ValueError('At least one nonempty input is required')
|
||||||
|
return records, inputs
|
||||||
|
|
||||||
|
|
||||||
|
def load_mm_program_inputs(report_path):
|
||||||
|
"""Read every MM observation without turning a program into import records.
|
||||||
|
|
||||||
|
Reconcile the complete output against its frozen raw source before any DB
|
||||||
|
connection. The synthetic unknown status is an audit placeholder only.
|
||||||
|
"""
|
||||||
|
from paper_atlas.collectors import mm2026
|
||||||
|
|
||||||
|
report_path = Path(report_path).absolute()
|
||||||
|
raw_report = mm2026.safe_read(report_path, 256*1024)
|
||||||
|
report = json.loads(raw_report)
|
||||||
|
if (report.get('source') != 'mm_program' or report.get('records') != 2143
|
||||||
|
or report.get('program_observations') != 4326 or len(report.get('files', [])) != 28
|
||||||
|
or report.get('source_body_sha256') != mm2026.BODY_SHA
|
||||||
|
or report.get('source_manifest_sha256') != mm2026.MANIFEST_SHA
|
||||||
|
or report.get('database_written') is not False or report.get('approved_for_import') != 0):
|
||||||
|
raise ValueError('Unreviewed MM observation report')
|
||||||
|
root = report_path.parent
|
||||||
|
raw_source = mm2026.safe_read(root/'evidence/program-data.json', 12*1024**2)
|
||||||
|
raw_receipts = mm2026.safe_read(root/'evidence/manifest.json', 128*1024)
|
||||||
|
if digest(raw_source) != mm2026.BODY_SHA or digest(raw_receipts) != mm2026.MANIFEST_SHA:
|
||||||
|
raise ValueError('MM raw evidence changed')
|
||||||
|
expected_rows, _ = mm2026.parse_program(raw_source)
|
||||||
|
expected = {r['paper_id']:r for r in expected_rows}
|
||||||
|
records, inputs, seen, tracks = [], [], set(), set()
|
||||||
|
for item in report['files']:
|
||||||
|
track = item.get('track_literal')
|
||||||
|
if (track not in mm2026.TRACK_COUNTS or track in tracks
|
||||||
|
or item.get('format') != 'mm_program_observation_v1_not_ingestion_record'
|
||||||
|
or item.get('approved_for_import') != 0):
|
||||||
|
raise ValueError('Duplicate or unreviewed MM track format')
|
||||||
|
tracks.add(track)
|
||||||
|
# Stored absolute paths are provenance, not instructions to read elsewhere.
|
||||||
|
path = root/Path(item['file']).name
|
||||||
|
if path.suffix != '.jsonl':
|
||||||
|
raise ValueError('Invalid MM observation filename')
|
||||||
|
raw = mm2026.safe_read(path, 16*1024**2)
|
||||||
|
lines = raw.splitlines(keepends=True)
|
||||||
|
if digest(raw) != item.get('file_sha256') or len(lines) != item.get('count') or len(lines) != mm2026.TRACK_COUNTS[track]:
|
||||||
|
raise ValueError('MM observation file count/hash mismatch')
|
||||||
|
for number,line in enumerate(lines, 1):
|
||||||
|
row = json.loads(line)
|
||||||
|
pid = row.get('paper_id')
|
||||||
|
if not isinstance(pid,str) or pid in seen or row != expected.get(pid) or row['track'] != track:
|
||||||
|
raise ValueError('MM observation differs from frozen source or repeats an ID')
|
||||||
|
seen.add(pid)
|
||||||
|
records.append({'source':'mm_program','external_id':'2026/'+pid,
|
||||||
|
'title':row['title'],'authors':row['authors'],'venue':'MM','year':2026,'track':track,
|
||||||
|
'status':'unknown','identifiers':{},'file':str(path),'line':number,
|
||||||
|
'raw_line_sha256':digest(line),'source_record_hits':[],
|
||||||
|
'identifier_hits':[],'title_candidate_ids':[],
|
||||||
|
'input_format':'mm_program_observation_v1_not_ingestion_record',
|
||||||
|
'program_id':pid,'program_contexts':row['contexts'],
|
||||||
|
'source_observed_status':row['observed_status'],
|
||||||
|
'missing_source_fields':row['missing_fields'],
|
||||||
|
'publication_identity_verified':False,'final_decision_verified':False,
|
||||||
|
'contribution_type_verified':False})
|
||||||
|
inputs.append({'file':str(path),'file_sha256':digest(raw),'count':len(lines),
|
||||||
|
'manifest_sha256':digest(raw_report),'manifest':{
|
||||||
|
'source':'mm_program','venue':'MM','year':2026,'track':track,'status':'unknown',
|
||||||
|
'format':'mm_program_observation_v1_not_ingestion_record',
|
||||||
|
'scope':'Read-only lookup input only; unknown is not a source decision.',
|
||||||
|
'source_report':str(report_path),'source_body_sha256':digest(raw_source)}})
|
||||||
|
if seen != expected.keys() or tracks != mm2026.TRACK_COUNTS.keys():
|
||||||
|
raise ValueError('Incomplete MM observation input; all28tracks required')
|
||||||
|
return records, inputs
|
||||||
|
|
||||||
|
|
||||||
|
def _lookup(connection, values, columns, lateral):
|
||||||
|
"""Bound scalar VALUES allow the existing text codec to encode each key."""
|
||||||
|
width = len(columns)
|
||||||
|
for start in range(0, len(values), 100):
|
||||||
|
batch = values[start:start + 100]
|
||||||
|
sql = 'WITH keys(' + ','.join(columns) + ') AS (VALUES ' + ','.join(
|
||||||
|
'(' + ','.join(['?'] * width) + ')' for _ in batch) + ') SELECT keys.n,hit.* FROM keys JOIN LATERAL (' + lateral + ') hit ON true'
|
||||||
|
for row in connection.execute(sql, tuple(value for item in batch for value in item)):
|
||||||
|
yield dict(row)
|
||||||
|
|
||||||
|
|
||||||
|
def find_candidates(connection, records):
|
||||||
|
source_keys = [(i, row['source'], row['external_id']) for i, row in enumerate(records)]
|
||||||
|
for hit in _lookup(connection, source_keys, ('n', 'source', 'external_id'), '''
|
||||||
|
SELECT r.id,r.paper_id FROM sources s JOIN source_records r ON r.source_id=s.id
|
||||||
|
WHERE s.name=keys.source AND r.external_id=keys.external_id'''):
|
||||||
|
records[hit.pop('n')]['source_record_hits'].append(hit)
|
||||||
|
identifiers = []
|
||||||
|
for i, row in enumerate(records):
|
||||||
|
pairs = {(row['source'], row['external_id']), *row['identifiers'].items()}
|
||||||
|
identifiers.extend((i, scheme, value) for scheme, value in sorted(pairs))
|
||||||
|
for hit in _lookup(connection, identifiers, ('n', 'scheme', 'value'), '''
|
||||||
|
SELECT scheme,value,paper_id FROM identifiers
|
||||||
|
WHERE scheme=keys.scheme AND value=keys.value'''):
|
||||||
|
records[hit.pop('n')]['identifier_hits'].append(hit)
|
||||||
|
title_keys = [(i, normalized(row['title'])) for i, row in enumerate(records)]
|
||||||
|
for hit in _lookup(connection, title_keys, ('n', 'title_key'), '''
|
||||||
|
SELECT id AS paper_id FROM papers WHERE title_key=keys.title_key ORDER BY id LIMIT 21'''):
|
||||||
|
records[hit['n']]['title_candidate_ids'].append(hit['paper_id'])
|
||||||
|
|
||||||
|
|
||||||
|
def entity_packet(connection, paper_id):
|
||||||
|
paper = connection.execute('SELECT * FROM papers WHERE id=?', (paper_id,)).fetchone()
|
||||||
|
if paper is None:
|
||||||
|
raise RuntimeError('Candidate disappeared from read-only snapshot')
|
||||||
|
sources = [dict(r) for r in connection.execute('''SELECT r.*,s.name AS source
|
||||||
|
FROM source_records r JOIN sources s ON s.id=r.source_id
|
||||||
|
WHERE r.paper_id=? ORDER BY r.id LIMIT 101''', (paper_id,))]
|
||||||
|
attempts = [dict(r) for r in connection.execute('''SELECT s.*,v.acronym AS venue,e.year,e.track,e.event_key
|
||||||
|
FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id
|
||||||
|
WHERE s.paper_id=? ORDER BY s.id LIMIT 201''', (paper_id,))]
|
||||||
|
packet = {'paper': dict(paper), 'identifiers': [dict(r) for r in connection.execute(
|
||||||
|
'SELECT * FROM identifiers WHERE paper_id=? ORDER BY scheme,value', (paper_id,))],
|
||||||
|
'source_records': sources[:100], 'source_records_truncated': len(sources) > 100,
|
||||||
|
'attempts': attempts[:200], 'attempts_truncated': len(attempts) > 200}
|
||||||
|
for attempt in packet['attempts']:
|
||||||
|
observations = [dict(r) for r in connection.execute(
|
||||||
|
'SELECT * FROM decisions WHERE submission_id=? ORDER BY id LIMIT 201', (attempt['id'],))]
|
||||||
|
attempt['decisions'], attempt['decisions_truncated'] = observations[:200], len(observations) > 200
|
||||||
|
provenance = [dict(r) for r in connection.execute('''SELECT ss.*,r.external_id,s.name AS source
|
||||||
|
FROM submission_sources ss JOIN source_records r ON r.id=ss.source_record_id
|
||||||
|
JOIN sources s ON s.id=r.source_id WHERE ss.submission_id=?
|
||||||
|
ORDER BY ss.source_record_id,ss.source_submission_id LIMIT 101''', (attempt['id'],))]
|
||||||
|
attempt['sources'], attempt['sources_truncated'] = provenance[:100], len(provenance) > 100
|
||||||
|
for decision in attempt['decisions']:
|
||||||
|
provenance = [dict(r) for r in connection.execute('''SELECT ds.*,r.external_id,s.name AS source
|
||||||
|
FROM decision_sources ds JOIN source_records r ON r.id=ds.source_record_id
|
||||||
|
JOIN sources s ON s.id=r.source_id WHERE ds.decision_id=?
|
||||||
|
ORDER BY ds.source_record_id,ds.source_decision_id LIMIT 101''', (decision['id'],))]
|
||||||
|
decision['sources'], decision['sources_truncated'] = provenance[:100], len(provenance) > 100
|
||||||
|
return packet
|
||||||
|
|
||||||
|
|
||||||
|
def classify(record, entities):
|
||||||
|
anchors = {r['paper_id'] for r in record['source_record_hits'] + record['identifier_hits']}
|
||||||
|
titles = record['title_candidate_ids']
|
||||||
|
flags = []
|
||||||
|
own_authors = authors_multiset(record['authors'])
|
||||||
|
if not own_authors:
|
||||||
|
flags.append('missing_source_authors')
|
||||||
|
if any(n > 1 for n in own_authors.values()):
|
||||||
|
flags.append('duplicate_normalized_author_names')
|
||||||
|
if len(normalized(record['title'])) < 25:
|
||||||
|
flags.append('short_title')
|
||||||
|
comparisons = []
|
||||||
|
for pid in sorted(anchors | set(titles)):
|
||||||
|
entity = entities[pid]
|
||||||
|
other = authors_multiset(entity['authors'])
|
||||||
|
comparisons.append({'paper_id': pid,
|
||||||
|
'normalized_title_equal': normalized(record['title']) == normalized(entity['title']),
|
||||||
|
'nonempty_author_multiset_equal': bool(own_authors) and own_authors == other,
|
||||||
|
'author_name_overlap': sorted(own_authors.keys() & other.keys())})
|
||||||
|
if len(anchors) > 1:
|
||||||
|
kind = 'conflicting_existing_bindings_hold'
|
||||||
|
elif record['source_record_hits']:
|
||||||
|
kind = 'existing_source_record_reconcile_no_replay'
|
||||||
|
elif anchors:
|
||||||
|
kind = 'existing_identifier_binding_requires_review'
|
||||||
|
elif len(titles) >= 21:
|
||||||
|
kind = 'title_candidate_limit_reached_hold'
|
||||||
|
elif not titles:
|
||||||
|
kind = 'no_exact_match_not_proof_of_novelty'
|
||||||
|
elif len(titles) > 1:
|
||||||
|
kind = 'multiple_exact_title_entities_hold'
|
||||||
|
elif comparisons[0]['nonempty_author_multiset_equal']:
|
||||||
|
kind = 'one_exact_title_author_multiset_requires_review'
|
||||||
|
elif comparisons[0]['author_name_overlap']:
|
||||||
|
kind = 'partial_author_overlap_requires_review'
|
||||||
|
else:
|
||||||
|
kind = 'no_author_agreement_hold'
|
||||||
|
return {'kind': kind, 'flags': flags, 'comparisons': comparisons,
|
||||||
|
'title_candidates_truncated': len(titles) >= 21,
|
||||||
|
'decision': 'not_approved_for_import'}
|
||||||
|
|
||||||
|
|
||||||
|
def _atomic_json(path, value):
|
||||||
|
temporary = path.with_suffix(path.suffix + '.part')
|
||||||
|
with temporary.open('w', encoding='utf-8') as stream:
|
||||||
|
json.dump(value, stream, ensure_ascii=False, indent=2)
|
||||||
|
stream.write('\n')
|
||||||
|
temporary.replace(path)
|
||||||
|
|
||||||
|
|
||||||
|
def audit(files, out, *, pg_service, pg_schema, mm_program_report=None):
|
||||||
|
# Reject input corruption before opening a database connection or reserving output.
|
||||||
|
if bool(files) == bool(mm_program_report):
|
||||||
|
raise ValueError('Choose source files or one MM program report')
|
||||||
|
records, inputs = load_mm_program_inputs(mm_program_report) if mm_program_report else load_inputs(files)
|
||||||
|
out = Path(out)
|
||||||
|
out.mkdir(parents=True, exist_ok=False)
|
||||||
|
(out / 'audit.lock').write_text(timestamp() + '\n')
|
||||||
|
connection = None
|
||||||
|
try:
|
||||||
|
connection = ReadConnection(pg_service, pg_schema)
|
||||||
|
connection.execute("SET LOCAL statement_timeout='60s'")
|
||||||
|
if connection.execute('SHOW transaction_read_only').fetchone()[0] != 'on':
|
||||||
|
raise RuntimeError('A read-only transaction is required')
|
||||||
|
snapshot = dict(connection.execute('''SELECT current_database() AS database,current_schema() AS schema,
|
||||||
|
transaction_timestamp()::text AS transaction_at,pg_current_snapshot()::text AS snapshot''').fetchone())
|
||||||
|
find_candidates(connection, records)
|
||||||
|
ids = sorted({hit['paper_id'] for row in records for hit in row['source_record_hits'] + row['identifier_hits']} |
|
||||||
|
{pid for row in records for pid in row['title_candidate_ids']})
|
||||||
|
entities, index = {}, {}
|
||||||
|
for number, pid in enumerate(ids, 1):
|
||||||
|
packet = entity_packet(connection, pid)
|
||||||
|
relative = Path('entities') / str(pid // 10000) / f'{pid}.json.gz'
|
||||||
|
target = out / relative
|
||||||
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
raw = gzip.compress(json.dumps(packet, ensure_ascii=False).encode(), mtime=0)
|
||||||
|
target.write_bytes(raw)
|
||||||
|
index[str(pid)] = {'file': relative.as_posix(), 'sha256': digest(raw)}
|
||||||
|
entities[pid] = {'title': packet['paper']['title'],
|
||||||
|
'authors': json.loads(packet['paper']['authors_json'])}
|
||||||
|
if number % 250 == 0 or number == len(ids):
|
||||||
|
print(json.dumps({'phase': 'entity_evidence', 'completed': number, 'total': len(ids)}), flush=True)
|
||||||
|
kinds = Counter()
|
||||||
|
for row in records:
|
||||||
|
row.update(classify(row, entities))
|
||||||
|
kinds[row['kind']] += 1
|
||||||
|
report = {'status': 'completed_read_only_candidates_not_import_approval',
|
||||||
|
'started_snapshot': snapshot, 'completed_at': timestamp(), 'transaction_read_only': True,
|
||||||
|
'database_written': False, 'approved_for_import': 0, 'records': len(records),
|
||||||
|
'kind_counts': dict(kinds), 'inputs': inputs, 'entities': index, 'entries': records,
|
||||||
|
'limitations': ['Exact keys only; no match does not establish a new work.',
|
||||||
|
'Name agreement does not prove person or scientific-work identity.',
|
||||||
|
'Bound identifiers can be incorrect; review titles, authors and original evidence.',
|
||||||
|
'Source/history limits are explicit in compressed entity packets.',
|
||||||
|
'Program observations and unknown audit status do not establish contribution type, publication or a final decision.',
|
||||||
|
'Not a final whole-database deduplication or coverage audit.']}
|
||||||
|
_atomic_json(out / 'postgresql-preflight.json', report)
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
_atomic_json(out / 'failed.json', {'status': 'failed_no_import_approval',
|
||||||
|
'error_type': type(error).__name__, 'database_written': False})
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
if connection is not None:
|
||||||
|
connection.close()
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
|
inputs = parser.add_mutually_exclusive_group(required=True)
|
||||||
|
inputs.add_argument('--files', nargs='+', type=Path)
|
||||||
|
inputs.add_argument('--mm-program-report', type=Path,
|
||||||
|
help='Read all frozen MM program observations directly; never generate an import queue')
|
||||||
|
parser.add_argument('--pg-service', required=True)
|
||||||
|
parser.add_argument('--pg-schema', required=True)
|
||||||
|
parser.add_argument('--out', required=True, type=Path)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
report = audit(args.files, args.out, pg_service=args.pg_service, pg_schema=args.pg_schema,
|
||||||
|
mm_program_report=args.mm_program_report)
|
||||||
|
print(json.dumps({k: report[k] for k in ('status', 'records', 'kind_counts', 'approved_for_import')}, ensure_ascii=False))
|
||||||
@@ -13,6 +13,7 @@ COMMANDS = {
|
|||||||
('download',): ('paper_atlas.io.snapshots', {}, '下载并校验固定版本快照'),
|
('download',): ('paper_atlas.io.snapshots', {}, '下载并校验固定版本快照'),
|
||||||
('collect','pmlr'): ('paper_atlas.collectors.pmlr', {}, 'PMLR 官方元数据'),
|
('collect','pmlr'): ('paper_atlas.collectors.pmlr', {}, 'PMLR 官方元数据'),
|
||||||
('collect','acl'): ('paper_atlas.collectors.acl', {}, 'ACL Anthology XML'),
|
('collect','acl'): ('paper_atlas.collectors.acl', {}, 'ACL Anthology XML'),
|
||||||
|
('collect','springer-selected'): ('paper_atlas.collectors.springer_selected', {}, '核验限定的会后选集章节与原会议关系'),
|
||||||
('collect','iclr-archive'): ('paper_atlas.collectors.iclr_archive', {}, '核对ICLR 2014–2016官方历史录用档案'),
|
('collect','iclr-archive'): ('paper_atlas.collectors.iclr_archive', {}, '核对ICLR 2014–2016官方历史录用档案'),
|
||||||
('collect','iclr-program'): ('paper_atlas.collectors.iclr', {}, 'ICLR 官方公开程序'),
|
('collect','iclr-program'): ('paper_atlas.collectors.iclr', {}, 'ICLR 官方公开程序'),
|
||||||
('collect','iclr-snapshot'): ('paper_atlas.collectors.openreview', {}, 'ICLR 公开投稿与决定快照'),
|
('collect','iclr-snapshot'): ('paper_atlas.collectors.openreview', {}, 'ICLR 公开投稿与决定快照'),
|
||||||
@@ -22,11 +23,25 @@ COMMANDS = {
|
|||||||
('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'),
|
('collect','colm'): ('paper_atlas.collectors.proceedings', {'source':'colm'}, 'COLM 官方论文集'),
|
||||||
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
|
('collect','neurips'): ('paper_atlas.collectors.proceedings', {'source':'neurips'}, 'NeurIPS 官方论文集'),
|
||||||
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
|
('collect','mlsys'): ('paper_atlas.collectors.proceedings', {'source':'mlsys'}, 'MLSys 官方历年标题与摘要'),
|
||||||
|
('collect','ismir2026-program'): ('paper_atlas.collectors.ismir2026', {}, '核验 ISMIR 2026 公开节目与录用名单并保留原始摘要、来源差异和自愿公开评审'),
|
||||||
|
('collect','icaps2026-registry'): ('paper_atlas.collectors.icaps_registry2026', {}, '核验 ICAPS 2026 Crossref 注册元数据并保留独立版本'),
|
||||||
|
('collect','eccv2026-program'): ('paper_atlas.collectors.eccv2026', {}, '核验ECCV2026暂定节目并断点采集公开元数据'),
|
||||||
|
('collect','icassp2026-registry'): ('paper_atlas.collectors.icassp2026', {}, '核验 ICASSP 2026 冻结注册论文元数据'),
|
||||||
|
('collect','icdm2026-program'): ('paper_atlas.collectors.icdm2026', {}, '核验 ICDM2026 冻结日程及明确的研究/应用轨道'),
|
||||||
|
('collect','sdm2026-proceedings'): ('paper_atlas.collectors.sdm2026', {}, '核验 SDM 2026 注册论文并单独保留公开日程版本'),
|
||||||
|
('collect','icaps2026-program'): ('paper_atlas.collectors.icaps2026', {}, '核验 ICAPS 2026 官方录用名单并分开既有论文展示'),
|
||||||
|
('collect','rss2026-proceedings'): ('paper_atlas.collectors.rss2026', {}, '核验 RSS 2026 官方论文集与独立录用表'),
|
||||||
|
('collect','recsys2026-program'): ('paper_atlas.collectors.recsys2026', {}, '核验 RecSys 2026 公开录用贡献并保留 DOI 来源版本'),
|
||||||
|
('collect','mm2026-program'): ('paper_atlas.collectors.mm2026', {}, '恢复 MM 2026 全部来源 track 与重复展示上下文,待贡献和身份核查'),
|
||||||
|
('collect','mm2026-publications'): ('paper_atlas.collectors.mm_publications2026', {}, '核验 MM 2026 全部注册出版条目并分别保留节目分轨、摘要与版本差异'),
|
||||||
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
('collect','cvf'): ('paper_atlas.collectors.vision', {'source':'cvf'}, 'CVPR / ICCV 官方开放目录'),
|
||||||
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
|
('collect','ecva'): ('paper_atlas.collectors.vision', {'source':'ecva'}, 'ECCV 官方开放目录'),
|
||||||
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
|
('collect','aamas-proceedings'): ('paper_atlas.collectors.aamas', {}, '按官方目录区分 AAMAS 发表形式和摘要'),
|
||||||
|
('collect','corl2026-program'): ('paper_atlas.collectors.corl2026', {}, '核验 CoRL 2026 官方论坛名单与分字段公开档案'),
|
||||||
|
('collect','neurips-program'): ('paper_atlas.collectors.neurips_program', {}, '按公开节目来源区分 NeurIPS 2026 投稿与重复展示'),
|
||||||
('collect','www-archive'): ('paper_atlas.collectors.www_archive', {}, '核验早期WWW官方档案并分开待核查条目'),
|
('collect','www-archive'): ('paper_atlas.collectors.www_archive', {}, '核验早期WWW官方档案并分开待核查条目'),
|
||||||
('collect','dblp'): ('paper_atlas.collectors.dblp_records', {}, '核对并整理 DBLP 历史发表记录'),
|
('collect','dblp'): ('paper_atlas.collectors.dblp_records', {}, '核对并整理 DBLP 历史发表记录'),
|
||||||
|
('collect','dblp-family'): ('paper_atlas.collectors.dblp_recovery', {}, '从固定快照恢复指定 DBLP family 的原始候选'),
|
||||||
('catalogue','dblp'): ('paper_atlas.collectors.dblp', {}, '盘点 DBLP 历史会议目录'),
|
('catalogue','dblp'): ('paper_atlas.collectors.dblp', {}, '盘点 DBLP 历史会议目录'),
|
||||||
('import','arxiv'): ('paper_atlas.ingestion.arxiv', {}, '导入 arXiv 元数据 ZIP'),
|
('import','arxiv'): ('paper_atlas.ingestion.arxiv', {}, '导入 arXiv 元数据 ZIP'),
|
||||||
('import','records'): ('paper_atlas.ingestion.records', {}, '导入核验过的会议文件'),
|
('import','records'): ('paper_atlas.ingestion.records', {}, '导入核验过的会议文件'),
|
||||||
@@ -43,6 +58,7 @@ COMMANDS = {
|
|||||||
('audit','arxiv'): ('paper_atlas.audits.arxiv', {}, '核验 arXiv 原始编号、重复行与 ZIP 完整性'),
|
('audit','arxiv'): ('paper_atlas.audits.arxiv', {}, '核验 arXiv 原始编号、重复行与 ZIP 完整性'),
|
||||||
('audit','arxiv-tags'): ('paper_atlas.audits.arxiv_tags', {}, '逐记录核验 arXiv 类别及出处完整性'),
|
('audit','arxiv-tags'): ('paper_atlas.audits.arxiv_tags', {}, '逐记录核验 arXiv 类别及出处完整性'),
|
||||||
('audit','public-openreview'): ('paper_atlas.audits.public_openreview', {}, '核对公开负面决定与已有投稿历史'),
|
('audit','public-openreview'): ('paper_atlas.audits.public_openreview', {}, '核对公开负面决定与已有投稿历史'),
|
||||||
|
('audit','source-identities'): ('paper_atlas.audits.source_identity', {}, '只读核对来源文件的身份候选与既有投稿历史'),
|
||||||
('audit','histories'): ('paper_atlas.audits.histories', {}, '只读核对跨会议投稿历史及决定来源关系'),
|
('audit','histories'): ('paper_atlas.audits.histories', {}, '只读核对跨会议投稿历史及决定来源关系'),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -136,6 +136,9 @@ def read_pages(cache=None, snapshots=None):
|
|||||||
|
|
||||||
|
|
||||||
def _collect(year, out, *, cache=None, snapshots=None):
|
def _collect(year, out, *, cache=None, snapshots=None):
|
||||||
|
if year == 2026:
|
||||||
|
from paper_atlas.collectors.aamas2026 import collect as collect_2026
|
||||||
|
return collect_2026(out, cache=cache, snapshots=snapshots)
|
||||||
if year == 2025:
|
if year == 2025:
|
||||||
from paper_atlas.collectors.aamas2025 import collect as collect_2025
|
from paper_atlas.collectors.aamas2025 import collect as collect_2025
|
||||||
return collect_2025(out, cache=cache, snapshots=snapshots)
|
return collect_2025(out, cache=cache, snapshots=snapshots)
|
||||||
@@ -193,8 +196,8 @@ def _collect(year, out, *, cache=None, snapshots=None):
|
|||||||
|
|
||||||
|
|
||||||
def collect(year, out, *, cache=None, snapshots=None):
|
def collect(year, out, *, cache=None, snapshots=None):
|
||||||
if year not in (2012, 2025):
|
if year not in (2012, 2025, 2026):
|
||||||
raise ValueError('Only independently reviewed AAMAS 2012/2025 layouts are supported')
|
raise ValueError('Only independently reviewed AAMAS 2012/2025/2026 layouts are supported')
|
||||||
out = Path(out)
|
out = Path(out)
|
||||||
out.mkdir(parents=True, exist_ok=True)
|
out.mkdir(parents=True, exist_ok=True)
|
||||||
with (out/'aamas.collect.lock').open('a') as lock:
|
with (out/'aamas.collect.lock').open('a') as lock:
|
||||||
@@ -204,7 +207,7 @@ def collect(year, out, *, cache=None, snapshots=None):
|
|||||||
|
|
||||||
def main(argv=None, *, prog=None):
|
def main(argv=None, *, prog=None):
|
||||||
parser = argparse.ArgumentParser(prog=prog)
|
parser = argparse.ArgumentParser(prog=prog)
|
||||||
parser.add_argument('--year', type=int, choices=[2012, 2025], required=True)
|
parser.add_argument('--year', type=int, choices=[2012, 2025, 2026], required=True)
|
||||||
parser.add_argument('--out', required=True)
|
parser.add_argument('--out', required=True)
|
||||||
source = parser.add_mutually_exclusive_group(required=True)
|
source = parser.add_mutually_exclusive_group(required=True)
|
||||||
source.add_argument('--cache')
|
source.add_argument('--cache')
|
||||||
|
|||||||
@@ -0,0 +1,219 @@
|
|||||||
|
"""Reviewed AAMAS 2026 HTML metadata; retractions stay outside accepted queues."""
|
||||||
|
from collections import Counter, defaultdict
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
from paper_atlas.collectors.aamas2025 import one
|
||||||
|
from paper_atlas.db.submissions import timestamp
|
||||||
|
from paper_atlas.io.html import Document, Node, clean
|
||||||
|
from paper_atlas.io.http import fetch
|
||||||
|
|
||||||
|
ROOT = 'https://www.ifaamas.org/Proceedings/aamas2026/'
|
||||||
|
URL = ROOT+'forms/contents.htm'
|
||||||
|
TITLE = 'AAMAS 2026: Proceedings of the 2026 International Conference on Autonomous Agents and Multiagent Systems'
|
||||||
|
SECTIONS = {
|
||||||
|
'K': ('Keynote Talks', 'keynote', 'keynote', 3),
|
||||||
|
'R': ('Research Paper Track', 'full_paper', 'proceedings_unspecified', 338),
|
||||||
|
'1': ('Extended Abstracts', 'extended_abstract', 'proceedings_unspecified', 185),
|
||||||
|
'A': ('AAAI Track', 'full_paper', 'aaai_transfer', 19),
|
||||||
|
'B': ('Blue Sky Ideas Track', 'blue_sky_paper', 'blue_sky', 17),
|
||||||
|
'C': ('Doctoral Consortium', 'doctoral_abstract', 'doctoral', 34),
|
||||||
|
'D': ('Demonstration Track', 'demo', 'demo', 44),
|
||||||
|
'J': ('JAAMAS Track', 'journal_presentation', 'jaamas', 3),
|
||||||
|
}
|
||||||
|
PDF = re.compile(r'\.\./pdfs/([A-Z]{4}[0-9]{4})\.pdf')
|
||||||
|
RETRACTION_TITLE = 'Unifying Model-Free Efficiency and Model-Based Representations via Latent Dynamics'
|
||||||
|
WITHDRAWAL_TITLE = 'Cognitive Heterogeneity and Behavioral Biases in Multi-Stage Supply Chains: Evidence from LLM-Based Agent Simulations'
|
||||||
|
|
||||||
|
|
||||||
|
def parse_authors(paragraph):
|
||||||
|
font = one([n for n in paragraph.walk() if n.tag == 'font' and
|
||||||
|
n.attrs.get('color', '').lower() == '#222222'], 'ordered author block')
|
||||||
|
authors, affiliations = [], []
|
||||||
|
name, affiliation = '', ''
|
||||||
|
def flush():
|
||||||
|
nonlocal name, affiliation
|
||||||
|
if clean(name) or clean(affiliation):
|
||||||
|
if not clean(name):
|
||||||
|
raise ValueError('Affiliation without author')
|
||||||
|
authors.append(clean(name)); affiliations.append(clean(affiliation))
|
||||||
|
name, affiliation = '', ''
|
||||||
|
for child in font.children:
|
||||||
|
if isinstance(child, str):
|
||||||
|
name += child
|
||||||
|
elif child.tag == 'br':
|
||||||
|
flush()
|
||||||
|
elif child.tag == 'i':
|
||||||
|
affiliation += child.text()
|
||||||
|
else:
|
||||||
|
raise ValueError('Unreviewed author markup: '+child.tag)
|
||||||
|
flush()
|
||||||
|
if not authors:
|
||||||
|
raise ValueError('Empty author list')
|
||||||
|
return authors, affiliations
|
||||||
|
|
||||||
|
|
||||||
|
def parse_contents(html, evidence):
|
||||||
|
doc = Document(html)
|
||||||
|
if [clean(n.text()) for n in doc.root.walk() if n.tag == 'title'] != [TITLE]:
|
||||||
|
raise ValueError('Unexpected AAMAS proceedings title/year')
|
||||||
|
section = None
|
||||||
|
headers, records, excluded, held = [], [], [], []
|
||||||
|
counts, seen_pages, seen_pdf_ids = Counter(), set(), set()
|
||||||
|
for node in doc.root.walk():
|
||||||
|
if node.tag != 'p' or any(n is not node and n.tag == 'p' for n in node.walk()):
|
||||||
|
continue
|
||||||
|
anchors = [n.attrs['name'] for n in node.walk() if n.tag == 'a' and n.attrs.get('name') in SECTIONS]
|
||||||
|
if anchors:
|
||||||
|
section = one(anchors, 'section anchor')
|
||||||
|
if clean(node.text()) != SECTIONS[section][0]:
|
||||||
|
raise ValueError('Publisher section heading differs from reviewed layout')
|
||||||
|
headers.append(section)
|
||||||
|
page_fonts = [n for n in node.walk() if n.tag == 'font' and
|
||||||
|
n.attrs.get('color', '').lower() == '#444444']
|
||||||
|
if not page_fonts:
|
||||||
|
continue
|
||||||
|
page_text = clean(one(page_fonts, 'page marker').text())
|
||||||
|
page_match = re.fullmatch(r'\(Page ([1-9][0-9]*)\)', page_text)
|
||||||
|
if not page_match or section not in SECTIONS:
|
||||||
|
raise ValueError('Invalid page or paper outside reviewed sections')
|
||||||
|
page = int(page_match[1])
|
||||||
|
if page in seen_pages:
|
||||||
|
raise ValueError('Repeated publisher page ID')
|
||||||
|
seen_pages.add(page)
|
||||||
|
title_node = one([n for n in node.walk() if n.tag == 'strong'], 'paper title')
|
||||||
|
title = clean(title_node.text())
|
||||||
|
if not title:
|
||||||
|
raise ValueError('Empty paper title')
|
||||||
|
authors, affiliations = parse_authors(node)
|
||||||
|
link = one([n for n in node.walk() if n.tag == 'a' and 'href' in n.attrs], 'paper link')
|
||||||
|
href = link.attrs['href']
|
||||||
|
title_nodes = {id(n) for n in title_node.walk()}
|
||||||
|
retractions = [clean(n.text()) for n in link.walk() if n.tag == 'i' and id(n) not in title_nodes]
|
||||||
|
heading, publication_format, track, _ = SECTIONS[section]
|
||||||
|
counts[section] += 1
|
||||||
|
metadata = {'publisher_page': page, 'section_url': URL, 'section_anchor': section,
|
||||||
|
'section_heading': heading, 'publication_format': publication_format,
|
||||||
|
'author_affiliations': affiliations, 'source_evidence': evidence,
|
||||||
|
'decision_date_precision': 'not_supplied',
|
||||||
|
'missing_abstract_reason': 'Publisher HTML supplies no abstract; research PDF not downloaded.',
|
||||||
|
'publication_evidence': 'Published proceedings entry, not an original decision note.'}
|
||||||
|
withdrawn = retractions == ['Withdrawn June 8, 2026']
|
||||||
|
if withdrawn:
|
||||||
|
if (section, page, title, href) != ('1', 3178, WITHDRAWAL_TITLE, ''):
|
||||||
|
raise ValueError('Unreviewed withdrawal notice')
|
||||||
|
metadata.update({'publication_state': 'withdrawn',
|
||||||
|
'withdrawal_date': '2026-06-08',
|
||||||
|
'raw_publication_state': retractions[0],
|
||||||
|
'decision_date_precision': 'day',
|
||||||
|
'publication_evidence': 'Explicit dated publisher withdrawal notice, not an original review decision note.'})
|
||||||
|
elif retractions:
|
||||||
|
if (section, page, title, href, retractions) != ('R', 238, RETRACTION_TITLE, '', ['Retracted June 8, 2026']):
|
||||||
|
raise ValueError('Unreviewed retraction or publication status markup')
|
||||||
|
held.append({'source': 'aamas_proceedings', 'external_id': '2026/p238',
|
||||||
|
'title': title, 'authors': authors, 'year': 2026, 'track': track,
|
||||||
|
'url': URL+'#R', 'abstract': None, 'identifiers': {},
|
||||||
|
'observed_at': evidence['observed_at'], 'metadata': metadata,
|
||||||
|
'publication_state': 'retracted', 'retraction_date': '2026-06-08',
|
||||||
|
'raw_publication_state': retractions[0],
|
||||||
|
'hold_reason': 'Explicit retraction; not an accepted/withdrawn submission assertion. Dedicated decision modeling and identity review required.'})
|
||||||
|
continue
|
||||||
|
if not withdrawn:
|
||||||
|
match = PDF.fullmatch(href)
|
||||||
|
if not match or match[1] in seen_pdf_ids:
|
||||||
|
raise ValueError('Invalid or repeated publisher PDF ID')
|
||||||
|
seen_pdf_ids.add(match[1])
|
||||||
|
metadata.update({'publisher_document_id': match[1], 'paper_url': ROOT+'pdfs/'+match[1]+'.pdf'})
|
||||||
|
if section == 'K':
|
||||||
|
excluded.append({'publisher_page': page, 'title': title, 'authors': authors,
|
||||||
|
'metadata': metadata, 'reason': 'Publisher keynote, not a research submission.'})
|
||||||
|
continue
|
||||||
|
metadata['track_scope_note'] = ('Published format known; individual main/special submission track not established.'
|
||||||
|
if track == 'proceedings_unspecified' else 'Explicit publisher section.')
|
||||||
|
if section == 'A':
|
||||||
|
metadata['prior_submission_note'] = 'AAAI transfer section does not supply original submission IDs or individual prior decision notes; none inferred.'
|
||||||
|
if section == 'J':
|
||||||
|
metadata['track_scope_note'] = 'Journal presentation; not another independent journal review or research-track acceptance.'
|
||||||
|
records.append({'source': 'aamas_proceedings', 'external_id': '2026/p'+str(page),
|
||||||
|
'url': URL+'#'+section, 'title': title, 'abstract': None,
|
||||||
|
'authors': authors, 'identifiers': {}, 'venue': 'AAMAS', 'venue_aliases': [],
|
||||||
|
'year': 2026, 'track': track, 'status': 'withdrawn' if withdrawn else 'accepted',
|
||||||
|
'observed_at': evidence['observed_at'],
|
||||||
|
'raw_decision': retractions[0] if withdrawn else 'listed in official AAMAS 2026 published proceedings '+heading,
|
||||||
|
'metadata': metadata,
|
||||||
|
'tags': [{'namespace': 'aamas_proceedings:publication_format', 'value': publication_format}]})
|
||||||
|
if headers != list(SECTIONS) or dict(counts) != {k: v[3] for k, v in SECTIONS.items()}:
|
||||||
|
raise ValueError('Publisher section inventory differs from reviewed 2026 layout')
|
||||||
|
if (len(records), len(excluded), len(held)) != (639, 3, 1):
|
||||||
|
raise ValueError('Unexpected accepted/keynote/retraction partition')
|
||||||
|
if Counter(r['status'] for r in records) != {'accepted': 638, 'withdrawn': 1}:
|
||||||
|
raise ValueError('Unexpected publisher publication-state partition')
|
||||||
|
return records, excluded, held
|
||||||
|
|
||||||
|
|
||||||
|
def read_page(cache, snapshots):
|
||||||
|
if bool(cache) == bool(snapshots):
|
||||||
|
raise ValueError('Choose exactly one cache or frozen snapshot directory')
|
||||||
|
if snapshots:
|
||||||
|
base = Path(snapshots).resolve()
|
||||||
|
frozen = json.loads((base/'receipts.json').read_text())['receipts']
|
||||||
|
r = one([r for r in frozen if r['url'] == URL], 'frozen response receipt')
|
||||||
|
filename = Path(r['file'])
|
||||||
|
if (filename.is_absolute() or '..' in filename.parts or
|
||||||
|
not (base/filename).resolve().is_relative_to(base) or r.get('status') != 200):
|
||||||
|
raise ValueError('Unsafe or unsuccessful publisher response')
|
||||||
|
data = (base/filename).read_bytes()
|
||||||
|
if len(data) > 8*1024**2 or len(data) != r['bytes'] or hashlib.sha256(data).hexdigest() != r['sha256']:
|
||||||
|
raise ValueError('Frozen publisher response checksum/size mismatch')
|
||||||
|
observed = r['finished_at']
|
||||||
|
else:
|
||||||
|
data = fetch(URL, cache, 8*1024**2, delay=1)
|
||||||
|
r = json.loads((Path(cache)/hashlib.sha256(URL.encode()).hexdigest()).with_suffix('.json').read_text())
|
||||||
|
observed = r['retrieved_at']
|
||||||
|
timestamp(observed)
|
||||||
|
return data, {'url': URL, 'sha256': hashlib.sha256(data).hexdigest(), 'bytes': len(data),
|
||||||
|
'file': 'evidence/'+hashlib.sha256(data).hexdigest()+'.html',
|
||||||
|
'observed_at': observed, 'http_status': 200, 'response_receipt': r}
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, cache=None, snapshots=None):
|
||||||
|
data, evidence = read_page(cache, snapshots)
|
||||||
|
records, excluded, held = parse_contents(data.decode('utf-8'), evidence)
|
||||||
|
grouped = defaultdict(list)
|
||||||
|
for row in records:
|
||||||
|
grouped[row['track']].append(row)
|
||||||
|
out = Path(out); out.mkdir(parents=True, exist_ok=True); files = []
|
||||||
|
with tempfile.TemporaryDirectory(prefix='.aamas2026-', dir=out) as temporary:
|
||||||
|
stage = Path(temporary); (stage/'evidence').mkdir()
|
||||||
|
(stage/evidence['file']).write_bytes(data)
|
||||||
|
for track, group in sorted(grouped.items()):
|
||||||
|
p = stage/f'aamas-proceedings-2026-{track}.jsonl'
|
||||||
|
p.write_text(''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in group))
|
||||||
|
manifest = {'source': 'aamas_proceedings', 'parser_version': 3, 'venue': 'AAMAS', 'year': 2026,
|
||||||
|
'track': track, 'count': len(group), 'file': str(out/p.name),
|
||||||
|
'file_sha256': hashlib.sha256(p.read_bytes()).hexdigest(), 'list_url': URL,
|
||||||
|
'status': 'partial', 'missing_abstract_count': len(group),
|
||||||
|
'status_counts': dict(Counter(r['status'] for r in group)),
|
||||||
|
'publication_format_counts': dict(Counter(r['metadata']['publication_format'] for r in group)),
|
||||||
|
'note': 'Reviewed TOC partition excluding keynote/retracted entries; identity and full submission history remain unverified.'}
|
||||||
|
p.with_suffix('.manifest.json').write_text(json.dumps(manifest, ensure_ascii=False, indent=2)+'\n')
|
||||||
|
files.append(manifest)
|
||||||
|
result = {'source': 'aamas_proceedings', 'year': 2026, 'finished_at': timestamp(),
|
||||||
|
'status': 'collected_pending_identity_review', 'records': len(records), 'files': files,
|
||||||
|
'excluded_publications': excluded, 'held_publications': held,
|
||||||
|
'status_counts': dict(Counter(r['status'] for r in records)),
|
||||||
|
'source_receipts': [evidence], 'database_written': False, 'pdfs_downloaded': False,
|
||||||
|
'note': 'Retraction evidence held separately; publication is not an original decision note, and absence is not rejection.'}
|
||||||
|
report = 'aamas-proceedings-2026.json'
|
||||||
|
(stage/report).write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n')
|
||||||
|
(out/'evidence').mkdir(exist_ok=True)
|
||||||
|
for p in (stage/'evidence').iterdir():
|
||||||
|
p.replace(out/'evidence'/p.name)
|
||||||
|
for p in stage.iterdir():
|
||||||
|
if p.is_file() and p.name != report:
|
||||||
|
p.replace(out/p.name)
|
||||||
|
(stage/report).replace(out/report)
|
||||||
|
return result
|
||||||
@@ -6,6 +6,7 @@ import xml.etree.ElementTree as ET
|
|||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
from paper_atlas.io.git_metadata import checkout_metadata
|
from paper_atlas.io.git_metadata import checkout_metadata
|
||||||
|
from paper_atlas.collectors.acl_historical import event_scope, material_reason
|
||||||
|
|
||||||
VENUES={'acl':'ACL','emnlp':'EMNLP','naacl':'NAACL','eacl':'EACL','coling':'COLING','conll':'CoNLL'}
|
VENUES={'acl':'ACL','emnlp':'EMNLP','naacl':'NAACL','eacl':'EACL','coling':'COLING','conll':'CoNLL'}
|
||||||
|
|
||||||
@@ -29,7 +30,10 @@ def paper_id(collection,volume,number):
|
|||||||
return f'{collection}-{int(volume):0{width}d}{int(number):0{digits}d}'
|
return f'{collection}-{int(volume):0{width}d}{int(number):0{digits}d}'
|
||||||
|
|
||||||
|
|
||||||
def volume_scope(meta,volume):
|
def volume_scope(meta,volume,collection=None):
|
||||||
|
historical=event_scope(meta,volume,collection)
|
||||||
|
if historical:
|
||||||
|
return historical['canonical_venue'],historical['venue_aliases'],historical['track']
|
||||||
codes=[text(n).lower() for n in meta.findall('venue')]
|
codes=[text(n).lower() for n in meta.findall('venue')]
|
||||||
book=text(meta.find('booktitle'))
|
book=text(meta.find('booktitle'))
|
||||||
lower=book.lower()
|
lower=book.lower()
|
||||||
@@ -59,7 +63,7 @@ def collect_xml(path,out,commit):
|
|||||||
vid=volume.attrib['id']
|
vid=volume.attrib['id']
|
||||||
meta=volume.find('meta')
|
meta=volume.find('meta')
|
||||||
if meta is None:raise ValueError('Volume metadata missing')
|
if meta is None:raise ValueError('Volume metadata missing')
|
||||||
scope=volume_scope(meta,vid)
|
scope=volume_scope(meta,vid,collection)
|
||||||
key=f'{collection}-{vid}'
|
key=f'{collection}-{vid}'
|
||||||
if scope is None:
|
if scope is None:
|
||||||
results.append({'volume':key,'status':'excluded_by_venue','venue_codes':[text(n) for n in meta.findall('venue')],'booktitle':text(meta.find('booktitle')),'year':text(meta.find('year'))})
|
results.append({'volume':key,'status':'excluded_by_venue','venue_codes':[text(n) for n in meta.findall('venue')],'booktitle':text(meta.find('booktitle')),'year':text(meta.find('year'))})
|
||||||
@@ -70,12 +74,18 @@ def collect_xml(path,out,commit):
|
|||||||
year=int(year_text)
|
year=int(year_text)
|
||||||
records=[]
|
records=[]
|
||||||
frontmatter=0
|
frontmatter=0
|
||||||
|
excluded_materials=[]
|
||||||
|
historical=event_scope(meta,vid,collection)
|
||||||
for paper in volume.findall('paper'):
|
for paper in volume.findall('paper'):
|
||||||
pid=paper.attrib['id']
|
pid=paper.attrib['id']
|
||||||
if int(pid)==0:
|
if int(pid)==0:
|
||||||
frontmatter+=1
|
frontmatter+=1
|
||||||
continue
|
continue
|
||||||
identifier=paper_id(collection,vid,pid)
|
identifier=paper_id(collection,vid,pid)
|
||||||
|
reason=material_reason(collection,vid,paper)
|
||||||
|
if reason:
|
||||||
|
excluded_materials.append({'external_id':identifier,'title':text(paper.find('title')),'reason':reason})
|
||||||
|
continue
|
||||||
doi=text(paper.find('doi'))
|
doi=text(paper.find('doi'))
|
||||||
if doi.startswith('10.18653/v1/') and doi.split('10.18653/v1/',1)[1].casefold()!=identifier.casefold():
|
if doi.startswith('10.18653/v1/') and doi.split('10.18653/v1/',1)[1].casefold()!=identifier.casefold():
|
||||||
raise ValueError('Official DOI disagrees with constructed Anthology ID: '+identifier)
|
raise ValueError('Official DOI disagrees with constructed Anthology ID: '+identifier)
|
||||||
@@ -83,6 +93,11 @@ def collect_xml(path,out,commit):
|
|||||||
if not title:raise ValueError('Missing paper title: '+identifier)
|
if not title:raise ValueError('Missing paper title: '+identifier)
|
||||||
authors=[{'given':text(a.find('first')),'family':text(a.find('last'))} for a in paper.findall('author')]
|
authors=[{'given':text(a.find('first')),'family':text(a.find('last'))} for a in paper.findall('author')]
|
||||||
records.append({'source':'acl_anthology','external_id':identifier,'url':f'https://aclanthology.org/{identifier}/','title':title,'abstract':text(paper.find('abstract')) or None,'authors':authors,'identifiers':{'acl':identifier,'doi':doi},'venue':venue,'venue_aliases':aliases if len(aliases)>1 else [],'year':year,'track':track,'status':'accepted','raw_decision':'listed in authoritative ACL Anthology proceedings XML','metadata':{'collection':collection,'volume':vid,'booktitle':text(meta.find('booktitle')),'venue_codes':[text(n) for n in meta.findall('venue')],'pages':text(paper.find('pages')),'publisher':text(meta.find('publisher')),'metadata_commit':commit,'source_xml':f'https://github.com/acl-org/acl-anthology/blob/{commit}/data/xml/{collection}.xml'}})
|
records.append({'source':'acl_anthology','external_id':identifier,'url':f'https://aclanthology.org/{identifier}/','title':title,'abstract':text(paper.find('abstract')) or None,'authors':authors,'identifiers':{'acl':identifier,'doi':doi},'venue':venue,'venue_aliases':aliases if len(aliases)>1 else [],'year':year,'track':track,'status':'accepted','raw_decision':'listed in authoritative ACL Anthology proceedings XML','metadata':{'collection':collection,'volume':vid,'booktitle':text(meta.find('booktitle')),'venue_codes':[text(n) for n in meta.findall('venue')],'pages':text(paper.find('pages')),'publisher':text(meta.find('publisher')),'metadata_commit':commit,'source_xml':f'https://github.com/acl-org/acl-anthology/blob/{commit}/data/xml/{collection}.xml'}})
|
||||||
|
if historical:
|
||||||
|
records[-1]['metadata']['historical_event_scope']=historical
|
||||||
|
if historical.get('publication_format'):
|
||||||
|
records[-1]['metadata']['publication_format']=historical['publication_format']
|
||||||
|
records[-1]['tags']=[{'namespace':'acl_anthology:publication_format','value':historical['publication_format']}]
|
||||||
if not records:
|
if not records:
|
||||||
results.append({'volume':key,'status':'unavailable','reason':'no research paper elements after excluding front matter'})
|
results.append({'volume':key,'status':'unavailable','reason':'no research paper elements after excluding front matter'})
|
||||||
continue
|
continue
|
||||||
@@ -93,7 +108,7 @@ def collect_xml(path,out,commit):
|
|||||||
for record in records:output.write(json.dumps(record,ensure_ascii=False)+'\n')
|
for record in records:output.write(json.dumps(record,ensure_ascii=False)+'\n')
|
||||||
part.replace(target)
|
part.replace(target)
|
||||||
missing=sum(not row['abstract'] for row in records)
|
missing=sum(not row['abstract'] for row in records)
|
||||||
manifest={'source':'acl_anthology','venue':venue,'year':year,'track':track,'count':len(records),'xml_paper_elements':len(volume.findall('paper')),'excluded_frontmatter':frontmatter,'missing_abstract_count':missing,'list_url':f'https://aclanthology.org/{key}/','metadata_commit':commit,'file':str(target),'status':'partial' if missing else 'collected'}
|
manifest={'source':'acl_anthology','venue':venue,'year':year,'track':track,'count':len(records),'xml_paper_elements':len(volume.findall('paper')),'excluded_frontmatter':frontmatter,'excluded_materials':excluded_materials,'missing_abstract_count':missing,'list_url':f'https://aclanthology.org/{key}/','metadata_commit':commit,'file':str(target),'status':'partial' if missing else 'collected'}
|
||||||
target.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2))
|
target.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2))
|
||||||
results.append(manifest)
|
results.append(manifest)
|
||||||
print(json.dumps(manifest),flush=True)
|
print(json.dumps(manifest),flush=True)
|
||||||
|
|||||||
@@ -0,0 +1,76 @@
|
|||||||
|
"""Reviewed edition-specific scope; never alias entire source classifications."""
|
||||||
|
|
||||||
|
HISTORY_URL = 'https://aclanthology.org/www.mt-archive.info/70/ACL-1982-Walker-1.pdf'
|
||||||
|
NOTICE_URL = 'https://aclanthology.org/J77-4001.pdf'
|
||||||
|
BOOK = 'Theoretical Issues in Natural Language Processing-2'
|
||||||
|
MATERIAL_TITLE = 'List of questions suggested for consideration in each session'
|
||||||
|
|
||||||
|
|
||||||
|
def text(node):
|
||||||
|
return ' '.join(''.join(node.itertext()).split()) if node is not None else ''
|
||||||
|
|
||||||
|
|
||||||
|
def event_scope(meta, volume, collection):
|
||||||
|
if (collection, volume) == ('1963.earlymt', '1'):
|
||||||
|
return amtcl1963_scope(meta)
|
||||||
|
if (collection, volume) != ('T78', '1'):
|
||||||
|
return None
|
||||||
|
if ([text(n) for n in meta.findall('venue')] != ['tinlap']
|
||||||
|
or [text(n) for n in meta.findall('year')] != ['1978']
|
||||||
|
or [text(n) for n in meta.findall('booktitle')] != [BOOK]
|
||||||
|
or [(text(n.find('first')), text(n.find('last'))) for n in meta.findall('editor')]
|
||||||
|
!= [('David L.', 'Waltz')]):
|
||||||
|
raise ValueError('Reviewed T78-1 scope metadata changed; independent review required')
|
||||||
|
return {
|
||||||
|
'collection': 'T78', 'volume': '1', 'original_venue': 'TINLAP',
|
||||||
|
'event_year': 1978, 'event_dates': ['1978-07-25', '1978-07-27'],
|
||||||
|
'relationship': 'ACL annual meeting took the form of TINLAP-2',
|
||||||
|
'canonical_venue': 'ACL+TINLAP', 'venue_aliases': ['ACL', 'TINLAP'],
|
||||||
|
'track': 'proceedings_unspecified',
|
||||||
|
'track_scope_note': 'Invited and submitted contributions are described at volume level; individual roles are not identified in XML.',
|
||||||
|
'evidence': [
|
||||||
|
{'url': NOTICE_URL, 'sha256': '14b5deb471c55afb07d789529b95d0462a2a1a77df3dfba7fa1f5d3ab75eb411',
|
||||||
|
'pdf_pages': [9, 10], 'printed_pages': ['9', '10'],
|
||||||
|
'scope': 'Contemporary annual meeting notice, not individual decision notes; ACM confirmation was pending in this notice.'},
|
||||||
|
{'url': HISTORY_URL, 'sha256': '35c21bd4983975bcaccbb94bd8ab91c9019fc190feec1791255407cf18a7e49a',
|
||||||
|
'pdf_pages': [3], 'printed_pages': ['91'],
|
||||||
|
'scope': 'Retrospective annual meeting table and TNLP abbreviation confirm the 1978 event association.'},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def amtcl1963_scope(meta):
|
||||||
|
expected = {
|
||||||
|
'venue': ['earlymt'], 'year': ['1963'],
|
||||||
|
'booktitle': ['Proceedings of the Annual meeting of the Association for Machine Translation and Computational Linguistics'],
|
||||||
|
'address': ['Denver, Colorado'], 'month': ['25-26 August'],
|
||||||
|
}
|
||||||
|
if any([text(n) for n in meta.findall(field)] != values
|
||||||
|
for field, values in expected.items()):
|
||||||
|
raise ValueError('Reviewed AMTCL 1963 scope metadata changed; independent review required')
|
||||||
|
return {
|
||||||
|
'collection': '1963.earlymt', 'volume': '1', 'original_venue': 'EarlyMT',
|
||||||
|
'original_event_name': 'Association for Machine Translation and Computational Linguistics',
|
||||||
|
'event_year': 1963, 'event_dates': ['1963-08-25', '1963-08-26'],
|
||||||
|
'relationship': 'Annual meeting under the former name of ACL; EarlyMT is the source classification, not a global ACL alias.',
|
||||||
|
'canonical_venue': 'ACL', 'venue_aliases': ['ACL', 'AMTCL'],
|
||||||
|
'track': 'proceedings_unspecified',
|
||||||
|
'track_scope_note': 'Published abstracts with named authors; individual submission tracks and original decision notes are not supplied.',
|
||||||
|
'publication_format': 'published_abstract',
|
||||||
|
'evidence': [
|
||||||
|
{'url': 'https://2023.aclweb.org/participants/about_acl/',
|
||||||
|
'sha256': '0f6bc260c2b34c9e2a51d971aeb43c3270a5b15b6db1d6d41e50669f316345f3',
|
||||||
|
'scope': 'Official ACL history identifies AMTCL as the former name, changed to ACL in 1968.'},
|
||||||
|
{'url': 'https://github.com/acl-org/acl-anthology/blob/a7be623d01c48c61405dec31c871121cdbed5a01/data/xml/1963.earlymt.xml',
|
||||||
|
'sha256': '4a4626d792ac507de0281cb358769f7246f4a0d6b6a9176ac02e61545c1635fc',
|
||||||
|
'scope': 'Official XML supplies the exact annual meeting title, dates, location and 34 contributions; hash is XML, not HTML.'},
|
||||||
|
],
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def material_reason(collection, volume, paper):
|
||||||
|
if (collection, volume, paper.attrib.get('id')) != ('T78', '1', '36'):
|
||||||
|
return None
|
||||||
|
if text(paper.find('title')) != MATERIAL_TITLE or paper.findall('author'):
|
||||||
|
raise ValueError('Reviewed T78-1036 material changed; independent review required')
|
||||||
|
return 'Session question list, not an authored research contribution; no decision inferred.'
|
||||||
@@ -0,0 +1,183 @@
|
|||||||
|
"""Frozen CoRL 2026 official program joined to attributed public archive metadata."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import unicodedata
|
||||||
|
from urllib.parse import parse_qs, urlsplit
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, clean
|
||||||
|
|
||||||
|
SOURCE = 'corl_program_archive'
|
||||||
|
OFFICIAL = 'https://2026.corl.org/program/accepted-papers'
|
||||||
|
REVISION = '75b3f7a04b3a77c454540fe2506bfa615b8e038e'
|
||||||
|
ARCHIVE = f'https://raw.githubusercontent.com/gisbi-kim/conference-explorers/{REVISION}/corl2026/data.json'
|
||||||
|
OFFICIAL_SHA = '3f026639c1a777b0b5fbd132e1935a27cdd519ca0a04e60ca3b650a3bc7eaef4'
|
||||||
|
FILTERED_SHA = '6e6d7ecd57ee139c61283188fec39ebe9dc7f5c30ec14c8c5839d1277b03a588'
|
||||||
|
RAW_SHA = '1e96ed2dffcd96c2584a7ba020201c831e49ae1a69eae3ef1553fd5a34802810'
|
||||||
|
BLOB = '322eeef71126b85e13a98fdce9c73fac81934d3f'
|
||||||
|
FIELDS = {'id', 'title', 'openreview', 'poster_session', 'poster_day', 'poster_date',
|
||||||
|
'poster_time', 'poster_venue', 'poster_number', 'index', 'presentation_type',
|
||||||
|
'arxiv', 'matched_title', 'abstract', 'metadata_source', 'author_source',
|
||||||
|
'abstract_source', 'identity_note', 'authors', 'openreview_updated', 'metadata_checked'}
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def comparable(text):
|
||||||
|
return ''.join(c for c in unicodedata.normalize('NFKD', text).casefold() if c.isalnum())
|
||||||
|
|
||||||
|
|
||||||
|
def forum_id(url):
|
||||||
|
if not isinstance(url, str):
|
||||||
|
raise ValueError('Forum URL must be text')
|
||||||
|
u = urlsplit(url); q = parse_qs(u.query, keep_blank_values=True)
|
||||||
|
if (u.scheme != 'https' or u.netloc != 'openreview.net' or u.path != '/forum'
|
||||||
|
or u.fragment or set(q) != {'id'} or len(q['id']) != 1
|
||||||
|
or not re.fullmatch(r'[A-Za-z0-9_-]{8,32}', q['id'][0])):
|
||||||
|
raise ValueError('Invalid original forum identity')
|
||||||
|
return q['id'][0]
|
||||||
|
|
||||||
|
|
||||||
|
def parse_program(raw):
|
||||||
|
embeds = [n.attrs['data-code'] for n in Document(raw).root.walk() if 'data-code' in n.attrs]
|
||||||
|
if len(embeds) != 1:
|
||||||
|
raise ValueError('Expected one official embedded program')
|
||||||
|
doc = Document(embeds[0]); nodes = list(doc.root.walk())
|
||||||
|
if [clean(n.text()) for n in nodes if n.tag == 'h1'] != ['CoRL 2026 Accepted Papers']:
|
||||||
|
raise ValueError('Unexpected program edition')
|
||||||
|
summary = [clean(n.text()) for n in nodes if n.tag == 'p' and n.attrs.get('class') == 'sub']
|
||||||
|
if len(summary) != 1 or not summary[0].startswith('687 accepted papers · 21 orals · 72 spotlights · 687 posters'):
|
||||||
|
raise ValueError('Reviewed program census changed')
|
||||||
|
papers = {}; sessions = {}; occurrences = []
|
||||||
|
for section in (n for n in nodes if n.tag == 'section' and 'session' in n.attrs.get('class', '').split()):
|
||||||
|
sid = section.attrs.get('data-sess', section.attrs.get('id'))
|
||||||
|
if not isinstance(sid, str) or not re.fullmatch(r'(?:O[1-7]|S[1-8]|post-p[1-3])', sid) or sid in sessions:
|
||||||
|
raise ValueError('Unexpected or repeated presentation session')
|
||||||
|
kind = 'oral' if sid.startswith('O') else 'spotlight' if sid.startswith('S') else 'poster'
|
||||||
|
headings = [clean(n.text()) for n in section.walk() if n.tag == 'h3']
|
||||||
|
if len(headings) != 1:
|
||||||
|
raise ValueError('Ambiguous session heading')
|
||||||
|
sessions[sid] = {'kind': kind, 'heading': headings[0]}
|
||||||
|
seen = set()
|
||||||
|
for link in (n for n in section.walk() if n.tag == 'a' and 'openreview.net' in n.attrs.get('href', '')):
|
||||||
|
fid = forum_id(link.attrs['href']); title = clean(link.text())
|
||||||
|
if fid in seen or not title:
|
||||||
|
raise ValueError('Repeated or empty paper in session')
|
||||||
|
seen.add(fid)
|
||||||
|
paper = papers.setdefault(fid, {'title': title, 'presentations': []})
|
||||||
|
if paper['title'] != title:
|
||||||
|
raise ValueError('Conflicting official titles share a forum')
|
||||||
|
event = {'session_id': sid, **sessions[sid]}
|
||||||
|
paper['presentations'].append(event); occurrences.append((fid, sid, kind))
|
||||||
|
counts = Counter(kind for _, _, kind in occurrences)
|
||||||
|
if len(papers) != 687 or len(sessions) != 18 or counts != {'poster': 687, 'oral': 21, 'spotlight': 72}:
|
||||||
|
raise ValueError('Incomplete official program')
|
||||||
|
for paper in papers.values():
|
||||||
|
kinds = Counter(p['kind'] for p in paper['presentations'])
|
||||||
|
if kinds['poster'] != 1 or len(paper['presentations']) not in (1, 2):
|
||||||
|
raise ValueError('Invalid poster/talk multiplicity')
|
||||||
|
return papers
|
||||||
|
|
||||||
|
|
||||||
|
def join_metadata(program, archive):
|
||||||
|
if (archive.get('repository') != 'gisbi-kim/conference-explorers'
|
||||||
|
or archive.get('revision') != REVISION or archive.get('source_url') != ARCHIVE
|
||||||
|
or archive.get('source_data_sha256') != RAW_SHA or archive.get('source_git_blob_sha1') != BLOB
|
||||||
|
or not isinstance(archive.get('papers'), list)):
|
||||||
|
raise ValueError('Unpinned public metadata provenance')
|
||||||
|
indexed = {}
|
||||||
|
for row in archive['papers']:
|
||||||
|
if not isinstance(row, dict) or set(row) - FIELDS:
|
||||||
|
raise ValueError('Unexpected metadata/profile fields')
|
||||||
|
fid = forum_id(row.get('openreview'))
|
||||||
|
if fid != row.get('id') or fid in indexed:
|
||||||
|
raise ValueError('Repeated or mismatched archive forum')
|
||||||
|
if any(forum_id(row.get(key)) != fid for key in ('metadata_source', 'author_source', 'abstract_source')):
|
||||||
|
raise ValueError('Field source belongs to another submission')
|
||||||
|
if (not isinstance(row.get('title'), str) or not row['title'].strip()
|
||||||
|
or not isinstance(row.get('abstract'), str) or not row['abstract'].strip()
|
||||||
|
or not isinstance(row.get('authors'), list) or not row['authors']
|
||||||
|
or any(not isinstance(a, str) or not a.strip() or '@' in a or 'anonymous' in a.casefold() for a in row['authors'])):
|
||||||
|
raise ValueError('Incomplete scholarly metadata')
|
||||||
|
indexed[fid] = row
|
||||||
|
if set(indexed) != set(program):
|
||||||
|
raise ValueError('Official and archive forum sets differ')
|
||||||
|
return indexed
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, official_snapshots, archive_snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
official_root = Path(official_snapshots); archive_root = Path(archive_snapshots)
|
||||||
|
official = official_root/'accepted.html'; filtered = archive_root/'explorer-paper-metadata-filtered-v1.json'
|
||||||
|
if official.is_symlink() or filtered.is_symlink():
|
||||||
|
raise ValueError('Snapshot symlinks rejected')
|
||||||
|
raw = official.read_bytes(); data = filtered.read_bytes()
|
||||||
|
if len(raw) > 8*1024**2 or len(data) > 8*1024**2 or digest(raw) != OFFICIAL_SHA or digest(data) != FILTERED_SHA:
|
||||||
|
raise ValueError('Reviewed snapshot hash changed')
|
||||||
|
receipts = json.loads((official_root/'research.json').read_bytes())['receipts']
|
||||||
|
matches = [r for r in receipts if r.get('url') == OFFICIAL]
|
||||||
|
if len(matches) != 1:
|
||||||
|
raise ValueError('Missing official source receipt')
|
||||||
|
receipt = matches[0]; observed = receipt['at']
|
||||||
|
if (receipt.get('final_url') != OFFICIAL or receipt['http_status'] != 200
|
||||||
|
or receipt['sha256'] != digest(raw) or receipt['bytes'] != len(raw)
|
||||||
|
or datetime.fromisoformat(observed.replace('Z', '+00:00')).utcoffset() is None):
|
||||||
|
raise ValueError('Invalid official receipt')
|
||||||
|
program = parse_program(raw.decode('utf-8')); archive = json.loads(data); indexed = join_metadata(program, archive)
|
||||||
|
evidence = out/'evidence'; evidence.mkdir()
|
||||||
|
sources = {}
|
||||||
|
for body, suffix, url, name in [(raw, 'html', OFFICIAL, 'program'), (data, 'json', ARCHIVE, 'archive')]:
|
||||||
|
rel = f'evidence/{digest(body)}.{suffix}'; (out/rel).write_bytes(body)
|
||||||
|
sources[name] = {'file': rel, 'sha256': digest(body), 'url': url}
|
||||||
|
rows = []; variants = []
|
||||||
|
for fid in sorted(program):
|
||||||
|
p = program[fid]; a = indexed[fid]; variant = comparable(p['title']) != comparable(a['title'])
|
||||||
|
if variant: variants.append({'forum': fid, 'official_title': p['title'], 'archive_title': a['title']})
|
||||||
|
rows.append({'source': SOURCE, 'external_id': f'2026/{fid}', 'url': a['openreview'],
|
||||||
|
'title': a['title'], 'authors': a['authors'], 'abstract': a['abstract'],
|
||||||
|
'identifiers': {'openreview': fid}, 'submission_identifiers': [{'scheme': 'openreview', 'value': fid}],
|
||||||
|
'venue': 'CoRL', 'year': 2026, 'track': 'main', 'status': 'accepted',
|
||||||
|
'raw_decision': 'listed in official CoRL 2026 accepted program', 'evidence_url': OFFICIAL,
|
||||||
|
'observed_at': observed, 'tags': [], 'metadata': {
|
||||||
|
'official_program': p, 'archive_fields': a, 'title_variant_requires_review': variant,
|
||||||
|
'source_evidence': sources, 'archive_revision': REVISION,
|
||||||
|
'field_provenance': {k: a[k] for k in ('metadata_source', 'author_source', 'abstract_source')},
|
||||||
|
'original_decision_signature_verified': False,
|
||||||
|
'scope': 'Acceptance from official forum listing; title/authors/abstract are curator snapshot versions, not live original metadata. arXiv links remain unapproved candidates. No reject inference or person aliases.'}})
|
||||||
|
if len(variants) != 34:
|
||||||
|
raise ValueError('Reviewed title-variant census changed')
|
||||||
|
path = out/'corl-program-archive-2026-main.jsonl'
|
||||||
|
payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
|
||||||
|
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'CoRL', 'year': 2026, 'track': 'main',
|
||||||
|
'count': len(rows), 'file': str(path), 'file_sha256': digest(payload), 'list_url': OFFICIAL,
|
||||||
|
'status': 'partial', 'missing_abstract_count': 0}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
|
||||||
|
report = {'status': 'collected_pending_identity_review', 'records': len(rows), 'presentations': 780,
|
||||||
|
'files': [manifest], 'title_variants': variants, 'source_evidence': sources,
|
||||||
|
'database_written': False, 'pdfs_downloaded': False, 'public_rejected_availability': 'unknown',
|
||||||
|
'scope': 'Source collection only; requires identity and metadata-version review before import; no exhaustive deduplication or rejected-set claim.'}
|
||||||
|
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
|
parser.add_argument('--out', type=Path, required=True)
|
||||||
|
parser.add_argument('--official-snapshots', type=Path, required=True)
|
||||||
|
parser.add_argument('--archive-snapshots', type=Path, required=True)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
report = collect(args.out, official_snapshots=args.official_snapshots, archive_snapshots=args.archive_snapshots)
|
||||||
|
print(json.dumps({'status': report['status'], 'records': report['records'], 'database_written': False}))
|
||||||
@@ -20,7 +20,7 @@ DTD_MD5='96e4283e71886dec2173718957293648'
|
|||||||
DTD_NAMES={'dblp.dtd','dblp-2023-06-28.dtd'}
|
DTD_NAMES={'dblp.dtd','dblp-2023-06-28.dtd'}
|
||||||
RELEASE='https://doi.org/10.4230/dblp.xml.2026-11-01'
|
RELEASE='https://doi.org/10.4230/dblp.xml.2026-11-01'
|
||||||
# Broad discovery candidates, including historical names; not a venue mapping.
|
# Broad discovery candidates, including historical names; not a venue mapping.
|
||||||
FAMILIES=set('nips icml iclr colm colt aistats uai clear acl emnlp naacl eacl coling conll cvpr iccv eccv aaai ijcai ecai kdd sigir www wsdm recsys cikm icdm sdm ecml pkdd ecmlpkdd icra iros rss corl aips icaps ecp ewsp kr atal aamas icassp interspeech eurospeech icslp ismir miccai fat facct lrec mm mlsys'.split())
|
FAMILIES=set('nips icml iclr colm colt aistats uai clear acl emnlp naacl eacl coling conll cvpr iccv eccv aaai ijcai ecai kdd sigir www wsdm recsys cikm icdm sdm ecml pkdd ecmlpkdd icra iros rss corl aips icaps ecp ewsp kr atal aamas ifaamas icassp interspeech eurospeech icslp ismir miccai fat facct lrec mm mlsys'.split())
|
||||||
|
|
||||||
|
|
||||||
def records(archive,dtd_bytes):
|
def records(archive,dtd_bytes):
|
||||||
|
|||||||
@@ -0,0 +1,156 @@
|
|||||||
|
"""Recover raw family candidates from a pinned snapshot and its existing catalogue.
|
||||||
|
|
||||||
|
The old candidate file is not an exhaustive source: earlier discovery filters
|
||||||
|
can omit an entire namespace. This command checks the complete snapshot and
|
||||||
|
catalogue, without assigning venues, tracks, acceptance or paper identities.
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from contextlib import closing
|
||||||
|
import fcntl
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import sqlite3
|
||||||
|
import tempfile
|
||||||
|
|
||||||
|
from paper_atlas.collectors.dblp import DTD_MD5, FAMILIES, RELEASE, XML_SHA, family, records
|
||||||
|
from paper_atlas.db.submissions import timestamp
|
||||||
|
from paper_atlas.io.snapshots import digest_file
|
||||||
|
|
||||||
|
|
||||||
|
def catalogue_evidence(catalogue, families, expected_sha, expected_dtd_md5):
|
||||||
|
manifest_path = catalogue.with_name('dblp-ai-candidates.manifest.json')
|
||||||
|
manifest = json.loads(manifest_path.read_text())
|
||||||
|
if (manifest.get('snapshot_sha256') != expected_sha
|
||||||
|
or manifest.get('dtd_md5') != expected_dtd_md5):
|
||||||
|
raise ValueError('Catalogue belongs to a different snapshot or DTD')
|
||||||
|
totals = manifest.get('record_types', {})
|
||||||
|
if (set(totals) != {'proceedings', 'inproceedings'}
|
||||||
|
or any(type(n) is not int or n < 0 for n in totals.values())
|
||||||
|
or type(manifest.get('records_scanned')) is not int
|
||||||
|
or sum(totals.values()) != manifest['records_scanned']):
|
||||||
|
raise ValueError('Incomplete catalogue record totals')
|
||||||
|
counts = Counter()
|
||||||
|
volumes = {}
|
||||||
|
# The catalogue is evidence, never a writable recovery destination.
|
||||||
|
with closing(sqlite3.connect(catalogue.resolve().as_uri() + '?mode=ro', uri=True)) as c:
|
||||||
|
for name in families:
|
||||||
|
found = c.execute('SELECT record_type,count FROM family_counts WHERE family=?',
|
||||||
|
(name,)).fetchall()
|
||||||
|
if not found or not any(n for _, n in found):
|
||||||
|
raise ValueError('Family absent from complete catalogue: ' + name)
|
||||||
|
for kind, count in found:
|
||||||
|
if kind not in totals or type(count) is not int or count < 0:
|
||||||
|
raise ValueError('Invalid catalogue family count')
|
||||||
|
counts[(name, kind)] = count
|
||||||
|
for key, raw in c.execute('SELECT key,metadata_json FROM proceedings WHERE family=?',
|
||||||
|
(name,)):
|
||||||
|
row = json.loads(raw)
|
||||||
|
if (family(key) != name or row.get('key') != key
|
||||||
|
or row.get('record_type') != 'proceedings'):
|
||||||
|
raise ValueError('Invalid catalogue proceedings evidence')
|
||||||
|
volumes[key] = row
|
||||||
|
if len(volumes) != sum(counts[(name, 'proceedings')] for name in families):
|
||||||
|
raise ValueError('Catalogue proceedings count differs from volume records')
|
||||||
|
return manifest, digest_file(manifest_path)['sha256'], counts, volumes
|
||||||
|
|
||||||
|
|
||||||
|
def recover(archive, dtd, catalogue, out, families, *, expected_sha=XML_SHA,
|
||||||
|
expected_dtd_md5=DTD_MD5):
|
||||||
|
families = list(families)
|
||||||
|
if (not families or any(not isinstance(name, str)
|
||||||
|
or not re.fullmatch(r'[a-z][a-z0-9-]*', name)
|
||||||
|
or name not in FAMILIES for name in families)):
|
||||||
|
raise ValueError('Select an explicitly supported discovery family')
|
||||||
|
families = sorted(set(families))
|
||||||
|
archive, dtd, catalogue = Path(archive), Path(dtd), Path(catalogue)
|
||||||
|
output = Path(out).absolute()
|
||||||
|
if output.exists() or output.is_symlink():
|
||||||
|
raise ValueError('Recovery output must not already exist')
|
||||||
|
output.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
lock_path = output.parent / ('.' + output.name + '.recovery.lock')
|
||||||
|
with lock_path.open('a') as lock:
|
||||||
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||||
|
if output.exists() or output.is_symlink():
|
||||||
|
raise ValueError('Recovery output appeared before scan')
|
||||||
|
actual = digest_file(archive)
|
||||||
|
if actual['sha256'] != expected_sha:
|
||||||
|
raise ValueError('DBLP snapshot checksum mismatch')
|
||||||
|
if dtd.stat().st_size > 128 * 1024 or digest_file(dtd)['md5'] != expected_dtd_md5:
|
||||||
|
raise ValueError('DBLP DTD checksum mismatch or size limit')
|
||||||
|
manifest, manifest_sha, expected, volumes = catalogue_evidence(
|
||||||
|
catalogue, families, expected_sha, expected_dtd_md5)
|
||||||
|
totals, found = Counter(), Counter()
|
||||||
|
seen_keys, found_volumes = set(), set()
|
||||||
|
with tempfile.TemporaryDirectory(prefix='.dblp-family-', dir=output.parent) as tmp:
|
||||||
|
stage = Path(tmp) / 'recovered'
|
||||||
|
stage.mkdir()
|
||||||
|
target = stage / 'dblp-family-candidates.jsonl'
|
||||||
|
with target.open('w') as stream:
|
||||||
|
for row in records(archive, dtd.read_bytes()):
|
||||||
|
totals[row['record_type']] += 1
|
||||||
|
name = family(row['key'])
|
||||||
|
if name in families:
|
||||||
|
if row['key'] in seen_keys:
|
||||||
|
raise ValueError('Duplicate raw bibliographic key')
|
||||||
|
seen_keys.add(row['key'])
|
||||||
|
found[(name, row['record_type'])] += 1
|
||||||
|
if row['record_type'] == 'proceedings':
|
||||||
|
if volumes.get(row['key']) != row:
|
||||||
|
raise ValueError('Raw proceedings differs from catalogue evidence')
|
||||||
|
found_volumes.add(row['key'])
|
||||||
|
stream.write(json.dumps(row, ensure_ascii=False) + '\n')
|
||||||
|
scanned = sum(totals.values())
|
||||||
|
if scanned % 250000 == 0:
|
||||||
|
print(json.dumps({'status': 'scanning_raw_family_candidates',
|
||||||
|
'records_scanned': scanned,
|
||||||
|
'recovered_records': sum(found.values())}), flush=True)
|
||||||
|
if (totals != Counter(manifest['record_types'])
|
||||||
|
or sum(totals.values()) != manifest['records_scanned']):
|
||||||
|
raise ValueError('Complete snapshot totals differ from catalogue')
|
||||||
|
if found != expected or found_volumes != volumes.keys():
|
||||||
|
raise ValueError('Recovered family totals differ from catalogue')
|
||||||
|
report = {
|
||||||
|
'format_version': 1, 'source': 'dblp',
|
||||||
|
'status': 'recovered_raw_candidates_pending_scope_review',
|
||||||
|
'finished_at': timestamp(), 'source_release': RELEASE,
|
||||||
|
'snapshot_sha256': actual['sha256'], 'dtd_md5': expected_dtd_md5,
|
||||||
|
'catalogue_manifest_sha256': manifest_sha, 'families': families,
|
||||||
|
'records_scanned': sum(totals.values()), 'record_types': dict(totals),
|
||||||
|
'recovered_records': sum(found.values()),
|
||||||
|
'recovered_publications': sum(found[(n, 'inproceedings')] for n in families),
|
||||||
|
'recovered_volumes': len(found_volumes),
|
||||||
|
'family_counts': [{'family': name, 'record_type': kind, 'count': count}
|
||||||
|
for (name, kind), count in sorted(found.items())],
|
||||||
|
'old_candidate_family_counts': {
|
||||||
|
name: manifest.get('candidate_family_counts', {}).get(name, 0)
|
||||||
|
for name in families},
|
||||||
|
'catalogue_volumes': [volumes[k] for k in sorted(volumes)],
|
||||||
|
'file': str(output / target.name),
|
||||||
|
'file_sha256': digest_file(target)['sha256'], 'database_written': False,
|
||||||
|
'note': 'Raw bibliographic candidates, including volume records and possible '
|
||||||
|
'keynotes/workshops. No venue, track, acceptance or work identity '
|
||||||
|
'inferred. Requires primary-source scope and identity review; '
|
||||||
|
'not a normalized import queue. Existing imported sources must '
|
||||||
|
'not be replayed.',
|
||||||
|
}
|
||||||
|
# Deliberately not an import-records sidecar manifest.
|
||||||
|
(stage / 'recovery-manifest.json').write_text(
|
||||||
|
json.dumps(report, ensure_ascii=False, indent=2) + '\n')
|
||||||
|
if output.exists() or output.is_symlink():
|
||||||
|
raise ValueError('Recovery output appeared during scan')
|
||||||
|
stage.rename(output)
|
||||||
|
return report
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
|
parser.add_argument('--archive', required=True, type=Path)
|
||||||
|
parser.add_argument('--dtd', required=True, type=Path)
|
||||||
|
parser.add_argument('--catalogue', required=True, type=Path)
|
||||||
|
parser.add_argument('--family', required=True, action='append', choices=sorted(FAMILIES))
|
||||||
|
parser.add_argument('--out', required=True, type=Path)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
print(json.dumps(recover(args.archive, args.dtd, args.catalogue, args.out, args.family),
|
||||||
|
ensure_ascii=False))
|
||||||
@@ -0,0 +1,254 @@
|
|||||||
|
"""ECCV 2026 frozen preliminary program and resumable public metadata pages."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import fcntl
|
||||||
|
from html import unescape
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, Node, clean
|
||||||
|
from paper_atlas.io.html_cache import PublicHTMLCache, checked_html, digest
|
||||||
|
|
||||||
|
SOURCE = 'eccv_program'
|
||||||
|
ORIGIN = 'https://eccv.ecva.net'
|
||||||
|
ACCEPTED = ORIGIN+'/Conferences/2026/AcceptedPapers'
|
||||||
|
TRACK = 'program_unclassified'
|
||||||
|
MANIFEST_SHA = '04f9734783bbbfa2e19f936d8f5a7aecfb524b0ceef33fc5d9fb6276c7b6678a'
|
||||||
|
PAGES = {'accepted.html': ACCEPTED, 'policies.html': ORIGIN+'/Conferences/2026/SubmissionPolicies',
|
||||||
|
'faq.html': ORIGIN+'/Conferences/2026/FAQs'}
|
||||||
|
|
||||||
|
|
||||||
|
def one(items):
|
||||||
|
if len(items) != 1:
|
||||||
|
raise ValueError('Expected one reviewed metadata field')
|
||||||
|
return items[0]
|
||||||
|
|
||||||
|
|
||||||
|
def has_class(node, value):
|
||||||
|
return value in node.attrs.get('class', '').split()
|
||||||
|
|
||||||
|
|
||||||
|
def parse_program(raw):
|
||||||
|
nodes = list(Document(raw).root.walk())
|
||||||
|
if len([n for n in nodes if n.tag == 'h2' and clean(n.text()) == 'ECCV 2026 Accepted Papers:']) != 1:
|
||||||
|
raise ValueError('Unexpected ECCV program edition')
|
||||||
|
if 'This paper list is preliminary pending publisher checks.' not in clean(Document(raw).root.text()):
|
||||||
|
raise ValueError('Reviewed provisional-publication notice changed')
|
||||||
|
table = one([n for n in nodes if n.tag == 'table' and n.attrs.get('id') == 'event-list-2026-poster-nodates-filter-vslinks-table'])
|
||||||
|
papers = {}; occurrences = 0
|
||||||
|
for tr in (n for n in table.walk() if n.tag == 'tr'):
|
||||||
|
cells = [n for n in tr.children if isinstance(n, Node) and n.tag == 'td']
|
||||||
|
if not cells:
|
||||||
|
continue
|
||||||
|
if len(cells) != 3:
|
||||||
|
raise ValueError('Unexpected program columns')
|
||||||
|
links = [n for n in cells[0].walk() if n.tag == 'a' and re.fullmatch(r'/virtual/2026/poster/[1-9][0-9]*', n.attrs.get('href', ''))]
|
||||||
|
link = one(links); pid = link.attrs['href'].rsplit('/', 1)[1]
|
||||||
|
authors = clean(one([n for n in cells[0].walk() if n.tag == 'i']).text())
|
||||||
|
row = {'title': clean(link.text()), 'authors_literal': authors,
|
||||||
|
'keywords_literal': clean(cells[1].text()), 'presentation_literal': clean(cells[2].text())}
|
||||||
|
if not row['title'] or not authors:
|
||||||
|
raise ValueError('Incomplete public program fields')
|
||||||
|
occurrences += 1
|
||||||
|
if pid in papers:
|
||||||
|
if papers[pid]['listing'] != row:
|
||||||
|
raise ValueError('Conflicting entries share a program ID')
|
||||||
|
papers[pid]['row_ordinals'].append(occurrences)
|
||||||
|
else:
|
||||||
|
papers[pid] = {'url': ORIGIN+link.attrs['href'], 'listing': row, 'row_ordinals': [occurrences]}
|
||||||
|
if occurrences != 2851 or len(papers) != 2834 or Counter(len(v['row_ordinals']) for v in papers.values()) != {1: 2817, 2: 17}:
|
||||||
|
raise ValueError('Incomplete reviewed program census')
|
||||||
|
return papers
|
||||||
|
|
||||||
|
|
||||||
|
def parse_paper(raw, url, program):
|
||||||
|
nodes = list(Document(raw).root.walk())
|
||||||
|
title = clean(one([n for n in nodes if n.tag == 'h1' and has_class(n, 'event-title')]).text())
|
||||||
|
authors_literal = clean(one([n for n in nodes if n.tag == 'div' and has_class(n, 'event-organizers')]).text())
|
||||||
|
abstract_node = one([n for n in nodes if n.tag == 'div' and has_class(n, 'abstract-text-inner')])
|
||||||
|
abstract = clean(abstract_node.text())
|
||||||
|
markup = sorted({n.tag for n in abstract_node.walk() if n.tag in ('math','sup','sub','svg','img')})
|
||||||
|
canonical = one([n.attrs.get('href') for n in nodes if n.tag == 'link' and n.attrs.get('rel') == 'canonical'])
|
||||||
|
scripts = [n for n in nodes if n.tag == 'script' and n.attrs.get('type') == 'application/ld+json']
|
||||||
|
ld = one([json.loads(''.join(x for x in n.children if isinstance(x, str))) for n in scripts])
|
||||||
|
authors = ld.get('author')
|
||||||
|
if (canonical != url or ld.get('@type') != 'CreativeWork' or ld.get('creditText') != 'ECCV 2026'
|
||||||
|
or not title or not isinstance(ld.get('name'), str) or not abstract
|
||||||
|
or not isinstance(authors, list) or not authors
|
||||||
|
or any(not isinstance(a, dict) or a.get('@type') != 'Person' or not isinstance(a.get('name'), str)
|
||||||
|
or not clean(a['name']) or '@' in a['name'] for a in authors)):
|
||||||
|
raise ValueError('Public visible/structured metadata identity or edition disagrees')
|
||||||
|
# Prefer source text that already agrees. Poster 3844 intentionally displays
|
||||||
|
# literal '&', while 4199 double-encodes an author's apostrophe in JSON-LD.
|
||||||
|
# Decode only to test agreement with visible text, never to rewrite it.
|
||||||
|
def agrees_with_display(value, expected):
|
||||||
|
if clean(value) == expected:
|
||||||
|
return True
|
||||||
|
for _ in range(2):
|
||||||
|
value = unescape(value)
|
||||||
|
if clean(value) == expected:
|
||||||
|
return True
|
||||||
|
return False
|
||||||
|
|
||||||
|
raw_names = [a['name'] for a in authors]
|
||||||
|
names = [clean(name) for name in raw_names]
|
||||||
|
if not agrees_with_display(ld['name'], title):
|
||||||
|
raise ValueError('Public visible and structured titles disagree')
|
||||||
|
if ' ⋅ '.join(names) != authors_literal:
|
||||||
|
visible_names = authors_literal.split(' ⋅ ')
|
||||||
|
if len(visible_names) != len(raw_names) or any(
|
||||||
|
not agrees_with_display(raw_name, visible_name)
|
||||||
|
for raw_name, visible_name in zip(raw_names, visible_names)):
|
||||||
|
raise ValueError('Complete visible and structured authors disagree')
|
||||||
|
names = visible_names
|
||||||
|
if any('@' in name for name in names) or ' ⋅ '.join(names) != authors_literal:
|
||||||
|
raise ValueError('Complete visible and structured authors disagree')
|
||||||
|
flags = ['structured_abstract_markup_requires_review'] if markup else []
|
||||||
|
if raw_names != names or ld['name'] != title:
|
||||||
|
flags.append('jsonld_display_text_normalized_requires_review')
|
||||||
|
if title != program['listing']['title']:
|
||||||
|
flags.append('program_and_page_title_versions_differ_requires_review')
|
||||||
|
if authors_literal != program['listing']['authors_literal']:
|
||||||
|
flags.append('program_and_page_author_versions_differ_requires_review')
|
||||||
|
return {'title': title, 'authors': names, 'authors_literal': authors_literal, 'abstract': abstract,
|
||||||
|
'page_metadata_dates': {k: ld.get(k) for k in ('datePublished', 'dateModified')}, 'review_flags': flags, 'abstract_markup_tags': markup,
|
||||||
|
'jsonld_raw_text': {'name': ld['name'], 'author_names': raw_names}}
|
||||||
|
|
||||||
|
|
||||||
|
def official_snapshot(root):
|
||||||
|
root = Path(root); manifest = root/'manifest.json'
|
||||||
|
if root.is_symlink() or manifest.is_symlink() or digest(manifest.read_bytes()) != MANIFEST_SHA:
|
||||||
|
raise ValueError('Reviewed official snapshot manifest changed')
|
||||||
|
m = json.loads(manifest.read_bytes())
|
||||||
|
if datetime.fromisoformat(m['at'].replace('Z', '+00:00')).utcoffset() is None:
|
||||||
|
raise ValueError('Official snapshot requires a timezone')
|
||||||
|
if len(m['sources']) != 3 or {r['file'] for r in m['sources']} != set(PAGES):
|
||||||
|
raise ValueError('Official source set changed')
|
||||||
|
pages = {}; evidence = {}
|
||||||
|
for r in m['sources']:
|
||||||
|
name = r['file']; p = root/name
|
||||||
|
if p.is_symlink() or p.stat().st_size > 8*1024**2:
|
||||||
|
raise ValueError('Unsafe official snapshot')
|
||||||
|
raw = p.read_bytes()
|
||||||
|
if r['url'] != PAGES[name] or r['final_url'] != r['url'] or r['http_status'] != 200 or r['bytes'] != len(raw) or r['sha256'] != digest(raw):
|
||||||
|
raise ValueError('Invalid official source receipt')
|
||||||
|
pages[name] = raw; evidence[name] = {'url': r['url'], 'sha256': digest(raw)}
|
||||||
|
program = parse_program(pages['accepted.html'].decode('utf-8'))
|
||||||
|
return m, pages, evidence, program
|
||||||
|
|
||||||
|
|
||||||
|
def paper_snapshots(root, program):
|
||||||
|
root = Path(root); path = root/'manifest.json'
|
||||||
|
if root.is_symlink() or path.is_symlink():
|
||||||
|
raise ValueError('Paper snapshot symlinks rejected')
|
||||||
|
if path.stat().st_size > 4*1024**2:
|
||||||
|
raise ValueError('Oversized paper manifest')
|
||||||
|
m = json.loads(path.read_bytes()); result = {}
|
||||||
|
for r in m['records']:
|
||||||
|
pid = r['paper_id']
|
||||||
|
if pid not in program or pid in result or r['file'] != pid+'.html':
|
||||||
|
raise ValueError('Foreign or repeated paper snapshot')
|
||||||
|
p = root/r['file']
|
||||||
|
if p.is_symlink() or p.stat().st_size > 2*1024**2:
|
||||||
|
raise ValueError('Unsafe paper snapshot')
|
||||||
|
receipt = {**r, 'retrieved_at': r.get('retrieved_at', m['at'])}
|
||||||
|
result[pid] = checked_html(p.read_bytes(), receipt, program[pid]['url'], 2*1024**2)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, official_snapshots, paper_snapshots_dir=None, cache=None, max_new_pages=None):
|
||||||
|
if (paper_snapshots_dir is None) == (cache is None):
|
||||||
|
raise ValueError('Choose paper snapshots or public metadata cache')
|
||||||
|
if max_new_pages is not None and (type(max_new_pages) is not int or max_new_pages < 0 or cache is None):
|
||||||
|
raise ValueError('New-page budget requires cache mode and a nonnegative integer')
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
lock = None
|
||||||
|
try:
|
||||||
|
official, pages, source_evidence, program = official_snapshot(official_snapshots)
|
||||||
|
directory = out/'evidence'; directory.mkdir()
|
||||||
|
for name, raw in pages.items():
|
||||||
|
rel = 'evidence/'+digest(raw)+'.html'; (out/rel).write_bytes(raw); source_evidence[name]['file'] = rel
|
||||||
|
snapshots = paper_snapshots(paper_snapshots_dir, program) if paper_snapshots_dir is not None else None
|
||||||
|
client = None
|
||||||
|
if cache is not None:
|
||||||
|
cache = Path(cache); cache.mkdir(parents=True, exist_ok=True)
|
||||||
|
if cache.is_symlink() or (cache/'.collector.lock').is_symlink():
|
||||||
|
raise ValueError('Cache lock symlinks rejected')
|
||||||
|
lock = (cache/'.collector.lock').open('a+')
|
||||||
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||||
|
client = PublicHTMLCache(cache, ORIGIN)
|
||||||
|
rows = []; missing = []; fetched = 0; reused = 0
|
||||||
|
for pid in sorted(program, key=int):
|
||||||
|
p = program[pid]
|
||||||
|
if snapshots is not None:
|
||||||
|
response = snapshots.get(pid)
|
||||||
|
else:
|
||||||
|
response = client.cached(p['url'])
|
||||||
|
if response is not None:
|
||||||
|
reused += 1
|
||||||
|
elif max_new_pages is None or fetched < max_new_pages:
|
||||||
|
response = client.get(p['url']); fetched += 1
|
||||||
|
if response is None:
|
||||||
|
missing.append(pid); continue
|
||||||
|
raw, receipt = response; fields = parse_paper(raw.decode('utf-8'), p['url'], p)
|
||||||
|
rel = 'evidence/'+digest(raw)+'.html'; (out/rel).write_bytes(raw)
|
||||||
|
keyword = p['listing']['keywords_literal']
|
||||||
|
rows.append({'source': SOURCE, 'external_id': '2026/'+pid, 'url': p['url'],
|
||||||
|
'title': fields['title'], 'authors': fields['authors'], 'abstract': fields['abstract'],
|
||||||
|
'identifiers': {}, 'submission_identifiers': [], 'venue': 'ECCV', 'year': 2026,
|
||||||
|
'track': TRACK, 'status': 'accepted', 'observed_at': official['at'],
|
||||||
|
'raw_decision': 'listed in frozen preliminary ECCV 2026 accepted program, pending publisher checks',
|
||||||
|
'evidence_url': ACCEPTED,
|
||||||
|
'tags': [{'namespace': SOURCE+':keyword', 'value': keyword}] if keyword else [],
|
||||||
|
'metadata': {'official_program': p, 'public_page_fields': fields,
|
||||||
|
'public_page_receipt': receipt, 'public_page_evidence': {'file': rel, 'sha256': digest(raw), 'url': p['url']},
|
||||||
|
'program_source_evidence': source_evidence, 'publication_state': 'preliminary_pending_publisher_checks',
|
||||||
|
'original_decision_signature_verified': False,
|
||||||
|
'program_id_semantics': 'Official program locator, not a verified original submission number or cross-platform identifier.',
|
||||||
|
'abstract_quality': 'Public display literal retained, including broken hyphenation/missing spaces; no automatic text repair.',
|
||||||
|
'date_semantics': 'JSON-LD dates describe the website entry, not verified paper publication or decision dates.',
|
||||||
|
'label_semantics': 'Official keyword namespace independent of track, contribution/research type and custom labels.',
|
||||||
|
'scope': 'Public metadata only; no protected live content, reviews, person profiles or PDFs. Requires work identity, metadata version and complete history review before import.'}})
|
||||||
|
if len(rows) % 10 == 0:
|
||||||
|
print(json.dumps({'phase': 'public_metadata', 'records': len(rows), 'new_pages': fetched,
|
||||||
|
'cached_pages': reused, 'expected_program_ids': 2834, 'at': datetime.now(timezone.utc).isoformat()}), flush=True)
|
||||||
|
files = []
|
||||||
|
if rows:
|
||||||
|
path = out/'eccv-program-2026-unclassified.jsonl'; payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
|
||||||
|
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ECCV', 'year': 2026, 'track': TRACK,
|
||||||
|
'count': len(rows), 'file': str(path), 'file_sha256': digest(payload), 'list_url': ACCEPTED,
|
||||||
|
'status': 'partial', 'missing_abstract_count': 0, 'source_expected_count': 2834}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n'); files.append(manifest)
|
||||||
|
report = {'status': 'collected_pending_identity_review' if rows else 'no_public_metadata_collected',
|
||||||
|
'records': len(rows), 'files': files, 'expected_program_ids': 2834, 'program_occurrences': 2851,
|
||||||
|
'duplicate_program_occurrences': 17, 'uncollected_program_ids': missing,
|
||||||
|
'new_pages': fetched, 'reused_pages': reused, 'database_written': False,
|
||||||
|
'public_review_visibility': 'not_public_per_2026_policy', 'public_rejected_decision_metadata': 'unknown',
|
||||||
|
'scope': 'Always partial: preliminary program, unresolved track/identity/version coverage; not final whole-database deduplication.'}
|
||||||
|
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__,
|
||||||
|
'database_written': False, 'scope': 'Preserve this output and verified cache; resume with a new output directory after inspection.'})+'\n')
|
||||||
|
raise
|
||||||
|
finally:
|
||||||
|
if lock is not None:
|
||||||
|
lock.close()
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
|
parser.add_argument('--official-snapshots', required=True)
|
||||||
|
group = parser.add_mutually_exclusive_group(required=True)
|
||||||
|
group.add_argument('--paper-snapshots')
|
||||||
|
group.add_argument('--cache')
|
||||||
|
parser.add_argument('--max-new-pages', type=int)
|
||||||
|
parser.add_argument('--out', required=True)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
print(json.dumps(collect(args.out, official_snapshots=args.official_snapshots,
|
||||||
|
paper_snapshots_dir=args.paper_snapshots, cache=args.cache, max_new_pages=args.max_new_pages), ensure_ascii=False))
|
||||||
@@ -0,0 +1,155 @@
|
|||||||
|
"""ICAPS 2026 frozen official listing, with unresolved track membership explicit."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, Node, clean
|
||||||
|
|
||||||
|
SOURCE = 'icaps_official_program'
|
||||||
|
ROOT = 'https://icaps26.icaps-conference.org/'
|
||||||
|
PAGES = {
|
||||||
|
'accepted.html': (ROOT+'program/accepted/', '235543c3e5917ecb3e10a1429a4754f9cb7fae9844b54404076254584da86443'),
|
||||||
|
'cfp.html': (ROOT+'calls/cfp/', 'bdcf5a5b3b4144d6c6ecf3e324c856363f4f4e992e592c1f63988980434ecd5d'),
|
||||||
|
'previously-published.html': (ROOT+'calls/previously_published_research/', 'ecd60c8c056c9aa73bac16eb11f220c756ddff562511f0ec42ada41919fb1946'),
|
||||||
|
}
|
||||||
|
MAIN = 'Papers accepted to the Main Tracks'
|
||||||
|
PREVIOUS = 'Previously Published Papers'
|
||||||
|
TRACK = 'main_tracks_unclassified'
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def authors_from_literal(literal):
|
||||||
|
# This frozen listing uses commas between authors and a final "and".
|
||||||
|
# Preserve the complete literal too; never infer person-name aliases.
|
||||||
|
if literal.count(' and ') != 1 or ', and ' in literal:
|
||||||
|
raise ValueError('Unreviewed author-list punctuation')
|
||||||
|
first, last = literal.rsplit(' and ', 1)
|
||||||
|
authors = first.split(', ') + [last]
|
||||||
|
if any(not a or a != clean(a) or ',' in a or '@' in a or 'anonymous' in a.casefold() for a in authors):
|
||||||
|
raise ValueError('Incomplete or ambiguous author list')
|
||||||
|
if ', '.join(authors[:-1])+' and '+authors[-1] != literal:
|
||||||
|
raise ValueError('Author literal cannot be reconstructed')
|
||||||
|
return authors
|
||||||
|
|
||||||
|
|
||||||
|
def parse_listing(raw):
|
||||||
|
nodes = list(Document(raw).root.walk())
|
||||||
|
headings = [clean(n.text()) for n in nodes if n.tag == 'h1']
|
||||||
|
if headings != ['List of Papers Accepted to ICAPS 2026']:
|
||||||
|
raise ValueError('Unexpected official conference edition')
|
||||||
|
section = None; sections = []; rows = []; seen = set()
|
||||||
|
for n in nodes:
|
||||||
|
if n.tag == 'h2':
|
||||||
|
section = clean(n.text()); sections.append(section)
|
||||||
|
if n.tag != 'div' or 'paper' not in n.attrs.get('class', '').split():
|
||||||
|
continue
|
||||||
|
if section not in (MAIN, PREVIOUS):
|
||||||
|
raise ValueError('Paper belongs to an unreviewed section')
|
||||||
|
# Require the reviewed title and complete-author blocks; fail on layout drift.
|
||||||
|
blocks = [b for b in n.children if isinstance(b, Node)]
|
||||||
|
if len(blocks) != 2 or any(b.tag != 'div' for b in blocks):
|
||||||
|
raise ValueError('Changed paper metadata layout')
|
||||||
|
titles = [b for b in blocks[0].walk() if b.tag == 'strong']
|
||||||
|
if len(titles) != 1 or any(b.tag == 'a' for b in n.walk()):
|
||||||
|
raise ValueError('Changed paper title/link layout')
|
||||||
|
title = clean(titles[0].text()); literal = clean(blocks[1].text())
|
||||||
|
if not title or clean(blocks[0].text()) != title or title.casefold() in seen:
|
||||||
|
raise ValueError('Empty or repeated paper title')
|
||||||
|
seen.add(title.casefold())
|
||||||
|
rows.append({'section': section, 'title': title, 'authors': authors_from_literal(literal),
|
||||||
|
'authors_literal': literal})
|
||||||
|
if sections != [MAIN, PREVIOUS] or Counter(r['section'] for r in rows) != {MAIN: 90, PREVIOUS: 10}:
|
||||||
|
raise ValueError('Incomplete reviewed official listing')
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
root = Path(snapshots)
|
||||||
|
manifest_path = root/'manifest.json'
|
||||||
|
if manifest_path.is_symlink():
|
||||||
|
raise ValueError('Snapshot symlinks rejected')
|
||||||
|
receipt = json.loads(manifest_path.read_bytes())
|
||||||
|
observed = receipt['at']
|
||||||
|
if (receipt.get('status') != 'collected_not_parsed_not_imported'
|
||||||
|
or receipt.get('database_written') is not False
|
||||||
|
or datetime.fromisoformat(observed.replace('Z', '+00:00')).utcoffset() is None):
|
||||||
|
raise ValueError('Invalid reviewed snapshot receipt')
|
||||||
|
receipts = receipt['sources']
|
||||||
|
if len(receipts) != 3 or {r['file'] for r in receipts} != set(PAGES):
|
||||||
|
raise ValueError('Unexpected snapshot source set')
|
||||||
|
bodies = {}; evidence = {}; directory = out/'evidence'; directory.mkdir()
|
||||||
|
for name, (url, expected) in PAGES.items():
|
||||||
|
matches = [r for r in receipts if r['file'] == name]
|
||||||
|
if len(matches) != 1:
|
||||||
|
raise ValueError('Repeated snapshot receipt')
|
||||||
|
r = matches[0]; p = root/name
|
||||||
|
if p.is_symlink() or p.stat().st_size > 2*1024**2:
|
||||||
|
raise ValueError('Unsafe or oversized snapshot')
|
||||||
|
raw = p.read_bytes()
|
||||||
|
if digest(raw) != expected or r['sha256'] != expected or r['bytes'] != len(raw):
|
||||||
|
raise ValueError('Reviewed snapshot hash changed')
|
||||||
|
if r['url'] != url or r['final_url'] != url:
|
||||||
|
raise ValueError('Unexpected snapshot URL or redirect')
|
||||||
|
rel = f'evidence/{expected}.html'; (out/rel).write_bytes(raw)
|
||||||
|
bodies[name] = raw.decode('utf-8')
|
||||||
|
evidence[name] = {'url': url, 'file': rel, 'sha256': expected}
|
||||||
|
entries = parse_listing(bodies['accepted.html']); rows = []; previous = []
|
||||||
|
for entry in entries:
|
||||||
|
entry_key = digest(json.dumps([entry['title'], entry['authors_literal']], ensure_ascii=False, separators=(',', ':')).encode())
|
||||||
|
if entry['section'] == PREVIOUS:
|
||||||
|
previous.append({**entry, 'entry_key': entry_key,
|
||||||
|
'presentation_kind': 'previously_published_research',
|
||||||
|
'evidence_url': PAGES['accepted.html'][0], 'observed_at': observed,
|
||||||
|
'scope': 'Official presentation listing, not evidence of new archival publication or a new research-paper acceptance.'})
|
||||||
|
continue
|
||||||
|
rows.append({'source': SOURCE, 'external_id': '2026/main-tracks/'+entry_key,
|
||||||
|
'url': PAGES['accepted.html'][0], 'title': entry['title'], 'authors': entry['authors'],
|
||||||
|
'abstract': None, 'identifiers': {}, 'submission_identifiers': [],
|
||||||
|
'venue': 'ICAPS', 'year': 2026, 'track': TRACK, 'status': 'accepted',
|
||||||
|
'raw_decision': 'listed in official ICAPS 2026 Papers accepted to the Main Tracks',
|
||||||
|
'evidence_url': PAGES['accepted.html'][0], 'observed_at': observed, 'tags': [],
|
||||||
|
'metadata': {'official_listing': entry, 'source_evidence': evidence,
|
||||||
|
'source_entry_key': {'algorithm': 'sha256-json-title-and-literal-authors', 'value': entry_key,
|
||||||
|
'scope': 'Snapshot entry locator only; not an original submission ID, DOI, or cross-source work identity.'},
|
||||||
|
'track_assignment': 'Unresolved among main and four special tracks; do not relabel all records as main.',
|
||||||
|
'abstract_availability': 'not_on_official_listing',
|
||||||
|
'original_decision_signature_verified': False,
|
||||||
|
'scope': 'Official listing evidence only; requires current identity review before import. No absence-based rejection or new-paper inference.'}})
|
||||||
|
path = out/'icaps-official-program-2026-main-tracks-unclassified.jsonl'
|
||||||
|
payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
|
||||||
|
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICAPS', 'year': 2026, 'track': TRACK,
|
||||||
|
'count': 90, 'file': str(path), 'file_sha256': digest(payload), 'list_url': PAGES['accepted.html'][0],
|
||||||
|
'status': 'partial', 'missing_abstract_count': 90}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
|
||||||
|
sidecar = out/'previously-published-presentations.json'
|
||||||
|
sidecar.write_text(json.dumps({'records': previous, 'count': 10, 'ingestion_ready': False,
|
||||||
|
'scope': 'Presentation evidence retained separately; not emitted as new accepted research-paper records.'}, ensure_ascii=False, indent=2)+'\n')
|
||||||
|
report = {'status': 'collected_pending_identity_review', 'records': 90, 'previously_published_presentations': 10,
|
||||||
|
'files': [manifest], 'previously_published_file': sidecar.name, 'previously_published_sha256': digest(sidecar.read_bytes()),
|
||||||
|
'source_evidence': evidence, 'database_written': False, 'pdfs_downloaded': False,
|
||||||
|
'public_rejected_availability': 'unknown', 'missing_abstract_count': 90,
|
||||||
|
'scope': 'Partial official list; individual tracks and stable identities unresolved; no all-paper, public-negative or final-dedup claim.'}
|
||||||
|
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
|
parser.add_argument('--snapshots', required=True)
|
||||||
|
parser.add_argument('--out', required=True)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
|
||||||
@@ -0,0 +1,131 @@
|
|||||||
|
"""Frozen Crossref ICAPS volume 36 metadata, kept separate from program versions."""
|
||||||
|
import argparse
|
||||||
|
from datetime import datetime, date
|
||||||
|
import hashlib
|
||||||
|
from html import unescape
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
from paper_atlas.io.html import clean
|
||||||
|
|
||||||
|
SOURCE = 'crossref_icaps_registry'
|
||||||
|
SNAPSHOT_SHA = 'c7480a09f71680d457a16cc1eb35a479742b7510eaad5f236ce54c96e8a29a82'
|
||||||
|
RAW_RESPONSE_SHA = 'ac5268333b1ecdfa2e385680e2e02b5f6639939c3193738d213e66bb53e42a54'
|
||||||
|
URL = 'https://api.crossref.org/journals/2334-0843/works?filter=from-pub-date%3A2026-01-01%2Cuntil-pub-date%3A2026-12-31&rows=200'
|
||||||
|
CONTAINER = 'Proceedings of the International Conference on Automated Planning and Scheduling'
|
||||||
|
TRACK = 'main_tracks_unclassified'
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def abstract_text(raw):
|
||||||
|
if not isinstance(raw, str):
|
||||||
|
raise ValueError('Missing registry abstract')
|
||||||
|
match = re.fullmatch(r'<jats:p>(.*?)</jats:p>', raw, re.S)
|
||||||
|
if not match or re.search(r'<[^>]*>', match[1]):
|
||||||
|
raise ValueError('Unreviewed abstract markup; preserve rather than flatten math')
|
||||||
|
text = clean(unescape(match[1]))
|
||||||
|
if not text:
|
||||||
|
raise ValueError('Empty registry abstract')
|
||||||
|
return text
|
||||||
|
|
||||||
|
|
||||||
|
def registry_papers(snapshot):
|
||||||
|
observed = snapshot['at']
|
||||||
|
if (snapshot.get('status') != 'registry_snapshot_not_identity_approved_not_imported'
|
||||||
|
or snapshot.get('url') != URL or snapshot.get('raw_response_sha256') != RAW_RESPONSE_SHA
|
||||||
|
or snapshot.get('database_written') is not False
|
||||||
|
or datetime.fromisoformat(observed.replace('Z', '+00:00')).utcoffset() is None
|
||||||
|
or snapshot.get('total_results') != 91 or len(snapshot.get('records', [])) != 91):
|
||||||
|
raise ValueError('Unreviewed registry snapshot receipt')
|
||||||
|
papers = []; excluded = []; seen = set()
|
||||||
|
for row in snapshot['records']:
|
||||||
|
doi = row.get('DOI')
|
||||||
|
if not isinstance(doi, str) or doi in seen:
|
||||||
|
raise ValueError('Missing or repeated registry DOI')
|
||||||
|
seen.add(doi)
|
||||||
|
if row.get('volume') == '35':
|
||||||
|
excluded.append(doi)
|
||||||
|
continue
|
||||||
|
if (row.get('volume') != '36' or row.get('issue') != '1'
|
||||||
|
or row.get('container-title') != [CONTAINER] or row.get('type') != 'journal-article'
|
||||||
|
or not re.fullmatch(r'10\.1609/icaps\.v36i1\.[0-9]+', doi)
|
||||||
|
or row.get('URL') != 'https://doi.org/'+doi
|
||||||
|
or not isinstance(row.get('title'), list) or len(row['title']) != 1
|
||||||
|
or not isinstance(row['title'][0], str) or not clean(row['title'][0])):
|
||||||
|
raise ValueError('Unreviewed publication volume, DOI, type or title')
|
||||||
|
day, = row['published']['date-parts']
|
||||||
|
if len(day) != 3 or day[0] != 2026 or any(type(x) is not int for x in day):
|
||||||
|
raise ValueError('Unreviewed publication date')
|
||||||
|
publication_date = date(*day).isoformat()
|
||||||
|
authors = row.get('authors')
|
||||||
|
if (not isinstance(authors, list) or not authors
|
||||||
|
or any(not isinstance(a, dict) or set(a)-{'given','family','name','sequence'}
|
||||||
|
or not a.get('given') or not a.get('family') for a in authors)):
|
||||||
|
raise ValueError('Incomplete registry authors')
|
||||||
|
names = [clean(a['given']+' '+a['family']) for a in authors]
|
||||||
|
if (len(set(n.casefold() for n in names)) != len(names)
|
||||||
|
or any('@' in n or 'anonymous' in n.casefold() for n in names)
|
||||||
|
or [a.get('sequence') for a in authors] != ['first']+['additional']*(len(authors)-1)):
|
||||||
|
raise ValueError('Unreviewed registry author order or identity')
|
||||||
|
papers.append({'original': row, 'title': row['title'][0], 'authors': names,
|
||||||
|
'abstract': abstract_text(row.get('abstract')), 'publication_date': publication_date})
|
||||||
|
if len(papers) != 90 or len(excluded) != 1:
|
||||||
|
raise ValueError('Incomplete reviewed volume census')
|
||||||
|
return sorted(papers, key=lambda p: p['original']['DOI']), excluded
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
path = Path(snapshots)/'crossref-filtered.json'
|
||||||
|
if path.is_symlink() or path.stat().st_size > 4*1024**2:
|
||||||
|
raise ValueError('Unsafe or oversized registry snapshot')
|
||||||
|
raw = path.read_bytes()
|
||||||
|
if digest(raw) != SNAPSHOT_SHA:
|
||||||
|
raise ValueError('Reviewed registry snapshot hash changed')
|
||||||
|
snapshot = json.loads(raw); papers, excluded = registry_papers(snapshot)
|
||||||
|
directory = out/'evidence'; directory.mkdir(); rel = f'evidence/{digest(raw)}.json'; (out/rel).write_bytes(raw)
|
||||||
|
evidence = {'file': rel, 'sha256': digest(raw), 'url': URL}; rows = []
|
||||||
|
for paper in papers:
|
||||||
|
original = paper['original']; doi = original['DOI']
|
||||||
|
rows.append({'source': SOURCE, 'external_id': doi, 'url': original['URL'],
|
||||||
|
'title': paper['title'], 'authors': paper['authors'], 'abstract': paper['abstract'],
|
||||||
|
'identifiers': {'doi': doi}, 'submission_identifiers': [{'scheme': 'publication_doi', 'value': doi}],
|
||||||
|
'venue': 'ICAPS', 'year': 2026, 'track': TRACK, 'status': 'accepted',
|
||||||
|
'raw_decision': 'publication recorded in Crossref ICAPS volume 36 issue 1; not an original signed acceptance decision',
|
||||||
|
'evidence_url': URL, 'observed_at': snapshot['at'], 'tags': [],
|
||||||
|
'metadata': {'registry_fields': original, 'source_evidence': evidence,
|
||||||
|
'abstract_transform': 'Remove one reviewed jats:p wrapper, decode character entities and normalize whitespace; original JATS retained.',
|
||||||
|
'publication_date': paper['publication_date'], 'original_decision_signature_verified': False,
|
||||||
|
'track_assignment': 'Unresolved main/special-track membership; no per-paper track inferred from topic or publication order.',
|
||||||
|
'program_relationship': 'Separate registry version; not substituted into official-listing records. Title/author variants require review.',
|
||||||
|
'scope': 'Registered publication metadata, not original review decision or current live publisher page. Requires current work-identity and history review before import.'}})
|
||||||
|
path = out/'icaps-crossref-registry-2026-main-tracks-unclassified.jsonl'
|
||||||
|
payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
|
||||||
|
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICAPS', 'year': 2026, 'track': TRACK,
|
||||||
|
'count': 90, 'file': str(path), 'file_sha256': digest(payload), 'list_url': URL, 'status': 'partial', 'missing_abstract_count': 0}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
|
||||||
|
report = {'status': 'collected_pending_identity_review', 'records': 90, 'files': [manifest],
|
||||||
|
'excluded_volume35_dois': excluded, 'missing_abstract_count': 0, 'source_evidence': evidence,
|
||||||
|
'database_written': False, 'pdfs_downloaded': False, 'public_rejected_availability': 'unknown',
|
||||||
|
'scope': 'Crossref publication versions only; no automatic official-program linkage, person aliases, work merges or complete public-negative/final-dedup claim.'}
|
||||||
|
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
|
parser.add_argument('--snapshots', required=True)
|
||||||
|
parser.add_argument('--out', required=True)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
|
||||||
@@ -0,0 +1,172 @@
|
|||||||
|
"""Normalize the frozen ICASSP 2026 Crossref census without network or database writes."""
|
||||||
|
import argparse
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
from paper_atlas.io.html import clean
|
||||||
|
|
||||||
|
SOURCE = 'crossref_icassp_registry'
|
||||||
|
TRACK = 'proceedings_unclassified'
|
||||||
|
SNAPSHOT = 'icassp2026-registry-capture-v1'
|
||||||
|
MANIFEST_SHA = '1f8ffa040b1f682a4b65f476f4115cb95629056d8de5c324c8417d823846ae93'
|
||||||
|
CONTAINER = 'ICASSP 2026 - 2026 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)'
|
||||||
|
OTHER_CONTAINER = '2026 IEEE 6th International Conference in Power Engineering Applications (ICPEA)'
|
||||||
|
PREFIX = '10.1109/icassp55912.2026.'
|
||||||
|
NONPAPERS = {
|
||||||
|
PREFIX+'11463724': ('ICASSP 2026 Reviewers', 'xii-lxxxvi'),
|
||||||
|
PREFIX+'11464828': ('Author Index', 'A-1-A-152'),
|
||||||
|
PREFIX+'11464776': ('Title Page', 'i-ii'),
|
||||||
|
PREFIX+'11462944': ('ICASSP 2026 Committees', 'vi-xii'),
|
||||||
|
PREFIX+'11464852': ('ICASSP 2026 Organizing Committee', 'iii-iii'),
|
||||||
|
PREFIX+'11460797': ('ICASSP 2026 Cover Page', 'c1-c1'),
|
||||||
|
PREFIX+'11463523': ('Table of Contents', 'xlix-cdlxii'),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def load_snapshot(root):
|
||||||
|
root = Path(root); directory = root/SNAPSHOT; path = directory/'manifest.json'
|
||||||
|
if root.is_symlink() or directory.is_symlink() or path.is_symlink() or path.stat().st_size > 128*1024:
|
||||||
|
raise ValueError('Unsafe snapshot manifest')
|
||||||
|
raw = path.read_bytes()
|
||||||
|
if digest(raw) != MANIFEST_SHA:
|
||||||
|
raise ValueError('Reviewed manifest hash changed')
|
||||||
|
manifest = json.loads(raw)
|
||||||
|
if (manifest['status'] != 'registry_census_completed_not_import_approval'
|
||||||
|
or manifest['records_captured'] != 4645 or manifest['unique_dois'] != 4645
|
||||||
|
or manifest['database_written'] is not False or manifest['approved_for_import'] != 0
|
||||||
|
or len(manifest['pages']) != 10):
|
||||||
|
raise ValueError('Incomplete frozen census receipt')
|
||||||
|
files = {'manifest.json': raw}; rows = []
|
||||||
|
for index, receipt in enumerate(manifest['pages'], 1):
|
||||||
|
name = receipt['file']; path = directory/name
|
||||||
|
if (name != f'page-{index:03}.json' or path.is_symlink()
|
||||||
|
or path.stat().st_size > 4*1024**2):
|
||||||
|
raise ValueError('Unsafe snapshot page')
|
||||||
|
body = path.read_bytes()
|
||||||
|
if (receipt['http_status'] != 200 or receipt['url'] != receipt['final_url']
|
||||||
|
or not receipt['url'].startswith('https://api.crossref.org/works?')
|
||||||
|
or datetime.fromisoformat(receipt['at']).utcoffset() is None
|
||||||
|
or receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)):
|
||||||
|
raise ValueError('Snapshot receipt/body disagree')
|
||||||
|
data = json.loads(body); message = data['message']
|
||||||
|
expected = 500 if index < 10 else 145
|
||||||
|
if data['status'] != 'ok' or message['total-results'] != 4645 or len(message['items']) != expected:
|
||||||
|
raise ValueError('Incomplete registry page')
|
||||||
|
files[name] = body
|
||||||
|
for original in message['items']:
|
||||||
|
rows.append((original, receipt))
|
||||||
|
return manifest, files, rows
|
||||||
|
|
||||||
|
|
||||||
|
def registry_papers(rows):
|
||||||
|
seen = set(); papers = []; nonpapers = []; other = []
|
||||||
|
for original, receipt in rows:
|
||||||
|
doi = original.get('DOI')
|
||||||
|
if not isinstance(doi, str) or doi in seen:
|
||||||
|
raise ValueError('Missing or repeated registry DOI')
|
||||||
|
seen.add(doi)
|
||||||
|
if original.get('container-title') == [OTHER_CONTAINER]:
|
||||||
|
if doi.startswith(PREFIX):
|
||||||
|
raise ValueError('Conference DOI/container conflict')
|
||||||
|
other.append(doi)
|
||||||
|
continue
|
||||||
|
if (original.get('container-title') != [CONTAINER]
|
||||||
|
or original.get('event', {}).get('name') != CONTAINER
|
||||||
|
or not re.fullmatch(re.escape(PREFIX)+r'[0-9]+', doi)
|
||||||
|
or original.get('publisher') != 'IEEE' or original.get('type') != 'proceedings-article'
|
||||||
|
or original.get('URL') != 'https://doi.org/'+doi
|
||||||
|
or original.get('published') != {'date-parts': [[2026, 5, 3]]}
|
||||||
|
or original.get('event', {}).get('start') != {'date-parts': [[2026, 5, 3]]}
|
||||||
|
or original.get('event', {}).get('end') != {'date-parts': [[2026, 5, 8]]}):
|
||||||
|
raise ValueError('Wrong registered conference, DOI, publisher or dates')
|
||||||
|
if doi in NONPAPERS:
|
||||||
|
title, page = NONPAPERS[doi]
|
||||||
|
if original.get('title') != [title] or original.get('page') != page or original.get('author'):
|
||||||
|
raise ValueError('Changed reviewed nonpaper evidence')
|
||||||
|
nonpapers.append({'doi': doi, 'title': title, 'page': page})
|
||||||
|
continue
|
||||||
|
titles = original.get('title')
|
||||||
|
if not isinstance(titles, list) or len(titles) != 1 or not isinstance(titles[0], str) or not clean(titles[0]):
|
||||||
|
raise ValueError('Missing registry title')
|
||||||
|
authors = original.get('author')
|
||||||
|
if not isinstance(authors, list) or not authors or original.get('abstract'):
|
||||||
|
raise ValueError('Missing authors or changed abstract availability')
|
||||||
|
names = []; flags = []
|
||||||
|
for position, author in enumerate(authors):
|
||||||
|
if (not isinstance(author, dict) or not isinstance(author.get('family'), str)
|
||||||
|
or not clean(author['family']) or ('given' in author and not isinstance(author['given'], str))
|
||||||
|
or author.get('sequence') != ('first' if position == 0 else 'additional')):
|
||||||
|
raise ValueError('Incomplete registry author or sequence')
|
||||||
|
name = clean(' '.join(x for x in (author.get('given'), author['family']) if x))
|
||||||
|
if not name or '@' in name or name.casefold() == 'anonymous':
|
||||||
|
raise ValueError('Unreviewed author identity')
|
||||||
|
if not author.get('given'):
|
||||||
|
flags.append({'kind': 'registry_single_name_field_not_split', 'author_position': position+1})
|
||||||
|
names.append(name)
|
||||||
|
papers.append({'original': original, 'receipt': receipt, 'title': clean(titles[0]), 'authors': names, 'review_flags': flags})
|
||||||
|
if len(seen) != 4645 or len(papers) != 4582 or len(other) != 56 or {r['doi'] for r in nonpapers} != set(NONPAPERS):
|
||||||
|
raise ValueError('Incomplete reviewed registry partition')
|
||||||
|
return sorted(papers, key=lambda p: p['original']['DOI']), nonpapers, other
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
manifest, files, items = load_snapshot(snapshots)
|
||||||
|
papers, nonpapers, other = registry_papers(items)
|
||||||
|
evidence_dir = out/'evidence'; evidence_dir.mkdir()
|
||||||
|
for name, body in files.items():
|
||||||
|
(evidence_dir/name).write_bytes(body)
|
||||||
|
rows = []
|
||||||
|
for paper in papers:
|
||||||
|
original = paper['original']; receipt = paper['receipt']; doi = original['DOI']
|
||||||
|
rows.append({'source': SOURCE, 'external_id': doi, 'url': original['URL'],
|
||||||
|
'title': paper['title'], 'authors': paper['authors'], 'abstract': None,
|
||||||
|
'identifiers': {'doi': doi}, 'submission_identifiers': [{'scheme': 'publication_doi', 'value': doi}],
|
||||||
|
'venue': 'ICASSP', 'year': 2026, 'track': TRACK, 'status': 'accepted',
|
||||||
|
'raw_decision': 'Publication registered in Crossref ICASSP 2026 proceedings; not an original signed decision.',
|
||||||
|
'evidence_url': receipt['url'], 'observed_at': receipt['at'], 'tags': [],
|
||||||
|
'metadata': {'registry_fields': original, 'review_flags': paper['review_flags'],
|
||||||
|
'source_evidence': {'file': 'evidence/'+receipt['file'], 'sha256': receipt['sha256'], 'url': receipt['url']},
|
||||||
|
'source_manifest': {'file': 'evidence/manifest.json', 'sha256': MANIFEST_SHA},
|
||||||
|
'original_decision_signature_verified': False,
|
||||||
|
'abstract_availability': 'Absent in frozen registry deposit; no abstract inferred or fused from program versions.',
|
||||||
|
'author_transform': 'Whitespace normalization of given + family in registered order; single fields retained, not guessed or split; all original author fields retained.',
|
||||||
|
'publication_dates': {k: original[k] for k in ('published', 'published-print', 'published-online') if k in original},
|
||||||
|
'date_scope': 'Registered event starts May 3; conference website says May 4–8. Source values preserved without substitution.',
|
||||||
|
'scope': 'Registered publication version only. Requires work/DOI/history review before import. No automatic program match, aliases, track/type or public-negative inference.'}})
|
||||||
|
path = out/'icassp-crossref-registry-2026-unclassified.jsonl'
|
||||||
|
raw = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(raw)
|
||||||
|
file_manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICASSP', 'year': 2026, 'track': TRACK,
|
||||||
|
'count': len(rows), 'file': str(path), 'file_sha256': digest(raw), 'list_url': manifest['pages'][0]['url'],
|
||||||
|
'status': 'partial', 'missing_abstract_count': len(rows)}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(file_manifest, indent=2)+'\n')
|
||||||
|
report = {'status': 'collected_pending_identity_review', 'records': len(rows), 'files': [file_manifest],
|
||||||
|
'filtered_registry_records': 4645, 'excluded_nonpaper_items': nonpapers, 'excluded_other_conference_dois': other,
|
||||||
|
'missing_abstract_count': len(rows), 'single_name_author_fields': sum(len(p['review_flags']) for p in papers),
|
||||||
|
'database_written': False, 'network_requests': 0, 'pdfs_downloaded': False, 'approved_for_import': 0,
|
||||||
|
'public_rejected_availability': 'unknown', 'source_manifest_sha256': MANIFEST_SHA,
|
||||||
|
'scope': 'Registry census, not completeness of program, abstracts, tracks, negative decisions or work identities. Official program has4589 titles;174 unmatched normalized title candidates remain separate source discrepancies, not rejection or withdrawal evidence.'}
|
||||||
|
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
|
parser.add_argument('--snapshots', required=True)
|
||||||
|
parser.add_argument('--out', required=True)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
|
||||||
@@ -0,0 +1,191 @@
|
|||||||
|
"""Frozen ICDM2026 in-person program, with explicit source track evidence."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, Node, clean
|
||||||
|
|
||||||
|
SOURCE = 'icdm_program'
|
||||||
|
SNAPSHOT = 'remaining2026-official-leads-v1'
|
||||||
|
MANIFEST_SHA = '0e13313d94e3bff17f071d3a1384f6ed83ece156239fe5bc4633cc7a34fb8e04'
|
||||||
|
PROGRAM = 'https://icdm2026.neu.edu.cn/DetailedSessions/list.htm'
|
||||||
|
PAGES = {
|
||||||
|
'detailed-sessions': PROGRAM,
|
||||||
|
'research-policy': 'https://icdm2026.neu.edu.cn/CallforResearchTrack/list.htm',
|
||||||
|
'applied-policy': 'https://icdm2026.neu.edu.cn/CallforAppliedTrack/list.htm',
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def one(items):
|
||||||
|
if len(items) != 1:
|
||||||
|
raise ValueError('Expected exactly one reviewed field')
|
||||||
|
return items[0]
|
||||||
|
|
||||||
|
|
||||||
|
def has_class(node, name):
|
||||||
|
return name in node.attrs.get('class', '').split()
|
||||||
|
|
||||||
|
|
||||||
|
def field(node, name):
|
||||||
|
return clean(one([n for n in node.walk() if has_class(n, name)]).text())
|
||||||
|
|
||||||
|
|
||||||
|
def load_snapshot(root):
|
||||||
|
root = Path(root); folder = root/SNAPSHOT; manifest = folder/'manifest.json'
|
||||||
|
if root.is_symlink() or folder.is_symlink() or manifest.is_symlink() or manifest.stat().st_size > 256*1024:
|
||||||
|
raise ValueError('Unsafe source manifest')
|
||||||
|
raw = manifest.read_bytes()
|
||||||
|
if digest(raw) != MANIFEST_SHA:
|
||||||
|
raise ValueError('Frozen manifest hash changed')
|
||||||
|
data = json.loads(raw)
|
||||||
|
if data['status'] != 'research_completed_not_coverage_approval' or data['year'] != 2026:
|
||||||
|
raise ValueError('Wrong source edition or incomplete research run')
|
||||||
|
receipts = [r for r in data['entries'] if r['venue'] == 'ICDM']
|
||||||
|
if len(receipts) != 3 or {r['label'] for r in receipts} != PAGES.keys():
|
||||||
|
raise ValueError('Incomplete ICDM source set')
|
||||||
|
files = {'manifest.json': raw}; indexed = {}
|
||||||
|
for receipt in receipts:
|
||||||
|
filename = receipt['file']; path = folder/filename
|
||||||
|
if Path(filename).name != filename or path.is_symlink() or path.stat().st_size > 8*1024**2:
|
||||||
|
raise ValueError('Unsafe source file')
|
||||||
|
body = path.read_bytes()
|
||||||
|
if (receipt['url'] != PAGES[receipt['label']] or receipt['final_url'] != receipt['url']
|
||||||
|
or receipt['http_status'] != 200 or receipt['bytes'] != len(body)
|
||||||
|
or receipt['sha256'] != digest(body)
|
||||||
|
or not receipt['content_type'].lower().startswith('text/html')
|
||||||
|
or datetime.fromisoformat(receipt['at']).utcoffset() is None):
|
||||||
|
raise ValueError('Source receipt/body mismatch')
|
||||||
|
if filename in files:
|
||||||
|
raise ValueError('Repeated source filename')
|
||||||
|
files[filename] = body; indexed[receipt['label']] = receipt
|
||||||
|
return indexed, files
|
||||||
|
|
||||||
|
|
||||||
|
def parse_program(raw):
|
||||||
|
doc = Document(raw.decode('utf-8'))
|
||||||
|
main = one([n for n in doc.root.walk() if n.tag == 'main' and n.attrs.get('id') == 'session-details'])
|
||||||
|
text = clean(main.text())
|
||||||
|
if ('IEEE ICDM 2026' not in text or 'November 13–15, 2026' not in text
|
||||||
|
or 'In-person session presentation list.' not in text or '34 sessions · 324 papers' not in text):
|
||||||
|
raise ValueError('Wrong edition or unreviewed program scope')
|
||||||
|
sessions = [n for n in main.walk() if n.tag == 'details' and has_class(n, 'sd-session')]
|
||||||
|
expected = {f'L{i}' for i in range(1, 21)} | {f'S{i}' for i in range(1, 15)}
|
||||||
|
if len(sessions) != 34 or {n.attrs.get('data-session-id') for n in sessions} != expected:
|
||||||
|
raise ValueError('Incomplete or repeated session census')
|
||||||
|
records = []; seen = set()
|
||||||
|
for session in sessions:
|
||||||
|
sid = session.attrs['data-session-id']; track = session.attrs.get('data-track')
|
||||||
|
form = session.attrs.get('data-format')
|
||||||
|
if (session.attrs.get('id') != sid or track not in ('research', 'applied')
|
||||||
|
or form not in ('long', 'short')):
|
||||||
|
raise ValueError('Invalid session ID, track or presentation label')
|
||||||
|
label = 'Research Track' if track == 'research' else 'Applied Track'
|
||||||
|
if field(session, 'sd-track') != label:
|
||||||
|
raise ValueError('Visible track and source attribute disagree')
|
||||||
|
count = re.fullmatch(r'(Long|Short) · ([1-9][0-9]*) papers', field(session, 'sd-format'))
|
||||||
|
if count is None or count[1].lower() != form:
|
||||||
|
raise ValueError('Visible presentation label disagrees')
|
||||||
|
rows = [n for n in session.walk() if n.tag == 'tr' and 'data-paper-id' in n.attrs]
|
||||||
|
if len(rows) != int(count[2]):
|
||||||
|
raise ValueError('Session paper count changed')
|
||||||
|
note = field(session, 'sd-table-note')
|
||||||
|
if not note.startswith(f'{form.title()} Presentation ({len(rows)} papers) · '):
|
||||||
|
raise ValueError('Unreviewed presentation-note semantics')
|
||||||
|
context = {'id': sid, 'track_literal': track, 'presentation_literal': form,
|
||||||
|
'title': field(session, 'sd-session-title'),
|
||||||
|
'time_literal': field(session, 'sd-session-time'), 'room_literal': field(session, 'sd-room')}
|
||||||
|
for row in rows:
|
||||||
|
pid = row.attrs['data-paper-id']
|
||||||
|
cells = [n for n in row.children if isinstance(n, Node) and n.tag in ('th', 'td')]
|
||||||
|
if (not re.fullmatch(r'DM[1-9][0-9]*', pid) or pid in seen
|
||||||
|
or row.attrs.get('id') != 'paper-'+pid or len(cells) != 4
|
||||||
|
or [clean(n.text()) for n in cells] != [field(row, k) for k in
|
||||||
|
('sd-time', 'sd-paper-id', 'sd-paper-title', 'sd-authors')]
|
||||||
|
or field(row, 'sd-paper-id') != pid):
|
||||||
|
raise ValueError('Repeated/malformed program paper identity or cells')
|
||||||
|
seen.add(pid)
|
||||||
|
title = field(row, 'sd-paper-title'); authors_literal = field(row, 'sd-authors')
|
||||||
|
authors = [name.strip() for name in authors_literal.split(';')]
|
||||||
|
if not title or any(not name for name in authors):
|
||||||
|
raise ValueError('Missing program title or author')
|
||||||
|
records.append({'paper_id_literal': pid, 'title': title, 'authors_literal': authors_literal,
|
||||||
|
'source_reported_authors': authors, 'time_literal': field(row, 'sd-time'),
|
||||||
|
'session': context.copy(), 'source_position': len(records)+1})
|
||||||
|
if (len(records) != 324 or len([n for n in main.walk() if 'data-paper-id' in n.attrs]) != 324
|
||||||
|
or Counter(r['session']['track_literal'] for r in records) != {'research': 241, 'applied': 83}
|
||||||
|
or Counter(r['session']['presentation_literal'] for r in records) != {'long': 168, 'short': 156}):
|
||||||
|
raise ValueError('Incomplete reviewed paper/track/presentation census')
|
||||||
|
return records
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
receipts, files = load_snapshot(snapshots)
|
||||||
|
program = receipts['detailed-sessions']; observations = parse_program(files[program['file']])
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
evidence = []; directory = out/'evidence'; directory.mkdir()
|
||||||
|
for name, raw in files.items():
|
||||||
|
rel = 'evidence/'+digest(raw)+Path(name).suffix; (out/rel).write_bytes(raw)
|
||||||
|
evidence.append({'file': name, 'output_file': rel, 'sha256': digest(raw)})
|
||||||
|
outputs = []
|
||||||
|
for track in ('research', 'applied'):
|
||||||
|
rows = []
|
||||||
|
for item in observations:
|
||||||
|
if item['session']['track_literal'] != track:
|
||||||
|
continue
|
||||||
|
pid = item['paper_id_literal']; url = PROGRAM+'#paper-'+pid
|
||||||
|
rows.append({'source': SOURCE, 'external_id': '2026/'+pid, 'url': url,
|
||||||
|
'title': item['title'], 'authors': item['source_reported_authors'], 'abstract': None,
|
||||||
|
'venue': 'ICDM', 'year': 2026, 'track': track, 'status': 'accepted',
|
||||||
|
'identifiers': {}, 'submission_identifiers': [], 'observed_at': program['at'],
|
||||||
|
'evidence_url': url,
|
||||||
|
'raw_decision': 'Listed in official ICDM2026 in-person '+track+' program; not an original signed decision or verified publisher record.',
|
||||||
|
'tags': [{'namespace': SOURCE+':session_track', 'value': track},
|
||||||
|
{'namespace': SOURCE+':presentation', 'value': item['session']['presentation_literal']}],
|
||||||
|
'metadata': {'program_fields': item, 'program_receipt': program,
|
||||||
|
'original_decision_signature_verified': False,
|
||||||
|
'program_id_scope': 'Source program locator only; not asserted original submission ID or publication DOI.',
|
||||||
|
'author_scope': 'Source-reported program names; no publisher completeness or person alias assertion.',
|
||||||
|
'abstract_availability': 'No abstracts in this frozen program table.',
|
||||||
|
'track_evidence': 'Explicit data-track attribute agrees with visible session label; not inferred from topic.',
|
||||||
|
'paper_type': None,
|
||||||
|
'scope': 'In-person program only; excludes other program sections. Long/short describe presentation format. Requires work/version and publication review before import.'}})
|
||||||
|
path = out/f'icdm-program-2026-{track}.jsonl'
|
||||||
|
payload = ''.join(json.dumps(row, ensure_ascii=False)+'\n' for row in rows).encode(); path.write_bytes(payload)
|
||||||
|
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'ICDM', 'year': 2026,
|
||||||
|
'track': track, 'count': len(rows), 'file': str(path), 'file_sha256': digest(payload),
|
||||||
|
'list_url': PROGRAM, 'status': 'partial', 'missing_abstract_count': len(rows),
|
||||||
|
'source_expected_count': len(rows), 'scope': 'Frozen in-person program section, not all accepted/published papers.'}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n'); outputs.append(manifest)
|
||||||
|
report = {'status': 'collected_pending_identity_review', 'records': 324, 'sessions': 34,
|
||||||
|
'files': outputs, 'track_counts': {'research': 241, 'applied': 83},
|
||||||
|
'presentation_counts': {'long': 168, 'short': 156}, 'missing_abstract_count': 324,
|
||||||
|
'evidence': evidence, 'public_rejected_metadata': 'unknown_no_public_export_verified',
|
||||||
|
'negative_policy_scope': 'Research triple-blind and applied single-blind concern review-time visibility; neither proves final public decision coverage.',
|
||||||
|
'database_written': False, 'network_requests': 0, 'pdfs_downloaded': False,
|
||||||
|
'scope': 'In-person program observations only. No work merge, inferred author alias, publication identity, original signed decision or final whole-database audit.'}
|
||||||
|
temporary = out/'collection-report.json.part'; temporary.write_text(json.dumps(report, indent=2)+'\n')
|
||||||
|
temporary.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status': 'failed', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
|
parser.add_argument('--snapshots', required=True, type=Path)
|
||||||
|
parser.add_argument('--out', required=True, type=Path)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
|
||||||
@@ -0,0 +1,208 @@
|
|||||||
|
"""Frozen ISMIR 2026 accepted program, abstracts and opt-in review provenance."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import unicodedata
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, Node, clean
|
||||||
|
|
||||||
|
SOURCE = 'ismir_program'
|
||||||
|
TRACK = 'program_unclassified'
|
||||||
|
PAPERS = 'https://ismir2026program.ismir.net/papers.html'
|
||||||
|
ACCEPTED = 'https://ismir2026.ismir.net/accepted-papers'
|
||||||
|
SNAPSHOTS = {
|
||||||
|
'remaining2026-listings-research-v2': ('fea7b710587636db06794ee195cd083634c41e9054a217859d9dd0ccdd1afab9',
|
||||||
|
'research_completed_not_coverage_or_import_approval', {
|
||||||
|
'ismir-accepted.html': ACCEPTED,
|
||||||
|
'ismir-call.html': 'https://ismir2026.ismir.net/authors/call-for-papers'}),
|
||||||
|
'ismir2026-script-research-v1': ('5885b4bcfc2fc65374e5dc7a0d74f4eb86a0a36b8689f9fe8edbd10d49a08794',
|
||||||
|
'public_script_research_only', {'papers.js': 'https://ismir2026program.ismir.net/static/js/papers.js'}),
|
||||||
|
'ismir2026-json-research-v1': ('8be3e23277c199720264d940917118572655d18c0f9ac6db1b9cd30a6561b34d',
|
||||||
|
'public_metadata_json_research_only', {'papers.json': 'https://ismir2026program.ismir.net/papers.json'}),
|
||||||
|
}
|
||||||
|
REVIEW_KEYS = {'meta_review','publish_reviews','review1','review2','review3','review4'}
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def one(items):
|
||||||
|
if len(items) != 1:
|
||||||
|
raise ValueError('Expected exactly one reviewed source field')
|
||||||
|
return items[0]
|
||||||
|
|
||||||
|
|
||||||
|
def title_key(text):
|
||||||
|
return ''.join(c for c in unicodedata.normalize('NFKD', text).casefold() if c.isalnum())
|
||||||
|
|
||||||
|
|
||||||
|
def load_snapshots(root):
|
||||||
|
root = Path(root)
|
||||||
|
if root.is_symlink():
|
||||||
|
raise ValueError('Unsafe snapshot root')
|
||||||
|
result = {}
|
||||||
|
for name, (expected, status, urls) in SNAPSHOTS.items():
|
||||||
|
directory = root/name; path = directory/'manifest.json'
|
||||||
|
if directory.is_symlink() or path.is_symlink() or path.stat().st_size > 128*1024:
|
||||||
|
raise ValueError('Unsafe snapshot manifest')
|
||||||
|
raw = path.read_bytes()
|
||||||
|
if digest(raw) != expected:
|
||||||
|
raise ValueError('Reviewed manifest hash changed')
|
||||||
|
manifest = json.loads(raw)
|
||||||
|
if manifest['status'] != status or manifest['database_written'] is not False:
|
||||||
|
raise ValueError('Unreviewed snapshot scope')
|
||||||
|
files = {'manifest.json': raw}; receipts = {}
|
||||||
|
for filename, url in urls.items():
|
||||||
|
receipt = one([r for r in manifest['entries'] if r.get('file') == filename])
|
||||||
|
path = directory/filename
|
||||||
|
if path.is_symlink() or path.stat().st_size > 8*1024**2:
|
||||||
|
raise ValueError('Unsafe snapshot source file')
|
||||||
|
body = path.read_bytes()
|
||||||
|
if (receipt['url'] != url or receipt['final_url'] != url or receipt['http_status'] != 200
|
||||||
|
or receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)
|
||||||
|
or datetime.fromisoformat(receipt['at']).utcoffset() is None):
|
||||||
|
raise ValueError('Source receipt/body disagree')
|
||||||
|
files[filename] = body; receipts[filename] = receipt
|
||||||
|
result[name] = (files, receipts)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def author_names(literal):
|
||||||
|
# Institution names can contain semicolons; split only outside parentheses.
|
||||||
|
parts = []; depth = 0; start = 0
|
||||||
|
for index, char in enumerate(literal):
|
||||||
|
if char == '(': depth += 1
|
||||||
|
elif char == ')':
|
||||||
|
depth -= 1
|
||||||
|
if depth < 0: raise ValueError('Unbalanced author affiliation')
|
||||||
|
elif char == ';' and depth == 0:
|
||||||
|
parts.append(literal[start:index]); start = index+1
|
||||||
|
if depth:
|
||||||
|
raise ValueError('Unbalanced author affiliation')
|
||||||
|
parts.append(literal[start:])
|
||||||
|
names = [clean(p.split('(', 1)[0]) for p in parts]
|
||||||
|
if any(not name for name in names):
|
||||||
|
raise ValueError('Missing accepted-list author')
|
||||||
|
return names
|
||||||
|
|
||||||
|
|
||||||
|
def accepted_listing(raw):
|
||||||
|
doc = Document(raw.decode('utf-8')); tables = []
|
||||||
|
for table in doc.root.walk():
|
||||||
|
if table.tag != 'table': continue
|
||||||
|
rows = []
|
||||||
|
for row in table.walk():
|
||||||
|
if row.tag == 'tr':
|
||||||
|
rows.append([clean(n.text()) for n in row.children if isinstance(n, Node) and n.tag in ('th','td')])
|
||||||
|
if rows and rows[0] == ['Paper Title', 'Authors']:
|
||||||
|
tables.append(rows)
|
||||||
|
rows = one(tables)[1:]; result = {}
|
||||||
|
if len(rows) != 140:
|
||||||
|
raise ValueError('Incomplete official accepted-list census')
|
||||||
|
for row in rows:
|
||||||
|
if len(row) != 2 or not row[0] or not row[1] or title_key(row[0]) in result:
|
||||||
|
raise ValueError('Missing or ambiguous accepted-list title')
|
||||||
|
result[title_key(row[0])] = {'title': row[0], 'authors_literal': row[1], 'author_names': author_names(row[1])}
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def program_observations(rows, listing):
|
||||||
|
if not isinstance(rows, list) or len(rows) != 140:
|
||||||
|
raise ValueError('Incomplete public program census')
|
||||||
|
observations = []; seen = set(); used = set()
|
||||||
|
for original in rows:
|
||||||
|
if not isinstance(original, dict) or not isinstance(original.get('content'), dict):
|
||||||
|
raise ValueError('Invalid public program record')
|
||||||
|
pid = original.get('id'); content = original.get('content', {})
|
||||||
|
if not isinstance(pid, str) or not re.fullmatch(r'[1-9][0-9]*', pid) or pid in seen or original.get('forum') != pid:
|
||||||
|
raise ValueError('Missing/repeated numeric program locator, not an OpenReview ID')
|
||||||
|
seen.add(pid); title = content.get('title'); authors = content.get('authors'); abstract = content.get('abstract')
|
||||||
|
if (not isinstance(title, str) or not clean(title) or not isinstance(abstract, str) or not abstract.strip()
|
||||||
|
or not isinstance(authors, list) or not authors or any(not isinstance(a, str) or not clean(a) for a in authors)):
|
||||||
|
raise ValueError('Missing program title, author sequence or abstract')
|
||||||
|
key = title_key(title)
|
||||||
|
if key not in listing or key in used:
|
||||||
|
raise ValueError('Program/accepted-list title membership mismatch')
|
||||||
|
used.add(key); accepted = listing[key]
|
||||||
|
keywords = content.get('keywords'); presentation = content.get('paper_presentation')
|
||||||
|
session = content.get('session'); reviews = original.get('reviews')
|
||||||
|
if (not isinstance(keywords, list) or any(not isinstance(k, str) for k in keywords)
|
||||||
|
or presentation not in ('Remote','In-person','Remote during oral, In-person during nominee')
|
||||||
|
or not isinstance(session, list) or len(session) != 1 or not isinstance(session[0], str) or session[0] != original.get('session')
|
||||||
|
or not re.fullmatch(r'(?:[1-9]|1[0-4])', session[0])):
|
||||||
|
raise ValueError('Unreviewed source keywords, presentation or session')
|
||||||
|
if (not isinstance(reviews, dict) or set(reviews) != REVIEW_KEYS
|
||||||
|
or any(not isinstance(v, str) for v in reviews.values())
|
||||||
|
or reviews['publish_reviews'] not in ('Yes','No')):
|
||||||
|
raise ValueError('Unreviewed opt-in review structure')
|
||||||
|
review_text = [reviews[k] for k in REVIEW_KEYS-{'publish_reviews'}]
|
||||||
|
if (reviews['publish_reviews'] == 'No' and any(review_text)) or (reviews['publish_reviews'] == 'Yes' and not any(review_text)):
|
||||||
|
raise ValueError('Review publication preference/body disagree')
|
||||||
|
flags = []
|
||||||
|
if title != accepted['title']: flags.append('program_and_accepted_title_literal_versions_differ')
|
||||||
|
if authors != accepted['author_names']: flags.append('program_and_accepted_author_sequence_versions_differ')
|
||||||
|
observations.append({'original': original, 'accepted_listing': accepted, 'review_flags': flags})
|
||||||
|
if (used != listing.keys() or sum('program_and_accepted_title_literal_versions_differ' in o['review_flags'] for o in observations) != 2
|
||||||
|
or {o['original']['id'] for o in observations if 'program_and_accepted_author_sequence_versions_differ' in o['review_flags']} != {'60','126','433'}
|
||||||
|
or Counter(o['original']['reviews']['publish_reviews'] for o in observations) != {'Yes':101,'No':39}):
|
||||||
|
raise ValueError('Unreviewed source-version or opt-in census')
|
||||||
|
return observations
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists(): raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
sources = load_snapshots(snapshots)
|
||||||
|
official, _ = sources['remaining2026-listings-research-v2']; script, _ = sources['ismir2026-script-research-v1']
|
||||||
|
program, receipts = sources['ismir2026-json-research-v1']; receipt = receipts['papers.json']
|
||||||
|
if b"d3.json('papers.json')" not in script['papers.js'] or b'href="poster_${openreview.id}.html"' not in script['papers.js']:
|
||||||
|
raise ValueError('Unreviewed public program data path or poster link template')
|
||||||
|
observations = program_observations(json.loads(program['papers.json']), accepted_listing(official['ismir-accepted.html']))
|
||||||
|
evidence = out/'evidence'; evidence.mkdir()
|
||||||
|
for name, (files, _) in sources.items():
|
||||||
|
directory = evidence/name; directory.mkdir()
|
||||||
|
for filename, body in files.items(): (directory/filename).write_bytes(body)
|
||||||
|
rows = []
|
||||||
|
for observation in observations:
|
||||||
|
original = observation['original']; content = original['content']; pid = original['id']
|
||||||
|
tags = [{'namespace':SOURCE+':presentation','value':content['paper_presentation']}]
|
||||||
|
tags += [{'namespace':SOURCE+':keyword','value':value} for value in dict.fromkeys(clean(k) for k in content['keywords'] if clean(k))]
|
||||||
|
rows.append({'source':SOURCE,'external_id':'2026/'+pid,'url':'https://ismir2026program.ismir.net/poster_'+pid+'.html',
|
||||||
|
'title':content['title'],'authors':content['authors'],'abstract':content['abstract'],
|
||||||
|
'identifiers':{},'submission_identifiers':[],'venue':'ISMIR','year':2026,'track':TRACK,'status':'accepted',
|
||||||
|
'raw_decision':'Present in official ISMIR2026 accepted list and public virtual program; not an original signed decision.',
|
||||||
|
'evidence_url':ACCEPTED,'observed_at':receipt['at'],'tags':tags,
|
||||||
|
'metadata':{'program_entry':original,'accepted_listing':observation['accepted_listing'],'review_flags':observation['review_flags'],
|
||||||
|
'program_receipt':receipt,'source_evidence':{'file':'evidence/ismir2026-json-research-v1/papers.json','sha256':receipt['sha256'],'url':receipt['url']},
|
||||||
|
'original_decision_signature_verified':False,'paper_type':None,
|
||||||
|
'id_scope':'Numeric miniconf id/forum fields are program locators, not original OpenReview submission IDs; no inferred DOI or arxiv ID.',
|
||||||
|
'track_scope':'Unclassified regular accepted-list program; no workshop or research-method class inferred from topic/session/remote presentation.',
|
||||||
|
'review_scope':'101 source-opt-in review sets retained as raw public text;39No contain no review text. Reviewer recommendations/meta-review prose are not separately inferred final negative decisions.',
|
||||||
|
'abstract_transform':'Original JSON abstract string retained verbatim, including math, newlines and Unicode; no markup flattening or PDF extraction.',
|
||||||
|
'program_listing_scope':'Source versions remain separate.2title literal differences and3author-list differences flagged, not silently fused or resolved.',
|
||||||
|
'scope':'Pending work/history review. Poster URL follows frozen official JS template; poster pages not downloaded. No work merge, aliases, negative decision or final coverage/dedup claim.'}})
|
||||||
|
path = out/'ismir-program-2026-unclassified.jsonl';raw=''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in rows).encode();path.write_bytes(raw)
|
||||||
|
manifest={'source':SOURCE,'parser_version':1,'venue':'ISMIR','year':2026,'track':TRACK,'count':140,'file':str(path),
|
||||||
|
'file_sha256':digest(raw),'list_url':ACCEPTED,'status':'partial','missing_abstract_count':0,'source_expected_count':140}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)+'\n')
|
||||||
|
report={'status':'collected_pending_identity_review','records':140,'files':[manifest],'missing_abstract_count':0,
|
||||||
|
'literal_title_version_differences':2,'author_sequence_version_differences':3,'review_optin_yes':101,'review_optin_no':39,
|
||||||
|
'database_written':False,'network_requests':0,'pdfs_downloaded':False,'approved_for_import':0,
|
||||||
|
'public_rejected_availability':'unknown_accepted_program_reviews_are_not_a_rejected_export',
|
||||||
|
'scope':'140accepted observations;101opt-in review sets, not101rejected records. Source keyword/presentation tags are namespace-scoped; no original OpenReview ID assumption or source-version fusion.'}
|
||||||
|
part=out/'collection-report.json.part';part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n');part.replace(out/'collection-report.json');return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser=argparse.ArgumentParser(prog=prog,description=__doc__);parser.add_argument('--snapshots',required=True);parser.add_argument('--out',required=True)
|
||||||
|
args=parser.parse_args(argv);print(json.dumps(collect(args.out,snapshots=args.snapshots),ensure_ascii=False))
|
||||||
@@ -0,0 +1,202 @@
|
|||||||
|
"""Restore frozen ACM MM 2026 program observations, not publication decisions."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
SOURCE = 'mm_program'
|
||||||
|
SNAPSHOT = 'mm2026-program-and-public-probes-v1'
|
||||||
|
URL = 'https://2026.acmmm.org/site/program-data.json'
|
||||||
|
BODY_SHA = '08e2976ecba2d0d535000b0935f41c85076ec674e518a1df2c5843d9bde4d083'
|
||||||
|
MANIFEST_SHA = 'e43ac6cdfc86bac61305553a044dd6148800b5aa41a5d1bf06c486e90d0ed7b8'
|
||||||
|
TRACK_COUNTS = {'MRAC':35,'SUMAC':9,'NeuroMM':10,'IMPACT':8,'RichMediaGAI':10,
|
||||||
|
'MMSports':16,'MUWS':5,'EReL-MIR':7,'GreenMM':10,'DFF':12,'Main Track':1703,
|
||||||
|
'Grand Challenge':97,'Doctoral Symposium':4,'Datasets':50,'Interactive Art':23,
|
||||||
|
'Reproducibility':1,'Demo and Video':27,'Open Source':14,'Industry Talk':7,
|
||||||
|
'BNI':25,'QoEVMA':4,'HUMA':14,'WIPAI':8,'AMI':11,'Mulsemedia':9,
|
||||||
|
'UAVM':11,'MCHM':7,'JAV-CG':6}
|
||||||
|
CORE = ('paper_id','title','authors','affiliations','abstract','track')
|
||||||
|
PRESENTATION = ('kind','start_time','end_time','poster_id','oral_id')
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def safe_read(path, limit):
|
||||||
|
if any(p.is_symlink() for p in (path, *path.parents)) or not path.is_file() or path.stat().st_size > limit:
|
||||||
|
raise ValueError('Unsafe snapshot path or size')
|
||||||
|
return path.read_bytes()
|
||||||
|
|
||||||
|
|
||||||
|
def load_snapshot(root):
|
||||||
|
directory = Path(root)/SNAPSHOT
|
||||||
|
raw = safe_read(directory/'manifest.json', 128*1024)
|
||||||
|
if digest(raw) != MANIFEST_SHA:
|
||||||
|
raise ValueError('Reviewed manifest hash changed')
|
||||||
|
manifest = json.loads(raw)
|
||||||
|
if manifest.get('database_written') is not False or len(manifest['entries']) != 5:
|
||||||
|
raise ValueError('Unreviewed manifest scope')
|
||||||
|
evidence = {'manifest.json':raw}
|
||||||
|
for receipt in manifest['entries']:
|
||||||
|
name = receipt['file']
|
||||||
|
if Path(name).name != name or name in evidence:
|
||||||
|
raise ValueError('Unsafe or duplicate receipt name')
|
||||||
|
body = safe_read(directory/name, 12*1024**2)
|
||||||
|
if (receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)
|
||||||
|
or receipt['url'] != receipt['final_url']
|
||||||
|
or not receipt['content_type'].startswith('application/json')
|
||||||
|
or datetime.fromisoformat(receipt['at']).utcoffset() is None):
|
||||||
|
raise ValueError('Snapshot receipt/body disagree')
|
||||||
|
if name == 'program-data.json':
|
||||||
|
if receipt['url'] != URL or receipt['http_status'] != 200 or digest(body) != BODY_SHA:
|
||||||
|
raise ValueError('Wrong program source')
|
||||||
|
elif receipt['http_status'] != 403:
|
||||||
|
raise ValueError('Unreviewed public-access result')
|
||||||
|
evidence[name] = body
|
||||||
|
if 'program-data.json' not in evidence:
|
||||||
|
raise ValueError('Missing program source')
|
||||||
|
return evidence
|
||||||
|
|
||||||
|
|
||||||
|
def without_emails(value):
|
||||||
|
"""Keep public session structure; contact arrays remain in raw evidence only."""
|
||||||
|
if isinstance(value, dict):
|
||||||
|
return {k:without_emails(v) for k,v in value.items() if k != 'emails'}
|
||||||
|
if isinstance(value, list):
|
||||||
|
return [without_emails(v) for v in value]
|
||||||
|
return value
|
||||||
|
|
||||||
|
|
||||||
|
def program_locations(node, pointer=''):
|
||||||
|
"""Independent structural guard against silently skipping a new nesting form."""
|
||||||
|
if isinstance(node, dict):
|
||||||
|
if 'paper_id' in node:
|
||||||
|
yield pointer
|
||||||
|
for key,value in node.items():
|
||||||
|
escaped = key.replace('~','~0').replace('/','~1')
|
||||||
|
yield from program_locations(value, pointer+'/'+escaped)
|
||||||
|
elif isinstance(node, list):
|
||||||
|
for i,value in enumerate(node):
|
||||||
|
yield from program_locations(value, pointer+'/'+str(i))
|
||||||
|
|
||||||
|
|
||||||
|
def parse_program(raw):
|
||||||
|
data = json.loads(raw)
|
||||||
|
if (data.get('generated_at') != '2026-11-14T18:36:26.860677+00:00'
|
||||||
|
or data.get('days') != [1,2,3,4,5] or len(data.get('rooms',[])) != 20
|
||||||
|
or len(data.get('entries',[])) != 157):
|
||||||
|
raise ValueError('Unreviewed program edition or census')
|
||||||
|
records = {}; sessions = []; seen = []
|
||||||
|
|
||||||
|
def observe(paper, pointer, session_pointer, subsession_pointer=None):
|
||||||
|
if not set(CORE) <= paper.keys() or set(paper)-set(CORE)-set(PRESENTATION)-{'emails'}:
|
||||||
|
raise ValueError('Unreviewed program fields')
|
||||||
|
for key in ('paper_id','title','abstract','track'):
|
||||||
|
if not isinstance(paper[key],str):
|
||||||
|
raise ValueError('Wrong program field type')
|
||||||
|
for key in ('authors','affiliations'):
|
||||||
|
if not isinstance(paper[key],list) or any(not isinstance(v,str) for v in paper[key]):
|
||||||
|
raise ValueError('Wrong author or affiliation list')
|
||||||
|
if not paper['paper_id'].strip() or not paper['title'].strip() or paper['track'] not in TRACK_COUNTS:
|
||||||
|
raise ValueError('Missing identity/title or unreviewed source track')
|
||||||
|
core = {k:paper[k] for k in CORE}
|
||||||
|
pid = paper['paper_id']; seen.append(pointer)
|
||||||
|
if pid not in records:
|
||||||
|
missing = [k for k in ('authors','abstract') if not paper[k]]
|
||||||
|
records[pid] = {**core,'source':SOURCE,'year':2026,'venue':'MM',
|
||||||
|
'source_locator':{'namespace':'mm_program:program_id','value':pid},
|
||||||
|
'observed_status':'listed_in_official_program',
|
||||||
|
'publication_identity_verified':False,'final_decision_verified':False,
|
||||||
|
'approved_for_import':False,'missing_fields':missing,
|
||||||
|
'tags':[{'namespace':'mm_program:track','value':paper['track']}],
|
||||||
|
'contexts':[]}
|
||||||
|
elif {k:records[pid][k] for k in CORE} != core:
|
||||||
|
raise ValueError('Conflicting core versions for one program ID; review required')
|
||||||
|
records[pid]['contexts'].append({'source_pointer':pointer,'session_pointer':session_pointer,
|
||||||
|
'subsession_pointer':subsession_pointer,
|
||||||
|
'presentation':{k:paper[k] for k in PRESENTATION if k in paper}})
|
||||||
|
|
||||||
|
for i,entry in enumerate(data['entries']):
|
||||||
|
pointer = '/entries/'+str(i)
|
||||||
|
if entry['track'] != '':
|
||||||
|
raise ValueError('Unreviewed outer track; do not override paper.track')
|
||||||
|
session = without_emails({k:v for k,v in entry.items() if k not in ('papers','subsessions')})
|
||||||
|
session.update(source_pointer=pointer,subsessions=[])
|
||||||
|
for j,paper in enumerate(entry.get('papers') or []):
|
||||||
|
observe(paper,pointer+'/papers/'+str(j),pointer)
|
||||||
|
for j,sub in enumerate(entry.get('subsessions') or []):
|
||||||
|
sp = pointer+'/subsessions/'+str(j)
|
||||||
|
saved = without_emails({k:v for k,v in sub.items() if k != 'items'})
|
||||||
|
saved.update(source_pointer=sp,nonpaper_items=[])
|
||||||
|
for k,item in enumerate(sub.get('items') or []):
|
||||||
|
ip = sp+'/items/'+str(k)
|
||||||
|
if 'paper_id' in item:
|
||||||
|
observe(item,ip,pointer,sp)
|
||||||
|
else:
|
||||||
|
saved['nonpaper_items'].append({'source_pointer':ip,'fields':without_emails(item)})
|
||||||
|
session['subsessions'].append(saved)
|
||||||
|
sessions.append(session)
|
||||||
|
if set(seen) != set(program_locations(data)) or len(set(seen)) != len(seen):
|
||||||
|
raise ValueError('Unhandled or duplicated program location')
|
||||||
|
items = list(records.values())
|
||||||
|
if (len(seen) != 4326 or len(items) != 2143
|
||||||
|
or Counter(r['track'] for r in items) != TRACK_COUNTS
|
||||||
|
or sum(not r['abstract'] for r in items) != 216
|
||||||
|
or {r['paper_id'] for r in items if not r['authors']} != {'mfp4971','mfp7801'}):
|
||||||
|
raise ValueError('Program count or missing-field census changed')
|
||||||
|
return items,sessions
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists() or out.is_symlink():
|
||||||
|
raise ValueError('Output exists; use a fresh directory')
|
||||||
|
if any(p.is_symlink() for p in out.parents):
|
||||||
|
raise ValueError('Unsafe output parent')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
evidence = load_snapshot(snapshots)
|
||||||
|
rows,sessions = parse_program(evidence['program-data.json'])
|
||||||
|
(out/'evidence').mkdir()
|
||||||
|
for name,body in evidence.items():
|
||||||
|
(out/'evidence'/name).write_bytes(body)
|
||||||
|
outputs = []
|
||||||
|
for track,count in TRACK_COUNTS.items():
|
||||||
|
slug = re.sub('[^a-z0-9]+','_',track.lower()).strip('_')
|
||||||
|
path = out/('mm-program-2026-'+slug+'.jsonl')
|
||||||
|
raw = ''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in rows if r['track']==track).encode()
|
||||||
|
path.write_bytes(raw)
|
||||||
|
outputs.append({'file':str(path),'file_sha256':digest(raw),'count':count,
|
||||||
|
'track_literal':track,'format':'mm_program_observation_v1_not_ingestion_record',
|
||||||
|
'status':'pending_contribution_and_identity_review','approved_for_import':0})
|
||||||
|
session_raw = ''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in sessions).encode()
|
||||||
|
(out/'sessions.jsonl').write_bytes(session_raw)
|
||||||
|
report = {'status':'collected_pending_contribution_and_identity_review','source':SOURCE,
|
||||||
|
'records':2143,'program_observations':4326,'extra_presentation_observations':2183,
|
||||||
|
'track_counts':TRACK_COUNTS,'files':outputs,'session_count':157,
|
||||||
|
'sessions_file_sha256':digest(session_raw),'missing_abstract_count':216,
|
||||||
|
'missing_author_program_ids':['mfp4971','mfp7801'],
|
||||||
|
'source_body_sha256':BODY_SHA,'source_manifest_sha256':MANIFEST_SHA,
|
||||||
|
'database_written':False,'pdfs_downloaded':False,'network_requests':0,'approved_for_import':0,
|
||||||
|
'public_negative_access':{'anonymous_notes_probes':4,'http_statuses':[403]*4,
|
||||||
|
'negative_decisions_inferred':0,'scope':'Observed endpoints only; not proof of zero public negative decisions.'},
|
||||||
|
'scope':'Program IDs are source locators, not scientific-work or original submission IDs. All28source tracks retained; presentation times/kinds/locators and session context kept separately. No publication status, paper type, DOI, arxiv/OpenReview ID, rejection or withdrawal inferred. Blank fields are not negative decisions. Contribution/publication/work/history review and final whole-database audit remain.'}
|
||||||
|
part = out/'collection-report.json.part'
|
||||||
|
part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n')
|
||||||
|
part.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog,description=__doc__)
|
||||||
|
parser.add_argument('--snapshots',required=True)
|
||||||
|
parser.add_argument('--out',required=True)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
print(json.dumps(collect(args.out,snapshots=args.snapshots),ensure_ascii=False))
|
||||||
@@ -0,0 +1,336 @@
|
|||||||
|
"""Frozen MM 2026 registered publications with separately attributed program evidence.
|
||||||
|
|
||||||
|
All publications survive normalization. A program metadata match is not a
|
||||||
|
scientific-work identity decision; publisher article type is not research type.
|
||||||
|
"""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter, defaultdict
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
from html.parser import HTMLParser
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import unicodedata
|
||||||
|
from urllib.parse import parse_qs, urlsplit
|
||||||
|
|
||||||
|
from paper_atlas.collectors import mm2026 as program
|
||||||
|
|
||||||
|
SOURCE = 'mm_registry_program'
|
||||||
|
SNAPSHOT = 'mm2026-container-registry-v2'
|
||||||
|
MANIFEST_SHA = '128b9d02c72261a66edb9a4537d118bdccab3ed80d49a1e971fefdac399f9855'
|
||||||
|
PARENT_SHA = 'bc3ce7da2914f9ddd198c928b3acb2734a8370add3348c23c5f9f12135381628'
|
||||||
|
PARENT_DOI = '10.1145/3767308'
|
||||||
|
CONTAINER = 'Proceedings of the 34th ACM International Conference on Multimedia'
|
||||||
|
EVENT = "MM '26: The 34th ACM International Conference on Multimedia"
|
||||||
|
TRACKS = {'Main Track':'main','Grand Challenge':'grand_challenge',
|
||||||
|
'Interactive Art':'interactive_art','Datasets':'datasets','Reproducibility':'reproducibility',
|
||||||
|
'BNI':'bni','Open Source':'open_source','Doctoral Symposium':'doctoral_symposium',
|
||||||
|
'Demo and Video':'demo_and_video','Industry Talk':'industry_talk'}
|
||||||
|
BIBLIOGRAPHY = {'DOI','URL','title','original-title','subtitle','short-title','author',
|
||||||
|
'publisher','member','type','container-title','short-container-title','event','abstract',
|
||||||
|
'page','ISBN','isbn-type','published','published-print','published-online','issued',
|
||||||
|
'license','resource','relation','alternative-id'}
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def key(text):
|
||||||
|
return ''.join(c for c in unicodedata.normalize('NFKD',text).casefold() if c.isalnum())
|
||||||
|
|
||||||
|
|
||||||
|
def author_names(values):
|
||||||
|
return Counter(key(v) for v in values)
|
||||||
|
|
||||||
|
|
||||||
|
def load_registry(root, parent_path):
|
||||||
|
directory = Path(root)/SNAPSHOT
|
||||||
|
raw_manifest = program.safe_read(directory/'manifest.json', 128*1024)
|
||||||
|
raw_parent = program.safe_read(Path(parent_path), 128*1024)
|
||||||
|
if digest(raw_manifest) != MANIFEST_SHA or digest(raw_parent) != PARENT_SHA:
|
||||||
|
raise ValueError('Reviewed registry or parent hash changed')
|
||||||
|
manifest = json.loads(raw_manifest);parent = json.loads(raw_parent)['message']
|
||||||
|
if (manifest['status'] != 'complete_exact_container_registry_capture_not_work_or_track_approval'
|
||||||
|
or manifest['unique_registry_DOIs'] != 1918 or manifest['expected_registry_items'] != 1918
|
||||||
|
or len(manifest['pages']) != 8 or manifest['database_written'] is not False
|
||||||
|
or manifest['approved_for_import'] != 0 or manifest['parent_body_sha256'] != PARENT_SHA
|
||||||
|
or parent['DOI'] != PARENT_DOI or parent['type'] != 'proceedings'
|
||||||
|
or parent['title'] != [CONTAINER] or parent['event']['name'] != EVENT
|
||||||
|
or parent['member'] != '320' or parent['ISBN'] != ['9798400722134']):
|
||||||
|
raise ValueError('Incomplete reviewed registry or wrong proceedings')
|
||||||
|
evidence = {'manifest.json':raw_manifest,'parent.json':raw_parent};rows=[];seen=set()
|
||||||
|
for number,receipt in enumerate(manifest['pages'],1):
|
||||||
|
name=receipt['file']
|
||||||
|
if name != f'page-{number:03}.json':
|
||||||
|
raise ValueError('Unexpected registry page filename')
|
||||||
|
body=program.safe_read(directory/name,8*1024**2)
|
||||||
|
url=urlsplit(receipt['url']);query=parse_qs(url.query)
|
||||||
|
expected_filter='container-title:'+CONTAINER+',member:320'
|
||||||
|
if (receipt['http_status'] != 200 or receipt['url'] != receipt['final_url']
|
||||||
|
or url.scheme != 'https' or url.netloc != 'api.crossref.org' or url.path != '/works'
|
||||||
|
or set(query) != {'filter','rows','cursor'} or query['filter'] != [expected_filter]
|
||||||
|
or query['rows'] != ['250'] or not query['cursor'][0]
|
||||||
|
or datetime.fromisoformat(receipt['at']).utcoffset() is None
|
||||||
|
or receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)):
|
||||||
|
raise ValueError('Registry receipt and source body disagree')
|
||||||
|
obj=json.loads(body);message=obj['message'];expected=250 if number<8 else 168
|
||||||
|
if (obj['status'] != 'ok' or message['total-results'] != 1918
|
||||||
|
or len(message['items']) != expected or receipt['items'] != expected):
|
||||||
|
raise ValueError('Incomplete registry pagination')
|
||||||
|
evidence[name]=body
|
||||||
|
for position,original in enumerate(message['items']):
|
||||||
|
doi=original.get('DOI')
|
||||||
|
if not isinstance(doi,str) or not re.fullmatch(r'10\.1145/3767308\.[0-9]+',doi) or doi in seen:
|
||||||
|
raise ValueError('Missing, foreign or repeated publication DOI')
|
||||||
|
seen.add(doi)
|
||||||
|
if (original.get('URL') != 'https://doi.org/'+doi or original.get('type') != 'proceedings-article'
|
||||||
|
or original.get('container-title') != [CONTAINER] or original.get('event',{}).get('name') != EVENT
|
||||||
|
or original.get('member') != '320' or original.get('publisher') != 'ACM'
|
||||||
|
or original.get('published') != {'date-parts':[[2026,11,9]]} or original.get('abstract')):
|
||||||
|
raise ValueError('Wrong registered publication context or changed abstract availability')
|
||||||
|
titles=original.get('title');authors=original.get('author')
|
||||||
|
if (not isinstance(titles,list) or len(titles)!=1 or not isinstance(titles[0],str) or not titles[0].strip()
|
||||||
|
or not isinstance(authors,list) or not authors):
|
||||||
|
raise ValueError('Missing publication title or authors')
|
||||||
|
names=[];flags=[]
|
||||||
|
for i,author in enumerate(authors,1):
|
||||||
|
family=author.get('family');given=author.get('given','')
|
||||||
|
if not isinstance(family,str) or not family.strip() or not isinstance(given,str):
|
||||||
|
raise ValueError('Invalid registered author fields')
|
||||||
|
name=' '.join(x.strip() for x in (given,family) if x.strip())
|
||||||
|
if '@' in name or name.casefold()=='anonymous':
|
||||||
|
raise ValueError('Unreviewed author identity')
|
||||||
|
names.append(name)
|
||||||
|
if not given.strip():flags.append({'kind':'single_registered_name_field_not_split','author_position':i})
|
||||||
|
if re.search(r'<[^>]+>|&[A-Za-z#0-9]+;',titles[0]):
|
||||||
|
flags.append({'kind':'literal_title_markup_or_entity_retained_requires_review'})
|
||||||
|
canonical=json.dumps(original,ensure_ascii=False,sort_keys=True,separators=(',',':')).encode()
|
||||||
|
rows.append({'doi':doi,'title':titles[0],'authors':names,'flags':flags,
|
||||||
|
'fields':{k:v for k,v in original.items() if k in BIBLIOGRAPHY},
|
||||||
|
'original_field_names':sorted(original),'original_canonical_json_sha256':digest(canonical),
|
||||||
|
'receipt':receipt,'source_position':position})
|
||||||
|
if len(rows)!=1918:
|
||||||
|
raise ValueError('Incomplete registered publication census')
|
||||||
|
return evidence,rows
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
class _TitleFormatting(HTMLParser):
|
||||||
|
"""Decode text once, removing only balanced, attribute-free reviewed tags."""
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__(convert_charrefs=True)
|
||||||
|
self.parts=[];self.stack=[];self.valid=True
|
||||||
|
|
||||||
|
def handle_data(self, text):
|
||||||
|
self.parts.append(text)
|
||||||
|
|
||||||
|
def handle_starttag(self, tag, attrs):
|
||||||
|
if tag not in {'i','scp','sup','sub'} or attrs:
|
||||||
|
self.valid=False
|
||||||
|
else:
|
||||||
|
self.stack.append(tag)
|
||||||
|
|
||||||
|
def handle_endtag(self, tag):
|
||||||
|
if not self.stack or self.stack[-1]!=tag:
|
||||||
|
self.valid=False
|
||||||
|
else:
|
||||||
|
self.stack.pop()
|
||||||
|
|
||||||
|
def handle_startendtag(self, tag, attrs):
|
||||||
|
self.valid=False
|
||||||
|
|
||||||
|
def handle_comment(self, text):
|
||||||
|
self.valid=False
|
||||||
|
|
||||||
|
def handle_decl(self, text):
|
||||||
|
self.valid=False
|
||||||
|
|
||||||
|
def handle_pi(self, text):
|
||||||
|
self.valid=False
|
||||||
|
|
||||||
|
def unknown_decl(self, text):
|
||||||
|
self.valid=False
|
||||||
|
|
||||||
|
|
||||||
|
def formatting_text(title):
|
||||||
|
# HTMLParser may silently discard a trailing incomplete start tag.
|
||||||
|
# Check lexical coverage before parsing so no title content can disappear.
|
||||||
|
literal=re.sub(r'</?(?:i|scp|sup|sub)\s*>','',title,flags=re.I)
|
||||||
|
if '<' in literal or '>' in literal:
|
||||||
|
return None
|
||||||
|
parser=_TitleFormatting()
|
||||||
|
parser.feed(title);parser.close()
|
||||||
|
text=''.join(parser.parts)
|
||||||
|
return text if parser.valid and not parser.stack and text.strip() and '<' not in text and '>' not in text else None
|
||||||
|
|
||||||
|
|
||||||
|
FORMATTING_MATCHES = {'10.1145/3767308.3834907': 'mfp0053',
|
||||||
|
'10.1145/3767308.3834936': 'mfp0229',
|
||||||
|
'10.1145/3767308.3834946': 'mfp0297',
|
||||||
|
'10.1145/3767308.3834956': 'mfp0339',
|
||||||
|
'10.1145/3767308.3835035': 'mfp0798',
|
||||||
|
'10.1145/3767308.3835055': 'mfp0903',
|
||||||
|
'10.1145/3767308.3835101': 'mfp1175',
|
||||||
|
'10.1145/3767308.3835374': 'mfp2617',
|
||||||
|
'10.1145/3767308.3835424': 'mfp2909',
|
||||||
|
'10.1145/3767308.3835452': 'mfp3048',
|
||||||
|
'10.1145/3767308.3835460': 'mfp3109',
|
||||||
|
'10.1145/3767308.3835544': 'mfp3554',
|
||||||
|
'10.1145/3767308.3835591': 'mfp3783',
|
||||||
|
'10.1145/3767308.3835667': 'mfp4173',
|
||||||
|
'10.1145/3767308.3835763': 'mfp4778',
|
||||||
|
'10.1145/3767308.3835895': 'mfp5537',
|
||||||
|
'10.1145/3767308.3835959': 'mfp5823',
|
||||||
|
'10.1145/3767308.3836215': 'mfp7332',
|
||||||
|
'10.1145/3767308.3836272': 'mfp7635',
|
||||||
|
'10.1145/3767308.3836293': 'mfp7732',
|
||||||
|
'10.1145/3767308.3836447': 'mfp8648',
|
||||||
|
'10.1145/3767308.3836600': 'mfp9815',
|
||||||
|
'10.1145/3767308.3838656': 'mds177',
|
||||||
|
'10.1145/3767308.3839012': 'mde41',
|
||||||
|
'10.1145/3767308.3839024': 'mde15',
|
||||||
|
'10.1145/3767308.3839028': 'mde02'}
|
||||||
|
|
||||||
|
|
||||||
|
def match_program(rows, observations):
|
||||||
|
titles=defaultdict(list);formatted=defaultdict(list)
|
||||||
|
for p in observations:
|
||||||
|
titles[key(p['title'])].append(p)
|
||||||
|
text=formatting_text(p['title'])
|
||||||
|
if text is not None:formatted[key(text)].append(p)
|
||||||
|
matched=set();kinds=Counter()
|
||||||
|
for row in rows:
|
||||||
|
row.pop('program',None);row.pop('formatting_comparison',None)
|
||||||
|
candidates=titles[key(row['title'])]
|
||||||
|
method='literal_title'
|
||||||
|
# A literal-title author disagreement must not be bypassed by fallback.
|
||||||
|
if not candidates:
|
||||||
|
text=formatting_text(row['title'])
|
||||||
|
if text is not None:
|
||||||
|
candidates=formatted[key(text)]
|
||||||
|
if candidates:method='reviewed_title_formatting'
|
||||||
|
exact=[p for p in candidates if author_names(p['authors'])==author_names(row['authors'])]
|
||||||
|
if len(exact)==1:
|
||||||
|
row['kind']=('unique_program_format_title_fullauthor_agreement_not_identity_approval'
|
||||||
|
if method=='reviewed_title_formatting' else 'unique_program_title_fullauthor_agreement_not_identity_approval')
|
||||||
|
row['program']=exact[0];matched.add(exact[0]['paper_id'])
|
||||||
|
if exact[0]['track'] not in TRACKS:
|
||||||
|
raise ValueError('New registered program track requires review')
|
||||||
|
elif len(exact)>1:row['kind']='multiple_program_ids_same_title_authors_hold'
|
||||||
|
elif candidates:
|
||||||
|
row['kind']=('program_format_author_version_requires_review'
|
||||||
|
if method=='reviewed_title_formatting' else 'program_author_version_requires_review')
|
||||||
|
else:row['kind']='no_exact_program_title_requires_review'
|
||||||
|
row['program_title_candidates']=[p['paper_id'] for p in candidates]
|
||||||
|
row['program_exact_candidates']=[p['paper_id'] for p in exact]
|
||||||
|
if method=='reviewed_title_formatting':
|
||||||
|
row['formatting_comparison']={'method':'balanced_attribute_free_i_scp_sup_sub_and_single_entity_decode_then_nfkd_alnum',
|
||||||
|
'registry_comparison_text':formatting_text(row['title']),
|
||||||
|
'program_comparison_texts':{p['paper_id']:formatting_text(p['title']) for p in candidates},
|
||||||
|
'raw_titles_preserved':True,'work_identity_approved':False}
|
||||||
|
kinds[row['kind']]+=1
|
||||||
|
return dict(kinds),[p for p in observations if p['paper_id'] not in matched]
|
||||||
|
|
||||||
|
|
||||||
|
def correlate(rows, observations):
|
||||||
|
kinds,unmatched=match_program(rows,observations)
|
||||||
|
expected={'unique_program_title_fullauthor_agreement_not_identity_approval':1726,
|
||||||
|
'unique_program_format_title_fullauthor_agreement_not_identity_approval':26,
|
||||||
|
'program_author_version_requires_review':122,
|
||||||
|
'program_format_author_version_requires_review':1,
|
||||||
|
'no_exact_program_title_requires_review':43}
|
||||||
|
actual={r['doi']:r['program']['paper_id'] for r in rows
|
||||||
|
if r['kind']=='unique_program_format_title_fullauthor_agreement_not_identity_approval'}
|
||||||
|
if kinds != expected or actual != FORMATTING_MATCHES:
|
||||||
|
raise ValueError('Reviewed program/publication correspondence changed')
|
||||||
|
return kinds,unmatched
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, registry_snapshots, program_snapshots, parent_snapshot):
|
||||||
|
out=Path(out)
|
||||||
|
if out.exists() or out.is_symlink():
|
||||||
|
raise ValueError('Output exists; inspect prior operation, never overwrite')
|
||||||
|
if any(p.is_symlink() for p in out.parents):
|
||||||
|
raise ValueError('Unsafe output parent')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
registry_evidence,publications=load_registry(registry_snapshots,parent_snapshot)
|
||||||
|
program_evidence=program.load_snapshot(program_snapshots)
|
||||||
|
observations,_=program.parse_program(program_evidence['program-data.json'])
|
||||||
|
kinds,unmatched=correlate(publications,observations)
|
||||||
|
for group,files in [('registry',registry_evidence),('program',program_evidence)]:
|
||||||
|
directory=out/'evidence'/group;directory.mkdir(parents=True)
|
||||||
|
for name,body in files.items():(directory/name).write_bytes(body)
|
||||||
|
buckets=defaultdict(list);missing=0;copied=0;known_tracks=Counter()
|
||||||
|
for item in sorted(publications,key=lambda r:r['doi']):
|
||||||
|
p=item.get('program');doi=item['doi'];receipt=item['receipt']
|
||||||
|
track=TRACKS[p['track']] if p else 'proceedings_unclassified'
|
||||||
|
abstract=p['abstract'] if p and p['abstract'] else None
|
||||||
|
missing+=abstract is None;copied+=abstract is not None
|
||||||
|
if p:known_tracks[p['track']]+=1
|
||||||
|
tags=[{'namespace':SOURCE+':crossref:type','value':'proceedings-article'}]
|
||||||
|
if p:tags.append({'namespace':SOURCE+':mm_program:track','value':p['track']})
|
||||||
|
buckets[track].append({'source':SOURCE,'external_id':doi,'url':'https://doi.org/'+doi,
|
||||||
|
'title':item['title'],'authors':item['authors'],'abstract':abstract,
|
||||||
|
'identifiers':{'doi':doi},'submission_identifiers':[{'scheme':'publication_doi','value':doi}],
|
||||||
|
'venue':'MM','year':2026,'track':track,'status':'accepted',
|
||||||
|
'raw_decision':'Registered publication in MM2026 proceedings; not an original signed decision.',
|
||||||
|
'evidence_url':receipt['url'],'observed_at':receipt['at'],'tags':tags,
|
||||||
|
'metadata':{**({'program_title_formatting_comparison':item['formatting_comparison']} if 'formatting_comparison' in item else {}),'registry_fields':item['fields'],'registry_original_field_names':item['original_field_names'],
|
||||||
|
'registry_source_evidence':{'file':'evidence/registry/'+receipt['file'],'sha256':receipt['sha256'],
|
||||||
|
'url':receipt['url'],'json_pointer':'/message/items/'+str(item['source_position']),
|
||||||
|
'canonical_object_json_sha256':item['original_canonical_json_sha256']},
|
||||||
|
'registry_manifest':{'file':'evidence/registry/manifest.json','sha256':MANIFEST_SHA},
|
||||||
|
'parent_evidence':{'file':'evidence/registry/parent.json','sha256':PARENT_SHA},
|
||||||
|
'program_source_evidence':{'file':'evidence/program/program-data.json','sha256':program.BODY_SHA},
|
||||||
|
'program_fields':p,'program_match_kind':item['kind'],
|
||||||
|
'program_title_candidates':item['program_title_candidates'],
|
||||||
|
'program_exact_candidates':item['program_exact_candidates'],
|
||||||
|
'abstract_source':('official_program_reviewed_title_formatting_correspondence' if item.get('formatting_comparison') else 'official_program_exact_metadata_correspondence') if abstract is not None else 'not_available_from_reviewed_sources',
|
||||||
|
'registry_abstract_available':False,'original_decision_signature_verified':False,
|
||||||
|
'scientific_work_identity_approved':False,'research_type_classified':False,
|
||||||
|
'review_flags':item['flags'],
|
||||||
|
'scope':'Derived publication/program metadata with separate original evidence. All registered articles retained, including art/industry/demo; bibliographic type is not research type. Program correlation does not approve work identity or person aliases. Publication dates are not decision dates. DOI is publication evidence, not original OpenReview ID.'}})
|
||||||
|
outputs=[]
|
||||||
|
for track,rows in sorted(buckets.items()):
|
||||||
|
path=out/f'mm-publications-2026-{track}.jsonl'
|
||||||
|
raw=''.join(json.dumps(row,ensure_ascii=False)+'\n' for row in rows).encode();path.write_bytes(raw)
|
||||||
|
manifest={'source':SOURCE,'parser_version':2,'venue':'MM','year':2026,'track':track,'count':len(rows),
|
||||||
|
'file':str(path),'file_sha256':digest(raw),'list_url':'https://doi.org/'+PARENT_DOI,
|
||||||
|
'status':'partial','missing_abstract_count':sum(row['abstract'] is None for row in rows),
|
||||||
|
'scope':'Registered container partition only; not allMMprogram/workshops, independent works or public-negative coverage.'}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,ensure_ascii=False,indent=2)+'\n');outputs.append(manifest)
|
||||||
|
report={'status':'collected_publications_pending_work_identity_review','records':1918,'unique_registered_dois':1918,
|
||||||
|
'files':outputs,'program_ids':2143,'program_observations':4326,'program_match_counts':kinds,
|
||||||
|
'matched_program_track_counts':dict(known_tracks),'unmatched_program':unmatched,
|
||||||
|
'unmatched_program_count':len(unmatched),'registry_missing_abstract_count':1918,
|
||||||
|
'abstracts_from_exact_program_correspondence':copied,'output_missing_abstract_count':missing,
|
||||||
|
'abstracts_from_literal_program_correspondence':copied-26,'abstracts_from_reviewed_title_formatting':26,
|
||||||
|
'registered_single_name_fields':sum(f['kind']=='single_registered_name_field_not_split' for r in publications for f in r['flags']),
|
||||||
|
'registered_title_markup_flags':sum(any(f['kind']=='literal_title_markup_or_entity_retained_requires_review' for f in r['flags']) for r in publications),
|
||||||
|
'database_written':False,'approved_for_import':0,'network_requests':0,'pdfs_downloaded':False,
|
||||||
|
'public_negative_availability':'unknown_no_negative_decisions_inferred',
|
||||||
|
'scope':'1918registeredpublication DOIs, not1918proven distinct scientificworks.166title/authorvariants remainunclassified,391unmatchedprogram items retained.26formattingcorrespondences retainedseparately;rawtitlesunchanged. All28programtracks remain independent census; article type never implies research. Per-work/source/DOI/history review andfinalwholeDBaudit remain.'}
|
||||||
|
if (sum(f['count'] for f in outputs)!=1918 or copied!=1743 or missing!=175 or len(unmatched)!=391):
|
||||||
|
raise ValueError('Reviewed output partition or abstract census changed')
|
||||||
|
part=out/'collection-report.json.part';part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n');part.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser=argparse.ArgumentParser(prog=prog,description=__doc__)
|
||||||
|
parser.add_argument('--registry-snapshots',required=True)
|
||||||
|
parser.add_argument('--program-snapshots',required=True)
|
||||||
|
parser.add_argument('--parent-snapshot',required=True)
|
||||||
|
parser.add_argument('--out',required=True)
|
||||||
|
args=parser.parse_args(argv)
|
||||||
|
report=collect(args.out,registry_snapshots=args.registry_snapshots,
|
||||||
|
program_snapshots=args.program_snapshots,parent_snapshot=args.parent_snapshot)
|
||||||
|
print(json.dumps({k:report[k] for k in ('status','records','files','program_match_counts',
|
||||||
|
'abstracts_from_exact_program_correspondence','output_missing_abstract_count',
|
||||||
|
'unmatched_program_count','database_written','approved_for_import')},ensure_ascii=False))
|
||||||
@@ -0,0 +1,218 @@
|
|||||||
|
"""Official NeurIPS program metadata, with presentations separate from submissions."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter, defaultdict
|
||||||
|
import fcntl
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import tempfile
|
||||||
|
from urllib.parse import parse_qs, urlsplit
|
||||||
|
|
||||||
|
from paper_atlas.db.submissions import timestamp
|
||||||
|
from paper_atlas.io.http import fetch
|
||||||
|
|
||||||
|
URL = 'https://neurips.cc/static/virtual/data/neurips-2026-orals-posters.json'
|
||||||
|
GROUPS = {
|
||||||
|
'https://openreview.net/group?id=NeurIPS.cc/2026/Conference': 'main',
|
||||||
|
'https://openreview.net/group?id=NeurIPS.cc/2026/Evaluations_and_Datasets_Track': 'evaluations_datasets',
|
||||||
|
'https://openreview.net/group?id=NeurIPS.cc/2026/Position_Paper_Track': 'position',
|
||||||
|
}
|
||||||
|
ACCEPTED = {'Accept', 'Accept (poster)', 'Accept (spotlight)', 'Accept (oral)',
|
||||||
|
'Accept (Spotlight)', 'Accept (Oral)'}
|
||||||
|
LIMIT = 64*1024**2
|
||||||
|
|
||||||
|
|
||||||
|
def forum_id(url):
|
||||||
|
if not isinstance(url, str):
|
||||||
|
return None
|
||||||
|
try:
|
||||||
|
parsed = urlsplit(url)
|
||||||
|
except ValueError:
|
||||||
|
return None
|
||||||
|
query = parse_qs(parsed.query, keep_blank_values=True)
|
||||||
|
if (parsed.scheme != 'https' or parsed.netloc != 'openreview.net' or
|
||||||
|
parsed.path != '/forum' or parsed.fragment or set(query) != {'id'} or
|
||||||
|
len(query['id']) != 1 or not re.fullmatch(r'[A-Za-z0-9_-]{8,32}', query['id'][0])):
|
||||||
|
return None
|
||||||
|
return query['id'][0]
|
||||||
|
|
||||||
|
|
||||||
|
def parse_program(data, evidence):
|
||||||
|
root = json.loads(data)
|
||||||
|
if (not isinstance(root, dict) or not {'count', 'next', 'previous', 'results'} <= set(root) or
|
||||||
|
not isinstance(root.get('results'), list) or
|
||||||
|
type(root.get('count')) is not int or root['count'] != len(root['results']) or
|
||||||
|
root.get('next') is not None or root.get('previous') is not None):
|
||||||
|
raise ValueError('Expected one complete, nonpaginated public program snapshot')
|
||||||
|
if not root['results']:
|
||||||
|
raise ValueError('Empty program snapshot')
|
||||||
|
events, grouped, held, duplicates = {}, defaultdict(list), [], []
|
||||||
|
for index, event in enumerate(root['results']):
|
||||||
|
if not isinstance(event, dict) or type(event.get('id')) is not int or event['id'] <= 0:
|
||||||
|
raise ValueError('Missing stable event ID')
|
||||||
|
eid = event['id']
|
||||||
|
if eid in events:
|
||||||
|
first_index, first = events[eid]
|
||||||
|
if first != event:
|
||||||
|
raise ValueError('Conflicting payloads share one publisher event ID')
|
||||||
|
duplicates.append({'event_id': eid, 'first_row': first_index, 'duplicate_row': index})
|
||||||
|
continue
|
||||||
|
events[eid] = index, event
|
||||||
|
origin = event.get('sourceurl')
|
||||||
|
reason = None
|
||||||
|
if not isinstance(origin, str) or origin not in GROUPS:
|
||||||
|
reason = ('journal_presentation_requires_original_publication_relation'
|
||||||
|
if isinstance(origin, str) and origin.startswith('journal-track/') else
|
||||||
|
'bridging_future_scope_requires_review' if origin == 'neurips2026-bridging-the-future' else
|
||||||
|
'unreviewed_or_missing_submission_origin')
|
||||||
|
elif event.get('visible') is not True:
|
||||||
|
reason = 'program_visibility_requires_review'
|
||||||
|
elif not isinstance(event.get('decision'), str) or event['decision'] not in ACCEPTED:
|
||||||
|
reason = 'unreviewed_decision_label'
|
||||||
|
elif event.get('event_type') not in ('Poster', 'Oral'):
|
||||||
|
reason = 'unreviewed_presentation_type'
|
||||||
|
forum = forum_id(event.get('paper_url'))
|
||||||
|
if reason is None and forum is None:
|
||||||
|
reason = 'missing_or_invalid_original_submission_link'
|
||||||
|
if reason:
|
||||||
|
held.append({'row': index, 'event_id': eid, 'reason': reason, 'raw_event': event})
|
||||||
|
continue
|
||||||
|
if (not isinstance(event.get('name'), str) or not event['name'].strip() or
|
||||||
|
not isinstance(event.get('authors'), list) or not event['authors'] or
|
||||||
|
any(not isinstance(a, dict) or not isinstance(a.get('fullname'), str) or
|
||||||
|
not a['fullname'].strip() for a in event['authors'])):
|
||||||
|
raise ValueError('Empty title or incomplete ordered author list')
|
||||||
|
if not isinstance(event.get('virtualsite_url'), str) or not re.fullmatch(
|
||||||
|
r'/virtual/2026/(?:poster|oral)/'+str(eid), event['virtualsite_url']):
|
||||||
|
raise ValueError('Publisher event URL differs from year/type/ID')
|
||||||
|
grouped[forum].append((index, event))
|
||||||
|
records = []
|
||||||
|
folded = 0
|
||||||
|
for forum, observations in sorted(grouped.items()):
|
||||||
|
# Signed sourceid values are preserved verbatim in the presentation
|
||||||
|
# evidence; they are not independent submissions or scientific works.
|
||||||
|
signatures = {json.dumps([e['name'], e['authors'], e['sourceurl'], e['decision']],
|
||||||
|
ensure_ascii=False, sort_keys=True) for _, e in observations}
|
||||||
|
if len(signatures) != 1:
|
||||||
|
raise ValueError('Title, authors, origin or decision disagree for one forum')
|
||||||
|
posters = [e for _, e in observations if e['event_type'] == 'Poster']
|
||||||
|
event = posters[0] if posters else observations[0][1]
|
||||||
|
track = GROUPS[event['sourceurl']]
|
||||||
|
presentations = []
|
||||||
|
for index, e in observations:
|
||||||
|
presentations.append({'row': index, 'event_id': e['id'], 'uid': e.get('uid'),
|
||||||
|
'sourceid': e.get('sourceid'), 'event_type': e['event_type'],
|
||||||
|
'eventtype': e.get('eventtype'), 'program_url': e['virtualsite_url'],
|
||||||
|
'session': e.get('session'), 'starttime': e.get('starttime'),
|
||||||
|
'endtime': e.get('endtime'), 'parent_id': e.get('parent_id'),
|
||||||
|
'raw_decision': e['decision']})
|
||||||
|
formats = sorted({e['event_type'] for _, e in observations})
|
||||||
|
records.append({
|
||||||
|
'source': 'neurips_program', 'external_id': '2026/'+forum,
|
||||||
|
'url': event['paper_url'], 'title': event['name'], 'abstract': None,
|
||||||
|
'authors': [a['fullname'] for a in event['authors']],
|
||||||
|
'identifiers': {'openreview': forum}, 'venue': 'NeurIPS', 'venue_aliases': [],
|
||||||
|
'year': 2026, 'track': track, 'status': 'accepted',
|
||||||
|
'observed_at': evidence['observed_at'], 'raw_decision': event['decision'],
|
||||||
|
'metadata': {'source_group_url': event['sourceurl'], 'forum_id': forum,
|
||||||
|
'author_affiliations': [a.get('institution') for a in event['authors']],
|
||||||
|
'presentations': presentations, 'source_evidence': evidence,
|
||||||
|
'publication_evidence': 'Explicit accepted label in official conference program; not an original decision note or proof of proceedings publication.',
|
||||||
|
'decision_date_precision': 'not_supplied',
|
||||||
|
'missing_abstract_reason': 'Program JSON does not contain abstracts; paper HTML/PDF not fetched by this command.',
|
||||||
|
'identity_scope_note': 'One record per original forum, not one scientific work per event. Other venues/versions require independent identity review.'},
|
||||||
|
'tags': [{'namespace': 'neurips_program:presentation_format', 'value': value}
|
||||||
|
for value in formats]})
|
||||||
|
folded += len(observations)-1
|
||||||
|
if not records:
|
||||||
|
raise ValueError('No reviewed submission origins found')
|
||||||
|
if root['count'] != len(records)+len(held)+len(duplicates)+folded:
|
||||||
|
raise ValueError('Program partition failed')
|
||||||
|
return records, held, duplicates, folded
|
||||||
|
|
||||||
|
|
||||||
|
def read_program(cache, snapshots):
|
||||||
|
if bool(cache) == bool(snapshots):
|
||||||
|
raise ValueError('Choose exactly one cache or frozen snapshot directory')
|
||||||
|
if snapshots:
|
||||||
|
base = Path(snapshots).resolve()
|
||||||
|
matches = [r for r in json.loads((base/'receipts.json').read_text())['receipts'] if r['url'] == URL]
|
||||||
|
if len(matches) != 1:
|
||||||
|
raise ValueError('Expected one official program receipt')
|
||||||
|
r = matches[0]; filename = Path(r['file'])
|
||||||
|
if (filename.is_absolute() or '..' in filename.parts or
|
||||||
|
not (base/filename).resolve().is_relative_to(base) or r.get('status') != 200):
|
||||||
|
raise ValueError('Unsafe or unsuccessful frozen program response')
|
||||||
|
data = (base/filename).read_bytes()
|
||||||
|
if len(data) > LIMIT or len(data) != r['bytes'] or hashlib.sha256(data).hexdigest() != r['sha256']:
|
||||||
|
raise ValueError('Frozen program checksum/size mismatch')
|
||||||
|
observed = r['finished_at']
|
||||||
|
else:
|
||||||
|
data = fetch(URL, cache, LIMIT, delay=1)
|
||||||
|
r = json.loads((Path(cache)/hashlib.sha256(URL.encode()).hexdigest()).with_suffix('.json').read_text())
|
||||||
|
observed = r['retrieved_at']
|
||||||
|
timestamp(observed)
|
||||||
|
return data, {'url': URL, 'sha256': hashlib.sha256(data).hexdigest(), 'bytes': len(data),
|
||||||
|
'file': 'evidence/'+hashlib.sha256(data).hexdigest()+'.json',
|
||||||
|
'observed_at': observed, 'http_status': 200, 'response_receipt': r}
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, cache=None, snapshots=None):
|
||||||
|
out = Path(out); out.mkdir(parents=True, exist_ok=True)
|
||||||
|
report_name = 'neurips-program-2026.json'
|
||||||
|
with (out/'neurips-program.collect.lock').open('a') as lock:
|
||||||
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||||
|
if (out/report_name).exists():
|
||||||
|
raise ValueError('Completed collection exists; use a new directory and reconcile before refresh')
|
||||||
|
data, evidence = read_program(cache, snapshots)
|
||||||
|
records, held, duplicates, folded = parse_program(data, evidence)
|
||||||
|
groups = defaultdict(list)
|
||||||
|
for r in records:
|
||||||
|
groups[r['track']].append(r)
|
||||||
|
files = []
|
||||||
|
with tempfile.TemporaryDirectory(prefix='.neurips-program-', dir=out) as temporary:
|
||||||
|
stage = Path(temporary); (stage/'evidence').mkdir()
|
||||||
|
(stage/evidence['file']).write_bytes(data)
|
||||||
|
for track, group in sorted(groups.items()):
|
||||||
|
p = stage/f'neurips-program-2026-{track}.jsonl'
|
||||||
|
p.write_text(''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in group))
|
||||||
|
manifest = {'source': 'neurips_program', 'parser_version': 1, 'venue': 'NeurIPS',
|
||||||
|
'year': 2026, 'track': track, 'count': len(group), 'file': str(out/p.name),
|
||||||
|
'file_sha256': hashlib.sha256(p.read_bytes()).hexdigest(), 'list_url': URL,
|
||||||
|
'status': 'partial', 'missing_abstract_count': len(group),
|
||||||
|
'note': 'Reviewed forum-level program partition; not full current proceedings, abstracts, public rejection coverage or scientific-work dedup.'}
|
||||||
|
p.with_suffix('.manifest.json').write_text(json.dumps(manifest, ensure_ascii=False, indent=2)+'\n')
|
||||||
|
files.append(manifest)
|
||||||
|
result = {'source': 'neurips_program', 'year': 2026, 'finished_at': timestamp(),
|
||||||
|
'status': 'collected_pending_identity_review', 'records': len(records), 'files': files,
|
||||||
|
'held_presentations': held, 'exact_duplicate_event_rows': duplicates,
|
||||||
|
'folded_extra_presentation_rows': folded, 'source_receipts': [evidence],
|
||||||
|
'database_written': False, 'pdfs_downloaded': False,
|
||||||
|
'note': 'Public program only; rejected opt-in availability needs a separate source. Journals, bridging and missing origins remain held; absence is not rejection.'}
|
||||||
|
(stage/report_name).write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n')
|
||||||
|
(out/'evidence').mkdir(exist_ok=True)
|
||||||
|
for p in (stage/'evidence').iterdir():
|
||||||
|
p.replace(out/'evidence'/p.name)
|
||||||
|
for p in stage.iterdir():
|
||||||
|
if p.is_file() and p.name != report_name:
|
||||||
|
p.replace(out/p.name)
|
||||||
|
(stage/report_name).replace(out/report_name)
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog)
|
||||||
|
parser.add_argument('--out', required=True)
|
||||||
|
source = parser.add_mutually_exclusive_group(required=True)
|
||||||
|
source.add_argument('--cache')
|
||||||
|
source.add_argument('--snapshots')
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
result = collect(args.out, cache=args.cache, snapshots=args.snapshots)
|
||||||
|
print(json.dumps({'source': result['source'], 'year': 2026, 'status': result['status'],
|
||||||
|
'records': result['records'], 'files': result['files'],
|
||||||
|
'held_presentations': len(result['held_presentations']),
|
||||||
|
'exact_duplicate_event_rows': len(result['exact_duplicate_event_rows']),
|
||||||
|
'folded_extra_presentation_rows': result['folded_extra_presentation_rows'],
|
||||||
|
'report_file': str(Path(args.out)/'neurips-program-2026.json'),
|
||||||
|
'database_written': False, 'pdfs_downloaded': False}, ensure_ascii=False))
|
||||||
@@ -0,0 +1,172 @@
|
|||||||
|
"""Frozen RecSys 2026 accepted-contribution observations, preserving source variants."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import datetime
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, Node, clean
|
||||||
|
|
||||||
|
SOURCE = 'recsys_program'
|
||||||
|
SNAPSHOT = 'remaining2026-listings-research-v2'
|
||||||
|
MANIFEST_SHA = 'fea7b710587636db06794ee195cd083634c41e9054a217859d9dd0ccdd1afab9'
|
||||||
|
URL = 'https://recsys.acm.org/recsys26/contributions/'
|
||||||
|
TRACKS = [('long', 'Long Papers', 65), ('short', 'Short Papers', 29),
|
||||||
|
('past_present_future', 'PPF Papers', 13), ('reproducibility', 'Reproducibility', 4),
|
||||||
|
('resource', 'Resource', 7), ('industry', 'Industry', 83),
|
||||||
|
('research_practice_notes', 'R&P Notes', 34), ('demo', 'Demos', 18),
|
||||||
|
('doctoral_symposium', 'Doctoral', 10)]
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def one(items):
|
||||||
|
if len(items) != 1:
|
||||||
|
raise ValueError('Expected exactly one reviewed source field')
|
||||||
|
return items[0]
|
||||||
|
|
||||||
|
|
||||||
|
def load_snapshot(root):
|
||||||
|
root = Path(root); directory = root/SNAPSHOT; path = directory/'manifest.json'
|
||||||
|
if root.is_symlink() or directory.is_symlink() or path.is_symlink() or path.stat().st_size > 128*1024:
|
||||||
|
raise ValueError('Unsafe snapshot manifest')
|
||||||
|
raw = path.read_bytes()
|
||||||
|
if digest(raw) != MANIFEST_SHA:
|
||||||
|
raise ValueError('Reviewed manifest hash changed')
|
||||||
|
manifest = json.loads(raw)
|
||||||
|
if manifest['status'] != 'research_completed_not_coverage_or_import_approval' or manifest['database_written'] is not False:
|
||||||
|
raise ValueError('Unreviewed snapshot scope')
|
||||||
|
receipt = one([e for e in manifest['entries'] if e['venue'] == 'RecSys' and e['label'] == 'contributions'])
|
||||||
|
name = receipt['file']; path = directory/name
|
||||||
|
if Path(name).name != name or path.is_symlink() or path.stat().st_size > 4*1024**2:
|
||||||
|
raise ValueError('Unsafe snapshot body')
|
||||||
|
body = path.read_bytes()
|
||||||
|
if (receipt['url'] != URL or receipt['final_url'] != URL or receipt['http_status'] != 200
|
||||||
|
or not receipt['content_type'].startswith('text/html') or receipt['bytes'] != len(body)
|
||||||
|
or receipt['sha256'] != digest(body) or datetime.fromisoformat(receipt['at']).utcoffset() is None):
|
||||||
|
raise ValueError('Snapshot receipt/body disagree')
|
||||||
|
return receipt, {'manifest.json': raw, name: body}, body
|
||||||
|
|
||||||
|
|
||||||
|
def body_text(node, doi_url):
|
||||||
|
if not isinstance(node, Node):
|
||||||
|
return node
|
||||||
|
if node.tag in ('script', 'style') or (node.tag == 'a' and node.attrs.get('href') == doi_url):
|
||||||
|
return ''
|
||||||
|
text = ''.join(body_text(c, doi_url) for c in node.children)
|
||||||
|
return ' '+text+' ' if node.tag in ('p','div','li','br','section') else text
|
||||||
|
|
||||||
|
|
||||||
|
def parse_program(raw):
|
||||||
|
doc = Document(raw.decode('utf-8')); nodes = list(doc.root.walk())
|
||||||
|
if not any(n.tag == 'title' and 'RecSys 26 - Accepted Contributions' in n.text() for n in nodes):
|
||||||
|
raise ValueError('Wrong conference edition')
|
||||||
|
menu = one([n for n in nodes if n.tag == 'ul' and n.attrs.get('id') == 'tab-1'])
|
||||||
|
labels = [(n.attrs.get('href'), clean(n.text())) for n in menu.walk() if n.tag == 'a']
|
||||||
|
if labels != [(f'#content-tab-1-{i}', label) for i, (_,label,_) in enumerate(TRACKS)]:
|
||||||
|
raise ValueError('Unreviewed track menu')
|
||||||
|
observations = []; seen = set()
|
||||||
|
for tab, (track, label, count) in enumerate(TRACKS):
|
||||||
|
section = one([n for n in nodes if n.tag == 'div' and n.attrs.get('id') == f'content-tab-1-{tab}'])
|
||||||
|
listing = one([n for n in section.walk() if n.tag == 'ul' and n.attrs.get('id') == f'accordion-{tab+1}'])
|
||||||
|
rows = [n for n in listing.children if isinstance(n, Node) and n.tag == 'li']
|
||||||
|
if len(rows) != count:
|
||||||
|
raise ValueError('Incomplete reviewed track census')
|
||||||
|
for position, row in enumerate(rows, 1):
|
||||||
|
anchor = one([n for n in row.children if isinstance(n, Node) and n.tag == 'a'])
|
||||||
|
locator = anchor.attrs.get('rel', '').removeprefix('#')
|
||||||
|
if not re.fullmatch(f'accordion-{tab+1}-slide-[0-9]+', locator) or locator in seen:
|
||||||
|
raise ValueError('Missing or repeated program locator')
|
||||||
|
seen.add(locator)
|
||||||
|
types = [n for n in anchor.walk() if n.tag == 'span' and n.attrs.get('class') == 'paper-type']
|
||||||
|
type_node = one(types); type_literal = type_node.attrs.get('title'); abbreviation = clean(type_node.text())
|
||||||
|
expected_type = [('Research','RES'),('Research','RES'),('PastPresentFuture','PPF'),
|
||||||
|
('Research','REPR'),('Research','RSC'),('Research','IND'),('Research','RPN'),('Research','DEMO'),('Doctoral Symposium','DS')][tab]
|
||||||
|
if (type_literal,abbreviation) != expected_type:
|
||||||
|
raise ValueError('Unreviewed source contribution type')
|
||||||
|
title = clean(''.join(c.text() if isinstance(c, Node) else c for c in anchor.children
|
||||||
|
if not isinstance(c, Node) or c.tag not in ('span','em','br')))
|
||||||
|
authors_literal = clean(one([n for n in anchor.walk() if n.tag == 'em']).text())
|
||||||
|
if not authors_literal.startswith('by '):
|
||||||
|
raise ValueError('Missing program author prefix')
|
||||||
|
authors_literal = authors_literal[3:]
|
||||||
|
authors = [n.strip() for n in re.split(r',\s+|\s+and\s+', authors_literal)]
|
||||||
|
if not title or any(not n or '@' in n for n in authors):
|
||||||
|
raise ValueError('Missing title or program author')
|
||||||
|
body = one([n for n in row.children if isinstance(n, Node) and n.tag == 'div'])
|
||||||
|
doi_url = one([n.attrs.get('href') for n in body.walk() if n.tag == 'a'
|
||||||
|
and n.attrs.get('href', '').startswith('https://doi.org/')])
|
||||||
|
if not re.fullmatch(r'https://doi.org/10\.1145/3773078\.[0-9]+', doi_url):
|
||||||
|
raise ValueError('Wrong publication DOI family')
|
||||||
|
abstract = clean(body_text(body, doi_url))
|
||||||
|
if len(abstract) < 100 or any(n.tag in ('math','sup','sub','img','svg') for n in body.walk()):
|
||||||
|
raise ValueError('Missing abstract or unreviewed structured markup')
|
||||||
|
observations.append({'doi': doi_url.removeprefix('https://doi.org/'), 'tab': tab, 'position': position,
|
||||||
|
'locator': locator, 'title': title, 'authors': authors, 'authors_literal': authors_literal,
|
||||||
|
'abstract': abstract, 'type_labels': [type_literal], 'source_type_abbreviation': abbreviation,
|
||||||
|
'track': track, 'track_label': label})
|
||||||
|
counts = Counter(r['doi'] for r in observations)
|
||||||
|
if len(observations) != 263 or len(counts) != 260 or {d:n for d,n in counts.items() if n>1} != {
|
||||||
|
'10.1145/3773078.3831839':2,'10.1145/3773078.3831819':2,'10.1145/3773078.3831861':2}:
|
||||||
|
raise ValueError('Unreviewed DOI observation census')
|
||||||
|
for item in observations:
|
||||||
|
item['review_flags'] = ['repeated_program_doi_preserve_versions_requires_review'] if counts[item['doi']] > 1 else []
|
||||||
|
return observations
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
receipt, files, body = load_snapshot(snapshots); observations = parse_program(body)
|
||||||
|
evidence = out/'evidence'; evidence.mkdir()
|
||||||
|
for name, raw in files.items():
|
||||||
|
(evidence/name).write_bytes(raw)
|
||||||
|
outputs = []
|
||||||
|
for track,label,count in TRACKS:
|
||||||
|
rows = []
|
||||||
|
for item in observations:
|
||||||
|
if item['track'] != track:
|
||||||
|
continue
|
||||||
|
doi = item['doi']; url = URL+'#'+item['locator']
|
||||||
|
rows.append({'source': SOURCE, 'external_id': '2026/'+item['locator'], 'url': url,
|
||||||
|
'title': item['title'], 'authors': item['authors'], 'abstract': item['abstract'],
|
||||||
|
'identifiers': {'doi': doi}, 'submission_identifiers': [{'scheme':'publication_doi','value':doi}],
|
||||||
|
'venue':'RecSys', 'year':2026, 'track':track, 'status':'accepted', 'observed_at':receipt['at'],
|
||||||
|
'evidence_url':URL, 'raw_decision':'Listed as an accepted contribution in the official RecSys2026 '+label+' section; not a signed review decision.',
|
||||||
|
'tags':[{'namespace':SOURCE+':contribution_type','value':item['type_labels'][0]}],
|
||||||
|
'metadata':{'program_fields':item, 'program_receipt':receipt,
|
||||||
|
'source_evidence':{'file':'evidence/'+receipt['file'],'sha256':receipt['sha256'],'url':URL},
|
||||||
|
'source_manifest':{'file':'evidence/manifest.json','sha256':MANIFEST_SHA},
|
||||||
|
'original_decision_signature_verified':False,
|
||||||
|
'author_scope':'Source-reported comma/and-delimited names; original literal retained. Not verified publisher person identities or aliases.',
|
||||||
|
'doi_scope':'DOI asserted by official program hyperlink, not independently verified live publisher metadata.',
|
||||||
|
'abstract_transform':'HTML display text and whitespace normalization; source retained. No PDF or cross-source fusion.',
|
||||||
|
'scope':'Program observation keyed by source locator. Repeated DOI variants retained separately and flagged, not counted as distinct scientific works. Requires DOI/work/history review before import.'}})
|
||||||
|
path = out/f'recsys-program-2026-{track}.jsonl';raw=''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in rows).encode();path.write_bytes(raw)
|
||||||
|
manifest={'source':SOURCE,'parser_version':1,'venue':'RecSys','year':2026,'track':track,'count':len(rows),
|
||||||
|
'file':str(path),'file_sha256':digest(raw),'list_url':URL,'status':'partial','missing_abstract_count':0,
|
||||||
|
'scope':'Official accepted-contribution observations; unique DOI counts differ from observation counts.'}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)+'\n');outputs.append(manifest)
|
||||||
|
report={'status':'collected_pending_identity_review','records':263,'unique_dois':260,'files':outputs,
|
||||||
|
'track_counts':{track:count for track,_,count in TRACKS},'missing_abstract_count':0,
|
||||||
|
'duplicate_doi_observations':6,'repeated_dois':{d:n for d,n in Counter(r['doi'] for r in observations).items() if n>1},
|
||||||
|
'source_manifest_sha256':MANIFEST_SHA,'database_written':False,'network_requests':0,'pdfs_downloaded':False,
|
||||||
|
'approved_for_import':0,'public_rejected_availability':'unknown_no_public_export_verified',
|
||||||
|
'scope':'263 program observations for260 DOIs, not263 unique papers or final work identities. Track/type labels are source scoped. Not complete public-negative coverage or a final whole-database audit.'}
|
||||||
|
part=out/'collection-report.json.part';part.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n');part.replace(out/'collection-report.json');return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status':'failed_not_complete','error_type':type(error).__name__,'database_written':False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser=argparse.ArgumentParser(prog=prog,description=__doc__)
|
||||||
|
parser.add_argument('--snapshots',required=True);parser.add_argument('--out',required=True)
|
||||||
|
args=parser.parse_args(argv);print(json.dumps(collect(args.out,snapshots=args.snapshots),ensure_ascii=False))
|
||||||
@@ -0,0 +1,222 @@
|
|||||||
|
"""Reviewed RSS 2026 publisher metadata, with separate program evidence."""
|
||||||
|
REVIEWED_AUTHOR_VARIANTS = {16: '04de7e813b7c68b5c1764e90fe41bcd2e45259d46749bb2cc54bac5150c8f25b', 27: '975f6adc4d15045f9c8829689a759c5e3a44eecb3410a18ff99b37226d9c8b97', 78: '8208d0950468295c2b73dae28658566b1c726f65a4167f490039cf3449aa2db4', 132: '31eee6289de404f3d859dd0d9de4dd40e968f86b89312de41df87ae01c529711', 199: 'ced0bc9e38b28b4a490799fd94315f104ece0df28be428b3cc417db8ea797b9d', 204: '7c6959e6668bf44af09959388f6b9e2d883a76106bf7f191dc9949368c969ee5'}
|
||||||
|
import argparse
|
||||||
|
from datetime import datetime, date
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import unicodedata
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, Node, clean
|
||||||
|
from paper_atlas.io.http import fetch
|
||||||
|
|
||||||
|
SOURCE = 'rss_proceedings'
|
||||||
|
ROOT = 'https://www.roboticsproceedings.org/rss22/'
|
||||||
|
HEADING = 'Robotics: Science and Systems XXII'
|
||||||
|
PAGES = {'index.html': ROOT,
|
||||||
|
'accepted.html': 'https://roboticsconference.org/2026/program/papers/',
|
||||||
|
'cfp.html': 'https://roboticsconference.org/2026/information/cfp/',
|
||||||
|
'sponsor-demos.html': 'https://roboticsconference.org/2026/program/demos/'}
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def one(values):
|
||||||
|
if len(values) != 1:
|
||||||
|
raise ValueError('Expected one reviewed HTML field')
|
||||||
|
return values[0]
|
||||||
|
|
||||||
|
|
||||||
|
def comparable(value):
|
||||||
|
return unicodedata.normalize('NFC', clean(value))
|
||||||
|
|
||||||
|
|
||||||
|
def parse_lists(index_html, program_html, policy_html, sponsors_html):
|
||||||
|
policy = clean(Document(policy_html).root.text())
|
||||||
|
for text in ('RSS 2026 will not have a separate category for demo papers.',
|
||||||
|
'papers will only be public after acceptance', 'reviews will not be publicly visible'):
|
||||||
|
if text not in policy:
|
||||||
|
raise ValueError('Reviewed 2026 publication/demo/privacy policy changed')
|
||||||
|
if [clean(n.text()) for n in Document(sponsors_html).root.walk() if n.tag == 'h1'] != ['Sponsor Demos']:
|
||||||
|
raise ValueError('Sponsor activity page changed; never classify these as papers')
|
||||||
|
nodes = list(Document(index_html).root.walk())
|
||||||
|
if [clean(n.text()) for n in nodes if n.tag == 'h1'] != [HEADING]:
|
||||||
|
raise ValueError('Unexpected publisher edition')
|
||||||
|
index = {}
|
||||||
|
for row in (n for n in nodes if n.tag == 'tr'):
|
||||||
|
links = [n for n in row.walk() if n.tag == 'a' and re.fullmatch(r'p[0-9]{3}\.html', n.attrs.get('href', ''))]
|
||||||
|
if not links:
|
||||||
|
continue
|
||||||
|
link = one(links); pid = int(link.attrs['href'][1:4])
|
||||||
|
if pid in index:
|
||||||
|
raise ValueError('Repeated publisher ID')
|
||||||
|
index[pid] = {'title': clean(link.text()), 'authors': clean(one([n for n in row.walk() if n.tag == 'i']).text())}
|
||||||
|
program = {}
|
||||||
|
if [clean(n.text()) for n in Document(program_html).root.walk() if n.tag == 'h1'] != ['Accepted Papers']:
|
||||||
|
raise ValueError('Unexpected accepted program')
|
||||||
|
for row in (n for n in Document(program_html).root.walk() if n.tag == 'tr'):
|
||||||
|
cells = [n for n in row.children if isinstance(n, Node) and n.tag == 'td']
|
||||||
|
if not cells:
|
||||||
|
continue
|
||||||
|
if len(cells) != 4 or not re.fullmatch('[1-9][0-9]*', clean(cells[0].text())):
|
||||||
|
raise ValueError('Unexpected program row')
|
||||||
|
pid = int(clean(cells[0].text()))
|
||||||
|
if pid in program:
|
||||||
|
raise ValueError('Repeated program ID')
|
||||||
|
preview = clean(' '.join(n for n in cells[3].children if isinstance(n, str)))
|
||||||
|
full = one([clean(n.text()) for n in cells[3].walk() if n.tag == 'div'
|
||||||
|
and 'display:none' in n.attrs.get('style', '').replace(' ', '')])
|
||||||
|
if not preview or not full.startswith(preview):
|
||||||
|
raise ValueError('Author preview is not a prefix of complete author evidence')
|
||||||
|
links = [n.attrs.get('href') for n in cells[2].walk() if n.tag == 'a']
|
||||||
|
if links != [f'/2026/program/papers/{pid}/']:
|
||||||
|
raise ValueError('Program display link changed')
|
||||||
|
program[pid] = {'display_id': pid, 'title': clean(cells[2].text()), 'authors_complete': full,
|
||||||
|
'authors_preview': preview, 'session': clean(cells[1].text()),
|
||||||
|
'program_url': 'https://roboticsconference.org'+links[0]}
|
||||||
|
if set(index) != set(program) or set(index) != set(range(1, 211)):
|
||||||
|
raise ValueError('Reviewed 210-entry source census changed')
|
||||||
|
return index, program
|
||||||
|
|
||||||
|
|
||||||
|
def parse_paper(html, pid, index, program):
|
||||||
|
nodes = list(Document(html).root.walk()); meta = {}
|
||||||
|
for n in nodes:
|
||||||
|
if n.tag == 'meta' and n.attrs.get('name', '').startswith('citation_'):
|
||||||
|
meta.setdefault(n.attrs['name'], []).append(n.attrs.get('content', ''))
|
||||||
|
title = one(meta.get('citation_title', [])); authors = meta.get('citation_author', [])
|
||||||
|
day = one(meta.get('citation_publication_date', []))
|
||||||
|
if not re.fullmatch(r'2026/[0-9]{2}/[0-9]{2}', day):
|
||||||
|
raise ValueError('Unexpected publication date')
|
||||||
|
date.fromisoformat(day.replace('/', '-'))
|
||||||
|
if (meta.get('citation_volume') != ['22'] or meta.get('citation_conference_title') != [HEADING]
|
||||||
|
or [clean(n.text()) for n in nodes if n.tag == 'h1'] != [HEADING]
|
||||||
|
or not authors or any(not clean(a) for a in authors)
|
||||||
|
or comparable(title) != comparable(index['title'])
|
||||||
|
or comparable(', '.join(authors)) != comparable(index['authors'])
|
||||||
|
or [comparable(n.text()) for n in nodes if n.tag == 'h3'] != [comparable(title)]):
|
||||||
|
raise ValueError('Publisher citation/index/visible metadata disagree')
|
||||||
|
pdf = one(meta.get('citation_pdf_url', []))
|
||||||
|
if pdf not in (ROOT+f'p{pid:03d}.pdf', ROOT.replace('https:', 'http:')+f'p{pid:03d}.pdf'):
|
||||||
|
raise ValueError('Citation PDF ID mismatch; PDFs are never fetched')
|
||||||
|
bib = one([n.text() for n in nodes if n.tag == 'pre'])
|
||||||
|
doi = one(re.findall(r'^\s*DOI\s*=\s*\{([^{}]+)\}', bib, re.I | re.M))
|
||||||
|
year = one(re.findall(r'^\s*YEAR\s*=\s*\{([^{}]+)\}', bib, re.I | re.M))
|
||||||
|
if doi != f'10.15607/RSS.2026.XXII.{pid:03d}' or year != '2026':
|
||||||
|
raise ValueError('BibTeX DOI/year mismatch')
|
||||||
|
variants = []
|
||||||
|
if comparable(', '.join(authors)) != comparable(program['authors_complete']):
|
||||||
|
key = digest(json.dumps([authors, program['authors_complete']], ensure_ascii=False, separators=(',', ':')).encode())
|
||||||
|
if REVIEWED_AUTHOR_VARIANTS.get(pid) != key:
|
||||||
|
raise ValueError('Unreviewed complete author variant')
|
||||||
|
variants.append('reviewed_program_author_name_variant_not_person_alias')
|
||||||
|
if comparable(title) != comparable(program['title']):
|
||||||
|
if (pid, title, program['title']) != (89,
|
||||||
|
'RLinf-VLA: A Unified and Efficient Framework for Reinforcement Learning of Vision-Language-Action Models',
|
||||||
|
'RLux-VLA: A Unified and Efficient Framework for Reinforcement Learning of Vision-Language-Action Models'):
|
||||||
|
raise ValueError('Unreviewed program title variant')
|
||||||
|
variants.append('reviewed_program_title_variant_not_erratum')
|
||||||
|
inside = False; parts = []; markers = []
|
||||||
|
for n in (n for n in nodes if n.tag == 'p'):
|
||||||
|
labels = [clean(b.text()) for b in n.walk() if b.tag == 'b']
|
||||||
|
if 'Abstract:' in labels:
|
||||||
|
inside = True; markers.append('start'); value = clean(n.text()).removeprefix('Abstract:').strip()
|
||||||
|
if value: parts.append(value)
|
||||||
|
elif 'Download:' in labels:
|
||||||
|
inside = False; markers.append('end')
|
||||||
|
elif inside:
|
||||||
|
parts.append(clean(n.text()))
|
||||||
|
if markers != ['start', 'end'] or not any(parts):
|
||||||
|
raise ValueError('Missing or changed publisher abstract layout')
|
||||||
|
return {'title': title, 'authors': authors, 'abstract': '\n\n'.join(v for v in parts if v),
|
||||||
|
'doi_raw': doi, 'publication_date': day, 'bibtex': bib, 'program_variants': variants}
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots=None, cache=None):
|
||||||
|
if (snapshots is None) == (cache is None):
|
||||||
|
raise ValueError('Specify snapshots or cache')
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output directory must not exist; inspect partial/completed runs first')
|
||||||
|
out.parent.mkdir(parents=True, exist_ok=True)
|
||||||
|
out.mkdir()
|
||||||
|
evidence = {}; receipts = {}
|
||||||
|
if snapshots is not None:
|
||||||
|
root = Path(snapshots)
|
||||||
|
try:
|
||||||
|
for r in json.loads((root/'receipts.json').read_bytes()):
|
||||||
|
name = r['file']
|
||||||
|
if not isinstance(name, str) or Path(name).name != name or name in receipts:
|
||||||
|
raise ValueError('Unsafe or repeated snapshot receipt')
|
||||||
|
receipts[name] = r
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||||
|
raise
|
||||||
|
def page(name, url):
|
||||||
|
if snapshots is not None:
|
||||||
|
path = root/name
|
||||||
|
if path.is_symlink(): raise ValueError('Snapshot symlink rejected')
|
||||||
|
raw = path.read_bytes(); r = receipts[name]
|
||||||
|
if (r['url'] != url or r.get('final_url', url) != url or r['http_status'] != 200
|
||||||
|
or r['bytes'] != len(raw) or r['sha256'] != digest(raw)):
|
||||||
|
raise ValueError('Snapshot hash/receipt mismatch')
|
||||||
|
at = r['at']
|
||||||
|
else:
|
||||||
|
raw = fetch(url, cache, limit=8*1024*1024)
|
||||||
|
sidecar = json.loads((Path(cache)/digest(url.encode())).with_suffix('.json').read_bytes())
|
||||||
|
if sidecar['sha256'] != digest(raw): raise ValueError('Cache receipt mismatch')
|
||||||
|
at = sidecar['retrieved_at']
|
||||||
|
if len(raw) > 8*1024*1024 or datetime.fromisoformat(at.replace('Z', '+00:00')).utcoffset() is None:
|
||||||
|
raise ValueError('Invalid source size/timezone')
|
||||||
|
relative = 'evidence/'+digest(raw)+'.html'; target = out/relative
|
||||||
|
target.parent.mkdir(exist_ok=True); target.write_bytes(raw)
|
||||||
|
evidence[name] = {'file': relative, 'sha256': digest(raw), 'url': url, 'observed_at': at}
|
||||||
|
return raw.decode('utf-8')
|
||||||
|
try:
|
||||||
|
pages = {name: page(name, url) for name, url in PAGES.items()}
|
||||||
|
index, program = parse_lists(pages['index.html'], pages['accepted.html'], pages['cfp.html'], pages['sponsor-demos.html'])
|
||||||
|
records = []
|
||||||
|
for pid in sorted(index):
|
||||||
|
name = f'p{pid:03d}.html'; fields = parse_paper(page(name, ROOT+name), pid, index[pid], program[pid])
|
||||||
|
row = {'source': SOURCE, 'external_id': f'2026/p{pid:03d}', 'url': ROOT+name,
|
||||||
|
'title': fields['title'], 'abstract': fields['abstract'], 'authors': fields['authors'],
|
||||||
|
'identifiers': {'doi': fields['doi_raw'].lower()}, 'venue': 'RSS', 'venue_aliases': [],
|
||||||
|
'year': 2026, 'track': 'main', 'status': 'accepted',
|
||||||
|
'raw_decision': 'listed in official RSS 2026 proceedings and accepted papers',
|
||||||
|
'observed_at': evidence[name]['observed_at'],
|
||||||
|
'metadata': {'publisher_date': fields['publication_date'], 'publisher_doi_raw': fields['doi_raw'],
|
||||||
|
'original_bibtex': fields['bibtex'], 'program': program[pid],
|
||||||
|
'program_variants': fields['program_variants'], 'source_evidence': evidence[name],
|
||||||
|
'index_evidence': evidence['index.html'], 'program_evidence': evidence['accepted.html'],
|
||||||
|
'policy_evidence': evidence['cfp.html'],
|
||||||
|
'scope': '2026 Science/Systems papers; no separate demo-paper category. Program display ID is not an original submission ID.'}}
|
||||||
|
records.append(row)
|
||||||
|
path = out/'rss-proceedings-2026-main.jsonl'
|
||||||
|
raw = b''.join((json.dumps(r, ensure_ascii=False)+'\n').encode() for r in records)
|
||||||
|
path.write_bytes(raw)
|
||||||
|
manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'RSS', 'year': 2026, 'track': 'main',
|
||||||
|
'count': 210, 'file': str(path), 'file_sha256': digest(raw), 'list_url': ROOT,
|
||||||
|
'status': 'partial', 'missing_abstract_count': 0}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest, indent=2)+'\n')
|
||||||
|
report = {'status': 'collected_pending_identity_review', 'source': SOURCE, 'records': 210,
|
||||||
|
'files': [manifest], 'source_evidence': evidence, 'database_written': False, 'pdfs_downloaded': False,
|
||||||
|
'limitations': 'Publisher metadata only; not database import, scientific-work deduplication, private reviews or a public rejection set.'}
|
||||||
|
temporary = out/'collection-report.json.part'; temporary.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n')
|
||||||
|
temporary.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
p = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
|
p.add_argument('--out', required=True, type=Path)
|
||||||
|
g = p.add_mutually_exclusive_group(required=True)
|
||||||
|
g.add_argument('--snapshots', type=Path); g.add_argument('--cache', type=Path)
|
||||||
|
args = p.parse_args(argv)
|
||||||
|
report = collect(args.out, snapshots=args.snapshots, cache=args.cache)
|
||||||
|
print(json.dumps({'status': report['status'], 'records': report['records'], 'database_written': False}))
|
||||||
@@ -0,0 +1,313 @@
|
|||||||
|
"""Frozen SDM 2026 publication records and separate public program observations."""
|
||||||
|
import argparse
|
||||||
|
from collections import Counter
|
||||||
|
from datetime import date, datetime
|
||||||
|
import hashlib
|
||||||
|
from html.parser import HTMLParser
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
from urllib.parse import parse_qs, urljoin, urlsplit
|
||||||
|
|
||||||
|
from paper_atlas.io.html import Document, clean
|
||||||
|
|
||||||
|
SOURCE = 'crossref_sdm_registry'
|
||||||
|
TRACK = 'proceedings_unclassified'
|
||||||
|
BOOK = '10.1137/1.9781611979329'
|
||||||
|
ISBN = '9781611979329'
|
||||||
|
TITLE = 'Proceedings of the 2026 SIAM International Conference on Data Mining (SDM)'
|
||||||
|
QUERY = 'https://api.crossref.org/works?filter=isbn%3A9781611979329&rows=1000'
|
||||||
|
PROGRAM = 'https://meetings.siam.org/program.cfm?CONFCODE=sdm26'
|
||||||
|
SNAPSHOTS = {
|
||||||
|
'sdm2026-official-research-v1': ('a39a6ee6a12b69051ba2dad7c9cbc5b0fe215af85100c7aa2f5166d44c22fe61', 'sources'),
|
||||||
|
'sdm2026-crossref-research-v1': ('51d337a153a1046e4a00ebc031707faf72e733577e4fc243a31672ee93277da5', 'sources'),
|
||||||
|
'sdm2026-program-research-v1': ('689a2aaaa15be532957c4c88e03e1e2e87909334b6f529be16905d8b9cc1fb77', 'sessions'),
|
||||||
|
'sdm2026-public-abstract-research-v1': ('8e1c29611e145cbfa54ec4afaa2efe152557b4e04e339901af5cd33abefc29a5', 'records'),
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def one(items):
|
||||||
|
if len(items) != 1:
|
||||||
|
raise ValueError('Expected exactly one reviewed source field')
|
||||||
|
return items[0]
|
||||||
|
|
||||||
|
|
||||||
|
def load_snapshot(root, name):
|
||||||
|
expected, key = SNAPSHOTS[name]
|
||||||
|
directory = root/name; path = directory/'manifest.json'
|
||||||
|
if root.is_symlink() or directory.is_symlink() or path.is_symlink() or path.stat().st_size > 256*1024:
|
||||||
|
raise ValueError('Unsafe snapshot manifest')
|
||||||
|
raw = path.read_bytes()
|
||||||
|
if digest(raw) != expected:
|
||||||
|
raise ValueError('Reviewed snapshot manifest hash changed')
|
||||||
|
manifest = json.loads(raw)
|
||||||
|
if datetime.fromisoformat(manifest['at']).utcoffset() is None:
|
||||||
|
raise ValueError('Snapshot timestamp requires timezone')
|
||||||
|
files = {'manifest.json': raw}
|
||||||
|
for receipt in manifest[key]:
|
||||||
|
if 'file' not in receipt:
|
||||||
|
# A recorded 403 is an access observation, not an empty data set.
|
||||||
|
continue
|
||||||
|
filename = receipt['file']; path = directory/filename
|
||||||
|
if Path(filename).name != filename or path.is_symlink() or path.stat().st_size > 24*1024**2:
|
||||||
|
raise ValueError('Unsafe snapshot source file')
|
||||||
|
body = path.read_bytes()
|
||||||
|
if (receipt['http_status'] != 200 or receipt['url'] != receipt['final_url']
|
||||||
|
or receipt['bytes'] != len(body) or receipt['sha256'] != digest(body)):
|
||||||
|
raise ValueError('Snapshot receipt/body disagree')
|
||||||
|
if filename in files:
|
||||||
|
raise ValueError('Repeated snapshot source file')
|
||||||
|
files[filename] = body
|
||||||
|
return manifest, files
|
||||||
|
|
||||||
|
|
||||||
|
def registry_papers(message):
|
||||||
|
items = message.get('items', [])
|
||||||
|
if message.get('total-results') != 55 or len(items) != 55:
|
||||||
|
raise ValueError('Incomplete exact-ISBN registry census')
|
||||||
|
if Counter(x.get('type') for x in items) != {'book-chapter': 53, 'book': 1, 'other': 1}:
|
||||||
|
raise ValueError('Unreviewed publication item types')
|
||||||
|
papers = []; excluded = []; seen = set()
|
||||||
|
for original in items:
|
||||||
|
doi = original.get('DOI')
|
||||||
|
if not isinstance(doi, str) or doi in seen or original.get('ISBN') != [ISBN]:
|
||||||
|
raise ValueError('Missing/repeated DOI or wrong ISBN')
|
||||||
|
seen.add(doi)
|
||||||
|
if doi in (BOOK, BOOK+'.fm'):
|
||||||
|
expected = ('book', [TITLE]) if doi == BOOK else ('other', ['Front Matter'])
|
||||||
|
if (original['type'], original.get('title')) != expected:
|
||||||
|
raise ValueError('Unreviewed excluded volume/front matter')
|
||||||
|
excluded.append({'doi': doi, 'type': original['type'], 'title': original['title']})
|
||||||
|
continue
|
||||||
|
if (original.get('type') != 'book-chapter' or original.get('container-title') != [TITLE]
|
||||||
|
or not re.fullmatch(re.escape(BOOK)+r'\.[1-9][0-9]*', doi)
|
||||||
|
or original.get('URL') != 'https://doi.org/'+doi
|
||||||
|
or original.get('publisher') != 'Society for Industrial and Applied Mathematics'
|
||||||
|
or original.get('relation', {}).get('is-part-of') != [
|
||||||
|
{'id-type': 'doi', 'id': 'https://doi.org/'+BOOK, 'asserted-by': 'object'}]):
|
||||||
|
raise ValueError('Wrong publication edition, publisher or parent DOI')
|
||||||
|
title = one(original.get('title', []))
|
||||||
|
if not isinstance(title, str) or not clean(title):
|
||||||
|
raise ValueError('Missing registry title')
|
||||||
|
flags = []
|
||||||
|
if '<' in title or '>' in title:
|
||||||
|
match = re.fullmatch(r'([^<>]+)<sup>\*</sup>\s*', title, re.S)
|
||||||
|
if not match:
|
||||||
|
raise ValueError('Unreviewed title markup')
|
||||||
|
comparison_title = clean(match[1])
|
||||||
|
flags.append('title_sup_asterisk_footnote_requires_review')
|
||||||
|
else:
|
||||||
|
comparison_title = clean(Document(title).root.text())
|
||||||
|
displayed_title = clean(Document(title).root.text())
|
||||||
|
authors = original.get('author')
|
||||||
|
if (not isinstance(authors, list) or not authors
|
||||||
|
or any(not isinstance(a.get('given'), str) or not isinstance(a.get('family'), str)
|
||||||
|
or not clean(a['given']) or not clean(a['family']) for a in authors)
|
||||||
|
or [a.get('sequence') for a in authors] != ['first']+['additional']*(len(authors)-1)):
|
||||||
|
raise ValueError('Incomplete registry authors or order evidence')
|
||||||
|
names = [clean(a['given']+' '+a['family']) for a in authors]
|
||||||
|
if any('@' in n or 'anonymous' in n.casefold() for n in names):
|
||||||
|
raise ValueError('Unreviewed registry author identity')
|
||||||
|
dates = {}
|
||||||
|
for field in ('published', 'published-print', 'published-online', 'issued'):
|
||||||
|
if field in original:
|
||||||
|
parts = one(original[field]['date-parts'])
|
||||||
|
if len(parts) != 3 or any(type(x) is not int for x in parts) or parts[0] != 2026:
|
||||||
|
raise ValueError('Unreviewed registry publication date')
|
||||||
|
dates[field] = date(*parts).isoformat()
|
||||||
|
if not dates or original.get('abstract'):
|
||||||
|
raise ValueError('Reviewed registry date/abstract availability changed')
|
||||||
|
papers.append({'original': original, 'title': displayed_title, 'comparison_title': comparison_title,
|
||||||
|
'authors': names, 'review_flags': flags, 'publication_dates': dates})
|
||||||
|
if {p['original']['DOI'] for p in papers} != {BOOK+'.'+str(n) for n in range(1, 54)} or len(excluded) != 2:
|
||||||
|
raise ValueError('Incomplete reviewed 53-chapter DOI set')
|
||||||
|
return sorted(papers, key=lambda p: int(p['original']['DOI'].rsplit('.', 1)[1])), excluded
|
||||||
|
|
||||||
|
|
||||||
|
class SessionParser(HTMLParser):
|
||||||
|
"""The source omits closing dt/dd tags; each next dt starts a new event."""
|
||||||
|
def __init__(self):
|
||||||
|
super().__init__(); self.rows = []; self.current = None; self.strong = False; self.author = False
|
||||||
|
|
||||||
|
def finish(self):
|
||||||
|
if self.current is not None:
|
||||||
|
match = re.fullmatch(r'(\d{1,2}:\d{2}-\d{1,2}:\d{2})\s+(.+)', clean(self.current.pop('timed_title')))
|
||||||
|
if not match or not clean(self.current['authors_literal']):
|
||||||
|
raise ValueError('Incomplete contributed program event')
|
||||||
|
self.current['time_literal'], self.current['title'] = match.groups()
|
||||||
|
self.current['authors_literal'] = clean(self.current['authors_literal'])
|
||||||
|
self.rows.append(self.current)
|
||||||
|
self.current = None; self.strong = False; self.author = False
|
||||||
|
|
||||||
|
def handle_starttag(self, tag, attrs):
|
||||||
|
attrs = dict(attrs)
|
||||||
|
if tag == 'dt':
|
||||||
|
self.finish()
|
||||||
|
self.current = {'timed_title': '', 'authors_literal': '', 'cancelled_presentation': False, 'abstract_links': []}
|
||||||
|
if self.current is not None:
|
||||||
|
if tag == 'strong': self.strong = True
|
||||||
|
elif tag == 'dd': self.author = True
|
||||||
|
elif tag == 's': self.current['cancelled_presentation'] = True
|
||||||
|
elif tag == 'a' and attrs.get('href', '').startswith('dsp_talk.cfm?p='):
|
||||||
|
self.current['abstract_links'].append(attrs['href'])
|
||||||
|
|
||||||
|
def handle_endtag(self, tag):
|
||||||
|
if tag == 'strong': self.strong = False
|
||||||
|
elif tag == 'dl': self.finish()
|
||||||
|
|
||||||
|
def handle_data(self, text):
|
||||||
|
if self.current is not None:
|
||||||
|
if self.strong: self.current['timed_title'] += text
|
||||||
|
elif self.author: self.current['authors_literal'] += text
|
||||||
|
elif clean(text) == 'Cancelled': self.current['cancelled_presentation'] = True
|
||||||
|
|
||||||
|
|
||||||
|
def talk_fields(raw, session_url):
|
||||||
|
nodes = list(Document(raw.decode('utf-8')).root.walk())
|
||||||
|
if one([clean(n.text()) for n in nodes if n.tag == 'h1']) != '2026 SIAM International Conference on Data Mining':
|
||||||
|
raise ValueError('Wrong program page edition')
|
||||||
|
title = clean(one([n for n in nodes if n.tag == 'div' and n.attrs.get('class') == 'ptitle']).text())
|
||||||
|
abstract_node = one([n for n in nodes if n.tag == 'div' and n.attrs.get('class') == 'abstract'])
|
||||||
|
abstract = clean(abstract_node.text())
|
||||||
|
if not title or not abstract.startswith('Abstract.') or not abstract[len('Abstract.'):].strip():
|
||||||
|
raise ValueError('Missing public program title/abstract')
|
||||||
|
reference = one([n for n in nodes if n.tag == 'div' and n.attrs.get('class') == 'sesref'])
|
||||||
|
link = one([n.attrs.get('href', '') for n in reference.walk() if n.tag == 'a'])
|
||||||
|
def session_key(url):
|
||||||
|
u = urlsplit(url)
|
||||||
|
query = {k.lower(): v for k, v in parse_qs(u.query).items()}
|
||||||
|
if u.scheme != 'https' or u.netloc != 'meetings.siam.org' or u.path != '/sess/dsp_programsess.cfm' or set(query) != {'sessioncode'}:
|
||||||
|
raise ValueError('Foreign program session link')
|
||||||
|
return query['sessioncode']
|
||||||
|
if session_key(link) != session_key(session_url):
|
||||||
|
raise ValueError('Program page belongs to a different session')
|
||||||
|
authors = []
|
||||||
|
for n in nodes:
|
||||||
|
if n.tag == 'li':
|
||||||
|
# Source rows contain name, affiliations, country and contact email.
|
||||||
|
# Only the independently checked leading name is a metadata field.
|
||||||
|
literal = clean(n.text()); name, separator, _ = literal.partition(',')
|
||||||
|
if not separator or not name or '@' in name:
|
||||||
|
raise ValueError('Unreviewed program author row')
|
||||||
|
authors.append(name)
|
||||||
|
if not authors:
|
||||||
|
raise ValueError('Missing source-reported program authors')
|
||||||
|
markup = sorted({n.tag for n in abstract_node.walk() if n.tag in ('math', 'sup', 'sub', 'svg', 'img')})
|
||||||
|
return {'title': title, 'source_reported_authors': authors, 'abstract': abstract[len('Abstract.'):].strip(),
|
||||||
|
'abstract_markup_tags': markup, 'review_flags': ['structured_abstract_markup_requires_review'] if markup else [],
|
||||||
|
'author_completeness': 'Program-reported names only; not asserted to be the complete or final publisher author sequence.'}
|
||||||
|
|
||||||
|
|
||||||
|
def program_observations(snapshots, papers):
|
||||||
|
official, official_files = snapshots['sdm2026-official-research-v1']
|
||||||
|
sessions, session_files = snapshots['sdm2026-program-research-v1']
|
||||||
|
pages, page_files = snapshots['sdm2026-public-abstract-research-v1']
|
||||||
|
main_nodes = list(Document(official_files['program.html'].decode()).root.walk())
|
||||||
|
links = {urljoin(PROGRAM, n.attrs.get('href', '')): clean(n.text()) for n in main_nodes
|
||||||
|
if n.tag == 'a' and re.match(r'CP[1-9]\s', clean(n.text()))}
|
||||||
|
if len(links) != 9 or {s['url']: s['title'] for s in sessions['sessions']} != links:
|
||||||
|
raise ValueError('Contributed session membership disagrees with main program')
|
||||||
|
receipts = {r['url']: r for r in pages['records']}
|
||||||
|
if len(receipts) != len(pages['records']) or len(receipts) != 52:
|
||||||
|
raise ValueError('Incomplete or repeated public talk receipts')
|
||||||
|
rows = []; used = set()
|
||||||
|
for session in sessions['sessions']:
|
||||||
|
parser = SessionParser(); parser.feed(session_files[session['file']].decode()); parser.finish()
|
||||||
|
for position, event in enumerate(parser.rows, 1):
|
||||||
|
urls = [urljoin(session['url'], link) for link in event.pop('abstract_links')]
|
||||||
|
event.update(record_kind='source_observation_not_import_record', session=session['title'],
|
||||||
|
session_url=session['url'], session_sha256=session['sha256'], session_position=position)
|
||||||
|
if not urls and event['cancelled_presentation']:
|
||||||
|
event.update(program_page=None, public_page_fields=None)
|
||||||
|
else:
|
||||||
|
url = one(urls)
|
||||||
|
if url not in receipts or url in used:
|
||||||
|
raise ValueError('Missing or repeated public program page')
|
||||||
|
used.add(url); receipt = receipts[url]
|
||||||
|
if receipt['session_url'] != session['url']:
|
||||||
|
raise ValueError('Public page receipt session disagrees')
|
||||||
|
fields = talk_fields(page_files[receipt['file']], session['url'])
|
||||||
|
if fields['title'] != event['title']:
|
||||||
|
fields['review_flags'].append('session_page_title_versions_differ_requires_review')
|
||||||
|
event.update(program_page=url, public_page_receipt=receipt, public_page_fields=fields)
|
||||||
|
candidates = []
|
||||||
|
for paper in papers:
|
||||||
|
if paper['comparison_title'].casefold() == event['title'].casefold():
|
||||||
|
fields = event['public_page_fields']
|
||||||
|
candidates.append({'registry_doi_candidate': paper['original']['DOI'],
|
||||||
|
'comparison': 'title_candidate_only_not_identity_approval',
|
||||||
|
'registry_title_review_flags': paper['review_flags'],
|
||||||
|
'author_sequence_exact': fields['source_reported_authors'] == paper['authors'] if fields else None})
|
||||||
|
event['registry_candidates'] = candidates
|
||||||
|
event['scope'] = 'Separate program version; cancellation is presentation status only. No original submission ID, decision, work identity or author alias inferred.'
|
||||||
|
rows.append(event)
|
||||||
|
if len(rows) != 53 or used != set(receipts) or sum(r['cancelled_presentation'] for r in rows) != 1:
|
||||||
|
raise ValueError('Incomplete reviewed program event census')
|
||||||
|
return rows
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, snapshots):
|
||||||
|
out = Path(out)
|
||||||
|
if out.exists():
|
||||||
|
raise ValueError('Output exists; inspect previous operation before retrying')
|
||||||
|
out.mkdir(parents=True)
|
||||||
|
try:
|
||||||
|
loaded = {name: load_snapshot(Path(snapshots), name) for name in SNAPSHOTS}
|
||||||
|
registry_manifest, registry_files = loaded['sdm2026-crossref-research-v1']
|
||||||
|
message = json.loads(registry_files['isbn-works.json'])['message']
|
||||||
|
papers, excluded = registry_papers(message)
|
||||||
|
observations = program_observations(loaded, papers)
|
||||||
|
evidence = []; directory = out/'evidence'; directory.mkdir()
|
||||||
|
for name, (manifest, files) in loaded.items():
|
||||||
|
for filename, raw in files.items():
|
||||||
|
rel = 'evidence/'+digest(raw)+Path(filename).suffix
|
||||||
|
(out/rel).write_bytes(raw)
|
||||||
|
evidence.append({'snapshot': name, 'file': filename, 'output_file': rel, 'sha256': digest(raw)})
|
||||||
|
receipt = one([r for r in registry_manifest['sources'] if r.get('file') == 'isbn-works.json'])
|
||||||
|
rows = []
|
||||||
|
for p in papers:
|
||||||
|
original = p['original']; doi = original['DOI']
|
||||||
|
rows.append({'source': SOURCE, 'external_id': doi, 'url': original['URL'], 'title': p['title'],
|
||||||
|
'authors': p['authors'], 'abstract': None, 'identifiers': {'doi': doi},
|
||||||
|
'submission_identifiers': [{'scheme': 'publication_doi', 'value': doi}],
|
||||||
|
'venue': 'SDM', 'year': 2026, 'track': TRACK, 'status': 'accepted', 'tags': [],
|
||||||
|
'observed_at': receipt['at'], 'evidence_url': QUERY,
|
||||||
|
'raw_decision': 'Registered research chapter in the exact SDM2026 proceedings ISBN; not an original signed review decision.',
|
||||||
|
'metadata': {'registry_fields': original, 'publication_dates': p['publication_dates'],
|
||||||
|
'registry_response_sha256': digest(registry_files['isbn-works.json']), 'review_flags': p['review_flags'],
|
||||||
|
'original_decision_signature_verified': False,
|
||||||
|
'abstract_availability': 'No abstract in frozen Crossref deposit; public program abstracts remain separate source versions.',
|
||||||
|
'title_transform': 'Display text with footnote asterisk retained; raw title preserved. Footnote-free text is used only to list review candidates.',
|
||||||
|
'scope': 'Requires fresh work/DOI/history review before import. No automatic program fusion, track, person alias or negative decision inference.'}})
|
||||||
|
path = out/'sdm-crossref-registry-2026-unclassified.jsonl'
|
||||||
|
payload = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in rows).encode(); path.write_bytes(payload)
|
||||||
|
file_manifest = {'source': SOURCE, 'parser_version': 1, 'venue': 'SDM', 'year': 2026, 'track': TRACK,
|
||||||
|
'count': 53, 'file': str(path), 'file_sha256': digest(payload), 'list_url': QUERY, 'status': 'partial', 'missing_abstract_count': 53}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(file_manifest, indent=2)+'\n')
|
||||||
|
sidecar = out/'program-observations.jsonl'
|
||||||
|
raw = ''.join(json.dumps(r, ensure_ascii=False)+'\n' for r in observations).encode(); sidecar.write_bytes(raw)
|
||||||
|
report = {'status': 'collected_pending_identity_review', 'records': 53, 'files': [file_manifest],
|
||||||
|
'missing_abstract_count': 53, 'excluded_registry_items': excluded,
|
||||||
|
'program_observations': {'file': str(sidecar), 'sha256': digest(raw), 'count': 53, 'abstract_pages': 52,
|
||||||
|
'cancelled_presentations': 1, 'not_an_import_manifest': True},
|
||||||
|
'program_registry_author_sequence_differences': sum(c['author_sequence_exact'] is False for o in observations for c in o['registry_candidates']),
|
||||||
|
'evidence': evidence, 'public_rejected_metadata': 'unknown_anonymous_query_http403',
|
||||||
|
'database_written': False, 'network_requests': 0, 'pdfs_downloaded': False,
|
||||||
|
'scope': 'Registry publication records and program source observations remain separate; no work merge, metadata fusion or final coverage/dedup claim.'}
|
||||||
|
part = out/'collection-report.json.part'; part.write_text(json.dumps(report, ensure_ascii=False, indent=2)+'\n'); part.replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
except Exception as error:
|
||||||
|
(out/'failed.json').write_text(json.dumps({'status': 'failed_not_complete', 'error_type': type(error).__name__, 'database_written': False})+'\n')
|
||||||
|
raise
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog, description=__doc__)
|
||||||
|
parser.add_argument('--snapshots', required=True)
|
||||||
|
parser.add_argument('--out', required=True)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
print(json.dumps(collect(args.out, snapshots=args.snapshots), ensure_ascii=False))
|
||||||
@@ -0,0 +1,231 @@
|
|||||||
|
"""Reviewed postconference selected volumes from frozen public HTML metadata."""
|
||||||
|
import argparse
|
||||||
|
from collections import defaultdict
|
||||||
|
import fcntl
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import tempfile
|
||||||
|
from urllib.parse import parse_qs, urlsplit
|
||||||
|
|
||||||
|
from paper_atlas.db.submissions import timestamp
|
||||||
|
from paper_atlas.io.html import Document, clean
|
||||||
|
|
||||||
|
SOURCE = 'springer_selected_volume'
|
||||||
|
BOOK = '10.1007/978-1-4612-2660-4'
|
||||||
|
BOOK_URL = 'https://link.springer.com/book/'+BOOK
|
||||||
|
HISTORY_URL = 'https://www.gatsby.ucl.ac.uk/aistats/workshops.html'
|
||||||
|
HISTORY_SHA = '6db205b9dbc82b4a096556efd2aa67e065cad8e0bd6351176b1962ffd1fca179'
|
||||||
|
SCOPE = {
|
||||||
|
'venue': 'AISTATS', 'year': 1993, 'track': 'selected_volume',
|
||||||
|
'event_key': 'aistats-1993-selected-volume-1994',
|
||||||
|
'publication_year': 1994, 'book_doi': BOOK,
|
||||||
|
'book_title': 'Selecting Models from Data', 'expected_chapters': 49,
|
||||||
|
'relationship': 'Selected workshop papers were revised and rereviewed for later publication; this is not the original complete workshop decision list.',
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def snapshot_index(root):
|
||||||
|
root = Path(root).resolve()
|
||||||
|
receipts = json.loads((root/'receipts.json').read_bytes())['responses']
|
||||||
|
index = {}
|
||||||
|
for receipt in receipts:
|
||||||
|
url = receipt['url']
|
||||||
|
if url in index:
|
||||||
|
raise ValueError('Duplicate snapshot URL')
|
||||||
|
index[url] = (root, receipt)
|
||||||
|
return index
|
||||||
|
|
||||||
|
|
||||||
|
def read_page(index, url):
|
||||||
|
if url not in index:
|
||||||
|
raise ValueError('Missing required public HTML snapshot: '+url)
|
||||||
|
root, receipt = index[url]
|
||||||
|
filename = Path(receipt['file'])
|
||||||
|
path = root/filename
|
||||||
|
content_type = {k.lower():v for k,v in receipt.get('headers', {}).items()}.get('content-type', '')
|
||||||
|
requested, final = urlsplit(url), urlsplit(receipt.get('final_url', ''))
|
||||||
|
query = parse_qs(final.query, keep_blank_values=True)
|
||||||
|
original_query = parse_qs(requested.query, keep_blank_values=True)
|
||||||
|
# Springer returns the public metadata with an explicit cookie diagnostic.
|
||||||
|
# Retain that final URL verbatim; accept no other redirect or changed page.
|
||||||
|
if query != original_query and requested.netloc == 'link.springer.com':
|
||||||
|
if (query.pop('error', None) != ['cookies_not_supported']
|
||||||
|
or not re.fullmatch(r'[0-9a-f-]{36}', ''.join(query.pop('code', [])))):
|
||||||
|
raise ValueError('Unreviewed Springer redirect query')
|
||||||
|
if (filename.is_absolute() or '..' in filename.parts or not path.resolve().is_relative_to(root)
|
||||||
|
or receipt.get('http_status') != 200
|
||||||
|
or (final.scheme, final.netloc, final.path) != (requested.scheme, requested.netloc, requested.path)
|
||||||
|
or final.fragment or query != original_query
|
||||||
|
or 'text/html' not in content_type.lower()):
|
||||||
|
raise ValueError('Unsafe, redirected or unsuccessful public HTML snapshot')
|
||||||
|
raw = path.read_bytes()
|
||||||
|
digest = hashlib.sha256(raw).hexdigest()
|
||||||
|
if len(raw) > 2_000_000 or len(raw) != receipt['bytes'] or digest != receipt['sha256']:
|
||||||
|
raise ValueError('Snapshot checksum or size differs')
|
||||||
|
evidence = {'url':url, 'sha256':digest, 'bytes':len(raw),
|
||||||
|
'file':'evidence/'+digest+'.html', 'http_status':200,
|
||||||
|
'observed_at':timestamp(receipt['at']), 'response_receipt':receipt}
|
||||||
|
return raw, evidence
|
||||||
|
|
||||||
|
|
||||||
|
def parse_toc(html, *, excluded_materials=None):
|
||||||
|
entries = {}
|
||||||
|
for node in Document(html).root.walk():
|
||||||
|
if node.tag != 'li' or node.attrs.get('data-test') != 'chapter':
|
||||||
|
continue
|
||||||
|
links = [n for n in node.walk() if n.tag == 'a' and re.fullmatch(
|
||||||
|
r'/chapter/'+re.escape(BOOK)+r'_[1-9]\d*', n.attrs.get('href', ''))]
|
||||||
|
if not links:
|
||||||
|
headings = [clean(n.text()) for n in node.walk() if n.tag in ('h3','h4')]
|
||||||
|
frontmatter = [n.attrs.get('href','') for n in node.walk()
|
||||||
|
if n.tag == 'a' and n.attrs.get('data-test') == 'front-matter-pdf']
|
||||||
|
if headings == ['Front Matter'] and len(frontmatter) == 1:
|
||||||
|
if excluded_materials is not None:
|
||||||
|
excluded_materials.append({'title':'Front Matter', 'url':frontmatter[0],
|
||||||
|
'reason':'Section front matter without a chapter DOI; not an authored chapter.'})
|
||||||
|
continue
|
||||||
|
if len(links) != 1:
|
||||||
|
raise ValueError('Chapter entry lacks one reviewed-book DOI')
|
||||||
|
link, = links
|
||||||
|
doi = link.attrs['href'].removeprefix('/chapter/')
|
||||||
|
if doi in entries or not clean(link.text()):
|
||||||
|
raise ValueError('Duplicate DOI or missing chapter title')
|
||||||
|
entries[doi] = clean(link.text())
|
||||||
|
return entries
|
||||||
|
|
||||||
|
|
||||||
|
def parse_chapter(html, doi):
|
||||||
|
nodes = list(Document(html).root.walk())
|
||||||
|
meta = defaultdict(list)
|
||||||
|
for node in nodes:
|
||||||
|
if node.tag == 'meta' and node.attrs.get('name', '').startswith('citation_'):
|
||||||
|
meta[node.attrs['name']].append(node.attrs.get('content', ''))
|
||||||
|
def one(key):
|
||||||
|
values = meta[key]
|
||||||
|
if len(values) != 1 or not values[0]:
|
||||||
|
raise ValueError('Missing or ambiguous citation field: '+key)
|
||||||
|
return values[0]
|
||||||
|
if (one('citation_doi') != doi or one('citation_inbook_title') != SCOPE['book_title']
|
||||||
|
or one('citation_publication_date') != str(SCOPE['publication_year'])):
|
||||||
|
raise ValueError('Chapter DOI, book or publication year differs from reviewed scope')
|
||||||
|
raw_authors = meta['citation_author']
|
||||||
|
if not raw_authors:
|
||||||
|
raise ValueError('Missing named chapter authors')
|
||||||
|
authors = []
|
||||||
|
for raw in raw_authors:
|
||||||
|
if raw.count(',') != 1:
|
||||||
|
raise ValueError('Unreviewed citation author format')
|
||||||
|
family, given = [clean(s) for s in raw.split(',', 1)]
|
||||||
|
if not family or not given:
|
||||||
|
raise ValueError('Incomplete citation author')
|
||||||
|
authors.append({'given':given, 'family':family})
|
||||||
|
abstract_nodes = [n for n in nodes if n.attrs.get('id') == 'Abs1-content']
|
||||||
|
if len(abstract_nodes) > 1:
|
||||||
|
raise ValueError('Ambiguous abstract section')
|
||||||
|
abstract = clean(abstract_nodes[0].text()) if abstract_nodes else None
|
||||||
|
first, last = one('citation_firstpage'), one('citation_lastpage')
|
||||||
|
if not first.isdigit() or not last.isdigit() or int(first) > int(last):
|
||||||
|
raise ValueError('Invalid chapter pages')
|
||||||
|
return {'doi':doi, 'title':one('citation_title'), 'authors':authors,
|
||||||
|
'authors_citation_raw':list(raw_authors), 'abstract':abstract or None,
|
||||||
|
'first_page':first, 'last_page':last}
|
||||||
|
|
||||||
|
|
||||||
|
def record_for(chapter, evidence, scope_evidence):
|
||||||
|
return {
|
||||||
|
'source':SOURCE, 'external_id':chapter['doi'],
|
||||||
|
'url':'https://link.springer.com/chapter/'+chapter['doi'],
|
||||||
|
'title':chapter['title'], 'abstract':chapter['abstract'], 'authors':chapter['authors'],
|
||||||
|
'identifiers':{'doi':chapter['doi']},
|
||||||
|
'submission_identifiers':[{'scheme':'publication_doi','value':chapter['doi']}],
|
||||||
|
**{k:SCOPE[k] for k in ('venue','year','track','event_key')},
|
||||||
|
'status':'accepted', 'observed_at':evidence['observed_at'],
|
||||||
|
'raw_decision':'listed in the 1994 selected postconference volume; original 1993 review decision not supplied',
|
||||||
|
'tags':[{'namespace':SOURCE+':publication_format','value':'selected_volume_chapter'}],
|
||||||
|
'metadata':{
|
||||||
|
'publication_format':'selected_volume_chapter', 'bibliographic_year':1994,
|
||||||
|
'related_workshop_year':1993, 'book_doi':BOOK, 'book_title':SCOPE['book_title'],
|
||||||
|
'year_basis':'Reviewed workshop year; bibliographic publication year is separately retained.',
|
||||||
|
'first_page':chapter['first_page'], 'last_page':chapter['last_page'],
|
||||||
|
'authors_citation_raw':chapter['authors_citation_raw'],
|
||||||
|
'publication_evidence':'Published selected-volume chapter, not an original workshop decision note.',
|
||||||
|
'scope_relationship':SCOPE['relationship'], 'decision_date_precision':'not_supplied',
|
||||||
|
'source_evidence':evidence, 'scope_evidence':scope_evidence,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def collect(out, *, toc_snapshots, chapter_snapshots):
|
||||||
|
toc_index, chapter_index = snapshot_index(toc_snapshots), snapshot_index(chapter_snapshots)
|
||||||
|
frozen = {}
|
||||||
|
def read(index, url):
|
||||||
|
raw, evidence = read_page(index, url)
|
||||||
|
frozen[evidence['file']] = raw
|
||||||
|
return raw.decode('utf-8'), evidence
|
||||||
|
history, history_evidence = read(toc_index, HISTORY_URL)
|
||||||
|
if hashlib.sha256(history.encode()).hexdigest() != HISTORY_SHA:
|
||||||
|
raise ValueError('Reviewed selected-volume relationship evidence changed')
|
||||||
|
scope_evidence = [history_evidence]
|
||||||
|
titles, excluded = {}, []
|
||||||
|
for page in range(1, 4):
|
||||||
|
html, evidence = read(toc_index, BOOK_URL+('?page='+str(page) if page > 1 else ''))
|
||||||
|
scope_evidence.append(evidence)
|
||||||
|
entries = parse_toc(html, excluded_materials=excluded)
|
||||||
|
if titles.keys() & entries.keys():
|
||||||
|
raise ValueError('DOI repeated across TOC pages')
|
||||||
|
titles.update(entries)
|
||||||
|
expected = {BOOK+'_'+str(i) for i in range(1, 50)}
|
||||||
|
if titles.keys() != expected:
|
||||||
|
raise ValueError('Reviewed 49-chapter inventory changed')
|
||||||
|
chapter_urls = {'https://link.springer.com/chapter/'+doi for doi in expected}
|
||||||
|
if chapter_index.keys() != chapter_urls:
|
||||||
|
raise ValueError('Missing or extraneous chapter snapshot')
|
||||||
|
records = []
|
||||||
|
for doi in sorted(expected, key=lambda s:int(s.rsplit('_',1)[1])):
|
||||||
|
html, evidence = read(chapter_index, 'https://link.springer.com/chapter/'+doi)
|
||||||
|
chapter = parse_chapter(html, doi)
|
||||||
|
if chapter['title'] != titles[doi]:
|
||||||
|
raise ValueError('Chapter title disagrees with official TOC')
|
||||||
|
records.append(record_for(chapter, evidence, scope_evidence))
|
||||||
|
out = Path(out)
|
||||||
|
out.mkdir(parents=True, exist_ok=True)
|
||||||
|
with (out/'springer-selected.collect.lock').open('a') as lock:
|
||||||
|
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
||||||
|
if any(p.name != 'springer-selected.collect.lock' for p in out.iterdir()):
|
||||||
|
raise FileExistsError('Existing output must be inspected, not overwritten')
|
||||||
|
with tempfile.TemporaryDirectory(prefix='.springer-selected-', dir=out) as temporary:
|
||||||
|
stage = Path(temporary)
|
||||||
|
(stage/'evidence').mkdir()
|
||||||
|
for filename, raw in frozen.items():
|
||||||
|
(stage/filename).write_bytes(raw)
|
||||||
|
path = stage/'springer-aistats-1993-selected-volume.jsonl'
|
||||||
|
path.write_text(''.join(json.dumps(r,ensure_ascii=False)+'\n' for r in records))
|
||||||
|
manifest = {'source':SOURCE, 'parser_version':1,
|
||||||
|
**{k:SCOPE[k] for k in ('venue','year','track','event_key')},
|
||||||
|
'count':len(records), 'file':str(out/path.name),
|
||||||
|
'file_sha256':hashlib.sha256(path.read_bytes()).hexdigest(),
|
||||||
|
'list_url':BOOK_URL, 'status':'partial',
|
||||||
|
'excluded_materials':excluded,
|
||||||
|
'missing_abstract_count':sum(not r['abstract'] for r in records)}
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest,indent=2)+'\n')
|
||||||
|
report = {'status':'collected_pending_identity_review', 'records':len(records),
|
||||||
|
'files':[manifest], 'reviewed_scope':SCOPE, 'database_written':False,
|
||||||
|
'pdfs_downloaded':False, 'scope_evidence':scope_evidence}
|
||||||
|
(stage/'collection-report.json').write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n')
|
||||||
|
for item in stage.iterdir():
|
||||||
|
if item.name != 'collection-report.json':
|
||||||
|
item.replace(out/item.name)
|
||||||
|
(stage/'collection-report.json').replace(out/'collection-report.json')
|
||||||
|
return report
|
||||||
|
|
||||||
|
|
||||||
|
def main(argv=None, *, prog=None):
|
||||||
|
parser = argparse.ArgumentParser(prog=prog)
|
||||||
|
parser.add_argument('--toc-snapshots', required=True)
|
||||||
|
parser.add_argument('--chapter-snapshots', required=True)
|
||||||
|
parser.add_argument('--out', required=True)
|
||||||
|
args = parser.parse_args(argv)
|
||||||
|
print(json.dumps(collect(args.out, toc_snapshots=args.toc_snapshots,
|
||||||
|
chapter_snapshots=args.chapter_snapshots),ensure_ascii=False))
|
||||||
@@ -4,7 +4,7 @@ import re
|
|||||||
from paper_atlas.io.publication_ids import mlsys_id
|
from paper_atlas.io.publication_ids import mlsys_id
|
||||||
|
|
||||||
|
|
||||||
PUBLICATIONS={'pmlr','acl_anthology','colm','neurips','mlsys','cvf','ecva','dblp','aamas_proceedings'}
|
PUBLICATIONS={'pmlr','acl_anthology','colm','neurips','mlsys','cvf','ecva','dblp','aamas_proceedings','rss_proceedings'}
|
||||||
|
|
||||||
|
|
||||||
def timestamp(value=None):
|
def timestamp(value=None):
|
||||||
|
|||||||
@@ -0,0 +1,106 @@
|
|||||||
|
"""Immutable, receipt-checked public HTML cache; no expiry or silent refresh."""
|
||||||
|
from datetime import datetime, timezone
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import time
|
||||||
|
import urllib.request
|
||||||
|
import urllib.error
|
||||||
|
from urllib.parse import urlsplit
|
||||||
|
|
||||||
|
|
||||||
|
def digest(raw):
|
||||||
|
return hashlib.sha256(raw).hexdigest()
|
||||||
|
|
||||||
|
|
||||||
|
def checked_html(raw, receipt, url, limit):
|
||||||
|
if (len(raw) > limit or receipt.get('url') != url or receipt.get('final_url') != url
|
||||||
|
or receipt.get('http_status') != 200 or receipt.get('bytes') != len(raw)
|
||||||
|
or receipt.get('sha256') != digest(raw)
|
||||||
|
or not str(receipt.get('content_type', '')).lower().startswith('text/html')
|
||||||
|
or datetime.fromisoformat(receipt['retrieved_at'].replace('Z', '+00:00')).utcoffset() is None):
|
||||||
|
raise ValueError('Invalid public HTML receipt, response or timestamp')
|
||||||
|
if any(v in raw.lower() for v in (b'<title>just a moment', b'cf-chl-', b'botstopper')):
|
||||||
|
raise ValueError('Access challenge; stop without bypass')
|
||||||
|
return raw, receipt
|
||||||
|
|
||||||
|
|
||||||
|
class NoRedirect(urllib.request.HTTPRedirectHandler):
|
||||||
|
def redirect_request(self, *args, **kwargs):
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
class PublicHTMLCache:
|
||||||
|
"""Caller holds a single-writer lock and restricts the allowed URL paths."""
|
||||||
|
def __init__(self, directory, origin, *, limit=2*1024**2, delay=1):
|
||||||
|
self.root = Path(directory)
|
||||||
|
self.root.mkdir(parents=True, exist_ok=True)
|
||||||
|
if self.root.is_symlink():
|
||||||
|
raise ValueError('Cache symlinks rejected')
|
||||||
|
self.origin = origin
|
||||||
|
self.limit = limit
|
||||||
|
self.delay = delay
|
||||||
|
self.opener = urllib.request.build_opener(NoRedirect())
|
||||||
|
|
||||||
|
def paths(self, url):
|
||||||
|
u = urlsplit(url)
|
||||||
|
if u.scheme != 'https' or u.username or u.password or u.fragment or u.scheme+'://'+u.netloc != self.origin:
|
||||||
|
raise ValueError('Unapproved public metadata origin')
|
||||||
|
key = digest(url.encode())
|
||||||
|
return self.root/(key+'.html'), self.root/(key+'.json'), self.root/(key+'.failed.json')
|
||||||
|
|
||||||
|
def cached(self, url):
|
||||||
|
body, receipt, failure = self.paths(url)
|
||||||
|
parts = (body.with_suffix('.html.part'), receipt.with_suffix('.json.part'))
|
||||||
|
if any(p.is_symlink() for p in (body, receipt, failure, *parts)):
|
||||||
|
raise ValueError('Cache symlinks rejected')
|
||||||
|
if any(p.exists() for p in parts):
|
||||||
|
raise ValueError('Incomplete cache part; preserve and inspect orphan before recovery')
|
||||||
|
if failure.exists():
|
||||||
|
raise ValueError('Previous access failure requires inspection; no automatic retry')
|
||||||
|
if body.exists() != receipt.exists():
|
||||||
|
raise ValueError('Incomplete cache pair; preserve and inspect orphan before recovery')
|
||||||
|
if not body.exists():
|
||||||
|
return None
|
||||||
|
if body.stat().st_size > self.limit or receipt.stat().st_size > 16*1024:
|
||||||
|
raise ValueError('Oversized cache entry')
|
||||||
|
return checked_html(body.read_bytes(), json.loads(receipt.read_bytes()), url, self.limit)
|
||||||
|
|
||||||
|
def get(self, url):
|
||||||
|
cached = self.cached(url)
|
||||||
|
if cached is not None:
|
||||||
|
return cached
|
||||||
|
body, receipt, failure = self.paths(url)
|
||||||
|
try:
|
||||||
|
request = urllib.request.Request(url, headers={'User-Agent': 'PaperAtlas/0.1 (public scholarly metadata research)'})
|
||||||
|
with self.opener.open(request, timeout=30) as response:
|
||||||
|
rec = {'url': url, 'final_url': response.geturl(), 'http_status': response.status,
|
||||||
|
'content_type': response.headers.get('Content-Type', '')}
|
||||||
|
if rec['final_url'] != url or rec['http_status'] != 200 or not rec['content_type'].lower().startswith('text/html'):
|
||||||
|
raise ValueError('Unexpected metadata redirect, HTTP status or content type')
|
||||||
|
raw = response.read(self.limit+1)
|
||||||
|
if len(raw) > self.limit:
|
||||||
|
raise ValueError('Metadata response exceeds size limit')
|
||||||
|
expected = response.headers.get('Content-Length')
|
||||||
|
if expected is not None and int(expected) != len(raw):
|
||||||
|
raise ConnectionError('Incomplete metadata response')
|
||||||
|
rec.update(bytes=len(raw), sha256=digest(raw), retrieved_at=datetime.now(timezone.utc).isoformat())
|
||||||
|
checked_html(raw, rec, url, self.limit)
|
||||||
|
except (urllib.error.HTTPError, ValueError) as error:
|
||||||
|
if isinstance(error, urllib.error.HTTPError):
|
||||||
|
status = error.code
|
||||||
|
retry_after = error.headers.get('Retry-After') if error.headers else None
|
||||||
|
error.close()
|
||||||
|
else:
|
||||||
|
status = None
|
||||||
|
retry_after = None
|
||||||
|
# Access errors remain sticky until the operator verifies availability.
|
||||||
|
failure.write_text(json.dumps({'url': url, 'http_status': status, 'retry_after': retry_after, 'error_type': type(error).__name__,
|
||||||
|
'scope': 'Inspect access/redirect/response failure before moving this marker aside; no bypass or automatic retry.'})+'\n')
|
||||||
|
raise
|
||||||
|
part = body.with_suffix('.html.part')
|
||||||
|
part.write_bytes(raw); part.replace(body)
|
||||||
|
part = receipt.with_suffix('.json.part')
|
||||||
|
part.write_text(json.dumps(rec, indent=2)+'\n'); part.replace(receipt)
|
||||||
|
time.sleep(self.delay)
|
||||||
|
return raw, rec
|
||||||
BIN
Binary file not shown.
Vendored
+263
@@ -0,0 +1,263 @@
|
|||||||
|
<?xml version='1.0' encoding='UTF-8'?>
|
||||||
|
<collection id="1963.earlymt">
|
||||||
|
<volume id="1" ingest-date="2021-11-14" type="proceedings">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1/ -->
|
||||||
|
<meta>
|
||||||
|
<booktitle>Proceedings of the Annual meeting of the Association for Machine Translation and Computational Linguistics</booktitle>
|
||||||
|
<address>Denver, Colorado</address>
|
||||||
|
<month>25-26 August</month>
|
||||||
|
<year>1963</year>
|
||||||
|
<pdf hash="2ff28cb1"/>
|
||||||
|
<venue>earlymt</venue>
|
||||||
|
</meta>
|
||||||
|
<paper id="1">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.1/ -->
|
||||||
|
<title>Necessity of introducing some information provided by transformational analysis into <fixed-case>MT</fixed-case> algorithms</title>
|
||||||
|
<author><first>Irena</first><last>Bellert</last></author>
|
||||||
|
<abstract>A few examples of ambiguous English constructions and their Polish equivalents are discussed in terms of the correlation between their respective phrase-marker representations and transformational analyses. It is shown by these examples that such an investigation can reveal interesting facts for MT, and therefore should be carried out for any pair of languages for which a given MT program is being constructed. If the phrase-marker of the English construction is set into one-to-one correspondence with the phrasemarker of the Polish equivalent construction, whatever particular transformational analysis of this construction is to be taken into account, then the ambiguous phrasemarker representation can be used as a syntactical model for MT algorithms with good results. If the phrase-marker of the English construction is set into one-to-many correspondence with the phrasemarkers of the Polish equivalents, according to the transformational analyses of this construction, then the ambiguous phrase-structure representation has to be resolved in terms of transformational analysis, for only then is it possible to assign the corresponding phrase structure representation to the Polish equivalents. A tentative scheme of syntactical recognition is provided for the multiply ambiguous adjectival construction in English1 (which proved to belong to the latter case) by means of introducing some information obtained from the transformational analysis of this construction.</abstract>
|
||||||
|
<bibkey>bellert-1963-necessity</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="2">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.2/ -->
|
||||||
|
<title>The use of a random access device for dictionary lookup</title>
|
||||||
|
<author><first>Robert S.</first><last>Betz</last></author>
|
||||||
|
<author><first>Walter</first><last>Hoffman</last></author>
|
||||||
|
<abstract>The purpose of this paper will be to present a scheme to locate for single textual items and idioms in textual order their corresponding dictionary entries stored in an IBM 1301 random access mechanism. Textual items are considered to be 24 characters in length (left justified with following blanks). A dictionary entry consists of a 24 character Russian form, grammar information for the form and a set of translations for that form. Dictionary entries are packed into sequential tracks of the 1301. This paper will cover the method used for dictionary storage. The lookup for a textual item I first consists of a search for the first track that the dictionary entry E (if one exists) for I could be stored in. Once a track has been determined its contents are searched in core by a bisection convergence technique to find E. If E cannot be found, a “no entry” indication is made. If E is found a further search is made of the dictionary to find the longest sequence of text, starting with the first item I, that has a dictionary entry. The last such entry found is picked up. Included in the presentation will be examples of the dictionary lookup output for actual text.</abstract>
|
||||||
|
<bibkey>betz-hoffman-1963-use</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="3">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.3/ -->
|
||||||
|
<title>Generative processes for <fixed-case>R</fixed-case>ussian impersonal sentences</title>
|
||||||
|
<author><first>C. G.</first><last>Borkowski</last></author>
|
||||||
|
<author><first>L. R.</first><last>Micklesen</last></author>
|
||||||
|
<abstract>Impersonal sentences of Russian are those traditionally construed to consist of predicates only. Ever since the first Russian grammar was compiled, they have continued to pose a problem for grammarians. This paper is intended to be a review and evaluation of all types of the so-called impersonal sentences in the Russian language. The investigation of these sentences has been conducted in terms of their relationships to basic (kernel) sentences. Our paper attempts to define the origin for such impersonal sentences, i.e., how such sentences might be derived within the framework of a generative grammar from a set of rules possessing maximal simplicity and maximal generative power. The long-range aim of this investigation involves the most efficient manipulation of such sentences in a recognition device for Russian-English MT.</abstract>
|
||||||
|
<bibkey>borkowski-micklesen-1963-generative</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="4">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.4/ -->
|
||||||
|
<title>Concerning the role of sub-grammars in machine translation</title>
|
||||||
|
<author><first>Joyce M.</first><last>Brady</last></author>
|
||||||
|
<author><first>William B.</first><last>Estes</last></author>
|
||||||
|
<abstract>The comprehensive grammars being developed at the Linguistics Research Center of the University of Texas will be too large for easy access and manipulation in either experimental programs or practical translation. It is necessary, therefore, to devise some reliable method for selecting subsets of the grammar rules which will be reasonably adequate for a given purpose. Since the majority of the rules are dictionary rules, this problem is closely related both to the problem of constructing microglossaries and to the subsequent problem of choosing a particular microglossary suitable to a given text. Our current approach to this problem entails the construction of key word lists in the first stage of analysis which guide the computer in its choice of a previously constructed microglossary. Work to date indicates adaptations of this technique may not only contribute to the solution of storage and access problems but also facilitate analysis and simplify problems of semantic resolution.</abstract>
|
||||||
|
<bibkey>brady-estes-1963-concerning</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="5">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.5/ -->
|
||||||
|
<title>Word-meaning and sentence-meaning</title>
|
||||||
|
<author><first>Elinor K.</first><last>Charney</last></author>
|
||||||
|
<abstract>A theory of semantics is presented which (1) defines the meanings of the most frequently occurring semantic morphemes (‘all’, ‘unless’, ‘only’, ‘if’, ‘not’, etc.), (2) explains their role, as semantically interdependent structural-constants, in giving rise to sentence-meanings, (3) suggests a possible approach to a sentenceby-sentence recognition program, and (4) offers a feasible method of coordinating among different language systems synonymous sentences whose grammatical features and structural-constants do not bear a one-to-one correspondence to one another. The theory applies only to morphemes that function as structural-constants and their interlocking relationships, denotative terms being treated as variables whose ranges alone have structural significance in sentencemeaning. The basic views underlying the theory are: In any given sentence, it is the particular configuration of structural-constants in combination with specific grammatical features which produces the sentencemeaning; the defined meaning of each individual structural-constant remains constant. The word-meanings of this type of morpheme, thus, must be carefully distinguished from the sentence-meanings that configuration of these morphemes produce. Sentence-synonymy is not based upon word-synonymy alone. Contrary to the popular view that the meanings of all of the individual words must be known before the sentencemeaning can be known, it is shown that one must comprehend the total configuration of structural-constants and syntactical features in a sentence in order to comprehend the correct sentence-meaning and that this understanding of the sentence as a whole must precede the determination of the correct semantic interpretation of these critical morphemes. In fact, the structural features that produce the sentence-meanings may restrict the possible meanings of even the denotative terms since a structural feature may demand, for example, a verbal rather than a noun phrase as an indispensable feature of the configuration. Two or more synonymous sentences whose denotative terms are everywhere the same but whose structural configurations are not isomorphic express the same fundamental sentence-meaning. The fundamental sentence-meanings can be explicitly formulated, and serve as the mapping functions to co-ordinate morphemically-unlike synonymous sentences within a language system or from one system to another. The research goal of the author is to establish empirically these translation rules that state formally the structural characteristics of the sentence configurations whose sentence-meanings, as wholes, are related as synonymous.</abstract>
|
||||||
|
<bibkey>charney-1963-word</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="6">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.6/ -->
|
||||||
|
<title>Translating ordinary language into symbolic logic</title>
|
||||||
|
<author><first>Jared L.</first><last>Darlington</last></author>
|
||||||
|
<abstract>The paper describes a computer program, written in COMIT, for translating ordinary English into the notation of propositional logic and first-order functional logic. The program is designed to provide an ordinary language input to a COMIT program for the David-Putnam proof-procedure algorithm. The entire set of operations which are performed on an input sentence or argument are divided into three stages. In Stage I, an input sentence ‘S’, such as “The composer who wrote ‘Alcina’ wrote some operas in English,” is rewritten in a quasi-logical notation, “The X/A such that X/A is a composer and X/A wrote Alcina wrote some X/B such that X/B is an opera and X/B is in English.” The quasi-logical notation serves as an intermediate language between logic and ordinary English. In Stage II, S is translated into the logical notation of propositional functions and quantifiers, or of propositional logic, whichever is appropriate. In Stage III, S is run through the proof-procedure program and evaluated. (The sample sentence quoted is of course ‘invalid’, i.e. nontautological.) The COMIT program for Stage III is complete, that for Stage II is almost complete, and that for Stage I is incomplete. The paper describes the work done to date on the programs for Stages I and II.</abstract>
|
||||||
|
<bibkey>darlington-1963-translating</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="7">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.7/ -->
|
||||||
|
<title>The graphic structure of word-breaking</title>
|
||||||
|
<author><first>J. L.</first><last>Dolby</last></author>
|
||||||
|
<author><first>H. L.</first><last>Resnikoff</last></author>
|
||||||
|
<abstract>In a recent paper1 the authors have shown that it is possible to determine the possible parts of speech of English words from an analysis of the written form. This determination depends upon the ability to determine the number of graphic syllables in the word. It is natural, then, to speculate as to the nature of graphic syllabification and the relation of this phenomenon to the practice of word-breaking in dictionaries and style manuals. It is not at all clear at the start that dictionary wordbreaking is subject to any fixed structure. In fact, certain forms cannot be broken uniquely in isolation since the dictionary provides different forms depending upon whether the word is used as a noun or a verb. However, it is shown in this paper that letter strings can be decomposed into 3 sets of roughly the same size in the following manner: in the first, strings are never broken in English words; in the second, the strings are always broken in English words; and in the third, both situations occur. Rules for breaking vowel strings are obtained by a study of the CVC forms. Breaks involving consonants can be determined by noting whether or not the consonant string occurs in penultimate position with the final c. The final e in compounds also serves to identify the forms that are generally split off from the rest of the word. A thorough analysis is made of the accuracy of the rules given when applied to the 12,000 words of the Government Printing Office Style Manual Supplement on word-breaking. Comparisons are also drawn between this source and several American dictionaries on the basis of a random sample of 500 words.</abstract>
|
||||||
|
<bibkey>dolby-resnikoff-1963-graphic</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="8">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.8/ -->
|
||||||
|
<title>Writing of <fixed-case>C</fixed-case>hinese recognition grammar for machine translation</title>
|
||||||
|
<author><first>Ching-yi</first><last>Dougherty</last></author>
|
||||||
|
<abstract>Our approach to this problem is based on the stratificational grammar outlined and the procedures proposed by Dr. Sydney Lamb. How the theory and the procedures can be applied to written Chinese is briefly discussed. For the time being our research is limited to the particular kind of written Chinese found in chemical and biochemical journals. First the Chinese lexes are classified by detailed syntactical analysis, then binary grammar rules are constructed for joining two primary or constitute classes. How a more and more refined classification can eliminate one by one the ambiguity resulting from all possible constructions arising from juxtaposition of two distributional classes is discussed in detail.</abstract>
|
||||||
|
<bibkey>dougherty-1963-writing</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="9">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.9/ -->
|
||||||
|
<title>The behavior of <fixed-case>E</fixed-case>nglish articles</title>
|
||||||
|
<author><first>H. P.</first><last>Edmundson</last></author>
|
||||||
|
<abstract>Machine translation has often been conceived as consisting of three steps: analysis of source-language sentence, transformation of analyzed pieces, and synthesis of target-language sentence. This paper is concerned with one aspect of the last step, namely, the rules of behavior of English articles. Since the classical definitions of definite and indefinite articles are operationally imprecise, proper mechanistic rules must be formulated in order to permit the automatic insertion or non-insertion of English articles. The rules discussed are of syntactic origin; however, note is also taken of their semantic aspects. This paper describes the methods used to derive these rules and offers ideas for further research.</abstract>
|
||||||
|
<bibkey>edmundson-1963-behavior</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="10">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.10/ -->
|
||||||
|
<title>On representing syntactic structure</title>
|
||||||
|
<author><first>E. R.</first><last>Gammon</last></author>
|
||||||
|
<abstract>The idea of sentence depth of Yngve (A Model and an Hypothesis for Language Structure, Proc. Am. Phil. Soc., Vol. 104, No. 5, Oct. 1960) is extended to the notion of “distance” between constituents of a construction. The distance between constituents is defined as a weighted sum of the number of IC cuts separating them. Yngve’s depth is then a maximum distance from a sentence to any of its words. Various systems of weighting cuts are investigated. For example, in endocentric structures we may require that the distance from an attribute to the structure exceeds the distance from the head to the structure, and in exocentric structures that the distances from each constituent to the structure are equal. Representations of constructions are considered which preserve the distance between constituents. It is shown that it is impossible to represent some sentences in Euclidean space with exact distances, but a representation may be found if only relative order is preserved. If more general spaces are used then exact distances may be represented. It follows that for a wide class of sentence types, there is a weighting, and a space, in which the distance preserving representations are identical with the diagrams of traditional grammar.</abstract>
|
||||||
|
<bibkey>gammon-1963-representing</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="11">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.11/ -->
|
||||||
|
<title>La traduction automatique et l’enseignement du russe</title>
|
||||||
|
<author><first>Yves</first><last>Gentilhomme</last></author>
|
||||||
|
<abstract>Les recherches effectuées depuis quelques années en vue de la Traduction Automatique ont conduit à des méthodes de travail et à des résultats intéressant la pédagogie des langues. Une expérience d’enseignement du russe a l’usage des scientifiques fondée sur ces données a été poursuivie pendant deux ans à Paris (Centre National de la Recherche Scientifique et Faculté des Sciences), et a abouti à la publication d’un manuel. Le present compte-rendu a pour objet de préciser les principes généraux utilisés, la réaction des étudiants et le rendement pédagogique obtenu. 1. Graphes morphologiques: Les mots d’une même famille. Notion de base. La double ramification. Les graphes abstraits. Les néologismes scientifiques. 2. Graphes syntaxiques: La double structure d’une phrase. Multiplicité des modèles. Point de vue psychologique. Notion de fonction. Continuité et discontinuité. 3. Les séparateurs: La segmentation d’une phrase. Le vocabulaire prioritaire. 4. Théorie de la valence: macro et microcontexte. Qu’est-ce-que “connaître un mot”? 5. Point de vue de l’étudiant; point de vue du traducteur humain; et point de vue de l’Enseignant.</abstract>
|
||||||
|
<bibkey>gentilhomme-1963-la</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="12">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.12/ -->
|
||||||
|
<title>Word and context association by means of linear networks</title>
|
||||||
|
<author><first>Vincent E.</first><last>Giuliano</last></author>
|
||||||
|
<abstract>This paper is concerned with the use of electrical networks for the automatic recognition of statistical associations among words and contexts present in written text. A general mathematical theory is proposed for association by means of linear transformations, and it is shown that this theory can be realized through use of passive linear electrical networks. Several smallscale experimental associative networks have been built, and are briefly described in the paper; one such device will be demonstrated in the course of the oral presentation of the paper. Some of the devices generate measures of association among index terms used to characterize a document collection, and between the index terms and the documents themselves. Another uses syntactic proximity within sentences as a criterion for the generation of word association measures. Examples are given of associations produced by these network devices. It is conjectured that the networkproduced association measures reflect two distinct types of linguistic association—“synonymy” association which reflects similarity of meaning, and “contiguity” association which reflects real-world relationships among designata.</abstract>
|
||||||
|
<bibkey>giuliano-1963-word</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="13">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.13/ -->
|
||||||
|
<title>A study of the combinatorial properties of <fixed-case>R</fixed-case>ussian nouns</title>
|
||||||
|
<author><first>Kenneth E.</first><last>Harper</last></author>
|
||||||
|
<abstract>A statistical study was made of the extent to which Russian nouns enter into certain kinds of syntactic combination. The basis of the study was a corpus of 180,000 running words of Russian physics text prepared for analysis by the Automatic Language Data Processing group at The Rand Corporation; for each sentence of text the syntactic dependency of each word had been previously coded. A data retrieval program was applied, showing for each noun in text the number of occurrences (a) with at least one genitive noun dependent, (b) with at least one adjective dependent, and (c) with either type of dependent. A listing of all nouns in text (64,026 occurrences of 2,993 nouns) was prepared, ordered by frequency, and showing counts for a, b, and c above. Separate listings were prepared, showing for each noun that occurred 50 times or more the probability P that it would be modified in each of these three ways; these listings were ordered on P. The data suggests, among others, the following conclusions: there is statistical significance in the variability with which nouns enter into the given combinations; the partial interchangeability of adjective and genitive noun modification is supported; a general correspondence exists between combinatorial groupings of nouns and morphological or semantic groupings (concrete nouns have low P for genitive complementation, abstract nouns have high P, etc); the use of words in a given field of discourse can be determined empirically (e.g., the use of deverbative nouns either to indicate a process or the result of a process). It is suggested that the distributional approach is a useful supplement to traditional syntactic and semantic classification schemes, and that it is of direct utility in automatic parsing programs.</abstract>
|
||||||
|
<bibkey>harper-1963-study</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="14">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.14/ -->
|
||||||
|
<title>Connectability calculations, syntactic functions, and <fixed-case>R</fixed-case>ussian syntax</title>
|
||||||
|
<author><first>David G.</first><last>Hays</last></author>
|
||||||
|
<abstract>A program for sentence-structure determination can be divided into routines for analysis of word order and for testing the grammatical connectability of pairs of sentence members. The present paper describes a connectability-test routine that uses the technique called code matching. This technique requires elaborate descriptions of individual items, say the words or morphemes listed in a dictionary, but it avoids the use of large tables or complicated programs for testing connectability. Development of the technique also leads to a certain clarification of the linguistic concepts of function, exocentrism, and homography. In the present paper, a format for the description of Russian items is offered and a program for testing the connectability of pairs of Russian items is sketched. This system recognizes nine dominative functions: subjective; first, second, and third complementary; first, second, and third auxiliary; modifying; and predicative. The nature of a program for testing connectability with respect to coordinative functions (coordination, apposition, etc.) is suggested.</abstract>
|
||||||
|
<bibkey>hays-1963-connectability</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="15">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.15/ -->
|
||||||
|
<title>Punctuation and automatic syntactic analysis</title>
|
||||||
|
<author><first>Lydia</first><last>Hirschberg</last></author>
|
||||||
|
<abstract>In this paper we discuss how algorithms for automatic analysis can take advantage of information carried by the punctuation marks. We neglect stylistic aspects of punctuation because they lack universality of usage and we restrict ourselves to those rules which any punctuation must observe in order to be intelligible. This involves a concept we call “coherence” of punctuation. In order to define “coherence”, we introduce two characteristics, which we prove to be mutually independent, namely “separating power” and “syntactic function”. The separating power is defined by three experimental laws expressing the fact that two punctuation marks of different separating power prevent to a different extent syntactic links from crossing them. These laws are defined independently of any particular grammatical character of the punctuation marks or of the attached grammatical syntagms. On the other hand, whichever grammatical system we choose, we may assimilate the punctuation marks to the ordinary words, to the extent that we can assign to them a known grammatical character and function, well defined in any particular context. They differ however from the other words by their large number of homographs and synonyms i.e. by the fact that almost every punctuation mark can occur with almost every grammatical value in each particular case, and in quite similar contexts. The syntactic functions, in general, and in particular those of the punctuation marks, can be ordered according to an arbitrary scale of decreasing “value” of syntactic links, where the “value” of a link is directly related to the number of syntactic conditions the links must satisfy. The law of coherence, then, shows that in a given context, a particular punctuation mark cannot indistinctly represent all its homographs, so that a certain number of assumptions about its syntactic nature and function can be discarded. This law can be stated as follows: “When moving from a punctuation mark to its immediate (left or right) neighbor in any text, the separating power cannot increase if the value of the syntactic function increases and vice-versa”. In addition we review two related topics, namely the stylistic character of punctuation and the necessity and existence of intrinsic criteria of grammatically, i.e. independent of punctuation. We propose such a criterion, and suggest a formalism related to the parenthesis free notation of logic.</abstract>
|
||||||
|
<bibkey>hirschberg-1963-punctuation</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="16">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.16/ -->
|
||||||
|
<title>Application of decision tables to syntactic analysis</title>
|
||||||
|
<author><first>Walter</first><last>Hoffman</last></author>
|
||||||
|
<author><first>Amelia</first><last>Janiotis</last></author>
|
||||||
|
<author><first>Sidney</first><last>Simon</last></author>
|
||||||
|
<abstract>Decision tables have recently become an object of investigation as a possible means of improving problem formulation of data processing procedures. The initial emphasis for this new tool came from systems analysts who were primarily concerned with business data processing problems. The purpose of this paper is to investigate the suitability of decision tables as a means of expressing syntactic relations as an alternative to customary flow charting techniques. The history of decision tables will be briefly reviewed and several kinds of decision tables will be defined. As an example, parts of the predicative blocking routine developed at Wayne State University will be presented as formulated with the aid of decision tables. The aim of the predicative blocking routine is to group a predicative form together with its modal and temporal auxiliaries, infinitive complements, and negative particle, if any of these exist. The object of the search is to define such a syntactic block, but it may turn out instead that an infinitive phrase is defined or that a possible predicative form turns out to be an adverb.</abstract>
|
||||||
|
<bibkey>hoffman-etal-1963-application</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="17">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.17/ -->
|
||||||
|
<title>Simultaneous computation of lexical and extralinguistic information measures in dialogue</title>
|
||||||
|
<author><first>Joseph</first><last>Jaffe</last></author>
|
||||||
|
<abstract>An approach to the study of information processing in verbal interaction is described. It compares patterns of two indices of dispersion in recorded dialogue. The lexical measure is the mean segmental type—token ratio, based on 25-word segments of the running conversation. It is computed from a key punched transcript of the dialogue without regard to the speaker of the words. The extralinguistic measure is the H statistic, computed from the temporal pattern of the interaction. The latter is prepared from a two-channel tape recording by a special analogue to digital converter (AVTA system) which key punches the state of the vocal transaction 200 times per minute. Probabilities of the four possible states (either A or B speaking, neither speaking, both speaking) are the basis for the computation. All analyses are done on the IBM 7090. The methodology is part of an investigation of information processing in dyadic systems, aimed toward the reclassification of pathological communication.</abstract>
|
||||||
|
<bibkey>jaffe-1963-simultaneous</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="18">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.18/ -->
|
||||||
|
<title>Design of a generalized information system</title>
|
||||||
|
<author><first>Ronald W.</first><last>Jonas</last></author>
|
||||||
|
<abstract>While mechanical translation research involves the design of a computer system which simulates language processes, there is the associated problem of collecting the language data which are to be used in translation. Because large quantities of information will be needed, the computer may be useful for data accumulation and verification. A generalized information system should be able to accept the many types of data which a linguist encodes. A suitable means of communication between the linguist and the system has to be established. This may be achieved with a central input, called Linguistic Requests, and a central output, called Information Displays. The requests should be coordinated so that all possible inputs to the system are compatible, and the displays should be composed by the system such that they are clearly understandable. An information system should be interpretive of the linguist’s needs by allowing him to program the data manipulation. The key to such a scheme is that the linguist be permitted to classify his data freely and to retrieve it as he chooses. He should have at his disposal selecting, sorting, and displaying functions with which he can verify data, select data for introduction to a mechanical translation system, and perform other activities necessary in his research. Such an information system has been designed at the Linguistics Research Center of The University of Texas.</abstract>
|
||||||
|
<bibkey>jonas-1963-design</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="19">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.19/ -->
|
||||||
|
<title>Some experiments performed with an automatic paraphraser</title>
|
||||||
|
<author><first>Sheldon</first><last>Klein</last></author>
|
||||||
|
<abstract>The automatic paraphrasing system used in the experiments described herein consisted of a phrase structure, grammatically correct nonsense generator coupled with a monitoring system that required the dependency relations of the sentence in production to be in harmony with those of a source text. The output sentences also appeared to be logically consistent with the content of that source. Dependency was treated as a binary relation, transitive except across most verbs and prepositions. Five experiments in paraphrasing were performed with this basic system. The first attempted to paraphrase without the operation of the dependency monitoring system, yielding grammatically correct nonsense. The second experiment included the operation of the monitoring system and yielded logically consistent paraphrases of the source text. The third and fourth experiments demanded that the monitoring system permit the production of only those sentences whose dependency relations were non-existent in the source text. While these latter outputs were seemingly nonsensical, they bore a special logical relationship to the source. The fifth experiment demanded that the monitoring system permit the production of sentences whose dependency relations were the converse of those in the source. This restriction was equivalent to turning the dependency tree of the source text upside down. The output of this experiment consisted only of kernel type sentences which, if read backwards, were logically consistent with the source. The results of these experiments determine some formal properties of dependency and engender some comments about the role of dependency in phrase structure and transformational models of language.</abstract>
|
||||||
|
<bibkey>klein-1963-experiments</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="20">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.20/ -->
|
||||||
|
<title>Interlingual correspondence at the syntactic level</title>
|
||||||
|
<author><first>Edward S.</first><last>Klima</last></author>
|
||||||
|
<abstract>The paper will investigate a few major construction types in several related European languages: relative clauses, attributive phrases, and certain instances of coordinate conjunction involving these constructions. In each of the languages independently, the constructions will be described as resulting from syntactic mechanisms further analyzable into chains of partially ordered operations on more basic structures. Pairs of sentences equivalent in two languages will be examined. Sentences will be considered equivalent if they are acceptable translations of one another. The examples used will, in fact, be drawn primarily from standard translations of scholarly and literary prose. Equivalence between whole sentences can be further analyzed, as will be shown, into general equivalence 1) between the chains of operations describing the constructions and 2) between certain elements (e.g., lexical items) in the more basic underlying structures. It will be seen that superficial differences in the ultimate shape of certain translation pairs can be accounted for as the result of minor differences in the particular operations involved or in the basic underlying structure. We shall examine two languages (e.g., French and German) in which attributive phrase formation and relative clause formation on the whole correspond and in which, in a more or less abstract way, the rules of relative clause formation are included as intermediate links in the chain of operations describing attributive phrases. The fact that in particular cases a relative clause in the one language corresponds to an attributive phrase in the other will be found to result from, e.g., differences in the choice of perfect auxiliary in the two languages.</abstract>
|
||||||
|
<bibkey>klima-1963-interlingual</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="21">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.21/ -->
|
||||||
|
<title>Sentence structure diagrams</title>
|
||||||
|
<author><first>Susumu</first><last>Kuno</last></author>
|
||||||
|
<abstract>A system for automatically producing a sentence structure diagram for each analysis of a given sentence has been added to the program of the multiple-path syntactic analyzer. A structure code, consisting of a series of structure symbols or phrase markers that identify the successive higher-order structures to which the word in question belongs, is assigned to each word of the sentence. The set of structure codes for the words of a given sentence is equivalent to an explicit tree diagram of the sentence structure, but more compact and easier to lay out on conventional printers. The diagramming system makes some experimental assumptions about the dependencies of certain structures upon higher-level structures. All the major syntactic components of a sentence (i.e., subject, verb, object, complement, period, or question mark) are represented in the current system as occurring on the same level, all being dependent on the topmost level, “sentence”. A floating structure such as a prepositional phrase or adverbial phrase or clause, whose dependency is not determined in the analyzer, is represented as depending upon the nearest preceding structure modifiable by such a floating structure. Different assumptions as to structural dependencies would yield different diagrams without requiring modification on the main flow of the diagramming program. The diagrams thus obtained contribute greatly to the rapid and accurate evaluation of the analysis results, and they are also useful for obtaining basic syntactic patterns of analyzed structures, and for detecting the head of each identified structure.</abstract>
|
||||||
|
<bibkey>kuno-1963-sentence</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="22">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.22/ -->
|
||||||
|
<title>Linguistic structure and machine translation</title>
|
||||||
|
<author><first>Sydney M.</first><last>Lamb</last></author>
|
||||||
|
<abstract>If one understands the nature of linguistic structure, one will know what design features an adequate machine translation system must have. To put it the other way around, it is futile to attempt the construction of a machine translation system without a knowledge of what the structure of language is like. This principle means that if someone wants to construct a machine translation system, the most important thing he must do is to understand the structure of language. Any MT system, whether by conscious intention on the part of its creators or not, is based upon some view of the nature of linguistic structure. By making explicit the underlying theory for various MT systems which have been proposed we can determine whether or not they are adequate. Similarly, by observing linguistic phenomena we can determine what properties an adequate theory of language must have, and such determination will show what features an MT system must have in order to be adequate. It can be shown that some of the approaches to MT now being pursued must necessarily fail because their underlying linguistic theories are inadequate to account for various well-known linguistic phenomena.</abstract>
|
||||||
|
<bibkey>lamb-1963-linguistic</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="23">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.23/ -->
|
||||||
|
<title>On redundancy in artificial languages</title>
|
||||||
|
<author><first>W. P.</first><last>Lehmann</last></author>
|
||||||
|
<abstract>Artificial languages are one concern of work in computational linguistics, if only as a mnemonic device for interlinguas which will be developed. Even if it does not gain wider use, the structure of an artificial language is of general interest. In contrast to the artificial languages which have been widely proposed, linguistic principles underlying a welldesigned artificial language and its usefulness are wellestablished, particularly through Trubetzkoy’s article, TCLP 8.5-21. which indicates phonological limitations for such a language. Since Trubetzkoy’s specifications yield a total of approximately 11,000 morphemes, if an artificial language incorporated the degree of redundancy found in natural languages it would be severely handicapped by the size of its lexicon. The paper discusses the problem particularly with regard to suprasegmentals, which Trubetzkoy almost entirely ignored.</abstract>
|
||||||
|
<bibkey>lehmann-1963-redundancy</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="24">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.24/ -->
|
||||||
|
<title>A procedure for automatic sentence structure analysis</title>
|
||||||
|
<author><first>D.</first><last>Lieberman</last></author>
|
||||||
|
<abstract>The two main considerations in the design of this procedure were the economical recognition and representation of multiple readings of syntactically ambiguous sentences, and general applicability to “all” languages (English, Russian, Chinese). The following features will be discussed: types of structural descriptions, form of linguistic rules, use of linguistic heuristics to achieve economical multiple analyses, application to linguistic research and application to production MT systems. Also, the relation between this procedure and other existing sentence analysis procedures will be discussed.</abstract>
|
||||||
|
<bibkey>lieberman-1963-procedure</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="25">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.25/ -->
|
||||||
|
<title>An algorithm for the translation of <fixed-case>R</fixed-case>ussian inorganic-chemistry terms</title>
|
||||||
|
<author><first>L. R.</first><last>Micklesen</last></author>
|
||||||
|
<author><first>P. H.</first><last>Smith</last></author>
|
||||||
|
<abstract>An algorithm has been devised, and a computer program written, to translate certain recurring types of inorganic-chemistry terms from Russian to English. The terms arc all noun-phrases, and several different types of such phrases have been included in the program. Examples are: AZOTNONATRIEVA4 SOL6 sodium nitrate SOL6 ZAKISI/OKISI JELEZA ferrous/ferric salt ZAKISNA4 OKISNA4 SOL6 JELEZA GIDRAT ZAKISI/OKISI JELEZA ferrous/ferric salt etc., where the stems underlined may be replaced by any of a number of other stems (up to 65 in some positions) in the particular type. Translation of each type encounters problems common to almost all the types: (1) The Russian noun is translated as an English adjective, while the noun of the resulting English phrase is found among the modifiers of the Russian noun. (2) The Russian noun (English adjective) may be a metal with more than one valence state, the state indicated (if at all) by the modifiers. (3) The number of the resulting English noun-phrase is determined by some member of the Russian phrase other than the noun. (4) The phrase elements may occur compounded in the chemical phrase but free in other contexts, and dictionary storage must provide for this. The program permits translation of conjoined phrase elements as well. The paper also includes an investigation into the deeper grammatical implications of this type of chemical nomenclature, and some excursions into the semantic correlations involved.</abstract>
|
||||||
|
<bibkey>micklesen-smith-1963-algorithm</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="26">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.26/ -->
|
||||||
|
<title>The application of table processing concepts to the Sakai translation technique</title>
|
||||||
|
<author><first>A.</first><last>Opler</last></author>
|
||||||
|
<author><first>R.</first><last>Silverstone</last></author>
|
||||||
|
<author><first>Y.</first><last>Saleh</last></author>
|
||||||
|
<author><first>M.</first><last>Hildebran</last></author>
|
||||||
|
<author><first>I.</first><last>Slutzky</last></author>
|
||||||
|
<abstract>In 1961, I. Sakai described a new technique for the mechanical translation of languages. The method utilizes large tables which contain the syntactic rules of the source and target languages. As part of a study of the AN/GSQ-16 Lexical Processing Machine, a modification of the Sakai method was developed. Five of six planned table scanning phases were implemented and tested. Our translation system (1) converts input text to syntactic and semantic codes with a dictionary scan, (2) clears syntactic ambiguities where resolution by adjacent words is effective, (3) resolves residual syntactic ambiguities by determining the longest meaningful semantic unit, (4) reorders word sequence according to the rules of the target language and (5) produces the final target language translation. French to English was the source-target pair selected for the study. An Input Dictionary of 3,000 French stems was prepared and 17,000 entries comprised the Input Product Table (allowable syntactic combinations ). Since Sakai was working with highly dissimilar languages, he found it necessary to use an intermediate language. Because of the structural similarity between French and English, we found an intermediate language was unnecessary. The method proved straightforward to implement using the table lookup logic of the Lexical Processor. The translation was actually performed on an IBM 1401 which we programmed to simulate the concept of the AN/GSQ-16 Lexical Processor. In our implementation magnetic tapes replaced the photoscopic storage disk.</abstract>
|
||||||
|
<bibkey>opler-etal-1963-application</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="27">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.27/ -->
|
||||||
|
<title><fixed-case>S</fixed-case>lavic languages—comparative morphosyntactic research</title>
|
||||||
|
<author><first>Milos</first><last>Pacak</last></author>
|
||||||
|
<abstract>An appropriate goal for present-day linguistics is the development of a general theory of relations between languages. One necessary requirement in the development of such a theory is the identification and classification of inflected forms in terms of their morphosyntactic properties in a set of presumably related languages. According to Sapir, “all languages differ from one another, but certain ones differ far more than others”. As for the Slavic languages he might well have said that they are all alike, but some are more alike than others. The similarities stemming from their common origin and from subsequent parallel development enable us to group them into a number of more or less homogeneous types. The experimental comparative research at The Georgetown University was focused on a group of four Slavic languages, namely, Russian, Czech, Polish and Serbocroatian. The first step in the comparative procedure here described is the morphosyntactic analysis of each of the four languages individually. The analysis should be based on the complementary distribution of inflectional morphemes. The properties whose distribution must be determined are: 1) the graphemic shape of the inflectional morphemes, 2) the establishment of distributional classes and subclasses of stem morphemes and (on the basis of 1 and 2), 3) the morphosyntactic function of inflectional morphemes which is determined by the distributional subclass of the stem morpheme. f(x,y)-l, where x is the distributional subclass of the stem morpheme (which is a constant) and y is the given inflectional morpheme (which is a free variable). On the basis of this preliminary analysis the patterns of absolute equivalence, partial equivalence, and absolute difference can be established for each class of inflected forms in each language under study. Once this has been accomplished, the results can be used in order to determine the extent of distributional equivalences among the individual languages. The applicability of this procedure was tested on the class of adjectivals. Within the frame of adjectivals the following morphosyntactic properties were analyzed within each language languages: 1) the category of gender, 2) the category of animateness, 3) the category of case and murder. The product of this comparative analysis is a set of formation rules which embody a system for the identification of the inflected forms. The detailed result will be presented in an additional report.</abstract>
|
||||||
|
<bibkey>pacak-1963-slavic</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="28">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.28/ -->
|
||||||
|
<title>Types of language hierarchy</title>
|
||||||
|
<author><first>E. D.</first><last>Pendergraft</last></author>
|
||||||
|
<abstract>Various relations lead to hierarchical systems of linguistic description. This paper considers briefly a typology of descriptive metalanguages based on such relations and sketches possible consequences for computational linguistics. Its scope is accordingly limited to metalanguages having operational interpretations which specify individual linguistic processes and structural interpretations which specify language data of individual languages. Immediate-constituent, context-free metalanguages are used to illustrate hierarchical types.</abstract>
|
||||||
|
<bibkey>pendergraft-1963-types</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="29">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.29/ -->
|
||||||
|
<title>Path economization in exhaustive left-to-right syntactic analysis</title>
|
||||||
|
<author><first>Warren J.</first><last>Plath</last></author>
|
||||||
|
<abstract>In exhaustive left-to-right syntactic analysis using the predictive approach, each path of syntactic connection which originates at the beginning of a sentence must be followed until it is clear whether or not it will lead to the production of a well-formed analysis. The original scheme of following each path until it terminates either in an analysis or in a grammatical inconsistency has been considerably improved through the. incorporation of two path-testing techniques. Using the first technique, the program abandons a path as unproductive whenever a situation is detected where the prediction pool contains more predictions of a given type than can possibly be fulfilled by the remaining words in the sentence. Employment of the second technique, which is based on periodic comparison of the current prediction pool with pools formed on earlier productive paths, eliminates repeated analysis of identical right-hand segments which belong to distinct paths. Taken together, the two path-testing procedures frequently enable the program to terminate the processing of a path well before its end has been reached. For most sentences, this means a considerable reduction in the total path length traversed, accompanied by a corresponding increase in the speed of analysis. Comparison of runs performed using both versions of the program indicates that employment of the new techniques reduces the average running time per sentence to less than one-fifth of its former value.</abstract>
|
||||||
|
<bibkey>plath-1963-path</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="30">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.30/ -->
|
||||||
|
<title>A computer representation for semantic information</title>
|
||||||
|
<author><first>Bertram</first><last>Raphael</last></author>
|
||||||
|
<abstract>This paper deals with the problem of representing in a useful form, within a digital computer, the information content of statements in natural language. The model proposed consists of words and list-structure associations between words. Statements in simple English are thought of as describing relations between objects in the real world. Sentences are analyzed by matching them against members of a list of formats, each of which determines a unique relation. These relations are stored on description-lists associated with those words which denote objects (or sets of objects). A LISP computer program uses this model in the context of a simple question-answering system. Functions are provided which may grow, search, and modify this model. Formats and functions dealing with set-relations, part-whole and numeric relations, and left-toright spatial relations have been included in the system, which is being expanded to handle other types of relations. All functions which operate on the model report information concerning their actions to the programmer, so that the applicability and limitations of this kind of model may more easily be evaluated.</abstract>
|
||||||
|
<bibkey>raphael-1963-computer</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="31">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.31/ -->
|
||||||
|
<title>Specifications for generative grammars used in language data processing</title>
|
||||||
|
<author><first>Robert</first><last>Tabory</last></author>
|
||||||
|
<abstract>It becomes more and more evident that successful pragmatics (i.e. automatic recognition and production procedures for sentences) cannot be performed without previously written generative grammars for the languages involved, using an underlying meta-theoretical framework proposed by the present school of mathematical linguistics. Two aspects of grammar writing are examined: 1. A taxonomy over the non-terminal vocabulary, using a subscripting system for signs and fitting into the more general string taxonomy of phrase structure components. The resulting more complex lexical organization is studied. 2. A command syntax for phrase structure components limiting the full, not necessarily needed generative power of these grammars. The proposed restrictions correspond to a priori linguistic intuition. Applicational order and location of the rules is studied. Finally, the recognitional power and generative capacity of a computer are examined, the machine being structured according to a Newell-Shaw-Simon list system. It is well known that pushdown stores are particular cases of list structures, that context-free grammars are particular cases of phrase structure grammars and that pushdown stores are the generative devices for context-free grammars.</abstract>
|
||||||
|
<bibkey>tabory-1963-specifications</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="32">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.32/ -->
|
||||||
|
<title>Collecting linguistic data for the grammar of a language</title>
|
||||||
|
<author><first>Wayne</first><last>Tosh</last></author>
|
||||||
|
<abstract>Establishing the grammatical description of a language is one of the major tasks facing the technician in machine translation. Another is that of creating the system of programs with which to carry out the translation process. The Linguistics Research Center of The University of Texas recognizes the advantages in maintaining the specialties of linguistic research and computer programming as two separate areas of endeavor. We regard the linguistic task as a problem in convergence. We do not expect ever to have a final description of a language (except theoretically for a given point in the history of that language). We do expect, however, to begin with almost immediate application of the very first grammatical description. We shall make repeated revisions of the grammar as we learn how to make it approximate better the language text fed into the computer. The grammatical description of any one language is based primarily on specific text evidence. We are not attempting to describe “the language”. We are, however, attempting to make descriptive decisions sufficiently general that new text evidence does not require extensive revision of earlier descriptions. Corpora selected for description are chosen so as to have similar texts within the same scientific discipline for the several languages. Tree diagrams are drawn for each sentence in detail. The diagrams are inspected for consistency before corresponding phrase-structure rules are compiled in the computer. The grammar is then verified in the computer system and revised as necessary.</abstract>
|
||||||
|
<bibkey>tosh-1963-collecting</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="33">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.33/ -->
|
||||||
|
<title>Derivational suffixes in <fixed-case>R</fixed-case>ussian general vocabulary and in chemical nomenclature</title>
|
||||||
|
<author><first>John H.</first><last>Wahlgren</last></author>
|
||||||
|
<abstract>A grammar based upon a conventional morphemic analysis of Russian will have a rather large inventory of derivational suffixes. A relatively small number of these recur with sufficient generality to acquire lexemic status (i.e., to be what is usually termed “productive”). Names of chemical substances in Russian may likewise be analyzed as combinations of roots or stems with derivational affixes, in particular, suffixes. The number of productive suffixes in the chemical nomenclature is considerably larger than in the general vocabulary. These suffixes derive from adoption into Russian of an international system of chemical nomenclature. A grammar of this system is basically independent of any grammar of Russian. It must, however, be consistently incorporated into the grammar and dictionary which are to serve in a machine translation system for texts in the source language containing chemical names. Grammatical analysis of chemical suffixes and connected study of general Russian derivational suffixes has raised certain practical problems and theoretical questions concerning the nature of derivation. On the practical side, where a complex and highly productive system is involved, effective means of detecting and dealing with homography have required development. Theoretical consideration has been given to the question of grammaticality in chemical names and to problems of sememic analysis and classification of root and stem lexemes into tactic classes on the basis of cooccurrence with derivational suffixes.</abstract>
|
||||||
|
<bibkey>wahlgren-1963-derivational</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="34">
|
||||||
|
<!-- https://aclanthology.org/1963.earlymt-1.34/ -->
|
||||||
|
<title>On the order of clauses</title>
|
||||||
|
<author><first>Victor H.</first><last>Yngve</last></author>
|
||||||
|
<abstract>We used to think that the output of a translation machine would be stylistically inelegant, but this would be tolerable if only the message got across. We now find that getting the message across accurately is difficult, but we may be able to have stylistic elegance in the output since much of style reflects depth phenomena and thus is systematic. As an example, the order of the clauses in many twoclause sentences can be reversed without a change of meaning, but the same is not normally true of sentences with more than two clauses. The meaning usually changes when the clause order is changed. Equivalently, there appear to be severe restrictions on clause order for any given meaning. These restrictions appear to follow from depth considerations. The idea is being investigated that there is a normal depth-related clause order and any deviations from this order must be signalled by special syntactic or semantic devices. The nature of these devices is being explored. When translating multi-clause sentences, there may be trouble due to the fact that the clause types of the two languages are not exactly parallel. Therefore the list of allowed and preferred clause orders in the two languages will not be equivalent and the special syntactic and semantic devices available to signal deviations from the normal order will be different. Thus one would predict that multi-clause sentences in language A often have to be split into two or more sentences when translated into language B, while at the same time multi-clause sentences in language B will often have to be broken into two or more sentences when translating into language A.</abstract>
|
||||||
|
<bibkey>yngve-1963-order</bibkey>
|
||||||
|
</paper>
|
||||||
|
</volume>
|
||||||
|
</collection>
|
||||||
Vendored
+273
@@ -0,0 +1,273 @@
|
|||||||
|
<?xml version='1.0' encoding='UTF-8'?>
|
||||||
|
<collection id="T78">
|
||||||
|
<volume id="1" type="proceedings">
|
||||||
|
<!-- https://aclanthology.org/T78-1/ -->
|
||||||
|
<meta>
|
||||||
|
<booktitle>Theoretical Issues in Natural Language <fixed-case>P</fixed-case>rocessing-2</booktitle>
|
||||||
|
<editor id="david-l-waltz"><first>David L.</first><last>Waltz</last></editor>
|
||||||
|
<venue>tinlap</venue>
|
||||||
|
<year>1978</year>
|
||||||
|
</meta>
|
||||||
|
<frontmatter>
|
||||||
|
<!-- https://aclanthology.org/T78-1000/ -->
|
||||||
|
<pdf hash="fb6164ad"/>
|
||||||
|
<bibkey>tinlap-1978-theoretical</bibkey>
|
||||||
|
</frontmatter>
|
||||||
|
<paper id="1">
|
||||||
|
<!-- https://aclanthology.org/T78-1001/ -->
|
||||||
|
<title>Testing The Psychological Reality of a Representational Model</title>
|
||||||
|
<author><first>Dedre</first><last>Gentner</last></author>
|
||||||
|
<pdf hash="f723a956"/>
|
||||||
|
<bibkey>gentner-1978-testing-psychological</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="2">
|
||||||
|
<!-- https://aclanthology.org/T78-1002/ -->
|
||||||
|
<title>What Makes Something “Ad Hoc”</title>
|
||||||
|
<author id="roger-c-schank"><first>Roger C.</first><last>Schank</last></author>
|
||||||
|
<pdf hash="636853f9"/>
|
||||||
|
<bibkey>schank-1978-makes-something</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="3">
|
||||||
|
<!-- https://aclanthology.org/T78-1003/ -->
|
||||||
|
<title>The Relation of Grammar to Cognition–a Synopsis</title>
|
||||||
|
<author><first>Leonard</first><last>Talmy</last></author>
|
||||||
|
<pdf hash="101c67dc"/>
|
||||||
|
<bibkey>talmy-1978-relation-grammar</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="4">
|
||||||
|
<!-- https://aclanthology.org/T78-1004/ -->
|
||||||
|
<title>On primitives, prototypes, and other semantic anomalies</title>
|
||||||
|
<author><first>Terry</first><last>Winograd</last></author>
|
||||||
|
<pdf hash="f8e47023"/>
|
||||||
|
<bibkey>winograd-1978-primitives-prototypes</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="5">
|
||||||
|
<!-- https://aclanthology.org/T78-1005/ -->
|
||||||
|
<title>Taxonomic Lattice Structures for Situation Recognition</title>
|
||||||
|
<author id="william-a-woods"><first>William A.</first><last>Woods</last></author>
|
||||||
|
<pdf hash="92957ddf"/>
|
||||||
|
<bibkey>woods-1978-taxonomic-lattice</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="6">
|
||||||
|
<!-- https://aclanthology.org/T78-1006/ -->
|
||||||
|
<title>Description Formation and Discourse Model Synthesis</title>
|
||||||
|
<author id="bonnie-webber"><first>Bonnie Lynn</first><last>Webber</last></author>
|
||||||
|
<pdf hash="ffded1d2"/>
|
||||||
|
<bibkey>webber-1978-description-formation</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="7">
|
||||||
|
<!-- https://aclanthology.org/T78-1007/ -->
|
||||||
|
<title>The Processing of Referring Expressions within a Semantic Network</title>
|
||||||
|
<author><first>John R.</first><last>Anderson</last></author>
|
||||||
|
<pdf hash="8bcc083a"/>
|
||||||
|
<bibkey>anderson-1978-processing-referring</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="8">
|
||||||
|
<!-- https://aclanthology.org/T78-1008/ -->
|
||||||
|
<title>Reference Diaries</title>
|
||||||
|
<author><first>Herbert H.</first><last>Clark</last></author>
|
||||||
|
<author><first>Catherine</first><last>Marshall</last></author>
|
||||||
|
<pdf hash="d06b7ea2"/>
|
||||||
|
<bibkey>clark-marshall-1978-reference-diaries</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="9">
|
||||||
|
<!-- https://aclanthology.org/T78-1009/ -->
|
||||||
|
<title>Subsequent Reference: Syntactic and Rhetorical Constraints</title>
|
||||||
|
<author id="david-d-mcdonald"><first>David D.</first><last>McDonald</last></author>
|
||||||
|
<pdf hash="2d424559"/>
|
||||||
|
<bibkey>mcdonald-1978-subsequent-reference</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="10">
|
||||||
|
<!-- https://aclanthology.org/T78-1010/ -->
|
||||||
|
<title>Some Psycholinguistic Constraints on the Construction and Interpretation of Definite Descriptions</title>
|
||||||
|
<author><first>Andrew</first><last>Ortony</last></author>
|
||||||
|
<pdf hash="69807727"/>
|
||||||
|
<bibkey>ortony-1978-psycholinguistic-constraints</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="11">
|
||||||
|
<!-- https://aclanthology.org/T78-1011/ -->
|
||||||
|
<title>Bound Variables and Other Anaphors</title>
|
||||||
|
<author id="barbara-h-partee"><first>Barbara H.</first><last>Partee</last></author>
|
||||||
|
<pdf hash="e8e6a510"/>
|
||||||
|
<bibkey>partee-1978-bound-variables</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="12">
|
||||||
|
<!-- https://aclanthology.org/T78-1012/ -->
|
||||||
|
<title>The Use of Focus as a Tool for Disambiguation of Definite Noun Phrases</title>
|
||||||
|
<author id="candace-l-sidner"><first>Candace L.</first><last>Sidner</last></author>
|
||||||
|
<pdf hash="7b96d2b7"/>
|
||||||
|
<bibkey>sidner-1978-use-focus</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="13">
|
||||||
|
<!-- https://aclanthology.org/T78-1013/ -->
|
||||||
|
<title>Focusing in Dialog</title>
|
||||||
|
<author id="barbara-j-grosz"><first>Barbara J.</first><last>Grosz</last></author>
|
||||||
|
<pdf hash="8c8d4c2e"/>
|
||||||
|
<bibkey>grosz-1978-focusing-dialog</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="14">
|
||||||
|
<!-- https://aclanthology.org/T78-1014/ -->
|
||||||
|
<title>Topic Levels</title>
|
||||||
|
<author><first>Joseph E.</first><last>Grimes</last></author>
|
||||||
|
<pdf hash="f485b2ba"/>
|
||||||
|
<bibkey>grimes-1978-topic-levels</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="15">
|
||||||
|
<!-- https://aclanthology.org/T78-1015/ -->
|
||||||
|
<title>Toward a Rational Model of Discourse Comprehension</title>
|
||||||
|
<author><first>Jerry L.</first><last>Morgan</last></author>
|
||||||
|
<pdf hash="5e12a597"/>
|
||||||
|
<bibkey>morgan-1978-toward-rational</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="16">
|
||||||
|
<!-- https://aclanthology.org/T78-1016/ -->
|
||||||
|
<title>Assent and Compliance in Children’s Language Comprehension</title>
|
||||||
|
<author><first>David R.</first><last>Olson</last></author>
|
||||||
|
<pdf hash="708bc4a0"/>
|
||||||
|
<bibkey>olson-1978-assent-compliance</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="17">
|
||||||
|
<!-- https://aclanthology.org/T78-1017/ -->
|
||||||
|
<title>Speech Acts as a Basis for Understanding Dialogue Coherence</title>
|
||||||
|
<author id="c-raymond-perrault"><first>C. Raymond</first><last>Perrault</last></author>
|
||||||
|
<author id="james-allen"><first>James F.</first><last>Allen</last></author>
|
||||||
|
<pdf hash="d111402b"/>
|
||||||
|
<bibkey>perrault-allen-1978-speech</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="18">
|
||||||
|
<!-- https://aclanthology.org/T78-1018/ -->
|
||||||
|
<title>A Framework for Comparing Language Experiences (with particular emphasis on: The Effect of Audience on Discourse Models)</title>
|
||||||
|
<author><first>Andee</first><last>Rubin</last></author>
|
||||||
|
<pdf hash="5d9a6150"/>
|
||||||
|
<bibkey>rubin-1978-framework-comparing</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="19">
|
||||||
|
<!-- https://aclanthology.org/T78-1019/ -->
|
||||||
|
<title>Intentlonallty and Human Conversations</title>
|
||||||
|
<author id="jaime-g-carbonell"><first>Jaime G.</first><last>Carbonell Jr</last></author>
|
||||||
|
<pdf hash="c50fc619"/>
|
||||||
|
<bibkey>carbonell-jr-1978-intentlonallty-human</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="20">
|
||||||
|
<!-- https://aclanthology.org/T78-1020/ -->
|
||||||
|
<title>On the Interdependence of Language and Perception</title>
|
||||||
|
<author id="david-l-waltz"><first>David L.</first><last>Waltz</last></author>
|
||||||
|
<pdf hash="3c708191"/>
|
||||||
|
<bibkey>waltz-1978-interdependence-language</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="21">
|
||||||
|
<!-- https://aclanthology.org/T78-1021/ -->
|
||||||
|
<title>The Problem of Naming Shapes: Vision-Language Interface</title>
|
||||||
|
<author id="ruzena-bajcsy"><first>R.</first><last>Bajcsy</last></author>
|
||||||
|
<author id="aravind-joshi"><first>A. K.</first><last>Joshi</last></author>
|
||||||
|
<pdf hash="50331b4e"/>
|
||||||
|
<bibkey>bajcsy-joshi-1978-problem-naming</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="22">
|
||||||
|
<!-- https://aclanthology.org/T78-1022/ -->
|
||||||
|
<title>An Argument about the Composition of Conceptual Structure</title>
|
||||||
|
<author><first>Ray</first><last>Jackendoff</last></author>
|
||||||
|
<pdf hash="57d001f8"/>
|
||||||
|
<bibkey>jackendoff-1978-argument-composition</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="23">
|
||||||
|
<!-- https://aclanthology.org/T78-1023/ -->
|
||||||
|
<title>On the Ontological Status of Visual Mental Images</title>
|
||||||
|
<author><first>Stephen Michael</first><last>Kosslyn</last></author>
|
||||||
|
<pdf hash="fd06e265"/>
|
||||||
|
<bibkey>kosslyn-1978-ontological-status</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="24">
|
||||||
|
<!-- https://aclanthology.org/T78-1024/ -->
|
||||||
|
<title>What has language to do with perception? Some speculations on the <i>Lingua Mentis</i></title>
|
||||||
|
<author><first>Zenon W.</first><last>Pylyshyn</last></author>
|
||||||
|
<pdf hash="a2f3a044"/>
|
||||||
|
<bibkey>pylyshyn-1978-language-perception</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="25">
|
||||||
|
<!-- https://aclanthology.org/T78-1025/ -->
|
||||||
|
<title>Semantic Primitives in Language and Vision</title>
|
||||||
|
<author id="yorick-wilks"><first>Yorick</first><last>Wilks</last></author>
|
||||||
|
<pdf hash="b4e13896"/>
|
||||||
|
<bibkey>wilks-1978-semantic-primitives</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="26">
|
||||||
|
<!-- https://aclanthology.org/T78-1026/ -->
|
||||||
|
<title>A Note on Partial Match of Descriptions. Can One Simultaneously Question (Retrieve) and Inform (Update)?</title>
|
||||||
|
<author id="aravind-joshi"><first>Aravind K.</first><last>Joshi</last></author>
|
||||||
|
<pdf hash="74969243"/>
|
||||||
|
<bibkey>joshi-1978-note-partial</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="27">
|
||||||
|
<!-- https://aclanthology.org/T78-1027/ -->
|
||||||
|
<title>With a Spoon in Hand This Must Be the Eating Frame</title>
|
||||||
|
<author><first>Eugene</first><last>Charniak</last></author>
|
||||||
|
<pdf hash="f1be68bb"/>
|
||||||
|
<bibkey>charniak-1978-spoon-hand</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="28">
|
||||||
|
<!-- https://aclanthology.org/T78-1028/ -->
|
||||||
|
<title>Fragments of a Theory of Human Plausible Reasoning</title>
|
||||||
|
<author><first>Allan</first><last>Collins</last></author>
|
||||||
|
<pdf hash="67b4bca2"/>
|
||||||
|
<bibkey>collins-1978-fragments-theory</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="29">
|
||||||
|
<!-- https://aclanthology.org/T78-1029/ -->
|
||||||
|
<title>Indirect Responses to Loaded Questions</title>
|
||||||
|
<author><first>S. Jerrold</first><last>Kaplan</last></author>
|
||||||
|
<pdf hash="135037f9"/>
|
||||||
|
<bibkey>kaplan-1978-indirect-responses</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="30">
|
||||||
|
<!-- https://aclanthology.org/T78-1030/ -->
|
||||||
|
<title>On Reasoning by Default</title>
|
||||||
|
<author><first>Raymond</first><last>Reiter</last></author>
|
||||||
|
<pdf hash="0de53d33"/>
|
||||||
|
<bibkey>reiter-1978-reasoning-default</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="31">
|
||||||
|
<!-- https://aclanthology.org/T78-1031/ -->
|
||||||
|
<title>Path-Based and Node-Based Inference in Semantic Networks</title>
|
||||||
|
<author id="stuart-c-shapiro"><first>Stuart C.</first><last>Shapiro</last></author>
|
||||||
|
<pdf hash="2ddc5535"/>
|
||||||
|
<bibkey>shapiro-1978-path-based</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="32">
|
||||||
|
<!-- https://aclanthology.org/T78-1032/ -->
|
||||||
|
<title>The Representation of Derivable Information in Memory: When What Might Have Been Left Unsaid Is Said</title>
|
||||||
|
<author><first>Rand J.</first><last>Spiro</last></author>
|
||||||
|
<author><first>Joseph</first><last>Esposito</last></author>
|
||||||
|
<author><first>Richard J.</first><last>Vondruska</last></author>
|
||||||
|
<pdf hash="7758fb45"/>
|
||||||
|
<bibkey>spiro-etal-1978-representation-derivable</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="33">
|
||||||
|
<!-- https://aclanthology.org/T78-1033/ -->
|
||||||
|
<title>A Heuristic for Paradigms</title>
|
||||||
|
<author><first>Joseph E.</first><last>Grimes</last></author>
|
||||||
|
<pdf hash="53b42c16"/>
|
||||||
|
<bibkey>grimes-1978-heuristic-paradigms</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="34">
|
||||||
|
<!-- https://aclanthology.org/T78-1034/ -->
|
||||||
|
<title>A Computational Account of Some Constraints on Language</title>
|
||||||
|
<author id="mitch-marcus"><first>Mitchell</first><last>Marcus</last></author>
|
||||||
|
<pdf hash="c9724c43"/>
|
||||||
|
<bibkey>marcus-1978-computational-account</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="35">
|
||||||
|
<!-- https://aclanthology.org/T78-1035/ -->
|
||||||
|
<title>Remarks on Processing, Constraints, and the Lexicon</title>
|
||||||
|
<author id="thomas-wasow"><first>Thomas</first><last>Wasow</last></author>
|
||||||
|
<pdf hash="8beed275"/>
|
||||||
|
<bibkey>wasow-1978-remarks-processing</bibkey>
|
||||||
|
</paper>
|
||||||
|
<paper id="36">
|
||||||
|
<!-- https://aclanthology.org/T78-1036/ -->
|
||||||
|
<title>List of questions suggested for consideration in each session</title>
|
||||||
|
<pdf hash="1950b908"/>
|
||||||
|
<bibkey>nn-1978-list-questions</bibkey>
|
||||||
|
</paper>
|
||||||
|
</volume>
|
||||||
|
</collection>
|
||||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
Vendored
BIN
Binary file not shown.
+706
@@ -0,0 +1,706 @@
|
|||||||
|
{
|
||||||
|
"status": "reviewed26_presentation_only_title_correspondences_not_work_identity_approval",
|
||||||
|
"gold": [
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3834907",
|
||||||
|
"title": "H\n <sup>2</sup>\n AL: Hyperbolic Hierarchy-aware Aggregative Learning for Registration-based Few-shot Medical Image Segmentation",
|
||||||
|
"authors": [
|
||||||
|
"Jia Wang",
|
||||||
|
"Jiaming Cai",
|
||||||
|
"Zunying Hu",
|
||||||
|
"Zhanjie Wu",
|
||||||
|
"Jinyuan Liu",
|
||||||
|
"Hua Cheng",
|
||||||
|
"Yun Peng",
|
||||||
|
"Xin Fan"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-004.json",
|
||||||
|
"source_position": 246,
|
||||||
|
"registry_object_sha256": "dd4a627c13ca806e15e59d3ae894c9216d14d33c85bc5ce7362b51fad8c8a692",
|
||||||
|
"program_id": "mfp0053",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "516fac88f1b7a2821c1e2b97cc41530f32abbe2514b65b5823427ff3ecdc9a23",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/129/papers/19",
|
||||||
|
"/entries/129/subsessions/1/items/7"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3834936",
|
||||||
|
"title": "M\n <sup>3</sup>\n TR: Temporal Retrieval Enhanced Multi-Modal Micro-video Popularity Prediction",
|
||||||
|
"authors": [
|
||||||
|
"Jiacheng Lu",
|
||||||
|
"Weijian Wang",
|
||||||
|
"Mingyuan Xiao",
|
||||||
|
"Yang Hua",
|
||||||
|
"Tao Song",
|
||||||
|
"Bo Peng",
|
||||||
|
"Cheng Hua",
|
||||||
|
"Haibing Guan"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-005.json",
|
||||||
|
"source_position": 25,
|
||||||
|
"registry_object_sha256": "93340f8b923d7cc0d848941107351fb661d95effa35b1869921e7163da36bae4",
|
||||||
|
"program_id": "mfp0229",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "6968e6a2d364075020e0b6c95c9076742531681232c40df511a2e8992c6b25f9",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/109/papers/27",
|
||||||
|
"/entries/109/subsessions/2/items/3"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3834946",
|
||||||
|
"title": "<i>PAST:</i>\n Prompt-Adaptive Sampling Termination for Efficient Diffusion Model",
|
||||||
|
"authors": [
|
||||||
|
"Renye Yan",
|
||||||
|
"Jikang Cheng",
|
||||||
|
"You Wu",
|
||||||
|
"Wei Peng",
|
||||||
|
"Zongwei Wang",
|
||||||
|
"Ling Liang",
|
||||||
|
"Yimao Cai"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-005.json",
|
||||||
|
"source_position": 35,
|
||||||
|
"registry_object_sha256": "4210191c2871090711820aef9aff06289694380c9d0194ec9b8b1e5e40fad00d",
|
||||||
|
"program_id": "mfp0297",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "b47bd7502bf3a8e9e9dc87e3d9552efe928cc455f843d527a4860c430c5e9490",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/61/papers/40",
|
||||||
|
"/entries/61/subsessions/3/items/4"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3834956",
|
||||||
|
"title": "<i>SwiftExplorer:</i>\n Training-free diffusion model alignment with Swift Diversity Exploration",
|
||||||
|
"authors": [
|
||||||
|
"Renye Yan",
|
||||||
|
"Jikang Cheng",
|
||||||
|
"You Wu",
|
||||||
|
"Baojin Huang",
|
||||||
|
"Wei Peng",
|
||||||
|
"Zongwei Wang",
|
||||||
|
"Ling Liang",
|
||||||
|
"Yimao Cai"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-006.json",
|
||||||
|
"source_position": 235,
|
||||||
|
"registry_object_sha256": "7f2812ca4072437ecaaa585e85139e7cba0e0d721f9a2c0ac4e9897739faed47",
|
||||||
|
"program_id": "mfp0339",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "f7809f57290e44fe291ffdc07ea956e5484aa491390cd385db75d056f95b840a",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/88/papers/48",
|
||||||
|
"/entries/88/subsessions/4/items/0"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3835035",
|
||||||
|
"title": "<i>Balancing Efficiency and Efficacy:</i>\n Training-Free Attention-Guided Switching Between Explicit and Latent Thoughts for MLLMs",
|
||||||
|
"authors": [
|
||||||
|
"Haoqian Kang",
|
||||||
|
"Liupeng Li",
|
||||||
|
"Kuofeng Gao",
|
||||||
|
"Jinpeng Wang",
|
||||||
|
"Zhenyu Lu",
|
||||||
|
"Bin Chen",
|
||||||
|
"Ke Chen",
|
||||||
|
"Yaowei Wang"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-004.json",
|
||||||
|
"source_position": 8,
|
||||||
|
"registry_object_sha256": "3d76f692754dc159642eeb1706491631267d437a95c4459674361ebf35f416b9",
|
||||||
|
"program_id": "mfp0798",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "477e42fa3d5c8236d6415ff18ba26d799d659b9e2f3f09c1d80bc0971dbca125",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/88/papers/39",
|
||||||
|
"/entries/88/subsessions/3/items/3"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3835055",
|
||||||
|
"title": "TM\n <sup>2</sup>\n Q-Det: Boosting Video Object Detection via Text-Guided Spatiotemporal Mamba and Multimodal Query Learning",
|
||||||
|
"authors": [
|
||||||
|
"Qiang Qi",
|
||||||
|
"Wenqi Shang",
|
||||||
|
"Xiao Wang",
|
||||||
|
"Shuyuan Lin"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-004.json",
|
||||||
|
"source_position": 28,
|
||||||
|
"registry_object_sha256": "7e3c3e6acdad588970c681bdb92969f5134086baa2a5a7b9414871135d5ea137",
|
||||||
|
"program_id": "mfp0903",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "8549d38e781896079ab04ef6a771a4061a0fe42df2fa5c5c03a037d59cc9f548",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/129/papers/55",
|
||||||
|
"/entries/129/subsessions/4/items/7"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3835101",
|
||||||
|
"title": "<scp>ConfAvatar:</scp>\n Confidence-Aware Self-Bootstrapping for Monocular 4D Head Avatar Reconstruction",
|
||||||
|
"authors": [
|
||||||
|
"Zifan Zheng",
|
||||||
|
"Yu Luo",
|
||||||
|
"Yuxin Huang",
|
||||||
|
"Chuancheng Shi",
|
||||||
|
"Mohammad Masbaul Alam Polash",
|
||||||
|
"Zhiyong Wang",
|
||||||
|
"Kun Hu"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-008.json",
|
||||||
|
"source_position": 82,
|
||||||
|
"registry_object_sha256": "b838756bd0b6af036e408936f8f13a6398d83e524ce0282167199eca092bc7d8",
|
||||||
|
"program_id": "mfp1175",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "cb8dbe67e16e163e5b136ad3401fc57c827905e9704d7c0a84961bc8f12565f4",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/109/papers/86",
|
||||||
|
"/entries/109/subsessions/7/items/2"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3835374",
|
||||||
|
"title": "<scp>DateGBC:</scp>\n Deep Fuzzy Granular-Ball Clustering via Adaptive Topology Evolution for Unsupervised Image Segmentation",
|
||||||
|
"authors": [
|
||||||
|
"Qianpeng Chong",
|
||||||
|
"Jiakun Wen",
|
||||||
|
"Guangyi Wei",
|
||||||
|
"Wenyi Zeng"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-006.json",
|
||||||
|
"source_position": 59,
|
||||||
|
"registry_object_sha256": "a7fe694a04449e841be5181888ae35786d0a4938d87eca1fffb0776d4d350e30",
|
||||||
|
"program_id": "mfp2617",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "b57b7026c750aeebb47598860b82980215c9e4e4935f6e8611a31b2fe64f73b9",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/61/papers/80",
|
||||||
|
"/entries/61/subsessions/6/items/8"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3835424",
|
||||||
|
"title": "<i>RTCBench</i>\n : A Scalable Benchmarking Framework for Multi-Party Video Conferencing",
|
||||||
|
"authors": [
|
||||||
|
"Evan Ram",
|
||||||
|
"Tuan Tran",
|
||||||
|
"Youngwook Son",
|
||||||
|
"Sangtae Ha"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-005.json",
|
||||||
|
"source_position": 108,
|
||||||
|
"registry_object_sha256": "b21198e0eba35d175bc83292be8c87bca2b80a9c8e01f40b640a4e9efc7ecde5",
|
||||||
|
"program_id": "mfp2909",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "2398ec6539c50b14c7ed4fa773277cc860151ab77b5430b49d935f88e90a487e",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/129/papers/130",
|
||||||
|
"/entries/129/subsessions/8/items/22"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3835452",
|
||||||
|
"title": "<scp>HeadRouter:</scp>\n Dynamic Head-Weight Routing for Task-Adaptive Audio Token Pruning in Large Audio Language Models",
|
||||||
|
"authors": [
|
||||||
|
"He Peize",
|
||||||
|
"Yaodi Luo",
|
||||||
|
"Xiaoqian Liu",
|
||||||
|
"Xuyang Liu",
|
||||||
|
"Jiahang Deng",
|
||||||
|
"Yaosong Du",
|
||||||
|
"Li Bangyu",
|
||||||
|
"Xiyan Gui",
|
||||||
|
"Yuxuan Chen",
|
||||||
|
"Linfeng Zhang"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-004.json",
|
||||||
|
"source_position": 167,
|
||||||
|
"registry_object_sha256": "ee31b12972ea5844a50c6c63a6fc3430875c3388a4bf261f8e76cab31112d351",
|
||||||
|
"program_id": "mfp3048",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "c455de7c27f43d0fb49990ead5fca795334c0b613eafa2b3c6150eb147d9c3fa",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/46/papers/78",
|
||||||
|
"/entries/46/subsessions/6/items/6"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3835460",
|
||||||
|
"title": "<i>CoAnchor:</i>\n Robust Collaborative Perception under Spatio-Temporal Misalignment via Object-Level Anchors",
|
||||||
|
"authors": [
|
||||||
|
"Chi Li",
|
||||||
|
"Rui Lin",
|
||||||
|
"Aobo Ji",
|
||||||
|
"Dongzhu Xu"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-004.json",
|
||||||
|
"source_position": 175,
|
||||||
|
"registry_object_sha256": "7c41b0dcb65ad27a1f0961f095889ccc73044776250ec816cf95fc25c30da7f1",
|
||||||
|
"program_id": "mfp3109",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "5f3d97775d83528d6f41fb2fc481d1848cac7c3f9de1d469039450f9b2ecbff1",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/109/papers/121",
|
||||||
|
"/entries/109/subsessions/8/items/13"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3835544",
|
||||||
|
"title": "<i>OC-GS:</i>\n Object-Centric Representation and Demand Alignment for Immersive 3D Gaussian Streaming",
|
||||||
|
"authors": [
|
||||||
|
"Boyan Li",
|
||||||
|
"Ruijun Mao",
|
||||||
|
"Zhicheng Liang",
|
||||||
|
"Dayou Zhang",
|
||||||
|
"Fangxin Wang"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-006.json",
|
||||||
|
"source_position": 129,
|
||||||
|
"registry_object_sha256": "afc0c7db304251511eef5e223f027dc6314dd70895955ff1a1cb0a461dbfdbe9",
|
||||||
|
"program_id": "mfp3554",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "00b2ffe1073feede8447bac439ac7842c1ba0841cb6ee38ec1b5e06c3a3dc22e",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/46/papers/153",
|
||||||
|
"/entries/46/subsessions/9/items/21"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3835591",
|
||||||
|
"title": "DP\n <sup>2</sup>\n -NeRF: Dual-Prior Probabilistic Neural Radiance Fields for Event-Guided Motion Deblurring",
|
||||||
|
"authors": [
|
||||||
|
"Jiafeng Huang",
|
||||||
|
"Wenzhen Jia",
|
||||||
|
"Shilong Dong",
|
||||||
|
"Shengjie Zhao"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-007.json",
|
||||||
|
"source_position": 97,
|
||||||
|
"registry_object_sha256": "145025e784ec251247f8cd86d0dbee1749666ec71f35566c20e41edd0a2347ce",
|
||||||
|
"program_id": "mfp3783",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "0b2d267736cfe506de666618820e2934c23395bd25dab63c5a347710445a361c",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/33/papers/158",
|
||||||
|
"/entries/33/subsessions/10/items/2"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3835667",
|
||||||
|
"title": "MMCoT\n <sup>3</sup>\n : Empowering 3D Scene Understanding with 3D-Text Interleaved Multi-Modal Chain-of-Thought Reasoning",
|
||||||
|
"authors": [
|
||||||
|
"Yongqi Shan",
|
||||||
|
"Yifan Wang",
|
||||||
|
"Yunzhi Zhuge",
|
||||||
|
"Lijun Wang",
|
||||||
|
"Huchuan Lu"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-005.json",
|
||||||
|
"source_position": 209,
|
||||||
|
"registry_object_sha256": "8b5ad8d891692e441b6e210304dcd35103835fd35d086c860c7611bf5385a6d9",
|
||||||
|
"program_id": "mfp4173",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "4924721a2c3328e6bca661654569bca86d3fa69aa5f6c9e3719f827c5f0d7639",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/88/papers/114",
|
||||||
|
"/entries/88/subsessions/8/items/6"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3835763",
|
||||||
|
"title": "<scp>AudioKV:</scp>\n KV Cache Eviction in Efficient Large Audio Language Models",
|
||||||
|
"authors": [
|
||||||
|
"Yuxuan Wang",
|
||||||
|
"He Peize",
|
||||||
|
"Xiyan Gui",
|
||||||
|
"Xiaoqian Liu",
|
||||||
|
"Junhao He",
|
||||||
|
"Xuyang Liu",
|
||||||
|
"Xuming Hu",
|
||||||
|
"Linfeng Zhang"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-006.json",
|
||||||
|
"source_position": 192,
|
||||||
|
"registry_object_sha256": "70054b0a5a2586fd08e75494eee08a26498730554601a5fe2e4efddcaad309de",
|
||||||
|
"program_id": "mfp4778",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "274b360da553dfc7afdb9072505c2957cec4213e096373e2edbf58236a3fc3a2",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/129/papers/100",
|
||||||
|
"/entries/129/subsessions/7/items/16"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3835895",
|
||||||
|
"title": "F\n <sup>2</sup>\n -Adapter: A Focus-and-Forget Framework for Generalizable CLIP-based Face Forgery Detection",
|
||||||
|
"authors": [
|
||||||
|
"Zan Gao",
|
||||||
|
"Xin Duan",
|
||||||
|
"Nian Hu",
|
||||||
|
"Hao Jia",
|
||||||
|
"Yibo Zhao",
|
||||||
|
"Cong Liu"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-001.json",
|
||||||
|
"source_position": 39,
|
||||||
|
"registry_object_sha256": "16bdbcc4398df4d9556c14d4d4203127349070669f5d0e1c4bf87756e241b698",
|
||||||
|
"program_id": "mfp5537",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "b3ef626b66d93ac37a73caf15a4cf9ff92241209c80ee66be5575e6b56f9c899",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/88/papers/184",
|
||||||
|
"/entries/88/subsessions/11/items/6"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3835959",
|
||||||
|
"title": "UPR-VAD:\n <i>U</i>\n ncertainty-guided Signal\n <i>P</i>\n urification and\n <i>R</i>\n egularization for Weakly-Supervised\n <i>V</i>\n ideo\n <i>A</i>\n nomaly\n <i>D</i>\n etection",
|
||||||
|
"authors": [
|
||||||
|
"Weiliang Huang",
|
||||||
|
"Weichao Cai",
|
||||||
|
"Bob Zhang",
|
||||||
|
"Chao Huang",
|
||||||
|
"Biao Xue",
|
||||||
|
"Rongxin Zhang"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-007.json",
|
||||||
|
"source_position": 21,
|
||||||
|
"registry_object_sha256": "e96f9e9d67cce31a9235bdc7dc5dba551676f3d36ba72654ae3d5a98ae8d5174",
|
||||||
|
"program_id": "mfp5823",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "d8469bcb68f8cc19070f3bf9a866c3377ba27fe6445b5bf9337b7599a6881592",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/88/papers/66",
|
||||||
|
"/entries/88/subsessions/5/items/6"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3836215",
|
||||||
|
"title": "BLM\n <sub>1</sub>\n : A Boundless Large Model for Cross-Space, Cross-Task, and Cross-Embodiment Learning",
|
||||||
|
"authors": [
|
||||||
|
"Wentao Tan",
|
||||||
|
"Bowen Wang",
|
||||||
|
"Heng Zhi",
|
||||||
|
"Chenyv Liu",
|
||||||
|
"Zhe Li",
|
||||||
|
"Jian Liu",
|
||||||
|
"Zengrong Lin",
|
||||||
|
"Yukun Dai",
|
||||||
|
"Yipeng Chen",
|
||||||
|
"Wenjie Yang",
|
||||||
|
"Enci Xie",
|
||||||
|
"Hao Xue",
|
||||||
|
"Baixu Ji",
|
||||||
|
"Chen Xu",
|
||||||
|
"Zhibin Wang",
|
||||||
|
"Tianshi Wang",
|
||||||
|
"Lei Zhu",
|
||||||
|
"Heng Tao Shen"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-003.json",
|
||||||
|
"source_position": 46,
|
||||||
|
"registry_object_sha256": "b4b51e5d0a4c0a330d01e7638c0bcc73871d750312081bafbd3b8a10529153f2",
|
||||||
|
"program_id": "mfp7332",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "aa458905209ad562117d72dd33d4d61e000b0d9efaa55dd9672919f40e6cc2fd",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/33/papers/13",
|
||||||
|
"/entries/33/subsessions/1/items/1",
|
||||||
|
"/entries/47/papers/0"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3836272",
|
||||||
|
"title": "SEE: Structure-aware Exploring & Exploiting for Long-horizon GUI Agent Trajectory Synthesis",
|
||||||
|
"authors": [
|
||||||
|
"Zhuohang Fan",
|
||||||
|
"Beichen Zhang",
|
||||||
|
"Yuanfa Li",
|
||||||
|
"Changqiao Wu",
|
||||||
|
"Wei Liu",
|
||||||
|
"Jian Luan",
|
||||||
|
"Weigang Zhang"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-002.json",
|
||||||
|
"source_position": 127,
|
||||||
|
"registry_object_sha256": "66eebbdef9e2d07fe08f3cb41cd577095f282f1256bdb73fe1e0d4c5d6e55d51",
|
||||||
|
"program_id": "mfp7635",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "df11d727337f397bab72977bcdce4246b388ef0c8e1198ac47fdc32e68a3aff4",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/88/papers/131",
|
||||||
|
"/entries/88/subsessions/8/items/23"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3836293",
|
||||||
|
"title": "Video\n <sup>*</sup>\n Bench: Benchmarking Fine-grained Video Perception with \"Think with Videos\"",
|
||||||
|
"authors": [
|
||||||
|
"Ruizhe Chen",
|
||||||
|
"Zhiting Fan",
|
||||||
|
"Yang Shi",
|
||||||
|
"Enxin Song",
|
||||||
|
"Wenhao Chai",
|
||||||
|
"Tongkun Guan",
|
||||||
|
"Songtao Jiang",
|
||||||
|
"Ruilin Luo",
|
||||||
|
"Yuanxing Zhang",
|
||||||
|
"Shuai Bai",
|
||||||
|
"Sibo Song",
|
||||||
|
"Zhibo Yang",
|
||||||
|
"Zuozhu Liu"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-002.json",
|
||||||
|
"source_position": 148,
|
||||||
|
"registry_object_sha256": "7332426edb3d15499f175fb98c093b78df21ea95fdd9cfea33da5e925cbec171",
|
||||||
|
"program_id": "mfp7732",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "f6e4df110fe85d7f6d34eb712cd338443c2e34e924393a4e65e07a0c7b152c9e",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/46/papers/198",
|
||||||
|
"/entries/46/subsessions/11/items/20"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3836447",
|
||||||
|
"title": "<i>Insert In Style</i>\n : A Zero-Shot Generative Framework for Harmonious Cross-Domain Object Composition",
|
||||||
|
"authors": [
|
||||||
|
"Raghu Vamsi Chittersu",
|
||||||
|
"Yuvraj Singh Rathore",
|
||||||
|
"Pranav Adlinge",
|
||||||
|
"Eswar Sai Pashaula",
|
||||||
|
"Kunal Swami"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-002.json",
|
||||||
|
"source_position": 183,
|
||||||
|
"registry_object_sha256": "92b2c094b96cebbace0150805b65b0a1bc3ca99ed42124bcb8de442a81094e1b",
|
||||||
|
"program_id": "mfp8648",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "39d3f21f774010980f8b30f2fd800b5243ea7ddc6e091b4d8769de3dee622273",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/107/papers/4",
|
||||||
|
"/entries/117/papers/30",
|
||||||
|
"/entries/117/subsessions/2/items/6"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3836600",
|
||||||
|
"title": "<scp>EEGaze</scp>\n : Multimodal Attention Prediction in Visual Search via EEG-Based Saliency Refinement",
|
||||||
|
"authors": [
|
||||||
|
"Mansi Sharma",
|
||||||
|
"Dinar Orazgaliyev",
|
||||||
|
"Antonio Krüger",
|
||||||
|
"Philipp Müller"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-003.json",
|
||||||
|
"source_position": 140,
|
||||||
|
"registry_object_sha256": "30b99b234aa2b944d6890119b2524d3b3c3c425bde22f20e813b9517b8f8536b",
|
||||||
|
"program_id": "mfp9815",
|
||||||
|
"track_literal": "Main Track",
|
||||||
|
"track": "main",
|
||||||
|
"abstract_sha256": "47dcb456aee600663d0df59429c1745b335dbae9bdef2dd1063265429fff49fb",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/46/papers/219",
|
||||||
|
"/entries/46/subsessions/12/items/17"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3838656",
|
||||||
|
"title": "<scp>MedMKG:</scp>\n Benchmarking Medical Knowledge Exploitation with Multimodal Knowledge Graph",
|
||||||
|
"authors": [
|
||||||
|
"Xiaochen Wang",
|
||||||
|
"Yuan Zhong",
|
||||||
|
"Lingwei Zhang",
|
||||||
|
"Lisong Dai",
|
||||||
|
"Ting Wang",
|
||||||
|
"Fenglong Ma"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-003.json",
|
||||||
|
"source_position": 225,
|
||||||
|
"registry_object_sha256": "b910b66541ca65127646d35d0cd5737a56ae2e0f311a79526005dec75fca739f",
|
||||||
|
"program_id": "mds177",
|
||||||
|
"track_literal": "Datasets",
|
||||||
|
"track": "datasets",
|
||||||
|
"abstract_sha256": "5bee3b5e9ff60690509d1f34953f3dcc185c50861e112d2c4a3baf5673f6b945",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/46/papers/23",
|
||||||
|
"/entries/46/subsessions/1/items/11"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3839012",
|
||||||
|
"title": "<i>EULALens</i>\n : SEE What YOU Agree!",
|
||||||
|
"authors": [
|
||||||
|
"Abhijeet Narang",
|
||||||
|
"Hagar Elbatanouny",
|
||||||
|
"Vanishri Shekhawat",
|
||||||
|
"Hasan Al-Nashash",
|
||||||
|
"Abhinav Dhall",
|
||||||
|
"Usman Tariq"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-007.json",
|
||||||
|
"source_position": 238,
|
||||||
|
"registry_object_sha256": "81dd29dafd6228f51ae2f32e2bb713272a88df78c78347b9170b54f83db00c84",
|
||||||
|
"program_id": "mde41",
|
||||||
|
"track_literal": "Demo and Video",
|
||||||
|
"track": "demo_and_video",
|
||||||
|
"abstract_sha256": "9acbb264b8019916aad0845aeae4eb5a9c94903ac5bca3cfb612c857306f6d55",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/94/papers/7"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3839024",
|
||||||
|
"title": "<scp>ThreatVerse:</scp>\n A Closed-Loop World Model Simulator for Autonomous Driving Safety Evaluation",
|
||||||
|
"authors": [
|
||||||
|
"Jiangfan Liu",
|
||||||
|
"Zexuan Cui",
|
||||||
|
"Tianyuan Zhang",
|
||||||
|
"Zonglei Jing",
|
||||||
|
"Yaoyuan Zhang",
|
||||||
|
"Jiakai Wang",
|
||||||
|
"Jinyang Guo",
|
||||||
|
"Yuqing Ma",
|
||||||
|
"Lu Sheng",
|
||||||
|
"Aishan Liu",
|
||||||
|
"Xianglong Liu"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-008.json",
|
||||||
|
"source_position": 0,
|
||||||
|
"registry_object_sha256": "d0571f4562b30e4e7fb2e3aa826068ac14a540d1ae70e8cf0b11fd73c67f26ee",
|
||||||
|
"program_id": "mde15",
|
||||||
|
"track_literal": "Demo and Video",
|
||||||
|
"track": "demo_and_video",
|
||||||
|
"abstract_sha256": "02fc9b31d2b7667ea13f00c74f923a375373042171aaa1ff1f2010ee9697211a",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/49/papers/4"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"doi": "10.1145/3767308.3839028",
|
||||||
|
"title": "<scp>FinAlogy:</scp>\n A Visual Analogy Retrieval System for Financial K-Line Analysis",
|
||||||
|
"authors": [
|
||||||
|
"Xiaoyan Yu",
|
||||||
|
"Ziya Zhao",
|
||||||
|
"Yunshan Ma",
|
||||||
|
"Yifan Wei",
|
||||||
|
"Dianbo Sui",
|
||||||
|
"Tat-Seng Chua"
|
||||||
|
],
|
||||||
|
"kind": "unique_program_format_title_fullauthor_agreement_not_identity_approval",
|
||||||
|
"source_page": "page-008.json",
|
||||||
|
"source_position": 4,
|
||||||
|
"registry_object_sha256": "b18c8e2821873a59318f6f09d1f453dd02195ccedad5c8e0ce1da60ac07466fb",
|
||||||
|
"program_id": "mde02",
|
||||||
|
"track_literal": "Demo and Video",
|
||||||
|
"track": "demo_and_video",
|
||||||
|
"abstract_sha256": "22f978dd02c05439663c5724f25bb997098fd85f5558d01d836711252d119931",
|
||||||
|
"program_context_pointers": [
|
||||||
|
"/entries/49/papers/0"
|
||||||
|
],
|
||||||
|
"single_name_positions": []
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"author_disagreement_hold": {
|
||||||
|
"doi": "10.1145/3767308.3836308",
|
||||||
|
"program_id": "mfp7801"
|
||||||
|
},
|
||||||
|
"scope": "Onlytitlecomparisonchanges;originalregistryandprogramfixturesremainunchanged;sourceABandtrackcomefromtheexplicitprogramID.",
|
||||||
|
"held_gold": {
|
||||||
|
"doi": "10.1145/3767308.3836308",
|
||||||
|
"title": "Dynamic Late Fusion Multiple Kernel\n <i>k</i>\n -means Clustering based on Adaptive Sample Selection and Alignment",
|
||||||
|
"authors": [
|
||||||
|
"Zhirong Tian",
|
||||||
|
"Yi Zhang",
|
||||||
|
"Weixuan Liang",
|
||||||
|
"Miaomiao Li",
|
||||||
|
"Xinwang Liu"
|
||||||
|
],
|
||||||
|
"kind": "program_format_author_version_requires_review",
|
||||||
|
"source_page": "page-001.json",
|
||||||
|
"source_position": 82,
|
||||||
|
"registry_object_sha256": "26a3c1715d276b5fbe988596e3115091086158f6ef12600a36e5206785fc9dfb",
|
||||||
|
"program_id": null,
|
||||||
|
"track_literal": null,
|
||||||
|
"track": "proceedings_unclassified",
|
||||||
|
"abstract_sha256": null,
|
||||||
|
"program_context_pointers": [],
|
||||||
|
"single_name_positions": []
|
||||||
|
}
|
||||||
|
}
|
||||||
BIN
Binary file not shown.
+15
@@ -0,0 +1,15 @@
|
|||||||
|
{
|
||||||
|
"receipts": [
|
||||||
|
{
|
||||||
|
"url": "https://neurips.cc/static/virtual/data/neurips-2026-orals-posters.json",
|
||||||
|
"final_url": "https://neurips.cc/static/virtual/data/neurips-2026-orals-posters.json",
|
||||||
|
"status": 200,
|
||||||
|
"content_type": "application/json",
|
||||||
|
"file": "program-json.response",
|
||||||
|
"bytes": 20447021,
|
||||||
|
"sha256": "879293a863057fa05370443b3b5ddee12a10eeb37c810abc6c1d53f71f545997",
|
||||||
|
"started_at": "2026-11-09T17:57:23.825509+00:00",
|
||||||
|
"finished_at": "2026-11-09T17:57:40.125580+00:00"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
@@ -0,0 +1,167 @@
|
|||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import sqlite3
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from paper_atlas.collectors import aamas, aamas2026
|
||||||
|
from paper_atlas.ingestion.records import ingest_file
|
||||||
|
|
||||||
|
|
||||||
|
FIXTURE = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/aamas2026-fixture.json.gz').read_bytes()))
|
||||||
|
EVIDENCE = {'url': aamas2026.URL, 'observed_at': FIXTURE['receipts']['receipts'][0]['finished_at']}
|
||||||
|
|
||||||
|
|
||||||
|
class Aamas2026Tests(unittest.TestCase):
|
||||||
|
def parse(self, html=None):
|
||||||
|
return aamas2026.parse_contents(FIXTURE['contents'] if html is None else html, EVIDENCE)
|
||||||
|
|
||||||
|
def snapshots(self, folder):
|
||||||
|
folder.mkdir()
|
||||||
|
(folder/'contents.html').write_text(FIXTURE['contents'])
|
||||||
|
(folder/'receipts.json').write_text(json.dumps(FIXTURE['receipts']))
|
||||||
|
return folder
|
||||||
|
|
||||||
|
def test_every_publisher_title_author_affiliation_and_page_preserved(self):
|
||||||
|
rows, excluded, held = self.parse()
|
||||||
|
self.assertEqual((len(rows), len(excluded), len(held)), (639, 3, 1))
|
||||||
|
expected = {r['publisher_page']: r for r in FIXTURE['expected_entries']}
|
||||||
|
self.assertEqual(len(expected), 643)
|
||||||
|
all_rows = rows+excluded+held
|
||||||
|
self.assertEqual({r['metadata']['publisher_page'] for r in all_rows}, set(expected))
|
||||||
|
for r in all_rows:
|
||||||
|
e = expected[r['metadata']['publisher_page']]
|
||||||
|
self.assertEqual(r['title'], e['title'])
|
||||||
|
self.assertEqual(r['authors'], e['authors'])
|
||||||
|
self.assertEqual(r['metadata']['author_affiliations'], e['affiliations'])
|
||||||
|
self.assertEqual(r['metadata']['section_heading'], e['section'])
|
||||||
|
for r in rows:
|
||||||
|
self.assertIsNone(r['abstract'])
|
||||||
|
self.assertEqual(r['identifiers'], {})
|
||||||
|
if r['external_id'] != '2026/p3178':
|
||||||
|
self.assertEqual(r['status'], 'accepted')
|
||||||
|
self.assertNotIn('decision_date', r)
|
||||||
|
|
||||||
|
def test_explicit_withdrawal_preserves_status_date_and_no_pdf(self):
|
||||||
|
rows, _, _ = self.parse()
|
||||||
|
row, = [r for r in rows if r['external_id'] == '2026/p3178']
|
||||||
|
self.assertEqual(row['status'], 'withdrawn')
|
||||||
|
self.assertEqual(row['metadata']['withdrawal_date'], '2026-06-08')
|
||||||
|
self.assertNotIn('decision_date', row)
|
||||||
|
self.assertEqual(row['raw_decision'], 'Withdrawn June 8, 2026')
|
||||||
|
self.assertNotIn('paper_url', row['metadata'])
|
||||||
|
self.assertEqual(row['metadata']['publication_state'], 'withdrawn')
|
||||||
|
self.assertEqual(sum(r['status'] == 'accepted' for r in rows), 638)
|
||||||
|
|
||||||
|
def test_withdrawal_ingestion_retains_date_precision_without_invented_acceptance(self):
|
||||||
|
rows, _, _ = self.parse()
|
||||||
|
row, = [r for r in rows if r['external_id'] == '2026/p3178']
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); path = root/'withdrawn.jsonl'; db = root/'sample.sqlite'
|
||||||
|
path.write_text(json.dumps(row, ensure_ascii=False)+'\n')
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps({
|
||||||
|
'source': 'aamas_proceedings', 'list_url': aamas2026.URL,
|
||||||
|
'venue': 'AAMAS', 'year': 2026, 'track': row['track'],
|
||||||
|
'count': 1, 'status': 'partial', 'missing_abstract_count': 1,
|
||||||
|
'file_sha256': hashlib.sha256(path.read_bytes()).hexdigest()}))
|
||||||
|
ingest_file(db, path)
|
||||||
|
with sqlite3.connect(db) as c:
|
||||||
|
self.assertEqual(c.execute('SELECT status,raw_value,decision_date FROM decisions').fetchall(),
|
||||||
|
[('withdrawn', 'Withdrawn June 8, 2026', None)])
|
||||||
|
metadata = json.loads(c.execute('SELECT metadata_json FROM source_records').fetchone()[0])
|
||||||
|
self.assertEqual(metadata['withdrawal_date'], '2026-06-08')
|
||||||
|
self.assertEqual(metadata['decision_date_precision'], 'day')
|
||||||
|
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(), [])
|
||||||
|
|
||||||
|
def test_retraction_is_not_accepted_or_withdrawn_and_retains_date(self):
|
||||||
|
rows, _, held = self.parse()
|
||||||
|
self.assertNotIn('2026/p238', {r['external_id'] for r in rows})
|
||||||
|
h, = held
|
||||||
|
self.assertEqual(h['publication_state'], 'retracted')
|
||||||
|
self.assertEqual(h['retraction_date'], '2026-06-08')
|
||||||
|
self.assertEqual(h['raw_publication_state'], 'Retracted June 8, 2026')
|
||||||
|
self.assertNotIn('status', h)
|
||||||
|
self.assertNotIn('paper_url', h['metadata'])
|
||||||
|
self.assertEqual(h['metadata']['source_evidence'], EVIDENCE)
|
||||||
|
|
||||||
|
def test_tracks_formats_and_prior_decisions_are_not_conflated(self):
|
||||||
|
rows, _, _ = self.parse()
|
||||||
|
counts = {t: sum(r['track'] == t for r in rows) for t in {r['track'] for r in rows}}
|
||||||
|
self.assertEqual(counts, {'proceedings_unspecified': 522, 'aaai_transfer': 19,
|
||||||
|
'blue_sky': 17, 'doctoral': 34, 'demo': 44, 'jaamas': 3})
|
||||||
|
for row in rows:
|
||||||
|
self.assertEqual(row['tags'][0]['namespace'], 'aamas_proceedings:publication_format')
|
||||||
|
if row['track'] == 'aaai_transfer':
|
||||||
|
self.assertIn('none inferred', row['metadata']['prior_submission_note'])
|
||||||
|
self.assertNotIn('decision_events', row)
|
||||||
|
if row['track'] == 'jaamas':
|
||||||
|
self.assertEqual(row['metadata']['publication_format'], 'journal_presentation')
|
||||||
|
self.assertIn('not another independent', row['metadata']['track_scope_note'])
|
||||||
|
|
||||||
|
def test_refuses_changed_year_section_page_link_author_and_retraction(self):
|
||||||
|
html = FIXTURE['contents']
|
||||||
|
replacements = [('AAMAS 2026:', 'AAMAS 2027:'),
|
||||||
|
('name="R" id="R"></a><font color="#010101">Research Paper Track',
|
||||||
|
'name="R" id="R"></a><font color="#010101">Workshop Track'),
|
||||||
|
('(Page 4)', '(Page 1)'),
|
||||||
|
('../pdfs/HQQZ1937.pdf', 'https://other.example/paper.pdf'),
|
||||||
|
('Michael Wooldridge <i>', '<i>'),
|
||||||
|
('Retracted June 8, 2026', 'Withdrawn June 8, 2026')]
|
||||||
|
for old, new in replacements:
|
||||||
|
self.assertIn(old, html)
|
||||||
|
with self.subTest(change=old), self.assertRaises(ValueError):
|
||||||
|
self.parse(html.replace(old, new, 1))
|
||||||
|
|
||||||
|
def test_frozen_collection_is_partial_with_full_held_evidence(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp, patch.object(aamas2026, 'fetch', side_effect=AssertionError('No HTTP')):
|
||||||
|
root = Path(tmp)
|
||||||
|
report = aamas.collect(2026, root/'out', snapshots=self.snapshots(root/'source'))
|
||||||
|
self.assertEqual(report['records'], 639)
|
||||||
|
self.assertEqual(report['status_counts'], {'accepted': 638, 'withdrawn': 1})
|
||||||
|
self.assertEqual(len(report['files']), 6)
|
||||||
|
self.assertEqual(len(report['held_publications']), 1)
|
||||||
|
self.assertFalse(report['database_written'])
|
||||||
|
self.assertFalse(report['pdfs_downloaded'])
|
||||||
|
for m in report['files']:
|
||||||
|
self.assertEqual(m['status'], 'partial')
|
||||||
|
self.assertEqual(m['missing_abstract_count'], m['count'])
|
||||||
|
self.assertEqual(hashlib.sha256(Path(m['file']).read_bytes()).hexdigest(), m['file_sha256'])
|
||||||
|
self.assertTrue((root/'out/aamas-proceedings-2026.json').is_file())
|
||||||
|
|
||||||
|
def test_frozen_response_rejects_tamper_unsafe_path_status_and_symlink(self):
|
||||||
|
for mode in ('tamper', 'path', 'status', 'symlink'):
|
||||||
|
with self.subTest(mode=mode), tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); source = self.snapshots(root/'source')
|
||||||
|
r = json.loads((source/'receipts.json').read_text())
|
||||||
|
if mode == 'tamper':
|
||||||
|
(source/'contents.html').write_text('bad')
|
||||||
|
elif mode == 'path':
|
||||||
|
r['receipts'][0]['file'] = '../outside.html'
|
||||||
|
elif mode == 'status':
|
||||||
|
r['receipts'][0]['status'] = 403
|
||||||
|
else:
|
||||||
|
(root/'outside.html').write_text(FIXTURE['contents'])
|
||||||
|
(source/'contents.html').unlink()
|
||||||
|
(source/'contents.html').symlink_to(root/'outside.html')
|
||||||
|
(source/'receipts.json').write_text(json.dumps(r))
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
aamas.collect(2026, root/'out', snapshots=source)
|
||||||
|
self.assertFalse((root/'out/aamas-proceedings-2026.json').exists())
|
||||||
|
|
||||||
|
def test_live_mode_fetches_only_html_and_preserves_receipt(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); cache = root/'cache'; cache.mkdir()
|
||||||
|
receipt = {'retrieved_at': EVIDENCE['observed_at'], 'url': aamas2026.URL}
|
||||||
|
(cache/hashlib.sha256(aamas2026.URL.encode()).hexdigest()).with_suffix('.json').write_text(json.dumps(receipt))
|
||||||
|
with patch.object(aamas2026, 'fetch', return_value=FIXTURE['contents'].encode()) as fetch:
|
||||||
|
report = aamas.collect(2026, root/'out', cache=cache)
|
||||||
|
fetch.assert_called_once_with(aamas2026.URL, cache, 8*1024**2, delay=1)
|
||||||
|
self.assertEqual(report['source_receipts'][0]['response_receipt'], receipt)
|
||||||
|
|
||||||
|
def test_frozen_fixture_is_original_hashed_response(self):
|
||||||
|
r, = FIXTURE['receipts']['receipts']
|
||||||
|
self.assertEqual(hashlib.sha256(FIXTURE['contents'].encode()).hexdigest(), r['sha256'])
|
||||||
|
self.assertEqual(r['bytes'], len(FIXTURE['contents'].encode()))
|
||||||
@@ -99,7 +99,7 @@ class AamasProceedingsTests(unittest.TestCase):
|
|||||||
aamas.collect(2012, root/'out', snapshots=source)
|
aamas.collect(2012, root/'out', snapshots=source)
|
||||||
self.assertFalse((root/'out/aamas-proceedings-2012.json').exists())
|
self.assertFalse((root/'out/aamas-proceedings-2012.json').exists())
|
||||||
with self.assertRaises(ValueError):
|
with self.assertRaises(ValueError):
|
||||||
aamas.collect(2026, root/'future', snapshots=source)
|
aamas.collect(2027, root/'future', snapshots=source)
|
||||||
self.assertFalse((root/'future').exists())
|
self.assertFalse((root/'future').exists())
|
||||||
|
|
||||||
def test_ids_colliding_across_sections_do_not_produce_a_completed_collection(self):
|
def test_ids_colliding_across_sections_do_not_produce_a_completed_collection(self):
|
||||||
|
|||||||
@@ -0,0 +1,81 @@
|
|||||||
|
"""Exact AMTCL volume provenance and native metadata/history preservation."""
|
||||||
|
import copy
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sqlite3
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
import xml.etree.ElementTree as ET
|
||||||
|
|
||||||
|
from paper_atlas.collectors.acl import collect_xml, volume_scope, text
|
||||||
|
from paper_atlas.collectors.acl_historical import event_scope
|
||||||
|
from paper_atlas.ingestion.records import ingest_file
|
||||||
|
|
||||||
|
FIXTURE = Path(__file__).parent/'fixtures/acl-1963-earlymt.xml'
|
||||||
|
COMMIT = 'a7be623d01c48c61405dec31c871121cdbed5a01'
|
||||||
|
|
||||||
|
|
||||||
|
class Acl1963Tests(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.volume = ET.parse(FIXTURE).getroot().find('volume')
|
||||||
|
self.meta = self.volume.find('meta')
|
||||||
|
|
||||||
|
def test_official_snapshot_and_exact_scope(self):
|
||||||
|
self.assertEqual(hashlib.sha256(FIXTURE.read_bytes()).hexdigest(), '4a4626d792ac507de0281cb358769f7246f4a0d6b6a9176ac02e61545c1635fc')
|
||||||
|
self.assertEqual(len(self.volume.findall('paper')), 34)
|
||||||
|
self.assertEqual(volume_scope(self.meta, '1', '1963.earlymt'), ('ACL', ['ACL', 'AMTCL'], 'proceedings_unspecified'))
|
||||||
|
scope = event_scope(self.meta, '1', '1963.earlymt')
|
||||||
|
self.assertEqual(scope['original_venue'], 'EarlyMT')
|
||||||
|
self.assertEqual(scope['publication_format'], 'published_abstract')
|
||||||
|
self.assertEqual(scope['event_dates'], ['1963-08-25', '1963-08-26'])
|
||||||
|
|
||||||
|
def test_other_earlymt_collections_and_volumes_stay_excluded(self):
|
||||||
|
for collection, volume in [('1962.earlymt', '1'), ('1971.earlymt', '1'), ('1963.earlymt', '2'), ('1963.other', '1')]:
|
||||||
|
with self.subTest(collection=collection, volume=volume):
|
||||||
|
self.assertIsNone(volume_scope(self.meta, volume, collection))
|
||||||
|
self.assertIsNone(volume_scope(self.meta, '1'))
|
||||||
|
|
||||||
|
def test_scope_metadata_drift_and_duplicates_are_rejected(self):
|
||||||
|
for field in ['venue', 'year', 'booktitle', 'address', 'month']:
|
||||||
|
for duplicate in [False, True]:
|
||||||
|
meta = copy.deepcopy(self.meta)
|
||||||
|
if duplicate:
|
||||||
|
ET.SubElement(meta, field).text = text(meta.find(field))
|
||||||
|
else:
|
||||||
|
meta.find(field).text = 'changed'
|
||||||
|
with self.subTest(field=field, duplicate=duplicate), self.assertRaises(ValueError):
|
||||||
|
volume_scope(meta, '1', '1963.earlymt')
|
||||||
|
|
||||||
|
def test_every_original_contribution_survives_collection(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
manifest, = collect_xml(FIXTURE, tmp, COMMIT)
|
||||||
|
rows = [json.loads(raw) for raw in Path(manifest['file']).read_text().splitlines()]
|
||||||
|
self.assertEqual(manifest['count'], 34)
|
||||||
|
self.assertEqual(manifest['missing_abstract_count'], 0)
|
||||||
|
self.assertEqual(manifest['excluded_materials'], [])
|
||||||
|
for paper, row in zip(self.volume.findall('paper'), rows):
|
||||||
|
self.assertEqual(row['external_id'], '1963.earlymt-1.'+paper.attrib['id'])
|
||||||
|
self.assertEqual(row['title'], text(paper.find('title')))
|
||||||
|
self.assertEqual(row['abstract'], text(paper.find('abstract')))
|
||||||
|
self.assertEqual(row['authors'], [{'given':text(a.find('first')), 'family':text(a.find('last'))} for a in paper.findall('author')])
|
||||||
|
self.assertEqual(row['metadata']['venue_codes'], ['earlymt'])
|
||||||
|
self.assertEqual(row['metadata']['publication_format'], 'published_abstract')
|
||||||
|
self.assertEqual(row['tags'], [{'namespace':'acl_anthology:publication_format','value':'published_abstract'}])
|
||||||
|
self.assertEqual((row['venue'], row['track'], row['status']), ('ACL', 'proceedings_unspecified', 'accepted'))
|
||||||
|
|
||||||
|
def test_native_ingestion_is_idempotent_and_preserves_abstracts_and_history(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp)
|
||||||
|
manifest, = collect_xml(FIXTURE, root/'source', COMMIT)
|
||||||
|
db = root/'sample.sqlite'
|
||||||
|
ingest_file(db, manifest['file'])
|
||||||
|
ingest_file(db, manifest['file'])
|
||||||
|
with sqlite3.connect(db) as c:
|
||||||
|
for table in ['papers', 'source_records', 'submissions', 'decisions']:
|
||||||
|
self.assertEqual(c.execute('SELECT count(*) FROM '+table).fetchone()[0], 34)
|
||||||
|
self.assertEqual(c.execute("SELECT count(*) FROM papers WHERE abstract IS NOT NULL AND abstract!=''").fetchone()[0], 34)
|
||||||
|
self.assertEqual(c.execute("SELECT DISTINCT status FROM decisions").fetchall(), [('accepted',)])
|
||||||
|
self.assertEqual(c.execute("SELECT v.acronym,e.year,e.track FROM editions e JOIN venues v ON v.id=e.venue_id").fetchall(), [('ACL',1963,'proceedings_unspecified')])
|
||||||
|
self.assertIsNone(c.execute("SELECT 1 FROM venue_aliases WHERE alias='EarlyMT'").fetchone())
|
||||||
|
self.assertEqual(c.execute("SELECT count(*) FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id WHERE t.namespace='acl_anthology:publication_format' AND t.value='published_abstract'").fetchone()[0], 34)
|
||||||
@@ -0,0 +1,114 @@
|
|||||||
|
"""Pinned T78 evidence, edition-local aliases and an explicitly identified material."""
|
||||||
|
import copy
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sqlite3
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
import xml.etree.ElementTree as ET
|
||||||
|
|
||||||
|
from paper_atlas.collectors.acl import collect_xml, volume_scope
|
||||||
|
from paper_atlas.collectors.acl_historical import event_scope, material_reason
|
||||||
|
from paper_atlas.db.store import Store
|
||||||
|
from paper_atlas.ingestion.records import ingest_file
|
||||||
|
|
||||||
|
FIXTURE = Path(__file__).parent/'fixtures/acl-t78.xml'
|
||||||
|
COMMIT = 'a7be623d01c48c61405dec31c871121cdbed5a01'
|
||||||
|
|
||||||
|
|
||||||
|
class AclHistoricalTests(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.root = ET.parse(FIXTURE).getroot()
|
||||||
|
self.volume = self.root.find('volume')
|
||||||
|
self.meta = self.volume.find('meta')
|
||||||
|
|
||||||
|
def test_frozen_volume_matches_official_snapshot(self):
|
||||||
|
self.assertEqual(hashlib.sha256(FIXTURE.read_bytes()).hexdigest(), 'f712f94cff0cba05f786b7ac31dad2ea9408fde2a17a04531130df623f57dd84')
|
||||||
|
self.assertEqual(len(self.volume.findall('paper')), 36)
|
||||||
|
self.assertEqual(volume_scope(self.meta, '1', 'T78'), ('ACL+TINLAP', ['ACL', 'TINLAP'], 'proceedings_unspecified'))
|
||||||
|
|
||||||
|
def test_unknown_year_collection_or_volume_is_not_globally_aliased(self):
|
||||||
|
for collection, volume in [('T75', '1'), ('T87', '1'), ('T78', '2'), ('C65', '1')]:
|
||||||
|
with self.subTest(collection=collection, volume=volume):
|
||||||
|
self.assertIsNone(event_scope(self.meta, volume, collection))
|
||||||
|
self.assertIsNone(volume_scope(self.meta, volume, collection))
|
||||||
|
self.assertIsNone(volume_scope(self.meta, '1'))
|
||||||
|
|
||||||
|
def test_known_volume_scope_rejects_critical_metadata_drift(self):
|
||||||
|
for field, replacement in [('venue', 'acl'), ('year', '1979'), ('booktitle', 'Another TINLAP'), ('editor/first', 'Another')]:
|
||||||
|
changed = copy.deepcopy(self.meta)
|
||||||
|
changed.find(field).text = replacement
|
||||||
|
with self.subTest(field=field), self.assertRaises(ValueError):
|
||||||
|
event_scope(changed, '1', 'T78')
|
||||||
|
changed = copy.deepcopy(self.meta)
|
||||||
|
ET.SubElement(changed, 'year').text = '1978'
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
event_scope(changed, '1', 'T78')
|
||||||
|
|
||||||
|
def test_evidence_retains_original_series_dates_and_role_boundary(self):
|
||||||
|
scope = event_scope(self.meta, '1', 'T78')
|
||||||
|
self.assertEqual(scope['original_venue'], 'TINLAP')
|
||||||
|
self.assertEqual(scope['event_dates'], ['1978-07-25', '1978-07-27'])
|
||||||
|
self.assertEqual([e['pdf_pages'] for e in scope['evidence']], [[9, 10], [3]])
|
||||||
|
self.assertIn('not identified', scope['track_scope_note'])
|
||||||
|
self.assertIn('pending', scope['evidence'][0]['scope'])
|
||||||
|
|
||||||
|
def test_only_exact_session_questions_material_is_excluded(self):
|
||||||
|
paper = copy.deepcopy(self.volume.findall('paper')[-1])
|
||||||
|
self.assertIn('Session question list', material_reason('T78', '1', paper))
|
||||||
|
self.assertIsNone(material_reason('C65', '1', paper))
|
||||||
|
self.assertIsNone(material_reason('T78', '2', paper))
|
||||||
|
paper.set('id', '37')
|
||||||
|
self.assertIsNone(material_reason('T78', '1', paper))
|
||||||
|
paper.set('id', '36')
|
||||||
|
ET.SubElement(paper, 'author')
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
material_reason('T78', '1', paper)
|
||||||
|
|
||||||
|
def test_all_authored_source_fields_and_material_receipt_survive_collection(self):
|
||||||
|
with tempfile.TemporaryDirectory() as temporary:
|
||||||
|
manifest, = collect_xml(FIXTURE, temporary, COMMIT)
|
||||||
|
rows = [json.loads(raw) for raw in Path(manifest['file']).read_text().splitlines()]
|
||||||
|
self.assertEqual(manifest['count'], 35)
|
||||||
|
self.assertEqual(manifest['xml_paper_elements'], 36)
|
||||||
|
self.assertEqual(manifest['missing_abstract_count'], 35)
|
||||||
|
self.assertEqual(manifest['excluded_materials'][0]['external_id'], 'T78-1036')
|
||||||
|
for number, row in enumerate(rows, 1):
|
||||||
|
paper = self.volume.findall('paper')[number-1]
|
||||||
|
self.assertEqual(row['external_id'], f'T78-1{number:03d}')
|
||||||
|
self.assertEqual(row['title'], ' '.join(''.join(paper.find('title').itertext()).split()))
|
||||||
|
self.assertEqual(row['metadata']['venue_codes'], ['tinlap'])
|
||||||
|
self.assertEqual(row['metadata']['historical_event_scope']['original_venue'], 'TINLAP')
|
||||||
|
self.assertEqual(row['metadata']['metadata_commit'], COMMIT)
|
||||||
|
self.assertEqual((row['status'], row['track'], row['abstract']), ('accepted', 'proceedings_unspecified', None))
|
||||||
|
|
||||||
|
def test_old_coling_scope_is_unchanged_by_calendar_replacement_evidence(self):
|
||||||
|
meta = ET.fromstring('<meta><venue>coling</venue><year>1965</year><booktitle>COLING 1965</booktitle></meta>')
|
||||||
|
self.assertEqual(volume_scope(meta, '1', 'C65'), ('COLING', ['COLING'], 'main'))
|
||||||
|
self.assertIsNone(event_scope(meta, '1', 'C65'))
|
||||||
|
|
||||||
|
def test_native_ingestion_does_not_alias_other_tinlap_years_or_infer_negative_status(self):
|
||||||
|
with tempfile.TemporaryDirectory() as temporary:
|
||||||
|
root = Path(temporary)
|
||||||
|
manifest, = collect_xml(FIXTURE, root/'source', COMMIT)
|
||||||
|
db = root/'sample.sqlite'
|
||||||
|
store = Store(db)
|
||||||
|
pid, rid = store.record('test_control', 'tinlap1975', 'https://example.org/tinlap1975',
|
||||||
|
'An earlier TINLAP contribution', None, ['Jane Doe'])
|
||||||
|
store.decision(pid, rid, 'TINLAP', 1975, 'main', 'tinlap1975', 'https://example.org/tinlap1975', 'accepted', 'test control')
|
||||||
|
store.conn.commit()
|
||||||
|
store.conn.close()
|
||||||
|
ingest_file(db, manifest['file'])
|
||||||
|
ingest_file(db, manifest['file'])
|
||||||
|
with sqlite3.connect(db) as c:
|
||||||
|
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0], 36)
|
||||||
|
self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0], 36)
|
||||||
|
self.assertEqual(c.execute('SELECT count(*) FROM decisions').fetchone()[0], 36)
|
||||||
|
self.assertEqual(c.execute("SELECT count(*) FROM decisions WHERE status!='accepted'").fetchone()[0], 0)
|
||||||
|
self.assertIsNone(c.execute("SELECT 1 FROM source_records WHERE external_id='T78-1036'").fetchone())
|
||||||
|
aliases = c.execute('SELECT v.acronym,a.alias FROM venue_aliases a JOIN venues v ON v.id=a.venue_id ORDER BY a.alias').fetchall()
|
||||||
|
self.assertEqual(aliases, [('ACL+TINLAP', 'ACL'), ('ACL+TINLAP', 'TINLAP')])
|
||||||
|
metadata = json.loads(c.execute("SELECT metadata_json FROM source_records WHERE external_id='T78-1001'").fetchone()[0])
|
||||||
|
self.assertEqual(metadata['historical_event_scope']['event_year'], 1978)
|
||||||
|
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(), [])
|
||||||
@@ -0,0 +1,75 @@
|
|||||||
|
import gzip
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import corl2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class CoRL2026Test(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.bundle = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/corl2026-source.json.gz').read_bytes()))
|
||||||
|
|
||||||
|
def test_official_census_and_presentation_dedup(self):
|
||||||
|
program = c.parse_program(self.bundle['official_html'])
|
||||||
|
self.assertEqual(len(program), 687)
|
||||||
|
self.assertEqual(sum(len(p['presentations']) for p in program.values()), 780)
|
||||||
|
self.assertEqual(sum(len(p['presentations']) == 2 for p in program.values()), 93)
|
||||||
|
|
||||||
|
def test_conflicting_official_title_is_not_silently_folded(self):
|
||||||
|
html = self.bundle['official_html']
|
||||||
|
title = 'Diagnosing Compositional Generalization in Sequential Robot Tasks'
|
||||||
|
self.assertIn(title, html)
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.parse_program(html.replace(title, 'Different paper', 1))
|
||||||
|
|
||||||
|
def test_foreign_forum_source_is_rejected(self):
|
||||||
|
program = c.parse_program(self.bundle['official_html']); archive = json.loads(self.bundle['filtered'])
|
||||||
|
archive['papers'][0]['author_source'] = 'https://openreview.net/forum?id=WrongPaper'
|
||||||
|
with self.assertRaisesRegex(ValueError, 'another submission'):
|
||||||
|
c.join_metadata(program, archive)
|
||||||
|
|
||||||
|
def test_unknown_profile_fields_are_rejected(self):
|
||||||
|
program = c.parse_program(self.bundle['official_html']); archive = json.loads(self.bundle['filtered'])
|
||||||
|
archive['papers'][0]['authorids'] = ['profile']
|
||||||
|
with self.assertRaisesRegex(ValueError, 'profile'):
|
||||||
|
c.join_metadata(program, archive)
|
||||||
|
|
||||||
|
def test_source_sets_must_agree(self):
|
||||||
|
program = c.parse_program(self.bundle['official_html']); archive = json.loads(self.bundle['filtered'])
|
||||||
|
archive['papers'].pop()
|
||||||
|
with self.assertRaisesRegex(ValueError, 'forum sets'):
|
||||||
|
c.join_metadata(program, archive)
|
||||||
|
|
||||||
|
def test_complete_snapshot_and_explicit_version_provenance(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); official = root/'official'; archive = root/'archive'
|
||||||
|
official.mkdir(); archive.mkdir()
|
||||||
|
(official/'accepted.html').write_text(self.bundle['official_html'])
|
||||||
|
(official/'research.json').write_text(json.dumps(self.bundle['research']))
|
||||||
|
(archive/'explorer-paper-metadata-filtered-v1.json').write_text(self.bundle['filtered'])
|
||||||
|
out = root/'out'; report = c.collect(out, official_snapshots=official, archive_snapshots=archive)
|
||||||
|
self.assertEqual(report['records'], 687); self.assertEqual(len(report['title_variants']), 34)
|
||||||
|
self.assertEqual(report['public_rejected_availability'], 'unknown')
|
||||||
|
rows = [json.loads(l) for l in (out/'corl-program-archive-2026-main.jsonl').read_text().splitlines()]
|
||||||
|
self.assertEqual(len({r['external_id'] for r in rows}), 687)
|
||||||
|
self.assertTrue(all(set(r['identifiers']) == {'openreview'} for r in rows))
|
||||||
|
self.assertTrue(all(r['submission_identifiers'] == [{'scheme': 'openreview', 'value': r['identifiers']['openreview']}] for r in rows))
|
||||||
|
self.assertTrue(all(not r['tags'] and not r['metadata']['original_decision_signature_verified'] for r in rows))
|
||||||
|
self.assertTrue(all(r['title'] == r['metadata']['archive_fields']['title'] for r in rows))
|
||||||
|
self.assertTrue(all(r['authors'] and r['abstract'] for r in rows))
|
||||||
|
self.assertFalse(report['database_written']); self.assertFalse(report['pdfs_downloaded'])
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||||
|
c.collect(out, official_snapshots=official, archive_snapshots=archive)
|
||||||
|
(official/'accepted.html').write_text('tampered')
|
||||||
|
with self.assertRaisesRegex(ValueError, 'hash changed'):
|
||||||
|
c.collect(root/'failed', official_snapshots=official, archive_snapshots=archive)
|
||||||
|
self.assertTrue((root/'failed/failed.json').exists())
|
||||||
|
self.assertFalse((root/'failed/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_forum_identity_validation(self):
|
||||||
|
for bad in ['https://openreview.net/challenge?redirect=forum', 'https://evil.test/forum?id=abcdefgh', 'https://openreview.net/forum?id=abcdefgh&other=1']:
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.forum_id(bad)
|
||||||
@@ -0,0 +1,141 @@
|
|||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sqlite3
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from paper_atlas.collectors.dblp import catalogue
|
||||||
|
from paper_atlas.collectors.dblp_recovery import recover
|
||||||
|
from paper_atlas.collectors.dblp_scope import scope
|
||||||
|
from paper_atlas.ingestion.records import ingest_file
|
||||||
|
|
||||||
|
|
||||||
|
class DblpRecoveryTests(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.addCleanup(self.tmp.cleanup)
|
||||||
|
self.root = Path(self.tmp.name)
|
||||||
|
self.dtd = self.root / 'dblp.dtd'
|
||||||
|
self.dtd.write_bytes(b'<!ENTITY eacute "é">')
|
||||||
|
self.archive = self.root / 'dblp.xml.gz'
|
||||||
|
# These include a keynote and a workshop, neither a main-paper inference.
|
||||||
|
body = '''<proceedings key="conf/ifaamas/2025"><title>AAMAS 2025</title><year>2025</year></proceedings>
|
||||||
|
<inproceedings key="conf/ifaamas/keynote"><author>René</author><title>Keynote: Agents</title><year>2025</year><booktitle>AAMAS</booktitle><crossref>conf/ifaamas/2025</crossref></inproceedings>
|
||||||
|
<inproceedings key="conf/ifaamas/paper"><author>Alice Doe</author><title>Research <i>agents</i></title><year>2025</year><booktitle>AAMAS</booktitle><crossref>conf/ifaamas/2025</crossref><ee>https://dl.acm.org/doi/10.5555/3709347.123</ee></inproceedings>
|
||||||
|
<proceedings key="conf/ifaamas/2025w"><title>AAMAS Workshop</title><year>2025</year></proceedings>
|
||||||
|
<inproceedings key="conf/ifaamas/workshop"><title>Workshop paper</title><year>2025</year><booktitle>AAMAS Workshop</booktitle><crossref>conf/ifaamas/2025w</crossref></inproceedings>
|
||||||
|
<inproceedings key="conf/unrelated/other"><title>Other family</title><year>2025</year></inproceedings>'''
|
||||||
|
with gzip.open(self.archive, 'wb') as stream:
|
||||||
|
stream.write(('<?xml version="1.0"?><!DOCTYPE dblp SYSTEM "dblp.dtd"><dblp>'
|
||||||
|
+ body + '</dblp>').encode())
|
||||||
|
self.sha = hashlib.sha256(self.archive.read_bytes()).hexdigest()
|
||||||
|
self.md5 = hashlib.md5(self.dtd.read_bytes()).hexdigest()
|
||||||
|
self.manifest = catalogue(self.archive, self.dtd, self.root / 'catalogue',
|
||||||
|
self.sha, self.md5)
|
||||||
|
self.catalogue = Path(self.manifest['catalogue'])
|
||||||
|
self.manifest_path = self.catalogue.with_name('dblp-ai-candidates.manifest.json')
|
||||||
|
|
||||||
|
def run_recovery(self, **kwargs):
|
||||||
|
options = dict(expected_sha=self.sha, expected_dtd_md5=self.md5)
|
||||||
|
options.update(kwargs)
|
||||||
|
return recover(self.archive, self.dtd, self.catalogue, self.root / 'recovered',
|
||||||
|
['ifaamas'], **options)
|
||||||
|
|
||||||
|
def test_fresh_discovery_and_recovery_from_old_candidate_omission(self):
|
||||||
|
self.assertEqual(self.manifest['candidate_family_counts']['ifaamas'], 3)
|
||||||
|
# Simulate an older filter without altering its complete catalogue.
|
||||||
|
manifest = dict(self.manifest)
|
||||||
|
manifest['candidate_family_counts'] = {}
|
||||||
|
self.manifest_path.write_text(json.dumps(manifest))
|
||||||
|
candidate = Path(manifest['file'])
|
||||||
|
candidate.write_text('old incomplete candidate file\n')
|
||||||
|
before = {p: p.read_bytes() for p in [self.archive, self.dtd, self.catalogue,
|
||||||
|
self.manifest_path, candidate]}
|
||||||
|
with patch('urllib.request.urlopen', side_effect=AssertionError('No network')):
|
||||||
|
report = self.run_recovery()
|
||||||
|
rows = [json.loads(line) for line in Path(report['file']).read_text().splitlines()]
|
||||||
|
self.assertEqual(report['recovered_publications'], 3)
|
||||||
|
self.assertEqual(report['recovered_volumes'], 2)
|
||||||
|
self.assertEqual(report['recovered_records'], 5)
|
||||||
|
self.assertEqual(report['records_scanned'], 6)
|
||||||
|
self.assertEqual(report['old_candidate_family_counts'], {'ifaamas': 0})
|
||||||
|
self.assertFalse(report['database_written'])
|
||||||
|
self.assertEqual(next(r for r in rows if r['key'].endswith('/keynote'))['fields']['author'], ['René'])
|
||||||
|
self.assertTrue(any(r['key'].endswith('/workshop') for r in rows))
|
||||||
|
self.assertTrue(all(not ({'source', 'status', 'venue', 'track'} & r.keys()) for r in rows))
|
||||||
|
publication = next(r for r in rows if r['key'].endswith('/paper'))
|
||||||
|
volume = next(r for r in rows if r['key'] == 'conf/ifaamas/2025')
|
||||||
|
self.assertEqual(scope(publication, volume), (None, 'outside_target_family'))
|
||||||
|
self.assertEqual({p: p.read_bytes() for p in before}, before)
|
||||||
|
self.assertEqual(hashlib.sha256(Path(report['file']).read_bytes()).hexdigest(),
|
||||||
|
report['file_sha256'])
|
||||||
|
# Raw recovery files cannot accidentally enter the normalized writer.
|
||||||
|
with patch('paper_atlas.ingestion.records.open_store') as writer:
|
||||||
|
with self.assertRaisesRegex(ValueError, 'source manifest'):
|
||||||
|
ingest_file(self.root / 'never.sqlite', report['file'])
|
||||||
|
writer.assert_not_called()
|
||||||
|
self.assertFalse((self.root / 'never.sqlite').exists())
|
||||||
|
|
||||||
|
def test_checksum_mismatch_does_not_publish(self):
|
||||||
|
for options in [{'expected_sha': '0' * 64}, {'expected_dtd_md5': '0' * 32}]:
|
||||||
|
with self.subTest(options=options), self.assertRaisesRegex(ValueError, 'checksum'):
|
||||||
|
self.run_recovery(**options)
|
||||||
|
self.assertFalse((self.root / 'recovered').exists())
|
||||||
|
|
||||||
|
def test_wrong_catalogue_snapshot_and_incomplete_totals_are_rejected(self):
|
||||||
|
for changes in [{'snapshot_sha256': '0' * 64}, {'dtd_md5': '0' * 32},
|
||||||
|
{'records_scanned': 99}, {'record_types': {'proceedings': 2}}]:
|
||||||
|
manifest = {**self.manifest, **changes}
|
||||||
|
self.manifest_path.write_text(json.dumps(manifest))
|
||||||
|
with self.subTest(changes=changes), self.assertRaises(ValueError):
|
||||||
|
self.run_recovery()
|
||||||
|
self.assertFalse((self.root / 'recovered').exists())
|
||||||
|
|
||||||
|
def test_changed_family_count_does_not_publish_partial_result(self):
|
||||||
|
with sqlite3.connect(self.catalogue) as c:
|
||||||
|
c.execute("UPDATE family_counts SET count=count+1 WHERE family='ifaamas' AND record_type='inproceedings'")
|
||||||
|
with self.assertRaisesRegex(ValueError, 'family totals'):
|
||||||
|
self.run_recovery()
|
||||||
|
self.assertFalse((self.root / 'recovered').exists())
|
||||||
|
|
||||||
|
def test_changed_volume_metadata_is_detected(self):
|
||||||
|
with sqlite3.connect(self.catalogue) as c:
|
||||||
|
row = json.loads(c.execute("SELECT metadata_json FROM proceedings WHERE key='conf/ifaamas/2025'").fetchone()[0])
|
||||||
|
row['fields']['title'] = ['Wrong volume']
|
||||||
|
c.execute("UPDATE proceedings SET metadata_json=? WHERE key='conf/ifaamas/2025'", (json.dumps(row),))
|
||||||
|
with self.assertRaisesRegex(ValueError, 'proceedings differs'):
|
||||||
|
self.run_recovery()
|
||||||
|
self.assertFalse((self.root / 'recovered').exists())
|
||||||
|
|
||||||
|
def test_truncated_gzip_never_publishes(self):
|
||||||
|
self.archive.write_bytes(self.archive.read_bytes()[:-8])
|
||||||
|
revised_sha = hashlib.sha256(self.archive.read_bytes()).hexdigest()
|
||||||
|
self.manifest_path.write_text(json.dumps({**self.manifest, 'snapshot_sha256': revised_sha}))
|
||||||
|
with self.assertRaises((EOFError, OSError)):
|
||||||
|
self.run_recovery(expected_sha=revised_sha)
|
||||||
|
self.assertFalse((self.root / 'recovered').exists())
|
||||||
|
|
||||||
|
def test_existing_output_is_preserved_and_unknown_family_is_rejected(self):
|
||||||
|
out = self.root / 'recovered'
|
||||||
|
out.mkdir()
|
||||||
|
sentinel = out / 'keep'
|
||||||
|
sentinel.write_text('existing')
|
||||||
|
with self.assertRaisesRegex(ValueError, 'already exist'):
|
||||||
|
self.run_recovery()
|
||||||
|
self.assertEqual(sentinel.read_text(), 'existing')
|
||||||
|
for names in [[], ['../ifaamas'], ['not-a-supported-family'], [None]]:
|
||||||
|
with self.subTest(names=names), self.assertRaises(ValueError):
|
||||||
|
recover(self.archive, self.dtd, self.catalogue, self.root / 'other', names)
|
||||||
|
|
||||||
|
def test_requested_family_missing_from_catalogue_is_explicit(self):
|
||||||
|
with self.assertRaisesRegex(ValueError, 'absent from complete catalogue'):
|
||||||
|
recover(self.archive, self.dtd, self.catalogue, self.root / 'recovered',
|
||||||
|
['ifaamas', 'icml'], expected_sha=self.sha, expected_dtd_md5=self.md5)
|
||||||
|
self.assertFalse((self.root / 'recovered').exists())
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
unittest.main()
|
||||||
@@ -0,0 +1,139 @@
|
|||||||
|
import copy
|
||||||
|
import gzip
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from paper_atlas.collectors import eccv2026 as c
|
||||||
|
from paper_atlas.io.html_cache import PublicHTMLCache
|
||||||
|
|
||||||
|
|
||||||
|
class ECCV2026Tests(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.bundle = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/eccv2026-source.json.gz').read_bytes()))
|
||||||
|
cls.program = c.parse_program(cls.bundle['official_pages']['accepted.html'])
|
||||||
|
|
||||||
|
def snapshots(self, root):
|
||||||
|
official = root/'official'; paper = root/'paper'; official.mkdir(); paper.mkdir()
|
||||||
|
for name, raw in self.bundle['official_pages'].items():
|
||||||
|
(official/name).write_bytes(raw.encode('utf-8'))
|
||||||
|
(official/'manifest.json').write_bytes(self.bundle['official_manifest'].encode('utf-8'))
|
||||||
|
for pid, raw in self.bundle['paper_pages'].items():
|
||||||
|
(paper/(pid+'.html')).write_bytes(raw.encode('utf-8'))
|
||||||
|
(paper/'manifest.json').write_text(json.dumps(self.bundle['paper_manifest']))
|
||||||
|
return official, paper
|
||||||
|
|
||||||
|
def test_complete_program_and_duplicate_occurrences_match_gold(self):
|
||||||
|
self.assertEqual(len(self.program), 2834)
|
||||||
|
self.assertEqual(sum(len(p['row_ordinals']) for p in self.program.values()), 2851)
|
||||||
|
for ordinal, gold in enumerate(self.bundle['inventory']['records'], 1):
|
||||||
|
item = self.program[gold['paper_id']]
|
||||||
|
self.assertEqual(item['listing'], {k: gold[k] for k in ('title','authors_literal','keywords_literal','presentation_literal')})
|
||||||
|
self.assertIn(ordinal, item['row_ordinals'])
|
||||||
|
|
||||||
|
def test_conflicting_duplicate_not_silently_folded(self):
|
||||||
|
title = 'Transferability Between Understanding and Generation in Unified Multimodal Models'
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Conflicting'):
|
||||||
|
c.parse_program(self.bundle['official_pages']['accepted.html'].replace('href="/virtual/2026/poster/3137">'+title, 'href="/virtual/2026/poster/3137">Conflicting title', 1))
|
||||||
|
|
||||||
|
def test_missing_member_and_changed_provisional_status_rejected(self):
|
||||||
|
html = self.bundle['official_pages']['accepted.html']
|
||||||
|
for changed in [html.replace('/virtual/2026/poster/4048', '/virtual/2024/poster/4048', 1),
|
||||||
|
html.replace('This paper list is preliminary pending publisher checks.', 'Final archival papers.')]:
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.parse_program(changed)
|
||||||
|
|
||||||
|
def test_public_page_fields_match_independent_sample_review(self):
|
||||||
|
for gold in self.bundle['paper_gold']['records']:
|
||||||
|
pid = gold['paper_id']; fields = c.parse_paper(self.bundle['paper_pages'][pid], gold['url'], self.program[pid])
|
||||||
|
self.assertEqual(fields['title'], gold['title']); self.assertEqual(fields['authors'], gold['authors'])
|
||||||
|
self.assertEqual(fields['abstract'], gold['abstract_literal']); self.assertFalse(fields['review_flags'])
|
||||||
|
self.assertIn('sig-nal', c.parse_paper(self.bundle['paper_pages']['4048'], self.program['4048']['url'], self.program['4048'])['abstract'])
|
||||||
|
|
||||||
|
def test_foreign_canonical_and_inconsistent_jsonld_author_rejected(self):
|
||||||
|
raw = self.bundle['paper_pages']['4048']; url = self.program['4048']['url']
|
||||||
|
for changed in [raw.replace(url, 'https://example.org/other'), raw.replace('Suhyeon Ha', 'Other Author', 1)]:
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.parse_paper(changed, url, self.program['4048'])
|
||||||
|
|
||||||
|
def test_program_and_page_versions_flagged_without_aliases(self):
|
||||||
|
program = copy.deepcopy(self.program['4048']); program['listing']['title'] = 'Earlier source title'
|
||||||
|
fields = c.parse_paper(self.bundle['paper_pages']['4048'], program['url'], program)
|
||||||
|
self.assertEqual(fields['review_flags'], ['program_and_page_title_versions_differ_requires_review'])
|
||||||
|
self.assertEqual(fields['title'], self.program['4048']['listing']['title'])
|
||||||
|
|
||||||
|
def test_partial_snapshot_collection_has_no_inferred_submission_ids(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); official, paper = self.snapshots(root)
|
||||||
|
report = c.collect(root/'out', official_snapshots=official, paper_snapshots_dir=paper)
|
||||||
|
self.assertEqual(report['records'], 3); self.assertEqual(len(report['uncollected_program_ids']), 2831)
|
||||||
|
self.assertEqual(report['public_rejected_decision_metadata'], 'unknown'); self.assertFalse(report['database_written'])
|
||||||
|
manifest, = report['files']; self.assertEqual(manifest['status'], 'partial'); self.assertEqual(manifest['source_expected_count'], 2834)
|
||||||
|
rows = [json.loads(l) for l in Path(manifest['file']).read_text().splitlines()]
|
||||||
|
self.assertEqual({r['external_id'] for r in rows}, {'2026/4048','2026/3137','2026/3652'})
|
||||||
|
self.assertTrue(all(not r['identifiers'] and not r['submission_identifiers'] and r['track']=='program_unclassified' for r in rows))
|
||||||
|
self.assertTrue(all(r['metadata']['publication_state']=='preliminary_pending_publisher_checks' and not r['metadata']['original_decision_signature_verified'] for r in rows))
|
||||||
|
self.assertTrue(all(t['namespace']=='eccv_program:keyword' for r in rows for t in r['tags']))
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||||
|
c.collect(root/'out', official_snapshots=official, paper_snapshots_dir=paper)
|
||||||
|
|
||||||
|
def test_zero_network_budget_reuses_immutable_cache_only(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); official, _ = self.snapshots(root); client = PublicHTMLCache(root/'cache', c.ORIGIN)
|
||||||
|
for r in self.bundle['paper_manifest']['records']:
|
||||||
|
body, receipt, _ = client.paths(r['url']); body.write_text(self.bundle['paper_pages'][r['paper_id']]); receipt.write_text(json.dumps({**r,'retrieved_at':self.bundle['paper_manifest']['at']}))
|
||||||
|
with patch.object(PublicHTMLCache, 'get', side_effect=AssertionError('network forbidden')):
|
||||||
|
report = c.collect(root/'out', official_snapshots=official, cache=root/'cache', max_new_pages=0)
|
||||||
|
self.assertEqual((report['records'],report['new_pages'],report['reused_pages']), (3,0,3))
|
||||||
|
|
||||||
|
def test_tampered_snapshot_preserves_failure_without_success(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); official, paper = self.snapshots(root); (paper/'4048.html').write_text('tampered')
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.collect(root/'out', official_snapshots=official, paper_snapshots_dir=paper)
|
||||||
|
self.assertTrue((root/'out/failed.json').exists()); self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_structured_math_markup_requires_explicit_review(self):
|
||||||
|
raw = self.bundle['paper_pages']['4048']
|
||||||
|
self.assertIn('<p>Proactive tamper', raw)
|
||||||
|
fields = c.parse_paper(raw.replace('<p>Proactive tamper', '<p>10<sup>6</sup> Proactive tamper', 1), self.program['4048']['url'], self.program['4048'])
|
||||||
|
self.assertIn('structured_abstract_markup_requires_review', fields['review_flags'])
|
||||||
|
self.assertEqual(fields['abstract_markup_tags'], ['sup'])
|
||||||
|
|
||||||
|
def test_fixture_preserves_exact_http_bytes_including_crlf(self):
|
||||||
|
import hashlib
|
||||||
|
for receipt in json.loads(self.bundle['official_manifest'])['sources']:
|
||||||
|
raw = self.bundle['official_pages'][receipt['file']].encode('utf-8')
|
||||||
|
self.assertEqual(len(raw), receipt['bytes'])
|
||||||
|
self.assertEqual(hashlib.sha256(raw).hexdigest(), receipt['sha256'])
|
||||||
|
for receipt in self.bundle['paper_manifest']['records']:
|
||||||
|
raw = self.bundle['paper_pages'][receipt['paper_id']].encode('utf-8')
|
||||||
|
self.assertEqual(hashlib.sha256(raw).hexdigest(), receipt['sha256'])
|
||||||
|
|
||||||
|
def test_real_double_encoded_author_requires_exact_visible_agreement(self):
|
||||||
|
import hashlib
|
||||||
|
sample = self.bundle['encoding_sample']; pid = sample['paper_id']; raw = sample['html']
|
||||||
|
self.assertEqual(hashlib.sha256(raw.encode('utf-8')).hexdigest(), sample['receipt']['sha256'])
|
||||||
|
self.assertEqual(sample['receipt']['sha256'], 'b575318fa569e83eea5e96f937dd60f078fbed482877318013f797c16a02ff09')
|
||||||
|
fields = c.parse_paper(raw, sample['receipt']['url'], self.program[pid])
|
||||||
|
self.assertEqual(fields['authors'], sample['gold']['authors'])
|
||||||
|
self.assertEqual(fields['jsonld_raw_text']['author_names'][0], 'Zi&#x27;ang Lu')
|
||||||
|
self.assertIn('jsonld_display_text_normalized_requires_review', fields['review_flags'])
|
||||||
|
# Real changes to the author or an extra encoding layer remain errors.
|
||||||
|
for replacement in ('Other Author', 'Zi&amp;#x27;ang Lu'):
|
||||||
|
with self.assertRaisesRegex(ValueError, 'authors disagree'):
|
||||||
|
c.parse_paper(raw.replace('Zi&#x27;ang Lu', replacement), sample['receipt']['url'], self.program[pid])
|
||||||
|
|
||||||
|
def test_real_already_agreeing_literal_title_is_not_decoded_again(self):
|
||||||
|
import hashlib
|
||||||
|
sample = self.bundle['literal_title_sample']; pid = sample['paper_id']; raw = sample['html']
|
||||||
|
self.assertEqual(hashlib.sha256(raw.encode('utf-8')).hexdigest(), 'c208525ad5f02b4ca274dff7e24a0392dabdc55bb3620497079a84df81a02a72')
|
||||||
|
fields = c.parse_paper(raw, sample['receipt']['url'], self.program[pid])
|
||||||
|
self.assertIn('&', fields['title'])
|
||||||
|
self.assertEqual(fields['title'], sample['gold']['visible_title'])
|
||||||
|
self.assertEqual(fields['jsonld_raw_text']['name'], sample['gold']['jsonld_title_raw'])
|
||||||
|
self.assertNotEqual(fields['title'], sample['gold']['blind_double_decoded_title'])
|
||||||
|
self.assertNotIn('jsonld_display_text_normalized_requires_review', fields['review_flags'])
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
from datetime import datetime, timezone
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import MagicMock
|
||||||
|
import urllib.error
|
||||||
|
|
||||||
|
from paper_atlas.io.html_cache import PublicHTMLCache, digest, NoRedirect
|
||||||
|
|
||||||
|
URL = 'https://eccv.ecva.net/virtual/2026/poster/4048'
|
||||||
|
|
||||||
|
|
||||||
|
class PublicHTMLCacheTests(unittest.TestCase):
|
||||||
|
def receipt(self, raw):
|
||||||
|
return {'url':URL,'final_url':URL,'http_status':200,'content_type':'text/html; charset=utf-8',
|
||||||
|
'bytes':len(raw),'sha256':digest(raw),'retrieved_at':datetime.now(timezone.utc).isoformat()}
|
||||||
|
|
||||||
|
def test_old_cache_reused_without_network_or_expiry(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0); raw = b'<html>metadata</html>'
|
||||||
|
body, receipt, _ = c.paths(URL); body.write_bytes(raw); receipt.write_text(json.dumps(self.receipt(raw))); os.utime(body,(0,0))
|
||||||
|
c.opener = MagicMock(); self.assertEqual(c.get(URL)[0], raw); c.opener.open.assert_not_called()
|
||||||
|
|
||||||
|
def test_pair_tampering_and_orphan_parts_fail_without_refetch(self):
|
||||||
|
for kind in ('hash','orphan','part'):
|
||||||
|
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0); body, receipt, _ = c.paths(URL); body.write_bytes(b'original')
|
||||||
|
if kind == 'hash': receipt.write_text(json.dumps(self.receipt(b'different')))
|
||||||
|
if kind == 'part': body.unlink(); body.with_suffix('.html.part').write_bytes(b'partial')
|
||||||
|
c.opener = MagicMock()
|
||||||
|
with self.assertRaises(ValueError): c.get(URL)
|
||||||
|
c.opener.open.assert_not_called()
|
||||||
|
|
||||||
|
def test_http_access_failure_sticky_without_automatic_retry(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0); c.opener = MagicMock()
|
||||||
|
c.opener.open.side_effect = urllib.error.HTTPError(URL,403,'Forbidden',{},None)
|
||||||
|
with self.assertRaises(urllib.error.HTTPError): c.get(URL)
|
||||||
|
with self.assertRaisesRegex(ValueError,'Previous access'): c.get(URL)
|
||||||
|
self.assertEqual(c.opener.open.call_count,1)
|
||||||
|
|
||||||
|
def test_redirect_nonhtml_challenge_and_oversize_rejected(self):
|
||||||
|
for kind in ('redirect','type','challenge','size'):
|
||||||
|
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c = PublicHTMLCache(tmp,'https://eccv.ecva.net',limit=100,delay=0); c.opener=MagicMock(); r=c.opener.open.return_value.__enter__.return_value
|
||||||
|
r.status=200; r.geturl.return_value=URL; r.headers={'Content-Type':'text/html'}; r.read.return_value=b'<html>metadata</html>'
|
||||||
|
if kind=='redirect':r.geturl.return_value='https://example.org/login'
|
||||||
|
if kind=='type':r.headers={'Content-Type':'application/pdf'}
|
||||||
|
if kind=='challenge':r.read.return_value=b'<title>Just a moment</title>'
|
||||||
|
if kind=='size':r.read.return_value=b'x'*101
|
||||||
|
with self.assertRaises(ValueError):c.get(URL)
|
||||||
|
self.assertFalse(c.paths(URL)[0].exists())
|
||||||
|
self.assertIsNone(NoRedirect().redirect_request(None,None,None,None,None,None))
|
||||||
|
|
||||||
|
def test_valid_response_has_atomic_body_and_receipt(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c=PublicHTMLCache(tmp,'https://eccv.ecva.net',delay=0);c.opener=MagicMock();r=c.opener.open.return_value.__enter__.return_value;r.status=200;r.geturl.return_value=URL;r.headers={'Content-Type':'text/html','Content-Length':'4'};r.read.return_value=b'data'
|
||||||
|
raw, receipt=c.get(URL);self.assertEqual(raw,b'data');self.assertEqual(receipt['sha256'],digest(raw));self.assertEqual(c.cached(URL),(raw,receipt));self.assertFalse(list(Path(tmp).glob('*.part')))
|
||||||
|
|
||||||
|
def test_foreign_origin_and_cache_symlink_rejected(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c=PublicHTMLCache(tmp,'https://eccv.ecva.net',delay=0)
|
||||||
|
with self.assertRaises(ValueError):c.get('https://example.org/other')
|
||||||
|
body,_,_=c.paths(URL);target=Path(tmp)/'outside';target.write_bytes(b'other');body.symlink_to(target)
|
||||||
|
with self.assertRaises(ValueError):c.get(URL)
|
||||||
|
|
||||||
|
def test_successful_fetch_publishes_receipt_and_reuses_without_refetch(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0)
|
||||||
|
c.opener = MagicMock(); response = c.opener.open.return_value.__enter__.return_value
|
||||||
|
response.status = 200; response.geturl.return_value = URL
|
||||||
|
raw = b'<html>public metadata</html>'
|
||||||
|
response.headers = {'Content-Type':'text/html','Content-Length':str(len(raw))}; response.read.return_value = raw
|
||||||
|
self.assertEqual(c.get(URL)[0], raw); self.assertEqual(c.get(URL)[0], raw)
|
||||||
|
self.assertEqual(c.opener.open.call_count, 1)
|
||||||
|
body, receipt, _ = c.paths(URL)
|
||||||
|
self.assertTrue(body.exists()); self.assertTrue(receipt.exists())
|
||||||
|
self.assertFalse(body.with_suffix('.html.part').exists())
|
||||||
|
self.assertFalse(receipt.with_suffix('.json.part').exists())
|
||||||
|
|
||||||
|
def test_changed_url_is_rejected_before_reading_body(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
c = PublicHTMLCache(tmp, 'https://eccv.ecva.net', delay=0)
|
||||||
|
c.opener = MagicMock(); response = c.opener.open.return_value.__enter__.return_value
|
||||||
|
response.status = 200; response.geturl.return_value = 'https://example.org/login'
|
||||||
|
response.headers = {'Content-Type':'text/html'}
|
||||||
|
with self.assertRaises(ValueError): c.get(URL)
|
||||||
|
response.read.assert_not_called()
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
import gzip
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import icaps2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class ICAPS2026Test(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.bundle = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/icaps2026-source.json.gz').read_bytes()))
|
||||||
|
|
||||||
|
def snapshots(self, root):
|
||||||
|
root.mkdir()
|
||||||
|
for name, html in self.bundle['pages'].items():
|
||||||
|
(root/name).write_text(html)
|
||||||
|
(root/'manifest.json').write_text(json.dumps(self.bundle['manifest']))
|
||||||
|
return root
|
||||||
|
|
||||||
|
def test_full_listing_matches_independently_recorded_literals(self):
|
||||||
|
rows = c.parse_listing(self.bundle['pages']['accepted.html'])
|
||||||
|
self.assertEqual([{k: r[k] for k in ('section', 'title', 'authors_literal')} for r in rows],
|
||||||
|
[{k: r[k] for k in ('section', 'title', 'authors_literal')} for r in self.bundle['expected']])
|
||||||
|
self.assertEqual(sum(r['section'] == c.MAIN for r in rows), 90)
|
||||||
|
self.assertEqual(sum(r['section'] == c.PREVIOUS for r in rows), 10)
|
||||||
|
self.assertTrue(all(len(r['authors']) >= 2 for r in rows))
|
||||||
|
|
||||||
|
def test_incomplete_and_duplicate_listing_rejected(self):
|
||||||
|
html = self.bundle['pages']['accepted.html']
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.parse_listing(html.replace('class=paper>', 'class=omitted>', 1))
|
||||||
|
with self.assertRaisesRegex(ValueError, 'repeated paper title'):
|
||||||
|
c.parse_listing(html.replace(self.bundle['expected'][1]['title'], self.bundle['expected'][0]['title']))
|
||||||
|
|
||||||
|
def test_changed_edition_or_section_rejected(self):
|
||||||
|
html = self.bundle['pages']['accepted.html']
|
||||||
|
for changed in [html.replace('ICAPS 2026</h1>', 'ICAPS 2027</h1>'),
|
||||||
|
html.replace(c.MAIN, 'Workshop Papers')]:
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.parse_listing(changed)
|
||||||
|
|
||||||
|
def test_ambiguous_author_lists_not_guessed(self):
|
||||||
|
for literal in ['Anonymous', 'Alice, and Bob', 'Alice and Bob and Carol', 'Alice, Bob and name@example.org']:
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.authors_from_literal(literal)
|
||||||
|
self.assertEqual(c.authors_from_literal('Augusto B. Corrêa, Simon Dold and Malte Helmert'),
|
||||||
|
['Augusto B. Corrêa', 'Simon Dold', 'Malte Helmert'])
|
||||||
|
|
||||||
|
def test_collection_separates_presentations_and_unresolved_tracks(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); src = self.snapshots(root/'src'); out = root/'out'
|
||||||
|
report = c.collect(out, snapshots=src)
|
||||||
|
self.assertEqual((report['records'], report['previously_published_presentations'], report['missing_abstract_count']), (90, 10, 90))
|
||||||
|
self.assertEqual(report['public_rejected_availability'], 'unknown')
|
||||||
|
rows = [json.loads(l) for l in Path(report['files'][0]['file']).read_text().splitlines()]
|
||||||
|
previous = json.loads((out/report['previously_published_file']).read_text())
|
||||||
|
self.assertFalse(previous['ingestion_ready']); self.assertEqual(previous['count'], 10)
|
||||||
|
self.assertFalse({r['title'] for r in rows} & {r['title'] for r in previous['records']})
|
||||||
|
self.assertTrue(all(r['track'] == 'main_tracks_unclassified' and r['status'] == 'accepted' for r in rows))
|
||||||
|
self.assertTrue(all(not r['identifiers'] and not r['submission_identifiers'] and r['abstract'] is None and not r['tags'] for r in rows))
|
||||||
|
self.assertEqual(len({r['external_id'] for r in rows}), 90)
|
||||||
|
self.assertTrue(all(r['metadata']['source_entry_key']['value'] in r['external_id'] for r in rows))
|
||||||
|
self.assertFalse(report['database_written']); self.assertFalse(report['pdfs_downloaded'])
|
||||||
|
self.assertEqual(report['files'][0]['status'], 'partial')
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||||
|
c.collect(out, snapshots=src)
|
||||||
|
|
||||||
|
def test_tampering_retains_failure_without_success_receipt(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); src = self.snapshots(root/'src')
|
||||||
|
(src/'accepted.html').write_text('tampered')
|
||||||
|
with self.assertRaisesRegex(ValueError, 'hash changed'):
|
||||||
|
c.collect(root/'out', snapshots=src)
|
||||||
|
self.assertTrue((root/'out/failed.json').exists())
|
||||||
|
self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_receipt_redirect_naive_time_and_duplicate_source_rejected(self):
|
||||||
|
for kind in ('redirect', 'time', 'duplicate'):
|
||||||
|
with self.subTest(kind=kind), tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); src = self.snapshots(root/'src'); p = src/'manifest.json'; r = json.loads(p.read_bytes())
|
||||||
|
if kind == 'redirect': r['sources'][0]['final_url'] = 'https://example.org/challenge'
|
||||||
|
if kind == 'time': r['at'] = '2026-11-16T12:00:00'
|
||||||
|
if kind == 'duplicate': r['sources'][1] = r['sources'][0]
|
||||||
|
p.write_text(json.dumps(r))
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.collect(root/'out', snapshots=src)
|
||||||
|
self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_snapshot_symlink_rejected(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); src = self.snapshots(root/'src'); p = src/'accepted.html'; raw = p.read_bytes(); p.unlink(); target = root/'external.html'; target.write_bytes(raw); p.symlink_to(target)
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Unsafe'):
|
||||||
|
c.collect(root/'out', snapshots=src)
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
import copy
|
||||||
|
import gzip
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import icaps_registry2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class ICAPSRegistry2026Test(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.raw = gzip.decompress((Path(__file__).parent/'fixtures/icaps2026-crossref.json.gz').read_bytes())
|
||||||
|
cls.snapshot = json.loads(cls.raw)
|
||||||
|
|
||||||
|
def test_exact_edition_not_publication_year_alone(self):
|
||||||
|
papers, excluded = c.registry_papers(self.snapshot)
|
||||||
|
self.assertEqual(len(papers), 90); self.assertEqual(len(excluded), 1)
|
||||||
|
self.assertTrue(all(p['original']['volume'] == '36' for p in papers))
|
||||||
|
self.assertEqual(excluded, [r['DOI'] for r in self.snapshot['records'] if r['volume'] == '35'])
|
||||||
|
self.assertTrue(all(p['abstract'] and p['authors'] for p in papers))
|
||||||
|
|
||||||
|
def test_full_collection_keeps_registered_versions_and_publication_identity(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); src = root/'src'; src.mkdir(); (src/'crossref-filtered.json').write_bytes(self.raw)
|
||||||
|
report = c.collect(root/'out', snapshots=src)
|
||||||
|
self.assertEqual((report['records'], report['missing_abstract_count']), (90, 0))
|
||||||
|
self.assertFalse(report['database_written']); self.assertFalse(report['pdfs_downloaded'])
|
||||||
|
self.assertEqual(report['public_rejected_availability'], 'unknown')
|
||||||
|
manifest, = report['files']; self.assertEqual(manifest['status'], 'partial')
|
||||||
|
rows = [json.loads(l) for l in Path(manifest['file']).read_text().splitlines()]
|
||||||
|
original = {r['DOI']: r for r in self.snapshot['records'] if r['volume'] == '36'}
|
||||||
|
self.assertEqual({r['external_id'] for r in rows}, set(original))
|
||||||
|
for r in rows:
|
||||||
|
doi = r['external_id']; o = original[doi]
|
||||||
|
self.assertEqual(r['metadata']['registry_fields'], o)
|
||||||
|
self.assertEqual(r['title'], o['title'][0])
|
||||||
|
self.assertEqual(r['authors'], [a['given']+' '+a['family'] for a in o['authors']])
|
||||||
|
self.assertEqual(r['identifiers'], {'doi': doi})
|
||||||
|
self.assertEqual(r['submission_identifiers'], [{'scheme': 'publication_doi', 'value': doi}])
|
||||||
|
self.assertFalse(r['tags']); self.assertFalse(r['metadata']['original_decision_signature_verified'])
|
||||||
|
self.assertEqual(r['track'], 'main_tracks_unclassified')
|
||||||
|
self.assertEqual(r['metadata']['publication_date'], '2026-06-08')
|
||||||
|
changed = next(r for r in rows if r['external_id'] == '10.1609/icaps.v36i1.42860')
|
||||||
|
self.assertEqual(changed['title'], 'Analyzing Planner Design Trade-Offs for MAPF Under ADG-Based Realistic Execution')
|
||||||
|
self.assertIn('Stephen F. Smith', changed['authors'])
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||||
|
c.collect(root/'out', snapshots=src)
|
||||||
|
|
||||||
|
def test_math_markup_is_not_flattened_as_plain_numbers(self):
|
||||||
|
self.assertEqual(c.abstract_text('<jats:p>A < B and C & D.\n Next line.</jats:p>'), 'A < B and C & D. Next line.')
|
||||||
|
for raw in ['<jats:p>10<sup>6</sup></jats:p>', '<jats:p>x</jats:p><jats:p>y</jats:p>', '<jats:p> </jats:p>', None]:
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.abstract_text(raw)
|
||||||
|
|
||||||
|
def test_invalid_or_repeated_doi_and_edition_rejected(self):
|
||||||
|
for kind in ('doi', 'repeat', 'volume', 'container', 'year'):
|
||||||
|
with self.subTest(kind=kind):
|
||||||
|
s = copy.deepcopy(self.snapshot); rows = [r for r in s['records'] if r['volume'] == '36']; r = rows[0]
|
||||||
|
if kind == 'doi': r['DOI'] = '10.9999/foreign'
|
||||||
|
if kind == 'repeat': rows[1]['DOI'] = r['DOI']
|
||||||
|
if kind == 'volume': r['volume'] = '37'
|
||||||
|
if kind == 'container': r['container-title'] = ['Other Conference']
|
||||||
|
if kind == 'year': r['published']['date-parts'] = [[2025, 6, 8]]
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.registry_papers(s)
|
||||||
|
|
||||||
|
def test_missing_authors_and_changed_order_evidence_rejected(self):
|
||||||
|
for kind in ('missing', 'sequence', 'profile'):
|
||||||
|
s = copy.deepcopy(self.snapshot); r = next(r for r in s['records'] if r['volume'] == '36')
|
||||||
|
if kind == 'missing': r['authors'][0].pop('family')
|
||||||
|
if kind == 'sequence': r['authors'][0]['sequence'] = 'additional'
|
||||||
|
if kind == 'profile': r['authors'][0]['ORCID'] = 'unreviewed-profile'
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.registry_papers(s)
|
||||||
|
|
||||||
|
def test_receipt_scope_and_timezone_rejected(self):
|
||||||
|
for key, value in [('at', '2026-11-17T00:00:00'), ('url', 'https://example.org/works'), ('raw_response_sha256', 'other')]:
|
||||||
|
s = copy.deepcopy(self.snapshot); s[key] = value
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.registry_papers(s)
|
||||||
|
|
||||||
|
def test_tamper_failure_retained_and_success_not_published(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); src = root/'src'; src.mkdir(); (src/'crossref-filtered.json').write_bytes(self.raw+b' ')
|
||||||
|
with self.assertRaisesRegex(ValueError, 'hash changed'):
|
||||||
|
c.collect(root/'out', snapshots=src)
|
||||||
|
self.assertTrue((root/'out/failed.json').exists())
|
||||||
|
self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_symlink_rejected(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); src = root/'src'; src.mkdir(); (root/'data.json').write_bytes(self.raw); (src/'crossref-filtered.json').symlink_to(root/'data.json')
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Unsafe'):
|
||||||
|
c.collect(root/'out', snapshots=src)
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
import base64
|
||||||
|
import copy
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import icassp2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class ICASSP2026Test(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/icassp2026-source.json.gz').read_bytes()))
|
||||||
|
cls.files = {name: base64.b64decode(body) for name, body in cls.fixture['files'].items()}
|
||||||
|
cls.items = []
|
||||||
|
cls.manifest = json.loads(cls.files[c.SNAPSHOT+'/manifest.json'])
|
||||||
|
for receipt in cls.manifest['pages']:
|
||||||
|
cls.items.extend((r, receipt) for r in json.loads(cls.files[c.SNAPSHOT+'/'+receipt['file']])['message']['items'])
|
||||||
|
|
||||||
|
def materialize(self, root):
|
||||||
|
for name, body in self.files.items():
|
||||||
|
path = root/name; path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(body)
|
||||||
|
|
||||||
|
def test_full_census_matches_independent_gold_and_preserves_source(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); src = root/'src'; self.materialize(src)
|
||||||
|
report = c.collect(root/'out', snapshots=src)
|
||||||
|
self.assertEqual((report['records'],report['missing_abstract_count'],report['single_name_author_fields']), (4582,4582,23))
|
||||||
|
self.assertEqual(len(report['excluded_nonpaper_items']),7); self.assertEqual(len(report['excluded_other_conference_dois']),56)
|
||||||
|
self.assertFalse(report['database_written']); self.assertFalse(report['pdfs_downloaded']); self.assertEqual(report['network_requests'],0)
|
||||||
|
self.assertEqual(report['approved_for_import'],0); self.assertEqual(report['public_rejected_availability'],'unknown')
|
||||||
|
manifest, = report['files']; path = Path(manifest['file']); raw = path.read_bytes()
|
||||||
|
self.assertEqual(hashlib.sha256(raw).hexdigest(),manifest['file_sha256']); self.assertEqual(manifest['status'],'partial')
|
||||||
|
rows = [json.loads(l) for l in raw.splitlines()]
|
||||||
|
actual = [{'doi':r['external_id'],'title':r['title'],'authors':r['authors']} for r in rows]
|
||||||
|
gold = sorted(self.fixture['independent_gold'],key=lambda r:r['doi'])
|
||||||
|
self.assertEqual(actual,gold)
|
||||||
|
original = {r['DOI']:r for r,_ in self.items}
|
||||||
|
evidence_hashes = {str(p.relative_to(root/'out')):hashlib.sha256(p.read_bytes()).hexdigest() for p in (root/'out/evidence').iterdir()}
|
||||||
|
for r in rows:
|
||||||
|
doi = r['external_id']; self.assertEqual(r['metadata']['registry_fields'],original[doi])
|
||||||
|
self.assertEqual(r['identifiers'],{'doi':doi}); self.assertEqual(r['submission_identifiers'],[{'scheme':'publication_doi','value':doi}])
|
||||||
|
self.assertEqual(r['track'],'proceedings_unclassified'); self.assertFalse(r['tags']); self.assertFalse(r['abstract'])
|
||||||
|
self.assertFalse(r['metadata']['original_decision_signature_verified'])
|
||||||
|
evidence = r['metadata']['source_evidence']
|
||||||
|
self.assertEqual(evidence_hashes[evidence['file']],evidence['sha256'])
|
||||||
|
self.assertIn('Fareedha',next(r for r in rows if r['external_id'].endswith('.11461474'))['authors'])
|
||||||
|
with self.assertRaisesRegex(ValueError,'Output exists'):
|
||||||
|
c.collect(root/'out',snapshots=src)
|
||||||
|
|
||||||
|
def test_wrong_conference_doi_date_and_unexpected_abstract_rejected(self):
|
||||||
|
for kind in ('container','event','doi','date','author','sequence','abstract'):
|
||||||
|
with self.subTest(kind=kind):
|
||||||
|
items = copy.deepcopy(self.items); r = next(r for r,_ in items if r.get('author') and r['DOI'].startswith(c.PREFIX))
|
||||||
|
if kind=='container': r['container-title']=['Foreign']
|
||||||
|
if kind=='event': r['event']['name']='Other'
|
||||||
|
if kind=='doi': r['DOI']='10.1109/foreign.2026.1'
|
||||||
|
if kind=='date': r['published']['date-parts']=[[2025,5,3]]
|
||||||
|
if kind=='author': r['author']=[]
|
||||||
|
if kind=='sequence': r['author'][0]['sequence']='additional'
|
||||||
|
if kind=='abstract': r['abstract']='new source version'
|
||||||
|
with self.assertRaises(ValueError): c.registry_papers(items)
|
||||||
|
|
||||||
|
def test_nonpaper_exclusion_requires_exact_reviewed_evidence(self):
|
||||||
|
for kind in ('title','page','author'):
|
||||||
|
items = copy.deepcopy(self.items); r = next(r for r,_ in items if r['DOI'] in c.NONPAPERS)
|
||||||
|
if kind=='title': r['title']=['A research paper']
|
||||||
|
if kind=='page': r['page']='1-5'
|
||||||
|
if kind=='author': r['author']=[{'family':'Smith'}]
|
||||||
|
with self.assertRaisesRegex(ValueError,'nonpaper'): c.registry_papers(items)
|
||||||
|
|
||||||
|
def test_duplicate_and_incomplete_census_rejected(self):
|
||||||
|
with self.assertRaisesRegex(ValueError,'repeated'): c.registry_papers(self.items+[self.items[0]])
|
||||||
|
with self.assertRaisesRegex(ValueError,'partition'): c.registry_papers(self.items[:-1])
|
||||||
|
|
||||||
|
def test_corruption_and_partial_input_preserve_failure_without_success(self):
|
||||||
|
for kind in ('manifest','body','missing'):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp); src=root/'src'; self.materialize(src)
|
||||||
|
path=src/c.SNAPSHOT/('manifest.json' if kind=='manifest' else 'page-002.json')
|
||||||
|
if kind=='missing': path.unlink()
|
||||||
|
else: path.write_bytes(path.read_bytes()+b' ')
|
||||||
|
with self.assertRaises((ValueError,FileNotFoundError)): c.collect(root/'out',snapshots=src)
|
||||||
|
self.assertTrue((root/'out/failed.json').exists()); self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_symlink_page_and_snapshot_rejected(self):
|
||||||
|
for kind in ('page','directory'):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp); src=root/'src'; self.materialize(src)
|
||||||
|
path=src/c.SNAPSHOT/('page-001.json' if kind=='page' else '')
|
||||||
|
renamed=path.with_name(path.name+'-original'); path.rename(renamed); path.symlink_to(renamed,target_is_directory=kind=='directory')
|
||||||
|
with self.assertRaisesRegex(ValueError,'Unsafe'): c.collect(root/'out',snapshots=src)
|
||||||
@@ -0,0 +1,113 @@
|
|||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import icdm2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class ICDM2026Tests(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/icdm2026-source.json.gz').read_bytes()))
|
||||||
|
cls.raw = cls.fixture['files'][c.SNAPSHOT+'/icdm-detailed-sessions.html'].encode()
|
||||||
|
cls.gold = cls.fixture['independent_inventory']
|
||||||
|
|
||||||
|
def snapshots(self, root):
|
||||||
|
for name, text in self.fixture['files'].items():
|
||||||
|
path = root/name; path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(text.encode())
|
||||||
|
|
||||||
|
def test_all_rows_match_separately_parsed_frozen_inventory(self):
|
||||||
|
result = c.parse_program(self.raw)
|
||||||
|
self.assertEqual(len(result), len(self.gold['entries']))
|
||||||
|
for position, (row, gold) in enumerate(zip(result, self.gold['entries']), 1):
|
||||||
|
for field in ('paper_id_literal', 'title', 'authors_literal', 'source_reported_authors', 'time_literal'):
|
||||||
|
self.assertEqual(row[field], gold[field])
|
||||||
|
self.assertEqual(row['source_position'], position)
|
||||||
|
for field, value in gold['session'].items():
|
||||||
|
self.assertEqual(row['session'][field], value)
|
||||||
|
|
||||||
|
def test_instantiable_collection_retains_scope_fields_and_source_bytes(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); self.snapshots(root/'src')
|
||||||
|
report = c.collect(root/'out', snapshots=root/'src')
|
||||||
|
self.assertEqual(report['track_counts'], self.gold['track_counts_literal'])
|
||||||
|
self.assertEqual(report['presentation_counts'], self.gold['presentation_counts_literal'])
|
||||||
|
self.assertEqual((report['records'], report['missing_abstract_count']), (324, 324))
|
||||||
|
self.assertFalse(report['database_written']); self.assertEqual(report['network_requests'], 0)
|
||||||
|
self.assertEqual(report['public_rejected_metadata'], 'unknown_no_public_export_verified')
|
||||||
|
all_rows = []
|
||||||
|
for manifest in report['files']:
|
||||||
|
path = Path(manifest['file']); raw = path.read_bytes()
|
||||||
|
self.assertEqual(hashlib.sha256(raw).hexdigest(), manifest['file_sha256'])
|
||||||
|
self.assertEqual(manifest['status'], 'partial')
|
||||||
|
rows = [json.loads(line) for line in raw.splitlines()]
|
||||||
|
self.assertEqual(len(rows), manifest['missing_abstract_count'])
|
||||||
|
for row in rows:
|
||||||
|
self.assertEqual((row['venue'], row['year'], row['track']), ('ICDM', 2026, manifest['track']))
|
||||||
|
self.assertIsNone(row['abstract']); self.assertIsNone(row['metadata']['paper_type'])
|
||||||
|
self.assertEqual(row['identifiers'], {}); self.assertEqual(row['submission_identifiers'], [])
|
||||||
|
self.assertFalse(row['metadata']['original_decision_signature_verified'])
|
||||||
|
self.assertEqual(row['external_id'], '2026/'+row['metadata']['program_fields']['paper_id_literal'])
|
||||||
|
self.assertTrue(row['url'].startswith(c.PROGRAM+'#paper-DM'))
|
||||||
|
self.assertTrue(all(t['namespace'].startswith(c.SOURCE+':') for t in row['tags']))
|
||||||
|
self.assertNotIn('doi', row['metadata']); self.assertNotIn('decision_date', row)
|
||||||
|
all_rows.extend(rows)
|
||||||
|
self.assertEqual(len({r['external_id'] for r in all_rows}), 324)
|
||||||
|
for item in report['evidence']:
|
||||||
|
raw = (root/'out'/item['output_file']).read_bytes()
|
||||||
|
self.assertEqual(raw, self.fixture['files'][c.SNAPSHOT+'/'+item['file']].encode())
|
||||||
|
self.assertTrue((root/'out/collection-report.json').is_file())
|
||||||
|
self.assertFalse((root/'out/collection-report.json.part').exists())
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||||
|
c.collect(root/'out', snapshots=root/'src')
|
||||||
|
|
||||||
|
def test_stale_or_different_conference_rejected(self):
|
||||||
|
for old, new in [(b'IEEE ICDM 2026', b'IEEE ICDM 2025'),
|
||||||
|
(b'In-person session presentation list.', b'Workshop presentation list.')]:
|
||||||
|
with self.subTest(old=old), self.assertRaises(ValueError):
|
||||||
|
c.parse_program(self.raw.replace(old, new))
|
||||||
|
|
||||||
|
def test_track_attribute_must_agree_with_visible_label(self):
|
||||||
|
changed = self.raw.replace(b'data-track="research"', b'data-track="applied"', 1)
|
||||||
|
with self.assertRaisesRegex(ValueError, 'track'):
|
||||||
|
c.parse_program(changed)
|
||||||
|
|
||||||
|
def test_presentation_attribute_is_not_inferred_from_session_id(self):
|
||||||
|
changed = self.raw.replace(b'data-format="long"', b'data-format="short"', 1)
|
||||||
|
with self.assertRaisesRegex(ValueError, 'presentation'):
|
||||||
|
c.parse_program(changed)
|
||||||
|
|
||||||
|
def test_repeated_paper_and_source_locator_disagreement_rejected(self):
|
||||||
|
for old, new in [(b'DM1278', b'DM827'), (b'id="paper-DM827"', b'id="paper-DM99999"')]:
|
||||||
|
with self.subTest(old=old), self.assertRaises(ValueError):
|
||||||
|
c.parse_program(self.raw.replace(old, new))
|
||||||
|
|
||||||
|
def test_lost_session_or_changed_session_counts_rejected(self):
|
||||||
|
for old, new in [(b'data-session-id="L1"', b'data-session-id="L2"'),
|
||||||
|
(b'Long \xc2\xb7 7 papers', b'Long \xc2\xb7 6 papers')]:
|
||||||
|
with self.subTest(old=old), self.assertRaises(ValueError):
|
||||||
|
c.parse_program(self.raw.replace(old, new, 1))
|
||||||
|
|
||||||
|
def test_empty_author_is_not_silently_dropped(self):
|
||||||
|
changed = self.raw.replace(b'Qishuang Fu; Andreas Deppeler;', b'Qishuang Fu;;', 1)
|
||||||
|
with self.assertRaisesRegex(ValueError, 'author'):
|
||||||
|
c.parse_program(changed)
|
||||||
|
|
||||||
|
def test_snapshot_manifest_and_body_corruption_rejected_before_output(self):
|
||||||
|
for name in ('manifest.json', 'icdm-detailed-sessions.html', 'icdm-research-policy.html', 'icdm-applied-policy.html'):
|
||||||
|
with self.subTest(name=name), tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); self.snapshots(root/'src'); path = root/'src'/c.SNAPSHOT/name
|
||||||
|
path.write_bytes(path.read_bytes()+b' ')
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
c.collect(root/'out', snapshots=root/'src')
|
||||||
|
self.assertFalse((root/'out').exists())
|
||||||
|
|
||||||
|
def test_snapshot_source_symlink_rejected(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); self.snapshots(root/'src'); path = root/'src'/c.SNAPSHOT/'icdm-detailed-sessions.html'
|
||||||
|
real = root/'real.html'; path.rename(real); path.symlink_to(real)
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Unsafe source'):
|
||||||
|
c.collect(root/'out', snapshots=root/'src')
|
||||||
@@ -0,0 +1,92 @@
|
|||||||
|
import base64
|
||||||
|
import copy
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import ismir2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class ISMIR2026Test(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture=json.loads(gzip.decompress((Path(__file__).parent/'fixtures/ismir2026-source.json.gz').read_bytes()))
|
||||||
|
cls.files={name:base64.b64decode(raw) for name,raw in cls.fixture['files'].items()}
|
||||||
|
cls.program=json.loads(cls.files['ismir2026-json-research-v1/papers.json'])
|
||||||
|
cls.accepted=cls.files['remaining2026-listings-research-v2/ismir-accepted.html']
|
||||||
|
|
||||||
|
def materialize(self,root):
|
||||||
|
for name,raw in self.files.items():
|
||||||
|
p=root/name;p.parent.mkdir(parents=True,exist_ok=True);p.write_bytes(raw)
|
||||||
|
|
||||||
|
def test140_source_versions_and_review_optins_against_independent_gold(self):
|
||||||
|
listing=c.accepted_listing(self.accepted);observations=c.program_observations(self.program,listing)
|
||||||
|
self.assertEqual(len(observations),140)
|
||||||
|
for observation,gold in zip(observations,self.fixture['independent_gold']):
|
||||||
|
r=observation['original'];v=r['content'];official=observation['accepted_listing']
|
||||||
|
self.assertEqual((r['id'],r['forum'],v['title'],v['authors'],v['abstract']),(gold['program_id'],gold['forum_literal'],gold['title'],gold['authors'],gold['abstract']))
|
||||||
|
self.assertEqual(official,{'title':gold['official_title'],'authors_literal':gold['official_authors_literal'],'author_names':gold['official_author_names']})
|
||||||
|
self.assertEqual(v['session'],gold['session_literal']);self.assertEqual(v['paper_presentation'],gold['presentation_literal']);self.assertEqual(v['keywords'],gold['keywords'])
|
||||||
|
self.assertEqual(sum('program_and_accepted_title_literal_versions_differ' in r['review_flags'] for r in observations),2)
|
||||||
|
self.assertEqual({r['original']['id'] for r in observations if 'program_and_accepted_author_sequence_versions_differ' in r['review_flags']},{'60','126','433'})
|
||||||
|
self.assertEqual(sum(r['original']['reviews']['publish_reviews']=='Yes' for r in observations),101)
|
||||||
|
|
||||||
|
def test_full_collection_keeps_abstract_bytes_review_text_tags_and_no_inferred_ids(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);src=root/'src';self.materialize(src);out=root/'out';report=c.collect(out,snapshots=src)
|
||||||
|
self.assertEqual((report['records'],report['missing_abstract_count'],report['review_optin_yes'],report['review_optin_no']),(140,0,101,39))
|
||||||
|
self.assertFalse(report['database_written']);self.assertFalse(report['pdfs_downloaded']);self.assertEqual(report['network_requests'],0);self.assertEqual(report['approved_for_import'],0)
|
||||||
|
self.assertEqual(report['public_rejected_availability'],'unknown_accepted_program_reviews_are_not_a_rejected_export')
|
||||||
|
manifest,=report['files'];raw=Path(manifest['file']).read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),manifest['file_sha256']);self.assertEqual(manifest['status'],'partial')
|
||||||
|
rows=[json.loads(l) for l in raw.splitlines()];self.assertEqual(len(rows),140)
|
||||||
|
for row,original,gold in zip(rows,self.program,self.fixture['independent_gold']):
|
||||||
|
self.assertEqual(row['metadata']['program_entry'],original);self.assertEqual(row['abstract'],gold['abstract']);self.assertEqual(row['title'],gold['title']);self.assertEqual(row['authors'],gold['authors'])
|
||||||
|
self.assertEqual(row['external_id'],'2026/'+gold['program_id']);self.assertEqual(row['identifiers'],{});self.assertEqual(row['submission_identifiers'],[])
|
||||||
|
self.assertEqual(row['status'],'accepted');self.assertEqual(row['track'],'program_unclassified');self.assertIsNone(row['metadata']['paper_type']);self.assertFalse(row['metadata']['original_decision_signature_verified'])
|
||||||
|
tags=[{'namespace':'ismir_program:presentation','value':gold['presentation_literal']}]+[{'namespace':'ismir_program:keyword','value':v} for v in dict.fromkeys(' '.join(k.split()) for k in gold['keywords'] if k.strip())]
|
||||||
|
self.assertEqual(row['tags'],tags);self.assertTrue(all(t['value'] for t in row['tags']))
|
||||||
|
if original['reviews']['publish_reviews']=='No':self.assertFalse(any(v for k,v in row['metadata']['program_entry']['reviews'].items() if k!='publish_reviews'))
|
||||||
|
for name,body in self.files.items():self.assertEqual((out/'evidence'/name).read_bytes(),body)
|
||||||
|
with self.assertRaisesRegex(ValueError,'Output exists'):c.collect(out,snapshots=src)
|
||||||
|
|
||||||
|
def test_duplicate_and_malformed_locator_or_incomplete_census_rejected(self):
|
||||||
|
listing=c.accepted_listing(self.accepted)
|
||||||
|
for kind in ('repeat','forum','non_numeric','missing_row'):
|
||||||
|
rows=copy.deepcopy(self.program)
|
||||||
|
if kind=='repeat':rows[1]['id']=rows[0]['id'];rows[1]['forum']=rows[0]['forum']
|
||||||
|
if kind=='forum':rows[0]['forum']='OriginalOpenReviewID'
|
||||||
|
if kind=='non_numeric':rows[0]['id']='abc';rows[0]['forum']='abc'
|
||||||
|
if kind=='missing_row':rows.pop()
|
||||||
|
with self.subTest(kind=kind),self.assertRaises(ValueError):c.program_observations(rows,listing)
|
||||||
|
|
||||||
|
def test_empty_fields_foreign_title_and_session_disagree_rejected(self):
|
||||||
|
listing=c.accepted_listing(self.accepted)
|
||||||
|
for field,value in [('authors',[]),('abstract',''),('title','A foreign paper'),('session',['99']),('keywords',[42])]:
|
||||||
|
rows=copy.deepcopy(self.program);rows[0]['content'][field]=value
|
||||||
|
with self.subTest(field=field),self.assertRaises(ValueError):c.program_observations(rows,listing)
|
||||||
|
|
||||||
|
def test_source_optin_no_cannot_publish_review_text(self):
|
||||||
|
rows=copy.deepcopy(self.program);r=next(r for r in rows if r['reviews']['publish_reviews']=='No');r['reviews']['meta_review']='Do not infer a negative decision'
|
||||||
|
with self.assertRaisesRegex(ValueError,'preference'):c.program_observations(rows,c.accepted_listing(self.accepted))
|
||||||
|
|
||||||
|
def test_parenthesized_affiliation_semicolons_do_not_split_people(self):
|
||||||
|
self.assertEqual(c.author_names('A Person (University A; Lab (X)); B Person (Institute B)'),['A Person','B Person'])
|
||||||
|
for raw in ['A Person (Broken','; A Person (Lab)','A Person )']:
|
||||||
|
with self.assertRaises(ValueError):c.author_names(raw)
|
||||||
|
|
||||||
|
def test_ambiguous_official_title_is_not_fused(self):
|
||||||
|
source=self.accepted.replace(b'FusID: Modality-Fused Semantic IDs for Generative Music Recommendation',b'Off-Manifold Robustness in Synthesizer Inversion with Joint Distribution Flow Matching',1)
|
||||||
|
with self.assertRaisesRegex(ValueError,'ambiguous'):c.accepted_listing(source)
|
||||||
|
|
||||||
|
def test_tamper_missing_file_and_symlink_fail_without_success_report(self):
|
||||||
|
for kind in ('manifest','body','missing','symlink'):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);src=root/'src';self.materialize(src);path=src/'ismir2026-json-research-v1'/('manifest.json' if kind=='manifest' else 'papers.json')
|
||||||
|
if kind=='missing':path.unlink()
|
||||||
|
elif kind=='symlink':other=root/'actual.json';path.rename(other);path.symlink_to(other)
|
||||||
|
else:path.write_bytes(path.read_bytes()+b' ')
|
||||||
|
with self.subTest(kind=kind),self.assertRaises((ValueError,FileNotFoundError)):c.collect(root/'out',snapshots=src)
|
||||||
|
self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
@@ -0,0 +1,146 @@
|
|||||||
|
import copy
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import mm2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class MM2026Test(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/mm2026-source.json.gz').read_bytes()))
|
||||||
|
cls.files = {name:body.encode('utf-8') for name,body in cls.fixture['files'].items()}
|
||||||
|
cls.raw = cls.files[c.SNAPSHOT+'/program-data.json']
|
||||||
|
cls.data = json.loads(cls.raw)
|
||||||
|
cls.gold = cls.fixture['independent_inventory']
|
||||||
|
|
||||||
|
def materialize(self, root):
|
||||||
|
for name,body in self.files.items():
|
||||||
|
path = root/name; path.parent.mkdir(parents=True,exist_ok=True); path.write_bytes(body)
|
||||||
|
|
||||||
|
def parse_changed(self, data):
|
||||||
|
return c.parse_program(json.dumps(data,ensure_ascii=False).encode())
|
||||||
|
|
||||||
|
def test_all2143_ids_and4326_locations_against_independent_inventory(self):
|
||||||
|
rows,sessions = c.parse_program(self.raw)
|
||||||
|
self.assertEqual(len(rows),2143); self.assertEqual(len(sessions),157)
|
||||||
|
self.assertEqual(sum(len(r['contexts']) for r in rows),4326)
|
||||||
|
actual = {r['paper_id']:r for r in rows}
|
||||||
|
self.assertEqual(set(actual),{g['paper_id'] for g in self.gold['inventory']})
|
||||||
|
for gold in self.gold['inventory']:
|
||||||
|
r = actual[gold['paper_id']]
|
||||||
|
self.assertEqual((r['title'],r['authors'],r['track'],r['affiliations']),
|
||||||
|
(gold['title'],gold['authors'],gold['track_literal'],gold['affiliations']))
|
||||||
|
self.assertEqual(hashlib.sha256(r['abstract'].encode()).hexdigest(),gold['abstract_sha256'])
|
||||||
|
self.assertEqual([x['source_pointer'] for x in r['contexts']],gold['locations'])
|
||||||
|
|
||||||
|
def test_every_presentation_and_session_context_is_preserved(self):
|
||||||
|
rows,sessions = c.parse_program(self.raw)
|
||||||
|
for r in rows:
|
||||||
|
for context in r['contexts']:
|
||||||
|
node = self.data
|
||||||
|
for key in context['source_pointer'].split('/')[1:]:
|
||||||
|
node = node[int(key)] if isinstance(node,list) else node[key]
|
||||||
|
self.assertEqual(context['presentation'],{k:node[k] for k in c.PRESENTATION if k in node})
|
||||||
|
self.assertEqual(r['paper_id'],node['paper_id'])
|
||||||
|
for i,session in enumerate(sessions):
|
||||||
|
original = self.data['entries'][i]
|
||||||
|
self.assertEqual(session['source_pointer'],'/entries/'+str(i))
|
||||||
|
self.assertEqual(session['track'],'')
|
||||||
|
for key in original.keys()-{'papers','subsessions'}:
|
||||||
|
self.assertEqual(session[key],original[key])
|
||||||
|
for j,sub in enumerate(original.get('subsessions') or []):
|
||||||
|
saved = session['subsessions'][j]
|
||||||
|
self.assertEqual(saved['title'],sub['title'])
|
||||||
|
expected = [x for x in sub.get('items') or [] if 'paper_id' not in x]
|
||||||
|
self.assertEqual([x['fields'] for x in saved['nonpaper_items']],expected)
|
||||||
|
|
||||||
|
def test_all_tracks_retained_without_publication_or_negative_inference(self):
|
||||||
|
rows,_ = c.parse_program(self.raw)
|
||||||
|
self.assertEqual({r['track'] for r in rows},set(self.gold['unique_id_track_counts']))
|
||||||
|
for r in rows:
|
||||||
|
self.assertFalse(r['approved_for_import']); self.assertFalse(r['final_decision_verified'])
|
||||||
|
self.assertFalse(r['publication_identity_verified'])
|
||||||
|
self.assertEqual(r['source_locator'],{'namespace':'mm_program:program_id','value':r['paper_id']})
|
||||||
|
self.assertEqual(r['tags'],[{'namespace':'mm_program:track','value':r['track']}])
|
||||||
|
self.assertEqual(r['observed_status'],'listed_in_official_program')
|
||||||
|
self.assertTrue({'status','identifiers','submission_identifiers','paper_type','emails'}.isdisjoint(r))
|
||||||
|
missing = [r for r in rows if not r['authors']]
|
||||||
|
self.assertEqual({r['paper_id'] for r in missing},{'mfp4971','mfp7801'})
|
||||||
|
self.assertTrue(all(r['missing_fields']==['authors','abstract'] for r in missing))
|
||||||
|
self.assertEqual(sum(not r['abstract'] for r in rows),216)
|
||||||
|
|
||||||
|
def test_collect_evidence_hashes_and_no_contact_arrays_in_normalized_output(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);self.materialize(root/'src')
|
||||||
|
report=c.collect(root/'out',snapshots=root/'src')
|
||||||
|
self.assertEqual((report['records'],report['program_observations'],report['extra_presentation_observations']),(2143,4326,2183))
|
||||||
|
self.assertFalse(report['database_written']);self.assertFalse(report['pdfs_downloaded'])
|
||||||
|
self.assertEqual(report['network_requests'],0);self.assertEqual(report['approved_for_import'],0)
|
||||||
|
self.assertEqual(report['public_negative_access']['http_statuses'],[403]*4)
|
||||||
|
self.assertEqual(report['track_counts'],self.gold['unique_id_track_counts'])
|
||||||
|
for name,body in self.files.items():
|
||||||
|
self.assertEqual((root/'out/evidence'/Path(name).name).read_bytes(),body)
|
||||||
|
n=0
|
||||||
|
for item in report['files']:
|
||||||
|
raw=Path(item['file']).read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),item['file_sha256'])
|
||||||
|
self.assertNotIn(b'"emails"',raw);self.assertEqual(len(raw.splitlines()),item['count']);n+=item['count']
|
||||||
|
self.assertEqual(item['format'],'mm_program_observation_v1_not_ingestion_record')
|
||||||
|
self.assertEqual(n,2143)
|
||||||
|
raw=(root/'out/sessions.jsonl').read_bytes()
|
||||||
|
self.assertEqual(hashlib.sha256(raw).hexdigest(),report['sessions_file_sha256']);self.assertNotIn(b'"emails"',raw)
|
||||||
|
with self.assertRaisesRegex(ValueError,'Output exists'):c.collect(root/'out',snapshots=root/'src')
|
||||||
|
|
||||||
|
def test_conflicting_core_version_is_not_silently_merged(self):
|
||||||
|
data=copy.deepcopy(self.data)
|
||||||
|
first=data['entries'][0]['papers'][0]
|
||||||
|
data['entries'][0]['papers'].append({**first,'abstract':'conflicting abstract'})
|
||||||
|
with self.assertRaisesRegex(ValueError,'Conflicting core'):self.parse_changed(data)
|
||||||
|
|
||||||
|
def test_unknown_nesting_cannot_silently_lose_observations(self):
|
||||||
|
data=copy.deepcopy(self.data)
|
||||||
|
moved=data['entries'][0]['papers'].pop()
|
||||||
|
data['entries'][0]['unreviewed_bucket']=[moved]
|
||||||
|
with self.assertRaisesRegex(ValueError,'Unhandled'):self.parse_changed(data)
|
||||||
|
|
||||||
|
def test_track_drift_unknown_fields_or_missing_title_require_review(self):
|
||||||
|
for kind in ('outer_track','paper_track','field','title','authors'):
|
||||||
|
data=copy.deepcopy(self.data);paper=data['entries'][0]['papers'][0]
|
||||||
|
if kind=='outer_track':data['entries'][0]['track']='Main Track'
|
||||||
|
elif kind=='paper_track':paper['track']='New Track'
|
||||||
|
elif kind=='field':paper['doi']='unreviewed'
|
||||||
|
elif kind=='title':paper['title']=''
|
||||||
|
else:paper['authors']='Unparsed author string'
|
||||||
|
with self.subTest(kind=kind),self.assertRaises(ValueError):self.parse_changed(data)
|
||||||
|
|
||||||
|
def test_edition_or_census_change_requires_review(self):
|
||||||
|
for kind in ('edition','count'):
|
||||||
|
data=copy.deepcopy(self.data)
|
||||||
|
if kind=='edition':data['generated_at']='2025-11-14T18:36:26.860677+00:00'
|
||||||
|
else:data['entries'][0]['papers'].pop()
|
||||||
|
with self.subTest(kind=kind),self.assertRaises(ValueError):self.parse_changed(data)
|
||||||
|
|
||||||
|
def test_corruption_missing_evidence_and_symlink_fail_without_completed_report(self):
|
||||||
|
for kind in ('manifest','program','probe','missing','symlink'):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);self.materialize(root/'src');directory=root/'src'/c.SNAPSHOT
|
||||||
|
name='manifest.json' if kind=='manifest' else 'openreview-rejected-probe.json' if kind=='probe' else 'program-data.json'
|
||||||
|
path=directory/name
|
||||||
|
if kind=='missing':path.unlink()
|
||||||
|
elif kind=='symlink':
|
||||||
|
original=directory/'original.json';path.rename(original);path.symlink_to(original)
|
||||||
|
else:path.write_bytes(path.read_bytes()+b' ')
|
||||||
|
with self.subTest(kind=kind),self.assertRaises((ValueError,FileNotFoundError)):c.collect(root/'out',snapshots=root/'src')
|
||||||
|
self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_literal_abstract_math_and_nonpaper_session_events_retained(self):
|
||||||
|
rows,sessions=c.parse_program(self.raw)
|
||||||
|
self.assertTrue(any('\\' in r['abstract'] for r in rows))
|
||||||
|
self.assertTrue(any('\n' in r['abstract'] for r in rows))
|
||||||
|
self.assertTrue(any(s['type']=='coffee_break' for s in sessions))
|
||||||
|
self.assertTrue(any(s['type']=='industry_talk' for s in sessions))
|
||||||
|
self.assertEqual(c.without_emails({'emails':['contact'],'items':[{'emails':['contact'],'title':'talk'}]}),{'items':[{'title':'talk'}]})
|
||||||
@@ -0,0 +1,273 @@
|
|||||||
|
from contextlib import redirect_stdout
|
||||||
|
import io
|
||||||
|
import os
|
||||||
|
import uuid
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import re
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import mm_publications2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class MMPublications2026Tests(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
fixtures=Path(__file__).parent/'fixtures'
|
||||||
|
cls.fixture=json.loads(gzip.decompress((fixtures/'mm2026-publications.json.gz').read_bytes()))
|
||||||
|
cls.program_fixture=json.loads(gzip.decompress((fixtures/'mm2026-source.json.gz').read_bytes()))
|
||||||
|
cls.format_gold=json.loads((fixtures/'mm2026-title-formatting-review.json').read_text())
|
||||||
|
overrides={g['doi']:g for g in [*cls.format_gold['gold'],cls.format_gold['held_gold']]}
|
||||||
|
cls.gold=[overrides.get(g['doi'],g) for g in cls.fixture['independent_gold']]
|
||||||
|
cls.temp=tempfile.TemporaryDirectory();cls.root=Path(cls.temp.name)
|
||||||
|
cls.materialize(cls.root/'registry',cls.fixture['files'])
|
||||||
|
cls.materialize(cls.root/'program',cls.program_fixture['files'])
|
||||||
|
cls.out=cls.root/'out'
|
||||||
|
cls.report=c.collect(cls.out,registry_snapshots=cls.root/'registry',program_snapshots=cls.root/'program',parent_snapshot=cls.root/'registry/parent.json')
|
||||||
|
cls.rows=[json.loads(line) for f in cls.report['files'] for line in Path(f['file']).read_bytes().splitlines()]
|
||||||
|
cls.by_doi={r['external_id']:r for r in cls.rows}
|
||||||
|
|
||||||
|
@classmethod
|
||||||
|
def tearDownClass(cls):
|
||||||
|
cls.temp.cleanup()
|
||||||
|
|
||||||
|
@staticmethod
|
||||||
|
def materialize(root,files):
|
||||||
|
for name,body in files.items():
|
||||||
|
path=root/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(body.encode('utf-8'))
|
||||||
|
|
||||||
|
def test_all1918_publications_against_independent_registry_program_gold(self):
|
||||||
|
self.assertEqual(len(self.rows),1918);self.assertEqual(len(self.by_doi),1918)
|
||||||
|
self.assertEqual(set(self.by_doi),{g['doi'] for g in self.gold})
|
||||||
|
for g in self.gold:
|
||||||
|
r=self.by_doi[g['doi']];m=r['metadata'];p=m['program_fields']
|
||||||
|
self.assertEqual((r['title'],r['authors'],r['track']),(g['title'],g['authors'],g['track']))
|
||||||
|
self.assertEqual(m['program_match_kind'],g['kind'])
|
||||||
|
self.assertEqual(m['registry_source_evidence']['json_pointer'],'/message/items/'+str(g['source_position']))
|
||||||
|
self.assertEqual(m['registry_source_evidence']['file'],'evidence/registry/'+g['source_page'])
|
||||||
|
self.assertEqual(m['registry_source_evidence']['canonical_object_json_sha256'],g['registry_object_sha256'])
|
||||||
|
self.assertEqual(p['paper_id'] if p else None,g['program_id'])
|
||||||
|
self.assertEqual([x['source_pointer'] for x in p['contexts']] if p else [],g['program_context_pointers'])
|
||||||
|
|
||||||
|
def test_abstract_source_is_separate_and_verbatim_not_fabricated_registry_abstract(self):
|
||||||
|
count=0
|
||||||
|
for g in self.gold:
|
||||||
|
r=self.by_doi[g['doi']];m=r['metadata']
|
||||||
|
self.assertFalse(m['registry_abstract_available']);self.assertFalse(m['registry_fields'].get('abstract'))
|
||||||
|
if g['abstract_sha256'] is not None:
|
||||||
|
count+=1;self.assertEqual(hashlib.sha256(r['abstract'].encode()).hexdigest(),g['abstract_sha256'])
|
||||||
|
self.assertEqual(r['abstract'],m['program_fields']['abstract'])
|
||||||
|
self.assertEqual(m['abstract_source'],'official_program_reviewed_title_formatting_correspondence' if g['kind'].startswith('unique_program_format_') else 'official_program_exact_metadata_correspondence')
|
||||||
|
else:
|
||||||
|
self.assertIsNone(r['abstract']);self.assertEqual(m['abstract_source'],'not_available_from_reviewed_sources')
|
||||||
|
self.assertEqual(count,1743);self.assertEqual(self.report['output_missing_abstract_count'],175)
|
||||||
|
|
||||||
|
def test_all_program_tracks_and_4326_contexts_survive_in_matches_or_unmatched(self):
|
||||||
|
programs={r['metadata']['program_fields']['paper_id']:r['metadata']['program_fields'] for r in self.rows if r['metadata']['program_fields']}
|
||||||
|
for p in self.report['unmatched_program']:
|
||||||
|
self.assertNotIn(p['paper_id'],programs);programs[p['paper_id']]=p
|
||||||
|
self.assertEqual(len(programs),2143);self.assertEqual(self.report['unmatched_program_count'],391)
|
||||||
|
gold=self.program_fixture['independent_inventory']['inventory']
|
||||||
|
self.assertEqual(set(programs),{g['paper_id'] for g in gold})
|
||||||
|
for g in gold:
|
||||||
|
p=programs[g['paper_id']]
|
||||||
|
self.assertEqual((p['title'],p['authors'],p['track']),(g['title'],g['authors'],g['track_literal']))
|
||||||
|
self.assertEqual([x['source_pointer'] for x in p['contexts']],g['locations'])
|
||||||
|
self.assertEqual(sum(len(p['contexts']) for p in programs.values()),4326)
|
||||||
|
self.assertEqual(len({p['track'] for p in programs.values()}),28)
|
||||||
|
|
||||||
|
def test_no_person_alias_original_submission_decision_or_research_type_inference(self):
|
||||||
|
for r in self.rows:
|
||||||
|
doi=r['external_id'];m=r['metadata']
|
||||||
|
self.assertEqual(r['identifiers'],{'doi':doi})
|
||||||
|
self.assertEqual(r['submission_identifiers'],[{'scheme':'publication_doi','value':doi}])
|
||||||
|
self.assertEqual((r['source'],r['venue'],r['year']),(c.SOURCE,'MM',2026))
|
||||||
|
self.assertEqual(r['status'],'accepted');self.assertIn('not an original signed decision',r['raw_decision'])
|
||||||
|
self.assertFalse(m['original_decision_signature_verified']);self.assertFalse(m['research_type_classified'])
|
||||||
|
self.assertFalse(m['scientific_work_identity_approved'])
|
||||||
|
self.assertTrue({'decided_at','decision_date','source_decision_id','paper_type'}.isdisjoint(r))
|
||||||
|
self.assertIn({'namespace':c.SOURCE+':crossref:type','value':'proceedings-article'},r['tags'])
|
||||||
|
self.assertTrue({x['namespace'] for x in r['tags']} <= {c.SOURCE+':crossref:type',c.SOURCE+':mm_program:track'})
|
||||||
|
for track in ('interactive_art','industry_talk','demo_and_video'):
|
||||||
|
self.assertTrue(any(r['track']==track for r in self.rows))
|
||||||
|
variants=[r for r in self.rows if r['track']=='proceedings_unclassified']
|
||||||
|
self.assertEqual(len(variants),166)
|
||||||
|
self.assertTrue(all(r['metadata']['program_fields'] is None and r['abstract'] is None for r in variants))
|
||||||
|
|
||||||
|
def test_original_bibliography_evidence_kept_without_duplicating_reference_payload(self):
|
||||||
|
authors_by_page={}
|
||||||
|
for name,body in self.fixture['files'].items():
|
||||||
|
actual=self.out/'evidence/registry'/('parent.json' if name=='parent.json' else Path(name).name)
|
||||||
|
self.assertEqual(actual.read_bytes(),body.encode())
|
||||||
|
if re.fullmatch(r'page-[0-9]+\.json',Path(name).name):
|
||||||
|
authors_by_page[str(actual.relative_to(self.out))]=[row['author'] for row in json.loads(body)['message']['items']]
|
||||||
|
for name,body in self.program_fixture['files'].items():
|
||||||
|
self.assertEqual((self.out/'evidence/program'/Path(name).name).read_bytes(),body.encode())
|
||||||
|
for r in self.rows:
|
||||||
|
m=r['metadata'];self.assertNotIn('reference',m['registry_fields'])
|
||||||
|
self.assertEqual(m['registry_fields']['title'],[r['title']])
|
||||||
|
self.assertEqual(m['registry_fields']['author'],authors_by_page[m['registry_source_evidence']['file']][int(m['registry_source_evidence']['json_pointer'].rsplit('/',1)[-1])])
|
||||||
|
for f in self.report['files']:
|
||||||
|
raw=Path(f['file']).read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),f['file_sha256'])
|
||||||
|
self.assertEqual(len(raw.splitlines()),f['count']);self.assertEqual(f['status'],'partial')
|
||||||
|
|
||||||
|
def test_eight_single_name_fields_preserved_without_splitting(self):
|
||||||
|
n=0
|
||||||
|
for g in self.gold:
|
||||||
|
r=self.by_doi[g['doi']];flags=r['metadata']['review_flags']
|
||||||
|
actual=[x['author_position'] for x in flags if x['kind']=='single_registered_name_field_not_split']
|
||||||
|
self.assertEqual(actual,g['single_name_positions']);n+=len(actual)
|
||||||
|
for i in actual:
|
||||||
|
self.assertEqual(r['authors'][i-1],r['metadata']['registry_fields']['author'][i-1]['family'].strip())
|
||||||
|
self.assertEqual(n,8)
|
||||||
|
|
||||||
|
def test_structured_title_markup_retained_and_flagged_without_flattening(self):
|
||||||
|
count=0
|
||||||
|
for g in self.gold:
|
||||||
|
if re.search(r'<[^>]+>|&[A-Za-z#0-9]+;',g['title']):
|
||||||
|
count+=1;r=self.by_doi[g['doi']]
|
||||||
|
self.assertEqual(r['title'],g['title'])
|
||||||
|
self.assertIn({'kind':'literal_title_markup_or_entity_retained_requires_review'},r['metadata']['review_flags'])
|
||||||
|
self.assertGreater(count,0);self.assertEqual(count,self.report['registered_title_markup_flags'])
|
||||||
|
|
||||||
|
|
||||||
|
def test_formatting_gold_preserves_raw_fields_and_copies_exact_program_evidence(self):
|
||||||
|
self.assertEqual(len(self.format_gold['gold']),26)
|
||||||
|
for gold in self.format_gold['gold']:
|
||||||
|
row=self.by_doi[gold['doi']];m=row['metadata'];p=m['program_fields']
|
||||||
|
self.assertEqual((row['title'],row['authors']),(gold['title'],gold['authors']))
|
||||||
|
self.assertEqual((p['paper_id'],row['track']),(gold['program_id'],gold['track']))
|
||||||
|
self.assertEqual(hashlib.sha256(row['abstract'].encode()).hexdigest(),gold['abstract_sha256'])
|
||||||
|
comparison=m['program_title_formatting_comparison']
|
||||||
|
self.assertTrue(comparison['raw_titles_preserved']);self.assertFalse(comparison['work_identity_approved'])
|
||||||
|
self.assertEqual(set(comparison['program_comparison_texts']),{p['paper_id']})
|
||||||
|
self.assertFalse(m['scientific_work_identity_approved'])
|
||||||
|
held=self.by_doi[self.format_gold['author_disagreement_hold']['doi']]
|
||||||
|
self.assertEqual(held['metadata']['program_title_candidates'],['mfp7801'])
|
||||||
|
self.assertEqual(held['metadata']['program_exact_candidates'],[])
|
||||||
|
self.assertEqual(held['metadata']['program_match_kind'],'program_format_author_version_requires_review')
|
||||||
|
self.assertIsNone(held['abstract']);self.assertEqual(held['track'],'proceedings_unclassified')
|
||||||
|
|
||||||
|
def test_reviewed_formatting_keeps_superscript_subscript_and_decodes_entities_once(self):
|
||||||
|
self.assertEqual(c.formatting_text('<i>A</i> H<sup>2</sup> B<sub>1</sub> & C'),'A H2 B1 & C')
|
||||||
|
self.assertEqual(c.formatting_text('A &amp; B'),'A & B')
|
||||||
|
|
||||||
|
def test_unknown_attributes_unbalanced_or_encoded_markup_not_discarded(self):
|
||||||
|
for title in ['A <model>B</model>','A <i class="x">B</i>','A <i>B','A </i>B',
|
||||||
|
'A <sup/>B','A <!--hidden-->B','<!DOCTYPE html>A','A <i>B</i>',
|
||||||
|
'A <sup','A <i>B</sup>','<![CDATA[A]]>']:
|
||||||
|
with self.subTest(title=title):self.assertIsNone(c.formatting_text(title))
|
||||||
|
|
||||||
|
def test_ambiguous_program_and_author_multiset_differences_remain_held(self):
|
||||||
|
def observation(pid,authors):
|
||||||
|
return {'paper_id':pid,'title':'M3 Model','authors':authors,'track':'Main Track'}
|
||||||
|
rows=[{'title':'M<sup>3</sup> Model','authors':['A','B']}]
|
||||||
|
kinds,_=c.match_program(rows,[observation('p1',['A','B']),observation('p2',['B','A'])])
|
||||||
|
self.assertEqual(kinds,{'multiple_program_ids_same_title_authors_hold':1})
|
||||||
|
self.assertNotIn('program',rows[0])
|
||||||
|
rows=[{'title':'M<sup>3</sup> Model','authors':['A','A','B']}]
|
||||||
|
c.match_program(rows,[observation('p1',['A','B'])])
|
||||||
|
self.assertEqual(rows[0]['kind'],'program_format_author_version_requires_review')
|
||||||
|
self.assertNotIn('program',rows[0])
|
||||||
|
|
||||||
|
def test_literal_title_author_disagreement_does_not_use_formatting_bypass(self):
|
||||||
|
rows=[{'title':'M<sup>3</sup> Model','authors':['A','B']}]
|
||||||
|
observations=[{'paper_id':'literal','title':'M<sup>3</sup> Model','authors':['Other'],'track':'Main Track'},
|
||||||
|
{'paper_id':'formatted','title':'M3 Model','authors':['A','B'],'track':'Main Track'}]
|
||||||
|
c.match_program(rows,observations)
|
||||||
|
self.assertEqual(rows[0]['kind'],'program_author_version_requires_review')
|
||||||
|
self.assertEqual(rows[0]['program_title_candidates'],['literal']);self.assertNotIn('program',rows[0])
|
||||||
|
|
||||||
|
|
||||||
|
def test_corrupt_registry_parent_or_program_keeps_failure_not_complete_output(self):
|
||||||
|
for kind in ('registry_manifest','registry_page','parent','program'):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);self.materialize(root/'registry',self.fixture['files']);self.materialize(root/'program',self.program_fixture['files'])
|
||||||
|
path={'registry_manifest':root/'registry'/c.SNAPSHOT/'manifest.json','registry_page':root/'registry'/c.SNAPSHOT/'page-001.json','parent':root/'registry/parent.json','program':root/'program'/c.program.SNAPSHOT/'program-data.json'}[kind]
|
||||||
|
path.write_bytes(path.read_bytes()+b' ')
|
||||||
|
with self.subTest(kind=kind),self.assertRaises(ValueError):
|
||||||
|
c.collect(root/'out',registry_snapshots=root/'registry',program_snapshots=root/'program',parent_snapshot=root/'registry/parent.json')
|
||||||
|
self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_symlink_source_rejected_and_no_previous_output_overwrite(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);self.materialize(root/'registry',self.fixture['files'])
|
||||||
|
path=root/'registry/parent.json';original=root/'registry/actual.json';path.rename(original);path.symlink_to(original)
|
||||||
|
with self.assertRaisesRegex(ValueError,'Unsafe'):
|
||||||
|
c.collect(root/'out',registry_snapshots=root/'registry',program_snapshots=self.root/'program',parent_snapshot=path)
|
||||||
|
with self.assertRaisesRegex(ValueError,'Output exists'):
|
||||||
|
c.collect(self.out,registry_snapshots=self.root/'registry',program_snapshots=self.root/'program',parent_snapshot=self.root/'registry/parent.json')
|
||||||
|
|
||||||
|
def test_complete_report_retains_source_partitions_not_import_or_final_dedup_claim(self):
|
||||||
|
self.assertEqual(self.report['program_match_counts'],{'unique_program_title_fullauthor_agreement_not_identity_approval':1726,'unique_program_format_title_fullauthor_agreement_not_identity_approval':26,'program_author_version_requires_review':122,'program_format_author_version_requires_review':1,'no_exact_program_title_requires_review':43})
|
||||||
|
self.assertEqual(self.report['registry_missing_abstract_count'],1918)
|
||||||
|
self.assertEqual(self.report['abstracts_from_exact_program_correspondence'],1743)
|
||||||
|
self.assertEqual(self.report['abstracts_from_literal_program_correspondence'],1717)
|
||||||
|
self.assertEqual(self.report['abstracts_from_reviewed_title_formatting'],26)
|
||||||
|
self.assertEqual(self.report['unmatched_program_count'],391)
|
||||||
|
self.assertEqual(sum(f['count'] for f in self.report['files']),1918)
|
||||||
|
self.assertEqual(self.report['approved_for_import'],0);self.assertFalse(self.report['database_written'])
|
||||||
|
self.assertFalse(self.report['pdfs_downloaded']);self.assertEqual(self.report['network_requests'],0)
|
||||||
|
self.assertIn('unknown',self.report['public_negative_availability'])
|
||||||
|
|
||||||
|
|
||||||
|
def assert_native_ingestion(self, pg_service=None):
|
||||||
|
# An isolated synthetic database exercises the writer contract, not a
|
||||||
|
# production identity approval. All source rows/track partitions run.
|
||||||
|
from paper_atlas.db.store import Store
|
||||||
|
from paper_atlas.db.backend import open_store
|
||||||
|
from paper_atlas.ingestion.records import ingest_file
|
||||||
|
from paper_atlas.postgres.client import connect
|
||||||
|
from paper_atlas.postgres.migration import migrate
|
||||||
|
from paper_atlas.postgres.schema import quote
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);db=root/'seed.sqlite';seed=self.rows[0]
|
||||||
|
store=Store(db)
|
||||||
|
paper,record=store.record('arxiv','synthetic-mm-anchor','https://example.org/arxiv',
|
||||||
|
seed['title'],'Existing canonical abstract',seed['authors'],seed['identifiers'])
|
||||||
|
store.decision(paper,record,'ICLR',2025,'main','synthetic-old-forum',
|
||||||
|
'https://example.org/decision','rejected','Reject')
|
||||||
|
before=tuple(store.conn.execute('SELECT * FROM papers WHERE id=?',(paper,)).fetchone())
|
||||||
|
store.conn.commit();store.conn.close()
|
||||||
|
schema='pa_mm_pub_'+uuid.uuid4().hex[:12] if pg_service else None
|
||||||
|
try:
|
||||||
|
with redirect_stdout(io.StringIO()):
|
||||||
|
if pg_service:migrate(db,pg_service,root/'migration.json',schema=schema,min_free_gib=0)
|
||||||
|
for f in self.report['files']:
|
||||||
|
ingest_file(None if pg_service else db,f['file'],pg_service=pg_service,pg_schema=schema)
|
||||||
|
store=open_store(None if pg_service else db,pg_service=pg_service,pg_schema=schema)
|
||||||
|
try:
|
||||||
|
conn=store.conn
|
||||||
|
self.assertEqual(tuple(conn.execute('SELECT * FROM papers WHERE id=?',(paper,)).fetchone()),before)
|
||||||
|
self.assertEqual(conn.execute('SELECT status FROM decisions WHERE submission_id=1').fetchone()[0],'rejected')
|
||||||
|
actual={r[0]:r for r in conn.execute("SELECT sr.external_id,sr.paper_id,sr.title,sr.abstract,sr.authors_json,sr.metadata_json,sr.id FROM source_records sr JOIN sources s ON s.id=sr.source_id WHERE s.name=?",(c.SOURCE,))}
|
||||||
|
self.assertEqual(set(actual),set(self.by_doi))
|
||||||
|
self.assertEqual(actual[seed['external_id']][1],paper)
|
||||||
|
for doi,row in self.by_doi.items():
|
||||||
|
got=actual[doi];self.assertEqual((got[2],got[3],json.loads(got[4])),(row['title'],row['abstract'],row['authors']))
|
||||||
|
meta=json.loads(got[5]);self.assertEqual(meta.pop('record_identifiers'),{'doi':doi,c.SOURCE:doi});self.assertEqual(meta,row['metadata'])
|
||||||
|
tags=conn.execute("SELECT t.namespace,t.value,count(*) FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id GROUP BY t.namespace,t.value").fetchall()
|
||||||
|
self.assertEqual(sum(r[2] for r in tags if r[0]==c.SOURCE+':crossref:type'),1918)
|
||||||
|
self.assertEqual(sum(r[2] for r in tags if r[0]==c.SOURCE+':mm_program:track'),1752)
|
||||||
|
self.assertEqual({r[0] for r in tags},{c.SOURCE+':crossref:type',c.SOURCE+':mm_program:track'})
|
||||||
|
decisions=conn.execute("SELECT d.status,d.decision_date,count(*) FROM decisions d JOIN submissions s ON s.id=d.submission_id JOIN source_records r ON r.id=s.source_record_id JOIN sources src ON src.id=r.source_id WHERE src.name=? GROUP BY d.status,d.decision_date",(c.SOURCE,)).fetchall()
|
||||||
|
self.assertEqual([tuple(r) for r in decisions],[('accepted',None,1918)])
|
||||||
|
self.assertEqual(conn.execute("SELECT count(*) FROM submission_identifiers WHERE scheme='publication_doi'").fetchone()[0],1918)
|
||||||
|
coverage=conn.execute("SELECT es.status,sum(es.imported_count) FROM edition_sources es JOIN sources s ON s.id=es.source_id WHERE s.name=? GROUP BY es.status",(c.SOURCE,)).fetchall()
|
||||||
|
self.assertEqual([tuple(r) for r in coverage],[('partial',1918)])
|
||||||
|
finally:store.conn.close()
|
||||||
|
finally:
|
||||||
|
if pg_service:
|
||||||
|
with connect(pg_service) as pg:
|
||||||
|
for name, in pg.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)',(schema,)).fetchall():
|
||||||
|
pg.execute('DROP SCHEMA '+quote(name)+' CASCADE')
|
||||||
|
|
||||||
|
def test_all1918_native_sqlite_source_tags_and_prior_rejection_survive(self):
|
||||||
|
self.assert_native_ingestion()
|
||||||
|
|
||||||
|
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'),'Requires approved remote PostgreSQL test service')
|
||||||
|
def test_all1918_native_postgres_source_tags_and_prior_rejection_survive(self):
|
||||||
|
self.assert_native_ingestion(os.environ['PAPER_ATLAS_TEST_PG_SERVICE'])
|
||||||
@@ -0,0 +1,101 @@
|
|||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from paper_atlas.collectors import mm2026
|
||||||
|
from paper_atlas.audits.source_identity import audit, load_inputs, load_mm_program_inputs
|
||||||
|
|
||||||
|
|
||||||
|
class MMProgramIdentityInputTests(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/mm2026-source.json.gz').read_bytes()))
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp = tempfile.TemporaryDirectory()
|
||||||
|
self.addCleanup(self.tmp.cleanup)
|
||||||
|
self.root = Path(self.tmp.name)
|
||||||
|
src = self.root/'source'
|
||||||
|
for name,body in self.fixture['files'].items():
|
||||||
|
path=src/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(body.encode())
|
||||||
|
mm2026.collect(self.root/'program', snapshots=src)
|
||||||
|
self.report_path = self.root/'program/collection-report.json'
|
||||||
|
|
||||||
|
def change_report(self, change):
|
||||||
|
report=json.loads(self.report_path.read_text());change(report)
|
||||||
|
self.report_path.write_text(json.dumps(report))
|
||||||
|
|
||||||
|
def assert_rejected_before_database(self):
|
||||||
|
with patch('paper_atlas.audits.source_identity.ReadConnection') as connection:
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
audit(None,self.root/'audit',pg_service='unused',pg_schema='research',mm_program_report=self.report_path)
|
||||||
|
connection.assert_not_called()
|
||||||
|
self.assertFalse((self.root/'audit').exists())
|
||||||
|
|
||||||
|
def test_all2143_programs_are_unknown_readonly_inputs_not_import_records(self):
|
||||||
|
rows,inputs=load_mm_program_inputs(self.report_path)
|
||||||
|
self.assertEqual(len(rows),2143);self.assertEqual(len(inputs),28)
|
||||||
|
self.assertEqual(sum(len(r['program_contexts']) for r in rows),4326)
|
||||||
|
actual={r['program_id']:r for r in rows}
|
||||||
|
for gold in self.fixture['independent_inventory']['inventory']:
|
||||||
|
r=actual[gold['paper_id']]
|
||||||
|
self.assertEqual((r['title'],r['authors'],r['track']),
|
||||||
|
(gold['title'],gold['authors'],gold['track_literal']))
|
||||||
|
self.assertEqual(r['external_id'],'2026/'+gold['paper_id'])
|
||||||
|
self.assertEqual(r['status'],'unknown');self.assertEqual(r['identifiers'],{})
|
||||||
|
self.assertEqual(r['source_observed_status'],'listed_in_official_program')
|
||||||
|
self.assertFalse(r['publication_identity_verified']);self.assertFalse(r['final_decision_verified'])
|
||||||
|
self.assertFalse(r['contribution_type_verified'])
|
||||||
|
self.assertEqual([c['source_pointer'] for c in r['program_contexts']],gold['locations'])
|
||||||
|
self.assertEqual({r['program_id'] for r in rows if not r['authors']},{'mfp4971','mfp7801'})
|
||||||
|
self.assertTrue(all('unknown is not a source decision' in i['manifest']['scope'] for i in inputs))
|
||||||
|
self.assertFalse(any(self.root.rglob('*.manifest.json')))
|
||||||
|
|
||||||
|
def test_untrusted_absolute_file_paths_are_not_followed(self):
|
||||||
|
self.change_report(lambda r:[f.update(file='/untrusted/'+Path(f['file']).name) for f in r['files']])
|
||||||
|
rows,inputs=load_mm_program_inputs(self.report_path)
|
||||||
|
self.assertEqual(len(rows),2143)
|
||||||
|
self.assertTrue(all(Path(i['file']).parent==self.report_path.parent for i in inputs))
|
||||||
|
|
||||||
|
def test_program_files_are_not_silently_accepted_by_regular_ingestion_input_loader(self):
|
||||||
|
report=json.loads(self.report_path.read_text())
|
||||||
|
with self.assertRaises(FileNotFoundError):load_inputs([report['files'][0]['file']])
|
||||||
|
|
||||||
|
def test_corrupt_jsonl_fails_before_connection_or_output(self):
|
||||||
|
report=json.loads(self.report_path.read_text());path=Path(report['files'][0]['file'])
|
||||||
|
path.write_bytes(path.read_bytes()+b' ')
|
||||||
|
self.assert_rejected_before_database()
|
||||||
|
|
||||||
|
def test_rehashed_forged_identifiers_decisions_and_contexts_rejected(self):
|
||||||
|
original=self.report_path.read_bytes();report=json.loads(original);path=Path(report['files'][0]['file']);raw=path.read_bytes()
|
||||||
|
for field,value in [('identifiers',{'openreview':'invented'}),('status','accepted'),('contexts',[])]:
|
||||||
|
path.write_bytes(raw);self.report_path.write_bytes(original)
|
||||||
|
lines=raw.splitlines();row=json.loads(lines[0]);row[field]=value;lines[0]=json.dumps(row).encode()
|
||||||
|
changed=b'\n'.join(lines)+b'\n';path.write_bytes(changed)
|
||||||
|
self.change_report(lambda r:r['files'][0].update(file_sha256=hashlib.sha256(changed).hexdigest()))
|
||||||
|
with self.subTest(field=field):self.assert_rejected_before_database()
|
||||||
|
|
||||||
|
def test_missing_or_repeated_track_rejected(self):
|
||||||
|
original=self.report_path.read_bytes()
|
||||||
|
for kind in ('missing','repeated'):
|
||||||
|
self.report_path.write_bytes(original)
|
||||||
|
if kind=='missing':self.change_report(lambda r:r['files'].pop())
|
||||||
|
else:self.change_report(lambda r:r['files'].__setitem__(1,r['files'][0]))
|
||||||
|
with self.subTest(kind=kind):self.assert_rejected_before_database()
|
||||||
|
|
||||||
|
def test_raw_source_corruption_or_symlink_rejected(self):
|
||||||
|
path=self.report_path.parent/'evidence/program-data.json';raw=path.read_bytes()
|
||||||
|
path.write_bytes(raw+b' ');self.assert_rejected_before_database()
|
||||||
|
path.write_bytes(raw);saved=path.with_name('original.json');path.rename(saved);path.symlink_to(saved)
|
||||||
|
self.assert_rejected_before_database()
|
||||||
|
|
||||||
|
def test_mixed_input_modes_rejected_before_connection(self):
|
||||||
|
with patch('paper_atlas.audits.source_identity.ReadConnection') as connection:
|
||||||
|
for files,program in [([],None),([Path('unused')],self.report_path)]:
|
||||||
|
with self.assertRaisesRegex(ValueError,'Choose source files'):
|
||||||
|
audit(files,self.root/'out',pg_service='unused',pg_schema='research',mm_program_report=program)
|
||||||
|
connection.assert_not_called()
|
||||||
@@ -0,0 +1,174 @@
|
|||||||
|
import copy
|
||||||
|
from collections import Counter
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
from urllib.parse import parse_qs, urlsplit
|
||||||
|
|
||||||
|
from paper_atlas.collectors import neurips_program as program
|
||||||
|
|
||||||
|
FIXTURES = Path(__file__).parent/'fixtures'
|
||||||
|
RAW = gzip.decompress((FIXTURES/'neurips2026-program.json.gz').read_bytes())
|
||||||
|
RECEIPTS = json.loads((FIXTURES/'neurips2026-receipts.json').read_text())
|
||||||
|
EVIDENCE = {'url': program.URL, 'observed_at': RECEIPTS['receipts'][0]['finished_at']}
|
||||||
|
|
||||||
|
|
||||||
|
class NeuripsProgramTests(unittest.TestCase):
|
||||||
|
def sample(self):
|
||||||
|
rows = json.loads(RAW)['results']
|
||||||
|
oral = next(r for r in rows if r['id'] == 154866)
|
||||||
|
poster = next(r for r in rows if r['paper_url'] == oral['paper_url'] and r['event_type'] == 'Poster')
|
||||||
|
return copy.deepcopy(poster), copy.deepcopy(oral)
|
||||||
|
|
||||||
|
def payload(self, rows):
|
||||||
|
return json.dumps({'count': len(rows), 'next': None, 'previous': None, 'results': rows})
|
||||||
|
|
||||||
|
def snapshots(self, folder):
|
||||||
|
folder.mkdir()
|
||||||
|
(folder/'program-json.response').write_bytes(RAW)
|
||||||
|
(folder/'receipts.json').write_text(json.dumps(RECEIPTS))
|
||||||
|
return folder
|
||||||
|
|
||||||
|
def test_full_snapshot_tracks_and_all_titles_authors_affiliations(self):
|
||||||
|
rows, held, duplicates, folded = program.parse_program(RAW, EVIDENCE)
|
||||||
|
self.assertEqual((len(rows), len(held), len(duplicates), folded), (9004, 139, 1, 132))
|
||||||
|
self.assertEqual(Counter(r['track'] for r in rows),
|
||||||
|
{'main': 7896, 'evaluations_datasets': 971, 'position': 137})
|
||||||
|
expected = {}
|
||||||
|
for e in json.loads(RAW)['results']:
|
||||||
|
if e['sourceurl'] in program.GROUPS and e['event_type'] == 'Poster':
|
||||||
|
forum = parse_qs(urlsplit(e['paper_url']).query)['id'][0]
|
||||||
|
expected[forum] = e
|
||||||
|
self.assertEqual(len(expected), 9004)
|
||||||
|
for r in rows:
|
||||||
|
e = expected[r['identifiers']['openreview']]
|
||||||
|
self.assertEqual(r['title'], e['name'])
|
||||||
|
self.assertEqual(r['authors'], [a['fullname'] for a in e['authors']])
|
||||||
|
self.assertEqual(r['metadata']['author_affiliations'], [a.get('institution') for a in e['authors']])
|
||||||
|
self.assertEqual(r['raw_decision'], e['decision'])
|
||||||
|
self.assertEqual(r['status'], 'accepted')
|
||||||
|
self.assertNotIn('decision_date', r)
|
||||||
|
self.assertIsNone(r['abstract'])
|
||||||
|
self.assertEqual(r['year'], 2026)
|
||||||
|
self.assertEqual(duplicates[0]['event_id'], 154866)
|
||||||
|
|
||||||
|
def test_same_forum_folds_presentations_and_preserves_signed_source_ids(self):
|
||||||
|
poster, oral = self.sample()
|
||||||
|
rows, held, duplicates, folded = program.parse_program(self.payload([oral, poster, oral]), EVIDENCE)
|
||||||
|
row, = rows
|
||||||
|
self.assertEqual((len(held), len(duplicates), folded), (0, 1, 1))
|
||||||
|
self.assertEqual({p['sourceid'] for p in row['metadata']['presentations']},
|
||||||
|
{poster['sourceid'], oral['sourceid']})
|
||||||
|
self.assertLess(oral['sourceid'], 0)
|
||||||
|
self.assertEqual({t['value'] for t in row['tags']}, {'Poster', 'Oral'})
|
||||||
|
self.assertTrue(all(t['namespace'] == 'neurips_program:presentation_format' for t in row['tags']))
|
||||||
|
|
||||||
|
def test_identical_title_and_authors_in_distinct_forums_remain_distinct(self):
|
||||||
|
poster, _ = self.sample(); other = copy.deepcopy(poster)
|
||||||
|
other.update(id=999999, uid='0'*32, sourceid=999999, paper_url='https://openreview.net/forum?id=otherForum7',
|
||||||
|
virtualsite_url='/virtual/2026/poster/999999')
|
||||||
|
rows, _, _, _ = program.parse_program(self.payload([poster, other]), EVIDENCE)
|
||||||
|
self.assertEqual(len(rows), 2)
|
||||||
|
self.assertEqual(len({r['external_id'] for r in rows}), 2)
|
||||||
|
|
||||||
|
def test_conflicting_event_ids_or_same_forum_bibliography_are_rejected(self):
|
||||||
|
poster, oral = self.sample()
|
||||||
|
for field in ('name', 'authors', 'sourceurl', 'decision'):
|
||||||
|
changed = copy.deepcopy(oral)
|
||||||
|
if field == 'authors':
|
||||||
|
changed['authors'][0]['fullname'] = 'Different Author'
|
||||||
|
elif field == 'sourceurl':
|
||||||
|
changed[field] = 'https://openreview.net/group?id=NeurIPS.cc/2026/Position_Paper_Track'
|
||||||
|
elif field == 'decision':
|
||||||
|
changed[field] = 'Accept (poster)'
|
||||||
|
else:
|
||||||
|
changed[field] += ' changed'
|
||||||
|
with self.subTest(field=field), self.assertRaises(ValueError):
|
||||||
|
program.parse_program(self.payload([poster, changed]), EVIDENCE)
|
||||||
|
changed = copy.deepcopy(poster); changed['name'] += ' changed'
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
program.parse_program(self.payload([poster, changed]), EVIDENCE)
|
||||||
|
|
||||||
|
def test_unreviewed_origin_decision_visibility_and_link_are_held(self):
|
||||||
|
poster, _ = self.sample()
|
||||||
|
mutations = [{'sourceurl': 'journal-track/NeurIPS_2026-Posters_TMLR.xlsx'},
|
||||||
|
{'sourceurl': 'neurips2026-bridging-the-future'},
|
||||||
|
{'sourceurl': None}, {'sourceurl': 'https://openreview.net/group?id=NeurIPS.cc/2027/Conference'},
|
||||||
|
{'decision': 'Reject'}, {'visible': False},
|
||||||
|
{'paper_url': 'https://other.example/forum?id=anotherID7'}]
|
||||||
|
for mutation in mutations:
|
||||||
|
changed = copy.deepcopy(poster); changed.update(id=999999, virtualsite_url='/virtual/2026/poster/999999')
|
||||||
|
changed.update(mutation)
|
||||||
|
rows, held, _, _ = program.parse_program(self.payload([poster, changed]), EVIDENCE)
|
||||||
|
self.assertEqual((len(rows), len(held)), (1, 1))
|
||||||
|
self.assertEqual(held[0]['raw_event'], changed)
|
||||||
|
|
||||||
|
def test_rejects_truncation_pagination_invalid_url_and_missing_authors(self):
|
||||||
|
poster, _ = self.sample()
|
||||||
|
raw = json.loads(self.payload([poster]))
|
||||||
|
bad = [dict(raw, count=2), dict(raw, next='https://example.com/next'),
|
||||||
|
{k: v for k, v in raw.items() if k != 'previous'}]
|
||||||
|
for payload in bad:
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
program.parse_program(json.dumps(payload), EVIDENCE)
|
||||||
|
for mutation in [{'authors': []}, {'virtualsite_url': '/virtual/2025/poster/154865'}, {'id': True}]:
|
||||||
|
changed = copy.deepcopy(poster); changed.update(mutation)
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
program.parse_program(self.payload([changed]), EVIDENCE)
|
||||||
|
|
||||||
|
def test_frozen_collection_is_partial_and_no_http_or_database_write(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp, patch.object(program, 'fetch', side_effect=AssertionError('No HTTP')):
|
||||||
|
root = Path(tmp); source = self.snapshots(root/'source')
|
||||||
|
report = program.collect(root/'out', snapshots=source)
|
||||||
|
self.assertEqual(report['records'], 9004)
|
||||||
|
self.assertEqual(len(report['files']), 3)
|
||||||
|
self.assertFalse(report['database_written'])
|
||||||
|
self.assertFalse(report['pdfs_downloaded'])
|
||||||
|
for m in report['files']:
|
||||||
|
self.assertEqual(m['status'], 'partial')
|
||||||
|
self.assertEqual(m['missing_abstract_count'], m['count'])
|
||||||
|
self.assertEqual(hashlib.sha256(Path(m['file']).read_bytes()).hexdigest(), m['file_sha256'])
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
program.collect(root/'out', snapshots=source)
|
||||||
|
|
||||||
|
def test_frozen_receipt_tamper_path_symlink_status_and_duplicate(self):
|
||||||
|
for mode in ('tamper', 'path', 'symlink', 'status', 'duplicate'):
|
||||||
|
with self.subTest(mode=mode), tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); source = self.snapshots(root/'source')
|
||||||
|
receipts = copy.deepcopy(RECEIPTS)
|
||||||
|
if mode == 'tamper':
|
||||||
|
(source/'program-json.response').write_text('bad')
|
||||||
|
elif mode == 'path':
|
||||||
|
receipts['receipts'][0]['file'] = '../outside.json'
|
||||||
|
elif mode == 'symlink':
|
||||||
|
(root/'outside.json').write_bytes(RAW)
|
||||||
|
(source/'program-json.response').unlink()
|
||||||
|
(source/'program-json.response').symlink_to(root/'outside.json')
|
||||||
|
elif mode == 'status':
|
||||||
|
receipts['receipts'][0]['status'] = 403
|
||||||
|
else:
|
||||||
|
receipts['receipts'].append(copy.deepcopy(receipts['receipts'][0]))
|
||||||
|
(source/'receipts.json').write_text(json.dumps(receipts))
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
program.collect(root/'out', snapshots=source)
|
||||||
|
self.assertFalse((root/'out/neurips-program-2026.json').exists())
|
||||||
|
|
||||||
|
def test_online_mode_only_requests_fixed_program_metadata(self):
|
||||||
|
poster, _ = self.sample(); raw = self.payload([poster]).encode()
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); cache = root/'cache'; cache.mkdir()
|
||||||
|
receipt = {'retrieved_at': EVIDENCE['observed_at'], 'url': program.URL}
|
||||||
|
(cache/hashlib.sha256(program.URL.encode()).hexdigest()).with_suffix('.json').write_text(json.dumps(receipt))
|
||||||
|
with patch.object(program, 'fetch', return_value=raw) as fetch:
|
||||||
|
report = program.collect(root/'out', cache=cache)
|
||||||
|
fetch.assert_called_once_with(program.URL, cache, program.LIMIT, delay=1)
|
||||||
|
self.assertEqual(report['records'], 1)
|
||||||
|
|
||||||
|
def test_frozen_fixture_preserves_actual_response_bytes(self):
|
||||||
|
receipt, = RECEIPTS['receipts']
|
||||||
|
self.assertEqual(hashlib.sha256(RAW).hexdigest(), receipt['sha256'])
|
||||||
|
self.assertEqual(len(RAW), receipt['bytes'])
|
||||||
@@ -0,0 +1,79 @@
|
|||||||
|
import base64
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import recsys2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class RecSys2026Test(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture=json.loads(gzip.decompress((Path(__file__).parent/'fixtures/recsys2026-source.json.gz').read_bytes()))
|
||||||
|
cls.files={n:base64.b64decode(raw) for n,raw in cls.fixture['files'].items()}
|
||||||
|
cls.body=cls.files[c.SNAPSHOT+'/recsys-contributions.html']
|
||||||
|
|
||||||
|
def materialize(self,root):
|
||||||
|
for name,raw in self.files.items():
|
||||||
|
path=root/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(raw)
|
||||||
|
|
||||||
|
def test_all263_observations_against_independent_gold(self):
|
||||||
|
items=c.parse_program(self.body)
|
||||||
|
keys=self.fixture['independent_gold'][0].keys()
|
||||||
|
self.assertEqual([{k:r[k] for k in keys} for r in items],self.fixture['independent_gold'])
|
||||||
|
self.assertEqual(len(items),263);self.assertEqual(len({r['doi'] for r in items}),260)
|
||||||
|
self.assertEqual(sum(bool(r['review_flags']) for r in items),6)
|
||||||
|
variants=[r for r in items if r['doi']=='10.1145/3773078.3831839']
|
||||||
|
self.assertNotEqual(variants[0]['title'],variants[1]['title']);self.assertNotEqual(variants[0]['authors'],variants[1]['authors'])
|
||||||
|
self.assertNotEqual(variants[0]['abstract'],variants[1]['abstract'])
|
||||||
|
|
||||||
|
def test_installed_format_tracks_evidence_and_original_fields(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);src=root/'src';self.materialize(src);out=root/'out'
|
||||||
|
report=c.collect(out,snapshots=src)
|
||||||
|
self.assertEqual((report['records'],report['unique_dois'],report['missing_abstract_count']),(263,260,0))
|
||||||
|
self.assertFalse(report['database_written']);self.assertFalse(report['pdfs_downloaded']);self.assertEqual(report['network_requests'],0);self.assertEqual(report['approved_for_import'],0)
|
||||||
|
self.assertEqual(report['track_counts'],{k:n for k,_,n in c.TRACKS});self.assertEqual(report['duplicate_doi_observations'],6)
|
||||||
|
gold={r['locator']:r for r in self.fixture['independent_gold']};seen=set()
|
||||||
|
for manifest in report['files']:
|
||||||
|
path=Path(manifest['file']);raw=path.read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),manifest['file_sha256'])
|
||||||
|
rows=[json.loads(l) for l in raw.splitlines()];self.assertEqual(len(rows),manifest['count']);self.assertEqual(manifest['status'],'partial')
|
||||||
|
for r in rows:
|
||||||
|
item=r['metadata']['program_fields'];locator=item['locator'];seen.add(locator);expected=gold[locator]
|
||||||
|
for key in expected:self.assertEqual(item[key],expected[key])
|
||||||
|
self.assertEqual(r['external_id'],'2026/'+locator);self.assertEqual(r['title'],expected['title']);self.assertEqual(r['authors'],expected['authors']);self.assertEqual(r['abstract'],expected['abstract'])
|
||||||
|
self.assertEqual(r['track'],manifest['track']);self.assertEqual(r['identifiers'],{'doi':expected['doi']});self.assertEqual(r['submission_identifiers'],[{'scheme':'publication_doi','value':expected['doi']}])
|
||||||
|
self.assertEqual(r['tags'],[{'namespace':'recsys_program:contribution_type','value':expected['type_labels'][0]}]);self.assertFalse(r['metadata']['original_decision_signature_verified'])
|
||||||
|
self.assertEqual(seen,set(gold));self.assertEqual((out/'evidence/recsys-contributions.html').read_bytes(),self.body)
|
||||||
|
with self.assertRaisesRegex(ValueError,'Output exists'):c.collect(out,snapshots=src)
|
||||||
|
|
||||||
|
def test_edition_and_track_label_drift_rejected(self):
|
||||||
|
for old,new in [(b'RecSys 26 - Accepted Contributions',b'RecSys 25 - Accepted Contributions'),(b'>Long Papers</a>',b'>Workshops</a>'),(b"title='Research'",b"title='Unreviewed'")]:
|
||||||
|
with self.subTest(old=old),self.assertRaises(ValueError):c.parse_program(self.body.replace(old,new,1))
|
||||||
|
|
||||||
|
def test_duplicate_locator_missing_author_and_foreign_doi_rejected(self):
|
||||||
|
for old,new in [(b'accordion-1-slide-2"',b'accordion-1-slide-1"'),(b'by Md Aminul Islam, Kathryn Vasilaky and Elena Zheleva',b'by '),(b'https://doi.org/10.1145/3773078.3831824',b'https://doi.org/10.1145/9999999.3831824')]:
|
||||||
|
with self.subTest(old=old),self.assertRaises(ValueError):c.parse_program(self.body.replace(old,new,1))
|
||||||
|
|
||||||
|
def test_math_markup_requires_review_and_is_not_flattened(self):
|
||||||
|
changed=self.body.replace(b'Learning-to-rank (LTR)',b'Learning-to-rank <sup>2</sup> (LTR)',1)
|
||||||
|
with self.assertRaisesRegex(ValueError,'markup'):c.parse_program(changed)
|
||||||
|
|
||||||
|
def test_corruption_and_missing_receipt_body_preserve_failure(self):
|
||||||
|
for kind in ('manifest','body','missing'):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);src=root/'src';self.materialize(src);path=src/c.SNAPSHOT/('manifest.json' if kind=='manifest' else 'recsys-contributions.html')
|
||||||
|
if kind=='missing':path.unlink()
|
||||||
|
else:path.write_bytes(path.read_bytes()+b' ')
|
||||||
|
with self.assertRaises((ValueError,FileNotFoundError)):c.collect(root/'out',snapshots=src)
|
||||||
|
self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_symlink_source_and_root_rejected(self):
|
||||||
|
for kind in ('source','root'):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);src=root/'src';self.materialize(src);path=src/c.SNAPSHOT/'recsys-contributions.html' if kind=='source' else src
|
||||||
|
original=path.with_name(path.name+'-original');path.rename(original);path.symlink_to(original,target_is_directory=kind=='root')
|
||||||
|
with self.assertRaisesRegex(ValueError,'Unsafe'):c.collect(root/'out',snapshots=src)
|
||||||
@@ -0,0 +1,100 @@
|
|||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
from paper_atlas.collectors import rss2026 as rss
|
||||||
|
|
||||||
|
FIXTURE = Path(__file__).parent/'fixtures/rss2026-source.json.gz'
|
||||||
|
|
||||||
|
class Rss2026Tests(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture=json.loads(gzip.decompress(FIXTURE.read_bytes()))
|
||||||
|
|
||||||
|
def setUp(self):
|
||||||
|
self.tmp=tempfile.TemporaryDirectory();self.addCleanup(self.tmp.cleanup)
|
||||||
|
self.root=Path(self.tmp.name);self.snap=self.root/'snapshots';self.snap.mkdir()
|
||||||
|
for name,text in self.fixture['pages'].items():(self.snap/name).write_bytes(text.encode())
|
||||||
|
(self.snap/'receipts.json').write_text(json.dumps(self.fixture['receipts']))
|
||||||
|
self.pages=self.fixture['pages'];self.index,self.program=self.lists()
|
||||||
|
|
||||||
|
def lists(self,**changes):
|
||||||
|
pages={**self.pages,**changes}
|
||||||
|
return rss.parse_lists(*(pages[n] for n in rss.PAGES))
|
||||||
|
|
||||||
|
def test_all210_records_match_independent_primary_inventory_and_evidence(self):
|
||||||
|
report=rss.collect(self.root/'out',snapshots=self.snap)
|
||||||
|
self.assertEqual(report['records'],210);self.assertFalse(report['database_written'])
|
||||||
|
self.assertFalse(report['pdfs_downloaded']);self.assertEqual(len(report['source_evidence']),214)
|
||||||
|
manifest,=report['files'];self.assertEqual(manifest['status'],'partial')
|
||||||
|
rows=[json.loads(l) for l in Path(manifest['file']).read_text().splitlines()]
|
||||||
|
for row,gold in zip(rows,self.fixture['independent_gold']):
|
||||||
|
self.assertEqual(row['external_id'],f"2026/p{gold['paper_id']:03d}")
|
||||||
|
for key in ('title','authors','abstract'):self.assertEqual(row[key],gold[key])
|
||||||
|
self.assertEqual(row['identifiers'],{'doi':gold['doi_raw'].lower()})
|
||||||
|
self.assertEqual(row['metadata']['publisher_date'],gold['publication_date'])
|
||||||
|
self.assertNotIn('submission_identifiers',row);self.assertEqual(row['track'],'main')
|
||||||
|
e=row['metadata']['source_evidence'];self.assertEqual(hashlib.sha256((self.root/'out'/e['file']).read_bytes()).hexdigest(),e['sha256'])
|
||||||
|
self.assertEqual(manifest['file_sha256'],hashlib.sha256(Path(manifest['file']).read_bytes()).hexdigest())
|
||||||
|
self.assertEqual(manifest['missing_abstract_count'],0)
|
||||||
|
|
||||||
|
def test_full_hidden_authors_are_separate_from_six_short_previews(self):
|
||||||
|
shortened=[p for p in self.program.values() if p['authors_complete']!=p['authors_preview']]
|
||||||
|
self.assertEqual(len(shortened),6)
|
||||||
|
p=rss.parse_paper(self.pages['p087.html'],87,self.index[87],self.program[87])
|
||||||
|
self.assertEqual(len(p['authors']),55)
|
||||||
|
self.assertFalse(p['program_variants'])
|
||||||
|
|
||||||
|
def test_six_reviewed_name_variants_preserve_original_strings(self):
|
||||||
|
for pid in rss.REVIEWED_AUTHOR_VARIANTS:
|
||||||
|
p=rss.parse_paper(self.pages[f'p{pid:03d}.html'],pid,self.index[pid],self.program[pid])
|
||||||
|
self.assertIn('reviewed_program_author_name_variant_not_person_alias',p['program_variants'])
|
||||||
|
self.assertNotEqual(', '.join(p['authors']),self.program[pid]['authors_complete'])
|
||||||
|
|
||||||
|
def test_reviewed_title_variant_is_not_an_automatic_erratum(self):
|
||||||
|
p=rss.parse_paper(self.pages['p089.html'],89,self.index[89],self.program[89])
|
||||||
|
self.assertTrue(p['title'].startswith('RLinf-VLA'))
|
||||||
|
self.assertTrue(self.program[89]['title'].startswith('RLux-VLA'))
|
||||||
|
self.assertIn('reviewed_program_title_variant_not_erratum',p['program_variants'])
|
||||||
|
|
||||||
|
def test_unreviewed_author_or_title_variant_stops(self):
|
||||||
|
for changed in [{'authors_complete':'An Unrelated Person'},{'title':'A different paper'}]:
|
||||||
|
with self.assertRaisesRegex(ValueError,'Unreviewed'):
|
||||||
|
rss.parse_paper(self.pages['p001.html'],1,self.index[1],{**self.program[1],**changed})
|
||||||
|
|
||||||
|
def test_demo_policy_cannot_be_reused_from2025(self):
|
||||||
|
with self.assertRaisesRegex(ValueError,'policy'):
|
||||||
|
self.lists(**{'cfp.html':self.pages['cfp.html'].replace('will not have a separate category','will have a separate category')})
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
self.lists(**{'sponsor-demos.html':self.pages['sponsor-demos.html'].replace('Sponsor Demos','Demo Papers')})
|
||||||
|
|
||||||
|
def test_wrong_doi_year_pdf_or_missing_abstract_is_rejected(self):
|
||||||
|
html=self.pages['p001.html']
|
||||||
|
for old,new in [('10.15607/RSS.2026.XXII.001','10.15607/RSS.2025.XXI.001'),
|
||||||
|
('2026/07/13','2025/07/13'),('rss22/p001.pdf','rss22/p002.pdf'),('Abstract:','Summary:')]:
|
||||||
|
self.assertIn(old,html)
|
||||||
|
with self.assertRaises(ValueError):rss.parse_paper(html.replace(old,new),1,self.index[1],self.program[1])
|
||||||
|
|
||||||
|
def test_index_census_and_preview_tampering_stop(self):
|
||||||
|
with self.assertRaises(ValueError):self.lists(**{'index.html':self.pages['index.html'].replace('p001.html','p999.html')})
|
||||||
|
bad=self.pages['accepted.html'].replace('Huayi Zhou, Kui Jia','Somebody Else',1)
|
||||||
|
with self.assertRaisesRegex(ValueError,'prefix'):self.lists(**{'accepted.html':bad})
|
||||||
|
|
||||||
|
def test_snapshot_tamper_has_no_completion_marker_and_never_fetches(self):
|
||||||
|
p=self.snap/'p001.html';p.write_bytes(p.read_bytes()+b'changed')
|
||||||
|
with patch.object(rss,'fetch',side_effect=AssertionError('network must not be used')):
|
||||||
|
with self.assertRaisesRegex(ValueError,'receipt'):rss.collect(self.root/'out',snapshots=self.snap)
|
||||||
|
self.assertFalse((self.root/'out/collection-report.json').exists())
|
||||||
|
self.assertTrue((self.root/'out/failed.json').exists())
|
||||||
|
|
||||||
|
def test_existing_output_and_unsafe_receipt_are_rejected(self):
|
||||||
|
out=self.root/'existing';out.mkdir()
|
||||||
|
with self.assertRaises(ValueError):rss.collect(out,snapshots=self.snap)
|
||||||
|
receipts=[*self.fixture['receipts'],{**self.fixture['receipts'][0],'file':'../outside'}]
|
||||||
|
(self.snap/'receipts.json').write_text(json.dumps(receipts))
|
||||||
|
with self.assertRaisesRegex(ValueError,'Unsafe'):rss.collect(self.root/'unsafe',snapshots=self.snap)
|
||||||
|
self.assertTrue((self.root/'unsafe/failed.json').exists())
|
||||||
|
self.assertFalse((self.root/'unsafe/collection-report.json').exists())
|
||||||
@@ -0,0 +1,59 @@
|
|||||||
|
"""RSS publisher/DBLP observations share one event without losing old decisions."""
|
||||||
|
from pathlib import Path
|
||||||
|
import json
|
||||||
|
import sqlite3
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from paper_atlas.db.submissions import identities_for_row
|
||||||
|
from tests.test_aamas_publication_identity import row, ingest
|
||||||
|
|
||||||
|
DOI='10.15607/rss.2026.xxii.041'
|
||||||
|
|
||||||
|
def rss(source,external,**kwargs):
|
||||||
|
return row(source,external,venue='RSS',year=2026,track='main',
|
||||||
|
identifiers=kwargs.pop('identifiers',{'doi':DOI}),**kwargs)
|
||||||
|
|
||||||
|
class RssPublicationIdentityTests(unittest.TestCase):
|
||||||
|
def test_only_verified_accepted_source_adds_publication_identity(self):
|
||||||
|
r=rss('rss_proceedings','2026/p041',identifiers={'doi':'https://doi.org/'+DOI.upper()})
|
||||||
|
self.assertEqual(identities_for_row(r),[('publication_doi',DOI)])
|
||||||
|
for status in ('rejected','desk_rejected','withdrawn','under_review','unknown'):
|
||||||
|
self.assertEqual(identities_for_row({**r,'status':status}),[])
|
||||||
|
self.assertEqual(identities_for_row({**r,'source':'unverified_rss_copy'}),[])
|
||||||
|
self.assertEqual(identities_for_row({**r,'identifiers':{'doi':'not-a-doi'}}),[])
|
||||||
|
|
||||||
|
def test_publisher_and_bibliography_both_orders_preserve_two_observations(self):
|
||||||
|
a=rss('rss_proceedings','2026/p041');b=rss('dblp','conf/rss/Example26')
|
||||||
|
for pair in ((a,b),(b,a)):
|
||||||
|
with self.subTest(first=pair[0]['source']),tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);db=root/'papers.sqlite'
|
||||||
|
for r in pair:ingest(db,root,r)
|
||||||
|
ingest(db,root,a)
|
||||||
|
with sqlite3.connect(db) as c:
|
||||||
|
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],1)
|
||||||
|
self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],1)
|
||||||
|
self.assertEqual(c.execute('SELECT count(*) FROM source_records').fetchone()[0],2)
|
||||||
|
self.assertEqual(c.execute('SELECT count(*) FROM submission_sources').fetchone()[0],2)
|
||||||
|
self.assertEqual(c.execute('SELECT count(*) FROM decisions').fetchone()[0],2)
|
||||||
|
self.assertEqual(c.execute('SELECT count(*) FROM decision_sources').fetchone()[0],2)
|
||||||
|
self.assertEqual(c.execute('SELECT scheme,value FROM submission_identifiers').fetchall(),[('publication_doi',DOI)])
|
||||||
|
metadata=[json.loads(r[0]) for r in c.execute('SELECT metadata_json FROM source_records')]
|
||||||
|
self.assertEqual({x['original_source'] for x in metadata},{'rss_proceedings','dblp'})
|
||||||
|
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(),[])
|
||||||
|
|
||||||
|
def test_prior_rejection_remains_independent_and_raw_evidence_unchanged(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root=Path(tmp);db=root/'papers.sqlite'
|
||||||
|
old=row('openreview_snapshot','old-forum',venue='ICLR',year=2025,track='main',
|
||||||
|
status='rejected',identifiers={'arxiv':'2501.00001','openreview':'old-forum'})
|
||||||
|
ingest(db,root,old)
|
||||||
|
with sqlite3.connect(db) as c:
|
||||||
|
before=c.execute('SELECT * FROM decisions').fetchall()
|
||||||
|
new=rss('rss_proceedings','2026/p041',identifiers={'arxiv':'2501.00001','doi':DOI})
|
||||||
|
ingest(db,root,new)
|
||||||
|
with sqlite3.connect(db) as c:
|
||||||
|
self.assertEqual(c.execute('SELECT * FROM decisions WHERE status=\'rejected\'').fetchall(),before)
|
||||||
|
self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],2)
|
||||||
|
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],1)
|
||||||
|
self.assertEqual(c.execute('SELECT count(*) FROM decisions WHERE status=\'accepted\'').fetchone()[0],1)
|
||||||
|
self.assertEqual(c.execute('PRAGMA foreign_key_check').fetchall(),[])
|
||||||
@@ -0,0 +1,142 @@
|
|||||||
|
import copy
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors import sdm2026 as c
|
||||||
|
|
||||||
|
|
||||||
|
class SDM2026Tests(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/sdm2026-source.json.gz').read_bytes()))
|
||||||
|
cls.message = json.loads(cls.fixture['files']['sdm2026-crossref-research-v1/isbn-works.json'])['message']
|
||||||
|
|
||||||
|
def snapshots(self, root):
|
||||||
|
for name, text in self.fixture['files'].items():
|
||||||
|
path = root/name; path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(text.encode('utf-8'))
|
||||||
|
|
||||||
|
def test_publication_census_excludes_book_and_front_matter_not_cancelled_paper(self):
|
||||||
|
papers, excluded = c.registry_papers(self.message)
|
||||||
|
self.assertEqual(len(papers), 53)
|
||||||
|
self.assertEqual({x['doi'] for x in excluded}, {c.BOOK, c.BOOK+'.fm'})
|
||||||
|
self.assertIn(c.BOOK+'.7', {p['original']['DOI'] for p in papers})
|
||||||
|
marked = [p for p in papers if p['review_flags']]
|
||||||
|
self.assertEqual({p['original']['DOI'] for p in marked}, {c.BOOK+'.12', c.BOOK+'.35'})
|
||||||
|
for p in marked:
|
||||||
|
self.assertTrue(p['title'].endswith('*'))
|
||||||
|
self.assertIn('<sup>*</sup>', p['original']['title'][0])
|
||||||
|
|
||||||
|
def test_full_collection_preserves_separate_source_versions_and_unknown_negatives(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); self.snapshots(root/'src')
|
||||||
|
report = c.collect(root/'out', snapshots=root/'src')
|
||||||
|
self.assertEqual((report['records'], report['missing_abstract_count']), (53, 53))
|
||||||
|
self.assertFalse(report['database_written']); self.assertEqual(report['network_requests'], 0)
|
||||||
|
self.assertEqual(report['public_rejected_metadata'], 'unknown_anonymous_query_http403')
|
||||||
|
self.assertEqual(report['program_registry_author_sequence_differences'], 20)
|
||||||
|
manifest, = report['files']; self.assertEqual(manifest['status'], 'partial')
|
||||||
|
rows = [json.loads(line) for line in Path(manifest['file']).read_text().splitlines()]
|
||||||
|
originals = {x['DOI']: x for x in self.message['items'] if x['type'] == 'book-chapter'}
|
||||||
|
for row in rows:
|
||||||
|
self.assertEqual(row['metadata']['registry_fields'], originals[row['external_id']])
|
||||||
|
self.assertIsNone(row['abstract']); self.assertFalse(row['tags'])
|
||||||
|
self.assertEqual(row['identifiers'], {'doi': row['external_id']})
|
||||||
|
self.assertEqual(row['submission_identifiers'], [{'scheme': 'publication_doi', 'value': row['external_id']}])
|
||||||
|
self.assertEqual((row['status'], row['track']), ('accepted', 'proceedings_unclassified'))
|
||||||
|
self.assertEqual(row['metadata']['publication_dates']['published-online'], '2026-11-12')
|
||||||
|
sidecar = report['program_observations']
|
||||||
|
self.assertTrue(sidecar['not_an_import_manifest'])
|
||||||
|
observations = [json.loads(line) for line in Path(sidecar['file']).read_text().splitlines()]
|
||||||
|
self.assertEqual(len(observations), 53)
|
||||||
|
self.assertEqual(sum(x['public_page_fields'] is not None for x in observations), 52)
|
||||||
|
for observation in observations:
|
||||||
|
self.assertNotIn('status', observation); self.assertNotIn('identifiers', observation)
|
||||||
|
self.assertEqual(observation['record_kind'], 'source_observation_not_import_record')
|
||||||
|
cancelled, = [x for x in observations if x['cancelled_presentation']]
|
||||||
|
self.assertEqual(cancelled['title'], 'A Trajectory View of Statistical Policy Comparison')
|
||||||
|
self.assertIsNone(cancelled['public_page_fields'])
|
||||||
|
# The real source has conflicting author sets under the same title.
|
||||||
|
conflict = next(x for x in observations if x['title'].startswith('Scaffold-Aware Generative'))
|
||||||
|
self.assertEqual(conflict['public_page_fields']['source_reported_authors'], ['Xin Wang', 'Qinwen Ge'])
|
||||||
|
candidate, = conflict['registry_candidates']; self.assertFalse(candidate['author_sequence_exact'])
|
||||||
|
registry_row = next(x for x in rows if x['external_id'] == candidate['registry_doi_candidate'])
|
||||||
|
self.assertEqual(registry_row['authors'], ['Xin Wang', 'Yu Wang', 'Yunchao Liu', 'Jens Meiler', 'Tyler Derr'])
|
||||||
|
# Every copied source and manifest remains byte-identical, including CRLF.
|
||||||
|
for evidence in report['evidence']:
|
||||||
|
body = (root/'out'/evidence['output_file']).read_bytes()
|
||||||
|
self.assertEqual(body, self.fixture['files'][evidence['snapshot']+'/'+evidence['file']].encode('utf-8'))
|
||||||
|
self.assertEqual(hashlib.sha256(body).hexdigest(), evidence['sha256'])
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Output exists'):
|
||||||
|
c.collect(root/'out', snapshots=root/'src')
|
||||||
|
|
||||||
|
def test_all_program_events_match_independent_inventory(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); self.snapshots(root)
|
||||||
|
loaded = {name: c.load_snapshot(root, name) for name in c.SNAPSHOTS}
|
||||||
|
papers, _ = c.registry_papers(self.message)
|
||||||
|
observations = c.program_observations(loaded, papers)
|
||||||
|
gold = {(x['session_url'], x['title']): x for x in self.fixture['program_gold']}
|
||||||
|
self.assertEqual({(x['session_url'], x['title']) for x in observations}, set(gold))
|
||||||
|
for item in observations:
|
||||||
|
original = gold[(item['session_url'], item['title'])]
|
||||||
|
for key in ('authors_literal', 'time_literal', 'cancelled_presentation'):
|
||||||
|
self.assertEqual(item[key], original[key])
|
||||||
|
|
||||||
|
def test_all_public_abstracts_and_source_reported_names_match_independent_gold(self):
|
||||||
|
files = self.fixture['files']
|
||||||
|
manifest = json.loads(files['sdm2026-public-abstract-research-v1/manifest.json'])
|
||||||
|
receipts = {r['url']: r for r in manifest['records']}
|
||||||
|
self.assertEqual(len(self.fixture['page_gold']), 52)
|
||||||
|
for gold in self.fixture['page_gold']:
|
||||||
|
receipt = receipts[gold['program_url']]
|
||||||
|
raw = files['sdm2026-public-abstract-research-v1/'+receipt['file']].encode('utf-8')
|
||||||
|
self.assertEqual(hashlib.sha256(raw).hexdigest(), gold['page_sha256'])
|
||||||
|
fields = c.talk_fields(raw, receipt['session_url'])
|
||||||
|
self.assertEqual(fields['title'], gold['title'])
|
||||||
|
self.assertEqual(fields['source_reported_authors'], gold['authors'])
|
||||||
|
self.assertEqual(fields['abstract'], gold['abstract'])
|
||||||
|
self.assertTrue(all('@' not in name for name in fields['source_reported_authors']))
|
||||||
|
|
||||||
|
def test_wrong_isbn_parent_doi_and_incomplete_census_rejected(self):
|
||||||
|
for kind in ('isbn', 'parent', 'repeat', 'type', 'year', 'author', 'markup', 'abstract', 'missing'):
|
||||||
|
with self.subTest(kind=kind):
|
||||||
|
message = copy.deepcopy(self.message)
|
||||||
|
row = next(x for x in message['items'] if x['type'] == 'book-chapter')
|
||||||
|
if kind == 'isbn': row['ISBN'] = ['9780000000000']
|
||||||
|
if kind == 'parent': row['relation']['is-part-of'][0]['id'] = 'https://doi.org/10.9999/other'
|
||||||
|
if kind == 'repeat': row['DOI'] = c.BOOK
|
||||||
|
if kind == 'type': row['type'] = 'other'
|
||||||
|
if kind == 'year': row['published']['date-parts'] = [[2025, 10, 31]]
|
||||||
|
if kind == 'author': row['author'][0]['sequence'] = 'additional'
|
||||||
|
if kind == 'markup': row['title'] = ['Changed <sup>2</sup> title']
|
||||||
|
if kind == 'abstract': row['abstract'] = 'Unreviewed new version'
|
||||||
|
if kind == 'missing': message['items'].pop()
|
||||||
|
with self.assertRaises(ValueError): c.registry_papers(message)
|
||||||
|
|
||||||
|
def test_foreign_session_and_changed_edition_rejected(self):
|
||||||
|
raw = self.fixture['files']['sdm2026-public-abstract-research-v1/162766.html'].encode()
|
||||||
|
url = 'https://meetings.siam.org/sess/dsp_programsess.cfm?SESSIONCODE=91511'
|
||||||
|
for changed in [raw.replace(b'sessioncode=91511', b'sessioncode=91507'),
|
||||||
|
raw.replace(b'2026 SIAM International Conference', b'2025 SIAM International Conference')]:
|
||||||
|
with self.assertRaises(ValueError): c.talk_fields(changed, url)
|
||||||
|
|
||||||
|
def test_corruption_preserves_failure_without_success_report(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); self.snapshots(root/'src')
|
||||||
|
(root/'src/sdm2026-public-abstract-research-v1/162766.html').write_bytes(b'corrupt')
|
||||||
|
with self.assertRaisesRegex(ValueError, 'receipt/body'):
|
||||||
|
c.collect(root/'out', snapshots=root/'src')
|
||||||
|
self.assertTrue((root/'out/failed.json').exists())
|
||||||
|
self.assertFalse((root/'out/collection-report.json').exists())
|
||||||
|
|
||||||
|
def test_source_symlink_rejected(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp); self.snapshots(root/'src')
|
||||||
|
p = root/'src/sdm2026-crossref-research-v1/isbn-works.json'
|
||||||
|
original = p.read_bytes(); p.unlink(); (root/'data.json').write_bytes(original); p.symlink_to(root/'data.json')
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Unsafe'):
|
||||||
|
c.collect(root/'out', snapshots=root/'src')
|
||||||
@@ -0,0 +1,148 @@
|
|||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
from unittest.mock import patch
|
||||||
|
|
||||||
|
from paper_atlas.audits.source_identity import audit, classify, load_inputs
|
||||||
|
from paper_atlas.postgres.client import ReadConnection
|
||||||
|
|
||||||
|
|
||||||
|
def record(**updates):
|
||||||
|
row = dict(source='reviewed_program', external_id='2026/forum-a',
|
||||||
|
title='A sufficiently specific scientific paper title', authors=['Jane Doe', 'John Smith'],
|
||||||
|
venue='NeurIPS', year=2026, track='main', status='accepted', identifiers={},
|
||||||
|
source_record_hits=[], identifier_hits=[], title_candidate_ids=[])
|
||||||
|
row.update(updates)
|
||||||
|
return row
|
||||||
|
|
||||||
|
|
||||||
|
def write_input(root, rows):
|
||||||
|
path = root / 'source.jsonl'
|
||||||
|
raw = b''.join((json.dumps(row) + '\n').encode() for row in rows)
|
||||||
|
path.write_bytes(raw)
|
||||||
|
manifest = {k: rows[0][k] for k in ('source', 'venue', 'year', 'track')}
|
||||||
|
manifest.update(count=len(rows), file_sha256=hashlib.sha256(raw).hexdigest(),
|
||||||
|
file='/ignored/untrusted/location.jsonl', status='partial')
|
||||||
|
path.with_suffix('.manifest.json').write_text(json.dumps(manifest))
|
||||||
|
return path
|
||||||
|
|
||||||
|
|
||||||
|
class CandidateTests(unittest.TestCase):
|
||||||
|
def test_duplicate_names_do_not_equal_distinct_author_multiset(self):
|
||||||
|
row = record(authors=['Jane Doe', 'Jane Doe'], title_candidate_ids=[7])
|
||||||
|
result = classify(row, {7: {'title': row['title'], 'authors': ['Jane Doe']}})
|
||||||
|
self.assertEqual(result['kind'], 'partial_author_overlap_requires_review')
|
||||||
|
self.assertIn('duplicate_normalized_author_names', result['flags'])
|
||||||
|
self.assertFalse(result['comparisons'][0]['nonempty_author_multiset_equal'])
|
||||||
|
|
||||||
|
def test_arxiv_author_representation_and_order_match_but_never_approve(self):
|
||||||
|
row = record(title_candidate_ids=[7])
|
||||||
|
result = classify(row, {7: {'title': row['title'], 'authors': [['Smith', 'John', ''], ['Doe', 'Jane', '']]}})
|
||||||
|
self.assertEqual(result['kind'], 'one_exact_title_author_multiset_requires_review')
|
||||||
|
self.assertEqual(result['decision'], 'not_approved_for_import')
|
||||||
|
|
||||||
|
def test_binding_conflict_overrides_existing_source(self):
|
||||||
|
row = record(source_record_hits=[{'paper_id': 7}], identifier_hits=[{'paper_id': 8}])
|
||||||
|
entities = {pid: {'title': row['title'], 'authors': row['authors']} for pid in (7, 8)}
|
||||||
|
self.assertEqual(classify(row, entities)['kind'], 'conflicting_existing_bindings_hold')
|
||||||
|
|
||||||
|
def test_wrong_doi_binding_retains_author_title_disagreement(self):
|
||||||
|
row = record(identifier_hits=[{'paper_id': 7}])
|
||||||
|
result = classify(row, {7: {'title': 'Completely different work', 'authors': ['Somebody Else']}})
|
||||||
|
self.assertEqual(result['kind'], 'existing_identifier_binding_requires_review')
|
||||||
|
self.assertFalse(result['comparisons'][0]['normalized_title_equal'])
|
||||||
|
self.assertEqual(result['comparisons'][0]['author_name_overlap'], [])
|
||||||
|
|
||||||
|
def test_no_hit_is_not_novelty_and_missing_authors_never_agree(self):
|
||||||
|
self.assertEqual(classify(record(), {})['kind'], 'no_exact_match_not_proof_of_novelty')
|
||||||
|
row = record(authors=[], title_candidate_ids=[7])
|
||||||
|
result = classify(row, {7: {'title': row['title'], 'authors': []}})
|
||||||
|
self.assertEqual(result['kind'], 'no_author_agreement_hold')
|
||||||
|
|
||||||
|
def test_title_candidate_cap_is_explicit(self):
|
||||||
|
row = record(title_candidate_ids=list(range(21)))
|
||||||
|
entities = {pid: {'title': row['title'], 'authors': row['authors']} for pid in range(21)}
|
||||||
|
self.assertTrue(classify(row, entities)['title_candidates_truncated'])
|
||||||
|
self.assertEqual(classify(row, entities)['kind'], 'title_candidate_limit_reached_hold')
|
||||||
|
|
||||||
|
|
||||||
|
class InputTests(unittest.TestCase):
|
||||||
|
def setUp(self):
|
||||||
|
self.temp = tempfile.TemporaryDirectory()
|
||||||
|
self.addCleanup(self.temp.cleanup)
|
||||||
|
self.root = Path(self.temp.name)
|
||||||
|
|
||||||
|
def test_raw_line_hash_and_manifest_path_not_followed(self):
|
||||||
|
path = write_input(self.root, [record()])
|
||||||
|
rows, inputs = load_inputs([path])
|
||||||
|
self.assertEqual(rows[0]['raw_line_sha256'], hashlib.sha256(path.read_bytes()).hexdigest())
|
||||||
|
self.assertEqual(inputs[0]['count'], 1)
|
||||||
|
|
||||||
|
def test_hash_tamper_rejected_before_database_connection(self):
|
||||||
|
path = write_input(self.root, [record()])
|
||||||
|
path.write_bytes(path.read_bytes().replace(b'Jane Doe', b'Jane Roe'))
|
||||||
|
with patch('paper_atlas.audits.source_identity.ReadConnection') as connection:
|
||||||
|
with self.assertRaisesRegex(ValueError, 'hash mismatch'):
|
||||||
|
audit([path], self.root / 'out', pg_service='test', pg_schema='research')
|
||||||
|
connection.assert_not_called()
|
||||||
|
|
||||||
|
def test_manifest_scope_and_count_tamper_and_duplicate_sources(self):
|
||||||
|
path = write_input(self.root, [record()])
|
||||||
|
manifest_path = path.with_suffix('.manifest.json')
|
||||||
|
manifest = json.loads(manifest_path.read_text())
|
||||||
|
for key, value in [('year', 2025), ('count', 2)]:
|
||||||
|
altered = dict(manifest, **{key: value})
|
||||||
|
manifest_path.write_text(json.dumps(altered))
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
load_inputs([path])
|
||||||
|
manifest_path.write_text(json.dumps(manifest))
|
||||||
|
with self.assertRaisesRegex(ValueError, 'Duplicate source'):
|
||||||
|
load_inputs([path, path])
|
||||||
|
|
||||||
|
def test_existing_output_never_replayed_and_failure_not_completion(self):
|
||||||
|
path = write_input(self.root, [record()])
|
||||||
|
out = self.root / 'out'
|
||||||
|
with patch('paper_atlas.audits.source_identity.ReadConnection', side_effect=RuntimeError('unavailable')):
|
||||||
|
with self.assertRaises(RuntimeError):
|
||||||
|
audit([path], out, pg_service='test', pg_schema='research')
|
||||||
|
self.assertTrue((out / 'failed.json').exists())
|
||||||
|
self.assertFalse((out / 'postgresql-preflight.json').exists())
|
||||||
|
with self.assertRaises(FileExistsError):
|
||||||
|
audit([path], out, pg_service='test', pg_schema='research')
|
||||||
|
|
||||||
|
|
||||||
|
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires the authorized remote PostgreSQL service')
|
||||||
|
class ReadOnlyIntegrationTests(unittest.TestCase):
|
||||||
|
def test_existing_binding_full_evidence_and_no_import_approval(self):
|
||||||
|
service = os.environ['PAPER_ATLAS_TEST_PG_SERVICE']
|
||||||
|
connection = ReadConnection(service, 'research')
|
||||||
|
try:
|
||||||
|
self.assertEqual(connection.execute('SHOW transaction_read_only').fetchone()[0], 'on')
|
||||||
|
hit = connection.execute("SELECT value,paper_id FROM identifiers WHERE scheme='arxiv' ORDER BY value LIMIT 1").fetchone()
|
||||||
|
self.assertIsNotNone(hit)
|
||||||
|
paper = connection.execute('SELECT title,authors_json FROM papers WHERE id=?', (hit['paper_id'],)).fetchone()
|
||||||
|
finally:
|
||||||
|
connection.close()
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp)
|
||||||
|
path = write_input(root, [record(title=paper['title'], authors=json.loads(paper['authors_json']),
|
||||||
|
identifiers={'arxiv': hit['value']})])
|
||||||
|
report = audit([path], root / 'out', pg_service=service, pg_schema='research')
|
||||||
|
self.assertTrue(report['transaction_read_only'])
|
||||||
|
self.assertFalse(report['database_written'])
|
||||||
|
self.assertEqual(report['approved_for_import'], 0)
|
||||||
|
row, = report['entries']
|
||||||
|
self.assertIn(hit['paper_id'], [r['paper_id'] for r in row['identifier_hits']])
|
||||||
|
shard = report['entities'][str(hit['paper_id'])]
|
||||||
|
raw = (root / 'out' / shard['file']).read_bytes()
|
||||||
|
self.assertEqual(hashlib.sha256(raw).hexdigest(), shard['sha256'])
|
||||||
|
packet = json.loads(gzip.decompress(raw))
|
||||||
|
self.assertEqual(packet['paper']['id'], hit['paper_id'])
|
||||||
|
self.assertEqual(packet['paper']['title'], paper['title'])
|
||||||
|
self.assertEqual(packet['paper']['authors_json'], paper['authors_json'])
|
||||||
|
self.assertIn('attempts', packet)
|
||||||
|
self.assertTrue((root / 'out/postgresql-preflight.json').is_file())
|
||||||
@@ -0,0 +1,160 @@
|
|||||||
|
"""Real 49-chapter snapshots compared with a separately frozen metadata audit."""
|
||||||
|
import copy
|
||||||
|
import gzip
|
||||||
|
import hashlib
|
||||||
|
import json
|
||||||
|
from pathlib import Path
|
||||||
|
import sqlite3
|
||||||
|
import tempfile
|
||||||
|
import unittest
|
||||||
|
|
||||||
|
from paper_atlas.collectors.springer_selected import (
|
||||||
|
BOOK, SOURCE, collect, parse_chapter, parse_toc, read_page, snapshot_index,
|
||||||
|
)
|
||||||
|
from paper_atlas.db.store import Store
|
||||||
|
from paper_atlas.ingestion.records import ingest_file
|
||||||
|
|
||||||
|
FIXTURE = Path(__file__).parent/'fixtures/springer-aistats1993-fixture.json.gz'
|
||||||
|
|
||||||
|
|
||||||
|
class SpringerSelectedTests(unittest.TestCase):
|
||||||
|
@classmethod
|
||||||
|
def setUpClass(cls):
|
||||||
|
raw = FIXTURE.read_bytes()
|
||||||
|
if hashlib.sha256(raw).hexdigest() != '94028fd37080f9d8027e5790e379cc151ee6eab17ed0d1dcc0f59e6e75f9d4d0':
|
||||||
|
raise AssertionError('Fixed source fixture differs')
|
||||||
|
cls.fixture = json.loads(gzip.decompress(raw))
|
||||||
|
|
||||||
|
def snapshots(self, root):
|
||||||
|
for group in ['toc','chapters']:
|
||||||
|
folder = root/group
|
||||||
|
folder.mkdir()
|
||||||
|
for name, text in self.fixture[group].items():
|
||||||
|
(folder/name).write_text(text)
|
||||||
|
return {'toc_snapshots':root/'toc','chapter_snapshots':root/'chapters'}
|
||||||
|
|
||||||
|
def test_all49_original_fields_against_independent_audit(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp)
|
||||||
|
report = collect(root/'out', **self.snapshots(root))
|
||||||
|
manifest, = report['files']
|
||||||
|
rows = [json.loads(s) for s in Path(manifest['file']).read_text().splitlines()]
|
||||||
|
self.assertEqual(len(rows), 49)
|
||||||
|
self.assertEqual(manifest['status'], 'partial')
|
||||||
|
self.assertEqual(manifest['missing_abstract_count'], 0)
|
||||||
|
self.assertTrue(manifest['excluded_materials'])
|
||||||
|
self.assertTrue(all(x['title'] == 'Front Matter' for x in manifest['excluded_materials']))
|
||||||
|
for row, expected in zip(rows, self.fixture['expected']):
|
||||||
|
for field in ['title','abstract']:
|
||||||
|
self.assertEqual(row[field], expected[field])
|
||||||
|
self.assertEqual(row['identifiers'], {'doi':expected['doi']})
|
||||||
|
self.assertEqual(row['submission_identifiers'], [{'scheme':'publication_doi','value':expected['doi']}])
|
||||||
|
self.assertEqual(row['external_id'], expected['doi'])
|
||||||
|
self.assertEqual(row['metadata']['authors_citation_raw'], expected['authors_citation_raw'])
|
||||||
|
self.assertEqual(row['authors'], [{'family':s.split(',',1)[0].strip(), 'given':s.split(',',1)[1].strip()} for s in expected['authors_citation_raw']])
|
||||||
|
self.assertEqual(row['metadata']['first_page'], expected['first_page'])
|
||||||
|
self.assertEqual(row['metadata']['last_page'], expected['last_page'])
|
||||||
|
self.assertEqual((row['year'],row['metadata']['bibliographic_year']), (1993,1994))
|
||||||
|
self.assertEqual(row['track'], 'selected_volume')
|
||||||
|
self.assertEqual(row['event_key'], 'aistats-1993-selected-volume-1994')
|
||||||
|
self.assertNotIn('decision_date', row)
|
||||||
|
self.assertIn('original 1993 review decision not supplied', row['raw_decision'])
|
||||||
|
for e in [row['metadata']['source_evidence'], *row['metadata']['scope_evidence']]:
|
||||||
|
self.assertEqual(hashlib.sha256((root/'out'/e['file']).read_bytes()).hexdigest(), e['sha256'])
|
||||||
|
self.assertFalse(report['database_written'])
|
||||||
|
with self.assertRaises(FileExistsError):
|
||||||
|
collect(root/'out', toc_snapshots=root/'toc', chapter_snapshots=root/'chapters')
|
||||||
|
|
||||||
|
def test_native_history_namespaces_and_idempotent_import(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp)
|
||||||
|
report = collect(root/'out', **self.snapshots(root))
|
||||||
|
manifest, = report['files']
|
||||||
|
first = json.loads(Path(manifest['file']).read_text().splitlines()[0])
|
||||||
|
db = root/'sample.sqlite'
|
||||||
|
store = Store(db)
|
||||||
|
pid, rid = store.record('prior_public','earlier','https://example.org/earlier',first['title'],first['abstract'],first['authors'])
|
||||||
|
store.decision(pid,rid,'AISTATS',1993,'main','earlier','https://example.org/earlier','rejected','original public rejection')
|
||||||
|
# Explicitly verified work identity; the short title alone must not
|
||||||
|
# force a merge or attach this publication DOI to the old attempt.
|
||||||
|
store.conn.execute('INSERT INTO identifiers VALUES(?,?,?)', ('doi',first['identifiers']['doi'],pid))
|
||||||
|
store.conn.commit()
|
||||||
|
old = store.conn.execute('SELECT * FROM decisions').fetchall()
|
||||||
|
store.conn.close()
|
||||||
|
ingest_file(db, manifest['file'])
|
||||||
|
ingest_file(db, manifest['file'])
|
||||||
|
with sqlite3.connect(db) as c:
|
||||||
|
self.assertEqual(c.execute('SELECT * FROM decisions WHERE status=\'rejected\'').fetchall(), [tuple(r) for r in old])
|
||||||
|
self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0], 49)
|
||||||
|
self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0], 50)
|
||||||
|
self.assertEqual(c.execute('SELECT count(*) FROM decisions').fetchone()[0], 50)
|
||||||
|
self.assertEqual(c.execute('SELECT e.year,e.track,e.event_key FROM editions e ORDER BY e.id').fetchall(), [(1993,'main',''),(1993,'selected_volume','aistats-1993-selected-volume-1994')])
|
||||||
|
self.assertEqual(c.execute('SELECT count(*) FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id WHERE t.namespace=?', (SOURCE+':publication_format',)).fetchone()[0], 49)
|
||||||
|
self.assertEqual(c.execute('SELECT count(*) FROM submission_identifiers WHERE scheme=\'publication_doi\'').fetchone()[0], 49)
|
||||||
|
self.assertIsNone(c.execute("SELECT 1 FROM submission_identifiers si JOIN submissions s ON s.id=si.submission_id JOIN editions e ON e.id=s.edition_id WHERE e.track='main'").fetchone())
|
||||||
|
|
||||||
|
def test_short_unanchored_title_keeps_identity_unconfirmed(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
store = Store(Path(tmp)/'identity.sqlite')
|
||||||
|
title = 'Statistical strategy: step 1'
|
||||||
|
authors = [{'given':'D. J.','family':'Hand'}]
|
||||||
|
old, _ = store.record('prior_public','earlier','https://example.org/old',title,'Same abstract',authors)
|
||||||
|
new, _ = store.record(SOURCE,BOOK+'_1','https://link.springer.com/chapter/'+BOOK+'_1',title,'Same abstract',authors,{'doi':BOOK+'_1'})
|
||||||
|
self.assertNotEqual(old, new)
|
||||||
|
self.assertEqual(store.conn.execute('SELECT count(*) FROM match_candidates WHERE method=\'exact_title_unconfirmed\'').fetchone()[0], 1)
|
||||||
|
store.conn.close()
|
||||||
|
|
||||||
|
def test_receipt_hash_path_redirect_and_content_type_guards(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp)
|
||||||
|
self.snapshots(root)
|
||||||
|
index = snapshot_index(root/'chapters')
|
||||||
|
url = 'https://link.springer.com/chapter/'+BOOK+'_1'
|
||||||
|
self.assertTrue(read_page(index,url)[0])
|
||||||
|
for key, value in [('sha256','0'*64),('file','../escape.html'),('final_url','https://example.org/chapter'),('http_status',403),('headers',{'Content-Type':'application/pdf'})]:
|
||||||
|
changed = copy.deepcopy(index)
|
||||||
|
changed[url][1][key] = value
|
||||||
|
with self.subTest(key=key), self.assertRaises((ValueError,KeyError)):
|
||||||
|
read_page(changed,url)
|
||||||
|
changed = copy.deepcopy(index)
|
||||||
|
changed[url][1]['final_url'] = url+'?page=2'
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
read_page(changed,url)
|
||||||
|
|
||||||
|
def test_missing_or_duplicate_snapshot_stops_collection(self):
|
||||||
|
with tempfile.TemporaryDirectory() as tmp:
|
||||||
|
root = Path(tmp)
|
||||||
|
args = self.snapshots(root)
|
||||||
|
path = root/'chapters/receipts.json'
|
||||||
|
d = json.loads(path.read_bytes())
|
||||||
|
original = copy.deepcopy(d)
|
||||||
|
d['responses'].pop()
|
||||||
|
path.write_text(json.dumps(d))
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
collect(root/'out', **args)
|
||||||
|
self.assertFalse((root/'out').exists())
|
||||||
|
original['responses'].append(original['responses'][0])
|
||||||
|
path.write_text(json.dumps(original))
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
snapshot_index(root/'chapters')
|
||||||
|
|
||||||
|
def test_book_doi_year_and_duplicate_citation_fields_reject_drift(self):
|
||||||
|
html = self.fixture['chapters']['chapter-01.html']
|
||||||
|
for key, value in [('citation_doi',BOOK+'_2'),('citation_inbook_title','Another Book'),('citation_publication_date','1993')]:
|
||||||
|
import re
|
||||||
|
changed, count = re.subn(r'(<meta[^>]*name="'+key+r'"[^>]*content=")[^"]*', lambda m:m[1]+value, html)
|
||||||
|
self.assertEqual(count, 1)
|
||||||
|
with self.subTest(key=key), self.assertRaises(ValueError):
|
||||||
|
parse_chapter(changed, BOOK+'_1')
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
parse_chapter(html+'<meta name="citation_doi" content="'+BOOK+'_1">', BOOK+'_1')
|
||||||
|
|
||||||
|
def test_missing_abstract_is_explicit_and_does_not_use_other_page_text(self):
|
||||||
|
html = self.fixture['chapters']['chapter-01.html'].replace('id="Abs1-content"','id="not-an-abstract"')
|
||||||
|
self.assertIsNone(parse_chapter(html, BOOK+'_1')['abstract'])
|
||||||
|
|
||||||
|
def test_toc_ignores_frontmatter_but_rejects_duplicate_publications(self):
|
||||||
|
html = self.fixture['toc']['springer-toc1.html']
|
||||||
|
self.assertEqual(len(parse_toc(html)), 17)
|
||||||
|
with self.assertRaises(ValueError):
|
||||||
|
parse_toc(html+html)
|
||||||
Reference in New Issue
Block a user