303 lines
30 KiB
YAML
303 lines
30 KiB
YAML
name: Metadata integrity checks
|
|
on:
|
|
push:
|
|
branches: [main, 'codex/**']
|
|
jobs:
|
|
metadata:
|
|
runs-on: ces-control
|
|
env:
|
|
ARCHIVE_SHA: ${{ github.sha }}
|
|
PYTHONDONTWRITEBYTECODE: '1'
|
|
steps:
|
|
- name: Verify metadata storage and query commands
|
|
run: |
|
|
python3 -B - <<'PY'
|
|
import base64, gzip, io, json, os, pathlib, re, subprocess, tarfile, tempfile, urllib.request
|
|
sha = os.environ['ARCHIVE_SHA']
|
|
if not re.fullmatch(r'[0-9a-fA-F]{40,64}', sha):
|
|
raise SystemExit('Invalid source commit')
|
|
# Public repository archive: no owner credentials or runner SSH keys.
|
|
url = 'https://gitea.entroia.com/Zayden-Jung/paper-atlas/archive/' + sha + '.tar.gz'
|
|
with urllib.request.urlopen(url, timeout=60) as response:
|
|
# Full publication-source fixtures retain original bibliographic evidence.
|
|
archive = response.read(24*1024*1024+1)
|
|
if len(archive)>24*1024*1024:
|
|
raise SystemExit('Unexpectedly large source archive')
|
|
with tempfile.TemporaryDirectory(prefix='paper-atlas-ci-') as tmp:
|
|
root = pathlib.Path(tmp)
|
|
with tarfile.open(fileobj=io.BytesIO(archive), mode='r:gz') as tar:
|
|
tar.extractall(root, filter='data')
|
|
candidates = list(root.glob('*/pyproject.toml'))
|
|
if len(candidates)!=1:
|
|
raise SystemExit('Unexpected repository archive layout')
|
|
src = candidates[0].parent
|
|
# Build and installation checks run only on the approved remote runner.
|
|
venv = root / 'venv'
|
|
subprocess.run(['python3','-B','-m','venv','--system-site-packages',str(venv)],check=True)
|
|
python = str(venv / 'bin/python')
|
|
subprocess.run([python,'-B','-m','pip','install','--only-binary=:all:','--no-compile','psycopg[binary]==3.3.8'],check=True)
|
|
testenv = os.environ.copy()
|
|
service_file = pathlib.Path('/etc/paper-atlas/pg_service.conf')
|
|
if service_file.is_file():
|
|
testenv.update(PGSERVICEFILE=str(service_file),PAPER_ATLAS_TEST_PG_SERVICE='foundry-research')
|
|
subprocess.run([python,'-B','-m','unittest','discover','-s','tests','-t','.','-v'],cwd=src,env=testenv,check=True)
|
|
wheels = root / 'wheels'
|
|
subprocess.run([python,'-B','-m','pip','wheel','--no-deps','--wheel-dir',str(wheels),str(src)],check=True)
|
|
wheel, = wheels.glob('paper_atlas-*.whl')
|
|
subprocess.run([python,'-B','-m','pip','install','--no-deps',str(wheel)],check=True)
|
|
outside = root / 'outside-checkout'
|
|
outside.mkdir()
|
|
mm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/mm2026-source.json.gz').read_bytes()))
|
|
mm_snapshots = root/'mm2026-snapshots'
|
|
for name, body in mm_fixture['files'].items():
|
|
path = mm_snapshots/name
|
|
path.parent.mkdir(parents=True,exist_ok=True)
|
|
path.write_bytes(body.encode('utf-8'))
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','mm2026-program','--snapshots',str(mm_snapshots),'--out',str(root/'installed-mm2026')],cwd=outside,check=True)
|
|
mm_report = json.loads((root/'installed-mm2026/collection-report.json').read_text())
|
|
assert (mm_report['records'],mm_report['program_observations'],mm_report['extra_presentation_observations']) == (2143,4326,2183)
|
|
assert len(mm_report['files']) == 28 and mm_report['missing_abstract_count'] == 216
|
|
assert mm_report['session_count'] == 157 and not mm_report['database_written'] and mm_report['network_requests'] == 0 and mm_report['approved_for_import'] == 0
|
|
mm_rows = {r['paper_id']:r for f in mm_report['files'] for r in map(json.loads,pathlib.Path(f['file']).read_text().splitlines())}
|
|
import hashlib
|
|
assert set(mm_rows) == {g['paper_id'] for g in mm_fixture['independent_inventory']['inventory']}
|
|
for gold in mm_fixture['independent_inventory']['inventory']:
|
|
row = mm_rows[gold['paper_id']]
|
|
assert (row['title'],row['authors'],row['affiliations'],row['track']) == (gold['title'],gold['authors'],gold['affiliations'],gold['track_literal'])
|
|
assert hashlib.sha256(row['abstract'].encode()).hexdigest() == gold['abstract_sha256']
|
|
assert [c['source_pointer'] for c in row['contexts']] == gold['locations']
|
|
assert not row['approved_for_import'] and not row['final_decision_verified'] and 'emails' not in row
|
|
mm_pub_fixture = json.loads(gzip.decompress((src/'tests/fixtures/mm2026-publications.json.gz').read_bytes()))
|
|
mm_pub_registry = root/'mm2026-publication-registry'
|
|
for name,body in mm_pub_fixture['files'].items():
|
|
path=mm_pub_registry/name
|
|
path.parent.mkdir(parents=True,exist_ok=True)
|
|
path.write_bytes(body.encode('utf-8'))
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','mm2026-publications','--registry-snapshots',str(mm_pub_registry),'--program-snapshots',str(mm_snapshots),'--parent-snapshot',str(mm_pub_registry/'parent.json'),'--out',str(root/'installed-mm2026-publications')],cwd=outside,check=True)
|
|
mm_pub_report=json.loads((root/'installed-mm2026-publications/collection-report.json').read_text())
|
|
assert (mm_pub_report['records'],mm_pub_report['unique_registered_dois'],mm_pub_report['abstracts_from_exact_program_correspondence'],mm_pub_report['output_missing_abstract_count'])==(1918,1918,1717,201)
|
|
assert mm_pub_report['unmatched_program_count']==417 and not mm_pub_report['database_written'] and mm_pub_report['approved_for_import']==0
|
|
mm_pub_rows={r['external_id']:r for f in mm_pub_report['files'] for r in map(json.loads,pathlib.Path(f['file']).read_text().splitlines())}
|
|
assert len(mm_pub_rows)==1918
|
|
for gold in mm_pub_fixture['independent_gold']:
|
|
row=mm_pub_rows[gold['doi']]
|
|
assert (row['title'],row['authors'],row['track'])==(gold['title'],gold['authors'],gold['track'])
|
|
assert (hashlib.sha256(row['abstract'].encode()).hexdigest() if row['abstract'] is not None else None)==gold['abstract_sha256']
|
|
assert row['metadata']['program_match_kind']==gold['kind'] and not row['metadata']['scientific_work_identity_approved'] and not row['metadata']['research_type_classified']
|
|
if service_file.is_file():
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','source-identities','--files',*[f['file'] for f in mm_pub_report['files']],'--pg-service','foundry-research','--pg-schema','research','--out',str(root/'installed-mm2026-publication-preflight')],cwd=outside,env=testenv,check=True)
|
|
mm_pub_preflight=json.loads((root/'installed-mm2026-publication-preflight/postgresql-preflight.json').read_text())
|
|
assert mm_pub_preflight['records']==1918 and mm_pub_preflight['transaction_read_only'] and not mm_pub_preflight['database_written'] and mm_pub_preflight['approved_for_import']==0
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','source-identities','--mm-program-report',str(root/'installed-mm2026/collection-report.json'),'--pg-service','foundry-research','--pg-schema','research','--out',str(root/'installed-mm2026-preflight')],cwd=outside,env=testenv,check=True)
|
|
mm_preflight = json.loads((root/'installed-mm2026-preflight/postgresql-preflight.json').read_text())
|
|
assert mm_preflight['records'] == 2143 and len(mm_preflight['inputs']) == 28
|
|
assert mm_preflight['transaction_read_only'] and not mm_preflight['database_written'] and mm_preflight['approved_for_import'] == 0
|
|
assert sum(len(r['program_contexts']) for r in mm_preflight['entries']) == 4326
|
|
assert all(r['status'] == 'unknown' and not r['identifiers'] and not r['contribution_type_verified'] and not r['publication_identity_verified'] and not r['final_decision_verified'] for r in mm_preflight['entries'])
|
|
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--help'],cwd=outside,check=True)
|
|
eccv_fixture = json.loads(gzip.decompress((src/'tests/fixtures/eccv2026-source.json.gz').read_bytes()))
|
|
eccv_official = root/'eccv2026-official'
|
|
eccv_papers = root/'eccv2026-paper-samples'
|
|
eccv_official.mkdir(); eccv_papers.mkdir()
|
|
for name, html in eccv_fixture['official_pages'].items():
|
|
(eccv_official/name).write_bytes(html.encode('utf-8'))
|
|
(eccv_official/'manifest.json').write_bytes(eccv_fixture['official_manifest'].encode('utf-8'))
|
|
for pid, html in eccv_fixture['paper_pages'].items():
|
|
(eccv_papers/(pid+'.html')).write_bytes(html.encode('utf-8'))
|
|
eccv_encoding = eccv_fixture['encoding_sample']
|
|
(eccv_papers/(eccv_encoding['paper_id']+'.html')).write_bytes(eccv_encoding['html'].encode('utf-8'))
|
|
eccv_fixture['paper_manifest']['records'].append({**eccv_encoding['receipt'], 'paper_id': eccv_encoding['paper_id'], 'file': eccv_encoding['paper_id']+'.html'})
|
|
eccv_literal = eccv_fixture['literal_title_sample']
|
|
(eccv_papers/(eccv_literal['paper_id']+'.html')).write_bytes(eccv_literal['html'].encode('utf-8'))
|
|
eccv_fixture['paper_manifest']['records'].append({**eccv_literal['receipt'], 'paper_id': eccv_literal['paper_id'], 'file': eccv_literal['paper_id']+'.html'})
|
|
(eccv_papers/'manifest.json').write_text(json.dumps(eccv_fixture['paper_manifest']))
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','eccv2026-program','--official-snapshots',str(eccv_official),'--paper-snapshots',str(eccv_papers),'--out',str(root/'installed-eccv2026')],cwd=outside,check=True)
|
|
eccv_report = json.loads((root/'installed-eccv2026/collection-report.json').read_text())
|
|
assert eccv_report['records'] == 5 and eccv_report['expected_program_ids'] == 2834
|
|
assert eccv_report['duplicate_program_occurrences'] == 17 and len(eccv_report['uncollected_program_ids']) == 2829
|
|
assert not eccv_report['database_written'] and eccv_report['files'][0]['track'] == 'program_unclassified'
|
|
eccv_rows = [json.loads(line) for line in pathlib.Path(eccv_report['files'][0]['file']).read_text().splitlines()]
|
|
eccv_encoded_row = next(row for row in eccv_rows if row['external_id'] == '2026/4199')
|
|
assert eccv_encoded_row['authors'] == eccv_encoding['gold']['authors']
|
|
assert 'jsonld_display_text_normalized_requires_review' in eccv_encoded_row['metadata']['public_page_fields']['review_flags']
|
|
eccv_literal_row = next(row for row in eccv_rows if row['external_id'] == '2026/3844')
|
|
assert eccv_literal_row['title'] == eccv_literal['gold']['visible_title']
|
|
assert '&' in eccv_literal_row['title']
|
|
icassp_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icassp2026-source.json.gz').read_bytes()))
|
|
icassp_snapshots = root/'icassp2026-snapshots'
|
|
icassp_snapshots.mkdir()
|
|
for name, encoded in icassp_fixture['files'].items():
|
|
target = icassp_snapshots/name
|
|
target.parent.mkdir(parents=True,exist_ok=True)
|
|
target.write_bytes(base64.b64decode(encoded))
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','icassp2026-registry','--snapshots',str(icassp_snapshots),'--out',str(root/'installed-icassp2026')],cwd=outside,check=True)
|
|
icassp_report = json.loads((root/'installed-icassp2026/collection-report.json').read_text())
|
|
assert icassp_report['records'] == 4582 and icassp_report['missing_abstract_count'] == 4582
|
|
assert icassp_report['single_name_author_fields'] == 23
|
|
assert len(icassp_report['excluded_nonpaper_items']) == 7 and len(icassp_report['excluded_other_conference_dois']) == 56
|
|
assert not icassp_report['database_written'] and icassp_report['network_requests'] == 0 and icassp_report['approved_for_import'] == 0
|
|
icassp_rows = [json.loads(line) for line in pathlib.Path(icassp_report['files'][0]['file']).read_text().splitlines()]
|
|
assert [{'doi':r['external_id'],'title':r['title'],'authors':r['authors']} for r in icassp_rows] == sorted(icassp_fixture['independent_gold'],key=lambda r:r['doi'])
|
|
assert all(r['abstract'] is None and r['track'] == 'proceedings_unclassified' for r in icassp_rows)
|
|
sdm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/sdm2026-source.json.gz').read_bytes()))
|
|
sdm_snapshots = root/'sdm2026-snapshots'
|
|
for name, text in sdm_fixture['files'].items():
|
|
target = sdm_snapshots/name
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_bytes(text.encode('utf-8'))
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','sdm2026-proceedings','--snapshots',str(sdm_snapshots),'--out',str(root/'installed-sdm2026')],cwd=outside,check=True)
|
|
sdm_report = json.loads((root/'installed-sdm2026/collection-report.json').read_text())
|
|
assert sdm_report['records'] == 53 and sdm_report['missing_abstract_count'] == 53
|
|
assert not sdm_report['database_written'] and sdm_report['network_requests'] == 0
|
|
assert sdm_report['program_observations']['count'] == 53 and sdm_report['program_observations']['abstract_pages'] == 52
|
|
assert sdm_report['program_registry_author_sequence_differences'] == 20
|
|
sdm_rows = [json.loads(line) for line in pathlib.Path(sdm_report['files'][0]['file']).read_text().splitlines()]
|
|
assert len(sdm_rows) == 53 and all(row['abstract'] is None for row in sdm_rows)
|
|
sdm_observations = [json.loads(line) for line in pathlib.Path(sdm_report['program_observations']['file']).read_text().splitlines()]
|
|
assert sum(row['cancelled_presentation'] for row in sdm_observations) == 1
|
|
assert all('status' not in row and 'identifiers' not in row for row in sdm_observations)
|
|
icdm_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icdm2026-source.json.gz').read_bytes()))
|
|
icdm_snapshots = root/'icdm2026-snapshots'
|
|
for name, text in icdm_fixture['files'].items():
|
|
target = icdm_snapshots/name
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
target.write_bytes(text.encode('utf-8'))
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','icdm2026-program','--snapshots',str(icdm_snapshots),'--out',str(root/'installed-icdm2026')],cwd=outside,check=True)
|
|
icdm_report = json.loads((root/'installed-icdm2026/collection-report.json').read_text())
|
|
assert icdm_report['records'] == 324 and icdm_report['missing_abstract_count'] == 324
|
|
assert icdm_report['track_counts'] == {'research':241,'applied':83}
|
|
assert icdm_report['presentation_counts'] == {'long':168,'short':156}
|
|
assert not icdm_report['database_written'] and icdm_report['network_requests'] == 0
|
|
icdm_rows = [json.loads(line) for f in icdm_report['files'] for line in pathlib.Path(f['file']).read_text().splitlines()]
|
|
assert len(icdm_rows) == 324 and all(row['abstract'] is None and not row['identifiers'] and not row['submission_identifiers'] for row in icdm_rows)
|
|
assert all(row['metadata']['paper_type'] is None and not row['metadata']['original_decision_signature_verified'] for row in icdm_rows)
|
|
icaps_fixture = json.loads(gzip.decompress((src/'tests/fixtures/icaps2026-source.json.gz').read_bytes()))
|
|
icaps_snapshots = root/'icaps2026-snapshots'
|
|
icaps_snapshots.mkdir()
|
|
for name, html in icaps_fixture['pages'].items():
|
|
(icaps_snapshots/name).write_text(html)
|
|
(icaps_snapshots/'manifest.json').write_text(json.dumps(icaps_fixture['manifest']))
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','icaps2026-program','--snapshots',str(icaps_snapshots),'--out',str(root/'installed-icaps2026')],cwd=outside,check=True)
|
|
icaps_report = json.loads((root/'installed-icaps2026/collection-report.json').read_text())
|
|
assert icaps_report['records'] == 90 and icaps_report['previously_published_presentations'] == 10
|
|
assert icaps_report['missing_abstract_count'] == 90 and not icaps_report['database_written']
|
|
assert icaps_report['files'][0]['track'] == 'main_tracks_unclassified'
|
|
registry_snapshots = root/'icaps2026-registry-snapshots'
|
|
registry_snapshots.mkdir()
|
|
(registry_snapshots/'crossref-filtered.json').write_bytes(gzip.decompress((src/'tests/fixtures/icaps2026-crossref.json.gz').read_bytes()))
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','icaps2026-registry','--snapshots',str(registry_snapshots),'--out',str(root/'installed-icaps2026-registry')],cwd=outside,check=True)
|
|
registry_report = json.loads((root/'installed-icaps2026-registry/collection-report.json').read_text())
|
|
assert registry_report['records'] == 90 and registry_report['missing_abstract_count'] == 0
|
|
assert len(registry_report['excluded_volume35_dois']) == 1 and not registry_report['database_written']
|
|
rss_fixture = json.loads(gzip.decompress((src/'tests/fixtures/rss2026-source.json.gz').read_bytes()))
|
|
rss_snapshots = root/'rss2026-snapshots'
|
|
rss_snapshots.mkdir()
|
|
for name, html in rss_fixture['pages'].items():
|
|
(rss_snapshots/name).write_bytes(html.encode('utf-8'))
|
|
(rss_snapshots/'receipts.json').write_text(json.dumps(rss_fixture['receipts']))
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','rss2026-proceedings','--snapshots',str(rss_snapshots),'--out',str(root/'installed-rss2026')],cwd=outside,check=True)
|
|
assert json.loads((root/'installed-rss2026/collection-report.json').read_text())['records'] == 210
|
|
recsys_fixture = json.loads(gzip.decompress((src/'tests/fixtures/recsys2026-source.json.gz').read_bytes()))
|
|
recsys_snapshots = root/'recsys2026-snapshots'
|
|
recsys_snapshots.mkdir()
|
|
for name,encoded in recsys_fixture['files'].items():
|
|
path = recsys_snapshots/name
|
|
path.parent.mkdir(parents=True,exist_ok=True)
|
|
path.write_bytes(base64.b64decode(encoded))
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','recsys2026-program','--snapshots',str(recsys_snapshots),'--out',str(root/'installed-recsys2026')],cwd=outside,check=True)
|
|
recsys_report = json.loads((root/'installed-recsys2026/collection-report.json').read_text())
|
|
assert (recsys_report['records'],recsys_report['unique_dois'],recsys_report['missing_abstract_count']) == (263,260,0)
|
|
assert recsys_report['duplicate_doi_observations'] == 6 and len(recsys_report['repeated_dois']) == 3
|
|
assert not recsys_report['database_written'] and recsys_report['network_requests'] == 0 and recsys_report['approved_for_import'] == 0
|
|
recsys_rows = [json.loads(line) for manifest in recsys_report['files'] for line in pathlib.Path(manifest['file']).read_text().splitlines()]
|
|
gold_keys = recsys_fixture['independent_gold'][0].keys()
|
|
assert [{k:r['metadata']['program_fields'][k] for k in gold_keys} for r in recsys_rows] == recsys_fixture['independent_gold']
|
|
ismir_fixture = json.loads(gzip.decompress((src/'tests/fixtures/ismir2026-source.json.gz').read_bytes()))
|
|
ismir_snapshots = root/'ismir2026-snapshots'
|
|
ismir_snapshots.mkdir()
|
|
for name,encoded in ismir_fixture['files'].items():
|
|
target = ismir_snapshots/name
|
|
target.parent.mkdir(parents=True,exist_ok=True)
|
|
target.write_bytes(base64.b64decode(encoded))
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','ismir2026-program','--snapshots',str(ismir_snapshots),'--out',str(root/'installed-ismir2026')],cwd=outside,check=True)
|
|
ismir_report = json.loads((root/'installed-ismir2026/collection-report.json').read_text())
|
|
assert (ismir_report['records'],ismir_report['missing_abstract_count'],ismir_report['review_optin_yes'],ismir_report['review_optin_no']) == (140,0,101,39)
|
|
assert ismir_report['literal_title_version_differences'] == 2 and ismir_report['author_sequence_version_differences'] == 3
|
|
assert not ismir_report['database_written'] and ismir_report['network_requests'] == 0 and ismir_report['approved_for_import'] == 0
|
|
ismir_rows = [json.loads(line) for line in pathlib.Path(ismir_report['files'][0]['file']).read_text().splitlines()]
|
|
assert [(r['external_id'],r['title'],r['authors'],r['abstract']) for r in ismir_rows] == [('2026/'+g['program_id'],g['title'],g['authors'],g['abstract']) for g in ismir_fixture['independent_gold']]
|
|
assert all(not r['identifiers'] and not r['submission_identifiers'] and r['metadata']['paper_type'] is None for r in ismir_rows)
|
|
corl_fixture = json.loads(gzip.decompress((src/'tests/fixtures/corl2026-source.json.gz').read_bytes()))
|
|
corl_official = root/'corl2026-official'
|
|
corl_archive = root/'corl2026-archive'
|
|
corl_official.mkdir()
|
|
corl_archive.mkdir()
|
|
(corl_official/'accepted.html').write_text(corl_fixture['official_html'])
|
|
(corl_official/'research.json').write_text(json.dumps(corl_fixture['research']))
|
|
(corl_archive/'explorer-paper-metadata-filtered-v1.json').write_text(corl_fixture['filtered'])
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','corl2026-program','--official-snapshots',str(corl_official),'--archive-snapshots',str(corl_archive),'--out',str(root/'installed-corl2026')],cwd=outside,check=True)
|
|
corl_report = json.loads((root/'installed-corl2026/collection-report.json').read_text())
|
|
assert corl_report['records'] == 687 and corl_report['presentations'] == 780
|
|
assert len(corl_report['title_variants']) == 34 and corl_report['public_rejected_availability'] == 'unknown'
|
|
assert not corl_report['database_written'] and not corl_report['pdfs_downloaded']
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','www-archive','--help'],cwd=outside,check=True)
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--help'],cwd=outside,check=True)
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','neurips-program','--help'],cwd=outside,check=True)
|
|
neurips_snapshots = root/'neurips2026-program-snapshots'
|
|
neurips_snapshots.mkdir()
|
|
(neurips_snapshots/'program-json.response').write_bytes(gzip.decompress((src/'tests/fixtures/neurips2026-program.json.gz').read_bytes()))
|
|
(neurips_snapshots/'receipts.json').write_bytes((src/'tests/fixtures/neurips2026-receipts.json').read_bytes())
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','neurips-program','--snapshots',str(neurips_snapshots),'--out',str(root/'installed-neurips2026-program')],cwd=outside,check=True)
|
|
neurips_report = json.loads((root/'installed-neurips2026-program/neurips-program-2026.json').read_text())
|
|
assert neurips_report['records'] == 9004 and len(neurips_report['held_presentations']) == 139
|
|
assert len(neurips_report['exact_duplicate_event_rows']) == 1 and neurips_report['folded_extra_presentation_rows'] == 132
|
|
assert not neurips_report['database_written'] and not neurips_report['pdfs_downloaded']
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','dblp-family','--help'],cwd=outside,check=True)
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--year','2025','--snapshots',str(src/'tests/fixtures/aamas2025'),'--out',str(root/'installed-aamas2025')],cwd=outside,check=True)
|
|
aamas2026 = json.loads(gzip.decompress((src/'tests/fixtures/aamas2026-fixture.json.gz').read_bytes()))
|
|
aamas2026_snapshots = root/'aamas2026-snapshots'
|
|
aamas2026_snapshots.mkdir()
|
|
(aamas2026_snapshots/'contents.html').write_text(aamas2026['contents'])
|
|
(aamas2026_snapshots/'receipts.json').write_text(json.dumps(aamas2026['receipts']))
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','aamas-proceedings','--year','2026','--snapshots',str(aamas2026_snapshots),'--out',str(root/'installed-aamas2026')],cwd=outside,check=True)
|
|
aamas2026_report = json.loads((root/'installed-aamas2026/aamas-proceedings-2026.json').read_text())
|
|
assert aamas2026_report['records'] == 639 and len(aamas2026_report['held_publications']) == 1
|
|
assert aamas2026_report['status_counts'] == {'accepted': 638, 'withdrawn': 1}
|
|
assert not aamas2026_report['database_written'] and not aamas2026_report['pdfs_downloaded']
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','iclr-archive','--help'],cwd=outside,check=True)
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','acl','--help'],cwd=outside,check=True)
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','springer-selected','--help'],cwd=outside,check=True)
|
|
springer = json.loads(gzip.decompress((src/'tests/fixtures/springer-aistats1993-fixture.json.gz').read_bytes()))
|
|
for group in ('toc','chapters'):
|
|
folder = root/'springer-snapshots'/group
|
|
folder.mkdir(parents=True)
|
|
for name, content in springer[group].items():
|
|
(folder/name).write_text(content)
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','springer-selected','--toc-snapshots',str(root/'springer-snapshots/toc'),'--chapter-snapshots',str(root/'springer-snapshots/chapters'),'--out',str(root/'installed-springer')],cwd=outside,check=True)
|
|
springer_report = json.loads((root/'installed-springer/collection-report.json').read_bytes())
|
|
assert springer_report['records'] == 49 and not springer_report['database_written']
|
|
springer_manifest, = springer_report['files']
|
|
assert springer_manifest['status'] == 'partial' and springer_manifest['missing_abstract_count'] == 0
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','postgres','--help'],cwd=outside,check=True)
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','edition-tracks','--help'],cwd=outside,check=True)
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','joint-editions','--help'],cwd=outside,check=True)
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','event-identity','--help'],cwd=outside,check=True)
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','import','records','--help'],cwd=outside,check=True)
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','import','arxiv','--help'],cwd=outside,check=True)
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','backfill','--help'],cwd=outside,check=True)
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','public-openreview','--help'],cwd=outside,check=True)
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','coverage','--help'],cwd=outside,check=True)
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','histories','--help'],cwd=outside,check=True)
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','source-identities','--help'],cwd=outside,check=True)
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','curate','identifiers','--help'],cwd=outside,check=True)
|
|
subprocess.run([python,'-I','-B','-c','from paper_atlas.postgres.writer import PostgresStore, WriteConnection'],cwd=outside,check=True)
|
|
smoke = "from paper_atlas.db.store import Store; from paper_atlas.config import DEFAULT_REGISTRY; import json; assert {'COLM','MM','MLSys'} <= {v['acronym'] for v in json.loads(DEFAULT_REGISTRY.read_text())['venues']}; s=Store('installed.sqlite'); s.conn.close()"
|
|
subprocess.run([python,'-I','-B','-c',smoke],cwd=outside,check=True)
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','query','--db','installed.sqlite','--stats'],cwd=outside,check=True)
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','annotate','init','--db','installed.sqlite'],cwd=outside,check=True)
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','annotate','list','--db','installed.sqlite','--paper','1'],cwd=outside,check=True)
|
|
subprocess.run([python,'-I','-B','-m','paper_atlas','audit','histories','--db','installed.sqlite','--examples','0'],cwd=outside,check=True)
|
|
PY
|