219 lines
16 KiB
Python
219 lines
16 KiB
Python
from contextlib import redirect_stdout
|
|
import io
|
|
import os
|
|
import uuid
|
|
import gzip
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
import tempfile
|
|
import unittest
|
|
|
|
from paper_atlas.collectors import mm_publications2026 as c
|
|
|
|
|
|
class MMPublications2026Tests(unittest.TestCase):
|
|
@classmethod
|
|
def setUpClass(cls):
|
|
fixtures=Path(__file__).parent/'fixtures'
|
|
cls.fixture=json.loads(gzip.decompress((fixtures/'mm2026-publications.json.gz').read_bytes()))
|
|
cls.program_fixture=json.loads(gzip.decompress((fixtures/'mm2026-source.json.gz').read_bytes()))
|
|
cls.temp=tempfile.TemporaryDirectory();cls.root=Path(cls.temp.name)
|
|
cls.materialize(cls.root/'registry',cls.fixture['files'])
|
|
cls.materialize(cls.root/'program',cls.program_fixture['files'])
|
|
cls.out=cls.root/'out'
|
|
cls.report=c.collect(cls.out,registry_snapshots=cls.root/'registry',program_snapshots=cls.root/'program',parent_snapshot=cls.root/'registry/parent.json')
|
|
cls.rows=[json.loads(line) for f in cls.report['files'] for line in Path(f['file']).read_bytes().splitlines()]
|
|
cls.by_doi={r['external_id']:r for r in cls.rows}
|
|
|
|
@classmethod
|
|
def tearDownClass(cls):
|
|
cls.temp.cleanup()
|
|
|
|
@staticmethod
|
|
def materialize(root,files):
|
|
for name,body in files.items():
|
|
path=root/name;path.parent.mkdir(parents=True,exist_ok=True);path.write_bytes(body.encode('utf-8'))
|
|
|
|
def test_all1918_publications_against_independent_registry_program_gold(self):
|
|
self.assertEqual(len(self.rows),1918);self.assertEqual(len(self.by_doi),1918)
|
|
self.assertEqual(set(self.by_doi),{g['doi'] for g in self.fixture['independent_gold']})
|
|
for g in self.fixture['independent_gold']:
|
|
r=self.by_doi[g['doi']];m=r['metadata'];p=m['program_fields']
|
|
self.assertEqual((r['title'],r['authors'],r['track']),(g['title'],g['authors'],g['track']))
|
|
self.assertEqual(m['program_match_kind'],g['kind'])
|
|
self.assertEqual(m['registry_source_evidence']['json_pointer'],'/message/items/'+str(g['source_position']))
|
|
self.assertEqual(m['registry_source_evidence']['file'],'evidence/registry/'+g['source_page'])
|
|
self.assertEqual(m['registry_source_evidence']['canonical_object_json_sha256'],g['registry_object_sha256'])
|
|
self.assertEqual(p['paper_id'] if p else None,g['program_id'])
|
|
self.assertEqual([x['source_pointer'] for x in p['contexts']] if p else [],g['program_context_pointers'])
|
|
|
|
def test_abstract_source_is_separate_and_verbatim_not_fabricated_registry_abstract(self):
|
|
count=0
|
|
for g in self.fixture['independent_gold']:
|
|
r=self.by_doi[g['doi']];m=r['metadata']
|
|
self.assertFalse(m['registry_abstract_available']);self.assertFalse(m['registry_fields'].get('abstract'))
|
|
if g['abstract_sha256'] is not None:
|
|
count+=1;self.assertEqual(hashlib.sha256(r['abstract'].encode()).hexdigest(),g['abstract_sha256'])
|
|
self.assertEqual(r['abstract'],m['program_fields']['abstract'])
|
|
self.assertEqual(m['abstract_source'],'official_program_exact_metadata_correspondence')
|
|
else:
|
|
self.assertIsNone(r['abstract']);self.assertEqual(m['abstract_source'],'not_available_from_reviewed_sources')
|
|
self.assertEqual(count,1717);self.assertEqual(self.report['output_missing_abstract_count'],201)
|
|
|
|
def test_all_program_tracks_and_4326_contexts_survive_in_matches_or_unmatched(self):
|
|
programs={r['metadata']['program_fields']['paper_id']:r['metadata']['program_fields'] for r in self.rows if r['metadata']['program_fields']}
|
|
for p in self.report['unmatched_program']:
|
|
self.assertNotIn(p['paper_id'],programs);programs[p['paper_id']]=p
|
|
self.assertEqual(len(programs),2143);self.assertEqual(self.report['unmatched_program_count'],417)
|
|
gold=self.program_fixture['independent_inventory']['inventory']
|
|
self.assertEqual(set(programs),{g['paper_id'] for g in gold})
|
|
for g in gold:
|
|
p=programs[g['paper_id']]
|
|
self.assertEqual((p['title'],p['authors'],p['track']),(g['title'],g['authors'],g['track_literal']))
|
|
self.assertEqual([x['source_pointer'] for x in p['contexts']],g['locations'])
|
|
self.assertEqual(sum(len(p['contexts']) for p in programs.values()),4326)
|
|
self.assertEqual(len({p['track'] for p in programs.values()}),28)
|
|
|
|
def test_no_person_alias_original_submission_decision_or_research_type_inference(self):
|
|
for r in self.rows:
|
|
doi=r['external_id'];m=r['metadata']
|
|
self.assertEqual(r['identifiers'],{'doi':doi})
|
|
self.assertEqual(r['submission_identifiers'],[{'scheme':'publication_doi','value':doi}])
|
|
self.assertEqual((r['source'],r['venue'],r['year']),(c.SOURCE,'MM',2026))
|
|
self.assertEqual(r['status'],'accepted');self.assertIn('not an original signed decision',r['raw_decision'])
|
|
self.assertFalse(m['original_decision_signature_verified']);self.assertFalse(m['research_type_classified'])
|
|
self.assertFalse(m['scientific_work_identity_approved'])
|
|
self.assertTrue({'decided_at','decision_date','source_decision_id','paper_type'}.isdisjoint(r))
|
|
self.assertIn({'namespace':c.SOURCE+':crossref:type','value':'proceedings-article'},r['tags'])
|
|
self.assertTrue({x['namespace'] for x in r['tags']} <= {c.SOURCE+':crossref:type',c.SOURCE+':mm_program:track'})
|
|
for track in ('interactive_art','industry_talk','demo_and_video'):
|
|
self.assertTrue(any(r['track']==track for r in self.rows))
|
|
variants=[r for r in self.rows if r['track']=='proceedings_unclassified']
|
|
self.assertEqual(len(variants),192)
|
|
self.assertTrue(all(r['metadata']['program_fields'] is None and r['abstract'] is None for r in variants))
|
|
|
|
def test_original_bibliography_evidence_kept_without_duplicating_reference_payload(self):
|
|
authors_by_page={}
|
|
for name,body in self.fixture['files'].items():
|
|
actual=self.out/'evidence/registry'/('parent.json' if name=='parent.json' else Path(name).name)
|
|
self.assertEqual(actual.read_bytes(),body.encode())
|
|
if re.fullmatch(r'page-[0-9]+\.json',Path(name).name):
|
|
authors_by_page[str(actual.relative_to(self.out))]=[row['author'] for row in json.loads(body)['message']['items']]
|
|
for name,body in self.program_fixture['files'].items():
|
|
self.assertEqual((self.out/'evidence/program'/Path(name).name).read_bytes(),body.encode())
|
|
for r in self.rows:
|
|
m=r['metadata'];self.assertNotIn('reference',m['registry_fields'])
|
|
self.assertEqual(m['registry_fields']['title'],[r['title']])
|
|
self.assertEqual(m['registry_fields']['author'],authors_by_page[m['registry_source_evidence']['file']][int(m['registry_source_evidence']['json_pointer'].rsplit('/',1)[-1])])
|
|
for f in self.report['files']:
|
|
raw=Path(f['file']).read_bytes();self.assertEqual(hashlib.sha256(raw).hexdigest(),f['file_sha256'])
|
|
self.assertEqual(len(raw.splitlines()),f['count']);self.assertEqual(f['status'],'partial')
|
|
|
|
def test_eight_single_name_fields_preserved_without_splitting(self):
|
|
n=0
|
|
for g in self.fixture['independent_gold']:
|
|
r=self.by_doi[g['doi']];flags=r['metadata']['review_flags']
|
|
actual=[x['author_position'] for x in flags if x['kind']=='single_registered_name_field_not_split']
|
|
self.assertEqual(actual,g['single_name_positions']);n+=len(actual)
|
|
for i in actual:
|
|
self.assertEqual(r['authors'][i-1],r['metadata']['registry_fields']['author'][i-1]['family'].strip())
|
|
self.assertEqual(n,8)
|
|
|
|
def test_structured_title_markup_retained_and_flagged_without_flattening(self):
|
|
count=0
|
|
for g in self.fixture['independent_gold']:
|
|
if re.search(r'<[^>]+>|&[A-Za-z#0-9]+;',g['title']):
|
|
count+=1;r=self.by_doi[g['doi']]
|
|
self.assertEqual(r['title'],g['title'])
|
|
self.assertIn({'kind':'literal_title_markup_or_entity_retained_requires_review'},r['metadata']['review_flags'])
|
|
self.assertGreater(count,0);self.assertEqual(count,self.report['registered_title_markup_flags'])
|
|
|
|
def test_corrupt_registry_parent_or_program_keeps_failure_not_complete_output(self):
|
|
for kind in ('registry_manifest','registry_page','parent','program'):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root=Path(tmp);self.materialize(root/'registry',self.fixture['files']);self.materialize(root/'program',self.program_fixture['files'])
|
|
path={'registry_manifest':root/'registry'/c.SNAPSHOT/'manifest.json','registry_page':root/'registry'/c.SNAPSHOT/'page-001.json','parent':root/'registry/parent.json','program':root/'program'/c.program.SNAPSHOT/'program-data.json'}[kind]
|
|
path.write_bytes(path.read_bytes()+b' ')
|
|
with self.subTest(kind=kind),self.assertRaises(ValueError):
|
|
c.collect(root/'out',registry_snapshots=root/'registry',program_snapshots=root/'program',parent_snapshot=root/'registry/parent.json')
|
|
self.assertTrue((root/'out/failed.json').exists());self.assertFalse((root/'out/collection-report.json').exists())
|
|
|
|
def test_symlink_source_rejected_and_no_previous_output_overwrite(self):
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root=Path(tmp);self.materialize(root/'registry',self.fixture['files'])
|
|
path=root/'registry/parent.json';original=root/'registry/actual.json';path.rename(original);path.symlink_to(original)
|
|
with self.assertRaisesRegex(ValueError,'Unsafe'):
|
|
c.collect(root/'out',registry_snapshots=root/'registry',program_snapshots=self.root/'program',parent_snapshot=path)
|
|
with self.assertRaisesRegex(ValueError,'Output exists'):
|
|
c.collect(self.out,registry_snapshots=self.root/'registry',program_snapshots=self.root/'program',parent_snapshot=self.root/'registry/parent.json')
|
|
|
|
def test_complete_report_retains_source_partitions_not_import_or_final_dedup_claim(self):
|
|
self.assertEqual(self.report['program_match_counts'],{'unique_program_title_fullauthor_agreement_not_identity_approval':1726,'program_author_version_requires_review':122,'no_exact_program_title_requires_review':70})
|
|
self.assertEqual(self.report['registry_missing_abstract_count'],1918)
|
|
self.assertEqual(self.report['abstracts_from_exact_program_correspondence'],1717)
|
|
self.assertEqual(sum(f['count'] for f in self.report['files']),1918)
|
|
self.assertEqual(self.report['approved_for_import'],0);self.assertFalse(self.report['database_written'])
|
|
self.assertFalse(self.report['pdfs_downloaded']);self.assertEqual(self.report['network_requests'],0)
|
|
self.assertIn('unknown',self.report['public_negative_availability'])
|
|
|
|
|
|
def assert_native_ingestion(self, pg_service=None):
|
|
# An isolated synthetic database exercises the writer contract, not a
|
|
# production identity approval. All source rows/track partitions run.
|
|
from paper_atlas.db.store import Store
|
|
from paper_atlas.db.backend import open_store
|
|
from paper_atlas.ingestion.records import ingest_file
|
|
from paper_atlas.postgres.client import connect
|
|
from paper_atlas.postgres.migration import migrate
|
|
from paper_atlas.postgres.schema import quote
|
|
with tempfile.TemporaryDirectory() as tmp:
|
|
root=Path(tmp);db=root/'seed.sqlite';seed=self.rows[0]
|
|
store=Store(db)
|
|
paper,record=store.record('arxiv','synthetic-mm-anchor','https://example.org/arxiv',
|
|
seed['title'],'Existing canonical abstract',seed['authors'],seed['identifiers'])
|
|
store.decision(paper,record,'ICLR',2025,'main','synthetic-old-forum',
|
|
'https://example.org/decision','rejected','Reject')
|
|
before=tuple(store.conn.execute('SELECT * FROM papers WHERE id=?',(paper,)).fetchone())
|
|
store.conn.commit();store.conn.close()
|
|
schema='pa_mm_pub_'+uuid.uuid4().hex[:12] if pg_service else None
|
|
try:
|
|
with redirect_stdout(io.StringIO()):
|
|
if pg_service:migrate(db,pg_service,root/'migration.json',schema=schema,min_free_gib=0)
|
|
for f in self.report['files']:
|
|
ingest_file(None if pg_service else db,f['file'],pg_service=pg_service,pg_schema=schema)
|
|
store=open_store(None if pg_service else db,pg_service=pg_service,pg_schema=schema)
|
|
try:
|
|
conn=store.conn
|
|
self.assertEqual(tuple(conn.execute('SELECT * FROM papers WHERE id=?',(paper,)).fetchone()),before)
|
|
self.assertEqual(conn.execute('SELECT status FROM decisions WHERE submission_id=1').fetchone()[0],'rejected')
|
|
actual={r[0]:r for r in conn.execute("SELECT sr.external_id,sr.paper_id,sr.title,sr.abstract,sr.authors_json,sr.metadata_json,sr.id FROM source_records sr JOIN sources s ON s.id=sr.source_id WHERE s.name=?",(c.SOURCE,))}
|
|
self.assertEqual(set(actual),set(self.by_doi))
|
|
self.assertEqual(actual[seed['external_id']][1],paper)
|
|
for doi,row in self.by_doi.items():
|
|
got=actual[doi];self.assertEqual((got[2],got[3],json.loads(got[4])),(row['title'],row['abstract'],row['authors']))
|
|
meta=json.loads(got[5]);self.assertEqual(meta.pop('record_identifiers'),{'doi':doi,c.SOURCE:doi});self.assertEqual(meta,row['metadata'])
|
|
tags=conn.execute("SELECT t.namespace,t.value,count(*) FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id GROUP BY t.namespace,t.value").fetchall()
|
|
self.assertEqual(sum(r[2] for r in tags if r[0]==c.SOURCE+':crossref:type'),1918)
|
|
self.assertEqual(sum(r[2] for r in tags if r[0]==c.SOURCE+':mm_program:track'),1726)
|
|
self.assertEqual({r[0] for r in tags},{c.SOURCE+':crossref:type',c.SOURCE+':mm_program:track'})
|
|
decisions=conn.execute("SELECT d.status,d.decision_date,count(*) FROM decisions d JOIN submissions s ON s.id=d.submission_id JOIN source_records r ON r.id=s.source_record_id JOIN sources src ON src.id=r.source_id WHERE src.name=? GROUP BY d.status,d.decision_date",(c.SOURCE,)).fetchall()
|
|
self.assertEqual([tuple(r) for r in decisions],[('accepted',None,1918)])
|
|
self.assertEqual(conn.execute("SELECT count(*) FROM submission_identifiers WHERE scheme='publication_doi'").fetchone()[0],1918)
|
|
coverage=conn.execute("SELECT es.status,sum(es.imported_count) FROM edition_sources es JOIN sources s ON s.id=es.source_id WHERE s.name=? GROUP BY es.status",(c.SOURCE,)).fetchall()
|
|
self.assertEqual([tuple(r) for r in coverage],[('partial',1918)])
|
|
finally:store.conn.close()
|
|
finally:
|
|
if pg_service:
|
|
with connect(pg_service) as pg:
|
|
for name, in pg.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)',(schema,)).fetchall():
|
|
pg.execute('DROP SCHEMA '+quote(name)+' CASCADE')
|
|
|
|
def test_all1918_native_sqlite_source_tags_and_prior_rejection_survive(self):
|
|
self.assert_native_ingestion()
|
|
|
|
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'),'Requires approved remote PostgreSQL test service')
|
|
def test_all1918_native_postgres_source_tags_and_prior_rejection_survive(self):
|
|
self.assert_native_ingestion(os.environ['PAPER_ATLAS_TEST_PG_SERVICE'])
|