import copy import gzip import hashlib import json from pathlib import Path import tempfile import unittest from paper_atlas.collectors import sdm2026 as c class SDM2026Tests(unittest.TestCase): @classmethod def setUpClass(cls): cls.fixture = json.loads(gzip.decompress((Path(__file__).parent/'fixtures/sdm2026-source.json.gz').read_bytes())) cls.message = json.loads(cls.fixture['files']['sdm2026-crossref-research-v1/isbn-works.json'])['message'] def snapshots(self, root): for name, text in self.fixture['files'].items(): path = root/name; path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(text.encode('utf-8')) def test_publication_census_excludes_book_and_front_matter_not_cancelled_paper(self): papers, excluded = c.registry_papers(self.message) self.assertEqual(len(papers), 53) self.assertEqual({x['doi'] for x in excluded}, {c.BOOK, c.BOOK+'.fm'}) self.assertIn(c.BOOK+'.7', {p['original']['DOI'] for p in papers}) marked = [p for p in papers if p['review_flags']] self.assertEqual({p['original']['DOI'] for p in marked}, {c.BOOK+'.12', c.BOOK+'.35'}) for p in marked: self.assertTrue(p['title'].endswith('*')) self.assertIn('*', p['original']['title'][0]) def test_full_collection_preserves_separate_source_versions_and_unknown_negatives(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp); self.snapshots(root/'src') report = c.collect(root/'out', snapshots=root/'src') self.assertEqual((report['records'], report['missing_abstract_count']), (53, 53)) self.assertFalse(report['database_written']); self.assertEqual(report['network_requests'], 0) self.assertEqual(report['public_rejected_metadata'], 'unknown_anonymous_query_http403') self.assertEqual(report['program_registry_author_sequence_differences'], 20) manifest, = report['files']; self.assertEqual(manifest['status'], 'partial') rows = [json.loads(line) for line in Path(manifest['file']).read_text().splitlines()] originals = {x['DOI']: x for x in self.message['items'] if x['type'] == 'book-chapter'} for row in rows: self.assertEqual(row['metadata']['registry_fields'], originals[row['external_id']]) self.assertIsNone(row['abstract']); self.assertFalse(row['tags']) self.assertEqual(row['identifiers'], {'doi': row['external_id']}) self.assertEqual(row['submission_identifiers'], [{'scheme': 'publication_doi', 'value': row['external_id']}]) self.assertEqual((row['status'], row['track']), ('accepted', 'proceedings_unclassified')) self.assertEqual(row['metadata']['publication_dates']['published-online'], '2026-11-12') sidecar = report['program_observations'] self.assertTrue(sidecar['not_an_import_manifest']) observations = [json.loads(line) for line in Path(sidecar['file']).read_text().splitlines()] self.assertEqual(len(observations), 53) self.assertEqual(sum(x['public_page_fields'] is not None for x in observations), 52) for observation in observations: self.assertNotIn('status', observation); self.assertNotIn('identifiers', observation) self.assertEqual(observation['record_kind'], 'source_observation_not_import_record') cancelled, = [x for x in observations if x['cancelled_presentation']] self.assertEqual(cancelled['title'], 'A Trajectory View of Statistical Policy Comparison') self.assertIsNone(cancelled['public_page_fields']) # The real source has conflicting author sets under the same title. conflict = next(x for x in observations if x['title'].startswith('Scaffold-Aware Generative')) self.assertEqual(conflict['public_page_fields']['source_reported_authors'], ['Xin Wang', 'Qinwen Ge']) candidate, = conflict['registry_candidates']; self.assertFalse(candidate['author_sequence_exact']) registry_row = next(x for x in rows if x['external_id'] == candidate['registry_doi_candidate']) self.assertEqual(registry_row['authors'], ['Xin Wang', 'Yu Wang', 'Yunchao Liu', 'Jens Meiler', 'Tyler Derr']) # Every copied source and manifest remains byte-identical, including CRLF. for evidence in report['evidence']: body = (root/'out'/evidence['output_file']).read_bytes() self.assertEqual(body, self.fixture['files'][evidence['snapshot']+'/'+evidence['file']].encode('utf-8')) self.assertEqual(hashlib.sha256(body).hexdigest(), evidence['sha256']) with self.assertRaisesRegex(ValueError, 'Output exists'): c.collect(root/'out', snapshots=root/'src') def test_all_program_events_match_independent_inventory(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp); self.snapshots(root) loaded = {name: c.load_snapshot(root, name) for name in c.SNAPSHOTS} papers, _ = c.registry_papers(self.message) observations = c.program_observations(loaded, papers) gold = {(x['session_url'], x['title']): x for x in self.fixture['program_gold']} self.assertEqual({(x['session_url'], x['title']) for x in observations}, set(gold)) for item in observations: original = gold[(item['session_url'], item['title'])] for key in ('authors_literal', 'time_literal', 'cancelled_presentation'): self.assertEqual(item[key], original[key]) def test_all_public_abstracts_and_source_reported_names_match_independent_gold(self): files = self.fixture['files'] manifest = json.loads(files['sdm2026-public-abstract-research-v1/manifest.json']) receipts = {r['url']: r for r in manifest['records']} self.assertEqual(len(self.fixture['page_gold']), 52) for gold in self.fixture['page_gold']: receipt = receipts[gold['program_url']] raw = files['sdm2026-public-abstract-research-v1/'+receipt['file']].encode('utf-8') self.assertEqual(hashlib.sha256(raw).hexdigest(), gold['page_sha256']) fields = c.talk_fields(raw, receipt['session_url']) self.assertEqual(fields['title'], gold['title']) self.assertEqual(fields['source_reported_authors'], gold['authors']) self.assertEqual(fields['abstract'], gold['abstract']) self.assertTrue(all('@' not in name for name in fields['source_reported_authors'])) def test_wrong_isbn_parent_doi_and_incomplete_census_rejected(self): for kind in ('isbn', 'parent', 'repeat', 'type', 'year', 'author', 'markup', 'abstract', 'missing'): with self.subTest(kind=kind): message = copy.deepcopy(self.message) row = next(x for x in message['items'] if x['type'] == 'book-chapter') if kind == 'isbn': row['ISBN'] = ['9780000000000'] if kind == 'parent': row['relation']['is-part-of'][0]['id'] = 'https://doi.org/10.9999/other' if kind == 'repeat': row['DOI'] = c.BOOK if kind == 'type': row['type'] = 'other' if kind == 'year': row['published']['date-parts'] = [[2025, 10, 31]] if kind == 'author': row['author'][0]['sequence'] = 'additional' if kind == 'markup': row['title'] = ['Changed 2 title'] if kind == 'abstract': row['abstract'] = 'Unreviewed new version' if kind == 'missing': message['items'].pop() with self.assertRaises(ValueError): c.registry_papers(message) def test_foreign_session_and_changed_edition_rejected(self): raw = self.fixture['files']['sdm2026-public-abstract-research-v1/162766.html'].encode() url = 'https://meetings.siam.org/sess/dsp_programsess.cfm?SESSIONCODE=91511' for changed in [raw.replace(b'sessioncode=91511', b'sessioncode=91507'), raw.replace(b'2026 SIAM International Conference', b'2025 SIAM International Conference')]: with self.assertRaises(ValueError): c.talk_fields(changed, url) def test_corruption_preserves_failure_without_success_report(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp); self.snapshots(root/'src') (root/'src/sdm2026-public-abstract-research-v1/162766.html').write_bytes(b'corrupt') with self.assertRaisesRegex(ValueError, 'receipt/body'): c.collect(root/'out', snapshots=root/'src') self.assertTrue((root/'out/failed.json').exists()) self.assertFalse((root/'out/collection-report.json').exists()) def test_source_symlink_rejected(self): with tempfile.TemporaryDirectory() as tmp: root = Path(tmp); self.snapshots(root/'src') p = root/'src/sdm2026-crossref-research-v1/isbn-works.json' original = p.read_bytes(); p.unlink(); (root/'data.json').write_bytes(original); p.symlink_to(root/'data.json') with self.assertRaisesRegex(ValueError, 'Unsafe'): c.collect(root/'out', snapshots=root/'src')