"""Whole-source scope correction, stale refresh, full history and atomicity on both engines.""" from contextlib import closing, redirect_stdout import copy import hashlib import io import json import os from pathlib import Path import tempfile import unittest from unittest.mock import patch import uuid from paper_atlas.db.backend import open_store from paper_atlas.db.edition_scopes import KEY from paper_atlas.db.source_revisions import available, require_row, require_manifest from paper_atlas.db.store import Store from paper_atlas.ingestion.source_revisions import proposal, apply_plan, run, validate from paper_atlas.ingestion.event_identity import database_identity, digest, values from paper_atlas.ingestion.records import coverage_options, ingest_file from paper_atlas.postgres.client import connect from paper_atlas.postgres.migration import migrate from paper_atlas.postgres.schema import quote URL = 'https://example.org/volume' def seed(path): old = [] for i in range(4): doi = '10.1234/publication'+str(i) old.append({'source': 'fixture', 'venue': 'MM', 'year': 2026, 'track': 'main' if i == 0 else 'unclassified', 'external_id': doi, 'url': 'https://doi.org/'+doi, 'title': 'Distinct publication '+str(i), 'authors': ['Alice Author', 'Bob Writer'], 'abstract': None if i == 1 else 'Source abstract '+str(i), 'identifiers': {'doi': doi}, 'submission_identifiers': {'publication_doi': doi}, 'status': 'accepted', 'raw_decision': 'Publication observation', 'evidence_url': URL, 'observed_at': '2026-06-01T00:00:00Z', 'tags': [{'namespace': 'fixture:track', 'value': 'main' if i == 0 else 'unclassified'}], 'metadata': {'original': i, 'program': None, 'scientific_work_identity_approved': False}}) current = copy.deepcopy(old) current[1].update(track='main', abstract='Reviewed program abstract with NUL\x00 and SOH\x01') current[1]['metadata']['program'] = {'id': 'p1', 'evidence_url': URL} current[1]['tags'][0]['value'] = 'main' store = Store(path) for row in old[:3]: p, r = store.record(row['source'], row['external_id'], row['url'], row['title'], row['abstract'], row['authors'], row['identifiers'], row['metadata']) store.decision(p, r, 'MM', 2026, row['track'], row['external_id'], URL, 'accepted', row['raw_decision'], identities=[('publication_doi', row['external_id'])], observed_at=row['observed_at']) store.record_tags(p, r, 'fixture', row['tags']) if row == old[1]: store.conn.execute('UPDATE papers SET abstract=? WHERE id=?', ('Canonical preprint abstract', p)) p, other = store.record('prior', 'old-forum', 'https://example.org/prior', row['title'], 'Canonical preprint abstract', row['authors'], row['identifiers']) store.decision(p, other, 'PRIOR', 2025, 'main', 'old-forum', 'https://example.org/prior', 'rejected', 'Reject', observed_at='2025-01-01T00:00:00Z') # An unrelated source already has an attempt in the populated target edition. p, r = store.record('other', 'other', 'https://example.org/other', 'Unrelated target paper', 'Other abstract', ['Someone Else']) store.decision(p, r, 'MM', 2026, 'main', 'other', 'https://example.org/other', 'accepted', 'Published') sid = store.conn.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0] for track, expected, imported in [('main', 1, 1), ('unclassified', 3, 2)]: eid = store.conn.execute('SELECT id FROM editions WHERE year=2026 AND track=?', (track,)).fetchone()[0] store.conn.execute("INSERT INTO edition_sources VALUES(?,?,?,'partial',?,?,?,'2026-06-01',NULL)", (eid, sid, URL, expected, imported, 'checkpoint')) store.conn.commit(); store.conn.close() return {'format': 1, 'source': 'fixture', 'venue': 'MM', 'year': 2026, 'event_key': '', 'list_url': URL, 'previous_rows': old, 'rows': current, 'evidence': [{'url': URL, 'sha256': 'a'*64}]} class RevisionCases: def store(self): return open_store(**self.target) def plan(self, c): return proposal(c, self.inventory, database_identity(c, self.target.get('db'))) def test_move_to_populated_target_preserves_full_history_and_before_images(self): with closing(self.store().conn) as c: plan = self.plan(c); self.assertEqual(len(plan['changes']), 1) self.assertEqual(len(plan['before']['packet']['submissions']), 5) before_papers = plan['before']['packet']['papers'] before_tags = plan['before']['packet']['paper_tags'] result = apply_plan(c, plan); c.commit() self.assertEqual((result['events_moved'], result['records_changed']), (1, 1)) self.assertEqual(result['protected_decisions'], 5) self.assertEqual(values(c.execute('SELECT * FROM papers ORDER BY id')), before_papers) self.assertEqual(values(c.execute('SELECT * FROM paper_tags ORDER BY paper_id,tag_id,evidence_key')), before_tags) self.assertEqual(c.execute("SELECT count(*) FROM decisions WHERE status='rejected'").fetchone()[0], 1) self.assertEqual(values(c.execute('SELECT status,expected_count,imported_count,cursor,last_success_at,last_error FROM edition_sources ORDER BY edition_id')), [['partial', 2, 2, 'checkpoint', '2026-06-01', None], ['partial', 2, 1, 'checkpoint', '2026-06-01', None]]) saved = json.loads(c.execute('SELECT plan_json FROM source_scope_revisions').fetchone()[0]) self.assertEqual(saved, plan) self.assertEqual(apply_plan(c, plan)['status'], 'already_applied') self.assertEqual(c.execute('SELECT count(*) FROM source_scope_revisions').fetchone()[0], 1) def test_stale_direct_store_and_jsonl_refresh_fail_but_current_refresh_succeeds(self): with closing(self.store().conn) as c: plan = self.plan(c); apply_plan(c, plan); c.commit() with closing(self.store().conn) as c: store = object.__new__(Store); store.conn = c; store.event_identity = True old = self.inventory['previous_rows'][1] with self.assertRaisesRegex(ValueError, 'Stale source'): store.record('fixture', old['external_id'], old['url'], old['title'], old['abstract'], old['authors'], old['identifiers'], old['metadata']) c.rollback() row = self.inventory['rows'][1] p, r = store.record('fixture', row['external_id'], row['url'], row['title'], row['abstract'], row['authors'], row['identifiers'], row['metadata']) self.assertEqual(c.execute('SELECT abstract FROM papers WHERE id=?', (p,)).fetchone()[0], 'Canonical preprint abstract') self.assertEqual(json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?', (r,)).fetchone()[0])[KEY][0]['target']['track'], 'main') c.rollback() path = self.root/'old.jsonl'; raw = (json.dumps(old)+'\n').encode(); path.write_bytes(raw) path.with_suffix('.manifest.json').write_text(json.dumps({'source': 'fixture', 'venue': 'MM', 'year': 2026, 'track': 'unclassified', 'list_url': URL, 'status': 'partial', 'count': 1, 'file_sha256': hashlib.sha256(raw).hexdigest()})) with self.assertRaisesRegex(ValueError, 'source snapshot'): ingest_file(self.target.get('db'), path, **{k: v for k, v in self.target.items() if k != 'db'}) def test_unregistered_member_and_changed_program_metadata_are_rejected(self): with closing(self.store().conn) as c: apply_plan(c, self.plan(c)); c.commit() sid = c.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0] cache = {}; unknown = copy.deepcopy(self.inventory['rows'][0]); unknown['external_id'] = '10.1234/unlisted' with self.assertRaisesRegex(ValueError, 'absent from current'): require_row(c, sid, unknown, {'list_url': URL}, cache) bad = copy.deepcopy(self.inventory['rows'][1]); bad['metadata']['program'] = None with self.assertRaisesRegex(ValueError, 'Stale source'): require_row(c, sid, bad, {'list_url': URL}, cache) good = copy.deepcopy(self.inventory['rows'][1]); good['metadata']['verified_work_anchor'] = {'reason': 'extra evidence'} require_row(c, sid, good, {'list_url': URL}, cache) with self.assertRaisesRegex(ValueError, 'coverage manifest'): require_manifest(c, sid, {'venue': 'MM', 'year': 2026, 'track': 'main', 'list_url': URL, 'count': 1, 'status': 'partial'}, cache) require_manifest(c, sid, {'venue': 'MM', 'year': 2026, 'track': 'main', 'list_url': URL, 'count': 2, 'status': 'partial'}, cache) def test_stale_plan_and_target_identifier_collision_prevent_writes(self): with closing(self.store().conn) as c: plan = self.plan(c) c.execute("UPDATE decisions SET raw_value='Additional evidence' WHERE status='rejected'") with self.assertRaisesRegex(ValueError, 'Stale'): apply_plan(c, plan) c.rollback() move = plan['changes'][0] target_sub = c.execute('SELECT id FROM submissions WHERE edition_id=? LIMIT 1', (move['target_edition_id'],)).fetchone()[0] c.execute('INSERT INTO submission_identifiers VALUES(?,?,?,?)', (move['target_edition_id'], 'publication_doi', move['external_id'], target_sub)) with self.assertRaisesRegex(ValueError, 'collision'): self.plan(c) c.rollback() self.assertFalse(available(c)) def test_failed_postcheck_rolls_back_rows_extension_and_coverage(self): with closing(self.store().conn) as c: plan = self.plan(c) with patch('paper_atlas.ingestion.source_revisions.verify', side_effect=ValueError('Injected verification failure')): with self.assertRaisesRegex(ValueError, 'Injected'): apply_plan(c, plan) c.rollback() self.assertFalse(available(c)) self.assertEqual(self.plan(c), plan) def test_cli_binds_database_and_file_hash_without_writing_during_preflight(self): inventory = self.root/'inventory.json'; inventory.write_text(json.dumps(self.inventory)) path = self.root/'plan.json' args = {k: v for k, v in self.target.items() if k != 'db'} run(self.target.get('db'), path, inventory=inventory, **args) with closing(self.store().conn) as c: self.assertFalse(available(c)) with self.assertRaisesRegex(ValueError, 'hash'): run(self.target.get('db'), self.root/'bad.json', plan=path, plan_sha256='0'*64, **args) report = run(self.target.get('db'), self.root/'applied.json', plan=path, plan_sha256=hashlib.sha256(path.read_bytes()).hexdigest(), **args) self.assertEqual(report['events_moved'], 1) def test_membership_identity_changes_and_incremental_shrink_remain_forbidden(self): for mutate in [lambda x: x['rows'].pop(), lambda x: x['rows'][0].update(title='Different work'), lambda x: x['rows'][0].update(status='rejected')]: bad = copy.deepcopy(self.inventory); mutate(bad) with self.assertRaises(ValueError): validate(bad) with self.assertRaisesRegex(ValueError, 'incremental'): coverage_options({'coverage_mode': 'incremental', 'venue': 'MM', 'year': 2026, 'track': 'unclassified', 'status': 'partial', 'count': 1, 'coverage_previous_count': 2, 'coverage_previous_expected_count': 3, 'coverage_expected_count': 2}) class SQLiteSourceRevisionTests(RevisionCases, unittest.TestCase): def setUp(self): temp = tempfile.TemporaryDirectory(); self.addCleanup(temp.cleanup); self.root = Path(temp.name) db = self.root/'source.sqlite'; self.inventory = seed(db); self.target = {'db': db} @unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires approved remote PostgreSQL service') def test_optional_revision_table_migrates_with_full_history(self): with closing(self.store().conn) as c: plan = self.plan(c); apply_plan(c, plan); c.commit() service = os.environ['PAPER_ATLAS_TEST_PG_SERVICE']; schema = 'pa_rev_copy_'+uuid.uuid4().hex[:10] self.addCleanup(drop_schema, service, schema) with redirect_stdout(io.StringIO()): migrate(self.target['db'], service, self.root/'copied.json', schema=schema, min_free_gib=0) with closing(open_store(pg_service=service, pg_schema=schema).conn) as c: self.assertEqual(json.loads(c.execute('SELECT plan_json FROM source_scope_revisions').fetchone()[0]), plan) self.assertEqual(c.execute('SELECT count(*) FROM source_scope_revisions').fetchone()[0], 1) def drop_schema(service, schema): with connect(service) as c: for name, in c.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)', (schema,)): c.execute('DROP SCHEMA '+quote(name)+' CASCADE') @unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires approved remote PostgreSQL service') class PostgreSQLSourceRevisionTests(RevisionCases, unittest.TestCase): def setUp(self): temp = tempfile.TemporaryDirectory(); self.addCleanup(temp.cleanup); self.root = Path(temp.name) db = self.root/'source.sqlite'; self.inventory = seed(db) service = os.environ['PAPER_ATLAS_TEST_PG_SERVICE']; schema = 'pa_rev_'+uuid.uuid4().hex[:12] self.addCleanup(drop_schema, service, schema) with redirect_stdout(io.StringIO()): migrate(db, service, self.root/'migration.json', schema=schema, min_free_gib=0) self.target = {'pg_service': service, 'pg_schema': schema}