Audit public decisions against PostgreSQL and recheck imports
Metadata integrity checks / metadata (push) Successful in 1m9s

This commit is contained in:
2026-11-07 16:01:27 +08:00
parent dc41dfa350
commit 6fa59c4d26
9 changed files with 153 additions and 23 deletions
+79 -1
View File
@@ -11,7 +11,7 @@ from unittest.mock import patch
import uuid
import zipfile
from paper_atlas.db.backend import open_store
from paper_atlas.db.backend import open_store, open_read_connection
from paper_atlas.db.store import Store
from paper_atlas.ingestion import arxiv
from paper_atlas.ingestion.backfill import backfill
@@ -27,6 +27,13 @@ class BackendSelectionTests(unittest.TestCase):
with self.assertRaisesRegex(ValueError, 'Choose exactly one'):
open_store(**kwargs)
def test_readonly_audit_does_not_create_an_absent_sqlite_database(self):
with tempfile.TemporaryDirectory() as tmp:
path = Path(tmp) / 'absent.sqlite'
with self.assertRaisesRegex(ValueError, 'Existing database'):
open_read_connection(path)
self.assertFalse(path.exists())
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires the approved remote PostgreSQL test service')
class PostgreSQLIngestionTests(unittest.TestCase):
@@ -138,6 +145,21 @@ class PostgreSQLIngestionTests(unittest.TestCase):
finally:
store.conn.close()
def test_changed_file_hash_rolls_back_data_before_commit(self):
path = self.file('changed', [self.row()])
manifest = json.loads(path.with_suffix('.manifest.json').read_text())
manifest['file_sha256'] = '0' * 64
path.with_suffix('.manifest.json').write_text(json.dumps(manifest))
with self.assertRaisesRegex(ValueError, 'hash differs'):
self.ingest(path)
store = self.store()
try:
self.assertEqual(store.conn.execute('SELECT count(*) FROM papers').fetchone()[0], 1)
self.assertEqual(store.conn.execute('SELECT count(*) FROM source_records').fetchone()[0], 1)
self.assertEqual(store.conn.execute('SELECT status,records_written FROM ingest_runs').fetchone(), ('failed', 0))
finally:
store.conn.close()
def test_writer_lock_survives_commit_and_rollback_but_readers_continue(self):
first = self.store()
try:
@@ -165,6 +187,62 @@ class PostgreSQLIngestionTests(unittest.TestCase):
with connect(self.service) as pg:
self.assertEqual(pg.execute('SELECT count(*) FROM information_schema.tables WHERE table_schema=%s', (staging,)).fetchone()[0], 0)
def test_negative_preflight_uses_current_pg_attempt_and_blocks_stale_import(self):
import hashlib
from paper_atlas.collectors.public_openreview import SOURCE, REVISION
from paper_atlas.audits.public_openreview import audit
row = self.row(SOURCE, 'negativeForum')
row['identifiers']['openreview'] = 'negativeForum'
row['metadata'] = {'evidence_kind': 'curated_public_status_assertion',
'source_revision': REVISION, 'original_decision_signature_verified': False}
path = self.file('negative', [row])
manifest = json.loads(path.with_suffix('.manifest.json').read_text())
manifest.update(file=str(path), file_sha256=hashlib.sha256(path.read_bytes()).hexdigest())
path.with_suffix('.manifest.json').write_text(json.dumps(manifest))
inventory = self.root / 'negative-inventory.json'
inventory.write_text(json.dumps({'source': SOURCE, 'revision': REVISION,
'normalized_records': 1, 'files': [manifest]}))
# An acceptance in another conference does not block this rejection.
self.ingest(self.file('other-conference', [self.row('official', 'acceptedElsewhere', 'accepted', 2026, 'COLM')]))
def preflight(name):
with redirect_stdout(io.StringIO()):
return audit(None, inventory, self.root / (name + '.json'), pg_service=self.service, pg_schema=self.schema)
first = preflight('before')
self.assertEqual(first['status'], 'ready_for_serial_import')
self.assertEqual(first['engine'], 'postgresql')
self.assertEqual(first['schema'], self.schema)
self.ingest(path)
self.assertEqual(preflight('existing')['existing_same_edition_attempts'], 1)
store = self.store()
try:
paper, record = store.conn.execute('''SELECT r.paper_id,r.id FROM source_records r
JOIN sources s ON s.id=r.source_id WHERE s.name=? AND r.external_id=?''', (SOURCE, row['external_id'])).fetchone()
store.decision(paper, record, 'ICLR', 2025, 'main', row['external_id'],
'https://example.org/formal', 'accepted', 'Accept', observed_at='2027-01-01T00:00:00Z')
# Equal observation time, but an explicitly earlier decision date.
# NULL dates must fall back to observed_at, not the empty string.
store.decision(paper, record, 'ICLR', 2025, 'main', row['external_id'],
'https://example.org/older', 'rejected', 'Reject', observed_at='2027-01-01T00:00:00Z',
decision_date='2026-12-31T00:00:00Z')
store.conn.commit()
finally:
store.conn.close()
blocked = preflight('blocked')
self.assertEqual(blocked['status'], 'blocked')
self.assertEqual(blocked['conflicts'][0]['existing_status'], 'accepted')
self.assertEqual(blocked['files'], [])
# The earlier ready report cannot authorize overwriting current status.
with self.assertRaisesRegex(ValueError, 'conflicts with current'):
self.ingest(path)
reader = open_read_connection(pg_service=self.service, pg_schema=self.schema)
try:
self.assertEqual(reader.execute('SHOW transaction_read_only').fetchone()[0], 'on')
from paper_atlas.db.submissions import latest_status
sid = reader.execute("SELECT submission_id FROM submission_identifiers WHERE scheme='openreview' AND value=?", ('negativeForum',)).fetchone()[0]
self.assertEqual(latest_status(reader, sid), 'accepted')
finally:
reader.close()
def test_backfill_merges_evidence_without_reimporting_or_losing_history(self):
files = []
for source, external, title in [('official', 'legacy', 'First bibliographic name'), ('openreview_snapshot', 'forum', 'Renamed submission')]: