Audit public decisions against PostgreSQL and recheck imports
Metadata integrity checks / metadata (push) Successful in 1m9s
Metadata integrity checks / metadata (push) Successful in 1m9s
This commit is contained in:
@@ -11,7 +11,7 @@ from unittest.mock import patch
|
||||
import uuid
|
||||
import zipfile
|
||||
|
||||
from paper_atlas.db.backend import open_store
|
||||
from paper_atlas.db.backend import open_store, open_read_connection
|
||||
from paper_atlas.db.store import Store
|
||||
from paper_atlas.ingestion import arxiv
|
||||
from paper_atlas.ingestion.backfill import backfill
|
||||
@@ -27,6 +27,13 @@ class BackendSelectionTests(unittest.TestCase):
|
||||
with self.assertRaisesRegex(ValueError, 'Choose exactly one'):
|
||||
open_store(**kwargs)
|
||||
|
||||
def test_readonly_audit_does_not_create_an_absent_sqlite_database(self):
|
||||
with tempfile.TemporaryDirectory() as tmp:
|
||||
path = Path(tmp) / 'absent.sqlite'
|
||||
with self.assertRaisesRegex(ValueError, 'Existing database'):
|
||||
open_read_connection(path)
|
||||
self.assertFalse(path.exists())
|
||||
|
||||
|
||||
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires the approved remote PostgreSQL test service')
|
||||
class PostgreSQLIngestionTests(unittest.TestCase):
|
||||
@@ -138,6 +145,21 @@ class PostgreSQLIngestionTests(unittest.TestCase):
|
||||
finally:
|
||||
store.conn.close()
|
||||
|
||||
def test_changed_file_hash_rolls_back_data_before_commit(self):
|
||||
path = self.file('changed', [self.row()])
|
||||
manifest = json.loads(path.with_suffix('.manifest.json').read_text())
|
||||
manifest['file_sha256'] = '0' * 64
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps(manifest))
|
||||
with self.assertRaisesRegex(ValueError, 'hash differs'):
|
||||
self.ingest(path)
|
||||
store = self.store()
|
||||
try:
|
||||
self.assertEqual(store.conn.execute('SELECT count(*) FROM papers').fetchone()[0], 1)
|
||||
self.assertEqual(store.conn.execute('SELECT count(*) FROM source_records').fetchone()[0], 1)
|
||||
self.assertEqual(store.conn.execute('SELECT status,records_written FROM ingest_runs').fetchone(), ('failed', 0))
|
||||
finally:
|
||||
store.conn.close()
|
||||
|
||||
def test_writer_lock_survives_commit_and_rollback_but_readers_continue(self):
|
||||
first = self.store()
|
||||
try:
|
||||
@@ -165,6 +187,62 @@ class PostgreSQLIngestionTests(unittest.TestCase):
|
||||
with connect(self.service) as pg:
|
||||
self.assertEqual(pg.execute('SELECT count(*) FROM information_schema.tables WHERE table_schema=%s', (staging,)).fetchone()[0], 0)
|
||||
|
||||
def test_negative_preflight_uses_current_pg_attempt_and_blocks_stale_import(self):
|
||||
import hashlib
|
||||
from paper_atlas.collectors.public_openreview import SOURCE, REVISION
|
||||
from paper_atlas.audits.public_openreview import audit
|
||||
row = self.row(SOURCE, 'negativeForum')
|
||||
row['identifiers']['openreview'] = 'negativeForum'
|
||||
row['metadata'] = {'evidence_kind': 'curated_public_status_assertion',
|
||||
'source_revision': REVISION, 'original_decision_signature_verified': False}
|
||||
path = self.file('negative', [row])
|
||||
manifest = json.loads(path.with_suffix('.manifest.json').read_text())
|
||||
manifest.update(file=str(path), file_sha256=hashlib.sha256(path.read_bytes()).hexdigest())
|
||||
path.with_suffix('.manifest.json').write_text(json.dumps(manifest))
|
||||
inventory = self.root / 'negative-inventory.json'
|
||||
inventory.write_text(json.dumps({'source': SOURCE, 'revision': REVISION,
|
||||
'normalized_records': 1, 'files': [manifest]}))
|
||||
# An acceptance in another conference does not block this rejection.
|
||||
self.ingest(self.file('other-conference', [self.row('official', 'acceptedElsewhere', 'accepted', 2026, 'COLM')]))
|
||||
def preflight(name):
|
||||
with redirect_stdout(io.StringIO()):
|
||||
return audit(None, inventory, self.root / (name + '.json'), pg_service=self.service, pg_schema=self.schema)
|
||||
first = preflight('before')
|
||||
self.assertEqual(first['status'], 'ready_for_serial_import')
|
||||
self.assertEqual(first['engine'], 'postgresql')
|
||||
self.assertEqual(first['schema'], self.schema)
|
||||
self.ingest(path)
|
||||
self.assertEqual(preflight('existing')['existing_same_edition_attempts'], 1)
|
||||
store = self.store()
|
||||
try:
|
||||
paper, record = store.conn.execute('''SELECT r.paper_id,r.id FROM source_records r
|
||||
JOIN sources s ON s.id=r.source_id WHERE s.name=? AND r.external_id=?''', (SOURCE, row['external_id'])).fetchone()
|
||||
store.decision(paper, record, 'ICLR', 2025, 'main', row['external_id'],
|
||||
'https://example.org/formal', 'accepted', 'Accept', observed_at='2027-01-01T00:00:00Z')
|
||||
# Equal observation time, but an explicitly earlier decision date.
|
||||
# NULL dates must fall back to observed_at, not the empty string.
|
||||
store.decision(paper, record, 'ICLR', 2025, 'main', row['external_id'],
|
||||
'https://example.org/older', 'rejected', 'Reject', observed_at='2027-01-01T00:00:00Z',
|
||||
decision_date='2026-12-31T00:00:00Z')
|
||||
store.conn.commit()
|
||||
finally:
|
||||
store.conn.close()
|
||||
blocked = preflight('blocked')
|
||||
self.assertEqual(blocked['status'], 'blocked')
|
||||
self.assertEqual(blocked['conflicts'][0]['existing_status'], 'accepted')
|
||||
self.assertEqual(blocked['files'], [])
|
||||
# The earlier ready report cannot authorize overwriting current status.
|
||||
with self.assertRaisesRegex(ValueError, 'conflicts with current'):
|
||||
self.ingest(path)
|
||||
reader = open_read_connection(pg_service=self.service, pg_schema=self.schema)
|
||||
try:
|
||||
self.assertEqual(reader.execute('SHOW transaction_read_only').fetchone()[0], 'on')
|
||||
from paper_atlas.db.submissions import latest_status
|
||||
sid = reader.execute("SELECT submission_id FROM submission_identifiers WHERE scheme='openreview' AND value=?", ('negativeForum',)).fetchone()[0]
|
||||
self.assertEqual(latest_status(reader, sid), 'accepted')
|
||||
finally:
|
||||
reader.close()
|
||||
|
||||
def test_backfill_merges_evidence_without_reimporting_or_losing_history(self):
|
||||
files = []
|
||||
for source, external, title in [('official', 'legacy', 'First bibliographic name'), ('openreview_snapshot', 'forum', 'Renamed submission')]:
|
||||
|
||||
Reference in New Issue
Block a user