Compare commits

...

1 Commits

Author SHA1 Message Date
codex-bot 3754545003 Fix incremental coverage preflight on PostgreSQL readonly rows
Metadata integrity checks / metadata (push) Successful in 7m15s
2026-11-18 13:34:09 +08:00
2 changed files with 32 additions and 2 deletions
+1 -1
View File
@@ -66,7 +66,7 @@ def require_incremental_parent(c, source, manifest, options):
if entries:
if len(entries) != 1 or entries[0][0] != manifest['list_url']:
raise ValueError('Incremental coverage requires one unambiguous source listing')
if tuple(entries[0][1:]) != ('partial', prior_expected, previous):
if tuple(entries[0][i] for i in (1, 2, 3)) != ('partial', prior_expected, previous):
raise ValueError('Incremental coverage parent changed or is not partial')
elif previous != 0 or prior_expected is not None:
raise ValueError('Incremental coverage parent is absent')
+31 -1
View File
@@ -15,7 +15,7 @@ from paper_atlas.db.backend import open_store, open_read_connection
from paper_atlas.db.store import Store
from paper_atlas.ingestion import arxiv
from paper_atlas.ingestion.backfill import backfill
from paper_atlas.ingestion.records import ingest_file
from paper_atlas.ingestion.records import coverage_options, ingest_file, require_incremental_parent
from paper_atlas.postgres.client import connect, ReadConnection
from paper_atlas.postgres.migration import migrate
from paper_atlas.postgres.schema import quote
@@ -114,6 +114,36 @@ class PostgreSQLIngestionTests(unittest.TestCase):
self.assertEqual(run[0],1)
self.assertEqual(json.loads(run[1])['coverage']['imported_count'],3)
def test_incremental_parent_preflight_uses_readonly_postgres_rows_and_keeps_guards(self):
rows = [self.row(source='dblp', external=key, status='accepted', year=2001, venue='UAI')
for key in ('preflight1', 'preflight2')]
path = self.file('preflight-parent', rows)
target = path.with_suffix('.manifest.json')
manifest = json.loads(target.read_text())
manifest['status'] = 'partial'
target.write_text(json.dumps(manifest))
self.ingest(path)
proposed = dict(manifest, coverage_mode='incremental', count=1,
coverage_previous_count=2, coverage_previous_expected_count=2,
coverage_expected_count=4)
with closing(ReadConnection(self.service, self.schema)) as c:
self.assertEqual(c.execute('SHOW transaction_read_only').fetchone()[0], 'on')
source = c.execute("SELECT id FROM sources WHERE name='dblp'").fetchone()[0]
before = {table: c.execute('SELECT count(*) FROM '+table).fetchone()[0]
for table in ('papers', 'source_records', 'submissions', 'decisions', 'ingest_runs')}
coverage = [dict(row) for row in c.execute('SELECT * FROM edition_sources ORDER BY edition_id,source_id,list_url')]
require_incremental_parent(c, source, proposed, coverage_options(proposed))
for changes, error in (({'coverage_previous_count': 1}, 'parent changed'),
({'coverage_previous_expected_count': 3}, 'parent changed'),
({'list_url': 'https://example.org/other-list'}, 'unambiguous source listing')):
with self.subTest(changes=changes):
stale = dict(proposed, **changes)
with self.assertRaisesRegex(ValueError, error):
require_incremental_parent(c, source, stale, coverage_options(stale))
self.assertEqual({table: c.execute('SELECT count(*) FROM '+table).fetchone()[0]
for table in before}, before)
self.assertEqual([dict(row) for row in c.execute('SELECT * FROM edition_sources ORDER BY edition_id,source_id,list_url')], coverage)
def test_incremental_postgres_bound_failure_rolls_back_rows_and_summary(self):
listing='https://example.org/uai/2001'
rows=[self.row(source='dblp',external=key,status='accepted',year=2001,venue='UAI')