From 3754545003e1f96b88c6533cc3a69dc557f07305 Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Wed, 18 Nov 2026 13:34:09 +0800 Subject: [PATCH] Fix incremental coverage preflight on PostgreSQL readonly rows --- paper_atlas/ingestion/records.py | 2 +- tests/test_postgres_ingestion.py | 32 +++++++++++++++++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/paper_atlas/ingestion/records.py b/paper_atlas/ingestion/records.py index 5b03604..aa37f4c 100644 --- a/paper_atlas/ingestion/records.py +++ b/paper_atlas/ingestion/records.py @@ -66,7 +66,7 @@ def require_incremental_parent(c, source, manifest, options): if entries: if len(entries) != 1 or entries[0][0] != manifest['list_url']: raise ValueError('Incremental coverage requires one unambiguous source listing') - if tuple(entries[0][1:]) != ('partial', prior_expected, previous): + if tuple(entries[0][i] for i in (1, 2, 3)) != ('partial', prior_expected, previous): raise ValueError('Incremental coverage parent changed or is not partial') elif previous != 0 or prior_expected is not None: raise ValueError('Incremental coverage parent is absent') diff --git a/tests/test_postgres_ingestion.py b/tests/test_postgres_ingestion.py index 95aaf55..706e737 100644 --- a/tests/test_postgres_ingestion.py +++ b/tests/test_postgres_ingestion.py @@ -15,7 +15,7 @@ from paper_atlas.db.backend import open_store, open_read_connection from paper_atlas.db.store import Store from paper_atlas.ingestion import arxiv from paper_atlas.ingestion.backfill import backfill -from paper_atlas.ingestion.records import ingest_file +from paper_atlas.ingestion.records import coverage_options, ingest_file, require_incremental_parent from paper_atlas.postgres.client import connect, ReadConnection from paper_atlas.postgres.migration import migrate from paper_atlas.postgres.schema import quote @@ -114,6 +114,36 @@ class PostgreSQLIngestionTests(unittest.TestCase): self.assertEqual(run[0],1) self.assertEqual(json.loads(run[1])['coverage']['imported_count'],3) + def test_incremental_parent_preflight_uses_readonly_postgres_rows_and_keeps_guards(self): + rows = [self.row(source='dblp', external=key, status='accepted', year=2001, venue='UAI') + for key in ('preflight1', 'preflight2')] + path = self.file('preflight-parent', rows) + target = path.with_suffix('.manifest.json') + manifest = json.loads(target.read_text()) + manifest['status'] = 'partial' + target.write_text(json.dumps(manifest)) + self.ingest(path) + proposed = dict(manifest, coverage_mode='incremental', count=1, + coverage_previous_count=2, coverage_previous_expected_count=2, + coverage_expected_count=4) + with closing(ReadConnection(self.service, self.schema)) as c: + self.assertEqual(c.execute('SHOW transaction_read_only').fetchone()[0], 'on') + source = c.execute("SELECT id FROM sources WHERE name='dblp'").fetchone()[0] + before = {table: c.execute('SELECT count(*) FROM '+table).fetchone()[0] + for table in ('papers', 'source_records', 'submissions', 'decisions', 'ingest_runs')} + coverage = [dict(row) for row in c.execute('SELECT * FROM edition_sources ORDER BY edition_id,source_id,list_url')] + require_incremental_parent(c, source, proposed, coverage_options(proposed)) + for changes, error in (({'coverage_previous_count': 1}, 'parent changed'), + ({'coverage_previous_expected_count': 3}, 'parent changed'), + ({'list_url': 'https://example.org/other-list'}, 'unambiguous source listing')): + with self.subTest(changes=changes): + stale = dict(proposed, **changes) + with self.assertRaisesRegex(ValueError, error): + require_incremental_parent(c, source, stale, coverage_options(stale)) + self.assertEqual({table: c.execute('SELECT count(*) FROM '+table).fetchone()[0] + for table in before}, before) + self.assertEqual([dict(row) for row in c.execute('SELECT * FROM edition_sources ORDER BY edition_id,source_id,list_url')], coverage) + def test_incremental_postgres_bound_failure_rolls_back_rows_and_summary(self): listing='https://example.org/uai/2001' rows=[self.row(source='dblp',external=key,status='accepted',year=2001,venue='UAI') -- 2.52.0