修复 PostgreSQL 只读增量覆盖预检 #74
@@ -66,7 +66,7 @@ def require_incremental_parent(c, source, manifest, options):
|
||||
if entries:
|
||||
if len(entries) != 1 or entries[0][0] != manifest['list_url']:
|
||||
raise ValueError('Incremental coverage requires one unambiguous source listing')
|
||||
if tuple(entries[0][1:]) != ('partial', prior_expected, previous):
|
||||
if tuple(entries[0][i] for i in (1, 2, 3)) != ('partial', prior_expected, previous):
|
||||
raise ValueError('Incremental coverage parent changed or is not partial')
|
||||
elif previous != 0 or prior_expected is not None:
|
||||
raise ValueError('Incremental coverage parent is absent')
|
||||
|
||||
@@ -15,7 +15,7 @@ from paper_atlas.db.backend import open_store, open_read_connection
|
||||
from paper_atlas.db.store import Store
|
||||
from paper_atlas.ingestion import arxiv
|
||||
from paper_atlas.ingestion.backfill import backfill
|
||||
from paper_atlas.ingestion.records import ingest_file
|
||||
from paper_atlas.ingestion.records import coverage_options, ingest_file, require_incremental_parent
|
||||
from paper_atlas.postgres.client import connect, ReadConnection
|
||||
from paper_atlas.postgres.migration import migrate
|
||||
from paper_atlas.postgres.schema import quote
|
||||
@@ -114,6 +114,36 @@ class PostgreSQLIngestionTests(unittest.TestCase):
|
||||
self.assertEqual(run[0],1)
|
||||
self.assertEqual(json.loads(run[1])['coverage']['imported_count'],3)
|
||||
|
||||
def test_incremental_parent_preflight_uses_readonly_postgres_rows_and_keeps_guards(self):
|
||||
rows = [self.row(source='dblp', external=key, status='accepted', year=2001, venue='UAI')
|
||||
for key in ('preflight1', 'preflight2')]
|
||||
path = self.file('preflight-parent', rows)
|
||||
target = path.with_suffix('.manifest.json')
|
||||
manifest = json.loads(target.read_text())
|
||||
manifest['status'] = 'partial'
|
||||
target.write_text(json.dumps(manifest))
|
||||
self.ingest(path)
|
||||
proposed = dict(manifest, coverage_mode='incremental', count=1,
|
||||
coverage_previous_count=2, coverage_previous_expected_count=2,
|
||||
coverage_expected_count=4)
|
||||
with closing(ReadConnection(self.service, self.schema)) as c:
|
||||
self.assertEqual(c.execute('SHOW transaction_read_only').fetchone()[0], 'on')
|
||||
source = c.execute("SELECT id FROM sources WHERE name='dblp'").fetchone()[0]
|
||||
before = {table: c.execute('SELECT count(*) FROM '+table).fetchone()[0]
|
||||
for table in ('papers', 'source_records', 'submissions', 'decisions', 'ingest_runs')}
|
||||
coverage = [dict(row) for row in c.execute('SELECT * FROM edition_sources ORDER BY edition_id,source_id,list_url')]
|
||||
require_incremental_parent(c, source, proposed, coverage_options(proposed))
|
||||
for changes, error in (({'coverage_previous_count': 1}, 'parent changed'),
|
||||
({'coverage_previous_expected_count': 3}, 'parent changed'),
|
||||
({'list_url': 'https://example.org/other-list'}, 'unambiguous source listing')):
|
||||
with self.subTest(changes=changes):
|
||||
stale = dict(proposed, **changes)
|
||||
with self.assertRaisesRegex(ValueError, error):
|
||||
require_incremental_parent(c, source, stale, coverage_options(stale))
|
||||
self.assertEqual({table: c.execute('SELECT count(*) FROM '+table).fetchone()[0]
|
||||
for table in before}, before)
|
||||
self.assertEqual([dict(row) for row in c.execute('SELECT * FROM edition_sources ORDER BY edition_id,source_id,list_url')], coverage)
|
||||
|
||||
def test_incremental_postgres_bound_failure_rolls_back_rows_and_summary(self):
|
||||
listing='https://example.org/uai/2001'
|
||||
rows=[self.row(source='dblp',external=key,status='accepted',year=2001,venue='UAI')
|
||||
|
||||
Reference in New Issue
Block a user