Merge pull request '保留同一来源分批导入后的覆盖统计' (#27) from codex/incremental-coverage into main
Metadata integrity checks / metadata (push) Successful in 1m20s
Metadata integrity checks / metadata (push) Successful in 1m20s
Reviewed-on: #27
This commit was merged in pull request #27.
This commit is contained in:
@@ -87,6 +87,82 @@ class PostgreSQLIngestionTests(unittest.TestCase):
|
||||
with redirect_stdout(io.StringIO()):
|
||||
ingest_file(None, path, pg_service=self.service, pg_schema=self.schema)
|
||||
|
||||
def test_incremental_source_coverage_counts_prior_chunks_and_refreshes_on_postgres(self):
|
||||
rows = [self.row(source='dblp',external=key,status='accepted',year=2001,venue='UAI')
|
||||
for key in ('incremental1','incremental2','incremental3')]
|
||||
listing = 'https://example.org/uai/2001'
|
||||
def manifest(path, **options):
|
||||
target=path.with_suffix('.manifest.json')
|
||||
value=json.loads(target.read_text())
|
||||
value.update(list_url=listing,status='partial',**options)
|
||||
target.write_text(json.dumps(value))
|
||||
first=self.file('initial',rows[:2]);manifest(first);self.ingest(first)
|
||||
second=self.file('append',[rows[2]])
|
||||
manifest(second,coverage_mode='incremental',coverage_previous_count=2,
|
||||
coverage_previous_expected_count=2,coverage_expected_count=4)
|
||||
self.ingest(second)
|
||||
manifest(second,coverage_mode='incremental',coverage_previous_count=3,
|
||||
coverage_previous_expected_count=4,coverage_expected_count=4)
|
||||
self.ingest(second)
|
||||
with closing(ReadConnection(self.service,self.schema)) as c:
|
||||
coverage=c.execute('SELECT status,expected_count,imported_count FROM edition_sources WHERE list_url=?',(listing,)).fetchone()
|
||||
self.assertEqual(tuple(coverage.values),('partial',4,3))
|
||||
stored=c.execute('SELECT abstract FROM source_records WHERE external_id=?',('incremental3',)).fetchone()
|
||||
self.assertEqual(stored[0],rows[2]['abstract'])
|
||||
self.assertEqual(c.execute("SELECT count(*) FROM source_records r JOIN sources s ON s.id=r.source_id WHERE s.name='dblp'").fetchone()[0],3)
|
||||
run=c.execute('SELECT records_written,checkpoint_json FROM ingest_runs ORDER BY id DESC LIMIT 1').fetchone()
|
||||
self.assertEqual(run[0],1)
|
||||
self.assertEqual(json.loads(run[1])['coverage']['imported_count'],3)
|
||||
|
||||
def test_incremental_postgres_bound_failure_rolls_back_rows_and_summary(self):
|
||||
listing='https://example.org/uai/2001'
|
||||
rows=[self.row(source='dblp',external=key,status='accepted',year=2001,venue='UAI')
|
||||
for key in ('bound1','bound2','bound3')]
|
||||
first=self.file('initial',rows[:2]);target=first.with_suffix('.manifest.json')
|
||||
value=json.loads(target.read_text());value.update(list_url=listing,status='partial');target.write_text(json.dumps(value))
|
||||
self.ingest(first)
|
||||
with closing(ReadConnection(self.service,self.schema)) as c:
|
||||
before={table:c.execute('SELECT count(*) FROM '+table).fetchone()[0]
|
||||
for table in ('papers','source_records','submissions','decisions')}
|
||||
path=self.file('overflow',[rows[2]]);target=path.with_suffix('.manifest.json')
|
||||
value=json.loads(target.read_text());value.update(list_url=listing,status='partial',coverage_mode='incremental',
|
||||
coverage_previous_count=2,coverage_previous_expected_count=2,coverage_expected_count=2)
|
||||
target.write_text(json.dumps(value))
|
||||
with self.assertRaisesRegex(ValueError,'source bounds'):self.ingest(path)
|
||||
with closing(ReadConnection(self.service,self.schema)) as c:
|
||||
self.assertEqual({table:c.execute('SELECT count(*) FROM '+table).fetchone()[0] for table in before},before)
|
||||
coverage=c.execute('SELECT status,expected_count,imported_count FROM edition_sources WHERE list_url=?',(listing,)).fetchone()
|
||||
self.assertEqual(tuple(coverage.values),('partial',2,2))
|
||||
self.assertEqual(c.execute('SELECT status FROM ingest_runs ORDER BY id DESC LIMIT 1').fetchone()[0],'failed')
|
||||
|
||||
def test_incremental_postgres_does_not_overwrite_a_concurrently_changed_parent(self):
|
||||
listing='https://example.org/uai/2001'
|
||||
rows=[self.row(source='dblp',external=key,status='accepted',year=2001,venue='UAI')
|
||||
for key in ('concurrent1','concurrent2','concurrent3')]
|
||||
first=self.file('initial',rows[:2]);target=first.with_suffix('.manifest.json')
|
||||
value=json.loads(target.read_text());value.update(list_url=listing,status='partial');target.write_text(json.dumps(value))
|
||||
self.ingest(first)
|
||||
path=self.file('racing',[rows[2]]);target=path.with_suffix('.manifest.json')
|
||||
value=json.loads(target.read_text());value.update(list_url=listing,status='partial',coverage_mode='incremental',
|
||||
coverage_previous_count=2,coverage_previous_expected_count=2,coverage_expected_count=4)
|
||||
target.write_text(json.dumps(value))
|
||||
original=Store.record
|
||||
def change_parent(store,*args,**kwargs):
|
||||
result=original(store,*args,**kwargs)
|
||||
# A separate committed SQL writer need not honor the native
|
||||
# advisory lock. Its update must survive our rejected file.
|
||||
with connect(self.service) as other:
|
||||
other.execute('SET search_path TO '+quote(self.schema)+',pg_catalog')
|
||||
other.execute('UPDATE edition_sources SET expected_count=99 WHERE list_url=%s',(listing,))
|
||||
return result
|
||||
with patch.object(Store,'record',change_parent):
|
||||
with self.assertRaisesRegex(ValueError,'parent changed during import'):self.ingest(path)
|
||||
with closing(ReadConnection(self.service,self.schema)) as c:
|
||||
coverage=c.execute('SELECT status,expected_count,imported_count FROM edition_sources WHERE list_url=?',(listing,)).fetchone()
|
||||
self.assertEqual(tuple(coverage.values),('partial',99,2))
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM source_records WHERE external_id=?',('concurrent3',)).fetchone()[0],0)
|
||||
self.assertEqual(c.execute('SELECT status FROM ingest_runs ORDER BY id DESC LIMIT 1').fetchone()[0],'failed')
|
||||
|
||||
def test_native_writes_keep_nul_historical_ids_and_later_acceptance(self):
|
||||
store = self.store()
|
||||
try:
|
||||
|
||||
Reference in New Issue
Block a user