|
|
|
@@ -15,7 +15,7 @@ from paper_atlas.db.backend import open_store, open_read_connection
|
|
|
|
|
from paper_atlas.db.store import Store
|
|
|
|
|
from paper_atlas.ingestion import arxiv
|
|
|
|
|
from paper_atlas.ingestion.backfill import backfill
|
|
|
|
|
from paper_atlas.ingestion.records import ingest_file
|
|
|
|
|
from paper_atlas.ingestion.records import coverage_options, ingest_file, require_incremental_parent
|
|
|
|
|
from paper_atlas.postgres.client import connect, ReadConnection
|
|
|
|
|
from paper_atlas.postgres.migration import migrate
|
|
|
|
|
from paper_atlas.postgres.schema import quote
|
|
|
|
@@ -114,6 +114,36 @@ class PostgreSQLIngestionTests(unittest.TestCase):
|
|
|
|
|
self.assertEqual(run[0],1)
|
|
|
|
|
self.assertEqual(json.loads(run[1])['coverage']['imported_count'],3)
|
|
|
|
|
|
|
|
|
|
def test_incremental_parent_preflight_uses_readonly_postgres_rows_and_keeps_guards(self):
|
|
|
|
|
rows = [self.row(source='dblp', external=key, status='accepted', year=2001, venue='UAI')
|
|
|
|
|
for key in ('preflight1', 'preflight2')]
|
|
|
|
|
path = self.file('preflight-parent', rows)
|
|
|
|
|
target = path.with_suffix('.manifest.json')
|
|
|
|
|
manifest = json.loads(target.read_text())
|
|
|
|
|
manifest['status'] = 'partial'
|
|
|
|
|
target.write_text(json.dumps(manifest))
|
|
|
|
|
self.ingest(path)
|
|
|
|
|
proposed = dict(manifest, coverage_mode='incremental', count=1,
|
|
|
|
|
coverage_previous_count=2, coverage_previous_expected_count=2,
|
|
|
|
|
coverage_expected_count=4)
|
|
|
|
|
with closing(ReadConnection(self.service, self.schema)) as c:
|
|
|
|
|
self.assertEqual(c.execute('SHOW transaction_read_only').fetchone()[0], 'on')
|
|
|
|
|
source = c.execute("SELECT id FROM sources WHERE name='dblp'").fetchone()[0]
|
|
|
|
|
before = {table: c.execute('SELECT count(*) FROM '+table).fetchone()[0]
|
|
|
|
|
for table in ('papers', 'source_records', 'submissions', 'decisions', 'ingest_runs')}
|
|
|
|
|
coverage = [dict(row) for row in c.execute('SELECT * FROM edition_sources ORDER BY edition_id,source_id,list_url')]
|
|
|
|
|
require_incremental_parent(c, source, proposed, coverage_options(proposed))
|
|
|
|
|
for changes, error in (({'coverage_previous_count': 1}, 'parent changed'),
|
|
|
|
|
({'coverage_previous_expected_count': 3}, 'parent changed'),
|
|
|
|
|
({'list_url': 'https://example.org/other-list'}, 'unambiguous source listing')):
|
|
|
|
|
with self.subTest(changes=changes):
|
|
|
|
|
stale = dict(proposed, **changes)
|
|
|
|
|
with self.assertRaisesRegex(ValueError, error):
|
|
|
|
|
require_incremental_parent(c, source, stale, coverage_options(stale))
|
|
|
|
|
self.assertEqual({table: c.execute('SELECT count(*) FROM '+table).fetchone()[0]
|
|
|
|
|
for table in before}, before)
|
|
|
|
|
self.assertEqual([dict(row) for row in c.execute('SELECT * FROM edition_sources ORDER BY edition_id,source_id,list_url')], coverage)
|
|
|
|
|
|
|
|
|
|
def test_incremental_postgres_bound_failure_rolls_back_rows_and_summary(self):
|
|
|
|
|
listing='https://example.org/uai/2001'
|
|
|
|
|
rows=[self.row(source='dblp',external=key,status='accepted',year=2001,venue='UAI')
|
|
|
|
|