Integrate optional provenance-backed paper annotations

# Conflicts:
#	README.md
#	paper_atlas/querying.py
This commit is contained in:
2026-11-08 21:01:26 +08:00
13 changed files with 607 additions and 6 deletions
+2 -2
View File
@@ -143,7 +143,7 @@ def migrate(db, service, out, *, schema='research', storage_path=None, min_free_
free_space(storage_path, min_free_gib)
control = schema + '_migration'
report = {'status': 'preflight', 'source': str(db), 'service': service, 'schema': schema,
'started_at': timestamp(), 'tables': {}, 'scope': 'All 21 business tables; FTS5 internals rebuilt as native PostgreSQL GIN.'}
'started_at': timestamp(), 'tables': {}, 'scope': 'All baseline business tables and recognized installed extensions; FTS5 rebuilt as native PostgreSQL GIN.'}
with Path(str(db) + '.maintenance.lock').open('a') as lock:
fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
if Path(str(db) + '-wal').exists() and Path(str(db) + '-wal').stat().st_size:
@@ -240,7 +240,7 @@ def migrate(db, service, out, *, schema='research', storage_path=None, min_free_
JOIN pg_am am ON am.oid=ix.relam WHERE ns.nspname=%s AND ix.relname='papers_search_gin' ''', (stage,)).fetchone()
if search != ('gin', True, True, 'papers'):
raise ValueError('Native full-text index is not valid and ready')
for table in TABLES:
for table in plan:
pg.execute('ANALYZE ' + qualify(stage, table))
invalid = pg.execute('''SELECT count(*) FROM pg_constraint co JOIN pg_class cl ON cl.oid=co.conrelid
JOIN pg_namespace ns ON ns.oid=cl.relnamespace WHERE ns.nspname=%s AND NOT co.convalidated''', (stage,)).fetchone()[0]
+4 -2
View File
@@ -8,7 +8,9 @@ TABLES = ('sources', 'papers', 'identifiers', 'source_records', 'venues', 'editi
'submission_sources', 'submission_identifiers', 'decision_sources',
'submission_merges', 'decision_merges', 'schema_migrations', 'tags',
'paper_tags', 'match_candidates', 'paper_merges', 'ingest_runs')
OPTIONAL_TABLES = ('annotation_events',)
CHECKS = {
'annotation_events': ["kind IN ('tag','research_type')", 'retracts_id IS NULL OR retracts_id<>id'],
'venues': ["venue_type IN ('conference','journal','review_platform')"],
'edition_sources': ["status IN ('pending','running','complete','partial','blocked','unavailable','not_yet_published')"],
'decisions': ["status IN ('accepted','rejected','desk_rejected','withdrawn','under_review','unknown')"],
@@ -50,10 +52,10 @@ def quote(name):
def inspect(connection):
business = {name for schema, name, kind, *_ in connection.execute('PRAGMA table_list')
if schema == 'main' and kind == 'table' and not name.startswith('sqlite_')}
if business != set(TABLES):
if not set(TABLES) <= business or business - set(TABLES) - set(OPTIONAL_TABLES):
raise ValueError('Unexpected business tables: ' + repr(sorted(business ^ set(TABLES))))
result = {}
for table in TABLES:
for table in (*TABLES, *(name for name in OPTIONAL_TABLES if name in business)):
columns = [{'name': r[1], 'type': r[2].upper(), 'notnull': bool(r[3]),
'default': r[4], 'pk': r[5]}
for r in connection.execute('PRAGMA table_info(' + quote(table) + ')')]