129 lines
6.3 KiB
Python
129 lines
6.3 KiB
Python
"""Versioned publication-source scopes and guards against stale source refreshes."""
|
|
import json
|
|
import re
|
|
|
|
TABLE = 'source_scope_revisions'
|
|
MARKER = 'source_scope_revisions_v1'
|
|
COLUMNS = ('version_sha256', 'source_id', 'venue_id', 'year', 'event_key', 'list_url',
|
|
'parent_sha256', 'records_json', 'plan_json', 'applied_at')
|
|
|
|
|
|
def available(c):
|
|
if getattr(c, 'postgres', False):
|
|
return bool(c.execute("SELECT 1 FROM information_schema.tables WHERE table_schema=current_schema() AND table_name=?", (TABLE,)).fetchone())
|
|
return bool(c.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,)).fetchone())
|
|
|
|
|
|
def initialize(c):
|
|
exists = available(c)
|
|
marker = bool(c.execute('SELECT 1 FROM schema_migrations WHERE name=?', (MARKER,)).fetchone())
|
|
if exists != marker:
|
|
raise ValueError('Source revision table/marker mismatch')
|
|
if exists:
|
|
result = c.execute('SELECT * FROM source_scope_revisions LIMIT 0')
|
|
description = result.cursor.description if getattr(c, 'postgres', False) else result.description
|
|
if tuple(x[0] for x in description) != COLUMNS:
|
|
raise ValueError('Unexpected source revision schema')
|
|
return
|
|
ddl = '''CREATE TABLE source_scope_revisions (
|
|
version_sha256 TEXT PRIMARY KEY,
|
|
source_id INTEGER NOT NULL REFERENCES sources(id),
|
|
venue_id INTEGER NOT NULL REFERENCES venues(id),
|
|
year INTEGER NOT NULL, event_key TEXT NOT NULL, list_url TEXT NOT NULL,
|
|
parent_sha256 TEXT UNIQUE REFERENCES source_scope_revisions(version_sha256),
|
|
records_json TEXT NOT NULL, plan_json TEXT NOT NULL,
|
|
applied_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP)'''
|
|
if getattr(c, 'postgres', False):
|
|
ddl = re.sub(r'\bINTEGER\b', 'BIGINT', ddl)
|
|
ddl = re.sub(r'\bTEXT\b', 'TEXT COLLATE "C"', ddl)
|
|
c.execute(ddl)
|
|
c.execute('CREATE INDEX source_scope_revisions_scope ON source_scope_revisions(source_id,venue_id,year,event_key,list_url)')
|
|
c.execute('INSERT INTO schema_migrations(name) VALUES(?)', (MARKER,))
|
|
|
|
|
|
def heads(c, source_id):
|
|
if not available(c):
|
|
return []
|
|
if not c.execute('SELECT 1 FROM schema_migrations WHERE name=?', (MARKER,)).fetchone():
|
|
raise ValueError('Source revision table/marker mismatch')
|
|
return c.execute('''SELECT v.version_sha256,v.venue_id,v.year,v.event_key,v.list_url
|
|
FROM source_scope_revisions v WHERE v.source_id=? AND NOT EXISTS
|
|
(SELECT 1 FROM source_scope_revisions child WHERE child.parent_sha256=v.version_sha256)
|
|
ORDER BY v.version_sha256''', (source_id,)).fetchall()
|
|
|
|
|
|
def current_rows(c, source_id, cache):
|
|
active = heads(c, source_id)
|
|
signatures = tuple(r[0] for r in active)
|
|
# A new head invalidates the cache, even when maintenance uses the same Store.
|
|
if cache.get('signatures') != (source_id, signatures):
|
|
records = {}; scopes = {}
|
|
for signature in signatures:
|
|
raw = c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone()[0]
|
|
scopes[signature] = json.loads(raw)
|
|
for row in scopes[signature]:
|
|
ext = row['external_id']
|
|
if ext in records and records[ext] != row:
|
|
raise ValueError('Conflicting active source snapshots')
|
|
records[ext] = row
|
|
cache.clear()
|
|
cache.update(signatures=(source_id, signatures), records=records, scopes=scopes)
|
|
return cache['records']
|
|
|
|
|
|
def subset(expected, actual):
|
|
"""Reviewed source fields must survive additional work-identity/curation evidence."""
|
|
return isinstance(actual, dict) and all(k in actual and actual[k] == v for k, v in expected.items())
|
|
|
|
|
|
def require_record(c, source_id, external, url, title, abstract, authors, identifiers, metadata, cache):
|
|
expected = current_rows(c, source_id, cache).get(external)
|
|
if expected is None:
|
|
return
|
|
actual = {'url': url, 'title': title, 'abstract': abstract, 'authors': authors}
|
|
if any(actual[k] != expected.get(k) for k in actual):
|
|
raise ValueError('Stale source snapshot: reconcile the source version before refresh')
|
|
if not subset(expected.get('metadata', {}), metadata or {}) or not subset(expected.get('identifiers', {}), identifiers or {}):
|
|
raise ValueError('Stale source snapshot metadata or identifiers')
|
|
|
|
|
|
def scope_head(c, source_id, manifest):
|
|
venue = c.execute('SELECT id FROM venues WHERE acronym=?', (manifest.get('venue'),)).fetchone()
|
|
if not venue:
|
|
return None
|
|
matches = [r for r in heads(c, source_id)
|
|
if [r[i] for i in range(1, 5)] == [venue[0], manifest.get('year'), manifest.get('event_key', ''), manifest.get('list_url')]]
|
|
if len(matches) > 1:
|
|
raise ValueError('Forked source revision history')
|
|
return matches[0][0] if matches else None
|
|
|
|
|
|
def require_manifest(c, source_id, manifest, cache):
|
|
current_rows(c, source_id, cache)
|
|
signature = scope_head(c, source_id, manifest)
|
|
if signature is None:
|
|
return
|
|
expected = sum(r['track'] == manifest.get('track') for r in cache['scopes'][signature])
|
|
bound = manifest.get('coverage_expected_count') if manifest.get('coverage_mode') == 'incremental' else manifest.get('count')
|
|
if expected < 1 or bound != expected or manifest.get('status') != 'partial':
|
|
raise ValueError('Source coverage manifest differs from current source snapshot')
|
|
|
|
|
|
def require_row(c, source_id, row, manifest, cache):
|
|
expected = current_rows(c, source_id, cache).get(row['external_id'])
|
|
scoped_manifest = {**manifest, 'venue': row['venue'], 'year': row['year'], 'event_key': row.get('event_key', '')}
|
|
signature = scope_head(c, source_id, scoped_manifest)
|
|
if expected is None:
|
|
if signature is not None:
|
|
raise ValueError('Publication absent from current source snapshot; reconcile source membership first')
|
|
return
|
|
if signature is None or not any(r['external_id'] == row['external_id'] for r in cache['scopes'][signature]):
|
|
raise ValueError('Source listing differs from current source snapshot')
|
|
for key, value in expected.items():
|
|
if key in ('metadata', 'identifiers'):
|
|
okay = subset(value, row.get(key))
|
|
else:
|
|
okay = row.get(key) == value
|
|
if not okay:
|
|
raise ValueError('Stale source snapshot field: ' + key)
|