"""Versioned publication-source scopes and guards against stale source refreshes.""" import json import re TABLE = 'source_scope_revisions' MARKER = 'source_scope_revisions_v1' COLUMNS = ('version_sha256', 'source_id', 'venue_id', 'year', 'event_key', 'list_url', 'parent_sha256', 'records_json', 'plan_json', 'applied_at') def available(c): if getattr(c, 'postgres', False): return bool(c.execute("SELECT 1 FROM information_schema.tables WHERE table_schema=current_schema() AND table_name=?", (TABLE,)).fetchone()) return bool(c.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,)).fetchone()) def initialize(c): exists = available(c) marker = bool(c.execute('SELECT 1 FROM schema_migrations WHERE name=?', (MARKER,)).fetchone()) if exists != marker: raise ValueError('Source revision table/marker mismatch') if exists: result = c.execute('SELECT * FROM source_scope_revisions LIMIT 0') description = result.cursor.description if getattr(c, 'postgres', False) else result.description if tuple(x[0] for x in description) != COLUMNS: raise ValueError('Unexpected source revision schema') return ddl = '''CREATE TABLE source_scope_revisions ( version_sha256 TEXT PRIMARY KEY, source_id INTEGER NOT NULL REFERENCES sources(id), venue_id INTEGER NOT NULL REFERENCES venues(id), year INTEGER NOT NULL, event_key TEXT NOT NULL, list_url TEXT NOT NULL, parent_sha256 TEXT UNIQUE REFERENCES source_scope_revisions(version_sha256), records_json TEXT NOT NULL, plan_json TEXT NOT NULL, applied_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP)''' if getattr(c, 'postgres', False): ddl = re.sub(r'\bINTEGER\b', 'BIGINT', ddl) ddl = re.sub(r'\bTEXT\b', 'TEXT COLLATE "C"', ddl) c.execute(ddl) c.execute('CREATE INDEX source_scope_revisions_scope ON source_scope_revisions(source_id,venue_id,year,event_key,list_url)') c.execute('INSERT INTO schema_migrations(name) VALUES(?)', (MARKER,)) def heads(c, source_id): if not available(c): return [] if not c.execute('SELECT 1 FROM schema_migrations WHERE name=?', (MARKER,)).fetchone(): raise ValueError('Source revision table/marker mismatch') return c.execute('''SELECT v.version_sha256,v.venue_id,v.year,v.event_key,v.list_url FROM source_scope_revisions v WHERE v.source_id=? AND NOT EXISTS (SELECT 1 FROM source_scope_revisions child WHERE child.parent_sha256=v.version_sha256) ORDER BY v.version_sha256''', (source_id,)).fetchall() def current_rows(c, source_id, cache): active = heads(c, source_id) signatures = tuple(r[0] for r in active) # A new head invalidates the cache, even when maintenance uses the same Store. if cache.get('signatures') != (source_id, signatures): records = {}; scopes = {} for signature in signatures: raw = c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone()[0] scopes[signature] = json.loads(raw) for row in scopes[signature]: ext = row['external_id'] if ext in records and records[ext] != row: raise ValueError('Conflicting active source snapshots') records[ext] = row cache.clear() cache.update(signatures=(source_id, signatures), records=records, scopes=scopes) return cache['records'] def subset(expected, actual): """Reviewed source fields must survive additional work-identity/curation evidence.""" return isinstance(actual, dict) and all(k in actual and actual[k] == v for k, v in expected.items()) def require_record(c, source_id, external, url, title, abstract, authors, identifiers, metadata, cache): expected = current_rows(c, source_id, cache).get(external) if expected is None: return actual = {'url': url, 'title': title, 'abstract': abstract, 'authors': authors} if any(actual[k] != expected.get(k) for k in actual): raise ValueError('Stale source snapshot: reconcile the source version before refresh') if not subset(expected.get('metadata', {}), metadata or {}) or not subset(expected.get('identifiers', {}), identifiers or {}): raise ValueError('Stale source snapshot metadata or identifiers') def scope_head(c, source_id, manifest): venue = c.execute('SELECT id FROM venues WHERE acronym=?', (manifest.get('venue'),)).fetchone() if not venue: return None matches = [r for r in heads(c, source_id) if [r[i] for i in range(1, 5)] == [venue[0], manifest.get('year'), manifest.get('event_key', ''), manifest.get('list_url')]] if len(matches) > 1: raise ValueError('Forked source revision history') return matches[0][0] if matches else None def require_manifest(c, source_id, manifest, cache): current_rows(c, source_id, cache) signature = scope_head(c, source_id, manifest) if signature is None: return expected = sum(r['track'] == manifest.get('track') for r in cache['scopes'][signature]) bound = manifest.get('coverage_expected_count') if manifest.get('coverage_mode') == 'incremental' else manifest.get('count') if expected < 1 or bound != expected or manifest.get('status') != 'partial': raise ValueError('Source coverage manifest differs from current source snapshot') def require_row(c, source_id, row, manifest, cache): expected = current_rows(c, source_id, cache).get(row['external_id']) scoped_manifest = {**manifest, 'venue': row['venue'], 'year': row['year'], 'event_key': row.get('event_key', '')} signature = scope_head(c, source_id, scoped_manifest) if expected is None: if signature is not None: raise ValueError('Publication absent from current source snapshot; reconcile source membership first') return if signature is None or not any(r['external_id'] == row['external_id'] for r in cache['scopes'][signature]): raise ValueError('Source listing differs from current source snapshot') for key, value in expected.items(): if key in ('metadata', 'identifiers'): okay = subset(value, row.get(key)) else: okay = row.get(key) == value if not okay: raise ValueError('Stale source snapshot field: ' + key)