Files
paper-atlas/paper_atlas/ingestion/event_identity.py
T
codex-bot 4182070e76
Metadata integrity checks / metadata (push) Successful in 1m45s
Scope incremental parents to events and verify integrated write safeguards
2026-11-08 21:04:05 +08:00

222 lines
14 KiB
Python

"""Explicit, plan-bound edition event-key upgrade; never reimport submissions."""
import argparse
from contextlib import closing
import hashlib
import json
from pathlib import Path
import sqlite3
import sys
from paper_atlas.db.backend import database_arguments, open_read_connection, require_sqlite_writable
from paper_atlas.db.events import has_event_key, pinned_www_fall
from paper_atlas.postgres.schema import TABLES, OPTIONAL_TABLES, quote
MARKER = 'edition_event_identity_v1:'
def digest(value):
return hashlib.sha256(json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(',', ':')).encode()).hexdigest()
def values(rows):
return [[r[i] for i in range(len(r))] for r in rows]
def exact_counts(c, *, page_size=5000, progress=None):
"""Count every row in the same snapshot through bounded primary-key pages."""
if not isinstance(page_size, int) or page_size < 1:
raise ValueError('Positive count page size required')
result = {}
if getattr(c, 'postgres', False):
optional = [table for table in OPTIONAL_TABLES if c.execute("SELECT 1 FROM information_schema.tables WHERE table_schema=current_schema() AND table_name=?", (table,)).fetchone()]
else:
optional = [table for table in OPTIONAL_TABLES if c.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)).fetchone()]
for table in (*TABLES, *optional):
if table == 'schema_migrations':
continue
if getattr(c, 'postgres', False):
columns = [r[0] for r in c.execute("SELECT a.attname FROM pg_constraint pc JOIN pg_attribute a ON a.attrelid=pc.conrelid AND a.attnum=ANY(pc.conkey) WHERE pc.conrelid=?::regclass AND pc.contype='p' ORDER BY array_position(pc.conkey,a.attnum)", (table,))]
else:
columns = [r[1] for r in sorted(c.execute('PRAGMA table_info('+quote(table)+')'), key=lambda r:r[5]) if r[5]]
if not columns:
raise ValueError('Primary key required for exact count: '+table)
fields = ','.join(quote(column) for column in columns)
last = None; count = 0; pages = 0
if progress:
progress({'table':table, 'rows_counted':0, 'complete':False})
while True:
where = (' WHERE ('+fields+') > ('+','.join('?' for _ in columns)+')') if last is not None else ''
batch = values(c.execute('SELECT '+fields+' FROM '+quote(table)+where+' ORDER BY '+fields+' LIMIT ?', (*last, page_size) if last is not None else (page_size,)).fetchall())
count += len(batch); pages += 1
complete = len(batch) < page_size
if progress and (complete or pages % 20 == 0):
progress({'table':table, 'rows_counted':count, 'complete':complete})
if complete:
break
last = batch[-1]
result[table] = count
return result
def inspect(c, *, progress=None):
enabled = has_event_key(c)
if getattr(c, 'postgres', False):
columns = [r[0] for r in c.execute("SELECT column_name FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='editions' ORDER BY ordinal_position")]
unique = [r[0] for r in c.execute("SELECT pg_get_constraintdef(oid) FROM pg_constraint WHERE conrelid='editions'::regclass AND contype='u'")]
expected_unique = ['UNIQUE (venue_id, year, track, event_key)' if enabled else 'UNIQUE (venue_id, year, track)']
else:
columns = [r[1] for r in c.execute('PRAGMA table_info(editions)')]
indexes = list(c.execute('PRAGMA index_list(editions)'))
unique = [tuple(r[2] for r in c.execute('PRAGMA index_info('+quote(index[1])+')')) for index in indexes if index[2]]
expected_unique = [('venue_id','year','track','event_key') if enabled else ('venue_id','year','track')]
if len(indexes) != 1:
raise ValueError('Unexpected edition indexes; require a reviewed migration')
if columns != ['id','venue_id','year','track'] + (['event_key'] if enabled else []) or unique != expected_unique:
raise ValueError('Unexpected edition schema; require a reviewed migration')
rows = values(c.execute('SELECT id,venue_id,year,track FROM editions ORDER BY id'))
keys = values(c.execute('SELECT id,event_key FROM editions ORDER BY id')) if enabled else [[r[0], ''] for r in rows]
counts = exact_counts(c, progress=progress)
targets = values(c.execute("SELECT e.id FROM editions e JOIN venues v ON v.id=e.venue_id WHERE v.acronym='WWW' AND e.year=1995 AND e.track='main' ORDER BY e.id"))
assignments = []
for (eid,) in targets:
current_key = dict(keys)[eid]
if current_key:
continue
evidence = values(c.execute('''SELECT s.id,src.name,r.id,r.external_id,r.metadata_json FROM submissions s
JOIN submission_sources ss ON ss.submission_id=s.id JOIN source_records r ON r.id=ss.source_record_id
JOIN sources src ON src.id=r.source_id WHERE s.edition_id=? ORDER BY s.id,r.id''', (eid,)))
count = c.execute('SELECT count(*) FROM submissions WHERE edition_id=?', (eid,)).fetchone()[0]
supported = {r[0] for r in evidence if pinned_www_fall(r[1], json.loads(r[4]))}
if not count or len(supported) != count:
raise ValueError('Existing unlabelled WWW1995 edition lacks complete pinned Fall evidence')
assignments.append({'edition_id':eid, 'event_key':'www-1995-fall', 'submissions':count,
'source_evidence_sha256':digest(evidence)})
# Protect all rows in the affected attempt history, including non-DBLP evidence.
target_ids = [r[0] for r in targets]
protected = {}
for table,sql in {
'submissions': 'SELECT s.* FROM submissions s WHERE s.edition_id IN ({}) ORDER BY s.id',
'decisions': 'SELECT d.* FROM decisions d JOIN submissions s ON s.id=d.submission_id WHERE s.edition_id IN ({}) ORDER BY d.id',
'submission_sources': 'SELECT ss.* FROM submission_sources ss JOIN submissions s ON s.id=ss.submission_id WHERE s.edition_id IN ({}) ORDER BY ss.submission_id,ss.source_record_id,ss.source_submission_id',
'decision_sources': 'SELECT ds.* FROM decision_sources ds JOIN decisions d ON d.id=ds.decision_id JOIN submissions s ON s.id=d.submission_id WHERE s.edition_id IN ({}) ORDER BY ds.decision_id,ds.source_record_id,ds.source_decision_id',
'source_records': 'SELECT DISTINCT r.* FROM source_records r JOIN submission_sources ss ON ss.source_record_id=r.id JOIN submissions s ON s.id=ss.submission_id WHERE s.edition_id IN ({}) ORDER BY r.id',
}.items():
records = values(c.execute(sql.format(','.join('?' for _ in target_ids)), target_ids)) if target_ids else []
protected[table] = {'count':len(records), 'sha256':digest(records)}
return {'event_column':enabled, 'editions_sha256':digest(rows), 'edition_ids':[r[0] for r in rows],
'event_keys':keys, 'counts':counts, 'assignments':assignments, 'protected':protected}
def database_identity(c, db=None):
if getattr(c, 'postgres', False):
r = c.execute('SELECT current_database(),current_schema()').fetchone()
return {'engine':'postgres', 'database':r[0], 'schema':r[1]}
return {'engine':'sqlite', 'path':str(Path(db).resolve())}
def plan(db=None, *, pg_service=None, pg_schema=None, progress=None):
with closing(open_read_connection(db, pg_service=pg_service, pg_schema=pg_schema)) as c:
return {'format':1, 'database':database_identity(c, db), 'before':inspect(c, progress=progress)}
def apply_plan(c, proposal, *, progress=None):
before = proposal['before']
if proposal.get('format') != 1:
raise ValueError('Unsupported event identity plan')
if getattr(c, 'postgres', False):
c.execute('LOCK TABLE editions IN ACCESS EXCLUSIVE MODE')
current = inspect(c, progress=progress)
signature = digest(proposal)
marker = MARKER + signature
if c.execute('SELECT 1 FROM schema_migrations WHERE name=?', (marker,)).fetchone():
expected_keys = dict(before['event_keys'])
expected_keys.update({a['edition_id']:a['event_key'] for a in before['assignments']})
if (not current['event_column'] or current['editions_sha256'] != before['editions_sha256']
or current['protected'] != before['protected'] or current['counts'] != before['counts']
or dict(current['event_keys']) != expected_keys):
raise ValueError('Applied plan no longer matches current data; generate a new plan')
return {'status':'already_applied', 'updated_editions':0, 'after':current}
if current != before:
raise ValueError('Event identity plan is stale; generate a new plan')
if not current['event_column']:
if getattr(c, 'postgres', False):
constraints = c.execute("SELECT conname,pg_get_constraintdef(oid) FROM pg_constraint WHERE conrelid='editions'::regclass AND contype='u'").fetchall()
matches = [name for name,definition in constraints if definition == 'UNIQUE (venue_id, year, track)']
if len(matches) != 1 or len(constraints) != 1:
raise ValueError('Unexpected legacy edition uniqueness')
c.execute("ALTER TABLE editions ADD COLUMN event_key TEXT COLLATE \"C\" NOT NULL DEFAULT ''")
c.execute('ALTER TABLE editions DROP CONSTRAINT '+quote(matches[0]))
c.execute('ALTER TABLE editions ADD CONSTRAINT editions_event_identity_unique UNIQUE(venue_id,year,track,event_key)')
else:
unique = [tuple(r[2] for r in c.execute('PRAGMA index_info('+quote(index[1])+')'))
for index in c.execute('PRAGMA index_list(editions)') if index[2]]
if unique != [('venue_id','year','track')]:
raise ValueError('Unexpected legacy edition uniqueness')
c.execute("CREATE TABLE editions_event_upgrade(id INTEGER PRIMARY KEY,venue_id INTEGER NOT NULL REFERENCES venues(id),year INTEGER NOT NULL,track TEXT NOT NULL DEFAULT 'main',event_key TEXT NOT NULL DEFAULT '',UNIQUE(venue_id,year,track,event_key))")
c.execute('INSERT INTO editions_event_upgrade(id,venue_id,year,track) SELECT id,venue_id,year,track FROM editions')
c.execute('DROP TABLE editions')
c.execute('ALTER TABLE editions_event_upgrade RENAME TO editions')
for item in before['assignments']:
result = c.execute("UPDATE editions SET event_key=? WHERE id=? AND event_key=''", (item['event_key'],item['edition_id']))
if result.rowcount != 1:
raise ValueError('Edition assignment changed during upgrade')
c.execute('INSERT INTO schema_migrations(name) VALUES(?)', (marker,))
after = inspect(c, progress=progress)
expected_keys = dict(before['event_keys'])
expected_keys.update({a['edition_id']:a['event_key'] for a in before['assignments']})
if (after['editions_sha256'] != before['editions_sha256'] or after['protected'] != before['protected']
or after['counts'] != before['counts'] or dict(after['event_keys']) != expected_keys):
raise ValueError('Event upgrade changed protected data')
if not getattr(c, 'postgres', False) and c.execute('PRAGMA foreign_key_check').fetchall():
raise ValueError('Foreign key violation after event upgrade')
return {'status':'applied', 'updated_editions':len(before['assignments']), 'after':after}
def upgrade(proposal, db=None, *, pg_service=None, pg_schema=None, progress=None):
if pg_service:
from paper_atlas.postgres.writer import WriteConnection
with closing(WriteConnection(pg_service, pg_schema)) as c:
if proposal['database'] != database_identity(c):
raise ValueError('Plan targets a different database')
try:
result = apply_plan(c, proposal, progress=progress); c.commit(); return result
except BaseException:
c.rollback(); raise
require_sqlite_writable(db)
if not Path(db).is_file():
raise ValueError('Existing SQLite sample required')
with closing(sqlite3.connect(db, timeout=30)) as c:
if proposal['database'] != database_identity(c, db):
raise ValueError('Plan targets a different database')
c.execute('PRAGMA foreign_keys=OFF') # Outside the transaction; no cascading deletes.
try:
c.execute('BEGIN IMMEDIATE')
result = apply_plan(c, proposal, progress=progress); c.commit(); return result
except BaseException:
c.rollback(); raise
finally:
c.execute('PRAGMA foreign_keys=ON')
def main(argv=None, *, prog=None):
p = argparse.ArgumentParser(prog=prog); database_arguments(p)
p.add_argument('--out', required=True)
p.add_argument('--apply-plan'); p.add_argument('--plan-sha256')
args = p.parse_args(argv)
if bool(args.apply_plan) != bool(args.plan_sha256):
p.error('--apply-plan and --plan-sha256 must be supplied together')
target = Path(args.out)
if target.exists():
raise ValueError('Refusing to overwrite an existing plan/report')
progress = lambda event: print(json.dumps({'phase':'exact_row_count', **event}), file=sys.stderr, flush=True)
if args.apply_plan:
raw = Path(args.apply_plan).read_bytes()
if hashlib.sha256(raw).hexdigest() != args.plan_sha256:
raise ValueError('Plan file SHA256 differs from approved bytes')
result = upgrade(json.loads(raw), args.db, pg_service=args.pg_service, pg_schema=args.pg_schema, progress=progress)
else:
result = plan(args.db, pg_service=args.pg_service, pg_schema=args.pg_schema, progress=progress)
target.write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n')
print(json.dumps({'out':str(target), 'sha256':hashlib.sha256(target.read_bytes()).hexdigest(),
'status':result.get('status','planned')}), flush=True)