222 lines
14 KiB
Python
222 lines
14 KiB
Python
"""Explicit, plan-bound edition event-key upgrade; never reimport submissions."""
|
|
import argparse
|
|
from contextlib import closing
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import sqlite3
|
|
import sys
|
|
|
|
from paper_atlas.db.backend import database_arguments, open_read_connection, require_sqlite_writable
|
|
from paper_atlas.db.events import has_event_key, pinned_www_fall
|
|
from paper_atlas.postgres.schema import TABLES, OPTIONAL_TABLES, quote
|
|
|
|
MARKER = 'edition_event_identity_v1:'
|
|
|
|
|
|
def digest(value):
|
|
return hashlib.sha256(json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(',', ':')).encode()).hexdigest()
|
|
|
|
|
|
def values(rows):
|
|
return [[r[i] for i in range(len(r))] for r in rows]
|
|
|
|
|
|
def exact_counts(c, *, page_size=5000, progress=None):
|
|
"""Count every row in the same snapshot through bounded primary-key pages."""
|
|
if not isinstance(page_size, int) or page_size < 1:
|
|
raise ValueError('Positive count page size required')
|
|
result = {}
|
|
if getattr(c, 'postgres', False):
|
|
optional = [table for table in OPTIONAL_TABLES if c.execute("SELECT 1 FROM information_schema.tables WHERE table_schema=current_schema() AND table_name=?", (table,)).fetchone()]
|
|
else:
|
|
optional = [table for table in OPTIONAL_TABLES if c.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (table,)).fetchone()]
|
|
for table in (*TABLES, *optional):
|
|
if table == 'schema_migrations':
|
|
continue
|
|
if getattr(c, 'postgres', False):
|
|
columns = [r[0] for r in c.execute("SELECT a.attname FROM pg_constraint pc JOIN pg_attribute a ON a.attrelid=pc.conrelid AND a.attnum=ANY(pc.conkey) WHERE pc.conrelid=?::regclass AND pc.contype='p' ORDER BY array_position(pc.conkey,a.attnum)", (table,))]
|
|
else:
|
|
columns = [r[1] for r in sorted(c.execute('PRAGMA table_info('+quote(table)+')'), key=lambda r:r[5]) if r[5]]
|
|
if not columns:
|
|
raise ValueError('Primary key required for exact count: '+table)
|
|
fields = ','.join(quote(column) for column in columns)
|
|
last = None; count = 0; pages = 0
|
|
if progress:
|
|
progress({'table':table, 'rows_counted':0, 'complete':False})
|
|
while True:
|
|
where = (' WHERE ('+fields+') > ('+','.join('?' for _ in columns)+')') if last is not None else ''
|
|
batch = values(c.execute('SELECT '+fields+' FROM '+quote(table)+where+' ORDER BY '+fields+' LIMIT ?', (*last, page_size) if last is not None else (page_size,)).fetchall())
|
|
count += len(batch); pages += 1
|
|
complete = len(batch) < page_size
|
|
if progress and (complete or pages % 20 == 0):
|
|
progress({'table':table, 'rows_counted':count, 'complete':complete})
|
|
if complete:
|
|
break
|
|
last = batch[-1]
|
|
result[table] = count
|
|
return result
|
|
|
|
|
|
def inspect(c, *, progress=None):
|
|
enabled = has_event_key(c)
|
|
if getattr(c, 'postgres', False):
|
|
columns = [r[0] for r in c.execute("SELECT column_name FROM information_schema.columns WHERE table_schema=current_schema() AND table_name='editions' ORDER BY ordinal_position")]
|
|
unique = [r[0] for r in c.execute("SELECT pg_get_constraintdef(oid) FROM pg_constraint WHERE conrelid='editions'::regclass AND contype='u'")]
|
|
expected_unique = ['UNIQUE (venue_id, year, track, event_key)' if enabled else 'UNIQUE (venue_id, year, track)']
|
|
else:
|
|
columns = [r[1] for r in c.execute('PRAGMA table_info(editions)')]
|
|
indexes = list(c.execute('PRAGMA index_list(editions)'))
|
|
unique = [tuple(r[2] for r in c.execute('PRAGMA index_info('+quote(index[1])+')')) for index in indexes if index[2]]
|
|
expected_unique = [('venue_id','year','track','event_key') if enabled else ('venue_id','year','track')]
|
|
if len(indexes) != 1:
|
|
raise ValueError('Unexpected edition indexes; require a reviewed migration')
|
|
if columns != ['id','venue_id','year','track'] + (['event_key'] if enabled else []) or unique != expected_unique:
|
|
raise ValueError('Unexpected edition schema; require a reviewed migration')
|
|
rows = values(c.execute('SELECT id,venue_id,year,track FROM editions ORDER BY id'))
|
|
keys = values(c.execute('SELECT id,event_key FROM editions ORDER BY id')) if enabled else [[r[0], ''] for r in rows]
|
|
counts = exact_counts(c, progress=progress)
|
|
targets = values(c.execute("SELECT e.id FROM editions e JOIN venues v ON v.id=e.venue_id WHERE v.acronym='WWW' AND e.year=1995 AND e.track='main' ORDER BY e.id"))
|
|
assignments = []
|
|
for (eid,) in targets:
|
|
current_key = dict(keys)[eid]
|
|
if current_key:
|
|
continue
|
|
evidence = values(c.execute('''SELECT s.id,src.name,r.id,r.external_id,r.metadata_json FROM submissions s
|
|
JOIN submission_sources ss ON ss.submission_id=s.id JOIN source_records r ON r.id=ss.source_record_id
|
|
JOIN sources src ON src.id=r.source_id WHERE s.edition_id=? ORDER BY s.id,r.id''', (eid,)))
|
|
count = c.execute('SELECT count(*) FROM submissions WHERE edition_id=?', (eid,)).fetchone()[0]
|
|
supported = {r[0] for r in evidence if pinned_www_fall(r[1], json.loads(r[4]))}
|
|
if not count or len(supported) != count:
|
|
raise ValueError('Existing unlabelled WWW1995 edition lacks complete pinned Fall evidence')
|
|
assignments.append({'edition_id':eid, 'event_key':'www-1995-fall', 'submissions':count,
|
|
'source_evidence_sha256':digest(evidence)})
|
|
# Protect all rows in the affected attempt history, including non-DBLP evidence.
|
|
target_ids = [r[0] for r in targets]
|
|
protected = {}
|
|
for table,sql in {
|
|
'submissions': 'SELECT s.* FROM submissions s WHERE s.edition_id IN ({}) ORDER BY s.id',
|
|
'decisions': 'SELECT d.* FROM decisions d JOIN submissions s ON s.id=d.submission_id WHERE s.edition_id IN ({}) ORDER BY d.id',
|
|
'submission_sources': 'SELECT ss.* FROM submission_sources ss JOIN submissions s ON s.id=ss.submission_id WHERE s.edition_id IN ({}) ORDER BY ss.submission_id,ss.source_record_id,ss.source_submission_id',
|
|
'decision_sources': 'SELECT ds.* FROM decision_sources ds JOIN decisions d ON d.id=ds.decision_id JOIN submissions s ON s.id=d.submission_id WHERE s.edition_id IN ({}) ORDER BY ds.decision_id,ds.source_record_id,ds.source_decision_id',
|
|
'source_records': 'SELECT DISTINCT r.* FROM source_records r JOIN submission_sources ss ON ss.source_record_id=r.id JOIN submissions s ON s.id=ss.submission_id WHERE s.edition_id IN ({}) ORDER BY r.id',
|
|
}.items():
|
|
records = values(c.execute(sql.format(','.join('?' for _ in target_ids)), target_ids)) if target_ids else []
|
|
protected[table] = {'count':len(records), 'sha256':digest(records)}
|
|
return {'event_column':enabled, 'editions_sha256':digest(rows), 'edition_ids':[r[0] for r in rows],
|
|
'event_keys':keys, 'counts':counts, 'assignments':assignments, 'protected':protected}
|
|
|
|
|
|
def database_identity(c, db=None):
|
|
if getattr(c, 'postgres', False):
|
|
r = c.execute('SELECT current_database(),current_schema()').fetchone()
|
|
return {'engine':'postgres', 'database':r[0], 'schema':r[1]}
|
|
return {'engine':'sqlite', 'path':str(Path(db).resolve())}
|
|
|
|
|
|
def plan(db=None, *, pg_service=None, pg_schema=None, progress=None):
|
|
with closing(open_read_connection(db, pg_service=pg_service, pg_schema=pg_schema)) as c:
|
|
return {'format':1, 'database':database_identity(c, db), 'before':inspect(c, progress=progress)}
|
|
|
|
|
|
def apply_plan(c, proposal, *, progress=None):
|
|
before = proposal['before']
|
|
if proposal.get('format') != 1:
|
|
raise ValueError('Unsupported event identity plan')
|
|
if getattr(c, 'postgres', False):
|
|
c.execute('LOCK TABLE editions IN ACCESS EXCLUSIVE MODE')
|
|
current = inspect(c, progress=progress)
|
|
signature = digest(proposal)
|
|
marker = MARKER + signature
|
|
if c.execute('SELECT 1 FROM schema_migrations WHERE name=?', (marker,)).fetchone():
|
|
expected_keys = dict(before['event_keys'])
|
|
expected_keys.update({a['edition_id']:a['event_key'] for a in before['assignments']})
|
|
if (not current['event_column'] or current['editions_sha256'] != before['editions_sha256']
|
|
or current['protected'] != before['protected'] or current['counts'] != before['counts']
|
|
or dict(current['event_keys']) != expected_keys):
|
|
raise ValueError('Applied plan no longer matches current data; generate a new plan')
|
|
return {'status':'already_applied', 'updated_editions':0, 'after':current}
|
|
if current != before:
|
|
raise ValueError('Event identity plan is stale; generate a new plan')
|
|
if not current['event_column']:
|
|
if getattr(c, 'postgres', False):
|
|
constraints = c.execute("SELECT conname,pg_get_constraintdef(oid) FROM pg_constraint WHERE conrelid='editions'::regclass AND contype='u'").fetchall()
|
|
matches = [name for name,definition in constraints if definition == 'UNIQUE (venue_id, year, track)']
|
|
if len(matches) != 1 or len(constraints) != 1:
|
|
raise ValueError('Unexpected legacy edition uniqueness')
|
|
c.execute("ALTER TABLE editions ADD COLUMN event_key TEXT COLLATE \"C\" NOT NULL DEFAULT ''")
|
|
c.execute('ALTER TABLE editions DROP CONSTRAINT '+quote(matches[0]))
|
|
c.execute('ALTER TABLE editions ADD CONSTRAINT editions_event_identity_unique UNIQUE(venue_id,year,track,event_key)')
|
|
else:
|
|
unique = [tuple(r[2] for r in c.execute('PRAGMA index_info('+quote(index[1])+')'))
|
|
for index in c.execute('PRAGMA index_list(editions)') if index[2]]
|
|
if unique != [('venue_id','year','track')]:
|
|
raise ValueError('Unexpected legacy edition uniqueness')
|
|
c.execute("CREATE TABLE editions_event_upgrade(id INTEGER PRIMARY KEY,venue_id INTEGER NOT NULL REFERENCES venues(id),year INTEGER NOT NULL,track TEXT NOT NULL DEFAULT 'main',event_key TEXT NOT NULL DEFAULT '',UNIQUE(venue_id,year,track,event_key))")
|
|
c.execute('INSERT INTO editions_event_upgrade(id,venue_id,year,track) SELECT id,venue_id,year,track FROM editions')
|
|
c.execute('DROP TABLE editions')
|
|
c.execute('ALTER TABLE editions_event_upgrade RENAME TO editions')
|
|
for item in before['assignments']:
|
|
result = c.execute("UPDATE editions SET event_key=? WHERE id=? AND event_key=''", (item['event_key'],item['edition_id']))
|
|
if result.rowcount != 1:
|
|
raise ValueError('Edition assignment changed during upgrade')
|
|
c.execute('INSERT INTO schema_migrations(name) VALUES(?)', (marker,))
|
|
after = inspect(c, progress=progress)
|
|
expected_keys = dict(before['event_keys'])
|
|
expected_keys.update({a['edition_id']:a['event_key'] for a in before['assignments']})
|
|
if (after['editions_sha256'] != before['editions_sha256'] or after['protected'] != before['protected']
|
|
or after['counts'] != before['counts'] or dict(after['event_keys']) != expected_keys):
|
|
raise ValueError('Event upgrade changed protected data')
|
|
if not getattr(c, 'postgres', False) and c.execute('PRAGMA foreign_key_check').fetchall():
|
|
raise ValueError('Foreign key violation after event upgrade')
|
|
return {'status':'applied', 'updated_editions':len(before['assignments']), 'after':after}
|
|
|
|
|
|
def upgrade(proposal, db=None, *, pg_service=None, pg_schema=None, progress=None):
|
|
if pg_service:
|
|
from paper_atlas.postgres.writer import WriteConnection
|
|
with closing(WriteConnection(pg_service, pg_schema)) as c:
|
|
if proposal['database'] != database_identity(c):
|
|
raise ValueError('Plan targets a different database')
|
|
try:
|
|
result = apply_plan(c, proposal, progress=progress); c.commit(); return result
|
|
except BaseException:
|
|
c.rollback(); raise
|
|
require_sqlite_writable(db)
|
|
if not Path(db).is_file():
|
|
raise ValueError('Existing SQLite sample required')
|
|
with closing(sqlite3.connect(db, timeout=30)) as c:
|
|
if proposal['database'] != database_identity(c, db):
|
|
raise ValueError('Plan targets a different database')
|
|
c.execute('PRAGMA foreign_keys=OFF') # Outside the transaction; no cascading deletes.
|
|
try:
|
|
c.execute('BEGIN IMMEDIATE')
|
|
result = apply_plan(c, proposal, progress=progress); c.commit(); return result
|
|
except BaseException:
|
|
c.rollback(); raise
|
|
finally:
|
|
c.execute('PRAGMA foreign_keys=ON')
|
|
|
|
|
|
def main(argv=None, *, prog=None):
|
|
p = argparse.ArgumentParser(prog=prog); database_arguments(p)
|
|
p.add_argument('--out', required=True)
|
|
p.add_argument('--apply-plan'); p.add_argument('--plan-sha256')
|
|
args = p.parse_args(argv)
|
|
if bool(args.apply_plan) != bool(args.plan_sha256):
|
|
p.error('--apply-plan and --plan-sha256 must be supplied together')
|
|
target = Path(args.out)
|
|
if target.exists():
|
|
raise ValueError('Refusing to overwrite an existing plan/report')
|
|
progress = lambda event: print(json.dumps({'phase':'exact_row_count', **event}), file=sys.stderr, flush=True)
|
|
if args.apply_plan:
|
|
raw = Path(args.apply_plan).read_bytes()
|
|
if hashlib.sha256(raw).hexdigest() != args.plan_sha256:
|
|
raise ValueError('Plan file SHA256 differs from approved bytes')
|
|
result = upgrade(json.loads(raw), args.db, pg_service=args.pg_service, pg_schema=args.pg_schema, progress=progress)
|
|
else:
|
|
result = plan(args.db, pg_service=args.pg_service, pg_schema=args.pg_schema, progress=progress)
|
|
target.write_text(json.dumps(result, ensure_ascii=False, indent=2)+'\n')
|
|
print(json.dumps({'out':str(target), 'sha256':hashlib.sha256(target.read_bytes()).hexdigest(),
|
|
'status':result.get('status','planned')}), flush=True)
|