205 lines
12 KiB
Python
205 lines
12 KiB
Python
"""Evidence-bound repair of two existing IJCAI/ECAI edition memberships."""
|
|
import argparse
|
|
from contextlib import closing
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from paper_atlas.db.backend import database_arguments, open_read_connection, open_store
|
|
from paper_atlas.db.submissions import timestamp
|
|
from paper_atlas.collectors.dblp import XML_SHA
|
|
|
|
YEARS = (2018, 2022)
|
|
TARGET = 'ECAI+IJCAI'
|
|
MEMBERS = {'ECAI', 'IJCAI'}
|
|
EVIDENCE = 'https://www.eurai.org/ecai/history'
|
|
MARKER = 'joint_ijcai_ecai_2018_2022_v1'
|
|
|
|
|
|
def _digest(c, sql, params=()):
|
|
sha = hashlib.sha256(); count = 0
|
|
for row in c.execute(sql, params):
|
|
sha.update((json.dumps([row[i] for i in range(len(row))], ensure_ascii=False,
|
|
separators=(',', ':')) + '\n').encode())
|
|
count += 1
|
|
if count > 100000:
|
|
raise ValueError('Joint repair evidence scope exceeded bounded row limit')
|
|
return {'rows': count, 'sha256': sha.hexdigest()}
|
|
|
|
|
|
def fingerprints(c, ids):
|
|
marks = ','.join('?' for _ in ids)
|
|
submissions = 'SELECT id FROM submissions WHERE edition_id IN (' + marks + ')'
|
|
papers = 'SELECT paper_id FROM submissions WHERE edition_id IN (' + marks + ')'
|
|
decisions = 'SELECT id FROM decisions WHERE submission_id IN (' + submissions + ')'
|
|
checks = {
|
|
'papers': ('id IN (' + papers + ')', 'id'),
|
|
'identifiers': ('paper_id IN (' + papers + ')', 'scheme,value'),
|
|
'source_records': ('paper_id IN (' + papers + ')', 'id'),
|
|
'paper_tags': ('paper_id IN (' + papers + ')', 'paper_id,tag_id,evidence_key'),
|
|
'submissions': ('edition_id IN (' + marks + ')', 'id'),
|
|
'decisions': ('submission_id IN (' + submissions + ')', 'id'),
|
|
'submission_sources': ('submission_id IN (' + submissions + ')', 'submission_id,source_record_id,source_submission_id'),
|
|
'submission_identifiers': ('edition_id IN (' + marks + ')', 'edition_id,scheme,value'),
|
|
'decision_sources': ('decision_id IN (' + decisions + ')', 'decision_id,source_record_id,source_decision_id'),
|
|
'edition_sources': ('edition_id IN (' + marks + ')', 'edition_id,source_id,list_url'),
|
|
}
|
|
return {table: _digest(c, 'SELECT * FROM ' + table + ' WHERE ' + where + ' ORDER BY ' + order, ids)
|
|
for table, (where, order) in checks.items()}
|
|
|
|
|
|
def inspect(c):
|
|
target = c.execute('SELECT id FROM venues WHERE acronym=?', (TARGET,)).fetchall()
|
|
if len(target) != 1:
|
|
raise ValueError('Exactly one existing canonical joint venue required')
|
|
target = target[0][0]
|
|
aliases = {r[0] for r in c.execute('SELECT alias FROM venue_aliases WHERE venue_id=?', (target,))}
|
|
if aliases != MEMBERS:
|
|
raise ValueError('Joint target must already have exactly ECAI and IJCAI aliases')
|
|
editions = []
|
|
for year in YEARS:
|
|
rows = c.execute('''SELECT e.id,e.venue_id,v.acronym FROM editions e JOIN venues v ON v.id=e.venue_id
|
|
WHERE v.acronym IN ('IJCAI','ECAI+IJCAI') AND e.year=? AND e.track='main' ''', (year,)).fetchall()
|
|
if len(rows) != 1:
|
|
raise ValueError('Expected one existing edition per year; missing or colliding editions require separate review')
|
|
edition, venue, acronym = (rows[0][i] for i in range(3))
|
|
count = c.execute('SELECT count(*) FROM submissions WHERE edition_id=?', (edition,)).fetchone()[0]
|
|
if not 0 < count <= 5000:
|
|
raise ValueError('Existing edition must have a nonempty bounded submission scope')
|
|
# Every existing submission must be supported by its own pinned DBLP
|
|
# volume record. No title/author matching, new attempts or text writes.
|
|
evidence = c.execute('''SELECT s.id,r.external_id,r.metadata_json FROM submissions s
|
|
JOIN submission_sources ss ON ss.submission_id=s.id
|
|
JOIN source_records r ON r.id=ss.source_record_id JOIN sources src ON src.id=r.source_id
|
|
WHERE s.edition_id=? AND src.name='dblp' ORDER BY s.id,r.id''', (edition,)).fetchall()
|
|
seen = set()
|
|
for row in evidence:
|
|
sid, external, metadata = (row[i] for i in range(3))
|
|
meta = json.loads(metadata)
|
|
bibliography = meta.get('bibliography') or {}
|
|
if (meta.get('snapshot_sha256') != XML_SHA
|
|
or meta.get('proceedings_key') != 'conf/ijcai/' + str(year)
|
|
or meta.get('bibliographic_year') != year
|
|
or bibliography.get('booktitle') not in (['IJCAI'], ['IJCAI/ECAI'])
|
|
or not external.startswith('conf/ijcai/')):
|
|
raise ValueError('Source evidence outside the exact verified IJCAI/ECAI volume')
|
|
seen.add(sid)
|
|
if len(seen) != count:
|
|
raise ValueError('Every submission must have matching original volume evidence')
|
|
editions.append({'id': edition, 'year': year, 'track': 'main', 'from_venue_id': venue,
|
|
'from_venue': acronym, 'target_venue_id': target, 'submissions': count,
|
|
'evidence_url': EVIDENCE, 'proceedings_key': 'conf/ijcai/' + str(year)})
|
|
ids = tuple(e['id'] for e in editions)
|
|
return {'format_version': 1, 'kind': MARKER, 'checked_at': timestamp(), 'editions': editions,
|
|
'content_fingerprints': fingerprints(c, ids), 'target_aliases': sorted(aliases),
|
|
'status': 'already_joint' if all(e['from_venue_id'] == target for e in editions) else 'ready',
|
|
'scope': 'Only these two existing edition venue references; no source text, paper, submission, decision, track, identifier or tag change.'}
|
|
|
|
|
|
def validate_plan(expected):
|
|
if not isinstance(expected, dict):
|
|
raise ValueError('Repair plan must be a JSON object')
|
|
if set(expected) != {'format_version','kind','checked_at','editions','content_fingerprints','target_aliases','status','scope','database'}:
|
|
raise ValueError('Unexpected repair plan fields')
|
|
if (type(expected['format_version']) is not int or expected['format_version'] != 1
|
|
or expected['kind'] != MARKER or expected['status'] not in ('ready','already_joint')):
|
|
raise ValueError('Unsupported repair plan')
|
|
if (not isinstance(expected['database'],dict) or not isinstance(expected['content_fingerprints'],dict)
|
|
or not isinstance(expected['editions'],list) or len(expected['editions']) != 2
|
|
or expected['target_aliases'] != sorted(MEMBERS)):
|
|
raise ValueError('Invalid repair plan structure')
|
|
for edition, year in zip(expected['editions'], YEARS):
|
|
if not isinstance(edition,dict) or edition.get('year') != year:
|
|
raise ValueError('Repair plan must name the two exact ordered years')
|
|
for key in ('id','year','from_venue_id','target_venue_id','submissions'):
|
|
if type(edition.get(key)) is not int or edition[key] <= 0:
|
|
raise ValueError('Repair plan IDs/counts must be positive integers')
|
|
|
|
|
|
def apply_plan(c, expected):
|
|
validate_plan(expected)
|
|
actual = inspect(c)
|
|
if actual['content_fingerprints'] != expected['content_fingerprints'] or actual['target_aliases'] != expected['target_aliases']:
|
|
raise ValueError('Stale repair plan: protected content or aliases changed')
|
|
if len(expected['editions']) != 2:
|
|
raise ValueError('Exactly two editions required')
|
|
for old, now in zip(expected['editions'], actual['editions']):
|
|
if set(old) != set(now):
|
|
raise ValueError('Unexpected edition fields')
|
|
for key in ('id','year','track','target_venue_id','submissions','evidence_url','proceedings_key'):
|
|
if old[key] != now[key]:
|
|
raise ValueError('Stale repair plan: edition differs')
|
|
if now['from_venue_id'] != old['from_venue_id'] and now['from_venue_id'] != now['target_venue_id']:
|
|
raise ValueError('Edition membership differs from plan')
|
|
if actual['status'] == 'already_joint':
|
|
return {**actual, 'status':'already_joint', 'updated_editions':0}
|
|
for edition in actual['editions']:
|
|
if edition['from_venue_id'] == edition['target_venue_id']:
|
|
continue
|
|
changed = c.execute('''UPDATE editions SET venue_id=? WHERE id=? AND venue_id=? AND year=? AND track=?''',
|
|
(edition['target_venue_id'],edition['id'],edition['from_venue_id'],edition['year'],edition['track']))
|
|
if changed.rowcount != 1:
|
|
raise ValueError('Concurrent edition change; entire repair must roll back')
|
|
after = inspect(c)
|
|
if after['status'] != 'already_joint' or after['content_fingerprints'] != actual['content_fingerprints']:
|
|
raise ValueError('Postcheck changed protected content; entire repair must roll back')
|
|
c.execute('INSERT INTO schema_migrations(name) VALUES(?) ON CONFLICT DO NOTHING', (MARKER,))
|
|
return {**after, 'status':'applied', 'updated_editions':sum(e['from_venue_id'] != e['target_venue_id'] for e in actual['editions'])}
|
|
|
|
|
|
def database_identity(c, db):
|
|
if getattr(c, 'postgres', False):
|
|
row = c.execute('SELECT current_database(),current_schema()').fetchone()
|
|
return {'engine':'postgresql','database':row[0],'schema':row[1]}
|
|
return {'engine':'sqlite','path':str(Path(db).resolve())}
|
|
|
|
|
|
def run(db, out, *, pg_service=None, pg_schema=None, plan=None, plan_sha256=None):
|
|
out = Path(out).resolve()
|
|
if not out.parent.is_dir():
|
|
raise ValueError('Report parent directory must already exist')
|
|
if out.exists():
|
|
raise ValueError('Preserve existing evidence report; choose a new output path')
|
|
expected = None
|
|
if plan:
|
|
raw = Path(plan).read_bytes()
|
|
if hashlib.sha256(raw).hexdigest() != plan_sha256:
|
|
raise ValueError('Repair plan hash differs before opening database')
|
|
expected = json.loads(raw)
|
|
validate_plan(expected)
|
|
elif plan_sha256:
|
|
raise ValueError('Plan hash requires an existing plan')
|
|
if db and out == Path(db).resolve():
|
|
raise ValueError('Report must not overwrite database')
|
|
writer = open_store(db, pg_service=pg_service, pg_schema=pg_schema) if expected is not None else None
|
|
c = writer.conn if writer else open_read_connection(db, pg_service=pg_service, pg_schema=pg_schema)
|
|
with closing(c):
|
|
try:
|
|
if expected is not None and not getattr(c, 'postgres', False):
|
|
c.commit(); c.execute('BEGIN IMMEDIATE')
|
|
identity = database_identity(c, db)
|
|
if expected is not None and expected['database'] != identity:
|
|
raise ValueError('Plan belongs to another database/schema')
|
|
report = apply_plan(c, expected) if expected is not None else inspect(c)
|
|
report['database'] = identity
|
|
if expected is not None:
|
|
report['plan_sha256'] = plan_sha256
|
|
if expected is not None:
|
|
c.commit()
|
|
except BaseException:
|
|
if expected is not None: c.rollback()
|
|
raise
|
|
out.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n')
|
|
print(json.dumps({'status':report['status'],'editions':len(report['editions']),
|
|
'updated_editions':report.get('updated_editions',0),'report':str(out)}))
|
|
return report
|
|
|
|
|
|
def main(argv=None, *, prog=None):
|
|
parser = argparse.ArgumentParser(prog=prog); database_arguments(parser)
|
|
parser.add_argument('--out', required=True)
|
|
parser.add_argument('--plan', help='Apply an already reviewed plan; omitted means read-only preflight')
|
|
parser.add_argument('--plan-sha256', help='Required exact plan SHA256 when applying')
|
|
args = parser.parse_args(argv)
|
|
run(args.db,args.out,pg_service=args.pg_service,pg_schema=args.pg_schema,plan=args.plan,plan_sha256=args.plan_sha256)
|