Files
paper-atlas/paper_atlas/ingestion/joint_editions.py
T
codex-bot 866aeaf068
Metadata integrity checks / metadata (push) Successful in 1m13s
Read PostgreSQL mapping rows by position during joint preflight
2026-11-08 19:10:10 +08:00

205 lines
12 KiB
Python

"""Evidence-bound repair of two existing IJCAI/ECAI edition memberships."""
import argparse
from contextlib import closing
import hashlib
import json
from pathlib import Path
from paper_atlas.db.backend import database_arguments, open_read_connection, open_store
from paper_atlas.db.submissions import timestamp
from paper_atlas.collectors.dblp import XML_SHA
YEARS = (2018, 2022)
TARGET = 'ECAI+IJCAI'
MEMBERS = {'ECAI', 'IJCAI'}
EVIDENCE = 'https://www.eurai.org/ecai/history'
MARKER = 'joint_ijcai_ecai_2018_2022_v1'
def _digest(c, sql, params=()):
sha = hashlib.sha256(); count = 0
for row in c.execute(sql, params):
sha.update((json.dumps([row[i] for i in range(len(row))], ensure_ascii=False,
separators=(',', ':')) + '\n').encode())
count += 1
if count > 100000:
raise ValueError('Joint repair evidence scope exceeded bounded row limit')
return {'rows': count, 'sha256': sha.hexdigest()}
def fingerprints(c, ids):
marks = ','.join('?' for _ in ids)
submissions = 'SELECT id FROM submissions WHERE edition_id IN (' + marks + ')'
papers = 'SELECT paper_id FROM submissions WHERE edition_id IN (' + marks + ')'
decisions = 'SELECT id FROM decisions WHERE submission_id IN (' + submissions + ')'
checks = {
'papers': ('id IN (' + papers + ')', 'id'),
'identifiers': ('paper_id IN (' + papers + ')', 'scheme,value'),
'source_records': ('paper_id IN (' + papers + ')', 'id'),
'paper_tags': ('paper_id IN (' + papers + ')', 'paper_id,tag_id,evidence_key'),
'submissions': ('edition_id IN (' + marks + ')', 'id'),
'decisions': ('submission_id IN (' + submissions + ')', 'id'),
'submission_sources': ('submission_id IN (' + submissions + ')', 'submission_id,source_record_id,source_submission_id'),
'submission_identifiers': ('edition_id IN (' + marks + ')', 'edition_id,scheme,value'),
'decision_sources': ('decision_id IN (' + decisions + ')', 'decision_id,source_record_id,source_decision_id'),
'edition_sources': ('edition_id IN (' + marks + ')', 'edition_id,source_id,list_url'),
}
return {table: _digest(c, 'SELECT * FROM ' + table + ' WHERE ' + where + ' ORDER BY ' + order, ids)
for table, (where, order) in checks.items()}
def inspect(c):
target = c.execute('SELECT id FROM venues WHERE acronym=?', (TARGET,)).fetchall()
if len(target) != 1:
raise ValueError('Exactly one existing canonical joint venue required')
target = target[0][0]
aliases = {r[0] for r in c.execute('SELECT alias FROM venue_aliases WHERE venue_id=?', (target,))}
if aliases != MEMBERS:
raise ValueError('Joint target must already have exactly ECAI and IJCAI aliases')
editions = []
for year in YEARS:
rows = c.execute('''SELECT e.id,e.venue_id,v.acronym FROM editions e JOIN venues v ON v.id=e.venue_id
WHERE v.acronym IN ('IJCAI','ECAI+IJCAI') AND e.year=? AND e.track='main' ''', (year,)).fetchall()
if len(rows) != 1:
raise ValueError('Expected one existing edition per year; missing or colliding editions require separate review')
edition, venue, acronym = (rows[0][i] for i in range(3))
count = c.execute('SELECT count(*) FROM submissions WHERE edition_id=?', (edition,)).fetchone()[0]
if not 0 < count <= 5000:
raise ValueError('Existing edition must have a nonempty bounded submission scope')
# Every existing submission must be supported by its own pinned DBLP
# volume record. No title/author matching, new attempts or text writes.
evidence = c.execute('''SELECT s.id,r.external_id,r.metadata_json FROM submissions s
JOIN submission_sources ss ON ss.submission_id=s.id
JOIN source_records r ON r.id=ss.source_record_id JOIN sources src ON src.id=r.source_id
WHERE s.edition_id=? AND src.name='dblp' ORDER BY s.id,r.id''', (edition,)).fetchall()
seen = set()
for row in evidence:
sid, external, metadata = (row[i] for i in range(3))
meta = json.loads(metadata)
bibliography = meta.get('bibliography') or {}
if (meta.get('snapshot_sha256') != XML_SHA
or meta.get('proceedings_key') != 'conf/ijcai/' + str(year)
or meta.get('bibliographic_year') != year
or bibliography.get('booktitle') not in (['IJCAI'], ['IJCAI/ECAI'])
or not external.startswith('conf/ijcai/')):
raise ValueError('Source evidence outside the exact verified IJCAI/ECAI volume')
seen.add(sid)
if len(seen) != count:
raise ValueError('Every submission must have matching original volume evidence')
editions.append({'id': edition, 'year': year, 'track': 'main', 'from_venue_id': venue,
'from_venue': acronym, 'target_venue_id': target, 'submissions': count,
'evidence_url': EVIDENCE, 'proceedings_key': 'conf/ijcai/' + str(year)})
ids = tuple(e['id'] for e in editions)
return {'format_version': 1, 'kind': MARKER, 'checked_at': timestamp(), 'editions': editions,
'content_fingerprints': fingerprints(c, ids), 'target_aliases': sorted(aliases),
'status': 'already_joint' if all(e['from_venue_id'] == target for e in editions) else 'ready',
'scope': 'Only these two existing edition venue references; no source text, paper, submission, decision, track, identifier or tag change.'}
def validate_plan(expected):
if not isinstance(expected, dict):
raise ValueError('Repair plan must be a JSON object')
if set(expected) != {'format_version','kind','checked_at','editions','content_fingerprints','target_aliases','status','scope','database'}:
raise ValueError('Unexpected repair plan fields')
if (type(expected['format_version']) is not int or expected['format_version'] != 1
or expected['kind'] != MARKER or expected['status'] not in ('ready','already_joint')):
raise ValueError('Unsupported repair plan')
if (not isinstance(expected['database'],dict) or not isinstance(expected['content_fingerprints'],dict)
or not isinstance(expected['editions'],list) or len(expected['editions']) != 2
or expected['target_aliases'] != sorted(MEMBERS)):
raise ValueError('Invalid repair plan structure')
for edition, year in zip(expected['editions'], YEARS):
if not isinstance(edition,dict) or edition.get('year') != year:
raise ValueError('Repair plan must name the two exact ordered years')
for key in ('id','year','from_venue_id','target_venue_id','submissions'):
if type(edition.get(key)) is not int or edition[key] <= 0:
raise ValueError('Repair plan IDs/counts must be positive integers')
def apply_plan(c, expected):
validate_plan(expected)
actual = inspect(c)
if actual['content_fingerprints'] != expected['content_fingerprints'] or actual['target_aliases'] != expected['target_aliases']:
raise ValueError('Stale repair plan: protected content or aliases changed')
if len(expected['editions']) != 2:
raise ValueError('Exactly two editions required')
for old, now in zip(expected['editions'], actual['editions']):
if set(old) != set(now):
raise ValueError('Unexpected edition fields')
for key in ('id','year','track','target_venue_id','submissions','evidence_url','proceedings_key'):
if old[key] != now[key]:
raise ValueError('Stale repair plan: edition differs')
if now['from_venue_id'] != old['from_venue_id'] and now['from_venue_id'] != now['target_venue_id']:
raise ValueError('Edition membership differs from plan')
if actual['status'] == 'already_joint':
return {**actual, 'status':'already_joint', 'updated_editions':0}
for edition in actual['editions']:
if edition['from_venue_id'] == edition['target_venue_id']:
continue
changed = c.execute('''UPDATE editions SET venue_id=? WHERE id=? AND venue_id=? AND year=? AND track=?''',
(edition['target_venue_id'],edition['id'],edition['from_venue_id'],edition['year'],edition['track']))
if changed.rowcount != 1:
raise ValueError('Concurrent edition change; entire repair must roll back')
after = inspect(c)
if after['status'] != 'already_joint' or after['content_fingerprints'] != actual['content_fingerprints']:
raise ValueError('Postcheck changed protected content; entire repair must roll back')
c.execute('INSERT INTO schema_migrations(name) VALUES(?) ON CONFLICT DO NOTHING', (MARKER,))
return {**after, 'status':'applied', 'updated_editions':sum(e['from_venue_id'] != e['target_venue_id'] for e in actual['editions'])}
def database_identity(c, db):
if getattr(c, 'postgres', False):
row = c.execute('SELECT current_database(),current_schema()').fetchone()
return {'engine':'postgresql','database':row[0],'schema':row[1]}
return {'engine':'sqlite','path':str(Path(db).resolve())}
def run(db, out, *, pg_service=None, pg_schema=None, plan=None, plan_sha256=None):
out = Path(out).resolve()
if not out.parent.is_dir():
raise ValueError('Report parent directory must already exist')
if out.exists():
raise ValueError('Preserve existing evidence report; choose a new output path')
expected = None
if plan:
raw = Path(plan).read_bytes()
if hashlib.sha256(raw).hexdigest() != plan_sha256:
raise ValueError('Repair plan hash differs before opening database')
expected = json.loads(raw)
validate_plan(expected)
elif plan_sha256:
raise ValueError('Plan hash requires an existing plan')
if db and out == Path(db).resolve():
raise ValueError('Report must not overwrite database')
writer = open_store(db, pg_service=pg_service, pg_schema=pg_schema) if expected is not None else None
c = writer.conn if writer else open_read_connection(db, pg_service=pg_service, pg_schema=pg_schema)
with closing(c):
try:
if expected is not None and not getattr(c, 'postgres', False):
c.commit(); c.execute('BEGIN IMMEDIATE')
identity = database_identity(c, db)
if expected is not None and expected['database'] != identity:
raise ValueError('Plan belongs to another database/schema')
report = apply_plan(c, expected) if expected is not None else inspect(c)
report['database'] = identity
if expected is not None:
report['plan_sha256'] = plan_sha256
if expected is not None:
c.commit()
except BaseException:
if expected is not None: c.rollback()
raise
out.write_text(json.dumps(report,ensure_ascii=False,indent=2)+'\n')
print(json.dumps({'status':report['status'],'editions':len(report['editions']),
'updated_editions':report.get('updated_editions',0),'report':str(out)}))
return report
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog); database_arguments(parser)
parser.add_argument('--out', required=True)
parser.add_argument('--plan', help='Apply an already reviewed plan; omitted means read-only preflight')
parser.add_argument('--plan-sha256', help='Required exact plan SHA256 when applying')
args = parser.parse_args(argv)
run(args.db,args.out,pg_service=args.pg_service,pg_schema=args.pg_schema,plan=args.plan,plan_sha256=args.plan_sha256)