"""Plan-bound publication scope revisions with complete before-images, never reimport.""" import argparse from collections import Counter from contextlib import closing import copy import hashlib import json from pathlib import Path import re from paper_atlas.db import source_revisions as versions from paper_atlas.db.backend import database_arguments, open_read_connection, open_store from paper_atlas.db.edition_scopes import KEY from paper_atlas.ingestion.edition_tracks import curation from paper_atlas.ingestion.event_identity import database_identity, digest, values MAX_ROWS = 10000 def validate(inventory): required = {'format', 'source', 'venue', 'year', 'event_key', 'list_url', 'previous_rows', 'rows', 'evidence'} if not isinstance(inventory, dict) or set(inventory) != required or inventory['format'] != 1: raise ValueError('Unsupported source revision inventory') if (type(inventory['year']) is not int or inventory['year'] < 2000 or not all(isinstance(inventory[k], str) and inventory[k] for k in ('source', 'venue', 'list_url')) or not isinstance(inventory['event_key'], str)): raise ValueError('Explicit publication source scope required') if not inventory['list_url'].startswith('https://'): raise ValueError('HTTPS source listing required') evidence = inventory['evidence'] if not isinstance(evidence, list) or not 1 <= len(evidence) <= 100: raise ValueError('Bounded source evidence required') for item in evidence: if (not isinstance(item, dict) or set(item) != {'url', 'sha256'} or not isinstance(item['url'], str) or not item['url'].startswith('https://') or not isinstance(item['sha256'], str) or not re.fullmatch('[0-9a-f]{64}', item['sha256'])): raise ValueError('Source evidence URL/hash required') snapshots = [] for key in ('previous_rows', 'rows'): items = inventory[key] if not isinstance(items, list) or not 1 <= len(items) <= MAX_ROWS: raise ValueError('Complete bounded source snapshots required') records = {} for row in items: if not isinstance(row, dict) or any(row.get(k) != inventory[k] for k in ('source', 'venue', 'year')): raise ValueError('Source snapshot scope mismatch') ext = row.get('external_id') if (not isinstance(ext, str) or ext in records or row.get('status') != 'accepted' or not re.fullmatch(r'10\.\d{4,9}/\S+', ext) or row.get('identifiers', {}).get('doi') != ext or not isinstance(row.get('track'), str) or not row['track'] or row.get('event_key', '') != inventory['event_key'] or not isinstance(row.get('title'), str) or not row['title'] or not isinstance(row.get('authors'), list) or not isinstance(row.get('metadata'), dict)): raise ValueError('Unique DOI-bearing accepted publication records required') records[ext] = row snapshots.append(records) previous, current = snapshots if set(previous) != set(current): raise ValueError('Source membership changed; this operation cannot add or remove publications') for ext, row in current.items(): before = previous[ext] stable = (set(before) | set(row)) - {'abstract', 'track', 'tags', 'metadata'} if any(before.get(k) != row.get(k) for k in stable): raise ValueError('Stable publication identity or decision changed') return previous, current def bounded(c, sql, params): result = values(c.execute(sql + ' LIMIT 100001', params)) if len(result) > 100000: raise ValueError('Source revision protection exceeds bounded scope') return result def capture(c, inventory): previous, current = validate(inventory) source = c.execute('SELECT id FROM sources WHERE name=?', (inventory['source'],)).fetchone() venue = c.execute('SELECT id FROM venues WHERE acronym=?', (inventory['venue'],)).fetchone() if not source or not venue: raise ValueError('Existing source and venue required') sid, vid = source[0], venue[0] editions = bounded(c, 'SELECT * FROM editions WHERE venue_id=? AND year=? AND event_key=? ORDER BY id', (vid, inventory['year'], inventory['event_key'])) by_track = {r[3]: r[0] for r in editions} # The reviewed scope includes all its existing target attempts, even when # they originate from another source. No empty-target shortcut is used. scope_subs = 'SELECT s.id FROM submissions s JOIN editions e ON e.id=s.edition_id WHERE e.venue_id=? AND e.year=? AND e.event_key=?' scope_params = (vid, inventory['year'], inventory['event_key']) scoped = bounded(c, 'SELECT * FROM submissions WHERE id IN (' + scope_subs + ') ORDER BY id', scope_params) own_records = bounded(c, '''SELECT DISTINCT r.* FROM source_records r JOIN submission_sources ss ON ss.source_record_id=r.id WHERE r.source_id=? AND ss.submission_id IN (''' + scope_subs + ') ORDER BY r.id', (sid, *scope_params)) if not own_records: raise ValueError('Existing source observations required') if any(r[2] not in current for r in own_records): raise ValueError('Snapshot omits an existing source observation') pids = sorted({r[1] for r in scoped}) if not pids or len(pids) > MAX_ROWS: raise ValueError('Bounded nonempty protected paper scope required') marks = ','.join('?' for _ in pids) papers = 'SELECT id FROM papers WHERE id IN (' + marks + ')' subs = 'SELECT id FROM submissions WHERE paper_id IN (' + papers + ')' decisions = 'SELECT id FROM decisions WHERE submission_id IN (' + subs + ')' queries = { 'papers': 'SELECT * FROM papers WHERE id IN (' + papers + ') ORDER BY id', 'identifiers': 'SELECT * FROM identifiers WHERE paper_id IN (' + papers + ') ORDER BY scheme,value', 'source_records': 'SELECT * FROM source_records WHERE paper_id IN (' + papers + ') ORDER BY id', 'submissions': 'SELECT * FROM submissions WHERE paper_id IN (' + papers + ') ORDER BY id', 'decisions': 'SELECT * FROM decisions WHERE submission_id IN (' + subs + ') ORDER BY id', 'submission_sources': 'SELECT * FROM submission_sources WHERE submission_id IN (' + subs + ') ORDER BY submission_id,source_record_id,source_submission_id', 'submission_identifiers': 'SELECT * FROM submission_identifiers WHERE submission_id IN (' + subs + ') ORDER BY edition_id,scheme,value', 'decision_sources': 'SELECT * FROM decision_sources WHERE decision_id IN (' + decisions + ') ORDER BY decision_id,source_record_id,source_decision_id', 'paper_tags': 'SELECT * FROM paper_tags WHERE paper_id IN (' + papers + ') ORDER BY paper_id,tag_id,evidence_key', } from paper_atlas.curation.annotations import available if available(c): queries['annotation_events'] = 'SELECT * FROM annotation_events WHERE paper_id IN (' + papers + ') ORDER BY id' packet = {k: bounded(c, q, pids) for k, q in queries.items()} coverage = bounded(c, '''SELECT es.* FROM edition_sources es JOIN editions e ON e.id=es.edition_id WHERE es.source_id=? AND e.venue_id=? AND e.year=? AND e.event_key=? ORDER BY es.edition_id,es.list_url''', (sid, *scope_params)) if not coverage or any(r[2] != inventory['list_url'] for r in coverage): raise ValueError('Unambiguous source listing required') if len({r[0] for r in coverage}) != len(coverage): raise ValueError('One source coverage entry per edition required') heads = [[r[i] for i in range(len(r))] for r in versions.heads(c, sid)] own_heads = [r for r in heads if r[1:] == [vid, inventory['year'], inventory['event_key'], inventory['list_url']]] if len(own_heads) > 1: raise ValueError('Forked source revision history') parent = own_heads[0][0] if own_heads else None return {'source_id': sid, 'venue_id': vid, 'editions': editions, 'coverage': coverage, 'packet': packet, 'own_records': own_records, 'revision_heads': heads, 'parent_sha256': parent, 'by_track': by_track, 'venue_names': sorted({inventory['venue']} | {r[0] for r in c.execute('SELECT alias FROM venue_aliases WHERE venue_id=?', (vid,))})} def changes(c, inventory, before): previous, current = validate(inventory) packet = before['packet']; result = []; sid = before['source_id'] editions = {r[0]: r for r in before['editions']} for raw in before['own_records']: ext = raw[2]; old = previous[ext]; new = current[ext] metadata = json.loads(raw[8]) candidates = [old, new] if not any(raw[4] == r['url'] and raw[5] == r['title'] and raw[6] == r.get('abstract') and json.loads(raw[7]) == r['authors'] and versions.subset(r['metadata'], metadata) for r in candidates): raise ValueError('Current raw source differs from both reviewed snapshots') if metadata.get('record_identifiers', {}).get('doi') != ext: raise ValueError('Current stable DOI differs from source evidence') attempts = [s for s in packet['submissions'] if s[3] == raw[0] and s[2] in editions] if len(attempts) != 1: raise ValueError('Exactly one publication attempt per source record required') sub = attempts[0]; origin = editions[sub[2]] if origin[3] not in {old['track'], new['track']} or sub[4] != ext or sub[1] != raw[3]: raise ValueError('Current source attempt identity/scope differs') if [x for x in packet['submission_sources'] if x[0] == sub[0]] != [[sub[0], raw[0], ext]]: raise ValueError('Shared-source attempt requires separate reconciliation') decisions = [d for d in packet['decisions'] if d[1] == sub[0]] if not decisions or any(d[2] != 'accepted' for d in decisions): raise ValueError('Only accepted publication observations can be revised') target = before['by_track'].get(new['track']) if target is None: raise ValueError('Reviewed target edition must already exist') identities = [x for x in packet['submission_identifiers'] if x[3] == sub[0]] if not identities or any(x[0] != sub[2] for x in identities): raise ValueError('Inconsistent scoped submission identifiers') for _, scheme, value, _ in identities: conflict = c.execute('SELECT submission_id FROM submission_identifiers WHERE edition_id=? AND scheme=? AND value=?', (target, scheme, value)).fetchone() if conflict and conflict[0] != sub[0]: raise ValueError('Target submission identifier collision') replacement = copy.deepcopy(metadata); replacement.update(new['metadata']) if sub[2] != target and KEY in metadata: raise ValueError('An already curated attempt needs explicit scope-chain reconciliation') if sub[2] != target or raw[6] != new.get('abstract') or replacement != metadata: result.append({'source_record_id': raw[0], 'paper_id': raw[3], 'submission_id': sub[0], 'external_id': ext, 'from_edition_id': sub[2], 'target_edition_id': target, 'from_track': origin[3], 'track': new['track'], 'new_abstract': new.get('abstract'), 'metadata': replacement}) old_expected = Counter(r['track'] for r in previous.values()); new_expected = Counter(r['track'] for r in current.values()) observed = Counter() own_ids = {r[0] for r in before['own_records']} for s in packet['submissions']: if s[3] in own_ids and s[2] in editions: observed[s[2]] += 1 coverage = [] for raw in before['coverage']: track = editions[raw[0]][3] if raw[3] != 'partial' or raw[8] is not None or raw[5] != observed[raw[0]]: raise ValueError('Coverage parent is not the verified partial source observation count') if raw[4] not in {old_expected[track], new_expected[track]}: raise ValueError('Coverage expected count differs from both source versions') count = raw[5] + sum((r['target_edition_id'] == raw[0]) - (r['from_edition_id'] == raw[0]) for r in result) if count < 0 or count > new_expected[track]: raise ValueError('Corrected coverage exceeds current source membership') coverage.append({'before': raw, 'expected_count': new_expected[track], 'imported_count': count}) covered = {r['before'][0] for r in coverage} if any(r['target_edition_id'] not in covered for r in result): raise ValueError('Target source coverage must already exist') return result, coverage def proposal(c, inventory, identity): before = capture(c, inventory) if before['parent_sha256']: saved = json.loads(c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (before['parent_sha256'],)).fetchone()[0]) if sorted(saved, key=lambda r: r['external_id']) != sorted(inventory['previous_rows'], key=lambda r: r['external_id']): raise ValueError('Previous snapshot differs from active source version') records, coverage = changes(c, inventory, before) return {'format': 1, 'database': identity, 'inventory': inventory, 'before': before, 'changes': records, 'coverage': coverage} def verify(c, plan, signature): after = capture(c, plan['inventory']); expected = copy.deepcopy(plan['before']['packet']) for change in plan['changes']: for row in expected['submissions']: if row[0] == change['submission_id']: row[2] = change['target_edition_id'] for row in expected['submission_identifiers']: if row[3] == change['submission_id']: row[0] = change['target_edition_id'] for row in expected['source_records']: if row[0] == change['source_record_id']: row[6] = change['new_abstract'] row[8] = encoded_metadata(plan, change, signature) expected['submission_identifiers'].sort(key=lambda r: (r[0], r[1], r[2])) if after['packet'] != expected: raise ValueError('Protected profiles/history/tags/identifiers differ; roll back') if after['editions'] != plan['before']['editions']: raise ValueError('Edition identities changed') expected_heads = [r for r in plan['before']['revision_heads'] if r[0] != plan['before']['parent_sha256']] inventory = plan['inventory'] expected_heads.append([signature, plan['before']['venue_id'], inventory['year'], inventory['event_key'], inventory['list_url']]) if after['revision_heads'] != sorted(expected_heads): raise ValueError('Active source version differs') expected_coverage = [] for item in plan['coverage']: row = item['before'][:]; row[4] = item['expected_count']; row[5] = item['imported_count'] expected_coverage.append(row) if after['coverage'] != expected_coverage: raise ValueError('Reconciled coverage differs; roll back') stored = c.execute('SELECT records_json,plan_json,parent_sha256 FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone() if (not stored or json.loads(stored[0]) != plan['inventory']['rows'] or json.loads(stored[1]) != plan or stored[2] != plan['before']['parent_sha256']): raise ValueError('Versioned source or complete before-images changed') return {'records_changed': len(plan['changes']), 'events_moved': sum(r['from_edition_id'] != r['target_edition_id'] for r in plan['changes']), 'protected_papers': len(expected['papers']), 'protected_decisions': len(expected['decisions'])} def encoded_metadata(plan, change, signature): metadata = copy.deepcopy(change['metadata']) if change['from_edition_id'] != change['target_edition_id']: inventory = plan['inventory'] evidence = inventory['evidence'][0] row = {**change, 'source': inventory['source'], 'doi': change['external_id'], 'venue_id': plan['before']['venue_id'], 'year': inventory['year'], 'event_key': inventory['event_key'], 'venue_names': plan['before']['venue_names'], 'section': change['track'], 'evidence_url': evidence['url'], 'evidence_sha256': evidence['sha256']} metadata[KEY] = [curation(row, change['target_edition_id'], signature)] return json.dumps(metadata, ensure_ascii=False) def apply_plan(c, plan): if plan.get('format') != 1: raise ValueError('Unsupported source revision plan') if not getattr(c, 'postgres', False) and not c.in_transaction: c.execute('BEGIN IMMEDIATE') signature = digest(plan) if versions.available(c) and c.execute('SELECT 1 FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone(): return {'status': 'already_applied', 'version_sha256': signature, **verify(c, plan, signature)} if getattr(c, 'postgres', False): c.execute('LOCK TABLE sources,papers,identifiers,source_records,venues,venue_aliases,editions,edition_sources,submissions,decisions,submission_sources,submission_identifiers,decision_sources,tags,paper_tags IN SHARE ROW EXCLUSIVE MODE') if proposal(c, plan['inventory'], plan['database']) != plan: raise ValueError('Stale source revision plan; create and review a fresh plan') versions.initialize(c) for row in plan['changes']: if row['from_edition_id'] != row['target_edition_id']: c.execute('UPDATE submission_identifiers SET edition_id=? WHERE submission_id=?', (row['target_edition_id'], row['submission_id'])) c.execute('UPDATE submissions SET edition_id=? WHERE id=? AND edition_id=?', (row['target_edition_id'], row['submission_id'], row['from_edition_id'])) c.execute('UPDATE source_records SET abstract=?,metadata_json=? WHERE id=?', (row['new_abstract'], encoded_metadata(plan, row, signature), row['source_record_id'])) for row in plan['coverage']: before = row['before'] c.execute('UPDATE edition_sources SET expected_count=?,imported_count=? WHERE edition_id=? AND source_id=? AND list_url=?', (row['expected_count'], row['imported_count'], before[0], before[1], before[2])) inventory = plan['inventory'] c.execute('''INSERT INTO source_scope_revisions(version_sha256,source_id,venue_id,year,event_key,list_url,parent_sha256,records_json,plan_json) VALUES(?,?,?,?,?,?,?,?,?)''', (signature, plan['before']['source_id'], plan['before']['venue_id'], inventory['year'], inventory['event_key'], inventory['list_url'], plan['before']['parent_sha256'], json.dumps(inventory['rows'], ensure_ascii=False), json.dumps(plan, ensure_ascii=False))) return {'status': 'applied', 'version_sha256': signature, **verify(c, plan, signature)} def run(db, out, *, pg_service=None, pg_schema=None, inventory=None, plan=None, plan_sha256=None): out = Path(out) if out.exists() or not out.parent.is_dir(): raise ValueError('New output in existing directory required') if bool(inventory) == bool(plan) or bool(plan) != bool(plan_sha256): raise ValueError('Choose inventory preflight or plan plus exact file hash') if inventory: with closing(open_read_connection(db, pg_service=pg_service, pg_schema=pg_schema)) as c: result = proposal(c, json.loads(Path(inventory).read_bytes()), database_identity(c, db)) else: raw = Path(plan).read_bytes() if hashlib.sha256(raw).hexdigest() != plan_sha256: raise ValueError('Plan file hash differs') proposed = json.loads(raw) with closing(open_store(db, pg_service=pg_service, pg_schema=pg_schema).conn) as c: try: if proposed['database'] != database_identity(c, db): raise ValueError('Plan belongs to another database') if not getattr(c, 'postgres', False): c.commit(); c.execute('BEGIN IMMEDIATE') result = apply_plan(c, proposed); c.commit() except BaseException: c.rollback(); raise out.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n') return result def main(argv=None, *, prog=None): parser = argparse.ArgumentParser(prog=prog); database_arguments(parser) parser.add_argument('--inventory'); parser.add_argument('--plan'); parser.add_argument('--plan-sha256'); parser.add_argument('--out', required=True) args = parser.parse_args(argv) report = run(args.db, args.out, pg_service=args.pg_service, pg_schema=args.pg_schema, inventory=args.inventory, plan=args.plan, plan_sha256=args.plan_sha256) print(json.dumps({'status': report.get('status', 'planned'), 'out': args.out}), flush=True)