Files
paper-atlas/paper_atlas/ingestion/source_revisions.py
T
codex-bot 2000d340fa
Metadata integrity checks / metadata (push) Successful in 7m16s
Add snapshot-bound publication source scope revisions
2026-11-18 06:59:21 +08:00

328 lines
21 KiB
Python

"""Plan-bound publication scope revisions with complete before-images, never reimport."""
import argparse
from collections import Counter
from contextlib import closing
import copy
import hashlib
import json
from pathlib import Path
import re
from paper_atlas.db import source_revisions as versions
from paper_atlas.db.backend import database_arguments, open_read_connection, open_store
from paper_atlas.db.edition_scopes import KEY
from paper_atlas.ingestion.edition_tracks import curation
from paper_atlas.ingestion.event_identity import database_identity, digest, values
MAX_ROWS = 10000
def validate(inventory):
required = {'format', 'source', 'venue', 'year', 'event_key', 'list_url',
'previous_rows', 'rows', 'evidence'}
if not isinstance(inventory, dict) or set(inventory) != required or inventory['format'] != 1:
raise ValueError('Unsupported source revision inventory')
if (type(inventory['year']) is not int or inventory['year'] < 2000
or not all(isinstance(inventory[k], str) and inventory[k] for k in ('source', 'venue', 'list_url'))
or not isinstance(inventory['event_key'], str)):
raise ValueError('Explicit publication source scope required')
if not inventory['list_url'].startswith('https://'):
raise ValueError('HTTPS source listing required')
evidence = inventory['evidence']
if not isinstance(evidence, list) or not 1 <= len(evidence) <= 100:
raise ValueError('Bounded source evidence required')
for item in evidence:
if (not isinstance(item, dict) or set(item) != {'url', 'sha256'}
or not isinstance(item['url'], str) or not item['url'].startswith('https://')
or not isinstance(item['sha256'], str) or not re.fullmatch('[0-9a-f]{64}', item['sha256'])):
raise ValueError('Source evidence URL/hash required')
snapshots = []
for key in ('previous_rows', 'rows'):
items = inventory[key]
if not isinstance(items, list) or not 1 <= len(items) <= MAX_ROWS:
raise ValueError('Complete bounded source snapshots required')
records = {}
for row in items:
if not isinstance(row, dict) or any(row.get(k) != inventory[k] for k in ('source', 'venue', 'year')):
raise ValueError('Source snapshot scope mismatch')
ext = row.get('external_id')
if (not isinstance(ext, str) or ext in records or row.get('status') != 'accepted'
or not re.fullmatch(r'10\.\d{4,9}/\S+', ext)
or row.get('identifiers', {}).get('doi') != ext
or not isinstance(row.get('track'), str) or not row['track']
or row.get('event_key', '') != inventory['event_key']
or not isinstance(row.get('title'), str) or not row['title']
or not isinstance(row.get('authors'), list)
or not isinstance(row.get('metadata'), dict)):
raise ValueError('Unique DOI-bearing accepted publication records required')
records[ext] = row
snapshots.append(records)
previous, current = snapshots
if set(previous) != set(current):
raise ValueError('Source membership changed; this operation cannot add or remove publications')
for ext, row in current.items():
before = previous[ext]
stable = (set(before) | set(row)) - {'abstract', 'track', 'tags', 'metadata'}
if any(before.get(k) != row.get(k) for k in stable):
raise ValueError('Stable publication identity or decision changed')
return previous, current
def bounded(c, sql, params):
result = values(c.execute(sql + ' LIMIT 100001', params))
if len(result) > 100000:
raise ValueError('Source revision protection exceeds bounded scope')
return result
def capture(c, inventory):
previous, current = validate(inventory)
source = c.execute('SELECT id FROM sources WHERE name=?', (inventory['source'],)).fetchone()
venue = c.execute('SELECT id FROM venues WHERE acronym=?', (inventory['venue'],)).fetchone()
if not source or not venue:
raise ValueError('Existing source and venue required')
sid, vid = source[0], venue[0]
editions = bounded(c, 'SELECT * FROM editions WHERE venue_id=? AND year=? AND event_key=? ORDER BY id',
(vid, inventory['year'], inventory['event_key']))
by_track = {r[3]: r[0] for r in editions}
# The reviewed scope includes all its existing target attempts, even when
# they originate from another source. No empty-target shortcut is used.
scope_subs = 'SELECT s.id FROM submissions s JOIN editions e ON e.id=s.edition_id WHERE e.venue_id=? AND e.year=? AND e.event_key=?'
scope_params = (vid, inventory['year'], inventory['event_key'])
scoped = bounded(c, 'SELECT * FROM submissions WHERE id IN (' + scope_subs + ') ORDER BY id', scope_params)
own_records = bounded(c, '''SELECT DISTINCT r.* FROM source_records r JOIN submission_sources ss ON ss.source_record_id=r.id
WHERE r.source_id=? AND ss.submission_id IN (''' + scope_subs + ') ORDER BY r.id', (sid, *scope_params))
if not own_records:
raise ValueError('Existing source observations required')
if any(r[2] not in current for r in own_records):
raise ValueError('Snapshot omits an existing source observation')
pids = sorted({r[1] for r in scoped})
if not pids or len(pids) > MAX_ROWS:
raise ValueError('Bounded nonempty protected paper scope required')
marks = ','.join('?' for _ in pids)
papers = 'SELECT id FROM papers WHERE id IN (' + marks + ')'
subs = 'SELECT id FROM submissions WHERE paper_id IN (' + papers + ')'
decisions = 'SELECT id FROM decisions WHERE submission_id IN (' + subs + ')'
queries = {
'papers': 'SELECT * FROM papers WHERE id IN (' + papers + ') ORDER BY id',
'identifiers': 'SELECT * FROM identifiers WHERE paper_id IN (' + papers + ') ORDER BY scheme,value',
'source_records': 'SELECT * FROM source_records WHERE paper_id IN (' + papers + ') ORDER BY id',
'submissions': 'SELECT * FROM submissions WHERE paper_id IN (' + papers + ') ORDER BY id',
'decisions': 'SELECT * FROM decisions WHERE submission_id IN (' + subs + ') ORDER BY id',
'submission_sources': 'SELECT * FROM submission_sources WHERE submission_id IN (' + subs + ') ORDER BY submission_id,source_record_id,source_submission_id',
'submission_identifiers': 'SELECT * FROM submission_identifiers WHERE submission_id IN (' + subs + ') ORDER BY edition_id,scheme,value',
'decision_sources': 'SELECT * FROM decision_sources WHERE decision_id IN (' + decisions + ') ORDER BY decision_id,source_record_id,source_decision_id',
'paper_tags': 'SELECT * FROM paper_tags WHERE paper_id IN (' + papers + ') ORDER BY paper_id,tag_id,evidence_key',
}
from paper_atlas.curation.annotations import available
if available(c):
queries['annotation_events'] = 'SELECT * FROM annotation_events WHERE paper_id IN (' + papers + ') ORDER BY id'
packet = {k: bounded(c, q, pids) for k, q in queries.items()}
coverage = bounded(c, '''SELECT es.* FROM edition_sources es JOIN editions e ON e.id=es.edition_id
WHERE es.source_id=? AND e.venue_id=? AND e.year=? AND e.event_key=? ORDER BY es.edition_id,es.list_url''', (sid, *scope_params))
if not coverage or any(r[2] != inventory['list_url'] for r in coverage):
raise ValueError('Unambiguous source listing required')
if len({r[0] for r in coverage}) != len(coverage):
raise ValueError('One source coverage entry per edition required')
heads = [[r[i] for i in range(len(r))] for r in versions.heads(c, sid)]
own_heads = [r for r in heads if r[1:] == [vid, inventory['year'], inventory['event_key'], inventory['list_url']]]
if len(own_heads) > 1:
raise ValueError('Forked source revision history')
parent = own_heads[0][0] if own_heads else None
return {'source_id': sid, 'venue_id': vid, 'editions': editions, 'coverage': coverage,
'packet': packet, 'own_records': own_records, 'revision_heads': heads,
'parent_sha256': parent, 'by_track': by_track,
'venue_names': sorted({inventory['venue']} | {r[0] for r in c.execute('SELECT alias FROM venue_aliases WHERE venue_id=?', (vid,))})}
def changes(c, inventory, before):
previous, current = validate(inventory)
packet = before['packet']; result = []; sid = before['source_id']
editions = {r[0]: r for r in before['editions']}
for raw in before['own_records']:
ext = raw[2]; old = previous[ext]; new = current[ext]
metadata = json.loads(raw[8])
candidates = [old, new]
if not any(raw[4] == r['url'] and raw[5] == r['title'] and raw[6] == r.get('abstract')
and json.loads(raw[7]) == r['authors'] and versions.subset(r['metadata'], metadata) for r in candidates):
raise ValueError('Current raw source differs from both reviewed snapshots')
if metadata.get('record_identifiers', {}).get('doi') != ext:
raise ValueError('Current stable DOI differs from source evidence')
attempts = [s for s in packet['submissions'] if s[3] == raw[0] and s[2] in editions]
if len(attempts) != 1:
raise ValueError('Exactly one publication attempt per source record required')
sub = attempts[0]; origin = editions[sub[2]]
if origin[3] not in {old['track'], new['track']} or sub[4] != ext or sub[1] != raw[3]:
raise ValueError('Current source attempt identity/scope differs')
if [x for x in packet['submission_sources'] if x[0] == sub[0]] != [[sub[0], raw[0], ext]]:
raise ValueError('Shared-source attempt requires separate reconciliation')
decisions = [d for d in packet['decisions'] if d[1] == sub[0]]
if not decisions or any(d[2] != 'accepted' for d in decisions):
raise ValueError('Only accepted publication observations can be revised')
target = before['by_track'].get(new['track'])
if target is None:
raise ValueError('Reviewed target edition must already exist')
identities = [x for x in packet['submission_identifiers'] if x[3] == sub[0]]
if not identities or any(x[0] != sub[2] for x in identities):
raise ValueError('Inconsistent scoped submission identifiers')
for _, scheme, value, _ in identities:
conflict = c.execute('SELECT submission_id FROM submission_identifiers WHERE edition_id=? AND scheme=? AND value=?', (target, scheme, value)).fetchone()
if conflict and conflict[0] != sub[0]:
raise ValueError('Target submission identifier collision')
replacement = copy.deepcopy(metadata); replacement.update(new['metadata'])
if sub[2] != target and KEY in metadata:
raise ValueError('An already curated attempt needs explicit scope-chain reconciliation')
if sub[2] != target or raw[6] != new.get('abstract') or replacement != metadata:
result.append({'source_record_id': raw[0], 'paper_id': raw[3], 'submission_id': sub[0],
'external_id': ext, 'from_edition_id': sub[2], 'target_edition_id': target,
'from_track': origin[3], 'track': new['track'],
'new_abstract': new.get('abstract'), 'metadata': replacement})
old_expected = Counter(r['track'] for r in previous.values()); new_expected = Counter(r['track'] for r in current.values())
observed = Counter()
own_ids = {r[0] for r in before['own_records']}
for s in packet['submissions']:
if s[3] in own_ids and s[2] in editions:
observed[s[2]] += 1
coverage = []
for raw in before['coverage']:
track = editions[raw[0]][3]
if raw[3] != 'partial' or raw[8] is not None or raw[5] != observed[raw[0]]:
raise ValueError('Coverage parent is not the verified partial source observation count')
if raw[4] not in {old_expected[track], new_expected[track]}:
raise ValueError('Coverage expected count differs from both source versions')
count = raw[5] + sum((r['target_edition_id'] == raw[0]) - (r['from_edition_id'] == raw[0]) for r in result)
if count < 0 or count > new_expected[track]:
raise ValueError('Corrected coverage exceeds current source membership')
coverage.append({'before': raw, 'expected_count': new_expected[track], 'imported_count': count})
covered = {r['before'][0] for r in coverage}
if any(r['target_edition_id'] not in covered for r in result):
raise ValueError('Target source coverage must already exist')
return result, coverage
def proposal(c, inventory, identity):
before = capture(c, inventory)
if before['parent_sha256']:
saved = json.loads(c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (before['parent_sha256'],)).fetchone()[0])
if sorted(saved, key=lambda r: r['external_id']) != sorted(inventory['previous_rows'], key=lambda r: r['external_id']):
raise ValueError('Previous snapshot differs from active source version')
records, coverage = changes(c, inventory, before)
return {'format': 1, 'database': identity, 'inventory': inventory, 'before': before,
'changes': records, 'coverage': coverage}
def verify(c, plan, signature):
after = capture(c, plan['inventory']); expected = copy.deepcopy(plan['before']['packet'])
for change in plan['changes']:
for row in expected['submissions']:
if row[0] == change['submission_id']:
row[2] = change['target_edition_id']
for row in expected['submission_identifiers']:
if row[3] == change['submission_id']:
row[0] = change['target_edition_id']
for row in expected['source_records']:
if row[0] == change['source_record_id']:
row[6] = change['new_abstract']
row[8] = encoded_metadata(plan, change, signature)
expected['submission_identifiers'].sort(key=lambda r: (r[0], r[1], r[2]))
if after['packet'] != expected:
raise ValueError('Protected profiles/history/tags/identifiers differ; roll back')
if after['editions'] != plan['before']['editions']:
raise ValueError('Edition identities changed')
expected_heads = [r for r in plan['before']['revision_heads'] if r[0] != plan['before']['parent_sha256']]
inventory = plan['inventory']
expected_heads.append([signature, plan['before']['venue_id'], inventory['year'], inventory['event_key'], inventory['list_url']])
if after['revision_heads'] != sorted(expected_heads):
raise ValueError('Active source version differs')
expected_coverage = []
for item in plan['coverage']:
row = item['before'][:]; row[4] = item['expected_count']; row[5] = item['imported_count']
expected_coverage.append(row)
if after['coverage'] != expected_coverage:
raise ValueError('Reconciled coverage differs; roll back')
stored = c.execute('SELECT records_json,plan_json,parent_sha256 FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone()
if (not stored or json.loads(stored[0]) != plan['inventory']['rows']
or json.loads(stored[1]) != plan or stored[2] != plan['before']['parent_sha256']):
raise ValueError('Versioned source or complete before-images changed')
return {'records_changed': len(plan['changes']), 'events_moved': sum(r['from_edition_id'] != r['target_edition_id'] for r in plan['changes']),
'protected_papers': len(expected['papers']), 'protected_decisions': len(expected['decisions'])}
def encoded_metadata(plan, change, signature):
metadata = copy.deepcopy(change['metadata'])
if change['from_edition_id'] != change['target_edition_id']:
inventory = plan['inventory']
evidence = inventory['evidence'][0]
row = {**change, 'source': inventory['source'], 'doi': change['external_id'],
'venue_id': plan['before']['venue_id'], 'year': inventory['year'],
'event_key': inventory['event_key'], 'venue_names': plan['before']['venue_names'],
'section': change['track'], 'evidence_url': evidence['url'], 'evidence_sha256': evidence['sha256']}
metadata[KEY] = [curation(row, change['target_edition_id'], signature)]
return json.dumps(metadata, ensure_ascii=False)
def apply_plan(c, plan):
if plan.get('format') != 1:
raise ValueError('Unsupported source revision plan')
if not getattr(c, 'postgres', False) and not c.in_transaction:
c.execute('BEGIN IMMEDIATE')
signature = digest(plan)
if versions.available(c) and c.execute('SELECT 1 FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone():
return {'status': 'already_applied', 'version_sha256': signature, **verify(c, plan, signature)}
if getattr(c, 'postgres', False):
c.execute('LOCK TABLE sources,papers,identifiers,source_records,venues,venue_aliases,editions,edition_sources,submissions,decisions,submission_sources,submission_identifiers,decision_sources,tags,paper_tags IN SHARE ROW EXCLUSIVE MODE')
if proposal(c, plan['inventory'], plan['database']) != plan:
raise ValueError('Stale source revision plan; create and review a fresh plan')
versions.initialize(c)
for row in plan['changes']:
if row['from_edition_id'] != row['target_edition_id']:
c.execute('UPDATE submission_identifiers SET edition_id=? WHERE submission_id=?', (row['target_edition_id'], row['submission_id']))
c.execute('UPDATE submissions SET edition_id=? WHERE id=? AND edition_id=?', (row['target_edition_id'], row['submission_id'], row['from_edition_id']))
c.execute('UPDATE source_records SET abstract=?,metadata_json=? WHERE id=?',
(row['new_abstract'], encoded_metadata(plan, row, signature), row['source_record_id']))
for row in plan['coverage']:
before = row['before']
c.execute('UPDATE edition_sources SET expected_count=?,imported_count=? WHERE edition_id=? AND source_id=? AND list_url=?',
(row['expected_count'], row['imported_count'], before[0], before[1], before[2]))
inventory = plan['inventory']
c.execute('''INSERT INTO source_scope_revisions(version_sha256,source_id,venue_id,year,event_key,list_url,parent_sha256,records_json,plan_json)
VALUES(?,?,?,?,?,?,?,?,?)''', (signature, plan['before']['source_id'], plan['before']['venue_id'], inventory['year'], inventory['event_key'],
inventory['list_url'], plan['before']['parent_sha256'], json.dumps(inventory['rows'], ensure_ascii=False), json.dumps(plan, ensure_ascii=False)))
return {'status': 'applied', 'version_sha256': signature, **verify(c, plan, signature)}
def run(db, out, *, pg_service=None, pg_schema=None, inventory=None, plan=None, plan_sha256=None):
out = Path(out)
if out.exists() or not out.parent.is_dir():
raise ValueError('New output in existing directory required')
if bool(inventory) == bool(plan) or bool(plan) != bool(plan_sha256):
raise ValueError('Choose inventory preflight or plan plus exact file hash')
if inventory:
with closing(open_read_connection(db, pg_service=pg_service, pg_schema=pg_schema)) as c:
result = proposal(c, json.loads(Path(inventory).read_bytes()), database_identity(c, db))
else:
raw = Path(plan).read_bytes()
if hashlib.sha256(raw).hexdigest() != plan_sha256:
raise ValueError('Plan file hash differs')
proposed = json.loads(raw)
with closing(open_store(db, pg_service=pg_service, pg_schema=pg_schema).conn) as c:
try:
if proposed['database'] != database_identity(c, db):
raise ValueError('Plan belongs to another database')
if not getattr(c, 'postgres', False):
c.commit(); c.execute('BEGIN IMMEDIATE')
result = apply_plan(c, proposed); c.commit()
except BaseException:
c.rollback(); raise
out.write_text(json.dumps(result, ensure_ascii=False, indent=2) + '\n')
return result
def main(argv=None, *, prog=None):
parser = argparse.ArgumentParser(prog=prog); database_arguments(parser)
parser.add_argument('--inventory'); parser.add_argument('--plan'); parser.add_argument('--plan-sha256'); parser.add_argument('--out', required=True)
args = parser.parse_args(argv)
report = run(args.db, args.out, pg_service=args.pg_service, pg_schema=args.pg_schema,
inventory=args.inventory, plan=args.plan, plan_sha256=args.plan_sha256)
print(json.dumps({'status': report.get('status', 'planned'), 'out': args.out}), flush=True)