Files
paper-atlas/paper_atlas/ingestion/edition_tracks.py
T
codex-bot d2c016e309
Metadata integrity checks / metadata (push) Successful in 2m3s
Reject target conflicts and post-application scope evidence drift
2026-11-08 23:48:40 +08:00

246 lines
17 KiB
Python

"""Snapshot-bound reclassification of existing attempts, without reimport or merge."""
import argparse
from collections import Counter
from contextlib import closing
import hashlib
import json
from pathlib import Path
import re
from paper_atlas.db.backend import database_arguments, open_read_connection, open_store, inserted_id
from paper_atlas.db.edition_scopes import KEY
from paper_atlas.ingestion.event_identity import database_identity, digest, values
MARKER = 'edition_tracks_v1:'
def validate(inventory):
if (not isinstance(inventory, dict) or inventory.get('format') != 1
or not isinstance(inventory.get('records'), list) or not 0 < len(inventory['records']) <= 10000):
raise ValueError('Nonempty bounded reviewed inventory required')
ids = set(); records = set()
for row in inventory['records']:
required = {'submission_id','source_record_id','external_id','source','doi','from_edition_id',
'year','venue','venue_names','track','section','evidence_url','evidence_sha256'}
if not isinstance(row, dict) or set(row) != required:
raise ValueError('Unexpected inventory record fields')
for field in ('submission_id','source_record_id','from_edition_id','year'):
if type(row[field]) is not int or row[field] <= 0:
raise ValueError('Positive integer inventory identity required')
if row['submission_id'] in ids or row['source_record_id'] in records:
raise ValueError('Repeated submission/source identity in inventory')
ids.add(row['submission_id']); records.add(row['source_record_id'])
for field in ('external_id','source','doi','venue','section','evidence_url'):
if not isinstance(row[field], str) or not row[field].strip():
raise ValueError('Missing inventory evidence field')
if (not isinstance(row['venue_names'], list) or not row['venue_names']
or not all(isinstance(v,str) and v for v in row['venue_names'])
or row['venue'] not in row['venue_names']
or not re.fullmatch(r'[a-z][a-z0-9_]{0,63}', row['track'])
or not re.fullmatch(r'[0-9a-f]{64}', row['evidence_sha256'])
or not row['evidence_url'].startswith('https://')):
raise ValueError('Invalid scope or evidence binding')
return inventory
def bounded(c, sql, params):
rows = values(c.execute(sql+' LIMIT 100001', params))
if len(rows) > 100000:
raise ValueError('Protected scope exceeds bounded reclassification limit')
return rows
def protect(c, ids):
marks = ','.join('?' for _ in ids)
subs = 'SELECT id FROM submissions WHERE id IN ('+marks+')'
papers = 'SELECT paper_id FROM submissions WHERE id IN ('+marks+')'
decisions = 'SELECT id FROM decisions WHERE submission_id IN ('+subs+')'
queries = {
'submissions': ('SELECT id,paper_id,source_record_id,source_submission_id,submitted_at FROM submissions WHERE id IN ('+marks+') ORDER BY id'),
'submission_identifiers': ('SELECT scheme,value,submission_id FROM submission_identifiers WHERE submission_id IN ('+subs+') ORDER BY submission_id,scheme,value'),
'submission_sources': ('SELECT * FROM submission_sources WHERE submission_id IN ('+subs+') ORDER BY submission_id,source_record_id,source_submission_id'),
'decisions': ('SELECT * FROM decisions WHERE submission_id IN ('+subs+') ORDER BY id'),
'decision_sources': ('SELECT * FROM decision_sources WHERE decision_id IN ('+decisions+') ORDER BY decision_id,source_record_id,source_decision_id'),
'papers': ('SELECT * FROM papers WHERE id IN ('+papers+') ORDER BY id'),
'identifiers': ('SELECT * FROM identifiers WHERE paper_id IN ('+papers+') ORDER BY scheme,value'),
'paper_tags': ('SELECT * FROM paper_tags WHERE paper_id IN ('+papers+') ORDER BY paper_id,tag_id,evidence_key'),
}
from paper_atlas.curation.annotations import available
if available(c):
queries['annotation_events'] = 'SELECT * FROM annotation_events WHERE paper_id IN ('+papers+') ORDER BY id'
result = {table: {'rows':len(rows), 'sha256':digest(rows)} for table, sql in queries.items()
for rows in [bounded(c,sql,ids)]}
rows = bounded(c, 'SELECT r.* FROM source_records r WHERE r.id IN (SELECT source_record_id FROM submission_sources WHERE submission_id IN ('+subs+')) ORDER BY r.id', ids)
# Only the reviewed curation field may change; every other metadata value
# and all raw text/IDs/retrieval times remain protected.
for row in rows:
metadata = json.loads(row[8]); metadata.pop(KEY,None); row[8] = metadata
result['source_records'] = {'rows':len(rows), 'sha256':digest(rows)}
return result
def inspect(c, inventory):
validate(inventory); changes=[]; ids=tuple(sorted(r['submission_id']for r in inventory['records']))
original = sorted({r['from_edition_id'] for r in inventory['records']})
actual_ids = {r[0] for r in c.execute('SELECT id FROM submissions WHERE edition_id IN ('+','.join('?'for _ in original)+')', original)}
if actual_ids != set(ids):
raise ValueError('Inventory must cover every attempt in each original edition')
for row in inventory['records']:
current = c.execute('''SELECT s.edition_id,s.source_record_id,s.source_submission_id,s.paper_id,
e.venue_id,e.year,e.track,e.event_key,v.acronym,r.external_id,r.metadata_json,src.name
FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id
JOIN source_records r ON r.id=s.source_record_id JOIN sources src ON src.id=r.source_id WHERE s.id=?''',(row['submission_id'],)).fetchone()
if not current or [current[i]for i in (0,1,2,5,8,9,11)] != [row[k]for k in ('from_edition_id','source_record_id','external_id','year','venue','external_id','source')]:
raise ValueError('Submission identity/scope differs from inventory')
aliases={r[0]for r in c.execute('SELECT alias FROM venue_aliases WHERE venue_id=?',(current[4],))}|{current[8]}
if set(row['venue_names'])!=aliases:
raise ValueError('Venue names differ from current aliases')
metadata=json.loads(current[10])
if KEY in metadata:
raise ValueError('Existing curated scope requires explicit reconciliation')
if (metadata.get('record_identifiers') or {}).get('doi') != row['doi']:
raise ValueError('Stable DOI differs from reviewed source evidence')
links=values(c.execute('SELECT source_record_id,source_submission_id FROM submission_sources WHERE submission_id=?',(row['submission_id'],)))
if links != [[row['source_record_id'],row['external_id']]]:
raise ValueError('Shared-source attempt requires separate reconciliation')
decisions=values(c.execute('SELECT status FROM decisions WHERE submission_id=?',(row['submission_id'],)))
if not decisions or any(d[0]!='accepted'for d in decisions):
raise ValueError('Only publication observations can use this track correction')
target=c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=? AND event_key=?',(current[4],current[5],row['track'],current[7])).fetchone()
if target and target[0] != current[0]:
if c.execute('SELECT 1 FROM submissions WHERE edition_id=? LIMIT 1',(target[0],)).fetchone():
raise ValueError('Nonempty target edition requires separate reconciliation')
if c.execute('SELECT 1 FROM submission_identifiers WHERE edition_id=? LIMIT 1',(target[0],)).fetchone():
raise ValueError('Target identifier scope already exists')
if c.execute('SELECT 1 FROM edition_sources WHERE edition_id=? LIMIT 1',(target[0],)).fetchone():
raise ValueError('Target coverage already exists; reconcile first')
if c.execute('SELECT 1 FROM submission_identifiers WHERE submission_id=? AND edition_id<>? LIMIT 1',(row['submission_id'],current[0])).fetchone():
raise ValueError('Existing identifier edition is inconsistent')
changes.append({**row,'from_track':current[6],'event_key':current[7], 'venue_id':current[4],
'target_edition_id':target[0]if target else None})
coverage=bounded(c,'SELECT * FROM edition_sources WHERE edition_id IN ('+','.join('?'for _ in original)+') ORDER BY edition_id,source_id,list_url',original)
if Counter(row[0]for row in coverage)!=Counter({edition:1 for edition in original}):
raise ValueError('Exactly one original coverage listing per edition required')
return {'records':changes, 'protected':protect(c,ids), 'coverage':coverage,
'editions':bounded(c,'SELECT * FROM editions WHERE id IN ('+','.join('?'for _ in original)+') ORDER BY id',original)}
def proposal(c, inventory, identity):
return {'format':1,'database':identity,'inventory':inventory,'before':inspect(c,inventory)}
def verify(c, plan):
records=plan['before']['records']; ids=tuple(sorted(r['submission_id']for r in records))
if protect(c,ids)!=plan['before']['protected']:
raise ValueError('Protected history/text/tags/identities changed; roll back')
for row in records:
event=c.execute('''SELECT e.id,e.venue_id,e.year,e.track,e.event_key FROM submissions s JOIN editions e ON e.id=s.edition_id WHERE s.id=?''',(row['submission_id'],)).fetchone()
if not event or [event[i]for i in range(1,5)] != [row['venue_id'],row['year'],row['track'],row['event_key']]:
raise ValueError('Target track assignment failed')
if c.execute('SELECT 1 FROM submission_identifiers WHERE submission_id=? AND edition_id<>? LIMIT 1',(row['submission_id'],event[0])).fetchone():
raise ValueError('Target identifier assignment failed')
metadata=json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(row['source_record_id'],)).fetchone()[0])
if row['track'] != row['from_track']:
if metadata.get(KEY) != [curation(row,event[0],digest(plan))]:
raise ValueError('Reviewed refresh protection missing or changed')
elif KEY in metadata:
raise ValueError('Unexpected scope revision on unchanged track')
original=sorted({r['from_edition_id']for r in records})
if bounded(c,'SELECT * FROM editions WHERE id IN ('+','.join('?'for _ in original)+') ORDER BY id',original)!=plan['before']['editions']:
raise ValueError('Original edition identities changed')
for old in plan['before']['coverage']:
edition,source,url,_,_,_,cursor,success,_=old
expected=Counter()
for row in records:
if row['from_edition_id']==edition:
target=c.execute('SELECT edition_id FROM submissions WHERE id=?',(row['submission_id'],)).fetchone()[0]
expected[target]+=1
for target,number in expected.items():
actual=c.execute('SELECT status,expected_count,imported_count,cursor,last_success_at,last_error FROM edition_sources WHERE edition_id=? AND source_id=? AND list_url=?',(target,source,url)).fetchone()
note='Track scope corrected from reviewed per-record evidence; original whole-volume coverage retained in plan '+digest(plan)
if actual is None or [actual[i]for i in range(6)]!=['partial',number,number,cursor,success,note]:
raise ValueError('Corrected coverage differs from reviewed scope')
return {'records_verified':len(records),'protected':plan['before']['protected']}
def curation(row, target, signature):
return {'external_id':row['external_id'], 'venue_names':sorted(set(row['venue_names'])),
'from':{'edition_id':row['from_edition_id'],'year':row['year'],'track':row['from_track'],'event_key':row['event_key']},
'target':{'edition_id':target,'venue_id':row['venue_id'],'year':row['year'],'track':row['track'],'event_key':row['event_key']},
'section':row['section'],'evidence_url':row['evidence_url'],'evidence_sha256':row['evidence_sha256'],'plan_digest':signature}
def apply_plan(c, plan):
if plan.get('format') != 1:
raise ValueError('Unsupported plan format')
signature=digest(plan);marker=MARKER+signature
if c.execute('SELECT 1 FROM schema_migrations WHERE name=?',(marker,)).fetchone():
return {'status':'already_applied','events_moved':0,**verify(c,plan)}
if inspect(c,plan['inventory']) != plan['before']:
raise ValueError('Stale track plan; create and review a fresh plan')
targets={}; counts=Counter(); moved=0
for row in plan['before']['records']:
key=(row['venue_id'],row['year'],row['track'],row['event_key'])
if key not in targets:
found=c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=? AND event_key=?',key).fetchone()
targets[key]=found[0]if found else inserted_id(c,'INSERT INTO editions(venue_id,year,track,event_key) VALUES(?,?,?,?)',key)
target=targets[key];counts[(row['from_edition_id'],target)]+=1
if target != row['from_edition_id']:
c.execute('UPDATE submission_identifiers SET edition_id=? WHERE submission_id=?',(target,row['submission_id']))
if c.execute('UPDATE submissions SET edition_id=? WHERE id=? AND edition_id=?',(target,row['submission_id'],row['from_edition_id'])).rowcount!=1:
raise ValueError('Concurrent submission change')
metadata=json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(row['source_record_id'],)).fetchone()[0]);metadata[KEY]=[curation(row,target,signature)]
c.execute('UPDATE source_records SET metadata_json=? WHERE id=?',(json.dumps(metadata,ensure_ascii=False),row['source_record_id']));moved+=1
# Preserve each original source URL and timestamps, but do not retain its
# old whole-volume count as main-track coverage. Partial is deliberate:
# this operation verifies scope, not abstracts or all possible sources.
for old in plan['before']['coverage']:
edition,source,url,status,expected,imported,cursor,success,error=old
own={r['submission_id'] for r in plan['before']['records']if r['from_edition_id']==edition}
source_name=c.execute('SELECT name FROM sources WHERE id=?',(source,)).fetchone()[0]
if {r['source']for r in plan['before']['records']if r['submission_id']in own}!={source_name}:
raise ValueError('Coverage includes a source outside the reviewed inventory')
for (origin,target),number in counts.items():
if origin!=edition:continue
note='Track scope corrected from reviewed per-record evidence; original whole-volume coverage retained in plan '+signature
c.execute('''INSERT INTO edition_sources(edition_id,source_id,list_url,status,expected_count,imported_count,cursor,last_success_at,last_error)
VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(edition_id,source_id,list_url) DO UPDATE SET status=excluded.status,
expected_count=excluded.expected_count,imported_count=excluded.imported_count,last_error=excluded.last_error''',
(target,source,url,'partial',number,number,cursor,success,note))
result=verify(c,plan)
c.execute('INSERT INTO schema_migrations(name) VALUES(?)',(marker,))
return {'status':'applied','events_moved':moved,**result}
def run(db,out,*,pg_service=None,pg_schema=None,inventory=None,plan=None,plan_sha256=None):
out=Path(out)
if out.exists() or not out.parent.is_dir():raise ValueError('New report path in existing directory required')
expected=None
if plan:
raw=Path(plan).read_bytes()
if hashlib.sha256(raw).hexdigest()!=plan_sha256:raise ValueError('Plan hash differs')
expected=json.loads(raw);validate(expected['inventory'])
elif plan_sha256:raise ValueError('Plan path required')
if bool(expected)==bool(inventory):raise ValueError('Choose inventory preflight or apply plan')
if expected is None:
with closing(open_read_connection(db,pg_service=pg_service,pg_schema=pg_schema))as c:
result=proposal(c,validate(json.loads(Path(inventory).read_bytes())),database_identity(c,db))
else:
store=open_store(db,pg_service=pg_service,pg_schema=pg_schema)
with closing(store.conn)as c:
try:
if expected['database']!=database_identity(c,db):raise ValueError('Plan belongs to another database')
if not getattr(c,'postgres',False):c.commit();c.execute('BEGIN IMMEDIATE')
result=apply_plan(c,expected);c.commit()
except BaseException:c.rollback();raise
out.write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n');return result
def main(argv=None,*,prog=None):
p=argparse.ArgumentParser(prog=prog);database_arguments(p)
p.add_argument('--inventory');p.add_argument('--plan');p.add_argument('--plan-sha256');p.add_argument('--out',required=True)
a=p.parse_args(argv)
if bool(a.plan)!=bool(a.plan_sha256):p.error('--plan and --plan-sha256 must be supplied together')
r=run(a.db,a.out,pg_service=a.pg_service,pg_schema=a.pg_schema,inventory=a.inventory,plan=a.plan,plan_sha256=a.plan_sha256)
print(json.dumps({'status':r.get('status','planned'),'events_moved':r.get('events_moved',0),'out':a.out}),flush=True)