246 lines
17 KiB
Python
246 lines
17 KiB
Python
"""Snapshot-bound reclassification of existing attempts, without reimport or merge."""
|
|
import argparse
|
|
from collections import Counter
|
|
from contextlib import closing
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
import re
|
|
|
|
from paper_atlas.db.backend import database_arguments, open_read_connection, open_store, inserted_id
|
|
from paper_atlas.db.edition_scopes import KEY
|
|
from paper_atlas.ingestion.event_identity import database_identity, digest, values
|
|
|
|
MARKER = 'edition_tracks_v1:'
|
|
|
|
|
|
def validate(inventory):
|
|
if (not isinstance(inventory, dict) or inventory.get('format') != 1
|
|
or not isinstance(inventory.get('records'), list) or not 0 < len(inventory['records']) <= 10000):
|
|
raise ValueError('Nonempty bounded reviewed inventory required')
|
|
ids = set(); records = set()
|
|
for row in inventory['records']:
|
|
required = {'submission_id','source_record_id','external_id','source','doi','from_edition_id',
|
|
'year','venue','venue_names','track','section','evidence_url','evidence_sha256'}
|
|
if not isinstance(row, dict) or set(row) != required:
|
|
raise ValueError('Unexpected inventory record fields')
|
|
for field in ('submission_id','source_record_id','from_edition_id','year'):
|
|
if type(row[field]) is not int or row[field] <= 0:
|
|
raise ValueError('Positive integer inventory identity required')
|
|
if row['submission_id'] in ids or row['source_record_id'] in records:
|
|
raise ValueError('Repeated submission/source identity in inventory')
|
|
ids.add(row['submission_id']); records.add(row['source_record_id'])
|
|
for field in ('external_id','source','doi','venue','section','evidence_url'):
|
|
if not isinstance(row[field], str) or not row[field].strip():
|
|
raise ValueError('Missing inventory evidence field')
|
|
if (not isinstance(row['venue_names'], list) or not row['venue_names']
|
|
or not all(isinstance(v,str) and v for v in row['venue_names'])
|
|
or row['venue'] not in row['venue_names']
|
|
or not re.fullmatch(r'[a-z][a-z0-9_]{0,63}', row['track'])
|
|
or not re.fullmatch(r'[0-9a-f]{64}', row['evidence_sha256'])
|
|
or not row['evidence_url'].startswith('https://')):
|
|
raise ValueError('Invalid scope or evidence binding')
|
|
return inventory
|
|
|
|
|
|
def bounded(c, sql, params):
|
|
rows = values(c.execute(sql+' LIMIT 100001', params))
|
|
if len(rows) > 100000:
|
|
raise ValueError('Protected scope exceeds bounded reclassification limit')
|
|
return rows
|
|
|
|
|
|
def protect(c, ids):
|
|
marks = ','.join('?' for _ in ids)
|
|
subs = 'SELECT id FROM submissions WHERE id IN ('+marks+')'
|
|
papers = 'SELECT paper_id FROM submissions WHERE id IN ('+marks+')'
|
|
decisions = 'SELECT id FROM decisions WHERE submission_id IN ('+subs+')'
|
|
queries = {
|
|
'submissions': ('SELECT id,paper_id,source_record_id,source_submission_id,submitted_at FROM submissions WHERE id IN ('+marks+') ORDER BY id'),
|
|
'submission_identifiers': ('SELECT scheme,value,submission_id FROM submission_identifiers WHERE submission_id IN ('+subs+') ORDER BY submission_id,scheme,value'),
|
|
'submission_sources': ('SELECT * FROM submission_sources WHERE submission_id IN ('+subs+') ORDER BY submission_id,source_record_id,source_submission_id'),
|
|
'decisions': ('SELECT * FROM decisions WHERE submission_id IN ('+subs+') ORDER BY id'),
|
|
'decision_sources': ('SELECT * FROM decision_sources WHERE decision_id IN ('+decisions+') ORDER BY decision_id,source_record_id,source_decision_id'),
|
|
'papers': ('SELECT * FROM papers WHERE id IN ('+papers+') ORDER BY id'),
|
|
'identifiers': ('SELECT * FROM identifiers WHERE paper_id IN ('+papers+') ORDER BY scheme,value'),
|
|
'paper_tags': ('SELECT * FROM paper_tags WHERE paper_id IN ('+papers+') ORDER BY paper_id,tag_id,evidence_key'),
|
|
}
|
|
from paper_atlas.curation.annotations import available
|
|
if available(c):
|
|
queries['annotation_events'] = 'SELECT * FROM annotation_events WHERE paper_id IN ('+papers+') ORDER BY id'
|
|
result = {table: {'rows':len(rows), 'sha256':digest(rows)} for table, sql in queries.items()
|
|
for rows in [bounded(c,sql,ids)]}
|
|
rows = bounded(c, 'SELECT r.* FROM source_records r WHERE r.id IN (SELECT source_record_id FROM submission_sources WHERE submission_id IN ('+subs+')) ORDER BY r.id', ids)
|
|
# Only the reviewed curation field may change; every other metadata value
|
|
# and all raw text/IDs/retrieval times remain protected.
|
|
for row in rows:
|
|
metadata = json.loads(row[8]); metadata.pop(KEY,None); row[8] = metadata
|
|
result['source_records'] = {'rows':len(rows), 'sha256':digest(rows)}
|
|
return result
|
|
|
|
|
|
def inspect(c, inventory):
|
|
validate(inventory); changes=[]; ids=tuple(sorted(r['submission_id']for r in inventory['records']))
|
|
original = sorted({r['from_edition_id'] for r in inventory['records']})
|
|
actual_ids = {r[0] for r in c.execute('SELECT id FROM submissions WHERE edition_id IN ('+','.join('?'for _ in original)+')', original)}
|
|
if actual_ids != set(ids):
|
|
raise ValueError('Inventory must cover every attempt in each original edition')
|
|
for row in inventory['records']:
|
|
current = c.execute('''SELECT s.edition_id,s.source_record_id,s.source_submission_id,s.paper_id,
|
|
e.venue_id,e.year,e.track,e.event_key,v.acronym,r.external_id,r.metadata_json,src.name
|
|
FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id
|
|
JOIN source_records r ON r.id=s.source_record_id JOIN sources src ON src.id=r.source_id WHERE s.id=?''',(row['submission_id'],)).fetchone()
|
|
if not current or [current[i]for i in (0,1,2,5,8,9,11)] != [row[k]for k in ('from_edition_id','source_record_id','external_id','year','venue','external_id','source')]:
|
|
raise ValueError('Submission identity/scope differs from inventory')
|
|
aliases={r[0]for r in c.execute('SELECT alias FROM venue_aliases WHERE venue_id=?',(current[4],))}|{current[8]}
|
|
if set(row['venue_names'])!=aliases:
|
|
raise ValueError('Venue names differ from current aliases')
|
|
metadata=json.loads(current[10])
|
|
if KEY in metadata:
|
|
raise ValueError('Existing curated scope requires explicit reconciliation')
|
|
if (metadata.get('record_identifiers') or {}).get('doi') != row['doi']:
|
|
raise ValueError('Stable DOI differs from reviewed source evidence')
|
|
links=values(c.execute('SELECT source_record_id,source_submission_id FROM submission_sources WHERE submission_id=?',(row['submission_id'],)))
|
|
if links != [[row['source_record_id'],row['external_id']]]:
|
|
raise ValueError('Shared-source attempt requires separate reconciliation')
|
|
decisions=values(c.execute('SELECT status FROM decisions WHERE submission_id=?',(row['submission_id'],)))
|
|
if not decisions or any(d[0]!='accepted'for d in decisions):
|
|
raise ValueError('Only publication observations can use this track correction')
|
|
target=c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=? AND event_key=?',(current[4],current[5],row['track'],current[7])).fetchone()
|
|
if target and target[0] != current[0]:
|
|
if c.execute('SELECT 1 FROM submissions WHERE edition_id=? LIMIT 1',(target[0],)).fetchone():
|
|
raise ValueError('Nonempty target edition requires separate reconciliation')
|
|
if c.execute('SELECT 1 FROM submission_identifiers WHERE edition_id=? LIMIT 1',(target[0],)).fetchone():
|
|
raise ValueError('Target identifier scope already exists')
|
|
if c.execute('SELECT 1 FROM edition_sources WHERE edition_id=? LIMIT 1',(target[0],)).fetchone():
|
|
raise ValueError('Target coverage already exists; reconcile first')
|
|
if c.execute('SELECT 1 FROM submission_identifiers WHERE submission_id=? AND edition_id<>? LIMIT 1',(row['submission_id'],current[0])).fetchone():
|
|
raise ValueError('Existing identifier edition is inconsistent')
|
|
changes.append({**row,'from_track':current[6],'event_key':current[7], 'venue_id':current[4],
|
|
'target_edition_id':target[0]if target else None})
|
|
coverage=bounded(c,'SELECT * FROM edition_sources WHERE edition_id IN ('+','.join('?'for _ in original)+') ORDER BY edition_id,source_id,list_url',original)
|
|
if Counter(row[0]for row in coverage)!=Counter({edition:1 for edition in original}):
|
|
raise ValueError('Exactly one original coverage listing per edition required')
|
|
return {'records':changes, 'protected':protect(c,ids), 'coverage':coverage,
|
|
'editions':bounded(c,'SELECT * FROM editions WHERE id IN ('+','.join('?'for _ in original)+') ORDER BY id',original)}
|
|
|
|
|
|
def proposal(c, inventory, identity):
|
|
return {'format':1,'database':identity,'inventory':inventory,'before':inspect(c,inventory)}
|
|
|
|
|
|
def verify(c, plan):
|
|
records=plan['before']['records']; ids=tuple(sorted(r['submission_id']for r in records))
|
|
if protect(c,ids)!=plan['before']['protected']:
|
|
raise ValueError('Protected history/text/tags/identities changed; roll back')
|
|
for row in records:
|
|
event=c.execute('''SELECT e.id,e.venue_id,e.year,e.track,e.event_key FROM submissions s JOIN editions e ON e.id=s.edition_id WHERE s.id=?''',(row['submission_id'],)).fetchone()
|
|
if not event or [event[i]for i in range(1,5)] != [row['venue_id'],row['year'],row['track'],row['event_key']]:
|
|
raise ValueError('Target track assignment failed')
|
|
if c.execute('SELECT 1 FROM submission_identifiers WHERE submission_id=? AND edition_id<>? LIMIT 1',(row['submission_id'],event[0])).fetchone():
|
|
raise ValueError('Target identifier assignment failed')
|
|
metadata=json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(row['source_record_id'],)).fetchone()[0])
|
|
if row['track'] != row['from_track']:
|
|
if metadata.get(KEY) != [curation(row,event[0],digest(plan))]:
|
|
raise ValueError('Reviewed refresh protection missing or changed')
|
|
elif KEY in metadata:
|
|
raise ValueError('Unexpected scope revision on unchanged track')
|
|
original=sorted({r['from_edition_id']for r in records})
|
|
if bounded(c,'SELECT * FROM editions WHERE id IN ('+','.join('?'for _ in original)+') ORDER BY id',original)!=plan['before']['editions']:
|
|
raise ValueError('Original edition identities changed')
|
|
for old in plan['before']['coverage']:
|
|
edition,source,url,_,_,_,cursor,success,_=old
|
|
expected=Counter()
|
|
for row in records:
|
|
if row['from_edition_id']==edition:
|
|
target=c.execute('SELECT edition_id FROM submissions WHERE id=?',(row['submission_id'],)).fetchone()[0]
|
|
expected[target]+=1
|
|
for target,number in expected.items():
|
|
actual=c.execute('SELECT status,expected_count,imported_count,cursor,last_success_at,last_error FROM edition_sources WHERE edition_id=? AND source_id=? AND list_url=?',(target,source,url)).fetchone()
|
|
note='Track scope corrected from reviewed per-record evidence; original whole-volume coverage retained in plan '+digest(plan)
|
|
if actual is None or [actual[i]for i in range(6)]!=['partial',number,number,cursor,success,note]:
|
|
raise ValueError('Corrected coverage differs from reviewed scope')
|
|
return {'records_verified':len(records),'protected':plan['before']['protected']}
|
|
|
|
|
|
def curation(row, target, signature):
|
|
return {'external_id':row['external_id'], 'venue_names':sorted(set(row['venue_names'])),
|
|
'from':{'edition_id':row['from_edition_id'],'year':row['year'],'track':row['from_track'],'event_key':row['event_key']},
|
|
'target':{'edition_id':target,'venue_id':row['venue_id'],'year':row['year'],'track':row['track'],'event_key':row['event_key']},
|
|
'section':row['section'],'evidence_url':row['evidence_url'],'evidence_sha256':row['evidence_sha256'],'plan_digest':signature}
|
|
|
|
|
|
def apply_plan(c, plan):
|
|
if plan.get('format') != 1:
|
|
raise ValueError('Unsupported plan format')
|
|
signature=digest(plan);marker=MARKER+signature
|
|
if c.execute('SELECT 1 FROM schema_migrations WHERE name=?',(marker,)).fetchone():
|
|
return {'status':'already_applied','events_moved':0,**verify(c,plan)}
|
|
if inspect(c,plan['inventory']) != plan['before']:
|
|
raise ValueError('Stale track plan; create and review a fresh plan')
|
|
targets={}; counts=Counter(); moved=0
|
|
for row in plan['before']['records']:
|
|
key=(row['venue_id'],row['year'],row['track'],row['event_key'])
|
|
if key not in targets:
|
|
found=c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=? AND event_key=?',key).fetchone()
|
|
targets[key]=found[0]if found else inserted_id(c,'INSERT INTO editions(venue_id,year,track,event_key) VALUES(?,?,?,?)',key)
|
|
target=targets[key];counts[(row['from_edition_id'],target)]+=1
|
|
if target != row['from_edition_id']:
|
|
c.execute('UPDATE submission_identifiers SET edition_id=? WHERE submission_id=?',(target,row['submission_id']))
|
|
if c.execute('UPDATE submissions SET edition_id=? WHERE id=? AND edition_id=?',(target,row['submission_id'],row['from_edition_id'])).rowcount!=1:
|
|
raise ValueError('Concurrent submission change')
|
|
metadata=json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(row['source_record_id'],)).fetchone()[0]);metadata[KEY]=[curation(row,target,signature)]
|
|
c.execute('UPDATE source_records SET metadata_json=? WHERE id=?',(json.dumps(metadata,ensure_ascii=False),row['source_record_id']));moved+=1
|
|
# Preserve each original source URL and timestamps, but do not retain its
|
|
# old whole-volume count as main-track coverage. Partial is deliberate:
|
|
# this operation verifies scope, not abstracts or all possible sources.
|
|
for old in plan['before']['coverage']:
|
|
edition,source,url,status,expected,imported,cursor,success,error=old
|
|
own={r['submission_id'] for r in plan['before']['records']if r['from_edition_id']==edition}
|
|
source_name=c.execute('SELECT name FROM sources WHERE id=?',(source,)).fetchone()[0]
|
|
if {r['source']for r in plan['before']['records']if r['submission_id']in own}!={source_name}:
|
|
raise ValueError('Coverage includes a source outside the reviewed inventory')
|
|
for (origin,target),number in counts.items():
|
|
if origin!=edition:continue
|
|
note='Track scope corrected from reviewed per-record evidence; original whole-volume coverage retained in plan '+signature
|
|
c.execute('''INSERT INTO edition_sources(edition_id,source_id,list_url,status,expected_count,imported_count,cursor,last_success_at,last_error)
|
|
VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(edition_id,source_id,list_url) DO UPDATE SET status=excluded.status,
|
|
expected_count=excluded.expected_count,imported_count=excluded.imported_count,last_error=excluded.last_error''',
|
|
(target,source,url,'partial',number,number,cursor,success,note))
|
|
result=verify(c,plan)
|
|
c.execute('INSERT INTO schema_migrations(name) VALUES(?)',(marker,))
|
|
return {'status':'applied','events_moved':moved,**result}
|
|
|
|
|
|
def run(db,out,*,pg_service=None,pg_schema=None,inventory=None,plan=None,plan_sha256=None):
|
|
out=Path(out)
|
|
if out.exists() or not out.parent.is_dir():raise ValueError('New report path in existing directory required')
|
|
expected=None
|
|
if plan:
|
|
raw=Path(plan).read_bytes()
|
|
if hashlib.sha256(raw).hexdigest()!=plan_sha256:raise ValueError('Plan hash differs')
|
|
expected=json.loads(raw);validate(expected['inventory'])
|
|
elif plan_sha256:raise ValueError('Plan path required')
|
|
if bool(expected)==bool(inventory):raise ValueError('Choose inventory preflight or apply plan')
|
|
if expected is None:
|
|
with closing(open_read_connection(db,pg_service=pg_service,pg_schema=pg_schema))as c:
|
|
result=proposal(c,validate(json.loads(Path(inventory).read_bytes())),database_identity(c,db))
|
|
else:
|
|
store=open_store(db,pg_service=pg_service,pg_schema=pg_schema)
|
|
with closing(store.conn)as c:
|
|
try:
|
|
if expected['database']!=database_identity(c,db):raise ValueError('Plan belongs to another database')
|
|
if not getattr(c,'postgres',False):c.commit();c.execute('BEGIN IMMEDIATE')
|
|
result=apply_plan(c,expected);c.commit()
|
|
except BaseException:c.rollback();raise
|
|
out.write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n');return result
|
|
|
|
|
|
def main(argv=None,*,prog=None):
|
|
p=argparse.ArgumentParser(prog=prog);database_arguments(p)
|
|
p.add_argument('--inventory');p.add_argument('--plan');p.add_argument('--plan-sha256');p.add_argument('--out',required=True)
|
|
a=p.parse_args(argv)
|
|
if bool(a.plan)!=bool(a.plan_sha256):p.error('--plan and --plan-sha256 must be supplied together')
|
|
r=run(a.db,a.out,pg_service=a.pg_service,pg_schema=a.pg_schema,inventory=a.inventory,plan=a.plan,plan_sha256=a.plan_sha256)
|
|
print(json.dumps({'status':r.get('status','planned'),'events_moved':r.get('events_moved',0),'out':a.out}),flush=True)
|