219 lines
11 KiB
Python
219 lines
11 KiB
Python
"""Append-only tag/type assertions and explicit retractions; no inferred labels."""
|
|
import argparse
|
|
from contextlib import closing
|
|
from importlib.resources import files
|
|
import json
|
|
import re
|
|
from urllib.parse import urlparse
|
|
|
|
from paper_atlas.db.backend import database_arguments, inserted_id, open_read_connection, open_store
|
|
from paper_atlas.db.submissions import timestamp
|
|
|
|
TABLE = 'annotation_events'
|
|
MIGRATION = 'annotation_events_v1'
|
|
COLUMNS = ('id', 'request_id', 'paper_id', 'original_paper_id', 'kind', 'namespace',
|
|
'value', 'source_record_id', 'evidence_key', 'evidence_url', 'reason',
|
|
'actor', 'observed_at', 'retracts_id')
|
|
ACTIVE = '''a.retracts_id IS NULL AND NOT EXISTS
|
|
(SELECT 1 FROM annotation_events removed WHERE removed.retracts_id=a.id)'''
|
|
|
|
|
|
def available(conn):
|
|
if getattr(conn, 'postgres', False):
|
|
if hasattr(conn, 'has_table'):
|
|
return conn.has_table(TABLE)
|
|
return bool(conn.execute("SELECT 1 FROM information_schema.tables WHERE table_schema=current_schema() AND table_name=?", (TABLE,)).fetchone())
|
|
return bool(conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,)).fetchone())
|
|
|
|
|
|
def _begin_write(conn):
|
|
if not getattr(conn, 'postgres', False) and not conn.in_transaction:
|
|
conn.execute('BEGIN IMMEDIATE')
|
|
|
|
|
|
def initialize(conn):
|
|
"""Explicit transactional schema extension; does not rescan or rewrite papers."""
|
|
_begin_write(conn)
|
|
exists = available(conn)
|
|
marker = conn.execute('SELECT 1 FROM schema_migrations WHERE name=?', (MIGRATION,)).fetchone()
|
|
if exists != bool(marker):
|
|
raise RuntimeError('Annotation table/upgrade marker mismatch; inspect before changing schema')
|
|
if exists:
|
|
result = conn.execute('SELECT * FROM annotation_events LIMIT 0')
|
|
description = result.cursor.description if getattr(conn, 'postgres', False) else result.description
|
|
if tuple(d[0] for d in description) != COLUMNS:
|
|
raise RuntimeError('Unexpected annotation schema')
|
|
return False
|
|
ddl = files('paper_atlas.db').joinpath('annotations.sql').read_text()
|
|
if getattr(conn, 'postgres', False):
|
|
ddl = re.sub(r'\bINTEGER\b', 'BIGINT', ddl)
|
|
ddl = re.sub(r'\bTEXT\b', 'TEXT COLLATE "C"', ddl)
|
|
ddl = ddl.replace('id BIGINT PRIMARY KEY', 'id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY', 1)
|
|
for statement in ddl.split(';'):
|
|
if statement.strip():
|
|
conn.execute(statement)
|
|
conn.execute('INSERT INTO schema_migrations(name) VALUES(?)', (MIGRATION,))
|
|
return True
|
|
|
|
|
|
def _text(value, name, limit=512):
|
|
if not isinstance(value, str) or not value.strip() or len(value) > limit:
|
|
raise ValueError(name + ' must be nonempty text of at most ' + str(limit) + ' characters')
|
|
return value
|
|
|
|
|
|
def _id(value, name):
|
|
if type(value) is not int or value <= 0:
|
|
raise ValueError(name + ' must be a positive integer')
|
|
return value
|
|
|
|
|
|
def _url(value):
|
|
if value is not None:
|
|
_text(value, 'evidence_url', 4096)
|
|
parsed = urlparse(value)
|
|
if parsed.scheme not in ('https', 'http') or not parsed.hostname or parsed.username or parsed.password:
|
|
raise ValueError('evidence_url must be an HTTP(S) URL without credentials')
|
|
return value
|
|
|
|
|
|
def _require(conn):
|
|
if not available(conn) or not conn.execute('SELECT 1 FROM schema_migrations WHERE name=?', (MIGRATION,)).fetchone():
|
|
raise RuntimeError('Run paper-atlas annotate init explicitly before adding annotations')
|
|
|
|
|
|
def _row(row):
|
|
return dict(zip(COLUMNS, (row[i] for i in range(len(COLUMNS)))))
|
|
|
|
|
|
def _replay(conn, request_id, payload):
|
|
old = conn.execute('SELECT * FROM annotation_events WHERE request_id=?', (request_id,)).fetchone()
|
|
if old is None:
|
|
return None
|
|
old = _row(old)
|
|
if any(old[k] != value for k, value in payload.items()):
|
|
raise ValueError('request_id already identifies different annotation content')
|
|
return old['id']
|
|
|
|
|
|
def assert_annotation(conn, paper, kind, namespace, value, *, request_id, evidence_key,
|
|
reason, actor, evidence_url=None, source_record_id=None):
|
|
_begin_write(conn)
|
|
_require(conn)
|
|
paper = _id(paper, 'paper')
|
|
if kind not in ('tag', 'research_type'):
|
|
raise ValueError('kind must be tag or research_type')
|
|
_text(namespace, 'namespace', 128)
|
|
if not re.fullmatch(r'[a-z][a-z0-9_]*:[a-z][a-z0-9_.-]*', namespace):
|
|
raise ValueError('namespace must have an explicit source:scope')
|
|
payload = dict(original_paper_id=paper, kind=kind, namespace=namespace,
|
|
value=_text(value, 'value'), request_id=_text(request_id, 'request_id', 128),
|
|
source_record_id=source_record_id, evidence_key=_text(evidence_key, 'evidence_key', 256),
|
|
reason=_text(reason, 'reason', 8192), actor=_text(actor, 'actor', 256),
|
|
evidence_url=_url(evidence_url), retracts_id=None)
|
|
if source_record_id is not None:
|
|
_id(source_record_id, 'source_record_id')
|
|
replay = _replay(conn, request_id, payload)
|
|
if replay is not None:
|
|
return replay
|
|
if not conn.execute('SELECT 1 FROM papers WHERE id=?', (paper,)).fetchone():
|
|
raise ValueError('Existing canonical paper required; no automatic identity merge')
|
|
prefix = namespace.split(':', 1)[0]
|
|
if prefix != 'custom' and source_record_id is None:
|
|
raise ValueError('Platform annotations require their own source record')
|
|
if source_record_id is not None:
|
|
source = conn.execute('''SELECT r.paper_id,s.name FROM source_records r
|
|
JOIN sources s ON s.id=r.source_id WHERE r.id=?''', (source_record_id,)).fetchone()
|
|
if not source or source[0] != paper or (prefix != 'custom' and prefix != source[1]):
|
|
raise ValueError('Annotation source record/namespace must belong to this paper and platform')
|
|
payload.update(paper_id=paper, observed_at=timestamp())
|
|
fields = tuple(payload)
|
|
return inserted_id(conn, 'INSERT INTO annotation_events(' + ','.join(fields) + ') VALUES(' +
|
|
','.join('?' for _ in fields) + ')', tuple(payload[k] for k in fields))
|
|
|
|
|
|
def retract_annotation(conn, event, *, request_id, reason, actor, evidence_url=None, custom_only=False):
|
|
_begin_write(conn)
|
|
_require(conn)
|
|
event = _id(event, 'event')
|
|
payload = dict(request_id=_text(request_id, 'request_id', 128), retracts_id=event,
|
|
reason=_text(reason, 'reason', 8192), actor=_text(actor, 'actor', 256), evidence_url=_url(evidence_url))
|
|
target = conn.execute('SELECT * FROM annotation_events WHERE id=?', (event,)).fetchone()
|
|
if target is None or target[-1] is not None:
|
|
raise ValueError('Retraction must reference an existing assertion')
|
|
target = _row(target)
|
|
if custom_only and not target['namespace'].startswith('custom:'):
|
|
raise ValueError('Manual CLI cannot retract platform evidence')
|
|
replay = _replay(conn, request_id, payload)
|
|
if replay is not None:
|
|
return replay
|
|
if conn.execute('SELECT 1 FROM annotation_events WHERE retracts_id=?', (event,)).fetchone():
|
|
raise ValueError('Assertion already retracted; its history is retained')
|
|
payload.update({k: target[k] for k in ('paper_id', 'original_paper_id', 'kind', 'namespace', 'value', 'source_record_id', 'evidence_key')})
|
|
payload['observed_at'] = timestamp()
|
|
fields = tuple(payload)
|
|
return inserted_id(conn, 'INSERT INTO annotation_events(' + ','.join(fields) + ') VALUES(' +
|
|
','.join('?' for _ in fields) + ')', tuple(payload[k] for k in fields))
|
|
|
|
|
|
def observations(conn, paper, *, kind=None, history=False):
|
|
if not available(conn):
|
|
return []
|
|
where = ['a.paper_id=?']; params = [_id(paper, 'paper')]
|
|
if not history:
|
|
where.append(ACTIVE)
|
|
if kind is not None:
|
|
if kind not in ('tag', 'research_type'):
|
|
raise ValueError('Invalid annotation kind')
|
|
where.append('a.kind=?'); params.append(kind)
|
|
return [_row(r) for r in conn.execute('SELECT a.* FROM annotation_events a WHERE ' +
|
|
' AND '.join(where) + ' ORDER BY a.id', params)]
|
|
|
|
|
|
def main(argv=None, *, prog=None):
|
|
parser = argparse.ArgumentParser(prog=prog, description='显式初始化、追加或撤回带证据的自定义标签和研究类型')
|
|
actions = parser.add_subparsers(dest='action', required=True)
|
|
for action in ('init', 'assert', 'retract', 'list'):
|
|
p = actions.add_parser(action)
|
|
database_arguments(p)
|
|
if action in ('assert', 'retract'):
|
|
for field in ('request-id', 'reason', 'actor'):
|
|
p.add_argument('--' + field, required=True)
|
|
p.add_argument('--evidence-url')
|
|
if action in ('assert', 'list'):
|
|
p.add_argument('--paper', type=int, required=True)
|
|
p.add_argument('--kind', choices=('tag', 'research_type'), required=action == 'assert')
|
|
if action == 'assert':
|
|
for field in ('namespace', 'value', 'evidence-key'):
|
|
p.add_argument('--' + field, required=True)
|
|
p.add_argument('--source-record', type=int)
|
|
if action == 'retract':
|
|
p.add_argument('--event', type=int, required=True)
|
|
if action == 'list':
|
|
p.add_argument('--history', action='store_true')
|
|
args = parser.parse_args(argv)
|
|
if args.action == 'list':
|
|
with closing(open_read_connection(args.db, pg_service=args.pg_service, pg_schema=args.pg_schema)) as conn:
|
|
result = observations(conn, args.paper, kind=args.kind, history=args.history)
|
|
else:
|
|
if args.action == 'assert' and not args.namespace.startswith('custom:'):
|
|
parser.error('Manual CLI requires custom:* namespace; platform assertions use validated source APIs')
|
|
with closing(open_store(args.db, pg_service=args.pg_service, pg_schema=args.pg_schema).conn) as conn:
|
|
try:
|
|
if not getattr(conn, 'postgres', False):
|
|
conn.commit()
|
|
conn.execute('BEGIN IMMEDIATE')
|
|
if args.action == 'init':
|
|
result = {'initialized': initialize(conn)}
|
|
else:
|
|
kwargs = dict(request_id=args.request_id, reason=args.reason, actor=args.actor, evidence_url=args.evidence_url)
|
|
event = (assert_annotation(conn, args.paper, args.kind, args.namespace, args.value,
|
|
evidence_key=args.evidence_key, source_record_id=args.source_record, **kwargs)
|
|
if args.action == 'assert' else retract_annotation(conn, args.event, custom_only=True, **kwargs))
|
|
result = {'event_id': event}
|
|
conn.commit()
|
|
except BaseException:
|
|
conn.rollback()
|
|
raise
|
|
print(json.dumps(result, ensure_ascii=False, indent=2))
|