From 27e336eee9d9ac3a01b5190a2f892368bd0bab11 Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Sun, 8 Nov 2026 18:38:48 +0800 Subject: [PATCH] Add evidence-bearing research types and retractable custom annotations --- .gitea/workflows/check.yml | 2 + README.md | 2 + docs/annotations.md | 107 ++++++++++++++ paper_atlas/cli.py | 1 + paper_atlas/curation/__init__.py | 1 + paper_atlas/curation/annotations.py | 218 +++++++++++++++++++++++++++ paper_atlas/db/annotations.sql | 15 ++ paper_atlas/db/store.py | 3 + paper_atlas/postgres/migration.py | 4 +- paper_atlas/postgres/schema.py | 6 +- paper_atlas/querying.py | 26 +++- pyproject.toml | 2 +- tests/test_annotations.py | 219 ++++++++++++++++++++++++++++ 13 files changed, 600 insertions(+), 6 deletions(-) create mode 100644 docs/annotations.md create mode 100644 paper_atlas/curation/__init__.py create mode 100644 paper_atlas/curation/annotations.py create mode 100644 paper_atlas/db/annotations.sql create mode 100644 tests/test_annotations.py diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index 950c95a..be24fb9 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -58,5 +58,7 @@ jobs: smoke = "from paper_atlas.db.store import Store; from paper_atlas.config import DEFAULT_REGISTRY; import json; assert {'COLM','MM','MLSys'} <= {v['acronym'] for v in json.loads(DEFAULT_REGISTRY.read_text())['venues']}; s=Store('installed.sqlite'); s.conn.close()" subprocess.run([python,'-I','-B','-c',smoke],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','query','--db','installed.sqlite','--stats'],cwd=outside,check=True) + subprocess.run([python,'-I','-B','-m','paper_atlas','annotate','init','--db','installed.sqlite'],cwd=outside,check=True) + subprocess.run([python,'-I','-B','-m','paper_atlas','annotate','list','--db','installed.sqlite','--paper','1'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','audit','histories','--db','installed.sqlite','--examples','0'],cwd=outside,check=True) PY diff --git a/README.md b/README.md index 09dcf66..f12719a 100644 --- a/README.md +++ b/README.md @@ -86,3 +86,5 @@ paper-atlas collect colm --cache /data/paper-index/raw/http-cache \ CI 在已授权的远程 runner 上运行测试、构建 wheel，并从源码目录外验证安装后的命令及随包资源。数据库全量去重和覆盖率是独立的数据验收步骤，不因代码测试通过而自动完成。 开发通过工作分支和 PR 交付，由用户手动合并。 + +研究类型与自定义标签可通过 [`annotate`](docs/annotations.md) 显式初始化、添加和撤回；两者独立查询，保留证据及完整更正历史，不从会议轨道推断研究类型。 diff --git a/docs/annotations.md b/docs/annotations.md new file mode 100644 index 0000000..fbb10b8 --- /dev/null +++ b/docs/annotations.md @@ -0,0 +1,107 @@ +# Research types and custom annotations + +Research type describes a claimed characteristic of the work (for example a survey, +benchmark, empirical study or replication). It is independent of topic, paper length, +conference track, presentation format and acceptance status. No type is inferred from +those fields. Multiple source-specific or human classifications may coexist; no +cross-platform vocabulary mapping or automatic classifier is provided. + +`annotation_events` stores both kinds with an explicit `kind` discriminator: +`research_type` and `tag` are separate domains, even when namespace/value strings +happen to coincide. Existing platform labels remain in `tags` / `paper_tags`. +Each assertion carries namespace, value, actor, reason, evidence key, optional source +record and evidence URL, and the time it was recorded. This is observation time, not +paper publication time or a conference decision date. + +Values are not forced into a global taxonomy: `custom:method=empirical` and +`platform:declared_type=empirical` retain separate provenance. An explicit platform +assertion API requires a source record belonging to that paper and matching the +namespace prefix. The manual CLI only accepts `custom:*`, and cannot retract platform +assertions. No source type has been automatically assigned to existing papers. + +## Initialize explicitly + +On the published PostgreSQL dataset, using its protected service file: + +```sh +PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas annotate init \ + --pg-service foundry-research --pg-schema research +``` + +This creates one optional table, its indexes, and an upgrade marker in one transaction, +using the existing dataset writer lock. It does not copy data, change existing papers +or decisions, or create a PostgreSQL instance. Repeating it is safe; an unexplained +pre-existing table/marker mismatch fails closed. Readers and ordinary imports do not +install the extension. Legacy datasets remain readable. A SQLite **sample** uses +`--db sample.sqlite`; the cutover SQLite backup remains protected against writes. + +## Add and retract evidence + +Use the canonical paper ID returned by `query`. For a reviewed classification: + +```sh +paper-atlas annotate assert --pg-service foundry-research --pg-schema research \ + --paper "$PAPER_ID" --kind research_type --namespace custom:method \ + --value empirical --evidence-key manual:abstract-review \ + --actor reviewer --reason 'Classified from the explicitly described study design' \ + --request-id "$REQUEST_ID" +``` + +`$REQUEST_ID` must be a new, stable identifier for this action (for example a UUID). +Retry with the same request ID and identical content after an interrupted response; +it returns the original event ID. Reusing that ID with changed content is rejected. +A topic annotation uses `--kind tag --namespace custom:topic --value robotics` instead. +Optional `--source-record` and `--evidence-url` attach additional evidence; the source +record must belong to this paper. URLs are retained as evidence and are not fetched. + +To withdraw one particular assertion, use its returned event ID: + +```sh +paper-atlas annotate retract --pg-service foundry-research --pg-schema research \ + --event "$EVENT_ID" --actor reviewer --reason 'Classification corrected after review' \ + --request-id "$RETRACTION_REQUEST_ID" +``` + +Retraction adds an event referencing that assertion, preserving its original content. +It does not remove another contributor's assertion of the same label, platform tags, +source metadata or submission/decision history. A retracted assertion can be asserted +again with a new request ID and evidence; it is a new observation. The API does not +update or delete past annotation events. Database administrators still control direct +SQL access. Work reviewed as the same paper may subsequently be merged by the existing +identity workflow: annotation IDs, original paper IDs, evidence and retraction links +are retained while the canonical paper reference follows the surviving entity. + +## Query and analyze + +```sh +paper-atlas query --pg-service foundry-research --pg-schema research \ + --research-type custom:method=empirical +paper-atlas query --pg-service foundry-research --pg-schema research \ + --tag custom:topic=robotics +paper-atlas query --pg-service foundry-research --pg-schema research \ + --paper "$PAPER_ID" --annotation-history +paper-atlas annotate list --pg-service foundry-research --pg-schema research \ + --paper "$PAPER_ID" --history +``` + +`--tag` includes original platform labels and active annotation tags. It never matches +research types. `--research-type` matches active type assertions only. Paper details +return separate `tags` (existing provenance), `annotation_tags`, and `research_types`; +`--annotation-history` additionally includes retractions. A missing extension yields +empty annotation results rather than changing the database. + +For direct SQL analysis, active research-type evidence is: + +```sql +SELECT a.* FROM research.annotation_events a +WHERE a.kind = 'research_type' AND a.retracts_id IS NULL + AND NOT EXISTS (SELECT 1 FROM research.annotation_events r WHERE r.retracts_id = a.id); +``` + +Several independent assertions can support one label; count distinct papers/labels +when appropriate rather than counting evidence rows as papers. Namespaces stay +separate. Standard paper queries decode the PostgreSQL text codec; direct SQL consumers +must honor the dataset's documented codec, including NUL/SOH values. SQLite-to-PostgreSQL +migration recognizes an installed annotation extension and verifies its full row hashes, +constraints and identity sequence along with the original business tables. An old +21-table snapshot stays a 21-table migration; no synthetic annotations are generated. diff --git a/paper_atlas/cli.py b/paper_atlas/cli.py index 549bfc0..eca3325 100644 --- a/paper_atlas/cli.py +++ b/paper_atlas/cli.py @@ -6,6 +6,7 @@ from paper_atlas import __version__ COMMANDS = { + ('annotate',): ('paper_atlas.curation.annotations', {}, '管理独立的研究类型与自定义标签证据'), ('query',): ('paper_atlas.querying', {}, '查询论文、标签和投稿历史'), ('coverage',): ('paper_atlas.audits.coverage', {}, '查看全部候选会议的覆盖情况'), ('download',): ('paper_atlas.io.snapshots', {}, '下载并校验固定版本快照'), diff --git a/paper_atlas/curation/__init__.py b/paper_atlas/curation/__init__.py new file mode 100644 index 0000000..27d2716 --- /dev/null +++ b/paper_atlas/curation/__init__.py @@ -0,0 +1 @@ +"""Evidence-bearing human and explicit source annotations, independent of outcomes.""" diff --git a/paper_atlas/curation/annotations.py b/paper_atlas/curation/annotations.py new file mode 100644 index 0000000..a8fa0d8 --- /dev/null +++ b/paper_atlas/curation/annotations.py @@ -0,0 +1,218 @@ +"""Append-only tag/type assertions and explicit retractions; no inferred labels.""" +import argparse +from contextlib import closing +from importlib.resources import files +import json +import re +from urllib.parse import urlparse + +from paper_atlas.db.backend import database_arguments, inserted_id, open_read_connection, open_store +from paper_atlas.db.submissions import timestamp + +TABLE = 'annotation_events' +MIGRATION = 'annotation_events_v1' +COLUMNS = ('id', 'request_id', 'paper_id', 'original_paper_id', 'kind', 'namespace', + 'value', 'source_record_id', 'evidence_key', 'evidence_url', 'reason', + 'actor', 'observed_at', 'retracts_id') +ACTIVE = '''a.retracts_id IS NULL AND NOT EXISTS + (SELECT 1 FROM annotation_events removed WHERE removed.retracts_id=a.id)''' + + +def available(conn): + if getattr(conn, 'postgres', False): + if hasattr(conn, 'has_table'): + return conn.has_table(TABLE) + return bool(conn.execute("SELECT 1 FROM information_schema.tables WHERE table_schema=current_schema() AND table_name=?", (TABLE,)).fetchone()) + return bool(conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,)).fetchone()) + + +def _begin_write(conn): + if not getattr(conn, 'postgres', False) and not conn.in_transaction: + conn.execute('BEGIN IMMEDIATE') + + +def initialize(conn): + """Explicit transactional schema extension; does not rescan or rewrite papers.""" + _begin_write(conn) + exists = available(conn) + marker = conn.execute('SELECT 1 FROM schema_migrations WHERE name=?', (MIGRATION,)).fetchone() + if exists != bool(marker): + raise RuntimeError('Annotation table/upgrade marker mismatch; inspect before changing schema') + if exists: + result = conn.execute('SELECT * FROM annotation_events LIMIT 0') + description = result.cursor.description if getattr(conn, 'postgres', False) else result.description + if tuple(d[0] for d in description) != COLUMNS: + raise RuntimeError('Unexpected annotation schema') + return False + ddl = files('paper_atlas.db').joinpath('annotations.sql').read_text() + if getattr(conn, 'postgres', False): + ddl = re.sub(r'\bINTEGER\b', 'BIGINT', ddl) + ddl = re.sub(r'\bTEXT\b', 'TEXT COLLATE "C"', ddl) + ddl = ddl.replace('id BIGINT PRIMARY KEY', 'id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY', 1) + for statement in ddl.split(';'): + if statement.strip(): + conn.execute(statement) + conn.execute('INSERT INTO schema_migrations(name) VALUES(?)', (MIGRATION,)) + return True + + +def _text(value, name, limit=512): + if not isinstance(value, str) or not value.strip() or len(value) > limit: + raise ValueError(name + ' must be nonempty text of at most ' + str(limit) + ' characters') + return value + + +def _id(value, name): + if type(value) is not int or value <= 0: + raise ValueError(name + ' must be a positive integer') + return value + + +def _url(value): + if value is not None: + _text(value, 'evidence_url', 4096) + parsed = urlparse(value) + if parsed.scheme not in ('https', 'http') or not parsed.hostname or parsed.username or parsed.password: + raise ValueError('evidence_url must be an HTTP(S) URL without credentials') + return value + + +def _require(conn): + if not available(conn): + raise RuntimeError('Run paper-atlas annotate init explicitly before adding annotations') + + +def _row(row): + return dict(zip(COLUMNS, (row[i] for i in range(len(COLUMNS))))) + + +def _replay(conn, request_id, payload): + old = conn.execute('SELECT * FROM annotation_events WHERE request_id=?', (request_id,)).fetchone() + if old is None: + return None + old = _row(old) + if any(old[k] != value for k, value in payload.items()): + raise ValueError('request_id already identifies different annotation content') + return old['id'] + + +def assert_annotation(conn, paper, kind, namespace, value, *, request_id, evidence_key, + reason, actor, evidence_url=None, source_record_id=None): + _begin_write(conn) + _require(conn) + paper = _id(paper, 'paper') + if kind not in ('tag', 'research_type'): + raise ValueError('kind must be tag or research_type') + _text(namespace, 'namespace', 128) + if not re.fullmatch(r'[a-z][a-z0-9_]*:[a-z][a-z0-9_.-]*', namespace): + raise ValueError('namespace must have an explicit source:scope') + payload = dict(original_paper_id=paper, kind=kind, namespace=namespace, + value=_text(value, 'value'), request_id=_text(request_id, 'request_id', 128), + source_record_id=source_record_id, evidence_key=_text(evidence_key, 'evidence_key', 256), + reason=_text(reason, 'reason', 8192), actor=_text(actor, 'actor', 256), + evidence_url=_url(evidence_url), retracts_id=None) + if source_record_id is not None: + _id(source_record_id, 'source_record_id') + replay = _replay(conn, request_id, payload) + if replay is not None: + return replay + if not conn.execute('SELECT 1 FROM papers WHERE id=?', (paper,)).fetchone(): + raise ValueError('Existing canonical paper required; no automatic identity merge') + prefix = namespace.split(':', 1)[0] + if prefix != 'custom' and source_record_id is None: + raise ValueError('Platform annotations require their own source record') + if source_record_id is not None: + source = conn.execute('''SELECT r.paper_id,s.name FROM source_records r + JOIN sources s ON s.id=r.source_id WHERE r.id=?''', (source_record_id,)).fetchone() + if not source or source[0] != paper or (prefix != 'custom' and prefix != source[1]): + raise ValueError('Annotation source record/namespace must belong to this paper and platform') + payload.update(paper_id=paper, observed_at=timestamp()) + fields = tuple(payload) + return inserted_id(conn, 'INSERT INTO annotation_events(' + ','.join(fields) + ') VALUES(' + + ','.join('?' for _ in fields) + ')', tuple(payload[k] for k in fields)) + + +def retract_annotation(conn, event, *, request_id, reason, actor, evidence_url=None, custom_only=False): + _begin_write(conn) + _require(conn) + event = _id(event, 'event') + payload = dict(request_id=_text(request_id, 'request_id', 128), retracts_id=event, + reason=_text(reason, 'reason', 8192), actor=_text(actor, 'actor', 256), evidence_url=_url(evidence_url)) + target = conn.execute('SELECT * FROM annotation_events WHERE id=?', (event,)).fetchone() + if target is None or target[-1] is not None: + raise ValueError('Retraction must reference an existing assertion') + target = _row(target) + if custom_only and not target['namespace'].startswith('custom:'): + raise ValueError('Manual CLI cannot retract platform evidence') + replay = _replay(conn, request_id, payload) + if replay is not None: + return replay + if conn.execute('SELECT 1 FROM annotation_events WHERE retracts_id=?', (event,)).fetchone(): + raise ValueError('Assertion already retracted; its history is retained') + payload.update({k: target[k] for k in ('paper_id', 'original_paper_id', 'kind', 'namespace', 'value', 'source_record_id', 'evidence_key')}) + payload['observed_at'] = timestamp() + fields = tuple(payload) + return inserted_id(conn, 'INSERT INTO annotation_events(' + ','.join(fields) + ') VALUES(' + + ','.join('?' for _ in fields) + ')', tuple(payload[k] for k in fields)) + + +def observations(conn, paper, *, kind=None, history=False): + if not available(conn): + return [] + where = ['a.paper_id=?']; params = [_id(paper, 'paper')] + if not history: + where.append(ACTIVE) + if kind is not None: + if kind not in ('tag', 'research_type'): + raise ValueError('Invalid annotation kind') + where.append('a.kind=?'); params.append(kind) + return [_row(r) for r in conn.execute('SELECT a.* FROM annotation_events a WHERE ' + + ' AND '.join(where) + ' ORDER BY a.id', params)] + + +def main(argv=None, *, prog=None): + parser = argparse.ArgumentParser(prog=prog, description='显式初始化、追加或撤回带证据的自定义标签和研究类型') + actions = parser.add_subparsers(dest='action', required=True) + for action in ('init', 'assert', 'retract', 'list'): + p = actions.add_parser(action) + database_arguments(p) + if action in ('assert', 'retract'): + for field in ('request-id', 'reason', 'actor'): + p.add_argument('--' + field, required=True) + p.add_argument('--evidence-url') + if action in ('assert', 'list'): + p.add_argument('--paper', type=int, required=True) + p.add_argument('--kind', choices=('tag', 'research_type'), required=action == 'assert') + if action == 'assert': + for field in ('namespace', 'value', 'evidence-key'): + p.add_argument('--' + field, required=True) + p.add_argument('--source-record', type=int) + if action == 'retract': + p.add_argument('--event', type=int, required=True) + if action == 'list': + p.add_argument('--history', action='store_true') + args = parser.parse_args(argv) + if args.action == 'list': + with closing(open_read_connection(args.db, pg_service=args.pg_service, pg_schema=args.pg_schema)) as conn: + result = observations(conn, args.paper, kind=args.kind, history=args.history) + else: + if args.action == 'assert' and not args.namespace.startswith('custom:'): + parser.error('Manual CLI requires custom:* namespace; platform assertions use validated source APIs') + with closing(open_store(args.db, pg_service=args.pg_service, pg_schema=args.pg_schema).conn) as conn: + try: + if not getattr(conn, 'postgres', False): + conn.commit() + conn.execute('BEGIN IMMEDIATE') + if args.action == 'init': + result = {'initialized': initialize(conn)} + else: + kwargs = dict(request_id=args.request_id, reason=args.reason, actor=args.actor, evidence_url=args.evidence_url) + event = (assert_annotation(conn, args.paper, args.kind, args.namespace, args.value, + evidence_key=args.evidence_key, source_record_id=args.source_record, **kwargs) + if args.action == 'assert' else retract_annotation(conn, args.event, custom_only=True, **kwargs)) + result = {'event_id': event} + conn.commit() + except BaseException: + conn.rollback() + raise + print(json.dumps(result, ensure_ascii=False, indent=2)) diff --git a/paper_atlas/db/annotations.sql b/paper_atlas/db/annotations.sql new file mode 100644 index 0000000..eabb873 --- /dev/null +++ b/paper_atlas/db/annotations.sql @@ -0,0 +1,15 @@ +CREATE TABLE annotation_events ( + id INTEGER PRIMARY KEY, + request_id TEXT NOT NULL UNIQUE, + paper_id INTEGER NOT NULL REFERENCES papers(id), + original_paper_id INTEGER NOT NULL, + kind TEXT NOT NULL CHECK(kind IN ('tag','research_type')), + namespace TEXT NOT NULL, value TEXT NOT NULL, + source_record_id INTEGER REFERENCES source_records(id), + evidence_key TEXT NOT NULL, evidence_url TEXT, + reason TEXT NOT NULL, actor TEXT NOT NULL, observed_at TEXT NOT NULL, + retracts_id INTEGER UNIQUE REFERENCES annotation_events(id), + CHECK(retracts_id IS NULL OR retracts_id<>id) +); +CREATE INDEX annotation_events_paper ON annotation_events(paper_id,kind,id); +CREATE INDEX annotation_events_label ON annotation_events(kind,namespace,value,paper_id); diff --git a/paper_atlas/db/store.py b/paper_atlas/db/store.py index 7d57533..e45ef4f 100644 --- a/paper_atlas/db/store.py +++ b/paper_atlas/db/store.py @@ -55,6 +55,9 @@ class Store: c.execute('UPDATE identifiers SET paper_id=? WHERE paper_id=?',(target,other)) c.execute('UPDATE source_records SET paper_id=? WHERE paper_id=?',(target,other)) c.execute('UPDATE submissions SET paper_id=? WHERE paper_id=?',(target,other)) + from paper_atlas.curation.annotations import available + if available(c): + c.execute('UPDATE annotation_events SET paper_id=? WHERE paper_id=?', (target,other)) c.execute('INSERT INTO paper_tags SELECT ?,tag_id,source_record_id,evidence_key FROM paper_tags WHERE paper_id=? ON CONFLICT DO NOTHING',(target,other)) c.execute('DELETE FROM paper_tags WHERE paper_id=?',(other,)) c.execute('UPDATE paper_merges SET surviving_paper_id=? WHERE surviving_paper_id=?',(target,other)) diff --git a/paper_atlas/postgres/migration.py b/paper_atlas/postgres/migration.py index 0a051ed..2865851 100644 --- a/paper_atlas/postgres/migration.py +++ b/paper_atlas/postgres/migration.py @@ -143,7 +143,7 @@ def migrate(db, service, out, *, schema='research', storage_path=None, min_free_ free_space(storage_path, min_free_gib) control = schema + '_migration' report = {'status': 'preflight', 'source': str(db), 'service': service, 'schema': schema, - 'started_at': timestamp(), 'tables': {}, 'scope': 'All 21 business tables; FTS5 internals rebuilt as native PostgreSQL GIN.'} + 'started_at': timestamp(), 'tables': {}, 'scope': 'All baseline business tables and recognized installed extensions; FTS5 rebuilt as native PostgreSQL GIN.'} with Path(str(db) + '.maintenance.lock').open('a') as lock: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) if Path(str(db) + '-wal').exists() and Path(str(db) + '-wal').stat().st_size: @@ -240,7 +240,7 @@ def migrate(db, service, out, *, schema='research', storage_path=None, min_free_ JOIN pg_am am ON am.oid=ix.relam WHERE ns.nspname=%s AND ix.relname='papers_search_gin' ''', (stage,)).fetchone() if search != ('gin', True, True, 'papers'): raise ValueError('Native full-text index is not valid and ready') - for table in TABLES: + for table in plan: pg.execute('ANALYZE ' + qualify(stage, table)) invalid = pg.execute('''SELECT count(*) FROM pg_constraint co JOIN pg_class cl ON cl.oid=co.conrelid JOIN pg_namespace ns ON ns.oid=cl.relnamespace WHERE ns.nspname=%s AND NOT co.convalidated''', (stage,)).fetchone()[0] diff --git a/paper_atlas/postgres/schema.py b/paper_atlas/postgres/schema.py index 90a97de..0120746 100644 --- a/paper_atlas/postgres/schema.py +++ b/paper_atlas/postgres/schema.py @@ -8,7 +8,9 @@ TABLES = ('sources', 'papers', 'identifiers', 'source_records', 'venues', 'editi 'submission_sources', 'submission_identifiers', 'decision_sources', 'submission_merges', 'decision_merges', 'schema_migrations', 'tags', 'paper_tags', 'match_candidates', 'paper_merges', 'ingest_runs') +OPTIONAL_TABLES = ('annotation_events',) CHECKS = { + 'annotation_events': ["kind IN ('tag','research_type')", 'retracts_id IS NULL OR retracts_id<>id'], 'venues': ["venue_type IN ('conference','journal','review_platform')"], 'edition_sources': ["status IN ('pending','running','complete','partial','blocked','unavailable','not_yet_published')"], 'decisions': ["status IN ('accepted','rejected','desk_rejected','withdrawn','under_review','unknown')"], @@ -50,10 +52,10 @@ def quote(name): def inspect(connection): business = {name for schema, name, kind, *_ in connection.execute('PRAGMA table_list') if schema == 'main' and kind == 'table' and not name.startswith('sqlite_')} - if business != set(TABLES): + if not set(TABLES) <= business or business - set(TABLES) - set(OPTIONAL_TABLES): raise ValueError('Unexpected business tables: ' + repr(sorted(business ^ set(TABLES)))) result = {} - for table in TABLES: + for table in (*TABLES, *(name for name in OPTIONAL_TABLES if name in business)): columns = [{'name': r[1], 'type': r[2].upper(), 'notnull': bool(r[3]), 'default': r[4], 'pk': r[5]} for r in connection.execute('PRAGMA table_info(' + quote(table) + ')')] diff --git a/paper_atlas/querying.py b/paper_atlas/querying.py index cc82979..a0f60c9 100644 --- a/paper_atlas/querying.py +++ b/paper_atlas/querying.py @@ -29,6 +29,8 @@ def main(argv=None, *, prog=None): parser.add_argument('--pg-schema', help='Optional PostgreSQL schema override; otherwise use the configured search_path') parser.add_argument('--query', help='Title/abstract search: FTS5 for SQLite, web-search syntax for PostgreSQL') parser.add_argument('--paper',type=int,help='Paper ID; include sources, tags and submission history') + parser.add_argument('--research-type', help='Independent type filter as namespace=value; no type inferred from track') + parser.add_argument('--annotation-history', action='store_true', help='Include retraction evidence for --paper') parser.add_argument('--tag', help='Filter as namespace=value, e.g. arxiv:category=cs.AI') parser.add_argument('--venue', help='Conference acronym') parser.add_argument('--year', type=int) @@ -47,6 +49,8 @@ def main(argv=None, *, prog=None): conn = sqlite3.connect(Path(args.db).resolve().as_uri()+'?mode=ro', uri=True) conn.row_factory = sqlite3.Row conn.execute('BEGIN') # One consistent read snapshot while ingestion continues. + from paper_atlas.curation.annotations import available, observations, ACTIVE + has_annotations = available(conn) if args.stats: result = {name:conn.execute('SELECT count(*) FROM '+name).fetchone()[0] for name in ('papers','source_records','submissions','decisions','tags','match_candidates')} result['records_by_source'] = [dict(r) for r in conn.execute('SELECT s.name,count(r.id) AS imported_records FROM sources s LEFT JOIN source_records r ON r.source_id=s.id GROUP BY s.id ORDER BY s.name')] @@ -69,8 +73,24 @@ def main(argv=None, *, prog=None): where.append('p.id IN (SELECT rowid FROM papers_fts WHERE papers_fts MATCH ?)') params.append(args.query) if args.tag: + if '=' not in args.tag: + parser.error('--tag requires namespace=value') namespace,value = args.tag.split('=',1) - where.append('EXISTS(SELECT 1 FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id WHERE pt.paper_id=p.id AND t.namespace=? AND t.value=?)'); params.extend([namespace,value]) + platform = 'EXISTS(SELECT 1 FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id WHERE pt.paper_id=p.id AND t.namespace=? AND t.value=?)' + params.extend([namespace,value]) + if has_annotations: + platform = '(' + platform + ' OR EXISTS(SELECT 1 FROM annotation_events a WHERE a.paper_id=p.id AND a.kind=\'tag\' AND a.namespace=? AND a.value=? AND ' + ACTIVE + '))' + params.extend([namespace,value]) + where.append(platform) + if args.research_type: + if '=' not in args.research_type: + parser.error('--research-type requires namespace=value') + namespace,value = args.research_type.split('=',1) + if has_annotations: + where.append('EXISTS(SELECT 1 FROM annotation_events a WHERE a.paper_id=p.id AND a.kind=\'research_type\' AND a.namespace=? AND a.value=? AND ' + ACTIVE + ')') + params.extend([namespace,value]) + else: + where.append('1=0') if args.outcome: where.append('p.id '+('IN' if args.outcome=='accepted' else 'NOT IN')+' ('+ACCEPTED_PAPERS+')') if args.venue or args.year or args.status or args.track: @@ -98,6 +118,10 @@ def main(argv=None, *, prog=None): paper['known_acceptances']=known_acceptances(conn,row['id']) paper['publication_outcome']='accepted' if paper['known_acceptances'] else 'no_known_acceptance' if args.paper is not None: + paper['research_types'] = observations(conn, row['id'], kind='research_type') + paper['annotation_tags'] = observations(conn, row['id'], kind='tag') + if args.annotation_history: + paper['annotation_history'] = observations(conn, row['id'], history=True) paper['tags'] = [dict(r) for r in conn.execute('SELECT t.namespace,t.value,pt.evidence_key FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id WHERE pt.paper_id=?',(row['id'],))] paper['sources'] = [dict(r) for r in conn.execute('SELECT s.name,r.external_id,r.url,r.title,r.abstract,r.retrieved_at FROM source_records r JOIN sources s ON s.id=r.source_id WHERE r.paper_id=?',(row['id'],))] submissions = [] diff --git a/pyproject.toml b/pyproject.toml index 0357a1c..9655ba8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,5 +21,5 @@ include = ["paper_atlas*"] exclude = ["tests*"] [tool.setuptools.package-data] -"paper_atlas.db" = ["schema.sql"] +"paper_atlas.db" = ["schema.sql", "annotations.sql"] "paper_atlas.resources" = ["venues.json"] diff --git a/tests/test_annotations.py b/tests/test_annotations.py new file mode 100644 index 0000000..2f4dcb7 --- /dev/null +++ b/tests/test_annotations.py @@ -0,0 +1,219 @@ +from contextlib import closing, redirect_stdout +import io +import json +import os +from pathlib import Path +import tempfile +import unittest +import uuid + +from paper_atlas import querying +from paper_atlas.curation.annotations import (assert_annotation, available, initialize, main, + observations, retract_annotation) +from paper_atlas.db.backend import open_store +from paper_atlas.db.store import Store +from paper_atlas.postgres import schema +from paper_atlas.postgres.client import connect +from paper_atlas.postgres.migration import migrate + + +class AnnotationTests(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory(); self.addCleanup(self.tmp.cleanup) + self.db = Path(self.tmp.name) / 'sample.sqlite' + self.store = Store(self.db); self.addCleanup(self.store.conn.close) + self.c = self.store.conn + self.paper, self.record = self.store.record('fixture', 'one', 'https://example.org/one', 'First research', None, []) + self.other, self.other_record = self.store.record('fixture', 'two', 'https://example.org/two', 'Other research', None, []) + self.store.record_tags(self.paper, self.record, 'fixture', [{'namespace':'fixture:topic','value':'original'}]) + self.c.commit() + + def add(self, **changes): + args = dict(paper=self.paper, kind='tag', namespace='custom:topic', value='reading', + request_id='request-one', evidence_key='manual:reading', reason='Reviewed metadata', actor='reviewer') + args.update(changes) + return assert_annotation(self.c, **args) + + def query(self, *args): + self.c.commit() + with redirect_stdout(io.StringIO()) as output: + querying.main(['--db', str(self.db), *map(str, args)]) + return json.loads(output.getvalue()) + + def test_read_does_not_install_extension_and_write_requires_explicit_init(self): + self.assertFalse(available(self.c)) + self.assertEqual(observations(self.c, self.paper), []) + with self.assertRaisesRegex(RuntimeError, 'init explicitly'): + self.add() + self.assertEqual(self.query('--research-type', 'custom:method=empirical'), []) + self.assertFalse(available(self.c)) + self.assertEqual(len(schema.inspect(self.c)), 21) + + def test_schema_upgrade_is_transactional_and_marker_mismatch_fails_closed(self): + self.assertTrue(initialize(self.c)); self.c.rollback() + self.assertFalse(available(self.c)) + self.c.execute("INSERT INTO schema_migrations(name) VALUES('annotation_events_v1')") + with self.assertRaisesRegex(RuntimeError, 'mismatch'): + initialize(self.c) + self.c.rollback() + self.assertTrue(initialize(self.c)); self.assertFalse(initialize(self.c)) + self.assertEqual(len(schema.inspect(self.c)), 22) + + def test_types_tags_and_platform_labels_are_independent_and_queryable(self): + initialize(self.c) + self.add() + self.add(kind='research_type', namespace='custom:topic', request_id='type', value='reading') + self.assertEqual(len(self.query('--tag', 'custom:topic=reading')), 1) + self.assertEqual(len(self.query('--research-type', 'custom:topic=reading')), 1) + self.assertEqual(len(self.query('--tag', 'fixture:topic=original')), 1) + detail = self.query('--paper', self.paper)[0] + self.assertEqual(len(detail['annotation_tags']), 1) + self.assertEqual(len(detail['research_types']), 1) + self.assertNotIn('annotation_history', detail) + self.assertEqual(detail['tags'][0]['namespace'], 'fixture:topic') + + def test_repeated_request_is_idempotent_and_changed_content_fails(self): + initialize(self.c); event = self.add() + self.assertEqual(self.add(), event) + for change in ({'value':'different'}, {'kind':'research_type'}, {'reason':'changed'}, {'paper':self.other}): + with self.assertRaisesRegex(ValueError, 'different annotation'): + self.add(**change) + self.assertEqual(self.c.execute('SELECT count(*) FROM annotation_events').fetchone()[0], 1) + + def test_retraction_keeps_original_and_other_assertions_of_same_label(self): + initialize(self.c); first = self.add(); second = self.add(request_id='second', evidence_key='independent') + kwargs = dict(request_id='retract', reason='No longer relevant', actor='reviewer') + removed = retract_annotation(self.c, first, **kwargs) + self.assertEqual(retract_annotation(self.c, first, **kwargs), removed) + self.assertEqual([r['id'] for r in observations(self.c, self.paper)], [second]) + self.assertEqual(len(self.query('--paper', self.paper, '--annotation-history')[0]['annotation_history']), 3) + with self.assertRaisesRegex(ValueError, 'already retracted'): + retract_annotation(self.c, first, **{**kwargs,'request_id':'duplicate'}) + with self.assertRaisesRegex(ValueError, 'existing assertion'): + retract_annotation(self.c, removed, **{**kwargs,'request_id':'nested'}) + + def test_source_assertions_require_matching_paper_and_platform(self): + initialize(self.c) + for change in ({'namespace':'fixture:type'}, {'namespace':'other:type','source_record_id':self.record}, + {'namespace':'fixture:type','source_record_id':self.other_record}): + with self.assertRaises(ValueError): + self.add(**change) + event = self.add(kind='research_type', namespace='fixture:declared_type', source_record_id=self.record) + with self.assertRaisesRegex(ValueError, 'Manual CLI'): + retract_annotation(self.c, event, request_id='cannot', reason='x', actor='y', custom_only=True) + self.assertEqual(observations(self.c, self.paper)[0]['source_record_id'], self.record) + + def test_platform_tag_refresh_preserves_both_custom_kinds(self): + initialize(self.c); self.add(); self.add(kind='research_type', request_id='type') + self.store.record_tags(self.paper, self.record, 'fixture', [{'namespace':'fixture:topic','value':'refreshed'}]) + self.assertEqual(len(observations(self.c, self.paper)), 2) + self.assertEqual(len(self.query('--tag','fixture:topic=refreshed')), 1) + self.assertEqual(self.query('--tag','fixture:topic=original'), []) + + def test_merge_preserves_retractions_evidence_and_original_paper_ids(self): + initialize(self.c); event = self.add(source_record_id=self.record) + removed = retract_annotation(self.c, event, request_id='withdraw', reason='corrected', actor='reviewer') + active = self.add(request_id='active', kind='research_type', value='survey') + self.store.merge(self.other, self.paper, 'Independently reviewed identity') + rows = observations(self.c, self.other, history=True) + self.assertEqual([r['id'] for r in rows], [event, removed, active]) + self.assertEqual({r['original_paper_id'] for r in rows}, {self.paper}) + self.assertEqual({r['paper_id'] for r in rows}, {self.other}) + self.assertEqual([r['id'] for r in observations(self.c, self.other)], [active]) + self.assertEqual(self.add(source_record_id=self.record), event) # exact retry survives canonical merge + self.assertEqual(self.c.execute('PRAGMA foreign_key_check').fetchall(), []) + + def test_invalid_input_does_not_create_events(self): + initialize(self.c) + for change in ({'paper':True}, {'paper':999999}, {'kind':'main'}, {'namespace':'arxiv'}, + {'namespace':'custom:topic;DROP'}, {'value':' '}, {'reason':''}, + {'actor':''}, {'request_id':''}, {'evidence_key':''}, {'source_record_id':True}, + {'evidence_url':'file:///etc/passwd'}, {'evidence_url':'https://user:secret@example.org/'}): + with self.subTest(change=change), self.assertRaises(ValueError): + self.add(**change) + self.assertEqual(self.c.execute('SELECT count(*) FROM annotation_events').fetchone()[0], 0) + + def test_cli_manual_scope_and_complete_assert_retract_lifecycle(self): + with redirect_stdout(io.StringIO()): + main(['init','--db',str(self.db)]) + main(['assert','--db',str(self.db),'--paper',str(self.paper),'--kind','research_type', + '--namespace','custom:method','--value','empirical','--evidence-key','manual:review', + '--request-id','cli-one','--reason','Read abstract','--actor','reviewer']) + self.assertEqual(len(self.query('--research-type','custom:method=empirical')), 1) + event = observations(self.c, self.paper)[0]['id'] + with redirect_stdout(io.StringIO()): + main(['retract','--db',str(self.db),'--event',str(event),'--request-id','cli-two', + '--reason','Corrected classification','--actor','reviewer']) + self.assertEqual(self.query('--research-type','custom:method=empirical'), []) + with redirect_stdout(io.StringIO()) as out: + main(['list','--db',str(self.db),'--paper',str(self.paper),'--history']) + self.assertEqual(len(json.loads(out.getvalue())), 2) + + +@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires approved remote PostgreSQL service') +class PostgreSQLAnnotationTests(unittest.TestCase): + def setUp(self): + self.service = os.environ['PAPER_ATLAS_TEST_PG_SERVICE'] + self.target = 'pa_anno_' + uuid.uuid4().hex[:12] + self.tmp = tempfile.TemporaryDirectory(); self.addCleanup(self.tmp.cleanup) + self.db = Path(self.tmp.name) / 'source.sqlite' + self.s = Store(self.db) + self.paper, self.record = self.s.record('fixture','one','https://example.org','Paper',None,[]) + self.s.conn.commit() + self.addCleanup(self.s.conn.close); self.addCleanup(self.cleanup) + + def cleanup(self): + with connect(self.service) as c: + for name, in c.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)', (self.target,)): + c.execute('DROP SCHEMA ' + schema.quote(name) + ' CASCADE') + + def migrate(self): + self.s.conn.commit() + with redirect_stdout(io.StringIO()): + return migrate(self.db, self.service, Path(self.tmp.name)/'report.json', schema=self.target, min_free_gib=0) + + def args(self, **changes): + a = dict(paper=self.paper,kind='research_type',namespace='custom:method',value='survey\x00and\x01', + request_id='native',evidence_key='manual',reason='Reviewed\x00evidence',actor='reviewer') + a.update(changes); return a + + def test_native_schema_upgrade_roundtrip_retraction_and_queries(self): + self.migrate() + with closing(open_store(pg_service=self.service, pg_schema=self.target).conn) as c: + self.assertTrue(initialize(c)); self.assertFalse(initialize(c)) + event = assert_annotation(c, **self.args()); c.commit() + self.assertEqual(assert_annotation(c, **self.args()), event) + self.assertEqual(observations(c,self.paper)[0]['value'], 'survey\x00and\x01') + c.commit() + with redirect_stdout(io.StringIO()) as out: + querying.main(['--pg-service',self.service,'--pg-schema',self.target,'--paper',str(self.paper)]) + self.assertEqual(json.loads(out.getvalue())[0]['research_types'][0]['id'], event) + with closing(open_store(pg_service=self.service,pg_schema=self.target).conn) as c: + retract_annotation(c,event,request_id='removed',reason='Changed',actor='reviewer');c.commit() + self.assertEqual(observations(c,self.paper), []) + self.assertEqual(len(observations(c,self.paper,history=True)), 2) + + def test_initialized_sqlite_extension_migrates_all_rows_hashes_and_identity(self): + initialize(self.s.conn) + event = assert_annotation(self.s.conn, **self.args()) + retract_annotation(self.s.conn,event,request_id='retracted',reason='Corrected',actor='reviewer') + report = self.migrate() + self.assertEqual(len(report['tables']),22) + self.assertEqual(report['tables']['annotation_events']['source_rows'],2) + self.assertEqual(report['tables']['annotation_events']['source_sha256'],report['tables']['annotation_events']['target_sha256']) + with closing(open_store(pg_service=self.service,pg_schema=self.target).conn) as c: + new = assert_annotation(c, **self.args(request_id='after-migration')) + self.assertGreater(new,2); c.commit() + self.assertEqual(len(observations(c,self.paper,history=True)),3) + + def test_native_upgrade_rollback_keeps_legacy_schema_usable(self): + self.migrate() + with closing(open_store(pg_service=self.service,pg_schema=self.target).conn) as c: + initialize(c); c.rollback() + self.assertFalse(available(c)) + self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],1) + self.assertTrue(initialize(c)) + with self.assertRaises(ValueError): + assert_annotation(c, **self.args(namespace='wrong:type',source_record_id=self.record)) + self.assertEqual(c.execute('SELECT count(*) FROM annotation_events').fetchone()[0],0) + c.rollback()