Freeze SQLite annotation fixtures before PostgreSQL migration

Reviewed-on: #17

.gitea/workflows/check.yml

@@ -53,8 +53,12 @@ jobs:
               subprocess.run([python,'-I','-B','-m','paper_atlas','import','arxiv','--help'],cwd=outside,check=True)
               subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','backfill','--help'],cwd=outside,check=True)
               subprocess.run([python,'-I','-B','-m','paper_atlas','audit','public-openreview','--help'],cwd=outside,check=True)
+              subprocess.run([python,'-I','-B','-m','paper_atlas','audit','histories','--help'],cwd=outside,check=True)
               subprocess.run([python,'-I','-B','-c','from paper_atlas.postgres.writer import PostgresStore, WriteConnection'],cwd=outside,check=True)
               smoke = "from paper_atlas.db.store import Store; from paper_atlas.config import DEFAULT_REGISTRY; import json; assert {'COLM','MM','MLSys'} <= {v['acronym'] for v in json.loads(DEFAULT_REGISTRY.read_text())['venues']}; s=Store('installed.sqlite'); s.conn.close()"
               subprocess.run([python,'-I','-B','-c',smoke],cwd=outside,check=True)
               subprocess.run([python,'-I','-B','-m','paper_atlas','query','--db','installed.sqlite','--stats'],cwd=outside,check=True)
+              subprocess.run([python,'-I','-B','-m','paper_atlas','annotate','init','--db','installed.sqlite'],cwd=outside,check=True)
+              subprocess.run([python,'-I','-B','-m','paper_atlas','annotate','list','--db','installed.sqlite','--paper','1'],cwd=outside,check=True)
+              subprocess.run([python,'-I','-B','-m','paper_atlas','audit','histories','--db','installed.sqlite','--examples','0'],cwd=outside,check=True)
           PY

README.md

@@ -86,3 +86,5 @@ paper-atlas collect colm --cache /data/paper-index/raw/http-cache \
 CI 在已授权的远程 runner 上运行测试、构建 wheel，并从源码目录外验证安装后的命令及随包资源。数据库全量去重和覆盖率是独立的数据验收步骤，不因代码测试通过而自动完成。
 
 开发通过工作分支和 PR 交付，由用户手动合并。
+
+研究类型与自定义标签可通过 [`annotate`](docs/annotations.md) 显式初始化、添加和撤回；两者独立查询，保留证据及完整更正历史，不从会议轨道推断研究类型。

docs/annotations.md

@@ -0,0 +1,107 @@
+# Research types and custom annotations
+
+Research type describes a claimed characteristic of the work (for example a survey,
+benchmark, empirical study or replication). It is independent of topic, paper length,
+conference track, presentation format and acceptance status. No type is inferred from
+those fields. Multiple source-specific or human classifications may coexist; no
+cross-platform vocabulary mapping or automatic classifier is provided.
+
+`annotation_events` stores both kinds with an explicit `kind` discriminator:
+`research_type` and `tag` are separate domains, even when namespace/value strings
+happen to coincide. Existing platform labels remain in `tags` / `paper_tags`.
+Each assertion carries namespace, value, actor, reason, evidence key, optional source
+record and evidence URL, and the time it was recorded. This is observation time, not
+paper publication time or a conference decision date.
+
+Values are not forced into a global taxonomy: `custom:method=empirical` and
+`platform:declared_type=empirical` retain separate provenance. An explicit platform
+assertion API requires a source record belonging to that paper and matching the
+namespace prefix. The manual CLI only accepts `custom:*`, and cannot retract platform
+assertions. No source type has been automatically assigned to existing papers.
+
+## Initialize explicitly
+
+On the published PostgreSQL dataset, using its protected service file:
+
+```sh
+PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas annotate init \
+  --pg-service foundry-research --pg-schema research
+```
+
+This creates one optional table, its indexes, and an upgrade marker in one transaction,
+using the existing dataset writer lock. It does not copy data, change existing papers
+or decisions, or create a PostgreSQL instance. Repeating it is safe; an unexplained
+pre-existing table/marker mismatch fails closed. Readers and ordinary imports do not
+install the extension. Legacy datasets remain readable. A SQLite **sample** uses
+`--db sample.sqlite`; the cutover SQLite backup remains protected against writes.
+
+## Add and retract evidence
+
+Use the canonical paper ID returned by `query`. For a reviewed classification:
+
+```sh
+paper-atlas annotate assert --pg-service foundry-research --pg-schema research \
+  --paper "$PAPER_ID" --kind research_type --namespace custom:method \
+  --value empirical --evidence-key manual:abstract-review \
+  --actor reviewer --reason 'Classified from the explicitly described study design' \
+  --request-id "$REQUEST_ID"
+```
+
+`$REQUEST_ID` must be a new, stable identifier for this action (for example a UUID).
+Retry with the same request ID and identical content after an interrupted response;
+it returns the original event ID. Reusing that ID with changed content is rejected.
+A topic annotation uses `--kind tag --namespace custom:topic --value robotics` instead.
+Optional `--source-record` and `--evidence-url` attach additional evidence; the source
+record must belong to this paper. URLs are retained as evidence and are not fetched.
+
+To withdraw one particular assertion, use its returned event ID:
+
+```sh
+paper-atlas annotate retract --pg-service foundry-research --pg-schema research \
+  --event "$EVENT_ID" --actor reviewer --reason 'Classification corrected after review' \
+  --request-id "$RETRACTION_REQUEST_ID"
+```
+
+Retraction adds an event referencing that assertion, preserving its original content.
+It does not remove another contributor's assertion of the same label, platform tags,
+source metadata or submission/decision history. A retracted assertion can be asserted
+again with a new request ID and evidence; it is a new observation. The API does not
+update or delete past annotation events. Database administrators still control direct
+SQL access. Work reviewed as the same paper may subsequently be merged by the existing
+identity workflow: annotation IDs, original paper IDs, evidence and retraction links
+are retained while the canonical paper reference follows the surviving entity.
+
+## Query and analyze
+
+```sh
+paper-atlas query --pg-service foundry-research --pg-schema research \
+  --research-type custom:method=empirical
+paper-atlas query --pg-service foundry-research --pg-schema research \
+  --tag custom:topic=robotics
+paper-atlas query --pg-service foundry-research --pg-schema research \
+  --paper "$PAPER_ID" --annotation-history
+paper-atlas annotate list --pg-service foundry-research --pg-schema research \
+  --paper "$PAPER_ID" --history
+```
+
+`--tag` includes original platform labels and active annotation tags. It never matches
+research types. `--research-type` matches active type assertions only. Paper details
+return separate `tags` (existing provenance), `annotation_tags`, and `research_types`;
+`--annotation-history` additionally includes retractions. A missing extension yields
+empty annotation results rather than changing the database.
+
+For direct SQL analysis, active research-type evidence is:
+
+```sql
+SELECT a.* FROM research.annotation_events a
+WHERE a.kind = 'research_type' AND a.retracts_id IS NULL
+  AND NOT EXISTS (SELECT 1 FROM research.annotation_events r WHERE r.retracts_id = a.id);
+```
+
+Several independent assertions can support one label; count distinct papers/labels
+when appropriate rather than counting evidence rows as papers. Namespaces stay
+separate. Standard paper queries decode the PostgreSQL text codec; direct SQL consumers
+must honor the dataset's documented codec, including NUL/SOH values. SQLite-to-PostgreSQL
+migration recognizes an installed annotation extension and verifies its full row hashes,
+constraints and identity sequence along with the original business tables. An old
+21-table snapshot stays a 21-table migration; no synthetic annotations are generated.

docs/data-sources.md

@@ -93,12 +93,6 @@ paper-atlas audit public-openreview --db /data/paper-index/papers.sqlite \
 
 输出按会议、届次及 main、position、datasets_benchmarks 分文件。workshop 默认进入单独的 held 清单；有明确 workshop 名称时可通过 `--include-workshops` 单列采集，不合并到主会。Retraction、未映射的轨道或负面标签也保留在 `held-records.jsonl`，不强改为 Reject 或 Withdrawn。当前决定表没有独立 retracted 状态，因此撤回发表记录仍需后续专门建模。
 
-`parser_version=2` 还会将明确负面标签但标题缺失、空白或 null 的记录放入 `held-records.jsonl`，保留原始状态、forum ID、固定来源版本和行号；不制造标题或导入不可识别的论文，也不让这一条缺失阻断其他可用记录。held 文件哈希与各原因数量保存在 inventory；非文本标题、错误身份、冲突届次和哈希不符仍使整批失败。旧年 ICLR 镜像的明确 `Withdraw` 标签归为 `withdrawn`，`Workshop`、`Poster Workshop`、`Oral Workshop` 均不作为拒稿。采集文件与导入主库仍是两个独立步骤。
-
-`parser_version=3` 增加五组已核实身份的 ICLR workshop 标签：2024 ME-FoMo / GenAI4DM 的 Withdrawn Submission、2025 AgenticAI 的 Reject / LMRL 的 Withdrawn Submission，以及 2026 LLM Reasoning 的 Withdrawn Submission。仅在 `--include-workshops` 下输出，并分别保存 `workshop_me_fomo`、`workshop_genai4dm`、`workshop_agenticai`、`workshop_lmrl`、`workshop_llm_reasoning`；默认仍暂存。标签必须与原文件会议、年份和原始 workshop track 一致，错年/错会议使采集失败，错轨道单独暂存，未知标签继续待核查，不用宽泛的 Reject/Withdraw 正则猜测。
-
-[OpenReview 官方 ICLR 活动目录](https://openreview.net/venue?id=ICLR.cc) 确认上述独立 workshop 身份；[ME-FoMo 2024](https://sites.google.com/view/me-fomo2024/call-for-papers)、[GenAI4DM 2024](https://sites.google.com/view/genai4dm-iclr2024)、[AgenticAI 2025](https://iclragenticai.github.io/) 和 [LLM Reasoning 2026](https://sites.google.com/view/iclr-2026-llmreasoning) 的活动网站也提供对应投稿入口。输出在 metadata 保存 `workshop_venueid` 和身份目录证据，决定仍只标为固定镜像的公开分类断言，保留原始标签、forum、来源行号与哈希，不声称核验原始决定签名。2026-11-08 核查时部分原站记录页要求浏览器验证；身份目录和主办方入口可读，不代表逐篇原始决定已可访问，也不外推其他 workshop 的公开政策。采集后仍须经过 PostgreSQL 身份/决定冲突预检、串行导入和独立验收。
-
 该来源明确标记为 `curated_public_status_assertion`：保留源 commit、文件哈希、行号、原标签和 forum ID，不声称已核验原始 PC 签名。只保存标题、摘要、公开作者名、关键词、领域、索引与决定来源，不缓存镜像中的个人资料、邮箱、性别、评审内容或 PDF。所有输出清单均为 partial；盘点中的零条记录仅描述这个固定镜像。
 
 入库前审计核验文件哈希、数量、轨道和来源声明，并用真实 forum ID 对比同届同轨道已有投稿。存在不同决定时阻止整批导入，交由来源和时序核查；另一会议的后续录用属于独立投稿历史，不构成该次拒稿的冲突。同一 forum 的镜像不新增投稿事件，但论文实体的跨平台去重仍须在全量结束后核验。审计通过的报告包含带 SHA-256 的导入队列；应在单写入维护锁下及时串行导入，主库变化后重新审计。

docs/usage.md

@@ -79,10 +79,10 @@ paper-atlas maintain merge --db /data/paper-index/samples/example.sqlite \
 ## 4. 查询与分析前检查
 
 ```sh
-paper-atlas query --db /data/paper-index/papers.sqlite --query '"language model"' --limit 20
-paper-atlas query --db /data/paper-index/papers.sqlite --tag arxiv:category=cs.AI
-paper-atlas query --db /data/paper-index/papers.sqlite --venue ICLR --year 2025 --status rejected
-paper-atlas query --db /data/paper-index/papers.sqlite --paper 123
+paper-atlas query --pg-service foundry-research --query '"language model"' --limit 20
+paper-atlas query --pg-service foundry-research --tag arxiv:category=cs.AI
+paper-atlas query --pg-service foundry-research --venue ICLR --year 2025 --status rejected
+paper-atlas query --pg-service foundry-research --paper 123
 paper-atlas coverage --db /data/paper-index/papers.sqlite --out /data/paper-index/collected/coverage.json
 ```
 
@@ -91,10 +91,26 @@ paper-atlas coverage --db /data/paper-index/papers.sqlite --out /data/paper-inde
 例如，查找在 ICLR 有拒稿记录、但在其他已知投稿中已有录用的论文：
 
 ```sh
-paper-atlas query --db /data/paper-index/papers.sqlite \\
+paper-atlas query --pg-service foundry-research \
   --venue ICLR --status rejected --outcome accepted
 ```
 
 `known_acceptances` 列出目前仍为录用状态的各次投稿及会议、年份、track 和决定证据，`publication_outcome` 汇总为 accepted 或 no_known_acceptance。汇总会检查所有会议和 track，不把某次拒稿覆盖到整篇论文；同一次投稿的后续撤稿也会更新该次状态并保留原决定。no_known_acceptance 只表示现有来源中没有查到录用，不能称为“从未录用”。
 
 `--stats` 中 papers 是当前规则下的论文实体数；source_records 是来源记录数；decisions 是决定观察数。一个投稿可能有多个来源和多条观察，不能相加当作论文数。
+
+## 5. 投稿历史关系审计
+
+```sh
+PGSERVICEFILE=/etc/paper-atlas/pg_service.conf paper-atlas audit histories \
+  --pg-service foundry-research --pg-schema research --examples 20 \
+  --out /data/paper-index/collected/history-audit.json
+```
+
+使用一个只读一致性快照，区分论文实体、投稿和决定观察；分别统计同次投稿曾出现负面与录用观察、不同投稿共存负面与当前已知录用、不同会议之间的共存。workshop/main 是不同 track，属于同一会议时不算跨会议；会议别名也不会增加一个会议。当前已知录用使用与 `query --outcome` 相同的“每次投稿最新观察”规则，后续撤稿的该次投稿不再算当前录用，但旧录用决定仍保留。
+
+报告不凭观察时间、届次年份或行顺序断言“拒稿发生在录用之前”。样例保留真实 `decision_date`、`observed_at`、原始标签、证据 URL 和决定来源，供进一步核查；日期缺失仍保持未知。`negative` 统计包含 rejected、desk_rejected 和 withdrawn，其原始状态不互相替换。
+
+总数覆盖快照内全部投稿，样例由 `--examples` 限定为 0–100 篇；每篇最多展示 20 次投稿、每次最多 50 条决定、每条决定最多 100 个来源，并明确标记截断。样例优先保留负面/录用的不同投稿对。四项来源关系检查报告缺失关联和来源论文归属不一致，不自动修复。
+
+这些统计依赖现有论文身份关联，不证明实体合并正确，也不证明原始决定签名、公开拒稿全集、标签类型或最终全库去重已完成。SQLite 独立样本可使用 `--db`；冻结备份不能代表当前 PostgreSQL 的历史情况。

paper_atlas/audits/histories.py

@@ -0,0 +1,196 @@
+"""Audit linked submission histories without inferring event chronology."""
+import argparse
+from collections import Counter
+import json
+from pathlib import Path
+import sqlite3
+
+from paper_atlas.db.backend import database_arguments, open_read_connection
+from paper_atlas.db.submissions import timestamp
+from paper_atlas.querying import LATEST_DECISION
+
+
+ATTEMPTS = '''SELECT s.paper_id,s.id AS submission_id,e.venue_id,v.acronym AS venue,e.year,e.track,
+    count(d.id) AS decision_observations,
+    MAX(CASE WHEN d.status IN ('rejected','desk_rejected','withdrawn') THEN 1 ELSE 0 END)
+        AS historical_negative,
+    MAX(CASE WHEN d.status='accepted' THEN 1 ELSE 0 END) AS historical_acceptance,
+    COALESCE(MAX(CASE WHEN ''' + LATEST_DECISION + ''' THEN d.status END),'unknown')
+        AS latest_observed_status
+    FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id
+    LEFT JOIN decisions d ON d.submission_id=s.id
+    GROUP BY s.paper_id,s.id,e.venue_id,v.acronym,e.year,e.track
+    ORDER BY s.paper_id,e.venue_id,s.id'''
+
+
+def _paper_counts(attempts):
+    negative = sum(row['historical_negative'] for row in attempts)
+    accepted = sum(row['latest_observed_status'] == 'accepted' for row in attempts)
+    overlap = sum(row['historical_negative'] and row['latest_observed_status'] == 'accepted'
+                  for row in attempts)
+    venues = {}
+    for row in attempts:
+        counts = venues.setdefault(row['venue_id'], [0, 0])
+        counts[0] += row['historical_negative']
+        counts[1] += row['latest_observed_status'] == 'accepted'
+    return {
+        'papers_with_submissions': 1,
+        'papers_with_multiple_submissions': int(len(attempts) > 1),
+        'papers_with_historical_negative': int(negative > 0),
+        'papers_with_current_known_acceptance': int(accepted > 0),
+        'papers_with_negative_and_acceptance_in_distinct_attempts': int(negative * accepted > overlap),
+        'papers_with_negative_and_acceptance_in_distinct_venues':
+            int(negative * accepted > sum(n * a for n, a in venues.values())),
+    }
+
+
+def _example(connection, attempts):
+    # Keep a negative/accepted pair visible even when a paper has many attempts.
+    accepted = [row for row in attempts if row['latest_observed_status'] == 'accepted']
+    first = accepted[0] if accepted else None
+    other_venue = next((row for row in accepted if row['venue_id'] != first['venue_id']), None)
+    pair = None
+    for negative in attempts:
+        if negative['historical_negative'] and first:
+            candidate = first if first['venue_id'] != negative['venue_id'] else other_venue
+            if candidate:
+                pair = (negative, candidate)
+                break
+    if pair is None:
+        for negative in attempts:
+            if negative['historical_negative'] and first:
+                candidate = first if first['submission_id'] != negative['submission_id'] else (
+                    accepted[1] if len(accepted) > 1 else None)
+                if candidate:
+                    pair = (negative, candidate)
+                    break
+    selected = {}
+    for row in list(pair or ()) + attempts:
+        if len(selected) < 20:
+            selected.setdefault(row['submission_id'], dict(row))
+    for row in selected.values():
+        row['decisions'] = []
+        for decision in connection.execute('''SELECT id,status,raw_value,evidence_url,
+                decision_date,observed_at FROM decisions WHERE submission_id=?
+                ORDER BY observed_at DESC,COALESCE(decision_date,observed_at) DESC,id DESC LIMIT 50''',
+                (row['submission_id'],)):
+            entry = dict(decision)
+            sources = [dict(source) for source in connection.execute('''
+                SELECT so.name,r.external_id,ds.source_decision_id FROM decision_sources ds
+                JOIN source_records r ON r.id=ds.source_record_id JOIN sources so ON so.id=r.source_id
+                WHERE ds.decision_id=? ORDER BY so.name,r.external_id LIMIT 101''', (entry['id'],))]
+            entry['sources'] = sources[:100]
+            entry['sources_truncated'] = len(sources) > 100
+            row['decisions'].append(entry)
+        row['decisions_truncated'] = row['decision_observations'] > len(row['decisions'])
+    paper_id = attempts[0]['paper_id']
+    return {'paper_id': paper_id,
+            'title': connection.execute('SELECT title FROM papers WHERE id=?', (paper_id,)).fetchone()[0],
+            'counts': _paper_counts(attempts), 'attempt_count': len(attempts),
+            'attempts': list(selected.values()), 'attempts_truncated': len(attempts) > len(selected),
+            'chronology': 'Not inferred from observed_at, edition year, or row order.'}
+
+
+def report(db=None, *, pg_service=None, pg_schema=None, examples=20):
+    if not 0 <= examples <= 100:
+        raise ValueError('Examples must be between 0 and 100')
+    started_at = timestamp()
+    connection = open_read_connection(db, pg_service=pg_service, pg_schema=pg_schema)
+    if not pg_service:
+        connection.row_factory = sqlite3.Row
+    try:
+        if pg_service:
+            identity = connection.execute('SELECT current_database(),current_schema()').fetchone()
+            database, schema = identity[0], identity[1]
+        else:
+            database, schema = str(Path(db).resolve()), None
+        names = ['papers_with_submissions', 'papers_with_multiple_submissions',
+                 'papers_with_historical_negative', 'papers_with_current_known_acceptance',
+                 'papers_with_negative_and_acceptance_in_distinct_attempts',
+                 'papers_with_negative_and_acceptance_in_distinct_venues']
+        totals = Counter({key: 0 for key in names})
+        totals.update(submissions=0, decision_observations=0,
+                      attempts_with_historical_negative=0,
+                      attempts_with_historical_negative_and_acceptance=0)
+        latest = Counter()
+        sample_attempts = []
+
+        def consume(rows):
+            if not rows:
+                return
+            counts = _paper_counts(rows)
+            totals.update(counts)
+            if counts['papers_with_negative_and_acceptance_in_distinct_attempts'] and len(sample_attempts) < examples:
+                sample_attempts.append(rows)
+
+        current = []
+        for raw in connection.execute(ATTEMPTS):
+            row = dict(raw)
+            if current and row['paper_id'] != current[0]['paper_id']:
+                consume(current)
+                current = []
+            current.append(row)
+            totals['submissions'] += 1
+            totals['decision_observations'] += row['decision_observations']
+            totals['attempts_with_historical_negative'] += row['historical_negative']
+            totals['attempts_with_historical_negative_and_acceptance'] += int(
+                row['historical_negative'] and row['historical_acceptance'])
+            latest[row['latest_observed_status']] += 1
+        consume(current)
+        # These are relationship checks, not a substitute for the final entity/FK audit.
+        integrity = {}
+        checks = {
+            'submissions_without_source_links': '''SELECT count(*) FROM submissions s
+                WHERE NOT EXISTS(SELECT 1 FROM submission_sources ss WHERE ss.submission_id=s.id)''',
+            'decisions_without_source_links': '''SELECT count(*) FROM decisions d
+                WHERE NOT EXISTS(SELECT 1 FROM decision_sources ds WHERE ds.decision_id=d.id)''',
+            'submission_source_paper_mismatches': '''SELECT count(*) FROM submission_sources ss
+                JOIN submissions s ON s.id=ss.submission_id JOIN source_records r ON r.id=ss.source_record_id
+                WHERE s.paper_id<>r.paper_id''',
+            'decision_source_paper_mismatches': '''SELECT count(*) FROM decision_sources ds
+                JOIN decisions d ON d.id=ds.decision_id JOIN submissions s ON s.id=d.submission_id
+                JOIN source_records r ON r.id=ds.source_record_id WHERE s.paper_id<>r.paper_id''',
+        }
+        for name, query in checks.items():
+            integrity[name] = connection.execute(query).fetchone()[0]
+        return {'report_started_at': started_at, 'checked_at': timestamp(),
+                'engine': 'postgresql' if pg_service else 'sqlite', 'database': database,
+                'schema': schema, 'pg_service': pg_service,
+                'status': 'observed_history_audit_not_final_dedup', 'counts': dict(totals),
+                'latest_attempt_status_counts': dict(sorted(latest.items())),
+                'evidence_relationship_checks': integrity,
+                'examples': [_example(connection, rows) for rows in sample_attempts],
+                'example_limit': examples,
+                'interpretation': [
+                    'Counts use existing canonical paper links, not independently verified real-world identity.',
+                    'Negative includes rejected, desk_rejected and withdrawn observations; these remain distinct decisions.',
+                    'Known acceptance follows the latest observed decision per attempt, matching query --outcome.',
+                    'Distinct attempts/venues require separate submission/venue IDs; aliases do not create new venues.',
+                    'Coexisting evidence does not prove rejection happened before acceptance. observed_at is collection time.',
+                    'No known acceptance is not proof of never accepted. Source signatures and completeness are not established here.',
+                    'Examples are bounded; counts include all attempts and observations in one read-only snapshot.',
+                    'This does not perform entity merging, final deduplication, or label namespace/type validation.',
+                ]}
+    finally:
+        connection.close()
+
+
+def main(argv=None, *, prog=None):
+    parser = argparse.ArgumentParser(prog=prog)
+    database_arguments(parser)
+    parser.add_argument('--examples', type=int, default=20)
+    parser.add_argument('--out')
+    args = parser.parse_args(argv)
+    result = report(args.db, pg_service=args.pg_service, pg_schema=args.pg_schema, examples=args.examples)
+    text = json.dumps(result, ensure_ascii=False, indent=2)
+    if args.out:
+        path = Path(args.out)
+        stage = path.with_suffix('.part')
+        stage.write_text(text + '\n')
+        stage.replace(path)
+    else:
+        print(text)
+
+
+if __name__ == '__main__':
+    main()

paper_atlas/audits/public_openreview.py

@@ -39,7 +39,9 @@ def audit(db, inventory, out, *, pg_service=None, pg_schema=None):
             if json.loads(path.with_suffix('.manifest.json').read_text()) != manifest:
                 raise ValueError('Collected manifest differs from inventory')
             count = 0
-            for line in path.read_text().splitlines():
+            # Unicode paragraph/line separators are valid inside JSON strings;
+            # split physical byte lines, not Unicode text boundaries.
+            for line in path.read_bytes().splitlines():
                 row = json.loads(line)
                 if any(row[key] != manifest[key] for key in ('source', 'venue', 'year', 'track')):
                     raise ValueError('Record edition differs from inventory')

paper_atlas/cli.py

@@ -6,6 +6,7 @@ from paper_atlas import __version__
 
 
 COMMANDS = {
+    ('annotate',): ('paper_atlas.curation.annotations', {}, '管理独立的研究类型与自定义标签证据'),
     ('query',): ('paper_atlas.querying', {}, '查询论文、标签和投稿历史'),
     ('coverage',): ('paper_atlas.audits.coverage', {}, '查看全部候选会议的覆盖情况'),
     ('download',): ('paper_atlas.io.snapshots', {}, '下载并校验固定版本快照'),
@@ -35,6 +36,7 @@ COMMANDS = {
     ('audit','arxiv'): ('paper_atlas.audits.arxiv', {}, '核验 arXiv 原始编号、重复行与 ZIP 完整性'),
     ('audit','arxiv-tags'): ('paper_atlas.audits.arxiv_tags', {}, '逐记录核验 arXiv 类别及出处完整性'),
     ('audit','public-openreview'): ('paper_atlas.audits.public_openreview', {}, '核对公开负面决定与已有投稿历史'),
+    ('audit','histories'): ('paper_atlas.audits.histories', {}, '只读核对跨会议投稿历史及决定来源关系'),
 }
 
 

paper_atlas/collectors/public_openreview.py

@@ -27,25 +27,11 @@ RAW_ROOT = 'https://raw.githubusercontent.com/papercopilot/openreview/' + REVISI
 TREE_URL = 'https://api.github.com/repos/papercopilot/openreview/git/trees/' + REVISION + '?recursive=1'
 NEGATIVE = {'reject': 'rejected', 'rejected': 'rejected',
             'desk reject': 'desk_rejected', 'desk rejected': 'desk_rejected',
-            'withdraw': 'withdrawn', 'withdrawn': 'withdrawn', 'withdrawal': 'withdrawn'}
+            'withdrawn': 'withdrawn', 'withdrawal': 'withdrawn'}
 TRACKS = {'main': 'main', 'Position': 'position',
           'Datasets & Benchmarks': 'datasets_benchmarks'}
 WORKSHOP_REJECT_LABELS = {"aim-fm workshop @ neurips'24 reject",
                           'coginterp @ neurips 2025 reject'}
-# Exact curated labels, tied to the year and raw workshop track. The official
-# venue directory establishes workshop identity, not an original decision signature.
-ICLR_WORKSHOP_LABELS = {
-    'iclr 2024 workshop mefomo withdrawn submission':
-        (2024, 'workshop_me_fomo', 'withdrawn', 'ME-FoMo'),
-    'iclr 2024 workshop genai4dm withdrawn submission':
-        (2024, 'workshop_genai4dm', 'withdrawn', 'GenAI4DM'),
-    'iclr 2025 workshop agenticai reject':
-        (2025, 'workshop_agenticai', 'rejected', 'AgenticAI'),
-    'iclr 2025 workshop lmrl withdrawn submission':
-        (2025, 'workshop_lmrl', 'withdrawn', 'LMRL'),
-    'iclr 2026 workshop llm reasoning withdrawn submission':
-        (2026, 'workshop_llm_reasoning', 'withdrawn', 'LLM_Reasoning'),
-}
 
 
 def download(url, limit):
@@ -77,20 +63,6 @@ def normalize(row, family, year, provenance, include_workshops=False):
     """Return a normalized assertion or a reason requiring separate review."""
     raw_status = row.get('status', '')
     status = negative_status(raw_status)
-    workshop = ICLR_WORKSHOP_LABELS.get(' '.join(raw_status.casefold().split()))
-    workshop_metadata = {}
-    if workshop:
-        expected_year, expected_track, status, group_name = workshop
-        if family != 'iclr' or year != expected_year:
-            raise ValueError('Workshop status label conflicts with source conference year')
-        if row.get('track') != expected_track:
-            return None, 'conflicting_workshop_track'
-        if not include_workshops:
-            return None, 'separate_workshop'
-        workshop_metadata = {
-            'workshop_venueid': f'ICLR.cc/{year}/Workshop/{group_name}',
-            'workshop_identity_evidence_url': 'https://openreview.net/venue?id=ICLR.cc',
-        }
     if status is None:
         if re.search(r'\bretract\w*\b', raw_status, re.I):
             return None, 'retracted'
@@ -118,9 +90,7 @@ def normalize(row, family, year, provenance, include_workshops=False):
         return None, 'unmapped_venue'
     title = row.get('title')
     abstract = row.get('abstract')
-    if title is None or (isinstance(title, str) and not title.strip()):
-        return None, 'missing_title'
-    if not isinstance(title, str):
+    if not isinstance(title, str) or not title.strip():
         raise ValueError('Public submission lacks a title')
     if abstract is not None and not isinstance(abstract, str):
         raise ValueError('Abstract must be text or null')
@@ -146,8 +116,7 @@ def normalize(row, family, year, provenance, include_workshops=False):
             'venue': venue, 'year': year, 'track': normalized_track,
             'status': status, 'raw_decision': raw_status, 'evidence_url': evidence,
             'observed_at': AS_OF, 'tags': tags,
-            'metadata': {**provenance, **workshop_metadata,
-                         'raw_track': track, 'raw_status': raw_status,
+            'metadata': {**provenance, 'raw_track': track, 'raw_status': raw_status,
                          'evidence_kind': 'curated_public_status_assertion',
                          'original_decision_signature_verified': False,
                          'source_snapshot_upper_bound': AS_OF}}, None
@@ -246,7 +215,7 @@ def _collect_files(out, selected, include_workshops, fetcher):
             name = f'public-openreview-{family}-{year}-{track}.jsonl'
             path = stage / name
             path.write_text(''.join(json.dumps(row, ensure_ascii=False) + '\n' for row in records))
-            manifest = {'source': SOURCE, 'parser_version': 3, 'venue': records[0]['venue'],
+            manifest = {'source': SOURCE, 'parser_version': 1, 'venue': records[0]['venue'],
                         'year': year, 'track': track, 'count': len(records), 'status': 'partial',
                         'file': str(out / name), 'file_sha256': hashlib.sha256(path.read_bytes()).hexdigest(),
                         'list_url': REPOSITORY + '/tree/' + REVISION, 'source_revision': REVISION,

paper_atlas/curation/__init__.py

@@ -0,0 +1 @@
+"""Evidence-bearing human and explicit source annotations, independent of outcomes."""

paper_atlas/curation/annotations.py

@@ -0,0 +1,218 @@
+"""Append-only tag/type assertions and explicit retractions; no inferred labels."""
+import argparse
+from contextlib import closing
+from importlib.resources import files
+import json
+import re
+from urllib.parse import urlparse
+
+from paper_atlas.db.backend import database_arguments, inserted_id, open_read_connection, open_store
+from paper_atlas.db.submissions import timestamp
+
+TABLE = 'annotation_events'
+MIGRATION = 'annotation_events_v1'
+COLUMNS = ('id', 'request_id', 'paper_id', 'original_paper_id', 'kind', 'namespace',
+           'value', 'source_record_id', 'evidence_key', 'evidence_url', 'reason',
+           'actor', 'observed_at', 'retracts_id')
+ACTIVE = '''a.retracts_id IS NULL AND NOT EXISTS
+    (SELECT 1 FROM annotation_events removed WHERE removed.retracts_id=a.id)'''
+
+
+def available(conn):
+    if getattr(conn, 'postgres', False):
+        if hasattr(conn, 'has_table'):
+            return conn.has_table(TABLE)
+        return bool(conn.execute("SELECT 1 FROM information_schema.tables WHERE table_schema=current_schema() AND table_name=?", (TABLE,)).fetchone())
+    return bool(conn.execute("SELECT 1 FROM sqlite_master WHERE type='table' AND name=?", (TABLE,)).fetchone())
+
+
+def _begin_write(conn):
+    if not getattr(conn, 'postgres', False) and not conn.in_transaction:
+        conn.execute('BEGIN IMMEDIATE')
+
+
+def initialize(conn):
+    """Explicit transactional schema extension; does not rescan or rewrite papers."""
+    _begin_write(conn)
+    exists = available(conn)
+    marker = conn.execute('SELECT 1 FROM schema_migrations WHERE name=?', (MIGRATION,)).fetchone()
+    if exists != bool(marker):
+        raise RuntimeError('Annotation table/upgrade marker mismatch; inspect before changing schema')
+    if exists:
+        result = conn.execute('SELECT * FROM annotation_events LIMIT 0')
+        description = result.cursor.description if getattr(conn, 'postgres', False) else result.description
+        if tuple(d[0] for d in description) != COLUMNS:
+            raise RuntimeError('Unexpected annotation schema')
+        return False
+    ddl = files('paper_atlas.db').joinpath('annotations.sql').read_text()
+    if getattr(conn, 'postgres', False):
+        ddl = re.sub(r'\bINTEGER\b', 'BIGINT', ddl)
+        ddl = re.sub(r'\bTEXT\b', 'TEXT COLLATE "C"', ddl)
+        ddl = ddl.replace('id BIGINT PRIMARY KEY', 'id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY', 1)
+    for statement in ddl.split(';'):
+        if statement.strip():
+            conn.execute(statement)
+    conn.execute('INSERT INTO schema_migrations(name) VALUES(?)', (MIGRATION,))
+    return True
+
+
+def _text(value, name, limit=512):
+    if not isinstance(value, str) or not value.strip() or len(value) > limit:
+        raise ValueError(name + ' must be nonempty text of at most ' + str(limit) + ' characters')
+    return value
+
+
+def _id(value, name):
+    if type(value) is not int or value <= 0:
+        raise ValueError(name + ' must be a positive integer')
+    return value
+
+
+def _url(value):
+    if value is not None:
+        _text(value, 'evidence_url', 4096)
+        parsed = urlparse(value)
+        if parsed.scheme not in ('https', 'http') or not parsed.hostname or parsed.username or parsed.password:
+            raise ValueError('evidence_url must be an HTTP(S) URL without credentials')
+    return value
+
+
+def _require(conn):
+    if not available(conn) or not conn.execute('SELECT 1 FROM schema_migrations WHERE name=?', (MIGRATION,)).fetchone():
+        raise RuntimeError('Run paper-atlas annotate init explicitly before adding annotations')
+
+
+def _row(row):
+    return dict(zip(COLUMNS, (row[i] for i in range(len(COLUMNS)))))
+
+
+def _replay(conn, request_id, payload):
+    old = conn.execute('SELECT * FROM annotation_events WHERE request_id=?', (request_id,)).fetchone()
+    if old is None:
+        return None
+    old = _row(old)
+    if any(old[k] != value for k, value in payload.items()):
+        raise ValueError('request_id already identifies different annotation content')
+    return old['id']
+
+
+def assert_annotation(conn, paper, kind, namespace, value, *, request_id, evidence_key,
+                      reason, actor, evidence_url=None, source_record_id=None):
+    _begin_write(conn)
+    _require(conn)
+    paper = _id(paper, 'paper')
+    if kind not in ('tag', 'research_type'):
+        raise ValueError('kind must be tag or research_type')
+    _text(namespace, 'namespace', 128)
+    if not re.fullmatch(r'[a-z][a-z0-9_]*:[a-z][a-z0-9_.-]*', namespace):
+        raise ValueError('namespace must have an explicit source:scope')
+    payload = dict(original_paper_id=paper, kind=kind, namespace=namespace,
+                   value=_text(value, 'value'), request_id=_text(request_id, 'request_id', 128),
+                   source_record_id=source_record_id, evidence_key=_text(evidence_key, 'evidence_key', 256),
+                   reason=_text(reason, 'reason', 8192), actor=_text(actor, 'actor', 256),
+                   evidence_url=_url(evidence_url), retracts_id=None)
+    if source_record_id is not None:
+        _id(source_record_id, 'source_record_id')
+    replay = _replay(conn, request_id, payload)
+    if replay is not None:
+        return replay
+    if not conn.execute('SELECT 1 FROM papers WHERE id=?', (paper,)).fetchone():
+        raise ValueError('Existing canonical paper required; no automatic identity merge')
+    prefix = namespace.split(':', 1)[0]
+    if prefix != 'custom' and source_record_id is None:
+        raise ValueError('Platform annotations require their own source record')
+    if source_record_id is not None:
+        source = conn.execute('''SELECT r.paper_id,s.name FROM source_records r
+            JOIN sources s ON s.id=r.source_id WHERE r.id=?''', (source_record_id,)).fetchone()
+        if not source or source[0] != paper or (prefix != 'custom' and prefix != source[1]):
+            raise ValueError('Annotation source record/namespace must belong to this paper and platform')
+    payload.update(paper_id=paper, observed_at=timestamp())
+    fields = tuple(payload)
+    return inserted_id(conn, 'INSERT INTO annotation_events(' + ','.join(fields) + ') VALUES(' +
+                       ','.join('?' for _ in fields) + ')', tuple(payload[k] for k in fields))
+
+
+def retract_annotation(conn, event, *, request_id, reason, actor, evidence_url=None, custom_only=False):
+    _begin_write(conn)
+    _require(conn)
+    event = _id(event, 'event')
+    payload = dict(request_id=_text(request_id, 'request_id', 128), retracts_id=event,
+                   reason=_text(reason, 'reason', 8192), actor=_text(actor, 'actor', 256), evidence_url=_url(evidence_url))
+    target = conn.execute('SELECT * FROM annotation_events WHERE id=?', (event,)).fetchone()
+    if target is None or target[-1] is not None:
+        raise ValueError('Retraction must reference an existing assertion')
+    target = _row(target)
+    if custom_only and not target['namespace'].startswith('custom:'):
+        raise ValueError('Manual CLI cannot retract platform evidence')
+    replay = _replay(conn, request_id, payload)
+    if replay is not None:
+        return replay
+    if conn.execute('SELECT 1 FROM annotation_events WHERE retracts_id=?', (event,)).fetchone():
+        raise ValueError('Assertion already retracted; its history is retained')
+    payload.update({k: target[k] for k in ('paper_id', 'original_paper_id', 'kind', 'namespace', 'value', 'source_record_id', 'evidence_key')})
+    payload['observed_at'] = timestamp()
+    fields = tuple(payload)
+    return inserted_id(conn, 'INSERT INTO annotation_events(' + ','.join(fields) + ') VALUES(' +
+                       ','.join('?' for _ in fields) + ')', tuple(payload[k] for k in fields))
+
+
+def observations(conn, paper, *, kind=None, history=False):
+    if not available(conn):
+        return []
+    where = ['a.paper_id=?']; params = [_id(paper, 'paper')]
+    if not history:
+        where.append(ACTIVE)
+    if kind is not None:
+        if kind not in ('tag', 'research_type'):
+            raise ValueError('Invalid annotation kind')
+        where.append('a.kind=?'); params.append(kind)
+    return [_row(r) for r in conn.execute('SELECT a.* FROM annotation_events a WHERE ' +
+                                         ' AND '.join(where) + ' ORDER BY a.id', params)]
+
+
+def main(argv=None, *, prog=None):
+    parser = argparse.ArgumentParser(prog=prog, description='显式初始化、追加或撤回带证据的自定义标签和研究类型')
+    actions = parser.add_subparsers(dest='action', required=True)
+    for action in ('init', 'assert', 'retract', 'list'):
+        p = actions.add_parser(action)
+        database_arguments(p)
+        if action in ('assert', 'retract'):
+            for field in ('request-id', 'reason', 'actor'):
+                p.add_argument('--' + field, required=True)
+            p.add_argument('--evidence-url')
+        if action in ('assert', 'list'):
+            p.add_argument('--paper', type=int, required=True)
+            p.add_argument('--kind', choices=('tag', 'research_type'), required=action == 'assert')
+        if action == 'assert':
+            for field in ('namespace', 'value', 'evidence-key'):
+                p.add_argument('--' + field, required=True)
+            p.add_argument('--source-record', type=int)
+        if action == 'retract':
+            p.add_argument('--event', type=int, required=True)
+        if action == 'list':
+            p.add_argument('--history', action='store_true')
+    args = parser.parse_args(argv)
+    if args.action == 'list':
+        with closing(open_read_connection(args.db, pg_service=args.pg_service, pg_schema=args.pg_schema)) as conn:
+            result = observations(conn, args.paper, kind=args.kind, history=args.history)
+    else:
+        if args.action == 'assert' and not args.namespace.startswith('custom:'):
+            parser.error('Manual CLI requires custom:* namespace; platform assertions use validated source APIs')
+        with closing(open_store(args.db, pg_service=args.pg_service, pg_schema=args.pg_schema).conn) as conn:
+            try:
+                if not getattr(conn, 'postgres', False):
+                    conn.commit()
+                    conn.execute('BEGIN IMMEDIATE')
+                if args.action == 'init':
+                    result = {'initialized': initialize(conn)}
+                else:
+                    kwargs = dict(request_id=args.request_id, reason=args.reason, actor=args.actor, evidence_url=args.evidence_url)
+                    event = (assert_annotation(conn, args.paper, args.kind, args.namespace, args.value,
+                              evidence_key=args.evidence_key, source_record_id=args.source_record, **kwargs)
+                             if args.action == 'assert' else retract_annotation(conn, args.event, custom_only=True, **kwargs))
+                    result = {'event_id': event}
+                conn.commit()
+            except BaseException:
+                conn.rollback()
+                raise
+    print(json.dumps(result, ensure_ascii=False, indent=2))

paper_atlas/db/annotations.sql

@@ -0,0 +1,15 @@
+CREATE TABLE annotation_events (
+ id INTEGER PRIMARY KEY,
+ request_id TEXT NOT NULL UNIQUE,
+ paper_id INTEGER NOT NULL REFERENCES papers(id),
+ original_paper_id INTEGER NOT NULL,
+ kind TEXT NOT NULL CHECK(kind IN ('tag','research_type')),
+ namespace TEXT NOT NULL, value TEXT NOT NULL,
+ source_record_id INTEGER REFERENCES source_records(id),
+ evidence_key TEXT NOT NULL, evidence_url TEXT,
+ reason TEXT NOT NULL, actor TEXT NOT NULL, observed_at TEXT NOT NULL,
+ retracts_id INTEGER UNIQUE REFERENCES annotation_events(id),
+ CHECK(retracts_id IS NULL OR retracts_id<>id)
+);
+CREATE INDEX annotation_events_paper ON annotation_events(paper_id,kind,id);
+CREATE INDEX annotation_events_label ON annotation_events(kind,namespace,value,paper_id);

paper_atlas/db/store.py

@@ -55,6 +55,9 @@ class Store:
         c.execute('UPDATE identifiers SET paper_id=? WHERE paper_id=?',(target,other))
         c.execute('UPDATE source_records SET paper_id=? WHERE paper_id=?',(target,other))
         c.execute('UPDATE submissions SET paper_id=? WHERE paper_id=?',(target,other))
+        from paper_atlas.curation.annotations import available
+        if available(c):
+            c.execute('UPDATE annotation_events SET paper_id=? WHERE paper_id=?', (target,other))
         c.execute('INSERT INTO paper_tags SELECT ?,tag_id,source_record_id,evidence_key FROM paper_tags WHERE paper_id=? ON CONFLICT DO NOTHING',(target,other))
         c.execute('DELETE FROM paper_tags WHERE paper_id=?',(other,))
         c.execute('UPDATE paper_merges SET surviving_paper_id=? WHERE surviving_paper_id=?',(target,other))

paper_atlas/postgres/migration.py

@@ -143,7 +143,7 @@ def migrate(db, service, out, *, schema='research', storage_path=None, min_free_
     free_space(storage_path, min_free_gib)
     control = schema + '_migration'
     report = {'status': 'preflight', 'source': str(db), 'service': service, 'schema': schema,
-              'started_at': timestamp(), 'tables': {}, 'scope': 'All 21 business tables; FTS5 internals rebuilt as native PostgreSQL GIN.'}
+              'started_at': timestamp(), 'tables': {}, 'scope': 'All baseline business tables and recognized installed extensions; FTS5 rebuilt as native PostgreSQL GIN.'}
     with Path(str(db) + '.maintenance.lock').open('a') as lock:
         fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
         if Path(str(db) + '-wal').exists() and Path(str(db) + '-wal').stat().st_size:
@@ -240,7 +240,7 @@ def migrate(db, service, out, *, schema='research', storage_path=None, min_free_
                     JOIN pg_am am ON am.oid=ix.relam WHERE ns.nspname=%s AND ix.relname='papers_search_gin' ''', (stage,)).fetchone()
                 if search != ('gin', True, True, 'papers'):
                     raise ValueError('Native full-text index is not valid and ready')
-                for table in TABLES:
+                for table in plan:
                     pg.execute('ANALYZE ' + qualify(stage, table))
                 invalid = pg.execute('''SELECT count(*) FROM pg_constraint co JOIN pg_class cl ON cl.oid=co.conrelid
                     JOIN pg_namespace ns ON ns.oid=cl.relnamespace WHERE ns.nspname=%s AND NOT co.convalidated''', (stage,)).fetchone()[0]

paper_atlas/postgres/schema.py

@@ -8,7 +8,9 @@ TABLES = ('sources', 'papers', 'identifiers', 'source_records', 'venues', 'editi
           'submission_sources', 'submission_identifiers', 'decision_sources',
           'submission_merges', 'decision_merges', 'schema_migrations', 'tags',
           'paper_tags', 'match_candidates', 'paper_merges', 'ingest_runs')
+OPTIONAL_TABLES = ('annotation_events',)
 CHECKS = {
+    'annotation_events': ["kind IN ('tag','research_type')", 'retracts_id IS NULL OR retracts_id<>id'],
     'venues': ["venue_type IN ('conference','journal','review_platform')"],
     'edition_sources': ["status IN ('pending','running','complete','partial','blocked','unavailable','not_yet_published')"],
     'decisions': ["status IN ('accepted','rejected','desk_rejected','withdrawn','under_review','unknown')"],
@@ -50,10 +52,10 @@ def quote(name):
 def inspect(connection):
     business = {name for schema, name, kind, *_ in connection.execute('PRAGMA table_list')
                 if schema == 'main' and kind == 'table' and not name.startswith('sqlite_')}
-    if business != set(TABLES):
+    if not set(TABLES) <= business or business - set(TABLES) - set(OPTIONAL_TABLES):
         raise ValueError('Unexpected business tables: ' + repr(sorted(business ^ set(TABLES))))
     result = {}
-    for table in TABLES:
+    for table in (*TABLES, *(name for name in OPTIONAL_TABLES if name in business)):
         columns = [{'name': r[1], 'type': r[2].upper(), 'notnull': bool(r[3]),
                     'default': r[4], 'pk': r[5]}
                    for r in connection.execute('PRAGMA table_info(' + quote(table) + ')')]

paper_atlas/querying.py

@@ -29,6 +29,8 @@ def main(argv=None, *, prog=None):
     parser.add_argument('--pg-schema', help='Optional PostgreSQL schema override; otherwise use the configured search_path')
     parser.add_argument('--query', help='Title/abstract search: FTS5 for SQLite, web-search syntax for PostgreSQL')
     parser.add_argument('--paper',type=int,help='Paper ID; include sources, tags and submission history')
+    parser.add_argument('--research-type', help='Independent type filter as namespace=value; no type inferred from track')
+    parser.add_argument('--annotation-history', action='store_true', help='Include retraction evidence for --paper')
     parser.add_argument('--tag', help='Filter as namespace=value, e.g. arxiv:category=cs.AI')
     parser.add_argument('--venue', help='Conference acronym')
     parser.add_argument('--year', type=int)
@@ -47,6 +49,8 @@ def main(argv=None, *, prog=None):
         conn = sqlite3.connect(Path(args.db).resolve().as_uri()+'?mode=ro', uri=True)
         conn.row_factory = sqlite3.Row
         conn.execute('BEGIN')  # One consistent read snapshot while ingestion continues.
+    from paper_atlas.curation.annotations import available, observations, ACTIVE
+    has_annotations = available(conn)
     if args.stats:
         result = {name:conn.execute('SELECT count(*) FROM '+name).fetchone()[0] for name in ('papers','source_records','submissions','decisions','tags','match_candidates')}
         result['records_by_source'] = [dict(r) for r in conn.execute('SELECT s.name,count(r.id) AS imported_records FROM sources s LEFT JOIN source_records r ON r.source_id=s.id GROUP BY s.id ORDER BY s.name')]
@@ -69,8 +73,24 @@ def main(argv=None, *, prog=None):
                 where.append('p.id IN (SELECT rowid FROM papers_fts WHERE papers_fts MATCH ?)')
             params.append(args.query)
         if args.tag:
+            if '=' not in args.tag:
+                parser.error('--tag requires namespace=value')
             namespace,value = args.tag.split('=',1)
-            where.append('EXISTS(SELECT 1 FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id WHERE pt.paper_id=p.id AND t.namespace=? AND t.value=?)'); params.extend([namespace,value])
+            platform = 'EXISTS(SELECT 1 FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id WHERE pt.paper_id=p.id AND t.namespace=? AND t.value=?)'
+            params.extend([namespace,value])
+            if has_annotations:
+                platform = '(' + platform + ' OR EXISTS(SELECT 1 FROM annotation_events a WHERE a.paper_id=p.id AND a.kind=\'tag\' AND a.namespace=? AND a.value=? AND ' + ACTIVE + '))'
+                params.extend([namespace,value])
+            where.append(platform)
+        if args.research_type:
+            if '=' not in args.research_type:
+                parser.error('--research-type requires namespace=value')
+            namespace,value = args.research_type.split('=',1)
+            if has_annotations:
+                where.append('EXISTS(SELECT 1 FROM annotation_events a WHERE a.paper_id=p.id AND a.kind=\'research_type\' AND a.namespace=? AND a.value=? AND ' + ACTIVE + ')')
+                params.extend([namespace,value])
+            else:
+                where.append('1=0')
         if args.outcome:
             where.append('p.id '+('IN' if args.outcome=='accepted' else 'NOT IN')+' ('+ACCEPTED_PAPERS+')')
         if args.venue or args.year or args.status or args.track:
@@ -98,6 +118,10 @@ def main(argv=None, *, prog=None):
             paper['known_acceptances']=known_acceptances(conn,row['id'])
             paper['publication_outcome']='accepted' if paper['known_acceptances'] else 'no_known_acceptance'
             if args.paper is not None:
+                paper['research_types'] = observations(conn, row['id'], kind='research_type')
+                paper['annotation_tags'] = observations(conn, row['id'], kind='tag')
+                if args.annotation_history:
+                    paper['annotation_history'] = observations(conn, row['id'], history=True)
                 paper['tags'] = [dict(r) for r in conn.execute('SELECT t.namespace,t.value,pt.evidence_key FROM paper_tags pt JOIN tags t ON t.id=pt.tag_id WHERE pt.paper_id=?',(row['id'],))]
                 paper['sources'] = [dict(r) for r in conn.execute('SELECT s.name,r.external_id,r.url,r.title,r.abstract,r.retrieved_at FROM source_records r JOIN sources s ON s.id=r.source_id WHERE r.paper_id=?',(row['id'],))]
                 submissions = []

pyproject.toml

@@ -21,5 +21,5 @@ include = ["paper_atlas*"]
 exclude = ["tests*"]
 
 [tool.setuptools.package-data]
-"paper_atlas.db" = ["schema.sql"]
+"paper_atlas.db" = ["schema.sql", "annotations.sql"]
 "paper_atlas.resources" = ["venues.json"]

tests/test_annotations.py

@@ -0,0 +1,227 @@
+from contextlib import closing, redirect_stdout
+import io
+import json
+import os
+from pathlib import Path
+import tempfile
+import unittest
+import uuid
+
+from paper_atlas import querying
+from paper_atlas.curation.annotations import (assert_annotation, available, initialize, main,
+                                             observations, retract_annotation)
+from paper_atlas.db.backend import open_store
+from paper_atlas.db.store import Store
+from paper_atlas.postgres import schema
+from paper_atlas.postgres.client import connect
+from paper_atlas.postgres.migration import migrate
+
+
+class AnnotationTests(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory(); self.addCleanup(self.tmp.cleanup)
+        self.db = Path(self.tmp.name) / 'sample.sqlite'
+        self.store = Store(self.db); self.addCleanup(self.store.conn.close)
+        self.c = self.store.conn
+        self.paper, self.record = self.store.record('fixture', 'one', 'https://example.org/one', 'First research', None, [])
+        self.other, self.other_record = self.store.record('fixture', 'two', 'https://example.org/two', 'Other research', None, [])
+        self.store.record_tags(self.paper, self.record, 'fixture', [{'namespace':'fixture:topic','value':'original'}])
+        self.c.commit()
+
+    def add(self, **changes):
+        args = dict(paper=self.paper, kind='tag', namespace='custom:topic', value='reading',
+                    request_id='request-one', evidence_key='manual:reading', reason='Reviewed metadata', actor='reviewer')
+        args.update(changes)
+        return assert_annotation(self.c, **args)
+
+    def query(self, *args):
+        self.c.commit()
+        with redirect_stdout(io.StringIO()) as output:
+            querying.main(['--db', str(self.db), *map(str, args)])
+        return json.loads(output.getvalue())
+
+    def test_read_does_not_install_extension_and_write_requires_explicit_init(self):
+        self.assertFalse(available(self.c))
+        self.assertEqual(observations(self.c, self.paper), [])
+        with self.assertRaisesRegex(RuntimeError, 'init explicitly'):
+            self.add()
+        self.assertEqual(self.query('--research-type', 'custom:method=empirical'), [])
+        self.assertFalse(available(self.c))
+        self.assertEqual(len(schema.inspect(self.c)), 21)
+
+    def test_schema_upgrade_is_transactional_and_marker_mismatch_fails_closed(self):
+        self.assertTrue(initialize(self.c)); self.c.rollback()
+        self.assertFalse(available(self.c))
+        self.c.execute("INSERT INTO schema_migrations(name) VALUES('annotation_events_v1')")
+        with self.assertRaisesRegex(RuntimeError, 'mismatch'):
+            initialize(self.c)
+        self.c.rollback()
+        self.assertTrue(initialize(self.c)); self.assertFalse(initialize(self.c))
+        self.assertEqual(len(schema.inspect(self.c)), 22)
+
+    def test_unmarked_table_cannot_receive_assertions(self):
+        initialize(self.c)
+        self.c.execute("DELETE FROM schema_migrations WHERE name='annotation_events_v1'")
+        with self.assertRaisesRegex(RuntimeError, 'init explicitly'):
+            self.add()
+        self.assertEqual(self.c.execute('SELECT count(*) FROM annotation_events').fetchone()[0], 0)
+
+    def test_types_tags_and_platform_labels_are_independent_and_queryable(self):
+        initialize(self.c)
+        self.add()
+        self.add(kind='research_type', namespace='custom:topic', request_id='type', value='reading')
+        self.assertEqual(len(self.query('--tag', 'custom:topic=reading')), 1)
+        self.assertEqual(len(self.query('--research-type', 'custom:topic=reading')), 1)
+        self.assertEqual(len(self.query('--tag', 'fixture:topic=original')), 1)
+        detail = self.query('--paper', self.paper)[0]
+        self.assertEqual(len(detail['annotation_tags']), 1)
+        self.assertEqual(len(detail['research_types']), 1)
+        self.assertNotIn('annotation_history', detail)
+        self.assertEqual(detail['tags'][0]['namespace'], 'fixture:topic')
+
+    def test_repeated_request_is_idempotent_and_changed_content_fails(self):
+        initialize(self.c); event = self.add()
+        self.assertEqual(self.add(), event)
+        for change in ({'value':'different'}, {'kind':'research_type'}, {'reason':'changed'}, {'paper':self.other}):
+            with self.assertRaisesRegex(ValueError, 'different annotation'):
+                self.add(**change)
+        self.assertEqual(self.c.execute('SELECT count(*) FROM annotation_events').fetchone()[0], 1)
+
+    def test_retraction_keeps_original_and_other_assertions_of_same_label(self):
+        initialize(self.c); first = self.add(); second = self.add(request_id='second', evidence_key='independent')
+        kwargs = dict(request_id='retract', reason='No longer relevant', actor='reviewer')
+        removed = retract_annotation(self.c, first, **kwargs)
+        self.assertEqual(retract_annotation(self.c, first, **kwargs), removed)
+        self.assertEqual([r['id'] for r in observations(self.c, self.paper)], [second])
+        self.assertEqual(len(self.query('--paper', self.paper, '--annotation-history')[0]['annotation_history']), 3)
+        with self.assertRaisesRegex(ValueError, 'already retracted'):
+            retract_annotation(self.c, first, **{**kwargs,'request_id':'duplicate'})
+        with self.assertRaisesRegex(ValueError, 'existing assertion'):
+            retract_annotation(self.c, removed, **{**kwargs,'request_id':'nested'})
+
+    def test_source_assertions_require_matching_paper_and_platform(self):
+        initialize(self.c)
+        for change in ({'namespace':'fixture:type'}, {'namespace':'other:type','source_record_id':self.record},
+                       {'namespace':'fixture:type','source_record_id':self.other_record}):
+            with self.assertRaises(ValueError):
+                self.add(**change)
+        event = self.add(kind='research_type', namespace='fixture:declared_type', source_record_id=self.record)
+        with self.assertRaisesRegex(ValueError, 'Manual CLI'):
+            retract_annotation(self.c, event, request_id='cannot', reason='x', actor='y', custom_only=True)
+        self.assertEqual(observations(self.c, self.paper)[0]['source_record_id'], self.record)
+
+    def test_platform_tag_refresh_preserves_both_custom_kinds(self):
+        initialize(self.c); self.add(); self.add(kind='research_type', request_id='type')
+        self.store.record_tags(self.paper, self.record, 'fixture', [{'namespace':'fixture:topic','value':'refreshed'}])
+        self.assertEqual(len(observations(self.c, self.paper)), 2)
+        self.assertEqual(len(self.query('--tag','fixture:topic=refreshed')), 1)
+        self.assertEqual(self.query('--tag','fixture:topic=original'), [])
+
+    def test_merge_preserves_retractions_evidence_and_original_paper_ids(self):
+        initialize(self.c); event = self.add(source_record_id=self.record)
+        removed = retract_annotation(self.c, event, request_id='withdraw', reason='corrected', actor='reviewer')
+        active = self.add(request_id='active', kind='research_type', value='survey')
+        self.store.merge(self.other, self.paper, 'Independently reviewed identity')
+        rows = observations(self.c, self.other, history=True)
+        self.assertEqual([r['id'] for r in rows], [event, removed, active])
+        self.assertEqual({r['original_paper_id'] for r in rows}, {self.paper})
+        self.assertEqual({r['paper_id'] for r in rows}, {self.other})
+        self.assertEqual([r['id'] for r in observations(self.c, self.other)], [active])
+        self.assertEqual(self.add(source_record_id=self.record), event)  # exact retry survives canonical merge
+        self.assertEqual(self.c.execute('PRAGMA foreign_key_check').fetchall(), [])
+
+    def test_invalid_input_does_not_create_events(self):
+        initialize(self.c)
+        for change in ({'paper':True}, {'paper':999999}, {'kind':'main'}, {'namespace':'arxiv'},
+                       {'namespace':'custom:topic;DROP'}, {'value':' '}, {'reason':''},
+                       {'actor':''}, {'request_id':''}, {'evidence_key':''}, {'source_record_id':True},
+                       {'evidence_url':'file:///etc/passwd'}, {'evidence_url':'https://user:secret@example.org/'}):
+            with self.subTest(change=change), self.assertRaises(ValueError):
+                self.add(**change)
+        self.assertEqual(self.c.execute('SELECT count(*) FROM annotation_events').fetchone()[0], 0)
+
+    def test_cli_manual_scope_and_complete_assert_retract_lifecycle(self):
+        with redirect_stdout(io.StringIO()):
+            main(['init','--db',str(self.db)])
+            main(['assert','--db',str(self.db),'--paper',str(self.paper),'--kind','research_type',
+                  '--namespace','custom:method','--value','empirical','--evidence-key','manual:review',
+                  '--request-id','cli-one','--reason','Read abstract','--actor','reviewer'])
+        self.assertEqual(len(self.query('--research-type','custom:method=empirical')), 1)
+        event = observations(self.c, self.paper)[0]['id']
+        with redirect_stdout(io.StringIO()):
+            main(['retract','--db',str(self.db),'--event',str(event),'--request-id','cli-two',
+                  '--reason','Corrected classification','--actor','reviewer'])
+        self.assertEqual(self.query('--research-type','custom:method=empirical'), [])
+        with redirect_stdout(io.StringIO()) as out:
+            main(['list','--db',str(self.db),'--paper',str(self.paper),'--history'])
+        self.assertEqual(len(json.loads(out.getvalue())), 2)
+
+
+@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'), 'Requires approved remote PostgreSQL service')
+class PostgreSQLAnnotationTests(unittest.TestCase):
+    def setUp(self):
+        self.service = os.environ['PAPER_ATLAS_TEST_PG_SERVICE']
+        self.target = 'pa_anno_' + uuid.uuid4().hex[:12]
+        self.tmp = tempfile.TemporaryDirectory(); self.addCleanup(self.tmp.cleanup)
+        self.db = Path(self.tmp.name) / 'source.sqlite'
+        self.s = Store(self.db)
+        self.paper, self.record = self.s.record('fixture','one','https://example.org','Paper',None,[])
+        self.s.conn.commit()
+        self.addCleanup(self.s.conn.close); self.addCleanup(self.cleanup)
+
+    def cleanup(self):
+        with connect(self.service) as c:
+            for name, in c.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)', (self.target,)):
+                c.execute('DROP SCHEMA ' + schema.quote(name) + ' CASCADE')
+
+    def migrate(self):
+        self.s.conn.commit()
+        self.s.conn.close()  # Freeze/checkpoint the fixture, just like a real source migration.
+        with redirect_stdout(io.StringIO()):
+            return migrate(self.db, self.service, Path(self.tmp.name)/'report.json', schema=self.target, min_free_gib=0)
+
+    def args(self, **changes):
+        a = dict(paper=self.paper,kind='research_type',namespace='custom:method',value='survey\x00and\x01',
+                 request_id='native',evidence_key='manual',reason='Reviewed\x00evidence',actor='reviewer')
+        a.update(changes); return a
+
+    def test_native_schema_upgrade_roundtrip_retraction_and_queries(self):
+        self.migrate()
+        with closing(open_store(pg_service=self.service, pg_schema=self.target).conn) as c:
+            self.assertTrue(initialize(c)); self.assertFalse(initialize(c))
+            event = assert_annotation(c, **self.args()); c.commit()
+            self.assertEqual(assert_annotation(c, **self.args()), event)
+            self.assertEqual(observations(c,self.paper)[0]['value'], 'survey\x00and\x01')
+            c.commit()
+        with redirect_stdout(io.StringIO()) as out:
+            querying.main(['--pg-service',self.service,'--pg-schema',self.target,'--paper',str(self.paper)])
+        self.assertEqual(json.loads(out.getvalue())[0]['research_types'][0]['id'], event)
+        with closing(open_store(pg_service=self.service,pg_schema=self.target).conn) as c:
+            retract_annotation(c,event,request_id='removed',reason='Changed',actor='reviewer');c.commit()
+            self.assertEqual(observations(c,self.paper), [])
+            self.assertEqual(len(observations(c,self.paper,history=True)), 2)
+
+    def test_initialized_sqlite_extension_migrates_all_rows_hashes_and_identity(self):
+        initialize(self.s.conn)
+        event = assert_annotation(self.s.conn, **self.args())
+        retract_annotation(self.s.conn,event,request_id='retracted',reason='Corrected',actor='reviewer')
+        report = self.migrate()
+        self.assertEqual(len(report['tables']),22)
+        self.assertEqual(report['tables']['annotation_events']['source_rows'],2)
+        self.assertEqual(report['tables']['annotation_events']['source_sha256'],report['tables']['annotation_events']['target_sha256'])
+        with closing(open_store(pg_service=self.service,pg_schema=self.target).conn) as c:
+            new = assert_annotation(c, **self.args(request_id='after-migration'))
+            self.assertGreater(new,2); c.commit()
+            self.assertEqual(len(observations(c,self.paper,history=True)),3)
+
+    def test_native_upgrade_rollback_keeps_legacy_schema_usable(self):
+        self.migrate()
+        with closing(open_store(pg_service=self.service,pg_schema=self.target).conn) as c:
+            initialize(c); c.rollback()
+            self.assertFalse(available(c))
+            self.assertEqual(c.execute('SELECT count(*) FROM papers').fetchone()[0],1)
+            self.assertTrue(initialize(c))
+            with self.assertRaises(ValueError):
+                assert_annotation(c, **self.args(namespace='wrong:type',source_record_id=self.record))
+            self.assertEqual(c.execute('SELECT count(*) FROM annotation_events').fetchone()[0],0)
+            c.rollback()

tests/test_histories.py

@@ -0,0 +1,114 @@
+from pathlib import Path
+import tempfile
+import unittest
+
+from paper_atlas.audits.histories import report
+from paper_atlas.db.store import Store
+
+
+def add_history_fixture(store):
+    def event(work, forum, venue, year, track, status, observed):
+        paper, record = store.record('history_fixture', work + '/' + forum,
+            'https://example.org/' + forum, work + ' title\x00 preserved', 'Abstract', ['Jane Doe'],
+            {'arxiv': work})
+        store.decision(paper, record, venue, year, track, forum,
+            'https://example.org/' + forum + '/' + str(status), status, status or '',
+            identities=[('openreview', forum)], observed_at=observed)
+        return paper
+
+    # The negative observation was collected later: observation order is not chronology.
+    event('2501.00001', 'HistoryA01', 'ICLR', 2024, 'main', 'rejected', '2026-02-01T00:00:00Z')
+    event('2501.00001', 'HistoryA02', 'ICML', 2025, 'main', 'accepted', '2026-01-01T00:00:00Z')
+    event('2501.00002', 'HistoryB01', 'ICLR', 2024, 'main', 'rejected', '2026-01-01T00:00:00Z')
+    event('2501.00002', 'HistoryB01', 'ICLR', 2024, 'main', 'accepted', '2026-02-01T00:00:00Z')
+    event('2501.00003', 'HistoryC01', 'ICLR', 2024, 'main', 'desk_rejected', '2026-01-01T00:00:00Z')
+    event('2501.00003', 'HistoryC02', 'ICLR', 2024, 'workshop_example', 'accepted', '2026-02-01T00:00:00Z')
+    event('2501.00004', 'HistoryD01', 'ICML', 2025, 'main', 'accepted', '2026-01-01T00:00:00Z')
+    event('2501.00004', 'HistoryD01', 'ICML', 2025, 'main', 'withdrawn', '2026-02-01T00:00:00Z')
+    event('2501.00004', 'HistoryD02', 'ICLR', 2024, 'main', 'rejected', '2026-01-01T00:00:00Z')
+    event('2501.00005', 'HistoryE01', 'COLM', 2026, 'main', None, '2026-01-01T00:00:00Z')
+    store.conn.commit()
+
+
+class HistoryTests(unittest.TestCase):
+    def setUp(self):
+        self.tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.tmp.cleanup)
+        self.db = Path(self.tmp.name) / 'sample.sqlite'
+        self.store = Store(self.db)
+        self.addCleanup(self.store.conn.close)
+
+    def test_distinct_attempts_venues_and_latest_acceptances_are_not_observation_counts(self):
+        add_history_fixture(self.store)
+        result = report(self.db)
+        self.assertEqual(result['counts'], {
+            'papers_with_submissions': 5, 'papers_with_multiple_submissions': 3,
+            'papers_with_historical_negative': 4, 'papers_with_current_known_acceptance': 3,
+            'papers_with_negative_and_acceptance_in_distinct_attempts': 2,
+            'papers_with_negative_and_acceptance_in_distinct_venues': 1,
+            'submissions': 8, 'decision_observations': 9,
+            'attempts_with_historical_negative': 5,
+            'attempts_with_historical_negative_and_acceptance': 2,
+        })
+        self.assertEqual(result['latest_attempt_status_counts'],
+                         {'accepted': 3, 'desk_rejected': 1, 'rejected': 2, 'unknown': 1, 'withdrawn': 1})
+        self.assertEqual(len(result['examples']), 2)
+        first = result['examples'][0]
+        self.assertIn('\x00', first['title'])
+        self.assertEqual({r['venue'] for r in first['attempts']}, {'ICLR', 'ICML'})
+        self.assertTrue(all(r['decisions'][0]['sources'] for r in first['attempts']))
+        self.assertTrue(all(r['decisions'][0]['decision_date'] is None for r in first['attempts']))
+        self.assertIn('Not inferred', first['chronology'])
+        self.assertEqual(result['status'], 'observed_history_audit_not_final_dedup')
+        self.assertTrue(all(v == 0 for v in result['evidence_relationship_checks'].values()))
+
+    def test_missing_and_mislinked_source_evidence_is_reported_without_repair(self):
+        add_history_fixture(self.store)
+        self.store.conn.execute('DELETE FROM decision_sources WHERE decision_id=1')
+        self.store.conn.execute('DELETE FROM submission_sources WHERE submission_id=1')
+        # Record3 belongs to paper2; move it to paper1 to create a real mismatch.
+        self.store.conn.execute('UPDATE source_records SET paper_id=1 WHERE id=3')
+        self.store.conn.commit()
+        result = report(self.db, examples=0)
+        checks = result['evidence_relationship_checks']
+        self.assertEqual(checks['decisions_without_source_links'], 1)
+        self.assertEqual(checks['submissions_without_source_links'], 1)
+        self.assertEqual(checks['submission_source_paper_mismatches'], 1)
+        self.assertEqual(checks['decision_source_paper_mismatches'], 2)
+        self.assertEqual(self.store.conn.execute('SELECT paper_id FROM source_records WHERE id=3').fetchone()[0], 1)
+
+    def test_example_limit_does_not_change_global_counts(self):
+        add_history_fixture(self.store)
+        none, one = report(self.db, examples=0), report(self.db, examples=1)
+        self.assertEqual(none['counts'], one['counts'])
+        self.assertEqual(none['examples'], [])
+        self.assertEqual(len(one['examples']), 1)
+
+    def test_empty_snapshot_and_invalid_limit_do_not_create_missing_database(self):
+        empty = report(self.db)
+        self.assertTrue(all(v == 0 for v in empty['counts'].values()))
+        self.assertEqual(empty['latest_attempt_status_counts'], {})
+        absent = self.db.parent / 'absent.sqlite'
+        with self.assertRaisesRegex(ValueError, 'Examples'):
+            report(absent, examples=-1)
+        with self.assertRaisesRegex(ValueError, 'Existing database'):
+            report(absent)
+        self.assertFalse(absent.exists())
+
+    def test_truncated_example_keeps_the_cross_venue_acceptance_visible(self):
+        paper, record = self.store.record('fixture', 'many', 'https://example.org/many',
+                                          'Many attempts', 'Abstract', [])
+        for index in range(26):
+            accepted = index == 25
+            forum = f'BoundedForum{index}'
+            self.store.decision(paper, record, 'ICML' if accepted else 'ICLR', 2025, 'main', forum,
+                                'https://example.org/' + forum, 'accepted' if accepted else 'rejected',
+                                'Accept' if accepted else 'Reject', identities=[('openreview', forum)])
+        self.store.conn.commit()
+        result = report(self.db)
+        example, = result['examples']
+        self.assertEqual((example['attempt_count'], len(example['attempts'])), (26, 20))
+        self.assertTrue(example['attempts_truncated'])
+        self.assertEqual({row['latest_observed_status'] for row in example['attempts']},
+                         {'accepted', 'rejected'})
+        self.assertEqual(result['counts']['submissions'], 26)

tests/test_postgres.py

@@ -175,6 +175,22 @@ class PostgreSQLIntegrationTests(unittest.TestCase):
         finally:
             reader.close()
 
+    def test_history_audit_matches_sqlite_for_actual_postgresql_snapshot(self):
+        from paper_atlas.audits.histories import report
+        from tests.test_histories import add_history_fixture
+        store = Store(self.db)
+        try:
+            add_history_fixture(store)
+        finally:
+            store.conn.close()
+        expected = report(self.db)
+        self.assertEqual(self.migrate()['status'], 'complete')
+        actual = report(pg_service=self.service, pg_schema=self.target)
+        self.assertEqual(actual['engine'], 'postgresql')
+        self.assertEqual(actual['schema'], self.target)
+        for key in ('counts', 'latest_attempt_status_counts', 'evidence_relationship_checks', 'examples'):
+            self.assertEqual(actual[key], expected[key], key)
+
     def interrupt_after_first_table(self):
         calls = 0
         def interrupt(*args):

tests/test_public_openreview.py

@@ -45,12 +45,10 @@ class PublicOpenReviewTests(unittest.TestCase):
 
     def test_only_explicit_decisions_and_verified_scope_become_events(self):
         for value in ('Pending', 'Active', 'Submitted to NeurIPS 2025', 'TBD', '',
-                      'Low score', 'ICML 2025 Retracted by Authors', 'Do not reject',
-                      'Workshop', 'Poster Workshop', 'Oral Workshop'):
+                      'Low score', 'ICML 2025 Retracted by Authors', 'Do not reject'):
             self.assertIsNone(negative_status(value), value)
         self.assertEqual(negative_status('Reject'), 'rejected')
         self.assertEqual(negative_status('Desk rejected'), 'desk_rejected')
-        self.assertEqual(negative_status('Withdraw'), 'withdrawn')
         self.assertEqual(negative_status('NeurIPS 2023 Conference Withdrawn Submission'), 'withdrawn')
         for raw, expected in [('main', 'main'), ('Position', 'position'),
                               ('Datasets & Benchmarks', 'datasets_benchmarks')]:
@@ -100,121 +98,6 @@ class PublicOpenReviewTests(unittest.TestCase):
                 self.assertEqual(list(Path(tmp).glob('*.jsonl')), [])
                 self.assertFalse((Path(tmp) / 'public-openreview-inventory.json').exists())
 
-    def test_missing_title_is_held_without_losing_valid_negative_records(self):
-        with tempfile.TemporaryDirectory() as tmp:
-            result = self.collect(tmp, {'venues/iclr/iclr2017.jsonl': [
-                self.row(id='SJUdkecgx', title=''),
-                self.row(id='NoTitle001', title=None),
-                self.row(id='Whitespace', title='  '),
-                self.row(id='ValidReject'),
-                self.row(id='Withdraw01', status='Withdraw'),
-                self.row(id='Workshop01', status='Workshop'),
-            ]}, families=['iclr'], years=[2017])
-            self.assertEqual(result['input_rows'], 6)
-            self.assertEqual(result['normalized_records'], 2)
-            self.assertEqual(result['source_files'][0]['excluded_or_held'],
-                             {'missing_title': 3, 'not_explicit_negative': 1})
-            held_path = Path(result['held_file'])
-            self.assertEqual(hashlib.sha256(held_path.read_bytes()).hexdigest(),
-                             result['held_file_sha256'])
-            held = [json.loads(line) for line in held_path.read_text().splitlines()]
-            self.assertEqual([row['id'] for row in held], ['SJUdkecgx', 'NoTitle001', 'Whitespace'])
-            self.assertTrue(all(row['reason'] == 'missing_title' and row['status'] == 'Reject'
-                                for row in held))
-            self.assertEqual([row['provenance']['source_line'] for row in held], [1, 2, 3])
-            self.assertNotIn('private@example.org', held_path.read_text())
-            self.assertNotIn('private review', held_path.read_text())
-            manifest = result['files'][0]
-            self.assertEqual((manifest['parser_version'], manifest['status']), (3, 'partial'))
-            rows = [json.loads(line) for line in Path(manifest['file']).read_text().splitlines()]
-            self.assertEqual([(row['identifiers']['openreview'], row['status']) for row in rows],
-                             [('ValidReject', 'rejected'), ('Withdraw01', 'withdrawn')])
-            for row in held:
-                self.assertEqual(row['provenance']['source_revision'], REVISION)
-
-    def test_nontext_title_still_fails_collection(self):
-        with tempfile.TemporaryDirectory() as tmp:
-            with self.assertRaisesRegex(ValueError, 'lacks a title'):
-                self.collect(tmp, {'venues/iclr/iclr2017.jsonl': [self.row(title=['invalid'])]},
-                             families=['iclr'], years=[2017])
-            self.assertFalse((Path(tmp) / 'public-openreview-inventory.json').exists())
-            self.assertEqual(list(Path(tmp).glob('*.jsonl')), [])
-
-    def workshop_cases(self):
-        return [
-            (2024, 'workshop_me_fomo', 'ICLR 2024 Workshop MEFoMo Withdrawn Submission', 'withdrawn', 'ME-FoMo'),
-            (2024, 'workshop_genai4dm', 'ICLR 2024 Workshop GenAI4DM Withdrawn Submission', 'withdrawn', 'GenAI4DM'),
-            (2025, 'workshop_agenticai', 'ICLR 2025 Workshop AgenticAI Reject', 'rejected', 'AgenticAI'),
-            (2025, 'workshop_lmrl', 'ICLR 2025 Workshop LMRL Withdrawn Submission', 'withdrawn', 'LMRL'),
-            (2026, 'workshop_llm_reasoning', 'ICLR 2026 Workshop LLM Reasoning Withdrawn Submission', 'withdrawn', 'LLM_Reasoning'),
-        ]
-
-    def test_iclr_workshop_labels_require_opt_in_and_retain_identity_evidence(self):
-        for year, track, label, status, group in self.workshop_cases():
-            with self.subTest(label=label):
-                raw = self.row(track=track, status=label)
-                self.assertEqual(normalize(raw, 'iclr', year, self.provenance()),
-                                 (None, 'separate_workshop'))
-                row, reason = normalize(raw, 'iclr', year, self.provenance(), True)
-                self.assertIsNone(reason)
-                self.assertEqual((row['venue'], row['year'], row['track'], row['status']),
-                                 ('ICLR', year, track, status))
-                self.assertEqual(row['raw_decision'], label)
-                self.assertEqual(row['metadata']['workshop_venueid'],
-                                 f'ICLR.cc/{year}/Workshop/{group}')
-                self.assertFalse(row['metadata']['original_decision_signature_verified'])
-                self.assertEqual(row['evidence_url'], 'https://example.org/pinned.jsonl#L7')
-                self.assertNotIn('private review', json.dumps(row))
-
-    def test_workshop_label_scope_conflicts_never_become_main_events(self):
-        for year, track, label, _, _ in self.workshop_cases():
-            for enabled in (False, True):
-                for wrong_track in ('main', 'workshop_other'):
-                    self.assertEqual(normalize(self.row(track=wrong_track, status=label),
-                                               'iclr', year, self.provenance(), enabled),
-                                     (None, 'conflicting_workshop_track'))
-                for family, wrong_year in [('icml', year), ('iclr', year + 1)]:
-                    with self.assertRaisesRegex(ValueError, 'conference year'):
-                        normalize(self.row(track=track, status=label), family,
-                                  wrong_year, self.provenance(), enabled)
-        row, reason = normalize(self.row(track='workshop_unknown',
-                                        status='ICLR 2026 Workshop Unknown Reject'),
-                                'iclr', 2026, self.provenance(), True)
-        self.assertIsNone(row)
-        self.assertEqual(reason, 'unmapped_negative_label')
-
-    def test_workshop_collection_and_import_keep_five_tracks_separate_from_main(self):
-        files = {}
-        for index, (year, track, label, _, _) in enumerate(self.workshop_cases()):
-            files.setdefault(f'venues/iclr/iclr{year}.jsonl', []).append(
-                self.row(id=f'Workshop{index:02}', track=track, status=label))
-        files['venues/iclr/iclr2024.jsonl'].append(self.row())
-        with tempfile.TemporaryDirectory() as tmp:
-            root = Path(tmp)
-            default = self.collect(root / 'default', files, families=['iclr'])
-            self.assertEqual(default['normalized_records'], 1)
-            held = [json.loads(line) for line in Path(default['held_file']).read_bytes().split(b'\n') if line]
-            self.assertEqual(len(held), 5)
-            self.assertTrue(all(row['reason'] == 'separate_workshop' for row in held))
-            selected = self.collect(root / 'selected', files, families=['iclr'], include_workshops=True)
-            self.assertEqual(selected['normalized_records'], 6)
-            self.assertEqual(len(selected['files']), 6)
-            self.assertEqual(Path(selected['held_file']).read_bytes(), b'')
-            db = root / 'db.sqlite'
-            for manifest in selected['files']:
-                self.assertEqual(manifest['parser_version'], 3)
-                ingest_file(db, manifest['file'])
-            store = Store(db)
-            try:
-                scopes = set(store.conn.execute('SELECT e.year,e.track,d.status FROM submissions s '
-                                               'JOIN editions e ON e.id=s.edition_id '
-                                               'JOIN decisions d ON d.submission_id=s.id'))
-                expected = {(year, track, status) for year, track, _, status, _ in self.workshop_cases()}
-                self.assertEqual(scopes, expected | {(2024, 'main', 'rejected')})
-                self.assertEqual(store.conn.execute('PRAGMA foreign_key_check').fetchall(), [])
-            finally:
-                store.conn.close()
-
     def test_mirror_reimport_preserves_one_attempt_and_later_acceptance_elsewhere(self):
         with tempfile.TemporaryDirectory() as tmp:
             root = Path(tmp)
@@ -269,6 +152,35 @@ class PublicOpenReviewTests(unittest.TestCase):
             finally:
                 store.conn.close()
 
+    def test_unicode_separators_in_abstract_survive_audit_and_import(self):
+        for separator in ('\u0085', '\u2028', '\u2029'):
+            with self.subTest(separator=repr(separator)), tempfile.TemporaryDirectory() as tmp:
+                root = Path(tmp)
+                abstract = 'First paragraph' + separator + 'Second paragraph'
+                result = self.collect(root / 'source', {
+                    'venues/icml/icml2025.jsonl': [self.row(abstract=abstract)]})
+                path = Path(result['files'][0]['file'])
+                data = path.read_bytes()
+                self.assertIn(separator.encode(), data)
+                self.assertEqual(data.count(b'\n'), 1)
+                db = root / 'db.sqlite'
+                store = Store(db)
+                store.conn.close()
+                inventory = root / 'source/public-openreview-inventory.json'
+                before = audit(db, inventory, root / 'before.json')
+                self.assertEqual(before['status'], 'ready_for_serial_import')
+                self.assertEqual(before['records_checked'], 1)
+                ingest_file(db, path)
+                after = audit(db, inventory, root / 'after.json')
+                self.assertEqual(after['status'], 'ready_for_serial_import')
+                self.assertEqual(after['existing_same_edition_attempts'], 1)
+                store = Store(db)
+                try:
+                    self.assertEqual(store.conn.execute(
+                        'SELECT abstract FROM source_records').fetchone()[0], abstract)
+                finally:
+                    store.conn.close()
+
 
 if __name__ == '__main__':
     unittest.main()