From 7035cf33b9cabf13184703d8b3318d646b67c5fe Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Sun, 8 Nov 2026 23:46:37 +0800 Subject: [PATCH] Correct existing attempt tracks with bound evidence and refresh guards --- .gitea/workflows/check.yml | 1 + README.md | 2 + docs/edition-tracks.md | 45 +++++ paper_atlas/cli.py | 1 + paper_atlas/db/edition_scopes.py | 38 ++++ paper_atlas/db/store.py | 7 +- paper_atlas/ingestion/edition_tracks.py | 243 ++++++++++++++++++++++++ paper_atlas/ingestion/records.py | 6 + tests/test_edition_tracks.py | 135 +++++++++++++ 9 files changed, 477 insertions(+), 1 deletion(-) create mode 100644 docs/edition-tracks.md create mode 100644 paper_atlas/db/edition_scopes.py create mode 100644 paper_atlas/ingestion/edition_tracks.py create mode 100644 tests/test_edition_tracks.py diff --git a/.gitea/workflows/check.yml b/.gitea/workflows/check.yml index 66daccd..204e40a 100644 --- a/.gitea/workflows/check.yml +++ b/.gitea/workflows/check.yml @@ -49,6 +49,7 @@ jobs: subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','postgres','--help'],cwd=outside,check=True) + subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','edition-tracks','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','joint-editions','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','event-identity','--help'],cwd=outside,check=True) subprocess.run([python,'-I','-B','-m','paper_atlas','import','records','--help'],cwd=outside,check=True) diff --git a/README.md b/README.md index 920965a..f6d8487 100644 --- a/README.md +++ b/README.md @@ -93,3 +93,5 @@ CI 在已授权的远程 runner 上运行测试、构建 wheel,并从源码目 研究类型与自定义标签可通过 [`annotate`](docs/annotations.md) 显式初始化、添加和撤回;两者独立查询,保留证据及完整更正历史,不从会议轨道推断研究类型。 整合后的运行版本依赖、交互约束和生产切换顺序见 [运行版本整合](docs/research-runtime.md)。 + +已导入整卷的逐条轨道修正见 [edition-tracks 维护流程](docs/edition-tracks.md):按完整证据生成只读计划,保留原投稿和决定 ID,事务性修正 edition、标识与覆盖,并防止旧整卷刷新覆盖分类。 diff --git a/docs/edition-tracks.md b/docs/edition-tracks.md new file mode 100644 index 0000000..769538b --- /dev/null +++ b/docs/edition-tracks.md @@ -0,0 +1,45 @@ +# 按证据修正既有投稿的轨道 + +`paper-atlas maintain edition-tracks` 用于已导入整卷论文集、后来取得逐条官方分区证据的情形。它更正原投稿的 edition 归属,保留投稿 ID、所有决定和来源关系;不会重新导入论文、合并实体、生成研究方法标签或推断拒稿。支持独立 SQLite 样本和共享 PostgreSQL;不得写入已冻结的生产 SQLite 备份。 + +## 证据与完整范围 + +先核实官方目录、稳定 DOI、原始来源记录和分区,保存原始页面及 SHA256。输入 JSON 为 `{"format":1,"records":[...]}`,每条必须包含: + +| 字段 | 含义 | +| --- | --- | +| `submission_id`、`source_record_id`、`external_id` | 原有投稿、主来源记录及主来源投稿标识 | +| `source`、`doi` | 原来源名称及其 metadata.record_identifiers 中的 DOI | +| `from_edition_id`、`year`、`venue` | 原 edition 及其真实年份和规范会议名 | +| `venue_names` | 规范会议名加全部已有别名;必须与库中一致 | +| `track` | 经核实的目标轨道,使用小写字母、数字、下划线;范围未明确可用 proceedings_unspecified | +| `section` | 官方分区原文或明确的未确定说明,不推断研究方法 | +| `evidence_url`、`evidence_sha256` | 已人工/独立核实的 HTTPS 原始证据和内容哈希 | + +输入必须列出每个原 edition 的**全部**投稿,包含留在原轨道的记录。工具核对身份、DOI、单来源发表观察、现有别名、完整范围以及目标冲突;不自动解释 HTML,也不把填写了 URL/哈希当成已证明内容。核查者应先独立证明每条分区对应关系。 + +多来源共同拥有的投稿、有非 accepted 决定历史的投稿、已有分类修订、非空目标 edition、目标已有覆盖或标识等情形会停止,需要专门核对。每个原 edition 须有一个明确来源清单;其他来源或多清单覆盖需先另行协调。本功能不是任意批量移动或跨会合并工具。 + +## 只读计划、事务执行和验收 + +```sh +paper-atlas maintain edition-tracks \ + --pg-service foundry-research --pg-schema research \ + --inventory reviewed-inventory.json --out plan.json + +sha256sum plan.json + +paper-atlas maintain edition-tracks \ + --pg-service foundry-research --pg-schema research \ + --plan plan.json --plan-sha256 <上一步的完整SHA256> --out applied.json +``` + +报告路径必须是已有目录中的新文件。预检只读。执行使用 PostgreSQL 单写入 advisory lock,或 SQLite BEGIN IMMEDIATE;整个分类修正、逐来源证据、标识、覆盖及验收在一个事务中完成。计划绑定数据库身份、全部来源文本/非修订元数据、标签、可选 annotation_events、投稿/决定/来源/标识内容及原覆盖快照;过期即停止。失败回滚数据,PostgreSQL 序列可能留下正常间隙,不能用 max(id) 当数量。 + +目标 edition 沿用原会议、年份、event_key,不删除或重命名原 edition。只更正既有 submissions.edition_id 和对应 submission_identifiers.edition_id。原文、来源检索时间、决定 ID/状态/证据及各表来源关系不变。 + +分类证据写入来源元数据的保留字段 `edition_scope_curation_v1`,包括原范围、目标 edition、官方分区、URL、哈希和计划摘要。普通来源导入不能伪造或替换此字段;后续 record 刷新保留修订。相同来源投稿通过旧范围调用 decision 时仍路由到修正后的 edition。旧整卷 main 文件通过 import records 刷新会报错并回滚,要求先按核实轨道重新整理,避免覆盖正确的统计。 + +覆盖保留原清单 URL、cursor、成功时间,按实际目标分配 expected_count/imported_count 并标 partial;本工具核实分类,未核实摘要或所有来源完整性。原覆盖整行保留在计划中,不能继续将整卷数当 main 论文数。应用后校验原 edition 身份、全部受保护内容、分类、标识、修订字段和覆盖;相同计划再次执行仅在这些验收条件仍成立时返回 already_applied。 + +生产使用前必须在已确认 runner 完成软件验证。正式数据修正后仍需独立验收,并在最终全库审计中复核;单元测试和应用回执不能代替生产验收。 diff --git a/paper_atlas/cli.py b/paper_atlas/cli.py index d68e147..fefb48c 100644 --- a/paper_atlas/cli.py +++ b/paper_atlas/cli.py @@ -27,6 +27,7 @@ COMMANDS = { ('catalogue','dblp'): ('paper_atlas.collectors.dblp', {}, '盘点 DBLP 历史会议目录'), ('import','arxiv'): ('paper_atlas.ingestion.arxiv', {}, '导入 arXiv 元数据 ZIP'), ('import','records'): ('paper_atlas.ingestion.records', {}, '导入核验过的会议文件'), + ('maintain','edition-tracks'): ('paper_atlas.ingestion.edition_tracks', {}, '按逐条来源证据修正既有投稿轨道并保护刷新'), ('maintain','joint-editions'): ('paper_atlas.ingestion.joint_editions', {}, '按已核验计划修正既有联合会议届次归属'), ('maintain','event-identity'): ('paper_atlas.ingestion.event_identity', {}, '按核验计划增加同年会议事件身份'), ('maintain','merge'): ('paper_atlas.ingestion.maintenance', {}, '协调单写入合并与恢复'), diff --git a/paper_atlas/db/edition_scopes.py b/paper_atlas/db/edition_scopes.py new file mode 100644 index 0000000..1781970 --- /dev/null +++ b/paper_atlas/db/edition_scopes.py @@ -0,0 +1,38 @@ +"""Retain reviewed event scopes when a source refresh repeats a whole-volume label.""" +import json + +KEY = 'edition_scope_curation_v1' + + +def preserve(metadata, previous): + old = json.loads(previous).get(KEY) if previous else None + if KEY in metadata and metadata[KEY] != old: + raise ValueError('Source import cannot supply or replace reviewed edition scope') + if old is not None: + metadata[KEY] = old + return metadata + + +def resolve(c, record, external, venue, year, track, event_key): + row = c.execute('SELECT metadata_json FROM source_records WHERE id=?', (record,)).fetchone() + entries = json.loads(row[0]).get(KEY, []) if row else [] + if not isinstance(entries, list): + raise ValueError('Invalid reviewed edition scope') + applicable = [] + for entry in entries: + if entry['external_id'] != external: + continue + old, target = entry['from'], entry['target'] + if int(year) != old['year'] or venue not in entry['venue_names']: + continue + if track not in (old['track'], target['track']): + continue + if event_key is not None and event_key != old['event_key']: + continue + current = c.execute('SELECT venue_id,year,track,event_key FROM editions WHERE id=?', (target['edition_id'],)).fetchone() + if not current or [current[i] for i in range(4)] != [target[k] for k in ('venue_id','year','track','event_key')]: + raise ValueError('Reviewed target edition has changed') + applicable.append(target['edition_id']) + if len(applicable) > 1: + raise ValueError('Ambiguous reviewed edition scope') + return applicable[0] if applicable else None diff --git a/paper_atlas/db/store.py b/paper_atlas/db/store.py index 17d187b..e625ca1 100644 --- a/paper_atlas/db/store.py +++ b/paper_atlas/db/store.py @@ -96,6 +96,8 @@ class Store: if found: matches.add(found[0]) old = c.execute('SELECT paper_id,title,abstract,retrieved_at,metadata_json FROM source_records WHERE source_id=? AND external_id=?',(source_id:=self.source(source,url),external_id)).fetchone() + from paper_atlas.db.edition_scopes import preserve + metadata = preserve(dict(metadata or {}), old[4] if old else None) if old: matches.add(old[0]) key = normalized(title) @@ -175,7 +177,10 @@ class Store: for alias in aliases or []: c.execute('INSERT INTO venue_aliases VALUES(?,?) ON CONFLICT DO NOTHING',(venue_id,alias)) from paper_atlas.db.events import find_edition - edition = find_edition(self,venue_id,venue,int(year),track,record,external_id,event_key) + from paper_atlas.db.edition_scopes import resolve + edition = resolve(c,record,external_id,venue,year,track,event_key) + if edition is None: + edition = find_edition(self,venue_id,venue,int(year),track,record,external_id,event_key) # A previous event in this row may already have merged the input paper. paper=c.execute('SELECT paper_id FROM source_records WHERE id=?',(record,)).fetchone()[0] submission=attach_submission(self,edition,paper,record,external_id,identities or []) diff --git a/paper_atlas/ingestion/edition_tracks.py b/paper_atlas/ingestion/edition_tracks.py new file mode 100644 index 0000000..ce39584 --- /dev/null +++ b/paper_atlas/ingestion/edition_tracks.py @@ -0,0 +1,243 @@ +"""Snapshot-bound reclassification of existing attempts, without reimport or merge.""" +import argparse +from collections import Counter +from contextlib import closing +import hashlib +import json +from pathlib import Path +import re + +from paper_atlas.db.backend import database_arguments, open_read_connection, open_store, inserted_id +from paper_atlas.db.edition_scopes import KEY +from paper_atlas.ingestion.event_identity import database_identity, digest, values + +MARKER = 'edition_tracks_v1:' + + +def validate(inventory): + if (not isinstance(inventory, dict) or inventory.get('format') != 1 + or not isinstance(inventory.get('records'), list) or not 0 < len(inventory['records']) <= 10000): + raise ValueError('Nonempty bounded reviewed inventory required') + ids = set(); records = set() + for row in inventory['records']: + required = {'submission_id','source_record_id','external_id','source','doi','from_edition_id', + 'year','venue','venue_names','track','section','evidence_url','evidence_sha256'} + if not isinstance(row, dict) or set(row) != required: + raise ValueError('Unexpected inventory record fields') + for field in ('submission_id','source_record_id','from_edition_id','year'): + if type(row[field]) is not int or row[field] <= 0: + raise ValueError('Positive integer inventory identity required') + if row['submission_id'] in ids or row['source_record_id'] in records: + raise ValueError('Repeated submission/source identity in inventory') + ids.add(row['submission_id']); records.add(row['source_record_id']) + for field in ('external_id','source','doi','venue','section','evidence_url'): + if not isinstance(row[field], str) or not row[field].strip(): + raise ValueError('Missing inventory evidence field') + if (not isinstance(row['venue_names'], list) or not row['venue_names'] + or not all(isinstance(v,str) and v for v in row['venue_names']) + or row['venue'] not in row['venue_names'] + or not re.fullmatch(r'[a-z][a-z0-9_]{0,63}', row['track']) + or not re.fullmatch(r'[0-9a-f]{64}', row['evidence_sha256']) + or not row['evidence_url'].startswith('https://')): + raise ValueError('Invalid scope or evidence binding') + return inventory + + +def bounded(c, sql, params): + rows = values(c.execute(sql+' LIMIT 100001', params)) + if len(rows) > 100000: + raise ValueError('Protected scope exceeds bounded reclassification limit') + return rows + + +def protect(c, ids): + marks = ','.join('?' for _ in ids) + subs = 'SELECT id FROM submissions WHERE id IN ('+marks+')' + papers = 'SELECT paper_id FROM submissions WHERE id IN ('+marks+')' + decisions = 'SELECT id FROM decisions WHERE submission_id IN ('+subs+')' + queries = { + 'submissions': ('SELECT id,paper_id,source_record_id,source_submission_id,submitted_at FROM submissions WHERE id IN ('+marks+') ORDER BY id'), + 'submission_identifiers': ('SELECT scheme,value,submission_id FROM submission_identifiers WHERE submission_id IN ('+subs+') ORDER BY submission_id,scheme,value'), + 'submission_sources': ('SELECT * FROM submission_sources WHERE submission_id IN ('+subs+') ORDER BY submission_id,source_record_id,source_submission_id'), + 'decisions': ('SELECT * FROM decisions WHERE submission_id IN ('+subs+') ORDER BY id'), + 'decision_sources': ('SELECT * FROM decision_sources WHERE decision_id IN ('+decisions+') ORDER BY decision_id,source_record_id,source_decision_id'), + 'papers': ('SELECT * FROM papers WHERE id IN ('+papers+') ORDER BY id'), + 'identifiers': ('SELECT * FROM identifiers WHERE paper_id IN ('+papers+') ORDER BY scheme,value'), + 'paper_tags': ('SELECT * FROM paper_tags WHERE paper_id IN ('+papers+') ORDER BY paper_id,tag_id,evidence_key'), + } + from paper_atlas.curation.annotations import available + if available(c): + queries['annotation_events'] = 'SELECT * FROM annotation_events WHERE paper_id IN ('+papers+') ORDER BY id' + result = {table: {'rows':len(rows), 'sha256':digest(rows)} for table, sql in queries.items() + for rows in [bounded(c,sql,ids)]} + rows = bounded(c, 'SELECT r.* FROM source_records r WHERE r.id IN (SELECT source_record_id FROM submission_sources WHERE submission_id IN ('+subs+')) ORDER BY r.id', ids) + # Only the reviewed curation field may change; every other metadata value + # and all raw text/IDs/retrieval times remain protected. + for row in rows: + metadata = json.loads(row[8]); metadata.pop(KEY,None); row[8] = metadata + result['source_records'] = {'rows':len(rows), 'sha256':digest(rows)} + return result + + +def inspect(c, inventory): + validate(inventory); changes=[]; ids=tuple(sorted(r['submission_id']for r in inventory['records'])) + original = sorted({r['from_edition_id'] for r in inventory['records']}) + actual_ids = {r[0] for r in c.execute('SELECT id FROM submissions WHERE edition_id IN ('+','.join('?'for _ in original)+')', original)} + if actual_ids != set(ids): + raise ValueError('Inventory must cover every attempt in each original edition') + for row in inventory['records']: + current = c.execute('''SELECT s.edition_id,s.source_record_id,s.source_submission_id,s.paper_id, + e.venue_id,e.year,e.track,e.event_key,v.acronym,r.external_id,r.metadata_json,src.name + FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id + JOIN source_records r ON r.id=s.source_record_id JOIN sources src ON src.id=r.source_id WHERE s.id=?''',(row['submission_id'],)).fetchone() + if not current or [current[i]for i in (0,1,2,5,8,9,11)] != [row[k]for k in ('from_edition_id','source_record_id','external_id','year','venue','external_id','source')]: + raise ValueError('Submission identity/scope differs from inventory') + aliases={r[0]for r in c.execute('SELECT alias FROM venue_aliases WHERE venue_id=?',(current[4],))}|{current[8]} + if set(row['venue_names'])!=aliases: + raise ValueError('Venue names differ from current aliases') + metadata=json.loads(current[10]) + if KEY in metadata: + raise ValueError('Existing curated scope requires explicit reconciliation') + if (metadata.get('record_identifiers') or {}).get('doi') != row['doi']: + raise ValueError('Stable DOI differs from reviewed source evidence') + links=values(c.execute('SELECT source_record_id,source_submission_id FROM submission_sources WHERE submission_id=?',(row['submission_id'],))) + if links != [[row['source_record_id'],row['external_id']]]: + raise ValueError('Shared-source attempt requires separate reconciliation') + decisions=values(c.execute('SELECT status FROM decisions WHERE submission_id=?',(row['submission_id'],))) + if not decisions or any(d[0]!='accepted'for d in decisions): + raise ValueError('Only publication observations can use this track correction') + target=c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=? AND event_key=?',(current[4],current[5],row['track'],current[7])).fetchone() + if target and target[0] != current[0]: + if c.execute('SELECT 1 FROM submissions WHERE edition_id=? LIMIT 1',(target[0],)).fetchone(): + raise ValueError('Nonempty target edition requires separate reconciliation') + if c.execute('SELECT 1 FROM submission_identifiers WHERE edition_id=? LIMIT 1',(target[0],)).fetchone(): + raise ValueError('Target identifier scope already exists') + if c.execute('SELECT 1 FROM edition_sources WHERE edition_id=? LIMIT 1',(target[0],)).fetchone(): + raise ValueError('Target coverage already exists; reconcile first') + if c.execute('SELECT 1 FROM submission_identifiers WHERE submission_id=? AND edition_id<>? LIMIT 1',(row['submission_id'],current[0])).fetchone(): + raise ValueError('Existing identifier edition is inconsistent') + changes.append({**row,'from_track':current[6],'event_key':current[7], 'venue_id':current[4], + 'target_edition_id':target[0]if target else None}) + coverage=bounded(c,'SELECT * FROM edition_sources WHERE edition_id IN ('+','.join('?'for _ in original)+') ORDER BY edition_id,source_id,list_url',original) + if Counter(row[0]for row in coverage)!=Counter({edition:1 for edition in original}): + raise ValueError('Exactly one original coverage listing per edition required') + return {'records':changes, 'protected':protect(c,ids), 'coverage':coverage, + 'editions':bounded(c,'SELECT * FROM editions WHERE id IN ('+','.join('?'for _ in original)+') ORDER BY id',original)} + + +def proposal(c, inventory, identity): + return {'format':1,'database':identity,'inventory':inventory,'before':inspect(c,inventory)} + + +def verify(c, plan): + records=plan['before']['records']; ids=tuple(sorted(r['submission_id']for r in records)) + if protect(c,ids)!=plan['before']['protected']: + raise ValueError('Protected history/text/tags/identities changed; roll back') + for row in records: + event=c.execute('''SELECT e.id,e.venue_id,e.year,e.track,e.event_key FROM submissions s JOIN editions e ON e.id=s.edition_id WHERE s.id=?''',(row['submission_id'],)).fetchone() + if not event or [event[i]for i in range(1,5)] != [row['venue_id'],row['year'],row['track'],row['event_key']]: + raise ValueError('Target track assignment failed') + if c.execute('SELECT 1 FROM submission_identifiers WHERE submission_id=? AND edition_id<>? LIMIT 1',(row['submission_id'],event[0])).fetchone(): + raise ValueError('Target identifier assignment failed') + if row['track'] != row['from_track']: + metadata=json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(row['source_record_id'],)).fetchone()[0]) + if metadata.get(KEY) != [curation(row,event[0],digest(plan))]: + raise ValueError('Reviewed refresh protection missing or changed') + original=sorted({r['from_edition_id']for r in records}) + if bounded(c,'SELECT * FROM editions WHERE id IN ('+','.join('?'for _ in original)+') ORDER BY id',original)!=plan['before']['editions']: + raise ValueError('Original edition identities changed') + for old in plan['before']['coverage']: + edition,source,url,_,_,_,cursor,success,_=old + expected=Counter() + for row in records: + if row['from_edition_id']==edition: + target=c.execute('SELECT edition_id FROM submissions WHERE id=?',(row['submission_id'],)).fetchone()[0] + expected[target]+=1 + for target,number in expected.items(): + actual=c.execute('SELECT status,expected_count,imported_count,cursor,last_success_at,last_error FROM edition_sources WHERE edition_id=? AND source_id=? AND list_url=?',(target,source,url)).fetchone() + note='Track scope corrected from reviewed per-record evidence; original whole-volume coverage retained in plan '+digest(plan) + if actual is None or [actual[i]for i in range(6)]!=['partial',number,number,cursor,success,note]: + raise ValueError('Corrected coverage differs from reviewed scope') + return {'records_verified':len(records),'protected':plan['before']['protected']} + + +def curation(row, target, signature): + return {'external_id':row['external_id'], 'venue_names':sorted(set(row['venue_names'])), + 'from':{'edition_id':row['from_edition_id'],'year':row['year'],'track':row['from_track'],'event_key':row['event_key']}, + 'target':{'edition_id':target,'venue_id':row['venue_id'],'year':row['year'],'track':row['track'],'event_key':row['event_key']}, + 'section':row['section'],'evidence_url':row['evidence_url'],'evidence_sha256':row['evidence_sha256'],'plan_digest':signature} + + +def apply_plan(c, plan): + if plan.get('format') != 1: + raise ValueError('Unsupported plan format') + signature=digest(plan);marker=MARKER+signature + if c.execute('SELECT 1 FROM schema_migrations WHERE name=?',(marker,)).fetchone(): + return {'status':'already_applied','events_moved':0,**verify(c,plan)} + if inspect(c,plan['inventory']) != plan['before']: + raise ValueError('Stale track plan; create and review a fresh plan') + targets={}; counts=Counter(); moved=0 + for row in plan['before']['records']: + key=(row['venue_id'],row['year'],row['track'],row['event_key']) + if key not in targets: + found=c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=? AND event_key=?',key).fetchone() + targets[key]=found[0]if found else inserted_id(c,'INSERT INTO editions(venue_id,year,track,event_key) VALUES(?,?,?,?)',key) + target=targets[key];counts[(row['from_edition_id'],target)]+=1 + if target != row['from_edition_id']: + c.execute('UPDATE submission_identifiers SET edition_id=? WHERE submission_id=?',(target,row['submission_id'])) + if c.execute('UPDATE submissions SET edition_id=? WHERE id=? AND edition_id=?',(target,row['submission_id'],row['from_edition_id'])).rowcount!=1: + raise ValueError('Concurrent submission change') + metadata=json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(row['source_record_id'],)).fetchone()[0]);metadata[KEY]=[curation(row,target,signature)] + c.execute('UPDATE source_records SET metadata_json=? WHERE id=?',(json.dumps(metadata,ensure_ascii=False),row['source_record_id']));moved+=1 + # Preserve each original source URL and timestamps, but do not retain its + # old whole-volume count as main-track coverage. Partial is deliberate: + # this operation verifies scope, not abstracts or all possible sources. + for old in plan['before']['coverage']: + edition,source,url,status,expected,imported,cursor,success,error=old + own={r['submission_id'] for r in plan['before']['records']if r['from_edition_id']==edition} + source_name=c.execute('SELECT name FROM sources WHERE id=?',(source,)).fetchone()[0] + if {r['source']for r in plan['before']['records']if r['submission_id']in own}!={source_name}: + raise ValueError('Coverage includes a source outside the reviewed inventory') + for (origin,target),number in counts.items(): + if origin!=edition:continue + note='Track scope corrected from reviewed per-record evidence; original whole-volume coverage retained in plan '+signature + c.execute('''INSERT INTO edition_sources(edition_id,source_id,list_url,status,expected_count,imported_count,cursor,last_success_at,last_error) + VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(edition_id,source_id,list_url) DO UPDATE SET status=excluded.status, + expected_count=excluded.expected_count,imported_count=excluded.imported_count,last_error=excluded.last_error''', + (target,source,url,'partial',number,number,cursor,success,note)) + result=verify(c,plan) + c.execute('INSERT INTO schema_migrations(name) VALUES(?)',(marker,)) + return {'status':'applied','events_moved':moved,**result} + + +def run(db,out,*,pg_service=None,pg_schema=None,inventory=None,plan=None,plan_sha256=None): + out=Path(out) + if out.exists() or not out.parent.is_dir():raise ValueError('New report path in existing directory required') + expected=None + if plan: + raw=Path(plan).read_bytes() + if hashlib.sha256(raw).hexdigest()!=plan_sha256:raise ValueError('Plan hash differs') + expected=json.loads(raw);validate(expected['inventory']) + elif plan_sha256:raise ValueError('Plan path required') + if bool(expected)==bool(inventory):raise ValueError('Choose inventory preflight or apply plan') + if expected is None: + with closing(open_read_connection(db,pg_service=pg_service,pg_schema=pg_schema))as c: + result=proposal(c,validate(json.loads(Path(inventory).read_bytes())),database_identity(c,db)) + else: + store=open_store(db,pg_service=pg_service,pg_schema=pg_schema) + with closing(store.conn)as c: + try: + if expected['database']!=database_identity(c,db):raise ValueError('Plan belongs to another database') + if not getattr(c,'postgres',False):c.commit();c.execute('BEGIN IMMEDIATE') + result=apply_plan(c,expected);c.commit() + except BaseException:c.rollback();raise + out.write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n');return result + + +def main(argv=None,*,prog=None): + p=argparse.ArgumentParser(prog=prog);database_arguments(p) + p.add_argument('--inventory');p.add_argument('--plan');p.add_argument('--plan-sha256');p.add_argument('--out',required=True) + a=p.parse_args(argv) + if bool(a.plan)!=bool(a.plan_sha256):p.error('--plan and --plan-sha256 must be supplied together') + r=run(a.db,a.out,pg_service=a.pg_service,pg_schema=a.pg_schema,inventory=a.inventory,plan=a.plan,plan_sha256=a.plan_sha256) + print(json.dumps({'status':r.get('status','planned'),'events_moved':r.get('events_moved',0),'out':a.out}),flush=True) diff --git a/paper_atlas/ingestion/records.py b/paper_atlas/ingestion/records.py index 63898ca..5b03604 100644 --- a/paper_atlas/ingestion/records.py +++ b/paper_atlas/ingestion/records.py @@ -134,6 +134,12 @@ def ingest_file(db,path, *, pg_service=None, pg_schema=None): if events and status is None:raise ValueError('Explicit decision event requires a status') if status is not None and status not in ('accepted','rejected','desk_rejected','withdrawn','under_review','unknown'):raise ValueError('Unsupported decision event') edition=store.decision(paper,record,row['venue'],row['year'],row['track'],external,event.get('evidence_url',row['url']),status,event.get('raw_value',''),row.get('venue_aliases'),identities=identities,observed_at=event.get('observed_at') or row_observed,decision_date=event.get('decision_date'),source_decision_id=event.get('source_decision_id',''),submitted_at=row.get('submitted_at'),guard_status=source_name in (PUBLIC_SOURCE,ARCHIVE_SOURCE),event_key=row.get('event_key')) + from paper_atlas.db.edition_scopes import KEY + scope_metadata=json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(record,)).fetchone()[0]) + if scope_metadata.get(KEY): + actual_track=c.execute('SELECT track FROM editions WHERE id=?',(edition,)).fetchone()[0] + if row['track']!=actual_track: + raise ValueError('Recollect this whole-volume source with reviewed per-record tracks before refresh') editions.add(edition) else: raise ValueError('This importer requires an explicit supported decision status') diff --git a/tests/test_edition_tracks.py b/tests/test_edition_tracks.py new file mode 100644 index 0000000..8d10273 --- /dev/null +++ b/tests/test_edition_tracks.py @@ -0,0 +1,135 @@ +"""Stable-attempt correction, rollback and stale whole-volume refresh on both engines.""" +from contextlib import closing, redirect_stdout +import copy +import hashlib +import io +import json +import os +from pathlib import Path +import tempfile +import unittest +import uuid + +from paper_atlas.db.backend import open_store +from paper_atlas.db.edition_scopes import KEY +from paper_atlas.db.store import Store +from paper_atlas.ingestion.edition_tracks import proposal,apply_plan,verify,validate,run +from paper_atlas.ingestion.event_identity import database_identity,values +from paper_atlas.ingestion.records import ingest_file +from paper_atlas.postgres.client import connect +from paper_atlas.postgres.migration import migrate +from paper_atlas.postgres.schema import quote + + +def seed(db): + s=Store(db); rows=[] + for i,track in enumerate(['main','survey','doctoral_consortium','proceedings_unspecified'],1): + external='entry'+str(i);doi='10.1234/entry'+str(i) + p,r=s.record('fixture',external,'https://example.org/'+external,'Publication '+external, + 'Abstract with NUL\x00 and SOH\x01',[],{'doi':doi}) + eid=s.decision(p,r,'JOINT',2022,'main',external,'https://example.org/'+external,'accepted','published',identities=[('doi',doi)],observed_at='2022-01-01T00:00:00Z') + sid=s.conn.execute('SELECT id FROM submissions WHERE source_record_id=?',(r,)).fetchone()[0] + s.record_tags(p,r,'fixture',[{'namespace':'fixture:subject','value':'Original'}]) + rows.append(dict(submission_id=sid,source_record_id=r,external_id=external,source='fixture',doi=doi, + from_edition_id=eid,year=2022,venue='JOINT',venue_names=['JOINT'],track=track, + section=track,evidence_url='https://example.org/proceedings/2022/',evidence_sha256='a'*64)) + source=s.conn.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0] + s.conn.execute("INSERT INTO edition_sources VALUES(?,?,?,'complete',4,4,'checkpoint','2022-01-01T00:00:00Z',NULL)",(eid,source,'https://example.org/volume')) + s.conn.commit();s.conn.close();return {'format':1,'records':rows} + + +class TrackCases: + def store(self):return open_store(**self.target) + + def plan(self,c):return proposal(c,self.inventory,database_identity(c,self.target.get('db'))) + + def test_move_preserves_all_history_tags_text_and_stable_identities(self): + with closing(self.store().conn)as c: + plan=self.plan(c);before=plan['before']['protected'];r=apply_plan(c,plan);c.commit() + self.assertEqual((r['events_moved'],r['records_verified']),(3,4)) + self.assertEqual(r['protected'],before) + self.assertEqual(apply_plan(c,plan)['status'],'already_applied') + self.assertEqual(values(c.execute('SELECT status,expected_count,imported_count FROM edition_sources ORDER BY edition_id')),[['partial',1,1]]*4) + self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],4) + self.assertEqual(c.execute('SELECT count(*) FROM decisions').fetchone()[0],4) + self.assertEqual(c.execute('SELECT count(*) FROM paper_merges').fetchone()[0],0) + + def test_source_record_refresh_keeps_scope_and_original_attempt(self): + with closing(self.store().conn)as c: + plan=self.plan(c);apply_plan(c,plan);c.commit() + with closing(self.store().conn)as c: + # Use Store methods with the same writer connection as production. + s=object.__new__(Store);s.conn=c;s.event_identity=True + r=self.inventory['records'][1];metadata=json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(r['source_record_id'],)).fetchone()[0]);saved=metadata[KEY] + p,record=s.record('fixture',r['external_id'],'https://example.org/new','Refreshed title',None,[],{'doi':r['doi']},{'new_source_field':True}) + eid=s.decision(p,record,'JOINT',2022,'main',r['external_id'],'https://example.org/entry2','accepted','published',identities=[('doi',r['doi'])],observed_at='2022-01-01T00:00:00Z') + self.assertEqual(c.execute('SELECT track FROM editions WHERE id=?',(eid,)).fetchone()[0],'survey') + self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],4) + self.assertEqual(json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(record,)).fetchone()[0])[KEY],saved) + with self.assertRaisesRegex(ValueError,'cannot supply'): + s.record('fixture',r['external_id'],'https://example.org/new','Forged',None,[],metadata={KEY:[]}) + c.rollback() + + def test_stale_snapshot_and_partial_inventory_are_rejected(self): + with closing(self.store().conn)as c: + plan=self.plan(c) + c.execute("UPDATE papers SET title='Changed' WHERE id=(SELECT paper_id FROM submissions WHERE id=?)",(self.inventory['records'][0]['submission_id'],)) + with self.assertRaisesRegex(ValueError,'Stale'):apply_plan(c,plan) + c.rollback() + bad=copy.deepcopy(self.inventory);bad['records'].pop() + with self.assertRaisesRegex(ValueError,'every attempt'):proposal(c,bad,database_identity(c,self.target.get('db'))) + bad=copy.deepcopy(self.inventory);bad['records'][0]['doi']='10.1234/wrong' + with self.assertRaisesRegex(ValueError,'DOI'):proposal(c,bad,database_identity(c,self.target.get('db'))) + + def test_failure_after_updates_rolls_back_all_scope_and_metadata_changes(self): + with closing(self.store().conn)as c: + plan=self.plan(c) + # A late coverage failure must roll back earlier moved attempts. + source=c.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0] + c.execute("INSERT INTO sources(name,homepage) VALUES('other','https://example.org')") + other=c.execute("SELECT id FROM sources WHERE name='other'").fetchone()[0] + c.execute('UPDATE edition_sources SET source_id=? WHERE source_id=?',(other,source));c.commit() + plan=self.plan(c) + try: + with self.assertRaisesRegex(ValueError,'outside'):apply_plan(c,plan) + finally:c.rollback() + self.assertEqual(self.plan(c)['before'],plan['before']) + self.assertEqual(c.execute('SELECT count(*) FROM editions').fetchone()[0],1) + + def test_import_old_whole_volume_fails_without_corrupting_coverage(self): + with closing(self.store().conn)as c: + plan=self.plan(c);apply_plan(c,plan);c.commit() + r=self.inventory['records'][1];path=self.root/'old.jsonl' + row=dict(source='fixture',external_id=r['external_id'],url='https://example.org/entry2',title='Old title',authors=[],identifiers={'doi':r['doi']},venue='JOINT',year=2022,track='main',status='accepted') + path.write_text(json.dumps(row)+'\n');path.with_suffix('.manifest.json').write_text(json.dumps(dict(source='fixture',venue='JOINT',year=2022,track='main',count=1,list_url='https://example.org/volume',status='complete'))) + with self.assertRaisesRegex(ValueError,'Recollect'):ingest_file(self.target.get('db'),path,**{k:v for k,v in self.target.items()if k!='db'}) + with closing(self.store().conn)as c:verify(c,plan) + + def test_cli_hash_binding_preflight_and_duplicate_inventory(self): + inventory=self.root/'inventory.json';inventory.write_text(json.dumps(self.inventory)) + planfile=self.root/'plan.json';p=run(self.target.get('db'),planfile,inventory=inventory,**{k:v for k,v in self.target.items()if k!='db'}) + with self.assertRaisesRegex(ValueError,'hash'):run(self.target.get('db'),self.root/'bad.json',plan=planfile,plan_sha256='0'*64,**{k:v for k,v in self.target.items()if k!='db'}) + report=run(self.target.get('db'),self.root/'applied.json',plan=planfile,plan_sha256=hashlib.sha256(planfile.read_bytes()).hexdigest(),**{k:v for k,v in self.target.items()if k!='db'}) + self.assertEqual(report['events_moved'],3) + bad=copy.deepcopy(self.inventory);bad['records'].append(bad['records'][0]) + with self.assertRaisesRegex(ValueError,'Repeated'):validate(bad) + + +class SQLiteTrackTests(TrackCases,unittest.TestCase): + def setUp(self): + t=tempfile.TemporaryDirectory();self.addCleanup(t.cleanup);self.root=Path(t.name);db=self.root/'source.sqlite' + self.inventory=seed(db);self.target={'db':db} + + +@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'),'Requires approved remote PostgreSQL service') +class PostgreSQLTrackTests(TrackCases,unittest.TestCase): + def setUp(self): + t=tempfile.TemporaryDirectory();self.addCleanup(t.cleanup);self.root=Path(t.name);db=self.root/'source.sqlite';self.inventory=seed(db) + self.service=os.environ['PAPER_ATLAS_TEST_PG_SERVICE'];self.schema='pa_tracks_'+uuid.uuid4().hex[:12];self.addCleanup(self.cleanup) + with redirect_stdout(io.StringIO()):migrate(db,self.service,self.root/'migration.json',schema=self.schema,min_free_gib=0) + self.target={'pg_service':self.service,'pg_schema':self.schema} + + def cleanup(self): + with connect(self.service)as c: + for name, in c.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)',(self.schema,)): + c.execute('DROP SCHEMA '+quote(name)+' CASCADE')