Correct existing attempt tracks with bound evidence and refresh guards
Metadata integrity checks / metadata (push) Successful in 1m59s

This commit is contained in:
2026-11-08 23:46:37 +08:00
parent a1bbd26a5c
commit 7035cf33b9
9 changed files with 477 additions and 1 deletions
+1
View File
@@ -49,6 +49,7 @@ jobs:
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','postgres','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','edition-tracks','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','joint-editions','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','event-identity','--help'],cwd=outside,check=True)
subprocess.run([python,'-I','-B','-m','paper_atlas','import','records','--help'],cwd=outside,check=True)
+2
View File
@@ -93,3 +93,5 @@ CI 在已授权的远程 runner 上运行测试、构建 wheel,并从源码目
研究类型与自定义标签可通过 [`annotate`](docs/annotations.md) 显式初始化、添加和撤回;两者独立查询,保留证据及完整更正历史,不从会议轨道推断研究类型。
整合后的运行版本依赖、交互约束和生产切换顺序见 [运行版本整合](docs/research-runtime.md)。
已导入整卷的逐条轨道修正见 [edition-tracks 维护流程](docs/edition-tracks.md):按完整证据生成只读计划,保留原投稿和决定 ID,事务性修正 edition、标识与覆盖,并防止旧整卷刷新覆盖分类。
+45
View File
@@ -0,0 +1,45 @@
# 按证据修正既有投稿的轨道
`paper-atlas maintain edition-tracks` 用于已导入整卷论文集、后来取得逐条官方分区证据的情形。它更正原投稿的 edition 归属,保留投稿 ID、所有决定和来源关系;不会重新导入论文、合并实体、生成研究方法标签或推断拒稿。支持独立 SQLite 样本和共享 PostgreSQL;不得写入已冻结的生产 SQLite 备份。
## 证据与完整范围
先核实官方目录、稳定 DOI、原始来源记录和分区,保存原始页面及 SHA256。输入 JSON 为 `{"format":1,"records":[...]}`,每条必须包含:
| 字段 | 含义 |
| --- | --- |
| `submission_id`、`source_record_id`、`external_id` | 原有投稿、主来源记录及主来源投稿标识 |
| `source`、`doi` | 原来源名称及其 metadata.record_identifiers 中的 DOI |
| `from_edition_id`、`year`、`venue` | 原 edition 及其真实年份和规范会议名 |
| `venue_names` | 规范会议名加全部已有别名;必须与库中一致 |
| `track` | 经核实的目标轨道,使用小写字母、数字、下划线;范围未明确可用 proceedings_unspecified |
| `section` | 官方分区原文或明确的未确定说明,不推断研究方法 |
| `evidence_url`、`evidence_sha256` | 已人工/独立核实的 HTTPS 原始证据和内容哈希 |
输入必须列出每个原 edition 的**全部**投稿,包含留在原轨道的记录。工具核对身份、DOI、单来源发表观察、现有别名、完整范围以及目标冲突;不自动解释 HTML,也不把填写了 URL/哈希当成已证明内容。核查者应先独立证明每条分区对应关系。
多来源共同拥有的投稿、有非 accepted 决定历史的投稿、已有分类修订、非空目标 edition、目标已有覆盖或标识等情形会停止,需要专门核对。每个原 edition 须有一个明确来源清单;其他来源或多清单覆盖需先另行协调。本功能不是任意批量移动或跨会合并工具。
## 只读计划、事务执行和验收
```sh
paper-atlas maintain edition-tracks \
--pg-service foundry-research --pg-schema research \
--inventory reviewed-inventory.json --out plan.json
sha256sum plan.json
paper-atlas maintain edition-tracks \
--pg-service foundry-research --pg-schema research \
--plan plan.json --plan-sha256 <上一步的完整SHA256> --out applied.json
```
报告路径必须是已有目录中的新文件。预检只读。执行使用 PostgreSQL 单写入 advisory lock,或 SQLite BEGIN IMMEDIATE;整个分类修正、逐来源证据、标识、覆盖及验收在一个事务中完成。计划绑定数据库身份、全部来源文本/非修订元数据、标签、可选 annotation_events、投稿/决定/来源/标识内容及原覆盖快照;过期即停止。失败回滚数据,PostgreSQL 序列可能留下正常间隙,不能用 max(id) 当数量。
目标 edition 沿用原会议、年份、event_key,不删除或重命名原 edition。只更正既有 submissions.edition_id 和对应 submission_identifiers.edition_id。原文、来源检索时间、决定 ID/状态/证据及各表来源关系不变。
分类证据写入来源元数据的保留字段 `edition_scope_curation_v1`,包括原范围、目标 edition、官方分区、URL、哈希和计划摘要。普通来源导入不能伪造或替换此字段;后续 record 刷新保留修订。相同来源投稿通过旧范围调用 decision 时仍路由到修正后的 edition。旧整卷 main 文件通过 import records 刷新会报错并回滚,要求先按核实轨道重新整理,避免覆盖正确的统计。
覆盖保留原清单 URL、cursor、成功时间,按实际目标分配 expected_count/imported_count 并标 partial;本工具核实分类,未核实摘要或所有来源完整性。原覆盖整行保留在计划中,不能继续将整卷数当 main 论文数。应用后校验原 edition 身份、全部受保护内容、分类、标识、修订字段和覆盖;相同计划再次执行仅在这些验收条件仍成立时返回 already_applied。
生产使用前必须在已确认 runner 完成软件验证。正式数据修正后仍需独立验收,并在最终全库审计中复核;单元测试和应用回执不能代替生产验收。
+1
View File
@@ -27,6 +27,7 @@ COMMANDS = {
('catalogue','dblp'): ('paper_atlas.collectors.dblp', {}, '盘点 DBLP 历史会议目录'),
('import','arxiv'): ('paper_atlas.ingestion.arxiv', {}, '导入 arXiv 元数据 ZIP'),
('import','records'): ('paper_atlas.ingestion.records', {}, '导入核验过的会议文件'),
('maintain','edition-tracks'): ('paper_atlas.ingestion.edition_tracks', {}, '按逐条来源证据修正既有投稿轨道并保护刷新'),
('maintain','joint-editions'): ('paper_atlas.ingestion.joint_editions', {}, '按已核验计划修正既有联合会议届次归属'),
('maintain','event-identity'): ('paper_atlas.ingestion.event_identity', {}, '按核验计划增加同年会议事件身份'),
('maintain','merge'): ('paper_atlas.ingestion.maintenance', {}, '协调单写入合并与恢复'),
+38
View File
@@ -0,0 +1,38 @@
"""Retain reviewed event scopes when a source refresh repeats a whole-volume label."""
import json
KEY = 'edition_scope_curation_v1'
def preserve(metadata, previous):
old = json.loads(previous).get(KEY) if previous else None
if KEY in metadata and metadata[KEY] != old:
raise ValueError('Source import cannot supply or replace reviewed edition scope')
if old is not None:
metadata[KEY] = old
return metadata
def resolve(c, record, external, venue, year, track, event_key):
row = c.execute('SELECT metadata_json FROM source_records WHERE id=?', (record,)).fetchone()
entries = json.loads(row[0]).get(KEY, []) if row else []
if not isinstance(entries, list):
raise ValueError('Invalid reviewed edition scope')
applicable = []
for entry in entries:
if entry['external_id'] != external:
continue
old, target = entry['from'], entry['target']
if int(year) != old['year'] or venue not in entry['venue_names']:
continue
if track not in (old['track'], target['track']):
continue
if event_key is not None and event_key != old['event_key']:
continue
current = c.execute('SELECT venue_id,year,track,event_key FROM editions WHERE id=?', (target['edition_id'],)).fetchone()
if not current or [current[i] for i in range(4)] != [target[k] for k in ('venue_id','year','track','event_key')]:
raise ValueError('Reviewed target edition has changed')
applicable.append(target['edition_id'])
if len(applicable) > 1:
raise ValueError('Ambiguous reviewed edition scope')
return applicable[0] if applicable else None
+6 -1
View File
@@ -96,6 +96,8 @@ class Store:
if found:
matches.add(found[0])
old = c.execute('SELECT paper_id,title,abstract,retrieved_at,metadata_json FROM source_records WHERE source_id=? AND external_id=?',(source_id:=self.source(source,url),external_id)).fetchone()
from paper_atlas.db.edition_scopes import preserve
metadata = preserve(dict(metadata or {}), old[4] if old else None)
if old:
matches.add(old[0])
key = normalized(title)
@@ -175,7 +177,10 @@ class Store:
for alias in aliases or []:
c.execute('INSERT INTO venue_aliases VALUES(?,?) ON CONFLICT DO NOTHING',(venue_id,alias))
from paper_atlas.db.events import find_edition
edition = find_edition(self,venue_id,venue,int(year),track,record,external_id,event_key)
from paper_atlas.db.edition_scopes import resolve
edition = resolve(c,record,external_id,venue,year,track,event_key)
if edition is None:
edition = find_edition(self,venue_id,venue,int(year),track,record,external_id,event_key)
# A previous event in this row may already have merged the input paper.
paper=c.execute('SELECT paper_id FROM source_records WHERE id=?',(record,)).fetchone()[0]
submission=attach_submission(self,edition,paper,record,external_id,identities or [])
+243
View File
@@ -0,0 +1,243 @@
"""Snapshot-bound reclassification of existing attempts, without reimport or merge."""
import argparse
from collections import Counter
from contextlib import closing
import hashlib
import json
from pathlib import Path
import re
from paper_atlas.db.backend import database_arguments, open_read_connection, open_store, inserted_id
from paper_atlas.db.edition_scopes import KEY
from paper_atlas.ingestion.event_identity import database_identity, digest, values
MARKER = 'edition_tracks_v1:'
def validate(inventory):
if (not isinstance(inventory, dict) or inventory.get('format') != 1
or not isinstance(inventory.get('records'), list) or not 0 < len(inventory['records']) <= 10000):
raise ValueError('Nonempty bounded reviewed inventory required')
ids = set(); records = set()
for row in inventory['records']:
required = {'submission_id','source_record_id','external_id','source','doi','from_edition_id',
'year','venue','venue_names','track','section','evidence_url','evidence_sha256'}
if not isinstance(row, dict) or set(row) != required:
raise ValueError('Unexpected inventory record fields')
for field in ('submission_id','source_record_id','from_edition_id','year'):
if type(row[field]) is not int or row[field] <= 0:
raise ValueError('Positive integer inventory identity required')
if row['submission_id'] in ids or row['source_record_id'] in records:
raise ValueError('Repeated submission/source identity in inventory')
ids.add(row['submission_id']); records.add(row['source_record_id'])
for field in ('external_id','source','doi','venue','section','evidence_url'):
if not isinstance(row[field], str) or not row[field].strip():
raise ValueError('Missing inventory evidence field')
if (not isinstance(row['venue_names'], list) or not row['venue_names']
or not all(isinstance(v,str) and v for v in row['venue_names'])
or row['venue'] not in row['venue_names']
or not re.fullmatch(r'[a-z][a-z0-9_]{0,63}', row['track'])
or not re.fullmatch(r'[0-9a-f]{64}', row['evidence_sha256'])
or not row['evidence_url'].startswith('https://')):
raise ValueError('Invalid scope or evidence binding')
return inventory
def bounded(c, sql, params):
rows = values(c.execute(sql+' LIMIT 100001', params))
if len(rows) > 100000:
raise ValueError('Protected scope exceeds bounded reclassification limit')
return rows
def protect(c, ids):
marks = ','.join('?' for _ in ids)
subs = 'SELECT id FROM submissions WHERE id IN ('+marks+')'
papers = 'SELECT paper_id FROM submissions WHERE id IN ('+marks+')'
decisions = 'SELECT id FROM decisions WHERE submission_id IN ('+subs+')'
queries = {
'submissions': ('SELECT id,paper_id,source_record_id,source_submission_id,submitted_at FROM submissions WHERE id IN ('+marks+') ORDER BY id'),
'submission_identifiers': ('SELECT scheme,value,submission_id FROM submission_identifiers WHERE submission_id IN ('+subs+') ORDER BY submission_id,scheme,value'),
'submission_sources': ('SELECT * FROM submission_sources WHERE submission_id IN ('+subs+') ORDER BY submission_id,source_record_id,source_submission_id'),
'decisions': ('SELECT * FROM decisions WHERE submission_id IN ('+subs+') ORDER BY id'),
'decision_sources': ('SELECT * FROM decision_sources WHERE decision_id IN ('+decisions+') ORDER BY decision_id,source_record_id,source_decision_id'),
'papers': ('SELECT * FROM papers WHERE id IN ('+papers+') ORDER BY id'),
'identifiers': ('SELECT * FROM identifiers WHERE paper_id IN ('+papers+') ORDER BY scheme,value'),
'paper_tags': ('SELECT * FROM paper_tags WHERE paper_id IN ('+papers+') ORDER BY paper_id,tag_id,evidence_key'),
}
from paper_atlas.curation.annotations import available
if available(c):
queries['annotation_events'] = 'SELECT * FROM annotation_events WHERE paper_id IN ('+papers+') ORDER BY id'
result = {table: {'rows':len(rows), 'sha256':digest(rows)} for table, sql in queries.items()
for rows in [bounded(c,sql,ids)]}
rows = bounded(c, 'SELECT r.* FROM source_records r WHERE r.id IN (SELECT source_record_id FROM submission_sources WHERE submission_id IN ('+subs+')) ORDER BY r.id', ids)
# Only the reviewed curation field may change; every other metadata value
# and all raw text/IDs/retrieval times remain protected.
for row in rows:
metadata = json.loads(row[8]); metadata.pop(KEY,None); row[8] = metadata
result['source_records'] = {'rows':len(rows), 'sha256':digest(rows)}
return result
def inspect(c, inventory):
validate(inventory); changes=[]; ids=tuple(sorted(r['submission_id']for r in inventory['records']))
original = sorted({r['from_edition_id'] for r in inventory['records']})
actual_ids = {r[0] for r in c.execute('SELECT id FROM submissions WHERE edition_id IN ('+','.join('?'for _ in original)+')', original)}
if actual_ids != set(ids):
raise ValueError('Inventory must cover every attempt in each original edition')
for row in inventory['records']:
current = c.execute('''SELECT s.edition_id,s.source_record_id,s.source_submission_id,s.paper_id,
e.venue_id,e.year,e.track,e.event_key,v.acronym,r.external_id,r.metadata_json,src.name
FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id
JOIN source_records r ON r.id=s.source_record_id JOIN sources src ON src.id=r.source_id WHERE s.id=?''',(row['submission_id'],)).fetchone()
if not current or [current[i]for i in (0,1,2,5,8,9,11)] != [row[k]for k in ('from_edition_id','source_record_id','external_id','year','venue','external_id','source')]:
raise ValueError('Submission identity/scope differs from inventory')
aliases={r[0]for r in c.execute('SELECT alias FROM venue_aliases WHERE venue_id=?',(current[4],))}|{current[8]}
if set(row['venue_names'])!=aliases:
raise ValueError('Venue names differ from current aliases')
metadata=json.loads(current[10])
if KEY in metadata:
raise ValueError('Existing curated scope requires explicit reconciliation')
if (metadata.get('record_identifiers') or {}).get('doi') != row['doi']:
raise ValueError('Stable DOI differs from reviewed source evidence')
links=values(c.execute('SELECT source_record_id,source_submission_id FROM submission_sources WHERE submission_id=?',(row['submission_id'],)))
if links != [[row['source_record_id'],row['external_id']]]:
raise ValueError('Shared-source attempt requires separate reconciliation')
decisions=values(c.execute('SELECT status FROM decisions WHERE submission_id=?',(row['submission_id'],)))
if not decisions or any(d[0]!='accepted'for d in decisions):
raise ValueError('Only publication observations can use this track correction')
target=c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=? AND event_key=?',(current[4],current[5],row['track'],current[7])).fetchone()
if target and target[0] != current[0]:
if c.execute('SELECT 1 FROM submissions WHERE edition_id=? LIMIT 1',(target[0],)).fetchone():
raise ValueError('Nonempty target edition requires separate reconciliation')
if c.execute('SELECT 1 FROM submission_identifiers WHERE edition_id=? LIMIT 1',(target[0],)).fetchone():
raise ValueError('Target identifier scope already exists')
if c.execute('SELECT 1 FROM edition_sources WHERE edition_id=? LIMIT 1',(target[0],)).fetchone():
raise ValueError('Target coverage already exists; reconcile first')
if c.execute('SELECT 1 FROM submission_identifiers WHERE submission_id=? AND edition_id<>? LIMIT 1',(row['submission_id'],current[0])).fetchone():
raise ValueError('Existing identifier edition is inconsistent')
changes.append({**row,'from_track':current[6],'event_key':current[7], 'venue_id':current[4],
'target_edition_id':target[0]if target else None})
coverage=bounded(c,'SELECT * FROM edition_sources WHERE edition_id IN ('+','.join('?'for _ in original)+') ORDER BY edition_id,source_id,list_url',original)
if Counter(row[0]for row in coverage)!=Counter({edition:1 for edition in original}):
raise ValueError('Exactly one original coverage listing per edition required')
return {'records':changes, 'protected':protect(c,ids), 'coverage':coverage,
'editions':bounded(c,'SELECT * FROM editions WHERE id IN ('+','.join('?'for _ in original)+') ORDER BY id',original)}
def proposal(c, inventory, identity):
return {'format':1,'database':identity,'inventory':inventory,'before':inspect(c,inventory)}
def verify(c, plan):
records=plan['before']['records']; ids=tuple(sorted(r['submission_id']for r in records))
if protect(c,ids)!=plan['before']['protected']:
raise ValueError('Protected history/text/tags/identities changed; roll back')
for row in records:
event=c.execute('''SELECT e.id,e.venue_id,e.year,e.track,e.event_key FROM submissions s JOIN editions e ON e.id=s.edition_id WHERE s.id=?''',(row['submission_id'],)).fetchone()
if not event or [event[i]for i in range(1,5)] != [row['venue_id'],row['year'],row['track'],row['event_key']]:
raise ValueError('Target track assignment failed')
if c.execute('SELECT 1 FROM submission_identifiers WHERE submission_id=? AND edition_id<>? LIMIT 1',(row['submission_id'],event[0])).fetchone():
raise ValueError('Target identifier assignment failed')
if row['track'] != row['from_track']:
metadata=json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(row['source_record_id'],)).fetchone()[0])
if metadata.get(KEY) != [curation(row,event[0],digest(plan))]:
raise ValueError('Reviewed refresh protection missing or changed')
original=sorted({r['from_edition_id']for r in records})
if bounded(c,'SELECT * FROM editions WHERE id IN ('+','.join('?'for _ in original)+') ORDER BY id',original)!=plan['before']['editions']:
raise ValueError('Original edition identities changed')
for old in plan['before']['coverage']:
edition,source,url,_,_,_,cursor,success,_=old
expected=Counter()
for row in records:
if row['from_edition_id']==edition:
target=c.execute('SELECT edition_id FROM submissions WHERE id=?',(row['submission_id'],)).fetchone()[0]
expected[target]+=1
for target,number in expected.items():
actual=c.execute('SELECT status,expected_count,imported_count,cursor,last_success_at,last_error FROM edition_sources WHERE edition_id=? AND source_id=? AND list_url=?',(target,source,url)).fetchone()
note='Track scope corrected from reviewed per-record evidence; original whole-volume coverage retained in plan '+digest(plan)
if actual is None or [actual[i]for i in range(6)]!=['partial',number,number,cursor,success,note]:
raise ValueError('Corrected coverage differs from reviewed scope')
return {'records_verified':len(records),'protected':plan['before']['protected']}
def curation(row, target, signature):
return {'external_id':row['external_id'], 'venue_names':sorted(set(row['venue_names'])),
'from':{'edition_id':row['from_edition_id'],'year':row['year'],'track':row['from_track'],'event_key':row['event_key']},
'target':{'edition_id':target,'venue_id':row['venue_id'],'year':row['year'],'track':row['track'],'event_key':row['event_key']},
'section':row['section'],'evidence_url':row['evidence_url'],'evidence_sha256':row['evidence_sha256'],'plan_digest':signature}
def apply_plan(c, plan):
if plan.get('format') != 1:
raise ValueError('Unsupported plan format')
signature=digest(plan);marker=MARKER+signature
if c.execute('SELECT 1 FROM schema_migrations WHERE name=?',(marker,)).fetchone():
return {'status':'already_applied','events_moved':0,**verify(c,plan)}
if inspect(c,plan['inventory']) != plan['before']:
raise ValueError('Stale track plan; create and review a fresh plan')
targets={}; counts=Counter(); moved=0
for row in plan['before']['records']:
key=(row['venue_id'],row['year'],row['track'],row['event_key'])
if key not in targets:
found=c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=? AND event_key=?',key).fetchone()
targets[key]=found[0]if found else inserted_id(c,'INSERT INTO editions(venue_id,year,track,event_key) VALUES(?,?,?,?)',key)
target=targets[key];counts[(row['from_edition_id'],target)]+=1
if target != row['from_edition_id']:
c.execute('UPDATE submission_identifiers SET edition_id=? WHERE submission_id=?',(target,row['submission_id']))
if c.execute('UPDATE submissions SET edition_id=? WHERE id=? AND edition_id=?',(target,row['submission_id'],row['from_edition_id'])).rowcount!=1:
raise ValueError('Concurrent submission change')
metadata=json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(row['source_record_id'],)).fetchone()[0]);metadata[KEY]=[curation(row,target,signature)]
c.execute('UPDATE source_records SET metadata_json=? WHERE id=?',(json.dumps(metadata,ensure_ascii=False),row['source_record_id']));moved+=1
# Preserve each original source URL and timestamps, but do not retain its
# old whole-volume count as main-track coverage. Partial is deliberate:
# this operation verifies scope, not abstracts or all possible sources.
for old in plan['before']['coverage']:
edition,source,url,status,expected,imported,cursor,success,error=old
own={r['submission_id'] for r in plan['before']['records']if r['from_edition_id']==edition}
source_name=c.execute('SELECT name FROM sources WHERE id=?',(source,)).fetchone()[0]
if {r['source']for r in plan['before']['records']if r['submission_id']in own}!={source_name}:
raise ValueError('Coverage includes a source outside the reviewed inventory')
for (origin,target),number in counts.items():
if origin!=edition:continue
note='Track scope corrected from reviewed per-record evidence; original whole-volume coverage retained in plan '+signature
c.execute('''INSERT INTO edition_sources(edition_id,source_id,list_url,status,expected_count,imported_count,cursor,last_success_at,last_error)
VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(edition_id,source_id,list_url) DO UPDATE SET status=excluded.status,
expected_count=excluded.expected_count,imported_count=excluded.imported_count,last_error=excluded.last_error''',
(target,source,url,'partial',number,number,cursor,success,note))
result=verify(c,plan)
c.execute('INSERT INTO schema_migrations(name) VALUES(?)',(marker,))
return {'status':'applied','events_moved':moved,**result}
def run(db,out,*,pg_service=None,pg_schema=None,inventory=None,plan=None,plan_sha256=None):
out=Path(out)
if out.exists() or not out.parent.is_dir():raise ValueError('New report path in existing directory required')
expected=None
if plan:
raw=Path(plan).read_bytes()
if hashlib.sha256(raw).hexdigest()!=plan_sha256:raise ValueError('Plan hash differs')
expected=json.loads(raw);validate(expected['inventory'])
elif plan_sha256:raise ValueError('Plan path required')
if bool(expected)==bool(inventory):raise ValueError('Choose inventory preflight or apply plan')
if expected is None:
with closing(open_read_connection(db,pg_service=pg_service,pg_schema=pg_schema))as c:
result=proposal(c,validate(json.loads(Path(inventory).read_bytes())),database_identity(c,db))
else:
store=open_store(db,pg_service=pg_service,pg_schema=pg_schema)
with closing(store.conn)as c:
try:
if expected['database']!=database_identity(c,db):raise ValueError('Plan belongs to another database')
if not getattr(c,'postgres',False):c.commit();c.execute('BEGIN IMMEDIATE')
result=apply_plan(c,expected);c.commit()
except BaseException:c.rollback();raise
out.write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n');return result
def main(argv=None,*,prog=None):
p=argparse.ArgumentParser(prog=prog);database_arguments(p)
p.add_argument('--inventory');p.add_argument('--plan');p.add_argument('--plan-sha256');p.add_argument('--out',required=True)
a=p.parse_args(argv)
if bool(a.plan)!=bool(a.plan_sha256):p.error('--plan and --plan-sha256 must be supplied together')
r=run(a.db,a.out,pg_service=a.pg_service,pg_schema=a.pg_schema,inventory=a.inventory,plan=a.plan,plan_sha256=a.plan_sha256)
print(json.dumps({'status':r.get('status','planned'),'events_moved':r.get('events_moved',0),'out':a.out}),flush=True)
+6
View File
@@ -134,6 +134,12 @@ def ingest_file(db,path, *, pg_service=None, pg_schema=None):
if events and status is None:raise ValueError('Explicit decision event requires a status')
if status is not None and status not in ('accepted','rejected','desk_rejected','withdrawn','under_review','unknown'):raise ValueError('Unsupported decision event')
edition=store.decision(paper,record,row['venue'],row['year'],row['track'],external,event.get('evidence_url',row['url']),status,event.get('raw_value',''),row.get('venue_aliases'),identities=identities,observed_at=event.get('observed_at') or row_observed,decision_date=event.get('decision_date'),source_decision_id=event.get('source_decision_id',''),submitted_at=row.get('submitted_at'),guard_status=source_name in (PUBLIC_SOURCE,ARCHIVE_SOURCE),event_key=row.get('event_key'))
from paper_atlas.db.edition_scopes import KEY
scope_metadata=json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(record,)).fetchone()[0])
if scope_metadata.get(KEY):
actual_track=c.execute('SELECT track FROM editions WHERE id=?',(edition,)).fetchone()[0]
if row['track']!=actual_track:
raise ValueError('Recollect this whole-volume source with reviewed per-record tracks before refresh')
editions.add(edition)
else:
raise ValueError('This importer requires an explicit supported decision status')
+135
View File
@@ -0,0 +1,135 @@
"""Stable-attempt correction, rollback and stale whole-volume refresh on both engines."""
from contextlib import closing, redirect_stdout
import copy
import hashlib
import io
import json
import os
from pathlib import Path
import tempfile
import unittest
import uuid
from paper_atlas.db.backend import open_store
from paper_atlas.db.edition_scopes import KEY
from paper_atlas.db.store import Store
from paper_atlas.ingestion.edition_tracks import proposal,apply_plan,verify,validate,run
from paper_atlas.ingestion.event_identity import database_identity,values
from paper_atlas.ingestion.records import ingest_file
from paper_atlas.postgres.client import connect
from paper_atlas.postgres.migration import migrate
from paper_atlas.postgres.schema import quote
def seed(db):
s=Store(db); rows=[]
for i,track in enumerate(['main','survey','doctoral_consortium','proceedings_unspecified'],1):
external='entry'+str(i);doi='10.1234/entry'+str(i)
p,r=s.record('fixture',external,'https://example.org/'+external,'Publication '+external,
'Abstract with NUL\x00 and SOH\x01',[],{'doi':doi})
eid=s.decision(p,r,'JOINT',2022,'main',external,'https://example.org/'+external,'accepted','published',identities=[('doi',doi)],observed_at='2022-01-01T00:00:00Z')
sid=s.conn.execute('SELECT id FROM submissions WHERE source_record_id=?',(r,)).fetchone()[0]
s.record_tags(p,r,'fixture',[{'namespace':'fixture:subject','value':'Original'}])
rows.append(dict(submission_id=sid,source_record_id=r,external_id=external,source='fixture',doi=doi,
from_edition_id=eid,year=2022,venue='JOINT',venue_names=['JOINT'],track=track,
section=track,evidence_url='https://example.org/proceedings/2022/',evidence_sha256='a'*64))
source=s.conn.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0]
s.conn.execute("INSERT INTO edition_sources VALUES(?,?,?,'complete',4,4,'checkpoint','2022-01-01T00:00:00Z',NULL)",(eid,source,'https://example.org/volume'))
s.conn.commit();s.conn.close();return {'format':1,'records':rows}
class TrackCases:
def store(self):return open_store(**self.target)
def plan(self,c):return proposal(c,self.inventory,database_identity(c,self.target.get('db')))
def test_move_preserves_all_history_tags_text_and_stable_identities(self):
with closing(self.store().conn)as c:
plan=self.plan(c);before=plan['before']['protected'];r=apply_plan(c,plan);c.commit()
self.assertEqual((r['events_moved'],r['records_verified']),(3,4))
self.assertEqual(r['protected'],before)
self.assertEqual(apply_plan(c,plan)['status'],'already_applied')
self.assertEqual(values(c.execute('SELECT status,expected_count,imported_count FROM edition_sources ORDER BY edition_id')),[['partial',1,1]]*4)
self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],4)
self.assertEqual(c.execute('SELECT count(*) FROM decisions').fetchone()[0],4)
self.assertEqual(c.execute('SELECT count(*) FROM paper_merges').fetchone()[0],0)
def test_source_record_refresh_keeps_scope_and_original_attempt(self):
with closing(self.store().conn)as c:
plan=self.plan(c);apply_plan(c,plan);c.commit()
with closing(self.store().conn)as c:
# Use Store methods with the same writer connection as production.
s=object.__new__(Store);s.conn=c;s.event_identity=True
r=self.inventory['records'][1];metadata=json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(r['source_record_id'],)).fetchone()[0]);saved=metadata[KEY]
p,record=s.record('fixture',r['external_id'],'https://example.org/new','Refreshed title',None,[],{'doi':r['doi']},{'new_source_field':True})
eid=s.decision(p,record,'JOINT',2022,'main',r['external_id'],'https://example.org/entry2','accepted','published',identities=[('doi',r['doi'])],observed_at='2022-01-01T00:00:00Z')
self.assertEqual(c.execute('SELECT track FROM editions WHERE id=?',(eid,)).fetchone()[0],'survey')
self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],4)
self.assertEqual(json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(record,)).fetchone()[0])[KEY],saved)
with self.assertRaisesRegex(ValueError,'cannot supply'):
s.record('fixture',r['external_id'],'https://example.org/new','Forged',None,[],metadata={KEY:[]})
c.rollback()
def test_stale_snapshot_and_partial_inventory_are_rejected(self):
with closing(self.store().conn)as c:
plan=self.plan(c)
c.execute("UPDATE papers SET title='Changed' WHERE id=(SELECT paper_id FROM submissions WHERE id=?)",(self.inventory['records'][0]['submission_id'],))
with self.assertRaisesRegex(ValueError,'Stale'):apply_plan(c,plan)
c.rollback()
bad=copy.deepcopy(self.inventory);bad['records'].pop()
with self.assertRaisesRegex(ValueError,'every attempt'):proposal(c,bad,database_identity(c,self.target.get('db')))
bad=copy.deepcopy(self.inventory);bad['records'][0]['doi']='10.1234/wrong'
with self.assertRaisesRegex(ValueError,'DOI'):proposal(c,bad,database_identity(c,self.target.get('db')))
def test_failure_after_updates_rolls_back_all_scope_and_metadata_changes(self):
with closing(self.store().conn)as c:
plan=self.plan(c)
# A late coverage failure must roll back earlier moved attempts.
source=c.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0]
c.execute("INSERT INTO sources(name,homepage) VALUES('other','https://example.org')")
other=c.execute("SELECT id FROM sources WHERE name='other'").fetchone()[0]
c.execute('UPDATE edition_sources SET source_id=? WHERE source_id=?',(other,source));c.commit()
plan=self.plan(c)
try:
with self.assertRaisesRegex(ValueError,'outside'):apply_plan(c,plan)
finally:c.rollback()
self.assertEqual(self.plan(c)['before'],plan['before'])
self.assertEqual(c.execute('SELECT count(*) FROM editions').fetchone()[0],1)
def test_import_old_whole_volume_fails_without_corrupting_coverage(self):
with closing(self.store().conn)as c:
plan=self.plan(c);apply_plan(c,plan);c.commit()
r=self.inventory['records'][1];path=self.root/'old.jsonl'
row=dict(source='fixture',external_id=r['external_id'],url='https://example.org/entry2',title='Old title',authors=[],identifiers={'doi':r['doi']},venue='JOINT',year=2022,track='main',status='accepted')
path.write_text(json.dumps(row)+'\n');path.with_suffix('.manifest.json').write_text(json.dumps(dict(source='fixture',venue='JOINT',year=2022,track='main',count=1,list_url='https://example.org/volume',status='complete')))
with self.assertRaisesRegex(ValueError,'Recollect'):ingest_file(self.target.get('db'),path,**{k:v for k,v in self.target.items()if k!='db'})
with closing(self.store().conn)as c:verify(c,plan)
def test_cli_hash_binding_preflight_and_duplicate_inventory(self):
inventory=self.root/'inventory.json';inventory.write_text(json.dumps(self.inventory))
planfile=self.root/'plan.json';p=run(self.target.get('db'),planfile,inventory=inventory,**{k:v for k,v in self.target.items()if k!='db'})
with self.assertRaisesRegex(ValueError,'hash'):run(self.target.get('db'),self.root/'bad.json',plan=planfile,plan_sha256='0'*64,**{k:v for k,v in self.target.items()if k!='db'})
report=run(self.target.get('db'),self.root/'applied.json',plan=planfile,plan_sha256=hashlib.sha256(planfile.read_bytes()).hexdigest(),**{k:v for k,v in self.target.items()if k!='db'})
self.assertEqual(report['events_moved'],3)
bad=copy.deepcopy(self.inventory);bad['records'].append(bad['records'][0])
with self.assertRaisesRegex(ValueError,'Repeated'):validate(bad)
class SQLiteTrackTests(TrackCases,unittest.TestCase):
def setUp(self):
t=tempfile.TemporaryDirectory();self.addCleanup(t.cleanup);self.root=Path(t.name);db=self.root/'source.sqlite'
self.inventory=seed(db);self.target={'db':db}
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'),'Requires approved remote PostgreSQL service')
class PostgreSQLTrackTests(TrackCases,unittest.TestCase):
def setUp(self):
t=tempfile.TemporaryDirectory();self.addCleanup(t.cleanup);self.root=Path(t.name);db=self.root/'source.sqlite';self.inventory=seed(db)
self.service=os.environ['PAPER_ATLAS_TEST_PG_SERVICE'];self.schema='pa_tracks_'+uuid.uuid4().hex[:12];self.addCleanup(self.cleanup)
with redirect_stdout(io.StringIO()):migrate(db,self.service,self.root/'migration.json',schema=self.schema,min_free_gib=0)
self.target={'pg_service':self.service,'pg_schema':self.schema}
def cleanup(self):
with connect(self.service)as c:
for name, in c.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)',(self.schema,)):
c.execute('DROP SCHEMA '+quote(name)+' CASCADE')