Correct existing attempt tracks with bound evidence and refresh guards
Metadata integrity checks / metadata (push) Successful in 1m59s
Metadata integrity checks / metadata (push) Successful in 1m59s
This commit is contained in:
@@ -49,6 +49,7 @@ jobs:
|
||||
subprocess.run([str(venv/'bin/paper-atlas'),'--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','collect','cvf','--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','postgres','--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','edition-tracks','--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','joint-editions','--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','maintain','event-identity','--help'],cwd=outside,check=True)
|
||||
subprocess.run([python,'-I','-B','-m','paper_atlas','import','records','--help'],cwd=outside,check=True)
|
||||
|
||||
@@ -93,3 +93,5 @@ CI 在已授权的远程 runner 上运行测试、构建 wheel,并从源码目
|
||||
研究类型与自定义标签可通过 [`annotate`](docs/annotations.md) 显式初始化、添加和撤回;两者独立查询,保留证据及完整更正历史,不从会议轨道推断研究类型。
|
||||
|
||||
整合后的运行版本依赖、交互约束和生产切换顺序见 [运行版本整合](docs/research-runtime.md)。
|
||||
|
||||
已导入整卷的逐条轨道修正见 [edition-tracks 维护流程](docs/edition-tracks.md):按完整证据生成只读计划,保留原投稿和决定 ID,事务性修正 edition、标识与覆盖,并防止旧整卷刷新覆盖分类。
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
# 按证据修正既有投稿的轨道
|
||||
|
||||
`paper-atlas maintain edition-tracks` 用于已导入整卷论文集、后来取得逐条官方分区证据的情形。它更正原投稿的 edition 归属,保留投稿 ID、所有决定和来源关系;不会重新导入论文、合并实体、生成研究方法标签或推断拒稿。支持独立 SQLite 样本和共享 PostgreSQL;不得写入已冻结的生产 SQLite 备份。
|
||||
|
||||
## 证据与完整范围
|
||||
|
||||
先核实官方目录、稳定 DOI、原始来源记录和分区,保存原始页面及 SHA256。输入 JSON 为 `{"format":1,"records":[...]}`,每条必须包含:
|
||||
|
||||
| 字段 | 含义 |
|
||||
| --- | --- |
|
||||
| `submission_id`、`source_record_id`、`external_id` | 原有投稿、主来源记录及主来源投稿标识 |
|
||||
| `source`、`doi` | 原来源名称及其 metadata.record_identifiers 中的 DOI |
|
||||
| `from_edition_id`、`year`、`venue` | 原 edition 及其真实年份和规范会议名 |
|
||||
| `venue_names` | 规范会议名加全部已有别名;必须与库中一致 |
|
||||
| `track` | 经核实的目标轨道,使用小写字母、数字、下划线;范围未明确可用 proceedings_unspecified |
|
||||
| `section` | 官方分区原文或明确的未确定说明,不推断研究方法 |
|
||||
| `evidence_url`、`evidence_sha256` | 已人工/独立核实的 HTTPS 原始证据和内容哈希 |
|
||||
|
||||
输入必须列出每个原 edition 的**全部**投稿,包含留在原轨道的记录。工具核对身份、DOI、单来源发表观察、现有别名、完整范围以及目标冲突;不自动解释 HTML,也不把填写了 URL/哈希当成已证明内容。核查者应先独立证明每条分区对应关系。
|
||||
|
||||
多来源共同拥有的投稿、有非 accepted 决定历史的投稿、已有分类修订、非空目标 edition、目标已有覆盖或标识等情形会停止,需要专门核对。每个原 edition 须有一个明确来源清单;其他来源或多清单覆盖需先另行协调。本功能不是任意批量移动或跨会合并工具。
|
||||
|
||||
## 只读计划、事务执行和验收
|
||||
|
||||
```sh
|
||||
paper-atlas maintain edition-tracks \
|
||||
--pg-service foundry-research --pg-schema research \
|
||||
--inventory reviewed-inventory.json --out plan.json
|
||||
|
||||
sha256sum plan.json
|
||||
|
||||
paper-atlas maintain edition-tracks \
|
||||
--pg-service foundry-research --pg-schema research \
|
||||
--plan plan.json --plan-sha256 <上一步的完整SHA256> --out applied.json
|
||||
```
|
||||
|
||||
报告路径必须是已有目录中的新文件。预检只读。执行使用 PostgreSQL 单写入 advisory lock,或 SQLite BEGIN IMMEDIATE;整个分类修正、逐来源证据、标识、覆盖及验收在一个事务中完成。计划绑定数据库身份、全部来源文本/非修订元数据、标签、可选 annotation_events、投稿/决定/来源/标识内容及原覆盖快照;过期即停止。失败回滚数据,PostgreSQL 序列可能留下正常间隙,不能用 max(id) 当数量。
|
||||
|
||||
目标 edition 沿用原会议、年份、event_key,不删除或重命名原 edition。只更正既有 submissions.edition_id 和对应 submission_identifiers.edition_id。原文、来源检索时间、决定 ID/状态/证据及各表来源关系不变。
|
||||
|
||||
分类证据写入来源元数据的保留字段 `edition_scope_curation_v1`,包括原范围、目标 edition、官方分区、URL、哈希和计划摘要。普通来源导入不能伪造或替换此字段;后续 record 刷新保留修订。相同来源投稿通过旧范围调用 decision 时仍路由到修正后的 edition。旧整卷 main 文件通过 import records 刷新会报错并回滚,要求先按核实轨道重新整理,避免覆盖正确的统计。
|
||||
|
||||
覆盖保留原清单 URL、cursor、成功时间,按实际目标分配 expected_count/imported_count 并标 partial;本工具核实分类,未核实摘要或所有来源完整性。原覆盖整行保留在计划中,不能继续将整卷数当 main 论文数。应用后校验原 edition 身份、全部受保护内容、分类、标识、修订字段和覆盖;相同计划再次执行仅在这些验收条件仍成立时返回 already_applied。
|
||||
|
||||
生产使用前必须在已确认 runner 完成软件验证。正式数据修正后仍需独立验收,并在最终全库审计中复核;单元测试和应用回执不能代替生产验收。
|
||||
@@ -27,6 +27,7 @@ COMMANDS = {
|
||||
('catalogue','dblp'): ('paper_atlas.collectors.dblp', {}, '盘点 DBLP 历史会议目录'),
|
||||
('import','arxiv'): ('paper_atlas.ingestion.arxiv', {}, '导入 arXiv 元数据 ZIP'),
|
||||
('import','records'): ('paper_atlas.ingestion.records', {}, '导入核验过的会议文件'),
|
||||
('maintain','edition-tracks'): ('paper_atlas.ingestion.edition_tracks', {}, '按逐条来源证据修正既有投稿轨道并保护刷新'),
|
||||
('maintain','joint-editions'): ('paper_atlas.ingestion.joint_editions', {}, '按已核验计划修正既有联合会议届次归属'),
|
||||
('maintain','event-identity'): ('paper_atlas.ingestion.event_identity', {}, '按核验计划增加同年会议事件身份'),
|
||||
('maintain','merge'): ('paper_atlas.ingestion.maintenance', {}, '协调单写入合并与恢复'),
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
"""Retain reviewed event scopes when a source refresh repeats a whole-volume label."""
|
||||
import json
|
||||
|
||||
KEY = 'edition_scope_curation_v1'
|
||||
|
||||
|
||||
def preserve(metadata, previous):
|
||||
old = json.loads(previous).get(KEY) if previous else None
|
||||
if KEY in metadata and metadata[KEY] != old:
|
||||
raise ValueError('Source import cannot supply or replace reviewed edition scope')
|
||||
if old is not None:
|
||||
metadata[KEY] = old
|
||||
return metadata
|
||||
|
||||
|
||||
def resolve(c, record, external, venue, year, track, event_key):
|
||||
row = c.execute('SELECT metadata_json FROM source_records WHERE id=?', (record,)).fetchone()
|
||||
entries = json.loads(row[0]).get(KEY, []) if row else []
|
||||
if not isinstance(entries, list):
|
||||
raise ValueError('Invalid reviewed edition scope')
|
||||
applicable = []
|
||||
for entry in entries:
|
||||
if entry['external_id'] != external:
|
||||
continue
|
||||
old, target = entry['from'], entry['target']
|
||||
if int(year) != old['year'] or venue not in entry['venue_names']:
|
||||
continue
|
||||
if track not in (old['track'], target['track']):
|
||||
continue
|
||||
if event_key is not None and event_key != old['event_key']:
|
||||
continue
|
||||
current = c.execute('SELECT venue_id,year,track,event_key FROM editions WHERE id=?', (target['edition_id'],)).fetchone()
|
||||
if not current or [current[i] for i in range(4)] != [target[k] for k in ('venue_id','year','track','event_key')]:
|
||||
raise ValueError('Reviewed target edition has changed')
|
||||
applicable.append(target['edition_id'])
|
||||
if len(applicable) > 1:
|
||||
raise ValueError('Ambiguous reviewed edition scope')
|
||||
return applicable[0] if applicable else None
|
||||
@@ -96,6 +96,8 @@ class Store:
|
||||
if found:
|
||||
matches.add(found[0])
|
||||
old = c.execute('SELECT paper_id,title,abstract,retrieved_at,metadata_json FROM source_records WHERE source_id=? AND external_id=?',(source_id:=self.source(source,url),external_id)).fetchone()
|
||||
from paper_atlas.db.edition_scopes import preserve
|
||||
metadata = preserve(dict(metadata or {}), old[4] if old else None)
|
||||
if old:
|
||||
matches.add(old[0])
|
||||
key = normalized(title)
|
||||
@@ -175,7 +177,10 @@ class Store:
|
||||
for alias in aliases or []:
|
||||
c.execute('INSERT INTO venue_aliases VALUES(?,?) ON CONFLICT DO NOTHING',(venue_id,alias))
|
||||
from paper_atlas.db.events import find_edition
|
||||
edition = find_edition(self,venue_id,venue,int(year),track,record,external_id,event_key)
|
||||
from paper_atlas.db.edition_scopes import resolve
|
||||
edition = resolve(c,record,external_id,venue,year,track,event_key)
|
||||
if edition is None:
|
||||
edition = find_edition(self,venue_id,venue,int(year),track,record,external_id,event_key)
|
||||
# A previous event in this row may already have merged the input paper.
|
||||
paper=c.execute('SELECT paper_id FROM source_records WHERE id=?',(record,)).fetchone()[0]
|
||||
submission=attach_submission(self,edition,paper,record,external_id,identities or [])
|
||||
|
||||
@@ -0,0 +1,243 @@
|
||||
"""Snapshot-bound reclassification of existing attempts, without reimport or merge."""
|
||||
import argparse
|
||||
from collections import Counter
|
||||
from contextlib import closing
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
|
||||
from paper_atlas.db.backend import database_arguments, open_read_connection, open_store, inserted_id
|
||||
from paper_atlas.db.edition_scopes import KEY
|
||||
from paper_atlas.ingestion.event_identity import database_identity, digest, values
|
||||
|
||||
MARKER = 'edition_tracks_v1:'
|
||||
|
||||
|
||||
def validate(inventory):
|
||||
if (not isinstance(inventory, dict) or inventory.get('format') != 1
|
||||
or not isinstance(inventory.get('records'), list) or not 0 < len(inventory['records']) <= 10000):
|
||||
raise ValueError('Nonempty bounded reviewed inventory required')
|
||||
ids = set(); records = set()
|
||||
for row in inventory['records']:
|
||||
required = {'submission_id','source_record_id','external_id','source','doi','from_edition_id',
|
||||
'year','venue','venue_names','track','section','evidence_url','evidence_sha256'}
|
||||
if not isinstance(row, dict) or set(row) != required:
|
||||
raise ValueError('Unexpected inventory record fields')
|
||||
for field in ('submission_id','source_record_id','from_edition_id','year'):
|
||||
if type(row[field]) is not int or row[field] <= 0:
|
||||
raise ValueError('Positive integer inventory identity required')
|
||||
if row['submission_id'] in ids or row['source_record_id'] in records:
|
||||
raise ValueError('Repeated submission/source identity in inventory')
|
||||
ids.add(row['submission_id']); records.add(row['source_record_id'])
|
||||
for field in ('external_id','source','doi','venue','section','evidence_url'):
|
||||
if not isinstance(row[field], str) or not row[field].strip():
|
||||
raise ValueError('Missing inventory evidence field')
|
||||
if (not isinstance(row['venue_names'], list) or not row['venue_names']
|
||||
or not all(isinstance(v,str) and v for v in row['venue_names'])
|
||||
or row['venue'] not in row['venue_names']
|
||||
or not re.fullmatch(r'[a-z][a-z0-9_]{0,63}', row['track'])
|
||||
or not re.fullmatch(r'[0-9a-f]{64}', row['evidence_sha256'])
|
||||
or not row['evidence_url'].startswith('https://')):
|
||||
raise ValueError('Invalid scope or evidence binding')
|
||||
return inventory
|
||||
|
||||
|
||||
def bounded(c, sql, params):
|
||||
rows = values(c.execute(sql+' LIMIT 100001', params))
|
||||
if len(rows) > 100000:
|
||||
raise ValueError('Protected scope exceeds bounded reclassification limit')
|
||||
return rows
|
||||
|
||||
|
||||
def protect(c, ids):
|
||||
marks = ','.join('?' for _ in ids)
|
||||
subs = 'SELECT id FROM submissions WHERE id IN ('+marks+')'
|
||||
papers = 'SELECT paper_id FROM submissions WHERE id IN ('+marks+')'
|
||||
decisions = 'SELECT id FROM decisions WHERE submission_id IN ('+subs+')'
|
||||
queries = {
|
||||
'submissions': ('SELECT id,paper_id,source_record_id,source_submission_id,submitted_at FROM submissions WHERE id IN ('+marks+') ORDER BY id'),
|
||||
'submission_identifiers': ('SELECT scheme,value,submission_id FROM submission_identifiers WHERE submission_id IN ('+subs+') ORDER BY submission_id,scheme,value'),
|
||||
'submission_sources': ('SELECT * FROM submission_sources WHERE submission_id IN ('+subs+') ORDER BY submission_id,source_record_id,source_submission_id'),
|
||||
'decisions': ('SELECT * FROM decisions WHERE submission_id IN ('+subs+') ORDER BY id'),
|
||||
'decision_sources': ('SELECT * FROM decision_sources WHERE decision_id IN ('+decisions+') ORDER BY decision_id,source_record_id,source_decision_id'),
|
||||
'papers': ('SELECT * FROM papers WHERE id IN ('+papers+') ORDER BY id'),
|
||||
'identifiers': ('SELECT * FROM identifiers WHERE paper_id IN ('+papers+') ORDER BY scheme,value'),
|
||||
'paper_tags': ('SELECT * FROM paper_tags WHERE paper_id IN ('+papers+') ORDER BY paper_id,tag_id,evidence_key'),
|
||||
}
|
||||
from paper_atlas.curation.annotations import available
|
||||
if available(c):
|
||||
queries['annotation_events'] = 'SELECT * FROM annotation_events WHERE paper_id IN ('+papers+') ORDER BY id'
|
||||
result = {table: {'rows':len(rows), 'sha256':digest(rows)} for table, sql in queries.items()
|
||||
for rows in [bounded(c,sql,ids)]}
|
||||
rows = bounded(c, 'SELECT r.* FROM source_records r WHERE r.id IN (SELECT source_record_id FROM submission_sources WHERE submission_id IN ('+subs+')) ORDER BY r.id', ids)
|
||||
# Only the reviewed curation field may change; every other metadata value
|
||||
# and all raw text/IDs/retrieval times remain protected.
|
||||
for row in rows:
|
||||
metadata = json.loads(row[8]); metadata.pop(KEY,None); row[8] = metadata
|
||||
result['source_records'] = {'rows':len(rows), 'sha256':digest(rows)}
|
||||
return result
|
||||
|
||||
|
||||
def inspect(c, inventory):
|
||||
validate(inventory); changes=[]; ids=tuple(sorted(r['submission_id']for r in inventory['records']))
|
||||
original = sorted({r['from_edition_id'] for r in inventory['records']})
|
||||
actual_ids = {r[0] for r in c.execute('SELECT id FROM submissions WHERE edition_id IN ('+','.join('?'for _ in original)+')', original)}
|
||||
if actual_ids != set(ids):
|
||||
raise ValueError('Inventory must cover every attempt in each original edition')
|
||||
for row in inventory['records']:
|
||||
current = c.execute('''SELECT s.edition_id,s.source_record_id,s.source_submission_id,s.paper_id,
|
||||
e.venue_id,e.year,e.track,e.event_key,v.acronym,r.external_id,r.metadata_json,src.name
|
||||
FROM submissions s JOIN editions e ON e.id=s.edition_id JOIN venues v ON v.id=e.venue_id
|
||||
JOIN source_records r ON r.id=s.source_record_id JOIN sources src ON src.id=r.source_id WHERE s.id=?''',(row['submission_id'],)).fetchone()
|
||||
if not current or [current[i]for i in (0,1,2,5,8,9,11)] != [row[k]for k in ('from_edition_id','source_record_id','external_id','year','venue','external_id','source')]:
|
||||
raise ValueError('Submission identity/scope differs from inventory')
|
||||
aliases={r[0]for r in c.execute('SELECT alias FROM venue_aliases WHERE venue_id=?',(current[4],))}|{current[8]}
|
||||
if set(row['venue_names'])!=aliases:
|
||||
raise ValueError('Venue names differ from current aliases')
|
||||
metadata=json.loads(current[10])
|
||||
if KEY in metadata:
|
||||
raise ValueError('Existing curated scope requires explicit reconciliation')
|
||||
if (metadata.get('record_identifiers') or {}).get('doi') != row['doi']:
|
||||
raise ValueError('Stable DOI differs from reviewed source evidence')
|
||||
links=values(c.execute('SELECT source_record_id,source_submission_id FROM submission_sources WHERE submission_id=?',(row['submission_id'],)))
|
||||
if links != [[row['source_record_id'],row['external_id']]]:
|
||||
raise ValueError('Shared-source attempt requires separate reconciliation')
|
||||
decisions=values(c.execute('SELECT status FROM decisions WHERE submission_id=?',(row['submission_id'],)))
|
||||
if not decisions or any(d[0]!='accepted'for d in decisions):
|
||||
raise ValueError('Only publication observations can use this track correction')
|
||||
target=c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=? AND event_key=?',(current[4],current[5],row['track'],current[7])).fetchone()
|
||||
if target and target[0] != current[0]:
|
||||
if c.execute('SELECT 1 FROM submissions WHERE edition_id=? LIMIT 1',(target[0],)).fetchone():
|
||||
raise ValueError('Nonempty target edition requires separate reconciliation')
|
||||
if c.execute('SELECT 1 FROM submission_identifiers WHERE edition_id=? LIMIT 1',(target[0],)).fetchone():
|
||||
raise ValueError('Target identifier scope already exists')
|
||||
if c.execute('SELECT 1 FROM edition_sources WHERE edition_id=? LIMIT 1',(target[0],)).fetchone():
|
||||
raise ValueError('Target coverage already exists; reconcile first')
|
||||
if c.execute('SELECT 1 FROM submission_identifiers WHERE submission_id=? AND edition_id<>? LIMIT 1',(row['submission_id'],current[0])).fetchone():
|
||||
raise ValueError('Existing identifier edition is inconsistent')
|
||||
changes.append({**row,'from_track':current[6],'event_key':current[7], 'venue_id':current[4],
|
||||
'target_edition_id':target[0]if target else None})
|
||||
coverage=bounded(c,'SELECT * FROM edition_sources WHERE edition_id IN ('+','.join('?'for _ in original)+') ORDER BY edition_id,source_id,list_url',original)
|
||||
if Counter(row[0]for row in coverage)!=Counter({edition:1 for edition in original}):
|
||||
raise ValueError('Exactly one original coverage listing per edition required')
|
||||
return {'records':changes, 'protected':protect(c,ids), 'coverage':coverage,
|
||||
'editions':bounded(c,'SELECT * FROM editions WHERE id IN ('+','.join('?'for _ in original)+') ORDER BY id',original)}
|
||||
|
||||
|
||||
def proposal(c, inventory, identity):
|
||||
return {'format':1,'database':identity,'inventory':inventory,'before':inspect(c,inventory)}
|
||||
|
||||
|
||||
def verify(c, plan):
|
||||
records=plan['before']['records']; ids=tuple(sorted(r['submission_id']for r in records))
|
||||
if protect(c,ids)!=plan['before']['protected']:
|
||||
raise ValueError('Protected history/text/tags/identities changed; roll back')
|
||||
for row in records:
|
||||
event=c.execute('''SELECT e.id,e.venue_id,e.year,e.track,e.event_key FROM submissions s JOIN editions e ON e.id=s.edition_id WHERE s.id=?''',(row['submission_id'],)).fetchone()
|
||||
if not event or [event[i]for i in range(1,5)] != [row['venue_id'],row['year'],row['track'],row['event_key']]:
|
||||
raise ValueError('Target track assignment failed')
|
||||
if c.execute('SELECT 1 FROM submission_identifiers WHERE submission_id=? AND edition_id<>? LIMIT 1',(row['submission_id'],event[0])).fetchone():
|
||||
raise ValueError('Target identifier assignment failed')
|
||||
if row['track'] != row['from_track']:
|
||||
metadata=json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(row['source_record_id'],)).fetchone()[0])
|
||||
if metadata.get(KEY) != [curation(row,event[0],digest(plan))]:
|
||||
raise ValueError('Reviewed refresh protection missing or changed')
|
||||
original=sorted({r['from_edition_id']for r in records})
|
||||
if bounded(c,'SELECT * FROM editions WHERE id IN ('+','.join('?'for _ in original)+') ORDER BY id',original)!=plan['before']['editions']:
|
||||
raise ValueError('Original edition identities changed')
|
||||
for old in plan['before']['coverage']:
|
||||
edition,source,url,_,_,_,cursor,success,_=old
|
||||
expected=Counter()
|
||||
for row in records:
|
||||
if row['from_edition_id']==edition:
|
||||
target=c.execute('SELECT edition_id FROM submissions WHERE id=?',(row['submission_id'],)).fetchone()[0]
|
||||
expected[target]+=1
|
||||
for target,number in expected.items():
|
||||
actual=c.execute('SELECT status,expected_count,imported_count,cursor,last_success_at,last_error FROM edition_sources WHERE edition_id=? AND source_id=? AND list_url=?',(target,source,url)).fetchone()
|
||||
note='Track scope corrected from reviewed per-record evidence; original whole-volume coverage retained in plan '+digest(plan)
|
||||
if actual is None or [actual[i]for i in range(6)]!=['partial',number,number,cursor,success,note]:
|
||||
raise ValueError('Corrected coverage differs from reviewed scope')
|
||||
return {'records_verified':len(records),'protected':plan['before']['protected']}
|
||||
|
||||
|
||||
def curation(row, target, signature):
|
||||
return {'external_id':row['external_id'], 'venue_names':sorted(set(row['venue_names'])),
|
||||
'from':{'edition_id':row['from_edition_id'],'year':row['year'],'track':row['from_track'],'event_key':row['event_key']},
|
||||
'target':{'edition_id':target,'venue_id':row['venue_id'],'year':row['year'],'track':row['track'],'event_key':row['event_key']},
|
||||
'section':row['section'],'evidence_url':row['evidence_url'],'evidence_sha256':row['evidence_sha256'],'plan_digest':signature}
|
||||
|
||||
|
||||
def apply_plan(c, plan):
|
||||
if plan.get('format') != 1:
|
||||
raise ValueError('Unsupported plan format')
|
||||
signature=digest(plan);marker=MARKER+signature
|
||||
if c.execute('SELECT 1 FROM schema_migrations WHERE name=?',(marker,)).fetchone():
|
||||
return {'status':'already_applied','events_moved':0,**verify(c,plan)}
|
||||
if inspect(c,plan['inventory']) != plan['before']:
|
||||
raise ValueError('Stale track plan; create and review a fresh plan')
|
||||
targets={}; counts=Counter(); moved=0
|
||||
for row in plan['before']['records']:
|
||||
key=(row['venue_id'],row['year'],row['track'],row['event_key'])
|
||||
if key not in targets:
|
||||
found=c.execute('SELECT id FROM editions WHERE venue_id=? AND year=? AND track=? AND event_key=?',key).fetchone()
|
||||
targets[key]=found[0]if found else inserted_id(c,'INSERT INTO editions(venue_id,year,track,event_key) VALUES(?,?,?,?)',key)
|
||||
target=targets[key];counts[(row['from_edition_id'],target)]+=1
|
||||
if target != row['from_edition_id']:
|
||||
c.execute('UPDATE submission_identifiers SET edition_id=? WHERE submission_id=?',(target,row['submission_id']))
|
||||
if c.execute('UPDATE submissions SET edition_id=? WHERE id=? AND edition_id=?',(target,row['submission_id'],row['from_edition_id'])).rowcount!=1:
|
||||
raise ValueError('Concurrent submission change')
|
||||
metadata=json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(row['source_record_id'],)).fetchone()[0]);metadata[KEY]=[curation(row,target,signature)]
|
||||
c.execute('UPDATE source_records SET metadata_json=? WHERE id=?',(json.dumps(metadata,ensure_ascii=False),row['source_record_id']));moved+=1
|
||||
# Preserve each original source URL and timestamps, but do not retain its
|
||||
# old whole-volume count as main-track coverage. Partial is deliberate:
|
||||
# this operation verifies scope, not abstracts or all possible sources.
|
||||
for old in plan['before']['coverage']:
|
||||
edition,source,url,status,expected,imported,cursor,success,error=old
|
||||
own={r['submission_id'] for r in plan['before']['records']if r['from_edition_id']==edition}
|
||||
source_name=c.execute('SELECT name FROM sources WHERE id=?',(source,)).fetchone()[0]
|
||||
if {r['source']for r in plan['before']['records']if r['submission_id']in own}!={source_name}:
|
||||
raise ValueError('Coverage includes a source outside the reviewed inventory')
|
||||
for (origin,target),number in counts.items():
|
||||
if origin!=edition:continue
|
||||
note='Track scope corrected from reviewed per-record evidence; original whole-volume coverage retained in plan '+signature
|
||||
c.execute('''INSERT INTO edition_sources(edition_id,source_id,list_url,status,expected_count,imported_count,cursor,last_success_at,last_error)
|
||||
VALUES(?,?,?,?,?,?,?,?,?) ON CONFLICT(edition_id,source_id,list_url) DO UPDATE SET status=excluded.status,
|
||||
expected_count=excluded.expected_count,imported_count=excluded.imported_count,last_error=excluded.last_error''',
|
||||
(target,source,url,'partial',number,number,cursor,success,note))
|
||||
result=verify(c,plan)
|
||||
c.execute('INSERT INTO schema_migrations(name) VALUES(?)',(marker,))
|
||||
return {'status':'applied','events_moved':moved,**result}
|
||||
|
||||
|
||||
def run(db,out,*,pg_service=None,pg_schema=None,inventory=None,plan=None,plan_sha256=None):
|
||||
out=Path(out)
|
||||
if out.exists() or not out.parent.is_dir():raise ValueError('New report path in existing directory required')
|
||||
expected=None
|
||||
if plan:
|
||||
raw=Path(plan).read_bytes()
|
||||
if hashlib.sha256(raw).hexdigest()!=plan_sha256:raise ValueError('Plan hash differs')
|
||||
expected=json.loads(raw);validate(expected['inventory'])
|
||||
elif plan_sha256:raise ValueError('Plan path required')
|
||||
if bool(expected)==bool(inventory):raise ValueError('Choose inventory preflight or apply plan')
|
||||
if expected is None:
|
||||
with closing(open_read_connection(db,pg_service=pg_service,pg_schema=pg_schema))as c:
|
||||
result=proposal(c,validate(json.loads(Path(inventory).read_bytes())),database_identity(c,db))
|
||||
else:
|
||||
store=open_store(db,pg_service=pg_service,pg_schema=pg_schema)
|
||||
with closing(store.conn)as c:
|
||||
try:
|
||||
if expected['database']!=database_identity(c,db):raise ValueError('Plan belongs to another database')
|
||||
if not getattr(c,'postgres',False):c.commit();c.execute('BEGIN IMMEDIATE')
|
||||
result=apply_plan(c,expected);c.commit()
|
||||
except BaseException:c.rollback();raise
|
||||
out.write_text(json.dumps(result,ensure_ascii=False,indent=2)+'\n');return result
|
||||
|
||||
|
||||
def main(argv=None,*,prog=None):
|
||||
p=argparse.ArgumentParser(prog=prog);database_arguments(p)
|
||||
p.add_argument('--inventory');p.add_argument('--plan');p.add_argument('--plan-sha256');p.add_argument('--out',required=True)
|
||||
a=p.parse_args(argv)
|
||||
if bool(a.plan)!=bool(a.plan_sha256):p.error('--plan and --plan-sha256 must be supplied together')
|
||||
r=run(a.db,a.out,pg_service=a.pg_service,pg_schema=a.pg_schema,inventory=a.inventory,plan=a.plan,plan_sha256=a.plan_sha256)
|
||||
print(json.dumps({'status':r.get('status','planned'),'events_moved':r.get('events_moved',0),'out':a.out}),flush=True)
|
||||
@@ -134,6 +134,12 @@ def ingest_file(db,path, *, pg_service=None, pg_schema=None):
|
||||
if events and status is None:raise ValueError('Explicit decision event requires a status')
|
||||
if status is not None and status not in ('accepted','rejected','desk_rejected','withdrawn','under_review','unknown'):raise ValueError('Unsupported decision event')
|
||||
edition=store.decision(paper,record,row['venue'],row['year'],row['track'],external,event.get('evidence_url',row['url']),status,event.get('raw_value',''),row.get('venue_aliases'),identities=identities,observed_at=event.get('observed_at') or row_observed,decision_date=event.get('decision_date'),source_decision_id=event.get('source_decision_id',''),submitted_at=row.get('submitted_at'),guard_status=source_name in (PUBLIC_SOURCE,ARCHIVE_SOURCE),event_key=row.get('event_key'))
|
||||
from paper_atlas.db.edition_scopes import KEY
|
||||
scope_metadata=json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(record,)).fetchone()[0])
|
||||
if scope_metadata.get(KEY):
|
||||
actual_track=c.execute('SELECT track FROM editions WHERE id=?',(edition,)).fetchone()[0]
|
||||
if row['track']!=actual_track:
|
||||
raise ValueError('Recollect this whole-volume source with reviewed per-record tracks before refresh')
|
||||
editions.add(edition)
|
||||
else:
|
||||
raise ValueError('This importer requires an explicit supported decision status')
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
"""Stable-attempt correction, rollback and stale whole-volume refresh on both engines."""
|
||||
from contextlib import closing, redirect_stdout
|
||||
import copy
|
||||
import hashlib
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
import uuid
|
||||
|
||||
from paper_atlas.db.backend import open_store
|
||||
from paper_atlas.db.edition_scopes import KEY
|
||||
from paper_atlas.db.store import Store
|
||||
from paper_atlas.ingestion.edition_tracks import proposal,apply_plan,verify,validate,run
|
||||
from paper_atlas.ingestion.event_identity import database_identity,values
|
||||
from paper_atlas.ingestion.records import ingest_file
|
||||
from paper_atlas.postgres.client import connect
|
||||
from paper_atlas.postgres.migration import migrate
|
||||
from paper_atlas.postgres.schema import quote
|
||||
|
||||
|
||||
def seed(db):
|
||||
s=Store(db); rows=[]
|
||||
for i,track in enumerate(['main','survey','doctoral_consortium','proceedings_unspecified'],1):
|
||||
external='entry'+str(i);doi='10.1234/entry'+str(i)
|
||||
p,r=s.record('fixture',external,'https://example.org/'+external,'Publication '+external,
|
||||
'Abstract with NUL\x00 and SOH\x01',[],{'doi':doi})
|
||||
eid=s.decision(p,r,'JOINT',2022,'main',external,'https://example.org/'+external,'accepted','published',identities=[('doi',doi)],observed_at='2022-01-01T00:00:00Z')
|
||||
sid=s.conn.execute('SELECT id FROM submissions WHERE source_record_id=?',(r,)).fetchone()[0]
|
||||
s.record_tags(p,r,'fixture',[{'namespace':'fixture:subject','value':'Original'}])
|
||||
rows.append(dict(submission_id=sid,source_record_id=r,external_id=external,source='fixture',doi=doi,
|
||||
from_edition_id=eid,year=2022,venue='JOINT',venue_names=['JOINT'],track=track,
|
||||
section=track,evidence_url='https://example.org/proceedings/2022/',evidence_sha256='a'*64))
|
||||
source=s.conn.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0]
|
||||
s.conn.execute("INSERT INTO edition_sources VALUES(?,?,?,'complete',4,4,'checkpoint','2022-01-01T00:00:00Z',NULL)",(eid,source,'https://example.org/volume'))
|
||||
s.conn.commit();s.conn.close();return {'format':1,'records':rows}
|
||||
|
||||
|
||||
class TrackCases:
|
||||
def store(self):return open_store(**self.target)
|
||||
|
||||
def plan(self,c):return proposal(c,self.inventory,database_identity(c,self.target.get('db')))
|
||||
|
||||
def test_move_preserves_all_history_tags_text_and_stable_identities(self):
|
||||
with closing(self.store().conn)as c:
|
||||
plan=self.plan(c);before=plan['before']['protected'];r=apply_plan(c,plan);c.commit()
|
||||
self.assertEqual((r['events_moved'],r['records_verified']),(3,4))
|
||||
self.assertEqual(r['protected'],before)
|
||||
self.assertEqual(apply_plan(c,plan)['status'],'already_applied')
|
||||
self.assertEqual(values(c.execute('SELECT status,expected_count,imported_count FROM edition_sources ORDER BY edition_id')),[['partial',1,1]]*4)
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],4)
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM decisions').fetchone()[0],4)
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM paper_merges').fetchone()[0],0)
|
||||
|
||||
def test_source_record_refresh_keeps_scope_and_original_attempt(self):
|
||||
with closing(self.store().conn)as c:
|
||||
plan=self.plan(c);apply_plan(c,plan);c.commit()
|
||||
with closing(self.store().conn)as c:
|
||||
# Use Store methods with the same writer connection as production.
|
||||
s=object.__new__(Store);s.conn=c;s.event_identity=True
|
||||
r=self.inventory['records'][1];metadata=json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(r['source_record_id'],)).fetchone()[0]);saved=metadata[KEY]
|
||||
p,record=s.record('fixture',r['external_id'],'https://example.org/new','Refreshed title',None,[],{'doi':r['doi']},{'new_source_field':True})
|
||||
eid=s.decision(p,record,'JOINT',2022,'main',r['external_id'],'https://example.org/entry2','accepted','published',identities=[('doi',r['doi'])],observed_at='2022-01-01T00:00:00Z')
|
||||
self.assertEqual(c.execute('SELECT track FROM editions WHERE id=?',(eid,)).fetchone()[0],'survey')
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM submissions').fetchone()[0],4)
|
||||
self.assertEqual(json.loads(c.execute('SELECT metadata_json FROM source_records WHERE id=?',(record,)).fetchone()[0])[KEY],saved)
|
||||
with self.assertRaisesRegex(ValueError,'cannot supply'):
|
||||
s.record('fixture',r['external_id'],'https://example.org/new','Forged',None,[],metadata={KEY:[]})
|
||||
c.rollback()
|
||||
|
||||
def test_stale_snapshot_and_partial_inventory_are_rejected(self):
|
||||
with closing(self.store().conn)as c:
|
||||
plan=self.plan(c)
|
||||
c.execute("UPDATE papers SET title='Changed' WHERE id=(SELECT paper_id FROM submissions WHERE id=?)",(self.inventory['records'][0]['submission_id'],))
|
||||
with self.assertRaisesRegex(ValueError,'Stale'):apply_plan(c,plan)
|
||||
c.rollback()
|
||||
bad=copy.deepcopy(self.inventory);bad['records'].pop()
|
||||
with self.assertRaisesRegex(ValueError,'every attempt'):proposal(c,bad,database_identity(c,self.target.get('db')))
|
||||
bad=copy.deepcopy(self.inventory);bad['records'][0]['doi']='10.1234/wrong'
|
||||
with self.assertRaisesRegex(ValueError,'DOI'):proposal(c,bad,database_identity(c,self.target.get('db')))
|
||||
|
||||
def test_failure_after_updates_rolls_back_all_scope_and_metadata_changes(self):
|
||||
with closing(self.store().conn)as c:
|
||||
plan=self.plan(c)
|
||||
# A late coverage failure must roll back earlier moved attempts.
|
||||
source=c.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0]
|
||||
c.execute("INSERT INTO sources(name,homepage) VALUES('other','https://example.org')")
|
||||
other=c.execute("SELECT id FROM sources WHERE name='other'").fetchone()[0]
|
||||
c.execute('UPDATE edition_sources SET source_id=? WHERE source_id=?',(other,source));c.commit()
|
||||
plan=self.plan(c)
|
||||
try:
|
||||
with self.assertRaisesRegex(ValueError,'outside'):apply_plan(c,plan)
|
||||
finally:c.rollback()
|
||||
self.assertEqual(self.plan(c)['before'],plan['before'])
|
||||
self.assertEqual(c.execute('SELECT count(*) FROM editions').fetchone()[0],1)
|
||||
|
||||
def test_import_old_whole_volume_fails_without_corrupting_coverage(self):
|
||||
with closing(self.store().conn)as c:
|
||||
plan=self.plan(c);apply_plan(c,plan);c.commit()
|
||||
r=self.inventory['records'][1];path=self.root/'old.jsonl'
|
||||
row=dict(source='fixture',external_id=r['external_id'],url='https://example.org/entry2',title='Old title',authors=[],identifiers={'doi':r['doi']},venue='JOINT',year=2022,track='main',status='accepted')
|
||||
path.write_text(json.dumps(row)+'\n');path.with_suffix('.manifest.json').write_text(json.dumps(dict(source='fixture',venue='JOINT',year=2022,track='main',count=1,list_url='https://example.org/volume',status='complete')))
|
||||
with self.assertRaisesRegex(ValueError,'Recollect'):ingest_file(self.target.get('db'),path,**{k:v for k,v in self.target.items()if k!='db'})
|
||||
with closing(self.store().conn)as c:verify(c,plan)
|
||||
|
||||
def test_cli_hash_binding_preflight_and_duplicate_inventory(self):
|
||||
inventory=self.root/'inventory.json';inventory.write_text(json.dumps(self.inventory))
|
||||
planfile=self.root/'plan.json';p=run(self.target.get('db'),planfile,inventory=inventory,**{k:v for k,v in self.target.items()if k!='db'})
|
||||
with self.assertRaisesRegex(ValueError,'hash'):run(self.target.get('db'),self.root/'bad.json',plan=planfile,plan_sha256='0'*64,**{k:v for k,v in self.target.items()if k!='db'})
|
||||
report=run(self.target.get('db'),self.root/'applied.json',plan=planfile,plan_sha256=hashlib.sha256(planfile.read_bytes()).hexdigest(),**{k:v for k,v in self.target.items()if k!='db'})
|
||||
self.assertEqual(report['events_moved'],3)
|
||||
bad=copy.deepcopy(self.inventory);bad['records'].append(bad['records'][0])
|
||||
with self.assertRaisesRegex(ValueError,'Repeated'):validate(bad)
|
||||
|
||||
|
||||
class SQLiteTrackTests(TrackCases,unittest.TestCase):
|
||||
def setUp(self):
|
||||
t=tempfile.TemporaryDirectory();self.addCleanup(t.cleanup);self.root=Path(t.name);db=self.root/'source.sqlite'
|
||||
self.inventory=seed(db);self.target={'db':db}
|
||||
|
||||
|
||||
@unittest.skipUnless(os.environ.get('PAPER_ATLAS_TEST_PG_SERVICE'),'Requires approved remote PostgreSQL service')
|
||||
class PostgreSQLTrackTests(TrackCases,unittest.TestCase):
|
||||
def setUp(self):
|
||||
t=tempfile.TemporaryDirectory();self.addCleanup(t.cleanup);self.root=Path(t.name);db=self.root/'source.sqlite';self.inventory=seed(db)
|
||||
self.service=os.environ['PAPER_ATLAS_TEST_PG_SERVICE'];self.schema='pa_tracks_'+uuid.uuid4().hex[:12];self.addCleanup(self.cleanup)
|
||||
with redirect_stdout(io.StringIO()):migrate(db,self.service,self.root/'migration.json',schema=self.schema,min_free_gib=0)
|
||||
self.target={'pg_service':self.service,'pg_schema':self.schema}
|
||||
|
||||
def cleanup(self):
|
||||
with connect(self.service)as c:
|
||||
for name, in c.execute('SELECT nspname FROM pg_namespace WHERE starts_with(nspname,%s)',(self.schema,)):
|
||||
c.execute('DROP SCHEMA '+quote(name)+' CASCADE')
|
||||
Reference in New Issue
Block a user