Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| dc4f61d2f6 | |||
| 04b2ed9e95 |
@@ -48,6 +48,8 @@ paper-atlas maintain source-revisions \
|
|||||||
|
|
||||||
同一事务显式增加可共享的可选表 `source_scope_revisions`,保持原 21 张业务表的身份。每个版本存储新来源全集、父版本、完整计划及修正前数据;应用接口只追加版本,不更新或删除旧版本。没有新建 PostgreSQL 实例。该表纳入 SQLite→PostgreSQL 的可选表迁移、全量内容哈希、约束与计数验收。`version_sha256` 是规范化计划内容哈希;CLI 的 `--plan-sha256` 是计划文件字节哈希,二者用途不同。
|
同一事务显式增加可共享的可选表 `source_scope_revisions`,保持原 21 张业务表的身份。每个版本存储新来源全集、父版本、完整计划及修正前数据;应用接口只追加版本,不更新或删除旧版本。没有新建 PostgreSQL 实例。该表纳入 SQLite→PostgreSQL 的可选表迁移、全量内容哈希、约束与计数验收。`version_sha256` 是规范化计划内容哈希;CLI 的 `--plan-sha256` 是计划文件字节哈希,二者用途不同。
|
||||||
|
|
||||||
|
源快照和计划各自在 UTF-8 JSON 达到 1 MiB 时由客户端压缩后写入,避免数据库在接收大参数时出现内存峰值。读取兼容既有纯 JSON 和带 `gzip+base64:v1:` 前缀的压缩值,完整内容和计划哈希均可按原口径核验。使用该表执行写入或刷新时,运行时需要支持两种存储格式。
|
||||||
|
|
||||||
提交前在同一事务比较全部保护数据与明确预期的变更;失败则全部回滚,包括新增表和版本记录。相同计划的重放还会重新验收当前状态;如后续数据已变化则拒绝,不会重复迁移或恢复旧状态。完整计划与 before-image 用于审查和恢复准备,本命令没有自动回滚历史版本的入口;不得手工删除版本来绕开刷新保护。
|
提交前在同一事务比较全部保护数据与明确预期的变更;失败则全部回滚,包括新增表和版本记录。相同计划的重放还会重新验收当前状态;如后续数据已变化则拒绝,不会重复迁移或恢复旧状态。完整计划与 before-image 用于审查和恢复准备,本命令没有自动回滚历史版本的入口;不得手工删除版本来绕开刷新保护。
|
||||||
|
|
||||||
## 后续刷新及限制
|
## 后续刷新及限制
|
||||||
|
|||||||
@@ -1,4 +1,6 @@
|
|||||||
"""Versioned publication-source scopes and guards against stale source refreshes."""
|
"""Versioned publication-source scopes and guards against stale source refreshes."""
|
||||||
|
import base64
|
||||||
|
import gzip
|
||||||
import json
|
import json
|
||||||
import re
|
import re
|
||||||
|
|
||||||
@@ -6,6 +8,24 @@ TABLE = 'source_scope_revisions'
|
|||||||
MARKER = 'source_scope_revisions_v1'
|
MARKER = 'source_scope_revisions_v1'
|
||||||
COLUMNS = ('version_sha256', 'source_id', 'venue_id', 'year', 'event_key', 'list_url',
|
COLUMNS = ('version_sha256', 'source_id', 'venue_id', 'year', 'event_key', 'list_url',
|
||||||
'parent_sha256', 'records_json', 'plan_json', 'applied_at')
|
'parent_sha256', 'records_json', 'plan_json', 'applied_at')
|
||||||
|
COMPRESSED_PREFIX = 'gzip+base64:v1:'
|
||||||
|
COMPRESSION_THRESHOLD = 1024 * 1024
|
||||||
|
|
||||||
|
|
||||||
|
def encode_payload(value):
|
||||||
|
"""Compress large revision payloads before sending them to the database."""
|
||||||
|
text = json.dumps(value, ensure_ascii=False, separators=(',', ':'))
|
||||||
|
raw = text.encode('utf-8')
|
||||||
|
if len(raw) < COMPRESSION_THRESHOLD:
|
||||||
|
return text
|
||||||
|
return COMPRESSED_PREFIX + base64.b64encode(gzip.compress(raw, compresslevel=1, mtime=0)).decode('ascii')
|
||||||
|
|
||||||
|
|
||||||
|
def decode_payload(text):
|
||||||
|
if text.startswith(COMPRESSED_PREFIX):
|
||||||
|
packed = base64.b64decode(text[len(COMPRESSED_PREFIX):], validate=True)
|
||||||
|
return json.loads(gzip.decompress(packed))
|
||||||
|
return json.loads(text)
|
||||||
|
|
||||||
|
|
||||||
def available(c):
|
def available(c):
|
||||||
@@ -60,7 +80,7 @@ def current_rows(c, source_id, cache):
|
|||||||
records = {}; scopes = {}
|
records = {}; scopes = {}
|
||||||
for signature in signatures:
|
for signature in signatures:
|
||||||
raw = c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone()[0]
|
raw = c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone()[0]
|
||||||
scopes[signature] = json.loads(raw)
|
scopes[signature] = decode_payload(raw)
|
||||||
for row in scopes[signature]:
|
for row in scopes[signature]:
|
||||||
ext = row['external_id']
|
ext = row['external_id']
|
||||||
if ext in records and records[ext] != row:
|
if ext in records and records[ext] != row:
|
||||||
|
|||||||
@@ -203,7 +203,7 @@ def changes(c, inventory, before):
|
|||||||
def proposal(c, inventory, identity):
|
def proposal(c, inventory, identity):
|
||||||
before = capture(c, inventory)
|
before = capture(c, inventory)
|
||||||
if before['parent_sha256']:
|
if before['parent_sha256']:
|
||||||
saved = json.loads(c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (before['parent_sha256'],)).fetchone()[0])
|
saved = versions.decode_payload(c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (before['parent_sha256'],)).fetchone()[0])
|
||||||
if sorted(saved, key=lambda r: r['external_id']) != sorted(inventory['previous_rows'], key=lambda r: r['external_id']):
|
if sorted(saved, key=lambda r: r['external_id']) != sorted(inventory['previous_rows'], key=lambda r: r['external_id']):
|
||||||
raise ValueError('Previous snapshot differs from active source version')
|
raise ValueError('Previous snapshot differs from active source version')
|
||||||
records, coverage = changes(c, inventory, before)
|
records, coverage = changes(c, inventory, before)
|
||||||
@@ -241,8 +241,8 @@ def verify(c, plan, signature):
|
|||||||
if after['coverage'] != expected_coverage:
|
if after['coverage'] != expected_coverage:
|
||||||
raise ValueError('Reconciled coverage differs; roll back')
|
raise ValueError('Reconciled coverage differs; roll back')
|
||||||
stored = c.execute('SELECT records_json,plan_json,parent_sha256 FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone()
|
stored = c.execute('SELECT records_json,plan_json,parent_sha256 FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone()
|
||||||
if (not stored or json.loads(stored[0]) != plan['inventory']['rows']
|
if (not stored or versions.decode_payload(stored[0]) != plan['inventory']['rows']
|
||||||
or json.loads(stored[1]) != plan or stored[2] != plan['before']['parent_sha256']):
|
or versions.decode_payload(stored[1]) != plan or stored[2] != plan['before']['parent_sha256']):
|
||||||
raise ValueError('Versioned source or complete before-images changed')
|
raise ValueError('Versioned source or complete before-images changed')
|
||||||
return {'records_changed': len(plan['changes']), 'events_moved': sum(r['from_edition_id'] != r['target_edition_id'] for r in plan['changes']),
|
return {'records_changed': len(plan['changes']), 'events_moved': sum(r['from_edition_id'] != r['target_edition_id'] for r in plan['changes']),
|
||||||
'protected_papers': len(expected['papers']), 'protected_decisions': len(expected['decisions'])}
|
'protected_papers': len(expected['papers']), 'protected_decisions': len(expected['decisions'])}
|
||||||
@@ -269,6 +269,9 @@ def apply_plan(c, plan):
|
|||||||
signature = digest(plan)
|
signature = digest(plan)
|
||||||
if versions.available(c) and c.execute('SELECT 1 FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone():
|
if versions.available(c) and c.execute('SELECT 1 FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone():
|
||||||
return {'status': 'already_applied', 'version_sha256': signature, **verify(c, plan, signature)}
|
return {'status': 'already_applied', 'version_sha256': signature, **verify(c, plan, signature)}
|
||||||
|
inventory = plan['inventory']
|
||||||
|
records_payload = versions.encode_payload(inventory['rows'])
|
||||||
|
plan_payload = versions.encode_payload(plan)
|
||||||
if getattr(c, 'postgres', False):
|
if getattr(c, 'postgres', False):
|
||||||
c.execute('LOCK TABLE sources,papers,identifiers,source_records,venues,venue_aliases,editions,edition_sources,submissions,decisions,submission_sources,submission_identifiers,decision_sources,tags,paper_tags IN SHARE ROW EXCLUSIVE MODE')
|
c.execute('LOCK TABLE sources,papers,identifiers,source_records,venues,venue_aliases,editions,edition_sources,submissions,decisions,submission_sources,submission_identifiers,decision_sources,tags,paper_tags IN SHARE ROW EXCLUSIVE MODE')
|
||||||
if proposal(c, plan['inventory'], plan['database']) != plan:
|
if proposal(c, plan['inventory'], plan['database']) != plan:
|
||||||
@@ -284,10 +287,10 @@ def apply_plan(c, plan):
|
|||||||
before = row['before']
|
before = row['before']
|
||||||
c.execute('UPDATE edition_sources SET expected_count=?,imported_count=? WHERE edition_id=? AND source_id=? AND list_url=?',
|
c.execute('UPDATE edition_sources SET expected_count=?,imported_count=? WHERE edition_id=? AND source_id=? AND list_url=?',
|
||||||
(row['expected_count'], row['imported_count'], before[0], before[1], before[2]))
|
(row['expected_count'], row['imported_count'], before[0], before[1], before[2]))
|
||||||
inventory = plan['inventory']
|
|
||||||
c.execute('''INSERT INTO source_scope_revisions(version_sha256,source_id,venue_id,year,event_key,list_url,parent_sha256,records_json,plan_json)
|
c.execute('''INSERT INTO source_scope_revisions(version_sha256,source_id,venue_id,year,event_key,list_url,parent_sha256,records_json,plan_json)
|
||||||
VALUES(?,?,?,?,?,?,?,?,?)''', (signature, plan['before']['source_id'], plan['before']['venue_id'], inventory['year'], inventory['event_key'],
|
VALUES(?,?,?,?,?,?,?,?,?)''', (signature, plan['before']['source_id'], plan['before']['venue_id'], inventory['year'], inventory['event_key'],
|
||||||
inventory['list_url'], plan['before']['parent_sha256'], json.dumps(inventory['rows'], ensure_ascii=False), json.dumps(plan, ensure_ascii=False)))
|
inventory['list_url'], plan['before']['parent_sha256'],
|
||||||
|
records_payload, plan_payload))
|
||||||
return {'status': 'applied', 'version_sha256': signature, **verify(c, plan, signature)}
|
return {'status': 'applied', 'version_sha256': signature, **verify(c, plan, signature)}
|
||||||
|
|
||||||
|
|
||||||
|
|||||||
@@ -13,7 +13,7 @@ import uuid
|
|||||||
|
|
||||||
from paper_atlas.db.backend import open_store
|
from paper_atlas.db.backend import open_store
|
||||||
from paper_atlas.db.edition_scopes import KEY
|
from paper_atlas.db.edition_scopes import KEY
|
||||||
from paper_atlas.db.source_revisions import available, require_row, require_manifest
|
from paper_atlas.db.source_revisions import available, decode_payload, require_row, require_manifest
|
||||||
from paper_atlas.db.store import Store
|
from paper_atlas.db.store import Store
|
||||||
from paper_atlas.ingestion.source_revisions import proposal, apply_plan, run, validate
|
from paper_atlas.ingestion.source_revisions import proposal, apply_plan, run, validate
|
||||||
from paper_atlas.ingestion.event_identity import database_identity, digest, values
|
from paper_atlas.ingestion.event_identity import database_identity, digest, values
|
||||||
@@ -69,6 +69,28 @@ class RevisionCases:
|
|||||||
def plan(self, c):
|
def plan(self, c):
|
||||||
return proposal(c, self.inventory, database_identity(c, self.target.get('db')))
|
return proposal(c, self.inventory, database_identity(c, self.target.get('db')))
|
||||||
|
|
||||||
|
def test_large_payload_round_trips_and_protects_next_revision(self):
|
||||||
|
for key in ('previous_rows', 'rows'):
|
||||||
|
self.inventory[key][-1]['metadata']['evidence'] = '来源原文' * 100000
|
||||||
|
with closing(self.store().conn) as c:
|
||||||
|
plan = self.plan(c)
|
||||||
|
applied = apply_plan(c, plan)
|
||||||
|
c.commit()
|
||||||
|
saved = c.execute('SELECT records_json,plan_json FROM source_scope_revisions').fetchone()
|
||||||
|
expected = (self.inventory['rows'], plan)
|
||||||
|
for i, value in enumerate(expected):
|
||||||
|
self.assertTrue(saved[i].startswith('gzip+base64:v1:'))
|
||||||
|
self.assertLess(len(saved[i]), len(json.dumps(value, ensure_ascii=False).encode()) // 10)
|
||||||
|
self.assertEqual(decode_payload(saved[i]), value)
|
||||||
|
self.assertEqual(apply_plan(c, plan)['status'], 'already_applied')
|
||||||
|
sid = c.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0]
|
||||||
|
require_row(c, sid, self.inventory['rows'][-1], {'list_url': URL}, {})
|
||||||
|
next_inventory = copy.deepcopy(self.inventory)
|
||||||
|
next_inventory['previous_rows'] = copy.deepcopy(self.inventory['rows'])
|
||||||
|
next_inventory['rows'][-1]['metadata']['reviewed'] = True
|
||||||
|
next_plan = proposal(c, next_inventory, database_identity(c, self.target.get('db')))
|
||||||
|
self.assertEqual(next_plan['before']['parent_sha256'], applied['version_sha256'])
|
||||||
|
|
||||||
def test_move_to_populated_target_preserves_full_history_and_before_images(self):
|
def test_move_to_populated_target_preserves_full_history_and_before_images(self):
|
||||||
with closing(self.store().conn) as c:
|
with closing(self.store().conn) as c:
|
||||||
plan = self.plan(c); self.assertEqual(len(plan['changes']), 1)
|
plan = self.plan(c); self.assertEqual(len(plan['changes']), 1)
|
||||||
|
|||||||
Reference in New Issue
Block a user