Compare commits

...

1 Commits

Author SHA1 Message Date
codex-bot dc4f61d2f6 Compress large source revision snapshots before PostgreSQL writes
Metadata integrity checks / metadata (push) Successful in 7m33s
2026-11-20 23:20:26 +08:00
4 changed files with 54 additions and 7 deletions
+2
View File
@@ -48,6 +48,8 @@ paper-atlas maintain source-revisions \
同一事务显式增加可共享的可选表 `source_scope_revisions`,保持原 21 张业务表的身份。每个版本存储新来源全集、父版本、完整计划及修正前数据;应用接口只追加版本,不更新或删除旧版本。没有新建 PostgreSQL 实例。该表纳入 SQLite→PostgreSQL 的可选表迁移、全量内容哈希、约束与计数验收。`version_sha256` 是规范化计划内容哈希;CLI 的 `--plan-sha256` 是计划文件字节哈希,二者用途不同。 同一事务显式增加可共享的可选表 `source_scope_revisions`,保持原 21 张业务表的身份。每个版本存储新来源全集、父版本、完整计划及修正前数据;应用接口只追加版本,不更新或删除旧版本。没有新建 PostgreSQL 实例。该表纳入 SQLite→PostgreSQL 的可选表迁移、全量内容哈希、约束与计数验收。`version_sha256` 是规范化计划内容哈希;CLI 的 `--plan-sha256` 是计划文件字节哈希,二者用途不同。
源快照和计划各自在 UTF-8 JSON 达到 1 MiB 时由客户端压缩后写入,避免数据库在接收大参数时出现内存峰值。读取兼容既有纯 JSON 和带 `gzip+base64:v1:` 前缀的压缩值,完整内容和计划哈希均可按原口径核验。使用该表执行写入或刷新时,运行时需要支持两种存储格式。
提交前在同一事务比较全部保护数据与明确预期的变更;失败则全部回滚,包括新增表和版本记录。相同计划的重放还会重新验收当前状态;如后续数据已变化则拒绝,不会重复迁移或恢复旧状态。完整计划与 before-image 用于审查和恢复准备,本命令没有自动回滚历史版本的入口;不得手工删除版本来绕开刷新保护。 提交前在同一事务比较全部保护数据与明确预期的变更;失败则全部回滚,包括新增表和版本记录。相同计划的重放还会重新验收当前状态;如后续数据已变化则拒绝,不会重复迁移或恢复旧状态。完整计划与 before-image 用于审查和恢复准备,本命令没有自动回滚历史版本的入口;不得手工删除版本来绕开刷新保护。
## 后续刷新及限制 ## 后续刷新及限制
+21 -1
View File
@@ -1,4 +1,6 @@
"""Versioned publication-source scopes and guards against stale source refreshes.""" """Versioned publication-source scopes and guards against stale source refreshes."""
import base64
import gzip
import json import json
import re import re
@@ -6,6 +8,24 @@ TABLE = 'source_scope_revisions'
MARKER = 'source_scope_revisions_v1' MARKER = 'source_scope_revisions_v1'
COLUMNS = ('version_sha256', 'source_id', 'venue_id', 'year', 'event_key', 'list_url', COLUMNS = ('version_sha256', 'source_id', 'venue_id', 'year', 'event_key', 'list_url',
'parent_sha256', 'records_json', 'plan_json', 'applied_at') 'parent_sha256', 'records_json', 'plan_json', 'applied_at')
COMPRESSED_PREFIX = 'gzip+base64:v1:'
COMPRESSION_THRESHOLD = 1024 * 1024
def encode_payload(value):
"""Compress large revision payloads before sending them to the database."""
text = json.dumps(value, ensure_ascii=False, separators=(',', ':'))
raw = text.encode('utf-8')
if len(raw) < COMPRESSION_THRESHOLD:
return text
return COMPRESSED_PREFIX + base64.b64encode(gzip.compress(raw, compresslevel=1, mtime=0)).decode('ascii')
def decode_payload(text):
if text.startswith(COMPRESSED_PREFIX):
packed = base64.b64decode(text[len(COMPRESSED_PREFIX):], validate=True)
return json.loads(gzip.decompress(packed))
return json.loads(text)
def available(c): def available(c):
@@ -60,7 +80,7 @@ def current_rows(c, source_id, cache):
records = {}; scopes = {} records = {}; scopes = {}
for signature in signatures: for signature in signatures:
raw = c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone()[0] raw = c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone()[0]
scopes[signature] = json.loads(raw) scopes[signature] = decode_payload(raw)
for row in scopes[signature]: for row in scopes[signature]:
ext = row['external_id'] ext = row['external_id']
if ext in records and records[ext] != row: if ext in records and records[ext] != row:
+8 -5
View File
@@ -203,7 +203,7 @@ def changes(c, inventory, before):
def proposal(c, inventory, identity): def proposal(c, inventory, identity):
before = capture(c, inventory) before = capture(c, inventory)
if before['parent_sha256']: if before['parent_sha256']:
saved = json.loads(c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (before['parent_sha256'],)).fetchone()[0]) saved = versions.decode_payload(c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (before['parent_sha256'],)).fetchone()[0])
if sorted(saved, key=lambda r: r['external_id']) != sorted(inventory['previous_rows'], key=lambda r: r['external_id']): if sorted(saved, key=lambda r: r['external_id']) != sorted(inventory['previous_rows'], key=lambda r: r['external_id']):
raise ValueError('Previous snapshot differs from active source version') raise ValueError('Previous snapshot differs from active source version')
records, coverage = changes(c, inventory, before) records, coverage = changes(c, inventory, before)
@@ -241,8 +241,8 @@ def verify(c, plan, signature):
if after['coverage'] != expected_coverage: if after['coverage'] != expected_coverage:
raise ValueError('Reconciled coverage differs; roll back') raise ValueError('Reconciled coverage differs; roll back')
stored = c.execute('SELECT records_json,plan_json,parent_sha256 FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone() stored = c.execute('SELECT records_json,plan_json,parent_sha256 FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone()
if (not stored or json.loads(stored[0]) != plan['inventory']['rows'] if (not stored or versions.decode_payload(stored[0]) != plan['inventory']['rows']
or json.loads(stored[1]) != plan or stored[2] != plan['before']['parent_sha256']): or versions.decode_payload(stored[1]) != plan or stored[2] != plan['before']['parent_sha256']):
raise ValueError('Versioned source or complete before-images changed') raise ValueError('Versioned source or complete before-images changed')
return {'records_changed': len(plan['changes']), 'events_moved': sum(r['from_edition_id'] != r['target_edition_id'] for r in plan['changes']), return {'records_changed': len(plan['changes']), 'events_moved': sum(r['from_edition_id'] != r['target_edition_id'] for r in plan['changes']),
'protected_papers': len(expected['papers']), 'protected_decisions': len(expected['decisions'])} 'protected_papers': len(expected['papers']), 'protected_decisions': len(expected['decisions'])}
@@ -269,6 +269,9 @@ def apply_plan(c, plan):
signature = digest(plan) signature = digest(plan)
if versions.available(c) and c.execute('SELECT 1 FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone(): if versions.available(c) and c.execute('SELECT 1 FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone():
return {'status': 'already_applied', 'version_sha256': signature, **verify(c, plan, signature)} return {'status': 'already_applied', 'version_sha256': signature, **verify(c, plan, signature)}
inventory = plan['inventory']
records_payload = versions.encode_payload(inventory['rows'])
plan_payload = versions.encode_payload(plan)
if getattr(c, 'postgres', False): if getattr(c, 'postgres', False):
c.execute('LOCK TABLE sources,papers,identifiers,source_records,venues,venue_aliases,editions,edition_sources,submissions,decisions,submission_sources,submission_identifiers,decision_sources,tags,paper_tags IN SHARE ROW EXCLUSIVE MODE') c.execute('LOCK TABLE sources,papers,identifiers,source_records,venues,venue_aliases,editions,edition_sources,submissions,decisions,submission_sources,submission_identifiers,decision_sources,tags,paper_tags IN SHARE ROW EXCLUSIVE MODE')
if proposal(c, plan['inventory'], plan['database']) != plan: if proposal(c, plan['inventory'], plan['database']) != plan:
@@ -284,10 +287,10 @@ def apply_plan(c, plan):
before = row['before'] before = row['before']
c.execute('UPDATE edition_sources SET expected_count=?,imported_count=? WHERE edition_id=? AND source_id=? AND list_url=?', c.execute('UPDATE edition_sources SET expected_count=?,imported_count=? WHERE edition_id=? AND source_id=? AND list_url=?',
(row['expected_count'], row['imported_count'], before[0], before[1], before[2])) (row['expected_count'], row['imported_count'], before[0], before[1], before[2]))
inventory = plan['inventory']
c.execute('''INSERT INTO source_scope_revisions(version_sha256,source_id,venue_id,year,event_key,list_url,parent_sha256,records_json,plan_json) c.execute('''INSERT INTO source_scope_revisions(version_sha256,source_id,venue_id,year,event_key,list_url,parent_sha256,records_json,plan_json)
VALUES(?,?,?,?,?,?,?,?,?)''', (signature, plan['before']['source_id'], plan['before']['venue_id'], inventory['year'], inventory['event_key'], VALUES(?,?,?,?,?,?,?,?,?)''', (signature, plan['before']['source_id'], plan['before']['venue_id'], inventory['year'], inventory['event_key'],
inventory['list_url'], plan['before']['parent_sha256'], json.dumps(inventory['rows'], ensure_ascii=False), json.dumps(plan, ensure_ascii=False))) inventory['list_url'], plan['before']['parent_sha256'],
records_payload, plan_payload))
return {'status': 'applied', 'version_sha256': signature, **verify(c, plan, signature)} return {'status': 'applied', 'version_sha256': signature, **verify(c, plan, signature)}
+23 -1
View File
@@ -13,7 +13,7 @@ import uuid
from paper_atlas.db.backend import open_store from paper_atlas.db.backend import open_store
from paper_atlas.db.edition_scopes import KEY from paper_atlas.db.edition_scopes import KEY
from paper_atlas.db.source_revisions import available, require_row, require_manifest from paper_atlas.db.source_revisions import available, decode_payload, require_row, require_manifest
from paper_atlas.db.store import Store from paper_atlas.db.store import Store
from paper_atlas.ingestion.source_revisions import proposal, apply_plan, run, validate from paper_atlas.ingestion.source_revisions import proposal, apply_plan, run, validate
from paper_atlas.ingestion.event_identity import database_identity, digest, values from paper_atlas.ingestion.event_identity import database_identity, digest, values
@@ -69,6 +69,28 @@ class RevisionCases:
def plan(self, c): def plan(self, c):
return proposal(c, self.inventory, database_identity(c, self.target.get('db'))) return proposal(c, self.inventory, database_identity(c, self.target.get('db')))
def test_large_payload_round_trips_and_protects_next_revision(self):
for key in ('previous_rows', 'rows'):
self.inventory[key][-1]['metadata']['evidence'] = '来源原文' * 100000
with closing(self.store().conn) as c:
plan = self.plan(c)
applied = apply_plan(c, plan)
c.commit()
saved = c.execute('SELECT records_json,plan_json FROM source_scope_revisions').fetchone()
expected = (self.inventory['rows'], plan)
for i, value in enumerate(expected):
self.assertTrue(saved[i].startswith('gzip+base64:v1:'))
self.assertLess(len(saved[i]), len(json.dumps(value, ensure_ascii=False).encode()) // 10)
self.assertEqual(decode_payload(saved[i]), value)
self.assertEqual(apply_plan(c, plan)['status'], 'already_applied')
sid = c.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0]
require_row(c, sid, self.inventory['rows'][-1], {'list_url': URL}, {})
next_inventory = copy.deepcopy(self.inventory)
next_inventory['previous_rows'] = copy.deepcopy(self.inventory['rows'])
next_inventory['rows'][-1]['metadata']['reviewed'] = True
next_plan = proposal(c, next_inventory, database_identity(c, self.target.get('db')))
self.assertEqual(next_plan['before']['parent_sha256'], applied['version_sha256'])
def test_move_to_populated_target_preserves_full_history_and_before_images(self): def test_move_to_populated_target_preserves_full_history_and_before_images(self):
with closing(self.store().conn) as c: with closing(self.store().conn) as c:
plan = self.plan(c); self.assertEqual(len(plan['changes']), 1) plan = self.plan(c); self.assertEqual(len(plan['changes']), 1)