diff --git a/docs/source-revisions.md b/docs/source-revisions.md index 971de5d..f04f769 100644 --- a/docs/source-revisions.md +++ b/docs/source-revisions.md @@ -48,6 +48,8 @@ paper-atlas maintain source-revisions \ 同一事务显式增加可共享的可选表 `source_scope_revisions`,保持原 21 张业务表的身份。每个版本存储新来源全集、父版本、完整计划及修正前数据;应用接口只追加版本,不更新或删除旧版本。没有新建 PostgreSQL 实例。该表纳入 SQLite→PostgreSQL 的可选表迁移、全量内容哈希、约束与计数验收。`version_sha256` 是规范化计划内容哈希;CLI 的 `--plan-sha256` 是计划文件字节哈希,二者用途不同。 +源快照和计划各自在 UTF-8 JSON 达到 1 MiB 时由客户端压缩后写入,避免数据库在接收大参数时出现内存峰值。读取兼容既有纯 JSON 和带 `gzip+base64:v1:` 前缀的压缩值,完整内容和计划哈希均可按原口径核验。使用该表执行写入或刷新时,运行时需要支持两种存储格式。 + 提交前在同一事务比较全部保护数据与明确预期的变更;失败则全部回滚,包括新增表和版本记录。相同计划的重放还会重新验收当前状态;如后续数据已变化则拒绝,不会重复迁移或恢复旧状态。完整计划与 before-image 用于审查和恢复准备,本命令没有自动回滚历史版本的入口;不得手工删除版本来绕开刷新保护。 ## 后续刷新及限制 diff --git a/paper_atlas/db/source_revisions.py b/paper_atlas/db/source_revisions.py index e6e71b2..fa8b725 100644 --- a/paper_atlas/db/source_revisions.py +++ b/paper_atlas/db/source_revisions.py @@ -1,4 +1,6 @@ """Versioned publication-source scopes and guards against stale source refreshes.""" +import base64 +import gzip import json import re @@ -6,6 +8,24 @@ TABLE = 'source_scope_revisions' MARKER = 'source_scope_revisions_v1' COLUMNS = ('version_sha256', 'source_id', 'venue_id', 'year', 'event_key', 'list_url', 'parent_sha256', 'records_json', 'plan_json', 'applied_at') +COMPRESSED_PREFIX = 'gzip+base64:v1:' +COMPRESSION_THRESHOLD = 1024 * 1024 + + +def encode_payload(value): + """Compress large revision payloads before sending them to the database.""" + text = json.dumps(value, ensure_ascii=False, separators=(',', ':')) + raw = text.encode('utf-8') + if len(raw) < COMPRESSION_THRESHOLD: + return text + return COMPRESSED_PREFIX + base64.b64encode(gzip.compress(raw, compresslevel=1, mtime=0)).decode('ascii') + + +def decode_payload(text): + if text.startswith(COMPRESSED_PREFIX): + packed = base64.b64decode(text[len(COMPRESSED_PREFIX):], validate=True) + return json.loads(gzip.decompress(packed)) + return json.loads(text) def available(c): @@ -60,7 +80,7 @@ def current_rows(c, source_id, cache): records = {}; scopes = {} for signature in signatures: raw = c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone()[0] - scopes[signature] = json.loads(raw) + scopes[signature] = decode_payload(raw) for row in scopes[signature]: ext = row['external_id'] if ext in records and records[ext] != row: diff --git a/paper_atlas/ingestion/source_revisions.py b/paper_atlas/ingestion/source_revisions.py index 28017dc..467f08d 100644 --- a/paper_atlas/ingestion/source_revisions.py +++ b/paper_atlas/ingestion/source_revisions.py @@ -203,7 +203,7 @@ def changes(c, inventory, before): def proposal(c, inventory, identity): before = capture(c, inventory) if before['parent_sha256']: - saved = json.loads(c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (before['parent_sha256'],)).fetchone()[0]) + saved = versions.decode_payload(c.execute('SELECT records_json FROM source_scope_revisions WHERE version_sha256=?', (before['parent_sha256'],)).fetchone()[0]) if sorted(saved, key=lambda r: r['external_id']) != sorted(inventory['previous_rows'], key=lambda r: r['external_id']): raise ValueError('Previous snapshot differs from active source version') records, coverage = changes(c, inventory, before) @@ -241,8 +241,8 @@ def verify(c, plan, signature): if after['coverage'] != expected_coverage: raise ValueError('Reconciled coverage differs; roll back') stored = c.execute('SELECT records_json,plan_json,parent_sha256 FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone() - if (not stored or json.loads(stored[0]) != plan['inventory']['rows'] - or json.loads(stored[1]) != plan or stored[2] != plan['before']['parent_sha256']): + if (not stored or versions.decode_payload(stored[0]) != plan['inventory']['rows'] + or versions.decode_payload(stored[1]) != plan or stored[2] != plan['before']['parent_sha256']): raise ValueError('Versioned source or complete before-images changed') return {'records_changed': len(plan['changes']), 'events_moved': sum(r['from_edition_id'] != r['target_edition_id'] for r in plan['changes']), 'protected_papers': len(expected['papers']), 'protected_decisions': len(expected['decisions'])} @@ -269,6 +269,9 @@ def apply_plan(c, plan): signature = digest(plan) if versions.available(c) and c.execute('SELECT 1 FROM source_scope_revisions WHERE version_sha256=?', (signature,)).fetchone(): return {'status': 'already_applied', 'version_sha256': signature, **verify(c, plan, signature)} + inventory = plan['inventory'] + records_payload = versions.encode_payload(inventory['rows']) + plan_payload = versions.encode_payload(plan) if getattr(c, 'postgres', False): c.execute('LOCK TABLE sources,papers,identifiers,source_records,venues,venue_aliases,editions,edition_sources,submissions,decisions,submission_sources,submission_identifiers,decision_sources,tags,paper_tags IN SHARE ROW EXCLUSIVE MODE') if proposal(c, plan['inventory'], plan['database']) != plan: @@ -284,10 +287,10 @@ def apply_plan(c, plan): before = row['before'] c.execute('UPDATE edition_sources SET expected_count=?,imported_count=? WHERE edition_id=? AND source_id=? AND list_url=?', (row['expected_count'], row['imported_count'], before[0], before[1], before[2])) - inventory = plan['inventory'] c.execute('''INSERT INTO source_scope_revisions(version_sha256,source_id,venue_id,year,event_key,list_url,parent_sha256,records_json,plan_json) VALUES(?,?,?,?,?,?,?,?,?)''', (signature, plan['before']['source_id'], plan['before']['venue_id'], inventory['year'], inventory['event_key'], - inventory['list_url'], plan['before']['parent_sha256'], json.dumps(inventory['rows'], ensure_ascii=False), json.dumps(plan, ensure_ascii=False))) + inventory['list_url'], plan['before']['parent_sha256'], + records_payload, plan_payload)) return {'status': 'applied', 'version_sha256': signature, **verify(c, plan, signature)} diff --git a/tests/test_source_revisions.py b/tests/test_source_revisions.py index 7d7b982..d53165e 100644 --- a/tests/test_source_revisions.py +++ b/tests/test_source_revisions.py @@ -13,7 +13,7 @@ import uuid from paper_atlas.db.backend import open_store from paper_atlas.db.edition_scopes import KEY -from paper_atlas.db.source_revisions import available, require_row, require_manifest +from paper_atlas.db.source_revisions import available, decode_payload, require_row, require_manifest from paper_atlas.db.store import Store from paper_atlas.ingestion.source_revisions import proposal, apply_plan, run, validate from paper_atlas.ingestion.event_identity import database_identity, digest, values @@ -69,6 +69,28 @@ class RevisionCases: def plan(self, c): return proposal(c, self.inventory, database_identity(c, self.target.get('db'))) + def test_large_payload_round_trips_and_protects_next_revision(self): + for key in ('previous_rows', 'rows'): + self.inventory[key][-1]['metadata']['evidence'] = '来源原文' * 100000 + with closing(self.store().conn) as c: + plan = self.plan(c) + applied = apply_plan(c, plan) + c.commit() + saved = c.execute('SELECT records_json,plan_json FROM source_scope_revisions').fetchone() + expected = (self.inventory['rows'], plan) + for i, value in enumerate(expected): + self.assertTrue(saved[i].startswith('gzip+base64:v1:')) + self.assertLess(len(saved[i]), len(json.dumps(value, ensure_ascii=False).encode()) // 10) + self.assertEqual(decode_payload(saved[i]), value) + self.assertEqual(apply_plan(c, plan)['status'], 'already_applied') + sid = c.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0] + require_row(c, sid, self.inventory['rows'][-1], {'list_url': URL}, {}) + next_inventory = copy.deepcopy(self.inventory) + next_inventory['previous_rows'] = copy.deepcopy(self.inventory['rows']) + next_inventory['rows'][-1]['metadata']['reviewed'] = True + next_plan = proposal(c, next_inventory, database_identity(c, self.target.get('db'))) + self.assertEqual(next_plan['before']['parent_sha256'], applied['version_sha256']) + def test_move_to_populated_target_preserves_full_history_and_before_images(self): with closing(self.store().conn) as c: plan = self.plan(c); self.assertEqual(len(plan['changes']), 1)