Compress large source revision snapshots before PostgreSQL writes
Metadata integrity checks / metadata (push) Successful in 7m33s

This commit is contained in:
2026-11-20 23:20:26 +08:00
parent 04b2ed9e95
commit dc4f61d2f6
4 changed files with 54 additions and 7 deletions
+23 -1
View File
@@ -13,7 +13,7 @@ import uuid
from paper_atlas.db.backend import open_store
from paper_atlas.db.edition_scopes import KEY
from paper_atlas.db.source_revisions import available, require_row, require_manifest
from paper_atlas.db.source_revisions import available, decode_payload, require_row, require_manifest
from paper_atlas.db.store import Store
from paper_atlas.ingestion.source_revisions import proposal, apply_plan, run, validate
from paper_atlas.ingestion.event_identity import database_identity, digest, values
@@ -69,6 +69,28 @@ class RevisionCases:
def plan(self, c):
return proposal(c, self.inventory, database_identity(c, self.target.get('db')))
def test_large_payload_round_trips_and_protects_next_revision(self):
for key in ('previous_rows', 'rows'):
self.inventory[key][-1]['metadata']['evidence'] = '来源原文' * 100000
with closing(self.store().conn) as c:
plan = self.plan(c)
applied = apply_plan(c, plan)
c.commit()
saved = c.execute('SELECT records_json,plan_json FROM source_scope_revisions').fetchone()
expected = (self.inventory['rows'], plan)
for i, value in enumerate(expected):
self.assertTrue(saved[i].startswith('gzip+base64:v1:'))
self.assertLess(len(saved[i]), len(json.dumps(value, ensure_ascii=False).encode()) // 10)
self.assertEqual(decode_payload(saved[i]), value)
self.assertEqual(apply_plan(c, plan)['status'], 'already_applied')
sid = c.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0]
require_row(c, sid, self.inventory['rows'][-1], {'list_url': URL}, {})
next_inventory = copy.deepcopy(self.inventory)
next_inventory['previous_rows'] = copy.deepcopy(self.inventory['rows'])
next_inventory['rows'][-1]['metadata']['reviewed'] = True
next_plan = proposal(c, next_inventory, database_identity(c, self.target.get('db')))
self.assertEqual(next_plan['before']['parent_sha256'], applied['version_sha256'])
def test_move_to_populated_target_preserves_full_history_and_before_images(self):
with closing(self.store().conn) as c:
plan = self.plan(c); self.assertEqual(len(plan['changes']), 1)