Compress large source revision snapshots before PostgreSQL writes
Metadata integrity checks / metadata (push) Successful in 7m33s
Metadata integrity checks / metadata (push) Successful in 7m33s
This commit is contained in:
@@ -13,7 +13,7 @@ import uuid
|
||||
|
||||
from paper_atlas.db.backend import open_store
|
||||
from paper_atlas.db.edition_scopes import KEY
|
||||
from paper_atlas.db.source_revisions import available, require_row, require_manifest
|
||||
from paper_atlas.db.source_revisions import available, decode_payload, require_row, require_manifest
|
||||
from paper_atlas.db.store import Store
|
||||
from paper_atlas.ingestion.source_revisions import proposal, apply_plan, run, validate
|
||||
from paper_atlas.ingestion.event_identity import database_identity, digest, values
|
||||
@@ -69,6 +69,28 @@ class RevisionCases:
|
||||
def plan(self, c):
|
||||
return proposal(c, self.inventory, database_identity(c, self.target.get('db')))
|
||||
|
||||
def test_large_payload_round_trips_and_protects_next_revision(self):
|
||||
for key in ('previous_rows', 'rows'):
|
||||
self.inventory[key][-1]['metadata']['evidence'] = '来源原文' * 100000
|
||||
with closing(self.store().conn) as c:
|
||||
plan = self.plan(c)
|
||||
applied = apply_plan(c, plan)
|
||||
c.commit()
|
||||
saved = c.execute('SELECT records_json,plan_json FROM source_scope_revisions').fetchone()
|
||||
expected = (self.inventory['rows'], plan)
|
||||
for i, value in enumerate(expected):
|
||||
self.assertTrue(saved[i].startswith('gzip+base64:v1:'))
|
||||
self.assertLess(len(saved[i]), len(json.dumps(value, ensure_ascii=False).encode()) // 10)
|
||||
self.assertEqual(decode_payload(saved[i]), value)
|
||||
self.assertEqual(apply_plan(c, plan)['status'], 'already_applied')
|
||||
sid = c.execute("SELECT id FROM sources WHERE name='fixture'").fetchone()[0]
|
||||
require_row(c, sid, self.inventory['rows'][-1], {'list_url': URL}, {})
|
||||
next_inventory = copy.deepcopy(self.inventory)
|
||||
next_inventory['previous_rows'] = copy.deepcopy(self.inventory['rows'])
|
||||
next_inventory['rows'][-1]['metadata']['reviewed'] = True
|
||||
next_plan = proposal(c, next_inventory, database_identity(c, self.target.get('db')))
|
||||
self.assertEqual(next_plan['before']['parent_sha256'], applied['version_sha256'])
|
||||
|
||||
def test_move_to_populated_target_preserves_full_history_and_before_images(self):
|
||||
with closing(self.store().conn) as c:
|
||||
plan = self.plan(c); self.assertEqual(len(plan['changes']), 1)
|
||||
|
||||
Reference in New Issue
Block a user