Preserve NUL text through PostgreSQL migration and resume
Metadata integrity checks / metadata (push) Successful in 1m7s

This commit is contained in:
2026-11-07 03:27:57 +08:00
parent 3f040b968f
commit 4e62710bba
6 changed files with 208 additions and 14 deletions
+17 -4
View File
@@ -1,6 +1,7 @@
"""Libpq service connections; credentials never belong in CLI arguments."""
from collections.abc import Mapping
import re
from paper_atlas.postgres import text
SEARCH_VECTOR = "to_tsvector('english'::regconfig,coalesce(title,'') || ' ' || coalesce(abstract,''))"
@@ -44,9 +45,9 @@ class Row(Mapping):
return len(self.columns)
def row_factory(cursor):
def row_factory(cursor, *, encoded=False):
columns = [column.name for column in cursor.description] if cursor.description else []
return lambda values: Row(columns, values)
return lambda values: Row(columns, tuple(map(text.decode, values)) if encoded else values)
class ReadConnection:
@@ -59,10 +60,21 @@ class ReadConnection:
from paper_atlas.postgres.schema import quote
if schema is not None:
self.connection.execute('SET LOCAL search_path TO ' + quote(schema) + ',pg_catalog')
self.connection.row_factory = row_factory
if not self.has_table('papers'):
self.close()
raise RuntimeError('PostgreSQL publication is not ready in the configured search_path')
current = self.connection.execute('SELECT current_schema()').fetchone()[0]
# Schema comments are readable by other projects granted table access;
# consumers need no privileges on the private migration controller.
annotation = self.connection.execute("SELECT obj_description(oid,'pg_namespace') FROM pg_namespace WHERE nspname=%s", (current,)).fetchone()[0]
self.encoded = False
if annotation and annotation.startswith('paper-atlas:text-codec='):
codec = annotation.removeprefix('paper-atlas:text-codec=')
if codec != text.CODEC:
self.close()
raise RuntimeError('Unsupported PostgreSQL text storage codec')
self.encoded = codec == text.CODEC
self.connection.row_factory = lambda cursor: row_factory(cursor, encoded=self.encoded)
def execute(self, query, params=()):
# All caller SQL is static; user values are always bound parameters.
@@ -73,7 +85,8 @@ class ReadConnection:
return self.connection.execute('''SELECT 1 FROM information_schema.tables
WHERE table_schema=current_schema() AND table_name=%s''',
(match[1],))
return self.connection.execute(query.replace('?', '%s'), params)
values = tuple(map(text.encode, params)) if self.encoded else params
return self.connection.execute(query.replace('?', '%s'), values)
def has_table(self, name):
return bool(self.connection.execute('''SELECT 1 FROM information_schema.tables
+55 -8
View File
@@ -14,6 +14,7 @@ import zlib
from paper_atlas.db.submissions import timestamp
from paper_atlas.postgres.client import connect, SEARCH_VECTOR
from paper_atlas.postgres.schema import TABLES, inspect, fingerprint, quote, select_sql, table_sql
from paper_atlas.postgres import text
def digest_file(path):
@@ -96,6 +97,41 @@ def sequences(pg, source, stage, plan):
return result
def upgrade_text_codec(pg, control, stage, plan):
"""Upgrade committed legacy staging rows once; never recopy their source."""
changed = {}
with pg.transaction():
copied = {r[0] for r in pg.execute('SELECT name FROM ' + qualify(control, 'tables'))}
if not copied <= plan.keys():
raise ValueError('Unknown committed staging tables')
# An older run may have reached constraint validation. Replacing SOH
# can temporarily equal another legacy key even though the final mapping
# is bijective. Rebuild unpublished constraints/indexes after the upgrade.
constraints = pg.execute('''SELECT cl.relname,co.conname FROM pg_constraint co
JOIN pg_class cl ON cl.oid=co.conrelid JOIN pg_namespace ns ON ns.oid=cl.relnamespace
WHERE ns.nspname=%s AND co.contype IN ('f','p','u')
ORDER BY CASE WHEN co.contype='f' THEN 0 ELSE 1 END''', (stage,)).fetchall()
for table, name in constraints:
pg.execute('ALTER TABLE ' + qualify(stage, table) + ' DROP CONSTRAINT ' + quote(name))
indexes = pg.execute('''SELECT cl.relname FROM pg_index i JOIN pg_class cl ON cl.oid=i.indexrelid
JOIN pg_namespace ns ON ns.oid=cl.relnamespace WHERE ns.nspname=%s AND i.indisunique''', (stage,)).fetchall()
for name, in indexes:
pg.execute('DROP INDEX ' + qualify(stage, name))
for table in plan:
if table not in copied:
continue
columns = [quote(c['name']) for c in plan[table]['columns'] if c['type'] == 'TEXT']
if columns:
assignments = ','.join(c + '=replace(' + c + ',%s,%s)' for c in columns)
predicate = ' OR '.join('strpos(' + c + ',%s)>0' for c in columns)
params = [v for c in columns for v in (text.ESCAPE, text.encode(text.ESCAPE))]
params += [text.ESCAPE] * len(columns)
changed[table] = pg.execute('UPDATE ' + qualify(stage, table) + ' SET ' + assignments +
' WHERE ' + predicate, params).rowcount
pg.execute('UPDATE ' + qualify(control, 'run') + ' SET text_codec=%s WHERE id=1', (text.CODEC,))
return changed
def migrate(db, service, out, *, schema='research', storage_path=None, min_free_gib=8):
if not re.fullmatch(r'[a-z][a-z0-9_]{0,35}', schema):
raise ValueError('Use a short lowercase target schema name')
@@ -129,20 +165,28 @@ def migrate(db, service, out, *, schema='research', storage_path=None, min_free_
'(id INTEGER PRIMARY KEY CHECK(id=1), source_json TEXT NOT NULL, stage TEXT NOT NULL, status TEXT NOT NULL)')
pg.execute('CREATE TABLE IF NOT EXISTS ' + qualify(control, 'tables') +
'(name TEXT PRIMARY KEY, rows BIGINT NOT NULL, sha256 TEXT NOT NULL, verified BOOLEAN NOT NULL DEFAULT false)')
run = pg.execute('SELECT source_json,stage,status FROM ' + qualify(control, 'run') + ' WHERE id=1').fetchone()
pg.execute('ALTER TABLE ' + qualify(control, 'run') + " ADD COLUMN IF NOT EXISTS text_codec TEXT NOT NULL DEFAULT 'legacy'")
run = pg.execute('SELECT source_json,stage,status,text_codec FROM ' + qualify(control, 'run') + ' WHERE id=1').fetchone()
if run:
if json.loads(run[0]) != identity or run[2] == 'complete':
raise ValueError('Resume requires the exact same unpublished SQLite snapshot')
stage = run[1]
if run[3] == 'legacy':
report['legacy_text_upgrade_rows'] = upgrade_text_codec(pg, control, stage, plan)
elif run[3] != text.CODEC:
raise ValueError('Unsupported staging text storage codec')
else:
stage = schema + '_stage_' + uuid.uuid4().hex[:12]
with pg.transaction():
pg.execute('CREATE SCHEMA ' + quote(stage))
for table, spec in plan.items():
pg.execute(table_sql(stage, table, spec))
pg.execute('INSERT INTO ' + qualify(control, 'run') + ' VALUES(1,%s,%s,%s)',
(json.dumps(identity, sort_keys=True), stage, 'copying'))
report.update(status='copying', stage=stage); save(out, report)
pg.execute('INSERT INTO ' + qualify(control, 'run') +
'(id,source_json,stage,status,text_codec) VALUES(1,%s,%s,%s,%s)',
(json.dumps(identity, sort_keys=True), stage, 'copying', text.CODEC))
report.update(status='copying', stage=stage, text_codec=text.CODEC,
content_hash_scope='Decoded original SQLite values; storage escapes are reversible.')
save(out, report)
try:
for table, spec in plan.items():
free_space(storage_path, min_free_gib)
@@ -157,7 +201,7 @@ def migrate(db, service, out, *, schema='research', storage_path=None, min_free_
columns = ','.join(quote(c['name']) for c in spec['columns'])
with pg.cursor().copy('COPY ' + qualify(stage, table) + '(' + columns + ') FROM STDIN') as copy:
for row in source.execute(select_sql(None, table, spec)):
copy.write_row(row); update_hash(digest, row); rows += 1
copy.write_row(tuple(map(text.encode, row))); update_hash(digest, row); rows += 1
if rows % 10000 == 0:
free_space(storage_path, min_free_gib)
if rows % 100000 == 0:
@@ -172,14 +216,16 @@ def migrate(db, service, out, *, schema='research', storage_path=None, min_free_
report['sequences'] = sequences(pg, source, stage, plan)
report['status'] = 'verifying_contents'; save(out, report)
for table, spec in plan.items():
digest = hashlib.sha256(); rows = 0
digest = hashlib.sha256(); storage_digest = hashlib.sha256(); rows = 0
with pg.transaction(), pg.cursor(name='verify_' + table) as cursor:
cursor.itersize = 512
cursor.execute(select_sql(stage, table, spec))
for row in cursor:
update_hash(digest, row); rows += 1
update_hash(storage_digest, row)
update_hash(digest, tuple(map(text.decode, row))); rows += 1
entry = report['tables'][table]
entry.update(target_rows=rows, target_sha256=digest.hexdigest())
entry.update(target_rows=rows, target_sha256=digest.hexdigest(),
target_storage_sha256=storage_digest.hexdigest())
if rows != entry['source_rows'] or digest.hexdigest() != entry['source_sha256']:
raise ValueError('Complete content verification failed for ' + table)
pg.execute('UPDATE ' + qualify(control, 'tables') + ' SET verified=true WHERE name=%s', (table,))
@@ -207,6 +253,7 @@ def migrate(db, service, out, *, schema='research', storage_path=None, min_free_
report.update(status='prepared', foreign_keys='all_validated', checks='all_validated', native_search_index=True)
save(out, report)
with pg.transaction():
pg.execute('COMMENT ON SCHEMA ' + quote(stage) + " IS 'paper-atlas:text-codec=" + text.CODEC + "'")
pg.execute('ALTER SCHEMA ' + quote(stage) + ' RENAME TO ' + quote(schema))
pg.execute('UPDATE ' + qualify(control, 'run') + " SET status='complete' WHERE id=1")
report.update(status='complete', finished_at=timestamp())
+31
View File
@@ -0,0 +1,31 @@
"""Order-preserving, reversible storage for SQLite text containing NUL.
PostgreSQL text forbids U+0000. Encode it as SOH/SOH, and encode a literal
SOH as SOH/STX. All other characters stay unchanged. The mapping preserves
binary lexicographic order, including string prefixes and composite keys.
"""
CODEC = 'nul-soh-v1'
ESCAPE = '\x01'
def encode(value):
if isinstance(value, str):
return value.replace(ESCAPE, ESCAPE + '\x02').replace('\x00', ESCAPE + ESCAPE)
return value
def decode(value):
if not isinstance(value, str) or ESCAPE not in value:
return value
result = []
start = 0
while (position := value.find(ESCAPE, start)) >= 0:
result.append(value[start:position])
following = value[position + 1:position + 2]
if following not in (ESCAPE, '\x02'):
raise ValueError('Invalid nul-soh-v1 text escape')
result.append('\x00' if following == ESCAPE else ESCAPE)
start = position + 2
result.append(value[start:])
return ''.join(result)