Preserve NUL text through PostgreSQL migration and resume
Metadata integrity checks / metadata (push) Successful in 1m7s

This commit is contained in:
2026-11-07 03:27:57 +08:00
parent 3f040b968f
commit 4e62710bba
6 changed files with 208 additions and 14 deletions
+17 -4
View File
@@ -1,6 +1,7 @@
"""Libpq service connections; credentials never belong in CLI arguments."""
from collections.abc import Mapping
import re
from paper_atlas.postgres import text
SEARCH_VECTOR = "to_tsvector('english'::regconfig,coalesce(title,'') || ' ' || coalesce(abstract,''))"
@@ -44,9 +45,9 @@ class Row(Mapping):
return len(self.columns)
def row_factory(cursor):
def row_factory(cursor, *, encoded=False):
columns = [column.name for column in cursor.description] if cursor.description else []
return lambda values: Row(columns, values)
return lambda values: Row(columns, tuple(map(text.decode, values)) if encoded else values)
class ReadConnection:
@@ -59,10 +60,21 @@ class ReadConnection:
from paper_atlas.postgres.schema import quote
if schema is not None:
self.connection.execute('SET LOCAL search_path TO ' + quote(schema) + ',pg_catalog')
self.connection.row_factory = row_factory
if not self.has_table('papers'):
self.close()
raise RuntimeError('PostgreSQL publication is not ready in the configured search_path')
current = self.connection.execute('SELECT current_schema()').fetchone()[0]
# Schema comments are readable by other projects granted table access;
# consumers need no privileges on the private migration controller.
annotation = self.connection.execute("SELECT obj_description(oid,'pg_namespace') FROM pg_namespace WHERE nspname=%s", (current,)).fetchone()[0]
self.encoded = False
if annotation and annotation.startswith('paper-atlas:text-codec='):
codec = annotation.removeprefix('paper-atlas:text-codec=')
if codec != text.CODEC:
self.close()
raise RuntimeError('Unsupported PostgreSQL text storage codec')
self.encoded = codec == text.CODEC
self.connection.row_factory = lambda cursor: row_factory(cursor, encoded=self.encoded)
def execute(self, query, params=()):
# All caller SQL is static; user values are always bound parameters.
@@ -73,7 +85,8 @@ class ReadConnection:
return self.connection.execute('''SELECT 1 FROM information_schema.tables
WHERE table_schema=current_schema() AND table_name=%s''',
(match[1],))
return self.connection.execute(query.replace('?', '%s'), params)
values = tuple(map(text.encode, params)) if self.encoded else params
return self.connection.execute(query.replace('?', '%s'), values)
def has_table(self, name):
return bool(self.connection.execute('''SELECT 1 FROM information_schema.tables