Preserve NUL text through PostgreSQL migration and resume
Metadata integrity checks / metadata (push) Successful in 1m7s

This commit is contained in:
2026-11-07 03:27:57 +08:00
parent 3f040b968f
commit 4e62710bba
6 changed files with 208 additions and 14 deletions
+31
View File
@@ -0,0 +1,31 @@
"""Order-preserving, reversible storage for SQLite text containing NUL.
PostgreSQL text forbids U+0000. Encode it as SOH/SOH, and encode a literal
SOH as SOH/STX. All other characters stay unchanged. The mapping preserves
binary lexicographic order, including string prefixes and composite keys.
"""
CODEC = 'nul-soh-v1'
ESCAPE = '\x01'
def encode(value):
if isinstance(value, str):
return value.replace(ESCAPE, ESCAPE + '\x02').replace('\x00', ESCAPE + ESCAPE)
return value
def decode(value):
if not isinstance(value, str) or ESCAPE not in value:
return value
result = []
start = 0
while (position := value.find(ESCAPE, start)) >= 0:
result.append(value[start:position])
following = value[position + 1:position + 2]
if following not in (ESCAPE, '\x02'):
raise ValueError('Invalid nul-soh-v1 text escape')
result.append('\x00' if following == ESCAPE else ESCAPE)
start = position + 2
result.append(value[start:])
return ''.join(result)