From bf8178354f7affa5f0cbbaf0fba23e03f29d69fc Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Sat, 7 Nov 2026 16:34:54 +0800 Subject: [PATCH] Preserve Unicode separators in public metadata preflight --- paper_atlas/audits/public_openreview.py | 4 +++- tests/test_public_openreview.py | 29 +++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/paper_atlas/audits/public_openreview.py b/paper_atlas/audits/public_openreview.py index d58fbe2..01d273e 100644 --- a/paper_atlas/audits/public_openreview.py +++ b/paper_atlas/audits/public_openreview.py @@ -39,7 +39,9 @@ def audit(db, inventory, out, *, pg_service=None, pg_schema=None): if json.loads(path.with_suffix('.manifest.json').read_text()) != manifest: raise ValueError('Collected manifest differs from inventory') count = 0 - for line in path.read_text().splitlines(): + # Unicode paragraph/line separators are valid inside JSON strings; + # split physical byte lines, not Unicode text boundaries. + for line in path.read_bytes().splitlines(): row = json.loads(line) if any(row[key] != manifest[key] for key in ('source', 'venue', 'year', 'track')): raise ValueError('Record edition differs from inventory') diff --git a/tests/test_public_openreview.py b/tests/test_public_openreview.py index 6471ab3..b817a1a 100644 --- a/tests/test_public_openreview.py +++ b/tests/test_public_openreview.py @@ -152,6 +152,35 @@ class PublicOpenReviewTests(unittest.TestCase): finally: store.conn.close() + def test_unicode_separators_in_abstract_survive_audit_and_import(self): + for separator in ('\u0085', '\u2028', '\u2029'): + with self.subTest(separator=repr(separator)), tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + abstract = 'First paragraph' + separator + 'Second paragraph' + result = self.collect(root / 'source', { + 'venues/icml/icml2025.jsonl': [self.row(abstract=abstract)]}) + path = Path(result['files'][0]['file']) + data = path.read_bytes() + self.assertIn(separator.encode(), data) + self.assertEqual(data.count(b'\n'), 1) + db = root / 'db.sqlite' + store = Store(db) + store.conn.close() + inventory = root / 'source/public-openreview-inventory.json' + before = audit(db, inventory, root / 'before.json') + self.assertEqual(before['status'], 'ready_for_serial_import') + self.assertEqual(before['records_checked'], 1) + ingest_file(db, path) + after = audit(db, inventory, root / 'after.json') + self.assertEqual(after['status'], 'ready_for_serial_import') + self.assertEqual(after['existing_same_edition_attempts'], 1) + store = Store(db) + try: + self.assertEqual(store.conn.execute( + 'SELECT abstract FROM source_records').fetchone()[0], abstract) + finally: + store.conn.close() + if __name__ == '__main__': unittest.main()