Preserve Unicode separators in public metadata preflight
Metadata integrity checks / metadata (push) Successful in 2m42s
Metadata integrity checks / metadata (push) Successful in 2m42s
This commit is contained in:
@@ -152,6 +152,35 @@ class PublicOpenReviewTests(unittest.TestCase):
|
||||
finally:
|
||||
store.conn.close()
|
||||
|
||||
def test_unicode_separators_in_abstract_survive_audit_and_import(self):
|
||||
for separator in ('\u0085', '\u2028', '\u2029'):
|
||||
with self.subTest(separator=repr(separator)), tempfile.TemporaryDirectory() as tmp:
|
||||
root = Path(tmp)
|
||||
abstract = 'First paragraph' + separator + 'Second paragraph'
|
||||
result = self.collect(root / 'source', {
|
||||
'venues/icml/icml2025.jsonl': [self.row(abstract=abstract)]})
|
||||
path = Path(result['files'][0]['file'])
|
||||
data = path.read_bytes()
|
||||
self.assertIn(separator.encode(), data)
|
||||
self.assertEqual(data.count(b'\n'), 1)
|
||||
db = root / 'db.sqlite'
|
||||
store = Store(db)
|
||||
store.conn.close()
|
||||
inventory = root / 'source/public-openreview-inventory.json'
|
||||
before = audit(db, inventory, root / 'before.json')
|
||||
self.assertEqual(before['status'], 'ready_for_serial_import')
|
||||
self.assertEqual(before['records_checked'], 1)
|
||||
ingest_file(db, path)
|
||||
after = audit(db, inventory, root / 'after.json')
|
||||
self.assertEqual(after['status'], 'ready_for_serial_import')
|
||||
self.assertEqual(after['existing_same_edition_attempts'], 1)
|
||||
store = Store(db)
|
||||
try:
|
||||
self.assertEqual(store.conn.execute(
|
||||
'SELECT abstract FROM source_records').fetchone()[0], abstract)
|
||||
finally:
|
||||
store.conn.close()
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user