Expose GPT-4o tokenizer IDs for graph input and output

test_syntax_dag.py

@@ -2,6 +2,8 @@ import unittest
 from dataclasses import replace
 from graphlib import CycleError
 
+import tiktoken
+
 from syntax_dag import Decoder, Edge, Encoder, Proposal
 
 
@@ -56,3 +58,21 @@ class SyntaxDAGTest(unittest.TestCase):
                      "Did the road flood because it rained?"):
             with self.subTest(text=text):
                 self.assertFalse(self.encoder.encode(text).proposals)
+
+
+    def test_model_token_ids(self):
+        tokenizer = tiktoken.get_encoding("o200k_base")
+        text = "中文 🙂 <|endoftext|>"
+        graph = self.encoder.encode(text)
+        self.assertEqual(graph.input_ids, tuple(tokenizer.encode_ordinary(text)))
+        self.assertEqual(tokenizer.decode(graph.input_ids), text)
+        self.assertEqual(self.decoder.decode_ids(graph), graph.input_ids)
+
+        graph = self.encoder.encode("The road flooded because it rained.")
+        committed = graph.commit(graph.proposals[0])
+        self.assertEqual(committed.input_ids, graph.input_ids)
+        self.assertNotEqual(self.decoder.decode_ids(committed), graph.input_ids)
+        self.assertEqual(
+            tokenizer.decode(self.decoder.decode_ids(committed)),
+            self.decoder.decode(committed),
+        )