From 53e0163941e709e132546f031a98973025f68f4a Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Wed, 4 Nov 2026 16:48:22 +0800 Subject: [PATCH] Expose GPT-4o tokenizer IDs for graph input and output --- README.md | 13 +++++++++++++ requirements.txt | 2 ++ syntax_dag.py | 29 +++++++++++++++++++++++++++-- test_syntax_dag.py | 20 ++++++++++++++++++++ 4 files changed, 62 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 474cec3..59256b7 100644 --- a/README.md +++ b/README.md @@ -34,3 +34,16 @@ The first rule covers simple affirmative past-tense statements with a trailing finite causal clause. Extraction is heuristic and depends on parser accuracy. Commits are in-memory graph edits. The prototype uses explicit authorization via the CLI flag and has no learned rewrite policy. + +## Model token IDs + +```sh +python -m syntax_dag --ids --commit 'The road flooded because it rained.' +``` + +JSON includes `encoding`, `input_ids`, `output_ids`, and the output `text`. +These are integer vocabulary IDs from tiktoken's `o200k_base` BPE encoding. +spaCy tokens still serve as syntax nodes. Their positions are independent of +BPE positions. A commit preserves the original input IDs, and the decoder +tokenizes the full rendered text so its output IDs reflect the new boundaries. +The tokenizer downloads its vocabulary on first use if it is absent from cache. diff --git a/requirements.txt b/requirements.txt index c5b63da..64049ee 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,3 +3,5 @@ en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_ # spaCy 3.8.11 imports Click when loading its CLI. click==8.1.8 + +tiktoken==0.12.0 diff --git a/syntax_dag.py b/syntax_dag.py index c429063..1262aa5 100644 --- a/syntax_dag.py +++ b/syntax_dag.py @@ -1,10 +1,15 @@ """Encode English text, commit supported graph edits, and decode a revision.""" import argparse +import json from dataclasses import dataclass, replace from graphlib import TopologicalSorter import spacy +import tiktoken + + +MODEL_ENCODING = "o200k_base" @dataclass(frozen=True) @@ -41,6 +46,7 @@ class SyntaxDAG: edges: tuple[Edge, ...] proposals: tuple[Proposal, ...] history: tuple[Proposal, ...] = () + input_ids: tuple[int, ...] = () def __post_init__(self): nodes = set(range(len(self.tokens))) | {clause.id for clause in self.clauses} @@ -67,6 +73,7 @@ class SyntaxDAG: class Encoder: def __init__(self): + self.tokenizer = tiktoken.get_encoding(MODEL_ENCODING) self.nlp = spacy.load("en_core_web_sm", disable=["ner"]) def encode(self, text: str) -> SyntaxDAG: @@ -98,7 +105,10 @@ class Encoder: ) proposals.append(Proposal(Edge(cause.id, effect.id, "causes"), marker.i, "trailing-because")) - return SyntaxDAG(tokens, tuple(clauses), tuple(edges), tuple(proposals)) + return SyntaxDAG( + tokens, tuple(clauses), tuple(edges), tuple(proposals), + input_ids=tuple(self.tokenizer.encode_ordinary(text)), + ) @staticmethod def _causal_marker(sentence): @@ -130,6 +140,10 @@ class Encoder: class Decoder: + def decode_ids(self, graph: SyntaxDAG) -> tuple[int, ...]: + tokenizer = tiktoken.get_encoding(MODEL_ENCODING) + return tuple(tokenizer.encode_ordinary(self.decode(graph))) + def decode(self, graph: SyntaxDAG) -> str: clauses = {clause.id: clause for clause in graph.clauses} edits = {} @@ -157,12 +171,23 @@ def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("text") parser.add_argument("--commit", action="store_true", help="Apply mined causal edits") + parser.add_argument("--ids", action="store_true", help="Emit model token IDs as JSON") args = parser.parse_args() graph = Encoder().encode(args.text) if args.commit: for proposal in graph.proposals: graph = graph.commit(proposal) - print(Decoder().decode(graph)) + decoder = Decoder() + text = decoder.decode(graph) + if args.ids: + print(json.dumps({ + "encoding": MODEL_ENCODING, + "input_ids": graph.input_ids, + "output_ids": decoder.decode_ids(graph), + "text": text, + }, ensure_ascii=False)) + else: + print(text) if __name__ == "__main__": diff --git a/test_syntax_dag.py b/test_syntax_dag.py index 38089c1..89ca4e0 100644 --- a/test_syntax_dag.py +++ b/test_syntax_dag.py @@ -2,6 +2,8 @@ import unittest from dataclasses import replace from graphlib import CycleError +import tiktoken + from syntax_dag import Decoder, Edge, Encoder, Proposal @@ -56,3 +58,21 @@ class SyntaxDAGTest(unittest.TestCase): "Did the road flood because it rained?"): with self.subTest(text=text): self.assertFalse(self.encoder.encode(text).proposals) + + + def test_model_token_ids(self): + tokenizer = tiktoken.get_encoding("o200k_base") + text = "δΈ­ζ–‡ πŸ™‚ <|endoftext|>" + graph = self.encoder.encode(text) + self.assertEqual(graph.input_ids, tuple(tokenizer.encode_ordinary(text))) + self.assertEqual(tokenizer.decode(graph.input_ids), text) + self.assertEqual(self.decoder.decode_ids(graph), graph.input_ids) + + graph = self.encoder.encode("The road flooded because it rained.") + committed = graph.commit(graph.proposals[0]) + self.assertEqual(committed.input_ids, graph.input_ids) + self.assertNotEqual(self.decoder.decode_ids(committed), graph.input_ids) + self.assertEqual( + tokenizer.decode(self.decoder.decode_ids(committed)), + self.decoder.decode(committed), + ) -- 2.52.0