Expose GPT-4o tokenizer IDs for graph input and output
Verify syntax DAG / verify (push) Successful in 37s
Verify syntax DAG / verify (pull_request) Successful in 23s

This commit is contained in:
2026-11-04 16:48:22 +08:00
parent 688613878f
commit 53e0163941
4 changed files with 62 additions and 2 deletions
+27 -2
View File
@@ -1,10 +1,15 @@
"""Encode English text, commit supported graph edits, and decode a revision."""
import argparse
import json
from dataclasses import dataclass, replace
from graphlib import TopologicalSorter
import spacy
import tiktoken
MODEL_ENCODING = "o200k_base"
@dataclass(frozen=True)
@@ -41,6 +46,7 @@ class SyntaxDAG:
edges: tuple[Edge, ...]
proposals: tuple[Proposal, ...]
history: tuple[Proposal, ...] = ()
input_ids: tuple[int, ...] = ()
def __post_init__(self):
nodes = set(range(len(self.tokens))) | {clause.id for clause in self.clauses}
@@ -67,6 +73,7 @@ class SyntaxDAG:
class Encoder:
def __init__(self):
self.tokenizer = tiktoken.get_encoding(MODEL_ENCODING)
self.nlp = spacy.load("en_core_web_sm", disable=["ner"])
def encode(self, text: str) -> SyntaxDAG:
@@ -98,7 +105,10 @@ class Encoder:
)
proposals.append(Proposal(Edge(cause.id, effect.id, "causes"),
marker.i, "trailing-because"))
return SyntaxDAG(tokens, tuple(clauses), tuple(edges), tuple(proposals))
return SyntaxDAG(
tokens, tuple(clauses), tuple(edges), tuple(proposals),
input_ids=tuple(self.tokenizer.encode_ordinary(text)),
)
@staticmethod
def _causal_marker(sentence):
@@ -130,6 +140,10 @@ class Encoder:
class Decoder:
def decode_ids(self, graph: SyntaxDAG) -> tuple[int, ...]:
tokenizer = tiktoken.get_encoding(MODEL_ENCODING)
return tuple(tokenizer.encode_ordinary(self.decode(graph)))
def decode(self, graph: SyntaxDAG) -> str:
clauses = {clause.id: clause for clause in graph.clauses}
edits = {}
@@ -157,12 +171,23 @@ def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("text")
parser.add_argument("--commit", action="store_true", help="Apply mined causal edits")
parser.add_argument("--ids", action="store_true", help="Emit model token IDs as JSON")
args = parser.parse_args()
graph = Encoder().encode(args.text)
if args.commit:
for proposal in graph.proposals:
graph = graph.commit(proposal)
print(Decoder().decode(graph))
decoder = Decoder()
text = decoder.decode(graph)
if args.ids:
print(json.dumps({
"encoding": MODEL_ENCODING,
"input_ids": graph.input_ids,
"output_ids": decoder.decode_ids(graph),
"text": text,
}, ensure_ascii=False))
else:
print(text)
if __name__ == "__main__":