Expose GPT-4o tokenizer IDs for graph input and output
This commit is contained in:
+27
-2
@@ -1,10 +1,15 @@
|
||||
"""Encode English text, commit supported graph edits, and decode a revision."""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from dataclasses import dataclass, replace
|
||||
from graphlib import TopologicalSorter
|
||||
|
||||
import spacy
|
||||
import tiktoken
|
||||
|
||||
|
||||
MODEL_ENCODING = "o200k_base"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -41,6 +46,7 @@ class SyntaxDAG:
|
||||
edges: tuple[Edge, ...]
|
||||
proposals: tuple[Proposal, ...]
|
||||
history: tuple[Proposal, ...] = ()
|
||||
input_ids: tuple[int, ...] = ()
|
||||
|
||||
def __post_init__(self):
|
||||
nodes = set(range(len(self.tokens))) | {clause.id for clause in self.clauses}
|
||||
@@ -67,6 +73,7 @@ class SyntaxDAG:
|
||||
|
||||
class Encoder:
|
||||
def __init__(self):
|
||||
self.tokenizer = tiktoken.get_encoding(MODEL_ENCODING)
|
||||
self.nlp = spacy.load("en_core_web_sm", disable=["ner"])
|
||||
|
||||
def encode(self, text: str) -> SyntaxDAG:
|
||||
@@ -98,7 +105,10 @@ class Encoder:
|
||||
)
|
||||
proposals.append(Proposal(Edge(cause.id, effect.id, "causes"),
|
||||
marker.i, "trailing-because"))
|
||||
return SyntaxDAG(tokens, tuple(clauses), tuple(edges), tuple(proposals))
|
||||
return SyntaxDAG(
|
||||
tokens, tuple(clauses), tuple(edges), tuple(proposals),
|
||||
input_ids=tuple(self.tokenizer.encode_ordinary(text)),
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _causal_marker(sentence):
|
||||
@@ -130,6 +140,10 @@ class Encoder:
|
||||
|
||||
|
||||
class Decoder:
|
||||
def decode_ids(self, graph: SyntaxDAG) -> tuple[int, ...]:
|
||||
tokenizer = tiktoken.get_encoding(MODEL_ENCODING)
|
||||
return tuple(tokenizer.encode_ordinary(self.decode(graph)))
|
||||
|
||||
def decode(self, graph: SyntaxDAG) -> str:
|
||||
clauses = {clause.id: clause for clause in graph.clauses}
|
||||
edits = {}
|
||||
@@ -157,12 +171,23 @@ def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("text")
|
||||
parser.add_argument("--commit", action="store_true", help="Apply mined causal edits")
|
||||
parser.add_argument("--ids", action="store_true", help="Emit model token IDs as JSON")
|
||||
args = parser.parse_args()
|
||||
graph = Encoder().encode(args.text)
|
||||
if args.commit:
|
||||
for proposal in graph.proposals:
|
||||
graph = graph.commit(proposal)
|
||||
print(Decoder().decode(graph))
|
||||
decoder = Decoder()
|
||||
text = decoder.decode(graph)
|
||||
if args.ids:
|
||||
print(json.dumps({
|
||||
"encoding": MODEL_ENCODING,
|
||||
"input_ids": graph.input_ids,
|
||||
"output_ids": decoder.decode_ids(graph),
|
||||
"text": text,
|
||||
}, ensure_ascii=False))
|
||||
else:
|
||||
print(text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user