Merge pull request 'Integrate the syntax DAG and model token IDs' (#3) from codex/dag-tokenizer-integration into main
Verify syntax DAG / verify (push) Successful in 15s
Verify syntax DAG / verify (push) Successful in 15s
Reviewed-on: #3
This commit was merged in pull request #3.
This commit is contained in:
@@ -34,3 +34,16 @@ The first rule covers simple affirmative past-tense statements with a trailing
|
|||||||
finite causal clause. Extraction is heuristic and depends on parser accuracy.
|
finite causal clause. Extraction is heuristic and depends on parser accuracy.
|
||||||
Commits are in-memory graph edits. The prototype uses explicit authorization
|
Commits are in-memory graph edits. The prototype uses explicit authorization
|
||||||
via the CLI flag and has no learned rewrite policy.
|
via the CLI flag and has no learned rewrite policy.
|
||||||
|
|
||||||
|
## Model token IDs
|
||||||
|
|
||||||
|
```sh
|
||||||
|
python -m syntax_dag --ids --commit 'The road flooded because it rained.'
|
||||||
|
```
|
||||||
|
|
||||||
|
JSON includes `encoding`, `input_ids`, `output_ids`, and the output `text`.
|
||||||
|
These are integer vocabulary IDs from tiktoken's `o200k_base` BPE encoding.
|
||||||
|
spaCy tokens still serve as syntax nodes. Their positions are independent of
|
||||||
|
BPE positions. A commit preserves the original input IDs, and the decoder
|
||||||
|
tokenizes the full rendered text so its output IDs reflect the new boundaries.
|
||||||
|
The tokenizer downloads its vocabulary on first use if it is absent from cache.
|
||||||
|
|||||||
@@ -3,3 +3,5 @@ en-core-web-sm @ https://github.com/explosion/spacy-models/releases/download/en_
|
|||||||
|
|
||||||
# spaCy 3.8.11 imports Click when loading its CLI.
|
# spaCy 3.8.11 imports Click when loading its CLI.
|
||||||
click==8.1.8
|
click==8.1.8
|
||||||
|
|
||||||
|
tiktoken==0.12.0
|
||||||
|
|||||||
+27
-2
@@ -1,10 +1,15 @@
|
|||||||
"""Encode English text, commit supported graph edits, and decode a revision."""
|
"""Encode English text, commit supported graph edits, and decode a revision."""
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
import json
|
||||||
from dataclasses import dataclass, replace
|
from dataclasses import dataclass, replace
|
||||||
from graphlib import TopologicalSorter
|
from graphlib import TopologicalSorter
|
||||||
|
|
||||||
import spacy
|
import spacy
|
||||||
|
import tiktoken
|
||||||
|
|
||||||
|
|
||||||
|
MODEL_ENCODING = "o200k_base"
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
@dataclass(frozen=True)
|
||||||
@@ -41,6 +46,7 @@ class SyntaxDAG:
|
|||||||
edges: tuple[Edge, ...]
|
edges: tuple[Edge, ...]
|
||||||
proposals: tuple[Proposal, ...]
|
proposals: tuple[Proposal, ...]
|
||||||
history: tuple[Proposal, ...] = ()
|
history: tuple[Proposal, ...] = ()
|
||||||
|
input_ids: tuple[int, ...] = ()
|
||||||
|
|
||||||
def __post_init__(self):
|
def __post_init__(self):
|
||||||
nodes = set(range(len(self.tokens))) | {clause.id for clause in self.clauses}
|
nodes = set(range(len(self.tokens))) | {clause.id for clause in self.clauses}
|
||||||
@@ -67,6 +73,7 @@ class SyntaxDAG:
|
|||||||
|
|
||||||
class Encoder:
|
class Encoder:
|
||||||
def __init__(self):
|
def __init__(self):
|
||||||
|
self.tokenizer = tiktoken.get_encoding(MODEL_ENCODING)
|
||||||
self.nlp = spacy.load("en_core_web_sm", disable=["ner"])
|
self.nlp = spacy.load("en_core_web_sm", disable=["ner"])
|
||||||
|
|
||||||
def encode(self, text: str) -> SyntaxDAG:
|
def encode(self, text: str) -> SyntaxDAG:
|
||||||
@@ -98,7 +105,10 @@ class Encoder:
|
|||||||
)
|
)
|
||||||
proposals.append(Proposal(Edge(cause.id, effect.id, "causes"),
|
proposals.append(Proposal(Edge(cause.id, effect.id, "causes"),
|
||||||
marker.i, "trailing-because"))
|
marker.i, "trailing-because"))
|
||||||
return SyntaxDAG(tokens, tuple(clauses), tuple(edges), tuple(proposals))
|
return SyntaxDAG(
|
||||||
|
tokens, tuple(clauses), tuple(edges), tuple(proposals),
|
||||||
|
input_ids=tuple(self.tokenizer.encode_ordinary(text)),
|
||||||
|
)
|
||||||
|
|
||||||
@staticmethod
|
@staticmethod
|
||||||
def _causal_marker(sentence):
|
def _causal_marker(sentence):
|
||||||
@@ -130,6 +140,10 @@ class Encoder:
|
|||||||
|
|
||||||
|
|
||||||
class Decoder:
|
class Decoder:
|
||||||
|
def decode_ids(self, graph: SyntaxDAG) -> tuple[int, ...]:
|
||||||
|
tokenizer = tiktoken.get_encoding(MODEL_ENCODING)
|
||||||
|
return tuple(tokenizer.encode_ordinary(self.decode(graph)))
|
||||||
|
|
||||||
def decode(self, graph: SyntaxDAG) -> str:
|
def decode(self, graph: SyntaxDAG) -> str:
|
||||||
clauses = {clause.id: clause for clause in graph.clauses}
|
clauses = {clause.id: clause for clause in graph.clauses}
|
||||||
edits = {}
|
edits = {}
|
||||||
@@ -157,12 +171,23 @@ def main():
|
|||||||
parser = argparse.ArgumentParser(description=__doc__)
|
parser = argparse.ArgumentParser(description=__doc__)
|
||||||
parser.add_argument("text")
|
parser.add_argument("text")
|
||||||
parser.add_argument("--commit", action="store_true", help="Apply mined causal edits")
|
parser.add_argument("--commit", action="store_true", help="Apply mined causal edits")
|
||||||
|
parser.add_argument("--ids", action="store_true", help="Emit model token IDs as JSON")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
graph = Encoder().encode(args.text)
|
graph = Encoder().encode(args.text)
|
||||||
if args.commit:
|
if args.commit:
|
||||||
for proposal in graph.proposals:
|
for proposal in graph.proposals:
|
||||||
graph = graph.commit(proposal)
|
graph = graph.commit(proposal)
|
||||||
print(Decoder().decode(graph))
|
decoder = Decoder()
|
||||||
|
text = decoder.decode(graph)
|
||||||
|
if args.ids:
|
||||||
|
print(json.dumps({
|
||||||
|
"encoding": MODEL_ENCODING,
|
||||||
|
"input_ids": graph.input_ids,
|
||||||
|
"output_ids": decoder.decode_ids(graph),
|
||||||
|
"text": text,
|
||||||
|
}, ensure_ascii=False))
|
||||||
|
else:
|
||||||
|
print(text)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|||||||
@@ -2,6 +2,8 @@ import unittest
|
|||||||
from dataclasses import replace
|
from dataclasses import replace
|
||||||
from graphlib import CycleError
|
from graphlib import CycleError
|
||||||
|
|
||||||
|
import tiktoken
|
||||||
|
|
||||||
from syntax_dag import Decoder, Edge, Encoder, Proposal
|
from syntax_dag import Decoder, Edge, Encoder, Proposal
|
||||||
|
|
||||||
|
|
||||||
@@ -56,3 +58,21 @@ class SyntaxDAGTest(unittest.TestCase):
|
|||||||
"Did the road flood because it rained?"):
|
"Did the road flood because it rained?"):
|
||||||
with self.subTest(text=text):
|
with self.subTest(text=text):
|
||||||
self.assertFalse(self.encoder.encode(text).proposals)
|
self.assertFalse(self.encoder.encode(text).proposals)
|
||||||
|
|
||||||
|
|
||||||
|
def test_model_token_ids(self):
|
||||||
|
tokenizer = tiktoken.get_encoding("o200k_base")
|
||||||
|
text = "中文 🙂 <|endoftext|>"
|
||||||
|
graph = self.encoder.encode(text)
|
||||||
|
self.assertEqual(graph.input_ids, tuple(tokenizer.encode_ordinary(text)))
|
||||||
|
self.assertEqual(tokenizer.decode(graph.input_ids), text)
|
||||||
|
self.assertEqual(self.decoder.decode_ids(graph), graph.input_ids)
|
||||||
|
|
||||||
|
graph = self.encoder.encode("The road flooded because it rained.")
|
||||||
|
committed = graph.commit(graph.proposals[0])
|
||||||
|
self.assertEqual(committed.input_ids, graph.input_ids)
|
||||||
|
self.assertNotEqual(self.decoder.decode_ids(committed), graph.input_ids)
|
||||||
|
self.assertEqual(
|
||||||
|
tokenizer.decode(self.decoder.decode_ids(committed)),
|
||||||
|
self.decoder.decode(committed),
|
||||||
|
)
|
||||||
|
|||||||
Reference in New Issue
Block a user