feat: generate unified AST and editor contracts from ASDL
Verify databases and deploy Aster VM / verify-and-deploy (push) Successful in 7m55s

This commit is contained in:
2026-10-19 14:31:28 +08:00
parent be4f1add58
commit f5268d2e09
40 changed files with 2517 additions and 172 deletions
+2
View File
@@ -7,6 +7,8 @@ license.workspace = true
[dependencies]
peg.workspace = true
serde.workspace = true
serde_json.workspace = true
[lints]
workspace = true
+38
View File
@@ -0,0 +1,38 @@
-- Endor AST schema. Source syntax belongs to lexer.peg and grammar.peg.
-- Wire contract: endor-ast-json/1, kind/data tagged sums and UTF-8 byte spans.
-- Builtins: string (owned text), span (UTF-8 byte range), schema_id (wire ID).
-- * is a sequence; ? is optional. Recursion is boxed by the Rust generator.
module Endor {
AstDocument = (schema_id schema, AstRoot root)
AstRoot = Module(Program body)
| Expression(Expr body)
| Interactive(InteractiveInput body)
Name = (string text, span span)
Program = (Decl* declarations, span span)
Decl = (DeclKind kind, span span)
DeclKind = Function(Function definition)
Function = (Name name, Parameter* parameters, Name return_type, Block body, span span)
Parameter = (Name name, Name type_name, span span)
Block = (Stmt* statements, span span)
Stmt = (StmtKind kind, span span)
StmtKind = Binding(Binding binding)
| Return(ReturnStatement statement)
| Expression(Expr expression)
| Declaration(Decl declaration)
Binding = (Name name, Expr value, span span)
ReturnStatement = (Expr? value, span span)
InteractiveInput = (Stmt* statements, span span)
Expr = (ExprKind kind, span span)
ExprKind = Name(string text)
| Number(string spelling)
| String(string value)
| Symbol(Name name)
| Unary(UnaryOp operation, Expr operand)
| Binary(BinaryOp operation, Expr left, Expr right)
| Member(Expr object, Name name)
| Call(Expr callee, Expr* arguments)
UnaryOp = Negate
BinaryOp = Add | Subtract | Multiply | Divide
}
+5 -3
View File
@@ -1,12 +1,14 @@
# frontend
独立源码前端:EndorReader → SourceUnit → 字符 PEG lexer → token → token PEG parser → 拥有文本的 AST。当前解析基础函数与表达式,不生成字节码、不进行类型/名称解析、不执行源码。语法是可继续调整的首版实现,当前契约见 [源码前端](../../docs/runtime/frontend.md)。
独立源码前端:EndorReader → SourceUnit → 字符 PEG lexer → token → token PEG parser → 拥有文本的 AST。AST 节点和 Rust/编辑器接口由 Endor.asdl 统一生成;支持模块、表达式及单次交互语法输入。当前解析基础函数与表达式,不生成字节码、不进行类型/名称解析、不执行源码。语法是可继续调整的首版实现,当前契约见 [源码前端](../../docs/runtime/frontend.md)。
| 内容 | 职责 |
| --- | --- |
| Cargo.toml | frontend 包与 peg 0.8.6 依赖,不依赖 core、Host 或 Storage |
| Cargo.toml | frontend 包与 peg 0.8.6、工作区已有 serde/serde_json 依赖,不依赖 core、Host 或 Storage |
| [Endor.asdl](Endor.asdl) | 唯一 AST 节点定义,供生成器读取 |
| [editor](editor/README.md) | 同源生成的 TypeScript 类型/遍历器和 JSON Schema |
| [src](src/README.md) | 阅读器、SourceUnit、token、源码范围、诊断、AST、词法规则独立在 [lexer.peg](src/lexer.peg),语法规则在 [grammar.peg](src/grammar.peg) |
| [tests](tests/README.md) | Unicode、优先级、函数、完整输入与错误位置验证 |
| [examples](examples/README.md) | 可交给 cli parse 的源码样例 |
宿主通过 EndorReader 读取字符串、流、文件或注入的 URI provider,取得 SourceUnit 后调用 lex_source、parse_source、parse_source_expression;低层 lex、parse、parse_expression 仍保留。读取契约见 [Endor 阅读器](../../docs/runtime/reader.md)。Objects 保持零依赖,Core 仅依赖本地 Objects;后续把 AST 转成已有字节码并交给 VM 解释执行;这是平台内部准备步骤,尚未接通。资源修订、依赖锁和解释器统一升级的设计方向见 [执行与版本模型](../../docs/architecture/execution-and-versions.md)。返回 [工作区](../README.md)。
宿主通过 EndorReader 读取字符串、流、文件或注入的 URI provider,取得 SourceUnit 后调用 lex_source、parse_source、parse_source_expression;统一入口 parse_ast/parse_source_ast 通过 ParseMode 返回 AstRoot;低层 lex、parse、parse_expression 仍保留。AST 定义、生成命令、Visitor 与 JSON 交换见 [AST 模型](../../docs/runtime/ast.md)。读取契约见 [Endor 阅读器](../../docs/runtime/reader.md)。Objects 保持零依赖,Core 仅依赖本地 Objects;后续把 AST 转成已有字节码并交给 VM 解释执行;这是平台内部准备步骤,尚未接通。资源修订、依赖锁和解释器统一升级的设计方向见 [执行与版本模型](../../docs/architecture/execution-and-versions.md)。返回 [工作区](../README.md)。
+14
View File
@@ -0,0 +1,14 @@
# 编辑器 AST 接口
这里的类型和遍历函数均从 [Endor.asdl](../Endor.asdl) 生成,不再维护第二份手写节点定义。
| 文件 | 职责 |
| --- | --- |
| [ast.ts](ast.ts) | TypeScript 节点类型、AST_SCHEMA_ID、Visitor 回调与 walk* 函数 |
| [ast.schema.json](ast.schema.json) | JSON Schema 2020-12;校验带 schema 标识的 AstDocument 结构 |
TypeScript 编译时类型不代替 JSON 校验。编辑器可使用现有 JSON Schema 工具校验文档,再交给类型与遍历接口;walk* 的回调返回 false 时跳过该节点的子树。可选字段允许省略或 null,Rust 的标准输出会写成 null。
所有位置为原文 UTF-8 字节范围,不是 JavaScript UTF-16 下标;编辑器需要以原文换算。JSON Schema 检查结构,父子范围关系和原文 UTF-8 边界由 Rust AstDocument::validate/validate_source 检查。schema 标识变化需要调用方升级契约,不提供隐式迁移。
在仓库根运行 `python3 scripts/generate_ast.py --write` 重新生成,--check 做静态同步检查。完整格式见 [AST 模型](../../../docs/runtime/ast.md)。返回 [frontend](../README.md)。
+601
View File
@@ -0,0 +1,601 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "urn:endor:ast:sha256:c0da1a2640ad85f2c8e8159a10756e649d3aab179823a7cd890d9ebbb7f5d46e",
"$ref": "#/$defs/AstDocument",
"$defs": {
"Span": {
"type": "object",
"properties": {
"start": {
"type": "integer",
"minimum": 0,
"maximum": 9007199254740991
},
"end": {
"type": "integer",
"minimum": 0,
"maximum": 9007199254740991
}
},
"required": [
"start",
"end"
],
"additionalProperties": false
},
"AstDocument": {
"type": "object",
"properties": {
"schema": {
"const": "urn:endor:ast:sha256:c0da1a2640ad85f2c8e8159a10756e649d3aab179823a7cd890d9ebbb7f5d46e"
},
"root": {
"$ref": "#/$defs/AstRoot"
}
},
"required": [
"schema",
"root"
],
"additionalProperties": false
},
"AstRoot": {
"oneOf": [
{
"type": "object",
"properties": {
"kind": {
"const": "Module"
},
"data": {
"$ref": "#/$defs/Program"
}
},
"required": [
"kind",
"data"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"kind": {
"const": "Expression"
},
"data": {
"$ref": "#/$defs/Expr"
}
},
"required": [
"kind",
"data"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"kind": {
"const": "Interactive"
},
"data": {
"$ref": "#/$defs/InteractiveInput"
}
},
"required": [
"kind",
"data"
],
"additionalProperties": false
}
]
},
"Name": {
"type": "object",
"properties": {
"text": {
"type": "string"
},
"span": {
"$ref": "#/$defs/Span"
}
},
"required": [
"text",
"span"
],
"additionalProperties": false
},
"Program": {
"type": "object",
"properties": {
"declarations": {
"type": "array",
"items": {
"$ref": "#/$defs/Decl"
}
},
"span": {
"$ref": "#/$defs/Span"
}
},
"required": [
"declarations",
"span"
],
"additionalProperties": false
},
"Decl": {
"type": "object",
"properties": {
"kind": {
"$ref": "#/$defs/DeclKind"
},
"span": {
"$ref": "#/$defs/Span"
}
},
"required": [
"kind",
"span"
],
"additionalProperties": false
},
"DeclKind": {
"oneOf": [
{
"type": "object",
"properties": {
"kind": {
"const": "Function"
},
"data": {
"$ref": "#/$defs/Function"
}
},
"required": [
"kind",
"data"
],
"additionalProperties": false
}
]
},
"Function": {
"type": "object",
"properties": {
"name": {
"$ref": "#/$defs/Name"
},
"parameters": {
"type": "array",
"items": {
"$ref": "#/$defs/Parameter"
}
},
"return_type": {
"$ref": "#/$defs/Name"
},
"body": {
"$ref": "#/$defs/Block"
},
"span": {
"$ref": "#/$defs/Span"
}
},
"required": [
"name",
"parameters",
"return_type",
"body",
"span"
],
"additionalProperties": false
},
"Parameter": {
"type": "object",
"properties": {
"name": {
"$ref": "#/$defs/Name"
},
"type_name": {
"$ref": "#/$defs/Name"
},
"span": {
"$ref": "#/$defs/Span"
}
},
"required": [
"name",
"type_name",
"span"
],
"additionalProperties": false
},
"Block": {
"type": "object",
"properties": {
"statements": {
"type": "array",
"items": {
"$ref": "#/$defs/Stmt"
}
},
"span": {
"$ref": "#/$defs/Span"
}
},
"required": [
"statements",
"span"
],
"additionalProperties": false
},
"Stmt": {
"type": "object",
"properties": {
"kind": {
"$ref": "#/$defs/StmtKind"
},
"span": {
"$ref": "#/$defs/Span"
}
},
"required": [
"kind",
"span"
],
"additionalProperties": false
},
"StmtKind": {
"oneOf": [
{
"type": "object",
"properties": {
"kind": {
"const": "Binding"
},
"data": {
"$ref": "#/$defs/Binding"
}
},
"required": [
"kind",
"data"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"kind": {
"const": "Return"
},
"data": {
"$ref": "#/$defs/ReturnStatement"
}
},
"required": [
"kind",
"data"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"kind": {
"const": "Expression"
},
"data": {
"$ref": "#/$defs/Expr"
}
},
"required": [
"kind",
"data"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"kind": {
"const": "Declaration"
},
"data": {
"$ref": "#/$defs/Decl"
}
},
"required": [
"kind",
"data"
],
"additionalProperties": false
}
]
},
"Binding": {
"type": "object",
"properties": {
"name": {
"$ref": "#/$defs/Name"
},
"value": {
"$ref": "#/$defs/Expr"
},
"span": {
"$ref": "#/$defs/Span"
}
},
"required": [
"name",
"value",
"span"
],
"additionalProperties": false
},
"ReturnStatement": {
"type": "object",
"properties": {
"value": {
"anyOf": [
{
"$ref": "#/$defs/Expr"
},
{
"type": "null"
}
]
},
"span": {
"$ref": "#/$defs/Span"
}
},
"required": [
"span"
],
"additionalProperties": false
},
"InteractiveInput": {
"type": "object",
"properties": {
"statements": {
"type": "array",
"items": {
"$ref": "#/$defs/Stmt"
}
},
"span": {
"$ref": "#/$defs/Span"
}
},
"required": [
"statements",
"span"
],
"additionalProperties": false
},
"Expr": {
"type": "object",
"properties": {
"kind": {
"$ref": "#/$defs/ExprKind"
},
"span": {
"$ref": "#/$defs/Span"
}
},
"required": [
"kind",
"span"
],
"additionalProperties": false
},
"ExprKind": {
"oneOf": [
{
"type": "object",
"properties": {
"kind": {
"const": "Name"
},
"data": {
"type": "string"
}
},
"required": [
"kind",
"data"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"kind": {
"const": "Number"
},
"data": {
"type": "string"
}
},
"required": [
"kind",
"data"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"kind": {
"const": "String"
},
"data": {
"type": "string"
}
},
"required": [
"kind",
"data"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"kind": {
"const": "Symbol"
},
"data": {
"$ref": "#/$defs/Name"
}
},
"required": [
"kind",
"data"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"kind": {
"const": "Unary"
},
"data": {
"type": "object",
"properties": {
"operation": {
"$ref": "#/$defs/UnaryOp"
},
"operand": {
"$ref": "#/$defs/Expr"
}
},
"required": [
"operation",
"operand"
],
"additionalProperties": false
}
},
"required": [
"kind",
"data"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"kind": {
"const": "Binary"
},
"data": {
"type": "object",
"properties": {
"operation": {
"$ref": "#/$defs/BinaryOp"
},
"left": {
"$ref": "#/$defs/Expr"
},
"right": {
"$ref": "#/$defs/Expr"
}
},
"required": [
"operation",
"left",
"right"
],
"additionalProperties": false
}
},
"required": [
"kind",
"data"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"kind": {
"const": "Member"
},
"data": {
"type": "object",
"properties": {
"object": {
"$ref": "#/$defs/Expr"
},
"name": {
"$ref": "#/$defs/Name"
}
},
"required": [
"object",
"name"
],
"additionalProperties": false
}
},
"required": [
"kind",
"data"
],
"additionalProperties": false
},
{
"type": "object",
"properties": {
"kind": {
"const": "Call"
},
"data": {
"type": "object",
"properties": {
"callee": {
"$ref": "#/$defs/Expr"
},
"arguments": {
"type": "array",
"items": {
"$ref": "#/$defs/Expr"
}
}
},
"required": [
"callee",
"arguments"
],
"additionalProperties": false
}
},
"required": [
"kind",
"data"
],
"additionalProperties": false
}
]
},
"UnaryOp": {
"enum": [
"Negate"
]
},
"BinaryOp": {
"enum": [
"Add",
"Subtract",
"Multiply",
"Divide"
]
}
}
}
+183
View File
@@ -0,0 +1,183 @@
// Generated by scripts/generate_ast.py from Endor.asdl. Do not edit.
export const AST_SCHEMA_ID = "urn:endor:ast:sha256:c0da1a2640ad85f2c8e8159a10756e649d3aab179823a7cd890d9ebbb7f5d46e" as const;
export interface Span { start: number; end: number; }
export type AstDocument = { schema: typeof AST_SCHEMA_ID; root: AstRoot; };
export type AstRoot = { kind: "Module"; data: Program; } | { kind: "Expression"; data: Expr; } | { kind: "Interactive"; data: InteractiveInput; };
export type Name = { text: string; span: Span; };
export type Program = { declarations: Array<Decl>; span: Span; };
export type Decl = { kind: DeclKind; span: Span; };
export type DeclKind = { kind: "Function"; data: Function; };
export type Function = { name: Name; parameters: Array<Parameter>; return_type: Name; body: Block; span: Span; };
export type Parameter = { name: Name; type_name: Name; span: Span; };
export type Block = { statements: Array<Stmt>; span: Span; };
export type Stmt = { kind: StmtKind; span: Span; };
export type StmtKind = { kind: "Binding"; data: Binding; } | { kind: "Return"; data: ReturnStatement; } | { kind: "Expression"; data: Expr; } | { kind: "Declaration"; data: Decl; };
export type Binding = { name: Name; value: Expr; span: Span; };
export type ReturnStatement = { value?: Expr | null; span: Span; };
export type InteractiveInput = { statements: Array<Stmt>; span: Span; };
export type Expr = { kind: ExprKind; span: Span; };
export type ExprKind = { kind: "Name"; data: string; } | { kind: "Number"; data: string; } | { kind: "String"; data: string; } | { kind: "Symbol"; data: Name; } | { kind: "Unary"; data: { operation: UnaryOp; operand: Expr; }; } | { kind: "Binary"; data: { operation: BinaryOp; left: Expr; right: Expr; }; } | { kind: "Member"; data: { object: Expr; name: Name; }; } | { kind: "Call"; data: { callee: Expr; arguments: Array<Expr>; }; };
export type UnaryOp = "Negate";
export type BinaryOp = "Add" | "Subtract" | "Multiply" | "Divide";
export interface Visitor {
visitSpan?(node: Span): boolean | void;
visitAstDocument?(node: AstDocument): boolean | void;
visitAstRoot?(node: AstRoot): boolean | void;
visitName?(node: Name): boolean | void;
visitProgram?(node: Program): boolean | void;
visitDecl?(node: Decl): boolean | void;
visitDeclKind?(node: DeclKind): boolean | void;
visitFunction?(node: Function): boolean | void;
visitParameter?(node: Parameter): boolean | void;
visitBlock?(node: Block): boolean | void;
visitStmt?(node: Stmt): boolean | void;
visitStmtKind?(node: StmtKind): boolean | void;
visitBinding?(node: Binding): boolean | void;
visitReturnStatement?(node: ReturnStatement): boolean | void;
visitInteractiveInput?(node: InteractiveInput): boolean | void;
visitExpr?(node: Expr): boolean | void;
visitExprKind?(node: ExprKind): boolean | void;
visitUnaryOp?(node: UnaryOp): boolean | void;
visitBinaryOp?(node: BinaryOp): boolean | void;
}
export function walkSpan(node: Span, visitor: Visitor): void { visitor.visitSpan?.(node); }
export function walkAstDocument(node: AstDocument, visitor: Visitor): void {
if (visitor.visitAstDocument?.(node) === false) return;
walkAstRoot(node.root, visitor);
}
export function walkAstRoot(node: AstRoot, visitor: Visitor): void {
if (visitor.visitAstRoot?.(node) === false) return;
switch (node.kind) {
case "Module":
walkProgram(node.data, visitor);
break;
case "Expression":
walkExpr(node.data, visitor);
break;
case "Interactive":
walkInteractiveInput(node.data, visitor);
break;
}
}
export function walkName(node: Name, visitor: Visitor): void {
if (visitor.visitName?.(node) === false) return;
walkSpan(node.span, visitor);
}
export function walkProgram(node: Program, visitor: Visitor): void {
if (visitor.visitProgram?.(node) === false) return;
for (const value of node.declarations) walkDecl(value, visitor);
walkSpan(node.span, visitor);
}
export function walkDecl(node: Decl, visitor: Visitor): void {
if (visitor.visitDecl?.(node) === false) return;
walkDeclKind(node.kind, visitor);
walkSpan(node.span, visitor);
}
export function walkDeclKind(node: DeclKind, visitor: Visitor): void {
if (visitor.visitDeclKind?.(node) === false) return;
switch (node.kind) {
case "Function":
walkFunction(node.data, visitor);
break;
}
}
export function walkFunction(node: Function, visitor: Visitor): void {
if (visitor.visitFunction?.(node) === false) return;
walkName(node.name, visitor);
for (const value of node.parameters) walkParameter(value, visitor);
walkName(node.return_type, visitor);
walkBlock(node.body, visitor);
walkSpan(node.span, visitor);
}
export function walkParameter(node: Parameter, visitor: Visitor): void {
if (visitor.visitParameter?.(node) === false) return;
walkName(node.name, visitor);
walkName(node.type_name, visitor);
walkSpan(node.span, visitor);
}
export function walkBlock(node: Block, visitor: Visitor): void {
if (visitor.visitBlock?.(node) === false) return;
for (const value of node.statements) walkStmt(value, visitor);
walkSpan(node.span, visitor);
}
export function walkStmt(node: Stmt, visitor: Visitor): void {
if (visitor.visitStmt?.(node) === false) return;
walkStmtKind(node.kind, visitor);
walkSpan(node.span, visitor);
}
export function walkStmtKind(node: StmtKind, visitor: Visitor): void {
if (visitor.visitStmtKind?.(node) === false) return;
switch (node.kind) {
case "Binding":
walkBinding(node.data, visitor);
break;
case "Return":
walkReturnStatement(node.data, visitor);
break;
case "Expression":
walkExpr(node.data, visitor);
break;
case "Declaration":
walkDecl(node.data, visitor);
break;
}
}
export function walkBinding(node: Binding, visitor: Visitor): void {
if (visitor.visitBinding?.(node) === false) return;
walkName(node.name, visitor);
walkExpr(node.value, visitor);
walkSpan(node.span, visitor);
}
export function walkReturnStatement(node: ReturnStatement, visitor: Visitor): void {
if (visitor.visitReturnStatement?.(node) === false) return;
if (node.value != null) walkExpr(node.value, visitor);
walkSpan(node.span, visitor);
}
export function walkInteractiveInput(node: InteractiveInput, visitor: Visitor): void {
if (visitor.visitInteractiveInput?.(node) === false) return;
for (const value of node.statements) walkStmt(value, visitor);
walkSpan(node.span, visitor);
}
export function walkExpr(node: Expr, visitor: Visitor): void {
if (visitor.visitExpr?.(node) === false) return;
walkExprKind(node.kind, visitor);
walkSpan(node.span, visitor);
}
export function walkExprKind(node: ExprKind, visitor: Visitor): void {
if (visitor.visitExprKind?.(node) === false) return;
switch (node.kind) {
case "Name":
break;
case "Number":
break;
case "String":
break;
case "Symbol":
walkName(node.data, visitor);
break;
case "Unary":
walkUnaryOp(node.data.operation, visitor);
walkExpr(node.data.operand, visitor);
break;
case "Binary":
walkBinaryOp(node.data.operation, visitor);
walkExpr(node.data.left, visitor);
walkExpr(node.data.right, visitor);
break;
case "Member":
walkExpr(node.data.object, visitor);
walkName(node.data.name, visitor);
break;
case "Call":
walkExpr(node.data.callee, visitor);
for (const value of node.data.arguments) walkExpr(value, visitor);
break;
}
}
export function walkUnaryOp(node: UnaryOp, visitor: Visitor): void {
if (visitor.visitUnaryOp?.(node) === false) return;
}
export function walkBinaryOp(node: BinaryOp, visitor: Visitor): void {
if (visitor.visitBinaryOp?.(node) === false) return;
}
+4 -3
View File
@@ -9,8 +9,9 @@
| [diagnostic.rs](diagnostic.rs) | UTF-8 字节范围、词法/语法诊断与一基行列转换 |
| [lexer.rs](lexer.rs) | lex/lex_source 接口、源码游标、逐 Token 调用、位置/诊断换算与字符串解码 |
| [lexer.peg](lexer.peg) | 字符输入的 PEG 词法规则:名称、数字、字符串/转义、标点、空白/注释和错误识别 |
| [ast.rs](ast.rs) | 拥有文本的函数、参数、绑定、return、表达式与运算符结构 |
| [parser.rs](parser.rs) | 解析 API、错误映射、AST 辅助函数,以及独立语法文件的编译期引入 |
| [ast.rs](ast.rs) | 生成类型的公开门面、Spanned 位置接口与便捷方法 |
| [ast](ast/README.md) | ASDL 生成的节点/遍历器与手写 JSON 校验接口 |
| [parser.rs](parser.rs) | 统一根/ParseMode 与具体解析 API、错误映射、AST 辅助函数,以及独立语法文件的编译期引入 |
| [grammar.peg](grammar.peg) | token 输入的 PEG 规则、运算符优先级与 AST 构造动作;保留 peg::parser! 外层声明 |
改输入适配时同步 [阅读器契约](../../../docs/runtime/reader.md) 与读取错误检查;改词法从 lexer.peg 进入,并同步 token、诊断和错误检查;改语法从 grammar.peg 进入,并同步 AST、[测试](../tests/README.md)、样例与 [前端契约](../../../docs/runtime/frontend.md)。词法与语法两份 PEG 动作均不做 VM 分配、I/O 或修改外部状态,避免回溯造成副作用。返回 [frontend](../README.md)。
改节点先编辑 ../Endor.asdl,再用 scripts/generate_ast.py 更新 Rust/编辑器产物,契约见 [AST 模型](../../../docs/runtime/ast.md)。改输入适配时同步 [阅读器契约](../../../docs/runtime/reader.md) 与读取错误检查;改词法从 lexer.peg 进入,并同步 token、诊断和错误检查;改语法从 grammar.peg 进入,并同步 AST、[测试](../tests/README.md)、样例与 [前端契约](../../../docs/runtime/frontend.md)。词法与语法两份 PEG 动作均不做 VM 分配、I/O 或修改外部状态,避免回溯造成副作用。返回 [frontend](../README.md)。
+30 -86
View File
@@ -1,90 +1,34 @@
use crate::Span;
//! AST types and walkers generated from the single Endor.asdl schema.
//! Runtime objects remain in the objects crate.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Name {
pub text: String,
pub span: Span,
mod generated;
pub mod visitor;
mod wire;
pub use generated::*;
pub use wire::AstError;
use crate::{SourceLocation, Span};
/// A syntax node's half-open byte range in its original UTF-8 source.
pub trait Spanned {
fn span(&self) -> Span;
fn source_text<'a>(&self, source: &'a str) -> Option<&'a str> {
self.span().slice(source)
}
fn location(&self, source: &str) -> Option<SourceLocation> {
self.span().location(source)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Program {
pub functions: Vec<Function>,
pub span: Span,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Function {
pub name: Name,
pub parameters: Vec<Parameter>,
pub return_type: Name,
pub body: Block,
pub span: Span,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Parameter {
pub name: Name,
pub type_name: Name,
pub span: Span,
}
/// Basic draft function body: zero or more bindings and one final return.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Block {
pub bindings: Vec<Binding>,
pub result: Expr,
pub return_span: Span,
pub span: Span,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Binding {
pub name: Name,
pub value: Expr,
pub span: Span,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Expr {
pub kind: ExprKind,
pub span: Span,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ExprKind {
Name(String),
/// The decimal spelling is retained, without f64 conversion.
Number(String),
String(String),
Symbol(Name),
Unary {
operation: UnaryOp,
operand: Box<Expr>,
},
Binary {
operation: BinaryOp,
left: Box<Expr>,
right: Box<Expr>,
},
Member {
object: Box<Expr>,
name: Name,
},
Call {
callee: Box<Expr>,
arguments: Vec<Expr>,
},
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum UnaryOp {
Negate,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BinaryOp {
Add,
Subtract,
Multiply,
Divide,
impl Program {
/// Iterate the currently supported top-level function declarations.
pub fn functions(&self) -> impl Iterator<Item = &Function> {
self.declarations.iter().map(|declaration| {
let DeclKind::Function(function) = &declaration.kind;
function
})
}
}
+11
View File
@@ -0,0 +1,11 @@
# ASDL 生成的 AST 与接口
本目录只保存 frontend 的语法数据与遍历/交换接口,不定义运行时对象。
| 文件 | 职责 |
| --- | --- |
| [generated.rs](generated.rs) | 从 Endor.asdl 生成的节点类型、Serde 实现、Spanned 实现和 AST_SCHEMA_ID |
| [visitor.rs](visitor.rs) | 同源生成的可失败 Visitor、节点进入/离开钩子与 walk_* 默认遍历 |
| [wire.rs](wire.rs) | 手写 AstDocument JSON 接口、schema ID、范围与父子位置校验 |
唯一节点定义源是 [Endor.asdl](../../Endor.asdl),生成命令为 `python3 scripts/generate_ast.py --write`,在仓库根运行;--check 仅检查产物是否同步,不执行语言或测试。不要直接编辑 generated.rs/visitor.rs。公开入口见 [ast.rs](../ast.rs),完整契约见 [AST 模型](../../../../docs/runtime/ast.md)。返回 [frontend/src](../README.md)。
+220
View File
@@ -0,0 +1,220 @@
// Generated by scripts/generate_ast.py from Endor.asdl. Do not edit.
use super::Spanned;
use crate::Span;
pub const AST_SCHEMA_ID: &str =
"urn:endor:ast:sha256:c0da1a2640ad85f2c8e8159a10756e649d3aab179823a7cd890d9ebbb7f5d46e";
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct AstDocument {
pub schema: String,
pub root: AstRoot,
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(tag = "kind", content = "data", deny_unknown_fields)]
pub enum AstRoot {
Module(Program),
Expression(Expr),
Interactive(InteractiveInput),
}
impl Spanned for AstRoot {
fn span(&self) -> Span {
match self {
Self::Module(node) => node.span(),
Self::Expression(node) => node.span(),
Self::Interactive(node) => node.span(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Name {
pub text: String,
pub span: Span,
}
impl Spanned for Name {
fn span(&self) -> Span {
self.span
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Program {
pub declarations: Vec<Decl>,
pub span: Span,
}
impl Spanned for Program {
fn span(&self) -> Span {
self.span
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Decl {
pub kind: DeclKind,
pub span: Span,
}
impl Spanned for Decl {
fn span(&self) -> Span {
self.span
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(tag = "kind", content = "data", deny_unknown_fields)]
pub enum DeclKind {
Function(Function),
}
impl Spanned for DeclKind {
fn span(&self) -> Span {
match self {
Self::Function(node) => node.span(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Function {
pub name: Name,
pub parameters: Vec<Parameter>,
pub return_type: Name,
pub body: Block,
pub span: Span,
}
impl Spanned for Function {
fn span(&self) -> Span {
self.span
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Parameter {
pub name: Name,
pub type_name: Name,
pub span: Span,
}
impl Spanned for Parameter {
fn span(&self) -> Span {
self.span
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Block {
pub statements: Vec<Stmt>,
pub span: Span,
}
impl Spanned for Block {
fn span(&self) -> Span {
self.span
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Stmt {
pub kind: StmtKind,
pub span: Span,
}
impl Spanned for Stmt {
fn span(&self) -> Span {
self.span
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(tag = "kind", content = "data", deny_unknown_fields)]
pub enum StmtKind {
Binding(Binding),
Return(ReturnStatement),
Expression(Expr),
Declaration(Decl),
}
impl Spanned for StmtKind {
fn span(&self) -> Span {
match self {
Self::Binding(node) => node.span(),
Self::Return(node) => node.span(),
Self::Expression(node) => node.span(),
Self::Declaration(node) => node.span(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Binding {
pub name: Name,
pub value: Expr,
pub span: Span,
}
impl Spanned for Binding {
fn span(&self) -> Span {
self.span
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ReturnStatement {
pub value: Option<Expr>,
pub span: Span,
}
impl Spanned for ReturnStatement {
fn span(&self) -> Span {
self.span
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct InteractiveInput {
pub statements: Vec<Stmt>,
pub span: Span,
}
impl Spanned for InteractiveInput {
fn span(&self) -> Span {
self.span
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Expr {
pub kind: ExprKind,
pub span: Span,
}
impl Spanned for Expr {
fn span(&self) -> Span {
self.span
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(tag = "kind", content = "data", deny_unknown_fields)]
pub enum ExprKind {
Name(String),
Number(String),
String(String),
Symbol(Name),
Unary {
operation: UnaryOp,
operand: Box<Expr>,
},
Binary {
operation: BinaryOp,
left: Box<Expr>,
right: Box<Expr>,
},
Member {
object: Box<Expr>,
name: Name,
},
Call {
callee: Box<Expr>,
arguments: Vec<Expr>,
},
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, Copy)]
pub enum UnaryOp {
Negate,
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, Copy)]
pub enum BinaryOp {
Add,
Subtract,
Multiply,
Divide,
}
+300
View File
@@ -0,0 +1,300 @@
// Generated by scripts/generate_ast.py from Endor.asdl. Do not edit.
use super::{
AstDocument, AstRoot, BinaryOp, Binding, Block, Decl, DeclKind, Expr, ExprKind, Function,
InteractiveInput, Name, Parameter, Program, ReturnStatement, Spanned, Stmt, StmtKind, UnaryOp,
};
use crate::Span;
pub trait Visitor {
type Error;
fn enter_node(&mut self, _span: Span) -> Result<(), Self::Error> {
Ok(())
}
fn leave_node(&mut self) {}
fn visit_span(&mut self, _span: &Span) -> Result<(), Self::Error> {
Ok(())
}
fn visit_ast_document(&mut self, node: &AstDocument) -> Result<(), Self::Error> {
walk_ast_document(self, node)
}
fn visit_ast_root(&mut self, node: &AstRoot) -> Result<(), Self::Error> {
self.enter_node(node.span())?;
let result = walk_ast_root(self, node);
self.leave_node();
result
}
fn visit_name(&mut self, node: &Name) -> Result<(), Self::Error> {
self.enter_node(node.span())?;
let result = walk_name(self, node);
self.leave_node();
result
}
fn visit_program(&mut self, node: &Program) -> Result<(), Self::Error> {
self.enter_node(node.span())?;
let result = walk_program(self, node);
self.leave_node();
result
}
fn visit_decl(&mut self, node: &Decl) -> Result<(), Self::Error> {
self.enter_node(node.span())?;
let result = walk_decl(self, node);
self.leave_node();
result
}
fn visit_decl_kind(&mut self, node: &DeclKind) -> Result<(), Self::Error> {
self.enter_node(node.span())?;
let result = walk_decl_kind(self, node);
self.leave_node();
result
}
fn visit_function(&mut self, node: &Function) -> Result<(), Self::Error> {
self.enter_node(node.span())?;
let result = walk_function(self, node);
self.leave_node();
result
}
fn visit_parameter(&mut self, node: &Parameter) -> Result<(), Self::Error> {
self.enter_node(node.span())?;
let result = walk_parameter(self, node);
self.leave_node();
result
}
fn visit_block(&mut self, node: &Block) -> Result<(), Self::Error> {
self.enter_node(node.span())?;
let result = walk_block(self, node);
self.leave_node();
result
}
fn visit_stmt(&mut self, node: &Stmt) -> Result<(), Self::Error> {
self.enter_node(node.span())?;
let result = walk_stmt(self, node);
self.leave_node();
result
}
fn visit_stmt_kind(&mut self, node: &StmtKind) -> Result<(), Self::Error> {
self.enter_node(node.span())?;
let result = walk_stmt_kind(self, node);
self.leave_node();
result
}
fn visit_binding(&mut self, node: &Binding) -> Result<(), Self::Error> {
self.enter_node(node.span())?;
let result = walk_binding(self, node);
self.leave_node();
result
}
fn visit_return_statement(&mut self, node: &ReturnStatement) -> Result<(), Self::Error> {
self.enter_node(node.span())?;
let result = walk_return_statement(self, node);
self.leave_node();
result
}
fn visit_interactive_input(&mut self, node: &InteractiveInput) -> Result<(), Self::Error> {
self.enter_node(node.span())?;
let result = walk_interactive_input(self, node);
self.leave_node();
result
}
fn visit_expr(&mut self, node: &Expr) -> Result<(), Self::Error> {
self.enter_node(node.span())?;
let result = walk_expr(self, node);
self.leave_node();
result
}
fn visit_expr_kind(&mut self, node: &ExprKind) -> Result<(), Self::Error> {
walk_expr_kind(self, node)
}
fn visit_unary_op(&mut self, node: &UnaryOp) -> Result<(), Self::Error> {
walk_unary_op(self, node)
}
fn visit_binary_op(&mut self, node: &BinaryOp) -> Result<(), Self::Error> {
walk_binary_op(self, node)
}
}
pub fn walk_ast_document<V: Visitor + ?Sized>(
visitor: &mut V,
node: &AstDocument,
) -> Result<(), V::Error> {
visitor.visit_ast_root(&node.root)?;
Ok(())
}
pub fn walk_ast_root<V: Visitor + ?Sized>(visitor: &mut V, node: &AstRoot) -> Result<(), V::Error> {
match node {
AstRoot::Module(value) => {
visitor.visit_program(value)?;
}
AstRoot::Expression(value) => {
visitor.visit_expr(value)?;
}
AstRoot::Interactive(value) => {
visitor.visit_interactive_input(value)?;
}
}
Ok(())
}
pub fn walk_name<V: Visitor + ?Sized>(visitor: &mut V, node: &Name) -> Result<(), V::Error> {
visitor.visit_span(&node.span)?;
Ok(())
}
pub fn walk_program<V: Visitor + ?Sized>(visitor: &mut V, node: &Program) -> Result<(), V::Error> {
for value in &node.declarations {
visitor.visit_decl(value)?;
}
visitor.visit_span(&node.span)?;
Ok(())
}
pub fn walk_decl<V: Visitor + ?Sized>(visitor: &mut V, node: &Decl) -> Result<(), V::Error> {
visitor.visit_decl_kind(&node.kind)?;
visitor.visit_span(&node.span)?;
Ok(())
}
pub fn walk_decl_kind<V: Visitor + ?Sized>(
visitor: &mut V,
node: &DeclKind,
) -> Result<(), V::Error> {
match node {
DeclKind::Function(value) => {
visitor.visit_function(value)?;
}
}
Ok(())
}
pub fn walk_function<V: Visitor + ?Sized>(
visitor: &mut V,
node: &Function,
) -> Result<(), V::Error> {
visitor.visit_name(&node.name)?;
for value in &node.parameters {
visitor.visit_parameter(value)?;
}
visitor.visit_name(&node.return_type)?;
visitor.visit_block(&node.body)?;
visitor.visit_span(&node.span)?;
Ok(())
}
pub fn walk_parameter<V: Visitor + ?Sized>(
visitor: &mut V,
node: &Parameter,
) -> Result<(), V::Error> {
visitor.visit_name(&node.name)?;
visitor.visit_name(&node.type_name)?;
visitor.visit_span(&node.span)?;
Ok(())
}
pub fn walk_block<V: Visitor + ?Sized>(visitor: &mut V, node: &Block) -> Result<(), V::Error> {
for value in &node.statements {
visitor.visit_stmt(value)?;
}
visitor.visit_span(&node.span)?;
Ok(())
}
pub fn walk_stmt<V: Visitor + ?Sized>(visitor: &mut V, node: &Stmt) -> Result<(), V::Error> {
visitor.visit_stmt_kind(&node.kind)?;
visitor.visit_span(&node.span)?;
Ok(())
}
pub fn walk_stmt_kind<V: Visitor + ?Sized>(
visitor: &mut V,
node: &StmtKind,
) -> Result<(), V::Error> {
match node {
StmtKind::Binding(value) => {
visitor.visit_binding(value)?;
}
StmtKind::Return(value) => {
visitor.visit_return_statement(value)?;
}
StmtKind::Expression(value) => {
visitor.visit_expr(value)?;
}
StmtKind::Declaration(value) => {
visitor.visit_decl(value)?;
}
}
Ok(())
}
pub fn walk_binding<V: Visitor + ?Sized>(visitor: &mut V, node: &Binding) -> Result<(), V::Error> {
visitor.visit_name(&node.name)?;
visitor.visit_expr(&node.value)?;
visitor.visit_span(&node.span)?;
Ok(())
}
pub fn walk_return_statement<V: Visitor + ?Sized>(
visitor: &mut V,
node: &ReturnStatement,
) -> Result<(), V::Error> {
if let Some(value) = &node.value {
visitor.visit_expr(value)?;
}
visitor.visit_span(&node.span)?;
Ok(())
}
pub fn walk_interactive_input<V: Visitor + ?Sized>(
visitor: &mut V,
node: &InteractiveInput,
) -> Result<(), V::Error> {
for value in &node.statements {
visitor.visit_stmt(value)?;
}
visitor.visit_span(&node.span)?;
Ok(())
}
pub fn walk_expr<V: Visitor + ?Sized>(visitor: &mut V, node: &Expr) -> Result<(), V::Error> {
visitor.visit_expr_kind(&node.kind)?;
visitor.visit_span(&node.span)?;
Ok(())
}
pub fn walk_expr_kind<V: Visitor + ?Sized>(
visitor: &mut V,
node: &ExprKind,
) -> Result<(), V::Error> {
match node {
ExprKind::Name(_) => {}
ExprKind::Number(_) => {}
ExprKind::String(_) => {}
ExprKind::Symbol(value) => {
visitor.visit_name(value)?;
}
ExprKind::Unary {
operation, operand, ..
} => {
visitor.visit_unary_op(operation)?;
visitor.visit_expr(operand)?;
}
ExprKind::Binary {
operation,
left,
right,
..
} => {
visitor.visit_binary_op(operation)?;
visitor.visit_expr(left)?;
visitor.visit_expr(right)?;
}
ExprKind::Member { object, name, .. } => {
visitor.visit_expr(object)?;
visitor.visit_name(name)?;
}
ExprKind::Call {
callee, arguments, ..
} => {
visitor.visit_expr(callee)?;
for value in arguments {
visitor.visit_expr(value)?;
}
}
}
Ok(())
}
pub fn walk_unary_op<V: Visitor + ?Sized>(
_visitor: &mut V,
_node: &UnaryOp,
) -> Result<(), V::Error> {
Ok(())
}
pub fn walk_binary_op<V: Visitor + ?Sized>(
_visitor: &mut V,
_node: &BinaryOp,
) -> Result<(), V::Error> {
Ok(())
}
+136
View File
@@ -0,0 +1,136 @@
use std::fmt;
use super::visitor::Visitor;
use super::{AST_SCHEMA_ID, AstDocument, AstRoot};
use crate::Span;
#[derive(Debug)]
pub enum AstError {
Json(serde_json::Error),
SchemaMismatch {
expected: &'static str,
found: String,
},
InvalidSpan(Span),
ChildOutsideParent {
parent: Span,
child: Span,
},
}
impl fmt::Display for AstError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Json(error) => write!(f, "invalid AST JSON: {error}"),
Self::SchemaMismatch { expected, found } => {
write!(f, "AST schema mismatch: expected {expected}, found {found}")
}
Self::InvalidSpan(span) => {
write!(f, "invalid AST source span: {}..{}", span.start, span.end)
}
Self::ChildOutsideParent { parent, child } => write!(
f,
"AST child span {}..{} is outside parent {}..{}",
child.start, child.end, parent.start, parent.end
),
}
}
}
impl std::error::Error for AstError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
Self::Json(error) => Some(error),
_ => None,
}
}
}
impl From<serde_json::Error> for AstError {
fn from(error: serde_json::Error) -> Self {
Self::Json(error)
}
}
impl AstDocument {
pub fn new(root: AstRoot) -> Result<Self, AstError> {
let document = Self {
schema: AST_SCHEMA_ID.into(),
root,
};
document.validate()?;
Ok(document)
}
/// Decode a versioned syntax tree. This does not compile or execute it.
pub fn from_json(text: &str) -> Result<Self, AstError> {
let document: Self = serde_json::from_str(text)?;
document.validate()?;
Ok(document)
}
pub fn to_json(&self) -> Result<String, AstError> {
self.validate()?;
Ok(serde_json::to_string(self)?)
}
/// Check schema identity, ranges, and parent/child containment.
pub fn validate(&self) -> Result<(), AstError> {
self.validate_with_source(None)
}
/// Additionally check bounds and UTF-8 boundaries against original text.
/// This does not check that the text and AST have identical semantics.
pub fn validate_source(&self, source: &str) -> Result<(), AstError> {
self.validate_with_source(Some(source))
}
fn validate_with_source(&self, source: Option<&str>) -> Result<(), AstError> {
if self.schema != AST_SCHEMA_ID {
return Err(AstError::SchemaMismatch {
expected: AST_SCHEMA_ID,
found: self.schema.clone(),
});
}
SpanValidator {
source,
parents: Vec::new(),
}
.visit_ast_root(&self.root)
}
}
struct SpanValidator<'a> {
source: Option<&'a str>,
parents: Vec<Span>,
}
impl Visitor for SpanValidator<'_> {
type Error = AstError;
fn enter_node(&mut self, span: Span) -> Result<(), AstError> {
// JSON offsets must also be exactly representable by editor JS numbers.
if span.start > span.end
|| span.end as u128 > 9_007_199_254_740_991
|| self
.source
.is_some_and(|source| span.slice(source).is_none())
{
return Err(AstError::InvalidSpan(span));
}
if let Some(parent) = self.parents.last().copied()
&& (span.start < parent.start || span.end > parent.end)
{
return Err(AstError::ChildOutsideParent {
parent,
child: span,
});
}
self.parents.push(span);
Ok(())
}
fn leave_node(&mut self) {
self.parents.pop();
}
}
+37 -22
View File
@@ -3,7 +3,8 @@ use std::fmt;
/// Half-open UTF-8 byte range in the original source, including trivia between
/// the first and last syntax token. EOF has an empty range at source.len().
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Span {
pub start: usize,
pub end: usize,
@@ -14,6 +15,16 @@ impl Span {
Self { start, end }
}
/// Safely read this range, rejecting reversed, out-of-bounds or split UTF-8 ranges.
pub fn slice(self, source: &str) -> Option<&str> {
source.get(self.start..self.end)
}
pub fn location(self, source: &str) -> Option<SourceLocation> {
self.slice(source)?;
Some(source_location(source, self.start))
}
pub(crate) fn through(self, last: Self) -> Self {
Self::new(self.start, last.end)
}
@@ -51,27 +62,7 @@ impl Diagnostic {
}
pub fn location(&self, source: &str) -> SourceLocation {
let mut location = SourceLocation { line: 1, column: 1 };
let mut previous_cr = false;
for (offset, character) in source.char_indices() {
if offset >= self.span.start {
break;
}
match character {
'\r' => {
location.line += 1;
location.column = 1;
}
'\n' if previous_cr => {}
'\n' => {
location.line += 1;
location.column = 1;
}
_ => location.column += 1,
}
previous_cr = character == '\r';
}
location
source_location(source, self.span.start)
}
}
@@ -86,3 +77,27 @@ impl fmt::Display for Diagnostic {
}
impl Error for Diagnostic {}
fn source_location(source: &str, position: usize) -> SourceLocation {
let mut location = SourceLocation { line: 1, column: 1 };
let mut previous_cr = false;
for (offset, character) in source.char_indices() {
if offset >= position {
break;
}
match character {
'\r' => {
location.line += 1;
location.column = 1;
}
'\n' if previous_cr => {}
'\n' => {
location.line += 1;
location.column = 1;
}
_ => location.column += 1,
}
previous_cr = character == '\r';
}
location
}
+27 -6
View File
@@ -42,9 +42,28 @@ peg::parser! {
= [Token { kind: TokenKind::Slash, .. }]
pub rule program() -> Program
= functions:function()+ {
let span = functions[0].span.through(functions.last().unwrap().span);
Program { functions, span }
= declarations:declaration()+ {
let span = declarations[0].span.through(declarations.last().unwrap().span);
Program { declarations, span }
}
rule declaration() -> Decl
= function:function() {
Decl { span: function.span, kind: DeclKind::Function(function) }
}
pub rule interactive() -> InteractiveInput
= statement:interactive_statement() {
InteractiveInput { span: statement.span, statements: vec![statement] }
}
rule interactive_statement() -> Stmt
= declaration:declaration() {
Stmt { span: declaration.span, kind: StmtKind::Declaration(declaration) }
}
/ binding:binding() { binding_statement(binding) }
/ expression:expression() {
Stmt { span: expression.span, kind: StmtKind::Expression(expression) }
}
rule function() -> Function
@@ -60,7 +79,9 @@ peg::parser! {
rule block() -> Block
= start:lbrace() bindings:binding()* returned:return_statement() end:rbrace() {
Block { bindings, result: returned.0, return_span: returned.1, span: start.through(end) }
let mut statements: Vec<Stmt> = bindings.into_iter().map(binding_statement).collect();
statements.push(returned);
Block { statements, span: start.through(end) }
}
rule binding() -> Binding
@@ -68,10 +89,10 @@ peg::parser! {
Binding { span: start.through(value.span), name, value }
}
rule return_statement() -> (Expr, Span)
rule return_statement() -> Stmt
= start:return_keyword() value:expression() {
let span = start.through(value.span);
(value, span)
Stmt { span, kind: StmtKind::Return(ReturnStatement { value: Some(value), span }) }
}
pub rule expression() -> Expr = precedence! {
+4 -1
View File
@@ -12,7 +12,10 @@ mod token;
pub use diagnostic::{Diagnostic, DiagnosticKind, SourceLocation, Span};
pub use lexer::{lex, lex_source};
pub use parser::{parse, parse_expression, parse_source, parse_source_expression};
pub use parser::{
ParseMode, parse, parse_ast, parse_expression, parse_source, parse_source_ast,
parse_source_expression,
};
pub use reader::{
EndorReader, EndorSourceProvider, ResolvedEndorSource, SourceReadError, SourceReadErrorKind,
};
+37 -1
View File
@@ -1,9 +1,38 @@
use crate::ast::{
BinaryOp, Binding, Block, Expr, ExprKind, Function, Name, Parameter, Program, UnaryOp,
AstRoot, BinaryOp, Binding, Block, Decl, DeclKind, Expr, ExprKind, Function, InteractiveInput,
Name, Parameter, Program, ReturnStatement, Stmt, StmtKind, UnaryOp,
};
use crate::lexer::decode_string;
use crate::{Diagnostic, DiagnosticKind, Span, Token, TokenKind, lex};
/// Select the syntax root without implying any execution or REPL session.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ParseMode {
Module,
Expression,
Interactive,
}
pub fn parse_ast(source: &str, mode: ParseMode) -> Result<AstRoot, Diagnostic> {
match mode {
ParseMode::Module => parse(source).map(AstRoot::Module),
ParseMode::Expression => parse_expression(source).map(AstRoot::Expression),
ParseMode::Interactive => {
let tokens = lex(source)?;
syntax::interactive(&tokens, source)
.map(AstRoot::Interactive)
.map_err(|error| diagnostic(error, &tokens, source.len()))
}
}
}
pub fn parse_source_ast(
source: &crate::SourceUnit,
mode: ParseMode,
) -> Result<AstRoot, crate::SourceDiagnostic> {
parse_ast(source.text(), mode).map_err(|error| source.annotate(error))
}
/// Parse a reader snapshot, retaining its origin and resolved revision on error.
pub fn parse_source(source: &crate::SourceUnit) -> Result<Program, crate::SourceDiagnostic> {
parse(source.text()).map_err(|error| source.annotate(error))
@@ -41,6 +70,13 @@ fn diagnostic(error: peg::error::ParseError<usize>, tokens: &[Token], end: usize
}
}
fn binding_statement(binding: Binding) -> Stmt {
Stmt {
span: binding.span,
kind: StmtKind::Binding(binding),
}
}
fn binary(left: Expr, operation: BinaryOp, right: Expr) -> Expr {
Expr {
span: left.span.through(right.span),
+2
View File
@@ -2,6 +2,8 @@
[syntax.rs](syntax.rs) 覆盖 UTF-8 token 范围、中文/日文名称、数字原文、负号、字符串转义与错误、算术优先级和结合性、成员/调用链、函数与绑定、上下文名称、完整输入、非法/未支持语法以及 CRLF 和 EOF 诊断位置。
[ast.rs](ast.rs) 覆盖根模式、声明/语句结构、生成 Visitor 与错误退出、Spanned 位置、带 schema 的 JSON 往返及错误输入;同时核对编辑器产物标识。
[lexical.rs](lexical.rs) 专门覆盖字符 PEG 迁移后的原文位置换算、首个错误及详细诊断、Unicode 非法转义、未闭合字符串、注释/除法/成员/数字/字符串边界和空输入/尾随注释。
[reader.rs](reader.rs) 覆盖字符串/分块流/文件、严格 UTF-8、读取失败、URI provider、修订标签、旧缓冲区保留及带来源的诊断。样例也通过 syntax.rs 解析。只验证语法和 AST,不宣称类型检查或运行求值通过。随工作区检查在 ces-control 执行,本地不运行;入口见 [验证](../../../docs/operations/validation.md)。返回 [frontend](../README.md)。
+181
View File
@@ -0,0 +1,181 @@
use frontend::ast::visitor::{Visitor, walk_expr, walk_function};
use frontend::ast::{
AST_SCHEMA_ID, AstDocument, AstError, AstRoot, Expr, ExprKind, Function, Spanned, StmtKind,
};
use frontend::{EndorReader, ParseMode, Span, parse_ast, parse_source_ast};
#[test]
fn parse_modes_produce_distinct_roots_and_preserve_source_diagnostics() {
let source = "function echo(value: String): String { let copy = value return copy }";
let root = parse_ast(source, ParseMode::Module).unwrap();
let AstRoot::Module(program) = &root else {
panic!("expected module")
};
let function = program.functions().next().unwrap();
assert_eq!(function.body.statements.len(), 2);
assert!(matches!(
function.body.statements[0].kind,
StmtKind::Binding(_)
));
assert!(matches!(
function.body.statements[1].kind,
StmtKind::Return(_)
));
AstDocument::new(root)
.unwrap()
.validate_source(source)
.unwrap();
let root = parse_ast("echo(\"日\")", ParseMode::Expression).unwrap();
assert!(matches!(root, AstRoot::Expression(_)));
for text in ["let value = \"日\"", "echo(value)", source] {
let root = parse_ast(text, ParseMode::Interactive).unwrap();
let AstRoot::Interactive(input) = &root else {
panic!("expected interactive")
};
assert_eq!(input.statements.len(), 1);
AstDocument::new(root)
.unwrap()
.validate_source(text)
.unwrap();
}
assert!(parse_ast("let x = 1 let y = 2", ParseMode::Interactive).is_err());
assert!(parse_ast("", ParseMode::Interactive).is_err());
let source = EndorReader::from_text("<session:1>", "\"日\" + )");
let error = parse_source_ast(&source, ParseMode::Expression).unwrap_err();
assert_eq!(error.location.line, 1);
assert_eq!(error.location.column, 7);
assert!(error.to_string().starts_with("<session:1>:1:7:"));
}
#[test]
fn generated_visitors_reach_names_and_unwind_node_hooks_on_error() {
#[derive(Default)]
struct Names {
functions: Vec<String>,
references: Vec<String>,
depth: usize,
stop: bool,
}
impl Visitor for Names {
type Error = &'static str;
fn enter_node(&mut self, _span: Span) -> Result<(), Self::Error> {
self.depth += 1;
Ok(())
}
fn leave_node(&mut self) {
self.depth -= 1;
}
fn visit_function(&mut self, node: &Function) -> Result<(), Self::Error> {
self.functions.push(node.name.text.clone());
walk_function(self, node)
}
fn visit_expr(&mut self, node: &Expr) -> Result<(), Self::Error> {
if let ExprKind::Name(name) = &node.kind {
if self.stop {
return Err("stopped at reference");
}
self.references.push(name.clone());
}
walk_expr(self, node)
}
}
let root = parse_ast(
"function echo(value: String): String { let copy = value return copy }",
ParseMode::Module,
)
.unwrap();
let mut names = Names::default();
names.visit_ast_root(&root).unwrap();
assert_eq!(names.functions, ["echo"]);
assert_eq!(names.references, ["value", "copy"]);
assert_eq!(names.depth, 0);
let mut stopped = Names {
stop: true,
..Names::default()
};
assert_eq!(stopped.visit_ast_root(&root), Err("stopped at reference"));
assert_eq!(stopped.depth, 0);
}
#[test]
fn spans_share_unicode_locations_and_safe_original_source_access() {
let source = " \r\n(値)";
let root = parse_ast(source, ParseMode::Expression).unwrap();
assert_eq!(root.source_text(source), Some("(値)"));
let location = root.location(source).unwrap();
assert_eq!((location.line, location.column), (2, 1));
assert_eq!(Span::new(1, 3).slice("値"), None);
assert_eq!(Span::new(2, 1).slice("値"), None);
assert_eq!(Span::new(0, 4).location("値"), None);
}
#[test]
fn versioned_json_roundtrips_all_roots_and_preserves_number_spelling() {
for (source, mode) in [
(
"function f(): Number { return 00012345678901234567890.00100 }",
ParseMode::Module,
),
("f(.名前, \"日\\n\")", ParseMode::Expression),
("let value = 1", ParseMode::Interactive),
] {
let document = AstDocument::new(parse_ast(source, mode).unwrap()).unwrap();
let json = document.to_json().unwrap();
let restored = AstDocument::from_json(&json).unwrap();
assert_eq!(restored, document);
restored.validate_source(source).unwrap();
assert!(json.contains(AST_SCHEMA_ID));
if mode == ParseMode::Module {
assert!(json.contains("00012345678901234567890.00100"));
}
}
let schema: serde_json::Value =
serde_json::from_str(include_str!("../editor/ast.schema.json")).unwrap();
assert_eq!(schema["$id"], AST_SCHEMA_ID);
assert_eq!(
schema["$defs"]["AstDocument"]["properties"]["schema"]["const"],
AST_SCHEMA_ID
);
assert!(include_str!("../editor/ast.ts").contains(AST_SCHEMA_ID));
}
#[test]
fn json_interfaces_reject_unknown_fields_schema_changes_and_invalid_ranges() {
let source = "(f(値))";
let document = AstDocument::new(parse_ast(source, ParseMode::Expression).unwrap()).unwrap();
let mut value = serde_json::to_value(&document).unwrap();
value["schema"] = "urn:endor:ast:unknown".into();
assert!(matches!(
AstDocument::from_json(&value.to_string()),
Err(AstError::SchemaMismatch { .. })
));
value = serde_json::to_value(&document).unwrap();
value["root"]["data"]["unexpected"] = true.into();
assert!(matches!(
AstDocument::from_json(&value.to_string()),
Err(AstError::Json(_))
));
value = serde_json::to_value(&document).unwrap();
value["root"]["data"]["span"]["end"] = 1.into();
assert!(matches!(
AstDocument::from_json(&value.to_string()),
Err(AstError::ChildOutsideParent { .. })
));
let AstRoot::Expression(mut expression) = document.root else {
unreachable!()
};
expression.span = Span::new(5, 4);
assert!(matches!(
AstDocument::new(AstRoot::Expression(expression)),
Err(AstError::InvalidSpan(_))
));
let root = AstRoot::Expression(Expr {
kind: ExprKind::Name("値".into()),
span: Span::new(1, 3),
});
assert!(matches!(
AstDocument::new(root).unwrap().validate_source("値"),
Err(AstError::InvalidSpan(_))
));
}
+41 -19
View File
@@ -1,6 +1,27 @@
use frontend::ast::{BinaryOp, Expr, ExprKind, UnaryOp};
use frontend::ast::{
BinaryOp, Binding, DeclKind, Expr, ExprKind, Function, Program, StmtKind, UnaryOp,
};
use frontend::{DiagnosticKind, Span, TokenKind, lex, parse, parse_expression};
fn function(program: &Program, index: usize) -> &Function {
let DeclKind::Function(function) = &program.declarations[index].kind;
function
}
fn binding(function: &Function, index: usize) -> &Binding {
let StmtKind::Binding(binding) = &function.body.statements[index].kind else {
panic!("expected binding")
};
binding
}
fn result(function: &Function) -> &Expr {
let StmtKind::Return(returned) = &function.body.statements.last().unwrap().kind else {
panic!("expected return")
};
returned.value.as_ref().unwrap()
}
fn binary(expression: &Expr, expected: BinaryOp) -> (&Expr, &Expr) {
let ExprKind::Binary {
operation,
@@ -174,7 +195,7 @@ fn member_and_call_chains_bind_more_tightly_than_unary_minus() {
#[test]
fn parses_typed_functions_bindings_and_multiple_top_level_definitions() {
let example = parse(include_str!("../examples/basic.endor")).unwrap();
assert_eq!(example.functions.len(), 2);
assert_eq!(example.declarations.len(), 2);
let source = r#"
// Basic draft syntax, including a Unicode name.
function discountedPrice(price: Number, discount: Number): Number {
@@ -184,25 +205,26 @@ fn parses_typed_functions_bindings_and_multiple_top_level_definitions() {
function 表示(): String { return "こんにちは" }
"#;
let program = parse(source).unwrap();
assert_eq!(program.functions.len(), 2);
let function = &program.functions[0];
assert_eq!(function.name.text, "discountedPrice");
assert_eq!(function.parameters.len(), 2);
assert_eq!(function.parameters[0].type_name.text, "Number");
assert_eq!(function.return_type.text, "Number");
assert_eq!(function.body.bindings[0].name.text, "amount");
binary(&function.body.bindings[0].value, BinaryOp::Subtract);
assert_eq!(function.body.result.kind, ExprKind::Name("amount".into()));
assert_eq!(program.declarations.len(), 2);
let first = function(&program, 0);
assert_eq!(first.name.text, "discountedPrice");
assert_eq!(first.parameters.len(), 2);
assert_eq!(first.parameters[0].type_name.text, "Number");
assert_eq!(first.return_type.text, "Number");
assert_eq!(binding(first, 0).name.text, "amount");
binary(&binding(first, 0).value, BinaryOp::Subtract);
assert_eq!(result(first).kind, ExprKind::Name("amount".into()));
assert_eq!(
&source[function.body.return_span.start..function.body.return_span.end],
&source[first.body.statements.last().unwrap().span.start
..first.body.statements.last().unwrap().span.end],
"return amount"
);
assert_eq!(
program.functions[1].body.result.kind,
result(function(&program, 1)).kind,
ExprKind::String("こんにちは".into())
);
assert_eq!(
&source[program.functions[1].span.start..program.functions[1].span.end],
&source[function(&program, 1).span.start..function(&program, 1).span.end],
"function 表示(): String { return \"こんにちは\" }"
);
// The output owns its text and can outlive the input buffer.
@@ -210,7 +232,7 @@ fn parses_typed_functions_bindings_and_multiple_top_level_definitions() {
let source = String::from("function f(): String { return \"ok\" }");
parse(&source).unwrap()
};
assert_eq!(owned.functions[0].name.text, "f");
assert_eq!(function(&owned, 0).name.text, "f");
}
#[test]
@@ -218,13 +240,13 @@ fn syntax_words_are_contextual_and_block_layout_does_not_depend_on_newlines() {
let program =
parse("function class(function: Type): Type { let agent = function return agent }")
.unwrap();
assert_eq!(program.functions[0].name.text, "class");
assert_eq!(program.functions[0].parameters[0].name.text, "function");
assert_eq!(program.functions[0].body.bindings[0].name.text, "agent");
assert_eq!(function(&program, 0).name.text, "class");
assert_eq!(function(&program, 0).parameters[0].name.text, "function");
assert_eq!(binding(function(&program, 0), 0).name.text, "agent");
let multiline =
parse("function f\r\n(\r\nx: Number\r\n): Number\r\n{\r\nreturn x\r\n}").unwrap();
assert_eq!(
multiline.functions[0].body.result.kind,
result(function(&multiline, 0)).kind,
ExprKind::Name("x".into())
);
}