feat: generate unified AST and editor contracts from ASDL
Verify databases and deploy Aster VM / verify-and-deploy (push) Successful in 7m55s
Verify databases and deploy Aster VM / verify-and-deploy (push) Successful in 7m55s
This commit is contained in:
Generated
+2
@@ -308,6 +308,8 @@ name = "frontend"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"peg",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
@@ -4,7 +4,7 @@ Rust 编写的 Endor 虚拟机实现。核心对象与对象存储位于独立 o
|
||||
|
||||
Endor 采用平台解释执行模型;AST 到字节码的生成属于解释器内部准备步骤,使用者无需先构建独立程序。资源修订与依赖锁用于选择用户代码,解释器由平台统一升级;相关设计方向见 [解释执行与平台资源版本](docs/architecture/execution-and-versions.md),资源版本管理尚未实现。
|
||||
|
||||
独立 frontend 已提供 Endor 阅读器、字符 PEG lexer、peg parser、基础函数/表达式 AST 与 cli parse 文件/stdin 入口。VM 从空堆启动;AST 到字节码生成、Python type/元类体系、数值算术和完整异常传播均未实现。数据库连接仅为需要数据库的特殊类保留;普通宿主不连接数据库;基础信息改用 host 包的 UTF-8 JSON 文件,不保存 VM 堆。当前边界见 [系统架构](docs/architecture/README.md)。
|
||||
独立 frontend 已提供 Endor 阅读器、字符 PEG lexer、peg parser、ASDL 生成的根/声明/语句/表达式 AST、遍历与编辑器接口,以及 cli parse 文件/stdin/JSON 入口。VM 从空堆启动;AST 到字节码生成、Python type/元类体系、数值算术和完整异常传播均未实现。数据库连接仅为需要数据库的特殊类保留;普通宿主不连接数据库;基础信息改用 host 包的 UTF-8 JSON 文件,不保存 VM 堆。当前边界见 [系统架构](docs/architecture/README.md)。
|
||||
|
||||
## 从哪里读
|
||||
|
||||
@@ -15,6 +15,7 @@ Endor 采用平台解释执行模型;AST 到字节码的生成属于解释器
|
||||
| 看核心对象与对象存储 | [objects](crates/objects/README.md) |
|
||||
| 看源码目录与 crate 边界 | [crates](crates/README.md) |
|
||||
| 看字符串、流、文件与平台 URI 如何进入 lexer | [Endor 阅读器](docs/runtime/reader.md) |
|
||||
| 看 AST 根节点、ASDL、遍历与 JSON/编辑器接口 | [AST 模型](docs/runtime/ast.md) |
|
||||
| 看源码词法、语法、AST 和解析命令 | [源码前端](docs/runtime/frontend.md) |
|
||||
| 看指令、代码对象、执行帧和执行循环 | [字节码 VM](docs/runtime/bytecode.md) |
|
||||
| 看 VM/帧/对象如何存活和销毁 | [生命周期](docs/runtime/lifecycle.md) |
|
||||
|
||||
+1
-1
@@ -6,7 +6,7 @@
|
||||
| --- | --- | --- |
|
||||
| [core](core/README.md) | core,库 endor_core | VM、执行帧与字节码求值循环,仅依赖本地 objects |
|
||||
| [objects](objects/README.md) | objects | 零依赖的核心对象、对象存储、代码对象与指令数据/静态验证 |
|
||||
| [frontend](frontend/README.md) | frontend | Endor 阅读器、字符 PEG lexer、token PEG parser、基础函数/表达式 AST,不执行源码 |
|
||||
| [frontend](frontend/README.md) | frontend | Endor 阅读器、字符 PEG lexer、token PEG parser、ASDL 生成的 AST/遍历/JSON 与编辑器接口,不执行源码 |
|
||||
| [host](host/README.md) | host | UTF-8 JSON 文件基础信息,与数据库分离 |
|
||||
| [storage](storage/README.md) | storage | 为数据库特殊类保留的显式连接适配,不提供基础信息存储 |
|
||||
| [cli](cli/README.md) | cli,二进制 cli | 引导演示、源码解析与文件信息命令,不使用数据库 |
|
||||
|
||||
@@ -6,6 +6,6 @@
|
||||
| --- | --- |
|
||||
| Cargo.toml | cli 二进制与 core/frontend/host 依赖;无 storage 或 Tokio 依赖 |
|
||||
| [src](src/README.md) | 参数路由、引导演示、文件/stdin 阅读、源码解析和退出码 |
|
||||
| [tests](tests/README.md) | parse 命令的 AST 输出、错误位置和严格 UTF-8 检查 |
|
||||
| [tests](tests/README.md) | parse 命令的 AST/JSON 输出、解析模式、错误位置和严格 UTF-8 检查 |
|
||||
|
||||
运行方式见 [运行指南](../../docs/operations/README.md),输入边界见 [配置](../../docs/operations/configuration.md)。parse 通过 EndorReader 严格读取 UTF-8 文件或 `-` 指定的 stdin 并输出 AST,不执行源码,见 [前端](../../docs/runtime/frontend.md);metadata put/get 和只导入缺失项的 import 均使用文件,见 [宿主信息](../../docs/operations/host-information.md)。返回 [工作区](../README.md)。
|
||||
运行方式见 [运行指南](../../docs/operations/README.md),输入边界见 [配置](../../docs/operations/configuration.md)。parse 通过 EndorReader 严格读取 UTF-8 文件或 `-` 指定的 stdin 并输出 AstRoot;--mode 选择 module/expression/interactive,--json 输出带 schema 的 AST JSON,不执行源码,见 [前端](../../docs/runtime/frontend.md);metadata put/get 和只导入缺失项的 import 均使用文件,见 [宿主信息](../../docs/operations/host-information.md)。返回 [工作区](../README.md)。
|
||||
|
||||
@@ -2,6 +2,6 @@
|
||||
|
||||
[main.rs](main.rs) 是唯一入口:同步 main 将 run 的结果转成进程退出码;run 处理 bootstrap、parse 文件/stdin、help、文件 metadata put/get/import,错误打印到 stderr 并返回失败。
|
||||
|
||||
bootstrap 创建并显式释放两个对象,不读取数据库环境、不打开数据库。parse 使用 EndorReader 和 parse_source 输出 AST;`-` 从 stdin 读到 EOF,来源为 `<stdin>`。词法/语法错误显示来源、行、列;不创建 VM 或执行源码。未知命令报错;metadata 通过 host::MetadataStore 读写文件,import 从标准输入接收旧记录且不覆盖已有文件。参数目前由 std::env::args 读取,非 Unicode 参数可能 panic,边界见 [配置](../../../docs/operations/configuration.md)。
|
||||
bootstrap 创建并显式释放两个对象,不读取数据库环境、不打开数据库。parse_command 使用 EndorReader 和 parse_source_ast 输出 AstRoot;--mode 选择 module/expression/interactive(默认 module),--json 通过 AstDocument 输出 JSON,格式见 [AST 模型](../../../docs/runtime/ast.md);`-` 从 stdin 读到 EOF,来源为 `<stdin>`。词法/语法错误显示来源、行、列;不创建 VM 或执行源码。未知命令报错;metadata 通过 host::MetadataStore 读写文件,import 从标准输入接收旧记录且不覆盖已有文件。参数目前由 std::env::args 读取,非 Unicode 参数可能 panic,边界见 [配置](../../../docs/operations/configuration.md)。
|
||||
|
||||
新增命令时更新用法、[运行指南](../../../docs/operations/README.md) 和 [smoke.py 的导航](../../../scripts/README.md)。返回 [cli](../README.md)。
|
||||
|
||||
+39
-11
@@ -2,10 +2,11 @@ use std::error::Error;
|
||||
use std::process::ExitCode;
|
||||
|
||||
use endor_core::{Value, Vm};
|
||||
use frontend::{EndorReader, parse_source};
|
||||
use frontend::ast::AstDocument;
|
||||
use frontend::{EndorReader, ParseMode, parse_source_ast};
|
||||
use host::MetadataStore;
|
||||
|
||||
const USAGE: &str = "Usage: cli [bootstrap | parse <file|-> | metadata put <key> <value> | metadata get <key> | metadata import | help]";
|
||||
const USAGE: &str = "Usage: cli [bootstrap | parse [--mode module|expression|interactive] [--json] <file|-> | metadata put <key> <value> | metadata get <key> | metadata import | help]";
|
||||
|
||||
fn main() -> ExitCode {
|
||||
match run() {
|
||||
@@ -46,15 +47,7 @@ fn run() -> Result<(), Box<dyn Error>> {
|
||||
println!("bootstrap uses no database");
|
||||
}
|
||||
["help"] | ["--help"] | ["-h"] => println!("{USAGE}"),
|
||||
["parse", path] => {
|
||||
let source = if *path == "-" {
|
||||
EndorReader::from_stream("<stdin>", std::io::stdin().lock())?
|
||||
} else {
|
||||
EndorReader::from_file(path)?
|
||||
};
|
||||
let program = parse_source(&source)?;
|
||||
println!("{program:#?}");
|
||||
}
|
||||
["parse", arguments @ ..] => parse_command(arguments)?,
|
||||
["metadata", "put", key, value] => {
|
||||
MetadataStore::from_env()?.put(key, value)?;
|
||||
println!("stored (file)");
|
||||
@@ -76,3 +69,38 @@ fn run() -> Result<(), Box<dyn Error>> {
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn parse_command(arguments: &[&str]) -> Result<(), Box<dyn Error>> {
|
||||
let mut mode = ParseMode::Module;
|
||||
let mut json = false;
|
||||
let mut path = None;
|
||||
let mut arguments = arguments.iter().copied();
|
||||
while let Some(argument) = arguments.next() {
|
||||
match argument {
|
||||
"--json" => json = true,
|
||||
"--mode" => {
|
||||
mode = match arguments.next() {
|
||||
Some("module") => ParseMode::Module,
|
||||
Some("expression") => ParseMode::Expression,
|
||||
Some("interactive") => ParseMode::Interactive,
|
||||
_ => return Err(USAGE.into()),
|
||||
}
|
||||
}
|
||||
value if !value.starts_with("--") && path.is_none() => path = Some(value),
|
||||
_ => return Err(USAGE.into()),
|
||||
}
|
||||
}
|
||||
let path = path.ok_or(USAGE)?;
|
||||
let source = if path == "-" {
|
||||
EndorReader::from_stream("<stdin>", std::io::stdin().lock())?
|
||||
} else {
|
||||
EndorReader::from_file(path)?
|
||||
};
|
||||
let root = parse_source_ast(&source, mode)?;
|
||||
if json {
|
||||
println!("{}", AstDocument::new(root)?.to_json()?);
|
||||
} else {
|
||||
println!("{root:#?}");
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
# CLI 命令检查
|
||||
|
||||
[parse.rs](parse.rs) 使用 Cargo 提供的 cli 测试二进制检查 parse 命令:合法 UTF-8 源码输出 AST 且不执行名称调用;非法语法输出文件/行/列并返回失败;非法 UTF-8 不替换字符;`parse -` 检查 stdin 输入、EOF 和带流名的错误。
|
||||
[parse.rs](parse.rs) 使用 Cargo 提供的 cli 测试二进制检查 parse 命令:合法 UTF-8 源码输出 AST 且不执行名称调用;非法语法输出文件/行/列并返回失败;非法 UTF-8 不替换字符;`parse -` 检查 stdin 输入、EOF 和带流名的错误;--json/--mode 检查表达式与交互根、schema 标识以及非法模式退出。
|
||||
|
||||
测试仅在 ces-control 的工作区检查中运行,本地不执行。测试临时文件使用独立名称并在结束时清理。返回 [cli](../README.md),验证约定见 [验证](../../../docs/operations/validation.md)。
|
||||
|
||||
@@ -105,3 +105,35 @@ fn parse_command_reads_stdin_to_eof_and_reports_its_origin() {
|
||||
.contains("<stdin>: invalid UTF-8 at byte 0")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn parse_command_exports_versioned_json_for_expression_and_interactive_inputs() {
|
||||
for (mode, source) in [("expression", "f(\"日\")"), ("interactive", "let x = 1")] {
|
||||
let source = SourceFile::new(source.as_bytes());
|
||||
let output = Command::new(env!("CARGO_BIN_EXE_cli"))
|
||||
.args(["parse", "--json", "--mode", mode])
|
||||
.arg(&source.0)
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"{}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
let document =
|
||||
frontend::ast::AstDocument::from_json(std::str::from_utf8(&output.stdout).unwrap())
|
||||
.unwrap();
|
||||
assert_eq!(document.schema, frontend::ast::AST_SCHEMA_ID);
|
||||
match (mode, document.root) {
|
||||
("expression", frontend::ast::AstRoot::Expression(_))
|
||||
| ("interactive", frontend::ast::AstRoot::Interactive(_)) => {}
|
||||
_ => panic!("incorrect parse mode"),
|
||||
}
|
||||
}
|
||||
let output = Command::new(env!("CARGO_BIN_EXE_cli"))
|
||||
.args(["parse", "--mode", "unknown", "-"])
|
||||
.output()
|
||||
.unwrap();
|
||||
assert!(!output.status.success());
|
||||
assert!(output.stdout.is_empty());
|
||||
}
|
||||
|
||||
@@ -7,6 +7,8 @@ license.workspace = true
|
||||
|
||||
[dependencies]
|
||||
peg.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
@@ -0,0 +1,38 @@
|
||||
-- Endor AST schema. Source syntax belongs to lexer.peg and grammar.peg.
|
||||
-- Wire contract: endor-ast-json/1, kind/data tagged sums and UTF-8 byte spans.
|
||||
-- Builtins: string (owned text), span (UTF-8 byte range), schema_id (wire ID).
|
||||
-- * is a sequence; ? is optional. Recursion is boxed by the Rust generator.
|
||||
module Endor {
|
||||
AstDocument = (schema_id schema, AstRoot root)
|
||||
AstRoot = Module(Program body)
|
||||
| Expression(Expr body)
|
||||
| Interactive(InteractiveInput body)
|
||||
|
||||
Name = (string text, span span)
|
||||
Program = (Decl* declarations, span span)
|
||||
Decl = (DeclKind kind, span span)
|
||||
DeclKind = Function(Function definition)
|
||||
Function = (Name name, Parameter* parameters, Name return_type, Block body, span span)
|
||||
Parameter = (Name name, Name type_name, span span)
|
||||
Block = (Stmt* statements, span span)
|
||||
Stmt = (StmtKind kind, span span)
|
||||
StmtKind = Binding(Binding binding)
|
||||
| Return(ReturnStatement statement)
|
||||
| Expression(Expr expression)
|
||||
| Declaration(Decl declaration)
|
||||
Binding = (Name name, Expr value, span span)
|
||||
ReturnStatement = (Expr? value, span span)
|
||||
InteractiveInput = (Stmt* statements, span span)
|
||||
|
||||
Expr = (ExprKind kind, span span)
|
||||
ExprKind = Name(string text)
|
||||
| Number(string spelling)
|
||||
| String(string value)
|
||||
| Symbol(Name name)
|
||||
| Unary(UnaryOp operation, Expr operand)
|
||||
| Binary(BinaryOp operation, Expr left, Expr right)
|
||||
| Member(Expr object, Name name)
|
||||
| Call(Expr callee, Expr* arguments)
|
||||
UnaryOp = Negate
|
||||
BinaryOp = Add | Subtract | Multiply | Divide
|
||||
}
|
||||
@@ -1,12 +1,14 @@
|
||||
# frontend
|
||||
|
||||
独立源码前端:EndorReader → SourceUnit → 字符 PEG lexer → token → token PEG parser → 拥有文本的 AST。当前解析基础函数与表达式,不生成字节码、不进行类型/名称解析、不执行源码。语法是可继续调整的首版实现,当前契约见 [源码前端](../../docs/runtime/frontend.md)。
|
||||
独立源码前端:EndorReader → SourceUnit → 字符 PEG lexer → token → token PEG parser → 拥有文本的 AST。AST 节点和 Rust/编辑器接口由 Endor.asdl 统一生成;支持模块、表达式及单次交互语法输入。当前解析基础函数与表达式,不生成字节码、不进行类型/名称解析、不执行源码。语法是可继续调整的首版实现,当前契约见 [源码前端](../../docs/runtime/frontend.md)。
|
||||
|
||||
| 内容 | 职责 |
|
||||
| --- | --- |
|
||||
| Cargo.toml | frontend 包与 peg 0.8.6 依赖,不依赖 core、Host 或 Storage |
|
||||
| Cargo.toml | frontend 包与 peg 0.8.6、工作区已有 serde/serde_json 依赖,不依赖 core、Host 或 Storage |
|
||||
| [Endor.asdl](Endor.asdl) | 唯一 AST 节点定义,供生成器读取 |
|
||||
| [editor](editor/README.md) | 同源生成的 TypeScript 类型/遍历器和 JSON Schema |
|
||||
| [src](src/README.md) | 阅读器、SourceUnit、token、源码范围、诊断、AST、词法规则独立在 [lexer.peg](src/lexer.peg),语法规则在 [grammar.peg](src/grammar.peg) |
|
||||
| [tests](tests/README.md) | Unicode、优先级、函数、完整输入与错误位置验证 |
|
||||
| [examples](examples/README.md) | 可交给 cli parse 的源码样例 |
|
||||
|
||||
宿主通过 EndorReader 读取字符串、流、文件或注入的 URI provider,取得 SourceUnit 后调用 lex_source、parse_source、parse_source_expression;低层 lex、parse、parse_expression 仍保留。读取契约见 [Endor 阅读器](../../docs/runtime/reader.md)。Objects 保持零依赖,Core 仅依赖本地 Objects;后续把 AST 转成已有字节码并交给 VM 解释执行;这是平台内部准备步骤,尚未接通。资源修订、依赖锁和解释器统一升级的设计方向见 [执行与版本模型](../../docs/architecture/execution-and-versions.md)。返回 [工作区](../README.md)。
|
||||
宿主通过 EndorReader 读取字符串、流、文件或注入的 URI provider,取得 SourceUnit 后调用 lex_source、parse_source、parse_source_expression;统一入口 parse_ast/parse_source_ast 通过 ParseMode 返回 AstRoot;低层 lex、parse、parse_expression 仍保留。AST 定义、生成命令、Visitor 与 JSON 交换见 [AST 模型](../../docs/runtime/ast.md)。读取契约见 [Endor 阅读器](../../docs/runtime/reader.md)。Objects 保持零依赖,Core 仅依赖本地 Objects;后续把 AST 转成已有字节码并交给 VM 解释执行;这是平台内部准备步骤,尚未接通。资源修订、依赖锁和解释器统一升级的设计方向见 [执行与版本模型](../../docs/architecture/execution-and-versions.md)。返回 [工作区](../README.md)。
|
||||
|
||||
@@ -0,0 +1,14 @@
|
||||
# 编辑器 AST 接口
|
||||
|
||||
这里的类型和遍历函数均从 [Endor.asdl](../Endor.asdl) 生成,不再维护第二份手写节点定义。
|
||||
|
||||
| 文件 | 职责 |
|
||||
| --- | --- |
|
||||
| [ast.ts](ast.ts) | TypeScript 节点类型、AST_SCHEMA_ID、Visitor 回调与 walk* 函数 |
|
||||
| [ast.schema.json](ast.schema.json) | JSON Schema 2020-12;校验带 schema 标识的 AstDocument 结构 |
|
||||
|
||||
TypeScript 编译时类型不代替 JSON 校验。编辑器可使用现有 JSON Schema 工具校验文档,再交给类型与遍历接口;walk* 的回调返回 false 时跳过该节点的子树。可选字段允许省略或 null,Rust 的标准输出会写成 null。
|
||||
|
||||
所有位置为原文 UTF-8 字节范围,不是 JavaScript UTF-16 下标;编辑器需要以原文换算。JSON Schema 检查结构,父子范围关系和原文 UTF-8 边界由 Rust AstDocument::validate/validate_source 检查。schema 标识变化需要调用方升级契约,不提供隐式迁移。
|
||||
|
||||
在仓库根运行 `python3 scripts/generate_ast.py --write` 重新生成,--check 做静态同步检查。完整格式见 [AST 模型](../../../docs/runtime/ast.md)。返回 [frontend](../README.md)。
|
||||
@@ -0,0 +1,601 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "urn:endor:ast:sha256:c0da1a2640ad85f2c8e8159a10756e649d3aab179823a7cd890d9ebbb7f5d46e",
|
||||
"$ref": "#/$defs/AstDocument",
|
||||
"$defs": {
|
||||
"Span": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"start": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"maximum": 9007199254740991
|
||||
},
|
||||
"end": {
|
||||
"type": "integer",
|
||||
"minimum": 0,
|
||||
"maximum": 9007199254740991
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"start",
|
||||
"end"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"AstDocument": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"schema": {
|
||||
"const": "urn:endor:ast:sha256:c0da1a2640ad85f2c8e8159a10756e649d3aab179823a7cd890d9ebbb7f5d46e"
|
||||
},
|
||||
"root": {
|
||||
"$ref": "#/$defs/AstRoot"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"schema",
|
||||
"root"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"AstRoot": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"kind": {
|
||||
"const": "Module"
|
||||
},
|
||||
"data": {
|
||||
"$ref": "#/$defs/Program"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"kind",
|
||||
"data"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"kind": {
|
||||
"const": "Expression"
|
||||
},
|
||||
"data": {
|
||||
"$ref": "#/$defs/Expr"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"kind",
|
||||
"data"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"kind": {
|
||||
"const": "Interactive"
|
||||
},
|
||||
"data": {
|
||||
"$ref": "#/$defs/InteractiveInput"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"kind",
|
||||
"data"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"Name": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"text": {
|
||||
"type": "string"
|
||||
},
|
||||
"span": {
|
||||
"$ref": "#/$defs/Span"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"text",
|
||||
"span"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"Program": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"declarations": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/Decl"
|
||||
}
|
||||
},
|
||||
"span": {
|
||||
"$ref": "#/$defs/Span"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"declarations",
|
||||
"span"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"Decl": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"kind": {
|
||||
"$ref": "#/$defs/DeclKind"
|
||||
},
|
||||
"span": {
|
||||
"$ref": "#/$defs/Span"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"kind",
|
||||
"span"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"DeclKind": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"kind": {
|
||||
"const": "Function"
|
||||
},
|
||||
"data": {
|
||||
"$ref": "#/$defs/Function"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"kind",
|
||||
"data"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"Function": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"$ref": "#/$defs/Name"
|
||||
},
|
||||
"parameters": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/Parameter"
|
||||
}
|
||||
},
|
||||
"return_type": {
|
||||
"$ref": "#/$defs/Name"
|
||||
},
|
||||
"body": {
|
||||
"$ref": "#/$defs/Block"
|
||||
},
|
||||
"span": {
|
||||
"$ref": "#/$defs/Span"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"name",
|
||||
"parameters",
|
||||
"return_type",
|
||||
"body",
|
||||
"span"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"Parameter": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"$ref": "#/$defs/Name"
|
||||
},
|
||||
"type_name": {
|
||||
"$ref": "#/$defs/Name"
|
||||
},
|
||||
"span": {
|
||||
"$ref": "#/$defs/Span"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"name",
|
||||
"type_name",
|
||||
"span"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"Block": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"statements": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/Stmt"
|
||||
}
|
||||
},
|
||||
"span": {
|
||||
"$ref": "#/$defs/Span"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"statements",
|
||||
"span"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"Stmt": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"kind": {
|
||||
"$ref": "#/$defs/StmtKind"
|
||||
},
|
||||
"span": {
|
||||
"$ref": "#/$defs/Span"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"kind",
|
||||
"span"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"StmtKind": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"kind": {
|
||||
"const": "Binding"
|
||||
},
|
||||
"data": {
|
||||
"$ref": "#/$defs/Binding"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"kind",
|
||||
"data"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"kind": {
|
||||
"const": "Return"
|
||||
},
|
||||
"data": {
|
||||
"$ref": "#/$defs/ReturnStatement"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"kind",
|
||||
"data"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"kind": {
|
||||
"const": "Expression"
|
||||
},
|
||||
"data": {
|
||||
"$ref": "#/$defs/Expr"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"kind",
|
||||
"data"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"kind": {
|
||||
"const": "Declaration"
|
||||
},
|
||||
"data": {
|
||||
"$ref": "#/$defs/Decl"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"kind",
|
||||
"data"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"Binding": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"name": {
|
||||
"$ref": "#/$defs/Name"
|
||||
},
|
||||
"value": {
|
||||
"$ref": "#/$defs/Expr"
|
||||
},
|
||||
"span": {
|
||||
"$ref": "#/$defs/Span"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"name",
|
||||
"value",
|
||||
"span"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"ReturnStatement": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"value": {
|
||||
"anyOf": [
|
||||
{
|
||||
"$ref": "#/$defs/Expr"
|
||||
},
|
||||
{
|
||||
"type": "null"
|
||||
}
|
||||
]
|
||||
},
|
||||
"span": {
|
||||
"$ref": "#/$defs/Span"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"span"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"InteractiveInput": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"statements": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/Stmt"
|
||||
}
|
||||
},
|
||||
"span": {
|
||||
"$ref": "#/$defs/Span"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"statements",
|
||||
"span"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"Expr": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"kind": {
|
||||
"$ref": "#/$defs/ExprKind"
|
||||
},
|
||||
"span": {
|
||||
"$ref": "#/$defs/Span"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"kind",
|
||||
"span"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
"ExprKind": {
|
||||
"oneOf": [
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"kind": {
|
||||
"const": "Name"
|
||||
},
|
||||
"data": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"kind",
|
||||
"data"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"kind": {
|
||||
"const": "Number"
|
||||
},
|
||||
"data": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"kind",
|
||||
"data"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"kind": {
|
||||
"const": "String"
|
||||
},
|
||||
"data": {
|
||||
"type": "string"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"kind",
|
||||
"data"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"kind": {
|
||||
"const": "Symbol"
|
||||
},
|
||||
"data": {
|
||||
"$ref": "#/$defs/Name"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"kind",
|
||||
"data"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"kind": {
|
||||
"const": "Unary"
|
||||
},
|
||||
"data": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"operation": {
|
||||
"$ref": "#/$defs/UnaryOp"
|
||||
},
|
||||
"operand": {
|
||||
"$ref": "#/$defs/Expr"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"operation",
|
||||
"operand"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"kind",
|
||||
"data"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"kind": {
|
||||
"const": "Binary"
|
||||
},
|
||||
"data": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"operation": {
|
||||
"$ref": "#/$defs/BinaryOp"
|
||||
},
|
||||
"left": {
|
||||
"$ref": "#/$defs/Expr"
|
||||
},
|
||||
"right": {
|
||||
"$ref": "#/$defs/Expr"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"operation",
|
||||
"left",
|
||||
"right"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"kind",
|
||||
"data"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"kind": {
|
||||
"const": "Member"
|
||||
},
|
||||
"data": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"object": {
|
||||
"$ref": "#/$defs/Expr"
|
||||
},
|
||||
"name": {
|
||||
"$ref": "#/$defs/Name"
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"object",
|
||||
"name"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"kind",
|
||||
"data"
|
||||
],
|
||||
"additionalProperties": false
|
||||
},
|
||||
{
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"kind": {
|
||||
"const": "Call"
|
||||
},
|
||||
"data": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"callee": {
|
||||
"$ref": "#/$defs/Expr"
|
||||
},
|
||||
"arguments": {
|
||||
"type": "array",
|
||||
"items": {
|
||||
"$ref": "#/$defs/Expr"
|
||||
}
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"callee",
|
||||
"arguments"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
},
|
||||
"required": [
|
||||
"kind",
|
||||
"data"
|
||||
],
|
||||
"additionalProperties": false
|
||||
}
|
||||
]
|
||||
},
|
||||
"UnaryOp": {
|
||||
"enum": [
|
||||
"Negate"
|
||||
]
|
||||
},
|
||||
"BinaryOp": {
|
||||
"enum": [
|
||||
"Add",
|
||||
"Subtract",
|
||||
"Multiply",
|
||||
"Divide"
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
// Generated by scripts/generate_ast.py from Endor.asdl. Do not edit.
|
||||
|
||||
export const AST_SCHEMA_ID = "urn:endor:ast:sha256:c0da1a2640ad85f2c8e8159a10756e649d3aab179823a7cd890d9ebbb7f5d46e" as const;
|
||||
export interface Span { start: number; end: number; }
|
||||
export type AstDocument = { schema: typeof AST_SCHEMA_ID; root: AstRoot; };
|
||||
export type AstRoot = { kind: "Module"; data: Program; } | { kind: "Expression"; data: Expr; } | { kind: "Interactive"; data: InteractiveInput; };
|
||||
export type Name = { text: string; span: Span; };
|
||||
export type Program = { declarations: Array<Decl>; span: Span; };
|
||||
export type Decl = { kind: DeclKind; span: Span; };
|
||||
export type DeclKind = { kind: "Function"; data: Function; };
|
||||
export type Function = { name: Name; parameters: Array<Parameter>; return_type: Name; body: Block; span: Span; };
|
||||
export type Parameter = { name: Name; type_name: Name; span: Span; };
|
||||
export type Block = { statements: Array<Stmt>; span: Span; };
|
||||
export type Stmt = { kind: StmtKind; span: Span; };
|
||||
export type StmtKind = { kind: "Binding"; data: Binding; } | { kind: "Return"; data: ReturnStatement; } | { kind: "Expression"; data: Expr; } | { kind: "Declaration"; data: Decl; };
|
||||
export type Binding = { name: Name; value: Expr; span: Span; };
|
||||
export type ReturnStatement = { value?: Expr | null; span: Span; };
|
||||
export type InteractiveInput = { statements: Array<Stmt>; span: Span; };
|
||||
export type Expr = { kind: ExprKind; span: Span; };
|
||||
export type ExprKind = { kind: "Name"; data: string; } | { kind: "Number"; data: string; } | { kind: "String"; data: string; } | { kind: "Symbol"; data: Name; } | { kind: "Unary"; data: { operation: UnaryOp; operand: Expr; }; } | { kind: "Binary"; data: { operation: BinaryOp; left: Expr; right: Expr; }; } | { kind: "Member"; data: { object: Expr; name: Name; }; } | { kind: "Call"; data: { callee: Expr; arguments: Array<Expr>; }; };
|
||||
export type UnaryOp = "Negate";
|
||||
export type BinaryOp = "Add" | "Subtract" | "Multiply" | "Divide";
|
||||
export interface Visitor {
|
||||
visitSpan?(node: Span): boolean | void;
|
||||
visitAstDocument?(node: AstDocument): boolean | void;
|
||||
visitAstRoot?(node: AstRoot): boolean | void;
|
||||
visitName?(node: Name): boolean | void;
|
||||
visitProgram?(node: Program): boolean | void;
|
||||
visitDecl?(node: Decl): boolean | void;
|
||||
visitDeclKind?(node: DeclKind): boolean | void;
|
||||
visitFunction?(node: Function): boolean | void;
|
||||
visitParameter?(node: Parameter): boolean | void;
|
||||
visitBlock?(node: Block): boolean | void;
|
||||
visitStmt?(node: Stmt): boolean | void;
|
||||
visitStmtKind?(node: StmtKind): boolean | void;
|
||||
visitBinding?(node: Binding): boolean | void;
|
||||
visitReturnStatement?(node: ReturnStatement): boolean | void;
|
||||
visitInteractiveInput?(node: InteractiveInput): boolean | void;
|
||||
visitExpr?(node: Expr): boolean | void;
|
||||
visitExprKind?(node: ExprKind): boolean | void;
|
||||
visitUnaryOp?(node: UnaryOp): boolean | void;
|
||||
visitBinaryOp?(node: BinaryOp): boolean | void;
|
||||
}
|
||||
export function walkSpan(node: Span, visitor: Visitor): void { visitor.visitSpan?.(node); }
|
||||
export function walkAstDocument(node: AstDocument, visitor: Visitor): void {
|
||||
if (visitor.visitAstDocument?.(node) === false) return;
|
||||
walkAstRoot(node.root, visitor);
|
||||
}
|
||||
export function walkAstRoot(node: AstRoot, visitor: Visitor): void {
|
||||
if (visitor.visitAstRoot?.(node) === false) return;
|
||||
switch (node.kind) {
|
||||
case "Module":
|
||||
walkProgram(node.data, visitor);
|
||||
break;
|
||||
case "Expression":
|
||||
walkExpr(node.data, visitor);
|
||||
break;
|
||||
case "Interactive":
|
||||
walkInteractiveInput(node.data, visitor);
|
||||
break;
|
||||
}
|
||||
}
|
||||
export function walkName(node: Name, visitor: Visitor): void {
|
||||
if (visitor.visitName?.(node) === false) return;
|
||||
walkSpan(node.span, visitor);
|
||||
}
|
||||
export function walkProgram(node: Program, visitor: Visitor): void {
|
||||
if (visitor.visitProgram?.(node) === false) return;
|
||||
for (const value of node.declarations) walkDecl(value, visitor);
|
||||
walkSpan(node.span, visitor);
|
||||
}
|
||||
export function walkDecl(node: Decl, visitor: Visitor): void {
|
||||
if (visitor.visitDecl?.(node) === false) return;
|
||||
walkDeclKind(node.kind, visitor);
|
||||
walkSpan(node.span, visitor);
|
||||
}
|
||||
export function walkDeclKind(node: DeclKind, visitor: Visitor): void {
|
||||
if (visitor.visitDeclKind?.(node) === false) return;
|
||||
switch (node.kind) {
|
||||
case "Function":
|
||||
walkFunction(node.data, visitor);
|
||||
break;
|
||||
}
|
||||
}
|
||||
export function walkFunction(node: Function, visitor: Visitor): void {
|
||||
if (visitor.visitFunction?.(node) === false) return;
|
||||
walkName(node.name, visitor);
|
||||
for (const value of node.parameters) walkParameter(value, visitor);
|
||||
walkName(node.return_type, visitor);
|
||||
walkBlock(node.body, visitor);
|
||||
walkSpan(node.span, visitor);
|
||||
}
|
||||
export function walkParameter(node: Parameter, visitor: Visitor): void {
|
||||
if (visitor.visitParameter?.(node) === false) return;
|
||||
walkName(node.name, visitor);
|
||||
walkName(node.type_name, visitor);
|
||||
walkSpan(node.span, visitor);
|
||||
}
|
||||
export function walkBlock(node: Block, visitor: Visitor): void {
|
||||
if (visitor.visitBlock?.(node) === false) return;
|
||||
for (const value of node.statements) walkStmt(value, visitor);
|
||||
walkSpan(node.span, visitor);
|
||||
}
|
||||
export function walkStmt(node: Stmt, visitor: Visitor): void {
|
||||
if (visitor.visitStmt?.(node) === false) return;
|
||||
walkStmtKind(node.kind, visitor);
|
||||
walkSpan(node.span, visitor);
|
||||
}
|
||||
export function walkStmtKind(node: StmtKind, visitor: Visitor): void {
|
||||
if (visitor.visitStmtKind?.(node) === false) return;
|
||||
switch (node.kind) {
|
||||
case "Binding":
|
||||
walkBinding(node.data, visitor);
|
||||
break;
|
||||
case "Return":
|
||||
walkReturnStatement(node.data, visitor);
|
||||
break;
|
||||
case "Expression":
|
||||
walkExpr(node.data, visitor);
|
||||
break;
|
||||
case "Declaration":
|
||||
walkDecl(node.data, visitor);
|
||||
break;
|
||||
}
|
||||
}
|
||||
export function walkBinding(node: Binding, visitor: Visitor): void {
|
||||
if (visitor.visitBinding?.(node) === false) return;
|
||||
walkName(node.name, visitor);
|
||||
walkExpr(node.value, visitor);
|
||||
walkSpan(node.span, visitor);
|
||||
}
|
||||
export function walkReturnStatement(node: ReturnStatement, visitor: Visitor): void {
|
||||
if (visitor.visitReturnStatement?.(node) === false) return;
|
||||
if (node.value != null) walkExpr(node.value, visitor);
|
||||
walkSpan(node.span, visitor);
|
||||
}
|
||||
export function walkInteractiveInput(node: InteractiveInput, visitor: Visitor): void {
|
||||
if (visitor.visitInteractiveInput?.(node) === false) return;
|
||||
for (const value of node.statements) walkStmt(value, visitor);
|
||||
walkSpan(node.span, visitor);
|
||||
}
|
||||
export function walkExpr(node: Expr, visitor: Visitor): void {
|
||||
if (visitor.visitExpr?.(node) === false) return;
|
||||
walkExprKind(node.kind, visitor);
|
||||
walkSpan(node.span, visitor);
|
||||
}
|
||||
export function walkExprKind(node: ExprKind, visitor: Visitor): void {
|
||||
if (visitor.visitExprKind?.(node) === false) return;
|
||||
switch (node.kind) {
|
||||
case "Name":
|
||||
break;
|
||||
case "Number":
|
||||
break;
|
||||
case "String":
|
||||
break;
|
||||
case "Symbol":
|
||||
walkName(node.data, visitor);
|
||||
break;
|
||||
case "Unary":
|
||||
walkUnaryOp(node.data.operation, visitor);
|
||||
walkExpr(node.data.operand, visitor);
|
||||
break;
|
||||
case "Binary":
|
||||
walkBinaryOp(node.data.operation, visitor);
|
||||
walkExpr(node.data.left, visitor);
|
||||
walkExpr(node.data.right, visitor);
|
||||
break;
|
||||
case "Member":
|
||||
walkExpr(node.data.object, visitor);
|
||||
walkName(node.data.name, visitor);
|
||||
break;
|
||||
case "Call":
|
||||
walkExpr(node.data.callee, visitor);
|
||||
for (const value of node.data.arguments) walkExpr(value, visitor);
|
||||
break;
|
||||
}
|
||||
}
|
||||
export function walkUnaryOp(node: UnaryOp, visitor: Visitor): void {
|
||||
if (visitor.visitUnaryOp?.(node) === false) return;
|
||||
}
|
||||
export function walkBinaryOp(node: BinaryOp, visitor: Visitor): void {
|
||||
if (visitor.visitBinaryOp?.(node) === false) return;
|
||||
}
|
||||
@@ -9,8 +9,9 @@
|
||||
| [diagnostic.rs](diagnostic.rs) | UTF-8 字节范围、词法/语法诊断与一基行列转换 |
|
||||
| [lexer.rs](lexer.rs) | lex/lex_source 接口、源码游标、逐 Token 调用、位置/诊断换算与字符串解码 |
|
||||
| [lexer.peg](lexer.peg) | 字符输入的 PEG 词法规则:名称、数字、字符串/转义、标点、空白/注释和错误识别 |
|
||||
| [ast.rs](ast.rs) | 拥有文本的函数、参数、绑定、return、表达式与运算符结构 |
|
||||
| [parser.rs](parser.rs) | 解析 API、错误映射、AST 辅助函数,以及独立语法文件的编译期引入 |
|
||||
| [ast.rs](ast.rs) | 生成类型的公开门面、Spanned 位置接口与便捷方法 |
|
||||
| [ast](ast/README.md) | ASDL 生成的节点/遍历器与手写 JSON 校验接口 |
|
||||
| [parser.rs](parser.rs) | 统一根/ParseMode 与具体解析 API、错误映射、AST 辅助函数,以及独立语法文件的编译期引入 |
|
||||
| [grammar.peg](grammar.peg) | token 输入的 PEG 规则、运算符优先级与 AST 构造动作;保留 peg::parser! 外层声明 |
|
||||
|
||||
改输入适配时同步 [阅读器契约](../../../docs/runtime/reader.md) 与读取错误检查;改词法从 lexer.peg 进入,并同步 token、诊断和错误检查;改语法从 grammar.peg 进入,并同步 AST、[测试](../tests/README.md)、样例与 [前端契约](../../../docs/runtime/frontend.md)。词法与语法两份 PEG 动作均不做 VM 分配、I/O 或修改外部状态,避免回溯造成副作用。返回 [frontend](../README.md)。
|
||||
改节点先编辑 ../Endor.asdl,再用 scripts/generate_ast.py 更新 Rust/编辑器产物,契约见 [AST 模型](../../../docs/runtime/ast.md)。改输入适配时同步 [阅读器契约](../../../docs/runtime/reader.md) 与读取错误检查;改词法从 lexer.peg 进入,并同步 token、诊断和错误检查;改语法从 grammar.peg 进入,并同步 AST、[测试](../tests/README.md)、样例与 [前端契约](../../../docs/runtime/frontend.md)。词法与语法两份 PEG 动作均不做 VM 分配、I/O 或修改外部状态,避免回溯造成副作用。返回 [frontend](../README.md)。
|
||||
|
||||
+30
-86
@@ -1,90 +1,34 @@
|
||||
use crate::Span;
|
||||
//! AST types and walkers generated from the single Endor.asdl schema.
|
||||
//! Runtime objects remain in the objects crate.
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Name {
|
||||
pub text: String,
|
||||
pub span: Span,
|
||||
mod generated;
|
||||
pub mod visitor;
|
||||
mod wire;
|
||||
|
||||
pub use generated::*;
|
||||
pub use wire::AstError;
|
||||
|
||||
use crate::{SourceLocation, Span};
|
||||
|
||||
/// A syntax node's half-open byte range in its original UTF-8 source.
|
||||
pub trait Spanned {
|
||||
fn span(&self) -> Span;
|
||||
|
||||
fn source_text<'a>(&self, source: &'a str) -> Option<&'a str> {
|
||||
self.span().slice(source)
|
||||
}
|
||||
|
||||
fn location(&self, source: &str) -> Option<SourceLocation> {
|
||||
self.span().location(source)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Program {
|
||||
pub functions: Vec<Function>,
|
||||
pub span: Span,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Function {
|
||||
pub name: Name,
|
||||
pub parameters: Vec<Parameter>,
|
||||
pub return_type: Name,
|
||||
pub body: Block,
|
||||
pub span: Span,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Parameter {
|
||||
pub name: Name,
|
||||
pub type_name: Name,
|
||||
pub span: Span,
|
||||
}
|
||||
|
||||
/// Basic draft function body: zero or more bindings and one final return.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Block {
|
||||
pub bindings: Vec<Binding>,
|
||||
pub result: Expr,
|
||||
pub return_span: Span,
|
||||
pub span: Span,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Binding {
|
||||
pub name: Name,
|
||||
pub value: Expr,
|
||||
pub span: Span,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Expr {
|
||||
pub kind: ExprKind,
|
||||
pub span: Span,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum ExprKind {
|
||||
Name(String),
|
||||
/// The decimal spelling is retained, without f64 conversion.
|
||||
Number(String),
|
||||
String(String),
|
||||
Symbol(Name),
|
||||
Unary {
|
||||
operation: UnaryOp,
|
||||
operand: Box<Expr>,
|
||||
},
|
||||
Binary {
|
||||
operation: BinaryOp,
|
||||
left: Box<Expr>,
|
||||
right: Box<Expr>,
|
||||
},
|
||||
Member {
|
||||
object: Box<Expr>,
|
||||
name: Name,
|
||||
},
|
||||
Call {
|
||||
callee: Box<Expr>,
|
||||
arguments: Vec<Expr>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum UnaryOp {
|
||||
Negate,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum BinaryOp {
|
||||
Add,
|
||||
Subtract,
|
||||
Multiply,
|
||||
Divide,
|
||||
impl Program {
|
||||
/// Iterate the currently supported top-level function declarations.
|
||||
pub fn functions(&self) -> impl Iterator<Item = &Function> {
|
||||
self.declarations.iter().map(|declaration| {
|
||||
let DeclKind::Function(function) = &declaration.kind;
|
||||
function
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
# ASDL 生成的 AST 与接口
|
||||
|
||||
本目录只保存 frontend 的语法数据与遍历/交换接口,不定义运行时对象。
|
||||
|
||||
| 文件 | 职责 |
|
||||
| --- | --- |
|
||||
| [generated.rs](generated.rs) | 从 Endor.asdl 生成的节点类型、Serde 实现、Spanned 实现和 AST_SCHEMA_ID |
|
||||
| [visitor.rs](visitor.rs) | 同源生成的可失败 Visitor、节点进入/离开钩子与 walk_* 默认遍历 |
|
||||
| [wire.rs](wire.rs) | 手写 AstDocument JSON 接口、schema ID、范围与父子位置校验 |
|
||||
|
||||
唯一节点定义源是 [Endor.asdl](../../Endor.asdl),生成命令为 `python3 scripts/generate_ast.py --write`,在仓库根运行;--check 仅检查产物是否同步,不执行语言或测试。不要直接编辑 generated.rs/visitor.rs。公开入口见 [ast.rs](../ast.rs),完整契约见 [AST 模型](../../../../docs/runtime/ast.md)。返回 [frontend/src](../README.md)。
|
||||
@@ -0,0 +1,220 @@
|
||||
// Generated by scripts/generate_ast.py from Endor.asdl. Do not edit.
|
||||
|
||||
use super::Spanned;
|
||||
use crate::Span;
|
||||
pub const AST_SCHEMA_ID: &str =
|
||||
"urn:endor:ast:sha256:c0da1a2640ad85f2c8e8159a10756e649d3aab179823a7cd890d9ebbb7f5d46e";
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct AstDocument {
|
||||
pub schema: String,
|
||||
pub root: AstRoot,
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(tag = "kind", content = "data", deny_unknown_fields)]
|
||||
pub enum AstRoot {
|
||||
Module(Program),
|
||||
Expression(Expr),
|
||||
Interactive(InteractiveInput),
|
||||
}
|
||||
impl Spanned for AstRoot {
|
||||
fn span(&self) -> Span {
|
||||
match self {
|
||||
Self::Module(node) => node.span(),
|
||||
Self::Expression(node) => node.span(),
|
||||
Self::Interactive(node) => node.span(),
|
||||
}
|
||||
}
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Name {
|
||||
pub text: String,
|
||||
pub span: Span,
|
||||
}
|
||||
impl Spanned for Name {
|
||||
fn span(&self) -> Span {
|
||||
self.span
|
||||
}
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Program {
|
||||
pub declarations: Vec<Decl>,
|
||||
pub span: Span,
|
||||
}
|
||||
impl Spanned for Program {
|
||||
fn span(&self) -> Span {
|
||||
self.span
|
||||
}
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Decl {
|
||||
pub kind: DeclKind,
|
||||
pub span: Span,
|
||||
}
|
||||
impl Spanned for Decl {
|
||||
fn span(&self) -> Span {
|
||||
self.span
|
||||
}
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(tag = "kind", content = "data", deny_unknown_fields)]
|
||||
pub enum DeclKind {
|
||||
Function(Function),
|
||||
}
|
||||
impl Spanned for DeclKind {
|
||||
fn span(&self) -> Span {
|
||||
match self {
|
||||
Self::Function(node) => node.span(),
|
||||
}
|
||||
}
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Function {
|
||||
pub name: Name,
|
||||
pub parameters: Vec<Parameter>,
|
||||
pub return_type: Name,
|
||||
pub body: Block,
|
||||
pub span: Span,
|
||||
}
|
||||
impl Spanned for Function {
|
||||
fn span(&self) -> Span {
|
||||
self.span
|
||||
}
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Parameter {
|
||||
pub name: Name,
|
||||
pub type_name: Name,
|
||||
pub span: Span,
|
||||
}
|
||||
impl Spanned for Parameter {
|
||||
fn span(&self) -> Span {
|
||||
self.span
|
||||
}
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Block {
|
||||
pub statements: Vec<Stmt>,
|
||||
pub span: Span,
|
||||
}
|
||||
impl Spanned for Block {
|
||||
fn span(&self) -> Span {
|
||||
self.span
|
||||
}
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Stmt {
|
||||
pub kind: StmtKind,
|
||||
pub span: Span,
|
||||
}
|
||||
impl Spanned for Stmt {
|
||||
fn span(&self) -> Span {
|
||||
self.span
|
||||
}
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(tag = "kind", content = "data", deny_unknown_fields)]
|
||||
pub enum StmtKind {
|
||||
Binding(Binding),
|
||||
Return(ReturnStatement),
|
||||
Expression(Expr),
|
||||
Declaration(Decl),
|
||||
}
|
||||
impl Spanned for StmtKind {
|
||||
fn span(&self) -> Span {
|
||||
match self {
|
||||
Self::Binding(node) => node.span(),
|
||||
Self::Return(node) => node.span(),
|
||||
Self::Expression(node) => node.span(),
|
||||
Self::Declaration(node) => node.span(),
|
||||
}
|
||||
}
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Binding {
|
||||
pub name: Name,
|
||||
pub value: Expr,
|
||||
pub span: Span,
|
||||
}
|
||||
impl Spanned for Binding {
|
||||
fn span(&self) -> Span {
|
||||
self.span
|
||||
}
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct ReturnStatement {
|
||||
pub value: Option<Expr>,
|
||||
pub span: Span,
|
||||
}
|
||||
impl Spanned for ReturnStatement {
|
||||
fn span(&self) -> Span {
|
||||
self.span
|
||||
}
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct InteractiveInput {
|
||||
pub statements: Vec<Stmt>,
|
||||
pub span: Span,
|
||||
}
|
||||
impl Spanned for InteractiveInput {
|
||||
fn span(&self) -> Span {
|
||||
self.span
|
||||
}
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Expr {
|
||||
pub kind: ExprKind,
|
||||
pub span: Span,
|
||||
}
|
||||
impl Spanned for Expr {
|
||||
fn span(&self) -> Span {
|
||||
self.span
|
||||
}
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(tag = "kind", content = "data", deny_unknown_fields)]
|
||||
pub enum ExprKind {
|
||||
Name(String),
|
||||
Number(String),
|
||||
String(String),
|
||||
Symbol(Name),
|
||||
Unary {
|
||||
operation: UnaryOp,
|
||||
operand: Box<Expr>,
|
||||
},
|
||||
Binary {
|
||||
operation: BinaryOp,
|
||||
left: Box<Expr>,
|
||||
right: Box<Expr>,
|
||||
},
|
||||
Member {
|
||||
object: Box<Expr>,
|
||||
name: Name,
|
||||
},
|
||||
Call {
|
||||
callee: Box<Expr>,
|
||||
arguments: Vec<Expr>,
|
||||
},
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, Copy)]
|
||||
pub enum UnaryOp {
|
||||
Negate,
|
||||
}
|
||||
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, Copy)]
|
||||
pub enum BinaryOp {
|
||||
Add,
|
||||
Subtract,
|
||||
Multiply,
|
||||
Divide,
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
// Generated by scripts/generate_ast.py from Endor.asdl. Do not edit.
|
||||
|
||||
use super::{
|
||||
AstDocument, AstRoot, BinaryOp, Binding, Block, Decl, DeclKind, Expr, ExprKind, Function,
|
||||
InteractiveInput, Name, Parameter, Program, ReturnStatement, Spanned, Stmt, StmtKind, UnaryOp,
|
||||
};
|
||||
use crate::Span;
|
||||
pub trait Visitor {
|
||||
type Error;
|
||||
fn enter_node(&mut self, _span: Span) -> Result<(), Self::Error> {
|
||||
Ok(())
|
||||
}
|
||||
fn leave_node(&mut self) {}
|
||||
fn visit_span(&mut self, _span: &Span) -> Result<(), Self::Error> {
|
||||
Ok(())
|
||||
}
|
||||
fn visit_ast_document(&mut self, node: &AstDocument) -> Result<(), Self::Error> {
|
||||
walk_ast_document(self, node)
|
||||
}
|
||||
fn visit_ast_root(&mut self, node: &AstRoot) -> Result<(), Self::Error> {
|
||||
self.enter_node(node.span())?;
|
||||
let result = walk_ast_root(self, node);
|
||||
self.leave_node();
|
||||
result
|
||||
}
|
||||
fn visit_name(&mut self, node: &Name) -> Result<(), Self::Error> {
|
||||
self.enter_node(node.span())?;
|
||||
let result = walk_name(self, node);
|
||||
self.leave_node();
|
||||
result
|
||||
}
|
||||
fn visit_program(&mut self, node: &Program) -> Result<(), Self::Error> {
|
||||
self.enter_node(node.span())?;
|
||||
let result = walk_program(self, node);
|
||||
self.leave_node();
|
||||
result
|
||||
}
|
||||
fn visit_decl(&mut self, node: &Decl) -> Result<(), Self::Error> {
|
||||
self.enter_node(node.span())?;
|
||||
let result = walk_decl(self, node);
|
||||
self.leave_node();
|
||||
result
|
||||
}
|
||||
fn visit_decl_kind(&mut self, node: &DeclKind) -> Result<(), Self::Error> {
|
||||
self.enter_node(node.span())?;
|
||||
let result = walk_decl_kind(self, node);
|
||||
self.leave_node();
|
||||
result
|
||||
}
|
||||
fn visit_function(&mut self, node: &Function) -> Result<(), Self::Error> {
|
||||
self.enter_node(node.span())?;
|
||||
let result = walk_function(self, node);
|
||||
self.leave_node();
|
||||
result
|
||||
}
|
||||
fn visit_parameter(&mut self, node: &Parameter) -> Result<(), Self::Error> {
|
||||
self.enter_node(node.span())?;
|
||||
let result = walk_parameter(self, node);
|
||||
self.leave_node();
|
||||
result
|
||||
}
|
||||
fn visit_block(&mut self, node: &Block) -> Result<(), Self::Error> {
|
||||
self.enter_node(node.span())?;
|
||||
let result = walk_block(self, node);
|
||||
self.leave_node();
|
||||
result
|
||||
}
|
||||
fn visit_stmt(&mut self, node: &Stmt) -> Result<(), Self::Error> {
|
||||
self.enter_node(node.span())?;
|
||||
let result = walk_stmt(self, node);
|
||||
self.leave_node();
|
||||
result
|
||||
}
|
||||
fn visit_stmt_kind(&mut self, node: &StmtKind) -> Result<(), Self::Error> {
|
||||
self.enter_node(node.span())?;
|
||||
let result = walk_stmt_kind(self, node);
|
||||
self.leave_node();
|
||||
result
|
||||
}
|
||||
fn visit_binding(&mut self, node: &Binding) -> Result<(), Self::Error> {
|
||||
self.enter_node(node.span())?;
|
||||
let result = walk_binding(self, node);
|
||||
self.leave_node();
|
||||
result
|
||||
}
|
||||
fn visit_return_statement(&mut self, node: &ReturnStatement) -> Result<(), Self::Error> {
|
||||
self.enter_node(node.span())?;
|
||||
let result = walk_return_statement(self, node);
|
||||
self.leave_node();
|
||||
result
|
||||
}
|
||||
fn visit_interactive_input(&mut self, node: &InteractiveInput) -> Result<(), Self::Error> {
|
||||
self.enter_node(node.span())?;
|
||||
let result = walk_interactive_input(self, node);
|
||||
self.leave_node();
|
||||
result
|
||||
}
|
||||
fn visit_expr(&mut self, node: &Expr) -> Result<(), Self::Error> {
|
||||
self.enter_node(node.span())?;
|
||||
let result = walk_expr(self, node);
|
||||
self.leave_node();
|
||||
result
|
||||
}
|
||||
fn visit_expr_kind(&mut self, node: &ExprKind) -> Result<(), Self::Error> {
|
||||
walk_expr_kind(self, node)
|
||||
}
|
||||
fn visit_unary_op(&mut self, node: &UnaryOp) -> Result<(), Self::Error> {
|
||||
walk_unary_op(self, node)
|
||||
}
|
||||
fn visit_binary_op(&mut self, node: &BinaryOp) -> Result<(), Self::Error> {
|
||||
walk_binary_op(self, node)
|
||||
}
|
||||
}
|
||||
pub fn walk_ast_document<V: Visitor + ?Sized>(
|
||||
visitor: &mut V,
|
||||
node: &AstDocument,
|
||||
) -> Result<(), V::Error> {
|
||||
visitor.visit_ast_root(&node.root)?;
|
||||
Ok(())
|
||||
}
|
||||
pub fn walk_ast_root<V: Visitor + ?Sized>(visitor: &mut V, node: &AstRoot) -> Result<(), V::Error> {
|
||||
match node {
|
||||
AstRoot::Module(value) => {
|
||||
visitor.visit_program(value)?;
|
||||
}
|
||||
AstRoot::Expression(value) => {
|
||||
visitor.visit_expr(value)?;
|
||||
}
|
||||
AstRoot::Interactive(value) => {
|
||||
visitor.visit_interactive_input(value)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub fn walk_name<V: Visitor + ?Sized>(visitor: &mut V, node: &Name) -> Result<(), V::Error> {
|
||||
visitor.visit_span(&node.span)?;
|
||||
Ok(())
|
||||
}
|
||||
pub fn walk_program<V: Visitor + ?Sized>(visitor: &mut V, node: &Program) -> Result<(), V::Error> {
|
||||
for value in &node.declarations {
|
||||
visitor.visit_decl(value)?;
|
||||
}
|
||||
visitor.visit_span(&node.span)?;
|
||||
Ok(())
|
||||
}
|
||||
pub fn walk_decl<V: Visitor + ?Sized>(visitor: &mut V, node: &Decl) -> Result<(), V::Error> {
|
||||
visitor.visit_decl_kind(&node.kind)?;
|
||||
visitor.visit_span(&node.span)?;
|
||||
Ok(())
|
||||
}
|
||||
pub fn walk_decl_kind<V: Visitor + ?Sized>(
|
||||
visitor: &mut V,
|
||||
node: &DeclKind,
|
||||
) -> Result<(), V::Error> {
|
||||
match node {
|
||||
DeclKind::Function(value) => {
|
||||
visitor.visit_function(value)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub fn walk_function<V: Visitor + ?Sized>(
|
||||
visitor: &mut V,
|
||||
node: &Function,
|
||||
) -> Result<(), V::Error> {
|
||||
visitor.visit_name(&node.name)?;
|
||||
for value in &node.parameters {
|
||||
visitor.visit_parameter(value)?;
|
||||
}
|
||||
visitor.visit_name(&node.return_type)?;
|
||||
visitor.visit_block(&node.body)?;
|
||||
visitor.visit_span(&node.span)?;
|
||||
Ok(())
|
||||
}
|
||||
pub fn walk_parameter<V: Visitor + ?Sized>(
|
||||
visitor: &mut V,
|
||||
node: &Parameter,
|
||||
) -> Result<(), V::Error> {
|
||||
visitor.visit_name(&node.name)?;
|
||||
visitor.visit_name(&node.type_name)?;
|
||||
visitor.visit_span(&node.span)?;
|
||||
Ok(())
|
||||
}
|
||||
pub fn walk_block<V: Visitor + ?Sized>(visitor: &mut V, node: &Block) -> Result<(), V::Error> {
|
||||
for value in &node.statements {
|
||||
visitor.visit_stmt(value)?;
|
||||
}
|
||||
visitor.visit_span(&node.span)?;
|
||||
Ok(())
|
||||
}
|
||||
pub fn walk_stmt<V: Visitor + ?Sized>(visitor: &mut V, node: &Stmt) -> Result<(), V::Error> {
|
||||
visitor.visit_stmt_kind(&node.kind)?;
|
||||
visitor.visit_span(&node.span)?;
|
||||
Ok(())
|
||||
}
|
||||
pub fn walk_stmt_kind<V: Visitor + ?Sized>(
|
||||
visitor: &mut V,
|
||||
node: &StmtKind,
|
||||
) -> Result<(), V::Error> {
|
||||
match node {
|
||||
StmtKind::Binding(value) => {
|
||||
visitor.visit_binding(value)?;
|
||||
}
|
||||
StmtKind::Return(value) => {
|
||||
visitor.visit_return_statement(value)?;
|
||||
}
|
||||
StmtKind::Expression(value) => {
|
||||
visitor.visit_expr(value)?;
|
||||
}
|
||||
StmtKind::Declaration(value) => {
|
||||
visitor.visit_decl(value)?;
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub fn walk_binding<V: Visitor + ?Sized>(visitor: &mut V, node: &Binding) -> Result<(), V::Error> {
|
||||
visitor.visit_name(&node.name)?;
|
||||
visitor.visit_expr(&node.value)?;
|
||||
visitor.visit_span(&node.span)?;
|
||||
Ok(())
|
||||
}
|
||||
pub fn walk_return_statement<V: Visitor + ?Sized>(
|
||||
visitor: &mut V,
|
||||
node: &ReturnStatement,
|
||||
) -> Result<(), V::Error> {
|
||||
if let Some(value) = &node.value {
|
||||
visitor.visit_expr(value)?;
|
||||
}
|
||||
visitor.visit_span(&node.span)?;
|
||||
Ok(())
|
||||
}
|
||||
pub fn walk_interactive_input<V: Visitor + ?Sized>(
|
||||
visitor: &mut V,
|
||||
node: &InteractiveInput,
|
||||
) -> Result<(), V::Error> {
|
||||
for value in &node.statements {
|
||||
visitor.visit_stmt(value)?;
|
||||
}
|
||||
visitor.visit_span(&node.span)?;
|
||||
Ok(())
|
||||
}
|
||||
pub fn walk_expr<V: Visitor + ?Sized>(visitor: &mut V, node: &Expr) -> Result<(), V::Error> {
|
||||
visitor.visit_expr_kind(&node.kind)?;
|
||||
visitor.visit_span(&node.span)?;
|
||||
Ok(())
|
||||
}
|
||||
pub fn walk_expr_kind<V: Visitor + ?Sized>(
|
||||
visitor: &mut V,
|
||||
node: &ExprKind,
|
||||
) -> Result<(), V::Error> {
|
||||
match node {
|
||||
ExprKind::Name(_) => {}
|
||||
ExprKind::Number(_) => {}
|
||||
ExprKind::String(_) => {}
|
||||
ExprKind::Symbol(value) => {
|
||||
visitor.visit_name(value)?;
|
||||
}
|
||||
ExprKind::Unary {
|
||||
operation, operand, ..
|
||||
} => {
|
||||
visitor.visit_unary_op(operation)?;
|
||||
visitor.visit_expr(operand)?;
|
||||
}
|
||||
ExprKind::Binary {
|
||||
operation,
|
||||
left,
|
||||
right,
|
||||
..
|
||||
} => {
|
||||
visitor.visit_binary_op(operation)?;
|
||||
visitor.visit_expr(left)?;
|
||||
visitor.visit_expr(right)?;
|
||||
}
|
||||
ExprKind::Member { object, name, .. } => {
|
||||
visitor.visit_expr(object)?;
|
||||
visitor.visit_name(name)?;
|
||||
}
|
||||
ExprKind::Call {
|
||||
callee, arguments, ..
|
||||
} => {
|
||||
visitor.visit_expr(callee)?;
|
||||
for value in arguments {
|
||||
visitor.visit_expr(value)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
pub fn walk_unary_op<V: Visitor + ?Sized>(
|
||||
_visitor: &mut V,
|
||||
_node: &UnaryOp,
|
||||
) -> Result<(), V::Error> {
|
||||
Ok(())
|
||||
}
|
||||
pub fn walk_binary_op<V: Visitor + ?Sized>(
|
||||
_visitor: &mut V,
|
||||
_node: &BinaryOp,
|
||||
) -> Result<(), V::Error> {
|
||||
Ok(())
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
use std::fmt;
|
||||
|
||||
use super::visitor::Visitor;
|
||||
use super::{AST_SCHEMA_ID, AstDocument, AstRoot};
|
||||
use crate::Span;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum AstError {
|
||||
Json(serde_json::Error),
|
||||
SchemaMismatch {
|
||||
expected: &'static str,
|
||||
found: String,
|
||||
},
|
||||
InvalidSpan(Span),
|
||||
ChildOutsideParent {
|
||||
parent: Span,
|
||||
child: Span,
|
||||
},
|
||||
}
|
||||
|
||||
impl fmt::Display for AstError {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Json(error) => write!(f, "invalid AST JSON: {error}"),
|
||||
Self::SchemaMismatch { expected, found } => {
|
||||
write!(f, "AST schema mismatch: expected {expected}, found {found}")
|
||||
}
|
||||
Self::InvalidSpan(span) => {
|
||||
write!(f, "invalid AST source span: {}..{}", span.start, span.end)
|
||||
}
|
||||
Self::ChildOutsideParent { parent, child } => write!(
|
||||
f,
|
||||
"AST child span {}..{} is outside parent {}..{}",
|
||||
child.start, child.end, parent.start, parent.end
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for AstError {
|
||||
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
|
||||
match self {
|
||||
Self::Json(error) => Some(error),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl From<serde_json::Error> for AstError {
|
||||
fn from(error: serde_json::Error) -> Self {
|
||||
Self::Json(error)
|
||||
}
|
||||
}
|
||||
|
||||
impl AstDocument {
|
||||
pub fn new(root: AstRoot) -> Result<Self, AstError> {
|
||||
let document = Self {
|
||||
schema: AST_SCHEMA_ID.into(),
|
||||
root,
|
||||
};
|
||||
document.validate()?;
|
||||
Ok(document)
|
||||
}
|
||||
|
||||
/// Decode a versioned syntax tree. This does not compile or execute it.
|
||||
pub fn from_json(text: &str) -> Result<Self, AstError> {
|
||||
let document: Self = serde_json::from_str(text)?;
|
||||
document.validate()?;
|
||||
Ok(document)
|
||||
}
|
||||
|
||||
pub fn to_json(&self) -> Result<String, AstError> {
|
||||
self.validate()?;
|
||||
Ok(serde_json::to_string(self)?)
|
||||
}
|
||||
|
||||
/// Check schema identity, ranges, and parent/child containment.
|
||||
pub fn validate(&self) -> Result<(), AstError> {
|
||||
self.validate_with_source(None)
|
||||
}
|
||||
|
||||
/// Additionally check bounds and UTF-8 boundaries against original text.
|
||||
/// This does not check that the text and AST have identical semantics.
|
||||
pub fn validate_source(&self, source: &str) -> Result<(), AstError> {
|
||||
self.validate_with_source(Some(source))
|
||||
}
|
||||
|
||||
fn validate_with_source(&self, source: Option<&str>) -> Result<(), AstError> {
|
||||
if self.schema != AST_SCHEMA_ID {
|
||||
return Err(AstError::SchemaMismatch {
|
||||
expected: AST_SCHEMA_ID,
|
||||
found: self.schema.clone(),
|
||||
});
|
||||
}
|
||||
SpanValidator {
|
||||
source,
|
||||
parents: Vec::new(),
|
||||
}
|
||||
.visit_ast_root(&self.root)
|
||||
}
|
||||
}
|
||||
|
||||
struct SpanValidator<'a> {
|
||||
source: Option<&'a str>,
|
||||
parents: Vec<Span>,
|
||||
}
|
||||
|
||||
impl Visitor for SpanValidator<'_> {
|
||||
type Error = AstError;
|
||||
|
||||
fn enter_node(&mut self, span: Span) -> Result<(), AstError> {
|
||||
// JSON offsets must also be exactly representable by editor JS numbers.
|
||||
if span.start > span.end
|
||||
|| span.end as u128 > 9_007_199_254_740_991
|
||||
|| self
|
||||
.source
|
||||
.is_some_and(|source| span.slice(source).is_none())
|
||||
{
|
||||
return Err(AstError::InvalidSpan(span));
|
||||
}
|
||||
if let Some(parent) = self.parents.last().copied()
|
||||
&& (span.start < parent.start || span.end > parent.end)
|
||||
{
|
||||
return Err(AstError::ChildOutsideParent {
|
||||
parent,
|
||||
child: span,
|
||||
});
|
||||
}
|
||||
self.parents.push(span);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn leave_node(&mut self) {
|
||||
self.parents.pop();
|
||||
}
|
||||
}
|
||||
@@ -3,7 +3,8 @@ use std::fmt;
|
||||
|
||||
/// Half-open UTF-8 byte range in the original source, including trivia between
|
||||
/// the first and last syntax token. EOF has an empty range at source.len().
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Span {
|
||||
pub start: usize,
|
||||
pub end: usize,
|
||||
@@ -14,6 +15,16 @@ impl Span {
|
||||
Self { start, end }
|
||||
}
|
||||
|
||||
/// Safely read this range, rejecting reversed, out-of-bounds or split UTF-8 ranges.
|
||||
pub fn slice(self, source: &str) -> Option<&str> {
|
||||
source.get(self.start..self.end)
|
||||
}
|
||||
|
||||
pub fn location(self, source: &str) -> Option<SourceLocation> {
|
||||
self.slice(source)?;
|
||||
Some(source_location(source, self.start))
|
||||
}
|
||||
|
||||
pub(crate) fn through(self, last: Self) -> Self {
|
||||
Self::new(self.start, last.end)
|
||||
}
|
||||
@@ -51,27 +62,7 @@ impl Diagnostic {
|
||||
}
|
||||
|
||||
pub fn location(&self, source: &str) -> SourceLocation {
|
||||
let mut location = SourceLocation { line: 1, column: 1 };
|
||||
let mut previous_cr = false;
|
||||
for (offset, character) in source.char_indices() {
|
||||
if offset >= self.span.start {
|
||||
break;
|
||||
}
|
||||
match character {
|
||||
'\r' => {
|
||||
location.line += 1;
|
||||
location.column = 1;
|
||||
}
|
||||
'\n' if previous_cr => {}
|
||||
'\n' => {
|
||||
location.line += 1;
|
||||
location.column = 1;
|
||||
}
|
||||
_ => location.column += 1,
|
||||
}
|
||||
previous_cr = character == '\r';
|
||||
}
|
||||
location
|
||||
source_location(source, self.span.start)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -86,3 +77,27 @@ impl fmt::Display for Diagnostic {
|
||||
}
|
||||
|
||||
impl Error for Diagnostic {}
|
||||
|
||||
fn source_location(source: &str, position: usize) -> SourceLocation {
|
||||
let mut location = SourceLocation { line: 1, column: 1 };
|
||||
let mut previous_cr = false;
|
||||
for (offset, character) in source.char_indices() {
|
||||
if offset >= position {
|
||||
break;
|
||||
}
|
||||
match character {
|
||||
'\r' => {
|
||||
location.line += 1;
|
||||
location.column = 1;
|
||||
}
|
||||
'\n' if previous_cr => {}
|
||||
'\n' => {
|
||||
location.line += 1;
|
||||
location.column = 1;
|
||||
}
|
||||
_ => location.column += 1,
|
||||
}
|
||||
previous_cr = character == '\r';
|
||||
}
|
||||
location
|
||||
}
|
||||
|
||||
@@ -42,9 +42,28 @@ peg::parser! {
|
||||
= [Token { kind: TokenKind::Slash, .. }]
|
||||
|
||||
pub rule program() -> Program
|
||||
= functions:function()+ {
|
||||
let span = functions[0].span.through(functions.last().unwrap().span);
|
||||
Program { functions, span }
|
||||
= declarations:declaration()+ {
|
||||
let span = declarations[0].span.through(declarations.last().unwrap().span);
|
||||
Program { declarations, span }
|
||||
}
|
||||
|
||||
rule declaration() -> Decl
|
||||
= function:function() {
|
||||
Decl { span: function.span, kind: DeclKind::Function(function) }
|
||||
}
|
||||
|
||||
pub rule interactive() -> InteractiveInput
|
||||
= statement:interactive_statement() {
|
||||
InteractiveInput { span: statement.span, statements: vec![statement] }
|
||||
}
|
||||
|
||||
rule interactive_statement() -> Stmt
|
||||
= declaration:declaration() {
|
||||
Stmt { span: declaration.span, kind: StmtKind::Declaration(declaration) }
|
||||
}
|
||||
/ binding:binding() { binding_statement(binding) }
|
||||
/ expression:expression() {
|
||||
Stmt { span: expression.span, kind: StmtKind::Expression(expression) }
|
||||
}
|
||||
|
||||
rule function() -> Function
|
||||
@@ -60,7 +79,9 @@ peg::parser! {
|
||||
|
||||
rule block() -> Block
|
||||
= start:lbrace() bindings:binding()* returned:return_statement() end:rbrace() {
|
||||
Block { bindings, result: returned.0, return_span: returned.1, span: start.through(end) }
|
||||
let mut statements: Vec<Stmt> = bindings.into_iter().map(binding_statement).collect();
|
||||
statements.push(returned);
|
||||
Block { statements, span: start.through(end) }
|
||||
}
|
||||
|
||||
rule binding() -> Binding
|
||||
@@ -68,10 +89,10 @@ peg::parser! {
|
||||
Binding { span: start.through(value.span), name, value }
|
||||
}
|
||||
|
||||
rule return_statement() -> (Expr, Span)
|
||||
rule return_statement() -> Stmt
|
||||
= start:return_keyword() value:expression() {
|
||||
let span = start.through(value.span);
|
||||
(value, span)
|
||||
Stmt { span, kind: StmtKind::Return(ReturnStatement { value: Some(value), span }) }
|
||||
}
|
||||
|
||||
pub rule expression() -> Expr = precedence! {
|
||||
|
||||
@@ -12,7 +12,10 @@ mod token;
|
||||
|
||||
pub use diagnostic::{Diagnostic, DiagnosticKind, SourceLocation, Span};
|
||||
pub use lexer::{lex, lex_source};
|
||||
pub use parser::{parse, parse_expression, parse_source, parse_source_expression};
|
||||
pub use parser::{
|
||||
ParseMode, parse, parse_ast, parse_expression, parse_source, parse_source_ast,
|
||||
parse_source_expression,
|
||||
};
|
||||
pub use reader::{
|
||||
EndorReader, EndorSourceProvider, ResolvedEndorSource, SourceReadError, SourceReadErrorKind,
|
||||
};
|
||||
|
||||
@@ -1,9 +1,38 @@
|
||||
use crate::ast::{
|
||||
BinaryOp, Binding, Block, Expr, ExprKind, Function, Name, Parameter, Program, UnaryOp,
|
||||
AstRoot, BinaryOp, Binding, Block, Decl, DeclKind, Expr, ExprKind, Function, InteractiveInput,
|
||||
Name, Parameter, Program, ReturnStatement, Stmt, StmtKind, UnaryOp,
|
||||
};
|
||||
use crate::lexer::decode_string;
|
||||
use crate::{Diagnostic, DiagnosticKind, Span, Token, TokenKind, lex};
|
||||
|
||||
/// Select the syntax root without implying any execution or REPL session.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ParseMode {
|
||||
Module,
|
||||
Expression,
|
||||
Interactive,
|
||||
}
|
||||
|
||||
pub fn parse_ast(source: &str, mode: ParseMode) -> Result<AstRoot, Diagnostic> {
|
||||
match mode {
|
||||
ParseMode::Module => parse(source).map(AstRoot::Module),
|
||||
ParseMode::Expression => parse_expression(source).map(AstRoot::Expression),
|
||||
ParseMode::Interactive => {
|
||||
let tokens = lex(source)?;
|
||||
syntax::interactive(&tokens, source)
|
||||
.map(AstRoot::Interactive)
|
||||
.map_err(|error| diagnostic(error, &tokens, source.len()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn parse_source_ast(
|
||||
source: &crate::SourceUnit,
|
||||
mode: ParseMode,
|
||||
) -> Result<AstRoot, crate::SourceDiagnostic> {
|
||||
parse_ast(source.text(), mode).map_err(|error| source.annotate(error))
|
||||
}
|
||||
|
||||
/// Parse a reader snapshot, retaining its origin and resolved revision on error.
|
||||
pub fn parse_source(source: &crate::SourceUnit) -> Result<Program, crate::SourceDiagnostic> {
|
||||
parse(source.text()).map_err(|error| source.annotate(error))
|
||||
@@ -41,6 +70,13 @@ fn diagnostic(error: peg::error::ParseError<usize>, tokens: &[Token], end: usize
|
||||
}
|
||||
}
|
||||
|
||||
fn binding_statement(binding: Binding) -> Stmt {
|
||||
Stmt {
|
||||
span: binding.span,
|
||||
kind: StmtKind::Binding(binding),
|
||||
}
|
||||
}
|
||||
|
||||
fn binary(left: Expr, operation: BinaryOp, right: Expr) -> Expr {
|
||||
Expr {
|
||||
span: left.span.through(right.span),
|
||||
|
||||
@@ -2,6 +2,8 @@
|
||||
|
||||
[syntax.rs](syntax.rs) 覆盖 UTF-8 token 范围、中文/日文名称、数字原文、负号、字符串转义与错误、算术优先级和结合性、成员/调用链、函数与绑定、上下文名称、完整输入、非法/未支持语法以及 CRLF 和 EOF 诊断位置。
|
||||
|
||||
[ast.rs](ast.rs) 覆盖根模式、声明/语句结构、生成 Visitor 与错误退出、Spanned 位置、带 schema 的 JSON 往返及错误输入;同时核对编辑器产物标识。
|
||||
|
||||
[lexical.rs](lexical.rs) 专门覆盖字符 PEG 迁移后的原文位置换算、首个错误及详细诊断、Unicode 非法转义、未闭合字符串、注释/除法/成员/数字/字符串边界和空输入/尾随注释。
|
||||
|
||||
[reader.rs](reader.rs) 覆盖字符串/分块流/文件、严格 UTF-8、读取失败、URI provider、修订标签、旧缓冲区保留及带来源的诊断。样例也通过 syntax.rs 解析。只验证语法和 AST,不宣称类型检查或运行求值通过。随工作区检查在 ces-control 执行,本地不运行;入口见 [验证](../../../docs/operations/validation.md)。返回 [frontend](../README.md)。
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
use frontend::ast::visitor::{Visitor, walk_expr, walk_function};
|
||||
use frontend::ast::{
|
||||
AST_SCHEMA_ID, AstDocument, AstError, AstRoot, Expr, ExprKind, Function, Spanned, StmtKind,
|
||||
};
|
||||
use frontend::{EndorReader, ParseMode, Span, parse_ast, parse_source_ast};
|
||||
|
||||
#[test]
|
||||
fn parse_modes_produce_distinct_roots_and_preserve_source_diagnostics() {
|
||||
let source = "function echo(value: String): String { let copy = value return copy }";
|
||||
let root = parse_ast(source, ParseMode::Module).unwrap();
|
||||
let AstRoot::Module(program) = &root else {
|
||||
panic!("expected module")
|
||||
};
|
||||
let function = program.functions().next().unwrap();
|
||||
assert_eq!(function.body.statements.len(), 2);
|
||||
assert!(matches!(
|
||||
function.body.statements[0].kind,
|
||||
StmtKind::Binding(_)
|
||||
));
|
||||
assert!(matches!(
|
||||
function.body.statements[1].kind,
|
||||
StmtKind::Return(_)
|
||||
));
|
||||
AstDocument::new(root)
|
||||
.unwrap()
|
||||
.validate_source(source)
|
||||
.unwrap();
|
||||
|
||||
let root = parse_ast("echo(\"日\")", ParseMode::Expression).unwrap();
|
||||
assert!(matches!(root, AstRoot::Expression(_)));
|
||||
for text in ["let value = \"日\"", "echo(value)", source] {
|
||||
let root = parse_ast(text, ParseMode::Interactive).unwrap();
|
||||
let AstRoot::Interactive(input) = &root else {
|
||||
panic!("expected interactive")
|
||||
};
|
||||
assert_eq!(input.statements.len(), 1);
|
||||
AstDocument::new(root)
|
||||
.unwrap()
|
||||
.validate_source(text)
|
||||
.unwrap();
|
||||
}
|
||||
assert!(parse_ast("let x = 1 let y = 2", ParseMode::Interactive).is_err());
|
||||
assert!(parse_ast("", ParseMode::Interactive).is_err());
|
||||
let source = EndorReader::from_text("<session:1>", "\"日\" + )");
|
||||
let error = parse_source_ast(&source, ParseMode::Expression).unwrap_err();
|
||||
assert_eq!(error.location.line, 1);
|
||||
assert_eq!(error.location.column, 7);
|
||||
assert!(error.to_string().starts_with("<session:1>:1:7:"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generated_visitors_reach_names_and_unwind_node_hooks_on_error() {
|
||||
#[derive(Default)]
|
||||
struct Names {
|
||||
functions: Vec<String>,
|
||||
references: Vec<String>,
|
||||
depth: usize,
|
||||
stop: bool,
|
||||
}
|
||||
impl Visitor for Names {
|
||||
type Error = &'static str;
|
||||
fn enter_node(&mut self, _span: Span) -> Result<(), Self::Error> {
|
||||
self.depth += 1;
|
||||
Ok(())
|
||||
}
|
||||
fn leave_node(&mut self) {
|
||||
self.depth -= 1;
|
||||
}
|
||||
fn visit_function(&mut self, node: &Function) -> Result<(), Self::Error> {
|
||||
self.functions.push(node.name.text.clone());
|
||||
walk_function(self, node)
|
||||
}
|
||||
fn visit_expr(&mut self, node: &Expr) -> Result<(), Self::Error> {
|
||||
if let ExprKind::Name(name) = &node.kind {
|
||||
if self.stop {
|
||||
return Err("stopped at reference");
|
||||
}
|
||||
self.references.push(name.clone());
|
||||
}
|
||||
walk_expr(self, node)
|
||||
}
|
||||
}
|
||||
let root = parse_ast(
|
||||
"function echo(value: String): String { let copy = value return copy }",
|
||||
ParseMode::Module,
|
||||
)
|
||||
.unwrap();
|
||||
let mut names = Names::default();
|
||||
names.visit_ast_root(&root).unwrap();
|
||||
assert_eq!(names.functions, ["echo"]);
|
||||
assert_eq!(names.references, ["value", "copy"]);
|
||||
assert_eq!(names.depth, 0);
|
||||
let mut stopped = Names {
|
||||
stop: true,
|
||||
..Names::default()
|
||||
};
|
||||
assert_eq!(stopped.visit_ast_root(&root), Err("stopped at reference"));
|
||||
assert_eq!(stopped.depth, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn spans_share_unicode_locations_and_safe_original_source_access() {
|
||||
let source = " \r\n(値)";
|
||||
let root = parse_ast(source, ParseMode::Expression).unwrap();
|
||||
assert_eq!(root.source_text(source), Some("(値)"));
|
||||
let location = root.location(source).unwrap();
|
||||
assert_eq!((location.line, location.column), (2, 1));
|
||||
assert_eq!(Span::new(1, 3).slice("値"), None);
|
||||
assert_eq!(Span::new(2, 1).slice("値"), None);
|
||||
assert_eq!(Span::new(0, 4).location("値"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn versioned_json_roundtrips_all_roots_and_preserves_number_spelling() {
|
||||
for (source, mode) in [
|
||||
(
|
||||
"function f(): Number { return 00012345678901234567890.00100 }",
|
||||
ParseMode::Module,
|
||||
),
|
||||
("f(.名前, \"日\\n\")", ParseMode::Expression),
|
||||
("let value = 1", ParseMode::Interactive),
|
||||
] {
|
||||
let document = AstDocument::new(parse_ast(source, mode).unwrap()).unwrap();
|
||||
let json = document.to_json().unwrap();
|
||||
let restored = AstDocument::from_json(&json).unwrap();
|
||||
assert_eq!(restored, document);
|
||||
restored.validate_source(source).unwrap();
|
||||
assert!(json.contains(AST_SCHEMA_ID));
|
||||
if mode == ParseMode::Module {
|
||||
assert!(json.contains("00012345678901234567890.00100"));
|
||||
}
|
||||
}
|
||||
let schema: serde_json::Value =
|
||||
serde_json::from_str(include_str!("../editor/ast.schema.json")).unwrap();
|
||||
assert_eq!(schema["$id"], AST_SCHEMA_ID);
|
||||
assert_eq!(
|
||||
schema["$defs"]["AstDocument"]["properties"]["schema"]["const"],
|
||||
AST_SCHEMA_ID
|
||||
);
|
||||
assert!(include_str!("../editor/ast.ts").contains(AST_SCHEMA_ID));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn json_interfaces_reject_unknown_fields_schema_changes_and_invalid_ranges() {
|
||||
let source = "(f(値))";
|
||||
let document = AstDocument::new(parse_ast(source, ParseMode::Expression).unwrap()).unwrap();
|
||||
let mut value = serde_json::to_value(&document).unwrap();
|
||||
value["schema"] = "urn:endor:ast:unknown".into();
|
||||
assert!(matches!(
|
||||
AstDocument::from_json(&value.to_string()),
|
||||
Err(AstError::SchemaMismatch { .. })
|
||||
));
|
||||
value = serde_json::to_value(&document).unwrap();
|
||||
value["root"]["data"]["unexpected"] = true.into();
|
||||
assert!(matches!(
|
||||
AstDocument::from_json(&value.to_string()),
|
||||
Err(AstError::Json(_))
|
||||
));
|
||||
value = serde_json::to_value(&document).unwrap();
|
||||
value["root"]["data"]["span"]["end"] = 1.into();
|
||||
assert!(matches!(
|
||||
AstDocument::from_json(&value.to_string()),
|
||||
Err(AstError::ChildOutsideParent { .. })
|
||||
));
|
||||
let AstRoot::Expression(mut expression) = document.root else {
|
||||
unreachable!()
|
||||
};
|
||||
expression.span = Span::new(5, 4);
|
||||
assert!(matches!(
|
||||
AstDocument::new(AstRoot::Expression(expression)),
|
||||
Err(AstError::InvalidSpan(_))
|
||||
));
|
||||
let root = AstRoot::Expression(Expr {
|
||||
kind: ExprKind::Name("値".into()),
|
||||
span: Span::new(1, 3),
|
||||
});
|
||||
assert!(matches!(
|
||||
AstDocument::new(root).unwrap().validate_source("値"),
|
||||
Err(AstError::InvalidSpan(_))
|
||||
));
|
||||
}
|
||||
@@ -1,6 +1,27 @@
|
||||
use frontend::ast::{BinaryOp, Expr, ExprKind, UnaryOp};
|
||||
use frontend::ast::{
|
||||
BinaryOp, Binding, DeclKind, Expr, ExprKind, Function, Program, StmtKind, UnaryOp,
|
||||
};
|
||||
use frontend::{DiagnosticKind, Span, TokenKind, lex, parse, parse_expression};
|
||||
|
||||
fn function(program: &Program, index: usize) -> &Function {
|
||||
let DeclKind::Function(function) = &program.declarations[index].kind;
|
||||
function
|
||||
}
|
||||
|
||||
fn binding(function: &Function, index: usize) -> &Binding {
|
||||
let StmtKind::Binding(binding) = &function.body.statements[index].kind else {
|
||||
panic!("expected binding")
|
||||
};
|
||||
binding
|
||||
}
|
||||
|
||||
fn result(function: &Function) -> &Expr {
|
||||
let StmtKind::Return(returned) = &function.body.statements.last().unwrap().kind else {
|
||||
panic!("expected return")
|
||||
};
|
||||
returned.value.as_ref().unwrap()
|
||||
}
|
||||
|
||||
fn binary(expression: &Expr, expected: BinaryOp) -> (&Expr, &Expr) {
|
||||
let ExprKind::Binary {
|
||||
operation,
|
||||
@@ -174,7 +195,7 @@ fn member_and_call_chains_bind_more_tightly_than_unary_minus() {
|
||||
#[test]
|
||||
fn parses_typed_functions_bindings_and_multiple_top_level_definitions() {
|
||||
let example = parse(include_str!("../examples/basic.endor")).unwrap();
|
||||
assert_eq!(example.functions.len(), 2);
|
||||
assert_eq!(example.declarations.len(), 2);
|
||||
let source = r#"
|
||||
// Basic draft syntax, including a Unicode name.
|
||||
function discountedPrice(price: Number, discount: Number): Number {
|
||||
@@ -184,25 +205,26 @@ fn parses_typed_functions_bindings_and_multiple_top_level_definitions() {
|
||||
function 表示(): String { return "こんにちは" }
|
||||
"#;
|
||||
let program = parse(source).unwrap();
|
||||
assert_eq!(program.functions.len(), 2);
|
||||
let function = &program.functions[0];
|
||||
assert_eq!(function.name.text, "discountedPrice");
|
||||
assert_eq!(function.parameters.len(), 2);
|
||||
assert_eq!(function.parameters[0].type_name.text, "Number");
|
||||
assert_eq!(function.return_type.text, "Number");
|
||||
assert_eq!(function.body.bindings[0].name.text, "amount");
|
||||
binary(&function.body.bindings[0].value, BinaryOp::Subtract);
|
||||
assert_eq!(function.body.result.kind, ExprKind::Name("amount".into()));
|
||||
assert_eq!(program.declarations.len(), 2);
|
||||
let first = function(&program, 0);
|
||||
assert_eq!(first.name.text, "discountedPrice");
|
||||
assert_eq!(first.parameters.len(), 2);
|
||||
assert_eq!(first.parameters[0].type_name.text, "Number");
|
||||
assert_eq!(first.return_type.text, "Number");
|
||||
assert_eq!(binding(first, 0).name.text, "amount");
|
||||
binary(&binding(first, 0).value, BinaryOp::Subtract);
|
||||
assert_eq!(result(first).kind, ExprKind::Name("amount".into()));
|
||||
assert_eq!(
|
||||
&source[function.body.return_span.start..function.body.return_span.end],
|
||||
&source[first.body.statements.last().unwrap().span.start
|
||||
..first.body.statements.last().unwrap().span.end],
|
||||
"return amount"
|
||||
);
|
||||
assert_eq!(
|
||||
program.functions[1].body.result.kind,
|
||||
result(function(&program, 1)).kind,
|
||||
ExprKind::String("こんにちは".into())
|
||||
);
|
||||
assert_eq!(
|
||||
&source[program.functions[1].span.start..program.functions[1].span.end],
|
||||
&source[function(&program, 1).span.start..function(&program, 1).span.end],
|
||||
"function 表示(): String { return \"こんにちは\" }"
|
||||
);
|
||||
// The output owns its text and can outlive the input buffer.
|
||||
@@ -210,7 +232,7 @@ fn parses_typed_functions_bindings_and_multiple_top_level_definitions() {
|
||||
let source = String::from("function f(): String { return \"ok\" }");
|
||||
parse(&source).unwrap()
|
||||
};
|
||||
assert_eq!(owned.functions[0].name.text, "f");
|
||||
assert_eq!(function(&owned, 0).name.text, "f");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -218,13 +240,13 @@ fn syntax_words_are_contextual_and_block_layout_does_not_depend_on_newlines() {
|
||||
let program =
|
||||
parse("function class(function: Type): Type { let agent = function return agent }")
|
||||
.unwrap();
|
||||
assert_eq!(program.functions[0].name.text, "class");
|
||||
assert_eq!(program.functions[0].parameters[0].name.text, "function");
|
||||
assert_eq!(program.functions[0].body.bindings[0].name.text, "agent");
|
||||
assert_eq!(function(&program, 0).name.text, "class");
|
||||
assert_eq!(function(&program, 0).parameters[0].name.text, "function");
|
||||
assert_eq!(binding(function(&program, 0), 0).name.text, "agent");
|
||||
let multiline =
|
||||
parse("function f\r\n(\r\nx: Number\r\n): Number\r\n{\r\nreturn x\r\n}").unwrap();
|
||||
assert_eq!(
|
||||
multiline.functions[0].body.result.kind,
|
||||
result(function(&multiline, 0)).kind,
|
||||
ExprKind::Name("x".into())
|
||||
);
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
| Number、String、Dict、Cell、Function、异常数据 | [基本对象](runtime/builtins.md) | [objects/src](../crates/objects/src/README.md) |
|
||||
| 字符串、文件、流与 endor URI 如何进入 lexer | [Endor 阅读器](runtime/reader.md) | [reader.rs](../crates/frontend/src/reader.rs)、[source.rs](../crates/frontend/src/source.rs) |
|
||||
| 源码如何分词、PEG 规则、AST 和诊断 | [源码前端](runtime/frontend.md) | [lexer.peg](../crates/frontend/src/lexer.peg)、[grammar.peg](../crates/frontend/src/grammar.peg)、[frontend](../crates/frontend/README.md)、[CLI](../crates/cli/README.md) |
|
||||
| AST 根/声明/语句、ASDL 生成、遍历、位置与 JSON/编辑器接口 | [AST 模型](runtime/ast.md) | [Endor.asdl](../crates/frontend/Endor.asdl)、[AST 实现](../crates/frontend/src/ast/README.md)、[editor](../crates/frontend/editor/README.md) |
|
||||
| 操作码、代码验证、局部槽、函数定义与调用协议 | [字节码 VM](runtime/bytecode.md) | instruction.rs/code.rs 在 [objects/src](../crates/objects/src/README.md),frame.rs/eval.rs 在 [core/src](../crates/core/src/README.md) |
|
||||
| 基础信息保存在哪里、旧库如何迁移 | [宿主文件信息](operations/host-information.md) | [host](../crates/host/README.md) |
|
||||
| UTF-8 在日本地区是否适用,其他编码怎样进入 VM | [文本编码](runtime/text-encoding.md) | [core](../crates/core/README.md)、[host](../crates/host/README.md) |
|
||||
|
||||
@@ -20,13 +20,13 @@ Server ─→ Core + Host
|
||||
未来数据库特殊类的宿主实现 ─→ Storage
|
||||
```
|
||||
|
||||
CLI、Server 依赖 Host 保存文件信息,不依赖 Storage;Core 只依赖本地 Objects,Objects 无依赖且不反向依赖 Core;二者均不依赖 Storage。CLI 的 parse 命令通过 Frontend 的 EndorReader 读取文件/stdin,再解析 AST;Frontend 的外部依赖只有 peg,读取适配使用标准库。平台 URI 的存储与授权由宿主 provider 提供,Core 不依赖 Frontend。HTTP、Tokio、SQLx、数据库、网络和浏览器能力不进入 core 或 objects;两包均无第三方依赖。包与文件定位见 [crates](../../crates/README.md)。
|
||||
CLI、Server 依赖 Host 保存文件信息,不依赖 Storage;Core 只依赖本地 Objects,Objects 无依赖且不反向依赖 Core;二者均不依赖 Storage。CLI 的 parse 命令通过 Frontend 的 EndorReader 读取文件/stdin,再解析 AST;Frontend 使用 peg 及工作区已有 serde/serde_json,读取适配使用标准库;Endor.asdl 在开发时生成 AST、遍历器与编辑器接口,普通构建不运行生成器。平台 URI 的存储与授权由宿主 provider 提供,Core 不依赖 Frontend。HTTP、Tokio、SQLx、数据库、网络和浏览器能力不进入 core 或 objects;两包均无第三方依赖。包与文件定位见 [crates](../../crates/README.md)。
|
||||
|
||||
| 组件 | 当前职责 | 全局契约 |
|
||||
| --- | --- | --- |
|
||||
| Core | Vm、执行帧、调用协议与取指求值循环;对象方法委托到 Objects | [字节码 VM](../runtime/bytecode.md) |
|
||||
| Objects | 核心对象、私有堆、ObjectStore、Number、代码对象、指令数据和静态验证;无求值器 | [对象模型](../runtime/objects.md) |
|
||||
| Frontend | Endor 阅读器、UTF-8 字符 PEG lexer、token PEG parser、基础函数/表达式 AST,不执行源码 | [源码前端](../runtime/frontend.md) |
|
||||
| Frontend | Endor 阅读器、UTF-8 字符 PEG lexer、token PEG parser、ASDL 生成的根/声明/语句/表达式 AST 与跨工具接口,不执行源码 | [源码前端](../runtime/frontend.md) |
|
||||
| Host | 基础信息的逐键 UTF-8 JSON 文件与旧库导入 | [宿主信息](../operations/host-information.md) |
|
||||
| Storage | 按显式配置建立 SQLite/PostgreSQL 连接池,不定义应用表 | [数据库特殊类](../runtime/database-objects.md) |
|
||||
| CLI | 对象引导演示、源码解析与文件信息命令 | [CLI](../../crates/cli/README.md) |
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
| Number、String、Dict、Cell、函数环境和异常数据 | [builtins.md](../runtime/builtins.md) | objects/src、objects/tests 与 core/tests 文件表 |
|
||||
| lexer 前的输入适配、SourceUnit 与来源诊断 | [reader.md](../runtime/reader.md) | frontend 的 reader/source、CLI 与读取检查 |
|
||||
| Token、lexer.peg 词法/grammar.peg 语法规则、AST 与源码诊断 | [frontend.md](../runtime/frontend.md) | frontend 源码/测试/样例和 CLI |
|
||||
| AST 节点/根模式、ASDL 生成、Visitor、位置与 JSON/编辑器接口 | [ast.md](../runtime/ast.md) | Endor.asdl、生成器、生成产物与 AST/CLI 检查 |
|
||||
| 指令编码、栈效果、调用协议与执行限制 | [bytecode.md](../runtime/bytecode.md) | instruction/code/frame/eval 文件职责 |
|
||||
| 宿主基础信息、JSON 文件和旧库导入 | [host-information.md](../operations/host-information.md) | host、cli、server 与脚本 README |
|
||||
| 地区与文本编码边界 | [text-encoding.md](../runtime/text-encoding.md) | 配置主题与实际 I/O 入口 |
|
||||
@@ -48,7 +49,9 @@ Rust 实现依赖模块可见性、trait 和 rustfmt,不照搬 CPython C 符
|
||||
|
||||
Endor 是由平台解释执行的语言,字节码生成是内部准备步骤。资源版本属于平台元数据,具体 URI 与持久化协议尚未定稿,不提前写进业务语法;首版暂不要求持久化字节码缓存,详见 [执行与版本模型](../architecture/execution-and-versions.md)。
|
||||
|
||||
独立 Frontend 已提供阅读器、lexer/parser 和 AST;后续按实际需求接入名称/闭包分析和字节码编译、扩充迭代器等指令、宿主回调、错误传播和资源边界。语法可继续调整,AST 到字节码和云服务机制尚未实现,不写成既有功能。
|
||||
AST 节点以 frontend/Endor.asdl 为唯一来源,Rust 类型/遍历器和 TypeScript/JSON Schema 为提交到 Git 的生成产物,不直接编辑;通过 scripts/generate_ast.py --write 更新,--check 做静态同步检查。生成器行为测试仅在指定远端执行。
|
||||
|
||||
独立 Frontend 已提供阅读器、lexer/parser 和 ASDL 生成的 AST/跨工具接口;后续按实际需求接入名称/闭包分析和字节码编译、扩充迭代器等指令、宿主回调、错误传播和资源边界。语法可继续调整,AST 到字节码和云服务机制尚未实现,不写成既有功能。
|
||||
|
||||
## 导出与迁移
|
||||
|
||||
|
||||
@@ -16,6 +16,7 @@ Objects 零依赖,统一拥有核心对象、Number、代码对象/指令数
|
||||
- Function、NativeFunction、Dict、Cell、异常数据、UTF-8 String 和无损表示优先的 Number。
|
||||
- 小型 wordcode、共享代码对象、操作数/控制流/栈校验、显式执行帧及 execute/call;MakeFunction 可在执行中创建带默认参数与共享闭包 Cell 的函数。对象返回值持有根,用后 release_value;借用读取不自动 retain。
|
||||
- EndorReader 在 lexer 前提供字符串/流/文件/宿主 URI provider 输入,产出不可变 SourceUnit,保存来源与不透明修订标签;带来源的词法/语法错误保留行列,CLI parse 支持文件和 stdin。没有平台 URI 后端或版本锁实现,见 [阅读器](../runtime/reader.md)。
|
||||
- Endor.asdl 是 AST 唯一定义源,生成 Rust 节点/Spanned/Visitor、TypeScript 类型/遍历器和 JSON Schema。AstRoot 支持 Module/Expression/Interactive,Program/Block 使用声明/语句列表;AstDocument 提供带 schema 的 JSON 及范围校验,CLI 支持 --mode/--json,见 [AST 模型](../runtime/ast.md)。生成产物需通过 scripts/generate_ast.py 更新,不能手改。
|
||||
- Frontend 使用 peg 0.8.6,词法规则在 lexer.peg(字符 → Token),语法规则在 grammar.peg(Token → AST);lexer.rs 保留逐 Token 游标与诊断位置换算,提供完整输入解析、基础函数/表达式 AST、Unicode 字节范围与行列诊断;CLI parse 可读 UTF-8 文件/stdin 输出 AST,不执行源码。当前语法以未完成设计稿为起点,可以继续调整,见 [源码前端](../runtime/frontend.md)。
|
||||
- Host 文件信息独立于数据库:host.version、deployment.release 与 metadata put/get 使用 UTF-8 JSON。旧 PostgreSQL 基础信息只读导入缺失文件,不删除原库,见 [文件信息](../operations/host-information.md)。
|
||||
- Storage 只提供显式 DatabaseConfig/DatabaseConnection;没有默认数据库、自动应用建表或基础信息 KV。特殊类的名称/接口尚未定义,没有将连接接入 VM。
|
||||
@@ -31,7 +32,7 @@ AST 到字节码生成、平台资源存储/修订与依赖锁、名称/闭
|
||||
|
||||
## 工作约定
|
||||
|
||||
本地不构建、不运行测试/冒烟,不下载依赖;只允许静态阅读、格式化、语法/链接检查。运行验证交给 ces-control 的 SQLite、真实 PostgreSQL 与宿主流程。按用户要求不主动跟踪 CI/部署,结果未查询就明确写未核实。
|
||||
本地不构建、不运行测试/冒烟,不下载依赖;允许静态阅读、格式化、ASDL 源码生成/同步检查、语法/链接检查。运行验证交给 ces-control 的 SQLite、真实 PostgreSQL 与宿主流程。按用户要求不主动跟踪 CI/部署,结果未查询就明确写未核实。
|
||||
|
||||
数据库、卷和已有资料保留;不得改其他项目或共享 Runner。研究材料与 AGENTS.md 不提交。Git 身份按本地工作约定使用专用 codex-bot,不能借用仓库所有者凭据。
|
||||
|
||||
@@ -43,4 +44,4 @@ Endor 由平台解释执行;AST 到字节码是解释器内部准备步骤,
|
||||
|
||||
## 下一步
|
||||
|
||||
前端已完成到 AST 的首版实现;下一步连接解释器内部的字节码生成:AST → 名称/局部槽与闭包分析 → 字节码 → 现有 VM,不改成 AST 直接解释。可以先连接字符串、名称、let、位置参数函数和调用,再补算术指令与数值语义。参数与返回类型目前只保存为语法节点,不引入 Python type/元类或借此实现整个语言类型系统。语法不必等外部设计稿全部定稿,按用户要求先实现可用子集、以后再调整;当前尚无源码执行命令。
|
||||
前端已完成 ASDL 单一来源、统一根/声明/语句层次、遍历及 JSON/编辑器接口;语法仍为可用子集,Interactive 仅解析单次提交,不是 REPL。下一步连接解释器内部的字节码生成:AST → 名称/局部槽与闭包分析 → 字节码 → 现有 VM,不改成 AST 直接解释。可以先连接字符串、名称、let、位置参数函数和调用,再补算术指令与数值语义。参数与返回类型目前只保存为语法节点,不引入 Python type/元类或借此实现整个语言类型系统。语法不必等外部设计稿全部定稿,按用户要求先实现可用子集、以后再调整;当前尚无源码执行命令。
|
||||
|
||||
@@ -14,11 +14,12 @@ cargo run -p cli -- bootstrap
|
||||
cargo run -p cli -- help
|
||||
cargo run -p cli -- parse crates/frontend/examples/basic.endor
|
||||
cargo run -p cli -- parse - < crates/frontend/examples/basic.endor
|
||||
cargo run -p cli -- parse --json crates/frontend/examples/basic.endor
|
||||
```
|
||||
|
||||
bootstrap 创建两个 Record,输出 runtime=EndorObject、type_system=disabled、objects_distinct=true 和 heap_objects=2,显式释放后输出 released heap_objects=0。默认无参数执行同一示例。
|
||||
|
||||
parse 通过 EndorReader 严格读取 UTF-8 源码文件;`parse -` 从 stdin 读到 EOF,输入来源为 `<stdin>`,没有 REPL 提示符或会话求值。成功时输出 Rust Debug 格式的 AST;失败时输出来源、行、列并返回非零退出码;阅读错误在进入 lexer 前返回。输入适配与 URI provider 见 [阅读器](../runtime/reader.md)。该命令只解析语法,不执行源码;语法与当前边界见 [源码前端](../runtime/frontend.md)。
|
||||
parse 通过 EndorReader 严格读取 UTF-8 源码文件;`parse -` 从 stdin 读到 EOF,输入来源为 `<stdin>`,没有 REPL 提示符或会话求值。默认解析 module 模式,成功时输出 Rust Debug 格式的 AstRoot;--mode 支持 expression/interactive,--json 输出带 schema 的 JSON,见 [AST 模型](../runtime/ast.md);失败时输出来源、行、列并返回非零退出码;阅读错误在进入 lexer 前返回。输入适配与 URI provider 见 [阅读器](../runtime/reader.md)。该命令只解析语法,不执行源码;语法与当前边界见 [源码前端](../runtime/frontend.md)。
|
||||
|
||||
CLI 不读取数据库环境,不创建基础信息数据库表。metadata put/get 通过 host 包写 UTF-8 JSON 文件,metadata import 仅导入缺失记录;示例、配置和协议见 [宿主信息](host-information.md)。输入与退出码见 [cli/src](../../crates/cli/src/README.md)。
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
## 执行约定
|
||||
|
||||
本地不运行 Cargo 构建、测试、Clippy、冒烟、Docker 或依赖下载;允许阅读、格式化和静态文件/文档检查。指定运行验证环境为 ces-control。按用户要求不主动追踪 CI/部署结果,有问题由用户反馈后再查。
|
||||
本地不运行 Cargo 构建、测试、Clippy、冒烟、Docker 或依赖下载;允许阅读、格式化、ASDL 源码生成/产物同步检查和静态文件/文档检查。指定运行验证环境为 ces-control。按用户要求不主动追踪 CI/部署结果,有问题由用户反馈后再查。
|
||||
|
||||
## 检查入口
|
||||
|
||||
@@ -16,7 +16,7 @@ cargo test -p storage --test postgres --locked -- --ignored
|
||||
python3 scripts/smoke.py
|
||||
```
|
||||
|
||||
check.sh 执行 cargo fmt --all --check、独立词法/语法文件的 rustfmt --edition 2024 --check crates/frontend/src/lexer.peg crates/frontend/src/grammar.peg、cargo test --workspace --locked 和 cargo clippy --workspace --all-targets --locked -- -D warnings。workspace 测试也包含 rustdoc 示例;显式忽略的 PostgreSQL 测试需要单独执行。
|
||||
check.sh 先运行 generate_ast.py --check 验证 ASDL 产物同步,以及仅远端执行的 test_generate_ast.py 生成器回归检查,再执行 cargo fmt --all --check、独立词法/语法文件的 rustfmt --edition 2024 --check crates/frontend/src/lexer.peg crates/frontend/src/grammar.peg、cargo test --workspace --locked 和 cargo clippy --workspace --all-targets --locked -- -D warnings。workspace 测试也包含 rustdoc 示例;显式忽略的 PostgreSQL 测试需要单独执行。
|
||||
|
||||
ci-ces-control.sh 负责准备全新专用临时 PostgreSQL 和测试 URL;成功路径使用真实数据库,不用模拟 socket 代替。测试库与生产数据不共享挂载。没有 ASTER_TEST_POSTGRES_URL 时,显式执行 PostgreSQL 测试会失败,不标记为通过。
|
||||
|
||||
@@ -30,6 +30,7 @@ ci-ces-control.sh 负责准备全新专用临时 PostgreSQL 和测试 URL;成
|
||||
| 基本对象 | Number 表示、字符串、Dict/Cell 引用与更新、函数环境、异常数据 | 同目录 builtins.rs |
|
||||
| 字节码 | 编解码、非法指令/控制流、跳转、调用、闭包、字节码创建函数/默认值/环境捕获、函数创建及后续失败清理、结果存活与限额 | 同目录 execution.rs |
|
||||
| Endor 阅读器 | 字符串、分块 UTF-8 流、文件、读取失败、URI provider、修订标签与来源诊断 | [frontend/tests](../../crates/frontend/tests/README.md) 的 reader.rs |
|
||||
| AST 与接口生成 | 三类根、声明/语句层次、Visitor 及失败退出、Unicode 位置、JSON 往返、schema/范围错误和跨语言产物标识 | frontend/tests/ast.rs 与 scripts/test_generate_ast.py,见 [AST 模型](../runtime/ast.md) |
|
||||
| 源码前端 | 字符 PEG 词法边界、逐 Token 原文位置与首个错误诊断、Unicode 范围、数字原文、字符串与错误、优先级/结合性、函数/绑定、完整输入与错误位置、源码样例 | [frontend/tests](../../crates/frontend/tests/README.md) |
|
||||
| CLI 解析 | AST 输出、未执行源码、文件/stdin 来源和行列错误、EOF 读取与非法 UTF-8 退出 | [cli/tests](../../crates/cli/tests/README.md) |
|
||||
| 公开示例 | 对象操作、手写指令执行与释放 | objects 与 core 的 src/lib.rs rustdoc,见 [objects/src](../../crates/objects/src/README.md)、[core/src](../../crates/core/src/README.md) |
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
| [基本对象与 Number](builtins.md) | 函数、原生函数、字典、Cell、异常数据、字符串与无损数字 |
|
||||
| [Endor 阅读器](reader.md) | SourceUnit、来源/修订标签、字符串/流/文件和宿主 URI provider |
|
||||
| [源码前端](frontend.md) | 字符 PEG lexer、token PEG parser、AST、基础函数/表达式语法与诊断 |
|
||||
| [AST 模型与 ASDL](ast.md) | 统一根、声明/语句/表达式、生成器、Visitor、位置和 JSON/编辑器接口 |
|
||||
| [字节码 VM](bytecode.md) | 指令编码、代码校验、局部槽、执行帧、函数创建、调用/返回与边界 |
|
||||
| [文本编码](text-encoding.md) | UTF-8、日文、外部 Shift_JIS/CP932 与宿主解码边界 |
|
||||
| [数据库特殊类](database-objects.md) | 显式连接、调用方数据模型、未来类接入与资源生命周期 |
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
# AST 模型、ASDL 与跨工具接口
|
||||
|
||||
Endor 的 AST 节点统一由 [Endor.asdl](../../crates/frontend/Endor.asdl) 定义。它描述节点及字段;lexer.peg 描述字符识别,grammar.peg 描述语法与节点构造。ASDL 与语法各自维护一个职责,不引入 CST 或 Python type/元类;AST 位于 frontend,运行时对象位于 objects。
|
||||
|
||||
## 生成关系
|
||||
|
||||
```text
|
||||
Endor.asdl → scripts/generate_ast.py
|
||||
├─ src/ast/generated.rs:Rust 节点、Serde 与 Spanned
|
||||
├─ src/ast/visitor.rs:Rust Visitor/walk_*
|
||||
├─ editor/ast.ts:TypeScript 类型、Visitor/walk*
|
||||
└─ editor/ast.schema.json:JSON Schema 2020-12
|
||||
```
|
||||
|
||||
路径均相对 crates/frontend。生成器使用 Python 标准库和 rustfmt,无额外 Python 包、build.rs 或构建时网络访问。产物提交到 Git,普通 Cargo 构建直接使用它们。开发修改 schema 后,在仓库根执行:
|
||||
|
||||
```sh
|
||||
python3 scripts/generate_ast.py --write
|
||||
python3 scripts/generate_ast.py --check
|
||||
```
|
||||
|
||||
这两条命令是源码生成/静态检查,可在本地使用;行为测试仍仅在指定远端执行。CI 检查生成同步,并运行生成器回归检查与工作区测试。
|
||||
|
||||
ASDL 支持 module、产品类型 `(Type field, ...)`、构造器联合 `A(...) | B(...)`、`*` 序列、`?` 可选字段、`--` 行注释。内建 string 是拥有文本的 String,span 对应共享的 UTF-8 Span,schema_id 是交换契约标识。生成器检查重复名称、字段、未知类型和不支持的语法;这是项目使用的 ASDL 子集,不声称兼容 CPython ASDL 的全部扩展。
|
||||
|
||||
Rust 用 struct/enum、Vec/Option 表达结构;生成器在按值依赖图的回边放置 Box,Vec 已经断开按值递归。增加节点后,Rust 类型、两种语言的遍历接口和 JSON Schema 由同一次生成更新。
|
||||
|
||||
## 根、声明、语句、表达式
|
||||
|
||||
| 层级 | 当前数据模型 |
|
||||
| --- | --- |
|
||||
| AstRoot | Module(Program)、Expression(Expr)、Interactive(InteractiveInput) |
|
||||
| Program | declarations: Vec<Decl> 与 span;当前声明种类为 Function |
|
||||
| Decl/DeclKind | 声明的位置及其具体种类;Function 保存参数、返回类型名称和 Block |
|
||||
| Block | statements: Vec<Stmt> 与 span,不再用固定的 bindings/result 字段表示 |
|
||||
| Stmt/StmtKind | Binding、Return、Expression、Declaration;每条语句有自身范围 |
|
||||
| ReturnStatement | 可选表达式与范围;当前源码中的 return 仍要求表达式 |
|
||||
| Expr/ExprKind | 名称、数字原文、字符串、符号、一元/二元运算、成员和调用 |
|
||||
|
||||
参数与返回类型仍为 Name,只保留语法,不实施类型或名称检查。Program::functions() 提供函数迭代便捷接口。旧的 Program.functions、Block.bindings/result/return_span 字段已移除;源码构造和使用方改为声明/语句列表。ExprKind 的既有表达式结构保留。
|
||||
|
||||
AST 数据模型允许一般语句列表,不代表 parser 接受了所有组合。模块仍要求一个或多个函数;函数体仍接受 let* 后跟一个带值 return。Interactive 模式接受一次函数声明、let 绑定或表达式,必须消费完整输入;不创建交互会话,不处理续行提示,不执行语言代码。多条交互提交由宿主分别解析。条件、循环、导入、类等尚未实现,也不加入 FunctionType 这种 Python 专用入口。
|
||||
|
||||
## 解析与位置
|
||||
|
||||
parse_ast(&str, ParseMode) 和 parse_source_ast(&SourceUnit, ParseMode) 返回 AstRoot;ParseMode 包含 Module、Expression、Interactive。原有 parse/parse_expression/parse_source/parse_source_expression 保留对应的具体返回类型。带来源入口的诊断仍记录名称/URI/修订和行列。
|
||||
|
||||
Spanned 为有位置的节点提供 span()、source_text(&str)、location(&str)。Span::slice/location 拒绝反向、越界或截断 UTF-8 字符的范围;行列是一基 Unicode 标量列,CRLF 算一次换行。Span 不携带源码身份,调用方需关联原始 SourceUnit;AST JSON 本身不含源码、文件路径、URI 或资源修订。
|
||||
|
||||
## 遍历与分析
|
||||
|
||||
生成的 Rust Visitor 有每个节点/种类的 visit_* 方法、默认 walk_* 函数,以及 enter_node/leave_node 钩子。Error 为关联类型,子节点失败时默认遍历返回错误,并配对调用已进入节点的离开钩子。可在 visit_function、visit_binding、visit_expr 等位置实现名称分析或字节码生成。
|
||||
|
||||
覆盖 visit_* 后,调用 walk_* 可继续访问子节点;覆盖方法自身的钩子管理由实现者决定,直接调用 walk_* 不会重复调用当前节点的 visit_*。这些接口只提供遍历,尚未实现名称绑定、闭包捕获分析或字节码生成。TypeScript Visitor 对应编辑器使用,回调返回 false 可跳过子树。
|
||||
|
||||
## JSON 交换
|
||||
|
||||
frontend 使用工作区已有的 serde/serde_json。AstDocument 是带 schema 和 root 的交换外壳,提供 new、to_json、from_json、validate、validate_source。构造/读取/写出时检查 schema 标识、位置顺序、父子范围包含关系及 JS 安全整数范围;validate_source 额外检查给定原文的长度和 UTF-8 字符边界,不比较原文与节点的程序含义。
|
||||
|
||||
构造器联合按 `{"kind":"Variant","data":...}` 编码,无字段的运算符联合使用字符串。产品类型使用字段对象;未知字段和未知构造器拒绝。可选字段可省略或为 null,标准写出为 null。数字保留源码拼写字符串,不经浮点转换。
|
||||
|
||||
AST_SCHEMA_ID 从 Endor.asdl 的 UTF-8 文本计算 SHA-256,包含注释与空白;生成器读取时使用 Python 的统一换行规则。它标识这套 AST 接口定义,区别于用户资源修订、编译器或 VM 版本。修改 schema 后旧标识被拒绝,尚无历史格式迁移。改变生成器的交换编码或内建位置类型契约时,也必须更新 ASDL 顶部的 Wire contract 版本说明,使标识随之变化。直接调用 Serde 是结构编解码,跨边界应使用 AstDocument 方法执行完整校验;JSON Schema 不表达所有位置约束。
|
||||
|
||||
CLI 可提供具体的交换输出:
|
||||
|
||||
```sh
|
||||
cli parse --json source.endor
|
||||
cli parse --mode expression --json -
|
||||
cli parse --mode interactive --json -
|
||||
```
|
||||
|
||||
文件/stdin 阅读保持原有 UTF-8 和错误来源规则,stdin 到 EOF 后才解析。普通 cli parse 输出 AstRoot 的 Debug 文本,稳定交换使用带 schema 的 JSON;所有入口均不执行源码。命令在指定远端运行。
|
||||
|
||||
检查源码见 [frontend/tests](../../crates/frontend/tests/README.md) 和 [生成器检查](../../scripts/test_generate_ast.py),边界见 [验证](../operations/validation.md)。返回 [运行时索引](README.md)。
|
||||
@@ -13,7 +13,7 @@ frontend 包提供 [Endor 阅读器](reader.md)、rust-peg 生成的字符 lexer
|
||||
| [lexer.peg](../../crates/frontend/src/lexer.peg) | str 字符输入,识别名称、数字、字符串/转义、标点、空白/注释及词法错误,生成 Token | [lexer.rs](../../crates/frontend/src/lexer.rs) 保留 lex/lex_source、游标、位置换算和字符串解码 |
|
||||
| [grammar.peg](../../crates/frontend/src/grammar.peg) | [Token] 输入,匹配函数/绑定/return、表达式优先级并构造 AST | [parser.rs](../../crates/frontend/src/parser.rs) 保留解析入口、诊断转换和 AST 辅助函数 |
|
||||
|
||||
lexer.rs 与 parser.rs 分别通过 include! 在编译期引入对应规则;两份 .peg 文件保留 peg::parser! 外层声明,直接参与编译,不新增 build.rs、生成脚本或依赖。远端检查脚本单独对两份规则执行 rustfmt。
|
||||
lexer.rs 与 parser.rs 分别通过 include! 在编译期引入对应规则;两份 .peg 文件保留 peg::parser! 外层声明,直接参与编译,自身不需要 build.rs 或生成脚本。AST 类型另由 Endor.asdl 离线生成,交换接口使用 serde/serde_json,见 [AST 模型](ast.md)。远端检查脚本单独对两份规则执行 rustfmt。
|
||||
|
||||
字符 PEG 的内部 next 规则使用 #[no_eof] 每次识别一个前缀,跳过空白/注释并返回一个 Token 或输入结束。Lexer 保存 source/offset,循环调用该规则,直到全输入结束或首个错误;公开 lex 仍扫描整份输入,不是增量/可恢复输入接口。规则中的 position! 相对本次输入切片,lexer.rs 将 Token 和诊断位置统一换算为原始源码 UTF-8 字节范围。
|
||||
|
||||
@@ -29,8 +29,10 @@ lexer.rs 与 parser.rs 分别通过 include! 在编译期引入对应规则;
|
||||
| lex(&str) | 产生 Vec<Token>;每个 token 只有种类和原始源码的字节范围,不复制文本 |
|
||||
| parse(&str) | 解析一个或多个完整函数定义,返回 Program;空输入和未消费的尾部内容报错 |
|
||||
| parse_expression(&str) | 解析一个完整表达式,不能忽略尾部 token |
|
||||
| parse_ast(&str, ParseMode) | 返回 Module/Expression/Interactive 三类 AstRoot,模式独立于输入来源 |
|
||||
| parse_source_ast(&SourceUnit, ParseMode) | 同一统一入口,失败保留来源、修订和行列 |
|
||||
|
||||
AST 的名称、数字原文和解码后的字符串均由 AST 自己持有,可以在释放输入后继续使用。Token/AST 的 Span 均为原始 UTF-8 源码的半开字节区间 start..end;节点跨度包含首尾 token 之间的空白及注释,不包含外侧 trivia。分组表达式的范围包括括号,Program 范围从第一个函数到最后一个函数。
|
||||
AST 结构及 Visitor、Spanned 和跨工具 JSON 契约见 [AST 模型](ast.md)。Program 使用 declarations,Block 使用 statements;ExprKind 保留既有表达式形状。AST 的名称、数字原文和解码后的字符串均由 AST 自己持有,可以在释放输入后继续使用。Token/AST 的 Span 均为原始 UTF-8 源码的半开字节区间 start..end;节点跨度包含首尾 token 之间的空白及注释,不包含外侧 trivia。分组表达式的范围包括括号,Program 范围从第一个函数到最后一个函数。
|
||||
|
||||
词法动作只返回 Token/诊断,语法动作只构造 AST;两者均不做 I/O、VM 分配或其他外部写入。PEG 回溯不会执行语言函数或产生宿主业务副作用。
|
||||
|
||||
@@ -54,7 +56,7 @@ block = "{" binding* "return" expression "}"
|
||||
binding = "let" name "=" expression
|
||||
```
|
||||
|
||||
parameters 和调用参数列表使用逗号分隔,不接受尾随逗号。函数体要求一个末尾 return;不支持独立表达式语句、重赋值、嵌套函数声明或默认参数源码写法。参数和返回类型目前只是名称节点;并不据此创建语言 type 对象或检查类型。
|
||||
parameters 和调用参数列表使用逗号分隔,不接受尾随逗号。函数体要求一个末尾 return;函数体尚不支持独立表达式语句、重赋值、嵌套函数声明或默认参数源码写法。新增 Interactive 解析模式接受一次函数声明、let 绑定或表达式,完整消费输入,但不创建 REPL 或执行代码;AST 可表示的语句组合比当前源码子集更广。参数和返回类型目前只是名称节点;并不据此创建语言 type 对象或检查类型。
|
||||
|
||||
表达式包括名称、数字、字符串、`.name` 符号、括号分组、成员访问与位置参数调用、一元负号以及四则运算。优先级从高到低为:
|
||||
|
||||
@@ -71,6 +73,6 @@ use、namespace、class、agent、完整类型语法、条件/循环、比较
|
||||
|
||||
词法和语法错误分别标记为 Lexical/Syntax。语法错误将 PEG 的 token 下标转换成原始源码 Span,保留期望项目列表;EOF 错误使用 source.len() 处的空区间。Diagnostic::location 返回一基行列,列按 Unicode 标量计数,CRLF 只算一次换行,不按字节列或字形簇计数。
|
||||
|
||||
`cli parse <file>` 与 `cli parse -` 成功时输出 Rust Debug 格式的 AST,失败时在 stderr 输出来源/行/列和诊断并返回非零退出码;`-` 表示从 stdin 读到 EOF,不是 REPL。阅读与来源包装规则见 [阅读器](reader.md)。AST Debug 文本不是稳定的序列化协议。命令不执行源码、不初始化 VM、不连接数据库;具体运行与本地禁测约定见 [运行](../operations/README.md) 和 [验证](../operations/validation.md)。
|
||||
`cli parse <file>` 与 `cli parse -` 成功时输出 Rust Debug 格式的 AST,失败时在 stderr 输出来源/行/列和诊断并返回非零退出码;`-` 表示从 stdin 读到 EOF,不是 REPL。阅读与来源包装规则见 [阅读器](reader.md)。可用 --mode 选择 module/expression/interactive,用 --json 输出带 schema 标识的 AstDocument;普通输出为 AstRoot Debug 文本,稳定交换格式见 [AST 模型](ast.md)。命令不执行源码、不初始化 VM、不连接数据库;具体运行与本地禁测约定见 [运行](../operations/README.md) 和 [验证](../operations/validation.md)。
|
||||
|
||||
后续需要将 AST 经名称/闭包分析转换为现有字节码,再交给 VM 解释执行。这是解释器内部的编译/执行准备步骤,不要求使用者先构建程序,见 [执行与版本模型](../architecture/execution-and-versions.md)。这条连接尚未实现;解析成功不表示表达式已有运行语义。VM 虽已支持 MakeFunction,但从源码创建函数仍需要字节码生成接入。解析器也不是不可信代码沙箱,当前没有通用输入大小或解析递归深度限额。
|
||||
|
||||
+4
-2
@@ -1,10 +1,12 @@
|
||||
# 检查与发布脚本
|
||||
|
||||
所有构建、测试、冒烟和部署脚本只在用户指定的远端环境运行。本目录 README 负责定位与副作用边界;全局流程见 [验证](../docs/operations/validation.md) 和 [部署](../docs/operations/deployment.md)。
|
||||
所有构建、测试、冒烟和部署脚本只在用户指定的远端环境运行。generate_ast.py 是源码生成/静态同步工具,可本地使用;test_generate_ast.py 仅远端执行。本目录 README 负责定位与副作用边界;全局流程见 [验证](../docs/operations/validation.md) 和 [部署](../docs/operations/deployment.md)。
|
||||
|
||||
| 文件 | 入口与副作用 |
|
||||
| --- | --- |
|
||||
| [check.sh](check.sh) | Cargo fmt 与独立 lexer.peg/grammar.peg 的 rustfmt 检查、workspace 测试、严格 Clippy;会使用 Cargo 依赖和构建产物 |
|
||||
| [generate_ast.py](generate_ast.py) | 从 Endor.asdl 生成并格式化 Rust/TypeScript/JSON Schema;--write 更新产物,--check 只比较,无依赖下载或语言执行 |
|
||||
| [test_generate_ast.py](test_generate_ast.py) | 远端 unittest:schema 扩展同步、递归装箱/遍历、非法定义拒绝 |
|
||||
| [check.sh](check.sh) | 先检查 ASDL 产物同步和生成器行为,再运行 Cargo fmt 与独立 lexer.peg/grammar.peg 的 rustfmt 检查、workspace 测试、严格 Clippy;会使用 Cargo 依赖和构建产物 |
|
||||
| [smoke.py](smoke.py) | 标准库 Python 脚本;构建后用 Cargo metadata 定位 cli/server 的 debug 二进制,支持 CARGO_TARGET_DIR;文件 metadata 跨进程与日文值;失效数据库变量下独立启动、不创建 SQLite 文件、HTTP 与 SIGTERM,不改生产配置 |
|
||||
| [ci-ces-control.sh](ci-ces-control.sh) | 需要 ASTER_RELEASE_SHA;构建 ci 镜像、创建临时网络/PostgreSQL/检查容器,运行全部检查;退出清理临时容器和网络,保留专属远端 Cargo 缓存卷 |
|
||||
| [deploy-ces-control.sh](deploy-ces-control.sh) | ces-control root 执行;校验已构建镜像、锁定发布、保留密码和数据、验证 HTTP 与无数据库 CLI 引导后切换 current,失败尝试恢复旧版;保留镜像;应用服务 aster 内调用 cli bootstrap,基础信息改写文件;一次性只读导入旧基础信息,不覆盖新文件或删除原库 |
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
python3 scripts/generate_ast.py --check
|
||||
python3 scripts/test_generate_ast.py
|
||||
cargo fmt --all --check
|
||||
rustfmt --edition 2024 --check crates/frontend/src/lexer.peg crates/frontend/src/grammar.peg
|
||||
cargo test --workspace --locked
|
||||
|
||||
@@ -0,0 +1,405 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate checked-in AST types, walkers and editor contracts from Endor.asdl.
|
||||
|
||||
Uses Python's standard library and rustfmt only. It never builds or runs Endor.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from dataclasses import dataclass
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
import re
|
||||
import subprocess
|
||||
|
||||
|
||||
BUILTINS = {"string": "String", "span": "Span", "schema_id": "String"}
|
||||
HEADER = "// Generated by scripts/generate_ast.py from Endor.asdl. Do not edit.\n"
|
||||
RESERVED = set("as break const continue crate else enum extern false fn for if impl in let loop match mod move mut pub ref return self Self static struct super trait true type unsafe use where while async await dyn abstract become box do final macro override priv typeof unsized virtual yield try gen".split())
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Field:
|
||||
type: str
|
||||
name: str
|
||||
quantifier: str = ""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Variant:
|
||||
name: str
|
||||
fields: tuple[Field, ...]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Definition:
|
||||
name: str
|
||||
fields: tuple[Field, ...] | None
|
||||
variants: tuple[Variant, ...] = ()
|
||||
|
||||
def all_fields(self):
|
||||
return self.fields if self.fields is not None else tuple(f for v in self.variants for f in v.fields)
|
||||
|
||||
def simple(self):
|
||||
return self.fields is None and all(not v.fields for v in self.variants)
|
||||
|
||||
|
||||
def parse_schema(text: str) -> dict[str, Definition]:
|
||||
"""Parse the project's ASDL subset; reject unknown syntax and field types."""
|
||||
text = re.sub(r"--[^\n]*", "", text)
|
||||
tokens = []
|
||||
cursor = 0
|
||||
pattern = re.compile(r"\s*([A-Za-z_][A-Za-z_0-9]*|[{}()=|,*?])")
|
||||
while cursor < len(text):
|
||||
if not text[cursor:].strip():
|
||||
break
|
||||
match = pattern.match(text, cursor)
|
||||
if not match:
|
||||
raise ValueError(f"unsupported ASDL syntax near {text[cursor:cursor + 30]!r}")
|
||||
tokens.append(match[1])
|
||||
cursor = match.end()
|
||||
index = 0
|
||||
|
||||
def peek():
|
||||
if index == len(tokens):
|
||||
raise ValueError("unexpected end of ASDL")
|
||||
return tokens[index]
|
||||
|
||||
def take(expected=None):
|
||||
nonlocal index
|
||||
if index == len(tokens):
|
||||
raise ValueError("unexpected end of ASDL")
|
||||
value = tokens[index]
|
||||
if expected is not None and value != expected:
|
||||
raise ValueError(f"expected {expected!r}, found {value!r}")
|
||||
index += 1
|
||||
return value
|
||||
|
||||
def identifier():
|
||||
value = take()
|
||||
if not re.fullmatch(r"[A-Za-z_][A-Za-z_0-9]*", value) or value in RESERVED:
|
||||
raise ValueError(f"invalid or reserved identifier: {value}")
|
||||
return value
|
||||
|
||||
def fields():
|
||||
take("(")
|
||||
result = []
|
||||
while peek() != ")":
|
||||
type_name = identifier()
|
||||
quantifier = take() if peek() in ("?", "*") else ""
|
||||
result.append(Field(type_name, identifier(), quantifier))
|
||||
if peek() != ",":
|
||||
break
|
||||
take(",")
|
||||
take(")")
|
||||
if len({f.name for f in result}) != len(result):
|
||||
raise ValueError("duplicate field name")
|
||||
return tuple(result)
|
||||
|
||||
take("module")
|
||||
identifier()
|
||||
take("{")
|
||||
definitions = {}
|
||||
while peek() != "}":
|
||||
name = identifier()
|
||||
if name in definitions or name in BUILTINS:
|
||||
raise ValueError(f"duplicate or builtin type: {name}")
|
||||
take("=")
|
||||
if peek() == "(":
|
||||
definition = Definition(name, fields())
|
||||
else:
|
||||
variants = []
|
||||
while True:
|
||||
variant = identifier()
|
||||
members = fields() if peek() == "(" else ()
|
||||
variants.append(Variant(variant, members))
|
||||
if peek() != "|":
|
||||
break
|
||||
take("|")
|
||||
if len({v.name for v in variants}) != len(variants):
|
||||
raise ValueError(f"duplicate constructor in {name}")
|
||||
definition = Definition(name, None, tuple(variants))
|
||||
definitions[name] = definition
|
||||
take("}")
|
||||
if index != len(tokens):
|
||||
raise ValueError("trailing ASDL input")
|
||||
for definition in definitions.values():
|
||||
for field in definition.all_fields():
|
||||
if field.type not in definitions and field.type not in BUILTINS:
|
||||
raise ValueError(f"undefined field type: {field.type}")
|
||||
return definitions
|
||||
|
||||
|
||||
def snake(name):
|
||||
return re.sub(r"(?<!^)(?=[A-Z])", "_", name).lower()
|
||||
|
||||
|
||||
def recursive_edges(definitions):
|
||||
"""Box DFS back edges in the by-value graph; Vec already breaks recursion."""
|
||||
active, done, boxed = set(), set(), set()
|
||||
|
||||
def visit(name):
|
||||
active.add(name)
|
||||
for field in definitions[name].all_fields():
|
||||
if field.quantifier == "*" or field.type not in definitions:
|
||||
continue
|
||||
if field.type in active:
|
||||
boxed.add((name, field.type))
|
||||
elif field.type not in done:
|
||||
visit(field.type)
|
||||
active.remove(name)
|
||||
done.add(name)
|
||||
|
||||
for name in definitions:
|
||||
if name not in done:
|
||||
visit(name)
|
||||
return boxed
|
||||
|
||||
|
||||
def spanned_types(definitions):
|
||||
result = {d.name for d in definitions.values() if d.fields is not None and Field("span", "span") in d.fields}
|
||||
while True:
|
||||
expanded = result | {
|
||||
d.name for d in definitions.values()
|
||||
if d.variants and all(len(v.fields) == 1 and not v.fields[0].quantifier and v.fields[0].type in result for v in d.variants)
|
||||
}
|
||||
if expanded == result:
|
||||
return result
|
||||
result = expanded
|
||||
|
||||
|
||||
def rust_types(definitions, schema_id):
|
||||
boxed = recursive_edges(definitions)
|
||||
spanned = spanned_types(definitions)
|
||||
|
||||
def field_type(owner, field):
|
||||
value = BUILTINS.get(field.type, field.type)
|
||||
if field.quantifier == "*":
|
||||
return f"Vec<{value}>"
|
||||
if (owner, field.type) in boxed:
|
||||
value = f"Box<{value}>"
|
||||
return f"Option<{value}>" if field.quantifier == "?" else value
|
||||
|
||||
lines = [HEADER, "use crate::Span;", "use super::Spanned;", f'pub const AST_SCHEMA_ID: &str = "{schema_id}";']
|
||||
for definition in definitions.values():
|
||||
derives = "Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize"
|
||||
if definition.simple():
|
||||
derives += ", Copy"
|
||||
lines.append(f"#[derive({derives})]")
|
||||
if not definition.simple():
|
||||
tag = 'tag = "kind", content = "data", ' if definition.fields is None else ""
|
||||
lines.append(f"#[serde({tag}deny_unknown_fields)]")
|
||||
if definition.fields is not None:
|
||||
lines.append(f"pub struct {definition.name} {{")
|
||||
lines.extend(f"pub {f.name}: {field_type(definition.name, f)}," for f in definition.fields)
|
||||
else:
|
||||
lines.append(f"pub enum {definition.name} {{")
|
||||
for variant in definition.variants:
|
||||
if not variant.fields:
|
||||
lines.append(f"{variant.name},")
|
||||
elif len(variant.fields) == 1:
|
||||
lines.append(f"{variant.name}({field_type(definition.name, variant.fields[0])}),")
|
||||
else:
|
||||
fields = ", ".join(f"{f.name}: {field_type(definition.name, f)}" for f in variant.fields)
|
||||
lines.append(f"{variant.name} {{ {fields} }},")
|
||||
lines.append("}")
|
||||
if definition.name in spanned:
|
||||
lines.append(f"impl Spanned for {definition.name} {{ fn span(&self) -> Span {{")
|
||||
if definition.fields is not None:
|
||||
lines.append("self.span")
|
||||
else:
|
||||
lines.append("match self {")
|
||||
lines.extend(f"Self::{v.name}(node) => node.span()," for v in definition.variants)
|
||||
lines.append("}")
|
||||
lines.append("} }")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def rust_visitor(definitions):
|
||||
spanned = spanned_types(definitions)
|
||||
names = ", ".join(definitions)
|
||||
lines = [HEADER, f"use super::{{{names}, Spanned}};", "use crate::Span;", "pub trait Visitor {", "type Error;",
|
||||
"fn enter_node(&mut self, _span: Span) -> Result<(), Self::Error> { Ok(()) }",
|
||||
"fn leave_node(&mut self) {}",
|
||||
"fn visit_span(&mut self, _span: &Span) -> Result<(), Self::Error> { Ok(()) }"]
|
||||
for name in definitions:
|
||||
lines.append(f"fn visit_{snake(name)}(&mut self, node: &{name}) -> Result<(), Self::Error> {{")
|
||||
if name in spanned:
|
||||
lines += ["self.enter_node(node.span())?;", f"let result = walk_{snake(name)}(self, node);", "self.leave_node();", "result"]
|
||||
else:
|
||||
lines.append(f"walk_{snake(name)}(self, node)")
|
||||
lines.append("}")
|
||||
lines.append("}")
|
||||
|
||||
def children(field, access):
|
||||
if field.type not in definitions and field.type != "span":
|
||||
return []
|
||||
method = f"visitor.visit_{snake(field.type)}"
|
||||
if field.quantifier == "*":
|
||||
return [f"for value in {access} {{ {method}(value)?; }}"]
|
||||
if field.quantifier == "?":
|
||||
return [f"if let Some(value) = {access} {{ {method}(value)?; }}"]
|
||||
return [f"{method}({access})?;"]
|
||||
|
||||
for definition in definitions.values():
|
||||
body = []
|
||||
if definition.fields is not None:
|
||||
for field in definition.fields:
|
||||
body += children(field, f"&node.{field.name}")
|
||||
elif not definition.simple():
|
||||
body.append("match node {")
|
||||
for variant in definition.variants:
|
||||
visited = [f for f in variant.fields if f.type in definitions or f.type == "span"]
|
||||
if not variant.fields:
|
||||
pattern = f"{definition.name}::{variant.name}"
|
||||
elif len(variant.fields) == 1:
|
||||
pattern = f"{definition.name}::{variant.name}({'value' if visited else '_'})"
|
||||
else:
|
||||
members = ", ".join(f.name for f in visited)
|
||||
pattern = f"{definition.name}::{variant.name} {{ " + (members + ", " if members else "") + ".. }"
|
||||
body.append(pattern + " => {")
|
||||
for field in visited:
|
||||
body += children(field, "value" if len(variant.fields) == 1 else field.name)
|
||||
body.append("}")
|
||||
body.append("}")
|
||||
visitor = "visitor" if any("visitor." in line for line in body) else "_visitor"
|
||||
node = "node" if body else "_node"
|
||||
lines.append(f"pub fn walk_{snake(definition.name)}<V: Visitor + ?Sized>({visitor}: &mut V, {node}: &{definition.name}) -> Result<(), V::Error> {{")
|
||||
lines.extend(body)
|
||||
lines += ["Ok(())", "}"]
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def editor_types(definitions, schema_id):
|
||||
def field_type(field):
|
||||
result = {"string": "string", "span": "Span", "schema_id": "typeof AST_SCHEMA_ID"}.get(field.type, field.type)
|
||||
if field.quantifier == "*":
|
||||
result = f"Array<{result}>"
|
||||
elif field.quantifier == "?":
|
||||
result += " | null"
|
||||
return result
|
||||
|
||||
def object_type(fields):
|
||||
return "{ " + " ".join(f"{f.name}{'?' if f.quantifier == '?' else ''}: {field_type(f)};" for f in fields) + " }"
|
||||
|
||||
lines = [HEADER, f'export const AST_SCHEMA_ID = "{schema_id}" as const;', "export interface Span { start: number; end: number; }"]
|
||||
for definition in definitions.values():
|
||||
if definition.fields is not None:
|
||||
lines.append(f"export type {definition.name} = {object_type(definition.fields)};")
|
||||
else:
|
||||
variants = []
|
||||
for variant in definition.variants:
|
||||
if definition.simple():
|
||||
value = json.dumps(variant.name)
|
||||
else:
|
||||
data = ""
|
||||
if variant.fields:
|
||||
result = field_type(variant.fields[0]) if len(variant.fields) == 1 else object_type(variant.fields)
|
||||
data = f" data: {result};"
|
||||
value = '{ kind: ' + json.dumps(variant.name) + ';' + data + ' }'
|
||||
variants.append(value)
|
||||
lines.append(f"export type {definition.name} = " + " | ".join(variants) + ";")
|
||||
lines += ["export interface Visitor {", "visitSpan?(node: Span): boolean | void;"]
|
||||
lines.extend(f"visit{name}?(node: {name}): boolean | void;" for name in definitions)
|
||||
lines.append("}")
|
||||
|
||||
def children(field, access):
|
||||
if field.type not in definitions and field.type != "span":
|
||||
return []
|
||||
function = "walkSpan" if field.type == "span" else f"walk{field.type}"
|
||||
if field.quantifier == "*":
|
||||
return [f"for (const value of {access}) {function}(value, visitor);"]
|
||||
if field.quantifier == "?":
|
||||
return [f"if ({access} != null) {function}({access}, visitor);"]
|
||||
return [f"{function}({access}, visitor);"]
|
||||
|
||||
lines.append("export function walkSpan(node: Span, visitor: Visitor): void { visitor.visitSpan?.(node); }")
|
||||
for definition in definitions.values():
|
||||
lines += [f"export function walk{definition.name}(node: {definition.name}, visitor: Visitor): void {{", f"if (visitor.visit{definition.name}?.(node) === false) return;"]
|
||||
if definition.fields is not None:
|
||||
for field in definition.fields:
|
||||
lines += children(field, "node." + field.name)
|
||||
elif not definition.simple():
|
||||
lines.append("switch (node.kind) {")
|
||||
for variant in definition.variants:
|
||||
lines.append(f"case {json.dumps(variant.name)}:")
|
||||
for field in variant.fields:
|
||||
access = "node.data" + ("." + field.name if len(variant.fields) > 1 else "")
|
||||
lines += children(field, access)
|
||||
lines.append("break;")
|
||||
lines.append("}")
|
||||
lines.append("}")
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def json_schema(definitions, schema_id):
|
||||
def field_schema(field):
|
||||
value = {"string": {"type": "string"}, "schema_id": {"const": schema_id}, "span": {"$ref": "#/$defs/Span"}}.get(field.type, {"$ref": f"#/$defs/{field.type}"})
|
||||
if field.quantifier == "*":
|
||||
value = {"type": "array", "items": value}
|
||||
elif field.quantifier == "?":
|
||||
value = {"anyOf": [value, {"type": "null"}]}
|
||||
return value
|
||||
|
||||
def object_schema(properties, required=None):
|
||||
return {"type": "object", "properties": properties, "required": list(properties) if required is None else required, "additionalProperties": False}
|
||||
|
||||
definitions_json = {"Span": object_schema({name: {"type": "integer", "minimum": 0, "maximum": 9007199254740991} for name in ["start", "end"]})}
|
||||
for definition in definitions.values():
|
||||
if definition.fields is not None:
|
||||
value = object_schema({f.name: field_schema(f) for f in definition.fields}, [f.name for f in definition.fields if f.quantifier != "?"])
|
||||
elif definition.simple():
|
||||
value = {"enum": [v.name for v in definition.variants]}
|
||||
else:
|
||||
variants = []
|
||||
for variant in definition.variants:
|
||||
properties = {"kind": {"const": variant.name}}
|
||||
if variant.fields:
|
||||
properties["data"] = field_schema(variant.fields[0]) if len(variant.fields) == 1 else object_schema({f.name: field_schema(f) for f in variant.fields}, [f.name for f in variant.fields if f.quantifier != "?"])
|
||||
variants.append(object_schema(properties))
|
||||
value = {"oneOf": variants}
|
||||
definitions_json[definition.name] = value
|
||||
return json.dumps({"$schema": "https://json-schema.org/draft/2020-12/schema", "$id": schema_id, "$ref": "#/$defs/AstDocument", "$defs": definitions_json}, ensure_ascii=False, indent=2) + "\n"
|
||||
|
||||
|
||||
def generate(text):
|
||||
definitions = parse_schema(text)
|
||||
for required in ("AstDocument", "AstRoot"):
|
||||
if required not in definitions:
|
||||
raise ValueError(f"missing required schema type: {required}")
|
||||
schema_id = "urn:endor:ast:sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
return {
|
||||
"crates/frontend/src/ast/generated.rs": rust_types(definitions, schema_id),
|
||||
"crates/frontend/src/ast/visitor.rs": rust_visitor(definitions),
|
||||
"crates/frontend/editor/ast.ts": editor_types(definitions, schema_id),
|
||||
"crates/frontend/editor/ast.schema.json": json_schema(definitions, schema_id),
|
||||
}
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
mode = parser.add_mutually_exclusive_group(required=True)
|
||||
mode.add_argument("--check", action="store_true")
|
||||
mode.add_argument("--write", action="store_true")
|
||||
args = parser.parse_args()
|
||||
root = Path(__file__).resolve().parents[1]
|
||||
outputs = generate((root / "crates/frontend/Endor.asdl").read_text(encoding="utf-8"))
|
||||
stale = []
|
||||
for relative, content in outputs.items():
|
||||
if relative.endswith(".rs"):
|
||||
content = subprocess.run(["rustfmt", "--edition", "2024", "--emit", "stdout"], input=content, text=True, capture_output=True, check=True).stdout
|
||||
path = root / relative
|
||||
if args.write:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(content, encoding="utf-8")
|
||||
elif not path.is_file() or path.read_text(encoding="utf-8") != content:
|
||||
stale.append(relative)
|
||||
if stale:
|
||||
parser.exit(1, "stale generated AST files:\n" + "\n".join(stale) + "\nRun python3 scripts/generate_ast.py --write\n")
|
||||
print("AST artifacts generated." if args.write else "AST artifacts match Endor.asdl.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,57 @@
|
||||
"""Generator regression checks for the designated remote validation environment."""
|
||||
|
||||
from pathlib import Path
|
||||
import json
|
||||
import re
|
||||
import unittest
|
||||
|
||||
from generate_ast import generate, parse_schema, recursive_edges
|
||||
|
||||
|
||||
SCHEMA = (Path(__file__).resolve().parents[1] / "crates/frontend/Endor.asdl").read_text(encoding="utf-8")
|
||||
|
||||
|
||||
class AstGenerationTests(unittest.TestCase):
|
||||
def test_editor_walkers_only_call_declared_functions(self):
|
||||
editor = generate(SCHEMA)["crates/frontend/editor/ast.ts"]
|
||||
declared = set(re.findall(r"export function (walk\w+)\(", editor))
|
||||
calls = set(re.findall(r"\b(walk\w+)\(", editor))
|
||||
self.assertLessEqual(calls, declared)
|
||||
self.assertIn("walkSpan(node.span, visitor)", editor)
|
||||
|
||||
def test_schema_changes_reach_all_languages_and_schema_identity(self):
|
||||
before = generate(SCHEMA)
|
||||
after = generate(SCHEMA.replace("UnaryOp = Negate", "UnaryOp = Negate | LogicalNot"))
|
||||
self.assertIn("LogicalNot", after["crates/frontend/src/ast/generated.rs"])
|
||||
self.assertIn('"LogicalNot"', after["crates/frontend/editor/ast.ts"])
|
||||
editor_schema = json.loads(after["crates/frontend/editor/ast.schema.json"])
|
||||
self.assertIn("LogicalNot", editor_schema["$defs"]["UnaryOp"]["enum"])
|
||||
self.assertNotEqual(before["crates/frontend/src/ast/generated.rs"], after["crates/frontend/src/ast/generated.rs"])
|
||||
self.assertEqual(after, generate(SCHEMA.replace("UnaryOp = Negate", "UnaryOp = Negate | LogicalNot")))
|
||||
|
||||
def test_recursive_optional_nodes_are_boxed_and_walked(self):
|
||||
extended = SCHEMA.rsplit("}", 1)[0] + "Link = (Link? next)\n}"
|
||||
outputs = generate(extended)
|
||||
self.assertIn("pub next: Option<Box<Link>>", outputs["crates/frontend/src/ast/generated.rs"])
|
||||
self.assertIn("visitor.visit_link(value)?", outputs["crates/frontend/src/ast/visitor.rs"])
|
||||
self.assertIn("next?: Link | null", outputs["crates/frontend/editor/ast.ts"])
|
||||
definitions = parse_schema(SCHEMA)
|
||||
self.assertIn(("ExprKind", "Expr"), recursive_edges(definitions))
|
||||
self.assertNotIn(("Expr", "ExprKind"), recursive_edges(definitions))
|
||||
|
||||
def test_invalid_schemas_fail_instead_of_emitting_partial_types(self):
|
||||
for schema in [
|
||||
"module Endor { Root = (Missing value) }",
|
||||
"module Endor { Root = (string x, string x) }",
|
||||
"module Endor { Root = A | A }",
|
||||
"module Endor { Root = A Root = B }",
|
||||
"module Endor { Root = (string value",
|
||||
"module Endor { Root = A } trailing",
|
||||
"module Endor { Root = @ }",
|
||||
]:
|
||||
with self.subTest(schema=schema), self.assertRaises(ValueError):
|
||||
parse_schema(schema)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user