feat: generate unified AST and editor contracts from ASDL
Verify databases and deploy Aster VM / verify-and-deploy (push) Successful in 7m55s

This commit is contained in:
2026-10-19 14:31:28 +08:00
parent be4f1add58
commit f5268d2e09
40 changed files with 2517 additions and 172 deletions
+4 -3
View File
@@ -9,8 +9,9 @@
| [diagnostic.rs](diagnostic.rs) | UTF-8 字节范围、词法/语法诊断与一基行列转换 |
| [lexer.rs](lexer.rs) | lex/lex_source 接口、源码游标、逐 Token 调用、位置/诊断换算与字符串解码 |
| [lexer.peg](lexer.peg) | 字符输入的 PEG 词法规则:名称、数字、字符串/转义、标点、空白/注释和错误识别 |
| [ast.rs](ast.rs) | 拥有文本的函数、参数、绑定、return、表达式与运算符结构 |
| [parser.rs](parser.rs) | 解析 API、错误映射、AST 辅助函数,以及独立语法文件的编译期引入 |
| [ast.rs](ast.rs) | 生成类型的公开门面、Spanned 位置接口与便捷方法 |
| [ast](ast/README.md) | ASDL 生成的节点/遍历器与手写 JSON 校验接口 |
| [parser.rs](parser.rs) | 统一根/ParseMode 与具体解析 API、错误映射、AST 辅助函数,以及独立语法文件的编译期引入 |
| [grammar.peg](grammar.peg) | token 输入的 PEG 规则、运算符优先级与 AST 构造动作;保留 peg::parser! 外层声明 |
改输入适配时同步 [阅读器契约](../../../docs/runtime/reader.md) 与读取错误检查;改词法从 lexer.peg 进入,并同步 token、诊断和错误检查;改语法从 grammar.peg 进入,并同步 AST、[测试](../tests/README.md)、样例与 [前端契约](../../../docs/runtime/frontend.md)。词法与语法两份 PEG 动作均不做 VM 分配、I/O 或修改外部状态,避免回溯造成副作用。返回 [frontend](../README.md)。
改节点先编辑 ../Endor.asdl,再用 scripts/generate_ast.py 更新 Rust/编辑器产物,契约见 [AST 模型](../../../docs/runtime/ast.md)。改输入适配时同步 [阅读器契约](../../../docs/runtime/reader.md) 与读取错误检查;改词法从 lexer.peg 进入,并同步 token、诊断和错误检查;改语法从 grammar.peg 进入,并同步 AST、[测试](../tests/README.md)、样例与 [前端契约](../../../docs/runtime/frontend.md)。词法与语法两份 PEG 动作均不做 VM 分配、I/O 或修改外部状态,避免回溯造成副作用。返回 [frontend](../README.md)。
+30 -86
View File
@@ -1,90 +1,34 @@
use crate::Span;
//! AST types and walkers generated from the single Endor.asdl schema.
//! Runtime objects remain in the objects crate.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Name {
pub text: String,
pub span: Span,
mod generated;
pub mod visitor;
mod wire;
pub use generated::*;
pub use wire::AstError;
use crate::{SourceLocation, Span};
/// A syntax node's half-open byte range in its original UTF-8 source.
pub trait Spanned {
fn span(&self) -> Span;
fn source_text<'a>(&self, source: &'a str) -> Option<&'a str> {
self.span().slice(source)
}
fn location(&self, source: &str) -> Option<SourceLocation> {
self.span().location(source)
}
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Program {
pub functions: Vec<Function>,
pub span: Span,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Function {
pub name: Name,
pub parameters: Vec<Parameter>,
pub return_type: Name,
pub body: Block,
pub span: Span,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Parameter {
pub name: Name,
pub type_name: Name,
pub span: Span,
}
/// Basic draft function body: zero or more bindings and one final return.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Block {
pub bindings: Vec<Binding>,
pub result: Expr,
pub return_span: Span,
pub span: Span,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Binding {
pub name: Name,
pub value: Expr,
pub span: Span,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Expr {
pub kind: ExprKind,
pub span: Span,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ExprKind {
Name(String),
/// The decimal spelling is retained, without f64 conversion.
Number(String),
String(String),
Symbol(Name),
Unary {
operation: UnaryOp,
operand: Box<Expr>,
},
Binary {
operation: BinaryOp,
left: Box<Expr>,
right: Box<Expr>,
},
Member {
object: Box<Expr>,
name: Name,
},
Call {
callee: Box<Expr>,
arguments: Vec<Expr>,
},
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum UnaryOp {
Negate,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BinaryOp {
Add,
Subtract,
Multiply,
Divide,
impl Program {
/// Iterate the currently supported top-level function declarations.
pub fn functions(&self) -> impl Iterator<Item = &Function> {
self.declarations.iter().map(|declaration| {
let DeclKind::Function(function) = &declaration.kind;
function
})
}
}
+11
View File
@@ -0,0 +1,11 @@
# ASDL 生成的 AST 与接口
本目录只保存 frontend 的语法数据与遍历/交换接口,不定义运行时对象。
| 文件 | 职责 |
| --- | --- |
| [generated.rs](generated.rs) | 从 Endor.asdl 生成的节点类型、Serde 实现、Spanned 实现和 AST_SCHEMA_ID |
| [visitor.rs](visitor.rs) | 同源生成的可失败 Visitor、节点进入/离开钩子与 walk_* 默认遍历 |
| [wire.rs](wire.rs) | 手写 AstDocument JSON 接口、schema ID、范围与父子位置校验 |
唯一节点定义源是 [Endor.asdl](../../Endor.asdl),生成命令为 `python3 scripts/generate_ast.py --write`,在仓库根运行;--check 仅检查产物是否同步,不执行语言或测试。不要直接编辑 generated.rs/visitor.rs。公开入口见 [ast.rs](../ast.rs),完整契约见 [AST 模型](../../../../docs/runtime/ast.md)。返回 [frontend/src](../README.md)。
+220
View File
@@ -0,0 +1,220 @@
// Generated by scripts/generate_ast.py from Endor.asdl. Do not edit.
use super::Spanned;
use crate::Span;
pub const AST_SCHEMA_ID: &str =
"urn:endor:ast:sha256:c0da1a2640ad85f2c8e8159a10756e649d3aab179823a7cd890d9ebbb7f5d46e";
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct AstDocument {
pub schema: String,
pub root: AstRoot,
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(tag = "kind", content = "data", deny_unknown_fields)]
pub enum AstRoot {
Module(Program),
Expression(Expr),
Interactive(InteractiveInput),
}
impl Spanned for AstRoot {
fn span(&self) -> Span {
match self {
Self::Module(node) => node.span(),
Self::Expression(node) => node.span(),
Self::Interactive(node) => node.span(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Name {
pub text: String,
pub span: Span,
}
impl Spanned for Name {
fn span(&self) -> Span {
self.span
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Program {
pub declarations: Vec<Decl>,
pub span: Span,
}
impl Spanned for Program {
fn span(&self) -> Span {
self.span
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Decl {
pub kind: DeclKind,
pub span: Span,
}
impl Spanned for Decl {
fn span(&self) -> Span {
self.span
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(tag = "kind", content = "data", deny_unknown_fields)]
pub enum DeclKind {
Function(Function),
}
impl Spanned for DeclKind {
fn span(&self) -> Span {
match self {
Self::Function(node) => node.span(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Function {
pub name: Name,
pub parameters: Vec<Parameter>,
pub return_type: Name,
pub body: Block,
pub span: Span,
}
impl Spanned for Function {
fn span(&self) -> Span {
self.span
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Parameter {
pub name: Name,
pub type_name: Name,
pub span: Span,
}
impl Spanned for Parameter {
fn span(&self) -> Span {
self.span
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Block {
pub statements: Vec<Stmt>,
pub span: Span,
}
impl Spanned for Block {
fn span(&self) -> Span {
self.span
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Stmt {
pub kind: StmtKind,
pub span: Span,
}
impl Spanned for Stmt {
fn span(&self) -> Span {
self.span
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(tag = "kind", content = "data", deny_unknown_fields)]
pub enum StmtKind {
Binding(Binding),
Return(ReturnStatement),
Expression(Expr),
Declaration(Decl),
}
impl Spanned for StmtKind {
fn span(&self) -> Span {
match self {
Self::Binding(node) => node.span(),
Self::Return(node) => node.span(),
Self::Expression(node) => node.span(),
Self::Declaration(node) => node.span(),
}
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Binding {
pub name: Name,
pub value: Expr,
pub span: Span,
}
impl Spanned for Binding {
fn span(&self) -> Span {
self.span
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ReturnStatement {
pub value: Option<Expr>,
pub span: Span,
}
impl Spanned for ReturnStatement {
fn span(&self) -> Span {
self.span
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct InteractiveInput {
pub statements: Vec<Stmt>,
pub span: Span,
}
impl Spanned for InteractiveInput {
fn span(&self) -> Span {
self.span
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Expr {
pub kind: ExprKind,
pub span: Span,
}
impl Spanned for Expr {
fn span(&self) -> Span {
self.span
}
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(tag = "kind", content = "data", deny_unknown_fields)]
pub enum ExprKind {
Name(String),
Number(String),
String(String),
Symbol(Name),
Unary {
operation: UnaryOp,
operand: Box<Expr>,
},
Binary {
operation: BinaryOp,
left: Box<Expr>,
right: Box<Expr>,
},
Member {
object: Box<Expr>,
name: Name,
},
Call {
callee: Box<Expr>,
arguments: Vec<Expr>,
},
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, Copy)]
pub enum UnaryOp {
Negate,
}
#[derive(Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize, Copy)]
pub enum BinaryOp {
Add,
Subtract,
Multiply,
Divide,
}
+300
View File
@@ -0,0 +1,300 @@
// Generated by scripts/generate_ast.py from Endor.asdl. Do not edit.
use super::{
AstDocument, AstRoot, BinaryOp, Binding, Block, Decl, DeclKind, Expr, ExprKind, Function,
InteractiveInput, Name, Parameter, Program, ReturnStatement, Spanned, Stmt, StmtKind, UnaryOp,
};
use crate::Span;
pub trait Visitor {
type Error;
fn enter_node(&mut self, _span: Span) -> Result<(), Self::Error> {
Ok(())
}
fn leave_node(&mut self) {}
fn visit_span(&mut self, _span: &Span) -> Result<(), Self::Error> {
Ok(())
}
fn visit_ast_document(&mut self, node: &AstDocument) -> Result<(), Self::Error> {
walk_ast_document(self, node)
}
fn visit_ast_root(&mut self, node: &AstRoot) -> Result<(), Self::Error> {
self.enter_node(node.span())?;
let result = walk_ast_root(self, node);
self.leave_node();
result
}
fn visit_name(&mut self, node: &Name) -> Result<(), Self::Error> {
self.enter_node(node.span())?;
let result = walk_name(self, node);
self.leave_node();
result
}
fn visit_program(&mut self, node: &Program) -> Result<(), Self::Error> {
self.enter_node(node.span())?;
let result = walk_program(self, node);
self.leave_node();
result
}
fn visit_decl(&mut self, node: &Decl) -> Result<(), Self::Error> {
self.enter_node(node.span())?;
let result = walk_decl(self, node);
self.leave_node();
result
}
fn visit_decl_kind(&mut self, node: &DeclKind) -> Result<(), Self::Error> {
self.enter_node(node.span())?;
let result = walk_decl_kind(self, node);
self.leave_node();
result
}
fn visit_function(&mut self, node: &Function) -> Result<(), Self::Error> {
self.enter_node(node.span())?;
let result = walk_function(self, node);
self.leave_node();
result
}
fn visit_parameter(&mut self, node: &Parameter) -> Result<(), Self::Error> {
self.enter_node(node.span())?;
let result = walk_parameter(self, node);
self.leave_node();
result
}
fn visit_block(&mut self, node: &Block) -> Result<(), Self::Error> {
self.enter_node(node.span())?;
let result = walk_block(self, node);
self.leave_node();
result
}
fn visit_stmt(&mut self, node: &Stmt) -> Result<(), Self::Error> {
self.enter_node(node.span())?;
let result = walk_stmt(self, node);
self.leave_node();
result
}
fn visit_stmt_kind(&mut self, node: &StmtKind) -> Result<(), Self::Error> {
self.enter_node(node.span())?;
let result = walk_stmt_kind(self, node);
self.leave_node();
result
}
fn visit_binding(&mut self, node: &Binding) -> Result<(), Self::Error> {
self.enter_node(node.span())?;
let result = walk_binding(self, node);
self.leave_node();
result
}
fn visit_return_statement(&mut self, node: &ReturnStatement) -> Result<(), Self::Error> {
self.enter_node(node.span())?;
let result = walk_return_statement(self, node);
self.leave_node();
result
}
fn visit_interactive_input(&mut self, node: &InteractiveInput) -> Result<(), Self::Error> {
self.enter_node(node.span())?;
let result = walk_interactive_input(self, node);
self.leave_node();
result
}
fn visit_expr(&mut self, node: &Expr) -> Result<(), Self::Error> {
self.enter_node(node.span())?;
let result = walk_expr(self, node);
self.leave_node();
result
}
fn visit_expr_kind(&mut self, node: &ExprKind) -> Result<(), Self::Error> {
walk_expr_kind(self, node)
}
fn visit_unary_op(&mut self, node: &UnaryOp) -> Result<(), Self::Error> {
walk_unary_op(self, node)
}
fn visit_binary_op(&mut self, node: &BinaryOp) -> Result<(), Self::Error> {
walk_binary_op(self, node)
}
}
pub fn walk_ast_document<V: Visitor + ?Sized>(
visitor: &mut V,
node: &AstDocument,
) -> Result<(), V::Error> {
visitor.visit_ast_root(&node.root)?;
Ok(())
}
pub fn walk_ast_root<V: Visitor + ?Sized>(visitor: &mut V, node: &AstRoot) -> Result<(), V::Error> {
match node {
AstRoot::Module(value) => {
visitor.visit_program(value)?;
}
AstRoot::Expression(value) => {
visitor.visit_expr(value)?;
}
AstRoot::Interactive(value) => {
visitor.visit_interactive_input(value)?;
}
}
Ok(())
}
pub fn walk_name<V: Visitor + ?Sized>(visitor: &mut V, node: &Name) -> Result<(), V::Error> {
visitor.visit_span(&node.span)?;
Ok(())
}
pub fn walk_program<V: Visitor + ?Sized>(visitor: &mut V, node: &Program) -> Result<(), V::Error> {
for value in &node.declarations {
visitor.visit_decl(value)?;
}
visitor.visit_span(&node.span)?;
Ok(())
}
pub fn walk_decl<V: Visitor + ?Sized>(visitor: &mut V, node: &Decl) -> Result<(), V::Error> {
visitor.visit_decl_kind(&node.kind)?;
visitor.visit_span(&node.span)?;
Ok(())
}
pub fn walk_decl_kind<V: Visitor + ?Sized>(
visitor: &mut V,
node: &DeclKind,
) -> Result<(), V::Error> {
match node {
DeclKind::Function(value) => {
visitor.visit_function(value)?;
}
}
Ok(())
}
pub fn walk_function<V: Visitor + ?Sized>(
visitor: &mut V,
node: &Function,
) -> Result<(), V::Error> {
visitor.visit_name(&node.name)?;
for value in &node.parameters {
visitor.visit_parameter(value)?;
}
visitor.visit_name(&node.return_type)?;
visitor.visit_block(&node.body)?;
visitor.visit_span(&node.span)?;
Ok(())
}
pub fn walk_parameter<V: Visitor + ?Sized>(
visitor: &mut V,
node: &Parameter,
) -> Result<(), V::Error> {
visitor.visit_name(&node.name)?;
visitor.visit_name(&node.type_name)?;
visitor.visit_span(&node.span)?;
Ok(())
}
pub fn walk_block<V: Visitor + ?Sized>(visitor: &mut V, node: &Block) -> Result<(), V::Error> {
for value in &node.statements {
visitor.visit_stmt(value)?;
}
visitor.visit_span(&node.span)?;
Ok(())
}
pub fn walk_stmt<V: Visitor + ?Sized>(visitor: &mut V, node: &Stmt) -> Result<(), V::Error> {
visitor.visit_stmt_kind(&node.kind)?;
visitor.visit_span(&node.span)?;
Ok(())
}
pub fn walk_stmt_kind<V: Visitor + ?Sized>(
visitor: &mut V,
node: &StmtKind,
) -> Result<(), V::Error> {
match node {
StmtKind::Binding(value) => {
visitor.visit_binding(value)?;
}
StmtKind::Return(value) => {
visitor.visit_return_statement(value)?;
}
StmtKind::Expression(value) => {
visitor.visit_expr(value)?;
}
StmtKind::Declaration(value) => {
visitor.visit_decl(value)?;
}
}
Ok(())
}
pub fn walk_binding<V: Visitor + ?Sized>(visitor: &mut V, node: &Binding) -> Result<(), V::Error> {
visitor.visit_name(&node.name)?;
visitor.visit_expr(&node.value)?;
visitor.visit_span(&node.span)?;
Ok(())
}
pub fn walk_return_statement<V: Visitor + ?Sized>(
visitor: &mut V,
node: &ReturnStatement,
) -> Result<(), V::Error> {
if let Some(value) = &node.value {
visitor.visit_expr(value)?;
}
visitor.visit_span(&node.span)?;
Ok(())
}
pub fn walk_interactive_input<V: Visitor + ?Sized>(
visitor: &mut V,
node: &InteractiveInput,
) -> Result<(), V::Error> {
for value in &node.statements {
visitor.visit_stmt(value)?;
}
visitor.visit_span(&node.span)?;
Ok(())
}
pub fn walk_expr<V: Visitor + ?Sized>(visitor: &mut V, node: &Expr) -> Result<(), V::Error> {
visitor.visit_expr_kind(&node.kind)?;
visitor.visit_span(&node.span)?;
Ok(())
}
pub fn walk_expr_kind<V: Visitor + ?Sized>(
visitor: &mut V,
node: &ExprKind,
) -> Result<(), V::Error> {
match node {
ExprKind::Name(_) => {}
ExprKind::Number(_) => {}
ExprKind::String(_) => {}
ExprKind::Symbol(value) => {
visitor.visit_name(value)?;
}
ExprKind::Unary {
operation, operand, ..
} => {
visitor.visit_unary_op(operation)?;
visitor.visit_expr(operand)?;
}
ExprKind::Binary {
operation,
left,
right,
..
} => {
visitor.visit_binary_op(operation)?;
visitor.visit_expr(left)?;
visitor.visit_expr(right)?;
}
ExprKind::Member { object, name, .. } => {
visitor.visit_expr(object)?;
visitor.visit_name(name)?;
}
ExprKind::Call {
callee, arguments, ..
} => {
visitor.visit_expr(callee)?;
for value in arguments {
visitor.visit_expr(value)?;
}
}
}
Ok(())
}
pub fn walk_unary_op<V: Visitor + ?Sized>(
_visitor: &mut V,
_node: &UnaryOp,
) -> Result<(), V::Error> {
Ok(())
}
pub fn walk_binary_op<V: Visitor + ?Sized>(
_visitor: &mut V,
_node: &BinaryOp,
) -> Result<(), V::Error> {
Ok(())
}
+136
View File
@@ -0,0 +1,136 @@
use std::fmt;
use super::visitor::Visitor;
use super::{AST_SCHEMA_ID, AstDocument, AstRoot};
use crate::Span;
#[derive(Debug)]
pub enum AstError {
Json(serde_json::Error),
SchemaMismatch {
expected: &'static str,
found: String,
},
InvalidSpan(Span),
ChildOutsideParent {
parent: Span,
child: Span,
},
}
impl fmt::Display for AstError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Json(error) => write!(f, "invalid AST JSON: {error}"),
Self::SchemaMismatch { expected, found } => {
write!(f, "AST schema mismatch: expected {expected}, found {found}")
}
Self::InvalidSpan(span) => {
write!(f, "invalid AST source span: {}..{}", span.start, span.end)
}
Self::ChildOutsideParent { parent, child } => write!(
f,
"AST child span {}..{} is outside parent {}..{}",
child.start, child.end, parent.start, parent.end
),
}
}
}
impl std::error::Error for AstError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
Self::Json(error) => Some(error),
_ => None,
}
}
}
impl From<serde_json::Error> for AstError {
fn from(error: serde_json::Error) -> Self {
Self::Json(error)
}
}
impl AstDocument {
pub fn new(root: AstRoot) -> Result<Self, AstError> {
let document = Self {
schema: AST_SCHEMA_ID.into(),
root,
};
document.validate()?;
Ok(document)
}
/// Decode a versioned syntax tree. This does not compile or execute it.
pub fn from_json(text: &str) -> Result<Self, AstError> {
let document: Self = serde_json::from_str(text)?;
document.validate()?;
Ok(document)
}
pub fn to_json(&self) -> Result<String, AstError> {
self.validate()?;
Ok(serde_json::to_string(self)?)
}
/// Check schema identity, ranges, and parent/child containment.
pub fn validate(&self) -> Result<(), AstError> {
self.validate_with_source(None)
}
/// Additionally check bounds and UTF-8 boundaries against original text.
/// This does not check that the text and AST have identical semantics.
pub fn validate_source(&self, source: &str) -> Result<(), AstError> {
self.validate_with_source(Some(source))
}
fn validate_with_source(&self, source: Option<&str>) -> Result<(), AstError> {
if self.schema != AST_SCHEMA_ID {
return Err(AstError::SchemaMismatch {
expected: AST_SCHEMA_ID,
found: self.schema.clone(),
});
}
SpanValidator {
source,
parents: Vec::new(),
}
.visit_ast_root(&self.root)
}
}
struct SpanValidator<'a> {
source: Option<&'a str>,
parents: Vec<Span>,
}
impl Visitor for SpanValidator<'_> {
type Error = AstError;
fn enter_node(&mut self, span: Span) -> Result<(), AstError> {
// JSON offsets must also be exactly representable by editor JS numbers.
if span.start > span.end
|| span.end as u128 > 9_007_199_254_740_991
|| self
.source
.is_some_and(|source| span.slice(source).is_none())
{
return Err(AstError::InvalidSpan(span));
}
if let Some(parent) = self.parents.last().copied()
&& (span.start < parent.start || span.end > parent.end)
{
return Err(AstError::ChildOutsideParent {
parent,
child: span,
});
}
self.parents.push(span);
Ok(())
}
fn leave_node(&mut self) {
self.parents.pop();
}
}
+37 -22
View File
@@ -3,7 +3,8 @@ use std::fmt;
/// Half-open UTF-8 byte range in the original source, including trivia between
/// the first and last syntax token. EOF has an empty range at source.len().
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Serialize, serde::Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Span {
pub start: usize,
pub end: usize,
@@ -14,6 +15,16 @@ impl Span {
Self { start, end }
}
/// Safely read this range, rejecting reversed, out-of-bounds or split UTF-8 ranges.
pub fn slice(self, source: &str) -> Option<&str> {
source.get(self.start..self.end)
}
pub fn location(self, source: &str) -> Option<SourceLocation> {
self.slice(source)?;
Some(source_location(source, self.start))
}
pub(crate) fn through(self, last: Self) -> Self {
Self::new(self.start, last.end)
}
@@ -51,27 +62,7 @@ impl Diagnostic {
}
pub fn location(&self, source: &str) -> SourceLocation {
let mut location = SourceLocation { line: 1, column: 1 };
let mut previous_cr = false;
for (offset, character) in source.char_indices() {
if offset >= self.span.start {
break;
}
match character {
'\r' => {
location.line += 1;
location.column = 1;
}
'\n' if previous_cr => {}
'\n' => {
location.line += 1;
location.column = 1;
}
_ => location.column += 1,
}
previous_cr = character == '\r';
}
location
source_location(source, self.span.start)
}
}
@@ -86,3 +77,27 @@ impl fmt::Display for Diagnostic {
}
impl Error for Diagnostic {}
fn source_location(source: &str, position: usize) -> SourceLocation {
let mut location = SourceLocation { line: 1, column: 1 };
let mut previous_cr = false;
for (offset, character) in source.char_indices() {
if offset >= position {
break;
}
match character {
'\r' => {
location.line += 1;
location.column = 1;
}
'\n' if previous_cr => {}
'\n' => {
location.line += 1;
location.column = 1;
}
_ => location.column += 1,
}
previous_cr = character == '\r';
}
location
}
+27 -6
View File
@@ -42,9 +42,28 @@ peg::parser! {
= [Token { kind: TokenKind::Slash, .. }]
pub rule program() -> Program
= functions:function()+ {
let span = functions[0].span.through(functions.last().unwrap().span);
Program { functions, span }
= declarations:declaration()+ {
let span = declarations[0].span.through(declarations.last().unwrap().span);
Program { declarations, span }
}
rule declaration() -> Decl
= function:function() {
Decl { span: function.span, kind: DeclKind::Function(function) }
}
pub rule interactive() -> InteractiveInput
= statement:interactive_statement() {
InteractiveInput { span: statement.span, statements: vec![statement] }
}
rule interactive_statement() -> Stmt
= declaration:declaration() {
Stmt { span: declaration.span, kind: StmtKind::Declaration(declaration) }
}
/ binding:binding() { binding_statement(binding) }
/ expression:expression() {
Stmt { span: expression.span, kind: StmtKind::Expression(expression) }
}
rule function() -> Function
@@ -60,7 +79,9 @@ peg::parser! {
rule block() -> Block
= start:lbrace() bindings:binding()* returned:return_statement() end:rbrace() {
Block { bindings, result: returned.0, return_span: returned.1, span: start.through(end) }
let mut statements: Vec<Stmt> = bindings.into_iter().map(binding_statement).collect();
statements.push(returned);
Block { statements, span: start.through(end) }
}
rule binding() -> Binding
@@ -68,10 +89,10 @@ peg::parser! {
Binding { span: start.through(value.span), name, value }
}
rule return_statement() -> (Expr, Span)
rule return_statement() -> Stmt
= start:return_keyword() value:expression() {
let span = start.through(value.span);
(value, span)
Stmt { span, kind: StmtKind::Return(ReturnStatement { value: Some(value), span }) }
}
pub rule expression() -> Expr = precedence! {
+4 -1
View File
@@ -12,7 +12,10 @@ mod token;
pub use diagnostic::{Diagnostic, DiagnosticKind, SourceLocation, Span};
pub use lexer::{lex, lex_source};
pub use parser::{parse, parse_expression, parse_source, parse_source_expression};
pub use parser::{
ParseMode, parse, parse_ast, parse_expression, parse_source, parse_source_ast,
parse_source_expression,
};
pub use reader::{
EndorReader, EndorSourceProvider, ResolvedEndorSource, SourceReadError, SourceReadErrorKind,
};
+37 -1
View File
@@ -1,9 +1,38 @@
use crate::ast::{
BinaryOp, Binding, Block, Expr, ExprKind, Function, Name, Parameter, Program, UnaryOp,
AstRoot, BinaryOp, Binding, Block, Decl, DeclKind, Expr, ExprKind, Function, InteractiveInput,
Name, Parameter, Program, ReturnStatement, Stmt, StmtKind, UnaryOp,
};
use crate::lexer::decode_string;
use crate::{Diagnostic, DiagnosticKind, Span, Token, TokenKind, lex};
/// Select the syntax root without implying any execution or REPL session.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ParseMode {
Module,
Expression,
Interactive,
}
pub fn parse_ast(source: &str, mode: ParseMode) -> Result<AstRoot, Diagnostic> {
match mode {
ParseMode::Module => parse(source).map(AstRoot::Module),
ParseMode::Expression => parse_expression(source).map(AstRoot::Expression),
ParseMode::Interactive => {
let tokens = lex(source)?;
syntax::interactive(&tokens, source)
.map(AstRoot::Interactive)
.map_err(|error| diagnostic(error, &tokens, source.len()))
}
}
}
pub fn parse_source_ast(
source: &crate::SourceUnit,
mode: ParseMode,
) -> Result<AstRoot, crate::SourceDiagnostic> {
parse_ast(source.text(), mode).map_err(|error| source.annotate(error))
}
/// Parse a reader snapshot, retaining its origin and resolved revision on error.
pub fn parse_source(source: &crate::SourceUnit) -> Result<Program, crate::SourceDiagnostic> {
parse(source.text()).map_err(|error| source.annotate(error))
@@ -41,6 +70,13 @@ fn diagnostic(error: peg::error::ParseError<usize>, tokens: &[Token], end: usize
}
}
fn binding_statement(binding: Binding) -> Stmt {
Stmt {
span: binding.span,
kind: StmtKind::Binding(binding),
}
}
fn binary(left: Expr, operation: BinaryOp, right: Expr) -> Expr {
Expr {
span: left.span.through(right.span),