feat: generate unified AST and editor contracts from ASDL
Verify databases and deploy Aster VM / verify-and-deploy (push) Successful in 7m55s
Verify databases and deploy Aster VM / verify-and-deploy (push) Successful in 7m55s
This commit is contained in:
@@ -2,6 +2,8 @@
|
||||
|
||||
[syntax.rs](syntax.rs) 覆盖 UTF-8 token 范围、中文/日文名称、数字原文、负号、字符串转义与错误、算术优先级和结合性、成员/调用链、函数与绑定、上下文名称、完整输入、非法/未支持语法以及 CRLF 和 EOF 诊断位置。
|
||||
|
||||
[ast.rs](ast.rs) 覆盖根模式、声明/语句结构、生成 Visitor 与错误退出、Spanned 位置、带 schema 的 JSON 往返及错误输入;同时核对编辑器产物标识。
|
||||
|
||||
[lexical.rs](lexical.rs) 专门覆盖字符 PEG 迁移后的原文位置换算、首个错误及详细诊断、Unicode 非法转义、未闭合字符串、注释/除法/成员/数字/字符串边界和空输入/尾随注释。
|
||||
|
||||
[reader.rs](reader.rs) 覆盖字符串/分块流/文件、严格 UTF-8、读取失败、URI provider、修订标签、旧缓冲区保留及带来源的诊断。样例也通过 syntax.rs 解析。只验证语法和 AST,不宣称类型检查或运行求值通过。随工作区检查在 ces-control 执行,本地不运行;入口见 [验证](../../../docs/operations/validation.md)。返回 [frontend](../README.md)。
|
||||
|
||||
@@ -0,0 +1,181 @@
|
||||
use frontend::ast::visitor::{Visitor, walk_expr, walk_function};
|
||||
use frontend::ast::{
|
||||
AST_SCHEMA_ID, AstDocument, AstError, AstRoot, Expr, ExprKind, Function, Spanned, StmtKind,
|
||||
};
|
||||
use frontend::{EndorReader, ParseMode, Span, parse_ast, parse_source_ast};
|
||||
|
||||
#[test]
|
||||
fn parse_modes_produce_distinct_roots_and_preserve_source_diagnostics() {
|
||||
let source = "function echo(value: String): String { let copy = value return copy }";
|
||||
let root = parse_ast(source, ParseMode::Module).unwrap();
|
||||
let AstRoot::Module(program) = &root else {
|
||||
panic!("expected module")
|
||||
};
|
||||
let function = program.functions().next().unwrap();
|
||||
assert_eq!(function.body.statements.len(), 2);
|
||||
assert!(matches!(
|
||||
function.body.statements[0].kind,
|
||||
StmtKind::Binding(_)
|
||||
));
|
||||
assert!(matches!(
|
||||
function.body.statements[1].kind,
|
||||
StmtKind::Return(_)
|
||||
));
|
||||
AstDocument::new(root)
|
||||
.unwrap()
|
||||
.validate_source(source)
|
||||
.unwrap();
|
||||
|
||||
let root = parse_ast("echo(\"日\")", ParseMode::Expression).unwrap();
|
||||
assert!(matches!(root, AstRoot::Expression(_)));
|
||||
for text in ["let value = \"日\"", "echo(value)", source] {
|
||||
let root = parse_ast(text, ParseMode::Interactive).unwrap();
|
||||
let AstRoot::Interactive(input) = &root else {
|
||||
panic!("expected interactive")
|
||||
};
|
||||
assert_eq!(input.statements.len(), 1);
|
||||
AstDocument::new(root)
|
||||
.unwrap()
|
||||
.validate_source(text)
|
||||
.unwrap();
|
||||
}
|
||||
assert!(parse_ast("let x = 1 let y = 2", ParseMode::Interactive).is_err());
|
||||
assert!(parse_ast("", ParseMode::Interactive).is_err());
|
||||
let source = EndorReader::from_text("<session:1>", "\"日\" + )");
|
||||
let error = parse_source_ast(&source, ParseMode::Expression).unwrap_err();
|
||||
assert_eq!(error.location.line, 1);
|
||||
assert_eq!(error.location.column, 7);
|
||||
assert!(error.to_string().starts_with("<session:1>:1:7:"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn generated_visitors_reach_names_and_unwind_node_hooks_on_error() {
|
||||
#[derive(Default)]
|
||||
struct Names {
|
||||
functions: Vec<String>,
|
||||
references: Vec<String>,
|
||||
depth: usize,
|
||||
stop: bool,
|
||||
}
|
||||
impl Visitor for Names {
|
||||
type Error = &'static str;
|
||||
fn enter_node(&mut self, _span: Span) -> Result<(), Self::Error> {
|
||||
self.depth += 1;
|
||||
Ok(())
|
||||
}
|
||||
fn leave_node(&mut self) {
|
||||
self.depth -= 1;
|
||||
}
|
||||
fn visit_function(&mut self, node: &Function) -> Result<(), Self::Error> {
|
||||
self.functions.push(node.name.text.clone());
|
||||
walk_function(self, node)
|
||||
}
|
||||
fn visit_expr(&mut self, node: &Expr) -> Result<(), Self::Error> {
|
||||
if let ExprKind::Name(name) = &node.kind {
|
||||
if self.stop {
|
||||
return Err("stopped at reference");
|
||||
}
|
||||
self.references.push(name.clone());
|
||||
}
|
||||
walk_expr(self, node)
|
||||
}
|
||||
}
|
||||
let root = parse_ast(
|
||||
"function echo(value: String): String { let copy = value return copy }",
|
||||
ParseMode::Module,
|
||||
)
|
||||
.unwrap();
|
||||
let mut names = Names::default();
|
||||
names.visit_ast_root(&root).unwrap();
|
||||
assert_eq!(names.functions, ["echo"]);
|
||||
assert_eq!(names.references, ["value", "copy"]);
|
||||
assert_eq!(names.depth, 0);
|
||||
let mut stopped = Names {
|
||||
stop: true,
|
||||
..Names::default()
|
||||
};
|
||||
assert_eq!(stopped.visit_ast_root(&root), Err("stopped at reference"));
|
||||
assert_eq!(stopped.depth, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn spans_share_unicode_locations_and_safe_original_source_access() {
|
||||
let source = " \r\n(値)";
|
||||
let root = parse_ast(source, ParseMode::Expression).unwrap();
|
||||
assert_eq!(root.source_text(source), Some("(値)"));
|
||||
let location = root.location(source).unwrap();
|
||||
assert_eq!((location.line, location.column), (2, 1));
|
||||
assert_eq!(Span::new(1, 3).slice("値"), None);
|
||||
assert_eq!(Span::new(2, 1).slice("値"), None);
|
||||
assert_eq!(Span::new(0, 4).location("値"), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn versioned_json_roundtrips_all_roots_and_preserves_number_spelling() {
|
||||
for (source, mode) in [
|
||||
(
|
||||
"function f(): Number { return 00012345678901234567890.00100 }",
|
||||
ParseMode::Module,
|
||||
),
|
||||
("f(.名前, \"日\\n\")", ParseMode::Expression),
|
||||
("let value = 1", ParseMode::Interactive),
|
||||
] {
|
||||
let document = AstDocument::new(parse_ast(source, mode).unwrap()).unwrap();
|
||||
let json = document.to_json().unwrap();
|
||||
let restored = AstDocument::from_json(&json).unwrap();
|
||||
assert_eq!(restored, document);
|
||||
restored.validate_source(source).unwrap();
|
||||
assert!(json.contains(AST_SCHEMA_ID));
|
||||
if mode == ParseMode::Module {
|
||||
assert!(json.contains("00012345678901234567890.00100"));
|
||||
}
|
||||
}
|
||||
let schema: serde_json::Value =
|
||||
serde_json::from_str(include_str!("../editor/ast.schema.json")).unwrap();
|
||||
assert_eq!(schema["$id"], AST_SCHEMA_ID);
|
||||
assert_eq!(
|
||||
schema["$defs"]["AstDocument"]["properties"]["schema"]["const"],
|
||||
AST_SCHEMA_ID
|
||||
);
|
||||
assert!(include_str!("../editor/ast.ts").contains(AST_SCHEMA_ID));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn json_interfaces_reject_unknown_fields_schema_changes_and_invalid_ranges() {
|
||||
let source = "(f(値))";
|
||||
let document = AstDocument::new(parse_ast(source, ParseMode::Expression).unwrap()).unwrap();
|
||||
let mut value = serde_json::to_value(&document).unwrap();
|
||||
value["schema"] = "urn:endor:ast:unknown".into();
|
||||
assert!(matches!(
|
||||
AstDocument::from_json(&value.to_string()),
|
||||
Err(AstError::SchemaMismatch { .. })
|
||||
));
|
||||
value = serde_json::to_value(&document).unwrap();
|
||||
value["root"]["data"]["unexpected"] = true.into();
|
||||
assert!(matches!(
|
||||
AstDocument::from_json(&value.to_string()),
|
||||
Err(AstError::Json(_))
|
||||
));
|
||||
value = serde_json::to_value(&document).unwrap();
|
||||
value["root"]["data"]["span"]["end"] = 1.into();
|
||||
assert!(matches!(
|
||||
AstDocument::from_json(&value.to_string()),
|
||||
Err(AstError::ChildOutsideParent { .. })
|
||||
));
|
||||
let AstRoot::Expression(mut expression) = document.root else {
|
||||
unreachable!()
|
||||
};
|
||||
expression.span = Span::new(5, 4);
|
||||
assert!(matches!(
|
||||
AstDocument::new(AstRoot::Expression(expression)),
|
||||
Err(AstError::InvalidSpan(_))
|
||||
));
|
||||
let root = AstRoot::Expression(Expr {
|
||||
kind: ExprKind::Name("値".into()),
|
||||
span: Span::new(1, 3),
|
||||
});
|
||||
assert!(matches!(
|
||||
AstDocument::new(root).unwrap().validate_source("値"),
|
||||
Err(AstError::InvalidSpan(_))
|
||||
));
|
||||
}
|
||||
@@ -1,6 +1,27 @@
|
||||
use frontend::ast::{BinaryOp, Expr, ExprKind, UnaryOp};
|
||||
use frontend::ast::{
|
||||
BinaryOp, Binding, DeclKind, Expr, ExprKind, Function, Program, StmtKind, UnaryOp,
|
||||
};
|
||||
use frontend::{DiagnosticKind, Span, TokenKind, lex, parse, parse_expression};
|
||||
|
||||
fn function(program: &Program, index: usize) -> &Function {
|
||||
let DeclKind::Function(function) = &program.declarations[index].kind;
|
||||
function
|
||||
}
|
||||
|
||||
fn binding(function: &Function, index: usize) -> &Binding {
|
||||
let StmtKind::Binding(binding) = &function.body.statements[index].kind else {
|
||||
panic!("expected binding")
|
||||
};
|
||||
binding
|
||||
}
|
||||
|
||||
fn result(function: &Function) -> &Expr {
|
||||
let StmtKind::Return(returned) = &function.body.statements.last().unwrap().kind else {
|
||||
panic!("expected return")
|
||||
};
|
||||
returned.value.as_ref().unwrap()
|
||||
}
|
||||
|
||||
fn binary(expression: &Expr, expected: BinaryOp) -> (&Expr, &Expr) {
|
||||
let ExprKind::Binary {
|
||||
operation,
|
||||
@@ -174,7 +195,7 @@ fn member_and_call_chains_bind_more_tightly_than_unary_minus() {
|
||||
#[test]
|
||||
fn parses_typed_functions_bindings_and_multiple_top_level_definitions() {
|
||||
let example = parse(include_str!("../examples/basic.endor")).unwrap();
|
||||
assert_eq!(example.functions.len(), 2);
|
||||
assert_eq!(example.declarations.len(), 2);
|
||||
let source = r#"
|
||||
// Basic draft syntax, including a Unicode name.
|
||||
function discountedPrice(price: Number, discount: Number): Number {
|
||||
@@ -184,25 +205,26 @@ fn parses_typed_functions_bindings_and_multiple_top_level_definitions() {
|
||||
function 表示(): String { return "こんにちは" }
|
||||
"#;
|
||||
let program = parse(source).unwrap();
|
||||
assert_eq!(program.functions.len(), 2);
|
||||
let function = &program.functions[0];
|
||||
assert_eq!(function.name.text, "discountedPrice");
|
||||
assert_eq!(function.parameters.len(), 2);
|
||||
assert_eq!(function.parameters[0].type_name.text, "Number");
|
||||
assert_eq!(function.return_type.text, "Number");
|
||||
assert_eq!(function.body.bindings[0].name.text, "amount");
|
||||
binary(&function.body.bindings[0].value, BinaryOp::Subtract);
|
||||
assert_eq!(function.body.result.kind, ExprKind::Name("amount".into()));
|
||||
assert_eq!(program.declarations.len(), 2);
|
||||
let first = function(&program, 0);
|
||||
assert_eq!(first.name.text, "discountedPrice");
|
||||
assert_eq!(first.parameters.len(), 2);
|
||||
assert_eq!(first.parameters[0].type_name.text, "Number");
|
||||
assert_eq!(first.return_type.text, "Number");
|
||||
assert_eq!(binding(first, 0).name.text, "amount");
|
||||
binary(&binding(first, 0).value, BinaryOp::Subtract);
|
||||
assert_eq!(result(first).kind, ExprKind::Name("amount".into()));
|
||||
assert_eq!(
|
||||
&source[function.body.return_span.start..function.body.return_span.end],
|
||||
&source[first.body.statements.last().unwrap().span.start
|
||||
..first.body.statements.last().unwrap().span.end],
|
||||
"return amount"
|
||||
);
|
||||
assert_eq!(
|
||||
program.functions[1].body.result.kind,
|
||||
result(function(&program, 1)).kind,
|
||||
ExprKind::String("こんにちは".into())
|
||||
);
|
||||
assert_eq!(
|
||||
&source[program.functions[1].span.start..program.functions[1].span.end],
|
||||
&source[function(&program, 1).span.start..function(&program, 1).span.end],
|
||||
"function 表示(): String { return \"こんにちは\" }"
|
||||
);
|
||||
// The output owns its text and can outlive the input buffer.
|
||||
@@ -210,7 +232,7 @@ fn parses_typed_functions_bindings_and_multiple_top_level_definitions() {
|
||||
let source = String::from("function f(): String { return \"ok\" }");
|
||||
parse(&source).unwrap()
|
||||
};
|
||||
assert_eq!(owned.functions[0].name.text, "f");
|
||||
assert_eq!(function(&owned, 0).name.text, "f");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -218,13 +240,13 @@ fn syntax_words_are_contextual_and_block_layout_does_not_depend_on_newlines() {
|
||||
let program =
|
||||
parse("function class(function: Type): Type { let agent = function return agent }")
|
||||
.unwrap();
|
||||
assert_eq!(program.functions[0].name.text, "class");
|
||||
assert_eq!(program.functions[0].parameters[0].name.text, "function");
|
||||
assert_eq!(program.functions[0].body.bindings[0].name.text, "agent");
|
||||
assert_eq!(function(&program, 0).name.text, "class");
|
||||
assert_eq!(function(&program, 0).parameters[0].name.text, "function");
|
||||
assert_eq!(binding(function(&program, 0), 0).name.text, "agent");
|
||||
let multiline =
|
||||
parse("function f\r\n(\r\nx: Number\r\n): Number\r\n{\r\nreturn x\r\n}").unwrap();
|
||||
assert_eq!(
|
||||
multiline.functions[0].body.result.kind,
|
||||
result(function(&multiline, 0)).kind,
|
||||
ExprKind::Name("x".into())
|
||||
);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user