Add token-based PEG frontend and CLI source parsing
Verify databases and deploy Aster VM / verify-and-deploy (push) Successful in 7m30s

This commit is contained in:
2026-10-19 02:48:23 +08:00
parent e24909a56e
commit 0b3a480555
36 changed files with 1146 additions and 33 deletions
+278
View File
@@ -0,0 +1,278 @@
use frontend::ast::{BinaryOp, Expr, ExprKind, UnaryOp};
use frontend::{DiagnosticKind, Span, TokenKind, lex, parse, parse_expression};
fn binary(expression: &Expr, expected: BinaryOp) -> (&Expr, &Expr) {
let ExprKind::Binary {
operation,
left,
right,
} = &expression.kind
else {
panic!("expected a binary expression: {expression:?}");
};
assert_eq!(*operation, expected);
(left, right)
}
fn number(expression: &Expr, expected: &str) {
assert_eq!(expression.kind, ExprKind::Number(expected.into()));
}
#[test]
fn unicode_tokens_refer_to_exact_original_byte_ranges() {
let source = "// 日本語\r\nlet 商品_2 = \"こんにちは🌸\"";
let tokens = lex(source).unwrap();
assert_eq!(tokens.len(), 4);
assert_eq!(tokens[0].kind, TokenKind::Name);
assert_eq!(&source[tokens[0].span.start..tokens[0].span.end], "let");
assert_eq!(&source[tokens[1].span.start..tokens[1].span.end], "商品_2");
assert_eq!(tokens[2].kind, TokenKind::Equal);
assert_eq!(tokens[3].kind, TokenKind::String);
for token in tokens {
assert!(source.is_char_boundary(token.span.start));
assert!(source.is_char_boundary(token.span.end));
}
}
#[test]
fn decimal_spelling_and_minus_tokens_are_preserved_without_float_conversion() {
let source = "-000123456789012345678901234567890.00100 / 7 // comment";
let tokens = lex(source).unwrap();
assert_eq!(
tokens.iter().map(|token| token.kind).collect::<Vec<_>>(),
vec![
TokenKind::Minus,
TokenKind::Number,
TokenKind::Slash,
TokenKind::Number,
]
);
assert_eq!(
&source[tokens[1].span.start..tokens[1].span.end],
"000123456789012345678901234567890.00100"
);
let expression = parse_expression(source).unwrap();
let (negative, divisor) = binary(&expression, BinaryOp::Divide);
let ExprKind::Unary {
operation: UnaryOp::Negate,
operand,
} = &negative.kind
else {
panic!("expected unary minus");
};
number(operand, "000123456789012345678901234567890.00100");
number(divisor, "7");
}
#[test]
fn strings_decode_only_the_supported_escapes_and_keep_comments_inside_strings() {
let expression = parse_expression(r#""商品\n\t\"\\// text""#).unwrap();
assert_eq!(
expression.kind,
ExprKind::String("商品\n\t\"\\// text".into())
);
assert_eq!(
expression.span,
Span::new(0, r#""商品\n\t\"\\// text""#.len())
);
}
#[test]
fn malformed_strings_numbers_and_unknown_characters_report_lexical_errors() {
for source in [
"\"unfinished",
"\"bad\\q\"",
"\"bad\nline\"",
"\"bad\rline\"",
"\"bad\0\"",
"\"trailing\\",
"1e3",
"12name",
"1_000",
"'text'",
";",
"🌸",
"\0",
] {
let error = lex(source).unwrap_err();
assert_eq!(error.kind, DiagnosticKind::Lexical, "{source:?}");
assert!(error.span.start <= error.span.end);
assert!(error.span.end <= source.len());
assert!(source.is_char_boundary(error.span.start));
assert!(source.is_char_boundary(error.span.end));
}
let source = "\"日\\q\"";
let error = lex(source).unwrap_err();
assert_eq!(&source[error.span.start..error.span.end], "\\q");
assert_eq!(error.location(source).column, 3);
}
#[test]
fn arithmetic_precedence_and_left_associativity_form_the_expected_tree() {
let expression = parse_expression("1 - 2 - 3 * 4 / 5 + 6").unwrap();
let (subtract, last) = binary(&expression, BinaryOp::Add);
number(last, "6");
let (first_subtract, divide) = binary(subtract, BinaryOp::Subtract);
let (first, second) = binary(first_subtract, BinaryOp::Subtract);
number(first, "1");
number(second, "2");
let (multiply, fifth) = binary(divide, BinaryOp::Divide);
number(fifth, "5");
let (third, fourth) = binary(multiply, BinaryOp::Multiply);
number(third, "3");
number(fourth, "4");
let grouped = parse_expression("(1 + 2) * 3").unwrap();
let (sum, _) = binary(&grouped, BinaryOp::Multiply);
binary(sum, BinaryOp::Add);
assert_eq!(sum.span, Span::new(0, 7));
let nested = parse_expression("--1").unwrap();
let ExprKind::Unary { operand, .. } = nested.kind else {
panic!("expected outer negate");
};
assert!(matches!(
operand.kind,
ExprKind::Unary {
operation: UnaryOp::Negate,
..
}
));
}
#[test]
fn member_and_call_chains_bind_more_tightly_than_unary_minus() {
let expression = parse_expression("-factory(1).build(.title)(\"日\", 2)").unwrap();
let ExprKind::Unary {
operation: UnaryOp::Negate,
operand,
} = expression.kind
else {
panic!("expected negate");
};
let ExprKind::Call { callee, arguments } = operand.kind else {
panic!("expected last call");
};
assert_eq!(arguments.len(), 2);
assert_eq!(arguments[0].kind, ExprKind::String("日".into()));
let ExprKind::Call { callee, arguments } = callee.kind else {
panic!("expected build call");
};
let ExprKind::Symbol(name) = &arguments[0].kind else {
panic!("expected a symbol");
};
assert_eq!(name.text, "title");
let ExprKind::Member { object, name } = callee.kind else {
panic!("expected member");
};
assert_eq!(name.text, "build");
let ExprKind::Call { callee, arguments } = object.kind else {
panic!("expected factory call");
};
assert_eq!(callee.kind, ExprKind::Name("factory".into()));
number(&arguments[0], "1");
}
#[test]
fn parses_typed_functions_bindings_and_multiple_top_level_definitions() {
let example = parse(include_str!("../examples/basic.endor")).unwrap();
assert_eq!(example.functions.len(), 2);
let source = r#"
// Basic draft syntax, including a Unicode name.
function discountedPrice(price: Number, discount: Number): Number {
let amount = price - discount
return amount
}
function 表示(): String { return "こんにちは" }
"#;
let program = parse(source).unwrap();
assert_eq!(program.functions.len(), 2);
let function = &program.functions[0];
assert_eq!(function.name.text, "discountedPrice");
assert_eq!(function.parameters.len(), 2);
assert_eq!(function.parameters[0].type_name.text, "Number");
assert_eq!(function.return_type.text, "Number");
assert_eq!(function.body.bindings[0].name.text, "amount");
binary(&function.body.bindings[0].value, BinaryOp::Subtract);
assert_eq!(function.body.result.kind, ExprKind::Name("amount".into()));
assert_eq!(
&source[function.body.return_span.start..function.body.return_span.end],
"return amount"
);
assert_eq!(
program.functions[1].body.result.kind,
ExprKind::String("こんにちは".into())
);
assert_eq!(
&source[program.functions[1].span.start..program.functions[1].span.end],
"function 表示(): String { return \"こんにちは\" }"
);
// The output owns its text and can outlive the input buffer.
let owned = {
let source = String::from("function f(): String { return \"ok\" }");
parse(&source).unwrap()
};
assert_eq!(owned.functions[0].name.text, "f");
}
#[test]
fn syntax_words_are_contextual_and_block_layout_does_not_depend_on_newlines() {
let program =
parse("function class(function: Type): Type { let agent = function return agent }")
.unwrap();
assert_eq!(program.functions[0].name.text, "class");
assert_eq!(program.functions[0].parameters[0].name.text, "function");
assert_eq!(program.functions[0].body.bindings[0].name.text, "agent");
let multiline =
parse("function f\r\n(\r\nx: Number\r\n): Number\r\n{\r\nreturn x\r\n}").unwrap();
assert_eq!(
multiline.functions[0].body.result.kind,
ExprKind::Name("x".into())
);
}
#[test]
fn parser_requires_complete_input_and_rejects_unimplemented_or_malformed_syntax() {
for source in [
"",
"// empty",
"let x = 1",
"use atelier:shop.data",
"class Product {}",
"agent Worker {}",
"function f() { return 1 }",
"function f(x): Number { return x }",
"function f(x: Number,): Number { return x }",
"function f(): Number {}",
"function f(): Number { return 1 let x = 2 }",
"function f(): Number { let x = 1 }",
"function f(): Number { return 1 } extra",
"function f(): Number { function g(): Number { return 1 } return 2 }",
] {
assert!(parse(source).is_err(), "{source}");
}
for source in [
"", "x y", "1 +", "+1", ".5", "1.", "f(1,)", "f(,1)", "f(1 2)", "(1", "obj.", "[1]",
"a == b", "1.2.3",
] {
assert!(parse_expression(source).is_err(), "{source}");
}
}
#[test]
fn syntax_errors_map_token_indexes_to_utf8_source_and_eof_locations() {
let source = "function f(): String {\r\n let 商品 = \"日\"\r\n return )\r\n}";
let error = parse(source).unwrap_err();
assert_eq!(error.kind, DiagnosticKind::Syntax);
assert_eq!(&source[error.span.start..error.span.end], ")");
assert_eq!(error.location(source).line, 3);
assert_eq!(error.location(source).column, 10);
assert!(!error.expected.is_empty());
let source = "function f(): Number { return 1\r\n ";
let error = parse(source).unwrap_err();
assert_eq!(error.span, Span::new(source.len(), source.len()));
assert_eq!(error.location(source).line, 2);
assert_eq!(error.location(source).column, 3);
let source = "\r\n\r\n🌸";
let error = lex(source).unwrap_err();
assert_eq!(error.location(source).line, 3);
assert_eq!(error.location(source).column, 1);
}