Add token-based PEG frontend and CLI source parsing
Verify databases and deploy Aster VM / verify-and-deploy (push) Successful in 7m30s

This commit is contained in:
2026-10-19 02:48:23 +08:00
parent e24909a56e
commit 0b3a480555
36 changed files with 1146 additions and 33 deletions
+12
View File
@@ -0,0 +1,12 @@
# Frontend 源码
| 文件 | 职责 |
| --- | --- |
| [lib.rs](lib.rs) | 公开 AST 模块、lex/parse/parse_expression 与 token、诊断导出 |
| [token.rs](token.rs) | 可复制的 token 种类和原始源码范围,文本由源码切片取得 |
| [diagnostic.rs](diagnostic.rs) | UTF-8 字节范围、词法/语法诊断与一基行列转换 |
| [lexer.rs](lexer.rs) | 名称、数字、字符串、标点、空白与注释扫描;字符串校验及解码 |
| [ast.rs](ast.rs) | 拥有文本的函数、参数、绑定、return、表达式与运算符结构 |
| [parser.rs](parser.rs) | token 输入的 peg::parser! 规则、优先级、AST 构造及错误映射 |
改词法时同步 token 和错误检查;改语法时同步 AST、PEG 规则、[测试](../tests/README.md)、样例与 [前端契约](../../../docs/runtime/frontend.md)。不在语法动作中做 VM 分配、I/O 或修改外部状态,避免回溯造成副作用。返回 [frontend](../README.md)。
+90
View File
@@ -0,0 +1,90 @@
use crate::Span;
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Name {
pub text: String,
pub span: Span,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Program {
pub functions: Vec<Function>,
pub span: Span,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Function {
pub name: Name,
pub parameters: Vec<Parameter>,
pub return_type: Name,
pub body: Block,
pub span: Span,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Parameter {
pub name: Name,
pub type_name: Name,
pub span: Span,
}
/// Basic draft function body: zero or more bindings and one final return.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Block {
pub bindings: Vec<Binding>,
pub result: Expr,
pub return_span: Span,
pub span: Span,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Binding {
pub name: Name,
pub value: Expr,
pub span: Span,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Expr {
pub kind: ExprKind,
pub span: Span,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ExprKind {
Name(String),
/// The decimal spelling is retained, without f64 conversion.
Number(String),
String(String),
Symbol(Name),
Unary {
operation: UnaryOp,
operand: Box<Expr>,
},
Binary {
operation: BinaryOp,
left: Box<Expr>,
right: Box<Expr>,
},
Member {
object: Box<Expr>,
name: Name,
},
Call {
callee: Box<Expr>,
arguments: Vec<Expr>,
},
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum UnaryOp {
Negate,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum BinaryOp {
Add,
Subtract,
Multiply,
Divide,
}
+88
View File
@@ -0,0 +1,88 @@
use std::error::Error;
use std::fmt;
/// Half-open UTF-8 byte range in the original source, including trivia between
/// the first and last syntax token. EOF has an empty range at source.len().
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct Span {
pub start: usize,
pub end: usize,
}
impl Span {
pub const fn new(start: usize, end: usize) -> Self {
Self { start, end }
}
pub(crate) fn through(self, last: Self) -> Self {
Self::new(self.start, last.end)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum DiagnosticKind {
Lexical,
Syntax,
}
/// Lines and Unicode scalar columns are one-based. CRLF counts as one newline.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct SourceLocation {
pub line: usize,
pub column: usize,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Diagnostic {
pub kind: DiagnosticKind,
pub span: Span,
pub message: String,
pub expected: Vec<String>,
}
impl Diagnostic {
pub(crate) fn lexical(span: Span, message: impl Into<String>) -> Self {
Self {
kind: DiagnosticKind::Lexical,
span,
message: message.into(),
expected: Vec::new(),
}
}
pub fn location(&self, source: &str) -> SourceLocation {
let mut location = SourceLocation { line: 1, column: 1 };
let mut previous_cr = false;
for (offset, character) in source.char_indices() {
if offset >= self.span.start {
break;
}
match character {
'\r' => {
location.line += 1;
location.column = 1;
}
'\n' if previous_cr => {}
'\n' => {
location.line += 1;
location.column = 1;
}
_ => location.column += 1,
}
previous_cr = character == '\r';
}
location
}
}
impl fmt::Display for Diagnostic {
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
formatter,
"{} at bytes {}..{}",
self.message, self.span.start, self.span.end
)
}
}
impl Error for Diagnostic {}
+183
View File
@@ -0,0 +1,183 @@
use crate::{Diagnostic, Span, Token, TokenKind};
/// Lex the basic Endorphin draft syntax without reading files or changing
/// locale. Positions always refer to bytes in this exact UTF-8 source.
pub fn lex(source: &str) -> Result<Vec<Token>, Diagnostic> {
let mut lexer = Lexer { source, offset: 0 };
let mut tokens = Vec::new();
while let Some(character) = lexer.peek() {
if matches!(character, ' ' | '\t' | '\r' | '\n') {
let _ = lexer.advance();
continue;
}
if lexer.remaining().starts_with("//") {
while lexer
.peek()
.is_some_and(|character| !matches!(character, '\r' | '\n'))
{
let _ = lexer.advance();
}
continue;
}
let start = lexer.offset;
let kind = match character {
character if name_start(character) => {
let _ = lexer.advance();
while lexer.peek().is_some_and(name_continue) {
let _ = lexer.advance();
}
TokenKind::Name
}
'0'..='9' => lexer.number()?,
'"' => lexer.string()?,
_ => {
let _ = lexer.advance();
match character {
'(' => TokenKind::LeftParen,
')' => TokenKind::RightParen,
'{' => TokenKind::LeftBrace,
'}' => TokenKind::RightBrace,
':' => TokenKind::Colon,
',' => TokenKind::Comma,
'.' => TokenKind::Dot,
'=' => TokenKind::Equal,
'+' => TokenKind::Plus,
'-' => TokenKind::Minus,
'*' => TokenKind::Star,
'/' => TokenKind::Slash,
_ => {
return Err(Diagnostic::lexical(
Span::new(start, lexer.offset),
"unexpected source character",
));
}
}
}
};
tokens.push(Token {
kind,
span: Span::new(start, lexer.offset),
});
}
Ok(tokens)
}
struct Lexer<'source> {
source: &'source str,
offset: usize,
}
impl<'source> Lexer<'source> {
fn remaining(&self) -> &'source str {
&self.source[self.offset..]
}
fn peek(&self) -> Option<char> {
self.remaining().chars().next()
}
fn advance(&mut self) -> Option<char> {
let character = self.peek()?;
self.offset += character.len_utf8();
Some(character)
}
fn digits(&mut self) {
while self
.peek()
.is_some_and(|character| character.is_ascii_digit())
{
let _ = self.advance();
}
}
fn number(&mut self) -> Result<TokenKind, Diagnostic> {
let start = self.offset;
self.digits();
let mut following = self.remaining().chars();
if following.next() == Some('.')
&& following
.next()
.is_some_and(|character| character.is_ascii_digit())
{
let _ = self.advance();
self.digits();
}
if self.peek().is_some_and(name_continue) {
while self.peek().is_some_and(name_continue) {
let _ = self.advance();
}
return Err(Diagnostic::lexical(
Span::new(start, self.offset),
"unsupported numeric suffix or missing token separator",
));
}
Ok(TokenKind::Number)
}
fn string(&mut self) -> Result<TokenKind, Diagnostic> {
let start = self.offset;
let _ = self.advance();
loop {
let position = self.offset;
match self.advance() {
Some('"') => return Ok(TokenKind::String),
Some('\\') => match self.advance() {
Some('n' | 't' | '"' | '\\') => {}
Some(_) => {
return Err(Diagnostic::lexical(
Span::new(position, self.offset),
"unsupported string escape",
));
}
None => {
return Err(Diagnostic::lexical(
Span::new(start, self.offset),
"unterminated string",
));
}
},
Some('\r' | '\n' | '\0') => {
return Err(Diagnostic::lexical(
Span::new(position, self.offset),
"unescaped newline or NUL in string",
));
}
None => {
return Err(Diagnostic::lexical(
Span::new(start, self.offset),
"unterminated string",
));
}
_ => {}
}
}
}
}
fn name_start(character: char) -> bool {
character == '_' || character.is_alphabetic()
}
fn name_continue(character: char) -> bool {
character == '_' || character.is_alphanumeric()
}
pub(crate) fn decode_string(raw: &str) -> String {
let mut result = String::new();
let mut characters = raw.chars();
while let Some(character) = characters.next() {
if character == '\\' {
result.push(match characters.next().expect("lexer validated escape") {
'n' => '\n',
't' => '\t',
'"' => '"',
'\\' => '\\',
_ => unreachable!("lexer validated escape"),
});
} else {
result.push(character);
}
}
result
}
+14
View File
@@ -0,0 +1,14 @@
//! UTF-8 source frontend: a handwritten lexer and token-based rust-peg parser.
//! Produces syntax only; it does not resolve types, generate code or execute VM
//! operations. AST data owns its text and does not borrow the source buffer.
pub mod ast;
mod diagnostic;
mod lexer;
mod parser;
mod token;
pub use diagnostic::{Diagnostic, DiagnosticKind, SourceLocation, Span};
pub use lexer::lex;
pub use parser::{parse, parse_expression};
pub use token::{Token, TokenKind};
+182
View File
@@ -0,0 +1,182 @@
use crate::ast::{
BinaryOp, Binding, Block, Expr, ExprKind, Function, Name, Parameter, Program, UnaryOp,
};
use crate::lexer::decode_string;
use crate::{Diagnostic, DiagnosticKind, Span, Token, TokenKind, lex};
/// Parse one or more basic typed function declarations. This is a syntax
/// operation: type names and name references are retained without resolution.
pub fn parse(source: &str) -> Result<Program, Diagnostic> {
let tokens = lex(source)?;
syntax::program(&tokens, source).map_err(|error| diagnostic(error, &tokens, source.len()))
}
/// Parse exactly one complete expression, useful independently of declarations.
pub fn parse_expression(source: &str) -> Result<Expr, Diagnostic> {
let tokens = lex(source)?;
syntax::expression(&tokens, source).map_err(|error| diagnostic(error, &tokens, source.len()))
}
fn diagnostic(error: peg::error::ParseError<usize>, tokens: &[Token], end: usize) -> Diagnostic {
let span = tokens
.get(error.location)
.map_or(Span::new(end, end), |token| token.span);
let expected: Vec<String> = error.expected.tokens().map(str::to_owned).collect();
Diagnostic {
kind: DiagnosticKind::Syntax,
span,
message: format!("expected {}", expected.join(" or ")),
expected,
}
}
fn binary(left: Expr, operation: BinaryOp, right: Expr) -> Expr {
Expr {
span: left.span.through(right.span),
kind: ExprKind::Binary {
operation,
left: Box::new(left),
right: Box::new(right),
},
}
}
enum Suffix {
Member(Name),
Call(Vec<Expr>, Span),
}
fn postfix(expression: Expr, suffix: Suffix) -> Expr {
match suffix {
Suffix::Member(name) => Expr {
span: expression.span.through(name.span),
kind: ExprKind::Member {
object: Box::new(expression),
name,
},
},
Suffix::Call(arguments, end) => Expr {
span: expression.span.through(end),
kind: ExprKind::Call {
callee: Box::new(expression),
arguments,
},
},
}
}
peg::parser! {
grammar syntax(source: &str) for [Token] {
rule keyword(word: &str) -> Span
= [Token { kind: TokenKind::Name, span } if &source[span.start..span.end] == word] { span }
rule function_keyword() -> Span
= quiet!{keyword("function")} / expected!("'function'")
rule let_keyword() -> Span
= quiet!{keyword("let")} / expected!("'let'")
rule return_keyword() -> Span
= quiet!{keyword("return")} / expected!("'return'")
rule name() -> Name
= span:quiet!{[Token { kind: TokenKind::Name, span }] { span }} {
Name { text: source[span.start..span.end].into(), span }
}
/ expected!("identifier")
rule lparen() -> Span
= quiet!{[Token { kind: TokenKind::LeftParen, span }] { span }} / expected!("'('")
rule rparen() -> Span
= quiet!{[Token { kind: TokenKind::RightParen, span }] { span }} / expected!("')'")
rule lbrace() -> Span
= quiet!{[Token { kind: TokenKind::LeftBrace, span }] { span }} / expected!("'{'")
rule rbrace() -> Span
= quiet!{[Token { kind: TokenKind::RightBrace, span }] { span }} / expected!("'}'")
rule colon()
= quiet!{[Token { kind: TokenKind::Colon, .. }]} / expected!("':'")
rule comma()
= quiet!{[Token { kind: TokenKind::Comma, .. }]} / expected!("','")
rule equal()
= quiet!{[Token { kind: TokenKind::Equal, .. }]} / expected!("'='")
rule dot() -> Span
= quiet!{[Token { kind: TokenKind::Dot, span }] { span }} / expected!("'.'")
rule plus()
= [Token { kind: TokenKind::Plus, .. }]
rule minus() -> Span
= [Token { kind: TokenKind::Minus, span }] { span }
rule star()
= [Token { kind: TokenKind::Star, .. }]
rule slash()
= [Token { kind: TokenKind::Slash, .. }]
pub rule program() -> Program
= functions:function()+ {
let span = functions[0].span.through(functions.last().unwrap().span);
Program { functions, span }
}
rule function() -> Function
= start:function_keyword() name:name() lparen()
parameters:(parameter() ** comma()) rparen() colon() return_type:name() body:block() {
Function { span: start.through(body.span), name, parameters, return_type, body }
}
rule parameter() -> Parameter
= name:name() colon() type_name:name() {
Parameter { span: name.span.through(type_name.span), name, type_name }
}
rule block() -> Block
= start:lbrace() bindings:binding()* returned:return_statement() end:rbrace() {
Block { bindings, result: returned.0, return_span: returned.1, span: start.through(end) }
}
rule binding() -> Binding
= start:let_keyword() name:name() equal() value:expression() {
Binding { span: start.through(value.span), name, value }
}
rule return_statement() -> (Expr, Span)
= start:return_keyword() value:expression() {
let span = start.through(value.span);
(value, span)
}
pub rule expression() -> Expr = precedence! {
left:(@) plus() right:@ { binary(left, BinaryOp::Add, right) }
left:(@) minus() right:@ { binary(left, BinaryOp::Subtract, right) }
--
left:(@) star() right:@ { binary(left, BinaryOp::Multiply, right) }
left:(@) slash() right:@ { binary(left, BinaryOp::Divide, right) }
--
start:minus() operand:@ {
Expr { span: start.through(operand.span), kind: ExprKind::Unary { operation: UnaryOp::Negate, operand: Box::new(operand) } }
}
--
expression:postfix_expression() { expression }
}
rule postfix_expression() -> Expr
= expression:primary() suffixes:suffix()* {
suffixes.into_iter().fold(expression, postfix)
}
rule suffix() -> Suffix
= dot() name:name() { Suffix::Member(name) }
/ lparen() arguments:(expression() ** comma()) end:rparen() { Suffix::Call(arguments, end) }
rule primary() -> Expr
= start:lparen() expression:expression() end:rparen() {
Expr { span: start.through(end), ..expression }
}
/ start:dot() name:name() {
Expr { span: start.through(name.span), kind: ExprKind::Symbol(name) }
}
/ [Token { kind: TokenKind::Number, span }] {
Expr { span, kind: ExprKind::Number(source[span.start..span.end].into()) }
}
/ [Token { kind: TokenKind::String, span }] {
Expr { span, kind: ExprKind::String(decode_string(&source[span.start + 1..span.end - 1])) }
}
/ name:name() { Expr { span: name.span, kind: ExprKind::Name(name.text) } }
}
}
+29
View File
@@ -0,0 +1,29 @@
use crate::Span;
/// Text is located by span in the original source. Names stay contextual: the
/// lexer does not turn syntax words into globally reserved keywords.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum TokenKind {
Name,
Number,
/// Span includes quotes. Escapes have been checked, not decoded.
String,
LeftParen,
RightParen,
LeftBrace,
RightBrace,
Colon,
Comma,
Dot,
Equal,
Plus,
Minus,
Star,
Slash,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct Token {
pub kind: TokenKind,
pub span: Span,
}