refactor: define lexical rules in a separate PEG grammar
Verify databases and deploy Aster VM / verify-and-deploy (push) Successful in 6m57s

This commit is contained in:
2026-10-19 13:56:05 +08:00
parent f77b8370ea
commit be4f1add58
18 changed files with 206 additions and 167 deletions
+3 -2
View File
@@ -7,9 +7,10 @@
| [reader.rs](reader.rs) | 字符串、流、文件读取、严格 UTF-8 解码与宿主 EndorSourceProvider 适配 |
| [token.rs](token.rs) | 可复制的 token 种类和原始源码范围,文本由源码切片取得 |
| [diagnostic.rs](diagnostic.rs) | UTF-8 字节范围、词法/语法诊断与一基行列转换 |
| [lexer.rs](lexer.rs) | 名称、数字、字符串、标点、空白与注释扫描;字符串校验及解码 |
| [lexer.rs](lexer.rs) | lex/lex_source 接口、源码游标、逐 Token 调用、位置/诊断换算与字符串解码 |
| [lexer.peg](lexer.peg) | 字符输入的 PEG 词法规则:名称、数字、字符串/转义、标点、空白/注释和错误识别 |
| [ast.rs](ast.rs) | 拥有文本的函数、参数、绑定、return、表达式与运算符结构 |
| [parser.rs](parser.rs) | 解析 API、错误映射、AST 辅助函数,以及独立语法文件的编译期引入 |
| [grammar.peg](grammar.peg) | token 输入的 PEG 规则、运算符优先级与 AST 构造动作;保留 peg::parser! 外层声明 |
改输入适配时同步 [阅读器契约](../../../docs/runtime/reader.md) 与读取错误检查;改词法时同步 token 和错误检查;改语法从 grammar.peg 进入,并同步 AST、[测试](../tests/README.md)、样例与 [前端契约](../../../docs/runtime/frontend.md)。不在语法动作中做 VM 分配、I/O 或修改外部状态,避免回溯造成副作用。返回 [frontend](../README.md)。
改输入适配时同步 [阅读器契约](../../../docs/runtime/reader.md) 与读取错误检查;改词法从 lexer.peg 进入,并同步 token、诊断和错误检查;改语法从 grammar.peg 进入,并同步 AST、[测试](../tests/README.md)、样例与 [前端契约](../../../docs/runtime/frontend.md)。词法与语法两份 PEG 动作均不做 VM 分配、I/O 或修改外部状态,避免回溯造成副作用。返回 [frontend](../README.md)。
+90
View File
@@ -0,0 +1,90 @@
peg::parser! {
grammar lexical() for str {
rule trivia()
= ([' ' | '\t' | '\r' | '\n'] / "//" [^'\r' | '\n']*)*
rule name_start()
= [character if character == '_' || character.is_alphabetic()]
rule name_continue()
= [character if character == '_' || character.is_alphanumeric()]
rule name() -> Result<TokenKind, Diagnostic>
= name_start() name_continue()* { Ok(TokenKind::Name) }
rule number() -> Result<TokenKind, Diagnostic>
= start:position!() ['0'..='9']+ ("." ['0'..='9']+)?
suffix:$(name_continue()*) end:position!() {
if suffix.is_empty() {
Ok(TokenKind::Number)
} else {
Err(Diagnostic::lexical(
Span::new(start, end),
"unsupported numeric suffix or missing token separator",
))
}
}
rule escape()
= "\\" ['n' | 't' | '"' | '\\']
rule string_character()
= [^'"' | '\\' | '\r' | '\n' | '\0']
rule string_end(start: usize) -> Result<(), Diagnostic>
= "\"" { Ok(()) }
/ position:position!() "\\" [_] end:position!() {
Err(Diagnostic::lexical(Span::new(position, end), "unsupported string escape"))
}
/ "\\" end:position!() ![_] {
Err(Diagnostic::lexical(Span::new(start, end), "unterminated string"))
}
/ position:position!() ['\r' | '\n' | '\0'] end:position!() {
Err(Diagnostic::lexical(Span::new(position, end), "unescaped newline or NUL in string"))
}
/ end:position!() ![_] {
Err(Diagnostic::lexical(Span::new(start, end), "unterminated string"))
}
rule string() -> Result<TokenKind, Diagnostic>
= start:position!() "\"" (escape() / string_character())* end:string_end(start) {
end.map(|()| TokenKind::String)
}
rule punctuation() -> TokenKind
= "(" { TokenKind::LeftParen }
/ ")" { TokenKind::RightParen }
/ "{" { TokenKind::LeftBrace }
/ "}" { TokenKind::RightBrace }
/ ":" { TokenKind::Colon }
/ "," { TokenKind::Comma }
/ "." { TokenKind::Dot }
/ "=" { TokenKind::Equal }
/ "+" { TokenKind::Plus }
/ "-" { TokenKind::Minus }
/ "*" { TokenKind::Star }
/ "/" { TokenKind::Slash }
rule kind() -> Result<TokenKind, Diagnostic>
= name()
/ number()
/ string()
/ kind:punctuation() { Ok(kind) }
/ start:position!() [_] end:position!() {
Err(Diagnostic::lexical(Span::new(start, end), "unexpected source character"))
}
rule token() -> Result<Token, Diagnostic>
= start:position!() kind:kind() end:position!() {
kind.map(|kind| Token { kind, span: Span::new(start, end) })
}
// Scan only one prefix. Errors are returned as values, so a matched
// malformed number/string cannot backtrack into another token kind.
#[no_eof]
pub rule next() -> (usize, Result<Option<Token>, Diagnostic>)
= trivia() result:(token:token() { token.map(Some) } / ![_] { Ok(None) }) end:position!() {
(end, result)
}
}
}
+30 -146
View File
@@ -10,59 +10,8 @@ pub fn lex_source(source: &crate::SourceUnit) -> Result<Vec<Token>, crate::Sourc
pub fn lex(source: &str) -> Result<Vec<Token>, Diagnostic> {
let mut lexer = Lexer { source, offset: 0 };
let mut tokens = Vec::new();
while let Some(character) = lexer.peek() {
if matches!(character, ' ' | '\t' | '\r' | '\n') {
let _ = lexer.advance();
continue;
}
if lexer.remaining().starts_with("//") {
while lexer
.peek()
.is_some_and(|character| !matches!(character, '\r' | '\n'))
{
let _ = lexer.advance();
}
continue;
}
let start = lexer.offset;
let kind = match character {
character if name_start(character) => {
let _ = lexer.advance();
while lexer.peek().is_some_and(name_continue) {
let _ = lexer.advance();
}
TokenKind::Name
}
'0'..='9' => lexer.number()?,
'"' => lexer.string()?,
_ => {
let _ = lexer.advance();
match character {
'(' => TokenKind::LeftParen,
')' => TokenKind::RightParen,
'{' => TokenKind::LeftBrace,
'}' => TokenKind::RightBrace,
':' => TokenKind::Colon,
',' => TokenKind::Comma,
'.' => TokenKind::Dot,
'=' => TokenKind::Equal,
'+' => TokenKind::Plus,
'-' => TokenKind::Minus,
'*' => TokenKind::Star,
'/' => TokenKind::Slash,
_ => {
return Err(Diagnostic::lexical(
Span::new(start, lexer.offset),
"unexpected source character",
));
}
}
}
};
tokens.push(Token {
kind,
span: Span::new(start, lexer.offset),
});
while let Some(token) = lexer.next()? {
tokens.push(token);
}
Ok(tokens)
}
@@ -72,100 +21,33 @@ struct Lexer<'source> {
offset: usize,
}
impl<'source> Lexer<'source> {
fn remaining(&self) -> &'source str {
&self.source[self.offset..]
impl Lexer<'_> {
fn next(&mut self) -> Result<Option<Token>, Diagnostic> {
let base = self.offset;
let (consumed, token) = lexical::next(&self.source[base..]).map_err(|error| {
let start = base + error.location.offset;
let end = start
+ self.source[start..]
.chars()
.next()
.map_or(0, char::len_utf8);
Diagnostic::lexical(Span::new(start, end), "unexpected source character")
})?;
self.offset += consumed;
token
.map(|token| {
token.map(|mut token| {
token.span.start += base;
token.span.end += base;
token
})
})
.map_err(|mut error| {
error.span.start += base;
error.span.end += base;
error
})
}
fn peek(&self) -> Option<char> {
self.remaining().chars().next()
}
fn advance(&mut self) -> Option<char> {
let character = self.peek()?;
self.offset += character.len_utf8();
Some(character)
}
fn digits(&mut self) {
while self
.peek()
.is_some_and(|character| character.is_ascii_digit())
{
let _ = self.advance();
}
}
fn number(&mut self) -> Result<TokenKind, Diagnostic> {
let start = self.offset;
self.digits();
let mut following = self.remaining().chars();
if following.next() == Some('.')
&& following
.next()
.is_some_and(|character| character.is_ascii_digit())
{
let _ = self.advance();
self.digits();
}
if self.peek().is_some_and(name_continue) {
while self.peek().is_some_and(name_continue) {
let _ = self.advance();
}
return Err(Diagnostic::lexical(
Span::new(start, self.offset),
"unsupported numeric suffix or missing token separator",
));
}
Ok(TokenKind::Number)
}
fn string(&mut self) -> Result<TokenKind, Diagnostic> {
let start = self.offset;
let _ = self.advance();
loop {
let position = self.offset;
match self.advance() {
Some('"') => return Ok(TokenKind::String),
Some('\\') => match self.advance() {
Some('n' | 't' | '"' | '\\') => {}
Some(_) => {
return Err(Diagnostic::lexical(
Span::new(position, self.offset),
"unsupported string escape",
));
}
None => {
return Err(Diagnostic::lexical(
Span::new(start, self.offset),
"unterminated string",
));
}
},
Some('\r' | '\n' | '\0') => {
return Err(Diagnostic::lexical(
Span::new(position, self.offset),
"unescaped newline or NUL in string",
));
}
None => {
return Err(Diagnostic::lexical(
Span::new(start, self.offset),
"unterminated string",
));
}
_ => {}
}
}
}
}
fn name_start(character: char) -> bool {
character == '_' || character.is_alphabetic()
}
fn name_continue(character: char) -> bool {
character == '_' || character.is_alphanumeric()
}
pub(crate) fn decode_string(raw: &str) -> String {
@@ -186,3 +68,5 @@ pub(crate) fn decode_string(raw: &str) -> String {
}
result
}
include!("lexer.peg");
+1 -1
View File
@@ -1,4 +1,4 @@
//! UTF-8 source reader, handwritten lexer and token-based rust-peg parser.
//! UTF-8 source reader and rust-peg rules for lexing and token-based parsing.
//! Produces syntax only; it does not resolve types, generate code or execute VM
//! operations. AST data owns its text and does not borrow the source buffer.