refactor: define lexical rules in a separate PEG grammar
Verify databases and deploy Aster VM / verify-and-deploy (push) Successful in 6m57s
Verify databases and deploy Aster VM / verify-and-deploy (push) Successful in 6m57s
This commit is contained in:
@@ -7,9 +7,10 @@
|
||||
| [reader.rs](reader.rs) | 字符串、流、文件读取、严格 UTF-8 解码与宿主 EndorSourceProvider 适配 |
|
||||
| [token.rs](token.rs) | 可复制的 token 种类和原始源码范围,文本由源码切片取得 |
|
||||
| [diagnostic.rs](diagnostic.rs) | UTF-8 字节范围、词法/语法诊断与一基行列转换 |
|
||||
| [lexer.rs](lexer.rs) | 名称、数字、字符串、标点、空白与注释扫描;字符串校验及解码 |
|
||||
| [lexer.rs](lexer.rs) | lex/lex_source 接口、源码游标、逐 Token 调用、位置/诊断换算与字符串解码 |
|
||||
| [lexer.peg](lexer.peg) | 字符输入的 PEG 词法规则:名称、数字、字符串/转义、标点、空白/注释和错误识别 |
|
||||
| [ast.rs](ast.rs) | 拥有文本的函数、参数、绑定、return、表达式与运算符结构 |
|
||||
| [parser.rs](parser.rs) | 解析 API、错误映射、AST 辅助函数,以及独立语法文件的编译期引入 |
|
||||
| [grammar.peg](grammar.peg) | token 输入的 PEG 规则、运算符优先级与 AST 构造动作;保留 peg::parser! 外层声明 |
|
||||
|
||||
改输入适配时同步 [阅读器契约](../../../docs/runtime/reader.md) 与读取错误检查;改词法时同步 token 和错误检查;改语法从 grammar.peg 进入,并同步 AST、[测试](../tests/README.md)、样例与 [前端契约](../../../docs/runtime/frontend.md)。不在语法动作中做 VM 分配、I/O 或修改外部状态,避免回溯造成副作用。返回 [frontend](../README.md)。
|
||||
改输入适配时同步 [阅读器契约](../../../docs/runtime/reader.md) 与读取错误检查;改词法从 lexer.peg 进入,并同步 token、诊断和错误检查;改语法从 grammar.peg 进入,并同步 AST、[测试](../tests/README.md)、样例与 [前端契约](../../../docs/runtime/frontend.md)。词法与语法两份 PEG 动作均不做 VM 分配、I/O 或修改外部状态,避免回溯造成副作用。返回 [frontend](../README.md)。
|
||||
|
||||
@@ -0,0 +1,90 @@
|
||||
peg::parser! {
|
||||
grammar lexical() for str {
|
||||
rule trivia()
|
||||
= ([' ' | '\t' | '\r' | '\n'] / "//" [^'\r' | '\n']*)*
|
||||
|
||||
rule name_start()
|
||||
= [character if character == '_' || character.is_alphabetic()]
|
||||
|
||||
rule name_continue()
|
||||
= [character if character == '_' || character.is_alphanumeric()]
|
||||
|
||||
rule name() -> Result<TokenKind, Diagnostic>
|
||||
= name_start() name_continue()* { Ok(TokenKind::Name) }
|
||||
|
||||
rule number() -> Result<TokenKind, Diagnostic>
|
||||
= start:position!() ['0'..='9']+ ("." ['0'..='9']+)?
|
||||
suffix:$(name_continue()*) end:position!() {
|
||||
if suffix.is_empty() {
|
||||
Ok(TokenKind::Number)
|
||||
} else {
|
||||
Err(Diagnostic::lexical(
|
||||
Span::new(start, end),
|
||||
"unsupported numeric suffix or missing token separator",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
rule escape()
|
||||
= "\\" ['n' | 't' | '"' | '\\']
|
||||
|
||||
rule string_character()
|
||||
= [^'"' | '\\' | '\r' | '\n' | '\0']
|
||||
|
||||
rule string_end(start: usize) -> Result<(), Diagnostic>
|
||||
= "\"" { Ok(()) }
|
||||
/ position:position!() "\\" [_] end:position!() {
|
||||
Err(Diagnostic::lexical(Span::new(position, end), "unsupported string escape"))
|
||||
}
|
||||
/ "\\" end:position!() ![_] {
|
||||
Err(Diagnostic::lexical(Span::new(start, end), "unterminated string"))
|
||||
}
|
||||
/ position:position!() ['\r' | '\n' | '\0'] end:position!() {
|
||||
Err(Diagnostic::lexical(Span::new(position, end), "unescaped newline or NUL in string"))
|
||||
}
|
||||
/ end:position!() ![_] {
|
||||
Err(Diagnostic::lexical(Span::new(start, end), "unterminated string"))
|
||||
}
|
||||
|
||||
rule string() -> Result<TokenKind, Diagnostic>
|
||||
= start:position!() "\"" (escape() / string_character())* end:string_end(start) {
|
||||
end.map(|()| TokenKind::String)
|
||||
}
|
||||
|
||||
rule punctuation() -> TokenKind
|
||||
= "(" { TokenKind::LeftParen }
|
||||
/ ")" { TokenKind::RightParen }
|
||||
/ "{" { TokenKind::LeftBrace }
|
||||
/ "}" { TokenKind::RightBrace }
|
||||
/ ":" { TokenKind::Colon }
|
||||
/ "," { TokenKind::Comma }
|
||||
/ "." { TokenKind::Dot }
|
||||
/ "=" { TokenKind::Equal }
|
||||
/ "+" { TokenKind::Plus }
|
||||
/ "-" { TokenKind::Minus }
|
||||
/ "*" { TokenKind::Star }
|
||||
/ "/" { TokenKind::Slash }
|
||||
|
||||
rule kind() -> Result<TokenKind, Diagnostic>
|
||||
= name()
|
||||
/ number()
|
||||
/ string()
|
||||
/ kind:punctuation() { Ok(kind) }
|
||||
/ start:position!() [_] end:position!() {
|
||||
Err(Diagnostic::lexical(Span::new(start, end), "unexpected source character"))
|
||||
}
|
||||
|
||||
rule token() -> Result<Token, Diagnostic>
|
||||
= start:position!() kind:kind() end:position!() {
|
||||
kind.map(|kind| Token { kind, span: Span::new(start, end) })
|
||||
}
|
||||
|
||||
// Scan only one prefix. Errors are returned as values, so a matched
|
||||
// malformed number/string cannot backtrack into another token kind.
|
||||
#[no_eof]
|
||||
pub rule next() -> (usize, Result<Option<Token>, Diagnostic>)
|
||||
= trivia() result:(token:token() { token.map(Some) } / ![_] { Ok(None) }) end:position!() {
|
||||
(end, result)
|
||||
}
|
||||
}
|
||||
}
|
||||
+30
-146
@@ -10,59 +10,8 @@ pub fn lex_source(source: &crate::SourceUnit) -> Result<Vec<Token>, crate::Sourc
|
||||
pub fn lex(source: &str) -> Result<Vec<Token>, Diagnostic> {
|
||||
let mut lexer = Lexer { source, offset: 0 };
|
||||
let mut tokens = Vec::new();
|
||||
while let Some(character) = lexer.peek() {
|
||||
if matches!(character, ' ' | '\t' | '\r' | '\n') {
|
||||
let _ = lexer.advance();
|
||||
continue;
|
||||
}
|
||||
if lexer.remaining().starts_with("//") {
|
||||
while lexer
|
||||
.peek()
|
||||
.is_some_and(|character| !matches!(character, '\r' | '\n'))
|
||||
{
|
||||
let _ = lexer.advance();
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let start = lexer.offset;
|
||||
let kind = match character {
|
||||
character if name_start(character) => {
|
||||
let _ = lexer.advance();
|
||||
while lexer.peek().is_some_and(name_continue) {
|
||||
let _ = lexer.advance();
|
||||
}
|
||||
TokenKind::Name
|
||||
}
|
||||
'0'..='9' => lexer.number()?,
|
||||
'"' => lexer.string()?,
|
||||
_ => {
|
||||
let _ = lexer.advance();
|
||||
match character {
|
||||
'(' => TokenKind::LeftParen,
|
||||
')' => TokenKind::RightParen,
|
||||
'{' => TokenKind::LeftBrace,
|
||||
'}' => TokenKind::RightBrace,
|
||||
':' => TokenKind::Colon,
|
||||
',' => TokenKind::Comma,
|
||||
'.' => TokenKind::Dot,
|
||||
'=' => TokenKind::Equal,
|
||||
'+' => TokenKind::Plus,
|
||||
'-' => TokenKind::Minus,
|
||||
'*' => TokenKind::Star,
|
||||
'/' => TokenKind::Slash,
|
||||
_ => {
|
||||
return Err(Diagnostic::lexical(
|
||||
Span::new(start, lexer.offset),
|
||||
"unexpected source character",
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
tokens.push(Token {
|
||||
kind,
|
||||
span: Span::new(start, lexer.offset),
|
||||
});
|
||||
while let Some(token) = lexer.next()? {
|
||||
tokens.push(token);
|
||||
}
|
||||
Ok(tokens)
|
||||
}
|
||||
@@ -72,100 +21,33 @@ struct Lexer<'source> {
|
||||
offset: usize,
|
||||
}
|
||||
|
||||
impl<'source> Lexer<'source> {
|
||||
fn remaining(&self) -> &'source str {
|
||||
&self.source[self.offset..]
|
||||
impl Lexer<'_> {
|
||||
fn next(&mut self) -> Result<Option<Token>, Diagnostic> {
|
||||
let base = self.offset;
|
||||
let (consumed, token) = lexical::next(&self.source[base..]).map_err(|error| {
|
||||
let start = base + error.location.offset;
|
||||
let end = start
|
||||
+ self.source[start..]
|
||||
.chars()
|
||||
.next()
|
||||
.map_or(0, char::len_utf8);
|
||||
Diagnostic::lexical(Span::new(start, end), "unexpected source character")
|
||||
})?;
|
||||
self.offset += consumed;
|
||||
token
|
||||
.map(|token| {
|
||||
token.map(|mut token| {
|
||||
token.span.start += base;
|
||||
token.span.end += base;
|
||||
token
|
||||
})
|
||||
})
|
||||
.map_err(|mut error| {
|
||||
error.span.start += base;
|
||||
error.span.end += base;
|
||||
error
|
||||
})
|
||||
}
|
||||
|
||||
fn peek(&self) -> Option<char> {
|
||||
self.remaining().chars().next()
|
||||
}
|
||||
|
||||
fn advance(&mut self) -> Option<char> {
|
||||
let character = self.peek()?;
|
||||
self.offset += character.len_utf8();
|
||||
Some(character)
|
||||
}
|
||||
|
||||
fn digits(&mut self) {
|
||||
while self
|
||||
.peek()
|
||||
.is_some_and(|character| character.is_ascii_digit())
|
||||
{
|
||||
let _ = self.advance();
|
||||
}
|
||||
}
|
||||
|
||||
fn number(&mut self) -> Result<TokenKind, Diagnostic> {
|
||||
let start = self.offset;
|
||||
self.digits();
|
||||
let mut following = self.remaining().chars();
|
||||
if following.next() == Some('.')
|
||||
&& following
|
||||
.next()
|
||||
.is_some_and(|character| character.is_ascii_digit())
|
||||
{
|
||||
let _ = self.advance();
|
||||
self.digits();
|
||||
}
|
||||
if self.peek().is_some_and(name_continue) {
|
||||
while self.peek().is_some_and(name_continue) {
|
||||
let _ = self.advance();
|
||||
}
|
||||
return Err(Diagnostic::lexical(
|
||||
Span::new(start, self.offset),
|
||||
"unsupported numeric suffix or missing token separator",
|
||||
));
|
||||
}
|
||||
Ok(TokenKind::Number)
|
||||
}
|
||||
|
||||
fn string(&mut self) -> Result<TokenKind, Diagnostic> {
|
||||
let start = self.offset;
|
||||
let _ = self.advance();
|
||||
loop {
|
||||
let position = self.offset;
|
||||
match self.advance() {
|
||||
Some('"') => return Ok(TokenKind::String),
|
||||
Some('\\') => match self.advance() {
|
||||
Some('n' | 't' | '"' | '\\') => {}
|
||||
Some(_) => {
|
||||
return Err(Diagnostic::lexical(
|
||||
Span::new(position, self.offset),
|
||||
"unsupported string escape",
|
||||
));
|
||||
}
|
||||
None => {
|
||||
return Err(Diagnostic::lexical(
|
||||
Span::new(start, self.offset),
|
||||
"unterminated string",
|
||||
));
|
||||
}
|
||||
},
|
||||
Some('\r' | '\n' | '\0') => {
|
||||
return Err(Diagnostic::lexical(
|
||||
Span::new(position, self.offset),
|
||||
"unescaped newline or NUL in string",
|
||||
));
|
||||
}
|
||||
None => {
|
||||
return Err(Diagnostic::lexical(
|
||||
Span::new(start, self.offset),
|
||||
"unterminated string",
|
||||
));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn name_start(character: char) -> bool {
|
||||
character == '_' || character.is_alphabetic()
|
||||
}
|
||||
|
||||
fn name_continue(character: char) -> bool {
|
||||
character == '_' || character.is_alphanumeric()
|
||||
}
|
||||
|
||||
pub(crate) fn decode_string(raw: &str) -> String {
|
||||
@@ -186,3 +68,5 @@ pub(crate) fn decode_string(raw: &str) -> String {
|
||||
}
|
||||
result
|
||||
}
|
||||
|
||||
include!("lexer.peg");
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! UTF-8 source reader, handwritten lexer and token-based rust-peg parser.
|
||||
//! UTF-8 source reader and rust-peg rules for lexing and token-based parsing.
|
||||
//! Produces syntax only; it does not resolve types, generate code or execute VM
|
||||
//! operations. AST data owns its text and does not borrow the source buffer.
|
||||
|
||||
|
||||
Reference in New Issue
Block a user