refactor: define lexical rules in a separate PEG grammar
Verify databases and deploy Aster VM / verify-and-deploy (push) Successful in 6m57s
Verify databases and deploy Aster VM / verify-and-deploy (push) Successful in 6m57s
This commit is contained in:
@@ -0,0 +1,90 @@
|
||||
peg::parser! {
|
||||
grammar lexical() for str {
|
||||
rule trivia()
|
||||
= ([' ' | '\t' | '\r' | '\n'] / "//" [^'\r' | '\n']*)*
|
||||
|
||||
rule name_start()
|
||||
= [character if character == '_' || character.is_alphabetic()]
|
||||
|
||||
rule name_continue()
|
||||
= [character if character == '_' || character.is_alphanumeric()]
|
||||
|
||||
rule name() -> Result<TokenKind, Diagnostic>
|
||||
= name_start() name_continue()* { Ok(TokenKind::Name) }
|
||||
|
||||
rule number() -> Result<TokenKind, Diagnostic>
|
||||
= start:position!() ['0'..='9']+ ("." ['0'..='9']+)?
|
||||
suffix:$(name_continue()*) end:position!() {
|
||||
if suffix.is_empty() {
|
||||
Ok(TokenKind::Number)
|
||||
} else {
|
||||
Err(Diagnostic::lexical(
|
||||
Span::new(start, end),
|
||||
"unsupported numeric suffix or missing token separator",
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
rule escape()
|
||||
= "\\" ['n' | 't' | '"' | '\\']
|
||||
|
||||
rule string_character()
|
||||
= [^'"' | '\\' | '\r' | '\n' | '\0']
|
||||
|
||||
rule string_end(start: usize) -> Result<(), Diagnostic>
|
||||
= "\"" { Ok(()) }
|
||||
/ position:position!() "\\" [_] end:position!() {
|
||||
Err(Diagnostic::lexical(Span::new(position, end), "unsupported string escape"))
|
||||
}
|
||||
/ "\\" end:position!() ![_] {
|
||||
Err(Diagnostic::lexical(Span::new(start, end), "unterminated string"))
|
||||
}
|
||||
/ position:position!() ['\r' | '\n' | '\0'] end:position!() {
|
||||
Err(Diagnostic::lexical(Span::new(position, end), "unescaped newline or NUL in string"))
|
||||
}
|
||||
/ end:position!() ![_] {
|
||||
Err(Diagnostic::lexical(Span::new(start, end), "unterminated string"))
|
||||
}
|
||||
|
||||
rule string() -> Result<TokenKind, Diagnostic>
|
||||
= start:position!() "\"" (escape() / string_character())* end:string_end(start) {
|
||||
end.map(|()| TokenKind::String)
|
||||
}
|
||||
|
||||
rule punctuation() -> TokenKind
|
||||
= "(" { TokenKind::LeftParen }
|
||||
/ ")" { TokenKind::RightParen }
|
||||
/ "{" { TokenKind::LeftBrace }
|
||||
/ "}" { TokenKind::RightBrace }
|
||||
/ ":" { TokenKind::Colon }
|
||||
/ "," { TokenKind::Comma }
|
||||
/ "." { TokenKind::Dot }
|
||||
/ "=" { TokenKind::Equal }
|
||||
/ "+" { TokenKind::Plus }
|
||||
/ "-" { TokenKind::Minus }
|
||||
/ "*" { TokenKind::Star }
|
||||
/ "/" { TokenKind::Slash }
|
||||
|
||||
rule kind() -> Result<TokenKind, Diagnostic>
|
||||
= name()
|
||||
/ number()
|
||||
/ string()
|
||||
/ kind:punctuation() { Ok(kind) }
|
||||
/ start:position!() [_] end:position!() {
|
||||
Err(Diagnostic::lexical(Span::new(start, end), "unexpected source character"))
|
||||
}
|
||||
|
||||
rule token() -> Result<Token, Diagnostic>
|
||||
= start:position!() kind:kind() end:position!() {
|
||||
kind.map(|kind| Token { kind, span: Span::new(start, end) })
|
||||
}
|
||||
|
||||
// Scan only one prefix. Errors are returned as values, so a matched
|
||||
// malformed number/string cannot backtrack into another token kind.
|
||||
#[no_eof]
|
||||
pub rule next() -> (usize, Result<Option<Token>, Diagnostic>)
|
||||
= trivia() result:(token:token() { token.map(Some) } / ![_] { Ok(None) }) end:position!() {
|
||||
(end, result)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user