peg::parser! { grammar lexical() for str { rule trivia() = ([' ' | '\t' | '\r' | '\n'] / "//" [^'\r' | '\n']*)* rule name_start() = [character if character == '_' || character.is_alphabetic()] rule name_continue() = [character if character == '_' || character.is_alphanumeric()] rule name() -> Result = name_start() name_continue()* { Ok(TokenKind::Name) } rule number() -> Result = start:position!() ['0'..='9']+ ("." ['0'..='9']+)? suffix:$(name_continue()*) end:position!() { if suffix.is_empty() { Ok(TokenKind::Number) } else { Err(Diagnostic::lexical( Span::new(start, end), "unsupported numeric suffix or missing token separator", )) } } rule escape() = "\\" ['n' | 't' | '"' | '\\'] rule string_character() = [^'"' | '\\' | '\r' | '\n' | '\0'] rule string_end(start: usize) -> Result<(), Diagnostic> = "\"" { Ok(()) } / position:position!() "\\" [_] end:position!() { Err(Diagnostic::lexical(Span::new(position, end), "unsupported string escape")) } / "\\" end:position!() ![_] { Err(Diagnostic::lexical(Span::new(start, end), "unterminated string")) } / position:position!() ['\r' | '\n' | '\0'] end:position!() { Err(Diagnostic::lexical(Span::new(position, end), "unescaped newline or NUL in string")) } / end:position!() ![_] { Err(Diagnostic::lexical(Span::new(start, end), "unterminated string")) } rule string() -> Result = start:position!() "\"" (escape() / string_character())* end:string_end(start) { end.map(|()| TokenKind::String) } rule punctuation() -> TokenKind = "(" { TokenKind::LeftParen } / ")" { TokenKind::RightParen } / "{" { TokenKind::LeftBrace } / "}" { TokenKind::RightBrace } / ":" { TokenKind::Colon } / "," { TokenKind::Comma } / "." { TokenKind::Dot } / "==" { TokenKind::EqualEqual } / "!=" { TokenKind::NotEqual } / "<=" { TokenKind::LessEqual } / ">=" { TokenKind::GreaterEqual } / "<" { TokenKind::Less } / ">" { TokenKind::Greater } / "=" { TokenKind::Equal } / "+" { TokenKind::Plus } / "-" { TokenKind::Minus } / "*" { TokenKind::Star } / "/" { TokenKind::Slash } rule kind() -> Result = name() / number() / string() / kind:punctuation() { Ok(kind) } / start:position!() [_] end:position!() { Err(Diagnostic::lexical(Span::new(start, end), "unexpected source character")) } rule token() -> Result = start:position!() kind:kind() end:position!() { kind.map(|kind| Token { kind, span: Span::new(start, end) }) } // Scan only one prefix. Errors are returned as values, so a matched // malformed number/string cannot backtrack into another token kind. #[no_eof] pub rule next() -> (usize, Result, Diagnostic>) = trivia() result:(token:token() { token.map(Some) } / ![_] { Ok(None) }) end:position!() { (end, result) } } }