Add token-based PEG frontend and CLI source parsing
Verify databases and deploy Aster VM / verify-and-deploy (push) Successful in 7m30s
Verify databases and deploy Aster VM / verify-and-deploy (push) Successful in 7m30s
This commit is contained in:
@@ -0,0 +1,29 @@
|
||||
use crate::Span;
|
||||
|
||||
/// Text is located by span in the original source. Names stay contextual: the
|
||||
/// lexer does not turn syntax words into globally reserved keywords.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum TokenKind {
|
||||
Name,
|
||||
Number,
|
||||
/// Span includes quotes. Escapes have been checked, not decoded.
|
||||
String,
|
||||
LeftParen,
|
||||
RightParen,
|
||||
LeftBrace,
|
||||
RightBrace,
|
||||
Colon,
|
||||
Comma,
|
||||
Dot,
|
||||
Equal,
|
||||
Plus,
|
||||
Minus,
|
||||
Star,
|
||||
Slash,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct Token {
|
||||
pub kind: TokenKind,
|
||||
pub span: Span,
|
||||
}
|
||||
Reference in New Issue
Block a user