Add token-based PEG frontend and CLI source parsing
Verify databases and deploy Aster VM / verify-and-deploy (push) Successful in 7m30s
Verify databases and deploy Aster VM / verify-and-deploy (push) Successful in 7m30s
This commit is contained in:
@@ -0,0 +1,12 @@
|
||||
# Frontend 源码
|
||||
|
||||
| 文件 | 职责 |
|
||||
| --- | --- |
|
||||
| [lib.rs](lib.rs) | 公开 AST 模块、lex/parse/parse_expression 与 token、诊断导出 |
|
||||
| [token.rs](token.rs) | 可复制的 token 种类和原始源码范围,文本由源码切片取得 |
|
||||
| [diagnostic.rs](diagnostic.rs) | UTF-8 字节范围、词法/语法诊断与一基行列转换 |
|
||||
| [lexer.rs](lexer.rs) | 名称、数字、字符串、标点、空白与注释扫描;字符串校验及解码 |
|
||||
| [ast.rs](ast.rs) | 拥有文本的函数、参数、绑定、return、表达式与运算符结构 |
|
||||
| [parser.rs](parser.rs) | token 输入的 peg::parser! 规则、优先级、AST 构造及错误映射 |
|
||||
|
||||
改词法时同步 token 和错误检查;改语法时同步 AST、PEG 规则、[测试](../tests/README.md)、样例与 [前端契约](../../../docs/runtime/frontend.md)。不在语法动作中做 VM 分配、I/O 或修改外部状态,避免回溯造成副作用。返回 [frontend](../README.md)。
|
||||
@@ -0,0 +1,90 @@
|
||||
use crate::Span;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Name {
|
||||
pub text: String,
|
||||
pub span: Span,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Program {
|
||||
pub functions: Vec<Function>,
|
||||
pub span: Span,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Function {
|
||||
pub name: Name,
|
||||
pub parameters: Vec<Parameter>,
|
||||
pub return_type: Name,
|
||||
pub body: Block,
|
||||
pub span: Span,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Parameter {
|
||||
pub name: Name,
|
||||
pub type_name: Name,
|
||||
pub span: Span,
|
||||
}
|
||||
|
||||
/// Basic draft function body: zero or more bindings and one final return.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Block {
|
||||
pub bindings: Vec<Binding>,
|
||||
pub result: Expr,
|
||||
pub return_span: Span,
|
||||
pub span: Span,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Binding {
|
||||
pub name: Name,
|
||||
pub value: Expr,
|
||||
pub span: Span,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Expr {
|
||||
pub kind: ExprKind,
|
||||
pub span: Span,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum ExprKind {
|
||||
Name(String),
|
||||
/// The decimal spelling is retained, without f64 conversion.
|
||||
Number(String),
|
||||
String(String),
|
||||
Symbol(Name),
|
||||
Unary {
|
||||
operation: UnaryOp,
|
||||
operand: Box<Expr>,
|
||||
},
|
||||
Binary {
|
||||
operation: BinaryOp,
|
||||
left: Box<Expr>,
|
||||
right: Box<Expr>,
|
||||
},
|
||||
Member {
|
||||
object: Box<Expr>,
|
||||
name: Name,
|
||||
},
|
||||
Call {
|
||||
callee: Box<Expr>,
|
||||
arguments: Vec<Expr>,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum UnaryOp {
|
||||
Negate,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum BinaryOp {
|
||||
Add,
|
||||
Subtract,
|
||||
Multiply,
|
||||
Divide,
|
||||
}
|
||||
@@ -0,0 +1,88 @@
|
||||
use std::error::Error;
|
||||
use std::fmt;
|
||||
|
||||
/// Half-open UTF-8 byte range in the original source, including trivia between
|
||||
/// the first and last syntax token. EOF has an empty range at source.len().
|
||||
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
|
||||
pub struct Span {
|
||||
pub start: usize,
|
||||
pub end: usize,
|
||||
}
|
||||
|
||||
impl Span {
|
||||
pub const fn new(start: usize, end: usize) -> Self {
|
||||
Self { start, end }
|
||||
}
|
||||
|
||||
pub(crate) fn through(self, last: Self) -> Self {
|
||||
Self::new(self.start, last.end)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum DiagnosticKind {
|
||||
Lexical,
|
||||
Syntax,
|
||||
}
|
||||
|
||||
/// Lines and Unicode scalar columns are one-based. CRLF counts as one newline.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct SourceLocation {
|
||||
pub line: usize,
|
||||
pub column: usize,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Diagnostic {
|
||||
pub kind: DiagnosticKind,
|
||||
pub span: Span,
|
||||
pub message: String,
|
||||
pub expected: Vec<String>,
|
||||
}
|
||||
|
||||
impl Diagnostic {
|
||||
pub(crate) fn lexical(span: Span, message: impl Into<String>) -> Self {
|
||||
Self {
|
||||
kind: DiagnosticKind::Lexical,
|
||||
span,
|
||||
message: message.into(),
|
||||
expected: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn location(&self, source: &str) -> SourceLocation {
|
||||
let mut location = SourceLocation { line: 1, column: 1 };
|
||||
let mut previous_cr = false;
|
||||
for (offset, character) in source.char_indices() {
|
||||
if offset >= self.span.start {
|
||||
break;
|
||||
}
|
||||
match character {
|
||||
'\r' => {
|
||||
location.line += 1;
|
||||
location.column = 1;
|
||||
}
|
||||
'\n' if previous_cr => {}
|
||||
'\n' => {
|
||||
location.line += 1;
|
||||
location.column = 1;
|
||||
}
|
||||
_ => location.column += 1,
|
||||
}
|
||||
previous_cr = character == '\r';
|
||||
}
|
||||
location
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for Diagnostic {
|
||||
fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(
|
||||
formatter,
|
||||
"{} at bytes {}..{}",
|
||||
self.message, self.span.start, self.span.end
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl Error for Diagnostic {}
|
||||
@@ -0,0 +1,183 @@
|
||||
use crate::{Diagnostic, Span, Token, TokenKind};
|
||||
|
||||
/// Lex the basic Endorphin draft syntax without reading files or changing
|
||||
/// locale. Positions always refer to bytes in this exact UTF-8 source.
|
||||
pub fn lex(source: &str) -> Result<Vec<Token>, Diagnostic> {
|
||||
let mut lexer = Lexer { source, offset: 0 };
|
||||
let mut tokens = Vec::new();
|
||||
while let Some(character) = lexer.peek() {
|
||||
if matches!(character, ' ' | '\t' | '\r' | '\n') {
|
||||
let _ = lexer.advance();
|
||||
continue;
|
||||
}
|
||||
if lexer.remaining().starts_with("//") {
|
||||
while lexer
|
||||
.peek()
|
||||
.is_some_and(|character| !matches!(character, '\r' | '\n'))
|
||||
{
|
||||
let _ = lexer.advance();
|
||||
}
|
||||
continue;
|
||||
}
|
||||
let start = lexer.offset;
|
||||
let kind = match character {
|
||||
character if name_start(character) => {
|
||||
let _ = lexer.advance();
|
||||
while lexer.peek().is_some_and(name_continue) {
|
||||
let _ = lexer.advance();
|
||||
}
|
||||
TokenKind::Name
|
||||
}
|
||||
'0'..='9' => lexer.number()?,
|
||||
'"' => lexer.string()?,
|
||||
_ => {
|
||||
let _ = lexer.advance();
|
||||
match character {
|
||||
'(' => TokenKind::LeftParen,
|
||||
')' => TokenKind::RightParen,
|
||||
'{' => TokenKind::LeftBrace,
|
||||
'}' => TokenKind::RightBrace,
|
||||
':' => TokenKind::Colon,
|
||||
',' => TokenKind::Comma,
|
||||
'.' => TokenKind::Dot,
|
||||
'=' => TokenKind::Equal,
|
||||
'+' => TokenKind::Plus,
|
||||
'-' => TokenKind::Minus,
|
||||
'*' => TokenKind::Star,
|
||||
'/' => TokenKind::Slash,
|
||||
_ => {
|
||||
return Err(Diagnostic::lexical(
|
||||
Span::new(start, lexer.offset),
|
||||
"unexpected source character",
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
tokens.push(Token {
|
||||
kind,
|
||||
span: Span::new(start, lexer.offset),
|
||||
});
|
||||
}
|
||||
Ok(tokens)
|
||||
}
|
||||
|
||||
struct Lexer<'source> {
|
||||
source: &'source str,
|
||||
offset: usize,
|
||||
}
|
||||
|
||||
impl<'source> Lexer<'source> {
|
||||
fn remaining(&self) -> &'source str {
|
||||
&self.source[self.offset..]
|
||||
}
|
||||
|
||||
fn peek(&self) -> Option<char> {
|
||||
self.remaining().chars().next()
|
||||
}
|
||||
|
||||
fn advance(&mut self) -> Option<char> {
|
||||
let character = self.peek()?;
|
||||
self.offset += character.len_utf8();
|
||||
Some(character)
|
||||
}
|
||||
|
||||
fn digits(&mut self) {
|
||||
while self
|
||||
.peek()
|
||||
.is_some_and(|character| character.is_ascii_digit())
|
||||
{
|
||||
let _ = self.advance();
|
||||
}
|
||||
}
|
||||
|
||||
fn number(&mut self) -> Result<TokenKind, Diagnostic> {
|
||||
let start = self.offset;
|
||||
self.digits();
|
||||
let mut following = self.remaining().chars();
|
||||
if following.next() == Some('.')
|
||||
&& following
|
||||
.next()
|
||||
.is_some_and(|character| character.is_ascii_digit())
|
||||
{
|
||||
let _ = self.advance();
|
||||
self.digits();
|
||||
}
|
||||
if self.peek().is_some_and(name_continue) {
|
||||
while self.peek().is_some_and(name_continue) {
|
||||
let _ = self.advance();
|
||||
}
|
||||
return Err(Diagnostic::lexical(
|
||||
Span::new(start, self.offset),
|
||||
"unsupported numeric suffix or missing token separator",
|
||||
));
|
||||
}
|
||||
Ok(TokenKind::Number)
|
||||
}
|
||||
|
||||
fn string(&mut self) -> Result<TokenKind, Diagnostic> {
|
||||
let start = self.offset;
|
||||
let _ = self.advance();
|
||||
loop {
|
||||
let position = self.offset;
|
||||
match self.advance() {
|
||||
Some('"') => return Ok(TokenKind::String),
|
||||
Some('\\') => match self.advance() {
|
||||
Some('n' | 't' | '"' | '\\') => {}
|
||||
Some(_) => {
|
||||
return Err(Diagnostic::lexical(
|
||||
Span::new(position, self.offset),
|
||||
"unsupported string escape",
|
||||
));
|
||||
}
|
||||
None => {
|
||||
return Err(Diagnostic::lexical(
|
||||
Span::new(start, self.offset),
|
||||
"unterminated string",
|
||||
));
|
||||
}
|
||||
},
|
||||
Some('\r' | '\n' | '\0') => {
|
||||
return Err(Diagnostic::lexical(
|
||||
Span::new(position, self.offset),
|
||||
"unescaped newline or NUL in string",
|
||||
));
|
||||
}
|
||||
None => {
|
||||
return Err(Diagnostic::lexical(
|
||||
Span::new(start, self.offset),
|
||||
"unterminated string",
|
||||
));
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn name_start(character: char) -> bool {
|
||||
character == '_' || character.is_alphabetic()
|
||||
}
|
||||
|
||||
fn name_continue(character: char) -> bool {
|
||||
character == '_' || character.is_alphanumeric()
|
||||
}
|
||||
|
||||
pub(crate) fn decode_string(raw: &str) -> String {
|
||||
let mut result = String::new();
|
||||
let mut characters = raw.chars();
|
||||
while let Some(character) = characters.next() {
|
||||
if character == '\\' {
|
||||
result.push(match characters.next().expect("lexer validated escape") {
|
||||
'n' => '\n',
|
||||
't' => '\t',
|
||||
'"' => '"',
|
||||
'\\' => '\\',
|
||||
_ => unreachable!("lexer validated escape"),
|
||||
});
|
||||
} else {
|
||||
result.push(character);
|
||||
}
|
||||
}
|
||||
result
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
//! UTF-8 source frontend: a handwritten lexer and token-based rust-peg parser.
|
||||
//! Produces syntax only; it does not resolve types, generate code or execute VM
|
||||
//! operations. AST data owns its text and does not borrow the source buffer.
|
||||
|
||||
pub mod ast;
|
||||
mod diagnostic;
|
||||
mod lexer;
|
||||
mod parser;
|
||||
mod token;
|
||||
|
||||
pub use diagnostic::{Diagnostic, DiagnosticKind, SourceLocation, Span};
|
||||
pub use lexer::lex;
|
||||
pub use parser::{parse, parse_expression};
|
||||
pub use token::{Token, TokenKind};
|
||||
@@ -0,0 +1,182 @@
|
||||
use crate::ast::{
|
||||
BinaryOp, Binding, Block, Expr, ExprKind, Function, Name, Parameter, Program, UnaryOp,
|
||||
};
|
||||
use crate::lexer::decode_string;
|
||||
use crate::{Diagnostic, DiagnosticKind, Span, Token, TokenKind, lex};
|
||||
|
||||
/// Parse one or more basic typed function declarations. This is a syntax
|
||||
/// operation: type names and name references are retained without resolution.
|
||||
pub fn parse(source: &str) -> Result<Program, Diagnostic> {
|
||||
let tokens = lex(source)?;
|
||||
syntax::program(&tokens, source).map_err(|error| diagnostic(error, &tokens, source.len()))
|
||||
}
|
||||
|
||||
/// Parse exactly one complete expression, useful independently of declarations.
|
||||
pub fn parse_expression(source: &str) -> Result<Expr, Diagnostic> {
|
||||
let tokens = lex(source)?;
|
||||
syntax::expression(&tokens, source).map_err(|error| diagnostic(error, &tokens, source.len()))
|
||||
}
|
||||
|
||||
fn diagnostic(error: peg::error::ParseError<usize>, tokens: &[Token], end: usize) -> Diagnostic {
|
||||
let span = tokens
|
||||
.get(error.location)
|
||||
.map_or(Span::new(end, end), |token| token.span);
|
||||
let expected: Vec<String> = error.expected.tokens().map(str::to_owned).collect();
|
||||
Diagnostic {
|
||||
kind: DiagnosticKind::Syntax,
|
||||
span,
|
||||
message: format!("expected {}", expected.join(" or ")),
|
||||
expected,
|
||||
}
|
||||
}
|
||||
|
||||
fn binary(left: Expr, operation: BinaryOp, right: Expr) -> Expr {
|
||||
Expr {
|
||||
span: left.span.through(right.span),
|
||||
kind: ExprKind::Binary {
|
||||
operation,
|
||||
left: Box::new(left),
|
||||
right: Box::new(right),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
enum Suffix {
|
||||
Member(Name),
|
||||
Call(Vec<Expr>, Span),
|
||||
}
|
||||
|
||||
fn postfix(expression: Expr, suffix: Suffix) -> Expr {
|
||||
match suffix {
|
||||
Suffix::Member(name) => Expr {
|
||||
span: expression.span.through(name.span),
|
||||
kind: ExprKind::Member {
|
||||
object: Box::new(expression),
|
||||
name,
|
||||
},
|
||||
},
|
||||
Suffix::Call(arguments, end) => Expr {
|
||||
span: expression.span.through(end),
|
||||
kind: ExprKind::Call {
|
||||
callee: Box::new(expression),
|
||||
arguments,
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
peg::parser! {
|
||||
grammar syntax(source: &str) for [Token] {
|
||||
rule keyword(word: &str) -> Span
|
||||
= [Token { kind: TokenKind::Name, span } if &source[span.start..span.end] == word] { span }
|
||||
|
||||
rule function_keyword() -> Span
|
||||
= quiet!{keyword("function")} / expected!("'function'")
|
||||
rule let_keyword() -> Span
|
||||
= quiet!{keyword("let")} / expected!("'let'")
|
||||
rule return_keyword() -> Span
|
||||
= quiet!{keyword("return")} / expected!("'return'")
|
||||
|
||||
rule name() -> Name
|
||||
= span:quiet!{[Token { kind: TokenKind::Name, span }] { span }} {
|
||||
Name { text: source[span.start..span.end].into(), span }
|
||||
}
|
||||
/ expected!("identifier")
|
||||
|
||||
rule lparen() -> Span
|
||||
= quiet!{[Token { kind: TokenKind::LeftParen, span }] { span }} / expected!("'('")
|
||||
rule rparen() -> Span
|
||||
= quiet!{[Token { kind: TokenKind::RightParen, span }] { span }} / expected!("')'")
|
||||
rule lbrace() -> Span
|
||||
= quiet!{[Token { kind: TokenKind::LeftBrace, span }] { span }} / expected!("'{'")
|
||||
rule rbrace() -> Span
|
||||
= quiet!{[Token { kind: TokenKind::RightBrace, span }] { span }} / expected!("'}'")
|
||||
rule colon()
|
||||
= quiet!{[Token { kind: TokenKind::Colon, .. }]} / expected!("':'")
|
||||
rule comma()
|
||||
= quiet!{[Token { kind: TokenKind::Comma, .. }]} / expected!("','")
|
||||
rule equal()
|
||||
= quiet!{[Token { kind: TokenKind::Equal, .. }]} / expected!("'='")
|
||||
rule dot() -> Span
|
||||
= quiet!{[Token { kind: TokenKind::Dot, span }] { span }} / expected!("'.'")
|
||||
rule plus()
|
||||
= [Token { kind: TokenKind::Plus, .. }]
|
||||
rule minus() -> Span
|
||||
= [Token { kind: TokenKind::Minus, span }] { span }
|
||||
rule star()
|
||||
= [Token { kind: TokenKind::Star, .. }]
|
||||
rule slash()
|
||||
= [Token { kind: TokenKind::Slash, .. }]
|
||||
|
||||
pub rule program() -> Program
|
||||
= functions:function()+ {
|
||||
let span = functions[0].span.through(functions.last().unwrap().span);
|
||||
Program { functions, span }
|
||||
}
|
||||
|
||||
rule function() -> Function
|
||||
= start:function_keyword() name:name() lparen()
|
||||
parameters:(parameter() ** comma()) rparen() colon() return_type:name() body:block() {
|
||||
Function { span: start.through(body.span), name, parameters, return_type, body }
|
||||
}
|
||||
|
||||
rule parameter() -> Parameter
|
||||
= name:name() colon() type_name:name() {
|
||||
Parameter { span: name.span.through(type_name.span), name, type_name }
|
||||
}
|
||||
|
||||
rule block() -> Block
|
||||
= start:lbrace() bindings:binding()* returned:return_statement() end:rbrace() {
|
||||
Block { bindings, result: returned.0, return_span: returned.1, span: start.through(end) }
|
||||
}
|
||||
|
||||
rule binding() -> Binding
|
||||
= start:let_keyword() name:name() equal() value:expression() {
|
||||
Binding { span: start.through(value.span), name, value }
|
||||
}
|
||||
|
||||
rule return_statement() -> (Expr, Span)
|
||||
= start:return_keyword() value:expression() {
|
||||
let span = start.through(value.span);
|
||||
(value, span)
|
||||
}
|
||||
|
||||
pub rule expression() -> Expr = precedence! {
|
||||
left:(@) plus() right:@ { binary(left, BinaryOp::Add, right) }
|
||||
left:(@) minus() right:@ { binary(left, BinaryOp::Subtract, right) }
|
||||
--
|
||||
left:(@) star() right:@ { binary(left, BinaryOp::Multiply, right) }
|
||||
left:(@) slash() right:@ { binary(left, BinaryOp::Divide, right) }
|
||||
--
|
||||
start:minus() operand:@ {
|
||||
Expr { span: start.through(operand.span), kind: ExprKind::Unary { operation: UnaryOp::Negate, operand: Box::new(operand) } }
|
||||
}
|
||||
--
|
||||
expression:postfix_expression() { expression }
|
||||
}
|
||||
|
||||
rule postfix_expression() -> Expr
|
||||
= expression:primary() suffixes:suffix()* {
|
||||
suffixes.into_iter().fold(expression, postfix)
|
||||
}
|
||||
|
||||
rule suffix() -> Suffix
|
||||
= dot() name:name() { Suffix::Member(name) }
|
||||
/ lparen() arguments:(expression() ** comma()) end:rparen() { Suffix::Call(arguments, end) }
|
||||
|
||||
rule primary() -> Expr
|
||||
= start:lparen() expression:expression() end:rparen() {
|
||||
Expr { span: start.through(end), ..expression }
|
||||
}
|
||||
/ start:dot() name:name() {
|
||||
Expr { span: start.through(name.span), kind: ExprKind::Symbol(name) }
|
||||
}
|
||||
/ [Token { kind: TokenKind::Number, span }] {
|
||||
Expr { span, kind: ExprKind::Number(source[span.start..span.end].into()) }
|
||||
}
|
||||
/ [Token { kind: TokenKind::String, span }] {
|
||||
Expr { span, kind: ExprKind::String(decode_string(&source[span.start + 1..span.end - 1])) }
|
||||
}
|
||||
/ name:name() { Expr { span: name.span, kind: ExprKind::Name(name.text) } }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
use crate::Span;
|
||||
|
||||
/// Text is located by span in the original source. Names stay contextual: the
|
||||
/// lexer does not turn syntax words into globally reserved keywords.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum TokenKind {
|
||||
Name,
|
||||
Number,
|
||||
/// Span includes quotes. Escapes have been checked, not decoded.
|
||||
String,
|
||||
LeftParen,
|
||||
RightParen,
|
||||
LeftBrace,
|
||||
RightBrace,
|
||||
Colon,
|
||||
Comma,
|
||||
Dot,
|
||||
Equal,
|
||||
Plus,
|
||||
Minus,
|
||||
Star,
|
||||
Slash,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct Token {
|
||||
pub kind: TokenKind,
|
||||
pub span: Span,
|
||||
}
|
||||
Reference in New Issue
Block a user