From 4bf889e5b71377cac98abdc0b4a8aef1362318f9 Mon Sep 17 00:00:00 2001 From: Codex Bot Date: Mon, 19 Oct 2026 02:56:33 +0800 Subject: [PATCH] Move PEG grammar into a dedicated source file --- crates/frontend/README.md | 2 +- crates/frontend/src/README.md | 5 +- crates/frontend/src/grammar.peg | 115 +++++++++++++++++++++++++++++++ crates/frontend/src/parser.rs | 116 +------------------------------- docs/README.md | 2 +- docs/operations/validation.md | 2 +- docs/runtime/frontend.md | 6 ++ scripts/README.md | 2 +- scripts/check.sh | 1 + 9 files changed, 130 insertions(+), 121 deletions(-) create mode 100644 crates/frontend/src/grammar.peg diff --git a/crates/frontend/README.md b/crates/frontend/README.md index 92e9942..d2c1e64 100644 --- a/crates/frontend/README.md +++ b/crates/frontend/README.md @@ -5,7 +5,7 @@ | 内容 | 职责 | | --- | --- | | Cargo.toml | frontend 包与 peg 0.8.6 依赖，不依赖 core、Host 或 Storage | -| [src](src/README.md) | token、源码范围、诊断、AST、词法与 PEG 规则 | +| [src](src/README.md) | token、源码范围、诊断、AST、词法；PEG 规则独立在 [grammar.peg](src/grammar.peg) | | [tests](tests/README.md) | Unicode、优先级、函数、完整输入与错误位置验证 | | [examples](examples/README.md) | 可交给 cli parse 的源码样例 | diff --git a/crates/frontend/src/README.md b/crates/frontend/src/README.md index 3b5928f..fd52a43 100644 --- a/crates/frontend/src/README.md +++ b/crates/frontend/src/README.md @@ -7,6 +7,7 @@ | [diagnostic.rs](diagnostic.rs) | UTF-8 字节范围、词法／语法诊断与一基行列转换 | | [lexer.rs](lexer.rs) | 名称、数字、字符串、标点、空白与注释扫描；字符串校验及解码 | | [ast.rs](ast.rs) | 拥有文本的函数、参数、绑定、return、表达式与运算符结构 | -| [parser.rs](parser.rs) | token 输入的 peg::parser! 规则、优先级、AST 构造及错误映射 | +| [parser.rs](parser.rs) | 解析 API、错误映射、AST 辅助函数，以及独立语法文件的编译期引入 | +| [grammar.peg](grammar.peg) | token 输入的 PEG 规则、运算符优先级与 AST 构造动作；保留 peg::parser! 外层声明 | -改词法时同步 token 和错误检查；改语法时同步 AST、PEG 规则、[测试](../tests/README.md)、样例与 [前端契约](../../../docs/runtime/frontend.md)。不在语法动作中做 VM 分配、I/O 或修改外部状态，避免回溯造成副作用。返回 [frontend](../README.md)。 +改词法时同步 token 和错误检查；改语法从 grammar.peg 进入，并同步 AST、[测试](../tests/README.md)、样例与 [前端契约](../../../docs/runtime/frontend.md)。不在语法动作中做 VM 分配、I/O 或修改外部状态，避免回溯造成副作用。返回 [frontend](../README.md)。 diff --git a/crates/frontend/src/grammar.peg b/crates/frontend/src/grammar.peg new file mode 100644 index 0000000..b75ad14 --- /dev/null +++ b/crates/frontend/src/grammar.peg @@ -0,0 +1,115 @@ +peg::parser! { + grammar syntax(source: &str) for [Token] { + rule keyword(word: &str) -> Span + = [Token { kind: TokenKind::Name, span } if &source[span.start..span.end] == word] { span } + + rule function_keyword() -> Span + = quiet!{keyword("function")} / expected!("'function'") + rule let_keyword() -> Span + = quiet!{keyword("let")} / expected!("'let'") + rule return_keyword() -> Span + = quiet!{keyword("return")} / expected!("'return'") + + rule name() -> Name + = span:quiet!{[Token { kind: TokenKind::Name, span }] { span }} { + Name { text: source[span.start..span.end].into(), span } + } + / expected!("identifier") + + rule lparen() -> Span + = quiet!{[Token { kind: TokenKind::LeftParen, span }] { span }} / expected!("'('") + rule rparen() -> Span + = quiet!{[Token { kind: TokenKind::RightParen, span }] { span }} / expected!("')'") + rule lbrace() -> Span + = quiet!{[Token { kind: TokenKind::LeftBrace, span }] { span }} / expected!("'{'") + rule rbrace() -> Span + = quiet!{[Token { kind: TokenKind::RightBrace, span }] { span }} / expected!("'}'") + rule colon() + = quiet!{[Token { kind: TokenKind::Colon, .. }]} / expected!("':'") + rule comma() + = quiet!{[Token { kind: TokenKind::Comma, .. }]} / expected!("','") + rule equal() + = quiet!{[Token { kind: TokenKind::Equal, .. }]} / expected!("'='") + rule dot() -> Span + = quiet!{[Token { kind: TokenKind::Dot, span }] { span }} / expected!("'.'") + rule plus() + = [Token { kind: TokenKind::Plus, .. }] + rule minus() -> Span + = [Token { kind: TokenKind::Minus, span }] { span } + rule star() + = [Token { kind: TokenKind::Star, .. }] + rule slash() + = [Token { kind: TokenKind::Slash, .. }] + + pub rule program() -> Program + = functions:function()+ { + let span = functions[0].span.through(functions.last().unwrap().span); + Program { functions, span } + } + + rule function() -> Function + = start:function_keyword() name:name() lparen() + parameters:(parameter() ** comma()) rparen() colon() return_type:name() body:block() { + Function { span: start.through(body.span), name, parameters, return_type, body } + } + + rule parameter() -> Parameter + = name:name() colon() type_name:name() { + Parameter { span: name.span.through(type_name.span), name, type_name } + } + + rule block() -> Block + = start:lbrace() bindings:binding()* returned:return_statement() end:rbrace() { + Block { bindings, result: returned.0, return_span: returned.1, span: start.through(end) } + } + + rule binding() -> Binding + = start:let_keyword() name:name() equal() value:expression() { + Binding { span: start.through(value.span), name, value } + } + + rule return_statement() -> (Expr, Span) + = start:return_keyword() value:expression() { + let span = start.through(value.span); + (value, span) + } + + pub rule expression() -> Expr = precedence! { + left:(@) plus() right:@ { binary(left, BinaryOp::Add, right) } + left:(@) minus() right:@ { binary(left, BinaryOp::Subtract, right) } + -- + left:(@) star() right:@ { binary(left, BinaryOp::Multiply, right) } + left:(@) slash() right:@ { binary(left, BinaryOp::Divide, right) } + -- + start:minus() operand:@ { + Expr { span: start.through(operand.span), kind: ExprKind::Unary { operation: UnaryOp::Negate, operand: Box::new(operand) } } + } + -- + expression:postfix_expression() { expression } + } + + rule postfix_expression() -> Expr + = expression:primary() suffixes:suffix()* { + suffixes.into_iter().fold(expression, postfix) + } + + rule suffix() -> Suffix + = dot() name:name() { Suffix::Member(name) } + / lparen() arguments:(expression() ** comma()) end:rparen() { Suffix::Call(arguments, end) } + + rule primary() -> Expr + = start:lparen() expression:expression() end:rparen() { + Expr { span: start.through(end), ..expression } + } + / start:dot() name:name() { + Expr { span: start.through(name.span), kind: ExprKind::Symbol(name) } + } + / [Token { kind: TokenKind::Number, span }] { + Expr { span, kind: ExprKind::Number(source[span.start..span.end].into()) } + } + / [Token { kind: TokenKind::String, span }] { + Expr { span, kind: ExprKind::String(decode_string(&source[span.start + 1..span.end - 1])) } + } + / name:name() { Expr { span: name.span, kind: ExprKind::Name(name.text) } } + } +} diff --git a/crates/frontend/src/parser.rs b/crates/frontend/src/parser.rs index 1103df6..1563c93 100644 --- a/crates/frontend/src/parser.rs +++ b/crates/frontend/src/parser.rs @@ -65,118 +65,4 @@ fn postfix(expression: Expr, suffix: Suffix) -> Expr { } } -peg::parser! { - grammar syntax(source: &str) for [Token] { - rule keyword(word: &str) -> Span - = [Token { kind: TokenKind::Name, span } if &source[span.start..span.end] == word] { span } - - rule function_keyword() -> Span - = quiet!{keyword("function")} / expected!("'function'") - rule let_keyword() -> Span - = quiet!{keyword("let")} / expected!("'let'") - rule return_keyword() -> Span - = quiet!{keyword("return")} / expected!("'return'") - - rule name() -> Name - = span:quiet!{[Token { kind: TokenKind::Name, span }] { span }} { - Name { text: source[span.start..span.end].into(), span } - } - / expected!("identifier") - - rule lparen() -> Span - = quiet!{[Token { kind: TokenKind::LeftParen, span }] { span }} / expected!("'('") - rule rparen() -> Span - = quiet!{[Token { kind: TokenKind::RightParen, span }] { span }} / expected!("')'") - rule lbrace() -> Span - = quiet!{[Token { kind: TokenKind::LeftBrace, span }] { span }} / expected!("'{'") - rule rbrace() -> Span - = quiet!{[Token { kind: TokenKind::RightBrace, span }] { span }} / expected!("'}'") - rule colon() - = quiet!{[Token { kind: TokenKind::Colon, .. }]} / expected!("':'") - rule comma() - = quiet!{[Token { kind: TokenKind::Comma, .. }]} / expected!("','") - rule equal() - = quiet!{[Token { kind: TokenKind::Equal, .. }]} / expected!("'='") - rule dot() -> Span - = quiet!{[Token { kind: TokenKind::Dot, span }] { span }} / expected!("'.'") - rule plus() - = [Token { kind: TokenKind::Plus, .. }] - rule minus() -> Span - = [Token { kind: TokenKind::Minus, span }] { span } - rule star() - = [Token { kind: TokenKind::Star, .. }] - rule slash() - = [Token { kind: TokenKind::Slash, .. }] - - pub rule program() -> Program - = functions:function()+ { - let span = functions[0].span.through(functions.last().unwrap().span); - Program { functions, span } - } - - rule function() -> Function - = start:function_keyword() name:name() lparen() - parameters:(parameter() ** comma()) rparen() colon() return_type:name() body:block() { - Function { span: start.through(body.span), name, parameters, return_type, body } - } - - rule parameter() -> Parameter - = name:name() colon() type_name:name() { - Parameter { span: name.span.through(type_name.span), name, type_name } - } - - rule block() -> Block - = start:lbrace() bindings:binding()* returned:return_statement() end:rbrace() { - Block { bindings, result: returned.0, return_span: returned.1, span: start.through(end) } - } - - rule binding() -> Binding - = start:let_keyword() name:name() equal() value:expression() { - Binding { span: start.through(value.span), name, value } - } - - rule return_statement() -> (Expr, Span) - = start:return_keyword() value:expression() { - let span = start.through(value.span); - (value, span) - } - - pub rule expression() -> Expr = precedence! { - left:(@) plus() right:@ { binary(left, BinaryOp::Add, right) } - left:(@) minus() right:@ { binary(left, BinaryOp::Subtract, right) } - -- - left:(@) star() right:@ { binary(left, BinaryOp::Multiply, right) } - left:(@) slash() right:@ { binary(left, BinaryOp::Divide, right) } - -- - start:minus() operand:@ { - Expr { span: start.through(operand.span), kind: ExprKind::Unary { operation: UnaryOp::Negate, operand: Box::new(operand) } } - } - -- - expression:postfix_expression() { expression } - } - - rule postfix_expression() -> Expr - = expression:primary() suffixes:suffix()* { - suffixes.into_iter().fold(expression, postfix) - } - - rule suffix() -> Suffix - = dot() name:name() { Suffix::Member(name) } - / lparen() arguments:(expression() ** comma()) end:rparen() { Suffix::Call(arguments, end) } - - rule primary() -> Expr - = start:lparen() expression:expression() end:rparen() { - Expr { span: start.through(end), ..expression } - } - / start:dot() name:name() { - Expr { span: start.through(name.span), kind: ExprKind::Symbol(name) } - } - / [Token { kind: TokenKind::Number, span }] { - Expr { span, kind: ExprKind::Number(source[span.start..span.end].into()) } - } - / [Token { kind: TokenKind::String, span }] { - Expr { span, kind: ExprKind::String(decode_string(&source[span.start + 1..span.end - 1])) } - } - / name:name() { Expr { span: name.span, kind: ExprKind::Name(name.text) } } - } -} +include!("grammar.peg"); diff --git a/docs/README.md b/docs/README.md index 1f677a2..d48a2c7 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,7 +9,7 @@ | 各个 crate 如何分层，哪些能力已实现 | [架构](architecture/README.md) | [crates](../crates/README.md) | | EndorObject、对象头、句柄、默认操作 | [对象模型](runtime/objects.md) | [core/src](../crates/core/src/README.md) | | Number、String、Dict、Cell、Function、异常数据 | [基本对象](runtime/builtins.md) | [core/src](../crates/core/src/README.md) | -| 源码如何分词、PEG 规则、AST 和诊断 | [源码前端](runtime/frontend.md) | [frontend](../crates/frontend/README.md)、[CLI](../crates/cli/README.md) | +| 源码如何分词、PEG 规则、AST 和诊断 | [源码前端](runtime/frontend.md) | [grammar.peg](../crates/frontend/src/grammar.peg)、[frontend](../crates/frontend/README.md)、[CLI](../crates/cli/README.md) | | 操作码、代码验证、局部槽、函数定义与调用协议 | [字节码 VM](runtime/bytecode.md) | instruction.rs / code.rs / frame.rs / eval.rs；见 [文件表](../crates/core/src/README.md) | | 基础信息保存在哪里、旧库如何迁移 | [宿主文件信息](operations/host-information.md) | [host](../crates/host/README.md) | | UTF-8 在日本地区是否适用，其他编码怎样进入 VM | [文本编码](runtime/text-encoding.md) | [core](../crates/core/README.md)、[host](../crates/host/README.md) | diff --git a/docs/operations/validation.md b/docs/operations/validation.md index f7144a4..c39969a 100644 --- a/docs/operations/validation.md +++ b/docs/operations/validation.md @@ -16,7 +16,7 @@ cargo test -p storage --test postgres --locked -- --ignored python3 scripts/smoke.py ``` -check.sh 执行 cargo fmt --all --check、cargo test --workspace --locked 和 cargo clippy --workspace --all-targets --locked -- -D warnings。workspace 测试也包含 rustdoc 示例；显式忽略的 PostgreSQL 测试需要单独执行。 +check.sh 执行 cargo fmt --all --check、独立语法文件的 rustfmt --edition 2024 --check crates/frontend/src/grammar.peg、cargo test --workspace --locked 和 cargo clippy --workspace --all-targets --locked -- -D warnings。workspace 测试也包含 rustdoc 示例；显式忽略的 PostgreSQL 测试需要单独执行。 ci-ces-control.sh 负责准备全新专用临时 PostgreSQL 和测试 URL；成功路径使用真实数据库，不用模拟 socket 代替。测试库与生产数据不共享挂载。没有 ASTER_TEST_POSTGRES_URL 时，显式执行 PostgreSQL 测试会失败，不标记为通过。 diff --git a/docs/runtime/frontend.md b/docs/runtime/frontend.md index edf6134..b494ef5 100644 --- a/docs/runtime/frontend.md +++ b/docs/runtime/frontend.md @@ -4,6 +4,12 @@ frontend 包提供手写 Rust lexer 和 rust-peg 生成的 token parser。当前 首版以 Endorphin-Language 设计稿的基础词法、表达式与函数章节为起点，语法可以继续修改；设计稿中的未完成部分不作为阻塞前端实现的前置条件，也不被整体视作已经实现的语言能力。这里维护当前解析契约；源码位置见 [frontend](../../crates/frontend/README.md)。 +## 语法文件 + +实际 PEG 规则集中在 [grammar.peg](../../crates/frontend/src/grammar.peg)，包含 token 匹配、函数／绑定／return、表达式优先级与 AST 构造动作。[parser.rs](../../crates/frontend/src/parser.rs) 保留解析入口、诊断转换和 AST 辅助函数，通过 include!("grammar.peg") 在编译期引入规则。 + +grammar.peg 保留 rust-peg 0.8 的 peg::parser! 外层声明，因此是直接参与编译的语法源文件；不另存一份说明性文法，也不新增生成脚本或 build-dependencies。修改规则后随 frontend 重新编译。远端检查脚本单独用 rustfmt 检查该扩展名文件的格式。 + ## 入口和所有权 | API | 输入与结果 | diff --git a/scripts/README.md b/scripts/README.md index c3d8997..4eb0aea 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -4,7 +4,7 @@ | 文件 | 入口与副作用 | | --- | --- | -| [check.sh](check.sh) | fmt 检查、workspace 测试、严格 Clippy；会使用 Cargo 依赖和构建产物 | +| [check.sh](check.sh) | Cargo fmt 与独立 grammar.peg 的 rustfmt 检查、workspace 测试、严格 Clippy；会使用 Cargo 依赖和构建产物 | | [smoke.py](smoke.py) | 标准库 Python 脚本；构建后用 Cargo metadata 定位 cli／server 的 debug 二进制，支持 CARGO_TARGET_DIR；文件 metadata 跨进程与日文值；失效数据库变量下独立启动、不创建 SQLite 文件、HTTP 与 SIGTERM，不改生产配置 | | [ci-ces-control.sh](ci-ces-control.sh) | 需要 ASTER_RELEASE_SHA；构建 ci 镜像、创建临时网络／PostgreSQL／检查容器，运行全部检查；退出清理临时容器和网络，保留专属远端 Cargo 缓存卷 | | [deploy-ces-control.sh](deploy-ces-control.sh) | ces-control root 执行；校验已构建镜像、锁定发布、保留密码和数据、验证 HTTP 与无数据库 CLI 引导后切换 current，失败尝试恢复旧版；保留镜像；应用服务 aster 内调用 cli bootstrap，基础信息改写文件；一次性只读导入旧基础信息，不覆盖新文件或删除原库 | diff --git a/scripts/check.sh b/scripts/check.sh index 80a1a65..40f25cc 100644 --- a/scripts/check.sh +++ b/scripts/check.sh @@ -1,5 +1,6 @@ #!/bin/sh set -eu cargo fmt --all --check +rustfmt --edition 2024 --check crates/frontend/src/grammar.peg cargo test --workspace --locked cargo clippy --workspace --all-targets --locked -- -D warnings