feat: generate unified AST and editor contracts from ASDL

scripts/README.md

@@ -1,10 +1,12 @@
 # 检查与发布脚本
 
-所有构建、测试、冒烟和部署脚本只在用户指定的远端环境运行。本目录 README 负责定位与副作用边界；全局流程见 [验证](../docs/operations/validation.md) 和 [部署](../docs/operations/deployment.md)。
+所有构建、测试、冒烟和部署脚本只在用户指定的远端环境运行。generate_ast.py 是源码生成／静态同步工具，可本地使用；test_generate_ast.py 仅远端执行。本目录 README 负责定位与副作用边界；全局流程见 [验证](../docs/operations/validation.md) 和 [部署](../docs/operations/deployment.md)。
 
 | 文件 | 入口与副作用 |
 | --- | --- |
-| [check.sh](check.sh) | Cargo fmt 与独立 lexer.peg／grammar.peg 的 rustfmt 检查、workspace 测试、严格 Clippy；会使用 Cargo 依赖和构建产物 |
+| [generate_ast.py](generate_ast.py) | 从 Endor.asdl 生成并格式化 Rust／TypeScript／JSON Schema；--write 更新产物，--check 只比较，无依赖下载或语言执行 |
+| [test_generate_ast.py](test_generate_ast.py) | 远端 unittest：schema 扩展同步、递归装箱／遍历、非法定义拒绝 |
+| [check.sh](check.sh) | 先检查 ASDL 产物同步和生成器行为，再运行 Cargo fmt 与独立 lexer.peg／grammar.peg 的 rustfmt 检查、workspace 测试、严格 Clippy；会使用 Cargo 依赖和构建产物 |
 | [smoke.py](smoke.py) | 标准库 Python 脚本；构建后用 Cargo metadata 定位 cli／server 的 debug 二进制，支持 CARGO_TARGET_DIR；文件 metadata 跨进程与日文值；失效数据库变量下独立启动、不创建 SQLite 文件、HTTP 与 SIGTERM，不改生产配置 |
 | [ci-ces-control.sh](ci-ces-control.sh) | 需要 ASTER_RELEASE_SHA；构建 ci 镜像、创建临时网络／PostgreSQL／检查容器，运行全部检查；退出清理临时容器和网络，保留专属远端 Cargo 缓存卷 |
 | [deploy-ces-control.sh](deploy-ces-control.sh) | ces-control root 执行；校验已构建镜像、锁定发布、保留密码和数据、验证 HTTP 与无数据库 CLI 引导后切换 current，失败尝试恢复旧版；保留镜像；应用服务 aster 内调用 cli bootstrap，基础信息改写文件；一次性只读导入旧基础信息，不覆盖新文件或删除原库 |

scripts/check.sh

@@ -1,5 +1,7 @@
 #!/bin/sh
 set -eu
+python3 scripts/generate_ast.py --check
+python3 scripts/test_generate_ast.py
 cargo fmt --all --check
 rustfmt --edition 2024 --check crates/frontend/src/lexer.peg crates/frontend/src/grammar.peg
 cargo test --workspace --locked

scripts/generate_ast.py

@@ -0,0 +1,405 @@
+#!/usr/bin/env python3
+"""Generate checked-in AST types, walkers and editor contracts from Endor.asdl.
+
+Uses Python's standard library and rustfmt only. It never builds or runs Endor.
+"""
+
+from __future__ import annotations
+
+import argparse
+from dataclasses import dataclass
+import hashlib
+import json
+from pathlib import Path
+import re
+import subprocess
+
+
+BUILTINS = {"string": "String", "span": "Span", "schema_id": "String"}
+HEADER = "// Generated by scripts/generate_ast.py from Endor.asdl. Do not edit.\n"
+RESERVED = set("as break const continue crate else enum extern false fn for if impl in let loop match mod move mut pub ref return self Self static struct super trait true type unsafe use where while async await dyn abstract become box do final macro override priv typeof unsized virtual yield try gen".split())
+
+
+@dataclass(frozen=True)
+class Field:
+    type: str
+    name: str
+    quantifier: str = ""
+
+
+@dataclass(frozen=True)
+class Variant:
+    name: str
+    fields: tuple[Field, ...]
+
+
+@dataclass(frozen=True)
+class Definition:
+    name: str
+    fields: tuple[Field, ...] | None
+    variants: tuple[Variant, ...] = ()
+
+    def all_fields(self):
+        return self.fields if self.fields is not None else tuple(f for v in self.variants for f in v.fields)
+
+    def simple(self):
+        return self.fields is None and all(not v.fields for v in self.variants)
+
+
+def parse_schema(text: str) -> dict[str, Definition]:
+    """Parse the project's ASDL subset; reject unknown syntax and field types."""
+    text = re.sub(r"--[^\n]*", "", text)
+    tokens = []
+    cursor = 0
+    pattern = re.compile(r"\s*([A-Za-z_][A-Za-z_0-9]*|[{}()=|,*?])")
+    while cursor < len(text):
+        if not text[cursor:].strip():
+            break
+        match = pattern.match(text, cursor)
+        if not match:
+            raise ValueError(f"unsupported ASDL syntax near {text[cursor:cursor + 30]!r}")
+        tokens.append(match[1])
+        cursor = match.end()
+    index = 0
+
+    def peek():
+        if index == len(tokens):
+            raise ValueError("unexpected end of ASDL")
+        return tokens[index]
+
+    def take(expected=None):
+        nonlocal index
+        if index == len(tokens):
+            raise ValueError("unexpected end of ASDL")
+        value = tokens[index]
+        if expected is not None and value != expected:
+            raise ValueError(f"expected {expected!r}, found {value!r}")
+        index += 1
+        return value
+
+    def identifier():
+        value = take()
+        if not re.fullmatch(r"[A-Za-z_][A-Za-z_0-9]*", value) or value in RESERVED:
+            raise ValueError(f"invalid or reserved identifier: {value}")
+        return value
+
+    def fields():
+        take("(")
+        result = []
+        while peek() != ")":
+            type_name = identifier()
+            quantifier = take() if peek() in ("?", "*") else ""
+            result.append(Field(type_name, identifier(), quantifier))
+            if peek() != ",":
+                break
+            take(",")
+        take(")")
+        if len({f.name for f in result}) != len(result):
+            raise ValueError("duplicate field name")
+        return tuple(result)
+
+    take("module")
+    identifier()
+    take("{")
+    definitions = {}
+    while peek() != "}":
+        name = identifier()
+        if name in definitions or name in BUILTINS:
+            raise ValueError(f"duplicate or builtin type: {name}")
+        take("=")
+        if peek() == "(":
+            definition = Definition(name, fields())
+        else:
+            variants = []
+            while True:
+                variant = identifier()
+                members = fields() if peek() == "(" else ()
+                variants.append(Variant(variant, members))
+                if peek() != "|":
+                    break
+                take("|")
+            if len({v.name for v in variants}) != len(variants):
+                raise ValueError(f"duplicate constructor in {name}")
+            definition = Definition(name, None, tuple(variants))
+        definitions[name] = definition
+    take("}")
+    if index != len(tokens):
+        raise ValueError("trailing ASDL input")
+    for definition in definitions.values():
+        for field in definition.all_fields():
+            if field.type not in definitions and field.type not in BUILTINS:
+                raise ValueError(f"undefined field type: {field.type}")
+    return definitions
+
+
+def snake(name):
+    return re.sub(r"(?<!^)(?=[A-Z])", "_", name).lower()
+
+
+def recursive_edges(definitions):
+    """Box DFS back edges in the by-value graph; Vec already breaks recursion."""
+    active, done, boxed = set(), set(), set()
+
+    def visit(name):
+        active.add(name)
+        for field in definitions[name].all_fields():
+            if field.quantifier == "*" or field.type not in definitions:
+                continue
+            if field.type in active:
+                boxed.add((name, field.type))
+            elif field.type not in done:
+                visit(field.type)
+        active.remove(name)
+        done.add(name)
+
+    for name in definitions:
+        if name not in done:
+            visit(name)
+    return boxed
+
+
+def spanned_types(definitions):
+    result = {d.name for d in definitions.values() if d.fields is not None and Field("span", "span") in d.fields}
+    while True:
+        expanded = result | {
+            d.name for d in definitions.values()
+            if d.variants and all(len(v.fields) == 1 and not v.fields[0].quantifier and v.fields[0].type in result for v in d.variants)
+        }
+        if expanded == result:
+            return result
+        result = expanded
+
+
+def rust_types(definitions, schema_id):
+    boxed = recursive_edges(definitions)
+    spanned = spanned_types(definitions)
+
+    def field_type(owner, field):
+        value = BUILTINS.get(field.type, field.type)
+        if field.quantifier == "*":
+            return f"Vec<{value}>"
+        if (owner, field.type) in boxed:
+            value = f"Box<{value}>"
+        return f"Option<{value}>" if field.quantifier == "?" else value
+
+    lines = [HEADER, "use crate::Span;", "use super::Spanned;", f'pub const AST_SCHEMA_ID: &str = "{schema_id}";']
+    for definition in definitions.values():
+        derives = "Debug, Clone, PartialEq, Eq, serde::Serialize, serde::Deserialize"
+        if definition.simple():
+            derives += ", Copy"
+        lines.append(f"#[derive({derives})]")
+        if not definition.simple():
+            tag = 'tag = "kind", content = "data", ' if definition.fields is None else ""
+            lines.append(f"#[serde({tag}deny_unknown_fields)]")
+        if definition.fields is not None:
+            lines.append(f"pub struct {definition.name} {{")
+            lines.extend(f"pub {f.name}: {field_type(definition.name, f)}," for f in definition.fields)
+        else:
+            lines.append(f"pub enum {definition.name} {{")
+            for variant in definition.variants:
+                if not variant.fields:
+                    lines.append(f"{variant.name},")
+                elif len(variant.fields) == 1:
+                    lines.append(f"{variant.name}({field_type(definition.name, variant.fields[0])}),")
+                else:
+                    fields = ", ".join(f"{f.name}: {field_type(definition.name, f)}" for f in variant.fields)
+                    lines.append(f"{variant.name} {{ {fields} }},")
+        lines.append("}")
+        if definition.name in spanned:
+            lines.append(f"impl Spanned for {definition.name} {{ fn span(&self) -> Span {{")
+            if definition.fields is not None:
+                lines.append("self.span")
+            else:
+                lines.append("match self {")
+                lines.extend(f"Self::{v.name}(node) => node.span()," for v in definition.variants)
+                lines.append("}")
+            lines.append("} }")
+    return "\n".join(lines) + "\n"
+
+
+def rust_visitor(definitions):
+    spanned = spanned_types(definitions)
+    names = ", ".join(definitions)
+    lines = [HEADER, f"use super::{{{names}, Spanned}};", "use crate::Span;", "pub trait Visitor {", "type Error;",
+             "fn enter_node(&mut self, _span: Span) -> Result<(), Self::Error> { Ok(()) }",
+             "fn leave_node(&mut self) {}",
+             "fn visit_span(&mut self, _span: &Span) -> Result<(), Self::Error> { Ok(()) }"]
+    for name in definitions:
+        lines.append(f"fn visit_{snake(name)}(&mut self, node: &{name}) -> Result<(), Self::Error> {{")
+        if name in spanned:
+            lines += ["self.enter_node(node.span())?;", f"let result = walk_{snake(name)}(self, node);", "self.leave_node();", "result"]
+        else:
+            lines.append(f"walk_{snake(name)}(self, node)")
+        lines.append("}")
+    lines.append("}")
+
+    def children(field, access):
+        if field.type not in definitions and field.type != "span":
+            return []
+        method = f"visitor.visit_{snake(field.type)}"
+        if field.quantifier == "*":
+            return [f"for value in {access} {{ {method}(value)?; }}"]
+        if field.quantifier == "?":
+            return [f"if let Some(value) = {access} {{ {method}(value)?; }}"]
+        return [f"{method}({access})?;"]
+
+    for definition in definitions.values():
+        body = []
+        if definition.fields is not None:
+            for field in definition.fields:
+                body += children(field, f"&node.{field.name}")
+        elif not definition.simple():
+            body.append("match node {")
+            for variant in definition.variants:
+                visited = [f for f in variant.fields if f.type in definitions or f.type == "span"]
+                if not variant.fields:
+                    pattern = f"{definition.name}::{variant.name}"
+                elif len(variant.fields) == 1:
+                    pattern = f"{definition.name}::{variant.name}({'value' if visited else '_'})"
+                else:
+                    members = ", ".join(f.name for f in visited)
+                    pattern = f"{definition.name}::{variant.name} {{ " + (members + ", " if members else "") + ".. }"
+                body.append(pattern + " => {")
+                for field in visited:
+                    body += children(field, "value" if len(variant.fields) == 1 else field.name)
+                body.append("}")
+            body.append("}")
+        visitor = "visitor" if any("visitor." in line for line in body) else "_visitor"
+        node = "node" if body else "_node"
+        lines.append(f"pub fn walk_{snake(definition.name)}<V: Visitor + ?Sized>({visitor}: &mut V, {node}: &{definition.name}) -> Result<(), V::Error> {{")
+        lines.extend(body)
+        lines += ["Ok(())", "}"]
+    return "\n".join(lines) + "\n"
+
+
+def editor_types(definitions, schema_id):
+    def field_type(field):
+        result = {"string": "string", "span": "Span", "schema_id": "typeof AST_SCHEMA_ID"}.get(field.type, field.type)
+        if field.quantifier == "*":
+            result = f"Array<{result}>"
+        elif field.quantifier == "?":
+            result += " | null"
+        return result
+
+    def object_type(fields):
+        return "{ " + " ".join(f"{f.name}{'?' if f.quantifier == '?' else ''}: {field_type(f)};" for f in fields) + " }"
+
+    lines = [HEADER, f'export const AST_SCHEMA_ID = "{schema_id}" as const;', "export interface Span { start: number; end: number; }"]
+    for definition in definitions.values():
+        if definition.fields is not None:
+            lines.append(f"export type {definition.name} = {object_type(definition.fields)};")
+        else:
+            variants = []
+            for variant in definition.variants:
+                if definition.simple():
+                    value = json.dumps(variant.name)
+                else:
+                    data = ""
+                    if variant.fields:
+                        result = field_type(variant.fields[0]) if len(variant.fields) == 1 else object_type(variant.fields)
+                        data = f" data: {result};"
+                    value = '{ kind: ' + json.dumps(variant.name) + ';' + data + ' }'
+                variants.append(value)
+            lines.append(f"export type {definition.name} = " + " | ".join(variants) + ";")
+    lines += ["export interface Visitor {", "visitSpan?(node: Span): boolean | void;"]
+    lines.extend(f"visit{name}?(node: {name}): boolean | void;" for name in definitions)
+    lines.append("}")
+
+    def children(field, access):
+        if field.type not in definitions and field.type != "span":
+            return []
+        function = "walkSpan" if field.type == "span" else f"walk{field.type}"
+        if field.quantifier == "*":
+            return [f"for (const value of {access}) {function}(value, visitor);"]
+        if field.quantifier == "?":
+            return [f"if ({access} != null) {function}({access}, visitor);"]
+        return [f"{function}({access}, visitor);"]
+
+    lines.append("export function walkSpan(node: Span, visitor: Visitor): void { visitor.visitSpan?.(node); }")
+    for definition in definitions.values():
+        lines += [f"export function walk{definition.name}(node: {definition.name}, visitor: Visitor): void {{", f"if (visitor.visit{definition.name}?.(node) === false) return;"]
+        if definition.fields is not None:
+            for field in definition.fields:
+                lines += children(field, "node." + field.name)
+        elif not definition.simple():
+            lines.append("switch (node.kind) {")
+            for variant in definition.variants:
+                lines.append(f"case {json.dumps(variant.name)}:")
+                for field in variant.fields:
+                    access = "node.data" + ("." + field.name if len(variant.fields) > 1 else "")
+                    lines += children(field, access)
+                lines.append("break;")
+            lines.append("}")
+        lines.append("}")
+    return "\n".join(lines) + "\n"
+
+
+def json_schema(definitions, schema_id):
+    def field_schema(field):
+        value = {"string": {"type": "string"}, "schema_id": {"const": schema_id}, "span": {"$ref": "#/$defs/Span"}}.get(field.type, {"$ref": f"#/$defs/{field.type}"})
+        if field.quantifier == "*":
+            value = {"type": "array", "items": value}
+        elif field.quantifier == "?":
+            value = {"anyOf": [value, {"type": "null"}]}
+        return value
+
+    def object_schema(properties, required=None):
+        return {"type": "object", "properties": properties, "required": list(properties) if required is None else required, "additionalProperties": False}
+
+    definitions_json = {"Span": object_schema({name: {"type": "integer", "minimum": 0, "maximum": 9007199254740991} for name in ["start", "end"]})}
+    for definition in definitions.values():
+        if definition.fields is not None:
+            value = object_schema({f.name: field_schema(f) for f in definition.fields}, [f.name for f in definition.fields if f.quantifier != "?"])
+        elif definition.simple():
+            value = {"enum": [v.name for v in definition.variants]}
+        else:
+            variants = []
+            for variant in definition.variants:
+                properties = {"kind": {"const": variant.name}}
+                if variant.fields:
+                    properties["data"] = field_schema(variant.fields[0]) if len(variant.fields) == 1 else object_schema({f.name: field_schema(f) for f in variant.fields}, [f.name for f in variant.fields if f.quantifier != "?"])
+                variants.append(object_schema(properties))
+            value = {"oneOf": variants}
+        definitions_json[definition.name] = value
+    return json.dumps({"$schema": "https://json-schema.org/draft/2020-12/schema", "$id": schema_id, "$ref": "#/$defs/AstDocument", "$defs": definitions_json}, ensure_ascii=False, indent=2) + "\n"
+
+
+def generate(text):
+    definitions = parse_schema(text)
+    for required in ("AstDocument", "AstRoot"):
+        if required not in definitions:
+            raise ValueError(f"missing required schema type: {required}")
+    schema_id = "urn:endor:ast:sha256:" + hashlib.sha256(text.encode("utf-8")).hexdigest()
+    return {
+        "crates/frontend/src/ast/generated.rs": rust_types(definitions, schema_id),
+        "crates/frontend/src/ast/visitor.rs": rust_visitor(definitions),
+        "crates/frontend/editor/ast.ts": editor_types(definitions, schema_id),
+        "crates/frontend/editor/ast.schema.json": json_schema(definitions, schema_id),
+    }
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    mode = parser.add_mutually_exclusive_group(required=True)
+    mode.add_argument("--check", action="store_true")
+    mode.add_argument("--write", action="store_true")
+    args = parser.parse_args()
+    root = Path(__file__).resolve().parents[1]
+    outputs = generate((root / "crates/frontend/Endor.asdl").read_text(encoding="utf-8"))
+    stale = []
+    for relative, content in outputs.items():
+        if relative.endswith(".rs"):
+            content = subprocess.run(["rustfmt", "--edition", "2024", "--emit", "stdout"], input=content, text=True, capture_output=True, check=True).stdout
+        path = root / relative
+        if args.write:
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_text(content, encoding="utf-8")
+        elif not path.is_file() or path.read_text(encoding="utf-8") != content:
+            stale.append(relative)
+    if stale:
+        parser.exit(1, "stale generated AST files:\n" + "\n".join(stale) + "\nRun python3 scripts/generate_ast.py --write\n")
+    print("AST artifacts generated." if args.write else "AST artifacts match Endor.asdl.")
+
+
+if __name__ == "__main__":
+    main()

scripts/test_generate_ast.py

@@ -0,0 +1,57 @@
+"""Generator regression checks for the designated remote validation environment."""
+
+from pathlib import Path
+import json
+import re
+import unittest
+
+from generate_ast import generate, parse_schema, recursive_edges
+
+
+SCHEMA = (Path(__file__).resolve().parents[1] / "crates/frontend/Endor.asdl").read_text(encoding="utf-8")
+
+
+class AstGenerationTests(unittest.TestCase):
+    def test_editor_walkers_only_call_declared_functions(self):
+        editor = generate(SCHEMA)["crates/frontend/editor/ast.ts"]
+        declared = set(re.findall(r"export function (walk\w+)\(", editor))
+        calls = set(re.findall(r"\b(walk\w+)\(", editor))
+        self.assertLessEqual(calls, declared)
+        self.assertIn("walkSpan(node.span, visitor)", editor)
+
+    def test_schema_changes_reach_all_languages_and_schema_identity(self):
+        before = generate(SCHEMA)
+        after = generate(SCHEMA.replace("UnaryOp = Negate", "UnaryOp = Negate | LogicalNot"))
+        self.assertIn("LogicalNot", after["crates/frontend/src/ast/generated.rs"])
+        self.assertIn('"LogicalNot"', after["crates/frontend/editor/ast.ts"])
+        editor_schema = json.loads(after["crates/frontend/editor/ast.schema.json"])
+        self.assertIn("LogicalNot", editor_schema["$defs"]["UnaryOp"]["enum"])
+        self.assertNotEqual(before["crates/frontend/src/ast/generated.rs"], after["crates/frontend/src/ast/generated.rs"])
+        self.assertEqual(after, generate(SCHEMA.replace("UnaryOp = Negate", "UnaryOp = Negate | LogicalNot")))
+
+    def test_recursive_optional_nodes_are_boxed_and_walked(self):
+        extended = SCHEMA.rsplit("}", 1)[0] + "Link = (Link? next)\n}"
+        outputs = generate(extended)
+        self.assertIn("pub next: Option<Box<Link>>", outputs["crates/frontend/src/ast/generated.rs"])
+        self.assertIn("visitor.visit_link(value)?", outputs["crates/frontend/src/ast/visitor.rs"])
+        self.assertIn("next?: Link | null", outputs["crates/frontend/editor/ast.ts"])
+        definitions = parse_schema(SCHEMA)
+        self.assertIn(("ExprKind", "Expr"), recursive_edges(definitions))
+        self.assertNotIn(("Expr", "ExprKind"), recursive_edges(definitions))
+
+    def test_invalid_schemas_fail_instead_of_emitting_partial_types(self):
+        for schema in [
+            "module Endor { Root = (Missing value) }",
+            "module Endor { Root = (string x, string x) }",
+            "module Endor { Root = A | A }",
+            "module Endor { Root = A Root = B }",
+            "module Endor { Root = (string value",
+            "module Endor { Root = A } trailing",
+            "module Endor { Root = @ }",
+        ]:
+            with self.subTest(schema=schema), self.assertRaises(ValueError):
+                parse_schema(schema)
+
+
+if __name__ == "__main__":
+    unittest.main()