ci: publish actual Endor example outputs as artifacts

scripts/README.md

@@ -6,9 +6,11 @@
 | --- | --- |
 | [generate_ast.py](generate_ast.py) | 从 Endor.asdl 生成并格式化 Rust／TypeScript／JSON Schema；--write 更新产物，--check 只比较，无依赖下载或语言执行 |
 | [test_generate_ast.py](test_generate_ast.py) | 远端 unittest：schema 扩展同步、递归装箱／遍历、非法定义拒绝 |
-| [check.sh](check.sh) | 先检查 ASDL 产物同步和生成器行为，再运行 Cargo fmt 与独立 lexer.peg／grammar.peg 的 rustfmt 检查、workspace 测试、严格 Clippy；会使用 Cargo 依赖和构建产物 |
+| [endor_examples.py](endor_examples.py) | 仅远端：构建 CLI，按 examples/cases.json 执行每个真实 Endor 文件，严格比较 stdout／stderr／退出码；保存源码、命令、实际输出、PASS／FAIL／NOT_RUN 的 Markdown 和 JSON，失败也写报告 |
+| [test_endor_examples.py](test_endor_examples.py) | 仅远端：报告清单覆盖、精确输出、不匹配、构建失败、超时部分输出与异常时仍保存双格式报告；模拟子进程，不启动 VM |
+| [check.sh](check.sh) | 先检查 ASDL 产物同步、生成器与输出报告行为，再运行 Cargo fmt 与独立 lexer.peg／grammar.peg 的 rustfmt 检查、workspace 测试、严格 Clippy；会使用 Cargo 依赖和构建产物 |
 | [smoke.py](smoke.py) | 标准库 Python 脚本；构建后用 Cargo metadata 定位 cli／server 的 debug 二进制，支持 CARGO_TARGET_DIR；文件 metadata 跨进程与日文值；失效数据库变量下独立启动、不创建 SQLite 文件、HTTP 与 SIGTERM，不改生产配置 |
-| [ci-ces-control.sh](ci-ces-control.sh) | 需要 ASTER_RELEASE_SHA；构建 ci 镜像、创建临时网络／PostgreSQL／检查容器，运行全部检查；退出清理临时容器和网络，保留专属远端 Cargo 缓存卷 |
+| [ci-ces-control.sh](ci-ces-control.sh) | 需要 ASTER_RELEASE_SHA；构建 ci 镜像、创建临时网络／PostgreSQL／检查容器；检查后独立执行 Endor 示例，失败不掩盖原检查状态；清理容器前复制两份报告到宿主 ci-output，提前失败也保留 NOT_RUN 说明；退出清理临时容器和网络，保留专属远端 Cargo 缓存卷 |
 | [deploy-ces-control.sh](deploy-ces-control.sh) | ces-control root 执行；校验已构建镜像、锁定发布、保留密码和数据、验证 HTTP 与无数据库 CLI 引导后切换 current，失败尝试恢复旧版；保留镜像；应用服务 aster 内调用 cli bootstrap，基础信息改写文件；一次性只读导入旧基础信息，不覆盖新文件或删除原库 |
 
 CI／部署使用的镜像与路径仅属于本项目，禁止改动其他项目和共享 Runner。smoke.py 临时选择端口后再启动宿主，存在小的端口竞争窗口，指定环境遇到冲突可重试。

scripts/check.sh

@@ -2,6 +2,7 @@
 set -eu
 python3 scripts/generate_ast.py --check
 python3 scripts/test_generate_ast.py
+python3 scripts/test_endor_examples.py
 cargo fmt --all --check
 rustfmt --edition 2024 --check crates/frontend/src/lexer.peg crates/frontend/src/grammar.peg
 cargo test --workspace --locked

scripts/ci-ces-control.sh

@@ -12,6 +12,12 @@ network="endorphin-aster-vm-ci-$ci_id"
 postgres="endorphin-aster-vm-ci-pg-$ci_id"
 checks="endorphin-aster-vm-ci-checks-$ci_id"
 image="endorphin-aster-vm-ci:$release_sha"
+report_dir="$PWD/ci-output"
+mkdir -p "$report_dir"
+# Reset both outputs before any build, so even an early failure has an honest
+# NOT_RUN artifact rather than a previous attempt's example results.
+printf '# Endor 示例实际运行结果\n\n提交：%s\n\n状态：NOT_RUN\n\nCI 尚未完成示例执行；请查看验证步骤日志。\n' "$release_sha" > "$report_dir/endor-examples.md"
+printf '{"format_version":1,"commit":"%s","status":"NOT_RUN","error":"CI did not finish example execution; inspect the verification log.","cases":[]}\n' "$release_sha" > "$report_dir/endor-examples.json"
 cleanup() {
     docker rm -f "$checks" "$postgres" >/dev/null 2>&1 || true
     docker network rm "$network" >/dev/null 2>&1 || true
@@ -41,12 +47,29 @@ if [ "$ready" != true ]; then
     exit 1
 fi
 
+check_status=0
 docker run --name "$checks" --init --network "$network" \
     --mount type=volume,src=endorphin-aster-vm-ci-cargo,dst=/usr/local/cargo/registry \
     --mount type=volume,src=endorphin-aster-vm-ci-target,dst=/build/target \
     --env ASTER_TEST_POSTGRES_URL='postgres://aster:aster-ci-only@postgres:5432/aster?sslmode=disable' \
+    --env ASTER_RELEASE_SHA="$release_sha" \
     "$image" sh -eu -c '
-        sh scripts/check.sh
+        checks=0
+        sh scripts/check.sh || checks=$?
+        examples=0
+        python3 scripts/endor_examples.py --output-dir /build/ci-output || examples=$?
+        test "$checks" -eq 0
+        test "$examples" -eq 0
         cargo test -p storage --test postgres --locked -- --ignored
         python3 scripts/smoke.py
-    '
+    ' || check_status=$?
+
+# Copy reports before the EXIT trap removes the stopped checks container.
+# Keep the original validation failure status if report extraction also fails.
+copy_status=0
+docker cp "$checks:/build/ci-output/endor-examples.md" "$report_dir/endor-examples.md" || copy_status=$?
+docker cp "$checks:/build/ci-output/endor-examples.json" "$report_dir/endor-examples.json" || copy_status=$?
+if [ "$check_status" -ne 0 ]; then
+    exit "$check_status"
+fi
+exit "$copy_status"

scripts/endor_examples.py

@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+"""Run Endor example files and retain actual output on the designated CI host.
+
+Never run locally: this builds the CLI and executes language code.
+"""
+
+import argparse
+from datetime import datetime, timezone
+import json
+import os
+from pathlib import Path
+import re
+import shlex
+import subprocess
+
+
+ROOT = Path(__file__).resolve().parents[1]
+EXAMPLES = ROOT / "crates/compiler/examples"
+
+
+def decoded(value):
+    return value.decode("utf-8", "replace") if isinstance(value, bytes) else value or ""
+
+
+def invoke(command, timeout):
+    """Keep stdout, stderr and failure state even for launch errors/timeouts."""
+    record = {"command": command, "exit_code": None, "stdout": "", "stderr": "", "error": None}
+    try:
+        result = subprocess.run(command, cwd=ROOT, capture_output=True, text=True,
+                                encoding="utf-8", errors="replace", timeout=timeout)
+        record.update(exit_code=result.returncode, stdout=result.stdout, stderr=result.stderr)
+    except subprocess.TimeoutExpired as error:
+        record.update(stdout=decoded(error.stdout), stderr=decoded(error.stderr),
+                      error=f"timed out after {timeout} seconds")
+    except OSError as error:
+        record["error"] = str(error)
+    return record
+
+
+def load_cases():
+    manifest = json.loads((EXAMPLES / "cases.json").read_text(encoding="utf-8"))
+    if manifest.get("format_version") != 1:
+        raise ValueError("unsupported example manifest version")
+    cases = manifest["cases"]
+    names, sources = set(), set()
+    for case in cases:
+        if not re.fullmatch(r"[a-z][a-z0-9_-]*", case["name"]) or case["name"] in names:
+            raise ValueError("invalid or duplicate example name")
+        source = EXAMPLES / case["source"]
+        if source.resolve().parent != EXAMPLES.resolve() or source.suffix != ".endor":
+            raise ValueError("example source must be a direct .endor file")
+        if case["mode"] not in ("run", "eval") or not isinstance(case["expected_stdout"], str):
+            raise ValueError("invalid example mode or expected output")
+        if case["source"] in sources:
+            raise ValueError("duplicate example source")
+        case["source_text"] = source.read_text(encoding="utf-8")
+        names.add(case["name"])
+        sources.add(case["source"])
+    if not cases or sources != {path.name for path in EXAMPLES.glob("*.endor")}:
+        raise ValueError("manifest must cover every .endor example exactly once")
+    return cases
+
+
+def fence(text, language="text"):
+    # Keep source/output literal even if future examples print Markdown fences.
+    longest = max((len(match[0]) for match in re.finditer(r"`+", text)), default=0)
+    delimiter = "`" * max(3, longest + 1)
+    return f"{delimiter}{language}\n{text}" + ("" if text.endswith("\n") else "\n") + delimiter
+
+
+def markdown(report):
+    lines = ["# Endor 示例实际运行结果", "", f"提交：`{report['commit']}`",
+             f"状态：**{report['status']}**", f"生成时间（UTC）：{report['generated_at']}",
+             "", "本报告仅覆盖 Endor 示例；整个 CI/CD 的状态以工作流结果为准。"]
+    if report.get("error"):
+        lines += ["", "报告阶段错误：", fence(report["error"])]
+    for stage in report["preparation"]:
+        lines += ["", "## 准备：" + shlex.join(stage["command"]),
+                  f"退出码：{stage['exit_code']}；错误：{stage['error'] or '无'}"]
+        if stage["exit_code"] != 0 or stage["error"]:
+            lines += ["", "stdout：", fence(stage["stdout"]), "", "stderr：", fence(stage["stderr"])]
+    for case in report["cases"]:
+        lines += ["", f"## {case['name']} — {case['status']}", "",
+                  "命令：", fence(shlex.join(case["command"]), "sh"), "", "源码：",
+                  fence(case["source_text"], "endor"), "", "预期 stdout：",
+                  fence(case["expected_stdout"]), "", "实际 stdout：", fence(case["stdout"]),
+                  "", "实际 stderr：", fence(case["stderr"]), "",
+                  f"预期退出码：0；实际退出码：{case['exit_code']}",
+                  f"错误：{case['error'] or '无'}"]
+    return "\n".join(lines) + "\n"
+
+
+def collect(report):
+    for case in load_cases():
+        report["cases"].append({**case, "status": "NOT_RUN", "exit_code": None,
+                                "stdout": "", "stderr": "", "error": None,
+                                "expected_stderr": "", "expected_exit_code": 0,
+                                "command": ["cli", case["mode"], "crates/compiler/examples/" + case["source"]]})
+    build = invoke(["cargo", "build", "-p", "cli", "--locked"], timeout=600)
+    report["preparation"].append(build)
+    if build["exit_code"] != 0 or build["error"]:
+        report.update(status="NOT_RUN", error="CLI build failed; examples were not executed.")
+        return
+    metadata = invoke(["cargo", "metadata", "--locked", "--no-deps", "--format-version", "1"], timeout=60)
+    report["preparation"].append(metadata)
+    if metadata["exit_code"] != 0 or metadata["error"]:
+        report.update(status="NOT_RUN", error="Cannot locate the compiled CLI.")
+        return
+    binary = Path(json.loads(metadata["stdout"])["target_directory"]) / "debug" / "cli"
+    for case in report["cases"]:
+        command = [str(binary), *case["command"][1:]]
+        actual = invoke(command, timeout=30)
+        case.update({key: actual[key] for key in ["exit_code", "stdout", "stderr", "error"]})
+        case["binary"] = str(binary)
+        case["status"] = "PASS" if (
+            case["exit_code"] == case["expected_exit_code"] and not case["error"]
+            and case["stdout"] == case["expected_stdout"]
+            and case["stderr"] == case["expected_stderr"]
+        ) else "FAIL"
+    report["status"] = "PASS" if all(case["status"] == "PASS" for case in report["cases"]) else "FAIL"
+
+
+def main():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--output-dir", type=Path, required=True)
+    args = parser.parse_args()
+    args.output_dir.mkdir(parents=True, exist_ok=True)
+    report = {"format_version": 1, "commit": os.environ.get("ASTER_RELEASE_SHA", "unknown"),
+              "generated_at": datetime.now(timezone.utc).isoformat(), "status": "NOT_RUN",
+              "error": None, "preparation": [], "cases": []}
+    try:
+        collect(report)
+    except Exception as error:
+        report.update(status="FAIL", error=str(error))
+    # Always persist diagnostics before returning a failing CI exit code.
+    args.output_dir.joinpath("endor-examples.json").write_text(
+        json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
+    rendered = markdown(report)
+    args.output_dir.joinpath("endor-examples.md").write_text(rendered, encoding="utf-8")
+    print(rendered)
+    return 0 if report["status"] == "PASS" else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/test_endor_examples.py

@@ -0,0 +1,80 @@
+"""Remote-only regression checks for example output capture and failure reports."""
+
+import contextlib
+import io
+import json
+from pathlib import Path
+import subprocess
+import tempfile
+import unittest
+from unittest.mock import patch
+
+import endor_examples as examples
+
+
+def result(stdout="", code=0):
+    return {"command": ["mock"], "exit_code": code, "stdout": stdout, "stderr": "", "error": None}
+
+
+def report():
+    return {"format_version": 1, "commit": "test", "generated_at": "test", "status": "NOT_RUN",
+            "preparation": [], "cases": [], "error": None}
+
+
+class ExampleReportTests(unittest.TestCase):
+    def test_manifest_runs_every_source_and_captures_exact_output(self):
+        cases = examples.load_cases()
+        responses = [result(), result(json.dumps({"target_directory": "/tmp/ci-target"}))]
+        responses += [result(case["expected_stdout"]) for case in cases]
+        collected = report()
+        with patch.object(examples, "invoke", side_effect=responses) as invoke:
+            examples.collect(collected)
+        self.assertEqual(collected["status"], "PASS")
+        self.assertEqual(invoke.call_count, len(cases) + 2)
+        for actual, expected in zip(collected["cases"], cases):
+            self.assertEqual(actual["stdout"], expected["expected_stdout"])
+            self.assertEqual(actual["status"], "PASS")
+            self.assertEqual(actual["binary"], "/tmp/ci-target/debug/cli")
+
+    def test_output_mismatch_fails_and_build_failure_never_claims_examples_ran(self):
+        cases = examples.load_cases()
+        responses = [result(), result(json.dumps({"target_directory": "/tmp/target"}))]
+        responses += [result("wrong\n")] + [result(case["expected_stdout"]) for case in cases[1:]]
+        collected = report()
+        with patch.object(examples, "invoke", side_effect=responses):
+            examples.collect(collected)
+        self.assertEqual(collected["status"], "FAIL")
+        self.assertEqual(collected["cases"][0]["stdout"], "wrong\n")
+        self.assertEqual(collected["cases"][0]["status"], "FAIL")
+        self.assertIn("wrong", examples.markdown(collected))
+        collected = report()
+        with patch.object(examples, "invoke", return_value=result(code=1)) as invoke:
+            examples.collect(collected)
+        self.assertEqual(invoke.call_count, 1)
+        self.assertEqual(collected["status"], "NOT_RUN")
+        self.assertTrue(all(case["status"] == "NOT_RUN" for case in collected["cases"]))
+
+    def test_timeout_keeps_partial_utf8_stdout_and_stderr(self):
+        failure = subprocess.TimeoutExpired(["cli"], 1, output="日".encode(), stderr=b"partial error")
+        with patch.object(examples.subprocess, "run", side_effect=failure):
+            captured = examples.invoke(["cli"], 1)
+        self.assertIsNone(captured["exit_code"])
+        self.assertEqual(captured["stdout"], "日")
+        self.assertEqual(captured["stderr"], "partial error")
+        self.assertIn("timed out", captured["error"])
+
+    def test_unexpected_failure_still_writes_both_artifacts(self):
+        with tempfile.TemporaryDirectory() as directory:
+            with patch("sys.argv", ["endor_examples.py", "--output-dir", directory]), \
+                 patch.object(examples, "collect", side_effect=ValueError("broken manifest")), \
+                 contextlib.redirect_stdout(io.StringIO()):
+                self.assertEqual(examples.main(), 1)
+            payload = json.loads(Path(directory, "endor-examples.json").read_text(encoding="utf-8"))
+            self.assertEqual(payload["status"], "FAIL")
+            self.assertEqual(payload["error"], "broken manifest")
+            self.assertIn("broken manifest", Path(directory, "endor-examples.md").read_text(encoding="utf-8"))
+        self.assertTrue(examples.fence("```\n日").startswith("````text\n"))
+
+
+if __name__ == "__main__":
+    unittest.main()