diff --git a/.dockerignore b/.dockerignore index 12ecee6..d6cd567 100644 --- a/.dockerignore +++ b/.dockerignore @@ -3,6 +3,7 @@ target data work outputs +ci-output .env *.db *.db-shm diff --git a/.gitea/workflows/README.md b/.gitea/workflows/README.md index 05a4fb8..c2cf76c 100644 --- a/.gitea/workflows/README.md +++ b/.gitea/workflows/README.md @@ -2,7 +2,9 @@ [ci-deploy.yml](ci-deploy.yml) 是当前唯一工作流:main push 或 workflow_dispatch 触发,运行于 ces-control,生产并发组串行且不取消正在运行的发布,任务超时 60 分钟。 -顺序为检出 → ci-ces-control.sh 验证 → 构建 SHA 生产镜像 → main 分支执行 deploy-ces-control.sh。设置 DOCKER_BUILDKIT=0 使用当前 builder 路径,检出不保留认证信息;工作流不更改共享 Runner。PostgreSQL 保留为预置资源,基础信息由文件保存;发布只为旧基础信息做一次只读导入,不再写入数据库。 +顺序为检出 → ci-ces-control.sh 验证及执行 Endor 示例 → 展示报告并上传 artifact → 构建 SHA 生产镜像 → main 分支执行 deploy-ces-control.sh。报告展示和上传使用 always(),验证失败也保留诊断。Artifact 名为 endor-execution-output-,包含 endor-examples.md 和 endor-examples.json,保留 30 天;从对应 Actions run 页面下载。使用 actions/upload-artifact@v3 连接当前 Gitea 的 artifact 服务,接口参考 [action 定义](https://github.com/actions/upload-artifact/blob/v3/action.yml)。 + +设置 DOCKER_BUILDKIT=0 使用当前 builder 路径,检出不保留认证信息;工作流不更改共享 Runner。PostgreSQL 保留为预置资源,基础信息由文件保存;发布只为旧基础信息做一次只读导入,不再写入数据库。 脚本职责见 [scripts](../../scripts/README.md),完整发布、保留数据和回退规则见 [部署](../../docs/operations/deployment.md)。main 推送会进入现有发布链,不能把推送成功写成 CI 或部署成功;按用户约定不主动跟踪状态。 diff --git a/.gitea/workflows/ci-deploy.yml b/.gitea/workflows/ci-deploy.yml index 5a17e13..f323878 100644 --- a/.gitea/workflows/ci-deploy.yml +++ b/.gitea/workflows/ci-deploy.yml @@ -25,6 +25,21 @@ jobs: - name: Verify SQLite, PostgreSQL and host behavior run: sh scripts/ci-ces-control.sh + - name: Show Endor example execution output + if: always() + run: cat ci-output/endor-examples.md + + - name: Upload Endor example execution output + if: always() + uses: actions/upload-artifact@v3 + with: + name: endor-execution-output-${{ gitea.sha }} + path: | + ci-output/endor-examples.md + ci-output/endor-examples.json + if-no-files-found: error + retention-days: 30 + - name: Build production image run: docker build --target runtime -t "endorphin-aster-vm:$ASTER_RELEASE_SHA" . diff --git a/.gitignore b/.gitignore index 4c5bf6b..7cbc573 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ /data/ /work/ /outputs/ +/ci-output/ .env *.db *.db-shm diff --git a/README.md b/README.md index bbe4624..ab96f21 100644 --- a/README.md +++ b/README.md @@ -44,4 +44,4 @@ Endor 采用平台解释执行模型;AST 到字节码的生成属于解释器 本地不编译、不测试、不下载依赖;可做文档和静态文件检查。运行验证交给 ces-control;按用户约定不主动跟踪 CI/部署结果。命令和使用条件见 [验证](docs/operations/validation.md)。 -文档随代码版本更新:就地解释放各目录 README,跨目录契约放 docs,对外阅读副本以仓库文档为准。忽略的 work、target、data 和 Git 内部目录不属于需要维护 README 的源码目录。 +文档随代码版本更新:就地解释放各目录 README,跨目录契约放 docs,对外阅读副本以仓库文档为准。忽略的 work、target、data、ci-output 和 Git 内部目录不属于需要维护 README 的源码目录。Endor 示例实际输出由 CI 保存为 artifact,查看方式见 [验证报告](docs/operations/validation.md)。 diff --git a/crates/compiler/examples/README.md b/crates/compiler/examples/README.md index d0d77c5..50fa90f 100644 --- a/crates/compiler/examples/README.md +++ b/crates/compiler/examples/README.md @@ -4,6 +4,7 @@ | 文件 | 输入模式与预期结果 | | --- | --- | +| [cases.json](cases.json) | CI 实际输出报告的用例清单;覆盖本目录每个 .endor 文件,指定 run/eval 和精确预期 stdout | | [hello.endor](hello.endor) | run;main 返回“你好,Endor 🌱” | | [calls.endor](calls.endor) | run;前向函数引用、参数和 let,返回“函数调用已接通” | | [shadowing.endor](shadowing.endor) | run;参数遮蔽全局函数名并传递函数值,返回“局部名称优先” | @@ -19,3 +20,5 @@ cli eval crates/compiler/examples/literal.endor ``` run 默认调用无参数 main,可用 --entry name 选择另一个无参数入口;eval 解析一个完整表达式,二者都支持 - 读取 stdin 到 EOF。类型名称暂只保留语法,Callable 不要求预先注册语言类。例子不依赖 print 或其他默认标准库。编译器尚无算术运行语义;前端 basic.endor 含算术,只是解析示例,不能混作可执行示例。规则见 [编译契约](../../../docs/runtime/compiler.md),返回 [compiler](../README.md)。 + +CI 使用 [endor_examples.py](../../../scripts/endor_examples.py) 再次执行真实 CLI 并保存实际结果,不只显示 Rust 测试名称。对应 Actions run 的 endor-execution-output- artifact 包含源码、命令、预期/实际 stdout、stderr、退出码和状态;位置与失败规则见 [验证报告](../../../docs/operations/validation.md)。新增 .endor 示例时同步 cases.json,未覆盖文件会让报告步骤失败。 diff --git a/crates/compiler/examples/cases.json b/crates/compiler/examples/cases.json new file mode 100644 index 0000000..c5303ce --- /dev/null +++ b/crates/compiler/examples/cases.json @@ -0,0 +1,35 @@ +{ + "format_version": 1, + "cases": [ + { + "name": "hello", + "mode": "run", + "source": "hello.endor", + "expected_stdout": "你好,Endor 🌱\n" + }, + { + "name": "calls", + "mode": "run", + "source": "calls.endor", + "expected_stdout": "函数调用已接通\n" + }, + { + "name": "shadowing", + "mode": "run", + "source": "shadowing.endor", + "expected_stdout": "局部名称优先\n" + }, + { + "name": "number", + "mode": "run", + "source": "number.endor", + "expected_stdout": "-123456789012345678901234567890.00100\n" + }, + { + "name": "literal", + "mode": "eval", + "source": "literal.endor", + "expected_stdout": "表达式 → 字节码 → VM\n" + } + ] +} diff --git a/docs/README.md b/docs/README.md index 7582762..d9f0d52 100644 --- a/docs/README.md +++ b/docs/README.md @@ -22,7 +22,7 @@ | LC_CTYPE、UTF-8、显式数据库配置和执行限额 | [配置](operations/configuration.md) | [core](../crates/core/README.md)、[storage](../crates/storage/README.md)、[server](../crates/server/README.md) | | CLI/HTTP 怎么运行,数据库在哪里使用 | [运行](operations/README.md) | [cli](../crates/cli/README.md)、[server](../crates/server/README.md) | | Docker、PostgreSQL、ces-control、回退与数据目录 | [部署](operations/deployment.md) | [scripts](../scripts/README.md)、[工作流](../.gitea/workflows/README.md) | -| 检查覆盖什么,结果如何确认 | [验证](operations/validation.md) | [objects/tests](../crates/objects/tests/README.md)、[core/tests](../crates/core/tests/README.md)、[storage/tests](../crates/storage/tests/README.md) | +| 检查覆盖什么,CI 中哪里查看 Endor 实际输出与 artifact | [验证](operations/validation.md) | [示例清单](../crates/compiler/examples/cases.json)、[报告脚本](../scripts/endor_examples.py)、[core/tests](../crates/core/tests/README.md)、[storage/tests](../crates/storage/tests/README.md) | | 改代码后文档写在哪里、怎样避免失效 | [维护规则](contributing/README.md) | 各目录 README 与本索引 | ## 多层目录 diff --git a/docs/contributing/handoff.md b/docs/contributing/handoff.md index cd39539..e15067b 100644 --- a/docs/contributing/handoff.md +++ b/docs/contributing/handoff.md @@ -22,6 +22,7 @@ Objects 零依赖,统一拥有核心对象、Number、代码对象/指令数 - Host 文件信息独立于数据库:host.version、deployment.release 与 metadata put/get 使用 UTF-8 JSON。旧 PostgreSQL 基础信息只读导入缺失文件,不删除原库,见 [文件信息](../operations/host-information.md)。 - Storage 只提供显式 DatabaseConfig/DatabaseConnection;没有默认数据库、自动应用建表或基础信息 KV。特殊类的名称/接口尚未定义,没有将连接接入 VM。 - CLI 提供引导、解析、源码执行与文件信息,Server HTTP 存活/只读 VM 信息;生产 PostgreSQL 作为未来特殊类的预置资源保留。 +- CI 使用 examples/cases.json 执行全部 Endor 示例,保存实际 stdout/stderr、退出码及状态;对应 run 页面展示 Markdown,并上传 endor-execution-output- artifact(Markdown/JSON,保留 30 天),失败也保留报告。查看方式见 [验证](../operations/validation.md),本地不运行报告脚本。 ## 资料位置 diff --git a/docs/operations/deployment.md b/docs/operations/deployment.md index 5635e54..5c773c0 100644 --- a/docs/operations/deployment.md +++ b/docs/operations/deployment.md @@ -30,7 +30,7 @@ docker compose -f docker-compose.postgres.yml exec aster cli bootstrap [工作流](../../.gitea/workflows/README.md) 响应 main push 和 workflow_dispatch,在 ces-control 上串行运行: -1. 检出代码,调用 ci-ces-control.sh;在 Docker 中检查 workspace(含 SQLite 适配)、独立 HTTP 冒烟及专用真实 PostgreSQL。 +1. 检出代码,调用 ci-ces-control.sh;在 Docker 中检查 workspace(含 SQLite 适配)、Endor 示例实际输出、独立 HTTP 冒烟及专用真实 PostgreSQL。清理检查容器前复制报告到宿主 ci-output,工作流无论验证成功或失败均展示并上传 endor-execution-output- artifact;字段与位置见 [验证报告](validation.md)。 2. 检查成功后构建 endorphin-aster-vm: 生产镜像。 3. main 上调用 deploy-ces-control.sh 更新 VM 宿主并保留预置 PostgreSQL。 diff --git a/docs/operations/validation.md b/docs/operations/validation.md index 93cbbee..a318a80 100644 --- a/docs/operations/validation.md +++ b/docs/operations/validation.md @@ -16,10 +16,21 @@ cargo test -p storage --test postgres --locked -- --ignored python3 scripts/smoke.py ``` -check.sh 先运行 generate_ast.py --check 验证 ASDL 产物同步,以及仅远端执行的 test_generate_ast.py 生成器回归检查,再执行 cargo fmt --all --check、独立词法/语法文件的 rustfmt --edition 2024 --check crates/frontend/src/lexer.peg crates/frontend/src/grammar.peg、cargo test --workspace --locked 和 cargo clippy --workspace --all-targets --locked -- -D warnings。workspace 测试也包含 rustdoc 示例;显式忽略的 PostgreSQL 测试需要单独执行。 +check.sh 先运行 generate_ast.py --check 验证 ASDL 产物同步,以及仅远端执行的 test_generate_ast.py 生成器回归检查、test_endor_examples.py 报告失败路径检查,再执行 cargo fmt --all --check、独立词法/语法文件的 rustfmt --edition 2024 --check crates/frontend/src/lexer.peg crates/frontend/src/grammar.peg、cargo test --workspace --locked 和 cargo clippy --workspace --all-targets --locked -- -D warnings。workspace 测试也包含 rustdoc 示例;显式忽略的 PostgreSQL 测试需要单独执行。 ci-ces-control.sh 负责准备全新专用临时 PostgreSQL 和测试 URL;成功路径使用真实数据库,不用模拟 socket 代替。测试库与生产数据不共享挂载。没有 ASTER_TEST_POSTGRES_URL 时,显式执行 PostgreSQL 测试会失败,不标记为通过。 +## Endor 示例实际输出 + +在对应 Gitea 仓库的 Actions → 本次提交的 run 页面,打开 Show Endor example execution output 步骤可看实际运行报告;下载 endor-execution-output- artifact 可保存两种格式: + +- endor-examples.md:逐例展示源码、执行命令、预期值、实际 stdout/stderr、退出码和 PASS/FAIL/NOT_RUN。 +- endor-examples.json:保留精确输出(包括换行)、提交 SHA、UTC 时间、构建/定位命令结果和逐例状态,供工具读取。 + +[endor_examples.py](../../scripts/endor_examples.py) 构建真实 CLI,通过 Cargo metadata 定位二进制并执行 [cases.json](../../crates/compiler/examples/cases.json) 中的全部示例;不以预期值冒充实际值。stdout 必须精确匹配、stderr 为空且退出码为 0 才通过。单例超时 30 秒;构建失败不运行示例并标记 NOT_RUN;报告本身的异常标记 FAIL。一个案例失败仍收集后续案例。示例报告不代表 workspace、数据库、宿主和部署全部通过,整体结果仍看工作流。 + +ci-ces-control.sh 在检查容器退出后、清理前将两份文件复制到宿主 ci-output;验证步骤提前失败时已有 NOT_RUN 占位说明。报告运行不掩盖前面检查的失败;后续数据库或宿主检查失败也不丢报告。工作流用 always() 展示并上传,artifact 保留 30 天,不写进 Git 或生产数据目录。普通 Actions 日志仍保留完整测试/构建信息;报告上传失败会使工作流失败。 + ## 覆盖范围与文件位置 | 范围 | 当前检查内容 | 入口 | diff --git a/scripts/README.md b/scripts/README.md index b54a413..c09425c 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -6,9 +6,11 @@ | --- | --- | | [generate_ast.py](generate_ast.py) | 从 Endor.asdl 生成并格式化 Rust/TypeScript/JSON Schema;--write 更新产物,--check 只比较,无依赖下载或语言执行 | | [test_generate_ast.py](test_generate_ast.py) | 远端 unittest:schema 扩展同步、递归装箱/遍历、非法定义拒绝 | -| [check.sh](check.sh) | 先检查 ASDL 产物同步和生成器行为,再运行 Cargo fmt 与独立 lexer.peg/grammar.peg 的 rustfmt 检查、workspace 测试、严格 Clippy;会使用 Cargo 依赖和构建产物 | +| [endor_examples.py](endor_examples.py) | 仅远端:构建 CLI,按 examples/cases.json 执行每个真实 Endor 文件,严格比较 stdout/stderr/退出码;保存源码、命令、实际输出、PASS/FAIL/NOT_RUN 的 Markdown 和 JSON,失败也写报告 | +| [test_endor_examples.py](test_endor_examples.py) | 仅远端:报告清单覆盖、精确输出、不匹配、构建失败、超时部分输出与异常时仍保存双格式报告;模拟子进程,不启动 VM | +| [check.sh](check.sh) | 先检查 ASDL 产物同步、生成器与输出报告行为,再运行 Cargo fmt 与独立 lexer.peg/grammar.peg 的 rustfmt 检查、workspace 测试、严格 Clippy;会使用 Cargo 依赖和构建产物 | | [smoke.py](smoke.py) | 标准库 Python 脚本;构建后用 Cargo metadata 定位 cli/server 的 debug 二进制,支持 CARGO_TARGET_DIR;文件 metadata 跨进程与日文值;失效数据库变量下独立启动、不创建 SQLite 文件、HTTP 与 SIGTERM,不改生产配置 | -| [ci-ces-control.sh](ci-ces-control.sh) | 需要 ASTER_RELEASE_SHA;构建 ci 镜像、创建临时网络/PostgreSQL/检查容器,运行全部检查;退出清理临时容器和网络,保留专属远端 Cargo 缓存卷 | +| [ci-ces-control.sh](ci-ces-control.sh) | 需要 ASTER_RELEASE_SHA;构建 ci 镜像、创建临时网络/PostgreSQL/检查容器;检查后独立执行 Endor 示例,失败不掩盖原检查状态;清理容器前复制两份报告到宿主 ci-output,提前失败也保留 NOT_RUN 说明;退出清理临时容器和网络,保留专属远端 Cargo 缓存卷 | | [deploy-ces-control.sh](deploy-ces-control.sh) | ces-control root 执行;校验已构建镜像、锁定发布、保留密码和数据、验证 HTTP 与无数据库 CLI 引导后切换 current,失败尝试恢复旧版;保留镜像;应用服务 aster 内调用 cli bootstrap,基础信息改写文件;一次性只读导入旧基础信息,不覆盖新文件或删除原库 | CI/部署使用的镜像与路径仅属于本项目,禁止改动其他项目和共享 Runner。smoke.py 临时选择端口后再启动宿主,存在小的端口竞争窗口,指定环境遇到冲突可重试。 diff --git a/scripts/check.sh b/scripts/check.sh index 2a5e793..32ac8a7 100644 --- a/scripts/check.sh +++ b/scripts/check.sh @@ -2,6 +2,7 @@ set -eu python3 scripts/generate_ast.py --check python3 scripts/test_generate_ast.py +python3 scripts/test_endor_examples.py cargo fmt --all --check rustfmt --edition 2024 --check crates/frontend/src/lexer.peg crates/frontend/src/grammar.peg cargo test --workspace --locked diff --git a/scripts/ci-ces-control.sh b/scripts/ci-ces-control.sh index 9df0f3d..eeb5273 100644 --- a/scripts/ci-ces-control.sh +++ b/scripts/ci-ces-control.sh @@ -12,6 +12,12 @@ network="endorphin-aster-vm-ci-$ci_id" postgres="endorphin-aster-vm-ci-pg-$ci_id" checks="endorphin-aster-vm-ci-checks-$ci_id" image="endorphin-aster-vm-ci:$release_sha" +report_dir="$PWD/ci-output" +mkdir -p "$report_dir" +# Reset both outputs before any build, so even an early failure has an honest +# NOT_RUN artifact rather than a previous attempt's example results. +printf '# Endor 示例实际运行结果\n\n提交:%s\n\n状态:NOT_RUN\n\nCI 尚未完成示例执行;请查看验证步骤日志。\n' "$release_sha" > "$report_dir/endor-examples.md" +printf '{"format_version":1,"commit":"%s","status":"NOT_RUN","error":"CI did not finish example execution; inspect the verification log.","cases":[]}\n' "$release_sha" > "$report_dir/endor-examples.json" cleanup() { docker rm -f "$checks" "$postgres" >/dev/null 2>&1 || true docker network rm "$network" >/dev/null 2>&1 || true @@ -41,12 +47,29 @@ if [ "$ready" != true ]; then exit 1 fi +check_status=0 docker run --name "$checks" --init --network "$network" \ --mount type=volume,src=endorphin-aster-vm-ci-cargo,dst=/usr/local/cargo/registry \ --mount type=volume,src=endorphin-aster-vm-ci-target,dst=/build/target \ --env ASTER_TEST_POSTGRES_URL='postgres://aster:aster-ci-only@postgres:5432/aster?sslmode=disable' \ + --env ASTER_RELEASE_SHA="$release_sha" \ "$image" sh -eu -c ' - sh scripts/check.sh + checks=0 + sh scripts/check.sh || checks=$? + examples=0 + python3 scripts/endor_examples.py --output-dir /build/ci-output || examples=$? + test "$checks" -eq 0 + test "$examples" -eq 0 cargo test -p storage --test postgres --locked -- --ignored python3 scripts/smoke.py - ' + ' || check_status=$? + +# Copy reports before the EXIT trap removes the stopped checks container. +# Keep the original validation failure status if report extraction also fails. +copy_status=0 +docker cp "$checks:/build/ci-output/endor-examples.md" "$report_dir/endor-examples.md" || copy_status=$? +docker cp "$checks:/build/ci-output/endor-examples.json" "$report_dir/endor-examples.json" || copy_status=$? +if [ "$check_status" -ne 0 ]; then + exit "$check_status" +fi +exit "$copy_status" diff --git a/scripts/endor_examples.py b/scripts/endor_examples.py new file mode 100644 index 0000000..cea42e6 --- /dev/null +++ b/scripts/endor_examples.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +"""Run Endor example files and retain actual output on the designated CI host. + +Never run locally: this builds the CLI and executes language code. +""" + +import argparse +from datetime import datetime, timezone +import json +import os +from pathlib import Path +import re +import shlex +import subprocess + + +ROOT = Path(__file__).resolve().parents[1] +EXAMPLES = ROOT / "crates/compiler/examples" + + +def decoded(value): + return value.decode("utf-8", "replace") if isinstance(value, bytes) else value or "" + + +def invoke(command, timeout): + """Keep stdout, stderr and failure state even for launch errors/timeouts.""" + record = {"command": command, "exit_code": None, "stdout": "", "stderr": "", "error": None} + try: + result = subprocess.run(command, cwd=ROOT, capture_output=True, text=True, + encoding="utf-8", errors="replace", timeout=timeout) + record.update(exit_code=result.returncode, stdout=result.stdout, stderr=result.stderr) + except subprocess.TimeoutExpired as error: + record.update(stdout=decoded(error.stdout), stderr=decoded(error.stderr), + error=f"timed out after {timeout} seconds") + except OSError as error: + record["error"] = str(error) + return record + + +def load_cases(): + manifest = json.loads((EXAMPLES / "cases.json").read_text(encoding="utf-8")) + if manifest.get("format_version") != 1: + raise ValueError("unsupported example manifest version") + cases = manifest["cases"] + names, sources = set(), set() + for case in cases: + if not re.fullmatch(r"[a-z][a-z0-9_-]*", case["name"]) or case["name"] in names: + raise ValueError("invalid or duplicate example name") + source = EXAMPLES / case["source"] + if source.resolve().parent != EXAMPLES.resolve() or source.suffix != ".endor": + raise ValueError("example source must be a direct .endor file") + if case["mode"] not in ("run", "eval") or not isinstance(case["expected_stdout"], str): + raise ValueError("invalid example mode or expected output") + if case["source"] in sources: + raise ValueError("duplicate example source") + case["source_text"] = source.read_text(encoding="utf-8") + names.add(case["name"]) + sources.add(case["source"]) + if not cases or sources != {path.name for path in EXAMPLES.glob("*.endor")}: + raise ValueError("manifest must cover every .endor example exactly once") + return cases + + +def fence(text, language="text"): + # Keep source/output literal even if future examples print Markdown fences. + longest = max((len(match[0]) for match in re.finditer(r"`+", text)), default=0) + delimiter = "`" * max(3, longest + 1) + return f"{delimiter}{language}\n{text}" + ("" if text.endswith("\n") else "\n") + delimiter + + +def markdown(report): + lines = ["# Endor 示例实际运行结果", "", f"提交:`{report['commit']}`", + f"状态:**{report['status']}**", f"生成时间(UTC):{report['generated_at']}", + "", "本报告仅覆盖 Endor 示例;整个 CI/CD 的状态以工作流结果为准。"] + if report.get("error"): + lines += ["", "报告阶段错误:", fence(report["error"])] + for stage in report["preparation"]: + lines += ["", "## 准备:" + shlex.join(stage["command"]), + f"退出码:{stage['exit_code']};错误:{stage['error'] or '无'}"] + if stage["exit_code"] != 0 or stage["error"]: + lines += ["", "stdout:", fence(stage["stdout"]), "", "stderr:", fence(stage["stderr"])] + for case in report["cases"]: + lines += ["", f"## {case['name']} — {case['status']}", "", + "命令:", fence(shlex.join(case["command"]), "sh"), "", "源码:", + fence(case["source_text"], "endor"), "", "预期 stdout:", + fence(case["expected_stdout"]), "", "实际 stdout:", fence(case["stdout"]), + "", "实际 stderr:", fence(case["stderr"]), "", + f"预期退出码:0;实际退出码:{case['exit_code']}", + f"错误:{case['error'] or '无'}"] + return "\n".join(lines) + "\n" + + +def collect(report): + for case in load_cases(): + report["cases"].append({**case, "status": "NOT_RUN", "exit_code": None, + "stdout": "", "stderr": "", "error": None, + "expected_stderr": "", "expected_exit_code": 0, + "command": ["cli", case["mode"], "crates/compiler/examples/" + case["source"]]}) + build = invoke(["cargo", "build", "-p", "cli", "--locked"], timeout=600) + report["preparation"].append(build) + if build["exit_code"] != 0 or build["error"]: + report.update(status="NOT_RUN", error="CLI build failed; examples were not executed.") + return + metadata = invoke(["cargo", "metadata", "--locked", "--no-deps", "--format-version", "1"], timeout=60) + report["preparation"].append(metadata) + if metadata["exit_code"] != 0 or metadata["error"]: + report.update(status="NOT_RUN", error="Cannot locate the compiled CLI.") + return + binary = Path(json.loads(metadata["stdout"])["target_directory"]) / "debug" / "cli" + for case in report["cases"]: + command = [str(binary), *case["command"][1:]] + actual = invoke(command, timeout=30) + case.update({key: actual[key] for key in ["exit_code", "stdout", "stderr", "error"]}) + case["binary"] = str(binary) + case["status"] = "PASS" if ( + case["exit_code"] == case["expected_exit_code"] and not case["error"] + and case["stdout"] == case["expected_stdout"] + and case["stderr"] == case["expected_stderr"] + ) else "FAIL" + report["status"] = "PASS" if all(case["status"] == "PASS" for case in report["cases"]) else "FAIL" + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--output-dir", type=Path, required=True) + args = parser.parse_args() + args.output_dir.mkdir(parents=True, exist_ok=True) + report = {"format_version": 1, "commit": os.environ.get("ASTER_RELEASE_SHA", "unknown"), + "generated_at": datetime.now(timezone.utc).isoformat(), "status": "NOT_RUN", + "error": None, "preparation": [], "cases": []} + try: + collect(report) + except Exception as error: + report.update(status="FAIL", error=str(error)) + # Always persist diagnostics before returning a failing CI exit code. + args.output_dir.joinpath("endor-examples.json").write_text( + json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + rendered = markdown(report) + args.output_dir.joinpath("endor-examples.md").write_text(rendered, encoding="utf-8") + print(rendered) + return 0 if report["status"] == "PASS" else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test_endor_examples.py b/scripts/test_endor_examples.py new file mode 100644 index 0000000..7b08bc3 --- /dev/null +++ b/scripts/test_endor_examples.py @@ -0,0 +1,80 @@ +"""Remote-only regression checks for example output capture and failure reports.""" + +import contextlib +import io +import json +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest.mock import patch + +import endor_examples as examples + + +def result(stdout="", code=0): + return {"command": ["mock"], "exit_code": code, "stdout": stdout, "stderr": "", "error": None} + + +def report(): + return {"format_version": 1, "commit": "test", "generated_at": "test", "status": "NOT_RUN", + "preparation": [], "cases": [], "error": None} + + +class ExampleReportTests(unittest.TestCase): + def test_manifest_runs_every_source_and_captures_exact_output(self): + cases = examples.load_cases() + responses = [result(), result(json.dumps({"target_directory": "/tmp/ci-target"}))] + responses += [result(case["expected_stdout"]) for case in cases] + collected = report() + with patch.object(examples, "invoke", side_effect=responses) as invoke: + examples.collect(collected) + self.assertEqual(collected["status"], "PASS") + self.assertEqual(invoke.call_count, len(cases) + 2) + for actual, expected in zip(collected["cases"], cases): + self.assertEqual(actual["stdout"], expected["expected_stdout"]) + self.assertEqual(actual["status"], "PASS") + self.assertEqual(actual["binary"], "/tmp/ci-target/debug/cli") + + def test_output_mismatch_fails_and_build_failure_never_claims_examples_ran(self): + cases = examples.load_cases() + responses = [result(), result(json.dumps({"target_directory": "/tmp/target"}))] + responses += [result("wrong\n")] + [result(case["expected_stdout"]) for case in cases[1:]] + collected = report() + with patch.object(examples, "invoke", side_effect=responses): + examples.collect(collected) + self.assertEqual(collected["status"], "FAIL") + self.assertEqual(collected["cases"][0]["stdout"], "wrong\n") + self.assertEqual(collected["cases"][0]["status"], "FAIL") + self.assertIn("wrong", examples.markdown(collected)) + collected = report() + with patch.object(examples, "invoke", return_value=result(code=1)) as invoke: + examples.collect(collected) + self.assertEqual(invoke.call_count, 1) + self.assertEqual(collected["status"], "NOT_RUN") + self.assertTrue(all(case["status"] == "NOT_RUN" for case in collected["cases"])) + + def test_timeout_keeps_partial_utf8_stdout_and_stderr(self): + failure = subprocess.TimeoutExpired(["cli"], 1, output="日".encode(), stderr=b"partial error") + with patch.object(examples.subprocess, "run", side_effect=failure): + captured = examples.invoke(["cli"], 1) + self.assertIsNone(captured["exit_code"]) + self.assertEqual(captured["stdout"], "日") + self.assertEqual(captured["stderr"], "partial error") + self.assertIn("timed out", captured["error"]) + + def test_unexpected_failure_still_writes_both_artifacts(self): + with tempfile.TemporaryDirectory() as directory: + with patch("sys.argv", ["endor_examples.py", "--output-dir", directory]), \ + patch.object(examples, "collect", side_effect=ValueError("broken manifest")), \ + contextlib.redirect_stdout(io.StringIO()): + self.assertEqual(examples.main(), 1) + payload = json.loads(Path(directory, "endor-examples.json").read_text(encoding="utf-8")) + self.assertEqual(payload["status"], "FAIL") + self.assertEqual(payload["error"], "broken manifest") + self.assertIn("broken manifest", Path(directory, "endor-examples.md").read_text(encoding="utf-8")) + self.assertTrue(examples.fence("```\n日").startswith("````text\n")) + + +if __name__ == "__main__": + unittest.main()