Add Endor source reader before lexing with provenance and URI provider

crates/README.md

@@ -5,7 +5,7 @@
 | 目录 | 包名／产物 | 职责 |
 | --- | --- | --- |
 | [core](core/README.md) | core，库 endor_core | 零外部依赖的对象运行时、堆与字节码 VM |
-| [frontend](frontend/README.md) | frontend | 手写 lexer、token PEG parser、基础函数／表达式 AST，不执行源码 |
+| [frontend](frontend/README.md) | frontend | Endor 阅读器、手写 lexer、token PEG parser、基础函数／表达式 AST，不执行源码 |
 | [host](host/README.md) | host | UTF-8 JSON 文件基础信息，与数据库分离 |
 | [storage](storage/README.md) | storage | 为数据库特殊类保留的显式连接适配，不提供基础信息存储 |
 | [cli](cli/README.md) | cli，二进制 cli | 引导演示、源码解析与文件信息命令，不使用数据库 |

crates/cli/README.md

@@ -5,7 +5,7 @@
 | 内容 | 职责 |
 | --- | --- |
 | Cargo.toml | cli 二进制与 core／frontend／host 依赖；无 storage 或 Tokio 依赖 |
-| [src](src/README.md) | 参数路由、引导演示、源码解析和退出码 |
+| [src](src/README.md) | 参数路由、引导演示、文件／stdin 阅读、源码解析和退出码 |
 | [tests](tests/README.md) | parse 命令的 AST 输出、错误位置和严格 UTF-8 检查 |
 
-运行方式见 [运行指南](../../docs/operations/README.md)，输入边界见 [配置](../../docs/operations/configuration.md)。parse 严格读取 UTF-8 文件并输出 AST，不执行源码，见 [前端](../../docs/runtime/frontend.md)；metadata put/get 和只导入缺失项的 import 均使用文件，见 [宿主信息](../../docs/operations/host-information.md)。返回 [工作区](../README.md)。
+运行方式见 [运行指南](../../docs/operations/README.md)，输入边界见 [配置](../../docs/operations/configuration.md)。parse 通过 EndorReader 严格读取 UTF-8 文件或 `-` 指定的 stdin 并输出 AST，不执行源码，见 [前端](../../docs/runtime/frontend.md)；metadata put/get 和只导入缺失项的 import 均使用文件，见 [宿主信息](../../docs/operations/host-information.md)。返回 [工作区](../README.md)。

crates/cli/src/README.md

@@ -1,7 +1,7 @@
 # CLI 入口
 
-[main.rs](main.rs) 是唯一入口：同步 main 将 run 的结果转成进程退出码；run 处理 bootstrap、parse 文件、help、文件 metadata put/get/import，错误打印到 stderr 并返回失败。
+[main.rs](main.rs) 是唯一入口：同步 main 将 run 的结果转成进程退出码；run 处理 bootstrap、parse 文件／stdin、help、文件 metadata put/get/import，错误打印到 stderr 并返回失败。
 
-bootstrap 创建并显式释放两个对象，不读取数据库环境、不打开数据库。parse 使用 frontend 输出 AST，词法／语法错误显示文件、行、列；不创建 VM 或执行源码。未知命令报错；metadata 通过 host::MetadataStore 读写文件，import 从标准输入接收旧记录且不覆盖已有文件。参数目前由 std::env::args 读取，非 Unicode 参数可能 panic，边界见 [配置](../../../docs/operations/configuration.md)。
+bootstrap 创建并显式释放两个对象，不读取数据库环境、不打开数据库。parse 使用 EndorReader 和 parse_source 输出 AST；`-` 从 stdin 读到 EOF，来源为 `<stdin>`。词法／语法错误显示来源、行、列；不创建 VM 或执行源码。未知命令报错；metadata 通过 host::MetadataStore 读写文件，import 从标准输入接收旧记录且不覆盖已有文件。参数目前由 std::env::args 读取，非 Unicode 参数可能 panic，边界见 [配置](../../../docs/operations/configuration.md)。
 
 新增命令时更新用法、[运行指南](../../../docs/operations/README.md) 和 [smoke.py 的导航](../../../scripts/README.md)。返回 [cli](../README.md)。

crates/cli/src/main.rs

@@ -2,9 +2,10 @@ use std::error::Error;
 use std::process::ExitCode;
 
 use endor_core::{Value, Vm};
+use frontend::{EndorReader, parse_source};
 use host::MetadataStore;
 
-const USAGE: &str = "Usage: cli [bootstrap | parse <file> | metadata put <key> <value> | metadata get <key> | metadata import | help]";
+const USAGE: &str = "Usage: cli [bootstrap | parse <file|-> | metadata put <key> <value> | metadata get <key> | metadata import | help]";
 
 fn main() -> ExitCode {
     match run() {
@@ -46,15 +47,12 @@ fn run() -> Result<(), Box<dyn Error>> {
         }
         ["help"] | ["--help"] | ["-h"] => println!("{USAGE}"),
         ["parse", path] => {
-            let source =
-                std::fs::read_to_string(path).map_err(|error| format!("{path}: {error}"))?;
-            let program = frontend::parse(&source).map_err(|error| {
-                let location = error.location(&source);
-                format!(
-                    "{path}:{}:{}: {}",
-                    location.line, location.column, error.message
-                )
-            })?;
+            let source = if *path == "-" {
+                EndorReader::from_stream("<stdin>", std::io::stdin().lock())?
+            } else {
+                EndorReader::from_file(path)?
+            };
+            let program = parse_source(&source)?;
             println!("{program:#?}");
         }
         ["metadata", "put", key, value] => {

crates/cli/tests/README.md

@@ -1,5 +1,5 @@
 # CLI 命令检查
 
-[parse.rs](parse.rs) 使用 Cargo 提供的 cli 测试二进制检查 parse 命令：合法 UTF-8 源码输出 AST 且不执行名称调用；非法语法输出文件／行／列并返回失败；非法 UTF-8 不替换字符。
+[parse.rs](parse.rs) 使用 Cargo 提供的 cli 测试二进制检查 parse 命令：合法 UTF-8 源码输出 AST 且不执行名称调用；非法语法输出文件／行／列并返回失败；非法 UTF-8 不替换字符；`parse -` 检查 stdin 输入、EOF 和带流名的错误。
 
 测试仅在 ces-control 的工作区检查中运行，本地不执行。测试临时文件使用独立名称并在结束时清理。返回 [cli](../README.md)，验证约定见 [验证](../../../docs/operations/validation.md)。

crates/cli/tests/parse.rs

@@ -1,5 +1,6 @@
+use std::io::Write;
 use std::path::PathBuf;
-use std::process::Command;
+use std::process::{Command, Stdio};
 use std::sync::atomic::{AtomicUsize, Ordering};
 
 static NEXT_FILE: AtomicUsize = AtomicUsize::new(0);
@@ -65,3 +66,42 @@ fn parse_command_reports_source_locations_and_rejects_invalid_utf8() {
     assert!(!output.status.success());
     assert!(output.stdout.is_empty());
 }
+
+fn parse_stdin(source: &[u8]) -> std::process::Output {
+    let mut child = Command::new(env!("CARGO_BIN_EXE_cli"))
+        .args(["parse", "-"])
+        .stdin(Stdio::piped())
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .spawn()
+        .unwrap();
+    let mut stdin = child.stdin.take().unwrap();
+    stdin.write_all(source).unwrap();
+    drop(stdin);
+    child.wait_with_output().unwrap()
+}
+
+#[test]
+fn parse_command_reads_stdin_to_eof_and_reports_its_origin() {
+    let output = parse_stdin("function 表示(): String { return \"日\" }".as_bytes());
+    assert!(
+        output.status.success(),
+        "{}",
+        String::from_utf8_lossy(&output.stderr)
+    );
+    assert!(String::from_utf8(output.stdout).unwrap().contains("表示"));
+    let output = parse_stdin(b"function f(): Number {\n  return )\n}");
+    assert!(!output.status.success());
+    assert!(
+        String::from_utf8(output.stderr)
+            .unwrap()
+            .contains("<stdin>:2:10:")
+    );
+    let output = parse_stdin(&[0xff]);
+    assert!(!output.status.success());
+    assert!(
+        String::from_utf8(output.stderr)
+            .unwrap()
+            .contains("<stdin>: invalid UTF-8 at byte 0")
+    );
+}

crates/frontend/README.md

@@ -1,12 +1,12 @@
 # frontend
 
-独立源码前端：手写 Rust lexer → token → rust-peg parser → 拥有文本的 AST。当前解析基础函数与表达式，不生成字节码、不进行类型／名称解析、不执行源码。语法是可继续调整的首版实现，当前契约见 [源码前端](../../docs/runtime/frontend.md)。
+独立源码前端：EndorReader → SourceUnit → 手写 Rust lexer → token → rust-peg parser → 拥有文本的 AST。当前解析基础函数与表达式，不生成字节码、不进行类型／名称解析、不执行源码。语法是可继续调整的首版实现，当前契约见 [源码前端](../../docs/runtime/frontend.md)。
 
 | 内容 | 职责 |
 | --- | --- |
 | Cargo.toml | frontend 包与 peg 0.8.6 依赖，不依赖 core、Host 或 Storage |
-| [src](src/README.md) | token、源码范围、诊断、AST、词法；PEG 规则独立在 [grammar.peg](src/grammar.peg) |
+| [src](src/README.md) | 阅读器、SourceUnit、token、源码范围、诊断、AST、词法；PEG 规则独立在 [grammar.peg](src/grammar.peg) |
 | [tests](tests/README.md) | Unicode、优先级、函数、完整输入与错误位置验证 |
 | [examples](examples/README.md) | 可交给 cli parse 的源码样例 |
 
-公开入口为 lex、parse、parse_expression；CLI 文件读取在宿主层。Core 仍为零外部依赖；未来编译器将 AST 转成已有字节码，不改为 AST 直接解释。返回 [工作区](../README.md)。
+宿主通过 EndorReader 读取字符串、流、文件或注入的 URI provider，取得 SourceUnit 后调用 lex_source、parse_source、parse_source_expression；低层 lex、parse、parse_expression 仍保留。读取契约见 [Endor 阅读器](../../docs/runtime/reader.md)。Core 仍为零外部依赖；未来编译器将 AST 转成已有字节码，不改为 AST 直接解释。返回 [工作区](../README.md)。

crates/frontend/src/README.md

@@ -2,7 +2,9 @@
 
 | 文件 | 职责 |
 | --- | --- |
-| [lib.rs](lib.rs) | 公开 AST 模块、lex／parse／parse_expression 与 token、诊断导出 |
+| [lib.rs](lib.rs) | 公开 AST 模块、阅读器／来源单元、低层及带来源的词法／语法入口与诊断导出 |
+| [source.rs](source.rs) | 不可变 SourceUnit、来源类别和携带来源／行列的 SourceDiagnostic |
+| [reader.rs](reader.rs) | 字符串、流、文件读取、严格 UTF-8 解码与宿主 EndorSourceProvider 适配 |
 | [token.rs](token.rs) | 可复制的 token 种类和原始源码范围，文本由源码切片取得 |
 | [diagnostic.rs](diagnostic.rs) | UTF-8 字节范围、词法／语法诊断与一基行列转换 |
 | [lexer.rs](lexer.rs) | 名称、数字、字符串、标点、空白与注释扫描；字符串校验及解码 |
@@ -10,4 +12,4 @@
 | [parser.rs](parser.rs) | 解析 API、错误映射、AST 辅助函数，以及独立语法文件的编译期引入 |
 | [grammar.peg](grammar.peg) | token 输入的 PEG 规则、运算符优先级与 AST 构造动作；保留 peg::parser! 外层声明 |
 
-改词法时同步 token 和错误检查；改语法从 grammar.peg 进入，并同步 AST、[测试](../tests/README.md)、样例与 [前端契约](../../../docs/runtime/frontend.md)。不在语法动作中做 VM 分配、I/O 或修改外部状态，避免回溯造成副作用。返回 [frontend](../README.md)。
+改输入适配时同步 [阅读器契约](../../../docs/runtime/reader.md) 与读取错误检查；改词法时同步 token 和错误检查；改语法从 grammar.peg 进入，并同步 AST、[测试](../tests/README.md)、样例与 [前端契约](../../../docs/runtime/frontend.md)。不在语法动作中做 VM 分配、I/O 或修改外部状态，避免回溯造成副作用。返回 [frontend](../README.md)。

crates/frontend/src/lexer.rs

@@ -1,5 +1,10 @@
 use crate::{Diagnostic, Span, Token, TokenKind};
 
+/// Lex a reader snapshot and preserve provenance in any diagnostic.
+pub fn lex_source(source: &crate::SourceUnit) -> Result<Vec<Token>, crate::SourceDiagnostic> {
+    lex(source.text()).map_err(|error| source.annotate(error))
+}
+
 /// Lex the basic Endorphin draft syntax without reading files or changing
 /// locale. Positions always refer to bytes in this exact UTF-8 source.
 pub fn lex(source: &str) -> Result<Vec<Token>, Diagnostic> {

crates/frontend/src/lib.rs

@@ -1,4 +1,4 @@
-//! UTF-8 source frontend: a handwritten lexer and token-based rust-peg parser.
+//! UTF-8 source reader, handwritten lexer and token-based rust-peg parser.
 //! Produces syntax only; it does not resolve types, generate code or execute VM
 //! operations. AST data owns its text and does not borrow the source buffer.
 
@@ -6,9 +6,15 @@ pub mod ast;
 mod diagnostic;
 mod lexer;
 mod parser;
+mod reader;
+mod source;
 mod token;
 
 pub use diagnostic::{Diagnostic, DiagnosticKind, SourceLocation, Span};
-pub use lexer::lex;
-pub use parser::{parse, parse_expression};
+pub use lexer::{lex, lex_source};
+pub use parser::{parse, parse_expression, parse_source, parse_source_expression};
+pub use reader::{
+    EndorReader, EndorSourceProvider, ResolvedEndorSource, SourceReadError, SourceReadErrorKind,
+};
+pub use source::{SourceDiagnostic, SourceOrigin, SourceUnit};
 pub use token::{Token, TokenKind};

crates/frontend/src/parser.rs

@@ -4,6 +4,17 @@ use crate::ast::{
 use crate::lexer::decode_string;
 use crate::{Diagnostic, DiagnosticKind, Span, Token, TokenKind, lex};
 
+/// Parse a reader snapshot, retaining its origin and resolved revision on error.
+pub fn parse_source(source: &crate::SourceUnit) -> Result<Program, crate::SourceDiagnostic> {
+    parse(source.text()).map_err(|error| source.annotate(error))
+}
+
+pub fn parse_source_expression(
+    source: &crate::SourceUnit,
+) -> Result<Expr, crate::SourceDiagnostic> {
+    parse_expression(source.text()).map_err(|error| source.annotate(error))
+}
+
 /// Parse one or more basic typed function declarations. This is a syntax
 /// operation: type names and name references are retained without resolution.
 pub fn parse(source: &str) -> Result<Program, Diagnostic> {

crates/frontend/src/reader.rs

@@ -0,0 +1,147 @@
+use std::error::Error;
+use std::fmt;
+use std::fs::File;
+use std::io::{self, Read};
+use std::path::Path;
+use std::string::FromUtf8Error;
+
+use crate::{SourceOrigin, SourceUnit};
+
+/// The host resolves names, authorizes access and selects a revision before
+/// returning its source bytes. No URI path layout or hash algorithm is assumed.
+pub trait EndorSourceProvider {
+    fn load(&self, uri: &str) -> io::Result<ResolvedEndorSource>;
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ResolvedEndorSource {
+    pub revision: String,
+    pub bytes: Vec<u8>,
+}
+
+/// Stateless source adapters, before lexing. Streams are read to EOF; session
+/// prompting and incomplete REPL input detection belong to the interactive host.
+#[derive(Debug, Clone, Copy, Default)]
+pub struct EndorReader;
+
+impl EndorReader {
+    pub fn from_text(name: impl Into<String>, text: impl Into<String>) -> SourceUnit {
+        SourceUnit::new(SourceOrigin::Text { name: name.into() }, text.into())
+    }
+
+    pub fn from_stream(
+        name: impl Into<String>,
+        reader: impl Read,
+    ) -> Result<SourceUnit, SourceReadError> {
+        Self::read(SourceOrigin::Stream { name: name.into() }, reader)
+    }
+
+    pub fn from_file(path: impl AsRef<Path>) -> Result<SourceUnit, SourceReadError> {
+        let path = path.as_ref();
+        let origin = SourceOrigin::File {
+            path: path.to_owned(),
+        };
+        let file = File::open(path).map_err(|error| SourceReadError {
+            origin: origin.to_string(),
+            kind: SourceReadErrorKind::Io(error),
+        })?;
+        Self::read(origin, file)
+    }
+
+    pub fn from_endor<P: EndorSourceProvider + ?Sized>(
+        uri: &str,
+        provider: &P,
+    ) -> Result<SourceUnit, SourceReadError> {
+        // Only recognize the scheme and reject empty/control-filled locators.
+        // Namespace syntax, revision selectors and percent decoding are host policy.
+        let valid = uri.split_once(':').is_some_and(|(scheme, address)| {
+            scheme.eq_ignore_ascii_case("endor") && !address.is_empty()
+        }) && !uri
+            .chars()
+            .any(|character| character.is_whitespace() || character.is_control());
+        if !valid {
+            return Err(SourceReadError {
+                origin: uri.into(),
+                kind: SourceReadErrorKind::InvalidEndorUri,
+            });
+        }
+        let resolved = provider.load(uri).map_err(|error| SourceReadError {
+            origin: uri.into(),
+            kind: SourceReadErrorKind::Io(error),
+        })?;
+        if resolved.revision.trim().is_empty() {
+            return Err(SourceReadError {
+                origin: uri.into(),
+                kind: SourceReadErrorKind::MissingRevision,
+            });
+        }
+        let origin = SourceOrigin::Endor {
+            uri: uri.into(),
+            revision: resolved.revision,
+        };
+        Self::decode(origin, resolved.bytes)
+    }
+
+    fn read(origin: SourceOrigin, mut reader: impl Read) -> Result<SourceUnit, SourceReadError> {
+        let mut bytes = Vec::new();
+        reader
+            .read_to_end(&mut bytes)
+            .map_err(|error| SourceReadError {
+                origin: origin.to_string(),
+                kind: SourceReadErrorKind::Io(error),
+            })?;
+        Self::decode(origin, bytes)
+    }
+
+    fn decode(origin: SourceOrigin, bytes: Vec<u8>) -> Result<SourceUnit, SourceReadError> {
+        let text = String::from_utf8(bytes).map_err(|error| SourceReadError {
+            origin: origin.to_string(),
+            kind: SourceReadErrorKind::InvalidUtf8(error),
+        })?;
+        Ok(SourceUnit::new(origin, text))
+    }
+}
+
+#[derive(Debug)]
+pub enum SourceReadErrorKind {
+    Io(io::Error),
+    InvalidUtf8(FromUtf8Error),
+    InvalidEndorUri,
+    MissingRevision,
+}
+
+#[derive(Debug)]
+pub struct SourceReadError {
+    pub origin: String,
+    pub kind: SourceReadErrorKind,
+}
+
+impl fmt::Display for SourceReadError {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(formatter, "{}: ", self.origin)?;
+        match &self.kind {
+            SourceReadErrorKind::Io(error) => write!(formatter, "{error}"),
+            SourceReadErrorKind::InvalidUtf8(error) => write!(
+                formatter,
+                "invalid UTF-8 at byte {}",
+                error.utf8_error().valid_up_to()
+            ),
+            SourceReadErrorKind::InvalidEndorUri => formatter.write_str(
+                "expected a nonempty endor URI without whitespace or control characters",
+            ),
+            SourceReadErrorKind::MissingRevision => {
+                formatter.write_str("source provider returned no resolved revision")
+            }
+        }
+    }
+}
+
+impl Error for SourceReadError {
+    fn source(&self) -> Option<&(dyn Error + 'static)> {
+        match &self.kind {
+            SourceReadErrorKind::Io(error) => Some(error),
+            SourceReadErrorKind::InvalidUtf8(error) => Some(error),
+            _ => None,
+        }
+    }
+}

crates/frontend/src/source.rs

@@ -0,0 +1,79 @@
+use std::error::Error;
+use std::fmt;
+use std::path::PathBuf;
+
+use crate::{Diagnostic, SourceLocation};
+
+/// Source provenance, independent of filesystem layout or VM heap handles.
+/// Endor revisions are opaque labels supplied by the host, not verified hashes.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum SourceOrigin {
+    Text { name: String },
+    Stream { name: String },
+    File { path: PathBuf },
+    Endor { uri: String, revision: String },
+}
+
+impl fmt::Display for SourceOrigin {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Text { name } | Self::Stream { name } => formatter.write_str(name),
+            Self::File { path } => write!(formatter, "{}", path.display()),
+            Self::Endor { uri, revision } => write!(formatter, "{uri} [revision {revision}]"),
+        }
+    }
+}
+
+/// An owned, immutable UTF-8 snapshot. Reader adapters preserve the text exactly
+/// so token byte offsets still refer to the original decoded source.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct SourceUnit {
+    origin: SourceOrigin,
+    text: String,
+}
+
+impl SourceUnit {
+    pub(crate) fn new(origin: SourceOrigin, text: String) -> Self {
+        Self { origin, text }
+    }
+
+    pub fn origin(&self) -> &SourceOrigin {
+        &self.origin
+    }
+
+    pub fn text(&self) -> &str {
+        &self.text
+    }
+
+    /// Bind a diagnostic to this snapshot before discarding its source buffer.
+    pub fn annotate(&self, diagnostic: Diagnostic) -> SourceDiagnostic {
+        SourceDiagnostic {
+            location: diagnostic.location(&self.text),
+            origin: self.origin.clone(),
+            diagnostic: Box::new(diagnostic),
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct SourceDiagnostic {
+    pub origin: SourceOrigin,
+    pub location: SourceLocation,
+    pub diagnostic: Box<Diagnostic>,
+}
+
+impl fmt::Display for SourceDiagnostic {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            formatter,
+            "{}:{}:{}: {}",
+            self.origin, self.location.line, self.location.column, self.diagnostic.message
+        )
+    }
+}
+
+impl Error for SourceDiagnostic {
+    fn source(&self) -> Option<&(dyn Error + 'static)> {
+        Some(self.diagnostic.as_ref())
+    }
+}

crates/frontend/tests/README.md

@@ -2,4 +2,4 @@
 
 [syntax.rs](syntax.rs) 覆盖 UTF-8 token 范围、中文／日文名称、数字原文、负号、字符串转义与错误、算术优先级和结合性、成员／调用链、函数与绑定、上下文名称、完整输入、非法／未支持语法以及 CRLF 和 EOF 诊断位置。
 
-样例也通过此文件解析。只验证语法和 AST，不宣称类型检查或运行求值通过。随工作区检查在 ces-control 执行，本地不运行；入口见 [验证](../../../docs/operations/validation.md)。返回 [frontend](../README.md)。
+[reader.rs](reader.rs) 覆盖字符串／分块流／文件、严格 UTF-8、读取失败、URI provider、修订标签、旧缓冲区保留及带来源的诊断。样例也通过 syntax.rs 解析。只验证语法和 AST，不宣称类型检查或运行求值通过。随工作区检查在 ces-control 执行，本地不运行；入口见 [验证](../../../docs/operations/validation.md)。返回 [frontend](../README.md)。

crates/frontend/tests/reader.rs

@@ -0,0 +1,283 @@
+use std::cell::Cell;
+use std::error::Error;
+use std::io::{self, Cursor, Read};
+use std::path::PathBuf;
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+use frontend::{
+    DiagnosticKind, EndorReader, EndorSourceProvider, ResolvedEndorSource, SourceOrigin,
+    SourceReadErrorKind, lex_source, parse_source, parse_source_expression,
+};
+
+struct OneByte<R>(R);
+
+impl<R: Read> Read for OneByte<R> {
+    fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
+        let count = buffer.len().min(1);
+        self.0.read(&mut buffer[..count])
+    }
+}
+
+#[test]
+fn strings_and_split_utf8_streams_produce_the_same_ast_and_exact_text() {
+    let text = "// 日本語\r\nfunction 表示(): String { return \"こんにちは🌸\" }\r\n";
+    let memory = EndorReader::from_text("<editor>", text);
+    let stream = EndorReader::from_stream("<pipe>", OneByte(Cursor::new(text.as_bytes()))).unwrap();
+    assert_eq!(memory.text(), text);
+    assert_eq!(stream.text(), text);
+    assert_eq!(
+        parse_source(&memory).unwrap(),
+        parse_source(&stream).unwrap()
+    );
+    assert_eq!(
+        memory.origin(),
+        &SourceOrigin::Text {
+            name: "<editor>".into()
+        }
+    );
+    assert_eq!(
+        stream.origin(),
+        &SourceOrigin::Stream {
+            name: "<pipe>".into()
+        }
+    );
+    assert_eq!(lex_source(&memory).unwrap(), lex_source(&stream).unwrap());
+    let empty = EndorReader::from_stream("<empty>", Cursor::new(b"")).unwrap();
+    assert!(empty.text().is_empty());
+    assert_eq!(
+        parse_source(&empty).unwrap_err().diagnostic.kind,
+        DiagnosticKind::Syntax
+    );
+    let bom = EndorReader::from_stream("<bom>", Cursor::new("\u{feff}1".as_bytes())).unwrap();
+    assert_eq!(bom.text(), "\u{feff}1");
+    let error = lex_source(&bom).unwrap_err();
+    assert_eq!(error.diagnostic.kind, DiagnosticKind::Lexical);
+    assert_eq!(error.diagnostic.span.end, 3);
+}
+
+#[test]
+fn source_diagnostics_keep_session_names_unicode_locations_and_original_offsets() {
+    let source = EndorReader::from_text("<session:one/input:2>", "\"日\" + )");
+    let error = parse_source_expression(&source).unwrap_err();
+    assert_eq!(error.location.line, 1);
+    assert_eq!(error.location.column, 7);
+    assert_eq!(
+        &source.text()[error.diagnostic.span.start..error.diagnostic.span.end],
+        ")"
+    );
+    assert_eq!(error.origin, source.origin().clone());
+    drop(source);
+    assert!(error.to_string().starts_with("<session:one/input:2>:1:7:"));
+    assert!(error.source().is_some());
+
+    let source = EndorReader::from_text("<editor>", "\r\n\"日\\q\"");
+    let error = lex_source(&source).unwrap_err();
+    assert_eq!(error.diagnostic.kind, DiagnosticKind::Lexical);
+    assert_eq!(error.location.line, 2);
+    assert_eq!(error.location.column, 3);
+    assert!(error.to_string().contains("unsupported string escape"));
+}
+
+#[test]
+fn incomplete_or_invalid_utf8_is_rejected_without_replacement_characters() {
+    for bytes in [vec![0xff], vec![0xe6, 0x97], vec![b'a', 0xff]] {
+        let error = EndorReader::from_stream("<bytes>", OneByte(Cursor::new(bytes))).unwrap_err();
+        assert!(matches!(error.kind, SourceReadErrorKind::InvalidUtf8(_)));
+        assert!(error.to_string().contains("<bytes>: invalid UTF-8 at byte"));
+        assert!(error.source().is_some());
+    }
+}
+
+struct FailingStream(bool);
+
+impl Read for FailingStream {
+    fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
+        if buffer.is_empty() {
+            return Ok(0);
+        }
+        if !self.0 {
+            self.0 = true;
+            buffer[0] = b'x';
+            return Ok(1);
+        }
+        Err(io::Error::other("stream failure"))
+    }
+}
+
+#[test]
+fn a_failed_stream_never_returns_partially_read_source() {
+    let error = EndorReader::from_stream("<failed>", FailingStream(false)).unwrap_err();
+    assert!(matches!(error.kind, SourceReadErrorKind::Io(_)));
+    assert!(error.to_string().contains("stream failure"));
+}
+
+static NEXT_FILE: AtomicUsize = AtomicUsize::new(0);
+
+struct SourceFile(PathBuf);
+
+impl SourceFile {
+    fn new(bytes: &[u8]) -> Self {
+        let path = std::env::temp_dir().join(format!(
+            "aster-reader-{}-{}.endor",
+            std::process::id(),
+            NEXT_FILE.fetch_add(1, Ordering::Relaxed)
+        ));
+        std::fs::write(&path, bytes).unwrap();
+        Self(path)
+    }
+}
+
+impl Drop for SourceFile {
+    fn drop(&mut self) {
+        let _ = std::fs::remove_file(&self.0);
+    }
+}
+
+#[test]
+fn file_sources_preserve_the_path_and_read_errors_are_not_lexical_errors() {
+    let file = SourceFile::new(b"function f(): Number { return 1 }\r\n");
+    let source = EndorReader::from_file(&file.0).unwrap();
+    assert_eq!(
+        source.origin(),
+        &SourceOrigin::File {
+            path: file.0.clone()
+        }
+    );
+    assert!(source.text().ends_with("\r\n"));
+    assert!(parse_source(&source).is_ok());
+    let invalid = SourceFile::new(&[0xff]);
+    let error = EndorReader::from_file(&invalid.0).unwrap_err();
+    assert!(matches!(error.kind, SourceReadErrorKind::InvalidUtf8(_)));
+    assert!(error.to_string().contains(&invalid.0.display().to_string()));
+    let missing = file.0.with_extension("missing");
+    let error = EndorReader::from_file(&missing).unwrap_err();
+    let SourceReadErrorKind::Io(error) = error.kind else {
+        panic!("expected I/O error");
+    };
+    assert_eq!(error.kind(), io::ErrorKind::NotFound);
+}
+
+struct Provider {
+    calls: Cell<usize>,
+}
+
+impl EndorSourceProvider for Provider {
+    fn load(&self, uri: &str) -> io::Result<ResolvedEndorSource> {
+        assert_eq!(uri, "endor:workspace/namespace/function");
+        let version = self.calls.get() + 1;
+        self.calls.set(version);
+        Ok(ResolvedEndorSource {
+            revision: format!("revision-{version}"),
+            bytes: format!("function f(): Number {{ return {version} }}").into_bytes(),
+        })
+    }
+}
+
+#[test]
+fn each_uri_read_resolves_once_and_old_snapshots_keep_their_original_revision() {
+    let provider = Provider {
+        calls: Cell::new(0),
+    };
+    let uri = "endor:workspace/namespace/function";
+    let first = EndorReader::from_endor(uri, &provider).unwrap();
+    let second = EndorReader::from_endor(uri, &provider).unwrap();
+    assert_eq!(provider.calls.get(), 2);
+    assert!(first.text().contains("return 1"));
+    assert!(second.text().contains("return 2"));
+    assert_eq!(
+        first.origin(),
+        &SourceOrigin::Endor {
+            uri: uri.into(),
+            revision: "revision-1".into()
+        }
+    );
+    assert_eq!(
+        second.origin(),
+        &SourceOrigin::Endor {
+            uri: uri.into(),
+            revision: "revision-2".into()
+        }
+    );
+    assert!(parse_source(&first).is_ok());
+    assert!(parse_source(&second).is_ok());
+    assert_eq!(provider.calls.get(), 2); // Parsing never resolves the URI again.
+}
+
+struct StaticProvider {
+    revision: &'static str,
+    bytes: &'static [u8],
+}
+
+impl EndorSourceProvider for StaticProvider {
+    fn load(&self, _uri: &str) -> io::Result<ResolvedEndorSource> {
+        Ok(ResolvedEndorSource {
+            revision: self.revision.into(),
+            bytes: self.bytes.into(),
+        })
+    }
+}
+
+#[test]
+fn uri_snapshots_require_a_revision_and_preserve_it_in_source_diagnostics() {
+    let uri = "endor:workspace/namespace/function";
+    let provider = StaticProvider {
+        revision: "rev-one",
+        bytes: b"function f(): Number { return ) }",
+    };
+    let source = EndorReader::from_endor(uri, &provider).unwrap();
+    let error = parse_source(&source).unwrap_err();
+    assert_eq!(error.origin, source.origin().clone());
+    assert!(error.to_string().contains("[revision rev-one]"));
+    let missing = StaticProvider {
+        revision: " \t",
+        bytes: b"1",
+    };
+    assert!(matches!(
+        EndorReader::from_endor(uri, &missing).unwrap_err().kind,
+        SourceReadErrorKind::MissingRevision
+    ));
+    let invalid = StaticProvider {
+        revision: "rev-two",
+        bytes: &[0xff],
+    };
+    let error = EndorReader::from_endor(uri, &invalid).unwrap_err();
+    assert!(matches!(error.kind, SourceReadErrorKind::InvalidUtf8(_)));
+    assert!(error.to_string().contains("[revision rev-two]"));
+}
+
+struct DeniedProvider;
+
+impl EndorSourceProvider for DeniedProvider {
+    fn load(&self, _uri: &str) -> io::Result<ResolvedEndorSource> {
+        Err(io::Error::new(
+            io::ErrorKind::PermissionDenied,
+            "host denied source access",
+        ))
+    }
+}
+
+#[test]
+fn invalid_uris_do_not_call_the_provider_and_denials_do_not_fall_back_to_files() {
+    let provider = Provider {
+        calls: Cell::new(0),
+    };
+    for uri in [
+        "",
+        "endor:",
+        "file:source",
+        "https://example.com",
+        "endor:space here",
+        "endor:bad\nname",
+    ] {
+        assert!(matches!(
+            EndorReader::from_endor(uri, &provider).unwrap_err().kind,
+            SourceReadErrorKind::InvalidEndorUri
+        ));
+    }
+    assert_eq!(provider.calls.get(), 0);
+    let error = EndorReader::from_endor("endor:anywhere", &DeniedProvider).unwrap_err();
+    let SourceReadErrorKind::Io(error) = error.kind else {
+        panic!("expected provider error");
+    };
+    assert_eq!(error.kind(), io::ErrorKind::PermissionDenied);
+}