Add Endor source reader before lexing with provenance and URI provider

crates/frontend/src/README.md

@@ -2,7 +2,9 @@
 
 | 文件 | 职责 |
 | --- | --- |
-| [lib.rs](lib.rs) | 公开 AST 模块、lex／parse／parse_expression 与 token、诊断导出 |
+| [lib.rs](lib.rs) | 公开 AST 模块、阅读器／来源单元、低层及带来源的词法／语法入口与诊断导出 |
+| [source.rs](source.rs) | 不可变 SourceUnit、来源类别和携带来源／行列的 SourceDiagnostic |
+| [reader.rs](reader.rs) | 字符串、流、文件读取、严格 UTF-8 解码与宿主 EndorSourceProvider 适配 |
 | [token.rs](token.rs) | 可复制的 token 种类和原始源码范围，文本由源码切片取得 |
 | [diagnostic.rs](diagnostic.rs) | UTF-8 字节范围、词法／语法诊断与一基行列转换 |
 | [lexer.rs](lexer.rs) | 名称、数字、字符串、标点、空白与注释扫描；字符串校验及解码 |
@@ -10,4 +12,4 @@
 | [parser.rs](parser.rs) | 解析 API、错误映射、AST 辅助函数，以及独立语法文件的编译期引入 |
 | [grammar.peg](grammar.peg) | token 输入的 PEG 规则、运算符优先级与 AST 构造动作；保留 peg::parser! 外层声明 |
 
-改词法时同步 token 和错误检查；改语法从 grammar.peg 进入，并同步 AST、[测试](../tests/README.md)、样例与 [前端契约](../../../docs/runtime/frontend.md)。不在语法动作中做 VM 分配、I/O 或修改外部状态，避免回溯造成副作用。返回 [frontend](../README.md)。
+改输入适配时同步 [阅读器契约](../../../docs/runtime/reader.md) 与读取错误检查；改词法时同步 token 和错误检查；改语法从 grammar.peg 进入，并同步 AST、[测试](../tests/README.md)、样例与 [前端契约](../../../docs/runtime/frontend.md)。不在语法动作中做 VM 分配、I/O 或修改外部状态，避免回溯造成副作用。返回 [frontend](../README.md)。

crates/frontend/src/lexer.rs

@@ -1,5 +1,10 @@
 use crate::{Diagnostic, Span, Token, TokenKind};
 
+/// Lex a reader snapshot and preserve provenance in any diagnostic.
+pub fn lex_source(source: &crate::SourceUnit) -> Result<Vec<Token>, crate::SourceDiagnostic> {
+    lex(source.text()).map_err(|error| source.annotate(error))
+}
+
 /// Lex the basic Endorphin draft syntax without reading files or changing
 /// locale. Positions always refer to bytes in this exact UTF-8 source.
 pub fn lex(source: &str) -> Result<Vec<Token>, Diagnostic> {

crates/frontend/src/lib.rs

@@ -1,4 +1,4 @@
-//! UTF-8 source frontend: a handwritten lexer and token-based rust-peg parser.
+//! UTF-8 source reader, handwritten lexer and token-based rust-peg parser.
 //! Produces syntax only; it does not resolve types, generate code or execute VM
 //! operations. AST data owns its text and does not borrow the source buffer.
 
@@ -6,9 +6,15 @@ pub mod ast;
 mod diagnostic;
 mod lexer;
 mod parser;
+mod reader;
+mod source;
 mod token;
 
 pub use diagnostic::{Diagnostic, DiagnosticKind, SourceLocation, Span};
-pub use lexer::lex;
-pub use parser::{parse, parse_expression};
+pub use lexer::{lex, lex_source};
+pub use parser::{parse, parse_expression, parse_source, parse_source_expression};
+pub use reader::{
+    EndorReader, EndorSourceProvider, ResolvedEndorSource, SourceReadError, SourceReadErrorKind,
+};
+pub use source::{SourceDiagnostic, SourceOrigin, SourceUnit};
 pub use token::{Token, TokenKind};

crates/frontend/src/parser.rs

@@ -4,6 +4,17 @@ use crate::ast::{
 use crate::lexer::decode_string;
 use crate::{Diagnostic, DiagnosticKind, Span, Token, TokenKind, lex};
 
+/// Parse a reader snapshot, retaining its origin and resolved revision on error.
+pub fn parse_source(source: &crate::SourceUnit) -> Result<Program, crate::SourceDiagnostic> {
+    parse(source.text()).map_err(|error| source.annotate(error))
+}
+
+pub fn parse_source_expression(
+    source: &crate::SourceUnit,
+) -> Result<Expr, crate::SourceDiagnostic> {
+    parse_expression(source.text()).map_err(|error| source.annotate(error))
+}
+
 /// Parse one or more basic typed function declarations. This is a syntax
 /// operation: type names and name references are retained without resolution.
 pub fn parse(source: &str) -> Result<Program, Diagnostic> {

crates/frontend/src/reader.rs

@@ -0,0 +1,147 @@
+use std::error::Error;
+use std::fmt;
+use std::fs::File;
+use std::io::{self, Read};
+use std::path::Path;
+use std::string::FromUtf8Error;
+
+use crate::{SourceOrigin, SourceUnit};
+
+/// The host resolves names, authorizes access and selects a revision before
+/// returning its source bytes. No URI path layout or hash algorithm is assumed.
+pub trait EndorSourceProvider {
+    fn load(&self, uri: &str) -> io::Result<ResolvedEndorSource>;
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct ResolvedEndorSource {
+    pub revision: String,
+    pub bytes: Vec<u8>,
+}
+
+/// Stateless source adapters, before lexing. Streams are read to EOF; session
+/// prompting and incomplete REPL input detection belong to the interactive host.
+#[derive(Debug, Clone, Copy, Default)]
+pub struct EndorReader;
+
+impl EndorReader {
+    pub fn from_text(name: impl Into<String>, text: impl Into<String>) -> SourceUnit {
+        SourceUnit::new(SourceOrigin::Text { name: name.into() }, text.into())
+    }
+
+    pub fn from_stream(
+        name: impl Into<String>,
+        reader: impl Read,
+    ) -> Result<SourceUnit, SourceReadError> {
+        Self::read(SourceOrigin::Stream { name: name.into() }, reader)
+    }
+
+    pub fn from_file(path: impl AsRef<Path>) -> Result<SourceUnit, SourceReadError> {
+        let path = path.as_ref();
+        let origin = SourceOrigin::File {
+            path: path.to_owned(),
+        };
+        let file = File::open(path).map_err(|error| SourceReadError {
+            origin: origin.to_string(),
+            kind: SourceReadErrorKind::Io(error),
+        })?;
+        Self::read(origin, file)
+    }
+
+    pub fn from_endor<P: EndorSourceProvider + ?Sized>(
+        uri: &str,
+        provider: &P,
+    ) -> Result<SourceUnit, SourceReadError> {
+        // Only recognize the scheme and reject empty/control-filled locators.
+        // Namespace syntax, revision selectors and percent decoding are host policy.
+        let valid = uri.split_once(':').is_some_and(|(scheme, address)| {
+            scheme.eq_ignore_ascii_case("endor") && !address.is_empty()
+        }) && !uri
+            .chars()
+            .any(|character| character.is_whitespace() || character.is_control());
+        if !valid {
+            return Err(SourceReadError {
+                origin: uri.into(),
+                kind: SourceReadErrorKind::InvalidEndorUri,
+            });
+        }
+        let resolved = provider.load(uri).map_err(|error| SourceReadError {
+            origin: uri.into(),
+            kind: SourceReadErrorKind::Io(error),
+        })?;
+        if resolved.revision.trim().is_empty() {
+            return Err(SourceReadError {
+                origin: uri.into(),
+                kind: SourceReadErrorKind::MissingRevision,
+            });
+        }
+        let origin = SourceOrigin::Endor {
+            uri: uri.into(),
+            revision: resolved.revision,
+        };
+        Self::decode(origin, resolved.bytes)
+    }
+
+    fn read(origin: SourceOrigin, mut reader: impl Read) -> Result<SourceUnit, SourceReadError> {
+        let mut bytes = Vec::new();
+        reader
+            .read_to_end(&mut bytes)
+            .map_err(|error| SourceReadError {
+                origin: origin.to_string(),
+                kind: SourceReadErrorKind::Io(error),
+            })?;
+        Self::decode(origin, bytes)
+    }
+
+    fn decode(origin: SourceOrigin, bytes: Vec<u8>) -> Result<SourceUnit, SourceReadError> {
+        let text = String::from_utf8(bytes).map_err(|error| SourceReadError {
+            origin: origin.to_string(),
+            kind: SourceReadErrorKind::InvalidUtf8(error),
+        })?;
+        Ok(SourceUnit::new(origin, text))
+    }
+}
+
+#[derive(Debug)]
+pub enum SourceReadErrorKind {
+    Io(io::Error),
+    InvalidUtf8(FromUtf8Error),
+    InvalidEndorUri,
+    MissingRevision,
+}
+
+#[derive(Debug)]
+pub struct SourceReadError {
+    pub origin: String,
+    pub kind: SourceReadErrorKind,
+}
+
+impl fmt::Display for SourceReadError {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(formatter, "{}: ", self.origin)?;
+        match &self.kind {
+            SourceReadErrorKind::Io(error) => write!(formatter, "{error}"),
+            SourceReadErrorKind::InvalidUtf8(error) => write!(
+                formatter,
+                "invalid UTF-8 at byte {}",
+                error.utf8_error().valid_up_to()
+            ),
+            SourceReadErrorKind::InvalidEndorUri => formatter.write_str(
+                "expected a nonempty endor URI without whitespace or control characters",
+            ),
+            SourceReadErrorKind::MissingRevision => {
+                formatter.write_str("source provider returned no resolved revision")
+            }
+        }
+    }
+}
+
+impl Error for SourceReadError {
+    fn source(&self) -> Option<&(dyn Error + 'static)> {
+        match &self.kind {
+            SourceReadErrorKind::Io(error) => Some(error),
+            SourceReadErrorKind::InvalidUtf8(error) => Some(error),
+            _ => None,
+        }
+    }
+}

crates/frontend/src/source.rs

@@ -0,0 +1,79 @@
+use std::error::Error;
+use std::fmt;
+use std::path::PathBuf;
+
+use crate::{Diagnostic, SourceLocation};
+
+/// Source provenance, independent of filesystem layout or VM heap handles.
+/// Endor revisions are opaque labels supplied by the host, not verified hashes.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum SourceOrigin {
+    Text { name: String },
+    Stream { name: String },
+    File { path: PathBuf },
+    Endor { uri: String, revision: String },
+}
+
+impl fmt::Display for SourceOrigin {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Self::Text { name } | Self::Stream { name } => formatter.write_str(name),
+            Self::File { path } => write!(formatter, "{}", path.display()),
+            Self::Endor { uri, revision } => write!(formatter, "{uri} [revision {revision}]"),
+        }
+    }
+}
+
+/// An owned, immutable UTF-8 snapshot. Reader adapters preserve the text exactly
+/// so token byte offsets still refer to the original decoded source.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct SourceUnit {
+    origin: SourceOrigin,
+    text: String,
+}
+
+impl SourceUnit {
+    pub(crate) fn new(origin: SourceOrigin, text: String) -> Self {
+        Self { origin, text }
+    }
+
+    pub fn origin(&self) -> &SourceOrigin {
+        &self.origin
+    }
+
+    pub fn text(&self) -> &str {
+        &self.text
+    }
+
+    /// Bind a diagnostic to this snapshot before discarding its source buffer.
+    pub fn annotate(&self, diagnostic: Diagnostic) -> SourceDiagnostic {
+        SourceDiagnostic {
+            location: diagnostic.location(&self.text),
+            origin: self.origin.clone(),
+            diagnostic: Box::new(diagnostic),
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct SourceDiagnostic {
+    pub origin: SourceOrigin,
+    pub location: SourceLocation,
+    pub diagnostic: Box<Diagnostic>,
+}
+
+impl fmt::Display for SourceDiagnostic {
+    fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result {
+        write!(
+            formatter,
+            "{}:{}:{}: {}",
+            self.origin, self.location.line, self.location.column, self.diagnostic.message
+        )
+    }
+}
+
+impl Error for SourceDiagnostic {
+    fn source(&self) -> Option<&(dyn Error + 'static)> {
+        Some(self.diagnostic.as_ref())
+    }
+}