Add Endor source reader before lexing with provenance and URI provider

crates/frontend/tests/reader.rs

@@ -0,0 +1,283 @@
+use std::cell::Cell;
+use std::error::Error;
+use std::io::{self, Cursor, Read};
+use std::path::PathBuf;
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+use frontend::{
+    DiagnosticKind, EndorReader, EndorSourceProvider, ResolvedEndorSource, SourceOrigin,
+    SourceReadErrorKind, lex_source, parse_source, parse_source_expression,
+};
+
+struct OneByte<R>(R);
+
+impl<R: Read> Read for OneByte<R> {
+    fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
+        let count = buffer.len().min(1);
+        self.0.read(&mut buffer[..count])
+    }
+}
+
+#[test]
+fn strings_and_split_utf8_streams_produce_the_same_ast_and_exact_text() {
+    let text = "// 日本語\r\nfunction 表示(): String { return \"こんにちは🌸\" }\r\n";
+    let memory = EndorReader::from_text("<editor>", text);
+    let stream = EndorReader::from_stream("<pipe>", OneByte(Cursor::new(text.as_bytes()))).unwrap();
+    assert_eq!(memory.text(), text);
+    assert_eq!(stream.text(), text);
+    assert_eq!(
+        parse_source(&memory).unwrap(),
+        parse_source(&stream).unwrap()
+    );
+    assert_eq!(
+        memory.origin(),
+        &SourceOrigin::Text {
+            name: "<editor>".into()
+        }
+    );
+    assert_eq!(
+        stream.origin(),
+        &SourceOrigin::Stream {
+            name: "<pipe>".into()
+        }
+    );
+    assert_eq!(lex_source(&memory).unwrap(), lex_source(&stream).unwrap());
+    let empty = EndorReader::from_stream("<empty>", Cursor::new(b"")).unwrap();
+    assert!(empty.text().is_empty());
+    assert_eq!(
+        parse_source(&empty).unwrap_err().diagnostic.kind,
+        DiagnosticKind::Syntax
+    );
+    let bom = EndorReader::from_stream("<bom>", Cursor::new("\u{feff}1".as_bytes())).unwrap();
+    assert_eq!(bom.text(), "\u{feff}1");
+    let error = lex_source(&bom).unwrap_err();
+    assert_eq!(error.diagnostic.kind, DiagnosticKind::Lexical);
+    assert_eq!(error.diagnostic.span.end, 3);
+}
+
+#[test]
+fn source_diagnostics_keep_session_names_unicode_locations_and_original_offsets() {
+    let source = EndorReader::from_text("<session:one/input:2>", "\"日\" + )");
+    let error = parse_source_expression(&source).unwrap_err();
+    assert_eq!(error.location.line, 1);
+    assert_eq!(error.location.column, 7);
+    assert_eq!(
+        &source.text()[error.diagnostic.span.start..error.diagnostic.span.end],
+        ")"
+    );
+    assert_eq!(error.origin, source.origin().clone());
+    drop(source);
+    assert!(error.to_string().starts_with("<session:one/input:2>:1:7:"));
+    assert!(error.source().is_some());
+
+    let source = EndorReader::from_text("<editor>", "\r\n\"日\\q\"");
+    let error = lex_source(&source).unwrap_err();
+    assert_eq!(error.diagnostic.kind, DiagnosticKind::Lexical);
+    assert_eq!(error.location.line, 2);
+    assert_eq!(error.location.column, 3);
+    assert!(error.to_string().contains("unsupported string escape"));
+}
+
+#[test]
+fn incomplete_or_invalid_utf8_is_rejected_without_replacement_characters() {
+    for bytes in [vec![0xff], vec![0xe6, 0x97], vec![b'a', 0xff]] {
+        let error = EndorReader::from_stream("<bytes>", OneByte(Cursor::new(bytes))).unwrap_err();
+        assert!(matches!(error.kind, SourceReadErrorKind::InvalidUtf8(_)));
+        assert!(error.to_string().contains("<bytes>: invalid UTF-8 at byte"));
+        assert!(error.source().is_some());
+    }
+}
+
+struct FailingStream(bool);
+
+impl Read for FailingStream {
+    fn read(&mut self, buffer: &mut [u8]) -> io::Result<usize> {
+        if buffer.is_empty() {
+            return Ok(0);
+        }
+        if !self.0 {
+            self.0 = true;
+            buffer[0] = b'x';
+            return Ok(1);
+        }
+        Err(io::Error::other("stream failure"))
+    }
+}
+
+#[test]
+fn a_failed_stream_never_returns_partially_read_source() {
+    let error = EndorReader::from_stream("<failed>", FailingStream(false)).unwrap_err();
+    assert!(matches!(error.kind, SourceReadErrorKind::Io(_)));
+    assert!(error.to_string().contains("stream failure"));
+}
+
+static NEXT_FILE: AtomicUsize = AtomicUsize::new(0);
+
+struct SourceFile(PathBuf);
+
+impl SourceFile {
+    fn new(bytes: &[u8]) -> Self {
+        let path = std::env::temp_dir().join(format!(
+            "aster-reader-{}-{}.endor",
+            std::process::id(),
+            NEXT_FILE.fetch_add(1, Ordering::Relaxed)
+        ));
+        std::fs::write(&path, bytes).unwrap();
+        Self(path)
+    }
+}
+
+impl Drop for SourceFile {
+    fn drop(&mut self) {
+        let _ = std::fs::remove_file(&self.0);
+    }
+}
+
+#[test]
+fn file_sources_preserve_the_path_and_read_errors_are_not_lexical_errors() {
+    let file = SourceFile::new(b"function f(): Number { return 1 }\r\n");
+    let source = EndorReader::from_file(&file.0).unwrap();
+    assert_eq!(
+        source.origin(),
+        &SourceOrigin::File {
+            path: file.0.clone()
+        }
+    );
+    assert!(source.text().ends_with("\r\n"));
+    assert!(parse_source(&source).is_ok());
+    let invalid = SourceFile::new(&[0xff]);
+    let error = EndorReader::from_file(&invalid.0).unwrap_err();
+    assert!(matches!(error.kind, SourceReadErrorKind::InvalidUtf8(_)));
+    assert!(error.to_string().contains(&invalid.0.display().to_string()));
+    let missing = file.0.with_extension("missing");
+    let error = EndorReader::from_file(&missing).unwrap_err();
+    let SourceReadErrorKind::Io(error) = error.kind else {
+        panic!("expected I/O error");
+    };
+    assert_eq!(error.kind(), io::ErrorKind::NotFound);
+}
+
+struct Provider {
+    calls: Cell<usize>,
+}
+
+impl EndorSourceProvider for Provider {
+    fn load(&self, uri: &str) -> io::Result<ResolvedEndorSource> {
+        assert_eq!(uri, "endor:workspace/namespace/function");
+        let version = self.calls.get() + 1;
+        self.calls.set(version);
+        Ok(ResolvedEndorSource {
+            revision: format!("revision-{version}"),
+            bytes: format!("function f(): Number {{ return {version} }}").into_bytes(),
+        })
+    }
+}
+
+#[test]
+fn each_uri_read_resolves_once_and_old_snapshots_keep_their_original_revision() {
+    let provider = Provider {
+        calls: Cell::new(0),
+    };
+    let uri = "endor:workspace/namespace/function";
+    let first = EndorReader::from_endor(uri, &provider).unwrap();
+    let second = EndorReader::from_endor(uri, &provider).unwrap();
+    assert_eq!(provider.calls.get(), 2);
+    assert!(first.text().contains("return 1"));
+    assert!(second.text().contains("return 2"));
+    assert_eq!(
+        first.origin(),
+        &SourceOrigin::Endor {
+            uri: uri.into(),
+            revision: "revision-1".into()
+        }
+    );
+    assert_eq!(
+        second.origin(),
+        &SourceOrigin::Endor {
+            uri: uri.into(),
+            revision: "revision-2".into()
+        }
+    );
+    assert!(parse_source(&first).is_ok());
+    assert!(parse_source(&second).is_ok());
+    assert_eq!(provider.calls.get(), 2); // Parsing never resolves the URI again.
+}
+
+struct StaticProvider {
+    revision: &'static str,
+    bytes: &'static [u8],
+}
+
+impl EndorSourceProvider for StaticProvider {
+    fn load(&self, _uri: &str) -> io::Result<ResolvedEndorSource> {
+        Ok(ResolvedEndorSource {
+            revision: self.revision.into(),
+            bytes: self.bytes.into(),
+        })
+    }
+}
+
+#[test]
+fn uri_snapshots_require_a_revision_and_preserve_it_in_source_diagnostics() {
+    let uri = "endor:workspace/namespace/function";
+    let provider = StaticProvider {
+        revision: "rev-one",
+        bytes: b"function f(): Number { return ) }",
+    };
+    let source = EndorReader::from_endor(uri, &provider).unwrap();
+    let error = parse_source(&source).unwrap_err();
+    assert_eq!(error.origin, source.origin().clone());
+    assert!(error.to_string().contains("[revision rev-one]"));
+    let missing = StaticProvider {
+        revision: " \t",
+        bytes: b"1",
+    };
+    assert!(matches!(
+        EndorReader::from_endor(uri, &missing).unwrap_err().kind,
+        SourceReadErrorKind::MissingRevision
+    ));
+    let invalid = StaticProvider {
+        revision: "rev-two",
+        bytes: &[0xff],
+    };
+    let error = EndorReader::from_endor(uri, &invalid).unwrap_err();
+    assert!(matches!(error.kind, SourceReadErrorKind::InvalidUtf8(_)));
+    assert!(error.to_string().contains("[revision rev-two]"));
+}
+
+struct DeniedProvider;
+
+impl EndorSourceProvider for DeniedProvider {
+    fn load(&self, _uri: &str) -> io::Result<ResolvedEndorSource> {
+        Err(io::Error::new(
+            io::ErrorKind::PermissionDenied,
+            "host denied source access",
+        ))
+    }
+}
+
+#[test]
+fn invalid_uris_do_not_call_the_provider_and_denials_do_not_fall_back_to_files() {
+    let provider = Provider {
+        calls: Cell::new(0),
+    };
+    for uri in [
+        "",
+        "endor:",
+        "file:source",
+        "https://example.com",
+        "endor:space here",
+        "endor:bad\nname",
+    ] {
+        assert!(matches!(
+            EndorReader::from_endor(uri, &provider).unwrap_err().kind,
+            SourceReadErrorKind::InvalidEndorUri
+        ));
+    }
+    assert_eq!(provider.calls.get(), 0);
+    let error = EndorReader::from_endor("endor:anywhere", &DeniedProvider).unwrap_err();
+    let SourceReadErrorKind::Io(error) = error.kind else {
+        panic!("expected provider error");
+    };
+    assert_eq!(error.kind(), io::ErrorKind::PermissionDenied);
+}