use crate::{Diagnostic, Span, Token, TokenKind}; /// Lex a reader snapshot and preserve provenance in any diagnostic. pub fn lex_source(source: &crate::SourceUnit) -> Result, crate::SourceDiagnostic> { lex(source.text()).map_err(|error| source.annotate(error)) } /// Lex the basic Endorphin draft syntax without reading files or changing /// locale. Positions always refer to bytes in this exact UTF-8 source. pub fn lex(source: &str) -> Result, Diagnostic> { let mut lexer = Lexer { source, offset: 0 }; let mut tokens = Vec::new(); while let Some(token) = lexer.next()? { tokens.push(token); } Ok(tokens) } struct Lexer<'source> { source: &'source str, offset: usize, } impl Lexer<'_> { fn next(&mut self) -> Result, Diagnostic> { let base = self.offset; let (consumed, token) = lexical::next(&self.source[base..]).map_err(|error| { let start = base + error.location.offset; let end = start + self.source[start..] .chars() .next() .map_or(0, char::len_utf8); Diagnostic::lexical(Span::new(start, end), "unexpected source character") })?; self.offset += consumed; token .map(|token| { token.map(|mut token| { token.span.start += base; token.span.end += base; token }) }) .map_err(|mut error| { error.span.start += base; error.span.end += base; error }) } } pub(crate) fn decode_string(raw: &str) -> String { let mut result = String::new(); let mut characters = raw.chars(); while let Some(character) = characters.next() { if character == '\\' { result.push(match characters.next().expect("lexer validated escape") { 'n' => '\n', 't' => '\t', '"' => '"', '\\' => '\\', _ => unreachable!("lexer validated escape"), }); } else { result.push(character); } } result } include!("lexer.peg");