endorphin-aster-vm/crates/frontend/src/lexer.rs

use crate::{Diagnostic, Span, Token, TokenKind};

/// Lex a reader snapshot and preserve provenance in any diagnostic.
pub fn lex_source(source: &crate::SourceUnit) -> Result<Vec<Token>, crate::SourceDiagnostic> {
    lex(source.text()).map_err(|error| source.annotate(error))
}

/// Lex the basic Endorphin draft syntax without reading files or changing
/// locale. Positions always refer to bytes in this exact UTF-8 source.
pub fn lex(source: &str) -> Result<Vec<Token>, Diagnostic> {
    let mut lexer = Lexer { source, offset: 0 };
    let mut tokens = Vec::new();
    while let Some(token) = lexer.next()? {
        tokens.push(token);
    }
    Ok(tokens)
}

struct Lexer<'source> {
    source: &'source str,
    offset: usize,
}

impl Lexer<'_> {
    fn next(&mut self) -> Result<Option<Token>, Diagnostic> {
        let base = self.offset;
        let (consumed, token) = lexical::next(&self.source[base..]).map_err(|error| {
            let start = base + error.location.offset;
            let end = start
                + self.source[start..]
                    .chars()
                    .next()
                    .map_or(0, char::len_utf8);
            Diagnostic::lexical(Span::new(start, end), "unexpected source character")
        })?;
        self.offset += consumed;
        token
            .map(|token| {
                token.map(|mut token| {
                    token.span.start += base;
                    token.span.end += base;
                    token
                })
            })
            .map_err(|mut error| {
                error.span.start += base;
                error.span.end += base;
                error
            })
    }
}

pub(crate) fn decode_string(raw: &str) -> String {
    let mut result = String::new();
    let mut characters = raw.chars();
    while let Some(character) = characters.next() {
        if character == '\\' {
            result.push(match characters.next().expect("lexer validated escape") {
                'n' => '\n',
                't' => '\t',
                '"' => '"',
                '\\' => '\\',
                _ => unreachable!("lexer validated escape"),
            });
        } else {
            result.push(character);
        }
    }
    result
}

include!("lexer.peg");