use frontend::{DiagnosticKind, Span, TokenKind, lex}; #[test] fn lexical_errors_keep_original_byte_ranges_after_multiple_tokens() { let prefix = "let 商品 = "; for (tail, offending, message) in [ ( "12.50名前 ;", "12.50名前", "unsupported numeric suffix or missing token separator", ), ("\"日\\q\" ;", "\\q", "unsupported string escape"), ("\"日\\🌸\" ;", "\\🌸", "unsupported string escape"), ("\"日\\\n\" ;", "\\\n", "unsupported string escape"), ("\"日\nrest\" ;", "\n", "unescaped newline or NUL in string"), ("\"日\rrest\" ;", "\r", "unescaped newline or NUL in string"), ("\"日\0rest\" ;", "\0", "unescaped newline or NUL in string"), ("\"未完", "\"未完", "unterminated string"), ("\"未完\\", "\"未完\\", "unterminated string"), ("🌸 ;", "🌸", "unexpected source character"), ] { let source = format!("{prefix}{tail}"); let error = lex(&source).unwrap_err(); let start = source.find(offending).unwrap(); assert_eq!(error.kind, DiagnosticKind::Lexical, "{source:?}"); assert_eq!(error.span, Span::new(start, start + offending.len())); assert_eq!(error.message, message, "{source:?}"); assert!(error.expected.is_empty()); } } #[test] fn token_boundaries_distinguish_members_division_comments_and_string_contents() { let source = " \t// 前置\r\n商品.価格 / 12.50// 行末\0\n\"//実文\" \r\n// 終端"; let tokens = lex(source).unwrap(); let spellings: Vec<_> = tokens .iter() .map(|token| (token.kind, &source[token.span.start..token.span.end])) .collect(); assert_eq!( spellings, vec![ (TokenKind::Name, "商品"), (TokenKind::Dot, "."), (TokenKind::Name, "価格"), (TokenKind::Slash, "/"), (TokenKind::Number, "12.50"), (TokenKind::String, "\"//実文\""), ] ); assert!(lex("").unwrap().is_empty()); assert!(lex(" \t\r\n// 注釈\0").unwrap().is_empty()); }