codex-bot
be4f1add58
Verify databases and deploy Aster VM / verify-and-deploy (push) Successful in 6m57s
54 lines
2.1 KiB
Rust
54 lines
2.1 KiB
Rust
use frontend::{DiagnosticKind, Span, TokenKind, lex};
|
|
|
|
#[test]
|
|
fn lexical_errors_keep_original_byte_ranges_after_multiple_tokens() {
|
|
let prefix = "let 商品 = ";
|
|
for (tail, offending, message) in [
|
|
(
|
|
"12.50名前 ;",
|
|
"12.50名前",
|
|
"unsupported numeric suffix or missing token separator",
|
|
),
|
|
("\"日\\q\" ;", "\\q", "unsupported string escape"),
|
|
("\"日\\🌸\" ;", "\\🌸", "unsupported string escape"),
|
|
("\"日\\\n\" ;", "\\\n", "unsupported string escape"),
|
|
("\"日\nrest\" ;", "\n", "unescaped newline or NUL in string"),
|
|
("\"日\rrest\" ;", "\r", "unescaped newline or NUL in string"),
|
|
("\"日\0rest\" ;", "\0", "unescaped newline or NUL in string"),
|
|
("\"未完", "\"未完", "unterminated string"),
|
|
("\"未完\\", "\"未完\\", "unterminated string"),
|
|
("🌸 ;", "🌸", "unexpected source character"),
|
|
] {
|
|
let source = format!("{prefix}{tail}");
|
|
let error = lex(&source).unwrap_err();
|
|
let start = source.find(offending).unwrap();
|
|
assert_eq!(error.kind, DiagnosticKind::Lexical, "{source:?}");
|
|
assert_eq!(error.span, Span::new(start, start + offending.len()));
|
|
assert_eq!(error.message, message, "{source:?}");
|
|
assert!(error.expected.is_empty());
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn token_boundaries_distinguish_members_division_comments_and_string_contents() {
|
|
let source = " \t// 前置\r\n商品.価格 / 12.50// 行末\0\n\"//実文\" \r\n// 終端";
|
|
let tokens = lex(source).unwrap();
|
|
let spellings: Vec<_> = tokens
|
|
.iter()
|
|
.map(|token| (token.kind, &source[token.span.start..token.span.end]))
|
|
.collect();
|
|
assert_eq!(
|
|
spellings,
|
|
vec![
|
|
(TokenKind::Name, "商品"),
|
|
(TokenKind::Dot, "."),
|
|
(TokenKind::Name, "価格"),
|
|
(TokenKind::Slash, "/"),
|
|
(TokenKind::Number, "12.50"),
|
|
(TokenKind::String, "\"//実文\""),
|
|
]
|
|
);
|
|
assert!(lex("").unwrap().is_empty());
|
|
assert!(lex(" \t\r\n// 注釈\0").unwrap().is_empty());
|
|
}
|