diff --git a/lib/std/zig/tokenizer.zig b/lib/std/zig/tokenizer.zig index 3008aecdc3..947c8b87fa 100644 --- a/lib/std/zig/tokenizer.zig +++ b/lib/std/zig/tokenizer.zig @@ -845,7 +845,7 @@ pub const Tokenizer = struct { self.index += 1; break; }, - '\t' => {}, + '\t', '\r' => {}, else => self.checkLiteralCharacter(), }, @@ -1936,6 +1936,10 @@ test "tokenizer - invalid builtin identifiers" { try testTokenize("@0()", &.{ .invalid, .integer_literal, .l_paren, .r_paren }); } +test "tokenizer - multiline string literal with a DOS-encoded CRLF line endings (issue #9257)" { + try testTokenize("\\\\foobar\r\n", &.{.multiline_string_literal_line}); +} + fn testTokenize(source: [:0]const u8, expected_tokens: []const Token.Tag) !void { var tokenizer = Tokenizer.init(source); for (expected_tokens) |expected_token_id| {