mirror/zig
1
0
Fork 0
mirror of https://github.com/ziglang/zig.git synced 2026-02-12 20:37:54 +00:00
Code Issues Packages Projects Releases Wiki Activity

Skip over CRs at the end of multiline literals

Browse Source 
Fixes #9257.
This is needed when tokenizing input containing DOS line endings, i.e.
the CRLF sequence.
This commit is contained in:
Daniele Cocca 2021-07-01 11:54:09 +01:00 committed by Veikka Tuominen
parent b936bbd2f1
commit 9de452f9a6
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
lib/std/zig/tokenizer.zig
View File

@ -845,7 +845,7 @@ pub const Tokenizer = struct {
self.index += 1;
break;
},
'\t' => {},
'\t', '\r' => {},
else => self.checkLiteralCharacter(),
},
@ -1936,6 +1936,10 @@ test "tokenizer - invalid builtin identifiers" {
try testTokenize("@0()", &.{ .invalid, .integer_literal, .l_paren, .r_paren });
}
test "tokenizer - multiline string literal with a DOS-encoded CRLF line endings (issue #9257)" {
try testTokenize("\\\\foobar\r\n", &.{.multiline_string_literal_line});
}
fn testTokenize(source: [:0]const u8, expected_tokens: []const Token.Tag) !void {
var tokenizer = Tokenizer.init(source);
for (expected_tokens) |expected_token_id| {