mirror/zig
1
0
Fork 0
mirror of https://github.com/ziglang/zig.git synced 2026-01-20 22:35:24 +00:00
Code Issues Packages Projects Releases Wiki Activity

tokenizer: detect null bytes before EOF

Browse Source 
Closes #13811
This commit is contained in:
Veikka Tuominen 2022-12-07 21:43:01 +02:00
parent 653c0bae0c
commit 6039554b26
1 changed files with 17 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

18
lib/std/zig/tokenizer.zig
View File

@ -425,7 +425,16 @@ pub const Tokenizer = struct {
const c = self.buffer[self.index];
switch (state) {
.start => switch (c) {
0 => break,
0 => {
if (self.index != self.buffer.len) {
result.tag = .invalid;
result.loc.start = self.index;
self.index += 1;
result.loc.end = self.index;
return result;
}
break;
},
' ', '\n', '\t', '\r' => {
result.loc.start = self.index + 1;
},
@ -1851,6 +1860,13 @@ test "saturating operators" {
try testTokenize("-|=", &.{.minus_pipe_equal});
}
test "null byte before eof" {
try testTokenize("123 \x00 456", &.{ .number_literal, .invalid, .number_literal });
try testTokenize("//\x00", &.{.invalid});
try testTokenize("\\\\\x00", &.{ .multiline_string_literal_line, .invalid });
try testTokenize("\x00", &.{.invalid});
}
fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !void {
var tokenizer = Tokenizer.init(source);
for (expected_token_tags) |expected_token_tag| {