mirror/zig
1
0
Fork 0
mirror of https://github.com/ziglang/zig.git synced 2026-02-21 00:35:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

stop tokenizer from recognizing lone @ or @ followed by a digit as a builtin

Browse Source
This commit is contained in:
Matthew Borkowski 2021-05-25 23:44:55 -04:00 committed by Andrew Kelley
parent 44de884980
commit f750618846
1 changed files with 10 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
lib/std/zig/tokenizer.zig
View File

@ -568,12 +568,14 @@ pub const Tokenizer = struct {
result.tag = .identifier;
state = .string_literal;
},
else => {
// reinterpret as a builtin
self.index -= 1;
'a'...'z', 'A'...'Z', '_' => {
state = .builtin;
result.tag = .builtin;
},
else => {
result.tag = .invalid;
break;
},
},
.ampersand => switch (c) {
@ -2053,6 +2055,11 @@ test "tokenizer - multi line string literal with only 1 backslash" {
try testTokenize("x \\\n;", &.{ .identifier, .invalid, .semicolon });
}
test "tokenizer - invalid builtin identifiers" {
try testTokenize("@()", &.{ .invalid, .l_paren, .r_paren });
try testTokenize("@0()", &.{ .invalid, .integer_literal, .l_paren, .r_paren });
}
fn testTokenize(source: []const u8, expected_tokens: []const Token.Tag) !void {
var tokenizer = Tokenizer.init(source);
for (expected_tokens) |expected_token_id| {