From f750618846f598836e5170c31ac4988df2d6f00d Mon Sep 17 00:00:00 2001 From: Matthew Borkowski Date: Tue, 25 May 2021 23:44:55 -0400 Subject: [PATCH] stop tokenizer from recognizing lone `@` or `@` followed by a digit as a builtin --- lib/std/zig/tokenizer.zig | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/std/zig/tokenizer.zig b/lib/std/zig/tokenizer.zig index 241d7b7e20..2aaa4cf512 100644 --- a/lib/std/zig/tokenizer.zig +++ b/lib/std/zig/tokenizer.zig @@ -568,12 +568,14 @@ pub const Tokenizer = struct { result.tag = .identifier; state = .string_literal; }, - else => { - // reinterpret as a builtin - self.index -= 1; + 'a'...'z', 'A'...'Z', '_' => { state = .builtin; result.tag = .builtin; }, + else => { + result.tag = .invalid; + break; + }, }, .ampersand => switch (c) { @@ -2053,6 +2055,11 @@ test "tokenizer - multi line string literal with only 1 backslash" { try testTokenize("x \\\n;", &.{ .identifier, .invalid, .semicolon }); } +test "tokenizer - invalid builtin identifiers" { + try testTokenize("@()", &.{ .invalid, .l_paren, .r_paren }); + try testTokenize("@0()", &.{ .invalid, .integer_literal, .l_paren, .r_paren }); +} + fn testTokenize(source: []const u8, expected_tokens: []const Token.Tag) !void { var tokenizer = Tokenizer.init(source); for (expected_tokens) |expected_token_id| {