From c1e8fdf812692edb9c7e496b00294aa749e81079 Mon Sep 17 00:00:00 2001 From: Brendan Hansknecht Date: Thu, 31 Oct 2019 21:53:46 -0700 Subject: [PATCH] add token for parsing pointer dereference --- doc/docgen.zig | 1 + lib/std/zig/parse.zig | 7 ++++--- lib/std/zig/render.zig | 9 +++++++-- lib/std/zig/tokenizer.zig | 7 +++++++ src/parser.cpp | 6 ++++-- src/tokenizer.cpp | 6 ++++++ src/tokenizer.hpp | 1 + test/stage1/behavior/bugs/3468.zig | 6 ++++++ 8 files changed, 36 insertions(+), 7 deletions(-) create mode 100644 test/stage1/behavior/bugs/3468.zig diff --git a/doc/docgen.zig b/doc/docgen.zig index bdc2604213..07a7f23968 100644 --- a/doc/docgen.zig +++ b/doc/docgen.zig @@ -917,6 +917,7 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Tok .LBracket, .RBracket, .Period, + .PeriodAsterisk, .Ellipsis2, .Ellipsis3, .Caret, diff --git a/lib/std/zig/parse.zig b/lib/std/zig/parse.zig index 71fa376361..4c6c461c01 100644 --- a/lib/std/zig/parse.zig +++ b/lib/std/zig/parse.zig @@ -2363,6 +2363,10 @@ fn parseSuffixOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { }; } + if (eatToken(it, .PeriodAsterisk)) |period_asterisk| { + break :blk OpAndToken{ .op = Op{ .Deref = {} }, .token = period_asterisk }; + } + if (eatToken(it, .Period)) |period| { if (try parseIdentifier(arena, it, tree)) |identifier| { // TODO: It's a bit weird to return an InfixOp from the SuffixOp parser. @@ -2378,9 +2382,6 @@ fn parseSuffixOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { }; return &node.base; } - if (eatToken(it, .Asterisk)) |asterisk| { - break :blk OpAndToken{ .op = Op{ .Deref = {} }, .token = asterisk }; - } if (eatToken(it, .QuestionMark)) |question_mark| { break :blk OpAndToken{ .op = Op{ .UnwrapOptional = {} }, .token = question_mark }; } diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index e5875fd04f..996cfc29e4 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -617,10 +617,15 @@ fn renderExpression( return renderToken(tree, stream, rbracket, indent, start_col, space); // ] }, - ast.Node.SuffixOp.Op.Deref, ast.Node.SuffixOp.Op.UnwrapOptional => { + ast.Node.SuffixOp.Op.Deref => { + try renderExpression(allocator, stream, tree, indent, start_col, suffix_op.lhs, Space.None); + return renderToken(tree, stream, suffix_op.rtoken, indent, start_col, space); // .* + }, + + ast.Node.SuffixOp.Op.UnwrapOptional => { try renderExpression(allocator, stream, tree, indent, start_col, suffix_op.lhs, Space.None); try renderToken(tree, stream, tree.prevToken(suffix_op.rtoken), indent, start_col, Space.None); // . - return renderToken(tree, stream, suffix_op.rtoken, indent, start_col, space); // * or ? + return renderToken(tree, stream, suffix_op.rtoken, indent, start_col, space); // ? }, @TagType(ast.Node.SuffixOp.Op).Slice => |range| { diff --git a/lib/std/zig/tokenizer.zig b/lib/std/zig/tokenizer.zig index 872893a607..672bd482da 100644 --- a/lib/std/zig/tokenizer.zig +++ b/lib/std/zig/tokenizer.zig @@ -103,6 +103,7 @@ pub const Token = struct { LBracket, RBracket, Period, + PeriodAsterisk, Ellipsis2, Ellipsis3, Caret, @@ -231,6 +232,7 @@ pub const Token = struct { .LBracket => "[", .RBracket => "]", .Period => ".", + .PeriodAsterisk => ".*", .Ellipsis2 => "..", .Ellipsis3 => "...", .Caret => "^", @@ -1033,6 +1035,11 @@ pub const Tokenizer = struct { '.' => { state = State.Period2; }, + '*' => { + result.id = Token.Id.PeriodAsterisk; + self.index += 1; + break; + }, else => { result.id = Token.Id.Period; break; diff --git a/src/parser.cpp b/src/parser.cpp index b6742ef1f1..a1ece3ed10 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -2693,10 +2693,12 @@ static AstNode *ast_parse_suffix_op(ParseContext *pc) { return res; } + Token *dot_asterisk = eat_token_if(pc, TokenIdDotStar); + if (dot_asterisk != nullptr) + return ast_create_node(pc, NodeTypePtrDeref, dot_asterisk); + Token *dot = eat_token_if(pc, TokenIdDot); if (dot != nullptr) { - if (eat_token_if(pc, TokenIdStar) != nullptr) - return ast_create_node(pc, NodeTypePtrDeref, dot); if (eat_token_if(pc, TokenIdQuestion) != nullptr) return ast_create_node(pc, NodeTypeUnwrapOptional, dot); diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index 475c284d27..674f0c2b96 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -583,6 +583,11 @@ void tokenize(Buf *buf, Tokenization *out) { t.state = TokenizeStateSawDotDot; set_token_id(&t, t.cur_tok, TokenIdEllipsis2); break; + case '*': + t.state = TokenizeStateStart; + set_token_id(&t, t.cur_tok, TokenIdDotStar); + end_token(&t); + break; default: t.pos -= 1; end_token(&t); @@ -1603,6 +1608,7 @@ const char * token_name(TokenId id) { case TokenIdDivEq: return "/="; case TokenIdDocComment: return "DocComment"; case TokenIdDot: return "."; + case TokenIdDotStar: return ".*"; case TokenIdEllipsis2: return ".."; case TokenIdEllipsis3: return "..."; case TokenIdEof: return "EOF"; diff --git a/src/tokenizer.hpp b/src/tokenizer.hpp index e33a82f31d..ee336123bb 100644 --- a/src/tokenizer.hpp +++ b/src/tokenizer.hpp @@ -44,6 +44,7 @@ enum TokenId { TokenIdDivEq, TokenIdDocComment, TokenIdDot, + TokenIdDotStar, TokenIdEllipsis2, TokenIdEllipsis3, TokenIdEof, diff --git a/test/stage1/behavior/bugs/3468.zig b/test/stage1/behavior/bugs/3468.zig new file mode 100644 index 0000000000..adf3db3306 --- /dev/null +++ b/test/stage1/behavior/bugs/3468.zig @@ -0,0 +1,6 @@ +// zig fmt: off +test "pointer deref next to assignment" { + var a:i32=2; + var b=&a; + b.*=3; +}