diff --git a/lib/std/zig/ast.zig b/lib/std/zig/ast.zig index 9072626c86..9f9064f88d 100644 --- a/lib/std/zig/ast.zig +++ b/lib/std/zig/ast.zig @@ -137,7 +137,6 @@ pub const Error = union(enum) { ExpectedCallOrFnProto: ExpectedCallOrFnProto, ExpectedSliceOrRBracket: ExpectedSliceOrRBracket, ExtraAlignQualifier: ExtraAlignQualifier, - ExtraNullQualifier: ExtraNullQualifier, ExtraConstQualifier: ExtraConstQualifier, ExtraVolatileQualifier: ExtraVolatileQualifier, ExtraAllowZeroQualifier: ExtraAllowZeroQualifier, @@ -185,7 +184,6 @@ pub const Error = union(enum) { .ExpectedCallOrFnProto => |*x| return x.render(tokens, stream), .ExpectedSliceOrRBracket => |*x| return x.render(tokens, stream), .ExtraAlignQualifier => |*x| return x.render(tokens, stream), - .ExtraNullQualifier => |*x| return x.render(tokens, stream), .ExtraConstQualifier => |*x| return x.render(tokens, stream), .ExtraVolatileQualifier => |*x| return x.render(tokens, stream), .ExtraAllowZeroQualifier => |*x| return x.render(tokens, stream), @@ -235,7 +233,6 @@ pub const Error = union(enum) { .ExpectedCallOrFnProto => |x| return x.node.firstToken(), .ExpectedSliceOrRBracket => |x| return x.token, .ExtraAlignQualifier => |x| return x.token, - .ExtraNullQualifier => |x| return x.token, .ExtraConstQualifier => |x| return x.token, .ExtraVolatileQualifier => |x| return x.token, .ExtraAllowZeroQualifier => |x| return x.token, @@ -296,7 +293,6 @@ pub const Error = union(enum) { pub const ExpectedPubItem = SimpleError("Expected function or variable declaration after pub"); pub const UnattachedDocComment = SimpleError("Unattached documentation comment"); pub const ExtraAlignQualifier = SimpleError("Extra align qualifier"); - pub const ExtraNullQualifier = SimpleError("Extra null qualifier"); pub const ExtraConstQualifier = SimpleError("Extra const qualifier"); pub const ExtraVolatileQualifier = SimpleError("Extra volatile qualifier"); pub const ExtraAllowZeroQualifier = SimpleError("Extra allowzero qualifier"); @@ -1535,7 +1531,7 @@ pub const Node = struct { }; pub const PrefixOp = struct { - base: Node, + base: Node = Node{ .id = .PrefixOp }, op_token: TokenIndex, op: Op, rhs: *Node, @@ -1558,15 +1554,15 @@ pub const Node = struct { pub const ArrayInfo = struct { len_expr: *Node, - null_token: ?TokenIndex, + sentinel: ?*Node, }; pub const PtrInfo = struct { - allowzero_token: ?TokenIndex, - align_info: ?Align, - const_token: ?TokenIndex, - volatile_token: ?TokenIndex, - null_token: ?TokenIndex, + allowzero_token: ?TokenIndex = null, + align_info: ?Align = null, + const_token: ?TokenIndex = null, + volatile_token: ?TokenIndex = null, + sentinel: ?*Node = null, pub const Align = struct { node: *Node, @@ -1585,6 +1581,11 @@ pub const Node = struct { switch (self.op) { // TODO https://github.com/ziglang/zig/issues/1107 Op.SliceType => |addr_of_info| { + if (addr_of_info.sentinel) |sentinel| { + if (i < 1) return sentinel; + i -= 1; + } + if (addr_of_info.align_info) |align_info| { if (i < 1) return align_info.node; i -= 1; @@ -1601,6 +1602,10 @@ pub const Node = struct { Op.ArrayType => |array_info| { if (i < 1) return array_info.len_expr; i -= 1; + if (array_info.sentinel) |sentinel| { + if (i < 1) return sentinel; + i -= 1; + } }, Op.AddressOf, diff --git a/lib/std/zig/parse.zig b/lib/std/zig/parse.zig index 1190b89089..fd15cda11a 100644 --- a/lib/std/zig/parse.zig +++ b/lib/std/zig/parse.zig @@ -2246,22 +2246,70 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node return &node.base; } + if (try parsePtrTypeStart(arena, it, tree)) |node| { + // If the token encountered was **, there will be two nodes instead of one. + // The attributes should be applied to the rightmost operator. + const prefix_op = node.cast(Node.PrefixOp).?; + var ptr_info = if (tree.tokens.at(prefix_op.op_token).id == .AsteriskAsterisk) + &prefix_op.rhs.cast(Node.PrefixOp).?.op.PtrType + else + &prefix_op.op.PtrType; + + while (true) { + if (eatToken(it, .Keyword_align)) |align_token| { + const lparen = try expectToken(it, tree, .LParen); + const expr_node = try expectNode(arena, it, tree, parseExpr, AstError{ + .ExpectedExpr = AstError.ExpectedExpr{ .token = it.index }, + }); + + // Optional bit range + const bit_range = if (eatToken(it, .Colon)) |_| bit_range_value: { + const range_start = try expectNode(arena, it, tree, parseIntegerLiteral, AstError{ + .ExpectedIntegerLiteral = AstError.ExpectedIntegerLiteral{ .token = it.index }, + }); + _ = try expectToken(it, tree, .Colon); + const range_end = try expectNode(arena, it, tree, parseIntegerLiteral, AstError{ + .ExpectedIntegerLiteral = AstError.ExpectedIntegerLiteral{ .token = it.index }, + }); + + break :bit_range_value Node.PrefixOp.PtrInfo.Align.BitRange{ + .start = range_start, + .end = range_end, + }; + } else null; + _ = try expectToken(it, tree, .RParen); + + ptr_info.align_info = Node.PrefixOp.PtrInfo.Align{ + .node = expr_node, + .bit_range = bit_range, + }; + + continue; + } + if (eatToken(it, .Keyword_const)) |const_token| { + ptr_info.const_token = const_token; + continue; + } + if (eatToken(it, .Keyword_volatile)) |volatile_token| { + ptr_info.volatile_token = volatile_token; + continue; + } + if (eatToken(it, .Keyword_allowzero)) |allowzero_token| { + ptr_info.allowzero_token = allowzero_token; + continue; + } + break; + } + + return node; + } + if (try parseArrayTypeStart(arena, it, tree)) |node| { switch (node.cast(Node.PrefixOp).?.op) { .ArrayType => {}, .SliceType => |*slice_type| { // Collect pointer qualifiers in any order, but disallow duplicates while (true) { - if (eatToken(it, .Keyword_null)) |null_token| { - if (slice_type.null_token != null) { - try tree.errors.push(AstError{ - .ExtraNullQualifier = AstError.ExtraNullQualifier{ .token = it.index }, - }); - return error.ParseError; - } - slice_type.null_token = null_token; - continue; - } if (try parseByteAlign(arena, it, tree)) |align_expr| { if (slice_type.align_info != null) { try tree.errors.push(AstError{ @@ -2313,68 +2361,6 @@ fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node return node; } - if (try parsePtrTypeStart(arena, it, tree)) |node| { - // If the token encountered was **, there will be two nodes instead of one. - // The attributes should be applied to the rightmost operator. - const prefix_op = node.cast(Node.PrefixOp).?; - var ptr_info = if (tree.tokens.at(prefix_op.op_token).id == .AsteriskAsterisk) - &prefix_op.rhs.cast(Node.PrefixOp).?.op.PtrType - else - &prefix_op.op.PtrType; - - while (true) { - if (eatToken(it, .Keyword_null)) |null_token| { - ptr_info.null_token = null_token; - continue; - } - if (eatToken(it, .Keyword_align)) |align_token| { - const lparen = try expectToken(it, tree, .LParen); - const expr_node = try expectNode(arena, it, tree, parseExpr, AstError{ - .ExpectedExpr = AstError.ExpectedExpr{ .token = it.index }, - }); - - // Optional bit range - const bit_range = if (eatToken(it, .Colon)) |_| bit_range_value: { - const range_start = try expectNode(arena, it, tree, parseIntegerLiteral, AstError{ - .ExpectedIntegerLiteral = AstError.ExpectedIntegerLiteral{ .token = it.index }, - }); - _ = try expectToken(it, tree, .Colon); - const range_end = try expectNode(arena, it, tree, parseIntegerLiteral, AstError{ - .ExpectedIntegerLiteral = AstError.ExpectedIntegerLiteral{ .token = it.index }, - }); - - break :bit_range_value Node.PrefixOp.PtrInfo.Align.BitRange{ - .start = range_start, - .end = range_end, - }; - } else null; - _ = try expectToken(it, tree, .RParen); - - ptr_info.align_info = Node.PrefixOp.PtrInfo.Align{ - .node = expr_node, - .bit_range = bit_range, - }; - - continue; - } - if (eatToken(it, .Keyword_const)) |const_token| { - ptr_info.const_token = const_token; - continue; - } - if (eatToken(it, .Keyword_volatile)) |volatile_token| { - ptr_info.volatile_token = volatile_token; - continue; - } - if (eatToken(it, .Keyword_allowzero)) |allowzero_token| { - ptr_info.allowzero_token = allowzero_token; - continue; - } - break; - } - - return node; - } - return null; } @@ -2473,14 +2459,19 @@ const AnnotatedParamList = struct { fn parseArrayTypeStart(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { const lbracket = eatToken(it, .LBracket) orelse return null; const expr = try parseExpr(arena, it, tree); + const sentinel = if (eatToken(it, .Colon)) |_| + try expectNode(arena, it, tree, parseExpr, AstError{ + .ExpectedExpr = .{ .token = it.index }, + }) + else + null; const rbracket = try expectToken(it, tree, .RBracket); - const null_token = eatToken(it, .Keyword_null); const op = if (expr) |len_expr| Node.PrefixOp.Op{ .ArrayType = .{ .len_expr = len_expr, - .null_token = null_token, + .sentinel = sentinel, }, } else @@ -2490,7 +2481,7 @@ fn parseArrayTypeStart(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*No .align_info = null, .const_token = null, .volatile_token = null, - .null_token = null, + .sentinel = sentinel, }, }; @@ -2510,49 +2501,76 @@ fn parseArrayTypeStart(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*No /// / PTRUNKNOWN /// / PTRC fn parsePtrTypeStart(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const token = eatAnnotatedToken(it, .Asterisk) orelse - eatAnnotatedToken(it, .AsteriskAsterisk) orelse - eatAnnotatedToken(it, .BracketStarBracket) orelse - eatAnnotatedToken(it, .BracketStarCBracket) orelse - return null; + if (eatToken(it, .Asterisk)) |asterisk| { + const sentinel = if (eatToken(it, .Colon)) |_| + try expectNode(arena, it, tree, parseExpr, AstError{ + .ExpectedExpr = .{ .token = it.index }, + }) + else + null; + const node = try arena.create(Node.PrefixOp); + node.* = .{ + .op_token = asterisk, + .op = .{ .PtrType = .{ .sentinel = sentinel } }, + .rhs = undefined, // set by caller + }; + return &node.base; + } - const node = try arena.create(Node.PrefixOp); - node.* = Node.PrefixOp{ - .base = Node{ .id = .PrefixOp }, - .op_token = token.index, - .op = Node.PrefixOp.Op{ - .PtrType = Node.PrefixOp.PtrInfo{ - .allowzero_token = null, - .align_info = null, - .const_token = null, - .volatile_token = null, - .null_token = null, - }, - }, - .rhs = undefined, // set by caller - }; + if (eatToken(it, .AsteriskAsterisk)) |double_asterisk| { + const node = try arena.create(Node.PrefixOp); + node.* = Node.PrefixOp{ + .op_token = double_asterisk, + .op = Node.PrefixOp.Op{ .PtrType = .{} }, + .rhs = undefined, // set by caller + }; - // Special case for **, which is its own token - if (token.ptr.id == .AsteriskAsterisk) { + // Special case for **, which is its own token const child = try arena.create(Node.PrefixOp); child.* = Node.PrefixOp{ - .base = Node{ .id = .PrefixOp }, - .op_token = token.index, - .op = Node.PrefixOp.Op{ - .PtrType = Node.PrefixOp.PtrInfo{ - .allowzero_token = null, - .align_info = null, - .const_token = null, - .volatile_token = null, - .null_token = null, - }, - }, + .op_token = double_asterisk, + .op = Node.PrefixOp.Op{ .PtrType = .{} }, .rhs = undefined, // set by caller }; node.rhs = &child.base; - } - return &node.base; + return &node.base; + } + if (eatToken(it, .LBracket)) |lbracket| { + const asterisk = eatToken(it, .Asterisk) orelse { + putBackToken(it, lbracket); + return null; + }; + if (eatToken(it, .Identifier)) |ident| { + if (!std.mem.eql(u8, tree.tokenSlice(ident), "c")) { + putBackToken(it, ident); + } else { + _ = try expectToken(it, tree, .RBracket); + const node = try arena.create(Node.PrefixOp); + node.* = .{ + .op_token = ident, + .op = .{ .PtrType = .{} }, + .rhs = undefined, // set by caller + }; + return &node.base; + } + } + const sentinel = if (eatToken(it, .Colon)) |_| + try expectNode(arena, it, tree, parseExpr, AstError{ + .ExpectedExpr = .{ .token = it.index }, + }) + else + null; + _ = try expectToken(it, tree, .RBracket); + const node = try arena.create(Node.PrefixOp); + node.* = .{ + .op_token = lbracket, + .op = .{ .PtrType = .{ .sentinel = sentinel } }, + .rhs = undefined, // set by caller + }; + return &node.base; + } + return null; } /// ContainerDeclAuto <- ContainerDeclType LBRACE ContainerMembers RBRACE diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index dab2d66319..d6e2456455 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -418,13 +418,26 @@ fn renderExpression( switch (prefix_op_node.op) { ast.Node.PrefixOp.Op.PtrType => |ptr_info| { - const star_offset = switch (tree.tokens.at(prefix_op_node.op_token).id) { - Token.Id.AsteriskAsterisk => @as(usize, 1), - else => @as(usize, 0), - }; - try renderTokenOffset(tree, stream, prefix_op_node.op_token, indent, start_col, Space.None, star_offset); // * - if (ptr_info.null_token) |null_token| { - try renderToken(tree, stream, null_token, indent, start_col, Space.Space); // null + const op_tok_id = tree.tokens.at(prefix_op_node.op_token).id; + switch (op_tok_id) { + .Asterisk, .AsteriskAsterisk => try stream.writeByte('*'), + .Identifier => try stream.write("[*c]"), + .LBracket => try stream.write("[*"), + else => unreachable, + } + if (ptr_info.sentinel) |sentinel| { + const colon_token = tree.prevToken(sentinel.firstToken()); + try renderToken(tree, stream, colon_token, indent, start_col, Space.None); // : + const sentinel_space = switch (op_tok_id) { + .LBracket => Space.None, + else => Space.Space, + }; + try renderExpression(allocator, stream, tree, indent, start_col, sentinel, sentinel_space); + } + switch (op_tok_id) { + .Asterisk, .AsteriskAsterisk, .Identifier => {}, + .LBracket => try stream.writeByte(']'), + else => unreachable, } if (ptr_info.allowzero_token) |allowzero_token| { try renderToken(tree, stream, allowzero_token, indent, start_col, Space.Space); // allowzero @@ -504,7 +517,10 @@ fn renderExpression( ast.Node.PrefixOp.Op.ArrayType => |array_info| { const lbracket = prefix_op_node.op_token; - const rbracket = tree.nextToken(array_info.len_expr.lastToken()); + const rbracket = tree.nextToken(if (array_info.sentinel) |sentinel| + sentinel.lastToken() + else + array_info.len_expr.lastToken()); try renderToken(tree, stream, lbracket, indent, start_col, Space.None); // [ @@ -519,10 +535,12 @@ fn renderExpression( if (ends_with_comment or starts_with_comment) { try stream.writeByteNTimes(' ', indent); } - try renderToken(tree, stream, rbracket, indent, start_col, Space.None); // ] - if (array_info.null_token) |null_token| { - try renderToken(tree, stream, null_token, indent, start_col, Space.Space); // null + if (array_info.sentinel) |sentinel| { + const colon_token = tree.prevToken(sentinel.firstToken()); + try renderToken(tree, stream, colon_token, indent, start_col, Space.None); // : + try renderExpression(allocator, stream, tree, indent, start_col, sentinel, Space.None); } + try renderToken(tree, stream, rbracket, indent, start_col, Space.None); // ] }, ast.Node.PrefixOp.Op.BitNot, ast.Node.PrefixOp.Op.BoolNot, diff --git a/lib/std/zig/tokenizer.zig b/lib/std/zig/tokenizer.zig index 2e89669e2c..0b2aea4cf6 100644 --- a/lib/std/zig/tokenizer.zig +++ b/lib/std/zig/tokenizer.zig @@ -143,8 +143,6 @@ pub const Token = struct { LineComment, DocComment, ContainerDocComment, - BracketStarBracket, - BracketStarCBracket, ShebangLine, Keyword_align, Keyword_allowzero, @@ -269,8 +267,6 @@ pub const Token = struct { .AngleBracketAngleBracketRight => ">>", .AngleBracketAngleBracketRightEqual => ">>=", .Tilde => "~", - .BracketStarBracket => "[*]", - .BracketStarCBracket => "[*c]", .Keyword_align => "align", .Keyword_allowzero => "allowzero", .Keyword_and => "and", @@ -400,9 +396,6 @@ pub const Tokenizer = struct { Period, Period2, SawAtSign, - LBracket, - LBracketStar, - LBracketStarC, }; pub fn next(self: *Tokenizer) Token { @@ -460,7 +453,9 @@ pub const Tokenizer = struct { break; }, '[' => { - state = State.LBracket; + result.id = .LBracket; + self.index += 1; + break; }, ']' => { result.id = Token.Id.RBracket; @@ -564,43 +559,6 @@ pub const Tokenizer = struct { }, }, - State.LBracket => switch (c) { - '*' => { - state = State.LBracketStar; - }, - else => { - result.id = Token.Id.LBracket; - break; - }, - }, - - State.LBracketStar => switch (c) { - 'c' => { - state = State.LBracketStarC; - }, - ']' => { - result.id = Token.Id.BracketStarBracket; - self.index += 1; - break; - }, - else => { - result.id = Token.Id.Invalid; - break; - }, - }, - - State.LBracketStarC => switch (c) { - ']' => { - result.id = Token.Id.BracketStarCBracket; - self.index += 1; - break; - }, - else => { - result.id = Token.Id.Invalid; - break; - }, - }, - State.Ampersand => switch (c) { '&' => { result.id = Token.Id.Invalid_ampersands; @@ -1227,8 +1185,6 @@ pub const Tokenizer = struct { State.CharLiteralEnd, State.CharLiteralUnicode, State.StringLiteralBackslash, - State.LBracketStar, - State.LBracketStarC, => { result.id = Token.Id.Invalid; }, @@ -1245,9 +1201,6 @@ pub const Tokenizer = struct { State.Slash => { result.id = Token.Id.Slash; }, - State.LBracket => { - result.id = Token.Id.LBracket; - }, State.Zero => { result.id = Token.Id.IntegerLiteral; }, @@ -1368,9 +1321,14 @@ test "tokenizer - unknown length pointer and then c pointer" { \\[*]u8 \\[*c]u8 , [_]Token.Id{ - Token.Id.BracketStarBracket, + Token.Id.LBracket, + Token.Id.Asterisk, + Token.Id.RBracket, Token.Id.Identifier, - Token.Id.BracketStarCBracket, + Token.Id.LBracket, + Token.Id.Asterisk, + Token.Id.Identifier, + Token.Id.RBracket, Token.Id.Identifier, }); } diff --git a/src-self-hosted/translate_c.zig b/src-self-hosted/translate_c.zig index 754224d92c..d08f7293a6 100644 --- a/src-self-hosted/translate_c.zig +++ b/src-self-hosted/translate_c.zig @@ -1105,19 +1105,32 @@ fn transCreateNodePtrType( is_const: bool, is_volatile: bool, op_tok_id: std.zig.Token.Id, - bytes: []const u8, ) !*ast.Node.PrefixOp { const node = try c.a().create(ast.Node.PrefixOp); + const op_token = switch (op_tok_id) { + .LBracket => blk: { + const lbracket = try appendToken(c, .LBracket, "["); + _ = try appendToken(c, .Asterisk, "*"); + _ = try appendToken(c, .RBracket, "]"); + break :blk lbracket; + }, + .Identifier => blk: { + _ = try appendToken(c, .LBracket, "["); + _ = try appendToken(c, .Asterisk, "*"); + const c_ident = try appendToken(c, .Identifier, "c"); + _ = try appendToken(c, .RBracket, "]"); + break :blk c_ident; + }, + .Asterisk => try appendToken(c, .Asterisk, "*"), + else => unreachable, + }; node.* = ast.Node.PrefixOp{ .base = ast.Node{ .id = .PrefixOp }, - .op_token = try appendToken(c, op_tok_id, bytes), + .op_token = op_token, .op = ast.Node.PrefixOp.Op{ - .PtrType = ast.Node.PrefixOp.PtrInfo{ - .allowzero_token = null, - .align_info = null, + .PtrType = .{ .const_token = if (is_const) try appendToken(c, .Keyword_const, "const") else null, .volatile_token = if (is_volatile) try appendToken(c, .Keyword_volatile, "volatile") else null, - .null_token = null, }, }, .rhs = undefined, // translate and set afterward @@ -1226,7 +1239,6 @@ fn transType(rp: RestorePoint, ty: *const ZigClangType, source_loc: ZigClangSour ZigClangQualType_isConstQualified(child_qt), ZigClangQualType_isVolatileQualified(child_qt), .Asterisk, - "*", ); optional_node.rhs = &pointer_node.base; pointer_node.rhs = try transQualType(rp, child_qt, source_loc); @@ -1236,8 +1248,7 @@ fn transType(rp: RestorePoint, ty: *const ZigClangType, source_loc: ZigClangSour rp.c, ZigClangQualType_isConstQualified(child_qt), ZigClangQualType_isVolatileQualified(child_qt), - .BracketStarCBracket, - "[*c]", + .Identifier, ); pointer_node.rhs = try transQualType(rp, child_qt, source_loc); return &pointer_node.base;