From 93384f7428c5186a1b44c6f5f8891ce40bc46e72 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 19 May 2020 21:11:30 -0400 Subject: [PATCH 01/38] use singly linked lists for std.zig.parse std.ast uses a singly linked list for lists of things. This is a breaking change to the self-hosted parser API. std.ast.Tree has been separated into a private "Parser" type which represents in-progress parsing, and std.ast.Tree which has only "output" data. This means cleaner, but breaking, API for parse results. Specifically, `tokens` and `errors` are no longer SegmentedList but a slice. The way to iterate over AST nodes has necessarily changed since lists of nodes are now singly linked lists rather than SegmentedList. From these changes, I observe the following on the self-hosted-parser benchmark from ziglang/gotta-go-fast: throughput: 45.6 MiB/s => 55.6 MiB/s maxrss: 359 KB => 342 KB This commit breaks the build; more updates are necessary to fix API usage of the self-hosted parser. --- lib/std/linked_list.zig | 55 +- lib/std/zig/ast.zig | 645 +++- lib/std/zig/parse.zig | 6188 ++++++++++++++++++----------------- lib/std/zig/parser_test.zig | 9 +- lib/std/zig/render.zig | 127 +- 5 files changed, 3651 insertions(+), 3373 deletions(-) diff --git a/lib/std/linked_list.zig b/lib/std/linked_list.zig index 50acbb2c9f..3152413d1c 100644 --- a/lib/std/linked_list.zig +++ b/lib/std/linked_list.zig @@ -21,6 +21,8 @@ pub fn SinglyLinkedList(comptime T: type) type { next: ?*Node = null, data: T, + pub const Data = T; + pub fn init(data: T) Node { return Node{ .data = data, @@ -51,25 +53,6 @@ pub fn SinglyLinkedList(comptime T: type) type { first: ?*Node = null, - /// Initialize a linked list. - /// - /// Returns: - /// An empty linked list. - pub fn init() Self { - return Self{ - .first = null, - }; - } - - /// Insert a new node after an existing one. - /// - /// Arguments: - /// node: Pointer to a node in the list. - /// new_node: Pointer to the new node to insert. - pub fn insertAfter(list: *Self, node: *Node, new_node: *Node) void { - node.insertAfter(new_node); - } - /// Insert a new node at the head. /// /// Arguments: @@ -104,40 +87,6 @@ pub fn SinglyLinkedList(comptime T: type) type { list.first = first.next; return first; } - - /// Allocate a new node. - /// - /// Arguments: - /// allocator: Dynamic memory allocator. - /// - /// Returns: - /// A pointer to the new node. - pub fn allocateNode(list: *Self, allocator: *Allocator) !*Node { - return allocator.create(Node); - } - - /// Deallocate a node. - /// - /// Arguments: - /// node: Pointer to the node to deallocate. - /// allocator: Dynamic memory allocator. - pub fn destroyNode(list: *Self, node: *Node, allocator: *Allocator) void { - allocator.destroy(node); - } - - /// Allocate and initialize a node and its data. - /// - /// Arguments: - /// data: The data to put inside the node. - /// allocator: Dynamic memory allocator. - /// - /// Returns: - /// A pointer to the new node. - pub fn createNode(list: *Self, data: T, allocator: *Allocator) !*Node { - var node = try list.allocateNode(allocator); - node.* = Node.init(data); - return node; - } }; } diff --git a/lib/std/zig/ast.zig b/lib/std/zig/ast.zig index fcbb090ace..8517b5b623 100644 --- a/lib/std/zig/ast.zig +++ b/lib/std/zig/ast.zig @@ -1,42 +1,39 @@ const std = @import("../std.zig"); const assert = std.debug.assert; const testing = std.testing; -const SegmentedList = std.SegmentedList; +const LinkedList = std.SinglyLinkedList; const mem = std.mem; const Token = std.zig.Token; pub const TokenIndex = usize; pub const Tree = struct { + /// Reference to externally-owned data. source: []const u8, - tokens: TokenList, - - /// undefined on parse error (errors not empty) + tokens: []const Token, + errors: []const Error, + /// undefined on parse error (when errors field is not empty) root_node: *Node.Root, - arena_allocator: std.heap.ArenaAllocator, - errors: ErrorList, + + arena: std.heap.ArenaAllocator.State, + gpa: *mem.Allocator, /// translate-c uses this to avoid having to emit correct newlines /// TODO get rid of this hack generated: bool = false, - pub const TokenList = SegmentedList(Token, 64); - pub const ErrorList = SegmentedList(Error, 0); - pub fn deinit(self: *Tree) void { - // Here we copy the arena allocator into stack memory, because - // otherwise it would destroy itself while it was still working. - var arena_allocator = self.arena_allocator; - arena_allocator.deinit(); - // self is destroyed + self.gpa.free(self.tokens); + self.gpa.free(self.errors); + self.arena.promote(self.gpa).deinit(); } - pub fn renderError(self: *Tree, parse_error: *Error, stream: var) !void { - return parse_error.render(&self.tokens, stream); + pub fn renderError(self: *Tree, parse_error: *const Error, stream: var) !void { + return parse_error.render(self.tokens, stream); } pub fn tokenSlice(self: *Tree, token_index: TokenIndex) []const u8 { - return self.tokenSlicePtr(self.tokens.at(token_index)); + return self.tokenSlicePtr(self.tokens[token_index]); } pub fn tokenSlicePtr(self: *Tree, token: *const Token) []const u8 { @@ -44,8 +41,8 @@ pub const Tree = struct { } pub fn getNodeSource(self: *const Tree, node: *const Node) []const u8 { - const first_token = self.tokens.at(node.firstToken()); - const last_token = self.tokens.at(node.lastToken()); + const first_token = self.tokens[node.firstToken()]; + const last_token = self.tokens[node.lastToken()]; return self.source[first_token.start..last_token.end]; } @@ -57,7 +54,7 @@ pub const Tree = struct { }; /// Return the Location of the token relative to the offset specified by `start_index`. - pub fn tokenLocationPtr(self: *Tree, start_index: usize, token: *const Token) Location { + pub fn tokenLocationPtr(self: *Tree, start_index: usize, token: Token) Location { var loc = Location{ .line = 0, .column = 0, @@ -85,11 +82,11 @@ pub const Tree = struct { } pub fn tokenLocation(self: *Tree, start_index: usize, token_index: TokenIndex) Location { - return self.tokenLocationPtr(start_index, self.tokens.at(token_index)); + return self.tokenLocationPtr(start_index, self.tokens[token_index]); } pub fn tokensOnSameLine(self: *Tree, token1_index: TokenIndex, token2_index: TokenIndex) bool { - return self.tokensOnSameLinePtr(self.tokens.at(token1_index), self.tokens.at(token2_index)); + return self.tokensOnSameLinePtr(self.tokens[token1_index], self.tokens[token2_index]); } pub fn tokensOnSameLinePtr(self: *Tree, token1: *const Token, token2: *const Token) bool { @@ -103,7 +100,7 @@ pub const Tree = struct { /// Skips over comments pub fn prevToken(self: *Tree, token_index: TokenIndex) TokenIndex { var index = token_index - 1; - while (self.tokens.at(index).id == Token.Id.LineComment) { + while (self.tokens[index].id == Token.Id.LineComment) { index -= 1; } return index; @@ -112,7 +109,7 @@ pub const Tree = struct { /// Skips over comments pub fn nextToken(self: *Tree, token_index: TokenIndex) TokenIndex { var index = token_index + 1; - while (self.tokens.at(index).id == Token.Id.LineComment) { + while (self.tokens[index].id == Token.Id.LineComment) { index += 1; } return index; @@ -169,7 +166,7 @@ pub const Error = union(enum) { DeclBetweenFields: DeclBetweenFields, InvalidAnd: InvalidAnd, - pub fn render(self: *const Error, tokens: *Tree.TokenList, stream: var) !void { + pub fn render(self: *const Error, tokens: []const Token, stream: var) !void { switch (self.*) { .InvalidToken => |*x| return x.render(tokens, stream), .ExpectedContainerMembers => |*x| return x.render(tokens, stream), @@ -324,7 +321,7 @@ pub const Error = union(enum) { pub const ExpectedCall = struct { node: *Node, - pub fn render(self: *const ExpectedCall, tokens: *Tree.TokenList, stream: var) !void { + pub fn render(self: *const ExpectedCall, tokens: []const Token, stream: var) !void { return stream.print("expected " ++ @tagName(@TagType(Node.SuffixOp.Op).Call) ++ ", found {}", .{ @tagName(self.node.id), }); @@ -334,7 +331,7 @@ pub const Error = union(enum) { pub const ExpectedCallOrFnProto = struct { node: *Node, - pub fn render(self: *const ExpectedCallOrFnProto, tokens: *Tree.TokenList, stream: var) !void { + pub fn render(self: *const ExpectedCallOrFnProto, tokens: []const Token, stream: var) !void { return stream.print("expected " ++ @tagName(@TagType(Node.SuffixOp.Op).Call) ++ " or " ++ @tagName(Node.Id.FnProto) ++ ", found {}", .{@tagName(self.node.id)}); } @@ -344,8 +341,8 @@ pub const Error = union(enum) { token: TokenIndex, expected_id: Token.Id, - pub fn render(self: *const ExpectedToken, tokens: *Tree.TokenList, stream: var) !void { - const found_token = tokens.at(self.token); + pub fn render(self: *const ExpectedToken, tokens: []const Token, stream: var) !void { + const found_token = tokens[self.token]; switch (found_token.id) { .Invalid => { return stream.print("expected '{}', found invalid bytes", .{self.expected_id.symbol()}); @@ -362,8 +359,8 @@ pub const Error = union(enum) { token: TokenIndex, end_id: Token.Id, - pub fn render(self: *const ExpectedCommaOrEnd, tokens: *Tree.TokenList, stream: var) !void { - const actual_token = tokens.at(self.token); + pub fn render(self: *const ExpectedCommaOrEnd, tokens: []const Token, stream: var) !void { + const actual_token = tokens[self.token]; return stream.print("expected ',' or '{}', found '{}'", .{ self.end_id.symbol(), actual_token.id.symbol(), @@ -377,8 +374,8 @@ pub const Error = union(enum) { token: TokenIndex, - pub fn render(self: *const ThisError, tokens: *Tree.TokenList, stream: var) !void { - const actual_token = tokens.at(self.token); + pub fn render(self: *const ThisError, tokens: []const Token, stream: var) !void { + const actual_token = tokens[self.token]; return stream.print(msg, .{actual_token.id.symbol()}); } }; @@ -390,7 +387,7 @@ pub const Error = union(enum) { token: TokenIndex, - pub fn render(self: *const ThisError, tokens: *Tree.TokenList, stream: var) !void { + pub fn render(self: *const ThisError, tokens: []const Token, stream: var) !void { return stream.writeAll(msg); } }; @@ -400,6 +397,23 @@ pub const Error = union(enum) { pub const Node = struct { id: Id, + /// All the child Node types use this same Iterator state for their iteration. + pub const Iterator = struct { + parent_node: *const Node, + node: ?*LinkedList(*Node).Node, + index: usize, + + pub fn next(it: *Iterator) ?*Node { + inline for (@typeInfo(Id).Enum.fields) |f| { + if (it.parent_node.id == @field(Id, f.name)) { + const T = @field(Node, f.name); + return @fieldParentPtr(T, "base", it.parent_node).iterateNext(it); + } + } + unreachable; + } + }; + pub const Id = enum { // Top level Root, @@ -473,11 +487,11 @@ pub const Node = struct { return null; } - pub fn iterate(base: *Node, index: usize) ?*Node { + pub fn iterate(base: *Node) Iterator { inline for (@typeInfo(Id).Enum.fields) |f| { if (base.id == @field(Id, f.name)) { const T = @field(Node, f.name); - return @fieldParentPtr(T, "base", base).iterate(index); + return @fieldParentPtr(T, "base", base).iterate(); } } unreachable; @@ -607,21 +621,35 @@ pub const Node = struct { decls: DeclList, eof_token: TokenIndex, - pub const DeclList = SegmentedList(*Node, 4); + pub const DeclList = LinkedList(*Node); - pub fn iterate(self: *Root, index: usize) ?*Node { - if (index < self.decls.len) { - return self.decls.at(index).*; - } - return null; + pub fn iterate(self: *const Root) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = self.decls.first }; + } + + pub fn iterateNext(self: *const Root, it: *Node.Iterator) ?*Node { + const decl = it.node orelse return null; + it.node = decl.next; + return decl.data; } pub fn firstToken(self: *const Root) TokenIndex { - return if (self.decls.len == 0) self.eof_token else (self.decls.at(0).*).firstToken(); + if (self.decls.first) |first| { + return first.data.firstToken(); + } else { + return self.eof_token; + } } pub fn lastToken(self: *const Root) TokenIndex { - return if (self.decls.len == 0) self.eof_token else (self.decls.at(self.decls.len - 1).*).lastToken(); + if (self.decls.first) |first| { + var node = first; + while (true) { + node = node.next orelse return node.data.lastToken(); + } + } else { + return self.eof_token; + } } }; @@ -642,8 +670,13 @@ pub const Node = struct { init_node: ?*Node, semicolon_token: TokenIndex, - pub fn iterate(self: *VarDecl, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const VarDecl) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const VarDecl, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; if (self.type_node) |type_node| { if (i < 1) return type_node; @@ -668,6 +701,7 @@ pub const Node = struct { return null; } + pub fn firstToken(self: *const VarDecl) TokenIndex { if (self.visib_token) |visib_token| return visib_token; if (self.thread_local_token) |thread_local_token| return thread_local_token; @@ -690,8 +724,13 @@ pub const Node = struct { expr: *Node, semicolon_token: TokenIndex, - pub fn iterate(self: *Use, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const Use) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const Use, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; if (i < 1) return self.expr; i -= 1; @@ -715,15 +754,16 @@ pub const Node = struct { decls: DeclList, rbrace_token: TokenIndex, - pub const DeclList = SegmentedList(*Node, 2); + pub const DeclList = LinkedList(*Node); - pub fn iterate(self: *ErrorSetDecl, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const ErrorSetDecl) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = self.decls.first }; + } - if (i < self.decls.len) return self.decls.at(i).*; - i -= self.decls.len; - - return null; + pub fn iterateNext(self: *const ErrorSetDecl, it: *Node.Iterator) ?*Node { + const decl = it.node orelse return null; + it.node = decl.next; + return decl.data; } pub fn firstToken(self: *const ErrorSetDecl) TokenIndex { @@ -752,8 +792,13 @@ pub const Node = struct { Type: *Node, }; - pub fn iterate(self: *ContainerDecl, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const ContainerDecl) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = self.fields_and_decls.first }; + } + + pub fn iterateNext(self: *const ContainerDecl, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; switch (self.init_arg_expr) { .Type => |t| { @@ -763,8 +808,10 @@ pub const Node = struct { .None, .Enum => {}, } - if (i < self.fields_and_decls.len) return self.fields_and_decls.at(i).*; - i -= self.fields_and_decls.len; + if (it.node) |child| { + it.node = child.next; + return child.data; + } return null; } @@ -790,8 +837,13 @@ pub const Node = struct { value_expr: ?*Node, align_expr: ?*Node, - pub fn iterate(self: *ContainerField, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const ContainerField) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const ContainerField, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; if (self.type_expr) |type_expr| { if (i < 1) return type_expr; @@ -837,8 +889,13 @@ pub const Node = struct { doc_comments: ?*DocComment, name_token: TokenIndex, - pub fn iterate(self: *ErrorTag, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const ErrorTag) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const ErrorTag, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; if (self.doc_comments) |comments| { if (i < 1) return &comments.base; @@ -861,7 +918,11 @@ pub const Node = struct { base: Node = Node{ .id = .Identifier }, token: TokenIndex, - pub fn iterate(self: *Identifier, index: usize) ?*Node { + pub fn iterate(self: *const Identifier) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const Identifier, it: *Node.Iterator) ?*Node { return null; } @@ -892,7 +953,7 @@ pub const Node = struct { is_extern_prototype: bool = false, // TODO: Remove once extern fn rewriting is is_async: bool = false, // TODO: remove once async fn rewriting is - pub const ParamList = SegmentedList(*Node, 2); + pub const ParamList = LinkedList(*Node); pub const ReturnType = union(enum) { Explicit: *Node, @@ -900,16 +961,24 @@ pub const Node = struct { Invalid: TokenIndex, }; - pub fn iterate(self: *FnProto, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const FnProto) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = self.params.first }; + } + + pub fn iterateNext(self: *const FnProto, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; if (self.lib_name) |lib_name| { if (i < 1) return lib_name; i -= 1; } - if (i < self.params.len) return self.params.at(self.params.len - i - 1).*; - i -= self.params.len; + if (it.node) |param| { + it.index -= 1; + it.node = param.next; + return param.data; + } if (self.align_expr) |align_expr| { if (i < 1) return align_expr; @@ -963,8 +1032,13 @@ pub const Node = struct { return_type: *Node, }; - pub fn iterate(self: *AnyFrameType, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const AnyFrameType) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const AnyFrameType, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; if (self.result) |result| { if (i < 1) return result.return_type; @@ -998,8 +1072,13 @@ pub const Node = struct { type_expr: *Node, }; - pub fn iterate(self: *ParamDecl, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const ParamDecl) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const ParamDecl, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; if (i < 1) { switch (self.param_type) { @@ -1039,13 +1118,14 @@ pub const Node = struct { pub const StatementList = Root.DeclList; - pub fn iterate(self: *Block, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const Block) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = self.statements.first }; + } - if (i < self.statements.len) return self.statements.at(i).*; - i -= self.statements.len; - - return null; + pub fn iterateNext(self: *const Block, it: *Node.Iterator) ?*Node { + const child = it.node orelse return null; + it.node = child.next; + return child.data; } pub fn firstToken(self: *const Block) TokenIndex { @@ -1067,8 +1147,13 @@ pub const Node = struct { payload: ?*Node, expr: *Node, - pub fn iterate(self: *Defer, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const Defer) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const Defer, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; if (i < 1) return self.expr; i -= 1; @@ -1091,8 +1176,13 @@ pub const Node = struct { comptime_token: TokenIndex, expr: *Node, - pub fn iterate(self: *Comptime, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const Comptime) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const Comptime, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; if (i < 1) return self.expr; i -= 1; @@ -1114,8 +1204,13 @@ pub const Node = struct { nosuspend_token: TokenIndex, expr: *Node, - pub fn iterate(self: *Nosuspend, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const Nosuspend) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const Nosuspend, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; if (i < 1) return self.expr; i -= 1; @@ -1138,8 +1233,13 @@ pub const Node = struct { error_symbol: *Node, rpipe: TokenIndex, - pub fn iterate(self: *Payload, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const Payload) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const Payload, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; if (i < 1) return self.error_symbol; i -= 1; @@ -1163,8 +1263,13 @@ pub const Node = struct { value_symbol: *Node, rpipe: TokenIndex, - pub fn iterate(self: *PointerPayload, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const PointerPayload) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const PointerPayload, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; if (i < 1) return self.value_symbol; i -= 1; @@ -1189,8 +1294,13 @@ pub const Node = struct { index_symbol: ?*Node, rpipe: TokenIndex, - pub fn iterate(self: *PointerIndexPayload, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const PointerIndexPayload) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const PointerIndexPayload, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; if (i < 1) return self.value_symbol; i -= 1; @@ -1218,8 +1328,13 @@ pub const Node = struct { payload: ?*Node, body: *Node, - pub fn iterate(self: *Else, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const Else) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const Else, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; if (self.payload) |payload| { if (i < 1) return payload; @@ -1250,16 +1365,24 @@ pub const Node = struct { cases: CaseList, rbrace: TokenIndex, - pub const CaseList = SegmentedList(*Node, 2); + pub const CaseList = LinkedList(*Node); - pub fn iterate(self: *Switch, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const Switch) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = self.cases.first }; + } + + pub fn iterateNext(self: *const Switch, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; if (i < 1) return self.expr; i -= 1; - if (i < self.cases.len) return self.cases.at(i).*; - i -= self.cases.len; + if (it.node) |child| { + it.index -= 1; + it.node = child.next; + return child.data; + } return null; } @@ -1280,13 +1403,21 @@ pub const Node = struct { payload: ?*Node, expr: *Node, - pub const ItemList = SegmentedList(*Node, 1); + pub const ItemList = LinkedList(*Node); - pub fn iterate(self: *SwitchCase, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const SwitchCase) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = self.items.first }; + } - if (i < self.items.len) return self.items.at(i).*; - i -= self.items.len; + pub fn iterateNext(self: *const SwitchCase, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; + + if (it.node) |child| { + it.index -= 1; + it.node = child.next; + return child.data; + } if (self.payload) |payload| { if (i < 1) return payload; @@ -1300,7 +1431,7 @@ pub const Node = struct { } pub fn firstToken(self: *const SwitchCase) TokenIndex { - return (self.items.at(0).*).firstToken(); + return self.items.first.?.data.firstToken(); } pub fn lastToken(self: *const SwitchCase) TokenIndex { @@ -1312,7 +1443,11 @@ pub const Node = struct { base: Node = Node{ .id = .SwitchElse }, token: TokenIndex, - pub fn iterate(self: *SwitchElse, index: usize) ?*Node { + pub fn iterate(self: *const SwitchElse) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const SwitchElse, it: *Node.Iterator) ?*Node { return null; } @@ -1336,8 +1471,13 @@ pub const Node = struct { body: *Node, @"else": ?*Else, - pub fn iterate(self: *While, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const While) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const While, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; if (i < 1) return self.condition; i -= 1; @@ -1394,8 +1534,13 @@ pub const Node = struct { body: *Node, @"else": ?*Else, - pub fn iterate(self: *For, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const For) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const For, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; if (i < 1) return self.array_expr; i -= 1; @@ -1443,8 +1588,13 @@ pub const Node = struct { body: *Node, @"else": ?*Else, - pub fn iterate(self: *If, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const If) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const If, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; if (i < 1) return self.condition; i -= 1; @@ -1531,8 +1681,13 @@ pub const Node = struct { UnwrapOptional, }; - pub fn iterate(self: *InfixOp, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const InfixOp) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const InfixOp, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; if (i < 1) return self.lhs; i -= 1; @@ -1649,8 +1804,13 @@ pub const Node = struct { }; }; - pub fn iterate(self: *PrefixOp, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const PrefixOp) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const PrefixOp, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; switch (self.op) { .PtrType, .SliceType => |addr_of_info| { @@ -1707,8 +1867,13 @@ pub const Node = struct { name_token: TokenIndex, expr: *Node, - pub fn iterate(self: *FieldInitializer, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const FieldInitializer) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const FieldInitializer, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; if (i < 1) return self.expr; i -= 1; @@ -1745,13 +1910,13 @@ pub const Node = struct { Deref, UnwrapOptional, - pub const InitList = SegmentedList(*Node, 2); + pub const InitList = LinkedList(*Node); pub const Call = struct { params: ParamList, async_token: ?TokenIndex, - pub const ParamList = SegmentedList(*Node, 2); + pub const ParamList = LinkedList(*Node); }; pub const Slice = struct { @@ -1761,8 +1926,20 @@ pub const Node = struct { }; }; - pub fn iterate(self: *SuffixOp, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const SuffixOp) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, + .node = switch(self.op) { + .Call => |call| call.params.first, + .ArrayInitializer => |ai| ai.first, + .StructInitializer => |si| si.first, + else => null, + }, + }; + } + + pub fn iterateNext(self: *const SuffixOp, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; switch (self.lhs) { .node => |node| { @@ -1773,9 +1950,12 @@ pub const Node = struct { } switch (self.op) { - .Call => |*call_info| { - if (i < call_info.params.len) return call_info.params.at(i).*; - i -= call_info.params.len; + .Call => |call_info| { + if (it.node) |child| { + it.index -= 1; + it.node = child.next; + return child.data; + } }, .ArrayAccess => |index_expr| { if (i < 1) return index_expr; @@ -1794,13 +1974,19 @@ pub const Node = struct { i -= 1; } }, - .ArrayInitializer => |*exprs| { - if (i < exprs.len) return exprs.at(i).*; - i -= exprs.len; + .ArrayInitializer => |exprs| { + if (it.node) |child| { + it.index -= 1; + it.node = child.next; + return child.data; + } }, - .StructInitializer => |*fields| { - if (i < fields.len) return fields.at(i).*; - i -= fields.len; + .StructInitializer => |fields| { + if (it.node) |child| { + it.index -= 1; + it.node = child.next; + return child.data; + } }, .UnwrapOptional, .Deref, @@ -1832,8 +2018,13 @@ pub const Node = struct { expr: *Node, rparen: TokenIndex, - pub fn iterate(self: *GroupedExpression, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const GroupedExpression) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const GroupedExpression, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; if (i < 1) return self.expr; i -= 1; @@ -1862,8 +2053,13 @@ pub const Node = struct { Return, }; - pub fn iterate(self: *ControlFlowExpression, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const ControlFlowExpression) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const ControlFlowExpression, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; switch (self.kind) { .Break, .Continue => |maybe_label| { @@ -1910,8 +2106,13 @@ pub const Node = struct { suspend_token: TokenIndex, body: ?*Node, - pub fn iterate(self: *Suspend, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const Suspend) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const Suspend, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; if (self.body) |body| { if (i < 1) return body; @@ -1938,7 +2139,11 @@ pub const Node = struct { base: Node = Node{ .id = .IntegerLiteral }, token: TokenIndex, - pub fn iterate(self: *IntegerLiteral, index: usize) ?*Node { + pub fn iterate(self: *const IntegerLiteral) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const IntegerLiteral, it: *Node.Iterator) ?*Node { return null; } @@ -1956,7 +2161,11 @@ pub const Node = struct { dot: TokenIndex, name: TokenIndex, - pub fn iterate(self: *EnumLiteral, index: usize) ?*Node { + pub fn iterate(self: *const EnumLiteral) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const EnumLiteral, it: *Node.Iterator) ?*Node { return null; } @@ -1973,7 +2182,11 @@ pub const Node = struct { base: Node = Node{ .id = .FloatLiteral }, token: TokenIndex, - pub fn iterate(self: *FloatLiteral, index: usize) ?*Node { + pub fn iterate(self: *const FloatLiteral) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const FloatLiteral, it: *Node.Iterator) ?*Node { return null; } @@ -1992,15 +2205,16 @@ pub const Node = struct { params: ParamList, rparen_token: TokenIndex, - pub const ParamList = SegmentedList(*Node, 2); + pub const ParamList = LinkedList(*Node); - pub fn iterate(self: *BuiltinCall, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const BuiltinCall) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = self.params.first }; + } - if (i < self.params.len) return self.params.at(i).*; - i -= self.params.len; - - return null; + pub fn iterateNext(self: *const BuiltinCall, it: *Node.Iterator) ?*Node { + const param = it.node orelse return null; + it.node = param.next; + return param.data; } pub fn firstToken(self: *const BuiltinCall) TokenIndex { @@ -2016,7 +2230,11 @@ pub const Node = struct { base: Node = Node{ .id = .StringLiteral }, token: TokenIndex, - pub fn iterate(self: *StringLiteral, index: usize) ?*Node { + pub fn iterate(self: *const StringLiteral) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const StringLiteral, it: *Node.Iterator) ?*Node { return null; } @@ -2033,18 +2251,25 @@ pub const Node = struct { base: Node = Node{ .id = .MultilineStringLiteral }, lines: LineList, - pub const LineList = SegmentedList(TokenIndex, 4); + pub const LineList = LinkedList(TokenIndex); - pub fn iterate(self: *MultilineStringLiteral, index: usize) ?*Node { + pub fn iterate(self: *const MultilineStringLiteral) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const MultilineStringLiteral, it: *Node.Iterator) ?*Node { return null; } pub fn firstToken(self: *const MultilineStringLiteral) TokenIndex { - return self.lines.at(0).*; + return self.lines.first.?.data; } pub fn lastToken(self: *const MultilineStringLiteral) TokenIndex { - return self.lines.at(self.lines.len - 1).*; + var node = self.lines.first.?; + while (true) { + node = node.next orelse return node.data; + } } }; @@ -2052,7 +2277,11 @@ pub const Node = struct { base: Node = Node{ .id = .CharLiteral }, token: TokenIndex, - pub fn iterate(self: *CharLiteral, index: usize) ?*Node { + pub fn iterate(self: *const CharLiteral) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const CharLiteral, it: *Node.Iterator) ?*Node { return null; } @@ -2069,7 +2298,11 @@ pub const Node = struct { base: Node = Node{ .id = .BoolLiteral }, token: TokenIndex, - pub fn iterate(self: *BoolLiteral, index: usize) ?*Node { + pub fn iterate(self: *const BoolLiteral) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const BoolLiteral, it: *Node.Iterator) ?*Node { return null; } @@ -2086,7 +2319,11 @@ pub const Node = struct { base: Node = Node{ .id = .NullLiteral }, token: TokenIndex, - pub fn iterate(self: *NullLiteral, index: usize) ?*Node { + pub fn iterate(self: *const NullLiteral) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const NullLiteral, it: *Node.Iterator) ?*Node { return null; } @@ -2103,7 +2340,11 @@ pub const Node = struct { base: Node = Node{ .id = .UndefinedLiteral }, token: TokenIndex, - pub fn iterate(self: *UndefinedLiteral, index: usize) ?*Node { + pub fn iterate(self: *const UndefinedLiteral) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const UndefinedLiteral, it: *Node.Iterator) ?*Node { return null; } @@ -2129,8 +2370,13 @@ pub const Node = struct { Return: *Node, }; - pub fn iterate(self: *AsmOutput, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const AsmOutput) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const AsmOutput, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; if (i < 1) return self.symbolic_name; i -= 1; @@ -2169,8 +2415,13 @@ pub const Node = struct { expr: *Node, rparen: TokenIndex, - pub fn iterate(self: *AsmInput, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const AsmInput) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const AsmInput, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; if (i < 1) return self.symbolic_name; i -= 1; @@ -2203,18 +2454,31 @@ pub const Node = struct { clobbers: ClobberList, rparen: TokenIndex, - pub const OutputList = SegmentedList(*AsmOutput, 2); - pub const InputList = SegmentedList(*AsmInput, 2); - pub const ClobberList = SegmentedList(*Node, 2); + pub const OutputList = LinkedList(*AsmOutput); + pub const InputList = LinkedList(*AsmInput); + pub const ClobberList = LinkedList(*Node); - pub fn iterate(self: *Asm, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const Asm) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null}; + } - if (i < self.outputs.len) return &self.outputs.at(i).*.base; - i -= self.outputs.len; + pub fn iterateNext(self: *const Asm, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; - if (i < self.inputs.len) return &self.inputs.at(i).*.base; - i -= self.inputs.len; + var output: ?*LinkedList(*AsmOutput).Node = self.outputs.first; + while (output) |o| { + if (i < 1) return &o.data.base; + i -= 1; + output = o.next; + } + + var input: ?*LinkedList(*AsmInput).Node = self.inputs.first; + while (input) |o| { + if (i < 1) return &o.data.base; + i -= 1; + input = o.next; + } return null; } @@ -2232,7 +2496,11 @@ pub const Node = struct { base: Node = Node{ .id = .Unreachable }, token: TokenIndex, - pub fn iterate(self: *Unreachable, index: usize) ?*Node { + pub fn iterate(self: *const Unreachable) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const Unreachable, it: *Node.Iterator) ?*Node { return null; } @@ -2249,7 +2517,11 @@ pub const Node = struct { base: Node = Node{ .id = .ErrorType }, token: TokenIndex, - pub fn iterate(self: *ErrorType, index: usize) ?*Node { + pub fn iterate(self: *const ErrorType) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const ErrorType, it: *Node.Iterator) ?*Node { return null; } @@ -2266,7 +2538,11 @@ pub const Node = struct { base: Node = Node{ .id = .VarType }, token: TokenIndex, - pub fn iterate(self: *VarType, index: usize) ?*Node { + pub fn iterate(self: *const VarType) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const VarType, it: *Node.Iterator) ?*Node { return null; } @@ -2283,18 +2559,25 @@ pub const Node = struct { base: Node = Node{ .id = .DocComment }, lines: LineList, - pub const LineList = SegmentedList(TokenIndex, 4); + pub const LineList = LinkedList(TokenIndex); - pub fn iterate(self: *DocComment, index: usize) ?*Node { + pub fn iterate(self: *const DocComment) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const DocComment, it: *Node.Iterator) ?*Node { return null; } pub fn firstToken(self: *const DocComment) TokenIndex { - return self.lines.at(0).*; + return self.lines.first.?.data; } pub fn lastToken(self: *const DocComment) TokenIndex { - return self.lines.at(self.lines.len - 1).*; + var node = self.lines.first.?; + while (true) { + node = node.next orelse return node.data; + } } }; @@ -2305,8 +2588,13 @@ pub const Node = struct { name: *Node, body_node: *Node, - pub fn iterate(self: *TestDecl, index: usize) ?*Node { - var i = index; + pub fn iterate(self: *const TestDecl) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const TestDecl, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; if (i < 1) return self.body_node; i -= 1; @@ -2331,5 +2619,6 @@ test "iterate" { .eof_token = 0, }; var base = &root.base; - testing.expect(base.iterate(0) == null); + var it = base.iterate(); + testing.expect(it.next() == null); } diff --git a/lib/std/zig/parse.zig b/lib/std/zig/parse.zig index 6dfd1120b6..d723ac934c 100644 --- a/lib/std/zig/parse.zig +++ b/lib/std/zig/parse.zig @@ -7,3303 +7,3349 @@ const Tree = ast.Tree; const AstError = ast.Error; const TokenIndex = ast.TokenIndex; const Token = std.zig.Token; -const TokenIterator = Tree.TokenList.Iterator; pub const Error = error{ParseError} || Allocator.Error; /// Result should be freed with tree.deinit() when there are /// no more references to any of the tokens or nodes. -pub fn parse(allocator: *Allocator, source: []const u8) Allocator.Error!*Tree { - const tree = blk: { - // This block looks unnecessary, but is a "foot-shield" to prevent the SegmentedLists - // from being initialized with a pointer to this `arena`, which is created on - // the stack. Following code should instead refer to `&tree.arena_allocator`, a - // pointer to data which lives safely on the heap and will outlive `parse`. See: - // https://github.com/ziglang/zig/commit/cb4fb14b6e66bd213575f69eec9598be8394fae6 - var arena = std.heap.ArenaAllocator.init(allocator); - errdefer arena.deinit(); - const tree = try arena.allocator.create(ast.Tree); - tree.* = .{ - .source = source, - .root_node = undefined, - .arena_allocator = arena, - .tokens = undefined, - .errors = undefined, - }; - break :blk tree; - }; - errdefer tree.deinit(); - const arena = &tree.arena_allocator.allocator; - - tree.tokens = ast.Tree.TokenList.init(arena); - tree.errors = ast.Tree.ErrorList.init(arena); +pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!*Tree { + // TODO optimization idea: ensureCapacity on the tokens list and + // then appendAssumeCapacity inside the loop. + var tokens = std.ArrayList(Token).init(gpa); + defer tokens.deinit(); var tokenizer = std.zig.Tokenizer.init(source); while (true) { - const tree_token = try tree.tokens.addOne(); + const tree_token = try tokens.addOne(); tree_token.* = tokenizer.next(); if (tree_token.id == .Eof) break; } - var it = tree.tokens.iterator(0); - while (it.peek().?.id == .LineComment) _ = it.next(); + var parser: Parser = .{ + .source = source, + .arena = std.heap.ArenaAllocator.init(gpa), + .gpa = gpa, + .tokens = tokens.items, + .errors = .{}, + .tok_i = 0, + }; + defer parser.errors.deinit(gpa); + errdefer parser.arena.deinit(); - tree.root_node = try parseRoot(arena, &it, tree); + while (tokens.items[parser.tok_i].id == .LineComment) parser.tok_i += 1; + const root_node = try parser.parseRoot(); + + const tree = try parser.arena.allocator.create(Tree); + tree.* = .{ + .gpa = gpa, + .source = source, + .tokens = tokens.toOwnedSlice(), + .errors = parser.errors.toOwnedSlice(gpa), + .root_node = root_node, + .arena = parser.arena.state, + }; return tree; } -/// Root <- skip ContainerMembers eof -fn parseRoot(arena: *Allocator, it: *TokenIterator, tree: *Tree) Allocator.Error!*Node.Root { - const node = try arena.create(Node.Root); - node.* = .{ - .decls = try parseContainerMembers(arena, it, tree, true), - // parseContainerMembers will try to skip as much - // invalid tokens as it can so this can only be the EOF - .eof_token = eatToken(it, .Eof).?, - }; - return node; -} +/// Represents in-progress parsing, will be converted to an ast.Tree after completion. +const Parser = struct { + arena: std.heap.ArenaAllocator, + gpa: *Allocator, + source: []const u8, + tokens: []const Token, + tok_i: TokenIndex, + errors: std.ArrayListUnmanaged(AstError), -/// ContainerMembers -/// <- TestDecl ContainerMembers -/// / TopLevelComptime ContainerMembers -/// / KEYWORD_pub? TopLevelDecl ContainerMembers -/// / ContainerField COMMA ContainerMembers -/// / ContainerField -/// / -fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree, top_level: bool) !Node.Root.DeclList { - var list = Node.Root.DeclList.init(arena); + /// Root <- skip ContainerMembers eof + fn parseRoot(p: *Parser) Allocator.Error!*Node.Root { + const node = try p.arena.allocator.create(Node.Root); + node.* = .{ + .decls = try parseContainerMembers(p, true), + // parseContainerMembers will try to skip as much + // invalid tokens as it can so this can only be the EOF + .eof_token = p.eatToken(.Eof).?, + }; + return node; + } - var field_state: union(enum) { - /// no fields have been seen - none, - /// currently parsing fields - seen, - /// saw fields and then a declaration after them. - /// payload is first token of previous declaration. - end: TokenIndex, - /// ther was a declaration between fields, don't report more errors - err, - } = .none; + fn llpush( + p: *Parser, + comptime T: type, + it: *?*std.SinglyLinkedList(T).Node, + data: T, + ) !*?*std.SinglyLinkedList(T).Node { + const llnode = try p.arena.allocator.create(std.SinglyLinkedList(T).Node); + llnode.* = .{ .data = data }; + it.* = llnode; + return &llnode.next; + } - while (true) { - if (try parseContainerDocComments(arena, it, tree)) |node| { - try list.push(node); - continue; - } + /// ContainerMembers + /// <- TestDecl ContainerMembers + /// / TopLevelComptime ContainerMembers + /// / KEYWORD_pub? TopLevelDecl ContainerMembers + /// / ContainerField COMMA ContainerMembers + /// / ContainerField + /// / + fn parseContainerMembers(p: *Parser, top_level: bool) !Node.Root.DeclList { + var list = Node.Root.DeclList{}; + var list_it = &list.first; - const doc_comments = try parseDocComment(arena, it, tree); + var field_state: union(enum) { + /// no fields have been seen + none, + /// currently parsing fields + seen, + /// saw fields and then a declaration after them. + /// payload is first token of previous declaration. + end: TokenIndex, + /// ther was a declaration between fields, don't report more errors + err, + } = .none; - if (parseTestDecl(arena, it, tree) catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - error.ParseError => { - findNextContainerMember(it); + while (true) { + if (try p.parseContainerDocComments()) |node| { + list_it = try p.llpush(*Node, list_it, node); continue; - }, - }) |node| { - if (field_state == .seen) { - field_state = .{ .end = node.firstToken() }; } - node.cast(Node.TestDecl).?.doc_comments = doc_comments; - try list.push(node); - continue; - } - if (parseTopLevelComptime(arena, it, tree) catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - error.ParseError => { - findNextContainerMember(it); + const doc_comments = try p.parseDocComment(); + + if (p.parseTestDecl() catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.ParseError => { + p.findNextContainerMember(); + continue; + }, + }) |node| { + if (field_state == .seen) { + field_state = .{ .end = node.firstToken() }; + } + node.cast(Node.TestDecl).?.doc_comments = doc_comments; + list_it = try p.llpush(*Node, list_it, node); continue; - }, - }) |node| { - if (field_state == .seen) { - field_state = .{ .end = node.firstToken() }; } - node.cast(Node.Comptime).?.doc_comments = doc_comments; - try list.push(node); - continue; - } - const visib_token = eatToken(it, .Keyword_pub); - - if (parseTopLevelDecl(arena, it, tree) catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - error.ParseError => { - findNextContainerMember(it); + if (p.parseTopLevelComptime() catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.ParseError => { + p.findNextContainerMember(); + continue; + }, + }) |node| { + if (field_state == .seen) { + field_state = .{ .end = node.firstToken() }; + } + node.cast(Node.Comptime).?.doc_comments = doc_comments; + list_it = try p.llpush(*Node, list_it, node); continue; - }, - }) |node| { - if (field_state == .seen) { - field_state = .{ .end = visib_token orelse node.firstToken() }; } - switch (node.id) { - .FnProto => { - node.cast(Node.FnProto).?.doc_comments = doc_comments; - node.cast(Node.FnProto).?.visib_token = visib_token; + + const visib_token = p.eatToken(.Keyword_pub); + + if (p.parseTopLevelDecl() catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.ParseError => { + p.findNextContainerMember(); + continue; }, - .VarDecl => { - node.cast(Node.VarDecl).?.doc_comments = doc_comments; - node.cast(Node.VarDecl).?.visib_token = visib_token; - }, - .Use => { - node.cast(Node.Use).?.doc_comments = doc_comments; - node.cast(Node.Use).?.visib_token = visib_token; - }, - else => unreachable, - } - try list.push(node); - if (try parseAppendedDocComment(arena, it, tree, node.lastToken())) |appended_comment| { + }) |node| { + if (field_state == .seen) { + field_state = .{ .end = visib_token orelse node.firstToken() }; + } switch (node.id) { - .FnProto => {}, - .VarDecl => node.cast(Node.VarDecl).?.doc_comments = appended_comment, - .Use => node.cast(Node.Use).?.doc_comments = appended_comment, + .FnProto => { + node.cast(Node.FnProto).?.doc_comments = doc_comments; + node.cast(Node.FnProto).?.visib_token = visib_token; + }, + .VarDecl => { + node.cast(Node.VarDecl).?.doc_comments = doc_comments; + node.cast(Node.VarDecl).?.visib_token = visib_token; + }, + .Use => { + node.cast(Node.Use).?.doc_comments = doc_comments; + node.cast(Node.Use).?.visib_token = visib_token; + }, else => unreachable, } - } - continue; - } - - if (visib_token != null) { - try tree.errors.push(.{ - .ExpectedPubItem = .{ .token = it.index }, - }); - // ignore this pub - continue; - } - - if (parseContainerField(arena, it, tree) catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - error.ParseError => { - // attempt to recover - findNextContainerMember(it); + list_it = try p.llpush(*Node, list_it, node); + if (try p.parseAppendedDocComment(node.lastToken())) |appended_comment| { + switch (node.id) { + .FnProto => {}, + .VarDecl => node.cast(Node.VarDecl).?.doc_comments = appended_comment, + .Use => node.cast(Node.Use).?.doc_comments = appended_comment, + else => unreachable, + } + } continue; - }, - }) |node| { - switch (field_state) { - .none => field_state = .seen, - .err, .seen => {}, - .end => |tok| { - try tree.errors.push(.{ - .DeclBetweenFields = .{ .token = tok }, - }); - // continue parsing, error will be reported later - field_state = .err; - }, } - const field = node.cast(Node.ContainerField).?; - field.doc_comments = doc_comments; - try list.push(node); - const comma = eatToken(it, .Comma) orelse { - // try to continue parsing - const index = it.index; - findNextContainerMember(it); - const next = it.peek().?.id; - switch (next) { - .Eof => break, - else => { - if (next == .RBrace) { - if (!top_level) break; - _ = nextToken(it); - } + if (visib_token != null) { + try p.errors.append(p.gpa, .{ + .ExpectedPubItem = .{ .token = p.tok_i }, + }); + // ignore this pub + continue; + } - // add error and continue - try tree.errors.push(.{ - .ExpectedToken = .{ .token = index, .expected_id = .Comma }, + if (p.parseContainerField() catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.ParseError => { + // attempt to recover + p.findNextContainerMember(); + continue; + }, + }) |node| { + switch (field_state) { + .none => field_state = .seen, + .err, .seen => {}, + .end => |tok| { + try p.errors.append(p.gpa, .{ + .DeclBetweenFields = .{ .token = tok }, }); - continue; + // continue parsing, error will be reported later + field_state = .err; }, } - }; - if (try parseAppendedDocComment(arena, it, tree, comma)) |appended_comment| - field.doc_comments = appended_comment; - continue; - } - // Dangling doc comment - if (doc_comments != null) { - try tree.errors.push(.{ - .UnattachedDocComment = .{ .token = doc_comments.?.firstToken() }, - }); - } + const field = node.cast(Node.ContainerField).?; + field.doc_comments = doc_comments; + list_it = try p.llpush(*Node, list_it, node); + const comma = p.eatToken(.Comma) orelse { + // try to continue parsing + const index = p.tok_i; + p.findNextContainerMember(); + const next = p.tokens[p.tok_i].id; + switch (next) { + .Eof => break, + else => { + if (next == .RBrace) { + if (!top_level) break; + _ = p.nextToken(); + } - const next = it.peek().?.id; - switch (next) { - .Eof => break, - .Keyword_comptime => { - _ = nextToken(it); - try tree.errors.push(.{ - .ExpectedBlockOrField = .{ .token = it.index }, + // add error and continue + try p.errors.append(p.gpa, .{ + .ExpectedToken = .{ .token = index, .expected_id = .Comma }, + }); + continue; + }, + } + }; + if (try p.parseAppendedDocComment(comma)) |appended_comment| + field.doc_comments = appended_comment; + continue; + } + + // Dangling doc comment + if (doc_comments != null) { + try p.errors.append(p.gpa, .{ + .UnattachedDocComment = .{ .token = doc_comments.?.firstToken() }, }); - }, - else => { - const index = it.index; - if (next == .RBrace) { - if (!top_level) break; - _ = nextToken(it); - } + } - // this was likely not supposed to end yet, - // try to find the next declaration - findNextContainerMember(it); - try tree.errors.push(.{ - .ExpectedContainerMembers = .{ .token = index }, + const next = p.tokens[p.tok_i].id; + switch (next) { + .Eof => break, + .Keyword_comptime => { + _ = p.nextToken(); + try p.errors.append(p.gpa, .{ + .ExpectedBlockOrField = .{ .token = p.tok_i }, + }); + }, + else => { + const index = p.tok_i; + if (next == .RBrace) { + if (!top_level) break; + _ = p.nextToken(); + } + + // this was likely not supposed to end yet, + // try to find the next declaration + p.findNextContainerMember(); + try p.errors.append(p.gpa, .{ + .ExpectedContainerMembers = .{ .token = index }, + }); + }, + } + } + + return list; + } + + /// Attempts to find next container member by searching for certain tokens + fn findNextContainerMember(p: *Parser) void { + var level: u32 = 0; + while (true) { + const tok = p.nextToken(); + switch (tok.ptr.id) { + // any of these can start a new top level declaration + .Keyword_test, + .Keyword_comptime, + .Keyword_pub, + .Keyword_export, + .Keyword_extern, + .Keyword_inline, + .Keyword_noinline, + .Keyword_usingnamespace, + .Keyword_threadlocal, + .Keyword_const, + .Keyword_var, + .Keyword_fn, + .Identifier, + => { + if (level == 0) { + p.putBackToken(tok.index); + return; + } + }, + .Comma, .Semicolon => { + // this decl was likely meant to end here + if (level == 0) { + return; + } + }, + .LParen, .LBracket, .LBrace => level += 1, + .RParen, .RBracket => { + if (level != 0) level -= 1; + }, + .RBrace => { + if (level == 0) { + // end of container, exit + p.putBackToken(tok.index); + return; + } + level -= 1; + }, + .Eof => { + p.putBackToken(tok.index); + return; + }, + else => {}, + } + } + } + + /// Attempts to find the next statement by searching for a semicolon + fn findNextStmt(p: *Parser) void { + var level: u32 = 0; + while (true) { + const tok = p.nextToken(); + switch (tok.ptr.id) { + .LBrace => level += 1, + .RBrace => { + if (level == 0) { + p.putBackToken(tok.index); + return; + } + level -= 1; + }, + .Semicolon => { + if (level == 0) { + return; + } + }, + .Eof => { + p.putBackToken(tok.index); + return; + }, + else => {}, + } + } + } + + /// Eat a multiline container doc comment + fn parseContainerDocComments(p: *Parser) !?*Node { + var lines = Node.DocComment.LineList{}; + var lines_it: *?*Node.DocComment.LineList.Node = &lines.first; + + while (p.eatToken(.ContainerDocComment)) |line| { + lines_it = try p.llpush(TokenIndex, lines_it, line); + } + + if (lines.first == null) return null; + + const node = try p.arena.allocator.create(Node.DocComment); + node.* = .{ + .lines = lines, + }; + return &node.base; + } + + /// TestDecl <- KEYWORD_test STRINGLITERALSINGLE Block + fn parseTestDecl(p: *Parser) !?*Node { + const test_token = p.eatToken(.Keyword_test) orelse return null; + const name_node = try p.expectNode(parseStringLiteralSingle, .{ + .ExpectedStringLiteral = .{ .token = p.tok_i }, + }); + const block_node = try p.expectNode(parseBlock, .{ + .ExpectedLBrace = .{ .token = p.tok_i }, + }); + + const test_node = try p.arena.allocator.create(Node.TestDecl); + test_node.* = .{ + .doc_comments = null, + .test_token = test_token, + .name = name_node, + .body_node = block_node, + }; + return &test_node.base; + } + + /// TopLevelComptime <- KEYWORD_comptime BlockExpr + fn parseTopLevelComptime(p: *Parser) !?*Node { + const tok = p.eatToken(.Keyword_comptime) orelse return null; + const lbrace = p.eatToken(.LBrace) orelse { + p.putBackToken(tok); + return null; + }; + p.putBackToken(lbrace); + const block_node = try p.expectNode(parseBlockExpr, .{ + .ExpectedLabelOrLBrace = .{ .token = p.tok_i }, + }); + + const comptime_node = try p.arena.allocator.create(Node.Comptime); + comptime_node.* = .{ + .doc_comments = null, + .comptime_token = tok, + .expr = block_node, + }; + return &comptime_node.base; + } + + /// TopLevelDecl + /// <- (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE? / (KEYWORD_inline / KEYWORD_noinline))? FnProto (SEMICOLON / Block) + /// / (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE?)? KEYWORD_threadlocal? VarDecl + /// / KEYWORD_usingnamespace Expr SEMICOLON + fn parseTopLevelDecl(p: *Parser) !?*Node { + var lib_name: ?*Node = null; + const extern_export_inline_token = blk: { + if (p.eatToken(.Keyword_export)) |token| break :blk token; + if (p.eatToken(.Keyword_extern)) |token| { + lib_name = try p.parseStringLiteralSingle(); + break :blk token; + } + if (p.eatToken(.Keyword_inline)) |token| break :blk token; + if (p.eatToken(.Keyword_noinline)) |token| break :blk token; + break :blk null; + }; + + if (try p.parseFnProto()) |node| { + const fn_node = node.cast(Node.FnProto).?; + fn_node.*.extern_export_inline_token = extern_export_inline_token; + fn_node.*.lib_name = lib_name; + if (p.eatToken(.Semicolon)) |_| return node; + + if (try p.expectNodeRecoverable(parseBlock, .{ + // since parseBlock only return error.ParseError on + // a missing '}' we can assume this function was + // supposed to end here. + .ExpectedSemiOrLBrace = .{ .token = p.tok_i }, + })) |body_node| { + fn_node.body_node = body_node; + } + return node; + } + + if (extern_export_inline_token) |token| { + if (p.tokens[token].id == .Keyword_inline or + p.tokens[token].id == .Keyword_noinline) + { + try p.errors.append(p.gpa, .{ + .ExpectedFn = .{ .token = p.tok_i }, }); - }, + return error.ParseError; + } } - } - return list; -} + const thread_local_token = p.eatToken(.Keyword_threadlocal); -/// Attempts to find next container member by searching for certain tokens -fn findNextContainerMember(it: *TokenIterator) void { - var level: u32 = 0; - while (true) { - const tok = nextToken(it); - switch (tok.ptr.id) { - // any of these can start a new top level declaration - .Keyword_test, - .Keyword_comptime, - .Keyword_pub, - .Keyword_export, - .Keyword_extern, - .Keyword_inline, - .Keyword_noinline, - .Keyword_usingnamespace, - .Keyword_threadlocal, - .Keyword_const, - .Keyword_var, - .Keyword_fn, - .Identifier, - => { - if (level == 0) { - putBackToken(it, tok.index); - return; - } - }, - .Comma, .Semicolon => { - // this decl was likely meant to end here - if (level == 0) { - return; - } - }, - .LParen, .LBracket, .LBrace => level += 1, - .RParen, .RBracket => { - if (level != 0) level -= 1; - }, - .RBrace => { - if (level == 0) { - // end of container, exit - putBackToken(it, tok.index); - return; - } - level -= 1; - }, - .Eof => { - putBackToken(it, tok.index); - return; - }, - else => {}, + if (try p.parseVarDecl()) |node| { + var var_decl = node.cast(Node.VarDecl).?; + var_decl.*.thread_local_token = thread_local_token; + var_decl.*.comptime_token = null; + var_decl.*.extern_export_token = extern_export_inline_token; + var_decl.*.lib_name = lib_name; + return node; } - } -} -/// Attempts to find the next statement by searching for a semicolon -fn findNextStmt(it: *TokenIterator) void { - var level: u32 = 0; - while (true) { - const tok = nextToken(it); - switch (tok.ptr.id) { - .LBrace => level += 1, - .RBrace => { - if (level == 0) { - putBackToken(it, tok.index); - return; - } - level -= 1; - }, - .Semicolon => { - if (level == 0) { - return; - } - }, - .Eof => { - putBackToken(it, tok.index); - return; - }, - else => {}, - } - } -} - -/// Eat a multiline container doc comment -fn parseContainerDocComments(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - var lines = Node.DocComment.LineList.init(arena); - while (eatToken(it, .ContainerDocComment)) |line| { - try lines.push(line); - } - - if (lines.len == 0) return null; - - const node = try arena.create(Node.DocComment); - node.* = .{ - .lines = lines, - }; - return &node.base; -} - -/// TestDecl <- KEYWORD_test STRINGLITERALSINGLE Block -fn parseTestDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const test_token = eatToken(it, .Keyword_test) orelse return null; - const name_node = try expectNode(arena, it, tree, parseStringLiteralSingle, .{ - .ExpectedStringLiteral = .{ .token = it.index }, - }); - const block_node = try expectNode(arena, it, tree, parseBlock, .{ - .ExpectedLBrace = .{ .token = it.index }, - }); - - const test_node = try arena.create(Node.TestDecl); - test_node.* = .{ - .doc_comments = null, - .test_token = test_token, - .name = name_node, - .body_node = block_node, - }; - return &test_node.base; -} - -/// TopLevelComptime <- KEYWORD_comptime BlockExpr -fn parseTopLevelComptime(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const tok = eatToken(it, .Keyword_comptime) orelse return null; - const lbrace = eatToken(it, .LBrace) orelse { - putBackToken(it, tok); - return null; - }; - putBackToken(it, lbrace); - const block_node = try expectNode(arena, it, tree, parseBlockExpr, .{ - .ExpectedLabelOrLBrace = .{ .token = it.index }, - }); - - const comptime_node = try arena.create(Node.Comptime); - comptime_node.* = .{ - .doc_comments = null, - .comptime_token = tok, - .expr = block_node, - }; - return &comptime_node.base; -} - -/// TopLevelDecl -/// <- (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE? / (KEYWORD_inline / KEYWORD_noinline))? FnProto (SEMICOLON / Block) -/// / (KEYWORD_export / KEYWORD_extern STRINGLITERALSINGLE?)? KEYWORD_threadlocal? VarDecl -/// / KEYWORD_usingnamespace Expr SEMICOLON -fn parseTopLevelDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - var lib_name: ?*Node = null; - const extern_export_inline_token = blk: { - if (eatToken(it, .Keyword_export)) |token| break :blk token; - if (eatToken(it, .Keyword_extern)) |token| { - lib_name = try parseStringLiteralSingle(arena, it, tree); - break :blk token; - } - if (eatToken(it, .Keyword_inline)) |token| break :blk token; - if (eatToken(it, .Keyword_noinline)) |token| break :blk token; - break :blk null; - }; - - if (try parseFnProto(arena, it, tree)) |node| { - const fn_node = node.cast(Node.FnProto).?; - fn_node.*.extern_export_inline_token = extern_export_inline_token; - fn_node.*.lib_name = lib_name; - if (eatToken(it, .Semicolon)) |_| return node; - - if (try expectNodeRecoverable(arena, it, tree, parseBlock, .{ - // since parseBlock only return error.ParseError on - // a missing '}' we can assume this function was - // supposed to end here. - .ExpectedSemiOrLBrace = .{ .token = it.index }, - })) |body_node| { - fn_node.body_node = body_node; - } - return node; - } - - if (extern_export_inline_token) |token| { - if (tree.tokens.at(token).id == .Keyword_inline or - tree.tokens.at(token).id == .Keyword_noinline) - { - try tree.errors.push(.{ - .ExpectedFn = .{ .token = it.index }, + if (thread_local_token != null) { + try p.errors.append(p.gpa, .{ + .ExpectedVarDecl = .{ .token = p.tok_i }, }); + // ignore this and try again; return error.ParseError; } - } - const thread_local_token = eatToken(it, .Keyword_threadlocal); - - if (try parseVarDecl(arena, it, tree)) |node| { - var var_decl = node.cast(Node.VarDecl).?; - var_decl.*.thread_local_token = thread_local_token; - var_decl.*.comptime_token = null; - var_decl.*.extern_export_token = extern_export_inline_token; - var_decl.*.lib_name = lib_name; - return node; - } - - if (thread_local_token != null) { - try tree.errors.push(.{ - .ExpectedVarDecl = .{ .token = it.index }, - }); - // ignore this and try again; - return error.ParseError; - } - - if (extern_export_inline_token) |token| { - try tree.errors.push(.{ - .ExpectedVarDeclOrFn = .{ .token = it.index }, - }); - // ignore this and try again; - return error.ParseError; - } - - return try parseUse(arena, it, tree); -} - -/// FnProto <- KEYWORD_fn IDENTIFIER? LPAREN ParamDeclList RPAREN ByteAlign? LinkSection? EXCLAMATIONMARK? (KEYWORD_var / TypeExpr) -fn parseFnProto(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - // TODO: Remove once extern/async fn rewriting is - var is_async = false; - var is_extern = false; - const cc_token: ?usize = blk: { - if (eatToken(it, .Keyword_extern)) |token| { - is_extern = true; - break :blk token; - } - if (eatToken(it, .Keyword_async)) |token| { - is_async = true; - break :blk token; - } - break :blk null; - }; - const fn_token = eatToken(it, .Keyword_fn) orelse { - if (cc_token) |token| - putBackToken(it, token); - return null; - }; - const name_token = eatToken(it, .Identifier); - const lparen = try expectToken(it, tree, .LParen); - const params = try parseParamDeclList(arena, it, tree); - const rparen = try expectToken(it, tree, .RParen); - const align_expr = try parseByteAlign(arena, it, tree); - const section_expr = try parseLinkSection(arena, it, tree); - const callconv_expr = try parseCallconv(arena, it, tree); - const exclamation_token = eatToken(it, .Bang); - - const return_type_expr = (try parseVarType(arena, it, tree)) orelse - try expectNodeRecoverable(arena, it, tree, parseTypeExpr, .{ - // most likely the user forgot to specify the return type. - // Mark return type as invalid and try to continue. - .ExpectedReturnType = .{ .token = it.index }, - }); - - // TODO https://github.com/ziglang/zig/issues/3750 - const R = Node.FnProto.ReturnType; - const return_type = if (return_type_expr == null) - R{ .Invalid = rparen } - else if (exclamation_token != null) - R{ .InferErrorSet = return_type_expr.? } - else - R{ .Explicit = return_type_expr.? }; - - const var_args_token = if (params.len > 0) blk: { - const param_type = params.at(params.len - 1).*.cast(Node.ParamDecl).?.param_type; - break :blk if (param_type == .var_args) param_type.var_args else null; - } else - null; - - const fn_proto_node = try arena.create(Node.FnProto); - fn_proto_node.* = .{ - .doc_comments = null, - .visib_token = null, - .fn_token = fn_token, - .name_token = name_token, - .params = params, - .return_type = return_type, - .var_args_token = var_args_token, - .extern_export_inline_token = null, - .body_node = null, - .lib_name = null, - .align_expr = align_expr, - .section_expr = section_expr, - .callconv_expr = callconv_expr, - .is_extern_prototype = is_extern, - .is_async = is_async, - }; - - return &fn_proto_node.base; -} - -/// VarDecl <- (KEYWORD_const / KEYWORD_var) IDENTIFIER (COLON TypeExpr)? ByteAlign? LinkSection? (EQUAL Expr)? SEMICOLON -fn parseVarDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const mut_token = eatToken(it, .Keyword_const) orelse - eatToken(it, .Keyword_var) orelse - return null; - - const name_token = try expectToken(it, tree, .Identifier); - const type_node = if (eatToken(it, .Colon) != null) - try expectNode(arena, it, tree, parseTypeExpr, .{ - .ExpectedTypeExpr = .{ .token = it.index }, - }) - else - null; - const align_node = try parseByteAlign(arena, it, tree); - const section_node = try parseLinkSection(arena, it, tree); - const eq_token = eatToken(it, .Equal); - const init_node = if (eq_token != null) blk: { - break :blk try expectNode(arena, it, tree, parseExpr, .{ - .ExpectedExpr = .{ .token = it.index }, - }); - } else null; - const semicolon_token = try expectToken(it, tree, .Semicolon); - - const node = try arena.create(Node.VarDecl); - node.* = .{ - .doc_comments = null, - .visib_token = null, - .thread_local_token = null, - .name_token = name_token, - .eq_token = eq_token, - .mut_token = mut_token, - .comptime_token = null, - .extern_export_token = null, - .lib_name = null, - .type_node = type_node, - .align_node = align_node, - .section_node = section_node, - .init_node = init_node, - .semicolon_token = semicolon_token, - }; - return &node.base; -} - -/// ContainerField <- KEYWORD_comptime? IDENTIFIER (COLON TypeExpr ByteAlign?)? (EQUAL Expr)? -fn parseContainerField(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const comptime_token = eatToken(it, .Keyword_comptime); - const name_token = eatToken(it, .Identifier) orelse { - if (comptime_token) |t| putBackToken(it, t); - return null; - }; - - var align_expr: ?*Node = null; - var type_expr: ?*Node = null; - if (eatToken(it, .Colon)) |_| { - if (eatToken(it, .Keyword_var)) |var_tok| { - const node = try arena.create(ast.Node.VarType); - node.* = .{ .token = var_tok }; - type_expr = &node.base; - } else { - type_expr = try expectNode(arena, it, tree, parseTypeExpr, .{ - .ExpectedTypeExpr = .{ .token = it.index }, + if (extern_export_inline_token) |token| { + try p.errors.append(p.gpa, .{ + .ExpectedVarDeclOrFn = .{ .token = p.tok_i }, }); - align_expr = try parseByteAlign(arena, it, tree); + // ignore this and try again; + return error.ParseError; } + + return p.parseUse(); } - const value_expr = if (eatToken(it, .Equal)) |_| - try expectNode(arena, it, tree, parseExpr, .{ - .ExpectedExpr = .{ .token = it.index }, - }) - else - null; + /// FnProto <- KEYWORD_fn IDENTIFIER? LPAREN ParamDeclList RPAREN ByteAlign? LinkSection? EXCLAMATIONMARK? (KEYWORD_var / TypeExpr) + fn parseFnProto(p: *Parser) !?*Node { + // TODO: Remove once extern/async fn rewriting is + var is_async = false; + var is_extern = false; + const cc_token: ?TokenIndex = blk: { + if (p.eatToken(.Keyword_extern)) |token| { + is_extern = true; + break :blk token; + } + if (p.eatToken(.Keyword_async)) |token| { + is_async = true; + break :blk token; + } + break :blk null; + }; + const fn_token = p.eatToken(.Keyword_fn) orelse { + if (cc_token) |token| + p.putBackToken(token); + return null; + }; + var var_args_token: ?TokenIndex = null; + const name_token = p.eatToken(.Identifier); + const lparen = try p.expectToken(.LParen); + const params = try p.parseParamDeclList(&var_args_token); + const rparen = try p.expectToken(.RParen); + const align_expr = try p.parseByteAlign(); + const section_expr = try p.parseLinkSection(); + const callconv_expr = try p.parseCallconv(); + const exclamation_token = p.eatToken(.Bang); - const node = try arena.create(Node.ContainerField); - node.* = .{ - .doc_comments = null, - .comptime_token = comptime_token, - .name_token = name_token, - .type_expr = type_expr, - .value_expr = value_expr, - .align_expr = align_expr, - }; - return &node.base; -} - -/// Statement -/// <- KEYWORD_comptime? VarDecl -/// / KEYWORD_comptime BlockExprStatement -/// / KEYWORD_nosuspend BlockExprStatement -/// / KEYWORD_suspend (SEMICOLON / BlockExprStatement) -/// / KEYWORD_defer BlockExprStatement -/// / KEYWORD_errdefer Payload? BlockExprStatement -/// / IfStatement -/// / LabeledStatement -/// / SwitchExpr -/// / AssignExpr SEMICOLON -fn parseStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?*Node { - const comptime_token = eatToken(it, .Keyword_comptime); - - const var_decl_node = try parseVarDecl(arena, it, tree); - if (var_decl_node) |node| { - const var_decl = node.cast(Node.VarDecl).?; - var_decl.comptime_token = comptime_token; - return node; - } - - if (comptime_token) |token| { - const block_expr = try expectNode(arena, it, tree, parseBlockExprStatement, .{ - .ExpectedBlockOrAssignment = .{ .token = it.index }, + const return_type_expr = (try p.parseVarType()) orelse + try p.expectNodeRecoverable(parseTypeExpr, .{ + // most likely the user forgot to specify the return type. + // Mark return type as invalid and try to continue. + .ExpectedReturnType = .{ .token = p.tok_i }, }); - const node = try arena.create(Node.Comptime); - node.* = .{ + // TODO https://github.com/ziglang/zig/issues/3750 + const R = Node.FnProto.ReturnType; + const return_type = if (return_type_expr == null) + R{ .Invalid = rparen } + else if (exclamation_token != null) + R{ .InferErrorSet = return_type_expr.? } + else + R{ .Explicit = return_type_expr.? }; + + const fn_proto_node = try p.arena.allocator.create(Node.FnProto); + fn_proto_node.* = .{ .doc_comments = null, - .comptime_token = token, - .expr = block_expr, + .visib_token = null, + .fn_token = fn_token, + .name_token = name_token, + .params = params, + .return_type = return_type, + .var_args_token = var_args_token, + .extern_export_inline_token = null, + .body_node = null, + .lib_name = null, + .align_expr = align_expr, + .section_expr = section_expr, + .callconv_expr = callconv_expr, + .is_extern_prototype = is_extern, + .is_async = is_async, }; - return &node.base; + + return &fn_proto_node.base; } - if (eatToken(it, .Keyword_nosuspend)) |nosuspend_token| { - const block_expr = try expectNode(arena, it, tree, parseBlockExprStatement, .{ - .ExpectedBlockOrAssignment = .{ .token = it.index }, - }); + /// VarDecl <- (KEYWORD_const / KEYWORD_var) IDENTIFIER (COLON TypeExpr)? ByteAlign? LinkSection? (EQUAL Expr)? SEMICOLON + fn parseVarDecl(p: *Parser) !?*Node { + const mut_token = p.eatToken(.Keyword_const) orelse + p.eatToken(.Keyword_var) orelse + return null; - const node = try arena.create(Node.Nosuspend); - node.* = .{ - .nosuspend_token = nosuspend_token, - .expr = block_expr, - }; - return &node.base; - } - - if (eatToken(it, .Keyword_suspend)) |suspend_token| { - const semicolon = eatToken(it, .Semicolon); - - const body_node = if (semicolon == null) blk: { - break :blk try expectNode(arena, it, tree, parseBlockExprStatement, .{ - .ExpectedBlockOrExpression = .{ .token = it.index }, - }); - } else null; - - const node = try arena.create(Node.Suspend); - node.* = .{ - .suspend_token = suspend_token, - .body = body_node, - }; - return &node.base; - } - - const defer_token = eatToken(it, .Keyword_defer) orelse eatToken(it, .Keyword_errdefer); - if (defer_token) |token| { - const payload = if (tree.tokens.at(token).id == .Keyword_errdefer) - try parsePayload(arena, it, tree) + const name_token = try p.expectToken(.Identifier); + const type_node = if (p.eatToken(.Colon) != null) + try p.expectNode(parseTypeExpr, .{ + .ExpectedTypeExpr = .{ .token = p.tok_i }, + }) else null; - const expr_node = try expectNode(arena, it, tree, parseBlockExprStatement, .{ - .ExpectedBlockOrExpression = .{ .token = it.index }, - }); - const node = try arena.create(Node.Defer); + const align_node = try p.parseByteAlign(); + const section_node = try p.parseLinkSection(); + const eq_token = p.eatToken(.Equal); + const init_node = if (eq_token != null) blk: { + break :blk try p.expectNode(parseExpr, .{ + .ExpectedExpr = .{ .token = p.tok_i }, + }); + } else null; + const semicolon_token = try p.expectToken(.Semicolon); + + const node = try p.arena.allocator.create(Node.VarDecl); node.* = .{ - .defer_token = token, - .expr = expr_node, - .payload = payload, + .doc_comments = null, + .visib_token = null, + .thread_local_token = null, + .name_token = name_token, + .eq_token = eq_token, + .mut_token = mut_token, + .comptime_token = null, + .extern_export_token = null, + .lib_name = null, + .type_node = type_node, + .align_node = align_node, + .section_node = section_node, + .init_node = init_node, + .semicolon_token = semicolon_token, }; return &node.base; } - if (try parseIfStatement(arena, it, tree)) |node| return node; - if (try parseLabeledStatement(arena, it, tree)) |node| return node; - if (try parseSwitchExpr(arena, it, tree)) |node| return node; - if (try parseAssignExpr(arena, it, tree)) |node| { - _ = try expectTokenRecoverable(it, tree, .Semicolon); - return node; - } - - return null; -} - -/// IfStatement -/// <- IfPrefix BlockExpr ( KEYWORD_else Payload? Statement )? -/// / IfPrefix AssignExpr ( SEMICOLON / KEYWORD_else Payload? Statement ) -fn parseIfStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const if_node = (try parseIfPrefix(arena, it, tree)) orelse return null; - const if_prefix = if_node.cast(Node.If).?; - - const block_expr = (try parseBlockExpr(arena, it, tree)); - const assign_expr = if (block_expr == null) - try expectNode(arena, it, tree, parseAssignExpr, .{ - .ExpectedBlockOrAssignment = .{ .token = it.index }, - }) - else - null; - - const semicolon = if (assign_expr != null) eatToken(it, .Semicolon) else null; - - const else_node = if (semicolon == null) blk: { - const else_token = eatToken(it, .Keyword_else) orelse break :blk null; - const payload = try parsePayload(arena, it, tree); - const else_body = try expectNode(arena, it, tree, parseStatement, .{ - .InvalidToken = .{ .token = it.index }, - }); - - const node = try arena.create(Node.Else); - node.* = .{ - .else_token = else_token, - .payload = payload, - .body = else_body, + /// ContainerField <- KEYWORD_comptime? IDENTIFIER (COLON TypeExpr ByteAlign?)? (EQUAL Expr)? + fn parseContainerField(p: *Parser) !?*Node { + const comptime_token = p.eatToken(.Keyword_comptime); + const name_token = p.eatToken(.Identifier) orelse { + if (comptime_token) |t| p.putBackToken(t); + return null; }; - break :blk node; - } else null; + var align_expr: ?*Node = null; + var type_expr: ?*Node = null; + if (p.eatToken(.Colon)) |_| { + if (p.eatToken(.Keyword_var)) |var_tok| { + const node = try p.arena.allocator.create(ast.Node.VarType); + node.* = .{ .token = var_tok }; + type_expr = &node.base; + } else { + type_expr = try p.expectNode(parseTypeExpr, .{ + .ExpectedTypeExpr = .{ .token = p.tok_i }, + }); + align_expr = try p.parseByteAlign(); + } + } - if (block_expr) |body| { - if_prefix.body = body; - if_prefix.@"else" = else_node; - return if_node; + const value_expr = if (p.eatToken(.Equal)) |_| + try p.expectNode(parseExpr, .{ + .ExpectedExpr = .{ .token = p.tok_i }, + }) + else + null; + + const node = try p.arena.allocator.create(Node.ContainerField); + node.* = .{ + .doc_comments = null, + .comptime_token = comptime_token, + .name_token = name_token, + .type_expr = type_expr, + .value_expr = value_expr, + .align_expr = align_expr, + }; + return &node.base; } - if (assign_expr) |body| { - if_prefix.body = body; - if (semicolon != null) return if_node; - if (else_node != null) { + /// Statement + /// <- KEYWORD_comptime? VarDecl + /// / KEYWORD_comptime BlockExprStatement + /// / KEYWORD_nosuspend BlockExprStatement + /// / KEYWORD_suspend (SEMICOLON / BlockExprStatement) + /// / KEYWORD_defer BlockExprStatement + /// / KEYWORD_errdefer Payload? BlockExprStatement + /// / IfStatement + /// / LabeledStatement + /// / SwitchExpr + /// / AssignExpr SEMICOLON + fn parseStatement(p: *Parser) Error!?*Node { + const comptime_token = p.eatToken(.Keyword_comptime); + + const var_decl_node = try p.parseVarDecl(); + if (var_decl_node) |node| { + const var_decl = node.cast(Node.VarDecl).?; + var_decl.comptime_token = comptime_token; + return node; + } + + if (comptime_token) |token| { + const block_expr = try p.expectNode(parseBlockExprStatement, .{ + .ExpectedBlockOrAssignment = .{ .token = p.tok_i }, + }); + + const node = try p.arena.allocator.create(Node.Comptime); + node.* = .{ + .doc_comments = null, + .comptime_token = token, + .expr = block_expr, + }; + return &node.base; + } + + if (p.eatToken(.Keyword_nosuspend)) |nosuspend_token| { + const block_expr = try p.expectNode(parseBlockExprStatement, .{ + .ExpectedBlockOrAssignment = .{ .token = p.tok_i }, + }); + + const node = try p.arena.allocator.create(Node.Nosuspend); + node.* = .{ + .nosuspend_token = nosuspend_token, + .expr = block_expr, + }; + return &node.base; + } + + if (p.eatToken(.Keyword_suspend)) |suspend_token| { + const semicolon = p.eatToken(.Semicolon); + + const body_node = if (semicolon == null) blk: { + break :blk try p.expectNode(parseBlockExprStatement, .{ + .ExpectedBlockOrExpression = .{ .token = p.tok_i }, + }); + } else null; + + const node = try p.arena.allocator.create(Node.Suspend); + node.* = .{ + .suspend_token = suspend_token, + .body = body_node, + }; + return &node.base; + } + + const defer_token = p.eatToken(.Keyword_defer) orelse p.eatToken(.Keyword_errdefer); + if (defer_token) |token| { + const payload = if (p.tokens[token].id == .Keyword_errdefer) + try p.parsePayload() + else + null; + const expr_node = try p.expectNode(parseBlockExprStatement, .{ + .ExpectedBlockOrExpression = .{ .token = p.tok_i }, + }); + const node = try p.arena.allocator.create(Node.Defer); + node.* = .{ + .defer_token = token, + .expr = expr_node, + .payload = payload, + }; + return &node.base; + } + + if (try p.parseIfStatement()) |node| return node; + if (try p.parseLabeledStatement()) |node| return node; + if (try p.parseSwitchExpr()) |node| return node; + if (try p.parseAssignExpr()) |node| { + _ = try p.expectTokenRecoverable(.Semicolon); + return node; + } + + return null; + } + + /// IfStatement + /// <- IfPrefix BlockExpr ( KEYWORD_else Payload? Statement )? + /// / IfPrefix AssignExpr ( SEMICOLON / KEYWORD_else Payload? Statement ) + fn parseIfStatement(p: *Parser) !?*Node { + const if_node = (try p.parseIfPrefix()) orelse return null; + const if_prefix = if_node.cast(Node.If).?; + + const block_expr = (try p.parseBlockExpr()); + const assign_expr = if (block_expr == null) + try p.expectNode(parseAssignExpr, .{ + .ExpectedBlockOrAssignment = .{ .token = p.tok_i }, + }) + else + null; + + const semicolon = if (assign_expr != null) p.eatToken(.Semicolon) else null; + + const else_node = if (semicolon == null) blk: { + const else_token = p.eatToken(.Keyword_else) orelse break :blk null; + const payload = try p.parsePayload(); + const else_body = try p.expectNode(parseStatement, .{ + .InvalidToken = .{ .token = p.tok_i }, + }); + + const node = try p.arena.allocator.create(Node.Else); + node.* = .{ + .else_token = else_token, + .payload = payload, + .body = else_body, + }; + + break :blk node; + } else null; + + if (block_expr) |body| { + if_prefix.body = body; if_prefix.@"else" = else_node; return if_node; } - try tree.errors.push(.{ - .ExpectedSemiOrElse = .{ .token = it.index }, - }); + + if (assign_expr) |body| { + if_prefix.body = body; + if (semicolon != null) return if_node; + if (else_node != null) { + if_prefix.@"else" = else_node; + return if_node; + } + try p.errors.append(p.gpa, .{ + .ExpectedSemiOrElse = .{ .token = p.tok_i }, + }); + } + + return if_node; } - return if_node; -} + /// LabeledStatement <- BlockLabel? (Block / LoopStatement) + fn parseLabeledStatement(p: *Parser) !?*Node { + var colon: TokenIndex = undefined; + const label_token = p.parseBlockLabel(&colon); -/// LabeledStatement <- BlockLabel? (Block / LoopStatement) -fn parseLabeledStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - var colon: TokenIndex = undefined; - const label_token = parseBlockLabel(arena, it, tree, &colon); + if (try p.parseBlock()) |node| { + node.cast(Node.Block).?.label = label_token; + return node; + } - if (try parseBlock(arena, it, tree)) |node| { - node.cast(Node.Block).?.label = label_token; - return node; + if (try p.parseLoopStatement()) |node| { + if (node.cast(Node.For)) |for_node| { + for_node.label = label_token; + } else if (node.cast(Node.While)) |while_node| { + while_node.label = label_token; + } else unreachable; + return node; + } + + if (label_token != null) { + try p.errors.append(p.gpa, .{ + .ExpectedLabelable = .{ .token = p.tok_i }, + }); + return error.ParseError; + } + + return null; } - if (try parseLoopStatement(arena, it, tree)) |node| { - if (node.cast(Node.For)) |for_node| { - for_node.label = label_token; - } else if (node.cast(Node.While)) |while_node| { - while_node.label = label_token; - } else unreachable; - return node; - } + /// LoopStatement <- KEYWORD_inline? (ForStatement / WhileStatement) + fn parseLoopStatement(p: *Parser) !?*Node { + const inline_token = p.eatToken(.Keyword_inline); - if (label_token != null) { - try tree.errors.push(.{ - .ExpectedLabelable = .{ .token = it.index }, + if (try p.parseForStatement()) |node| { + node.cast(Node.For).?.inline_token = inline_token; + return node; + } + + if (try p.parseWhileStatement()) |node| { + node.cast(Node.While).?.inline_token = inline_token; + return node; + } + if (inline_token == null) return null; + + // If we've seen "inline", there should have been a "for" or "while" + try p.errors.append(p.gpa, .{ + .ExpectedInlinable = .{ .token = p.tok_i }, }); return error.ParseError; } - return null; -} + /// ForStatement + /// <- ForPrefix BlockExpr ( KEYWORD_else Statement )? + /// / ForPrefix AssignExpr ( SEMICOLON / KEYWORD_else Statement ) + fn parseForStatement(p: *Parser) !?*Node { + const node = (try p.parseForPrefix()) orelse return null; + const for_prefix = node.cast(Node.For).?; -/// LoopStatement <- KEYWORD_inline? (ForStatement / WhileStatement) -fn parseLoopStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const inline_token = eatToken(it, .Keyword_inline); + if (try p.parseBlockExpr()) |block_expr_node| { + for_prefix.body = block_expr_node; - if (try parseForStatement(arena, it, tree)) |node| { - node.cast(Node.For).?.inline_token = inline_token; - return node; - } + if (p.eatToken(.Keyword_else)) |else_token| { + const statement_node = try p.expectNode(parseStatement, .{ + .InvalidToken = .{ .token = p.tok_i }, + }); - if (try parseWhileStatement(arena, it, tree)) |node| { - node.cast(Node.While).?.inline_token = inline_token; - return node; - } - if (inline_token == null) return null; + const else_node = try p.arena.allocator.create(Node.Else); + else_node.* = .{ + .else_token = else_token, + .payload = null, + .body = statement_node, + }; + for_prefix.@"else" = else_node; - // If we've seen "inline", there should have been a "for" or "while" - try tree.errors.push(.{ - .ExpectedInlinable = .{ .token = it.index }, - }); - return error.ParseError; -} - -/// ForStatement -/// <- ForPrefix BlockExpr ( KEYWORD_else Statement )? -/// / ForPrefix AssignExpr ( SEMICOLON / KEYWORD_else Statement ) -fn parseForStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const node = (try parseForPrefix(arena, it, tree)) orelse return null; - const for_prefix = node.cast(Node.For).?; - - if (try parseBlockExpr(arena, it, tree)) |block_expr_node| { - for_prefix.body = block_expr_node; - - if (eatToken(it, .Keyword_else)) |else_token| { - const statement_node = try expectNode(arena, it, tree, parseStatement, .{ - .InvalidToken = .{ .token = it.index }, - }); - - const else_node = try arena.create(Node.Else); - else_node.* = .{ - .else_token = else_token, - .payload = null, - .body = statement_node, - }; - for_prefix.@"else" = else_node; + return node; + } return node; } - return node; - } + if (try p.parseAssignExpr()) |assign_expr| { + for_prefix.body = assign_expr; - if (try parseAssignExpr(arena, it, tree)) |assign_expr| { - for_prefix.body = assign_expr; + if (p.eatToken(.Semicolon) != null) return node; - if (eatToken(it, .Semicolon) != null) return node; + if (p.eatToken(.Keyword_else)) |else_token| { + const statement_node = try p.expectNode(parseStatement, .{ + .ExpectedStatement = .{ .token = p.tok_i }, + }); - if (eatToken(it, .Keyword_else)) |else_token| { - const statement_node = try expectNode(arena, it, tree, parseStatement, .{ - .ExpectedStatement = .{ .token = it.index }, + const else_node = try p.arena.allocator.create(Node.Else); + else_node.* = .{ + .else_token = else_token, + .payload = null, + .body = statement_node, + }; + for_prefix.@"else" = else_node; + return node; + } + + try p.errors.append(p.gpa, .{ + .ExpectedSemiOrElse = .{ .token = p.tok_i }, }); - const else_node = try arena.create(Node.Else); - else_node.* = .{ - .else_token = else_token, - .payload = null, - .body = statement_node, - }; - for_prefix.@"else" = else_node; - return node; - } - - try tree.errors.push(.{ - .ExpectedSemiOrElse = .{ .token = it.index }, - }); - - return node; - } - - return null; -} - -/// WhileStatement -/// <- WhilePrefix BlockExpr ( KEYWORD_else Payload? Statement )? -/// / WhilePrefix AssignExpr ( SEMICOLON / KEYWORD_else Payload? Statement ) -fn parseWhileStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const node = (try parseWhilePrefix(arena, it, tree)) orelse return null; - const while_prefix = node.cast(Node.While).?; - - if (try parseBlockExpr(arena, it, tree)) |block_expr_node| { - while_prefix.body = block_expr_node; - - if (eatToken(it, .Keyword_else)) |else_token| { - const payload = try parsePayload(arena, it, tree); - - const statement_node = try expectNode(arena, it, tree, parseStatement, .{ - .InvalidToken = .{ .token = it.index }, - }); - - const else_node = try arena.create(Node.Else); - else_node.* = .{ - .else_token = else_token, - .payload = payload, - .body = statement_node, - }; - while_prefix.@"else" = else_node; - return node; } - return node; + return null; } - if (try parseAssignExpr(arena, it, tree)) |assign_expr_node| { - while_prefix.body = assign_expr_node; + /// WhileStatement + /// <- WhilePrefix BlockExpr ( KEYWORD_else Payload? Statement )? + /// / WhilePrefix AssignExpr ( SEMICOLON / KEYWORD_else Payload? Statement ) + fn parseWhileStatement(p: *Parser) !?*Node { + const node = (try p.parseWhilePrefix()) orelse return null; + const while_prefix = node.cast(Node.While).?; - if (eatToken(it, .Semicolon) != null) return node; + if (try p.parseBlockExpr()) |block_expr_node| { + while_prefix.body = block_expr_node; - if (eatToken(it, .Keyword_else)) |else_token| { - const payload = try parsePayload(arena, it, tree); + if (p.eatToken(.Keyword_else)) |else_token| { + const payload = try p.parsePayload(); - const statement_node = try expectNode(arena, it, tree, parseStatement, .{ - .ExpectedStatement = .{ .token = it.index }, - }); + const statement_node = try p.expectNode(parseStatement, .{ + .InvalidToken = .{ .token = p.tok_i }, + }); + + const else_node = try p.arena.allocator.create(Node.Else); + else_node.* = .{ + .else_token = else_token, + .payload = payload, + .body = statement_node, + }; + while_prefix.@"else" = else_node; + + return node; + } - const else_node = try arena.create(Node.Else); - else_node.* = .{ - .else_token = else_token, - .payload = payload, - .body = statement_node, - }; - while_prefix.@"else" = else_node; return node; } - try tree.errors.push(.{ - .ExpectedSemiOrElse = .{ .token = it.index }, - }); + if (try p.parseAssignExpr()) |assign_expr_node| { + while_prefix.body = assign_expr_node; - return node; + if (p.eatToken(.Semicolon) != null) return node; + + if (p.eatToken(.Keyword_else)) |else_token| { + const payload = try p.parsePayload(); + + const statement_node = try p.expectNode(parseStatement, .{ + .ExpectedStatement = .{ .token = p.tok_i }, + }); + + const else_node = try p.arena.allocator.create(Node.Else); + else_node.* = .{ + .else_token = else_token, + .payload = payload, + .body = statement_node, + }; + while_prefix.@"else" = else_node; + return node; + } + + try p.errors.append(p.gpa, .{ + .ExpectedSemiOrElse = .{ .token = p.tok_i }, + }); + + return node; + } + + return null; } - return null; -} - -/// BlockExprStatement -/// <- BlockExpr -/// / AssignExpr SEMICOLON -fn parseBlockExprStatement(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - if (try parseBlockExpr(arena, it, tree)) |node| return node; - if (try parseAssignExpr(arena, it, tree)) |node| { - _ = try expectTokenRecoverable(it, tree, .Semicolon); - return node; - } - return null; -} - -/// BlockExpr <- BlockLabel? Block -fn parseBlockExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?*Node { - var colon: TokenIndex = undefined; - const label_token = parseBlockLabel(arena, it, tree, &colon); - const block_node = (try parseBlock(arena, it, tree)) orelse { - if (label_token) |label| { - putBackToken(it, label + 1); // ":" - putBackToken(it, label); // IDENTIFIER + /// BlockExprStatement + /// <- BlockExpr + /// / AssignExpr SEMICOLON + fn parseBlockExprStatement(p: *Parser) !?*Node { + if (try p.parseBlockExpr()) |node| return node; + if (try p.parseAssignExpr()) |node| { + _ = try p.expectTokenRecoverable(.Semicolon); + return node; } return null; - }; - block_node.cast(Node.Block).?.label = label_token; - return block_node; -} - -/// AssignExpr <- Expr (AssignOp Expr)? -fn parseAssignExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - return parseBinOpExpr(arena, it, tree, parseAssignOp, parseExpr, .Once); -} - -/// Expr <- KEYWORD_try* BoolOrExpr -fn parseExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?*Node { - return parsePrefixOpExpr(arena, it, tree, parseTry, parseBoolOrExpr); -} - -/// BoolOrExpr <- BoolAndExpr (KEYWORD_or BoolAndExpr)* -fn parseBoolOrExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - return parseBinOpExpr( - arena, - it, - tree, - SimpleBinOpParseFn(.Keyword_or, Node.InfixOp.Op.BoolOr), - parseBoolAndExpr, - .Infinitely, - ); -} - -/// BoolAndExpr <- CompareExpr (KEYWORD_and CompareExpr)* -fn parseBoolAndExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - return parseBinOpExpr( - arena, - it, - tree, - SimpleBinOpParseFn(.Keyword_and, .BoolAnd), - parseCompareExpr, - .Infinitely, - ); -} - -/// CompareExpr <- BitwiseExpr (CompareOp BitwiseExpr)? -fn parseCompareExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - return parseBinOpExpr(arena, it, tree, parseCompareOp, parseBitwiseExpr, .Once); -} - -/// BitwiseExpr <- BitShiftExpr (BitwiseOp BitShiftExpr)* -fn parseBitwiseExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - return parseBinOpExpr(arena, it, tree, parseBitwiseOp, parseBitShiftExpr, .Infinitely); -} - -/// BitShiftExpr <- AdditionExpr (BitShiftOp AdditionExpr)* -fn parseBitShiftExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - return parseBinOpExpr(arena, it, tree, parseBitShiftOp, parseAdditionExpr, .Infinitely); -} - -/// AdditionExpr <- MultiplyExpr (AdditionOp MultiplyExpr)* -fn parseAdditionExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - return parseBinOpExpr(arena, it, tree, parseAdditionOp, parseMultiplyExpr, .Infinitely); -} - -/// MultiplyExpr <- PrefixExpr (MultiplyOp PrefixExpr)* -fn parseMultiplyExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - return parseBinOpExpr(arena, it, tree, parseMultiplyOp, parsePrefixExpr, .Infinitely); -} - -/// PrefixExpr <- PrefixOp* PrimaryExpr -fn parsePrefixExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - return parsePrefixOpExpr(arena, it, tree, parsePrefixOp, parsePrimaryExpr); -} - -/// PrimaryExpr -/// <- AsmExpr -/// / IfExpr -/// / KEYWORD_break BreakLabel? Expr? -/// / KEYWORD_comptime Expr -/// / KEYWORD_nosuspend Expr -/// / KEYWORD_continue BreakLabel? -/// / KEYWORD_resume Expr -/// / KEYWORD_return Expr? -/// / BlockLabel? LoopExpr -/// / Block -/// / CurlySuffixExpr -fn parsePrimaryExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - if (try parseAsmExpr(arena, it, tree)) |node| return node; - if (try parseIfExpr(arena, it, tree)) |node| return node; - - if (eatToken(it, .Keyword_break)) |token| { - const label = try parseBreakLabel(arena, it, tree); - const expr_node = try parseExpr(arena, it, tree); - const node = try arena.create(Node.ControlFlowExpression); - node.* = .{ - .ltoken = token, - .kind = .{ .Break = label }, - .rhs = expr_node, - }; - return &node.base; } - if (eatToken(it, .Keyword_comptime)) |token| { - const expr_node = try expectNode(arena, it, tree, parseExpr, .{ - .ExpectedExpr = .{ .token = it.index }, - }); - const node = try arena.create(Node.Comptime); - node.* = .{ - .doc_comments = null, - .comptime_token = token, - .expr = expr_node, - }; - return &node.base; - } - - if (eatToken(it, .Keyword_nosuspend)) |token| { - const expr_node = try expectNode(arena, it, tree, parseExpr, .{ - .ExpectedExpr = .{ .token = it.index }, - }); - const node = try arena.create(Node.Nosuspend); - node.* = .{ - .nosuspend_token = token, - .expr = expr_node, - }; - return &node.base; - } - - if (eatToken(it, .Keyword_continue)) |token| { - const label = try parseBreakLabel(arena, it, tree); - const node = try arena.create(Node.ControlFlowExpression); - node.* = .{ - .ltoken = token, - .kind = .{ .Continue = label }, - .rhs = null, - }; - return &node.base; - } - - if (eatToken(it, .Keyword_resume)) |token| { - const expr_node = try expectNode(arena, it, tree, parseExpr, .{ - .ExpectedExpr = .{ .token = it.index }, - }); - const node = try arena.create(Node.PrefixOp); - node.* = .{ - .op_token = token, - .op = .Resume, - .rhs = expr_node, - }; - return &node.base; - } - - if (eatToken(it, .Keyword_return)) |token| { - const expr_node = try parseExpr(arena, it, tree); - const node = try arena.create(Node.ControlFlowExpression); - node.* = .{ - .ltoken = token, - .kind = .Return, - .rhs = expr_node, - }; - return &node.base; - } - - var colon: TokenIndex = undefined; - const label = parseBlockLabel(arena, it, tree, &colon); - if (try parseLoopExpr(arena, it, tree)) |node| { - if (node.cast(Node.For)) |for_node| { - for_node.label = label; - } else if (node.cast(Node.While)) |while_node| { - while_node.label = label; - } else unreachable; - return node; - } - if (label) |token| { - putBackToken(it, token + 1); // ":" - putBackToken(it, token); // IDENTIFIER - } - - if (try parseBlock(arena, it, tree)) |node| return node; - if (try parseCurlySuffixExpr(arena, it, tree)) |node| return node; - - return null; -} - -/// IfExpr <- IfPrefix Expr (KEYWORD_else Payload? Expr)? -fn parseIfExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - return parseIf(arena, it, tree, parseExpr); -} - -/// Block <- LBRACE Statement* RBRACE -fn parseBlock(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const lbrace = eatToken(it, .LBrace) orelse return null; - - var statements = Node.Block.StatementList.init(arena); - while (true) { - const statement = (parseStatement(arena, it, tree) catch |err| switch (err) { - error.OutOfMemory => return error.OutOfMemory, - error.ParseError => { - // try to skip to the next statement - findNextStmt(it); - continue; - }, - }) orelse break; - try statements.push(statement); - } - - const rbrace = try expectToken(it, tree, .RBrace); - - const block_node = try arena.create(Node.Block); - block_node.* = .{ - .label = null, - .lbrace = lbrace, - .statements = statements, - .rbrace = rbrace, - }; - - return &block_node.base; -} - -/// LoopExpr <- KEYWORD_inline? (ForExpr / WhileExpr) -fn parseLoopExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const inline_token = eatToken(it, .Keyword_inline); - - if (try parseForExpr(arena, it, tree)) |node| { - node.cast(Node.For).?.inline_token = inline_token; - return node; - } - - if (try parseWhileExpr(arena, it, tree)) |node| { - node.cast(Node.While).?.inline_token = inline_token; - return node; - } - - if (inline_token == null) return null; - - // If we've seen "inline", there should have been a "for" or "while" - try tree.errors.push(.{ - .ExpectedInlinable = .{ .token = it.index }, - }); - return error.ParseError; -} - -/// ForExpr <- ForPrefix Expr (KEYWORD_else Expr)? -fn parseForExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const node = (try parseForPrefix(arena, it, tree)) orelse return null; - const for_prefix = node.cast(Node.For).?; - - const body_node = try expectNode(arena, it, tree, parseExpr, .{ - .ExpectedExpr = .{ .token = it.index }, - }); - for_prefix.body = body_node; - - if (eatToken(it, .Keyword_else)) |else_token| { - const body = try expectNode(arena, it, tree, parseExpr, .{ - .ExpectedExpr = .{ .token = it.index }, - }); - - const else_node = try arena.create(Node.Else); - else_node.* = .{ - .else_token = else_token, - .payload = null, - .body = body, - }; - - for_prefix.@"else" = else_node; - } - - return node; -} - -/// WhileExpr <- WhilePrefix Expr (KEYWORD_else Payload? Expr)? -fn parseWhileExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const node = (try parseWhilePrefix(arena, it, tree)) orelse return null; - const while_prefix = node.cast(Node.While).?; - - const body_node = try expectNode(arena, it, tree, parseExpr, .{ - .ExpectedExpr = .{ .token = it.index }, - }); - while_prefix.body = body_node; - - if (eatToken(it, .Keyword_else)) |else_token| { - const payload = try parsePayload(arena, it, tree); - const body = try expectNode(arena, it, tree, parseExpr, .{ - .ExpectedExpr = .{ .token = it.index }, - }); - - const else_node = try arena.create(Node.Else); - else_node.* = .{ - .else_token = else_token, - .payload = payload, - .body = body, - }; - - while_prefix.@"else" = else_node; - } - - return node; -} - -/// CurlySuffixExpr <- TypeExpr InitList? -fn parseCurlySuffixExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const type_expr = (try parseTypeExpr(arena, it, tree)) orelse return null; - const suffix_op = (try parseInitList(arena, it, tree)) orelse return type_expr; - suffix_op.lhs.node = type_expr; - return &suffix_op.base; -} - -/// InitList -/// <- LBRACE FieldInit (COMMA FieldInit)* COMMA? RBRACE -/// / LBRACE Expr (COMMA Expr)* COMMA? RBRACE -/// / LBRACE RBRACE -fn parseInitList(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node.SuffixOp { - const lbrace = eatToken(it, .LBrace) orelse return null; - var init_list = Node.SuffixOp.Op.InitList.init(arena); - - const op: Node.SuffixOp.Op = blk: { - if (try parseFieldInit(arena, it, tree)) |field_init| { - try init_list.push(field_init); - while (eatToken(it, .Comma)) |_| { - const next = (try parseFieldInit(arena, it, tree)) orelse break; - try init_list.push(next); + /// BlockExpr <- BlockLabel? Block + fn parseBlockExpr(p: *Parser) Error!?*Node { + var colon: TokenIndex = undefined; + const label_token = p.parseBlockLabel(&colon); + const block_node = (try p.parseBlock()) orelse { + if (label_token) |label| { + p.putBackToken(label + 1); // ":" + p.putBackToken(label); // IDENTIFIER } - break :blk .{ .StructInitializer = init_list }; - } - - if (try parseExpr(arena, it, tree)) |expr| { - try init_list.push(expr); - while (eatToken(it, .Comma)) |_| { - const next = (try parseExpr(arena, it, tree)) orelse break; - try init_list.push(next); - } - break :blk .{ .ArrayInitializer = init_list }; - } - - break :blk .{ .StructInitializer = init_list }; - }; - - const node = try arena.create(Node.SuffixOp); - node.* = .{ - .lhs = .{ .node = undefined }, // set by caller - .op = op, - .rtoken = try expectToken(it, tree, .RBrace), - }; - return node; -} - -/// TypeExpr <- PrefixTypeOp* ErrorUnionExpr -fn parseTypeExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?*Node { - return parsePrefixOpExpr(arena, it, tree, parsePrefixTypeOp, parseErrorUnionExpr); -} - -/// ErrorUnionExpr <- SuffixExpr (EXCLAMATIONMARK TypeExpr)? -fn parseErrorUnionExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const suffix_expr = (try parseSuffixExpr(arena, it, tree)) orelse return null; - - if (try SimpleBinOpParseFn(.Bang, Node.InfixOp.Op.ErrorUnion)(arena, it, tree)) |node| { - const error_union = node.cast(Node.InfixOp).?; - const type_expr = try expectNode(arena, it, tree, parseTypeExpr, .{ - .ExpectedTypeExpr = .{ .token = it.index }, - }); - error_union.lhs = suffix_expr; - error_union.rhs = type_expr; - return node; + return null; + }; + block_node.cast(Node.Block).?.label = label_token; + return block_node; } - return suffix_expr; -} + /// AssignExpr <- Expr (AssignOp Expr)? + fn parseAssignExpr(p: *Parser) !?*Node { + return p.parseBinOpExpr(parseAssignOp, parseExpr, .Once); + } -/// SuffixExpr -/// <- KEYWORD_async PrimaryTypeExpr SuffixOp* FnCallArguments -/// / PrimaryTypeExpr (SuffixOp / FnCallArguments)* -fn parseSuffixExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const maybe_async = eatToken(it, .Keyword_async); - if (maybe_async) |async_token| { - const token_fn = eatToken(it, .Keyword_fn); - if (token_fn != null) { - // TODO: remove this hack when async fn rewriting is - // HACK: If we see the keyword `fn`, then we assume that - // we are parsing an async fn proto, and not a call. - // We therefore put back all tokens consumed by the async - // prefix... - putBackToken(it, token_fn.?); - putBackToken(it, async_token); - return parsePrimaryTypeExpr(arena, it, tree); - } - var res = try expectNode(arena, it, tree, parsePrimaryTypeExpr, .{ - .ExpectedPrimaryTypeExpr = .{ .token = it.index }, - }); + /// Expr <- KEYWORD_try* BoolOrExpr + fn parseExpr(p: *Parser) Error!?*Node { + return p.parsePrefixOpExpr(parseTry, parseBoolOrExpr); + } - while (try parseSuffixOp(arena, it, tree)) |node| { - switch (node.id) { - .SuffixOp => node.cast(Node.SuffixOp).?.lhs = .{ .node = res }, - .InfixOp => node.cast(Node.InfixOp).?.lhs = res, - else => unreachable, - } - res = node; + /// BoolOrExpr <- BoolAndExpr (KEYWORD_or BoolAndExpr)* + fn parseBoolOrExpr(p: *Parser) !?*Node { + return p.parseBinOpExpr( + SimpleBinOpParseFn(.Keyword_or, Node.InfixOp.Op.BoolOr), + parseBoolAndExpr, + .Infinitely, + ); + } + + /// BoolAndExpr <- CompareExpr (KEYWORD_and CompareExpr)* + fn parseBoolAndExpr(p: *Parser) !?*Node { + return p.parseBinOpExpr( + SimpleBinOpParseFn(.Keyword_and, .BoolAnd), + parseCompareExpr, + .Infinitely, + ); + } + + /// CompareExpr <- BitwiseExpr (CompareOp BitwiseExpr)? + fn parseCompareExpr(p: *Parser) !?*Node { + return p.parseBinOpExpr(parseCompareOp, parseBitwiseExpr, .Once); + } + + /// BitwiseExpr <- BitShiftExpr (BitwiseOp BitShiftExpr)* + fn parseBitwiseExpr(p: *Parser) !?*Node { + return p.parseBinOpExpr(parseBitwiseOp, parseBitShiftExpr, .Infinitely); + } + + /// BitShiftExpr <- AdditionExpr (BitShiftOp AdditionExpr)* + fn parseBitShiftExpr(p: *Parser) !?*Node { + return p.parseBinOpExpr(parseBitShiftOp, parseAdditionExpr, .Infinitely); + } + + /// AdditionExpr <- MultiplyExpr (AdditionOp MultiplyExpr)* + fn parseAdditionExpr(p: *Parser) !?*Node { + return p.parseBinOpExpr(parseAdditionOp, parseMultiplyExpr, .Infinitely); + } + + /// MultiplyExpr <- PrefixExpr (MultiplyOp PrefixExpr)* + fn parseMultiplyExpr(p: *Parser) !?*Node { + return p.parseBinOpExpr(parseMultiplyOp, parsePrefixExpr, .Infinitely); + } + + /// PrefixExpr <- PrefixOp* PrimaryExpr + fn parsePrefixExpr(p: *Parser) !?*Node { + return p.parsePrefixOpExpr(parsePrefixOp, parsePrimaryExpr); + } + + /// PrimaryExpr + /// <- AsmExpr + /// / IfExpr + /// / KEYWORD_break BreakLabel? Expr? + /// / KEYWORD_comptime Expr + /// / KEYWORD_nosuspend Expr + /// / KEYWORD_continue BreakLabel? + /// / KEYWORD_resume Expr + /// / KEYWORD_return Expr? + /// / BlockLabel? LoopExpr + /// / Block + /// / CurlySuffixExpr + fn parsePrimaryExpr(p: *Parser) !?*Node { + if (try p.parseAsmExpr()) |node| return node; + if (try p.parseIfExpr()) |node| return node; + + if (p.eatToken(.Keyword_break)) |token| { + const label = try p.parseBreakLabel(); + const expr_node = try p.parseExpr(); + const node = try p.arena.allocator.create(Node.ControlFlowExpression); + node.* = .{ + .ltoken = token, + .kind = .{ .Break = label }, + .rhs = expr_node, + }; + return &node.base; } - const params = (try parseFnCallArguments(arena, it, tree)) orelse { - try tree.errors.push(.{ - .ExpectedParamList = .{ .token = it.index }, + if (p.eatToken(.Keyword_comptime)) |token| { + const expr_node = try p.expectNode(parseExpr, .{ + .ExpectedExpr = .{ .token = p.tok_i }, }); - // ignore this, continue parsing - return res; - }; - const node = try arena.create(Node.SuffixOp); - node.* = .{ - .lhs = .{ .node = res }, - .op = .{ - .Call = .{ - .params = params.list, - .async_token = async_token, - }, - }, - .rtoken = params.rparen, - }; - return &node.base; - } - if (try parsePrimaryTypeExpr(arena, it, tree)) |expr| { - var res = expr; + const node = try p.arena.allocator.create(Node.Comptime); + node.* = .{ + .doc_comments = null, + .comptime_token = token, + .expr = expr_node, + }; + return &node.base; + } + if (p.eatToken(.Keyword_nosuspend)) |token| { + const expr_node = try p.expectNode(parseExpr, .{ + .ExpectedExpr = .{ .token = p.tok_i }, + }); + const node = try p.arena.allocator.create(Node.Nosuspend); + node.* = .{ + .nosuspend_token = token, + .expr = expr_node, + }; + return &node.base; + } + + if (p.eatToken(.Keyword_continue)) |token| { + const label = try p.parseBreakLabel(); + const node = try p.arena.allocator.create(Node.ControlFlowExpression); + node.* = .{ + .ltoken = token, + .kind = .{ .Continue = label }, + .rhs = null, + }; + return &node.base; + } + + if (p.eatToken(.Keyword_resume)) |token| { + const expr_node = try p.expectNode(parseExpr, .{ + .ExpectedExpr = .{ .token = p.tok_i }, + }); + const node = try p.arena.allocator.create(Node.PrefixOp); + node.* = .{ + .op_token = token, + .op = .Resume, + .rhs = expr_node, + }; + return &node.base; + } + + if (p.eatToken(.Keyword_return)) |token| { + const expr_node = try p.parseExpr(); + const node = try p.arena.allocator.create(Node.ControlFlowExpression); + node.* = .{ + .ltoken = token, + .kind = .Return, + .rhs = expr_node, + }; + return &node.base; + } + + var colon: TokenIndex = undefined; + const label = p.parseBlockLabel(&colon); + if (try p.parseLoopExpr()) |node| { + if (node.cast(Node.For)) |for_node| { + for_node.label = label; + } else if (node.cast(Node.While)) |while_node| { + while_node.label = label; + } else unreachable; + return node; + } + if (label) |token| { + p.putBackToken(token + 1); // ":" + p.putBackToken(token); // IDENTIFIER + } + + if (try p.parseBlock()) |node| return node; + if (try p.parseCurlySuffixExpr()) |node| return node; + + return null; + } + + /// IfExpr <- IfPrefix Expr (KEYWORD_else Payload? Expr)? + fn parseIfExpr(p: *Parser) !?*Node { + return p.parseIf(parseExpr); + } + + /// Block <- LBRACE Statement* RBRACE + fn parseBlock(p: *Parser) !?*Node { + const lbrace = p.eatToken(.LBrace) orelse return null; + + var statements = Node.Block.StatementList{}; + var statements_it = &statements.first; while (true) { - if (try parseSuffixOp(arena, it, tree)) |node| { + const statement = (p.parseStatement() catch |err| switch (err) { + error.OutOfMemory => return error.OutOfMemory, + error.ParseError => { + // try to skip to the next statement + p.findNextStmt(); + continue; + }, + }) orelse break; + statements_it = try p.llpush(*Node, statements_it, statement); + } + + const rbrace = try p.expectToken(.RBrace); + + const block_node = try p.arena.allocator.create(Node.Block); + block_node.* = .{ + .label = null, + .lbrace = lbrace, + .statements = statements, + .rbrace = rbrace, + }; + + return &block_node.base; + } + + /// LoopExpr <- KEYWORD_inline? (ForExpr / WhileExpr) + fn parseLoopExpr(p: *Parser) !?*Node { + const inline_token = p.eatToken(.Keyword_inline); + + if (try p.parseForExpr()) |node| { + node.cast(Node.For).?.inline_token = inline_token; + return node; + } + + if (try p.parseWhileExpr()) |node| { + node.cast(Node.While).?.inline_token = inline_token; + return node; + } + + if (inline_token == null) return null; + + // If we've seen "inline", there should have been a "for" or "while" + try p.errors.append(p.gpa, .{ + .ExpectedInlinable = .{ .token = p.tok_i }, + }); + return error.ParseError; + } + + /// ForExpr <- ForPrefix Expr (KEYWORD_else Expr)? + fn parseForExpr(p: *Parser) !?*Node { + const node = (try p.parseForPrefix()) orelse return null; + const for_prefix = node.cast(Node.For).?; + + const body_node = try p.expectNode(parseExpr, .{ + .ExpectedExpr = .{ .token = p.tok_i }, + }); + for_prefix.body = body_node; + + if (p.eatToken(.Keyword_else)) |else_token| { + const body = try p.expectNode(parseExpr, .{ + .ExpectedExpr = .{ .token = p.tok_i }, + }); + + const else_node = try p.arena.allocator.create(Node.Else); + else_node.* = .{ + .else_token = else_token, + .payload = null, + .body = body, + }; + + for_prefix.@"else" = else_node; + } + + return node; + } + + /// WhileExpr <- WhilePrefix Expr (KEYWORD_else Payload? Expr)? + fn parseWhileExpr(p: *Parser) !?*Node { + const node = (try p.parseWhilePrefix()) orelse return null; + const while_prefix = node.cast(Node.While).?; + + const body_node = try p.expectNode(parseExpr, .{ + .ExpectedExpr = .{ .token = p.tok_i }, + }); + while_prefix.body = body_node; + + if (p.eatToken(.Keyword_else)) |else_token| { + const payload = try p.parsePayload(); + const body = try p.expectNode(parseExpr, .{ + .ExpectedExpr = .{ .token = p.tok_i }, + }); + + const else_node = try p.arena.allocator.create(Node.Else); + else_node.* = .{ + .else_token = else_token, + .payload = payload, + .body = body, + }; + + while_prefix.@"else" = else_node; + } + + return node; + } + + /// CurlySuffixExpr <- TypeExpr InitList? + fn parseCurlySuffixExpr(p: *Parser) !?*Node { + const type_expr = (try p.parseTypeExpr()) orelse return null; + const suffix_op = (try p.parseInitList()) orelse return type_expr; + suffix_op.lhs.node = type_expr; + return &suffix_op.base; + } + + /// InitList + /// <- LBRACE FieldInit (COMMA FieldInit)* COMMA? RBRACE + /// / LBRACE Expr (COMMA Expr)* COMMA? RBRACE + /// / LBRACE RBRACE + fn parseInitList(p: *Parser) !?*Node.SuffixOp { + const lbrace = p.eatToken(.LBrace) orelse return null; + var init_list = Node.SuffixOp.Op.InitList{}; + var init_list_it = &init_list.first; + + const op: Node.SuffixOp.Op = blk: { + if (try p.parseFieldInit()) |field_init| { + init_list_it = try p.llpush(*Node, init_list_it, field_init); + while (p.eatToken(.Comma)) |_| { + const next = (try p.parseFieldInit()) orelse break; + init_list_it = try p.llpush(*Node, init_list_it, next); + } + break :blk .{ .StructInitializer = init_list }; + } + + if (try p.parseExpr()) |expr| { + init_list_it = try p.llpush(*Node, init_list_it, expr); + while (p.eatToken(.Comma)) |_| { + const next = (try p.parseExpr()) orelse break; + init_list_it = try p.llpush(*Node, init_list_it, next); + } + break :blk .{ .ArrayInitializer = init_list }; + } + + break :blk .{ .StructInitializer = init_list }; + }; + + const node = try p.arena.allocator.create(Node.SuffixOp); + node.* = .{ + .lhs = .{ .node = undefined }, // set by caller + .op = op, + .rtoken = try p.expectToken(.RBrace), + }; + return node; + } + + /// TypeExpr <- PrefixTypeOp* ErrorUnionExpr + fn parseTypeExpr(p: *Parser) Error!?*Node { + return p.parsePrefixOpExpr(parsePrefixTypeOp, parseErrorUnionExpr); + } + + /// ErrorUnionExpr <- SuffixExpr (EXCLAMATIONMARK TypeExpr)? + fn parseErrorUnionExpr(p: *Parser) !?*Node { + const suffix_expr = (try p.parseSuffixExpr()) orelse return null; + + if (try SimpleBinOpParseFn(.Bang, Node.InfixOp.Op.ErrorUnion)(p)) |node| { + const error_union = node.cast(Node.InfixOp).?; + const type_expr = try p.expectNode(parseTypeExpr, .{ + .ExpectedTypeExpr = .{ .token = p.tok_i }, + }); + error_union.lhs = suffix_expr; + error_union.rhs = type_expr; + return node; + } + + return suffix_expr; + } + + /// SuffixExpr + /// <- KEYWORD_async PrimaryTypeExpr SuffixOp* FnCallArguments + /// / PrimaryTypeExpr (SuffixOp / FnCallArguments)* + fn parseSuffixExpr(p: *Parser) !?*Node { + const maybe_async = p.eatToken(.Keyword_async); + if (maybe_async) |async_token| { + const token_fn = p.eatToken(.Keyword_fn); + if (token_fn != null) { + // TODO: remove this hack when async fn rewriting is + // HACK: If we see the keyword `fn`, then we assume that + // we are parsing an async fn proto, and not a call. + // We therefore put back all tokens consumed by the async + // prefix... + p.putBackToken(token_fn.?); + p.putBackToken(async_token); + return p.parsePrimaryTypeExpr(); + } + var res = try p.expectNode(parsePrimaryTypeExpr, .{ + .ExpectedPrimaryTypeExpr = .{ .token = p.tok_i }, + }); + + while (try p.parseSuffixOp()) |node| { switch (node.id) { .SuffixOp => node.cast(Node.SuffixOp).?.lhs = .{ .node = res }, .InfixOp => node.cast(Node.InfixOp).?.lhs = res, else => unreachable, } res = node; - continue; } - if (try parseFnCallArguments(arena, it, tree)) |params| { - const call = try arena.create(Node.SuffixOp); - call.* = .{ - .lhs = .{ .node = res }, - .op = .{ - .Call = .{ - .params = params.list, - .async_token = null, - }, + + const params = (try p.parseFnCallArguments()) orelse { + try p.errors.append(p.gpa, .{ + .ExpectedParamList = .{ .token = p.tok_i }, + }); + // ignore this, continue parsing + return res; + }; + const node = try p.arena.allocator.create(Node.SuffixOp); + node.* = .{ + .lhs = .{ .node = res }, + .op = .{ + .Call = .{ + .params = params.list, + .async_token = async_token, }, - .rtoken = params.rparen, - }; - res = &call.base; - continue; - } - break; + }, + .rtoken = params.rparen, + }; + return &node.base; } - return res; + if (try p.parsePrimaryTypeExpr()) |expr| { + var res = expr; + + while (true) { + if (try p.parseSuffixOp()) |node| { + switch (node.id) { + .SuffixOp => node.cast(Node.SuffixOp).?.lhs = .{ .node = res }, + .InfixOp => node.cast(Node.InfixOp).?.lhs = res, + else => unreachable, + } + res = node; + continue; + } + if (try p.parseFnCallArguments()) |params| { + const call = try p.arena.allocator.create(Node.SuffixOp); + call.* = .{ + .lhs = .{ .node = res }, + .op = .{ + .Call = .{ + .params = params.list, + .async_token = null, + }, + }, + .rtoken = params.rparen, + }; + res = &call.base; + continue; + } + break; + } + return res; + } + + return null; } - return null; -} + /// PrimaryTypeExpr + /// <- BUILTINIDENTIFIER FnCallArguments + /// / CHAR_LITERAL + /// / ContainerDecl + /// / DOT IDENTIFIER + /// / ErrorSetDecl + /// / FLOAT + /// / FnProto + /// / GroupedExpr + /// / LabeledTypeExpr + /// / IDENTIFIER + /// / IfTypeExpr + /// / INTEGER + /// / KEYWORD_comptime TypeExpr + /// / KEYWORD_error DOT IDENTIFIER + /// / KEYWORD_false + /// / KEYWORD_null + /// / KEYWORD_anyframe + /// / KEYWORD_true + /// / KEYWORD_undefined + /// / KEYWORD_unreachable + /// / STRINGLITERAL + /// / SwitchExpr + fn parsePrimaryTypeExpr(p: *Parser) !?*Node { + if (try p.parseBuiltinCall()) |node| return node; + if (p.eatToken(.CharLiteral)) |token| { + const node = try p.arena.allocator.create(Node.CharLiteral); + node.* = .{ + .token = token, + }; + return &node.base; + } + if (try p.parseContainerDecl()) |node| return node; + if (try p.parseAnonLiteral()) |node| return node; + if (try p.parseErrorSetDecl()) |node| return node; + if (try p.parseFloatLiteral()) |node| return node; + if (try p.parseFnProto()) |node| return node; + if (try p.parseGroupedExpr()) |node| return node; + if (try p.parseLabeledTypeExpr()) |node| return node; + if (try p.parseIdentifier()) |node| return node; + if (try p.parseIfTypeExpr()) |node| return node; + if (try p.parseIntegerLiteral()) |node| return node; + if (p.eatToken(.Keyword_comptime)) |token| { + const expr = (try p.parseTypeExpr()) orelse return null; + const node = try p.arena.allocator.create(Node.Comptime); + node.* = .{ + .doc_comments = null, + .comptime_token = token, + .expr = expr, + }; + return &node.base; + } + if (p.eatToken(.Keyword_error)) |token| { + const period = try p.expectTokenRecoverable(.Period); + const identifier = try p.expectNodeRecoverable(parseIdentifier, .{ + .ExpectedIdentifier = .{ .token = p.tok_i }, + }); + const global_error_set = try p.createLiteral(Node.ErrorType, token); + if (period == null or identifier == null) return global_error_set; -/// PrimaryTypeExpr -/// <- BUILTINIDENTIFIER FnCallArguments -/// / CHAR_LITERAL -/// / ContainerDecl -/// / DOT IDENTIFIER -/// / ErrorSetDecl -/// / FLOAT -/// / FnProto -/// / GroupedExpr -/// / LabeledTypeExpr -/// / IDENTIFIER -/// / IfTypeExpr -/// / INTEGER -/// / KEYWORD_comptime TypeExpr -/// / KEYWORD_error DOT IDENTIFIER -/// / KEYWORD_false -/// / KEYWORD_null -/// / KEYWORD_anyframe -/// / KEYWORD_true -/// / KEYWORD_undefined -/// / KEYWORD_unreachable -/// / STRINGLITERAL -/// / SwitchExpr -fn parsePrimaryTypeExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - if (try parseBuiltinCall(arena, it, tree)) |node| return node; - if (eatToken(it, .CharLiteral)) |token| { - const node = try arena.create(Node.CharLiteral); + const node = try p.arena.allocator.create(Node.InfixOp); + node.* = .{ + .op_token = period.?, + .lhs = global_error_set, + .op = .Period, + .rhs = identifier.?, + }; + return &node.base; + } + if (p.eatToken(.Keyword_false)) |token| return p.createLiteral(Node.BoolLiteral, token); + if (p.eatToken(.Keyword_null)) |token| return p.createLiteral(Node.NullLiteral, token); + if (p.eatToken(.Keyword_anyframe)) |token| { + const node = try p.arena.allocator.create(Node.AnyFrameType); + node.* = .{ + .anyframe_token = token, + .result = null, + }; + return &node.base; + } + if (p.eatToken(.Keyword_true)) |token| return p.createLiteral(Node.BoolLiteral, token); + if (p.eatToken(.Keyword_undefined)) |token| return p.createLiteral(Node.UndefinedLiteral, token); + if (p.eatToken(.Keyword_unreachable)) |token| return p.createLiteral(Node.Unreachable, token); + if (try p.parseStringLiteral()) |node| return node; + if (try p.parseSwitchExpr()) |node| return node; + + return null; + } + + /// ContainerDecl <- (KEYWORD_extern / KEYWORD_packed)? ContainerDeclAuto + fn parseContainerDecl(p: *Parser) !?*Node { + const layout_token = p.eatToken(.Keyword_extern) orelse + p.eatToken(.Keyword_packed); + + const node = (try p.parseContainerDeclAuto()) orelse { + if (layout_token) |token| + p.putBackToken(token); + return null; + }; + node.cast(Node.ContainerDecl).?.*.layout_token = layout_token; + return node; + } + + /// ErrorSetDecl <- KEYWORD_error LBRACE IdentifierList RBRACE + fn parseErrorSetDecl(p: *Parser) !?*Node { + const error_token = p.eatToken(.Keyword_error) orelse return null; + if (p.eatToken(.LBrace) == null) { + // Might parse as `KEYWORD_error DOT IDENTIFIER` later in PrimaryTypeExpr, so don't error + p.putBackToken(error_token); + return null; + } + const decls = try p.parseErrorTagList(); + const rbrace = try p.expectToken(.RBrace); + + const node = try p.arena.allocator.create(Node.ErrorSetDecl); node.* = .{ - .token = token, + .error_token = error_token, + .decls = decls, + .rbrace_token = rbrace, }; return &node.base; } - if (try parseContainerDecl(arena, it, tree)) |node| return node; - if (try parseAnonLiteral(arena, it, tree)) |node| return node; - if (try parseErrorSetDecl(arena, it, tree)) |node| return node; - if (try parseFloatLiteral(arena, it, tree)) |node| return node; - if (try parseFnProto(arena, it, tree)) |node| return node; - if (try parseGroupedExpr(arena, it, tree)) |node| return node; - if (try parseLabeledTypeExpr(arena, it, tree)) |node| return node; - if (try parseIdentifier(arena, it, tree)) |node| return node; - if (try parseIfTypeExpr(arena, it, tree)) |node| return node; - if (try parseIntegerLiteral(arena, it, tree)) |node| return node; - if (eatToken(it, .Keyword_comptime)) |token| { - const expr = (try parseTypeExpr(arena, it, tree)) orelse return null; - const node = try arena.create(Node.Comptime); - node.* = .{ - .doc_comments = null, - .comptime_token = token, - .expr = expr, - }; - return &node.base; - } - if (eatToken(it, .Keyword_error)) |token| { - const period = try expectTokenRecoverable(it, tree, .Period); - const identifier = try expectNodeRecoverable(arena, it, tree, parseIdentifier, .{ - .ExpectedIdentifier = .{ .token = it.index }, + + /// GroupedExpr <- LPAREN Expr RPAREN + fn parseGroupedExpr(p: *Parser) !?*Node { + const lparen = p.eatToken(.LParen) orelse return null; + const expr = try p.expectNode(parseExpr, .{ + .ExpectedExpr = .{ .token = p.tok_i }, }); - const global_error_set = try createLiteral(arena, Node.ErrorType, token); - if (period == null or identifier == null) return global_error_set; + const rparen = try p.expectToken(.RParen); - const node = try arena.create(Node.InfixOp); + const node = try p.arena.allocator.create(Node.GroupedExpression); node.* = .{ - .op_token = period.?, - .lhs = global_error_set, - .op = .Period, - .rhs = identifier.?, + .lparen = lparen, + .expr = expr, + .rparen = rparen, }; return &node.base; } - if (eatToken(it, .Keyword_false)) |token| return createLiteral(arena, Node.BoolLiteral, token); - if (eatToken(it, .Keyword_null)) |token| return createLiteral(arena, Node.NullLiteral, token); - if (eatToken(it, .Keyword_anyframe)) |token| { - const node = try arena.create(Node.AnyFrameType); - node.* = .{ - .anyframe_token = token, - .result = null, - }; - return &node.base; + + /// IfTypeExpr <- IfPrefix TypeExpr (KEYWORD_else Payload? TypeExpr)? + fn parseIfTypeExpr(p: *Parser) !?*Node { + return p.parseIf(parseTypeExpr); } - if (eatToken(it, .Keyword_true)) |token| return createLiteral(arena, Node.BoolLiteral, token); - if (eatToken(it, .Keyword_undefined)) |token| return createLiteral(arena, Node.UndefinedLiteral, token); - if (eatToken(it, .Keyword_unreachable)) |token| return createLiteral(arena, Node.Unreachable, token); - if (try parseStringLiteral(arena, it, tree)) |node| return node; - if (try parseSwitchExpr(arena, it, tree)) |node| return node; - return null; -} + /// LabeledTypeExpr + /// <- BlockLabel Block + /// / BlockLabel? LoopTypeExpr + fn parseLabeledTypeExpr(p: *Parser) !?*Node { + var colon: TokenIndex = undefined; + const label = p.parseBlockLabel(&colon); -/// ContainerDecl <- (KEYWORD_extern / KEYWORD_packed)? ContainerDeclAuto -fn parseContainerDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const layout_token = eatToken(it, .Keyword_extern) orelse - eatToken(it, .Keyword_packed); + if (label) |token| { + if (try p.parseBlock()) |node| { + node.cast(Node.Block).?.label = token; + return node; + } + } - const node = (try parseContainerDeclAuto(arena, it, tree)) orelse { - if (layout_token) |token| - putBackToken(it, token); - return null; - }; - node.cast(Node.ContainerDecl).?.*.layout_token = layout_token; - return node; -} - -/// ErrorSetDecl <- KEYWORD_error LBRACE IdentifierList RBRACE -fn parseErrorSetDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const error_token = eatToken(it, .Keyword_error) orelse return null; - if (eatToken(it, .LBrace) == null) { - // Might parse as `KEYWORD_error DOT IDENTIFIER` later in PrimaryTypeExpr, so don't error - putBackToken(it, error_token); - return null; - } - const decls = try parseErrorTagList(arena, it, tree); - const rbrace = try expectToken(it, tree, .RBrace); - - const node = try arena.create(Node.ErrorSetDecl); - node.* = .{ - .error_token = error_token, - .decls = decls, - .rbrace_token = rbrace, - }; - return &node.base; -} - -/// GroupedExpr <- LPAREN Expr RPAREN -fn parseGroupedExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const lparen = eatToken(it, .LParen) orelse return null; - const expr = try expectNode(arena, it, tree, parseExpr, .{ - .ExpectedExpr = .{ .token = it.index }, - }); - const rparen = try expectToken(it, tree, .RParen); - - const node = try arena.create(Node.GroupedExpression); - node.* = .{ - .lparen = lparen, - .expr = expr, - .rparen = rparen, - }; - return &node.base; -} - -/// IfTypeExpr <- IfPrefix TypeExpr (KEYWORD_else Payload? TypeExpr)? -fn parseIfTypeExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - return parseIf(arena, it, tree, parseTypeExpr); -} - -/// LabeledTypeExpr -/// <- BlockLabel Block -/// / BlockLabel? LoopTypeExpr -fn parseLabeledTypeExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - var colon: TokenIndex = undefined; - const label = parseBlockLabel(arena, it, tree, &colon); - - if (label) |token| { - if (try parseBlock(arena, it, tree)) |node| { - node.cast(Node.Block).?.label = token; + if (try p.parseLoopTypeExpr()) |node| { + switch (node.id) { + .For => node.cast(Node.For).?.label = label, + .While => node.cast(Node.While).?.label = label, + else => unreachable, + } return node; } - } - if (try parseLoopTypeExpr(arena, it, tree)) |node| { - switch (node.id) { - .For => node.cast(Node.For).?.label = label, - .While => node.cast(Node.While).?.label = label, - else => unreachable, + if (label) |token| { + p.putBackToken(colon); + p.putBackToken(token); } - return node; + return null; } - if (label) |token| { - putBackToken(it, colon); - putBackToken(it, token); - } - return null; -} + /// LoopTypeExpr <- KEYWORD_inline? (ForTypeExpr / WhileTypeExpr) + fn parseLoopTypeExpr(p: *Parser) !?*Node { + const inline_token = p.eatToken(.Keyword_inline); -/// LoopTypeExpr <- KEYWORD_inline? (ForTypeExpr / WhileTypeExpr) -fn parseLoopTypeExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const inline_token = eatToken(it, .Keyword_inline); - - if (try parseForTypeExpr(arena, it, tree)) |node| { - node.cast(Node.For).?.inline_token = inline_token; - return node; - } - - if (try parseWhileTypeExpr(arena, it, tree)) |node| { - node.cast(Node.While).?.inline_token = inline_token; - return node; - } - - if (inline_token == null) return null; - - // If we've seen "inline", there should have been a "for" or "while" - try tree.errors.push(.{ - .ExpectedInlinable = .{ .token = it.index }, - }); - return error.ParseError; -} - -/// ForTypeExpr <- ForPrefix TypeExpr (KEYWORD_else TypeExpr)? -fn parseForTypeExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const node = (try parseForPrefix(arena, it, tree)) orelse return null; - const for_prefix = node.cast(Node.For).?; - - const type_expr = try expectNode(arena, it, tree, parseTypeExpr, .{ - .ExpectedTypeExpr = .{ .token = it.index }, - }); - for_prefix.body = type_expr; - - if (eatToken(it, .Keyword_else)) |else_token| { - const else_expr = try expectNode(arena, it, tree, parseTypeExpr, .{ - .ExpectedTypeExpr = .{ .token = it.index }, - }); - - const else_node = try arena.create(Node.Else); - else_node.* = .{ - .else_token = else_token, - .payload = null, - .body = else_expr, - }; - - for_prefix.@"else" = else_node; - } - - return node; -} - -/// WhileTypeExpr <- WhilePrefix TypeExpr (KEYWORD_else Payload? TypeExpr)? -fn parseWhileTypeExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const node = (try parseWhilePrefix(arena, it, tree)) orelse return null; - const while_prefix = node.cast(Node.While).?; - - const type_expr = try expectNode(arena, it, tree, parseTypeExpr, .{ - .ExpectedTypeExpr = .{ .token = it.index }, - }); - while_prefix.body = type_expr; - - if (eatToken(it, .Keyword_else)) |else_token| { - const payload = try parsePayload(arena, it, tree); - - const else_expr = try expectNode(arena, it, tree, parseTypeExpr, .{ - .ExpectedTypeExpr = .{ .token = it.index }, - }); - - const else_node = try arena.create(Node.Else); - else_node.* = .{ - .else_token = else_token, - .payload = null, - .body = else_expr, - }; - - while_prefix.@"else" = else_node; - } - - return node; -} - -/// SwitchExpr <- KEYWORD_switch LPAREN Expr RPAREN LBRACE SwitchProngList RBRACE -fn parseSwitchExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const switch_token = eatToken(it, .Keyword_switch) orelse return null; - _ = try expectToken(it, tree, .LParen); - const expr_node = try expectNode(arena, it, tree, parseExpr, .{ - .ExpectedExpr = .{ .token = it.index }, - }); - _ = try expectToken(it, tree, .RParen); - _ = try expectToken(it, tree, .LBrace); - const cases = try parseSwitchProngList(arena, it, tree); - const rbrace = try expectToken(it, tree, .RBrace); - - const node = try arena.create(Node.Switch); - node.* = .{ - .switch_token = switch_token, - .expr = expr_node, - .cases = cases, - .rbrace = rbrace, - }; - return &node.base; -} - -/// AsmExpr <- KEYWORD_asm KEYWORD_volatile? LPAREN Expr AsmOutput? RPAREN -fn parseAsmExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const asm_token = eatToken(it, .Keyword_asm) orelse return null; - const volatile_token = eatToken(it, .Keyword_volatile); - _ = try expectToken(it, tree, .LParen); - const template = try expectNode(arena, it, tree, parseExpr, .{ - .ExpectedExpr = .{ .token = it.index }, - }); - - const node = try arena.create(Node.Asm); - node.* = .{ - .asm_token = asm_token, - .volatile_token = volatile_token, - .template = template, - .outputs = Node.Asm.OutputList.init(arena), - .inputs = Node.Asm.InputList.init(arena), - .clobbers = Node.Asm.ClobberList.init(arena), - .rparen = undefined, - }; - - try parseAsmOutput(arena, it, tree, node); - node.rparen = try expectToken(it, tree, .RParen); - return &node.base; -} - -/// DOT IDENTIFIER -fn parseAnonLiteral(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const dot = eatToken(it, .Period) orelse return null; - - // anon enum literal - if (eatToken(it, .Identifier)) |name| { - const node = try arena.create(Node.EnumLiteral); - node.* = .{ - .dot = dot, - .name = name, - }; - return &node.base; - } - - // anon container literal - if (try parseInitList(arena, it, tree)) |node| { - node.lhs = .{ .dot = dot }; - return &node.base; - } - - putBackToken(it, dot); - return null; -} - -/// AsmOutput <- COLON AsmOutputList AsmInput? -fn parseAsmOutput(arena: *Allocator, it: *TokenIterator, tree: *Tree, asm_node: *Node.Asm) !void { - if (eatToken(it, .Colon) == null) return; - asm_node.outputs = try parseAsmOutputList(arena, it, tree); - try parseAsmInput(arena, it, tree, asm_node); -} - -/// AsmOutputItem <- LBRACKET IDENTIFIER RBRACKET STRINGLITERAL LPAREN (MINUSRARROW TypeExpr / IDENTIFIER) RPAREN -fn parseAsmOutputItem(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node.AsmOutput { - const lbracket = eatToken(it, .LBracket) orelse return null; - const name = try expectNode(arena, it, tree, parseIdentifier, .{ - .ExpectedIdentifier = .{ .token = it.index }, - }); - _ = try expectToken(it, tree, .RBracket); - - const constraint = try expectNode(arena, it, tree, parseStringLiteral, .{ - .ExpectedStringLiteral = .{ .token = it.index }, - }); - - _ = try expectToken(it, tree, .LParen); - const kind: Node.AsmOutput.Kind = blk: { - if (eatToken(it, .Arrow) != null) { - const return_ident = try expectNode(arena, it, tree, parseTypeExpr, .{ - .ExpectedTypeExpr = .{ .token = it.index }, - }); - break :blk .{ .Return = return_ident }; + if (try p.parseForTypeExpr()) |node| { + node.cast(Node.For).?.inline_token = inline_token; + return node; } - const variable = try expectNode(arena, it, tree, parseIdentifier, .{ - .ExpectedIdentifier = .{ .token = it.index }, - }); - break :blk .{ .Variable = variable.cast(Node.Identifier).? }; - }; - const rparen = try expectToken(it, tree, .RParen); - const node = try arena.create(Node.AsmOutput); - node.* = .{ - .lbracket = lbracket, - .symbolic_name = name, - .constraint = constraint, - .kind = kind, - .rparen = rparen, - }; - return node; -} + if (try p.parseWhileTypeExpr()) |node| { + node.cast(Node.While).?.inline_token = inline_token; + return node; + } -/// AsmInput <- COLON AsmInputList AsmClobbers? -fn parseAsmInput(arena: *Allocator, it: *TokenIterator, tree: *Tree, asm_node: *Node.Asm) !void { - if (eatToken(it, .Colon) == null) return; - asm_node.inputs = try parseAsmInputList(arena, it, tree); - try parseAsmClobbers(arena, it, tree, asm_node); -} + if (inline_token == null) return null; -/// AsmInputItem <- LBRACKET IDENTIFIER RBRACKET STRINGLITERAL LPAREN Expr RPAREN -fn parseAsmInputItem(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node.AsmInput { - const lbracket = eatToken(it, .LBracket) orelse return null; - const name = try expectNode(arena, it, tree, parseIdentifier, .{ - .ExpectedIdentifier = .{ .token = it.index }, - }); - _ = try expectToken(it, tree, .RBracket); - - const constraint = try expectNode(arena, it, tree, parseStringLiteral, .{ - .ExpectedStringLiteral = .{ .token = it.index }, - }); - - _ = try expectToken(it, tree, .LParen); - const expr = try expectNode(arena, it, tree, parseExpr, .{ - .ExpectedExpr = .{ .token = it.index }, - }); - const rparen = try expectToken(it, tree, .RParen); - - const node = try arena.create(Node.AsmInput); - node.* = .{ - .lbracket = lbracket, - .symbolic_name = name, - .constraint = constraint, - .expr = expr, - .rparen = rparen, - }; - return node; -} - -/// AsmClobbers <- COLON StringList -/// StringList <- (STRINGLITERAL COMMA)* STRINGLITERAL? -fn parseAsmClobbers(arena: *Allocator, it: *TokenIterator, tree: *Tree, asm_node: *Node.Asm) !void { - if (eatToken(it, .Colon) == null) return; - asm_node.clobbers = try ListParseFn( - Node.Asm.ClobberList, - parseStringLiteral, - )(arena, it, tree); -} - -/// BreakLabel <- COLON IDENTIFIER -fn parseBreakLabel(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - _ = eatToken(it, .Colon) orelse return null; - return try expectNode(arena, it, tree, parseIdentifier, .{ - .ExpectedIdentifier = .{ .token = it.index }, - }); -} - -/// BlockLabel <- IDENTIFIER COLON -fn parseBlockLabel(arena: *Allocator, it: *TokenIterator, tree: *Tree, colon_token: *TokenIndex) ?TokenIndex { - const identifier = eatToken(it, .Identifier) orelse return null; - if (eatToken(it, .Colon)) |colon| { - colon_token.* = colon; - return identifier; - } - putBackToken(it, identifier); - return null; -} - -/// FieldInit <- DOT IDENTIFIER EQUAL Expr -fn parseFieldInit(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const period_token = eatToken(it, .Period) orelse return null; - const name_token = eatToken(it, .Identifier) orelse { - // Because of anon literals `.{` is also valid. - putBackToken(it, period_token); - return null; - }; - const eq_token = eatToken(it, .Equal) orelse { - // `.Name` may also be an enum literal, which is a later rule. - putBackToken(it, name_token); - putBackToken(it, period_token); - return null; - }; - const expr_node = try expectNode(arena, it, tree, parseExpr, .{ - .ExpectedExpr = .{ .token = it.index }, - }); - - const node = try arena.create(Node.FieldInitializer); - node.* = .{ - .period_token = period_token, - .name_token = name_token, - .expr = expr_node, - }; - return &node.base; -} - -/// WhileContinueExpr <- COLON LPAREN AssignExpr RPAREN -fn parseWhileContinueExpr(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - _ = eatToken(it, .Colon) orelse return null; - _ = try expectToken(it, tree, .LParen); - const node = try expectNode(arena, it, tree, parseAssignExpr, .{ - .ExpectedExprOrAssignment = .{ .token = it.index }, - }); - _ = try expectToken(it, tree, .RParen); - return node; -} - -/// LinkSection <- KEYWORD_linksection LPAREN Expr RPAREN -fn parseLinkSection(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - _ = eatToken(it, .Keyword_linksection) orelse return null; - _ = try expectToken(it, tree, .LParen); - const expr_node = try expectNode(arena, it, tree, parseExpr, .{ - .ExpectedExpr = .{ .token = it.index }, - }); - _ = try expectToken(it, tree, .RParen); - return expr_node; -} - -/// CallConv <- KEYWORD_callconv LPAREN Expr RPAREN -fn parseCallconv(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - _ = eatToken(it, .Keyword_callconv) orelse return null; - _ = try expectToken(it, tree, .LParen); - const expr_node = try expectNode(arena, it, tree, parseExpr, .{ - .ExpectedExpr = .{ .token = it.index }, - }); - _ = try expectToken(it, tree, .RParen); - return expr_node; -} - -/// ParamDecl <- (KEYWORD_noalias / KEYWORD_comptime)? (IDENTIFIER COLON)? ParamType -fn parseParamDecl(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const doc_comments = try parseDocComment(arena, it, tree); - const noalias_token = eatToken(it, .Keyword_noalias); - const comptime_token = if (noalias_token == null) eatToken(it, .Keyword_comptime) else null; - const name_token = blk: { - const identifier = eatToken(it, .Identifier) orelse break :blk null; - if (eatToken(it, .Colon) != null) break :blk identifier; - putBackToken(it, identifier); // ParamType may also be an identifier - break :blk null; - }; - const param_type = (try parseParamType(arena, it, tree)) orelse { - // Only return cleanly if no keyword, identifier, or doc comment was found - if (noalias_token == null and - comptime_token == null and - name_token == null and - doc_comments == null) return null; - try tree.errors.push(.{ - .ExpectedParamType = .{ .token = it.index }, + // If we've seen "inline", there should have been a "for" or "while" + try p.errors.append(p.gpa, .{ + .ExpectedInlinable = .{ .token = p.tok_i }, }); return error.ParseError; - }; + } - const param_decl = try arena.create(Node.ParamDecl); - param_decl.* = .{ - .doc_comments = doc_comments, - .comptime_token = comptime_token, - .noalias_token = noalias_token, - .name_token = name_token, - .param_type = param_type, - }; - return ¶m_decl.base; -} + /// ForTypeExpr <- ForPrefix TypeExpr (KEYWORD_else TypeExpr)? + fn parseForTypeExpr(p: *Parser) !?*Node { + const node = (try p.parseForPrefix()) orelse return null; + const for_prefix = node.cast(Node.For).?; -/// ParamType -/// <- KEYWORD_var -/// / DOT3 -/// / TypeExpr -fn parseParamType(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?Node.ParamDecl.ParamType { - // TODO cast from tuple to error union is broken - const P = Node.ParamDecl.ParamType; - if (try parseVarType(arena, it, tree)) |node| return P{ .var_type = node }; - if (eatToken(it, .Ellipsis3)) |token| return P{ .var_args = token }; - if (try parseTypeExpr(arena, it, tree)) |node| return P{ .type_expr = node }; - return null; -} - -/// IfPrefix <- KEYWORD_if LPAREN Expr RPAREN PtrPayload? -fn parseIfPrefix(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const if_token = eatToken(it, .Keyword_if) orelse return null; - _ = try expectToken(it, tree, .LParen); - const condition = try expectNode(arena, it, tree, parseExpr, .{ - .ExpectedExpr = .{ .token = it.index }, - }); - _ = try expectToken(it, tree, .RParen); - const payload = try parsePtrPayload(arena, it, tree); - - const node = try arena.create(Node.If); - node.* = .{ - .if_token = if_token, - .condition = condition, - .payload = payload, - .body = undefined, // set by caller - .@"else" = null, - }; - return &node.base; -} - -/// WhilePrefix <- KEYWORD_while LPAREN Expr RPAREN PtrPayload? WhileContinueExpr? -fn parseWhilePrefix(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const while_token = eatToken(it, .Keyword_while) orelse return null; - - _ = try expectToken(it, tree, .LParen); - const condition = try expectNode(arena, it, tree, parseExpr, .{ - .ExpectedExpr = .{ .token = it.index }, - }); - _ = try expectToken(it, tree, .RParen); - - const payload = try parsePtrPayload(arena, it, tree); - const continue_expr = try parseWhileContinueExpr(arena, it, tree); - - const node = try arena.create(Node.While); - node.* = .{ - .label = null, - .inline_token = null, - .while_token = while_token, - .condition = condition, - .payload = payload, - .continue_expr = continue_expr, - .body = undefined, // set by caller - .@"else" = null, - }; - return &node.base; -} - -/// ForPrefix <- KEYWORD_for LPAREN Expr RPAREN PtrIndexPayload -fn parseForPrefix(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const for_token = eatToken(it, .Keyword_for) orelse return null; - - _ = try expectToken(it, tree, .LParen); - const array_expr = try expectNode(arena, it, tree, parseExpr, .{ - .ExpectedExpr = .{ .token = it.index }, - }); - _ = try expectToken(it, tree, .RParen); - - const payload = try expectNode(arena, it, tree, parsePtrIndexPayload, .{ - .ExpectedPayload = .{ .token = it.index }, - }); - - const node = try arena.create(Node.For); - node.* = .{ - .label = null, - .inline_token = null, - .for_token = for_token, - .array_expr = array_expr, - .payload = payload, - .body = undefined, // set by caller - .@"else" = null, - }; - return &node.base; -} - -/// Payload <- PIPE IDENTIFIER PIPE -fn parsePayload(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const lpipe = eatToken(it, .Pipe) orelse return null; - const identifier = try expectNode(arena, it, tree, parseIdentifier, .{ - .ExpectedIdentifier = .{ .token = it.index }, - }); - const rpipe = try expectToken(it, tree, .Pipe); - - const node = try arena.create(Node.Payload); - node.* = .{ - .lpipe = lpipe, - .error_symbol = identifier, - .rpipe = rpipe, - }; - return &node.base; -} - -/// PtrPayload <- PIPE ASTERISK? IDENTIFIER PIPE -fn parsePtrPayload(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const lpipe = eatToken(it, .Pipe) orelse return null; - const asterisk = eatToken(it, .Asterisk); - const identifier = try expectNode(arena, it, tree, parseIdentifier, .{ - .ExpectedIdentifier = .{ .token = it.index }, - }); - const rpipe = try expectToken(it, tree, .Pipe); - - const node = try arena.create(Node.PointerPayload); - node.* = .{ - .lpipe = lpipe, - .ptr_token = asterisk, - .value_symbol = identifier, - .rpipe = rpipe, - }; - return &node.base; -} - -/// PtrIndexPayload <- PIPE ASTERISK? IDENTIFIER (COMMA IDENTIFIER)? PIPE -fn parsePtrIndexPayload(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const lpipe = eatToken(it, .Pipe) orelse return null; - const asterisk = eatToken(it, .Asterisk); - const identifier = try expectNode(arena, it, tree, parseIdentifier, .{ - .ExpectedIdentifier = .{ .token = it.index }, - }); - - const index = if (eatToken(it, .Comma) == null) - null - else - try expectNode(arena, it, tree, parseIdentifier, .{ - .ExpectedIdentifier = .{ .token = it.index }, + const type_expr = try p.expectNode(parseTypeExpr, .{ + .ExpectedTypeExpr = .{ .token = p.tok_i }, }); + for_prefix.body = type_expr; - const rpipe = try expectToken(it, tree, .Pipe); - - const node = try arena.create(Node.PointerIndexPayload); - node.* = .{ - .lpipe = lpipe, - .ptr_token = asterisk, - .value_symbol = identifier, - .index_symbol = index, - .rpipe = rpipe, - }; - return &node.base; -} - -/// SwitchProng <- SwitchCase EQUALRARROW PtrPayload? AssignExpr -fn parseSwitchProng(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const node = (try parseSwitchCase(arena, it, tree)) orelse return null; - const arrow = try expectToken(it, tree, .EqualAngleBracketRight); - const payload = try parsePtrPayload(arena, it, tree); - const expr = try expectNode(arena, it, tree, parseAssignExpr, .{ - .ExpectedExprOrAssignment = .{ .token = it.index }, - }); - - const switch_case = node.cast(Node.SwitchCase).?; - switch_case.arrow_token = arrow; - switch_case.payload = payload; - switch_case.expr = expr; - - return node; -} - -/// SwitchCase -/// <- SwitchItem (COMMA SwitchItem)* COMMA? -/// / KEYWORD_else -fn parseSwitchCase(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - var list = Node.SwitchCase.ItemList.init(arena); - - if (try parseSwitchItem(arena, it, tree)) |first_item| { - try list.push(first_item); - while (eatToken(it, .Comma) != null) { - const next_item = (try parseSwitchItem(arena, it, tree)) orelse break; - try list.push(next_item); - } - } else if (eatToken(it, .Keyword_else)) |else_token| { - const else_node = try arena.create(Node.SwitchElse); - else_node.* = .{ - .token = else_token, - }; - try list.push(&else_node.base); - } else return null; - - const node = try arena.create(Node.SwitchCase); - node.* = .{ - .items = list, - .arrow_token = undefined, // set by caller - .payload = null, - .expr = undefined, // set by caller - }; - return &node.base; -} - -/// SwitchItem <- Expr (DOT3 Expr)? -fn parseSwitchItem(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const expr = (try parseExpr(arena, it, tree)) orelse return null; - if (eatToken(it, .Ellipsis3)) |token| { - const range_end = try expectNode(arena, it, tree, parseExpr, .{ - .ExpectedExpr = .{ .token = it.index }, - }); - - const node = try arena.create(Node.InfixOp); - node.* = .{ - .op_token = token, - .lhs = expr, - .op = .Range, - .rhs = range_end, - }; - return &node.base; - } - return expr; -} - -/// AssignOp -/// <- ASTERISKEQUAL -/// / SLASHEQUAL -/// / PERCENTEQUAL -/// / PLUSEQUAL -/// / MINUSEQUAL -/// / LARROW2EQUAL -/// / RARROW2EQUAL -/// / AMPERSANDEQUAL -/// / CARETEQUAL -/// / PIPEEQUAL -/// / ASTERISKPERCENTEQUAL -/// / PLUSPERCENTEQUAL -/// / MINUSPERCENTEQUAL -/// / EQUAL -fn parseAssignOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const token = nextToken(it); - const op: Node.InfixOp.Op = switch (token.ptr.id) { - .AsteriskEqual => .AssignMul, - .SlashEqual => .AssignDiv, - .PercentEqual => .AssignMod, - .PlusEqual => .AssignAdd, - .MinusEqual => .AssignSub, - .AngleBracketAngleBracketLeftEqual => .AssignBitShiftLeft, - .AngleBracketAngleBracketRightEqual => .AssignBitShiftRight, - .AmpersandEqual => .AssignBitAnd, - .CaretEqual => .AssignBitXor, - .PipeEqual => .AssignBitOr, - .AsteriskPercentEqual => .AssignMulWrap, - .PlusPercentEqual => .AssignAddWrap, - .MinusPercentEqual => .AssignSubWrap, - .Equal => .Assign, - else => { - putBackToken(it, token.index); - return null; - }, - }; - - const node = try arena.create(Node.InfixOp); - node.* = .{ - .op_token = token.index, - .lhs = undefined, // set by caller - .op = op, - .rhs = undefined, // set by caller - }; - return &node.base; -} - -/// CompareOp -/// <- EQUALEQUAL -/// / EXCLAMATIONMARKEQUAL -/// / LARROW -/// / RARROW -/// / LARROWEQUAL -/// / RARROWEQUAL -fn parseCompareOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const token = nextToken(it); - const op: Node.InfixOp.Op = switch (token.ptr.id) { - .EqualEqual => .EqualEqual, - .BangEqual => .BangEqual, - .AngleBracketLeft => .LessThan, - .AngleBracketRight => .GreaterThan, - .AngleBracketLeftEqual => .LessOrEqual, - .AngleBracketRightEqual => .GreaterOrEqual, - else => { - putBackToken(it, token.index); - return null; - }, - }; - - return try createInfixOp(arena, token.index, op); -} - -/// BitwiseOp -/// <- AMPERSAND -/// / CARET -/// / PIPE -/// / KEYWORD_orelse -/// / KEYWORD_catch Payload? -fn parseBitwiseOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const token = nextToken(it); - const op: Node.InfixOp.Op = switch (token.ptr.id) { - .Ampersand => .BitAnd, - .Caret => .BitXor, - .Pipe => .BitOr, - .Keyword_orelse => .UnwrapOptional, - .Keyword_catch => .{ .Catch = try parsePayload(arena, it, tree) }, - else => { - putBackToken(it, token.index); - return null; - }, - }; - - return try createInfixOp(arena, token.index, op); -} - -/// BitShiftOp -/// <- LARROW2 -/// / RARROW2 -fn parseBitShiftOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const token = nextToken(it); - const op: Node.InfixOp.Op = switch (token.ptr.id) { - .AngleBracketAngleBracketLeft => .BitShiftLeft, - .AngleBracketAngleBracketRight => .BitShiftRight, - else => { - putBackToken(it, token.index); - return null; - }, - }; - - return try createInfixOp(arena, token.index, op); -} - -/// AdditionOp -/// <- PLUS -/// / MINUS -/// / PLUS2 -/// / PLUSPERCENT -/// / MINUSPERCENT -fn parseAdditionOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const token = nextToken(it); - const op: Node.InfixOp.Op = switch (token.ptr.id) { - .Plus => .Add, - .Minus => .Sub, - .PlusPlus => .ArrayCat, - .PlusPercent => .AddWrap, - .MinusPercent => .SubWrap, - else => { - putBackToken(it, token.index); - return null; - }, - }; - - return try createInfixOp(arena, token.index, op); -} - -/// MultiplyOp -/// <- PIPE2 -/// / ASTERISK -/// / SLASH -/// / PERCENT -/// / ASTERISK2 -/// / ASTERISKPERCENT -fn parseMultiplyOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const token = nextToken(it); - const op: Node.InfixOp.Op = switch (token.ptr.id) { - .PipePipe => .MergeErrorSets, - .Asterisk => .Mul, - .Slash => .Div, - .Percent => .Mod, - .AsteriskAsterisk => .ArrayMult, - .AsteriskPercent => .MulWrap, - else => { - putBackToken(it, token.index); - return null; - }, - }; - - return try createInfixOp(arena, token.index, op); -} - -/// PrefixOp -/// <- EXCLAMATIONMARK -/// / MINUS -/// / TILDE -/// / MINUSPERCENT -/// / AMPERSAND -/// / KEYWORD_try -/// / KEYWORD_await -fn parsePrefixOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const token = nextToken(it); - const op: Node.PrefixOp.Op = switch (token.ptr.id) { - .Bang => .BoolNot, - .Minus => .Negation, - .Tilde => .BitNot, - .MinusPercent => .NegationWrap, - .Ampersand => .AddressOf, - .Keyword_try => .Try, - .Keyword_await => .Await, - else => { - putBackToken(it, token.index); - return null; - }, - }; - - const node = try arena.create(Node.PrefixOp); - node.* = .{ - .op_token = token.index, - .op = op, - .rhs = undefined, // set by caller - }; - return &node.base; -} - -// TODO: ArrayTypeStart is either an array or a slice, but const/allowzero only work on -// pointers. Consider updating this rule: -// ... -// / ArrayTypeStart -// / SliceTypeStart (ByteAlign / KEYWORD_const / KEYWORD_volatile / KEYWORD_allowzero)* -// / PtrTypeStart ... - -/// PrefixTypeOp -/// <- QUESTIONMARK -/// / KEYWORD_anyframe MINUSRARROW -/// / ArrayTypeStart (ByteAlign / KEYWORD_const / KEYWORD_volatile / KEYWORD_allowzero)* -/// / PtrTypeStart (KEYWORD_align LPAREN Expr (COLON INTEGER COLON INTEGER)? RPAREN / KEYWORD_const / KEYWORD_volatile / KEYWORD_allowzero)* -fn parsePrefixTypeOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - if (eatToken(it, .QuestionMark)) |token| { - const node = try arena.create(Node.PrefixOp); - node.* = .{ - .op_token = token, - .op = .OptionalType, - .rhs = undefined, // set by caller - }; - return &node.base; - } - - // TODO: Returning a AnyFrameType instead of PrefixOp makes casting and setting .rhs or - // .return_type more difficult for the caller (see parsePrefixOpExpr helper). - // Consider making the AnyFrameType a member of PrefixOp and add a - // PrefixOp.AnyFrameType variant? - if (eatToken(it, .Keyword_anyframe)) |token| { - const arrow = eatToken(it, .Arrow) orelse { - putBackToken(it, token); - return null; - }; - const node = try arena.create(Node.AnyFrameType); - node.* = .{ - .anyframe_token = token, - .result = .{ - .arrow_token = arrow, - .return_type = undefined, // set by caller - }, - }; - return &node.base; - } - - if (try parsePtrTypeStart(arena, it, tree)) |node| { - // If the token encountered was **, there will be two nodes instead of one. - // The attributes should be applied to the rightmost operator. - const prefix_op = node.cast(Node.PrefixOp).?; - var ptr_info = if (tree.tokens.at(prefix_op.op_token).id == .AsteriskAsterisk) - &prefix_op.rhs.cast(Node.PrefixOp).?.op.PtrType - else - &prefix_op.op.PtrType; - - while (true) { - if (eatToken(it, .Keyword_align)) |align_token| { - const lparen = try expectToken(it, tree, .LParen); - const expr_node = try expectNode(arena, it, tree, parseExpr, .{ - .ExpectedExpr = .{ .token = it.index }, - }); - - // Optional bit range - const bit_range = if (eatToken(it, .Colon)) |_| bit_range_value: { - const range_start = try expectNode(arena, it, tree, parseIntegerLiteral, .{ - .ExpectedIntegerLiteral = .{ .token = it.index }, - }); - _ = try expectToken(it, tree, .Colon); - const range_end = try expectNode(arena, it, tree, parseIntegerLiteral, .{ - .ExpectedIntegerLiteral = .{ .token = it.index }, - }); - - break :bit_range_value Node.PrefixOp.PtrInfo.Align.BitRange{ - .start = range_start, - .end = range_end, - }; - } else null; - _ = try expectToken(it, tree, .RParen); - - if (ptr_info.align_info != null) { - try tree.errors.push(.{ - .ExtraAlignQualifier = .{ .token = it.index - 1 }, - }); - continue; - } - - ptr_info.align_info = Node.PrefixOp.PtrInfo.Align{ - .node = expr_node, - .bit_range = bit_range, - }; - - continue; - } - if (eatToken(it, .Keyword_const)) |const_token| { - if (ptr_info.const_token != null) { - try tree.errors.push(.{ - .ExtraConstQualifier = .{ .token = it.index - 1 }, - }); - continue; - } - ptr_info.const_token = const_token; - continue; - } - if (eatToken(it, .Keyword_volatile)) |volatile_token| { - if (ptr_info.volatile_token != null) { - try tree.errors.push(.{ - .ExtraVolatileQualifier = .{ .token = it.index - 1 }, - }); - continue; - } - ptr_info.volatile_token = volatile_token; - continue; - } - if (eatToken(it, .Keyword_allowzero)) |allowzero_token| { - if (ptr_info.allowzero_token != null) { - try tree.errors.push(.{ - .ExtraAllowZeroQualifier = .{ .token = it.index - 1 }, - }); - continue; - } - ptr_info.allowzero_token = allowzero_token; - continue; - } - break; - } - - return node; - } - - if (try parseArrayTypeStart(arena, it, tree)) |node| { - switch (node.cast(Node.PrefixOp).?.op) { - .ArrayType => {}, - .SliceType => |*slice_type| { - // Collect pointer qualifiers in any order, but disallow duplicates - while (true) { - if (try parseByteAlign(arena, it, tree)) |align_expr| { - if (slice_type.align_info != null) { - try tree.errors.push(.{ - .ExtraAlignQualifier = .{ .token = it.index - 1 }, - }); - continue; - } - slice_type.align_info = Node.PrefixOp.PtrInfo.Align{ - .node = align_expr, - .bit_range = null, - }; - continue; - } - if (eatToken(it, .Keyword_const)) |const_token| { - if (slice_type.const_token != null) { - try tree.errors.push(.{ - .ExtraConstQualifier = .{ .token = it.index - 1 }, - }); - continue; - } - slice_type.const_token = const_token; - continue; - } - if (eatToken(it, .Keyword_volatile)) |volatile_token| { - if (slice_type.volatile_token != null) { - try tree.errors.push(.{ - .ExtraVolatileQualifier = .{ .token = it.index - 1 }, - }); - continue; - } - slice_type.volatile_token = volatile_token; - continue; - } - if (eatToken(it, .Keyword_allowzero)) |allowzero_token| { - if (slice_type.allowzero_token != null) { - try tree.errors.push(.{ - .ExtraAllowZeroQualifier = .{ .token = it.index - 1 }, - }); - continue; - } - slice_type.allowzero_token = allowzero_token; - continue; - } - break; - } - }, - else => unreachable, - } - return node; - } - - return null; -} - -/// SuffixOp -/// <- LBRACKET Expr (DOT2 (Expr (COLON Expr)?)?)? RBRACKET -/// / DOT IDENTIFIER -/// / DOTASTERISK -/// / DOTQUESTIONMARK -fn parseSuffixOp(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const OpAndToken = struct { - op: Node.SuffixOp.Op, - token: TokenIndex, - }; - const op_and_token: OpAndToken = blk: { - if (eatToken(it, .LBracket)) |_| { - const index_expr = try expectNode(arena, it, tree, parseExpr, .{ - .ExpectedExpr = .{ .token = it.index }, + if (p.eatToken(.Keyword_else)) |else_token| { + const else_expr = try p.expectNode(parseTypeExpr, .{ + .ExpectedTypeExpr = .{ .token = p.tok_i }, }); - if (eatToken(it, .Ellipsis2) != null) { - const end_expr = try parseExpr(arena, it, tree); - const sentinel: ?*ast.Node = if (eatToken(it, .Colon) != null) - try parseExpr(arena, it, tree) - else - null; - break :blk .{ - .op = .{ - .Slice = .{ - .start = index_expr, - .end = end_expr, - .sentinel = sentinel, - }, - }, - .token = try expectToken(it, tree, .RBracket), - }; - } - - break :blk .{ - .op = .{ .ArrayAccess = index_expr }, - .token = try expectToken(it, tree, .RBracket), + const else_node = try p.arena.allocator.create(Node.Else); + else_node.* = .{ + .else_token = else_token, + .payload = null, + .body = else_expr, }; + + for_prefix.@"else" = else_node; } - if (eatToken(it, .PeriodAsterisk)) |period_asterisk| { - break :blk .{ .op = .Deref, .token = period_asterisk }; - } + return node; + } - if (eatToken(it, .Period)) |period| { - if (try parseIdentifier(arena, it, tree)) |identifier| { - // TODO: It's a bit weird to return an InfixOp from the SuffixOp parser. - // Should there be an ast.Node.SuffixOp.FieldAccess variant? Or should - // this grammar rule be altered? - const node = try arena.create(Node.InfixOp); - node.* = .{ - .op_token = period, - .lhs = undefined, // set by caller - .op = .Period, - .rhs = identifier, - }; - return &node.base; - } - if (eatToken(it, .QuestionMark)) |question_mark| { - break :blk .{ .op = .UnwrapOptional, .token = question_mark }; - } - try tree.errors.push(.{ - .ExpectedSuffixOp = .{ .token = it.index }, + /// WhileTypeExpr <- WhilePrefix TypeExpr (KEYWORD_else Payload? TypeExpr)? + fn parseWhileTypeExpr(p: *Parser) !?*Node { + const node = (try p.parseWhilePrefix()) orelse return null; + const while_prefix = node.cast(Node.While).?; + + const type_expr = try p.expectNode(parseTypeExpr, .{ + .ExpectedTypeExpr = .{ .token = p.tok_i }, + }); + while_prefix.body = type_expr; + + if (p.eatToken(.Keyword_else)) |else_token| { + const payload = try p.parsePayload(); + + const else_expr = try p.expectNode(parseTypeExpr, .{ + .ExpectedTypeExpr = .{ .token = p.tok_i }, }); - return null; + + const else_node = try p.arena.allocator.create(Node.Else); + else_node.* = .{ + .else_token = else_token, + .payload = null, + .body = else_expr, + }; + + while_prefix.@"else" = else_node; } - return null; - }; + return node; + } - const node = try arena.create(Node.SuffixOp); - node.* = .{ - .lhs = undefined, // set by caller - .op = op_and_token.op, - .rtoken = op_and_token.token, - }; - return &node.base; -} + /// SwitchExpr <- KEYWORD_switch LPAREN Expr RPAREN LBRACE SwitchProngList RBRACE + fn parseSwitchExpr(p: *Parser) !?*Node { + const switch_token = p.eatToken(.Keyword_switch) orelse return null; + _ = try p.expectToken(.LParen); + const expr_node = try p.expectNode(parseExpr, .{ + .ExpectedExpr = .{ .token = p.tok_i }, + }); + _ = try p.expectToken(.RParen); + _ = try p.expectToken(.LBrace); + const cases = try p.parseSwitchProngList(); + const rbrace = try p.expectToken(.RBrace); -/// FnCallArguments <- LPAREN ExprList RPAREN -/// ExprList <- (Expr COMMA)* Expr? -fn parseFnCallArguments(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?AnnotatedParamList { - if (eatToken(it, .LParen) == null) return null; - const list = try ListParseFn(Node.FnProto.ParamList, parseExpr)(arena, it, tree); - const rparen = try expectToken(it, tree, .RParen); - return AnnotatedParamList{ .list = list, .rparen = rparen }; -} - -const AnnotatedParamList = struct { - list: Node.FnProto.ParamList, // NOTE: may also be any other type SegmentedList(*Node, 2) - rparen: TokenIndex, -}; - -/// ArrayTypeStart <- LBRACKET Expr? RBRACKET -fn parseArrayTypeStart(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const lbracket = eatToken(it, .LBracket) orelse return null; - const expr = try parseExpr(arena, it, tree); - const sentinel = if (eatToken(it, .Colon)) |_| - try expectNode(arena, it, tree, parseExpr, .{ - .ExpectedExpr = .{ .token = it.index }, - }) - else - null; - const rbracket = try expectToken(it, tree, .RBracket); - - const op: Node.PrefixOp.Op = if (expr) |len_expr| - .{ - .ArrayType = .{ - .len_expr = len_expr, - .sentinel = sentinel, - }, - } - else - .{ - .SliceType = Node.PrefixOp.PtrInfo{ - .allowzero_token = null, - .align_info = null, - .const_token = null, - .volatile_token = null, - .sentinel = sentinel, - }, - }; - - const node = try arena.create(Node.PrefixOp); - node.* = .{ - .op_token = lbracket, - .op = op, - .rhs = undefined, // set by caller - }; - return &node.base; -} - -/// PtrTypeStart -/// <- ASTERISK -/// / ASTERISK2 -/// / PTRUNKNOWN -/// / PTRC -fn parsePtrTypeStart(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - if (eatToken(it, .Asterisk)) |asterisk| { - const sentinel = if (eatToken(it, .Colon)) |_| - try expectNode(arena, it, tree, parseExpr, .{ - .ExpectedExpr = .{ .token = it.index }, - }) - else - null; - const node = try arena.create(Node.PrefixOp); + const node = try p.arena.allocator.create(Node.Switch); node.* = .{ - .op_token = asterisk, - .op = .{ .PtrType = .{ .sentinel = sentinel } }, - .rhs = undefined, // set by caller + .switch_token = switch_token, + .expr = expr_node, + .cases = cases, + .rbrace = rbrace, }; return &node.base; } - if (eatToken(it, .AsteriskAsterisk)) |double_asterisk| { - const node = try arena.create(Node.PrefixOp); + /// AsmExpr <- KEYWORD_asm KEYWORD_volatile? LPAREN Expr AsmOutput? RPAREN + fn parseAsmExpr(p: *Parser) !?*Node { + const asm_token = p.eatToken(.Keyword_asm) orelse return null; + const volatile_token = p.eatToken(.Keyword_volatile); + _ = try p.expectToken(.LParen); + const template = try p.expectNode(parseExpr, .{ + .ExpectedExpr = .{ .token = p.tok_i }, + }); + + const node = try p.arena.allocator.create(Node.Asm); node.* = .{ - .op_token = double_asterisk, - .op = .{ .PtrType = .{} }, - .rhs = undefined, // set by caller + .asm_token = asm_token, + .volatile_token = volatile_token, + .template = template, + .outputs = Node.Asm.OutputList{}, + .inputs = Node.Asm.InputList{}, + .clobbers = Node.Asm.ClobberList{}, + .rparen = undefined, }; - // Special case for **, which is its own token - const child = try arena.create(Node.PrefixOp); - child.* = .{ - .op_token = double_asterisk, - .op = .{ .PtrType = .{} }, - .rhs = undefined, // set by caller - }; - node.rhs = &child.base; - + try p.parseAsmOutput(node); + node.rparen = try p.expectToken(.RParen); return &node.base; } - if (eatToken(it, .LBracket)) |lbracket| { - const asterisk = eatToken(it, .Asterisk) orelse { - putBackToken(it, lbracket); - return null; - }; - if (eatToken(it, .Identifier)) |ident| { - if (!std.mem.eql(u8, tree.tokenSlice(ident), "c")) { - putBackToken(it, ident); - } else { - _ = try expectToken(it, tree, .RBracket); - const node = try arena.create(Node.PrefixOp); - node.* = .{ - .op_token = lbracket, - .op = .{ .PtrType = .{} }, - .rhs = undefined, // set by caller - }; - return &node.base; - } - } - const sentinel = if (eatToken(it, .Colon)) |_| - try expectNode(arena, it, tree, parseExpr, .{ - .ExpectedExpr = .{ .token = it.index }, - }) - else - null; - _ = try expectToken(it, tree, .RBracket); - const node = try arena.create(Node.PrefixOp); - node.* = .{ - .op_token = lbracket, - .op = .{ .PtrType = .{ .sentinel = sentinel } }, - .rhs = undefined, // set by caller - }; - return &node.base; - } - return null; -} -/// ContainerDeclAuto <- ContainerDeclType LBRACE ContainerMembers RBRACE -fn parseContainerDeclAuto(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const node = (try parseContainerDeclType(arena, it, tree)) orelse return null; - const lbrace = try expectToken(it, tree, .LBrace); - const members = try parseContainerMembers(arena, it, tree, false); - const rbrace = try expectToken(it, tree, .RBrace); + /// DOT IDENTIFIER + fn parseAnonLiteral(p: *Parser) !?*Node { + const dot = p.eatToken(.Period) orelse return null; - const decl_type = node.cast(Node.ContainerDecl).?; - decl_type.fields_and_decls = members; - decl_type.lbrace_token = lbrace; - decl_type.rbrace_token = rbrace; - - return node; -} - -/// ContainerDeclType -/// <- KEYWORD_struct -/// / KEYWORD_enum (LPAREN Expr RPAREN)? -/// / KEYWORD_union (LPAREN (KEYWORD_enum (LPAREN Expr RPAREN)? / Expr) RPAREN)? -fn parseContainerDeclType(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const kind_token = nextToken(it); - - const init_arg_expr = switch (kind_token.ptr.id) { - .Keyword_struct => Node.ContainerDecl.InitArg{ .None = {} }, - .Keyword_enum => blk: { - if (eatToken(it, .LParen) != null) { - const expr = try expectNode(arena, it, tree, parseExpr, .{ - .ExpectedExpr = .{ .token = it.index }, - }); - _ = try expectToken(it, tree, .RParen); - break :blk Node.ContainerDecl.InitArg{ .Type = expr }; - } - break :blk Node.ContainerDecl.InitArg{ .None = {} }; - }, - .Keyword_union => blk: { - if (eatToken(it, .LParen) != null) { - if (eatToken(it, .Keyword_enum) != null) { - if (eatToken(it, .LParen) != null) { - const expr = try expectNode(arena, it, tree, parseExpr, .{ - .ExpectedExpr = .{ .token = it.index }, - }); - _ = try expectToken(it, tree, .RParen); - _ = try expectToken(it, tree, .RParen); - break :blk Node.ContainerDecl.InitArg{ .Enum = expr }; - } - _ = try expectToken(it, tree, .RParen); - break :blk Node.ContainerDecl.InitArg{ .Enum = null }; - } - const expr = try expectNode(arena, it, tree, parseExpr, .{ - .ExpectedExpr = .{ .token = it.index }, - }); - _ = try expectToken(it, tree, .RParen); - break :blk Node.ContainerDecl.InitArg{ .Type = expr }; - } - break :blk Node.ContainerDecl.InitArg{ .None = {} }; - }, - else => { - putBackToken(it, kind_token.index); - return null; - }, - }; - - const node = try arena.create(Node.ContainerDecl); - node.* = .{ - .layout_token = null, - .kind_token = kind_token.index, - .init_arg_expr = init_arg_expr, - .fields_and_decls = undefined, // set by caller - .lbrace_token = undefined, // set by caller - .rbrace_token = undefined, // set by caller - }; - return &node.base; -} - -/// ByteAlign <- KEYWORD_align LPAREN Expr RPAREN -fn parseByteAlign(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - _ = eatToken(it, .Keyword_align) orelse return null; - _ = try expectToken(it, tree, .LParen); - const expr = try expectNode(arena, it, tree, parseExpr, .{ - .ExpectedExpr = .{ .token = it.index }, - }); - _ = try expectToken(it, tree, .RParen); - return expr; -} - -/// IdentifierList <- (IDENTIFIER COMMA)* IDENTIFIER? -/// Only ErrorSetDecl parses an IdentifierList -fn parseErrorTagList(arena: *Allocator, it: *TokenIterator, tree: *Tree) !Node.ErrorSetDecl.DeclList { - return try ListParseFn(Node.ErrorSetDecl.DeclList, parseErrorTag)(arena, it, tree); -} - -/// SwitchProngList <- (SwitchProng COMMA)* SwitchProng? -fn parseSwitchProngList(arena: *Allocator, it: *TokenIterator, tree: *Tree) !Node.Switch.CaseList { - return try ListParseFn(Node.Switch.CaseList, parseSwitchProng)(arena, it, tree); -} - -/// AsmOutputList <- (AsmOutputItem COMMA)* AsmOutputItem? -fn parseAsmOutputList(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!Node.Asm.OutputList { - return try ListParseFn(Node.Asm.OutputList, parseAsmOutputItem)(arena, it, tree); -} - -/// AsmInputList <- (AsmInputItem COMMA)* AsmInputItem? -fn parseAsmInputList(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!Node.Asm.InputList { - return try ListParseFn(Node.Asm.InputList, parseAsmInputItem)(arena, it, tree); -} - -/// ParamDeclList <- (ParamDecl COMMA)* ParamDecl? -fn parseParamDeclList(arena: *Allocator, it: *TokenIterator, tree: *Tree) !Node.FnProto.ParamList { - return try ListParseFn(Node.FnProto.ParamList, parseParamDecl)(arena, it, tree); -} - -fn ParseFn(comptime T: type) type { - return fn (*Allocator, *TokenIterator, *Tree) Error!T; -} - -const NodeParseFn = fn (*Allocator, *TokenIterator, *Tree) Error!?*Node; - -fn ListParseFn(comptime L: type, comptime nodeParseFn: var) ParseFn(L) { - return struct { - pub fn parse(arena: *Allocator, it: *TokenIterator, tree: *Tree) !L { - var list = L.init(arena); - while (try nodeParseFn(arena, it, tree)) |node| { - try list.push(node); - - switch (it.peek().?.id) { - .Comma => _ = nextToken(it), - // all possible delimiters - .Colon, .RParen, .RBrace, .RBracket => break, - else => { - // this is likely just a missing comma, - // continue parsing this list and give an error - try tree.errors.push(.{ - .ExpectedToken = .{ .token = it.index, .expected_id = .Comma }, - }); - }, - } - } - return list; - } - }.parse; -} - -fn SimpleBinOpParseFn(comptime token: Token.Id, comptime op: Node.InfixOp.Op) NodeParseFn { - return struct { - pub fn parse(arena: *Allocator, it: *TokenIterator, tree: *Tree) Error!?*Node { - const op_token = if (token == .Keyword_and) switch (it.peek().?.id) { - .Keyword_and => nextToken(it).index, - .Invalid_ampersands => blk: { - try tree.errors.push(.{ - .InvalidAnd = .{ .token = it.index }, - }); - break :blk nextToken(it).index; - }, - else => return null, - } else eatToken(it, token) orelse return null; - - const node = try arena.create(Node.InfixOp); + // anon enum literal + if (p.eatToken(.Identifier)) |name| { + const node = try p.arena.allocator.create(Node.EnumLiteral); node.* = .{ - .op_token = op_token, - .lhs = undefined, // set by caller - .op = op, + .dot = dot, + .name = name, + }; + return &node.base; + } + + // anon container literal + if (try p.parseInitList()) |node| { + node.lhs = .{ .dot = dot }; + return &node.base; + } + + p.putBackToken(dot); + return null; + } + + /// AsmOutput <- COLON AsmOutputList AsmInput? + fn parseAsmOutput(p: *Parser, asm_node: *Node.Asm) !void { + if (p.eatToken(.Colon) == null) return; + asm_node.outputs = try p.parseAsmOutputList(); + try p.parseAsmInput(asm_node); + } + + /// AsmOutputItem <- LBRACKET IDENTIFIER RBRACKET STRINGLITERAL LPAREN (MINUSRARROW TypeExpr / IDENTIFIER) RPAREN + fn parseAsmOutputItem(p: *Parser) !?*Node.AsmOutput { + const lbracket = p.eatToken(.LBracket) orelse return null; + const name = try p.expectNode(parseIdentifier, .{ + .ExpectedIdentifier = .{ .token = p.tok_i }, + }); + _ = try p.expectToken(.RBracket); + + const constraint = try p.expectNode(parseStringLiteral, .{ + .ExpectedStringLiteral = .{ .token = p.tok_i }, + }); + + _ = try p.expectToken(.LParen); + const kind: Node.AsmOutput.Kind = blk: { + if (p.eatToken(.Arrow) != null) { + const return_ident = try p.expectNode(parseTypeExpr, .{ + .ExpectedTypeExpr = .{ .token = p.tok_i }, + }); + break :blk .{ .Return = return_ident }; + } + const variable = try p.expectNode(parseIdentifier, .{ + .ExpectedIdentifier = .{ .token = p.tok_i }, + }); + break :blk .{ .Variable = variable.cast(Node.Identifier).? }; + }; + const rparen = try p.expectToken(.RParen); + + const node = try p.arena.allocator.create(Node.AsmOutput); + node.* = .{ + .lbracket = lbracket, + .symbolic_name = name, + .constraint = constraint, + .kind = kind, + .rparen = rparen, + }; + return node; + } + + /// AsmInput <- COLON AsmInputList AsmClobbers? + fn parseAsmInput(p: *Parser, asm_node: *Node.Asm) !void { + if (p.eatToken(.Colon) == null) return; + asm_node.inputs = try p.parseAsmInputList(); + try p.parseAsmClobbers(asm_node); + } + + /// AsmInputItem <- LBRACKET IDENTIFIER RBRACKET STRINGLITERAL LPAREN Expr RPAREN + fn parseAsmInputItem(p: *Parser) !?*Node.AsmInput { + const lbracket = p.eatToken(.LBracket) orelse return null; + const name = try p.expectNode(parseIdentifier, .{ + .ExpectedIdentifier = .{ .token = p.tok_i }, + }); + _ = try p.expectToken(.RBracket); + + const constraint = try p.expectNode(parseStringLiteral, .{ + .ExpectedStringLiteral = .{ .token = p.tok_i }, + }); + + _ = try p.expectToken(.LParen); + const expr = try p.expectNode(parseExpr, .{ + .ExpectedExpr = .{ .token = p.tok_i }, + }); + const rparen = try p.expectToken(.RParen); + + const node = try p.arena.allocator.create(Node.AsmInput); + node.* = .{ + .lbracket = lbracket, + .symbolic_name = name, + .constraint = constraint, + .expr = expr, + .rparen = rparen, + }; + return node; + } + + /// AsmClobbers <- COLON StringList + /// StringList <- (STRINGLITERAL COMMA)* STRINGLITERAL? + fn parseAsmClobbers(p: *Parser, asm_node: *Node.Asm) !void { + if (p.eatToken(.Colon) == null) return; + asm_node.clobbers = try ListParseFn( + Node.Asm.ClobberList, + parseStringLiteral, + )(p); + } + + /// BreakLabel <- COLON IDENTIFIER + fn parseBreakLabel(p: *Parser) !?*Node { + _ = p.eatToken(.Colon) orelse return null; + return p.expectNode(parseIdentifier, .{ + .ExpectedIdentifier = .{ .token = p.tok_i }, + }); + } + + /// BlockLabel <- IDENTIFIER COLON + fn parseBlockLabel(p: *Parser, colon_token: *TokenIndex) ?TokenIndex { + const identifier = p.eatToken(.Identifier) orelse return null; + if (p.eatToken(.Colon)) |colon| { + colon_token.* = colon; + return identifier; + } + p.putBackToken(identifier); + return null; + } + + /// FieldInit <- DOT IDENTIFIER EQUAL Expr + fn parseFieldInit(p: *Parser) !?*Node { + const period_token = p.eatToken(.Period) orelse return null; + const name_token = p.eatToken(.Identifier) orelse { + // Because of anon literals `.{` is also valid. + p.putBackToken(period_token); + return null; + }; + const eq_token = p.eatToken(.Equal) orelse { + // `.Name` may also be an enum literal, which is a later rule. + p.putBackToken(name_token); + p.putBackToken(period_token); + return null; + }; + const expr_node = try p.expectNode(parseExpr, .{ + .ExpectedExpr = .{ .token = p.tok_i }, + }); + + const node = try p.arena.allocator.create(Node.FieldInitializer); + node.* = .{ + .period_token = period_token, + .name_token = name_token, + .expr = expr_node, + }; + return &node.base; + } + + /// WhileContinueExpr <- COLON LPAREN AssignExpr RPAREN + fn parseWhileContinueExpr(p: *Parser) !?*Node { + _ = p.eatToken(.Colon) orelse return null; + _ = try p.expectToken(.LParen); + const node = try p.expectNode(parseAssignExpr, .{ + .ExpectedExprOrAssignment = .{ .token = p.tok_i }, + }); + _ = try p.expectToken(.RParen); + return node; + } + + /// LinkSection <- KEYWORD_linksection LPAREN Expr RPAREN + fn parseLinkSection(p: *Parser) !?*Node { + _ = p.eatToken(.Keyword_linksection) orelse return null; + _ = try p.expectToken(.LParen); + const expr_node = try p.expectNode(parseExpr, .{ + .ExpectedExpr = .{ .token = p.tok_i }, + }); + _ = try p.expectToken(.RParen); + return expr_node; + } + + /// CallConv <- KEYWORD_callconv LPAREN Expr RPAREN + fn parseCallconv(p: *Parser) !?*Node { + _ = p.eatToken(.Keyword_callconv) orelse return null; + _ = try p.expectToken(.LParen); + const expr_node = try p.expectNode(parseExpr, .{ + .ExpectedExpr = .{ .token = p.tok_i }, + }); + _ = try p.expectToken(.RParen); + return expr_node; + } + + /// ParamDecl <- (KEYWORD_noalias / KEYWORD_comptime)? (IDENTIFIER COLON)? ParamType + fn parseParamDecl(p: *Parser) !?*Node { + const doc_comments = try p.parseDocComment(); + const noalias_token = p.eatToken(.Keyword_noalias); + const comptime_token = if (noalias_token == null) p.eatToken(.Keyword_comptime) else null; + const name_token = blk: { + const identifier = p.eatToken(.Identifier) orelse break :blk null; + if (p.eatToken(.Colon) != null) break :blk identifier; + p.putBackToken(identifier); // ParamType may also be an identifier + break :blk null; + }; + const param_type = (try p.parseParamType()) orelse { + // Only return cleanly if no keyword, identifier, or doc comment was found + if (noalias_token == null and + comptime_token == null and + name_token == null and + doc_comments == null) return null; + try p.errors.append(p.gpa, .{ + .ExpectedParamType = .{ .token = p.tok_i }, + }); + return error.ParseError; + }; + + const param_decl = try p.arena.allocator.create(Node.ParamDecl); + param_decl.* = .{ + .doc_comments = doc_comments, + .comptime_token = comptime_token, + .noalias_token = noalias_token, + .name_token = name_token, + .param_type = param_type, + }; + return ¶m_decl.base; + } + + /// ParamType + /// <- KEYWORD_var + /// / DOT3 + /// / TypeExpr + fn parseParamType(p: *Parser) !?Node.ParamDecl.ParamType { + // TODO cast from tuple to error union is broken + const P = Node.ParamDecl.ParamType; + if (try p.parseVarType()) |node| return P{ .var_type = node }; + if (p.eatToken(.Ellipsis3)) |token| return P{ .var_args = token }; + if (try p.parseTypeExpr()) |node| return P{ .type_expr = node }; + return null; + } + + /// IfPrefix <- KEYWORD_if LPAREN Expr RPAREN PtrPayload? + fn parseIfPrefix(p: *Parser) !?*Node { + const if_token = p.eatToken(.Keyword_if) orelse return null; + _ = try p.expectToken(.LParen); + const condition = try p.expectNode(parseExpr, .{ + .ExpectedExpr = .{ .token = p.tok_i }, + }); + _ = try p.expectToken(.RParen); + const payload = try p.parsePtrPayload(); + + const node = try p.arena.allocator.create(Node.If); + node.* = .{ + .if_token = if_token, + .condition = condition, + .payload = payload, + .body = undefined, // set by caller + .@"else" = null, + }; + return &node.base; + } + + /// WhilePrefix <- KEYWORD_while LPAREN Expr RPAREN PtrPayload? WhileContinueExpr? + fn parseWhilePrefix(p: *Parser) !?*Node { + const while_token = p.eatToken(.Keyword_while) orelse return null; + + _ = try p.expectToken(.LParen); + const condition = try p.expectNode(parseExpr, .{ + .ExpectedExpr = .{ .token = p.tok_i }, + }); + _ = try p.expectToken(.RParen); + + const payload = try p.parsePtrPayload(); + const continue_expr = try p.parseWhileContinueExpr(); + + const node = try p.arena.allocator.create(Node.While); + node.* = .{ + .label = null, + .inline_token = null, + .while_token = while_token, + .condition = condition, + .payload = payload, + .continue_expr = continue_expr, + .body = undefined, // set by caller + .@"else" = null, + }; + return &node.base; + } + + /// ForPrefix <- KEYWORD_for LPAREN Expr RPAREN PtrIndexPayload + fn parseForPrefix(p: *Parser) !?*Node { + const for_token = p.eatToken(.Keyword_for) orelse return null; + + _ = try p.expectToken(.LParen); + const array_expr = try p.expectNode(parseExpr, .{ + .ExpectedExpr = .{ .token = p.tok_i }, + }); + _ = try p.expectToken(.RParen); + + const payload = try p.expectNode(parsePtrIndexPayload, .{ + .ExpectedPayload = .{ .token = p.tok_i }, + }); + + const node = try p.arena.allocator.create(Node.For); + node.* = .{ + .label = null, + .inline_token = null, + .for_token = for_token, + .array_expr = array_expr, + .payload = payload, + .body = undefined, // set by caller + .@"else" = null, + }; + return &node.base; + } + + /// Payload <- PIPE IDENTIFIER PIPE + fn parsePayload(p: *Parser) !?*Node { + const lpipe = p.eatToken(.Pipe) orelse return null; + const identifier = try p.expectNode(parseIdentifier, .{ + .ExpectedIdentifier = .{ .token = p.tok_i }, + }); + const rpipe = try p.expectToken(.Pipe); + + const node = try p.arena.allocator.create(Node.Payload); + node.* = .{ + .lpipe = lpipe, + .error_symbol = identifier, + .rpipe = rpipe, + }; + return &node.base; + } + + /// PtrPayload <- PIPE ASTERISK? IDENTIFIER PIPE + fn parsePtrPayload(p: *Parser) !?*Node { + const lpipe = p.eatToken(.Pipe) orelse return null; + const asterisk = p.eatToken(.Asterisk); + const identifier = try p.expectNode(parseIdentifier, .{ + .ExpectedIdentifier = .{ .token = p.tok_i }, + }); + const rpipe = try p.expectToken(.Pipe); + + const node = try p.arena.allocator.create(Node.PointerPayload); + node.* = .{ + .lpipe = lpipe, + .ptr_token = asterisk, + .value_symbol = identifier, + .rpipe = rpipe, + }; + return &node.base; + } + + /// PtrIndexPayload <- PIPE ASTERISK? IDENTIFIER (COMMA IDENTIFIER)? PIPE + fn parsePtrIndexPayload(p: *Parser) !?*Node { + const lpipe = p.eatToken(.Pipe) orelse return null; + const asterisk = p.eatToken(.Asterisk); + const identifier = try p.expectNode(parseIdentifier, .{ + .ExpectedIdentifier = .{ .token = p.tok_i }, + }); + + const index = if (p.eatToken(.Comma) == null) + null + else + try p.expectNode(parseIdentifier, .{ + .ExpectedIdentifier = .{ .token = p.tok_i }, + }); + + const rpipe = try p.expectToken(.Pipe); + + const node = try p.arena.allocator.create(Node.PointerIndexPayload); + node.* = .{ + .lpipe = lpipe, + .ptr_token = asterisk, + .value_symbol = identifier, + .index_symbol = index, + .rpipe = rpipe, + }; + return &node.base; + } + + /// SwitchProng <- SwitchCase EQUALRARROW PtrPayload? AssignExpr + fn parseSwitchProng(p: *Parser) !?*Node { + const node = (try p.parseSwitchCase()) orelse return null; + const arrow = try p.expectToken(.EqualAngleBracketRight); + const payload = try p.parsePtrPayload(); + const expr = try p.expectNode(parseAssignExpr, .{ + .ExpectedExprOrAssignment = .{ .token = p.tok_i }, + }); + + const switch_case = node.cast(Node.SwitchCase).?; + switch_case.arrow_token = arrow; + switch_case.payload = payload; + switch_case.expr = expr; + + return node; + } + + /// SwitchCase + /// <- SwitchItem (COMMA SwitchItem)* COMMA? + /// / KEYWORD_else + fn parseSwitchCase(p: *Parser) !?*Node { + var list = Node.SwitchCase.ItemList{}; + var list_it = &list.first; + + if (try p.parseSwitchItem()) |first_item| { + list_it = try p.llpush(*Node, list_it, first_item); + while (p.eatToken(.Comma) != null) { + const next_item = (try p.parseSwitchItem()) orelse break; + list_it = try p.llpush(*Node, list_it, next_item); + } + } else if (p.eatToken(.Keyword_else)) |else_token| { + const else_node = try p.arena.allocator.create(Node.SwitchElse); + else_node.* = .{ + .token = else_token, + }; + list_it = try p.llpush(*Node, list_it, &else_node.base); + } else return null; + + const node = try p.arena.allocator.create(Node.SwitchCase); + node.* = .{ + .items = list, + .arrow_token = undefined, // set by caller + .payload = null, + .expr = undefined, // set by caller + }; + return &node.base; + } + + /// SwitchItem <- Expr (DOT3 Expr)? + fn parseSwitchItem(p: *Parser) !?*Node { + const expr = (try p.parseExpr()) orelse return null; + if (p.eatToken(.Ellipsis3)) |token| { + const range_end = try p.expectNode(parseExpr, .{ + .ExpectedExpr = .{ .token = p.tok_i }, + }); + + const node = try p.arena.allocator.create(Node.InfixOp); + node.* = .{ + .op_token = token, + .lhs = expr, + .op = .Range, + .rhs = range_end, + }; + return &node.base; + } + return expr; + } + + /// AssignOp + /// <- ASTERISKEQUAL + /// / SLASHEQUAL + /// / PERCENTEQUAL + /// / PLUSEQUAL + /// / MINUSEQUAL + /// / LARROW2EQUAL + /// / RARROW2EQUAL + /// / AMPERSANDEQUAL + /// / CARETEQUAL + /// / PIPEEQUAL + /// / ASTERISKPERCENTEQUAL + /// / PLUSPERCENTEQUAL + /// / MINUSPERCENTEQUAL + /// / EQUAL + fn parseAssignOp(p: *Parser) !?*Node { + const token = p.nextToken(); + const op: Node.InfixOp.Op = switch (token.ptr.id) { + .AsteriskEqual => .AssignMul, + .SlashEqual => .AssignDiv, + .PercentEqual => .AssignMod, + .PlusEqual => .AssignAdd, + .MinusEqual => .AssignSub, + .AngleBracketAngleBracketLeftEqual => .AssignBitShiftLeft, + .AngleBracketAngleBracketRightEqual => .AssignBitShiftRight, + .AmpersandEqual => .AssignBitAnd, + .CaretEqual => .AssignBitXor, + .PipeEqual => .AssignBitOr, + .AsteriskPercentEqual => .AssignMulWrap, + .PlusPercentEqual => .AssignAddWrap, + .MinusPercentEqual => .AssignSubWrap, + .Equal => .Assign, + else => { + p.putBackToken(token.index); + return null; + }, + }; + + const node = try p.arena.allocator.create(Node.InfixOp); + node.* = .{ + .op_token = token.index, + .lhs = undefined, // set by caller + .op = op, + .rhs = undefined, // set by caller + }; + return &node.base; + } + + /// CompareOp + /// <- EQUALEQUAL + /// / EXCLAMATIONMARKEQUAL + /// / LARROW + /// / RARROW + /// / LARROWEQUAL + /// / RARROWEQUAL + fn parseCompareOp(p: *Parser) !?*Node { + const token = p.nextToken(); + const op: Node.InfixOp.Op = switch (token.ptr.id) { + .EqualEqual => .EqualEqual, + .BangEqual => .BangEqual, + .AngleBracketLeft => .LessThan, + .AngleBracketRight => .GreaterThan, + .AngleBracketLeftEqual => .LessOrEqual, + .AngleBracketRightEqual => .GreaterOrEqual, + else => { + p.putBackToken(token.index); + return null; + }, + }; + + return p.createInfixOp(token.index, op); + } + + /// BitwiseOp + /// <- AMPERSAND + /// / CARET + /// / PIPE + /// / KEYWORD_orelse + /// / KEYWORD_catch Payload? + fn parseBitwiseOp(p: *Parser) !?*Node { + const token = p.nextToken(); + const op: Node.InfixOp.Op = switch (token.ptr.id) { + .Ampersand => .BitAnd, + .Caret => .BitXor, + .Pipe => .BitOr, + .Keyword_orelse => .UnwrapOptional, + .Keyword_catch => .{ .Catch = try p.parsePayload() }, + else => { + p.putBackToken(token.index); + return null; + }, + }; + + return p.createInfixOp(token.index, op); + } + + /// BitShiftOp + /// <- LARROW2 + /// / RARROW2 + fn parseBitShiftOp(p: *Parser) !?*Node { + const token = p.nextToken(); + const op: Node.InfixOp.Op = switch (token.ptr.id) { + .AngleBracketAngleBracketLeft => .BitShiftLeft, + .AngleBracketAngleBracketRight => .BitShiftRight, + else => { + p.putBackToken(token.index); + return null; + }, + }; + + return p.createInfixOp(token.index, op); + } + + /// AdditionOp + /// <- PLUS + /// / MINUS + /// / PLUS2 + /// / PLUSPERCENT + /// / MINUSPERCENT + fn parseAdditionOp(p: *Parser) !?*Node { + const token = p.nextToken(); + const op: Node.InfixOp.Op = switch (token.ptr.id) { + .Plus => .Add, + .Minus => .Sub, + .PlusPlus => .ArrayCat, + .PlusPercent => .AddWrap, + .MinusPercent => .SubWrap, + else => { + p.putBackToken(token.index); + return null; + }, + }; + + return p.createInfixOp(token.index, op); + } + + /// MultiplyOp + /// <- PIPE2 + /// / ASTERISK + /// / SLASH + /// / PERCENT + /// / ASTERISK2 + /// / ASTERISKPERCENT + fn parseMultiplyOp(p: *Parser) !?*Node { + const token = p.nextToken(); + const op: Node.InfixOp.Op = switch (token.ptr.id) { + .PipePipe => .MergeErrorSets, + .Asterisk => .Mul, + .Slash => .Div, + .Percent => .Mod, + .AsteriskAsterisk => .ArrayMult, + .AsteriskPercent => .MulWrap, + else => { + p.putBackToken(token.index); + return null; + }, + }; + + return p.createInfixOp(token.index, op); + } + + /// PrefixOp + /// <- EXCLAMATIONMARK + /// / MINUS + /// / TILDE + /// / MINUSPERCENT + /// / AMPERSAND + /// / KEYWORD_try + /// / KEYWORD_await + fn parsePrefixOp(p: *Parser) !?*Node { + const token = p.nextToken(); + const op: Node.PrefixOp.Op = switch (token.ptr.id) { + .Bang => .BoolNot, + .Minus => .Negation, + .Tilde => .BitNot, + .MinusPercent => .NegationWrap, + .Ampersand => .AddressOf, + .Keyword_try => .Try, + .Keyword_await => .Await, + else => { + p.putBackToken(token.index); + return null; + }, + }; + + const node = try p.arena.allocator.create(Node.PrefixOp); + node.* = .{ + .op_token = token.index, + .op = op, + .rhs = undefined, // set by caller + }; + return &node.base; + } + + // TODO: ArrayTypeStart is either an array or a slice, but const/allowzero only work on + // pointers. Consider updating this rule: + // ... + // / ArrayTypeStart + // / SliceTypeStart (ByteAlign / KEYWORD_const / KEYWORD_volatile / KEYWORD_allowzero)* + // / PtrTypeStart ... + + /// PrefixTypeOp + /// <- QUESTIONMARK + /// / KEYWORD_anyframe MINUSRARROW + /// / ArrayTypeStart (ByteAlign / KEYWORD_const / KEYWORD_volatile / KEYWORD_allowzero)* + /// / PtrTypeStart (KEYWORD_align LPAREN Expr (COLON INTEGER COLON INTEGER)? RPAREN / KEYWORD_const / KEYWORD_volatile / KEYWORD_allowzero)* + fn parsePrefixTypeOp(p: *Parser) !?*Node { + if (p.eatToken(.QuestionMark)) |token| { + const node = try p.arena.allocator.create(Node.PrefixOp); + node.* = .{ + .op_token = token, + .op = .OptionalType, .rhs = undefined, // set by caller }; return &node.base; } - }.parse; -} -// Helper parsers not included in the grammar - -fn parseBuiltinCall(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const token = eatToken(it, .Builtin) orelse return null; - const params = (try parseFnCallArguments(arena, it, tree)) orelse { - try tree.errors.push(.{ - .ExpectedParamList = .{ .token = it.index }, - }); - - // lets pretend this was an identifier so we can continue parsing - const node = try arena.create(Node.Identifier); - node.* = .{ - .token = token, - }; - return &node.base; - }; - const node = try arena.create(Node.BuiltinCall); - node.* = .{ - .builtin_token = token, - .params = params.list, - .rparen_token = params.rparen, - }; - return &node.base; -} - -fn parseErrorTag(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const doc_comments = try parseDocComment(arena, it, tree); // no need to rewind on failure - const token = eatToken(it, .Identifier) orelse return null; - - const node = try arena.create(Node.ErrorTag); - node.* = .{ - .doc_comments = doc_comments, - .name_token = token, - }; - return &node.base; -} - -fn parseIdentifier(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const token = eatToken(it, .Identifier) orelse return null; - const node = try arena.create(Node.Identifier); - node.* = .{ - .token = token, - }; - return &node.base; -} - -fn parseVarType(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const token = eatToken(it, .Keyword_var) orelse return null; - const node = try arena.create(Node.VarType); - node.* = .{ - .token = token, - }; - return &node.base; -} - -fn createLiteral(arena: *Allocator, comptime T: type, token: TokenIndex) !*Node { - const result = try arena.create(T); - result.* = T{ - .base = Node{ .id = Node.typeToId(T) }, - .token = token, - }; - return &result.base; -} - -fn parseStringLiteralSingle(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - if (eatToken(it, .StringLiteral)) |token| { - const node = try arena.create(Node.StringLiteral); - node.* = .{ - .token = token, - }; - return &node.base; - } - return null; -} - -// string literal or multiline string literal -fn parseStringLiteral(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - if (try parseStringLiteralSingle(arena, it, tree)) |node| return node; - - if (eatToken(it, .MultilineStringLiteralLine)) |first_line| { - const node = try arena.create(Node.MultilineStringLiteral); - node.* = .{ - .lines = Node.MultilineStringLiteral.LineList.init(arena), - }; - try node.lines.push(first_line); - while (eatToken(it, .MultilineStringLiteralLine)) |line| - try node.lines.push(line); - - return &node.base; - } - - return null; -} - -fn parseIntegerLiteral(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const token = eatToken(it, .IntegerLiteral) orelse return null; - const node = try arena.create(Node.IntegerLiteral); - node.* = .{ - .token = token, - }; - return &node.base; -} - -fn parseFloatLiteral(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const token = eatToken(it, .FloatLiteral) orelse return null; - const node = try arena.create(Node.FloatLiteral); - node.* = .{ - .token = token, - }; - return &node.base; -} - -fn parseTry(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const token = eatToken(it, .Keyword_try) orelse return null; - const node = try arena.create(Node.PrefixOp); - node.* = .{ - .op_token = token, - .op = .Try, - .rhs = undefined, // set by caller - }; - return &node.base; -} - -fn parseUse(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node { - const token = eatToken(it, .Keyword_usingnamespace) orelse return null; - const node = try arena.create(Node.Use); - node.* = .{ - .doc_comments = null, - .visib_token = null, - .use_token = token, - .expr = try expectNode(arena, it, tree, parseExpr, .{ - .ExpectedExpr = .{ .token = it.index }, - }), - .semicolon_token = try expectToken(it, tree, .Semicolon), - }; - return &node.base; -} - -/// IfPrefix Body (KEYWORD_else Payload? Body)? -fn parseIf(arena: *Allocator, it: *TokenIterator, tree: *Tree, bodyParseFn: NodeParseFn) !?*Node { - const node = (try parseIfPrefix(arena, it, tree)) orelse return null; - const if_prefix = node.cast(Node.If).?; - - if_prefix.body = try expectNode(arena, it, tree, bodyParseFn, .{ - .InvalidToken = .{ .token = it.index }, - }); - - const else_token = eatToken(it, .Keyword_else) orelse return node; - const payload = try parsePayload(arena, it, tree); - const else_expr = try expectNode(arena, it, tree, bodyParseFn, .{ - .InvalidToken = .{ .token = it.index }, - }); - const else_node = try arena.create(Node.Else); - else_node.* = .{ - .else_token = else_token, - .payload = payload, - .body = else_expr, - }; - if_prefix.@"else" = else_node; - - return node; -} - -/// Eat a multiline doc comment -fn parseDocComment(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*Node.DocComment { - var lines = Node.DocComment.LineList.init(arena); - while (eatToken(it, .DocComment)) |line| { - try lines.push(line); - } - - if (lines.len == 0) return null; - - const node = try arena.create(Node.DocComment); - node.* = .{ - .lines = lines, - }; - return node; -} - -/// Eat a single-line doc comment on the same line as another node -fn parseAppendedDocComment(arena: *Allocator, it: *TokenIterator, tree: *Tree, after_token: TokenIndex) !?*Node.DocComment { - const comment_token = eatToken(it, .DocComment) orelse return null; - if (tree.tokensOnSameLine(after_token, comment_token)) { - const node = try arena.create(Node.DocComment); - node.* = .{ - .lines = Node.DocComment.LineList.init(arena), - }; - try node.lines.push(comment_token); - return node; - } - putBackToken(it, comment_token); - return null; -} - -/// Op* Child -fn parsePrefixOpExpr( - arena: *Allocator, - it: *TokenIterator, - tree: *Tree, - opParseFn: NodeParseFn, - childParseFn: NodeParseFn, -) Error!?*Node { - if (try opParseFn(arena, it, tree)) |first_op| { - var rightmost_op = first_op; - while (true) { - switch (rightmost_op.id) { - .PrefixOp => { - var prefix_op = rightmost_op.cast(Node.PrefixOp).?; - // If the token encountered was **, there will be two nodes - if (tree.tokens.at(prefix_op.op_token).id == .AsteriskAsterisk) { - rightmost_op = prefix_op.rhs; - prefix_op = rightmost_op.cast(Node.PrefixOp).?; - } - if (try opParseFn(arena, it, tree)) |rhs| { - prefix_op.rhs = rhs; - rightmost_op = rhs; - } else break; + // TODO: Returning a AnyFrameType instead of PrefixOp makes casting and setting .rhs or + // .return_type more difficult for the caller (see parsePrefixOpExpr helper). + // Consider making the AnyFrameType a member of PrefixOp and add a + // PrefixOp.AnyFrameType variant? + if (p.eatToken(.Keyword_anyframe)) |token| { + const arrow = p.eatToken(.Arrow) orelse { + p.putBackToken(token); + return null; + }; + const node = try p.arena.allocator.create(Node.AnyFrameType); + node.* = .{ + .anyframe_token = token, + .result = .{ + .arrow_token = arrow, + .return_type = undefined, // set by caller }, - .AnyFrameType => { - const prom = rightmost_op.cast(Node.AnyFrameType).?; - if (try opParseFn(arena, it, tree)) |rhs| { - prom.result.?.return_type = rhs; - rightmost_op = rhs; - } else break; + }; + return &node.base; + } + + if (try p.parsePtrTypeStart()) |node| { + // If the token encountered was **, there will be two nodes instead of one. + // The attributes should be applied to the rightmost operator. + const prefix_op = node.cast(Node.PrefixOp).?; + var ptr_info = if (p.tokens[prefix_op.op_token].id == .AsteriskAsterisk) + &prefix_op.rhs.cast(Node.PrefixOp).?.op.PtrType + else + &prefix_op.op.PtrType; + + while (true) { + if (p.eatToken(.Keyword_align)) |align_token| { + const lparen = try p.expectToken(.LParen); + const expr_node = try p.expectNode(parseExpr, .{ + .ExpectedExpr = .{ .token = p.tok_i }, + }); + + // Optional bit range + const bit_range = if (p.eatToken(.Colon)) |_| bit_range_value: { + const range_start = try p.expectNode(parseIntegerLiteral, .{ + .ExpectedIntegerLiteral = .{ .token = p.tok_i }, + }); + _ = try p.expectToken(.Colon); + const range_end = try p.expectNode(parseIntegerLiteral, .{ + .ExpectedIntegerLiteral = .{ .token = p.tok_i }, + }); + + break :bit_range_value Node.PrefixOp.PtrInfo.Align.BitRange{ + .start = range_start, + .end = range_end, + }; + } else null; + _ = try p.expectToken(.RParen); + + if (ptr_info.align_info != null) { + try p.errors.append(p.gpa, .{ + .ExtraAlignQualifier = .{ .token = p.tok_i - 1 }, + }); + continue; + } + + ptr_info.align_info = Node.PrefixOp.PtrInfo.Align{ + .node = expr_node, + .bit_range = bit_range, + }; + + continue; + } + if (p.eatToken(.Keyword_const)) |const_token| { + if (ptr_info.const_token != null) { + try p.errors.append(p.gpa, .{ + .ExtraConstQualifier = .{ .token = p.tok_i - 1 }, + }); + continue; + } + ptr_info.const_token = const_token; + continue; + } + if (p.eatToken(.Keyword_volatile)) |volatile_token| { + if (ptr_info.volatile_token != null) { + try p.errors.append(p.gpa, .{ + .ExtraVolatileQualifier = .{ .token = p.tok_i - 1 }, + }); + continue; + } + ptr_info.volatile_token = volatile_token; + continue; + } + if (p.eatToken(.Keyword_allowzero)) |allowzero_token| { + if (ptr_info.allowzero_token != null) { + try p.errors.append(p.gpa, .{ + .ExtraAllowZeroQualifier = .{ .token = p.tok_i - 1 }, + }); + continue; + } + ptr_info.allowzero_token = allowzero_token; + continue; + } + break; + } + + return node; + } + + if (try p.parseArrayTypeStart()) |node| { + switch (node.cast(Node.PrefixOp).?.op) { + .ArrayType => {}, + .SliceType => |*slice_type| { + // Collect pointer qualifiers in any order, but disallow duplicates + while (true) { + if (try p.parseByteAlign()) |align_expr| { + if (slice_type.align_info != null) { + try p.errors.append(p.gpa, .{ + .ExtraAlignQualifier = .{ .token = p.tok_i - 1 }, + }); + continue; + } + slice_type.align_info = Node.PrefixOp.PtrInfo.Align{ + .node = align_expr, + .bit_range = null, + }; + continue; + } + if (p.eatToken(.Keyword_const)) |const_token| { + if (slice_type.const_token != null) { + try p.errors.append(p.gpa, .{ + .ExtraConstQualifier = .{ .token = p.tok_i - 1 }, + }); + continue; + } + slice_type.const_token = const_token; + continue; + } + if (p.eatToken(.Keyword_volatile)) |volatile_token| { + if (slice_type.volatile_token != null) { + try p.errors.append(p.gpa, .{ + .ExtraVolatileQualifier = .{ .token = p.tok_i - 1 }, + }); + continue; + } + slice_type.volatile_token = volatile_token; + continue; + } + if (p.eatToken(.Keyword_allowzero)) |allowzero_token| { + if (slice_type.allowzero_token != null) { + try p.errors.append(p.gpa, .{ + .ExtraAllowZeroQualifier = .{ .token = p.tok_i - 1 }, + }); + continue; + } + slice_type.allowzero_token = allowzero_token; + continue; + } + break; + } }, else => unreachable, } + return node; } - // If any prefix op existed, a child node on the RHS is required - switch (rightmost_op.id) { - .PrefixOp => { - const prefix_op = rightmost_op.cast(Node.PrefixOp).?; - prefix_op.rhs = try expectNode(arena, it, tree, childParseFn, .{ - .InvalidToken = .{ .token = it.index }, - }); - }, - .AnyFrameType => { - const prom = rightmost_op.cast(Node.AnyFrameType).?; - prom.result.?.return_type = try expectNode(arena, it, tree, childParseFn, .{ - .InvalidToken = .{ .token = it.index }, - }); - }, - else => unreachable, - } - - return first_op; - } - - // Otherwise, the child node is optional - return try childParseFn(arena, it, tree); -} - -/// Child (Op Child)* -/// Child (Op Child)? -fn parseBinOpExpr( - arena: *Allocator, - it: *TokenIterator, - tree: *Tree, - opParseFn: NodeParseFn, - childParseFn: NodeParseFn, - chain: enum { - Once, - Infinitely, - }, -) Error!?*Node { - var res = (try childParseFn(arena, it, tree)) orelse return null; - - while (try opParseFn(arena, it, tree)) |node| { - const right = try expectNode(arena, it, tree, childParseFn, .{ - .InvalidToken = .{ .token = it.index }, - }); - const left = res; - res = node; - - const op = node.cast(Node.InfixOp).?; - op.*.lhs = left; - op.*.rhs = right; - - switch (chain) { - .Once => break, - .Infinitely => continue, - } - } - - return res; -} - -fn createInfixOp(arena: *Allocator, index: TokenIndex, op: Node.InfixOp.Op) !*Node { - const node = try arena.create(Node.InfixOp); - node.* = .{ - .op_token = index, - .lhs = undefined, // set by caller - .op = op, - .rhs = undefined, // set by caller - }; - return &node.base; -} - -fn eatToken(it: *TokenIterator, id: Token.Id) ?TokenIndex { - return if (eatAnnotatedToken(it, id)) |token| token.index else null; -} - -fn eatAnnotatedToken(it: *TokenIterator, id: Token.Id) ?AnnotatedToken { - return if (it.peek().?.id == id) nextToken(it) else null; -} - -fn expectToken(it: *TokenIterator, tree: *Tree, id: Token.Id) Error!TokenIndex { - return (try expectTokenRecoverable(it, tree, id)) orelse - error.ParseError; -} - -fn expectTokenRecoverable(it: *TokenIterator, tree: *Tree, id: Token.Id) !?TokenIndex { - const token = nextToken(it); - if (token.ptr.id != id) { - try tree.errors.push(.{ - .ExpectedToken = .{ .token = token.index, .expected_id = id }, - }); - // go back so that we can recover properly - putBackToken(it, token.index); return null; } - return token.index; -} -fn nextToken(it: *TokenIterator) AnnotatedToken { - const result = AnnotatedToken{ - .index = it.index, - .ptr = it.next().?, + /// SuffixOp + /// <- LBRACKET Expr (DOT2 (Expr (COLON Expr)?)?)? RBRACKET + /// / DOT IDENTIFIER + /// / DOTASTERISK + /// / DOTQUESTIONMARK + fn parseSuffixOp(p: *Parser) !?*Node { + const OpAndToken = struct { + op: Node.SuffixOp.Op, + token: TokenIndex, + }; + const op_and_token: OpAndToken = blk: { + if (p.eatToken(.LBracket)) |_| { + const index_expr = try p.expectNode(parseExpr, .{ + .ExpectedExpr = .{ .token = p.tok_i }, + }); + + if (p.eatToken(.Ellipsis2) != null) { + const end_expr = try p.parseExpr(); + const sentinel: ?*ast.Node = if (p.eatToken(.Colon) != null) + try p.parseExpr() + else + null; + break :blk .{ + .op = .{ + .Slice = .{ + .start = index_expr, + .end = end_expr, + .sentinel = sentinel, + }, + }, + .token = try p.expectToken(.RBracket), + }; + } + + break :blk .{ + .op = .{ .ArrayAccess = index_expr }, + .token = try p.expectToken(.RBracket), + }; + } + + if (p.eatToken(.PeriodAsterisk)) |period_asterisk| { + break :blk .{ .op = .Deref, .token = period_asterisk }; + } + + if (p.eatToken(.Period)) |period| { + if (try p.parseIdentifier()) |identifier| { + // TODO: It's a bit weird to return an InfixOp from the SuffixOp parser. + // Should there be an ast.Node.SuffixOp.FieldAccess variant? Or should + // this grammar rule be altered? + const node = try p.arena.allocator.create(Node.InfixOp); + node.* = .{ + .op_token = period, + .lhs = undefined, // set by caller + .op = .Period, + .rhs = identifier, + }; + return &node.base; + } + if (p.eatToken(.QuestionMark)) |question_mark| { + break :blk .{ .op = .UnwrapOptional, .token = question_mark }; + } + try p.errors.append(p.gpa, .{ + .ExpectedSuffixOp = .{ .token = p.tok_i }, + }); + return null; + } + + return null; + }; + + const node = try p.arena.allocator.create(Node.SuffixOp); + node.* = .{ + .lhs = undefined, // set by caller + .op = op_and_token.op, + .rtoken = op_and_token.token, + }; + return &node.base; + } + + /// FnCallArguments <- LPAREN ExprList RPAREN + /// ExprList <- (Expr COMMA)* Expr? + fn parseFnCallArguments(p: *Parser) !?AnnotatedParamList { + if (p.eatToken(.LParen) == null) return null; + const list = try ListParseFn(Node.FnProto.ParamList, parseExpr)(p); + const rparen = try p.expectToken(.RParen); + return AnnotatedParamList{ .list = list, .rparen = rparen }; + } + + const AnnotatedParamList = struct { + list: Node.FnProto.ParamList, // NOTE: may also be any other type SegmentedList(*Node, 2) + rparen: TokenIndex, }; - assert(result.ptr.id != .LineComment); - while (true) { - const next_tok = it.peek() orelse return result; - if (next_tok.id != .LineComment) return result; - _ = it.next(); + /// ArrayTypeStart <- LBRACKET Expr? RBRACKET + fn parseArrayTypeStart(p: *Parser) !?*Node { + const lbracket = p.eatToken(.LBracket) orelse return null; + const expr = try p.parseExpr(); + const sentinel = if (p.eatToken(.Colon)) |_| + try p.expectNode(parseExpr, .{ + .ExpectedExpr = .{ .token = p.tok_i }, + }) + else + null; + const rbracket = try p.expectToken(.RBracket); + + const op: Node.PrefixOp.Op = if (expr) |len_expr| + .{ + .ArrayType = .{ + .len_expr = len_expr, + .sentinel = sentinel, + }, + } + else + .{ + .SliceType = Node.PrefixOp.PtrInfo{ + .allowzero_token = null, + .align_info = null, + .const_token = null, + .volatile_token = null, + .sentinel = sentinel, + }, + }; + + const node = try p.arena.allocator.create(Node.PrefixOp); + node.* = .{ + .op_token = lbracket, + .op = op, + .rhs = undefined, // set by caller + }; + return &node.base; } -} -fn putBackToken(it: *TokenIterator, putting_back: TokenIndex) void { - while (true) { - const prev_tok = it.prev() orelse return; - if (prev_tok.id == .LineComment) continue; - assert(it.list.at(putting_back) == prev_tok); - return; + /// PtrTypeStart + /// <- ASTERISK + /// / ASTERISK2 + /// / PTRUNKNOWN + /// / PTRC + fn parsePtrTypeStart(p: *Parser) !?*Node { + if (p.eatToken(.Asterisk)) |asterisk| { + const sentinel = if (p.eatToken(.Colon)) |_| + try p.expectNode(parseExpr, .{ + .ExpectedExpr = .{ .token = p.tok_i }, + }) + else + null; + const node = try p.arena.allocator.create(Node.PrefixOp); + node.* = .{ + .op_token = asterisk, + .op = .{ .PtrType = .{ .sentinel = sentinel } }, + .rhs = undefined, // set by caller + }; + return &node.base; + } + + if (p.eatToken(.AsteriskAsterisk)) |double_asterisk| { + const node = try p.arena.allocator.create(Node.PrefixOp); + node.* = .{ + .op_token = double_asterisk, + .op = .{ .PtrType = .{} }, + .rhs = undefined, // set by caller + }; + + // Special case for **, which is its own token + const child = try p.arena.allocator.create(Node.PrefixOp); + child.* = .{ + .op_token = double_asterisk, + .op = .{ .PtrType = .{} }, + .rhs = undefined, // set by caller + }; + node.rhs = &child.base; + + return &node.base; + } + if (p.eatToken(.LBracket)) |lbracket| { + const asterisk = p.eatToken(.Asterisk) orelse { + p.putBackToken(lbracket); + return null; + }; + if (p.eatToken(.Identifier)) |ident| { + const token_slice = p.source[p.tokens[ident].start..p.tokens[ident].end]; + if (!std.mem.eql(u8, token_slice, "c")) { + p.putBackToken(ident); + } else { + _ = try p.expectToken(.RBracket); + const node = try p.arena.allocator.create(Node.PrefixOp); + node.* = .{ + .op_token = lbracket, + .op = .{ .PtrType = .{} }, + .rhs = undefined, // set by caller + }; + return &node.base; + } + } + const sentinel = if (p.eatToken(.Colon)) |_| + try p.expectNode(parseExpr, .{ + .ExpectedExpr = .{ .token = p.tok_i }, + }) + else + null; + _ = try p.expectToken(.RBracket); + const node = try p.arena.allocator.create(Node.PrefixOp); + node.* = .{ + .op_token = lbracket, + .op = .{ .PtrType = .{ .sentinel = sentinel } }, + .rhs = undefined, // set by caller + }; + return &node.base; + } + return null; } -} -const AnnotatedToken = struct { - index: TokenIndex, - ptr: *Token, + /// ContainerDeclAuto <- ContainerDeclType LBRACE ContainerMembers RBRACE + fn parseContainerDeclAuto(p: *Parser) !?*Node { + const node = (try p.parseContainerDeclType()) orelse return null; + const lbrace = try p.expectToken(.LBrace); + const members = try p.parseContainerMembers(false); + const rbrace = try p.expectToken(.RBrace); + + const decl_type = node.cast(Node.ContainerDecl).?; + decl_type.fields_and_decls = members; + decl_type.lbrace_token = lbrace; + decl_type.rbrace_token = rbrace; + + return node; + } + + /// ContainerDeclType + /// <- KEYWORD_struct + /// / KEYWORD_enum (LPAREN Expr RPAREN)? + /// / KEYWORD_union (LPAREN (KEYWORD_enum (LPAREN Expr RPAREN)? / Expr) RPAREN)? + fn parseContainerDeclType(p: *Parser) !?*Node { + const kind_token = p.nextToken(); + + const init_arg_expr = switch (kind_token.ptr.id) { + .Keyword_struct => Node.ContainerDecl.InitArg{ .None = {} }, + .Keyword_enum => blk: { + if (p.eatToken(.LParen) != null) { + const expr = try p.expectNode(parseExpr, .{ + .ExpectedExpr = .{ .token = p.tok_i }, + }); + _ = try p.expectToken(.RParen); + break :blk Node.ContainerDecl.InitArg{ .Type = expr }; + } + break :blk Node.ContainerDecl.InitArg{ .None = {} }; + }, + .Keyword_union => blk: { + if (p.eatToken(.LParen) != null) { + if (p.eatToken(.Keyword_enum) != null) { + if (p.eatToken(.LParen) != null) { + const expr = try p.expectNode(parseExpr, .{ + .ExpectedExpr = .{ .token = p.tok_i }, + }); + _ = try p.expectToken(.RParen); + _ = try p.expectToken(.RParen); + break :blk Node.ContainerDecl.InitArg{ .Enum = expr }; + } + _ = try p.expectToken(.RParen); + break :blk Node.ContainerDecl.InitArg{ .Enum = null }; + } + const expr = try p.expectNode(parseExpr, .{ + .ExpectedExpr = .{ .token = p.tok_i }, + }); + _ = try p.expectToken(.RParen); + break :blk Node.ContainerDecl.InitArg{ .Type = expr }; + } + break :blk Node.ContainerDecl.InitArg{ .None = {} }; + }, + else => { + p.putBackToken(kind_token.index); + return null; + }, + }; + + const node = try p.arena.allocator.create(Node.ContainerDecl); + node.* = .{ + .layout_token = null, + .kind_token = kind_token.index, + .init_arg_expr = init_arg_expr, + .fields_and_decls = undefined, // set by caller + .lbrace_token = undefined, // set by caller + .rbrace_token = undefined, // set by caller + }; + return &node.base; + } + + /// ByteAlign <- KEYWORD_align LPAREN Expr RPAREN + fn parseByteAlign(p: *Parser) !?*Node { + _ = p.eatToken(.Keyword_align) orelse return null; + _ = try p.expectToken(.LParen); + const expr = try p.expectNode(parseExpr, .{ + .ExpectedExpr = .{ .token = p.tok_i }, + }); + _ = try p.expectToken(.RParen); + return expr; + } + + /// IdentifierList <- (IDENTIFIER COMMA)* IDENTIFIER? + /// Only ErrorSetDecl parses an IdentifierList + fn parseErrorTagList(p: *Parser) !Node.ErrorSetDecl.DeclList { + return ListParseFn(Node.ErrorSetDecl.DeclList, parseErrorTag)(p); + } + + /// SwitchProngList <- (SwitchProng COMMA)* SwitchProng? + fn parseSwitchProngList(p: *Parser) !Node.Switch.CaseList { + return ListParseFn(Node.Switch.CaseList, parseSwitchProng)(p); + } + + /// AsmOutputList <- (AsmOutputItem COMMA)* AsmOutputItem? + fn parseAsmOutputList(p: *Parser) Error!Node.Asm.OutputList { + return ListParseFn(Node.Asm.OutputList, parseAsmOutputItem)(p); + } + + /// AsmInputList <- (AsmInputItem COMMA)* AsmInputItem? + fn parseAsmInputList(p: *Parser) Error!Node.Asm.InputList { + return ListParseFn(Node.Asm.InputList, parseAsmInputItem)(p); + } + + /// ParamDeclList <- (ParamDecl COMMA)* ParamDecl? + fn parseParamDeclList(p: *Parser, var_args_token: *?TokenIndex) !Node.FnProto.ParamList { + var list = Node.FnProto.ParamList{}; + var list_it = &list.first; + var last: ?*Node = null; + while (try p.parseParamDecl()) |node| { + last = node; + list_it = try p.llpush(*Node, list_it, node); + + switch (p.tokens[p.tok_i].id) { + .Comma => _ = p.nextToken(), + // all possible delimiters + .Colon, .RParen, .RBrace, .RBracket => break, + else => { + // this is likely just a missing comma, + // continue parsing this list and give an error + try p.errors.append(p.gpa, .{ + .ExpectedToken = .{ .token = p.tok_i, .expected_id = .Comma }, + }); + }, + } + } + if (last) |node| { + const param_type = node.cast(Node.ParamDecl).?.param_type; + if (param_type == .var_args) { + var_args_token.* = param_type.var_args; + } + } + return list; + } + + const NodeParseFn = fn (p: *Parser) Error!?*Node; + + fn ListParseFn(comptime L: type, comptime nodeParseFn: var) ParseFn(L) { + return struct { + pub fn parse(p: *Parser) !L { + var list = L{}; + var list_it = &list.first; + while (try nodeParseFn(p)) |node| { + list_it = try p.llpush(L.Node.Data, list_it, node); + + switch (p.tokens[p.tok_i].id) { + .Comma => _ = p.nextToken(), + // all possible delimiters + .Colon, .RParen, .RBrace, .RBracket => break, + else => { + // this is likely just a missing comma, + // continue parsing this list and give an error + try p.errors.append(p.gpa, .{ + .ExpectedToken = .{ .token = p.tok_i, .expected_id = .Comma }, + }); + }, + } + } + return list; + } + }.parse; + } + + fn SimpleBinOpParseFn(comptime token: Token.Id, comptime op: Node.InfixOp.Op) NodeParseFn { + return struct { + pub fn parse(p: *Parser) Error!?*Node { + const op_token = if (token == .Keyword_and) switch (p.tokens[p.tok_i].id) { + .Keyword_and => p.nextToken().index, + .Invalid_ampersands => blk: { + try p.errors.append(p.gpa, .{ + .InvalidAnd = .{ .token = p.tok_i }, + }); + break :blk p.nextToken().index; + }, + else => return null, + } else p.eatToken(token) orelse return null; + + const node = try p.arena.allocator.create(Node.InfixOp); + node.* = .{ + .op_token = op_token, + .lhs = undefined, // set by caller + .op = op, + .rhs = undefined, // set by caller + }; + return &node.base; + } + }.parse; + } + + // Helper parsers not included in the grammar + + fn parseBuiltinCall(p: *Parser) !?*Node { + const token = p.eatToken(.Builtin) orelse return null; + const params = (try p.parseFnCallArguments()) orelse { + try p.errors.append(p.gpa, .{ + .ExpectedParamList = .{ .token = p.tok_i }, + }); + + // lets pretend this was an identifier so we can continue parsing + const node = try p.arena.allocator.create(Node.Identifier); + node.* = .{ + .token = token, + }; + return &node.base; + }; + const node = try p.arena.allocator.create(Node.BuiltinCall); + node.* = .{ + .builtin_token = token, + .params = params.list, + .rparen_token = params.rparen, + }; + return &node.base; + } + + fn parseErrorTag(p: *Parser) !?*Node { + const doc_comments = try p.parseDocComment(); // no need to rewind on failure + const token = p.eatToken(.Identifier) orelse return null; + + const node = try p.arena.allocator.create(Node.ErrorTag); + node.* = .{ + .doc_comments = doc_comments, + .name_token = token, + }; + return &node.base; + } + + fn parseIdentifier(p: *Parser) !?*Node { + const token = p.eatToken(.Identifier) orelse return null; + const node = try p.arena.allocator.create(Node.Identifier); + node.* = .{ + .token = token, + }; + return &node.base; + } + + fn parseVarType(p: *Parser) !?*Node { + const token = p.eatToken(.Keyword_var) orelse return null; + const node = try p.arena.allocator.create(Node.VarType); + node.* = .{ + .token = token, + }; + return &node.base; + } + + fn createLiteral(p: *Parser, comptime T: type, token: TokenIndex) !*Node { + const result = try p.arena.allocator.create(T); + result.* = T{ + .base = Node{ .id = Node.typeToId(T) }, + .token = token, + }; + return &result.base; + } + + fn parseStringLiteralSingle(p: *Parser) !?*Node { + if (p.eatToken(.StringLiteral)) |token| { + const node = try p.arena.allocator.create(Node.StringLiteral); + node.* = .{ + .token = token, + }; + return &node.base; + } + return null; + } + + // string literal or multiline string literal + fn parseStringLiteral(p: *Parser) !?*Node { + if (try p.parseStringLiteralSingle()) |node| return node; + + if (p.eatToken(.MultilineStringLiteralLine)) |first_line| { + const node = try p.arena.allocator.create(Node.MultilineStringLiteral); + node.* = .{ + .lines = Node.MultilineStringLiteral.LineList{}, + }; + var lines_it = &node.lines.first; + lines_it = try p.llpush(TokenIndex, lines_it, first_line); + while (p.eatToken(.MultilineStringLiteralLine)) |line| + lines_it = try p.llpush(TokenIndex, lines_it, line); + + return &node.base; + } + + return null; + } + + fn parseIntegerLiteral(p: *Parser) !?*Node { + const token = p.eatToken(.IntegerLiteral) orelse return null; + const node = try p.arena.allocator.create(Node.IntegerLiteral); + node.* = .{ + .token = token, + }; + return &node.base; + } + + fn parseFloatLiteral(p: *Parser) !?*Node { + const token = p.eatToken(.FloatLiteral) orelse return null; + const node = try p.arena.allocator.create(Node.FloatLiteral); + node.* = .{ + .token = token, + }; + return &node.base; + } + + fn parseTry(p: *Parser) !?*Node { + const token = p.eatToken(.Keyword_try) orelse return null; + const node = try p.arena.allocator.create(Node.PrefixOp); + node.* = .{ + .op_token = token, + .op = .Try, + .rhs = undefined, // set by caller + }; + return &node.base; + } + + fn parseUse(p: *Parser) !?*Node { + const token = p.eatToken(.Keyword_usingnamespace) orelse return null; + const node = try p.arena.allocator.create(Node.Use); + node.* = .{ + .doc_comments = null, + .visib_token = null, + .use_token = token, + .expr = try p.expectNode(parseExpr, .{ + .ExpectedExpr = .{ .token = p.tok_i }, + }), + .semicolon_token = try p.expectToken(.Semicolon), + }; + return &node.base; + } + + /// IfPrefix Body (KEYWORD_else Payload? Body)? + fn parseIf(p: *Parser, bodyParseFn: NodeParseFn) !?*Node { + const node = (try p.parseIfPrefix()) orelse return null; + const if_prefix = node.cast(Node.If).?; + + if_prefix.body = try p.expectNode(bodyParseFn, .{ + .InvalidToken = .{ .token = p.tok_i }, + }); + + const else_token = p.eatToken(.Keyword_else) orelse return node; + const payload = try p.parsePayload(); + const else_expr = try p.expectNode(bodyParseFn, .{ + .InvalidToken = .{ .token = p.tok_i }, + }); + const else_node = try p.arena.allocator.create(Node.Else); + else_node.* = .{ + .else_token = else_token, + .payload = payload, + .body = else_expr, + }; + if_prefix.@"else" = else_node; + + return node; + } + + /// Eat a multiline doc comment + fn parseDocComment(p: *Parser) !?*Node.DocComment { + var lines = Node.DocComment.LineList{}; + var lines_it = &lines.first; + + while (p.eatToken(.DocComment)) |line| { + lines_it = try p.llpush(TokenIndex, lines_it, line); + } + + if (lines.first == null) return null; + + const node = try p.arena.allocator.create(Node.DocComment); + node.* = .{ + .lines = lines, + }; + return node; + } + + fn tokensOnSameLine(p: *Parser, token1: TokenIndex, token2: TokenIndex) bool { + return std.mem.indexOfScalar(u8, p.source[p.tokens[token1].end..p.tokens[token2].start], '\n') == null; + } + + /// Eat a single-line doc comment on the same line as another node + fn parseAppendedDocComment(p: *Parser, after_token: TokenIndex) !?*Node.DocComment { + const comment_token = p.eatToken(.DocComment) orelse return null; + if (p.tokensOnSameLine(after_token, comment_token)) { + var lines = Node.DocComment.LineList{}; + _ = try p.llpush(TokenIndex, &lines.first, comment_token); + + const node = try p.arena.allocator.create(Node.DocComment); + node.* = .{ .lines = lines }; + return node; + } + p.putBackToken(comment_token); + return null; + } + + /// Op* Child + fn parsePrefixOpExpr(p: *Parser, opParseFn: NodeParseFn, childParseFn: NodeParseFn) Error!?*Node { + if (try opParseFn(p)) |first_op| { + var rightmost_op = first_op; + while (true) { + switch (rightmost_op.id) { + .PrefixOp => { + var prefix_op = rightmost_op.cast(Node.PrefixOp).?; + // If the token encountered was **, there will be two nodes + if (p.tokens[prefix_op.op_token].id == .AsteriskAsterisk) { + rightmost_op = prefix_op.rhs; + prefix_op = rightmost_op.cast(Node.PrefixOp).?; + } + if (try opParseFn(p)) |rhs| { + prefix_op.rhs = rhs; + rightmost_op = rhs; + } else break; + }, + .AnyFrameType => { + const prom = rightmost_op.cast(Node.AnyFrameType).?; + if (try opParseFn(p)) |rhs| { + prom.result.?.return_type = rhs; + rightmost_op = rhs; + } else break; + }, + else => unreachable, + } + } + + // If any prefix op existed, a child node on the RHS is required + switch (rightmost_op.id) { + .PrefixOp => { + const prefix_op = rightmost_op.cast(Node.PrefixOp).?; + prefix_op.rhs = try p.expectNode(childParseFn, .{ + .InvalidToken = .{ .token = p.tok_i }, + }); + }, + .AnyFrameType => { + const prom = rightmost_op.cast(Node.AnyFrameType).?; + prom.result.?.return_type = try p.expectNode(childParseFn, .{ + .InvalidToken = .{ .token = p.tok_i }, + }); + }, + else => unreachable, + } + + return first_op; + } + + // Otherwise, the child node is optional + return childParseFn(p); + } + + /// Child (Op Child)* + /// Child (Op Child)? + fn parseBinOpExpr( + p: *Parser, + opParseFn: NodeParseFn, + childParseFn: NodeParseFn, + chain: enum { + Once, + Infinitely, + }, + ) Error!?*Node { + var res = (try childParseFn(p)) orelse return null; + + while (try opParseFn(p)) |node| { + const right = try p.expectNode(childParseFn, .{ + .InvalidToken = .{ .token = p.tok_i }, + }); + const left = res; + res = node; + + const op = node.cast(Node.InfixOp).?; + op.*.lhs = left; + op.*.rhs = right; + + switch (chain) { + .Once => break, + .Infinitely => continue, + } + } + + return res; + } + + fn createInfixOp(p: *Parser, index: TokenIndex, op: Node.InfixOp.Op) !*Node { + const node = try p.arena.allocator.create(Node.InfixOp); + node.* = .{ + .op_token = index, + .lhs = undefined, // set by caller + .op = op, + .rhs = undefined, // set by caller + }; + return &node.base; + } + + fn eatToken(p: *Parser, id: Token.Id) ?TokenIndex { + return if (p.eatAnnotatedToken(id)) |token| token.index else null; + } + + fn eatAnnotatedToken(p: *Parser, id: Token.Id) ?AnnotatedToken { + return if (p.tokens[p.tok_i].id == id) p.nextToken() else null; + } + + fn expectToken(p: *Parser, id: Token.Id) Error!TokenIndex { + return (try p.expectTokenRecoverable(id)) orelse + error.ParseError; + } + + fn expectTokenRecoverable(p: *Parser, id: Token.Id) !?TokenIndex { + const token = p.nextToken(); + if (token.ptr.id != id) { + try p.errors.append(p.gpa, .{ + .ExpectedToken = .{ .token = token.index, .expected_id = id }, + }); + // go back so that we can recover properly + p.putBackToken(token.index); + return null; + } + return token.index; + } + + fn nextToken(p: *Parser) AnnotatedToken { + const result = AnnotatedToken{ + .index = p.tok_i, + .ptr = &p.tokens[p.tok_i], + }; + if (p.tokens[p.tok_i].id == .Eof) { + return result; + } + p.tok_i += 1; + assert(result.ptr.id != .LineComment); + + while (true) { + const next_tok = p.tokens[p.tok_i]; + if (next_tok.id != .LineComment) return result; + p.tok_i += 1; + } + } + + fn putBackToken(p: *Parser, putting_back: TokenIndex) void { + while (p.tok_i > 0) { + p.tok_i -= 1; + const prev_tok = p.tokens[p.tok_i]; + if (prev_tok.id == .LineComment) continue; + assert(putting_back == p.tok_i); + return; + } + } + + const AnnotatedToken = struct { + index: TokenIndex, + ptr: *const Token, + }; + + fn expectNode( + p: *Parser, + parseFn: NodeParseFn, + /// if parsing fails + err: AstError, + ) Error!*Node { + return (try p.expectNodeRecoverable(parseFn, err)) orelse return error.ParseError; + } + + fn expectNodeRecoverable( + p: *Parser, + parseFn: NodeParseFn, + /// if parsing fails + err: AstError, + ) !?*Node { + return (try parseFn(p)) orelse { + try p.errors.append(p.gpa, err); + return null; + }; + } }; -fn expectNode( - arena: *Allocator, - it: *TokenIterator, - tree: *Tree, - parseFn: NodeParseFn, - err: AstError, // if parsing fails -) Error!*Node { - return (try expectNodeRecoverable(arena, it, tree, parseFn, err)) orelse - return error.ParseError; +fn ParseFn(comptime T: type) type { + return fn (p: *Parser) Error!T; } -fn expectNodeRecoverable( - arena: *Allocator, - it: *TokenIterator, - tree: *Tree, - parseFn: NodeParseFn, - err: AstError, // if parsing fails -) !?*Node { - return (try parseFn(arena, it, tree)) orelse { - try tree.errors.push(err); - return null; - }; -} test "std.zig.parser" { _ = @import("parser_test.zig"); diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index 0a3a47cf94..32667a02fb 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -3180,9 +3180,8 @@ fn testParse(source: []const u8, allocator: *mem.Allocator, anything_changed: *b const tree = try std.zig.parse(allocator, source); defer tree.deinit(); - var error_it = tree.errors.iterator(0); - while (error_it.next()) |parse_error| { - const token = tree.tokens.at(parse_error.loc()); + for (tree.errors) |*parse_error| { + const token = tree.tokens[parse_error.loc()]; const loc = tree.tokenLocation(0, parse_error.loc()); try stderr.print("(memory buffer):{}:{}: error: ", .{ loc.line + 1, loc.column + 1 }); try tree.renderError(parse_error, stderr); @@ -3271,8 +3270,6 @@ fn testError(source: []const u8, expected_errors: []const Error) !void { std.testing.expect(tree.errors.len == expected_errors.len); for (expected_errors) |expected, i| { - const err = tree.errors.at(i); - - std.testing.expect(expected == err.*); + std.testing.expect(expected == tree.errors[i]); } } diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index 2de5022876..42531b82cf 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -67,24 +67,21 @@ fn renderRoot( stream: var, tree: *ast.Tree, ) (@TypeOf(stream).Error || Error)!void { - var tok_it = tree.tokens.iterator(0); - // render all the line comments at the beginning of the file - while (tok_it.next()) |token| { + for (tree.tokens) |*token, i| { if (token.id != .LineComment) break; try stream.print("{}\n", .{mem.trimRight(u8, tree.tokenSlicePtr(token), " ")}); - if (tok_it.peek()) |next_token| { - const loc = tree.tokenLocationPtr(token.end, next_token); - if (loc.line >= 2) { - try stream.writeByte('\n'); - } + const next_token = &tree.tokens[i + 1]; + const loc = tree.tokenLocationPtr(token.end, next_token.*); + if (loc.line >= 2) { + try stream.writeByte('\n'); } } var start_col: usize = 0; - var it = tree.root_node.decls.iterator(0); + var it = tree.root_node.decls.first orelse return; while (true) { - var decl = (it.next() orelse return).*; + var decl = it.data; // This loop does the following: // @@ -103,7 +100,7 @@ fn renderRoot( while (token_index != 0) { token_index -= 1; - const token = tree.tokens.at(token_index); + const token = tree.tokens[token_index]; switch (token.id) { .LineComment => {}, .DocComment => { @@ -133,17 +130,18 @@ fn renderRoot( token_index = decl.firstToken(); while (!fmt_active) { - decl = (it.next() orelse { + it = it.next orelse { // If there's no next reformatted `decl`, just copy the // remaining input tokens and bail out. - const start = tree.tokens.at(copy_start_token_index).start; + const start = tree.tokens[copy_start_token_index].start; try copyFixingWhitespace(stream, tree.source[start..]); return; - }).*; + }; + decl = it.data; var decl_first_token_index = decl.firstToken(); while (token_index < decl_first_token_index) : (token_index += 1) { - const token = tree.tokens.at(token_index); + const token = tree.tokens[token_index]; switch (token.id) { .LineComment => {}, .Eof => unreachable, @@ -163,7 +161,7 @@ fn renderRoot( token_index = copy_end_token_index; while (token_index != 0) { token_index -= 1; - const token = tree.tokens.at(token_index); + const token = tree.tokens[token_index]; switch (token.id) { .LineComment => {}, .DocComment => { @@ -174,15 +172,14 @@ fn renderRoot( } } - const start = tree.tokens.at(copy_start_token_index).start; - const end = tree.tokens.at(copy_end_token_index).start; + const start = tree.tokens[copy_start_token_index].start; + const end = tree.tokens[copy_end_token_index].start; try copyFixingWhitespace(stream, tree.source[start..end]); } try renderTopLevelDecl(allocator, stream, tree, 0, &start_col, decl); - if (it.peek()) |next_decl| { - try renderExtraNewline(tree, stream, &start_col, next_decl.*); - } + it = it.next orelse return; + try renderExtraNewline(tree, stream, &start_col, it.data); } } @@ -191,13 +188,13 @@ fn renderExtraNewline(tree: *ast.Tree, stream: var, start_col: *usize, node: *as var prev_token = first_token; if (prev_token == 0) return; var newline_threshold: usize = 2; - while (tree.tokens.at(prev_token - 1).id == .DocComment) { - if (tree.tokenLocation(tree.tokens.at(prev_token - 1).end, prev_token).line == 1) { + while (tree.tokens[prev_token - 1].id == .DocComment) { + if (tree.tokenLocation(tree.tokens[prev_token - 1].end, prev_token).line == 1) { newline_threshold += 1; } prev_token -= 1; } - const prev_token_end = tree.tokens.at(prev_token - 1).end; + const prev_token_end = tree.tokens[prev_token - 1].end; const loc = tree.tokenLocation(prev_token_end, first_token); if (loc.line >= newline_threshold) { try stream.writeByte('\n'); @@ -262,7 +259,7 @@ fn renderContainerDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree, const src_has_trailing_comma = blk: { const maybe_comma = tree.nextToken(field.lastToken()); - break :blk tree.tokens.at(maybe_comma).id == .Comma; + break :blk tree.tokens[maybe_comma].id == .Comma; }; // The trailing comma is emitted at the end, but if it's not present @@ -426,13 +423,13 @@ fn renderExpression( try renderExpression(allocator, stream, tree, indent, start_col, infix_op_node.lhs, op_space); const after_op_space = blk: { - const loc = tree.tokenLocation(tree.tokens.at(infix_op_node.op_token).end, tree.nextToken(infix_op_node.op_token)); + const loc = tree.tokenLocation(tree.tokens[infix_op_node.op_token].end, tree.nextToken(infix_op_node.op_token)); break :blk if (loc.line == 0) op_space else Space.Newline; }; try renderToken(tree, stream, infix_op_node.op_token, indent, start_col, after_op_space); if (after_op_space == Space.Newline and - tree.tokens.at(tree.nextToken(infix_op_node.op_token)).id != .MultilineStringLiteralLine) + tree.tokens[tree.nextToken(infix_op_node.op_token)].id != .MultilineStringLiteralLine) { try stream.writeByteNTimes(' ', indent + indent_delta); start_col.* = indent + indent_delta; @@ -453,10 +450,10 @@ fn renderExpression( switch (prefix_op_node.op) { .PtrType => |ptr_info| { - const op_tok_id = tree.tokens.at(prefix_op_node.op_token).id; + const op_tok_id = tree.tokens[prefix_op_node.op_token].id; switch (op_tok_id) { .Asterisk, .AsteriskAsterisk => try stream.writeByte('*'), - .LBracket => if (tree.tokens.at(prefix_op_node.op_token + 2).id == .Identifier) + .LBracket => if (tree.tokens[prefix_op_node.op_token + 2].id == .Identifier) try stream.writeAll("[*c") else try stream.writeAll("[*"), @@ -568,8 +565,8 @@ fn renderExpression( try renderToken(tree, stream, lbracket, indent, start_col, Space.None); // [ - const starts_with_comment = tree.tokens.at(lbracket + 1).id == .LineComment; - const ends_with_comment = tree.tokens.at(rbracket - 1).id == .LineComment; + const starts_with_comment = tree.tokens[lbracket + 1].id == .LineComment; + const ends_with_comment = tree.tokens[rbracket - 1].id == .LineComment; const new_indent = if (ends_with_comment) indent + indent_delta else indent; const new_space = if (ends_with_comment) Space.Newline else Space.None; try renderExpression(allocator, stream, tree, new_indent, start_col, array_info.len_expr, new_space); @@ -630,7 +627,7 @@ fn renderExpression( const src_has_trailing_comma = blk: { const maybe_comma = tree.prevToken(suffix_op.rtoken); - break :blk tree.tokens.at(maybe_comma).id == .Comma; + break :blk tree.tokens[maybe_comma].id == .Comma; }; if (src_has_trailing_comma) { @@ -682,8 +679,8 @@ fn renderExpression( try renderExpression(allocator, stream, tree, indent, start_col, suffix_op.lhs.node, Space.None); try renderToken(tree, stream, lbracket, indent, start_col, Space.None); // [ - const starts_with_comment = tree.tokens.at(lbracket + 1).id == .LineComment; - const ends_with_comment = tree.tokens.at(rbracket - 1).id == .LineComment; + const starts_with_comment = tree.tokens[lbracket + 1].id == .LineComment; + const ends_with_comment = tree.tokens[rbracket - 1].id == .LineComment; const new_indent = if (ends_with_comment) indent + indent_delta else indent; const new_space = if (ends_with_comment) Space.Newline else Space.None; try renderExpression(allocator, stream, tree, new_indent, start_col, index_expr, new_space); @@ -750,11 +747,11 @@ fn renderExpression( const src_has_trailing_comma = blk: { const maybe_comma = tree.prevToken(suffix_op.rtoken); - break :blk tree.tokens.at(maybe_comma).id == .Comma; + break :blk tree.tokens[maybe_comma].id == .Comma; }; const src_same_line = blk: { - const loc = tree.tokenLocation(tree.tokens.at(lbrace).end, suffix_op.rtoken); + const loc = tree.tokenLocation(tree.tokens[lbrace].end, suffix_op.rtoken); break :blk loc.line == 0; }; @@ -858,7 +855,7 @@ fn renderExpression( try renderToken(tree, stream, lbrace, indent, start_col, Space.None); return renderToken(tree, stream, suffix_op.rtoken, indent, start_col, space); } - if (exprs.len == 1 and tree.tokens.at(exprs.at(0).*.lastToken() + 1).id == .RBrace) { + if (exprs.len == 1 and tree.tokens[exprs.at(0).*.lastToken() + 1].id == .RBrace) { const expr = exprs.at(0).*; switch (suffix_op.lhs) { @@ -883,17 +880,17 @@ fn renderExpression( const expr = it.next().?.*; if (it.peek()) |next_expr| { const expr_last_token = expr.*.lastToken() + 1; - const loc = tree.tokenLocation(tree.tokens.at(expr_last_token).end, next_expr.*.firstToken()); + const loc = tree.tokenLocation(tree.tokens[expr_last_token].end, next_expr.*.firstToken()); if (loc.line != 0) break :blk count; count += 1; } else { const expr_last_token = expr.*.lastToken(); - const loc = tree.tokenLocation(tree.tokens.at(expr_last_token).end, suffix_op.rtoken); + const loc = tree.tokenLocation(tree.tokens[expr_last_token].end, suffix_op.rtoken); if (loc.line == 0) { // all on one line const src_has_trailing_comma = trailblk: { const maybe_comma = tree.prevToken(suffix_op.rtoken); - break :trailblk tree.tokens.at(maybe_comma).id == .Comma; + break :trailblk tree.tokens[maybe_comma].id == .Comma; }; if (src_has_trailing_comma) { break :blk 1; // force row size 1 @@ -933,7 +930,7 @@ fn renderExpression( var new_indent = indent + indent_delta; - if (tree.tokens.at(tree.nextToken(lbrace)).id != .MultilineStringLiteralLine) { + if (tree.tokens[tree.nextToken(lbrace)].id != .MultilineStringLiteralLine) { try renderToken(tree, stream, lbrace, new_indent, start_col, Space.Newline); try stream.writeByteNTimes(' ', new_indent); } else { @@ -961,7 +958,7 @@ fn renderExpression( } col = 1; - if (tree.tokens.at(tree.nextToken(comma)).id != .MultilineStringLiteralLine) { + if (tree.tokens[tree.nextToken(comma)].id != .MultilineStringLiteralLine) { try renderToken(tree, stream, comma, new_indent, start_col, Space.Newline); // , } else { try renderToken(tree, stream, comma, new_indent, start_col, Space.None); // , @@ -1188,9 +1185,9 @@ fn renderExpression( var maybe_comma = tree.prevToken(container_decl.lastToken()); // Doc comments for a field may also appear after the comma, eg. // field_name: T, // comment attached to field_name - if (tree.tokens.at(maybe_comma).id == .DocComment) + if (tree.tokens[maybe_comma].id == .DocComment) maybe_comma = tree.prevToken(maybe_comma); - break :blk tree.tokens.at(maybe_comma).id == .Comma; + break :blk tree.tokens[maybe_comma].id == .Comma; }; // Check if the first declaration and the { are on the same line @@ -1285,7 +1282,7 @@ fn renderExpression( const src_has_trailing_comma = blk: { const maybe_comma = tree.prevToken(err_set_decl.rbrace_token); - break :blk tree.tokens.at(maybe_comma).id == .Comma; + break :blk tree.tokens[maybe_comma].id == .Comma; }; if (src_has_trailing_comma) { @@ -1317,7 +1314,7 @@ fn renderExpression( try renderExpression(allocator, stream, tree, indent, start_col, node.*, Space.None); const comma_token = tree.nextToken(node.*.lastToken()); - assert(tree.tokens.at(comma_token).id == .Comma); + assert(tree.tokens[comma_token].id == .Comma); try renderToken(tree, stream, comma_token, indent, start_col, Space.Space); // , try renderExtraNewline(tree, stream, start_col, next_node.*); } else { @@ -1342,7 +1339,7 @@ fn renderExpression( const multiline_str_literal = @fieldParentPtr(ast.Node.MultilineStringLiteral, "base", base); var skip_first_indent = true; - if (tree.tokens.at(multiline_str_literal.firstToken() - 1).id != .LineComment) { + if (tree.tokens[multiline_str_literal.firstToken() - 1].id != .LineComment) { try stream.print("\n", .{}); skip_first_indent = false; } @@ -1372,7 +1369,7 @@ fn renderExpression( if (builtin_call.params.len < 2) break :blk false; const last_node = builtin_call.params.at(builtin_call.params.len - 1).*; const maybe_comma = tree.nextToken(last_node.lastToken()); - break :blk tree.tokens.at(maybe_comma).id == .Comma; + break :blk tree.tokens[maybe_comma].id == .Comma; }; const lparen = tree.nextToken(builtin_call.builtin_token); @@ -1410,7 +1407,7 @@ fn renderExpression( const fn_proto = @fieldParentPtr(ast.Node.FnProto, "base", base); if (fn_proto.visib_token) |visib_token_index| { - const visib_token = tree.tokens.at(visib_token_index); + const visib_token = tree.tokens[visib_token_index]; assert(visib_token.id == .Keyword_pub or visib_token.id == .Keyword_export); try renderToken(tree, stream, visib_token_index, indent, start_col, Space.Space); // pub @@ -1433,7 +1430,7 @@ fn renderExpression( try renderToken(tree, stream, fn_proto.fn_token, indent, start_col, Space.Space); // fn break :blk tree.nextToken(fn_proto.fn_token); }; - assert(tree.tokens.at(lparen).id == .LParen); + assert(tree.tokens[lparen].id == .LParen); const rparen = tree.prevToken( // the first token for the annotation expressions is the left @@ -1449,10 +1446,10 @@ fn renderExpression( .InferErrorSet => |node| tree.prevToken(node.firstToken()), .Invalid => unreachable, }); - assert(tree.tokens.at(rparen).id == .RParen); + assert(tree.tokens[rparen].id == .RParen); const src_params_trailing_comma = blk: { - const maybe_comma = tree.tokens.at(rparen - 1).id; + const maybe_comma = tree.tokens[rparen - 1].id; break :blk maybe_comma == .Comma or maybe_comma == .LineComment; }; @@ -1591,7 +1588,7 @@ fn renderExpression( const src_has_trailing_comma = blk: { const last_node = switch_case.items.at(switch_case.items.len - 1).*; const maybe_comma = tree.nextToken(last_node.lastToken()); - break :blk tree.tokens.at(maybe_comma).id == .Comma; + break :blk tree.tokens[maybe_comma].id == .Comma; }; if (switch_case.items.len == 1 or !src_has_trailing_comma) { @@ -1940,7 +1937,7 @@ fn renderExpression( try renderExpression(allocator, stream, tree, indent_extra, start_col, node, Space.Newline); try stream.writeByteNTimes(' ', indent_once); const comma_or_colon = tree.nextToken(node.lastToken()); - break :blk switch (tree.tokens.at(comma_or_colon).id) { + break :blk switch (tree.tokens[comma_or_colon].id) { .Comma => tree.nextToken(comma_or_colon), else => comma_or_colon, }; @@ -1978,7 +1975,7 @@ fn renderExpression( try renderExpression(allocator, stream, tree, indent_extra, start_col, node, Space.Newline); try stream.writeByteNTimes(' ', indent_once); const comma_or_colon = tree.nextToken(node.lastToken()); - break :blk switch (tree.tokens.at(comma_or_colon).id) { + break :blk switch (tree.tokens[comma_or_colon].id) { .Comma => tree.nextToken(comma_or_colon), else => comma_or_colon, }; @@ -2174,7 +2171,7 @@ fn renderStatement( try renderExpression(allocator, stream, tree, indent, start_col, base, Space.None); const semicolon_index = tree.nextToken(base.lastToken()); - assert(tree.tokens.at(semicolon_index).id == .Semicolon); + assert(tree.tokens[semicolon_index].id == .Semicolon); try renderToken(tree, stream, semicolon_index, indent, start_col, Space.Newline); } else { try renderExpression(allocator, stream, tree, indent, start_col, base, Space.Newline); @@ -2212,13 +2209,13 @@ fn renderTokenOffset( return; } - var token = tree.tokens.at(token_index); + var token = tree.tokens[token_index]; try stream.writeAll(mem.trimRight(u8, tree.tokenSlicePtr(token)[token_skip_bytes..], " ")); if (space == Space.NoComment) return; - var next_token = tree.tokens.at(token_index + 1); + var next_token = tree.tokens[token_index + 1]; if (space == Space.Comma) switch (next_token.id) { .Comma => return renderToken(tree, stream, token_index + 1, indent, start_col, Space.Newline), @@ -2227,7 +2224,7 @@ fn renderTokenOffset( return renderToken(tree, stream, token_index + 1, indent, start_col, Space.Newline); }, else => { - if (token_index + 2 < tree.tokens.len and tree.tokens.at(token_index + 2).id == .MultilineStringLiteralLine) { + if (token_index + 2 < tree.tokens.len and tree.tokens[token_index + 2].id == .MultilineStringLiteralLine) { try stream.writeAll(","); return; } else { @@ -2244,7 +2241,7 @@ fn renderTokenOffset( const loc = tree.tokenLocationPtr(token.end, next_token); if (loc.line == 0) { offset += 1; - next_token = tree.tokens.at(token_index + offset); + next_token = tree.tokens[token_index + offset]; } } @@ -2277,7 +2274,7 @@ fn renderTokenOffset( Space.Newline => { offset += 1; token = next_token; - next_token = tree.tokens.at(token_index + offset); + next_token = tree.tokens[token_index + offset]; if (next_token.id != .LineComment) { try stream.writeByte('\n'); start_col.* = 0; @@ -2296,12 +2293,12 @@ fn renderTokenOffset( try stream.print(" {}", .{mem.trimRight(u8, tree.tokenSlicePtr(next_token), " ")}); offset = 2; token = next_token; - next_token = tree.tokens.at(token_index + offset); + next_token = tree.tokens[token_index + offset]; if (next_token.id != .LineComment) { switch (space) { Space.None, Space.Space => { try stream.writeByte('\n'); - const after_comment_token = tree.tokens.at(token_index + offset); + const after_comment_token = tree.tokens[token_index + offset]; const next_line_indent = switch (after_comment_token.id) { .RParen, .RBrace, .RBracket => indent, else => indent + indent_delta, @@ -2342,7 +2339,7 @@ fn renderTokenOffset( offset += 1; token = next_token; - next_token = tree.tokens.at(token_index + offset); + next_token = tree.tokens[token_index + offset]; if (next_token.id != .LineComment) { switch (space) { Space.Newline => { @@ -2357,7 +2354,7 @@ fn renderTokenOffset( Space.None, Space.Space => { try stream.writeByte('\n'); - const after_comment_token = tree.tokens.at(token_index + offset); + const after_comment_token = tree.tokens[token_index + offset]; const next_line_indent = switch (after_comment_token.id) { .RParen, .RBrace, .RBracket => blk: { if (indent > indent_delta) { From 82225f6568969856ffe9363d2b8b0f39548bd9df Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 19 May 2020 23:07:16 -0400 Subject: [PATCH 02/38] zig fmt parser tests are compiling again --- lib/std/zig/ast.zig | 4 +- lib/std/zig/render.zig | 419 +++++++++++++++++++++++------------------ 2 files changed, 238 insertions(+), 185 deletions(-) diff --git a/lib/std/zig/ast.zig b/lib/std/zig/ast.zig index 8517b5b623..846b6e5041 100644 --- a/lib/std/zig/ast.zig +++ b/lib/std/zig/ast.zig @@ -36,7 +36,7 @@ pub const Tree = struct { return self.tokenSlicePtr(self.tokens[token_index]); } - pub fn tokenSlicePtr(self: *Tree, token: *const Token) []const u8 { + pub fn tokenSlicePtr(self: *Tree, token: Token) []const u8 { return self.source[token.start..token.end]; } @@ -89,7 +89,7 @@ pub const Tree = struct { return self.tokensOnSameLinePtr(self.tokens[token1_index], self.tokens[token2_index]); } - pub fn tokensOnSameLinePtr(self: *Tree, token1: *const Token, token2: *const Token) bool { + pub fn tokensOnSameLinePtr(self: *Tree, token1: Token, token2: Token) bool { return mem.indexOfScalar(u8, self.source[token1.end..token2.start], '\n') == null; } diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index 42531b82cf..f27b6ed738 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -68,7 +68,7 @@ fn renderRoot( tree: *ast.Tree, ) (@TypeOf(stream).Error || Error)!void { // render all the line comments at the beginning of the file - for (tree.tokens) |*token, i| { + for (tree.tokens) |token, i| { if (token.id != .LineComment) break; try stream.print("{}\n", .{mem.trimRight(u8, tree.tokenSlicePtr(token), " ")}); const next_token = &tree.tokens[i + 1]; @@ -321,10 +321,10 @@ fn renderContainerDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree, .DocComment => { const comment = @fieldParentPtr(ast.Node.DocComment, "base", decl); - var it = comment.lines.iterator(0); - while (it.next()) |line_token_index| { - try renderToken(tree, stream, line_token_index.*, indent, start_col, .Newline); - if (it.peek()) |_| { + var it = comment.lines.first; + while (it) |node| : (it = node.next) { + try renderToken(tree, stream, node.data, indent, start_col, .Newline); + if (node.next != null) { try stream.writeByteNTimes(' ', indent); } } @@ -355,20 +355,21 @@ fn renderExpression( try renderToken(tree, stream, tree.nextToken(label), indent, start_col, Space.Space); } - if (block.statements.len == 0) { + if (block.statements.first == null) { try renderToken(tree, stream, block.lbrace, indent + indent_delta, start_col, Space.None); return renderToken(tree, stream, block.rbrace, indent, start_col, space); } else { const block_indent = indent + indent_delta; try renderToken(tree, stream, block.lbrace, block_indent, start_col, Space.Newline); - var it = block.statements.iterator(0); - while (it.next()) |statement| { + var it = block.statements.first; + while (it) |statement_node| : (it = statement_node.next) { + const statement = statement_node.data; try stream.writeByteNTimes(' ', block_indent); - try renderStatement(allocator, stream, tree, block_indent, start_col, statement.*); + try renderStatement(allocator, stream, tree, block_indent, start_col, statement); - if (it.peek()) |next_statement| { - try renderExtraNewline(tree, stream, start_col, next_statement.*); + if (statement_node.next) |next_statement| { + try renderExtraNewline(tree, stream, start_col, next_statement.data); } } @@ -620,7 +621,7 @@ fn renderExpression( const lparen = tree.nextToken(suffix_op.lhs.node.lastToken()); - if (call_info.params.len == 0) { + if (call_info.params.first == null) { try renderToken(tree, stream, lparen, indent, start_col, Space.None); return renderToken(tree, stream, suffix_op.rtoken, indent, start_col, space); } @@ -634,24 +635,26 @@ fn renderExpression( const new_indent = indent + indent_delta; try renderToken(tree, stream, lparen, new_indent, start_col, Space.Newline); - var it = call_info.params.iterator(0); + var it = call_info.params.first; while (true) { - const param_node = it.next().?; + const param_node_node = it.?; + it = param_node_node.next; + const param_node = param_node_node.data; - const param_node_new_indent = if (param_node.*.id == .MultilineStringLiteral) blk: { + const param_node_new_indent = if (param_node.id == .MultilineStringLiteral) blk: { break :blk indent; } else blk: { try stream.writeByteNTimes(' ', new_indent); break :blk new_indent; }; - if (it.peek()) |next_node| { - try renderExpression(allocator, stream, tree, param_node_new_indent, start_col, param_node.*, Space.None); - const comma = tree.nextToken(param_node.*.lastToken()); + if (it) |next_node| { + try renderExpression(allocator, stream, tree, param_node_new_indent, start_col, param_node, Space.None); + const comma = tree.nextToken(param_node.lastToken()); try renderToken(tree, stream, comma, new_indent, start_col, Space.Newline); // , - try renderExtraNewline(tree, stream, start_col, next_node.*); + try renderExtraNewline(tree, stream, start_col, next_node.data); } else { - try renderExpression(allocator, stream, tree, param_node_new_indent, start_col, param_node.*, Space.Comma); + try renderExpression(allocator, stream, tree, param_node_new_indent, start_col, param_node, Space.Comma); try stream.writeByteNTimes(' ', indent); return renderToken(tree, stream, suffix_op.rtoken, indent, start_col, space); } @@ -660,12 +663,13 @@ fn renderExpression( try renderToken(tree, stream, lparen, indent, start_col, Space.None); // ( - var it = call_info.params.iterator(0); - while (it.next()) |param_node| { - try renderExpression(allocator, stream, tree, indent, start_col, param_node.*, Space.None); + var it = call_info.params.first; + while (it) |param_node_node| : (it = param_node_node.next) { + const param_node = param_node_node.data; + try renderExpression(allocator, stream, tree, indent, start_col, param_node, Space.None); - if (it.peek() != null) { - const comma = tree.nextToken(param_node.*.lastToken()); + if (param_node_node.next != null) { + const comma = tree.nextToken(param_node.lastToken()); try renderToken(tree, stream, comma, indent, start_col, Space.Space); } } @@ -736,7 +740,7 @@ fn renderExpression( .node => |node| tree.nextToken(node.lastToken()), }; - if (field_inits.len == 0) { + if (field_inits.first == null) { switch (suffix_op.lhs) { .dot => |dot| try renderToken(tree, stream, dot, indent, start_col, Space.None), .node => |node| try renderExpression(allocator, stream, tree, indent, start_col, node, Space.None), @@ -757,18 +761,19 @@ fn renderExpression( const expr_outputs_one_line = blk: { // render field expressions until a LF is found - var it = field_inits.iterator(0); - while (it.next()) |field_init| { + var it = field_inits.first; + while (it) |field_init_node| : (it = field_init_node.next) { + const field_init = field_init_node.data; var find_stream = FindByteOutStream.init('\n'); var dummy_col: usize = 0; - try renderExpression(allocator, find_stream.outStream(), tree, 0, &dummy_col, field_init.*, Space.None); + try renderExpression(allocator, find_stream.outStream(), tree, 0, &dummy_col, field_init, Space.None); if (find_stream.byte_found) break :blk false; } break :blk true; }; - if (field_inits.len == 1) blk: { - const field_init = field_inits.at(0).*.cast(ast.Node.FieldInitializer).?; + if (field_inits.first != null and field_inits.first.?.next == null) blk: { + const field_init = field_inits.first.?.data.cast(ast.Node.FieldInitializer).?; if (field_init.expr.cast(ast.Node.SuffixOp)) |nested_suffix_op| { if (nested_suffix_op.op == .StructInitializer) { @@ -798,15 +803,16 @@ fn renderExpression( } try renderToken(tree, stream, lbrace, indent, start_col, Space.Space); - var it = field_inits.iterator(0); - while (it.next()) |field_init| { - if (it.peek() != null) { - try renderExpression(allocator, stream, tree, indent, start_col, field_init.*, Space.None); + var it = field_inits.first; + while (it) |field_init_node| : (it = field_init_node.next) { + const field_init = field_init_node.data; + if (field_init_node.next != null) { + try renderExpression(allocator, stream, tree, indent, start_col, field_init, Space.None); - const comma = tree.nextToken(field_init.*.lastToken()); + const comma = tree.nextToken(field_init.lastToken()); try renderToken(tree, stream, comma, indent, start_col, Space.Space); } else { - try renderExpression(allocator, stream, tree, indent, start_col, field_init.*, Space.Space); + try renderExpression(allocator, stream, tree, indent, start_col, field_init, Space.Space); } } @@ -821,19 +827,20 @@ fn renderExpression( } try renderToken(tree, stream, lbrace, new_indent, start_col, Space.Newline); - var it = field_inits.iterator(0); - while (it.next()) |field_init| { + var it = field_inits.first; + while (it) |field_init_node| : (it = field_init_node.next) { + const field_init = field_init_node.data; try stream.writeByteNTimes(' ', new_indent); - if (it.peek()) |next_field_init| { - try renderExpression(allocator, stream, tree, new_indent, start_col, field_init.*, Space.None); + if (field_init_node.next) |next_field_init| { + try renderExpression(allocator, stream, tree, new_indent, start_col, field_init, Space.None); - const comma = tree.nextToken(field_init.*.lastToken()); + const comma = tree.nextToken(field_init.lastToken()); try renderToken(tree, stream, comma, new_indent, start_col, Space.Newline); - try renderExtraNewline(tree, stream, start_col, next_field_init.*); + try renderExtraNewline(tree, stream, start_col, next_field_init.data); } else { - try renderExpression(allocator, stream, tree, new_indent, start_col, field_init.*, Space.Comma); + try renderExpression(allocator, stream, tree, new_indent, start_col, field_init, Space.Comma); } } @@ -847,7 +854,7 @@ fn renderExpression( .node => |node| tree.nextToken(node.lastToken()), }; - if (exprs.len == 0) { + if (exprs.first == null) { switch (suffix_op.lhs) { .dot => |dot| try renderToken(tree, stream, dot, indent, start_col, Space.None), .node => |node| try renderExpression(allocator, stream, tree, indent, start_col, node, Space.None), @@ -855,16 +862,17 @@ fn renderExpression( try renderToken(tree, stream, lbrace, indent, start_col, Space.None); return renderToken(tree, stream, suffix_op.rtoken, indent, start_col, space); } - if (exprs.len == 1 and tree.tokens[exprs.at(0).*.lastToken() + 1].id == .RBrace) { - const expr = exprs.at(0).*; - - switch (suffix_op.lhs) { - .dot => |dot| try renderToken(tree, stream, dot, indent, start_col, Space.None), - .node => |node| try renderExpression(allocator, stream, tree, indent, start_col, node, Space.None), + if (exprs.first) |first_expr_node| { + const expr = first_expr_node.data; + if (first_expr_node.next == null and tree.tokens[expr.lastToken() + 1].id == .RBrace) { + switch (suffix_op.lhs) { + .dot => |dot| try renderToken(tree, stream, dot, indent, start_col, Space.None), + .node => |node| try renderExpression(allocator, stream, tree, indent, start_col, node, Space.None), + } + try renderToken(tree, stream, lbrace, indent, start_col, Space.None); + try renderExpression(allocator, stream, tree, indent, start_col, expr, Space.None); + return renderToken(tree, stream, suffix_op.rtoken, indent, start_col, space); } - try renderToken(tree, stream, lbrace, indent, start_col, Space.None); - try renderExpression(allocator, stream, tree, indent, start_col, expr, Space.None); - return renderToken(tree, stream, suffix_op.rtoken, indent, start_col, space); } switch (suffix_op.lhs) { @@ -875,16 +883,18 @@ fn renderExpression( // scan to find row size const maybe_row_size: ?usize = blk: { var count: usize = 1; - var it = exprs.iterator(0); + var it = exprs.first; while (true) { - const expr = it.next().?.*; - if (it.peek()) |next_expr| { - const expr_last_token = expr.*.lastToken() + 1; - const loc = tree.tokenLocation(tree.tokens[expr_last_token].end, next_expr.*.firstToken()); + const expr_node = it.?; + it = expr_node.next; + const expr = expr_node.data; + if (expr_node.next) |next_expr| { + const expr_last_token = expr.lastToken() + 1; + const loc = tree.tokenLocation(tree.tokens[expr_last_token].end, next_expr.data.firstToken()); if (loc.line != 0) break :blk count; count += 1; } else { - const expr_last_token = expr.*.lastToken(); + const expr_last_token = expr.lastToken(); const loc = tree.tokenLocation(tree.tokens[expr_last_token].end, suffix_op.rtoken); if (loc.line == 0) { // all on one line @@ -905,7 +915,8 @@ fn renderExpression( if (maybe_row_size) |row_size| { // A place to store the width of each expression and its column's maximum - var widths = try allocator.alloc(usize, exprs.len + row_size); + const exprs_len = countLen(exprs.first); + var widths = try allocator.alloc(usize, exprs_len + row_size); defer allocator.free(widths); mem.set(usize, widths, 0); @@ -915,13 +926,14 @@ fn renderExpression( // Null stream for counting the printed length of each expression var counting_stream = std.io.countingOutStream(std.io.null_out_stream); - var it = exprs.iterator(0); + var it = exprs.first; var i: usize = 0; - while (it.next()) |expr| : (i += 1) { + while (it) |expr_node| : ({i += 1; it = expr_node.next;}) { + const expr = expr_node.data; counting_stream.bytes_written = 0; var dummy_col: usize = 0; - try renderExpression(allocator, counting_stream.outStream(), tree, indent, &dummy_col, expr.*, Space.None); + try renderExpression(allocator, counting_stream.outStream(), tree, indent, &dummy_col, expr, Space.None); const width = @intCast(usize, counting_stream.bytes_written); const col = i % row_size; column_widths[col] = std.math.max(column_widths[col], width); @@ -938,14 +950,16 @@ fn renderExpression( try renderToken(tree, stream, lbrace, new_indent, start_col, Space.None); } - it.set(0); + it = exprs.first; i = 0; var col: usize = 1; - while (it.next()) |expr| : (i += 1) { - if (it.peek()) |next_expr| { - try renderExpression(allocator, stream, tree, new_indent, start_col, expr.*, Space.None); + var last_node = it.?; + while (it) |expr_node| : ({i += 1; it = expr_node.next; last_node = expr_node;}) { + const expr = expr_node.data; + if (expr_node.next) |next_expr| { + try renderExpression(allocator, stream, tree, new_indent, start_col, expr, Space.None); - const comma = tree.nextToken(expr.*.lastToken()); + const comma = tree.nextToken(expr.lastToken()); if (col != row_size) { try renderToken(tree, stream, comma, new_indent, start_col, Space.Space); // , @@ -964,29 +978,29 @@ fn renderExpression( try renderToken(tree, stream, comma, new_indent, start_col, Space.None); // , } - try renderExtraNewline(tree, stream, start_col, next_expr.*); - if (next_expr.*.id != .MultilineStringLiteral) { + try renderExtraNewline(tree, stream, start_col, next_expr.data); + if (next_expr.data.id != .MultilineStringLiteral) { try stream.writeByteNTimes(' ', new_indent); } } else { - try renderExpression(allocator, stream, tree, new_indent, start_col, expr.*, Space.Comma); // , + try renderExpression(allocator, stream, tree, new_indent, start_col, expr, Space.Comma); // , } } - const last_node = it.prev().?; - if (last_node.*.id != .MultilineStringLiteral) { + if (last_node.data.id != .MultilineStringLiteral) { try stream.writeByteNTimes(' ', indent); } return renderToken(tree, stream, suffix_op.rtoken, indent, start_col, space); } else { try renderToken(tree, stream, lbrace, indent, start_col, Space.Space); - var it = exprs.iterator(0); - while (it.next()) |expr| { - if (it.peek()) |next_expr| { - try renderExpression(allocator, stream, tree, indent, start_col, expr.*, Space.None); - const comma = tree.nextToken(expr.*.lastToken()); + var it = exprs.first; + while (it) |expr_node| : (it = expr_node.next) { + const expr = expr_node.data; + if (expr_node.next) |next_expr| { + try renderExpression(allocator, stream, tree, indent, start_col, expr, Space.None); + const comma = tree.nextToken(expr.lastToken()); try renderToken(tree, stream, comma, indent, start_col, Space.Space); // , } else { - try renderExpression(allocator, stream, tree, indent, start_col, expr.*, Space.Space); + try renderExpression(allocator, stream, tree, indent, start_col, expr, Space.Space); } } @@ -1176,7 +1190,7 @@ fn renderExpression( }, } - if (container_decl.fields_and_decls.len == 0) { + if (container_decl.fields_and_decls.first == null) { try renderToken(tree, stream, container_decl.lbrace_token, indent + indent_delta, start_col, Space.None); // { return renderToken(tree, stream, container_decl.rbrace_token, indent, start_col, space); // } } @@ -1193,15 +1207,16 @@ fn renderExpression( // Check if the first declaration and the { are on the same line const src_has_newline = !tree.tokensOnSameLine( container_decl.lbrace_token, - container_decl.fields_and_decls.at(0).*.firstToken(), + container_decl.fields_and_decls.first.?.data.firstToken(), ); // We can only print all the elements in-line if all the // declarations inside are fields const src_has_only_fields = blk: { - var it = container_decl.fields_and_decls.iterator(0); - while (it.next()) |decl| { - if (decl.*.id != .ContainerField) break :blk false; + var it = container_decl.fields_and_decls.first; + while (it) |decl_node| : (it = decl_node.next) { + const decl = decl_node.data; + if (decl.id != .ContainerField) break :blk false; } break :blk true; }; @@ -1211,13 +1226,14 @@ fn renderExpression( const new_indent = indent + indent_delta; try renderToken(tree, stream, container_decl.lbrace_token, new_indent, start_col, .Newline); // { - var it = container_decl.fields_and_decls.iterator(0); - while (it.next()) |decl| { + var it = container_decl.fields_and_decls.first; + while (it) |decl_node| : (it = decl_node.next) { + const decl = decl_node.data; try stream.writeByteNTimes(' ', new_indent); - try renderContainerDecl(allocator, stream, tree, new_indent, start_col, decl.*, .Newline); + try renderContainerDecl(allocator, stream, tree, new_indent, start_col, decl, .Newline); - if (it.peek()) |next_decl| { - try renderExtraNewline(tree, stream, start_col, next_decl.*); + if (decl_node.next) |next_decl| { + try renderExtraNewline(tree, stream, start_col, next_decl.data); } } @@ -1230,10 +1246,11 @@ fn renderExpression( const new_indent = indent + indent_delta; try stream.writeByteNTimes(' ', new_indent); - var it = container_decl.fields_and_decls.iterator(0); - while (it.next()) |decl| { - const space_after_decl: Space = if (it.peek() == null) .Newline else .Space; - try renderContainerDecl(allocator, stream, tree, new_indent, start_col, decl.*, space_after_decl); + var it = container_decl.fields_and_decls.first; + while (it) |decl_node| : (it = decl_node.next) { + const decl = decl_node.data; + const space_after_decl: Space = if (decl_node.next == null) .Newline else .Space; + try renderContainerDecl(allocator, stream, tree, new_indent, start_col, decl, space_after_decl); } try stream.writeByteNTimes(' ', indent); @@ -1241,9 +1258,10 @@ fn renderExpression( // All the declarations on the same line try renderToken(tree, stream, container_decl.lbrace_token, indent, start_col, .Space); // { - var it = container_decl.fields_and_decls.iterator(0); - while (it.next()) |decl| { - try renderContainerDecl(allocator, stream, tree, indent, start_col, decl.*, .Space); + var it = container_decl.fields_and_decls.first; + while (it) |decl_node| : (it = decl_node.next) { + const decl = decl_node.data; + try renderContainerDecl(allocator, stream, tree, indent, start_col, decl, .Space); } } @@ -1255,14 +1273,14 @@ fn renderExpression( const lbrace = tree.nextToken(err_set_decl.error_token); - if (err_set_decl.decls.len == 0) { + if (err_set_decl.decls.first == null) { try renderToken(tree, stream, err_set_decl.error_token, indent, start_col, Space.None); try renderToken(tree, stream, lbrace, indent, start_col, Space.None); return renderToken(tree, stream, err_set_decl.rbrace_token, indent, start_col, space); } - if (err_set_decl.decls.len == 1) blk: { - const node = err_set_decl.decls.at(0).*; + if (err_set_decl.decls.first.?.next == null) blk: { + const node = err_set_decl.decls.first.?.data; // if there are any doc comments or same line comments // don't try to put it all on one line @@ -1289,17 +1307,18 @@ fn renderExpression( try renderToken(tree, stream, lbrace, indent, start_col, Space.Newline); // { const new_indent = indent + indent_delta; - var it = err_set_decl.decls.iterator(0); - while (it.next()) |node| { + var it = err_set_decl.decls.first; + while (it) |node_node| : (it = node_node.next) { + const node = node_node.data; try stream.writeByteNTimes(' ', new_indent); - if (it.peek()) |next_node| { - try renderExpression(allocator, stream, tree, new_indent, start_col, node.*, Space.None); - try renderToken(tree, stream, tree.nextToken(node.*.lastToken()), new_indent, start_col, Space.Newline); // , + if (node_node.next) |next_node| { + try renderExpression(allocator, stream, tree, new_indent, start_col, node, Space.None); + try renderToken(tree, stream, tree.nextToken(node.lastToken()), new_indent, start_col, Space.Newline); // , - try renderExtraNewline(tree, stream, start_col, next_node.*); + try renderExtraNewline(tree, stream, start_col, next_node.data); } else { - try renderExpression(allocator, stream, tree, new_indent, start_col, node.*, Space.Comma); + try renderExpression(allocator, stream, tree, new_indent, start_col, node, Space.Comma); } } @@ -1308,17 +1327,18 @@ fn renderExpression( } else { try renderToken(tree, stream, lbrace, indent, start_col, Space.Space); // { - var it = err_set_decl.decls.iterator(0); - while (it.next()) |node| { - if (it.peek()) |next_node| { - try renderExpression(allocator, stream, tree, indent, start_col, node.*, Space.None); + var it = err_set_decl.decls.first; + while (it) |node_node| : (it = node_node.next) { + const node = node_node.data; + if (node_node.next) |next_node| { + try renderExpression(allocator, stream, tree, indent, start_col, node, Space.None); - const comma_token = tree.nextToken(node.*.lastToken()); + const comma_token = tree.nextToken(node.lastToken()); assert(tree.tokens[comma_token].id == .Comma); try renderToken(tree, stream, comma_token, indent, start_col, Space.Space); // , - try renderExtraNewline(tree, stream, start_col, next_node.*); + try renderExtraNewline(tree, stream, start_col, next_node.data); } else { - try renderExpression(allocator, stream, tree, indent, start_col, node.*, Space.Space); + try renderExpression(allocator, stream, tree, indent, start_col, node, Space.Space); } } @@ -1344,9 +1364,9 @@ fn renderExpression( skip_first_indent = false; } - var i: usize = 0; - while (i < multiline_str_literal.lines.len) : (i += 1) { - const t = multiline_str_literal.lines.at(i).*; + var it = multiline_str_literal.lines.first; + while (it) |t_node| : (it = t_node.next) { + const t = t_node.data; if (!skip_first_indent) { try stream.writeByteNTimes(' ', indent + indent_delta); } @@ -1366,8 +1386,12 @@ fn renderExpression( try renderToken(tree, stream, builtin_call.builtin_token, indent, start_col, Space.None); // @name const src_params_trailing_comma = blk: { - if (builtin_call.params.len < 2) break :blk false; - const last_node = builtin_call.params.at(builtin_call.params.len - 1).*; + if (builtin_call.params.first == null or + builtin_call.params.first.?.next == null) + { + break :blk false; + } + const last_node = findLast(builtin_call.params.first.?).data; const maybe_comma = tree.nextToken(last_node.lastToken()); break :blk tree.tokens[maybe_comma].id == .Comma; }; @@ -1378,12 +1402,13 @@ fn renderExpression( try renderToken(tree, stream, lparen, indent, start_col, Space.None); // ( // render all on one line, no trailing comma - var it = builtin_call.params.iterator(0); - while (it.next()) |param_node| { - try renderExpression(allocator, stream, tree, indent, start_col, param_node.*, Space.None); + var it = builtin_call.params.first; + while (it) |param_node_node| : (it = param_node_node.next) { + const param_node = param_node_node.data; + try renderExpression(allocator, stream, tree, indent, start_col, param_node, Space.None); - if (it.peek() != null) { - const comma_token = tree.nextToken(param_node.*.lastToken()); + if (param_node_node.next != null) { + const comma_token = tree.nextToken(param_node.lastToken()); try renderToken(tree, stream, comma_token, indent, start_col, Space.Space); // , } } @@ -1392,10 +1417,11 @@ fn renderExpression( const new_indent = indent + indent_delta; try renderToken(tree, stream, lparen, new_indent, start_col, Space.Newline); // ( - var it = builtin_call.params.iterator(0); - while (it.next()) |param_node| { + var it = builtin_call.params.first; + while (it) |param_node_node| : (it = param_node_node.next) { + const param_node = param_node_node.data; try stream.writeByteNTimes(' ', new_indent); - try renderExpression(allocator, stream, tree, indent, start_col, param_node.*, Space.Comma); + try renderExpression(allocator, stream, tree, indent, start_col, param_node, Space.Comma); } try stream.writeByteNTimes(' ', indent); } @@ -1457,12 +1483,13 @@ fn renderExpression( try renderToken(tree, stream, lparen, indent, start_col, Space.None); // ( // render all on one line, no trailing comma - var it = fn_proto.params.iterator(0); - while (it.next()) |param_decl_node| { - try renderParamDecl(allocator, stream, tree, indent, start_col, param_decl_node.*, Space.None); + var it = fn_proto.params.first; + while (it) |param_decl_node_node| : (it = param_decl_node_node.next) { + const param_decl_node = param_decl_node_node.data; + try renderParamDecl(allocator, stream, tree, indent, start_col, param_decl_node, Space.None); - if (it.peek() != null) { - const comma = tree.nextToken(param_decl_node.*.lastToken()); + if (param_decl_node_node.next != null) { + const comma = tree.nextToken(param_decl_node.lastToken()); try renderToken(tree, stream, comma, indent, start_col, Space.Space); // , } } @@ -1471,10 +1498,11 @@ fn renderExpression( const new_indent = indent + indent_delta; try renderToken(tree, stream, lparen, new_indent, start_col, Space.Newline); // ( - var it = fn_proto.params.iterator(0); - while (it.next()) |param_decl_node| { + var it = fn_proto.params.first; + while (it) |param_decl_node_node| : (it = param_decl_node_node.next) { + const param_decl_node = param_decl_node_node.data; try stream.writeByteNTimes(' ', new_indent); - try renderParamDecl(allocator, stream, tree, new_indent, start_col, param_decl_node.*, Space.Comma); + try renderParamDecl(allocator, stream, tree, new_indent, start_col, param_decl_node, Space.Comma); } try stream.writeByteNTimes(' ', indent); } @@ -1553,7 +1581,7 @@ fn renderExpression( const rparen = tree.nextToken(switch_node.expr.lastToken()); const lbrace = tree.nextToken(rparen); - if (switch_node.cases.len == 0) { + if (switch_node.cases.first == null) { try renderExpression(allocator, stream, tree, indent, start_col, switch_node.expr, Space.None); try renderToken(tree, stream, rparen, indent, start_col, Space.Space); // ) try renderToken(tree, stream, lbrace, indent, start_col, Space.None); // { @@ -1567,13 +1595,14 @@ fn renderExpression( try renderToken(tree, stream, rparen, indent, start_col, Space.Space); // ) try renderToken(tree, stream, lbrace, new_indent, start_col, Space.Newline); // { - var it = switch_node.cases.iterator(0); - while (it.next()) |node| { + var it = switch_node.cases.first; + while (it) |node_node| : (it = node_node.next) { + const node = node_node.data; try stream.writeByteNTimes(' ', new_indent); - try renderExpression(allocator, stream, tree, new_indent, start_col, node.*, Space.Comma); + try renderExpression(allocator, stream, tree, new_indent, start_col, node, Space.Comma); - if (it.peek()) |next_node| { - try renderExtraNewline(tree, stream, start_col, next_node.*); + if (node_node.next) |next_node| { + try renderExtraNewline(tree, stream, start_col, next_node.data); } } @@ -1584,38 +1613,40 @@ fn renderExpression( .SwitchCase => { const switch_case = @fieldParentPtr(ast.Node.SwitchCase, "base", base); - assert(switch_case.items.len != 0); + assert(switch_case.items.first != null); const src_has_trailing_comma = blk: { - const last_node = switch_case.items.at(switch_case.items.len - 1).*; + const last_node = findLast(switch_case.items.first.?).data; const maybe_comma = tree.nextToken(last_node.lastToken()); break :blk tree.tokens[maybe_comma].id == .Comma; }; - if (switch_case.items.len == 1 or !src_has_trailing_comma) { - var it = switch_case.items.iterator(0); - while (it.next()) |node| { - if (it.peek()) |next_node| { - try renderExpression(allocator, stream, tree, indent, start_col, node.*, Space.None); + if (switch_case.items.first.?.next == null or !src_has_trailing_comma) { + var it = switch_case.items.first; + while (it) |node_node| : (it = node_node.next) { + const node = node_node.data; + if (node_node.next) |next_node| { + try renderExpression(allocator, stream, tree, indent, start_col, node, Space.None); - const comma_token = tree.nextToken(node.*.lastToken()); + const comma_token = tree.nextToken(node.lastToken()); try renderToken(tree, stream, comma_token, indent, start_col, Space.Space); // , - try renderExtraNewline(tree, stream, start_col, next_node.*); + try renderExtraNewline(tree, stream, start_col, next_node.data); } else { - try renderExpression(allocator, stream, tree, indent, start_col, node.*, Space.Space); + try renderExpression(allocator, stream, tree, indent, start_col, node, Space.Space); } } } else { - var it = switch_case.items.iterator(0); - while (it.next()) |node| { - if (it.peek()) |next_node| { - try renderExpression(allocator, stream, tree, indent, start_col, node.*, Space.None); + var it = switch_case.items.first; + while (it) |node_node| : (it = node_node.next) { + const node = node_node.data; + if (node_node.next) |next_node| { + try renderExpression(allocator, stream, tree, indent, start_col, node, Space.None); - const comma_token = tree.nextToken(node.*.lastToken()); + const comma_token = tree.nextToken(node.lastToken()); try renderToken(tree, stream, comma_token, indent, start_col, Space.Newline); // , - try renderExtraNewline(tree, stream, start_col, next_node.*); + try renderExtraNewline(tree, stream, start_col, next_node.data); try stream.writeByteNTimes(' ', indent); } else { - try renderExpression(allocator, stream, tree, indent, start_col, node.*, Space.Comma); + try renderExpression(allocator, stream, tree, indent, start_col, node, Space.Comma); try stream.writeByteNTimes(' ', indent); } } @@ -1887,7 +1918,7 @@ fn renderExpression( try renderToken(tree, stream, tree.nextToken(asm_node.asm_token), indent, start_col, Space.None); // ( } - if (asm_node.outputs.len == 0 and asm_node.inputs.len == 0 and asm_node.clobbers.len == 0) { + if (asm_node.outputs.first == null and asm_node.inputs.first == null and asm_node.clobbers.first == null) { try renderExpression(allocator, stream, tree, indent, start_col, asm_node.template, Space.None); return renderToken(tree, stream, asm_node.rparen, indent, start_col, space); } @@ -1907,7 +1938,7 @@ fn renderExpression( const colon1 = tree.nextToken(asm_node.template.lastToken()); const indent_extra = indent_once + 2; - const colon2 = if (asm_node.outputs.len == 0) blk: { + const colon2 = if (asm_node.outputs.first == null) blk: { try renderToken(tree, stream, colon1, indent, start_col, Space.Newline); // : try stream.writeByteNTimes(' ', indent_once); @@ -1915,21 +1946,23 @@ fn renderExpression( } else blk: { try renderToken(tree, stream, colon1, indent, start_col, Space.Space); // : - var it = asm_node.outputs.iterator(0); + var it = asm_node.outputs.first; while (true) { - const asm_output = it.next().?; - const node = &(asm_output.*).base; + const asm_output_node = it.?; + it = asm_output_node.next; + const asm_output = asm_output_node.data; + const node = &asm_output.base; - if (it.peek()) |next_asm_output| { + if (asm_output_node.next) |next_asm_output| { try renderExpression(allocator, stream, tree, indent_extra, start_col, node, Space.None); - const next_node = &(next_asm_output.*).base; + const next_node = &next_asm_output.data.base; - const comma = tree.prevToken(next_asm_output.*.firstToken()); + const comma = tree.prevToken(next_asm_output.data.firstToken()); try renderToken(tree, stream, comma, indent_extra, start_col, Space.Newline); // , try renderExtraNewline(tree, stream, start_col, next_node); try stream.writeByteNTimes(' ', indent_extra); - } else if (asm_node.inputs.len == 0 and asm_node.clobbers.len == 0) { + } else if (asm_node.inputs.first == null and asm_node.clobbers.first == null) { try renderExpression(allocator, stream, tree, indent_extra, start_col, node, Space.Newline); try stream.writeByteNTimes(' ', indent); return renderToken(tree, stream, asm_node.rparen, indent, start_col, space); @@ -1945,7 +1978,7 @@ fn renderExpression( } }; - const colon3 = if (asm_node.inputs.len == 0) blk: { + const colon3 = if (asm_node.inputs.first == null) blk: { try renderToken(tree, stream, colon2, indent, start_col, Space.Newline); // : try stream.writeByteNTimes(' ', indent_once); @@ -1953,21 +1986,22 @@ fn renderExpression( } else blk: { try renderToken(tree, stream, colon2, indent, start_col, Space.Space); // : - var it = asm_node.inputs.iterator(0); + var it = asm_node.inputs.first; while (true) { - const asm_input = it.next().?; - const node = &(asm_input.*).base; + const asm_input_node = it.?; + it = asm_input_node.next; + const node = &asm_input_node.data.base; - if (it.peek()) |next_asm_input| { + if (it) |next_asm_input| { try renderExpression(allocator, stream, tree, indent_extra, start_col, node, Space.None); - const next_node = &(next_asm_input.*).base; + const next_node = &next_asm_input.data.base; - const comma = tree.prevToken(next_asm_input.*.firstToken()); + const comma = tree.prevToken(next_asm_input.data.firstToken()); try renderToken(tree, stream, comma, indent_extra, start_col, Space.Newline); // , try renderExtraNewline(tree, stream, start_col, next_node); try stream.writeByteNTimes(' ', indent_extra); - } else if (asm_node.clobbers.len == 0) { + } else if (asm_node.clobbers.first == null) { try renderExpression(allocator, stream, tree, indent_extra, start_col, node, Space.Newline); try stream.writeByteNTimes(' ', indent); return renderToken(tree, stream, asm_node.rparen, indent, start_col, space); // ) @@ -1985,11 +2019,13 @@ fn renderExpression( try renderToken(tree, stream, colon3, indent, start_col, Space.Space); // : - var it = asm_node.clobbers.iterator(0); + var it = asm_node.clobbers.first; while (true) { - const clobber_node = it.next().?.*; + const clobber_node_node = it.?; + it = clobber_node_node.next; + const clobber_node = clobber_node_node.data; - if (it.peek() == null) { + if (it == null) { try renderExpression(allocator, stream, tree, indent_extra, start_col, clobber_node, Space.Newline); try stream.writeByteNTimes(' ', indent); return renderToken(tree, stream, asm_node.rparen, indent, start_col, space); @@ -2401,14 +2437,15 @@ fn renderDocComments( start_col: *usize, ) (@TypeOf(stream).Error || Error)!void { const comment = node.doc_comments orelse return; - var it = comment.lines.iterator(0); + var it = comment.lines.first; const first_token = node.firstToken(); - while (it.next()) |line_token_index| { - if (line_token_index.* < first_token) { - try renderToken(tree, stream, line_token_index.*, indent, start_col, Space.Newline); + while (it) |line_token_index_node| : (it = line_token_index_node.next) { + const line_token_index = line_token_index_node.data; + if (line_token_index < first_token) { + try renderToken(tree, stream, line_token_index, indent, start_col, Space.Newline); try stream.writeByteNTimes(' ', indent); } else { - try renderToken(tree, stream, line_token_index.*, indent, start_col, Space.NoComment); + try renderToken(tree, stream, line_token_index, indent, start_col, Space.NoComment); try stream.writeAll("\n"); try stream.writeByteNTimes(' ', indent); } @@ -2473,3 +2510,19 @@ fn copyFixingWhitespace(stream: var, slice: []const u8) @TypeOf(stream).Error!vo else => try stream.writeByte(byte), }; } + +fn countLen(node: ?*std.SinglyLinkedList(*ast.Node).Node) usize { + var count: usize = 0; + var it = node; + while (it) |n| : (it = n.next) { + count += 1; + } + return count; +} + +fn findLast(node: *std.SinglyLinkedList(*ast.Node).Node) *std.SinglyLinkedList(*ast.Node).Node { + var it = node; + while (true) { + it = it.next orelse return it; + } +} From 4617c5907a4b7933947a3223d19d637d143fd705 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 20 May 2020 00:19:00 -0400 Subject: [PATCH 03/38] fix zig fmt test regressions --- lib/std/zig/parse.zig | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/std/zig/parse.zig b/lib/std/zig/parse.zig index d723ac934c..7abd3b300c 100644 --- a/lib/std/zig/parse.zig +++ b/lib/std/zig/parse.zig @@ -3296,11 +3296,9 @@ const Parser = struct { .index = p.tok_i, .ptr = &p.tokens[p.tok_i], }; - if (p.tokens[p.tok_i].id == .Eof) { - return result; - } p.tok_i += 1; assert(result.ptr.id != .LineComment); + if (p.tok_i >= p.tokens.len) return result; while (true) { const next_tok = p.tokens[p.tok_i]; From 8c10178a1e209af53d28cc70343d0ce908bccc1b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 20 May 2020 13:53:53 -0400 Subject: [PATCH 04/38] golly jeepers it's taking a long time to update translate-c --- lib/std/linked_list.zig | 30 ++ lib/std/zig/render.zig | 23 +- src-self-hosted/main.zig | 14 +- src-self-hosted/translate_c.zig | 539 ++++++++++++++++++-------------- 4 files changed, 339 insertions(+), 267 deletions(-) diff --git a/lib/std/linked_list.zig b/lib/std/linked_list.zig index 3152413d1c..c4c56df181 100644 --- a/lib/std/linked_list.zig +++ b/lib/std/linked_list.zig @@ -49,6 +49,26 @@ pub fn SinglyLinkedList(comptime T: type) type { node.next = next_node.next; return next_node; } + + /// Iterate over the singly-linked list from this node, until the final node is found. + /// This operation is O(N). + pub fn findLast(node: *Node) *Node { + var it = node; + while (true) { + it = it.next orelse return it; + } + } + + /// Iterate over each next node, returning the count of all nodes except the starting one. + /// This operation is O(N). + pub fn countChildren(node: *const Node) usize { + var count: usize = 0; + var it: ?*const Node = node; + while (it) |n| : (it = n.next) { + count += 1; + } + return count; + } }; first: ?*Node = null, @@ -87,6 +107,16 @@ pub fn SinglyLinkedList(comptime T: type) type { list.first = first.next; return first; } + + /// Iterate over all nodes, returning the count. + /// This operation is O(N). + pub fn len(list: Self) usize { + if (list.first) |n| { + return 1 + n.countChildren(); + } else { + return 0; + } + } }; } diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index f27b6ed738..bcd7ef4887 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -915,8 +915,7 @@ fn renderExpression( if (maybe_row_size) |row_size| { // A place to store the width of each expression and its column's maximum - const exprs_len = countLen(exprs.first); - var widths = try allocator.alloc(usize, exprs_len + row_size); + var widths = try allocator.alloc(usize, exprs.len() + row_size); defer allocator.free(widths); mem.set(usize, widths, 0); @@ -1391,7 +1390,7 @@ fn renderExpression( { break :blk false; } - const last_node = findLast(builtin_call.params.first.?).data; + const last_node = builtin_call.params.first.?.findLast().data; const maybe_comma = tree.nextToken(last_node.lastToken()); break :blk tree.tokens[maybe_comma].id == .Comma; }; @@ -1615,7 +1614,7 @@ fn renderExpression( assert(switch_case.items.first != null); const src_has_trailing_comma = blk: { - const last_node = findLast(switch_case.items.first.?).data; + const last_node = switch_case.items.first.?.findLast().data; const maybe_comma = tree.nextToken(last_node.lastToken()); break :blk tree.tokens[maybe_comma].id == .Comma; }; @@ -2510,19 +2509,3 @@ fn copyFixingWhitespace(stream: var, slice: []const u8) @TypeOf(stream).Error!vo else => try stream.writeByte(byte), }; } - -fn countLen(node: ?*std.SinglyLinkedList(*ast.Node).Node) usize { - var count: usize = 0; - var it = node; - while (it) |n| : (it = n.next) { - count += 1; - } - return count; -} - -fn findLast(node: *std.SinglyLinkedList(*ast.Node).Node) *std.SinglyLinkedList(*ast.Node).Node { - var it = node; - while (true) { - it = it.next orelse return it; - } -} diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 4ef4acc24b..295d7b1337 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -600,8 +600,7 @@ pub fn cmdFmt(gpa: *Allocator, args: []const []const u8) !void { }; defer tree.deinit(); - var error_it = tree.errors.iterator(0); - while (error_it.next()) |parse_error| { + for (tree.errors) |parse_error| { try printErrMsgToFile(gpa, parse_error, tree, "", stderr_file, color); } if (tree.errors.len != 0) { @@ -701,8 +700,7 @@ fn fmtPath(fmt: *Fmt, file_path: []const u8, check_mode: bool) FmtError!void { }; defer tree.deinit(); - var error_it = tree.errors.iterator(0); - while (error_it.next()) |parse_error| { + for (tree.errors) |parse_error| { try printErrMsgToFile(fmt.gpa, parse_error, tree, file_path, std.io.getStdErr(), fmt.color); } if (tree.errors.len != 0) { @@ -730,7 +728,7 @@ fn fmtPath(fmt: *Fmt, file_path: []const u8, check_mode: bool) FmtError!void { fn printErrMsgToFile( gpa: *mem.Allocator, - parse_error: *const ast.Error, + parse_error: ast.Error, tree: *ast.Tree, path: []const u8, file: fs.File, @@ -745,15 +743,15 @@ fn printErrMsgToFile( const span_first = lok_token; const span_last = lok_token; - const first_token = tree.tokens.at(span_first); - const last_token = tree.tokens.at(span_last); + const first_token = tree.tokens[span_first]; + const last_token = tree.tokens[span_last]; const start_loc = tree.tokenLocationPtr(0, first_token); const end_loc = tree.tokenLocationPtr(first_token.end, last_token); var text_buf = std.ArrayList(u8).init(gpa); defer text_buf.deinit(); const out_stream = text_buf.outStream(); - try parse_error.render(&tree.tokens, out_stream); + try parse_error.render(tree.tokens, out_stream); const text = text_buf.span(); const stream = file.outStream(); diff --git a/src-self-hosted/translate_c.zig b/src-self-hosted/translate_c.zig index 8321061924..184c07cdc9 100644 --- a/src-self-hosted/translate_c.zig +++ b/src-self-hosted/translate_c.zig @@ -37,10 +37,10 @@ fn addrEql(a: usize, b: usize) bool { } const SymbolTable = std.StringHashMap(*ast.Node); -const AliasList = std.SegmentedList(struct { +const AliasList = std.ArrayList(struct { alias: []const u8, name: []const u8, -}, 4); +}); const Scope = struct { id: Id, @@ -64,40 +64,50 @@ const Scope = struct { const Block = struct { base: Scope, block_node: *ast.Node.Block, + statements_it: *?*std.SinglyLinkedList(*ast.Node), variables: AliasList, label: ?[]const u8, mangle_count: u32 = 0, /// Don't forget to set rbrace token and block_node later fn init(c: *Context, parent: *Scope, label: ?[]const u8) !*Block { - const block = try c.a().create(Block); + const block = try c.arena.create(Block); block.* = .{ .base = .{ .id = .Block, .parent = parent, }, .block_node = undefined, - .variables = AliasList.init(c.a()), + .statements_it = undefined, + .variables = AliasList.init(c.arena), .label = label, }; return block; } + fn pushStatement(self: *Block, c: *Context, stmt: *ast.Node) !void { + self.statements_it = c.llpush(*ast.Node, self.statements_it, stmt); + } + + fn setBlockNode(self: *Block, block: *ast.Node.Block) void { + self.block_node = block; + self.statements_it = &block.statements.first; + } + /// Given the desired name, return a name that does not shadow anything from outer scopes. /// Inserts the returned name into the scope. fn makeMangledName(scope: *Block, c: *Context, name: []const u8) ![]const u8 { var proposed_name = name; while (scope.contains(proposed_name)) { scope.mangle_count += 1; - proposed_name = try std.fmt.allocPrint(c.a(), "{}_{}", .{ name, scope.mangle_count }); + proposed_name = try std.fmt.allocPrint(c.arena, "{}_{}", .{ name, scope.mangle_count }); } - try scope.variables.push(.{ .name = name, .alias = proposed_name }); + try scope.variables.append(.{ .name = name, .alias = proposed_name }); return proposed_name; } fn getAlias(scope: *Block, name: []const u8) []const u8 { - var it = scope.variables.iterator(0); - while (it.next()) |p| { + for (scope.variables.items) |p| { if (mem.eql(u8, p.name, name)) return p.alias; } @@ -105,8 +115,7 @@ const Scope = struct { } fn localContains(scope: *Block, name: []const u8) bool { - var it = scope.variables.iterator(0); - while (it.next()) |p| { + for (scope.variables.items) |p| { if (mem.eql(u8, p.name, name)) return true; } @@ -132,8 +141,8 @@ const Scope = struct { .id = .Root, .parent = null, }, - .sym_table = SymbolTable.init(c.a()), - .macro_table = SymbolTable.init(c.a()), + .sym_table = SymbolTable.init(c.arena), + .macro_table = SymbolTable.init(c.arena), .context = c, }; } @@ -208,7 +217,10 @@ const Scope = struct { }; pub const Context = struct { - tree: *ast.Tree, + gpa: *mem.Allocator, + arena: *mem.Allocator, + tokens: std.ArrayListUnmanaged(Token), + errors: std.ArrayListUnmanaged(ast.Error), source_buffer: *std.ArrayList(u8), err: Error, source_manager: *ZigClangSourceManager, @@ -217,6 +229,8 @@ pub const Context = struct { global_scope: *Scope.Root, clang_context: *ZigClangASTContext, mangle_count: u32 = 0, + root_node: *ast.Node.Root, + root_decls_it: *?*std.SinglyLinkedList(*ast.Node).Node, /// This one is different than the root scope's name table. This contains /// a list of names that we found by visiting all the top level decls without @@ -224,18 +238,45 @@ pub const Context = struct { /// up front in a pre-processing step. global_names: std.StringHashMap(void), + /// Helper type to append elements to a singly linked list. + const LinkedListPusher = struct { + c: *Context, + it: *?*std.SinglyLinkedList(*ast.Node).Node, + + fn push(self: *LinkedListPusher, element: *ast.Node) !void { + self.it = try self.c.llpush(*ast.Node, self.it, element); + } + }; + + /// Helper function to append items to a singly linked list. + fn llpusher(c: *Context, list: *std.SinglyLinkedList(*ast.Node)) LinkedListPusher { + assert(list.first == null); + return .{ + .c = c, + .it = &list.first, + }; + } + + fn llpush( + c: *Context, + comptime T: type, + it: *?*std.SinglyLinkedList(T).Node, + data: T, + ) !*?*std.SinglyLinkedList(T).Node { + const llnode = try c.arena.create(std.SinglyLinkedList(T).Node); + llnode.* = .{ .data = data }; + it.* = llnode; + return &llnode.next; + } + fn getMangle(c: *Context) u32 { c.mangle_count += 1; return c.mangle_count; } - fn a(c: *Context) *mem.Allocator { - return &c.tree.arena_allocator.allocator; - } - /// Convert a null-terminated C string to a slice allocated in the arena fn str(c: *Context, s: [*:0]const u8) ![]u8 { - return mem.dupe(c.a(), u8, mem.spanZ(s)); + return mem.dupe(c.arena, u8, mem.spanZ(s)); } /// Convert a clang source location to a file:line:column string @@ -246,12 +287,12 @@ pub const Context = struct { const line = ZigClangSourceManager_getSpellingLineNumber(c.source_manager, spelling_loc); const column = ZigClangSourceManager_getSpellingColumnNumber(c.source_manager, spelling_loc); - return std.fmt.allocPrint(c.a(), "{}:{}:{}", .{ filename, line, column }); + return std.fmt.allocPrint(c.arena, "{}:{}:{}", .{ filename, line, column }); } }; pub fn translate( - backing_allocator: *mem.Allocator, + gpa: *mem.Allocator, args_begin: [*]?[*]const u8, args_end: [*]?[*]const u8, errors: *[]ClangErrMsg, @@ -269,47 +310,43 @@ pub fn translate( }; defer ZigClangASTUnit_delete(ast_unit); - const tree = blk: { - var tree_arena = std.heap.ArenaAllocator.init(backing_allocator); - errdefer tree_arena.deinit(); + var source_buffer = std.ArrayList(u8).init(gpa); + defer source_buffer.deinit(); - const tree = try tree_arena.allocator.create(ast.Tree); - tree.* = .{ - .source = undefined, // need to use toOwnedSlice later - .root_node = undefined, - .arena_allocator = tree_arena, - .tokens = undefined, // can't reference the allocator yet - .errors = undefined, // can't reference the allocator yet - .generated = true, - }; - break :blk tree; - }; - const arena = &tree.arena_allocator.allocator; // now we can reference the allocator - errdefer tree.arena_allocator.deinit(); - tree.tokens = ast.Tree.TokenList.init(arena); - tree.errors = ast.Tree.ErrorList.init(arena); + // For memory that has the same lifetime as the Tree that we return + // from this function. + var arena = std.heap.ArenaAllocator.init(gpa); + errdefer arena.deinit(); - tree.root_node = try arena.create(ast.Node.Root); - tree.root_node.* = .{ - .decls = ast.Node.Root.DeclList.init(arena), + const root_node = try arena.allocator.create(ast.Node.Root); + root_node.* = .{ + .decls = ast.Node.Root.DeclList{}, // initialized with the eof token at the end .eof_token = undefined, }; - var source_buffer = std.ArrayList(u8).init(arena); - var context = Context{ - .tree = tree, + .gpa = gpa, + .arena = &arena.allocator, .source_buffer = &source_buffer, .source_manager = ZigClangASTUnit_getSourceManager(ast_unit), .err = undefined, - .decl_table = DeclTable.init(arena), - .alias_list = AliasList.init(arena), - .global_scope = try arena.create(Scope.Root), + .decl_table = DeclTable.init(gpa), + .alias_list = AliasList.init(gpa), + .global_scope = try arena.allocator.create(Scope.Root), .clang_context = ZigClangASTUnit_getASTContext(ast_unit).?, - .global_names = std.StringHashMap(void).init(arena), + .global_names = std.StringHashMap(void).init(gpa), + .tokens = .{}, + .errors = .{}, + .root_node = root_node, + .root_decls_it = &root_node.decls.first, }; context.global_scope.* = Scope.Root.init(&context); + defer context.decl_table.deinit(); + defer context.alias_list.deinit(); + defer context.tokens.deinit(gpa); + defer context.errors.deinit(gpa); + defer context.global_names.deinit(); try prepopulateGlobalNameTable(ast_unit, &context); @@ -320,23 +357,30 @@ pub fn translate( try transPreprocessorEntities(&context, ast_unit); try addMacros(&context); - var it = context.alias_list.iterator(0); - while (it.next()) |alias| { + for (context.alias_list.items) |alias| { if (!context.global_scope.sym_table.contains(alias.alias)) { try createAlias(&context, alias); } } - tree.root_node.eof_token = try appendToken(&context, .Eof, ""); - tree.source = source_buffer.toOwnedSlice(); + root_node.eof_token = try appendToken(&context, .Eof, ""); if (false) { - std.debug.warn("debug source:\n{}\n==EOF==\ntokens:\n", .{tree.source}); - var i: usize = 0; - while (i < tree.tokens.len) : (i += 1) { - const token = tree.tokens.at(i); + std.debug.warn("debug source:\n{}\n==EOF==\ntokens:\n", .{source_buffer.items}); + for (context.tokens.items) |token| { std.debug.warn("{}\n", .{token}); } } + + const tree = try arena.allocator.create(ast.Tree); + tree.* = .{ + .gpa = gpa, + .source = try arena.allocator.dupe(u8, source_buffer.items), + .tokens = context.tokens.toOwnedSlice(gpa), + .errors = context.errors.toOwnedSlice(gpa), + .root_node = root_node, + .arena = arena.state, + .generated = true, + }; return tree; } @@ -493,17 +537,19 @@ fn visitFnDecl(c: *Context, fn_decl: *const ZigClangFunctionDecl) Error!void { const block_scope = try Scope.Block.init(rp.c, &c.global_scope.base, null); var scope = &block_scope.base; const block_node = try transCreateNodeBlock(rp.c, null); - block_scope.block_node = block_node; + block_scope.setBlockNode(block_node); - var it = proto_node.params.iterator(0); + var it = proto_node.params.first; var param_id: c_uint = 0; - while (it.next()) |p| { - const param = @fieldParentPtr(ast.Node.ParamDecl, "base", p.*); + var prev_node_link = &proto_node.params_first; + while (it) |p_node| : ({prev_node_link = &p_node.next; it = p_node.next;}) { + const p = p_node.data; + const param = @fieldParentPtr(ast.Node.ParamDecl, "base", p); const param_name = if (param.name_token) |name_tok| tokenSlice(c, name_tok) else if (param.param_type == .var_args) { - assert(it.next() == null); - _ = proto_node.params.pop(); + assert(p_node.next == null); + prev_node_link.* = null; break; } else return failDecl(c, fn_decl_loc, fn_name, "function {} parameter has no name", .{fn_name}); @@ -516,7 +562,7 @@ fn visitFnDecl(c: *Context, fn_decl: *const ZigClangFunctionDecl) Error!void { const arg_name = blk: { const param_prefix = if (is_const) "" else "arg_"; - const bare_arg_name = try std.fmt.allocPrint(c.a(), "{}{}", .{ param_prefix, mangled_param_name }); + const bare_arg_name = try std.fmt.allocPrint(c.arena, "{}{}", .{ param_prefix, mangled_param_name }); break :blk try block_scope.makeMangledName(c, bare_arg_name); }; @@ -525,7 +571,7 @@ fn visitFnDecl(c: *Context, fn_decl: *const ZigClangFunctionDecl) Error!void { node.eq_token = try appendToken(c, .Equal, "="); node.init_node = try transCreateNodeIdentifier(c, arg_name); node.semicolon_token = try appendToken(c, .Semicolon, ";"); - try block_node.statements.push(&node.base); + block_scope.block_statements_it = try c.llpush(*ast.Node, block_scope.block_statements_it, &node.base); param.name_token = try appendIdentifier(c, arg_name); _ = try appendToken(c, .Colon, ":"); } @@ -560,7 +606,7 @@ fn visitVarDecl(c: *Context, var_decl: *const ZigClangVarDecl) Error!void { // TODO https://github.com/ziglang/zig/issues/3756 // TODO https://github.com/ziglang/zig/issues/1802 - const checked_name = if (isZigPrimitiveType(var_name)) try std.fmt.allocPrint(c.a(), "{}_{}", .{ var_name, c.getMangle() }) else var_name; + const checked_name = if (isZigPrimitiveType(var_name)) try std.fmt.allocPrint(c.arena, "{}_{}", .{ var_name, c.getMangle() }) else var_name; const var_decl_loc = ZigClangVarDecl_getLocation(var_decl); const qual_type = ZigClangVarDecl_getTypeSourceInfo_getType(var_decl); @@ -620,7 +666,7 @@ fn visitVarDecl(c: *Context, var_decl: *const ZigClangVarDecl) Error!void { _ = try appendToken(rp.c, .LParen, "("); const expr = try transCreateNodeStringLiteral( rp.c, - try std.fmt.allocPrint(rp.c.a(), "\"{}\"", .{str_ptr[0..str_len]}), + try std.fmt.allocPrint(rp.c.arena, "\"{}\"", .{str_ptr[0..str_len]}), ); _ = try appendToken(rp.c, .RParen, ")"); @@ -643,7 +689,7 @@ fn visitVarDecl(c: *Context, var_decl: *const ZigClangVarDecl) Error!void { break :blk null; }; - const node = try c.a().create(ast.Node.VarDecl); + const node = try c.arena.create(ast.Node.VarDecl); node.* = .{ .doc_comments = null, .visib_token = visib_tok, @@ -702,7 +748,7 @@ fn transTypeDef(c: *Context, typedef_decl: *const ZigClangTypedefNameDecl, top_l // TODO https://github.com/ziglang/zig/issues/3756 // TODO https://github.com/ziglang/zig/issues/1802 - const checked_name = if (isZigPrimitiveType(typedef_name)) try std.fmt.allocPrint(c.a(), "{}_{}", .{ typedef_name, c.getMangle() }) else typedef_name; + const checked_name = if (isZigPrimitiveType(typedef_name)) try std.fmt.allocPrint(c.arena, "{}_{}", .{ typedef_name, c.getMangle() }) else typedef_name; if (checkForBuiltinTypedef(checked_name)) |builtin| { return transTypeDefAsBuiltin(c, typedef_decl, builtin); } @@ -745,7 +791,7 @@ fn transRecordDecl(c: *Context, record_decl: *const ZigClangRecordDecl) Error!?* // Record declarations such as `struct {...} x` have no name but they're not // anonymous hence here isAnonymousStructOrUnion is not needed if (bare_name.len == 0) { - bare_name = try std.fmt.allocPrint(c.a(), "unnamed_{}", .{c.getMangle()}); + bare_name = try std.fmt.allocPrint(c.arena, "unnamed_{}", .{c.getMangle()}); is_unnamed = true; } @@ -762,7 +808,7 @@ fn transRecordDecl(c: *Context, record_decl: *const ZigClangRecordDecl) Error!?* return null; } - const name = try std.fmt.allocPrint(c.a(), "{}_{}", .{ container_kind_name, bare_name }); + const name = try std.fmt.allocPrint(c.arena, "{}_{}", .{ container_kind_name, bare_name }); _ = try c.decl_table.put(@ptrToInt(ZigClangRecordDecl_getCanonicalDecl(record_decl)), name); const node = try transCreateNodeVarDecl(c, !is_unnamed, true, name); @@ -785,15 +831,16 @@ fn transRecordDecl(c: *Context, record_decl: *const ZigClangRecordDecl) Error!?* const container_tok = try appendToken(c, container_kind, container_kind_name); const lbrace_token = try appendToken(c, .LBrace, "{"); - const container_node = try c.a().create(ast.Node.ContainerDecl); + const container_node = try c.arena.create(ast.Node.ContainerDecl); container_node.* = .{ .layout_token = layout_tok, .kind_token = container_tok, .init_arg_expr = .None, - .fields_and_decls = ast.Node.ContainerDecl.DeclList.init(c.a()), + .fields_and_decls = ast.Node.ContainerDecl.DeclList{}, .lbrace_token = lbrace_token, .rbrace_token = undefined, }; + var container_fields_and_decls = c.llpusher(&container_node.fields_and_decls); var unnamed_field_count: u32 = 0; var it = ZigClangRecordDecl_field_begin(record_def); @@ -821,7 +868,7 @@ fn transRecordDecl(c: *Context, record_decl: *const ZigClangRecordDecl) Error!?* var raw_name = try c.str(ZigClangNamedDecl_getName_bytes_begin(@ptrCast(*const ZigClangNamedDecl, field_decl))); if (ZigClangFieldDecl_isAnonymousStructOrUnion(field_decl) or raw_name.len == 0) { // Context.getMangle() is not used here because doing so causes unpredictable field names for anonymous fields. - raw_name = try std.fmt.allocPrint(c.a(), "unnamed_{}", .{unnamed_field_count}); + raw_name = try std.fmt.allocPrint(c.arena, "unnamed_{}", .{unnamed_field_count}); unnamed_field_count += 1; is_anon = true; } @@ -851,7 +898,7 @@ fn transRecordDecl(c: *Context, record_decl: *const ZigClangRecordDecl) Error!?* break :blk null; }; - const field_node = try c.a().create(ast.Node.ContainerField); + const field_node = try c.arena.create(ast.Node.ContainerField); field_node.* = .{ .doc_comments = null, .comptime_token = null, @@ -868,7 +915,7 @@ fn transRecordDecl(c: *Context, record_decl: *const ZigClangRecordDecl) Error!?* ); } - try container_node.fields_and_decls.push(&field_node.base); + try container_fields_and_decls.push(&field_node.base); _ = try appendToken(c, .Comma, ","); } container_node.rbrace_token = try appendToken(c, .RBrace, "}"); @@ -892,11 +939,11 @@ fn transEnumDecl(c: *Context, enum_decl: *const ZigClangEnumDecl) Error!?*ast.No var bare_name = try c.str(ZigClangNamedDecl_getName_bytes_begin(@ptrCast(*const ZigClangNamedDecl, enum_decl))); var is_unnamed = false; if (bare_name.len == 0) { - bare_name = try std.fmt.allocPrint(c.a(), "unnamed_{}", .{c.getMangle()}); + bare_name = try std.fmt.allocPrint(c.arena, "unnamed_{}", .{c.getMangle()}); is_unnamed = true; } - const name = try std.fmt.allocPrint(c.a(), "enum_{}", .{bare_name}); + const name = try std.fmt.allocPrint(c.arena, "enum_{}", .{bare_name}); _ = try c.decl_table.put(@ptrToInt(ZigClangEnumDecl_getCanonicalDecl(enum_decl)), name); const node = try transCreateNodeVarDecl(c, !is_unnamed, true, name); node.eq_token = try appendToken(c, .Equal, "="); @@ -916,15 +963,16 @@ fn transEnumDecl(c: *Context, enum_decl: *const ZigClangEnumDecl) Error!?*ast.No const extern_tok = try appendToken(c, .Keyword_extern, "extern"); const container_tok = try appendToken(c, .Keyword_enum, "enum"); - const container_node = try c.a().create(ast.Node.ContainerDecl); + const container_node = try c.arena.create(ast.Node.ContainerDecl); container_node.* = .{ .layout_token = extern_tok, .kind_token = container_tok, .init_arg_expr = .None, - .fields_and_decls = ast.Node.ContainerDecl.DeclList.init(c.a()), + .fields_and_decls = ast.Node.ContainerDecl.DeclList{}, .lbrace_token = undefined, .rbrace_token = undefined, }; + var container_node_fields_and_decls = c.llpusher(&container_node.fields_and_decls); const int_type = ZigClangEnumDecl_getIntegerType(enum_decl); // The underlying type may be null in case of forward-declared enum @@ -971,7 +1019,7 @@ fn transEnumDecl(c: *Context, enum_decl: *const ZigClangEnumDecl) Error!?*ast.No } else null; - const field_node = try c.a().create(ast.Node.ContainerField); + const field_node = try c.arena.create(ast.Node.ContainerField); field_node.* = .{ .doc_comments = null, .comptime_token = null, @@ -981,7 +1029,7 @@ fn transEnumDecl(c: *Context, enum_decl: *const ZigClangEnumDecl) Error!?*ast.No .align_expr = null, }; - try container_node.fields_and_decls.push(&field_node.base); + try container_node_fields_and_decls.push(&field_node.base); _ = try appendToken(c, .Comma, ","); // In C each enum value is in the global namespace. So we put them there too. @@ -992,7 +1040,7 @@ fn transEnumDecl(c: *Context, enum_decl: *const ZigClangEnumDecl) Error!?*ast.No const enum_ident = try transCreateNodeIdentifier(c, name); const period_tok = try appendToken(c, .Period, "."); const field_ident = try transCreateNodeIdentifier(c, field_name); - const field_access_node = try c.a().create(ast.Node.InfixOp); + const field_access_node = try c.arena.create(ast.Node.InfixOp); field_access_node.* = .{ .op_token = period_tok, .lhs = enum_ident, @@ -1006,7 +1054,7 @@ fn transEnumDecl(c: *Context, enum_decl: *const ZigClangEnumDecl) Error!?*ast.No try addTopLevelDecl(c, field_name, &tld_node.base); } // make non exhaustive - const field_node = try c.a().create(ast.Node.ContainerField); + const field_node = try c.arena.create(ast.Node.ContainerField); field_node.* = .{ .doc_comments = null, .comptime_token = null, @@ -1016,7 +1064,7 @@ fn transEnumDecl(c: *Context, enum_decl: *const ZigClangEnumDecl) Error!?*ast.No .align_expr = null, }; - try container_node.fields_and_decls.push(&field_node.base); + try container_node_fields_and_decls.push(&field_node.base); _ = try appendToken(c, .Comma, ","); container_node.rbrace_token = try appendToken(c, .RBrace, "}"); @@ -1071,7 +1119,7 @@ fn transStmt( .ParenExprClass => { const expr = try transExpr(rp, scope, ZigClangParenExpr_getSubExpr(@ptrCast(*const ZigClangParenExpr, stmt)), .used, lrvalue); if (expr.id == .GroupedExpression) return maybeSuppressResult(rp, scope, result_used, expr); - const node = try rp.c.a().create(ast.Node.GroupedExpression); + const node = try rp.c.arena.create(ast.Node.GroupedExpression); node.* = .{ .lparen = try appendToken(rp.c, .LParen, "("), .expr = expr, @@ -1116,7 +1164,7 @@ fn transStmt( const source_expr = ZigClangOpaqueValueExpr_getSourceExpr(@ptrCast(*const ZigClangOpaqueValueExpr, stmt)).?; const expr = try transExpr(rp, scope, source_expr, .used, lrvalue); if (expr.id == .GroupedExpression) return maybeSuppressResult(rp, scope, result_used, expr); - const node = try rp.c.a().create(ast.Node.GroupedExpression); + const node = try rp.c.arena.create(ast.Node.GroupedExpression); node.* = .{ .lparen = try appendToken(rp.c, .LParen, "("), .expr = expr, @@ -1147,18 +1195,18 @@ fn transBinaryOperator( var op_token: ast.TokenIndex = undefined; var op_id: ast.Node.InfixOp.Op = undefined; switch (op) { - .Assign => return transCreateNodeAssign(rp, scope, result_used, ZigClangBinaryOperator_getLHS(stmt), ZigClangBinaryOperator_getRHS(stmt)), + .Assign => return try transCreateNodeAssign(rp, scope, result_used, ZigClangBinaryOperator_getLHS(stmt), ZigClangBinaryOperator_getRHS(stmt)), .Comma => { const block_scope = try scope.findBlockScope(rp.c); const expr = block_scope.base.parent == scope; const lparen = if (expr) blk: { const l = try appendToken(rp.c, .LParen, "("); - block_scope.block_node = try transCreateNodeBlock(rp.c, block_scope.label); + block_scope.setBlockNode(try transCreateNodeBlock(rp.c, block_scope.label)); break :blk l; } else undefined; const lhs = try transExpr(rp, &block_scope.base, ZigClangBinaryOperator_getLHS(stmt), .unused, .r_value); - try block_scope.block_node.statements.push(lhs); + try block_scope.statements.push(lhs); const rhs = try transExpr(rp, &block_scope.base, ZigClangBinaryOperator_getRHS(stmt), .used, .r_value); if (expr) { @@ -1168,7 +1216,7 @@ fn transBinaryOperator( try block_scope.block_node.statements.push(&break_node.base); block_scope.block_node.rbrace = try appendToken(rp.c, .RBrace, "}"); const rparen = try appendToken(rp.c, .RParen, ")"); - const grouped_expr = try rp.c.a().create(ast.Node.GroupedExpression); + const grouped_expr = try rp.c.arena.create(ast.Node.GroupedExpression); grouped_expr.* = .{ .lparen = lparen, .expr = &block_scope.block_node.base, @@ -1335,7 +1383,7 @@ fn transCompoundStmtInline( fn transCompoundStmt(rp: RestorePoint, scope: *Scope, stmt: *const ZigClangCompoundStmt) TransError!*ast.Node { const block_scope = try Scope.Block.init(rp.c, scope, null); - block_scope.block_node = try transCreateNodeBlock(rp.c, null); + block_scope.setBlockNode(try transCreateNodeBlock(rp.c, null)); try transCompoundStmtInline(rp, &block_scope.base, stmt, block_scope.block_node); block_scope.block_node.rbrace = try appendToken(rp.c, .RBrace, "}"); return &block_scope.block_node.base; @@ -1458,7 +1506,7 @@ fn transImplicitCastExpr( switch (ZigClangImplicitCastExpr_getCastKind(expr)) { .BitCast, .FloatingCast, .FloatingToIntegral, .IntegralToFloating, .IntegralCast, .PointerToIntegral, .IntegralToPointer => { const sub_expr_node = try transExpr(rp, scope, sub_expr, .used, .r_value); - return transCCast(rp, scope, ZigClangImplicitCastExpr_getBeginLoc(expr), dest_type, src_type, sub_expr_node); + return try transCCast(rp, scope, ZigClangImplicitCastExpr_getBeginLoc(expr), dest_type, src_type, sub_expr_node); }, .LValueToRValue, .NoOp, .FunctionToPointerDecay => { const sub_expr_node = try transExpr(rp, scope, sub_expr, .used, .r_value); @@ -1539,7 +1587,7 @@ fn transBoolExpr( if (grouped) { const rparen = try appendToken(rp.c, .RParen, ")"); - const grouped_expr = try rp.c.a().create(ast.Node.GroupedExpression); + const grouped_expr = try rp.c.arena.create(ast.Node.GroupedExpression); grouped_expr.* = .{ .lparen = lparen, .expr = node, @@ -1747,13 +1795,13 @@ fn transStringLiteral( len = 0; for (str) |c| len += escapeChar(c, &char_buf).len; - const buf = try rp.c.a().alloc(u8, len + "\"\"".len); + const buf = try rp.c.arena.alloc(u8, len + "\"\"".len); buf[0] = '"'; writeEscapedString(buf[1..], str); buf[buf.len - 1] = '"'; const token = try appendToken(rp.c, .StringLiteral, buf); - const node = try rp.c.a().create(ast.Node.StringLiteral); + const node = try rp.c.arena.create(ast.Node.StringLiteral); node.* = .{ .token = token, }; @@ -2038,13 +2086,13 @@ fn transInitListExprRecord( var raw_name = try rp.c.str(ZigClangNamedDecl_getName_bytes_begin(@ptrCast(*const ZigClangNamedDecl, field_decl))); if (ZigClangFieldDecl_isAnonymousStructOrUnion(field_decl)) { const name = rp.c.decl_table.get(@ptrToInt(ZigClangFieldDecl_getCanonicalDecl(field_decl))).?; - raw_name = try mem.dupe(rp.c.a(), u8, name.value); + raw_name = try mem.dupe(rp.c.arena, u8, name.value); } const field_name_tok = try appendIdentifier(rp.c, raw_name); _ = try appendToken(rp.c, .Equal, "="); - const field_init_node = try rp.c.a().create(ast.Node.FieldInitializer); + const field_init_node = try rp.c.arena.create(ast.Node.FieldInitializer); field_init_node.* = .{ .period_token = period_tok, .name_token = field_name_tok, @@ -2133,7 +2181,7 @@ fn transInitListExprArray( &filler_init_node.base else blk: { const mul_tok = try appendToken(rp.c, .AsteriskAsterisk, "**"); - const mul_node = try rp.c.a().create(ast.Node.InfixOp); + const mul_node = try rp.c.arena.create(ast.Node.InfixOp); mul_node.* = .{ .op_token = mul_tok, .lhs = &filler_init_node.base, @@ -2147,7 +2195,7 @@ fn transInitListExprArray( return rhs_node; } - const cat_node = try rp.c.a().create(ast.Node.InfixOp); + const cat_node = try rp.c.arena.create(ast.Node.InfixOp); cat_node.* = .{ .op_token = cat_tok, .lhs = &init_node.base, @@ -2382,7 +2430,7 @@ fn transForLoop( if (ZigClangForStmt_getInit(stmt)) |init| { block_scope = try Scope.Block.init(rp.c, scope, null); const block = try transCreateNodeBlock(rp.c, null); - block_scope.?.block_node = block; + block_scope.?.setBlockNode(block); loop_scope.parent = &block_scope.?.base; const result = try transStmt(rp, &block_scope.?.base, init, .unused, .r_value); if (result != &block.base) @@ -2445,7 +2493,7 @@ fn transSwitch( const block_scope = try Scope.Block.init(rp.c, &switch_scope.base, null); // tmp block that all statements will go before being picked up by a case or default const block = try transCreateNodeBlock(rp.c, null); - block_scope.block_node = block; + block_scope.setBlockNode(block); const switch_block = try transCreateNodeBlock(rp.c, null); try switch_block.statements.push(&switch_node.base); @@ -2479,7 +2527,7 @@ fn transCase( ) TransError!*ast.Node { const block_scope = scope.findBlockScope(rp.c) catch unreachable; const switch_scope = scope.getSwitch(); - const label = try std.fmt.allocPrint(rp.c.a(), "__case_{}", .{switch_scope.cases.len - @boolToInt(switch_scope.has_default)}); + const label = try std.fmt.allocPrint(rp.c.arena, "__case_{}", .{switch_scope.cases.len() - @boolToInt(switch_scope.has_default)}); _ = try appendToken(rp.c, .Semicolon, ";"); const expr = if (ZigClangCaseStmt_getRHS(stmt)) |rhs| blk: { @@ -2487,7 +2535,7 @@ fn transCase( const ellips = try appendToken(rp.c, .Ellipsis3, "..."); const rhs_node = try transExpr(rp, scope, rhs, .used, .r_value); - const node = try rp.c.a().create(ast.Node.InfixOp); + const node = try rp.c.arena.create(ast.Node.InfixOp); node.* = .{ .op_token = ellips, .lhs = lhs_node, @@ -2606,7 +2654,7 @@ fn transCharLiteral( } var char_buf: [4]u8 = undefined; const token = try appendTokenFmt(rp.c, .CharLiteral, "'{}'", .{escapeChar(@intCast(u8, val), &char_buf)}); - const node = try rp.c.a().create(ast.Node.CharLiteral); + const node = try rp.c.arena.create(ast.Node.CharLiteral); node.* = .{ .token = token, }; @@ -2646,7 +2694,7 @@ fn transStmtExpr(rp: RestorePoint, scope: *Scope, stmt: *const ZigClangStmtExpr, const lparen = try appendToken(rp.c, .LParen, "("); const block_scope = try Scope.Block.init(rp.c, scope, "blk"); const block = try transCreateNodeBlock(rp.c, "blk"); - block_scope.block_node = block; + block_scope.setBlockNode(block); var it = ZigClangCompoundStmt_body_begin(comp); const end_it = ZigClangCompoundStmt_body_end(comp); @@ -2661,7 +2709,7 @@ fn transStmtExpr(rp: RestorePoint, scope: *Scope, stmt: *const ZigClangStmtExpr, try block.statements.push(&break_node.base); block.rbrace = try appendToken(rp.c, .RBrace, "}"); const rparen = try appendToken(rp.c, .RParen, ")"); - const grouped_expr = try rp.c.a().create(ast.Node.GroupedExpression); + const grouped_expr = try rp.c.arena.create(ast.Node.GroupedExpression); grouped_expr.* = .{ .lparen = lparen, .expr = &block.base, @@ -2686,7 +2734,7 @@ fn transMemberExpr(rp: RestorePoint, scope: *Scope, stmt: *const ZigClangMemberE const field_decl = @ptrCast(*const struct_ZigClangFieldDecl, member_decl); if (ZigClangFieldDecl_isAnonymousStructOrUnion(field_decl)) { const name = rp.c.decl_table.get(@ptrToInt(ZigClangFieldDecl_getCanonicalDecl(field_decl))).?; - break :blk try mem.dupe(rp.c.a(), u8, name.value); + break :blk try mem.dupe(rp.c.arena, u8, name.value); } } const decl = @ptrCast(*const ZigClangNamedDecl, member_decl); @@ -2941,7 +2989,7 @@ fn transCreatePreCrement( // zig: break :blk _ref.* // zig: }) const block_scope = try Scope.Block.init(rp.c, scope, "blk"); - block_scope.block_node = try transCreateNodeBlock(rp.c, block_scope.label); + block_scope.setBlockNode(try transCreateNodeBlock(rp.c, block_scope.label)); const ref = try block_scope.makeMangledName(rp.c, "ref"); const node = try transCreateNodeVarDecl(rp.c, false, true, ref); @@ -2967,7 +3015,7 @@ fn transCreatePreCrement( block_scope.block_node.rbrace = try appendToken(rp.c, .RBrace, "}"); // semicolon must immediately follow rbrace because it is the last token in a block _ = try appendToken(rp.c, .Semicolon, ";"); - const grouped_expr = try rp.c.a().create(ast.Node.GroupedExpression); + const grouped_expr = try rp.c.arena.create(ast.Node.GroupedExpression); grouped_expr.* = .{ .lparen = try appendToken(rp.c, .LParen, "("), .expr = &block_scope.block_node.base, @@ -3007,7 +3055,7 @@ fn transCreatePostCrement( // zig: break :blk _tmp // zig: }) const block_scope = try Scope.Block.init(rp.c, scope, "blk"); - block_scope.block_node = try transCreateNodeBlock(rp.c, block_scope.label); + block_scope.setBlockNode(try transCreateNodeBlock(rp.c, block_scope.label)); const ref = try block_scope.makeMangledName(rp.c, "ref"); const node = try transCreateNodeVarDecl(rp.c, false, true, ref); @@ -3040,7 +3088,7 @@ fn transCreatePostCrement( try block_scope.block_node.statements.push(&break_node.base); _ = try appendToken(rp.c, .Semicolon, ";"); block_scope.block_node.rbrace = try appendToken(rp.c, .RBrace, "}"); - const grouped_expr = try rp.c.a().create(ast.Node.GroupedExpression); + const grouped_expr = try rp.c.arena.create(ast.Node.GroupedExpression); grouped_expr.* = .{ .lparen = try appendToken(rp.c, .LParen, "("), .expr = &block_scope.block_node.base, @@ -3106,7 +3154,7 @@ fn transCreateCompoundAssign( if ((is_mod or is_div) and is_signed) { const op_token = try appendToken(rp.c, .Equal, "="); - const op_node = try rp.c.a().create(ast.Node.InfixOp); + const op_node = try rp.c.arena.create(ast.Node.InfixOp); const builtin = if (is_mod) "@rem" else "@divTrunc"; const builtin_node = try transCreateNodeBuiltinFnCall(rp.c, builtin); const lhs_node = try transExpr(rp, scope, lhs, .used, .l_value); @@ -3152,7 +3200,7 @@ fn transCreateCompoundAssign( // zig: break :blk _ref.* // zig: }) const block_scope = try Scope.Block.init(rp.c, scope, "blk"); - block_scope.block_node = try transCreateNodeBlock(rp.c, block_scope.label); + block_scope.setBlockNode(try transCreateNodeBlock(rp.c, block_scope.label)); const ref = try block_scope.makeMangledName(rp.c, "ref"); const node = try transCreateNodeVarDecl(rp.c, false, true, ref); @@ -3169,7 +3217,7 @@ fn transCreateCompoundAssign( if ((is_mod or is_div) and is_signed) { const op_token = try appendToken(rp.c, .Equal, "="); - const op_node = try rp.c.a().create(ast.Node.InfixOp); + const op_node = try rp.c.arena.create(ast.Node.InfixOp); const builtin = if (is_mod) "@rem" else "@divTrunc"; const builtin_node = try transCreateNodeBuiltinFnCall(rp.c, builtin); try builtin_node.params.push(try transCreateNodePtrDeref(rp.c, lhs_node)); @@ -3211,7 +3259,7 @@ fn transCreateCompoundAssign( break_node.rhs = ref_node; try block_scope.block_node.statements.push(&break_node.base); block_scope.block_node.rbrace = try appendToken(rp.c, .RBrace, "}"); - const grouped_expr = try rp.c.a().create(ast.Node.GroupedExpression); + const grouped_expr = try rp.c.arena.create(ast.Node.GroupedExpression); grouped_expr.* = .{ .lparen = try appendToken(rp.c, .LParen, "("), .expr = &block_scope.block_node.base, @@ -3295,7 +3343,7 @@ fn transBreak(rp: RestorePoint, scope: *Scope) TransError!*ast.Node { fn transFloatingLiteral(rp: RestorePoint, scope: *Scope, stmt: *const ZigClangFloatingLiteral, used: ResultUsed) TransError!*ast.Node { // TODO use something more accurate const dbl = ZigClangAPFloat_getValueAsApproximateDouble(stmt); - const node = try rp.c.a().create(ast.Node.FloatLiteral); + const node = try rp.c.arena.create(ast.Node.FloatLiteral); node.* = .{ .token = try appendTokenFmt(rp.c, .FloatLiteral, "{d}", .{dbl}), }; @@ -3318,7 +3366,7 @@ fn transBinaryConditionalOperator(rp: RestorePoint, scope: *Scope, stmt: *const const lparen = try appendToken(rp.c, .LParen, "("); const block_scope = try Scope.Block.init(rp.c, scope, "blk"); - block_scope.block_node = try transCreateNodeBlock(rp.c, block_scope.label); + block_scope.setBlockNode(try transCreateNodeBlock(rp.c, block_scope.label)); const mangled_name = try block_scope.makeMangledName(rp.c, "cond_temp"); const tmp_var = try transCreateNodeVarDecl(rp.c, false, true, mangled_name); @@ -3351,7 +3399,7 @@ fn transBinaryConditionalOperator(rp: RestorePoint, scope: *Scope, stmt: *const try block_scope.block_node.statements.push(&break_node.base); block_scope.block_node.rbrace = try appendToken(rp.c, .RBrace, "}"); - const grouped_expr = try rp.c.a().create(ast.Node.GroupedExpression); + const grouped_expr = try rp.c.arena.create(ast.Node.GroupedExpression); grouped_expr.* = .{ .lparen = lparen, .expr = &block_scope.block_node.base, @@ -3384,7 +3432,7 @@ fn transConditionalOperator(rp: RestorePoint, scope: *Scope, stmt: *const ZigCla if (grouped) { const rparen = try appendToken(rp.c, .RParen, ")"); - const grouped_expr = try rp.c.a().create(ast.Node.GroupedExpression); + const grouped_expr = try rp.c.arena.create(ast.Node.GroupedExpression); grouped_expr.* = .{ .lparen = lparen, .expr = &if_node.base, @@ -3415,7 +3463,7 @@ fn maybeSuppressResult( } const lhs = try transCreateNodeIdentifier(rp.c, "_"); const op_token = try appendToken(rp.c, .Equal, "="); - const op_node = try rp.c.a().create(ast.Node.InfixOp); + const op_node = try rp.c.arena.create(ast.Node.InfixOp); op_node.* = .{ .op_token = op_token, .lhs = lhs, @@ -3426,7 +3474,7 @@ fn maybeSuppressResult( } fn addTopLevelDecl(c: *Context, name: []const u8, decl_node: *ast.Node) !void { - try c.tree.root_node.decls.push(decl_node); + c.root_decls_it = try c.llpush(*ast.Node, c.root_decls_it, decl_node); _ = try c.global_scope.sym_table.put(name, decl_node); } @@ -3524,7 +3572,7 @@ fn qualTypeToLog2IntRef(rp: RestorePoint, qt: ZigClangQualType, source_loc: ZigC if (int_bit_width != 0) { // we can perform the log2 now. const cast_bit_width = math.log2_int(u64, int_bit_width); - const node = try rp.c.a().create(ast.Node.IntegerLiteral); + const node = try rp.c.arena.create(ast.Node.IntegerLiteral); node.* = .{ .token = try appendTokenFmt(rp.c, .Identifier, "u{}", .{cast_bit_width}), }; @@ -3547,7 +3595,7 @@ fn qualTypeToLog2IntRef(rp: RestorePoint, qt: ZigClangQualType, source_loc: ZigC const import_fn_call = try transCreateNodeBuiltinFnCall(rp.c, "@import"); const std_token = try appendToken(rp.c, .StringLiteral, "\"std\""); - const std_node = try rp.c.a().create(ast.Node.StringLiteral); + const std_node = try rp.c.arena.create(ast.Node.StringLiteral); std_node.* = .{ .token = std_token, }; @@ -3749,7 +3797,7 @@ fn transCreateNodeAssign( // zig: break :blk _tmp // zig: }) const block_scope = try Scope.Block.init(rp.c, scope, "blk"); - block_scope.block_node = try transCreateNodeBlock(rp.c, block_scope.label); + block_scope.setBlockNode(try transCreateNodeBlock(rp.c, block_scope.label)); const tmp = try block_scope.makeMangledName(rp.c, "tmp"); const node = try transCreateNodeVarDecl(rp.c, false, true, tmp); @@ -3786,10 +3834,10 @@ fn transCreateNodeAssign( fn transCreateNodeBuiltinFnCall(c: *Context, name: []const u8) !*ast.Node.BuiltinCall { const builtin_token = try appendToken(c, .Builtin, name); _ = try appendToken(c, .LParen, "("); - const node = try c.a().create(ast.Node.BuiltinCall); + const node = try c.arena.create(ast.Node.BuiltinCall); node.* = .{ .builtin_token = builtin_token, - .params = ast.Node.BuiltinCall.ParamList.init(c.a()), + .params = ast.Node.BuiltinCall.ParamList{}, .rparen_token = undefined, // set after appending args }; return node; @@ -3797,12 +3845,12 @@ fn transCreateNodeBuiltinFnCall(c: *Context, name: []const u8) !*ast.Node.Builti fn transCreateNodeFnCall(c: *Context, fn_expr: *ast.Node) !*ast.Node.SuffixOp { _ = try appendToken(c, .LParen, "("); - const node = try c.a().create(ast.Node.SuffixOp); + const node = try c.arena.create(ast.Node.SuffixOp); node.* = .{ .lhs = .{ .node = fn_expr }, .op = .{ .Call = .{ - .params = ast.Node.SuffixOp.Op.Call.ParamList.init(c.a()), + .params = ast.Node.SuffixOp.Op.Call.ParamList{}, .async_token = null, }, }, @@ -3812,7 +3860,7 @@ fn transCreateNodeFnCall(c: *Context, fn_expr: *ast.Node) !*ast.Node.SuffixOp { } fn transCreateNodeFieldAccess(c: *Context, container: *ast.Node, field_name: []const u8) !*ast.Node { - const field_access_node = try c.a().create(ast.Node.InfixOp); + const field_access_node = try c.arena.create(ast.Node.InfixOp); field_access_node.* = .{ .op_token = try appendToken(c, .Period, "."), .lhs = container, @@ -3828,7 +3876,7 @@ fn transCreateNodePrefixOp( op_tok_id: std.zig.Token.Id, bytes: []const u8, ) !*ast.Node.PrefixOp { - const node = try c.a().create(ast.Node.PrefixOp); + const node = try c.arena.create(ast.Node.PrefixOp); node.* = .{ .op_token = try appendToken(c, op_tok_id, bytes), .op = op, @@ -3851,7 +3899,7 @@ fn transCreateNodeInfixOp( try appendToken(rp.c, .LParen, "(") else null; - const node = try rp.c.a().create(ast.Node.InfixOp); + const node = try rp.c.arena.create(ast.Node.InfixOp); node.* = .{ .op_token = op_token, .lhs = lhs_node, @@ -3860,7 +3908,7 @@ fn transCreateNodeInfixOp( }; if (!grouped) return maybeSuppressResult(rp, scope, used, &node.base); const rparen = try appendToken(rp.c, .RParen, ")"); - const grouped_expr = try rp.c.a().create(ast.Node.GroupedExpression); + const grouped_expr = try rp.c.arena.create(ast.Node.GroupedExpression); grouped_expr.* = .{ .lparen = lparen.?, .expr = &node.base, @@ -3904,7 +3952,7 @@ fn transCreateNodePtrType( is_volatile: bool, op_tok_id: std.zig.Token.Id, ) !*ast.Node.PrefixOp { - const node = try c.a().create(ast.Node.PrefixOp); + const node = try c.arena.create(ast.Node.PrefixOp); const op_token = switch (op_tok_id) { .LBracket => blk: { const lbracket = try appendToken(c, .LBracket, "["); @@ -3946,8 +3994,8 @@ fn transCreateNodeAPInt(c: *Context, int: *const ZigClangAPSInt) !*ast.Node { ZigClangAPSInt_free(aps_int); }; - const limbs = try c.a().alloc(math.big.Limb, num_limbs); - defer c.a().free(limbs); + const limbs = try c.arena.alloc(math.big.Limb, num_limbs); + defer c.arena.free(limbs); const data = ZigClangAPSInt_getRawData(aps_int); switch (@sizeOf(math.big.Limb)) { @@ -3972,12 +4020,12 @@ fn transCreateNodeAPInt(c: *Context, int: *const ZigClangAPSInt) !*ast.Node { } const big: math.big.int.Const = .{ .limbs = limbs, .positive = !is_negative }; - const str = big.toStringAlloc(c.a(), 10, false) catch |err| switch (err) { + const str = big.toStringAlloc(c.arena, 10, false) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, }; - defer c.a().free(str); + defer c.arena.free(str); const token = try appendToken(c, .IntegerLiteral, str); - const node = try c.a().create(ast.Node.IntegerLiteral); + const node = try c.arena.create(ast.Node.IntegerLiteral); node.* = .{ .token = token, }; @@ -3986,7 +4034,7 @@ fn transCreateNodeAPInt(c: *Context, int: *const ZigClangAPSInt) !*ast.Node { fn transCreateNodeReturnExpr(c: *Context) !*ast.Node.ControlFlowExpression { const ltoken = try appendToken(c, .Keyword_return, "return"); - const node = try c.a().create(ast.Node.ControlFlowExpression); + const node = try c.arena.create(ast.Node.ControlFlowExpression); node.* = .{ .ltoken = ltoken, .kind = .Return, @@ -3997,7 +4045,7 @@ fn transCreateNodeReturnExpr(c: *Context) !*ast.Node.ControlFlowExpression { fn transCreateNodeUndefinedLiteral(c: *Context) !*ast.Node { const token = try appendToken(c, .Keyword_undefined, "undefined"); - const node = try c.a().create(ast.Node.UndefinedLiteral); + const node = try c.arena.create(ast.Node.UndefinedLiteral); node.* = .{ .token = token, }; @@ -4006,7 +4054,7 @@ fn transCreateNodeUndefinedLiteral(c: *Context) !*ast.Node { fn transCreateNodeNullLiteral(c: *Context) !*ast.Node { const token = try appendToken(c, .Keyword_null, "null"); - const node = try c.a().create(ast.Node.NullLiteral); + const node = try c.arena.create(ast.Node.NullLiteral); node.* = .{ .token = token, }; @@ -4018,7 +4066,7 @@ fn transCreateNodeBoolLiteral(c: *Context, value: bool) !*ast.Node { try appendToken(c, .Keyword_true, "true") else try appendToken(c, .Keyword_false, "false"); - const node = try c.a().create(ast.Node.BoolLiteral); + const node = try c.arena.create(ast.Node.BoolLiteral); node.* = .{ .token = token, }; @@ -4027,11 +4075,11 @@ fn transCreateNodeBoolLiteral(c: *Context, value: bool) !*ast.Node { fn transCreateNodeArrayInitializer(c: *Context, ty: *ast.Node) !*ast.Node.SuffixOp { _ = try appendToken(c, .LBrace, "{"); - const node = try c.a().create(ast.Node.SuffixOp); + const node = try c.arena.create(ast.Node.SuffixOp); node.* = .{ .lhs = .{ .node = ty }, .op = .{ - .ArrayInitializer = ast.Node.SuffixOp.Op.InitList.init(c.a()), + .ArrayInitializer = ast.Node.SuffixOp.Op.InitList{}, }, .rtoken = undefined, // set after appending values }; @@ -4040,11 +4088,11 @@ fn transCreateNodeArrayInitializer(c: *Context, ty: *ast.Node) !*ast.Node.Suffix fn transCreateNodeStructInitializer(c: *Context, ty: *ast.Node) !*ast.Node.SuffixOp { _ = try appendToken(c, .LBrace, "{"); - const node = try c.a().create(ast.Node.SuffixOp); + const node = try c.arena.create(ast.Node.SuffixOp); node.* = .{ .lhs = .{ .node = ty }, .op = .{ - .StructInitializer = ast.Node.SuffixOp.Op.InitList.init(c.a()), + .StructInitializer = ast.Node.SuffixOp.Op.InitList{}, }, .rtoken = undefined, // set after appending values }; @@ -4053,7 +4101,7 @@ fn transCreateNodeStructInitializer(c: *Context, ty: *ast.Node) !*ast.Node.Suffi fn transCreateNodeInt(c: *Context, int: var) !*ast.Node { const token = try appendTokenFmt(c, .IntegerLiteral, "{}", .{int}); - const node = try c.a().create(ast.Node.IntegerLiteral); + const node = try c.arena.create(ast.Node.IntegerLiteral); node.* = .{ .token = token, }; @@ -4062,7 +4110,7 @@ fn transCreateNodeInt(c: *Context, int: var) !*ast.Node { fn transCreateNodeFloat(c: *Context, int: var) !*ast.Node { const token = try appendTokenFmt(c, .FloatLiteral, "{}", .{int}); - const node = try c.a().create(ast.Node.FloatLiteral); + const node = try c.arena.create(ast.Node.FloatLiteral); node.* = .{ .token = token, }; @@ -4085,20 +4133,22 @@ fn transCreateNodeMacroFn(c: *Context, name: []const u8, ref: *ast.Node, proto_a const name_tok = try appendIdentifier(c, name); _ = try appendToken(c, .LParen, "("); - var fn_params = ast.Node.FnProto.ParamList.init(c.a()); - var it = proto_alias.params.iterator(0); - while (it.next()) |pn| { - if (it.index != 0) { + var fn_params = ast.Node.FnProto.ParamList{}; + var fn_params_list = &fn_params.first; + var it = proto_alias.params.first; + while (it) |pn_node| : (it = pn_node.next) { + const pn = pn_node.data; + if (pn_node != proto_alias.params.first.?) { _ = try appendToken(c, .Comma, ","); } - const param = pn.*.cast(ast.Node.ParamDecl).?; + const param = pn.cast(ast.Node.ParamDecl).?; const param_name_tok = param.name_token orelse try appendTokenFmt(c, .Identifier, "arg_{}", .{c.getMangle()}); _ = try appendToken(c, .Colon, ":"); - const param_node = try c.a().create(ast.Node.ParamDecl); + const param_node = try c.arena.create(ast.Node.ParamDecl); param_node.* = .{ .doc_comments = null, .comptime_token = null, @@ -4106,12 +4156,12 @@ fn transCreateNodeMacroFn(c: *Context, name: []const u8, ref: *ast.Node, proto_a .name_token = param_name_tok, .param_type = param.param_type, }; - try fn_params.push(¶m_node.base); + fn_params_list = try c.llpush(*ast.Node, fn_params_list, ¶m_node.base); } _ = try appendToken(c, .RParen, ")"); - const fn_proto = try c.a().create(ast.Node.FnProto); + const fn_proto = try c.arena.create(ast.Node.FnProto); fn_proto.* = .{ .doc_comments = null, .visib_token = pub_tok, @@ -4129,24 +4179,28 @@ fn transCreateNodeMacroFn(c: *Context, name: []const u8, ref: *ast.Node, proto_a }; const block = try transCreateNodeBlock(c, null); + var block_statements_it = &block.statements.first; const return_expr = try transCreateNodeReturnExpr(c); const unwrap_expr = try transCreateNodeUnwrapNull(c, ref.cast(ast.Node.VarDecl).?.init_node.?); const call_expr = try transCreateNodeFnCall(c, unwrap_expr); - it = fn_params.iterator(0); - while (it.next()) |pn| { - if (it.index != 0) { + var call_params_it = &call_expr.op.Call.params.first; + + it = fn_params.first; + while (it) |pn_node| : (it = pn_node.next) { + const pn = pn_node.data; + if (fn_params.first.? != pn_node) { _ = try appendToken(c, .Comma, ","); } - const param = pn.*.cast(ast.Node.ParamDecl).?; - try call_expr.op.Call.params.push(try transCreateNodeIdentifier(c, tokenSlice(c, param.name_token.?))); + const param = pn.cast(ast.Node.ParamDecl).?; + call_params_it = try c.llpush(*ast.Node, call_params_it, try transCreateNodeIdentifier(c, tokenSlice(c, param.name_token.?)),); } call_expr.rtoken = try appendToken(c, .RParen, ")"); return_expr.rhs = &call_expr.base; _ = try appendToken(c, .Semicolon, ";"); block.rbrace = try appendToken(c, .RBrace, "}"); - try block.statements.push(&return_expr.base); + block_statements_it = try c.llpush(*ast.Node, block_statements_it, &return_expr.base); fn_proto.body_node = &block.base; return &fn_proto.base; } @@ -4154,7 +4208,7 @@ fn transCreateNodeMacroFn(c: *Context, name: []const u8, ref: *ast.Node, proto_a fn transCreateNodeUnwrapNull(c: *Context, wrapped: *ast.Node) !*ast.Node { _ = try appendToken(c, .Period, "."); const qm = try appendToken(c, .QuestionMark, "?"); - const node = try c.a().create(ast.Node.SuffixOp); + const node = try c.arena.create(ast.Node.SuffixOp); node.* = .{ .op = .UnwrapOptional, .lhs = .{ .node = wrapped }, @@ -4164,7 +4218,7 @@ fn transCreateNodeUnwrapNull(c: *Context, wrapped: *ast.Node) !*ast.Node { } fn transCreateNodeEnumLiteral(c: *Context, name: []const u8) !*ast.Node { - const node = try c.a().create(ast.Node.EnumLiteral); + const node = try c.arena.create(ast.Node.EnumLiteral); node.* = .{ .dot = try appendToken(c, .Period, "."), .name = try appendIdentifier(c, name), @@ -4173,7 +4227,7 @@ fn transCreateNodeEnumLiteral(c: *Context, name: []const u8) !*ast.Node { } fn transCreateNodeStringLiteral(c: *Context, str: []const u8) !*ast.Node { - const node = try c.a().create(ast.Node.StringLiteral); + const node = try c.arena.create(ast.Node.StringLiteral); node.* = .{ .token = try appendToken(c, .StringLiteral, str), }; @@ -4183,7 +4237,7 @@ fn transCreateNodeStringLiteral(c: *Context, str: []const u8) !*ast.Node { fn transCreateNodeIf(c: *Context) !*ast.Node.If { const if_tok = try appendToken(c, .Keyword_if, "if"); _ = try appendToken(c, .LParen, "("); - const node = try c.a().create(ast.Node.If); + const node = try c.arena.create(ast.Node.If); node.* = .{ .if_token = if_tok, .condition = undefined, @@ -4195,7 +4249,7 @@ fn transCreateNodeIf(c: *Context) !*ast.Node.If { } fn transCreateNodeElse(c: *Context) !*ast.Node.Else { - const node = try c.a().create(ast.Node.Else); + const node = try c.arena.create(ast.Node.Else); node.* = .{ .else_token = try appendToken(c, .Keyword_else, "else"), .payload = null, @@ -4210,11 +4264,11 @@ fn transCreateNodeBlock(c: *Context, label: ?[]const u8) !*ast.Node.Block { _ = try appendToken(c, .Colon, ":"); break :blk ll; } else null; - const block_node = try c.a().create(ast.Node.Block); + const block_node = try c.arena.create(ast.Node.Block); block_node.* = .{ .label = label_node, .lbrace = try appendToken(c, .LBrace, "{"), - .statements = ast.Node.Block.StatementList.init(c.a()), + .statements = ast.Node.Block.StatementList{}, .rbrace = undefined, }; return block_node; @@ -4226,7 +4280,7 @@ fn transCreateNodeBreak(c: *Context, label: ?[]const u8) !*ast.Node.ControlFlowE _ = try appendToken(c, .Colon, ":"); break :blk try transCreateNodeIdentifier(c, l); } else null; - const node = try c.a().create(ast.Node.ControlFlowExpression); + const node = try c.arena.create(ast.Node.ControlFlowExpression); node.* = .{ .ltoken = ltoken, .kind = .{ .Break = label_node }, @@ -4240,7 +4294,7 @@ fn transCreateNodeVarDecl(c: *Context, is_pub: bool, is_const: bool, name: []con const mut_tok = if (is_const) try appendToken(c, .Keyword_const, "const") else try appendToken(c, .Keyword_var, "var"); const name_tok = try appendIdentifier(c, name); - const node = try c.a().create(ast.Node.VarDecl); + const node = try c.arena.create(ast.Node.VarDecl); node.* = .{ .doc_comments = null, .visib_token = visib_tok, @@ -4264,7 +4318,7 @@ fn transCreateNodeWhile(c: *Context) !*ast.Node.While { const while_tok = try appendToken(c, .Keyword_while, "while"); _ = try appendToken(c, .LParen, "("); - const node = try c.a().create(ast.Node.While); + const node = try c.arena.create(ast.Node.While); node.* = .{ .label = null, .inline_token = null, @@ -4280,7 +4334,7 @@ fn transCreateNodeWhile(c: *Context) !*ast.Node.While { fn transCreateNodeContinue(c: *Context) !*ast.Node { const ltoken = try appendToken(c, .Keyword_continue, "continue"); - const node = try c.a().create(ast.Node.ControlFlowExpression); + const node = try c.arena.create(ast.Node.ControlFlowExpression); node.* = .{ .ltoken = ltoken, .kind = .{ .Continue = null }, @@ -4294,11 +4348,11 @@ fn transCreateNodeSwitch(c: *Context) !*ast.Node.Switch { const switch_tok = try appendToken(c, .Keyword_switch, "switch"); _ = try appendToken(c, .LParen, "("); - const node = try c.a().create(ast.Node.Switch); + const node = try c.arena.create(ast.Node.Switch); node.* = .{ .switch_token = switch_tok, .expr = undefined, - .cases = ast.Node.Switch.CaseList.init(c.a()), + .cases = ast.Node.Switch.CaseList{}, .rbrace = undefined, }; return node; @@ -4307,9 +4361,9 @@ fn transCreateNodeSwitch(c: *Context) !*ast.Node.Switch { fn transCreateNodeSwitchCase(c: *Context, lhs: *ast.Node) !*ast.Node.SwitchCase { const arrow_tok = try appendToken(c, .EqualAngleBracketRight, "=>"); - const node = try c.a().create(ast.Node.SwitchCase); + const node = try c.arena.create(ast.Node.SwitchCase); node.* = .{ - .items = ast.Node.SwitchCase.ItemList.init(c.a()), + .items = ast.Node.SwitchCase.ItemList{}, .arrow_token = arrow_tok, .payload = null, .expr = undefined, @@ -4319,7 +4373,7 @@ fn transCreateNodeSwitchCase(c: *Context, lhs: *ast.Node) !*ast.Node.SwitchCase } fn transCreateNodeSwitchElse(c: *Context) !*ast.Node { - const node = try c.a().create(ast.Node.SwitchElse); + const node = try c.arena.create(ast.Node.SwitchElse); node.* = .{ .token = try appendToken(c, .Keyword_else, "else"), }; @@ -4352,7 +4406,7 @@ fn transCreateNodeShiftOp( try cast_node.params.push(rhs); cast_node.rparen_token = try appendToken(rp.c, .RParen, ")"); - const node = try rp.c.a().create(ast.Node.InfixOp); + const node = try rp.c.arena.create(ast.Node.InfixOp); node.* = .{ .op_token = op_token, .lhs = lhs, @@ -4364,7 +4418,7 @@ fn transCreateNodeShiftOp( } fn transCreateNodePtrDeref(c: *Context, lhs: *ast.Node) !*ast.Node { - const node = try c.a().create(ast.Node.SuffixOp); + const node = try c.arena.create(ast.Node.SuffixOp); node.* = .{ .lhs = .{ .node = lhs }, .op = .Deref, @@ -4375,7 +4429,7 @@ fn transCreateNodePtrDeref(c: *Context, lhs: *ast.Node) !*ast.Node { fn transCreateNodeArrayAccess(c: *Context, lhs: *ast.Node) !*ast.Node.SuffixOp { _ = try appendToken(c, .LBrace, "["); - const node = try c.a().create(ast.Node.SuffixOp); + const node = try c.arena.create(ast.Node.SuffixOp); node.* = .{ .lhs = .{ .node = lhs }, .op = .{ @@ -4392,7 +4446,7 @@ const RestorePoint = struct { src_buf_index: usize, fn activate(self: RestorePoint) void { - self.c.tree.tokens.shrink(self.token_index); + self.c.tokens.shrink(self.c.gpa, self.token_index); self.c.source_buffer.shrink(self.src_buf_index); } }; @@ -4400,7 +4454,7 @@ const RestorePoint = struct { fn makeRestorePoint(c: *Context) RestorePoint { return RestorePoint{ .c = c, - .token_index = c.tree.tokens.len, + .token_index = c.tokens.items.len, .src_buf_index = c.source_buffer.items.len, }; } @@ -4647,7 +4701,8 @@ fn finishTransFnProto( const name_tok = if (fn_decl_context) |ctx| try appendIdentifier(rp.c, ctx.fn_name) else null; const lparen_tok = try appendToken(rp.c, .LParen, "("); - var fn_params = ast.Node.FnProto.ParamList.init(rp.c.a()); + var fn_params = ast.Node.FnProto.ParamList{}; + var fn_params_list = rp.c.llpusher(&fn_params); const param_count: usize = if (fn_proto_ty != null) ZigClangFunctionProtoType_getNumParams(fn_proto_ty.?) else 0; var i: usize = 0; @@ -4672,7 +4727,7 @@ fn finishTransFnProto( const type_node = try transQualType(rp, param_qt, source_loc); - const param_node = try rp.c.a().create(ast.Node.ParamDecl); + const param_node = try rp.c.arena.create(ast.Node.ParamDecl); param_node.* = .{ .doc_comments = null, .comptime_token = null, @@ -4680,7 +4735,7 @@ fn finishTransFnProto( .name_token = param_name_tok, .param_type = .{ .type_expr = type_node }, }; - try fn_params.push(¶m_node.base); + try fn_params_list.push(¶m_node.base); if (i + 1 < param_count) { _ = try appendToken(rp.c, .Comma, ","); @@ -4692,7 +4747,7 @@ fn finishTransFnProto( _ = try appendToken(rp.c, .Comma, ","); } - const var_arg_node = try rp.c.a().create(ast.Node.ParamDecl); + const var_arg_node = try rp.c.arena.create(ast.Node.ParamDecl); var_arg_node.* = .{ .doc_comments = null, .comptime_token = null, @@ -4700,7 +4755,7 @@ fn finishTransFnProto( .name_token = null, .param_type = .{ .var_args = try appendToken(rp.c, .Ellipsis3, "...") } }; - try fn_params.push(&var_arg_node.base); + try fn_params_list.push(&var_arg_node.base); } const rparen_tok = try appendToken(rp.c, .RParen, ")"); @@ -4713,7 +4768,7 @@ fn finishTransFnProto( _ = try appendToken(rp.c, .LParen, "("); const expr = try transCreateNodeStringLiteral( rp.c, - try std.fmt.allocPrint(rp.c.a(), "\"{}\"", .{str_ptr[0..str_len]}), + try std.fmt.allocPrint(rp.c.arena, "\"{}\"", .{str_ptr[0..str_len]}), ); _ = try appendToken(rp.c, .RParen, ")"); @@ -4767,7 +4822,7 @@ fn finishTransFnProto( } }; - const fn_proto = try rp.c.a().create(ast.Node.FnProto); + const fn_proto = try rp.c.arena.create(ast.Node.FnProto); fn_proto.* = .{ .doc_comments = null, .visib_token = pub_tok, @@ -4816,20 +4871,21 @@ pub fn failDecl(c: *Context, loc: ZigClangSourceLocation, name: []const u8, comp const semi_tok = try appendToken(c, .Semicolon, ";"); _ = try appendTokenFmt(c, .LineComment, "// {}", .{c.locStr(loc)}); - const msg_node = try c.a().create(ast.Node.StringLiteral); + const msg_node = try c.arena.create(ast.Node.StringLiteral); msg_node.* = .{ .token = msg_tok, }; - const call_node = try c.a().create(ast.Node.BuiltinCall); + const call_node = try c.arena.create(ast.Node.BuiltinCall); call_node.* = .{ .builtin_token = builtin_tok, - .params = ast.Node.BuiltinCall.ParamList.init(c.a()), + .params = ast.Node.BuiltinCall.ParamList{}, .rparen_token = rparen_tok, }; - try call_node.params.push(&msg_node.base); + var call_params_it = &call_node.params.first; + call_params_it = try c.llpush(*ast.Node, call_params_it, &msg_node.base); - const var_decl_node = try c.a().create(ast.Node.VarDecl); + const var_decl_node = try c.arena.create(ast.Node.VarDecl); var_decl_node.* = .{ .doc_comments = null, .visib_token = pub_tok, @@ -4861,9 +4917,9 @@ fn appendTokenFmt(c: *Context, token_id: Token.Id, comptime format: []const u8, try c.source_buffer.outStream().print(format, args); const end_index = c.source_buffer.items.len; - const token_index = c.tree.tokens.len; - const new_token = try c.tree.tokens.addOne(); - errdefer c.tree.tokens.shrink(token_index); + const token_index = c.tokens.items.len; + const new_token = try c.tokens.addOne(c.gpa); + errdefer c.tokens.shrink(c.gpa, token_index); new_token.* = .{ .id = token_id, @@ -4931,7 +4987,7 @@ fn appendIdentifier(c: *Context, name: []const u8) !ast.TokenIndex { fn transCreateNodeIdentifier(c: *Context, name: []const u8) !*ast.Node { const token_index = try appendIdentifier(c, name); - const identifier = try c.a().create(ast.Node.Identifier); + const identifier = try c.arena.create(ast.Node.Identifier); identifier.* = .{ .token = token_index, }; @@ -4940,7 +4996,7 @@ fn transCreateNodeIdentifier(c: *Context, name: []const u8) !*ast.Node { fn transCreateNodeIdentifierUnchecked(c: *Context, name: []const u8) !*ast.Node { const token_index = try appendTokenFmt(c, .Identifier, "{}", .{name}); - const identifier = try c.a().create(ast.Node.Identifier); + const identifier = try c.arena.create(ast.Node.Identifier); identifier.* = .{ .token = token_index, }; @@ -4955,7 +5011,7 @@ fn transPreprocessorEntities(c: *Context, unit: *ZigClangASTUnit) Error!void { // TODO if we see #undef, delete it from the table var it = ZigClangASTUnit_getLocalPreprocessingEntities_begin(unit); const it_end = ZigClangASTUnit_getLocalPreprocessingEntities_end(unit); - var tok_list = CTokenList.init(c.a()); + var tok_list = CTokenList.init(c.arena); const scope = c.global_scope; while (it.I != it_end.I) : (it.I += 1) { @@ -4970,7 +5026,7 @@ fn transPreprocessorEntities(c: *Context, unit: *ZigClangASTUnit) Error!void { const name = try c.str(raw_name); // TODO https://github.com/ziglang/zig/issues/3756 // TODO https://github.com/ziglang/zig/issues/1802 - const mangled_name = if (isZigPrimitiveType(name)) try std.fmt.allocPrint(c.a(), "{}_{}", .{ name, c.getMangle() }) else name; + const mangled_name = if (isZigPrimitiveType(name)) try std.fmt.allocPrint(c.arena, "{}_{}", .{ name, c.getMangle() }) else name; if (scope.containsNow(mangled_name)) { continue; } @@ -5078,7 +5134,8 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, .{}, ); } - var fn_params = ast.Node.FnProto.ParamList.init(c.a()); + var fn_params = ast.Node.FnProto.ParamList{}; + var fn_params_it = &fn_params.first; while (true) { const param_tok = it.next().?; if (param_tok.id != .Identifier) { @@ -5096,12 +5153,12 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, _ = try appendToken(c, .Colon, ":"); const token_index = try appendToken(c, .Keyword_var, "var"); - const identifier = try c.a().create(ast.Node.Identifier); + const identifier = try c.arena.create(ast.Node.Identifier); identifier.* = .{ .token = token_index, }; - const param_node = try c.a().create(ast.Node.ParamDecl); + const param_node = try c.arena.create(ast.Node.ParamDecl); param_node.* = .{ .doc_comments = null, .comptime_token = null, @@ -5109,7 +5166,7 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, .name_token = param_name_tok, .param_type = .{ .type_expr = &identifier.base }, }; - try fn_params.push(¶m_node.base); + fn_params_it = try c.llpush(*ast.Node, fn_params_it, ¶m_node.base); if (it.peek().?.id != .Comma) break; @@ -5131,8 +5188,9 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, const type_of = try transCreateNodeBuiltinFnCall(c, "@TypeOf"); type_of.rparen_token = try appendToken(c, .RParen, ")"); + var type_of_params = c.llpusher(&type_of.params); - const fn_proto = try c.a().create(ast.Node.FnProto); + const fn_proto = try c.arena.create(ast.Node.FnProto); fn_proto.* = .{ .visib_token = pub_tok, .extern_export_inline_token = inline_tok, @@ -5150,6 +5208,7 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, }; const block = try transCreateNodeBlock(c, null); + var block_statements = c.llpusher(&block.statements); const return_expr = try transCreateNodeReturnExpr(c); const expr = try parseCExpr(c, it, source, source_loc, scope); @@ -5165,16 +5224,16 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, _ = try appendToken(c, .Semicolon, ";"); const type_of_arg = if (expr.id != .Block) expr else blk: { const blk = @fieldParentPtr(ast.Node.Block, "base", expr); - const blk_last = blk.statements.at(blk.statements.len - 1).*; + const blk_last = blk.statements.first.?.findLast().data; std.debug.assert(blk_last.id == .ControlFlowExpression); const br = @fieldParentPtr(ast.Node.ControlFlowExpression, "base", blk_last); break :blk br.rhs.?; }; - try type_of.params.push(type_of_arg); + try type_of_params.push(type_of_arg); return_expr.rhs = expr; block.rbrace = try appendToken(c, .RBrace, "}"); - try block.statements.push(&return_expr.base); + try block_statements.push(&return_expr.base); fn_proto.body_node = &block.base; _ = try c.global_scope.macro_table.put(name, &fn_proto.base); } @@ -5208,14 +5267,14 @@ fn parseCExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, source_ .Comma => { _ = try appendToken(c, .Semicolon, ";"); const block_scope = try Scope.Block.init(c, scope, "blk"); - block_scope.block_node = try transCreateNodeBlock(c, block_scope.label); + block_scope.setBlockNode(try transCreateNodeBlock(c, block_scope.label)); var last = node; while (true) { // suppress result const lhs = try transCreateNodeIdentifier(c, "_"); const op_token = try appendToken(c, .Equal, "="); - const op_node = try c.a().create(ast.Node.InfixOp); + const op_node = try c.arena.create(ast.Node.InfixOp); op_node.* = .{ .op_token = op_token, .lhs = lhs, @@ -5253,11 +5312,11 @@ fn parseCNumLit(c: *Context, tok: *CToken, source: []const u8, source_loc: ZigCl switch (lit_bytes[1]) { '0'...'7' => { // Octal - lit_bytes = try std.fmt.allocPrint(c.a(), "0o{}", .{lit_bytes}); + lit_bytes = try std.fmt.allocPrint(c.arena, "0o{}", .{lit_bytes}); }, 'X' => { // Hexadecimal with capital X, valid in C but not in Zig - lit_bytes = try std.fmt.allocPrint(c.a(), "0x{}", .{lit_bytes[2..]}); + lit_bytes = try std.fmt.allocPrint(c.arena, "0x{}", .{lit_bytes[2..]}); }, else => {}, } @@ -5288,7 +5347,7 @@ fn parseCNumLit(c: *Context, tok: *CToken, source: []const u8, source_loc: ZigCl return &cast_node.base; } else if (tok.id == .FloatLiteral) { if (lit_bytes[0] == '.') - lit_bytes = try std.fmt.allocPrint(c.a(), "0{}", .{lit_bytes}); + lit_bytes = try std.fmt.allocPrint(c.arena, "0{}", .{lit_bytes}); if (tok.id.FloatLiteral == .None) { return transCreateNodeFloat(c, lit_bytes); } @@ -5318,7 +5377,7 @@ fn zigifyEscapeSequences(ctx: *Context, source_bytes: []const u8, name: []const break; } } else return source; - var bytes = try ctx.a().alloc(u8, source.len * 2); + var bytes = try ctx.arena.alloc(u8, source.len * 2); var state: enum { Start, Escape, @@ -5472,14 +5531,14 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, const first_tok = it.list.at(0); if (source[tok.start] != '\'' or source[tok.start + 1] == '\\' or tok.end - tok.start == 3) { const token = try appendToken(c, .CharLiteral, try zigifyEscapeSequences(c, source[tok.start..tok.end], source[first_tok.start..first_tok.end], source_loc)); - const node = try c.a().create(ast.Node.CharLiteral); + const node = try c.arena.create(ast.Node.CharLiteral); node.* = .{ .token = token, }; return &node.base; } else { const token = try appendTokenFmt(c, .IntegerLiteral, "0x{x}", .{source[tok.start+1..tok.end-1]}); - const node = try c.a().create(ast.Node.IntegerLiteral); + const node = try c.arena.create(ast.Node.IntegerLiteral); node.* = .{ .token = token, }; @@ -5489,7 +5548,7 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, .StringLiteral => { const first_tok = it.list.at(0); const token = try appendToken(c, .StringLiteral, try zigifyEscapeSequences(c, source[tok.start..tok.end], source[first_tok.start..first_tok.end], source_loc)); - const node = try c.a().create(ast.Node.StringLiteral); + const node = try c.arena.create(ast.Node.StringLiteral); node.* = .{ .token = token, }; @@ -5572,7 +5631,7 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, const type_info_node = try transCreateNodeBuiltinFnCall(c, "@typeInfo"); try type_info_node.params.push(inner_node); type_info_node.rparen_token = try appendToken(c, .LParen, ")"); - const cmp_node = try c.a().create(ast.Node.InfixOp); + const cmp_node = try c.arena.create(ast.Node.InfixOp); cmp_node.* = .{ .op_token = try appendToken(c, .EqualEqual, "=="), .lhs = &type_info_node.base, @@ -5597,7 +5656,7 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, as_node.rparen_token = try appendToken(c, .RParen, ")"); else_node.body = &as_node.base; - const group_node = try c.a().create(ast.Node.GroupedExpression); + const group_node = try c.arena.create(ast.Node.GroupedExpression); group_node.* = .{ .lparen = lparen, .expr = &if_node.base, @@ -5621,7 +5680,7 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, type_of_1.rparen_token = try appendToken(c, .RParen, ")"); type_info_1.rparen_token = try appendToken(c, .RParen, ")"); - const cmp_1 = try c.a().create(ast.Node.InfixOp); + const cmp_1 = try c.arena.create(ast.Node.InfixOp); cmp_1.* = .{ .op_token = try appendToken(c, .EqualEqual, "=="), .lhs = &type_info_1.base, @@ -5633,7 +5692,7 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, const period_tok = try appendToken(c, .Period, "."); const child_ident = try transCreateNodeIdentifier(c, "Child"); - const inner_node_child = try c.a().create(ast.Node.InfixOp); + const inner_node_child = try c.arena.create(ast.Node.InfixOp); inner_node_child.* = .{ .op_token = period_tok, .lhs = inner_node, @@ -5669,7 +5728,7 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, type_of_2.rparen_token = try appendToken(c, .RParen, ")"); type_info_2.rparen_token = try appendToken(c, .RParen, ")"); - const cmp_2 = try c.a().create(ast.Node.InfixOp); + const cmp_2 = try c.arena.create(ast.Node.InfixOp); cmp_2.* = .{ .op_token = try appendToken(c, .EqualEqual, "=="), .lhs = &type_info_2.base, @@ -5677,7 +5736,7 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, .rhs = try transCreateNodeEnumLiteral(c, "Int"), }; if_2.condition = &cmp_2.base; - const cmp_4 = try c.a().create(ast.Node.InfixOp); + const cmp_4 = try c.arena.create(ast.Node.InfixOp); cmp_4.* = .{ .op_token = try appendToken(c, .Keyword_and, "and"), .lhs = &cmp_2.base, @@ -5687,7 +5746,7 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, const type_info_3 = try transCreateNodeBuiltinFnCall(c, "@typeInfo"); try type_info_3.params.push(inner_node); type_info_3.rparen_token = try appendToken(c, .LParen, ")"); - const cmp_3 = try c.a().create(ast.Node.InfixOp); + const cmp_3 = try c.arena.create(ast.Node.InfixOp); cmp_3.* = .{ .op_token = try appendToken(c, .EqualEqual, "=="), .lhs = &type_info_3.base, @@ -5714,7 +5773,7 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, as.rparen_token = try appendToken(c, .RParen, ")"); else_2.body = &as.base; - const group_node = try c.a().create(ast.Node.GroupedExpression); + const group_node = try c.arena.create(ast.Node.GroupedExpression); group_node.* = .{ .lparen = lparen, .expr = &if_1.base, @@ -5740,7 +5799,7 @@ fn macroBoolToInt(c: *Context, node: *ast.Node) !*ast.Node { if (!isBoolRes(node)) { if (node.id != .InfixOp) return node; - const group_node = try c.a().create(ast.Node.GroupedExpression); + const group_node = try c.arena.create(ast.Node.GroupedExpression); group_node.* = .{ .lparen = try appendToken(c, .LParen, "("), .expr = node, @@ -5759,7 +5818,7 @@ fn macroIntToBool(c: *Context, node: *ast.Node) !*ast.Node { if (isBoolRes(node)) { if (node.id != .InfixOp) return node; - const group_node = try c.a().create(ast.Node.GroupedExpression); + const group_node = try c.arena.create(ast.Node.GroupedExpression); group_node.* = .{ .lparen = try appendToken(c, .LParen, "("), .expr = node, @@ -5770,14 +5829,14 @@ fn macroIntToBool(c: *Context, node: *ast.Node) !*ast.Node { const op_token = try appendToken(c, .BangEqual, "!="); const zero = try transCreateNodeInt(c, 0); - const res = try c.a().create(ast.Node.InfixOp); + const res = try c.arena.create(ast.Node.InfixOp); res.* = .{ .op_token = op_token, .lhs = node, .op = .BangEqual, .rhs = zero, }; - const group_node = try c.a().create(ast.Node.GroupedExpression); + const group_node = try c.arena.create(ast.Node.GroupedExpression); group_node.* = .{ .lparen = try appendToken(c, .LParen, "("), .expr = &res.base, @@ -5989,7 +6048,7 @@ fn parseCSuffixOpExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, const cast_fn = if (bool_op) macroIntToBool else macroBoolToInt; const lhs_node = try cast_fn(c, node); const rhs_node = try parseCPrefixOpExpr(c, it, source, source_loc, scope); - const op_node = try c.a().create(ast.Node.InfixOp); + const op_node = try c.arena.create(ast.Node.InfixOp); op_node.* = .{ .op_token = op_token, .lhs = lhs_node, @@ -6037,7 +6096,7 @@ fn parseCPrefixOpExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, } fn tokenSlice(c: *Context, token: ast.TokenIndex) []u8 { - const tok = c.tree.tokens.at(token); + const tok = c.tokens.items[token]; const slice = c.source_buffer.span()[tok.start..tok.end]; return if (mem.startsWith(u8, slice, "@\"")) slice[2 .. slice.len - 1] @@ -6060,9 +6119,10 @@ fn getContainer(c: *Context, node: *ast.Node) ?*ast.Node { return null; if (getContainerTypeOf(c, infix.lhs)) |ty_node| { if (ty_node.cast(ast.Node.ContainerDecl)) |container| { - var it = container.fields_and_decls.iterator(0); - while (it.next()) |field_ref| { - const field = field_ref.*.cast(ast.Node.ContainerField).?; + var it = container.fields_and_decls.first; + while (it) |field_ref_node| : (it = field_ref_node.next) { + const field_ref = field_ref_node.data; + const field = field_ref.cast(ast.Node.ContainerField).?; const ident = infix.rhs.cast(ast.Node.Identifier).?; if (mem.eql(u8, tokenSlice(c, field.name_token), tokenSlice(c, ident.token))) { return getContainer(c, field.type_expr.?); @@ -6087,9 +6147,10 @@ fn getContainerTypeOf(c: *Context, ref: *ast.Node) ?*ast.Node { return null; if (getContainerTypeOf(c, infix.lhs)) |ty_node| { if (ty_node.cast(ast.Node.ContainerDecl)) |container| { - var it = container.fields_and_decls.iterator(0); - while (it.next()) |field_ref| { - const field = field_ref.*.cast(ast.Node.ContainerField).?; + var it = container.fields_and_decls.first; + while (it) |field_ref_node| : (it = field_ref_node.next) { + const field_ref = field_ref_node.data; + const field = field_ref.cast(ast.Node.ContainerField).?; const ident = infix.rhs.cast(ast.Node.Identifier).?; if (mem.eql(u8, tokenSlice(c, field.name_token), tokenSlice(c, ident.token))) { return getContainer(c, field.type_expr.?); From b1bcdc96ca4451be95caa00fe71aafb578611e34 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 20 May 2020 15:57:02 -0400 Subject: [PATCH 05/38] arena allocator: smaller minimum allocation size --- lib/std/heap/arena_allocator.zig | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/lib/std/heap/arena_allocator.zig b/lib/std/heap/arena_allocator.zig index daed17d6b3..337edf6dc0 100644 --- a/lib/std/heap/arena_allocator.zig +++ b/lib/std/heap/arena_allocator.zig @@ -47,12 +47,8 @@ pub const ArenaAllocator = struct { fn createNode(self: *ArenaAllocator, prev_len: usize, minimum_size: usize) !*BufNode { const actual_min_size = minimum_size + @sizeOf(BufNode); - var len = prev_len; - while (true) { - len += len / 2; - len += mem.page_size - @rem(len, mem.page_size); - if (len >= actual_min_size) break; - } + const big_enough_len = prev_len + actual_min_size; + const len = big_enough_len + big_enough_len / 2; const buf = try self.child_allocator.alignedAlloc(u8, @alignOf(BufNode), len); const buf_node_slice = mem.bytesAsSlice(BufNode, buf[0..@sizeOf(BufNode)]); const buf_node = &buf_node_slice[0]; From 64c149ca19cc299485693acbda325eee51d37f28 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 20 May 2020 16:13:02 -0400 Subject: [PATCH 06/38] fields and decls: ArrayList appending, memcpy to ast arena This makes fields and decl ast nodes part of the Root and ContainerDecl AST nodes. Surprisingly, it's a performance regression from using a singly-linked list for these nodes: throughput: 76.5 MiB/s => 69.4 MiB/s However it has much better memory usage: maxrss: 392 KB => 77 KB It's also better API for consumers of the parser, since it is a flat list in memory. --- lib/std/zig/ast.zig | 106 +++++++++++++++++++++++++++++------------ lib/std/zig/parse.zig | 72 ++++++++++++++++------------ lib/std/zig/render.zig | 45 ++++++++--------- 3 files changed, 137 insertions(+), 86 deletions(-) diff --git a/lib/std/zig/ast.zig b/lib/std/zig/ast.zig index 846b6e5041..a1b2856718 100644 --- a/lib/std/zig/ast.zig +++ b/lib/std/zig/ast.zig @@ -6,6 +6,7 @@ const mem = std.mem; const Token = std.zig.Token; pub const TokenIndex = usize; +pub const NodeIndex = usize; pub const Tree = struct { /// Reference to externally-owned data. @@ -616,40 +617,62 @@ pub const Node = struct { } } + /// The decls data follows this struct in memory as an array of Node pointers. pub const Root = struct { base: Node = Node{ .id = .Root }, - decls: DeclList, eof_token: TokenIndex, + decls_len: NodeIndex, - pub const DeclList = LinkedList(*Node); + /// After this the caller must initialize the decls list. + pub fn create(allocator: *mem.Allocator, decls_len: NodeIndex, eof_token: TokenIndex) !*Root { + const bytes = try allocator.alignedAlloc(u8, @alignOf(Root), sizeInBytes(decls_len)); + const self = @ptrCast(*Root, bytes.ptr); + self.* = .{ + .eof_token = eof_token, + .decls_len = decls_len, + }; + return self; + } + + pub fn destroy(self: *Decl, allocator: *mem.Allocator) void { + const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.decls_len)]; + allocator.free(bytes); + } pub fn iterate(self: *const Root) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = self.decls.first }; + return .{ .parent_node = &self.base, .index = 0, .node = null }; } pub fn iterateNext(self: *const Root, it: *Node.Iterator) ?*Node { - const decl = it.node orelse return null; - it.node = decl.next; - return decl.data; + var i = it.index; + it.index += 1; + + if (i < self.decls_len) return self.declsConst()[i]; + return null; + } + + pub fn decls(self: *Root) []*Node { + const decls_start = @ptrCast([*]u8, self) + @sizeOf(Root); + return @ptrCast([*]*Node, decls_start)[0..self.decls_len]; + } + + pub fn declsConst(self: *const Root) []const *Node { + const decls_start = @ptrCast([*]const u8, self) + @sizeOf(Root); + return @ptrCast([*]const *Node, decls_start)[0..self.decls_len]; } pub fn firstToken(self: *const Root) TokenIndex { - if (self.decls.first) |first| { - return first.data.firstToken(); - } else { - return self.eof_token; - } + if (self.decls_len == 0) return self.eof_token; + return self.declsConst()[0].firstToken(); } pub fn lastToken(self: *const Root) TokenIndex { - if (self.decls.first) |first| { - var node = first; - while (true) { - node = node.next orelse return node.data.lastToken(); - } - } else { - return self.eof_token; - } + if (self.decls_len == 0) return self.eof_token; + return self.declsConst()[self.decls_len - 1].lastToken(); + } + + fn sizeInBytes(decls_len: NodeIndex) usize { + return @sizeOf(Root) + @sizeOf(*Node) * @as(usize, decls_len); } }; @@ -777,14 +800,12 @@ pub const Node = struct { pub const ContainerDecl = struct { base: Node = Node{ .id = .ContainerDecl }, - layout_token: ?TokenIndex, kind_token: TokenIndex, - init_arg_expr: InitArg, - fields_and_decls: DeclList, + layout_token: ?TokenIndex, lbrace_token: TokenIndex, rbrace_token: TokenIndex, - - pub const DeclList = Root.DeclList; + fields_and_decls_len: NodeIndex, + init_arg_expr: InitArg, pub const InitArg = union(enum) { None, @@ -792,8 +813,19 @@ pub const Node = struct { Type: *Node, }; + /// After this the caller must initialize the fields_and_decls list. + pub fn alloc(allocator: *mem.Allocator, fields_and_decls_len: NodeIndex) !*ContainerDecl { + const bytes = try allocator.alignedAlloc(u8, @alignOf(ContainerDecl), sizeInBytes(fields_and_decls_len)); + return @ptrCast(*ContainerDecl, bytes.ptr); + } + + pub fn free(self: *Decl, allocator: *mem.Allocator) void { + const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.fields_and_decls_len)]; + allocator.free(bytes); + } + pub fn iterate(self: *const ContainerDecl) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = self.fields_and_decls.first }; + return .{ .parent_node = &self.base, .index = 0, .node = null }; } pub fn iterateNext(self: *const ContainerDecl, it: *Node.Iterator) ?*Node { @@ -808,10 +840,8 @@ pub const Node = struct { .None, .Enum => {}, } - if (it.node) |child| { - it.node = child.next; - return child.data; - } + if (i < self.fields_and_decls_len) return self.fieldsAndDeclsConst()[i]; + i -= self.fields_and_decls_len; return null; } @@ -826,6 +856,20 @@ pub const Node = struct { pub fn lastToken(self: *const ContainerDecl) TokenIndex { return self.rbrace_token; } + + pub fn fieldsAndDecls(self: *ContainerDecl) []*Node { + const decls_start = @ptrCast([*]u8, self) + @sizeOf(ContainerDecl); + return @ptrCast([*]*Node, decls_start)[0..self.fields_and_decls_len]; + } + + pub fn fieldsAndDeclsConst(self: *const ContainerDecl) []const *Node { + const decls_start = @ptrCast([*]const u8, self) + @sizeOf(ContainerDecl); + return @ptrCast([*]const *Node, decls_start)[0..self.fields_and_decls_len]; + } + + fn sizeInBytes(fields_and_decls_len: NodeIndex) usize { + return @sizeOf(ContainerDecl) + @sizeOf(*Node) * @as(usize, fields_and_decls_len); + } }; pub const ContainerField = struct { @@ -1116,7 +1160,7 @@ pub const Node = struct { statements: StatementList, rbrace: TokenIndex, - pub const StatementList = Root.DeclList; + pub const StatementList = LinkedList(*Node); pub fn iterate(self: *const Block) Node.Iterator { return .{ .parent_node = &self.base, .index = 0, .node = self.statements.first }; @@ -2615,7 +2659,7 @@ pub const Node = struct { test "iterate" { var root = Node.Root{ .base = Node{ .id = Node.Id.Root }, - .decls = Node.Root.DeclList.init(std.testing.allocator), + .decls_len = 0, .eof_token = 0, }; var base = &root.base; diff --git a/lib/std/zig/parse.zig b/lib/std/zig/parse.zig index 7abd3b300c..3ae72cd021 100644 --- a/lib/std/zig/parse.zig +++ b/lib/std/zig/parse.zig @@ -63,16 +63,20 @@ const Parser = struct { /// Root <- skip ContainerMembers eof fn parseRoot(p: *Parser) Allocator.Error!*Node.Root { - const node = try p.arena.allocator.create(Node.Root); - node.* = .{ - .decls = try parseContainerMembers(p, true), - // parseContainerMembers will try to skip as much - // invalid tokens as it can so this can only be the EOF - .eof_token = p.eatToken(.Eof).?, - }; + const decls = try parseContainerMembers(p, true); + defer p.gpa.free(decls); + + // parseContainerMembers will try to skip as much + // invalid tokens as it can so this can only be the EOF + const eof_token = p.eatToken(.Eof).?; + + const node = try Node.Root.create(&p.arena.allocator, decls.len, eof_token); + std.mem.copy(*ast.Node, node.decls(), decls); + return node; } + /// Helper function for appending elements to a singly linked list. fn llpush( p: *Parser, comptime T: type, @@ -92,9 +96,9 @@ const Parser = struct { /// / ContainerField COMMA ContainerMembers /// / ContainerField /// / - fn parseContainerMembers(p: *Parser, top_level: bool) !Node.Root.DeclList { - var list = Node.Root.DeclList{}; - var list_it = &list.first; + fn parseContainerMembers(p: *Parser, top_level: bool) ![]*ast.Node { + var list = std.ArrayList(*ast.Node).init(p.gpa); + defer list.deinit(); var field_state: union(enum) { /// no fields have been seen @@ -110,7 +114,7 @@ const Parser = struct { while (true) { if (try p.parseContainerDocComments()) |node| { - list_it = try p.llpush(*Node, list_it, node); + try list.append(node); continue; } @@ -127,7 +131,7 @@ const Parser = struct { field_state = .{ .end = node.firstToken() }; } node.cast(Node.TestDecl).?.doc_comments = doc_comments; - list_it = try p.llpush(*Node, list_it, node); + try list.append(node); continue; } @@ -142,7 +146,7 @@ const Parser = struct { field_state = .{ .end = node.firstToken() }; } node.cast(Node.Comptime).?.doc_comments = doc_comments; - list_it = try p.llpush(*Node, list_it, node); + try list.append(node); continue; } @@ -173,7 +177,7 @@ const Parser = struct { }, else => unreachable, } - list_it = try p.llpush(*Node, list_it, node); + try list.append(node); if (try p.parseAppendedDocComment(node.lastToken())) |appended_comment| { switch (node.id) { .FnProto => {}, @@ -215,7 +219,7 @@ const Parser = struct { const field = node.cast(Node.ContainerField).?; field.doc_comments = doc_comments; - list_it = try p.llpush(*Node, list_it, node); + try list.append(node); const comma = p.eatToken(.Comma) orelse { // try to continue parsing const index = p.tok_i; @@ -275,7 +279,7 @@ const Parser = struct { } } - return list; + return list.toOwnedSlice(); } /// Attempts to find next container member by searching for certain tokens @@ -2778,24 +2782,36 @@ const Parser = struct { /// ContainerDeclAuto <- ContainerDeclType LBRACE ContainerMembers RBRACE fn parseContainerDeclAuto(p: *Parser) !?*Node { - const node = (try p.parseContainerDeclType()) orelse return null; + const container_decl_type = (try p.parseContainerDeclType()) orelse return null; const lbrace = try p.expectToken(.LBrace); const members = try p.parseContainerMembers(false); + defer p.gpa.free(members); const rbrace = try p.expectToken(.RBrace); - const decl_type = node.cast(Node.ContainerDecl).?; - decl_type.fields_and_decls = members; - decl_type.lbrace_token = lbrace; - decl_type.rbrace_token = rbrace; - - return node; + const node = try Node.ContainerDecl.alloc(&p.arena.allocator, members.len); + node.* = .{ + .layout_token = null, + .kind_token = container_decl_type.kind_token, + .init_arg_expr = container_decl_type.init_arg_expr, + .fields_and_decls_len = members.len, + .lbrace_token = lbrace, + .rbrace_token = rbrace, + }; + std.mem.copy(*ast.Node, node.fieldsAndDecls(), members); + return &node.base; } + /// Holds temporary data until we are ready to construct the full ContainerDecl AST node. + const ContainerDeclType = struct { + kind_token: TokenIndex, + init_arg_expr: ast.Node.ContainerDecl.InitArg, + }; + /// ContainerDeclType /// <- KEYWORD_struct /// / KEYWORD_enum (LPAREN Expr RPAREN)? /// / KEYWORD_union (LPAREN (KEYWORD_enum (LPAREN Expr RPAREN)? / Expr) RPAREN)? - fn parseContainerDeclType(p: *Parser) !?*Node { + fn parseContainerDeclType(p: *Parser) !?ContainerDeclType { const kind_token = p.nextToken(); const init_arg_expr = switch (kind_token.ptr.id) { @@ -2838,16 +2854,10 @@ const Parser = struct { }, }; - const node = try p.arena.allocator.create(Node.ContainerDecl); - node.* = .{ - .layout_token = null, + return ContainerDeclType{ .kind_token = kind_token.index, .init_arg_expr = init_arg_expr, - .fields_and_decls = undefined, // set by caller - .lbrace_token = undefined, // set by caller - .rbrace_token = undefined, // set by caller }; - return &node.base; } /// ByteAlign <- KEYWORD_align LPAREN Expr RPAREN diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index bcd7ef4887..cfae09dcd4 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -79,9 +79,10 @@ fn renderRoot( } var start_col: usize = 0; - var it = tree.root_node.decls.first orelse return; + var decl_i: ast.NodeIndex = 0; + const root_decls = tree.root_node.decls(); while (true) { - var decl = it.data; + var decl = root_decls[decl_i]; // This loop does the following: // @@ -130,14 +131,15 @@ fn renderRoot( token_index = decl.firstToken(); while (!fmt_active) { - it = it.next orelse { + decl_i += 1; + if (decl_i >= root_decls.len) { // If there's no next reformatted `decl`, just copy the // remaining input tokens and bail out. const start = tree.tokens[copy_start_token_index].start; try copyFixingWhitespace(stream, tree.source[start..]); return; - }; - decl = it.data; + } + decl = root_decls[decl_i]; var decl_first_token_index = decl.firstToken(); while (token_index < decl_first_token_index) : (token_index += 1) { @@ -178,8 +180,9 @@ fn renderRoot( } try renderTopLevelDecl(allocator, stream, tree, 0, &start_col, decl); - it = it.next orelse return; - try renderExtraNewline(tree, stream, &start_col, it.data); + decl_i += 1; + if (decl_i >= root_decls.len) return; + try renderExtraNewline(tree, stream, &start_col, root_decls[decl_i]); } } @@ -1189,7 +1192,7 @@ fn renderExpression( }, } - if (container_decl.fields_and_decls.first == null) { + if (container_decl.fields_and_decls_len == 0) { try renderToken(tree, stream, container_decl.lbrace_token, indent + indent_delta, start_col, Space.None); // { return renderToken(tree, stream, container_decl.rbrace_token, indent, start_col, space); // } } @@ -1203,18 +1206,18 @@ fn renderExpression( break :blk tree.tokens[maybe_comma].id == .Comma; }; + const fields_and_decls = container_decl.fieldsAndDecls(); + // Check if the first declaration and the { are on the same line const src_has_newline = !tree.tokensOnSameLine( container_decl.lbrace_token, - container_decl.fields_and_decls.first.?.data.firstToken(), + fields_and_decls[0].firstToken(), ); // We can only print all the elements in-line if all the // declarations inside are fields const src_has_only_fields = blk: { - var it = container_decl.fields_and_decls.first; - while (it) |decl_node| : (it = decl_node.next) { - const decl = decl_node.data; + for (fields_and_decls) |decl| { if (decl.id != .ContainerField) break :blk false; } break :blk true; @@ -1225,14 +1228,12 @@ fn renderExpression( const new_indent = indent + indent_delta; try renderToken(tree, stream, container_decl.lbrace_token, new_indent, start_col, .Newline); // { - var it = container_decl.fields_and_decls.first; - while (it) |decl_node| : (it = decl_node.next) { - const decl = decl_node.data; + for (fields_and_decls) |decl, i| { try stream.writeByteNTimes(' ', new_indent); try renderContainerDecl(allocator, stream, tree, new_indent, start_col, decl, .Newline); - if (decl_node.next) |next_decl| { - try renderExtraNewline(tree, stream, start_col, next_decl.data); + if (i + 1 < fields_and_decls.len) { + try renderExtraNewline(tree, stream, start_col, fields_and_decls[i + 1]); } } @@ -1245,10 +1246,8 @@ fn renderExpression( const new_indent = indent + indent_delta; try stream.writeByteNTimes(' ', new_indent); - var it = container_decl.fields_and_decls.first; - while (it) |decl_node| : (it = decl_node.next) { - const decl = decl_node.data; - const space_after_decl: Space = if (decl_node.next == null) .Newline else .Space; + for (fields_and_decls) |decl, i| { + const space_after_decl: Space = if (i + 1 >= fields_and_decls.len) .Newline else .Space; try renderContainerDecl(allocator, stream, tree, new_indent, start_col, decl, space_after_decl); } @@ -1257,9 +1256,7 @@ fn renderExpression( // All the declarations on the same line try renderToken(tree, stream, container_decl.lbrace_token, indent, start_col, .Space); // { - var it = container_decl.fields_and_decls.first; - while (it) |decl_node| : (it = decl_node.next) { - const decl = decl_node.data; + for (fields_and_decls) |decl| { try renderContainerDecl(allocator, stream, tree, indent, start_col, decl, .Space); } } From ad4b591db1155d52bfce526c695594cc2dda7f14 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 20 May 2020 16:30:28 -0400 Subject: [PATCH 07/38] small bump to ArenaAllocator minimum alloc size self-hosted parser benchmark: throughput: 69.4 MiB/s => 72.2 MiB/s --- lib/std/heap/arena_allocator.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/heap/arena_allocator.zig b/lib/std/heap/arena_allocator.zig index 337edf6dc0..b41399772a 100644 --- a/lib/std/heap/arena_allocator.zig +++ b/lib/std/heap/arena_allocator.zig @@ -46,7 +46,7 @@ pub const ArenaAllocator = struct { } fn createNode(self: *ArenaAllocator, prev_len: usize, minimum_size: usize) !*BufNode { - const actual_min_size = minimum_size + @sizeOf(BufNode); + const actual_min_size = minimum_size + (@sizeOf(BufNode) + 16); const big_enough_len = prev_len + actual_min_size; const len = big_enough_len + big_enough_len / 2; const buf = try self.child_allocator.alignedAlloc(u8, @alignOf(BufNode), len); From 5db9f306ba88087d269dc5fe5672a5af5ab55333 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 20 May 2020 16:54:21 -0400 Subject: [PATCH 08/38] update translate-c for new Root and ContainerDecl AST --- src-self-hosted/translate_c.zig | 86 +++++++++++++++------------------ 1 file changed, 39 insertions(+), 47 deletions(-) diff --git a/src-self-hosted/translate_c.zig b/src-self-hosted/translate_c.zig index 184c07cdc9..b55197c91f 100644 --- a/src-self-hosted/translate_c.zig +++ b/src-self-hosted/translate_c.zig @@ -229,8 +229,7 @@ pub const Context = struct { global_scope: *Scope.Root, clang_context: *ZigClangASTContext, mangle_count: u32 = 0, - root_node: *ast.Node.Root, - root_decls_it: *?*std.SinglyLinkedList(*ast.Node).Node, + root_decls: std.ArrayListUnmanaged(*ast.Node), /// This one is different than the root scope's name table. This contains /// a list of names that we found by visiting all the top level decls without @@ -318,13 +317,6 @@ pub fn translate( var arena = std.heap.ArenaAllocator.init(gpa); errdefer arena.deinit(); - const root_node = try arena.allocator.create(ast.Node.Root); - root_node.* = .{ - .decls = ast.Node.Root.DeclList{}, - // initialized with the eof token at the end - .eof_token = undefined, - }; - var context = Context{ .gpa = gpa, .arena = &arena.allocator, @@ -338,8 +330,7 @@ pub fn translate( .global_names = std.StringHashMap(void).init(gpa), .tokens = .{}, .errors = .{}, - .root_node = root_node, - .root_decls_it = &root_node.decls.first, + .root_decls = .{}, }; context.global_scope.* = Scope.Root.init(&context); defer context.decl_table.deinit(); @@ -347,6 +338,7 @@ pub fn translate( defer context.tokens.deinit(gpa); defer context.errors.deinit(gpa); defer context.global_names.deinit(); + defer context.root_decls.deinit(gpa); try prepopulateGlobalNameTable(ast_unit, &context); @@ -363,7 +355,10 @@ pub fn translate( } } - root_node.eof_token = try appendToken(&context, .Eof, ""); + const eof_token = try appendToken(&context, .Eof, ""); + const root_node = try ast.Node.Root.create(&arena.allocator, context.root_decls.items.len, eof_token); + mem.copy(*ast.Node, root_node.decls(), context.root_decls.items); + if (false) { std.debug.warn("debug source:\n{}\n==EOF==\ntokens:\n", .{source_buffer.items}); for (context.tokens.items) |token| { @@ -831,16 +826,8 @@ fn transRecordDecl(c: *Context, record_decl: *const ZigClangRecordDecl) Error!?* const container_tok = try appendToken(c, container_kind, container_kind_name); const lbrace_token = try appendToken(c, .LBrace, "{"); - const container_node = try c.arena.create(ast.Node.ContainerDecl); - container_node.* = .{ - .layout_token = layout_tok, - .kind_token = container_tok, - .init_arg_expr = .None, - .fields_and_decls = ast.Node.ContainerDecl.DeclList{}, - .lbrace_token = lbrace_token, - .rbrace_token = undefined, - }; - var container_fields_and_decls = c.llpusher(&container_node.fields_and_decls); + var fields_and_decls = std.ArrayList(*ast.Node).init(c.gpa); + defer fields_and_decls.deinit(); var unnamed_field_count: u32 = 0; var it = ZigClangRecordDecl_field_begin(record_def); @@ -915,10 +902,19 @@ fn transRecordDecl(c: *Context, record_decl: *const ZigClangRecordDecl) Error!?* ); } - try container_fields_and_decls.push(&field_node.base); + try fields_and_decls.append(&field_node.base); _ = try appendToken(c, .Comma, ","); } - container_node.rbrace_token = try appendToken(c, .RBrace, "}"); + const container_node = try ast.Node.ContainerDecl.alloc(c.arena, fields_and_decls.items.len); + container_node.* = .{ + .layout_token = layout_tok, + .kind_token = container_tok, + .init_arg_expr = .None, + .fields_and_decls_len = fields_and_decls.items.len, + .lbrace_token = lbrace_token, + .rbrace_token = try appendToken(c, .RBrace, "}"), + }; + mem.copy(*ast.Node, container_node.fieldsAndDecls(), fields_and_decls.items); semicolon = try appendToken(c, .Semicolon, ";"); break :blk &container_node.base; }; @@ -963,16 +959,8 @@ fn transEnumDecl(c: *Context, enum_decl: *const ZigClangEnumDecl) Error!?*ast.No const extern_tok = try appendToken(c, .Keyword_extern, "extern"); const container_tok = try appendToken(c, .Keyword_enum, "enum"); - const container_node = try c.arena.create(ast.Node.ContainerDecl); - container_node.* = .{ - .layout_token = extern_tok, - .kind_token = container_tok, - .init_arg_expr = .None, - .fields_and_decls = ast.Node.ContainerDecl.DeclList{}, - .lbrace_token = undefined, - .rbrace_token = undefined, - }; - var container_node_fields_and_decls = c.llpusher(&container_node.fields_and_decls); + var fields_and_decls = std.ArrayList(*ast.Node).init(c.gpa); + defer fields_and_decls.deinit(); const int_type = ZigClangEnumDecl_getIntegerType(enum_decl); // The underlying type may be null in case of forward-declared enum @@ -981,7 +969,7 @@ fn transEnumDecl(c: *Context, enum_decl: *const ZigClangEnumDecl) Error!?*ast.No // default to c_int since msvc and gcc default to different types _ = try appendToken(c, .LParen, "("); - container_node.init_arg_expr = .{ + const init_arg_expr = ast.Node.ContainerDecl.InitArg{ .Type = if (int_type.ptr != null and !isCBuiltinType(int_type, .UInt) and !isCBuiltinType(int_type, .Int)) @@ -997,7 +985,7 @@ fn transEnumDecl(c: *Context, enum_decl: *const ZigClangEnumDecl) Error!?*ast.No }; _ = try appendToken(c, .RParen, ")"); - container_node.lbrace_token = try appendToken(c, .LBrace, "{"); + const lbrace_token = try appendToken(c, .LBrace, "{"); it = ZigClangEnumDecl_enumerator_begin(enum_def); end_it = ZigClangEnumDecl_enumerator_end(enum_def); @@ -1029,7 +1017,7 @@ fn transEnumDecl(c: *Context, enum_decl: *const ZigClangEnumDecl) Error!?*ast.No .align_expr = null, }; - try container_node_fields_and_decls.push(&field_node.base); + try fields_and_decls.append(&field_node.base); _ = try appendToken(c, .Comma, ","); // In C each enum value is in the global namespace. So we put them there too. @@ -1064,10 +1052,18 @@ fn transEnumDecl(c: *Context, enum_decl: *const ZigClangEnumDecl) Error!?*ast.No .align_expr = null, }; - try container_node_fields_and_decls.push(&field_node.base); + try fields_and_decls.append(&field_node.base); _ = try appendToken(c, .Comma, ","); - container_node.rbrace_token = try appendToken(c, .RBrace, "}"); - + const container_node = try ast.Node.ContainerDecl.alloc(c.arena, fields_and_decls.items.len); + container_node.* = .{ + .layout_token = extern_tok, + .kind_token = container_tok, + .init_arg_expr = init_arg_expr, + .fields_and_decls_len = fields_and_decls.items.len, + .lbrace_token = lbrace_token, + .rbrace_token = try appendToken(c, .RBrace, "}"), + }; + mem.copy(*ast.Node, container_node.fieldsAndDecls(), fields_and_decls.items); break :blk &container_node.base; } else try transCreateNodeOpaqueType(c); @@ -3474,7 +3470,7 @@ fn maybeSuppressResult( } fn addTopLevelDecl(c: *Context, name: []const u8, decl_node: *ast.Node) !void { - c.root_decls_it = try c.llpush(*ast.Node, c.root_decls_it, decl_node); + try c.root_decls.append(c.gpa, decl_node); _ = try c.global_scope.sym_table.put(name, decl_node); } @@ -6119,9 +6115,7 @@ fn getContainer(c: *Context, node: *ast.Node) ?*ast.Node { return null; if (getContainerTypeOf(c, infix.lhs)) |ty_node| { if (ty_node.cast(ast.Node.ContainerDecl)) |container| { - var it = container.fields_and_decls.first; - while (it) |field_ref_node| : (it = field_ref_node.next) { - const field_ref = field_ref_node.data; + for (container.fieldsAndDecls()) |field_ref| { const field = field_ref.cast(ast.Node.ContainerField).?; const ident = infix.rhs.cast(ast.Node.Identifier).?; if (mem.eql(u8, tokenSlice(c, field.name_token), tokenSlice(c, ident.token))) { @@ -6147,9 +6141,7 @@ fn getContainerTypeOf(c: *Context, ref: *ast.Node) ?*ast.Node { return null; if (getContainerTypeOf(c, infix.lhs)) |ty_node| { if (ty_node.cast(ast.Node.ContainerDecl)) |container| { - var it = container.fields_and_decls.first; - while (it) |field_ref_node| : (it = field_ref_node.next) { - const field_ref = field_ref_node.data; + for (container.fieldsAndDecls()) |field_ref| { const field = field_ref.cast(ast.Node.ContainerField).?; const ident = infix.rhs.cast(ast.Node.Identifier).?; if (mem.eql(u8, tokenSlice(c, field.name_token), tokenSlice(c, ident.token))) { From 7c2c0e36f8378d8efaf64d859bdffb91007db82a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 20 May 2020 17:39:54 -0400 Subject: [PATCH 09/38] stage2 parser: different memory layout of ParamDecl Instead of being its own node, it's a struct inside FnProto. Instead of FnProto having a SinglyLinkedList of ParamDecl nodes, ParamDecls are appended directly in memory after the FnProto. throughput: 72.2 MiB/s => 72.9 MiB/s maxrss: 70 KB => 68 KB Importantly, the API is improved as well since the data is arranged linearly in memory. --- lib/std/zig/ast.zig | 151 ++++++++++++++++++++++++----------------- lib/std/zig/parse.zig | 56 ++++++++------- lib/std/zig/render.zig | 21 ++---- 3 files changed, 123 insertions(+), 105 deletions(-) diff --git a/lib/std/zig/ast.zig b/lib/std/zig/ast.zig index a1b2856718..b23f0705c0 100644 --- a/lib/std/zig/ast.zig +++ b/lib/std/zig/ast.zig @@ -477,7 +477,6 @@ pub const Node = struct { ErrorTag, AsmInput, AsmOutput, - ParamDecl, FieldInitializer, }; @@ -533,7 +532,6 @@ pub const Node = struct { switch (n.id) { .Root, .ContainerField, - .ParamDecl, .Block, .Payload, .PointerPayload, @@ -819,7 +817,7 @@ pub const Node = struct { return @ptrCast(*ContainerDecl, bytes.ptr); } - pub fn free(self: *Decl, allocator: *mem.Allocator) void { + pub fn free(self: *ContainerDecl, allocator: *mem.Allocator) void { const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.fields_and_decls_len)]; allocator.free(bytes); } @@ -979,13 +977,14 @@ pub const Node = struct { } }; + /// The params are directly after the FnProto in memory. pub const FnProto = struct { base: Node = Node{ .id = .FnProto }, doc_comments: ?*DocComment, visib_token: ?TokenIndex, fn_token: TokenIndex, name_token: ?TokenIndex, - params: ParamList, + params_len: NodeIndex, return_type: ReturnType, var_args_token: ?TokenIndex, extern_export_inline_token: ?TokenIndex, @@ -997,16 +996,75 @@ pub const Node = struct { is_extern_prototype: bool = false, // TODO: Remove once extern fn rewriting is is_async: bool = false, // TODO: remove once async fn rewriting is - pub const ParamList = LinkedList(*Node); - pub const ReturnType = union(enum) { Explicit: *Node, InferErrorSet: *Node, Invalid: TokenIndex, }; + pub const ParamDecl = struct { + doc_comments: ?*DocComment, + comptime_token: ?TokenIndex, + noalias_token: ?TokenIndex, + name_token: ?TokenIndex, + param_type: ParamType, + + pub const ParamType = union(enum) { + var_type: *Node, + var_args: TokenIndex, + type_expr: *Node, + }; + + pub fn iterate(self: *const ParamDecl) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const ParamDecl, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; + + if (i < 1) { + switch (self.param_type) { + .var_args => return null, + .var_type, .type_expr => |node| return node, + } + } + i -= 1; + + return null; + } + + pub fn firstToken(self: *const ParamDecl) TokenIndex { + if (self.comptime_token) |comptime_token| return comptime_token; + if (self.noalias_token) |noalias_token| return noalias_token; + if (self.name_token) |name_token| return name_token; + switch (self.param_type) { + .var_args => |tok| return tok, + .var_type, .type_expr => |node| return node.firstToken(), + } + } + + pub fn lastToken(self: *const ParamDecl) TokenIndex { + switch (self.param_type) { + .var_args => |tok| return tok, + .var_type, .type_expr => |node| return node.lastToken(), + } + } + }; + + /// After this the caller must initialize the params list. + pub fn alloc(allocator: *mem.Allocator, params_len: NodeIndex) !*FnProto { + const bytes = try allocator.alignedAlloc(u8, @alignOf(FnProto), sizeInBytes(params_len)); + return @ptrCast(*FnProto, bytes.ptr); + } + + pub fn free(self: *FnProto, allocator: *mem.Allocator) void { + const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.params_len)]; + allocator.free(bytes); + } + pub fn iterate(self: *const FnProto) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = self.params.first }; + return .{ .parent_node = &self.base, .index = 0, .node = null }; } pub fn iterateNext(self: *const FnProto, it: *Node.Iterator) ?*Node { @@ -1018,11 +1076,17 @@ pub const Node = struct { i -= 1; } - if (it.node) |param| { - it.index -= 1; - it.node = param.next; - return param.data; + if (i < self.params_len) { + switch (self.paramsConst()[i].param_type) { + .var_type => |n| return n, + .var_args => { + i += 1; + it.index += 1; + }, + .type_expr => |n| return n, + } } + i -= self.params_len; if (self.align_expr) |align_expr| { if (i < 1) return align_expr; @@ -1064,6 +1128,20 @@ pub const Node = struct { .Invalid => |tok| return tok, } } + + pub fn params(self: *FnProto) []ParamDecl { + const decls_start = @ptrCast([*]u8, self) + @sizeOf(FnProto); + return @ptrCast([*]ParamDecl, decls_start)[0..self.params_len]; + } + + pub fn paramsConst(self: *const FnProto) []const ParamDecl { + const decls_start = @ptrCast([*]const u8, self) + @sizeOf(FnProto); + return @ptrCast([*]const ParamDecl, decls_start)[0..self.params_len]; + } + + fn sizeInBytes(params_len: NodeIndex) usize { + return @sizeOf(FnProto) + @sizeOf(ParamDecl) * @as(usize, params_len); + } }; pub const AnyFrameType = struct { @@ -1102,57 +1180,6 @@ pub const Node = struct { } }; - pub const ParamDecl = struct { - base: Node = Node{ .id = .ParamDecl }, - doc_comments: ?*DocComment, - comptime_token: ?TokenIndex, - noalias_token: ?TokenIndex, - name_token: ?TokenIndex, - param_type: ParamType, - - pub const ParamType = union(enum) { - var_type: *Node, - var_args: TokenIndex, - type_expr: *Node, - }; - - pub fn iterate(self: *const ParamDecl) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; - } - - pub fn iterateNext(self: *const ParamDecl, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; - - if (i < 1) { - switch (self.param_type) { - .var_args => return null, - .var_type, .type_expr => |node| return node, - } - } - i -= 1; - - return null; - } - - pub fn firstToken(self: *const ParamDecl) TokenIndex { - if (self.comptime_token) |comptime_token| return comptime_token; - if (self.noalias_token) |noalias_token| return noalias_token; - if (self.name_token) |name_token| return name_token; - switch (self.param_type) { - .var_args => |tok| return tok, - .var_type, .type_expr => |node| return node.firstToken(), - } - } - - pub fn lastToken(self: *const ParamDecl) TokenIndex { - switch (self.param_type) { - .var_args => |tok| return tok, - .var_type, .type_expr => |node| return node.lastToken(), - } - } - }; - pub const Block = struct { base: Node = Node{ .id = .Block }, label: ?TokenIndex, diff --git a/lib/std/zig/parse.zig b/lib/std/zig/parse.zig index 3ae72cd021..8f171805ce 100644 --- a/lib/std/zig/parse.zig +++ b/lib/std/zig/parse.zig @@ -71,7 +71,7 @@ const Parser = struct { const eof_token = p.eatToken(.Eof).?; const node = try Node.Root.create(&p.arena.allocator, decls.len, eof_token); - std.mem.copy(*ast.Node, node.decls(), decls); + std.mem.copy(*Node, node.decls(), decls); return node; } @@ -96,8 +96,8 @@ const Parser = struct { /// / ContainerField COMMA ContainerMembers /// / ContainerField /// / - fn parseContainerMembers(p: *Parser, top_level: bool) ![]*ast.Node { - var list = std.ArrayList(*ast.Node).init(p.gpa); + fn parseContainerMembers(p: *Parser, top_level: bool) ![]*Node { + var list = std.ArrayList(*Node).init(p.gpa); defer list.deinit(); var field_state: union(enum) { @@ -522,6 +522,7 @@ const Parser = struct { const name_token = p.eatToken(.Identifier); const lparen = try p.expectToken(.LParen); const params = try p.parseParamDeclList(&var_args_token); + defer p.gpa.free(params); const rparen = try p.expectToken(.RParen); const align_expr = try p.parseByteAlign(); const section_expr = try p.parseLinkSection(); @@ -544,13 +545,13 @@ const Parser = struct { else R{ .Explicit = return_type_expr.? }; - const fn_proto_node = try p.arena.allocator.create(Node.FnProto); + const fn_proto_node = try Node.FnProto.alloc(&p.arena.allocator, params.len); fn_proto_node.* = .{ .doc_comments = null, .visib_token = null, .fn_token = fn_token, .name_token = name_token, - .params = params, + .params_len = params.len, .return_type = return_type, .var_args_token = var_args_token, .extern_export_inline_token = null, @@ -562,6 +563,7 @@ const Parser = struct { .is_extern_prototype = is_extern, .is_async = is_async, }; + std.mem.copy(Node.FnProto.ParamDecl, fn_proto_node.params(), params); return &fn_proto_node.base; } @@ -621,7 +623,7 @@ const Parser = struct { var type_expr: ?*Node = null; if (p.eatToken(.Colon)) |_| { if (p.eatToken(.Keyword_var)) |var_tok| { - const node = try p.arena.allocator.create(ast.Node.VarType); + const node = try p.arena.allocator.create(Node.VarType); node.* = .{ .token = var_tok }; type_expr = &node.base; } else { @@ -1936,7 +1938,7 @@ const Parser = struct { } /// ParamDecl <- (KEYWORD_noalias / KEYWORD_comptime)? (IDENTIFIER COLON)? ParamType - fn parseParamDecl(p: *Parser) !?*Node { + fn parseParamDecl(p: *Parser, list: *std.ArrayList(Node.FnProto.ParamDecl)) !bool { const doc_comments = try p.parseDocComment(); const noalias_token = p.eatToken(.Keyword_noalias); const comptime_token = if (noalias_token == null) p.eatToken(.Keyword_comptime) else null; @@ -1951,31 +1953,30 @@ const Parser = struct { if (noalias_token == null and comptime_token == null and name_token == null and - doc_comments == null) return null; + doc_comments == null) return false; try p.errors.append(p.gpa, .{ .ExpectedParamType = .{ .token = p.tok_i }, }); return error.ParseError; }; - const param_decl = try p.arena.allocator.create(Node.ParamDecl); - param_decl.* = .{ + (try list.addOne()).* = .{ .doc_comments = doc_comments, .comptime_token = comptime_token, .noalias_token = noalias_token, .name_token = name_token, .param_type = param_type, }; - return ¶m_decl.base; + return true; } /// ParamType /// <- KEYWORD_var /// / DOT3 /// / TypeExpr - fn parseParamType(p: *Parser) !?Node.ParamDecl.ParamType { + fn parseParamType(p: *Parser) !?Node.FnProto.ParamDecl.ParamType { // TODO cast from tuple to error union is broken - const P = Node.ParamDecl.ParamType; + const P = Node.FnProto.ParamDecl.ParamType; if (try p.parseVarType()) |node| return P{ .var_type = node }; if (p.eatToken(.Ellipsis3)) |token| return P{ .var_args = token }; if (try p.parseTypeExpr()) |node| return P{ .type_expr = node }; @@ -2587,7 +2588,7 @@ const Parser = struct { if (p.eatToken(.Ellipsis2) != null) { const end_expr = try p.parseExpr(); - const sentinel: ?*ast.Node = if (p.eatToken(.Colon) != null) + const sentinel: ?*Node = if (p.eatToken(.Colon) != null) try p.parseExpr() else null; @@ -2616,7 +2617,7 @@ const Parser = struct { if (p.eatToken(.Period)) |period| { if (try p.parseIdentifier()) |identifier| { // TODO: It's a bit weird to return an InfixOp from the SuffixOp parser. - // Should there be an ast.Node.SuffixOp.FieldAccess variant? Or should + // Should there be an Node.SuffixOp.FieldAccess variant? Or should // this grammar rule be altered? const node = try p.arena.allocator.create(Node.InfixOp); node.* = .{ @@ -2652,13 +2653,13 @@ const Parser = struct { /// ExprList <- (Expr COMMA)* Expr? fn parseFnCallArguments(p: *Parser) !?AnnotatedParamList { if (p.eatToken(.LParen) == null) return null; - const list = try ListParseFn(Node.FnProto.ParamList, parseExpr)(p); + const list = try ListParseFn(std.SinglyLinkedList(*Node), parseExpr)(p); const rparen = try p.expectToken(.RParen); return AnnotatedParamList{ .list = list, .rparen = rparen }; } const AnnotatedParamList = struct { - list: Node.FnProto.ParamList, // NOTE: may also be any other type SegmentedList(*Node, 2) + list: std.SinglyLinkedList(*Node), rparen: TokenIndex, }; @@ -2797,14 +2798,14 @@ const Parser = struct { .lbrace_token = lbrace, .rbrace_token = rbrace, }; - std.mem.copy(*ast.Node, node.fieldsAndDecls(), members); + std.mem.copy(*Node, node.fieldsAndDecls(), members); return &node.base; } /// Holds temporary data until we are ready to construct the full ContainerDecl AST node. const ContainerDeclType = struct { kind_token: TokenIndex, - init_arg_expr: ast.Node.ContainerDecl.InitArg, + init_arg_expr: Node.ContainerDecl.InitArg, }; /// ContainerDeclType @@ -2893,14 +2894,11 @@ const Parser = struct { } /// ParamDeclList <- (ParamDecl COMMA)* ParamDecl? - fn parseParamDeclList(p: *Parser, var_args_token: *?TokenIndex) !Node.FnProto.ParamList { - var list = Node.FnProto.ParamList{}; - var list_it = &list.first; - var last: ?*Node = null; - while (try p.parseParamDecl()) |node| { - last = node; - list_it = try p.llpush(*Node, list_it, node); + fn parseParamDeclList(p: *Parser, var_args_token: *?TokenIndex) ![]Node.FnProto.ParamDecl { + var list = std.ArrayList(Node.FnProto.ParamDecl).init(p.gpa); + defer list.deinit(); + while (try p.parseParamDecl(&list)) { switch (p.tokens[p.tok_i].id) { .Comma => _ = p.nextToken(), // all possible delimiters @@ -2914,13 +2912,13 @@ const Parser = struct { }, } } - if (last) |node| { - const param_type = node.cast(Node.ParamDecl).?.param_type; + if (list.items.len != 0) { + const param_type = list.items[list.items.len - 1].param_type; if (param_type == .var_args) { var_args_token.* = param_type.var_args; } } - return list; + return list.toOwnedSlice(); } const NodeParseFn = fn (p: *Parser) Error!?*Node; diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index cfae09dcd4..fbdfe51bc8 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -1479,13 +1479,11 @@ fn renderExpression( try renderToken(tree, stream, lparen, indent, start_col, Space.None); // ( // render all on one line, no trailing comma - var it = fn_proto.params.first; - while (it) |param_decl_node_node| : (it = param_decl_node_node.next) { - const param_decl_node = param_decl_node_node.data; - try renderParamDecl(allocator, stream, tree, indent, start_col, param_decl_node, Space.None); + for (fn_proto.params()) |param_decl, i| { + try renderParamDecl(allocator, stream, tree, indent, start_col, param_decl, Space.None); - if (param_decl_node_node.next != null) { - const comma = tree.nextToken(param_decl_node.lastToken()); + if (i + 1 < fn_proto.params_len) { + const comma = tree.nextToken(param_decl.lastToken()); try renderToken(tree, stream, comma, indent, start_col, Space.Space); // , } } @@ -1494,11 +1492,9 @@ fn renderExpression( const new_indent = indent + indent_delta; try renderToken(tree, stream, lparen, new_indent, start_col, Space.Newline); // ( - var it = fn_proto.params.first; - while (it) |param_decl_node_node| : (it = param_decl_node_node.next) { - const param_decl_node = param_decl_node_node.data; + for (fn_proto.params()) |param_decl| { try stream.writeByteNTimes(' ', new_indent); - try renderParamDecl(allocator, stream, tree, new_indent, start_col, param_decl_node, Space.Comma); + try renderParamDecl(allocator, stream, tree, new_indent, start_col, param_decl, Space.Comma); } try stream.writeByteNTimes(' ', indent); } @@ -2079,7 +2075,6 @@ fn renderExpression( .VarDecl, .Use, .TestDecl, - .ParamDecl, => unreachable, } } @@ -2162,11 +2157,9 @@ fn renderParamDecl( tree: *ast.Tree, indent: usize, start_col: *usize, - base: *ast.Node, + param_decl: ast.Node.FnProto.ParamDecl, space: Space, ) (@TypeOf(stream).Error || Error)!void { - const param_decl = @fieldParentPtr(ast.Node.ParamDecl, "base", base); - try renderDocComments(tree, stream, param_decl, indent, start_col); if (param_decl.comptime_token) |comptime_token| { From 897f23f20ff30e379e8d320cfd0013cb48dcc5b2 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 20 May 2020 19:18:14 -0400 Subject: [PATCH 10/38] stage2 parser: split off some SuffixOp AST nodes into separate tags These SuffixOp nodes have their own ast.Node tags now: * ArrayInitializer * ArrayInitializerDot * StructInitializer * StructInitializerDot Their sub-expression lists are general-purpose-allocator allocated and then copied into the arena after completion of parsing. throughput: 72.9 MiB/s => 74.4 MiB/s maxrss: 68 KB => 72 KB The API is also nicer since the sub expression lists are now flat arrays instead of singly linked lists. --- lib/std/zig/ast.zig | 171 +++++++++--- lib/std/zig/parse.zig | 130 ++++++--- lib/std/zig/render.zig | 578 +++++++++++++++++++++-------------------- 3 files changed, 524 insertions(+), 355 deletions(-) diff --git a/lib/std/zig/ast.zig b/lib/std/zig/ast.zig index b23f0705c0..e636710e08 100644 --- a/lib/std/zig/ast.zig +++ b/lib/std/zig/ast.zig @@ -429,6 +429,14 @@ pub const Node = struct { InfixOp, PrefixOp, SuffixOp, + /// This is a suffix operation but to save memory we have a dedicated Node id for it. + ArrayInitializer, + /// ArrayInitializer but with `.` instead of a left-hand-side operand. + ArrayInitializerDot, + /// This is a suffix operation but to save memory we have a dedicated Node id for it. + StructInitializer, + /// StructInitializer but with `.` instead of a left-hand-side operand. + StructInitializerDot, // Control flow Switch, @@ -1961,28 +1969,141 @@ pub const Node = struct { } }; + pub const ArrayInitializer = struct { + base: Node = Node{ .id = .ArrayInitializer }, + rtoken: TokenIndex, + lhs: *Node, + list: []*Node, + + pub fn iterate(self: *const ArrayInitializer) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const ArrayInitializer, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; + + if (i < 1) return self.lhs; + i -= 1; + + if (i < self.list.len) return self.list[i]; + i -= self.list.len; + + return null; + } + + pub fn firstToken(self: *const ArrayInitializer) TokenIndex { + return self.lhs.firstToken(); + } + + pub fn lastToken(self: *const ArrayInitializer) TokenIndex { + return self.rtoken; + } + }; + + pub const ArrayInitializerDot = struct { + base: Node = Node{ .id = .ArrayInitializerDot }, + dot: TokenIndex, + rtoken: TokenIndex, + list: []*Node, + + pub fn iterate(self: *const ArrayInitializerDot) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const ArrayInitializerDot, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; + + if (i < self.list.len) return self.list[i]; + i -= self.list.len; + + return null; + } + + pub fn firstToken(self: *const ArrayInitializerDot) TokenIndex { + return self.dot; + } + + pub fn lastToken(self: *const ArrayInitializerDot) TokenIndex { + return self.rtoken; + } + }; + + pub const StructInitializer = struct { + base: Node = Node{ .id = .StructInitializer }, + rtoken: TokenIndex, + lhs: *Node, + list: []*Node, + + pub fn iterate(self: *const StructInitializer) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const StructInitializer, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; + + if (i < 1) return self.lhs; + i -= 1; + + if (i < self.list.len) return self.list[i]; + i -= self.list.len; + + return null; + } + + pub fn firstToken(self: *const StructInitializer) TokenIndex { + return self.lhs.firstToken(); + } + + pub fn lastToken(self: *const StructInitializer) TokenIndex { + return self.rtoken; + } + }; + + pub const StructInitializerDot = struct { + base: Node = Node{ .id = .StructInitializerDot }, + dot: TokenIndex, + rtoken: TokenIndex, + list: []*Node, + + pub fn iterate(self: *const StructInitializerDot) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const StructInitializerDot, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; + + if (i < self.list.len) return self.list[i]; + i -= self.list.len; + + return null; + } + + pub fn firstToken(self: *const StructInitializerDot) TokenIndex { + return self.dot; + } + + pub fn lastToken(self: *const StructInitializerDot) TokenIndex { + return self.rtoken; + } + }; + pub const SuffixOp = struct { base: Node = Node{ .id = .SuffixOp }, - lhs: Lhs, op: Op, + lhs: *Node, rtoken: TokenIndex, - pub const Lhs = union(enum) { - node: *Node, - dot: TokenIndex, - }; - pub const Op = union(enum) { Call: Call, ArrayAccess: *Node, Slice: Slice, - ArrayInitializer: InitList, - StructInitializer: InitList, Deref, UnwrapOptional, - pub const InitList = LinkedList(*Node); - pub const Call = struct { params: ParamList, async_token: ?TokenIndex, @@ -2001,8 +2122,6 @@ pub const Node = struct { return .{ .parent_node = &self.base, .index = 0, .node = switch(self.op) { .Call => |call| call.params.first, - .ArrayInitializer => |ai| ai.first, - .StructInitializer => |si| si.first, else => null, }, }; @@ -2012,13 +2131,8 @@ pub const Node = struct { var i = it.index; it.index += 1; - switch (self.lhs) { - .node => |node| { - if (i == 0) return node; - i -= 1; - }, - .dot => {}, - } + if (i == 0) return self.lhs; + i -= 1; switch (self.op) { .Call => |call_info| { @@ -2045,20 +2159,6 @@ pub const Node = struct { i -= 1; } }, - .ArrayInitializer => |exprs| { - if (it.node) |child| { - it.index -= 1; - it.node = child.next; - return child.data; - } - }, - .StructInitializer => |fields| { - if (it.node) |child| { - it.index -= 1; - it.node = child.next; - return child.data; - } - }, .UnwrapOptional, .Deref, => {}, @@ -2072,10 +2172,7 @@ pub const Node = struct { .Call => |*call_info| if (call_info.async_token) |async_token| return async_token, else => {}, } - switch (self.lhs) { - .node => |node| return node.firstToken(), - .dot => |dot| return dot, - } + return self.lhs.firstToken(); } pub fn lastToken(self: *const SuffixOp) TokenIndex { diff --git a/lib/std/zig/parse.zig b/lib/std/zig/parse.zig index 8f171805ce..98224413b2 100644 --- a/lib/std/zig/parse.zig +++ b/lib/std/zig/parse.zig @@ -1287,50 +1287,105 @@ const Parser = struct { /// CurlySuffixExpr <- TypeExpr InitList? fn parseCurlySuffixExpr(p: *Parser) !?*Node { - const type_expr = (try p.parseTypeExpr()) orelse return null; - const suffix_op = (try p.parseInitList()) orelse return type_expr; - suffix_op.lhs.node = type_expr; - return &suffix_op.base; + const lhs = (try p.parseTypeExpr()) orelse return null; + const suffix_op = (try p.parseInitList(lhs)) orelse return lhs; + return suffix_op; } /// InitList /// <- LBRACE FieldInit (COMMA FieldInit)* COMMA? RBRACE /// / LBRACE Expr (COMMA Expr)* COMMA? RBRACE /// / LBRACE RBRACE - fn parseInitList(p: *Parser) !?*Node.SuffixOp { + fn parseInitList(p: *Parser, lhs: *Node) !?*Node { const lbrace = p.eatToken(.LBrace) orelse return null; - var init_list = Node.SuffixOp.Op.InitList{}; - var init_list_it = &init_list.first; + var init_list = std.ArrayList(*Node).init(p.gpa); + defer init_list.deinit(); - const op: Node.SuffixOp.Op = blk: { - if (try p.parseFieldInit()) |field_init| { - init_list_it = try p.llpush(*Node, init_list_it, field_init); - while (p.eatToken(.Comma)) |_| { - const next = (try p.parseFieldInit()) orelse break; - init_list_it = try p.llpush(*Node, init_list_it, next); - } - break :blk .{ .StructInitializer = init_list }; + if (try p.parseFieldInit()) |field_init| { + try init_list.append(field_init); + while (p.eatToken(.Comma)) |_| { + const next = (try p.parseFieldInit()) orelse break; + try init_list.append(next); } + const node = try p.arena.allocator.create(Node.StructInitializer); + node.* = .{ + .lhs = lhs, + .rtoken = try p.expectToken(.RBrace), + .list = try p.arena.allocator.dupe(*Node, init_list.items), + }; + return &node.base; + } - if (try p.parseExpr()) |expr| { - init_list_it = try p.llpush(*Node, init_list_it, expr); - while (p.eatToken(.Comma)) |_| { - const next = (try p.parseExpr()) orelse break; - init_list_it = try p.llpush(*Node, init_list_it, next); - } - break :blk .{ .ArrayInitializer = init_list }; + if (try p.parseExpr()) |expr| { + try init_list.append(expr); + while (p.eatToken(.Comma)) |_| { + const next = (try p.parseExpr()) orelse break; + try init_list.append(next); } + const node = try p.arena.allocator.create(Node.ArrayInitializer); + node.* = .{ + .lhs = lhs, + .rtoken = try p.expectToken(.RBrace), + .list = try p.arena.allocator.dupe(*Node, init_list.items), + }; + return &node.base; + } - break :blk .{ .StructInitializer = init_list }; - }; - - const node = try p.arena.allocator.create(Node.SuffixOp); + const node = try p.arena.allocator.create(Node.StructInitializer); node.* = .{ - .lhs = .{ .node = undefined }, // set by caller - .op = op, + .lhs = lhs, .rtoken = try p.expectToken(.RBrace), + .list = &[0]*Node{}, }; - return node; + return &node.base; + } + + /// InitList + /// <- LBRACE FieldInit (COMMA FieldInit)* COMMA? RBRACE + /// / LBRACE Expr (COMMA Expr)* COMMA? RBRACE + /// / LBRACE RBRACE + fn parseAnonInitList(p: *Parser, dot: TokenIndex) !?*Node { + const lbrace = p.eatToken(.LBrace) orelse return null; + var init_list = std.ArrayList(*Node).init(p.gpa); + defer init_list.deinit(); + + if (try p.parseFieldInit()) |field_init| { + try init_list.append(field_init); + while (p.eatToken(.Comma)) |_| { + const next = (try p.parseFieldInit()) orelse break; + try init_list.append(next); + } + const node = try p.arena.allocator.create(Node.StructInitializerDot); + node.* = .{ + .dot = dot, + .rtoken = try p.expectToken(.RBrace), + .list = try p.arena.allocator.dupe(*Node, init_list.items), + }; + return &node.base; + } + + if (try p.parseExpr()) |expr| { + try init_list.append(expr); + while (p.eatToken(.Comma)) |_| { + const next = (try p.parseExpr()) orelse break; + try init_list.append(next); + } + const node = try p.arena.allocator.create(Node.ArrayInitializerDot); + node.* = .{ + .dot = dot, + .rtoken = try p.expectToken(.RBrace), + .list = try p.arena.allocator.dupe(*Node, init_list.items), + }; + return &node.base; + } + + const node = try p.arena.allocator.create(Node.StructInitializerDot); + node.* = .{ + .dot = dot, + .rtoken = try p.expectToken(.RBrace), + .list = &[0]*Node{}, + }; + return &node.base; } /// TypeExpr <- PrefixTypeOp* ErrorUnionExpr @@ -1378,7 +1433,7 @@ const Parser = struct { while (try p.parseSuffixOp()) |node| { switch (node.id) { - .SuffixOp => node.cast(Node.SuffixOp).?.lhs = .{ .node = res }, + .SuffixOp => node.cast(Node.SuffixOp).?.lhs = res, .InfixOp => node.cast(Node.InfixOp).?.lhs = res, else => unreachable, } @@ -1394,7 +1449,7 @@ const Parser = struct { }; const node = try p.arena.allocator.create(Node.SuffixOp); node.* = .{ - .lhs = .{ .node = res }, + .lhs = res, .op = .{ .Call = .{ .params = params.list, @@ -1411,7 +1466,7 @@ const Parser = struct { while (true) { if (try p.parseSuffixOp()) |node| { switch (node.id) { - .SuffixOp => node.cast(Node.SuffixOp).?.lhs = .{ .node = res }, + .SuffixOp => node.cast(Node.SuffixOp).?.lhs = res, .InfixOp => node.cast(Node.InfixOp).?.lhs = res, else => unreachable, } @@ -1421,7 +1476,7 @@ const Parser = struct { if (try p.parseFnCallArguments()) |params| { const call = try p.arena.allocator.create(Node.SuffixOp); call.* = .{ - .lhs = .{ .node = res }, + .lhs = res, .op = .{ .Call = .{ .params = params.list, @@ -1757,10 +1812,8 @@ const Parser = struct { return &node.base; } - // anon container literal - if (try p.parseInitList()) |node| { - node.lhs = .{ .dot = dot }; - return &node.base; + if (try p.parseAnonInitList(dot)) |node| { + return node; } p.putBackToken(dot); @@ -3282,8 +3335,7 @@ const Parser = struct { } fn expectToken(p: *Parser, id: Token.Id) Error!TokenIndex { - return (try p.expectTokenRecoverable(id)) orelse - error.ParseError; + return (try p.expectTokenRecoverable(id)) orelse error.ParseError; } fn expectTokenRecoverable(p: *Parser, id: Token.Id) !?TokenIndex { diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index fbdfe51bc8..150f6f6016 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -611,6 +611,298 @@ fn renderExpression( return renderExpression(allocator, stream, tree, indent, start_col, prefix_op_node.rhs, space); }, + .ArrayInitializer, .ArrayInitializerDot => { + var rtoken: ast.TokenIndex = undefined; + var exprs: []*ast.Node = undefined; + const lhs: union(enum) {dot: ast.TokenIndex, node: *ast.Node } = switch (base.id){ + .ArrayInitializerDot => blk: { + const casted = @fieldParentPtr(ast.Node.ArrayInitializerDot, "base", base); + rtoken = casted.rtoken; + exprs = casted.list; + break :blk .{ .dot = casted.dot }; + }, + .ArrayInitializer => blk: { + const casted = @fieldParentPtr(ast.Node.ArrayInitializer, "base", base); + rtoken = casted.rtoken; + exprs = casted.list; + break :blk .{ .node = casted.lhs }; + }, + else => unreachable, + }; + + const lbrace = switch (lhs) { + .dot => |dot| tree.nextToken(dot), + .node => |node| tree.nextToken(node.lastToken()), + }; + + if (exprs.len == 0) { + switch (lhs) { + .dot => |dot| try renderToken(tree, stream, dot, indent, start_col, Space.None), + .node => |node| try renderExpression(allocator, stream, tree, indent, start_col, node, Space.None), + } + try renderToken(tree, stream, lbrace, indent, start_col, Space.None); + return renderToken(tree, stream, rtoken, indent, start_col, space); + } + + if (exprs.len == 1 and tree.tokens[exprs[0].lastToken() + 1].id == .RBrace) { + const expr = exprs[0]; + switch (lhs) { + .dot => |dot| try renderToken(tree, stream, dot, indent, start_col, Space.None), + .node => |node| try renderExpression(allocator, stream, tree, indent, start_col, node, Space.None), + } + try renderToken(tree, stream, lbrace, indent, start_col, Space.None); + try renderExpression(allocator, stream, tree, indent, start_col, expr, Space.None); + return renderToken(tree, stream, rtoken, indent, start_col, space); + } + + switch (lhs) { + .dot => |dot| try renderToken(tree, stream, dot, indent, start_col, Space.None), + .node => |node| try renderExpression(allocator, stream, tree, indent, start_col, node, Space.None), + } + + // scan to find row size + const maybe_row_size: ?usize = blk: { + var count: usize = 1; + for (exprs) |expr, i| { + if (i + 1 < exprs.len) { + const expr_last_token = expr.lastToken() + 1; + const loc = tree.tokenLocation(tree.tokens[expr_last_token].end, exprs[i+1].firstToken()); + if (loc.line != 0) break :blk count; + count += 1; + } else { + const expr_last_token = expr.lastToken(); + const loc = tree.tokenLocation(tree.tokens[expr_last_token].end, rtoken); + if (loc.line == 0) { + // all on one line + const src_has_trailing_comma = trailblk: { + const maybe_comma = tree.prevToken(rtoken); + break :trailblk tree.tokens[maybe_comma].id == .Comma; + }; + if (src_has_trailing_comma) { + break :blk 1; // force row size 1 + } else { + break :blk null; // no newlines + } + } + break :blk count; + } + } + unreachable; + }; + + if (maybe_row_size) |row_size| { + // A place to store the width of each expression and its column's maximum + var widths = try allocator.alloc(usize, exprs.len + row_size); + defer allocator.free(widths); + mem.set(usize, widths, 0); + + var expr_widths = widths[0 .. widths.len - row_size]; + var column_widths = widths[widths.len - row_size ..]; + + // Null stream for counting the printed length of each expression + var counting_stream = std.io.countingOutStream(std.io.null_out_stream); + + for (exprs) |expr, i| { + counting_stream.bytes_written = 0; + var dummy_col: usize = 0; + try renderExpression(allocator, counting_stream.outStream(), tree, indent, &dummy_col, expr, Space.None); + const width = @intCast(usize, counting_stream.bytes_written); + const col = i % row_size; + column_widths[col] = std.math.max(column_widths[col], width); + expr_widths[i] = width; + } + + var new_indent = indent + indent_delta; + + if (tree.tokens[tree.nextToken(lbrace)].id != .MultilineStringLiteralLine) { + try renderToken(tree, stream, lbrace, new_indent, start_col, Space.Newline); + try stream.writeByteNTimes(' ', new_indent); + } else { + new_indent -= indent_delta; + try renderToken(tree, stream, lbrace, new_indent, start_col, Space.None); + } + + var col: usize = 1; + for (exprs) |expr, i| { + if (i + 1 < exprs.len) { + const next_expr = exprs[i + 1]; + try renderExpression(allocator, stream, tree, new_indent, start_col, expr, Space.None); + + const comma = tree.nextToken(expr.lastToken()); + + if (col != row_size) { + try renderToken(tree, stream, comma, new_indent, start_col, Space.Space); // , + + const padding = column_widths[i % row_size] - expr_widths[i]; + try stream.writeByteNTimes(' ', padding); + + col += 1; + continue; + } + col = 1; + + if (tree.tokens[tree.nextToken(comma)].id != .MultilineStringLiteralLine) { + try renderToken(tree, stream, comma, new_indent, start_col, Space.Newline); // , + } else { + try renderToken(tree, stream, comma, new_indent, start_col, Space.None); // , + } + + try renderExtraNewline(tree, stream, start_col, next_expr); + if (next_expr.id != .MultilineStringLiteral) { + try stream.writeByteNTimes(' ', new_indent); + } + } else { + try renderExpression(allocator, stream, tree, new_indent, start_col, expr, Space.Comma); // , + } + } + if (exprs[exprs.len - 1].id != .MultilineStringLiteral) { + try stream.writeByteNTimes(' ', indent); + } + return renderToken(tree, stream, rtoken, indent, start_col, space); + } else { + try renderToken(tree, stream, lbrace, indent, start_col, Space.Space); + for (exprs) |expr, i| { + if (i + 1 < exprs.len) { + try renderExpression(allocator, stream, tree, indent, start_col, expr, Space.None); + const comma = tree.nextToken(expr.lastToken()); + try renderToken(tree, stream, comma, indent, start_col, Space.Space); // , + } else { + try renderExpression(allocator, stream, tree, indent, start_col, expr, Space.Space); + } + } + + return renderToken(tree, stream, rtoken, indent, start_col, space); + } + }, + + .StructInitializer, .StructInitializerDot => { + var rtoken: ast.TokenIndex = undefined; + var field_inits: []*ast.Node = undefined; + const lhs: union(enum) {dot: ast.TokenIndex, node: *ast.Node } = switch (base.id){ + .StructInitializerDot => blk: { + const casted = @fieldParentPtr(ast.Node.StructInitializerDot, "base", base); + rtoken = casted.rtoken; + field_inits = casted.list; + break :blk .{ .dot = casted.dot }; + }, + .StructInitializer => blk: { + const casted = @fieldParentPtr(ast.Node.StructInitializer, "base", base); + rtoken = casted.rtoken; + field_inits = casted.list; + break :blk .{ .node = casted.lhs }; + }, + else => unreachable, + }; + + const lbrace = switch (lhs) { + .dot => |dot| tree.nextToken(dot), + .node => |node| tree.nextToken(node.lastToken()), + }; + + if (field_inits.len == 0) { + switch (lhs) { + .dot => |dot| try renderToken(tree, stream, dot, indent, start_col, Space.None), + .node => |node| try renderExpression(allocator, stream, tree, indent, start_col, node, Space.None), + } + try renderToken(tree, stream, lbrace, indent + indent_delta, start_col, Space.None); + return renderToken(tree, stream, rtoken, indent, start_col, space); + } + + const src_has_trailing_comma = blk: { + const maybe_comma = tree.prevToken(rtoken); + break :blk tree.tokens[maybe_comma].id == .Comma; + }; + + const src_same_line = blk: { + const loc = tree.tokenLocation(tree.tokens[lbrace].end, rtoken); + break :blk loc.line == 0; + }; + + const expr_outputs_one_line = blk: { + // render field expressions until a LF is found + for (field_inits) |field_init| { + var find_stream = FindByteOutStream.init('\n'); + var dummy_col: usize = 0; + try renderExpression(allocator, find_stream.outStream(), tree, 0, &dummy_col, field_init, Space.None); + if (find_stream.byte_found) break :blk false; + } + break :blk true; + }; + + if (field_inits.len == 1) blk: { + const field_init = field_inits[0].cast(ast.Node.FieldInitializer).?; + + switch (field_init.expr.id) { + .StructInitializer, + .StructInitializerDot, + => break :blk, + + else => {}, + } + + // if the expression outputs to multiline, make this struct multiline + if (!expr_outputs_one_line or src_has_trailing_comma) { + break :blk; + } + + switch (lhs) { + .dot => |dot| try renderToken(tree, stream, dot, indent, start_col, Space.None), + .node => |node| try renderExpression(allocator, stream, tree, indent, start_col, node, Space.None), + } + try renderToken(tree, stream, lbrace, indent, start_col, Space.Space); + try renderExpression(allocator, stream, tree, indent, start_col, &field_init.base, Space.Space); + return renderToken(tree, stream, rtoken, indent, start_col, space); + } + + if (!src_has_trailing_comma and src_same_line and expr_outputs_one_line) { + // render all on one line, no trailing comma + switch (lhs) { + .dot => |dot| try renderToken(tree, stream, dot, indent, start_col, Space.None), + .node => |node| try renderExpression(allocator, stream, tree, indent, start_col, node, Space.None), + } + try renderToken(tree, stream, lbrace, indent, start_col, Space.Space); + + for (field_inits) |field_init, i| { + if (i + 1 < field_inits.len) { + try renderExpression(allocator, stream, tree, indent, start_col, field_init, Space.None); + + const comma = tree.nextToken(field_init.lastToken()); + try renderToken(tree, stream, comma, indent, start_col, Space.Space); + } else { + try renderExpression(allocator, stream, tree, indent, start_col, field_init, Space.Space); + } + } + + return renderToken(tree, stream, rtoken, indent, start_col, space); + } + + const new_indent = indent + indent_delta; + + switch (lhs) { + .dot => |dot| try renderToken(tree, stream, dot, new_indent, start_col, Space.None), + .node => |node| try renderExpression(allocator, stream, tree, new_indent, start_col, node, Space.None), + } + try renderToken(tree, stream, lbrace, new_indent, start_col, Space.Newline); + + for (field_inits) |field_init, i| { + try stream.writeByteNTimes(' ', new_indent); + + if (i + 1 < field_inits.len) { + try renderExpression(allocator, stream, tree, new_indent, start_col, field_init, Space.None); + + const comma = tree.nextToken(field_init.lastToken()); + try renderToken(tree, stream, comma, new_indent, start_col, Space.Newline); + + try renderExtraNewline(tree, stream, start_col, field_inits[i + 1]); + } else { + try renderExpression(allocator, stream, tree, new_indent, start_col, field_init, Space.Comma); + } + } + + try stream.writeByteNTimes(' ', indent); + return renderToken(tree, stream, rtoken, indent, start_col, space); + }, + .SuffixOp => { const suffix_op = @fieldParentPtr(ast.Node.SuffixOp, "base", base); @@ -620,9 +912,9 @@ fn renderExpression( try renderToken(tree, stream, async_token, indent, start_col, Space.Space); } - try renderExpression(allocator, stream, tree, indent, start_col, suffix_op.lhs.node, Space.None); + try renderExpression(allocator, stream, tree, indent, start_col, suffix_op.lhs, Space.None); - const lparen = tree.nextToken(suffix_op.lhs.node.lastToken()); + const lparen = tree.nextToken(suffix_op.lhs.lastToken()); if (call_info.params.first == null) { try renderToken(tree, stream, lparen, indent, start_col, Space.None); @@ -680,10 +972,10 @@ fn renderExpression( }, .ArrayAccess => |index_expr| { - const lbracket = tree.nextToken(suffix_op.lhs.node.lastToken()); + const lbracket = tree.nextToken(suffix_op.lhs.lastToken()); const rbracket = tree.nextToken(index_expr.lastToken()); - try renderExpression(allocator, stream, tree, indent, start_col, suffix_op.lhs.node, Space.None); + try renderExpression(allocator, stream, tree, indent, start_col, suffix_op.lhs, Space.None); try renderToken(tree, stream, lbracket, indent, start_col, Space.None); // [ const starts_with_comment = tree.tokens[lbracket + 1].id == .LineComment; @@ -701,18 +993,18 @@ fn renderExpression( }, .Deref => { - try renderExpression(allocator, stream, tree, indent, start_col, suffix_op.lhs.node, Space.None); + try renderExpression(allocator, stream, tree, indent, start_col, suffix_op.lhs, Space.None); return renderToken(tree, stream, suffix_op.rtoken, indent, start_col, space); // .* }, .UnwrapOptional => { - try renderExpression(allocator, stream, tree, indent, start_col, suffix_op.lhs.node, Space.None); + try renderExpression(allocator, stream, tree, indent, start_col, suffix_op.lhs, Space.None); try renderToken(tree, stream, tree.prevToken(suffix_op.rtoken), indent, start_col, Space.None); // . return renderToken(tree, stream, suffix_op.rtoken, indent, start_col, space); // ? }, .Slice => |range| { - try renderExpression(allocator, stream, tree, indent, start_col, suffix_op.lhs.node, Space.None); + try renderExpression(allocator, stream, tree, indent, start_col, suffix_op.lhs, Space.None); const lbracket = tree.prevToken(range.start.firstToken()); const dotdot = tree.nextToken(range.start.lastToken()); @@ -737,278 +1029,6 @@ fn renderExpression( return renderToken(tree, stream, suffix_op.rtoken, indent, start_col, space); // ] }, - .StructInitializer => |*field_inits| { - const lbrace = switch (suffix_op.lhs) { - .dot => |dot| tree.nextToken(dot), - .node => |node| tree.nextToken(node.lastToken()), - }; - - if (field_inits.first == null) { - switch (suffix_op.lhs) { - .dot => |dot| try renderToken(tree, stream, dot, indent, start_col, Space.None), - .node => |node| try renderExpression(allocator, stream, tree, indent, start_col, node, Space.None), - } - try renderToken(tree, stream, lbrace, indent + indent_delta, start_col, Space.None); - return renderToken(tree, stream, suffix_op.rtoken, indent, start_col, space); - } - - const src_has_trailing_comma = blk: { - const maybe_comma = tree.prevToken(suffix_op.rtoken); - break :blk tree.tokens[maybe_comma].id == .Comma; - }; - - const src_same_line = blk: { - const loc = tree.tokenLocation(tree.tokens[lbrace].end, suffix_op.rtoken); - break :blk loc.line == 0; - }; - - const expr_outputs_one_line = blk: { - // render field expressions until a LF is found - var it = field_inits.first; - while (it) |field_init_node| : (it = field_init_node.next) { - const field_init = field_init_node.data; - var find_stream = FindByteOutStream.init('\n'); - var dummy_col: usize = 0; - try renderExpression(allocator, find_stream.outStream(), tree, 0, &dummy_col, field_init, Space.None); - if (find_stream.byte_found) break :blk false; - } - break :blk true; - }; - - if (field_inits.first != null and field_inits.first.?.next == null) blk: { - const field_init = field_inits.first.?.data.cast(ast.Node.FieldInitializer).?; - - if (field_init.expr.cast(ast.Node.SuffixOp)) |nested_suffix_op| { - if (nested_suffix_op.op == .StructInitializer) { - break :blk; - } - } - - // if the expression outputs to multiline, make this struct multiline - if (!expr_outputs_one_line or src_has_trailing_comma) { - break :blk; - } - - switch (suffix_op.lhs) { - .dot => |dot| try renderToken(tree, stream, dot, indent, start_col, Space.None), - .node => |node| try renderExpression(allocator, stream, tree, indent, start_col, node, Space.None), - } - try renderToken(tree, stream, lbrace, indent, start_col, Space.Space); - try renderExpression(allocator, stream, tree, indent, start_col, &field_init.base, Space.Space); - return renderToken(tree, stream, suffix_op.rtoken, indent, start_col, space); - } - - if (!src_has_trailing_comma and src_same_line and expr_outputs_one_line) { - // render all on one line, no trailing comma - switch (suffix_op.lhs) { - .dot => |dot| try renderToken(tree, stream, dot, indent, start_col, Space.None), - .node => |node| try renderExpression(allocator, stream, tree, indent, start_col, node, Space.None), - } - try renderToken(tree, stream, lbrace, indent, start_col, Space.Space); - - var it = field_inits.first; - while (it) |field_init_node| : (it = field_init_node.next) { - const field_init = field_init_node.data; - if (field_init_node.next != null) { - try renderExpression(allocator, stream, tree, indent, start_col, field_init, Space.None); - - const comma = tree.nextToken(field_init.lastToken()); - try renderToken(tree, stream, comma, indent, start_col, Space.Space); - } else { - try renderExpression(allocator, stream, tree, indent, start_col, field_init, Space.Space); - } - } - - return renderToken(tree, stream, suffix_op.rtoken, indent, start_col, space); - } - - const new_indent = indent + indent_delta; - - switch (suffix_op.lhs) { - .dot => |dot| try renderToken(tree, stream, dot, new_indent, start_col, Space.None), - .node => |node| try renderExpression(allocator, stream, tree, new_indent, start_col, node, Space.None), - } - try renderToken(tree, stream, lbrace, new_indent, start_col, Space.Newline); - - var it = field_inits.first; - while (it) |field_init_node| : (it = field_init_node.next) { - const field_init = field_init_node.data; - try stream.writeByteNTimes(' ', new_indent); - - if (field_init_node.next) |next_field_init| { - try renderExpression(allocator, stream, tree, new_indent, start_col, field_init, Space.None); - - const comma = tree.nextToken(field_init.lastToken()); - try renderToken(tree, stream, comma, new_indent, start_col, Space.Newline); - - try renderExtraNewline(tree, stream, start_col, next_field_init.data); - } else { - try renderExpression(allocator, stream, tree, new_indent, start_col, field_init, Space.Comma); - } - } - - try stream.writeByteNTimes(' ', indent); - return renderToken(tree, stream, suffix_op.rtoken, indent, start_col, space); - }, - - .ArrayInitializer => |*exprs| { - const lbrace = switch (suffix_op.lhs) { - .dot => |dot| tree.nextToken(dot), - .node => |node| tree.nextToken(node.lastToken()), - }; - - if (exprs.first == null) { - switch (suffix_op.lhs) { - .dot => |dot| try renderToken(tree, stream, dot, indent, start_col, Space.None), - .node => |node| try renderExpression(allocator, stream, tree, indent, start_col, node, Space.None), - } - try renderToken(tree, stream, lbrace, indent, start_col, Space.None); - return renderToken(tree, stream, suffix_op.rtoken, indent, start_col, space); - } - if (exprs.first) |first_expr_node| { - const expr = first_expr_node.data; - if (first_expr_node.next == null and tree.tokens[expr.lastToken() + 1].id == .RBrace) { - switch (suffix_op.lhs) { - .dot => |dot| try renderToken(tree, stream, dot, indent, start_col, Space.None), - .node => |node| try renderExpression(allocator, stream, tree, indent, start_col, node, Space.None), - } - try renderToken(tree, stream, lbrace, indent, start_col, Space.None); - try renderExpression(allocator, stream, tree, indent, start_col, expr, Space.None); - return renderToken(tree, stream, suffix_op.rtoken, indent, start_col, space); - } - } - - switch (suffix_op.lhs) { - .dot => |dot| try renderToken(tree, stream, dot, indent, start_col, Space.None), - .node => |node| try renderExpression(allocator, stream, tree, indent, start_col, node, Space.None), - } - - // scan to find row size - const maybe_row_size: ?usize = blk: { - var count: usize = 1; - var it = exprs.first; - while (true) { - const expr_node = it.?; - it = expr_node.next; - const expr = expr_node.data; - if (expr_node.next) |next_expr| { - const expr_last_token = expr.lastToken() + 1; - const loc = tree.tokenLocation(tree.tokens[expr_last_token].end, next_expr.data.firstToken()); - if (loc.line != 0) break :blk count; - count += 1; - } else { - const expr_last_token = expr.lastToken(); - const loc = tree.tokenLocation(tree.tokens[expr_last_token].end, suffix_op.rtoken); - if (loc.line == 0) { - // all on one line - const src_has_trailing_comma = trailblk: { - const maybe_comma = tree.prevToken(suffix_op.rtoken); - break :trailblk tree.tokens[maybe_comma].id == .Comma; - }; - if (src_has_trailing_comma) { - break :blk 1; // force row size 1 - } else { - break :blk null; // no newlines - } - } - break :blk count; - } - } - }; - - if (maybe_row_size) |row_size| { - // A place to store the width of each expression and its column's maximum - var widths = try allocator.alloc(usize, exprs.len() + row_size); - defer allocator.free(widths); - mem.set(usize, widths, 0); - - var expr_widths = widths[0 .. widths.len - row_size]; - var column_widths = widths[widths.len - row_size ..]; - - // Null stream for counting the printed length of each expression - var counting_stream = std.io.countingOutStream(std.io.null_out_stream); - - var it = exprs.first; - var i: usize = 0; - - while (it) |expr_node| : ({i += 1; it = expr_node.next;}) { - const expr = expr_node.data; - counting_stream.bytes_written = 0; - var dummy_col: usize = 0; - try renderExpression(allocator, counting_stream.outStream(), tree, indent, &dummy_col, expr, Space.None); - const width = @intCast(usize, counting_stream.bytes_written); - const col = i % row_size; - column_widths[col] = std.math.max(column_widths[col], width); - expr_widths[i] = width; - } - - var new_indent = indent + indent_delta; - - if (tree.tokens[tree.nextToken(lbrace)].id != .MultilineStringLiteralLine) { - try renderToken(tree, stream, lbrace, new_indent, start_col, Space.Newline); - try stream.writeByteNTimes(' ', new_indent); - } else { - new_indent -= indent_delta; - try renderToken(tree, stream, lbrace, new_indent, start_col, Space.None); - } - - it = exprs.first; - i = 0; - var col: usize = 1; - var last_node = it.?; - while (it) |expr_node| : ({i += 1; it = expr_node.next; last_node = expr_node;}) { - const expr = expr_node.data; - if (expr_node.next) |next_expr| { - try renderExpression(allocator, stream, tree, new_indent, start_col, expr, Space.None); - - const comma = tree.nextToken(expr.lastToken()); - - if (col != row_size) { - try renderToken(tree, stream, comma, new_indent, start_col, Space.Space); // , - - const padding = column_widths[i % row_size] - expr_widths[i]; - try stream.writeByteNTimes(' ', padding); - - col += 1; - continue; - } - col = 1; - - if (tree.tokens[tree.nextToken(comma)].id != .MultilineStringLiteralLine) { - try renderToken(tree, stream, comma, new_indent, start_col, Space.Newline); // , - } else { - try renderToken(tree, stream, comma, new_indent, start_col, Space.None); // , - } - - try renderExtraNewline(tree, stream, start_col, next_expr.data); - if (next_expr.data.id != .MultilineStringLiteral) { - try stream.writeByteNTimes(' ', new_indent); - } - } else { - try renderExpression(allocator, stream, tree, new_indent, start_col, expr, Space.Comma); // , - } - } - if (last_node.data.id != .MultilineStringLiteral) { - try stream.writeByteNTimes(' ', indent); - } - return renderToken(tree, stream, suffix_op.rtoken, indent, start_col, space); - } else { - try renderToken(tree, stream, lbrace, indent, start_col, Space.Space); - var it = exprs.first; - while (it) |expr_node| : (it = expr_node.next) { - const expr = expr_node.data; - if (expr_node.next) |next_expr| { - try renderExpression(allocator, stream, tree, indent, start_col, expr, Space.None); - const comma = tree.nextToken(expr.lastToken()); - try renderToken(tree, stream, comma, indent, start_col, Space.Space); // , - } else { - try renderExpression(allocator, stream, tree, indent, start_col, expr, Space.Space); - } - } - - return renderToken(tree, stream, suffix_op.rtoken, indent, start_col, space); - } - }, } }, From 44aeb383289bc80644e3c6aef2b7b2494149c80b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 20 May 2020 21:27:08 -0400 Subject: [PATCH 11/38] make rand function public, fixes crypto benchmark --- lib/std/rand.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/rand.zig b/lib/std/rand.zig index 45a1172b3d..1e5bb37592 100644 --- a/lib/std/rand.zig +++ b/lib/std/rand.zig @@ -658,7 +658,7 @@ pub const Xoroshiro128 = struct { self.s[1] = s1; } - fn seed(self: *Xoroshiro128, init_s: u64) void { + pub fn seed(self: *Xoroshiro128, init_s: u64) void { // Xoroshiro requires 128-bits of seed. var gen = SplitMix64.init(init_s); From 84df1d4f3d0312553f5a3857ed67042319c20846 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 20 May 2020 22:41:18 -0400 Subject: [PATCH 12/38] stage2 parser: elide memcpy of large initialization lists throughput: 71.4 MiB/s => 72.9 MiB/s --- lib/std/zig/ast.zig | 6 +++++ lib/std/zig/parse.zig | 54 +++++++++++++++++++++++++++++++++++++++---- 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/lib/std/zig/ast.zig b/lib/std/zig/ast.zig index e636710e08..e80f772c43 100644 --- a/lib/std/zig/ast.zig +++ b/lib/std/zig/ast.zig @@ -18,6 +18,8 @@ pub const Tree = struct { arena: std.heap.ArenaAllocator.State, gpa: *mem.Allocator, + /// This keeps track of slices of memory that must be freed on deinit. + owned_memory: [][]u8, /// translate-c uses this to avoid having to emit correct newlines /// TODO get rid of this hack @@ -26,6 +28,10 @@ pub const Tree = struct { pub fn deinit(self: *Tree) void { self.gpa.free(self.tokens); self.gpa.free(self.errors); + for (self.owned_memory) |list| { + self.gpa.free(list); + } + self.gpa.free(self.owned_memory); self.arena.promote(self.gpa).deinit(); } diff --git a/lib/std/zig/parse.zig b/lib/std/zig/parse.zig index 98224413b2..b5561425fd 100644 --- a/lib/std/zig/parse.zig +++ b/lib/std/zig/parse.zig @@ -10,6 +10,13 @@ const Token = std.zig.Token; pub const Error = error{ParseError} || Allocator.Error; +/// This is the maximum length of a list that will be copied into the ast.Tree +/// arena when parsing. If the list is longer than this, the ast.Tree will have +/// a reference to the memory allocated in the general purpose allocator, and +/// will free it separately. Simply put, lists longer than this will elide the +/// memcpy(). +const large_list_len = 512; + /// Result should be freed with tree.deinit() when there are /// no more references to any of the tokens or nodes. pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!*Tree { @@ -32,6 +39,11 @@ pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!*Tree { .tokens = tokens.items, .errors = .{}, .tok_i = 0, + .owned_memory = .{}, + }; + defer parser.owned_memory.deinit(gpa); + errdefer for (parser.owned_memory.items) |list| { + gpa.free(list); }; defer parser.errors.deinit(gpa); errdefer parser.arena.deinit(); @@ -46,6 +58,7 @@ pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!*Tree { .source = source, .tokens = tokens.toOwnedSlice(), .errors = parser.errors.toOwnedSlice(gpa), + .owned_memory = parser.owned_memory.toOwnedSlice(gpa), .root_node = root_node, .arena = parser.arena.state, }; @@ -60,6 +73,7 @@ const Parser = struct { tokens: []const Token, tok_i: TokenIndex, errors: std.ArrayListUnmanaged(AstError), + owned_memory: std.ArrayListUnmanaged([]u8), /// Root <- skip ContainerMembers eof fn parseRoot(p: *Parser) Allocator.Error!*Node.Root { @@ -1307,11 +1321,19 @@ const Parser = struct { const next = (try p.parseFieldInit()) orelse break; try init_list.append(next); } + + const list = if (init_list.items.len > large_list_len) blk: { + try p.owned_memory.ensureCapacity(p.gpa, p.owned_memory.items.len + 1); + const list = init_list.toOwnedSlice(); + p.owned_memory.appendAssumeCapacity(std.mem.sliceAsBytes(list)); + break :blk list; + } else try p.arena.allocator.dupe(*Node, init_list.items); + const node = try p.arena.allocator.create(Node.StructInitializer); node.* = .{ .lhs = lhs, .rtoken = try p.expectToken(.RBrace), - .list = try p.arena.allocator.dupe(*Node, init_list.items), + .list = list, }; return &node.base; } @@ -1322,11 +1344,19 @@ const Parser = struct { const next = (try p.parseExpr()) orelse break; try init_list.append(next); } + + const list = if (init_list.items.len > large_list_len) blk: { + try p.owned_memory.ensureCapacity(p.gpa, p.owned_memory.items.len + 1); + const list = init_list.toOwnedSlice(); + p.owned_memory.appendAssumeCapacity(std.mem.sliceAsBytes(list)); + break :blk list; + } else try p.arena.allocator.dupe(*Node, init_list.items); + const node = try p.arena.allocator.create(Node.ArrayInitializer); node.* = .{ .lhs = lhs, .rtoken = try p.expectToken(.RBrace), - .list = try p.arena.allocator.dupe(*Node, init_list.items), + .list = list, }; return &node.base; } @@ -1355,11 +1385,19 @@ const Parser = struct { const next = (try p.parseFieldInit()) orelse break; try init_list.append(next); } + + const list = if (init_list.items.len > large_list_len) blk: { + try p.owned_memory.ensureCapacity(p.gpa, p.owned_memory.items.len + 1); + const list = init_list.toOwnedSlice(); + p.owned_memory.appendAssumeCapacity(std.mem.sliceAsBytes(list)); + break :blk list; + } else try p.arena.allocator.dupe(*Node, init_list.items); + const node = try p.arena.allocator.create(Node.StructInitializerDot); node.* = .{ .dot = dot, .rtoken = try p.expectToken(.RBrace), - .list = try p.arena.allocator.dupe(*Node, init_list.items), + .list = list, }; return &node.base; } @@ -1370,11 +1408,19 @@ const Parser = struct { const next = (try p.parseExpr()) orelse break; try init_list.append(next); } + + const list = if (init_list.items.len > large_list_len) blk: { + try p.owned_memory.ensureCapacity(p.gpa, p.owned_memory.items.len + 1); + const list = init_list.toOwnedSlice(); + p.owned_memory.appendAssumeCapacity(std.mem.sliceAsBytes(list)); + break :blk list; + } else try p.arena.allocator.dupe(*Node, init_list.items); + const node = try p.arena.allocator.create(Node.ArrayInitializerDot); node.* = .{ .dot = dot, .rtoken = try p.expectToken(.RBrace), - .list = try p.arena.allocator.dupe(*Node, init_list.items), + .list = list, }; return &node.base; } From 688aa114e434e05b96f916c168f177aa0484baec Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 20 May 2020 22:42:43 -0400 Subject: [PATCH 13/38] Revert "stage2 parser: elide memcpy of large initialization lists" This reverts commit 84df1d4f3d0312553f5a3857ed67042319c20846. Not worth the complexity! Always memcpy initialization lists into the arena. --- lib/std/zig/ast.zig | 6 ----- lib/std/zig/parse.zig | 54 ++++--------------------------------------- 2 files changed, 4 insertions(+), 56 deletions(-) diff --git a/lib/std/zig/ast.zig b/lib/std/zig/ast.zig index e80f772c43..e636710e08 100644 --- a/lib/std/zig/ast.zig +++ b/lib/std/zig/ast.zig @@ -18,8 +18,6 @@ pub const Tree = struct { arena: std.heap.ArenaAllocator.State, gpa: *mem.Allocator, - /// This keeps track of slices of memory that must be freed on deinit. - owned_memory: [][]u8, /// translate-c uses this to avoid having to emit correct newlines /// TODO get rid of this hack @@ -28,10 +26,6 @@ pub const Tree = struct { pub fn deinit(self: *Tree) void { self.gpa.free(self.tokens); self.gpa.free(self.errors); - for (self.owned_memory) |list| { - self.gpa.free(list); - } - self.gpa.free(self.owned_memory); self.arena.promote(self.gpa).deinit(); } diff --git a/lib/std/zig/parse.zig b/lib/std/zig/parse.zig index b5561425fd..98224413b2 100644 --- a/lib/std/zig/parse.zig +++ b/lib/std/zig/parse.zig @@ -10,13 +10,6 @@ const Token = std.zig.Token; pub const Error = error{ParseError} || Allocator.Error; -/// This is the maximum length of a list that will be copied into the ast.Tree -/// arena when parsing. If the list is longer than this, the ast.Tree will have -/// a reference to the memory allocated in the general purpose allocator, and -/// will free it separately. Simply put, lists longer than this will elide the -/// memcpy(). -const large_list_len = 512; - /// Result should be freed with tree.deinit() when there are /// no more references to any of the tokens or nodes. pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!*Tree { @@ -39,11 +32,6 @@ pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!*Tree { .tokens = tokens.items, .errors = .{}, .tok_i = 0, - .owned_memory = .{}, - }; - defer parser.owned_memory.deinit(gpa); - errdefer for (parser.owned_memory.items) |list| { - gpa.free(list); }; defer parser.errors.deinit(gpa); errdefer parser.arena.deinit(); @@ -58,7 +46,6 @@ pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!*Tree { .source = source, .tokens = tokens.toOwnedSlice(), .errors = parser.errors.toOwnedSlice(gpa), - .owned_memory = parser.owned_memory.toOwnedSlice(gpa), .root_node = root_node, .arena = parser.arena.state, }; @@ -73,7 +60,6 @@ const Parser = struct { tokens: []const Token, tok_i: TokenIndex, errors: std.ArrayListUnmanaged(AstError), - owned_memory: std.ArrayListUnmanaged([]u8), /// Root <- skip ContainerMembers eof fn parseRoot(p: *Parser) Allocator.Error!*Node.Root { @@ -1321,19 +1307,11 @@ const Parser = struct { const next = (try p.parseFieldInit()) orelse break; try init_list.append(next); } - - const list = if (init_list.items.len > large_list_len) blk: { - try p.owned_memory.ensureCapacity(p.gpa, p.owned_memory.items.len + 1); - const list = init_list.toOwnedSlice(); - p.owned_memory.appendAssumeCapacity(std.mem.sliceAsBytes(list)); - break :blk list; - } else try p.arena.allocator.dupe(*Node, init_list.items); - const node = try p.arena.allocator.create(Node.StructInitializer); node.* = .{ .lhs = lhs, .rtoken = try p.expectToken(.RBrace), - .list = list, + .list = try p.arena.allocator.dupe(*Node, init_list.items), }; return &node.base; } @@ -1344,19 +1322,11 @@ const Parser = struct { const next = (try p.parseExpr()) orelse break; try init_list.append(next); } - - const list = if (init_list.items.len > large_list_len) blk: { - try p.owned_memory.ensureCapacity(p.gpa, p.owned_memory.items.len + 1); - const list = init_list.toOwnedSlice(); - p.owned_memory.appendAssumeCapacity(std.mem.sliceAsBytes(list)); - break :blk list; - } else try p.arena.allocator.dupe(*Node, init_list.items); - const node = try p.arena.allocator.create(Node.ArrayInitializer); node.* = .{ .lhs = lhs, .rtoken = try p.expectToken(.RBrace), - .list = list, + .list = try p.arena.allocator.dupe(*Node, init_list.items), }; return &node.base; } @@ -1385,19 +1355,11 @@ const Parser = struct { const next = (try p.parseFieldInit()) orelse break; try init_list.append(next); } - - const list = if (init_list.items.len > large_list_len) blk: { - try p.owned_memory.ensureCapacity(p.gpa, p.owned_memory.items.len + 1); - const list = init_list.toOwnedSlice(); - p.owned_memory.appendAssumeCapacity(std.mem.sliceAsBytes(list)); - break :blk list; - } else try p.arena.allocator.dupe(*Node, init_list.items); - const node = try p.arena.allocator.create(Node.StructInitializerDot); node.* = .{ .dot = dot, .rtoken = try p.expectToken(.RBrace), - .list = list, + .list = try p.arena.allocator.dupe(*Node, init_list.items), }; return &node.base; } @@ -1408,19 +1370,11 @@ const Parser = struct { const next = (try p.parseExpr()) orelse break; try init_list.append(next); } - - const list = if (init_list.items.len > large_list_len) blk: { - try p.owned_memory.ensureCapacity(p.gpa, p.owned_memory.items.len + 1); - const list = init_list.toOwnedSlice(); - p.owned_memory.appendAssumeCapacity(std.mem.sliceAsBytes(list)); - break :blk list; - } else try p.arena.allocator.dupe(*Node, init_list.items); - const node = try p.arena.allocator.create(Node.ArrayInitializerDot); node.* = .{ .dot = dot, .rtoken = try p.expectToken(.RBrace), - .list = list, + .list = try p.arena.allocator.dupe(*Node, init_list.items), }; return &node.base; } From 140dc2f43e347ddddc2f6f344b388bdc74c97c56 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 20 May 2020 23:13:02 -0400 Subject: [PATCH 14/38] stage1: fix false positive redeclared variable compile error --- src/analyze.cpp | 21 ++++++++++++++++++--- src/ir.cpp | 15 ++++++++++++--- 2 files changed, 30 insertions(+), 6 deletions(-) diff --git a/src/analyze.cpp b/src/analyze.cpp index 5a2629a0ea..0fddc568a5 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -3612,6 +3612,12 @@ static void add_top_level_decl(CodeGen *g, ScopeDecls *decls_scope, Tld *tld) { auto entry = decls_scope->decl_table.put_unique(tld->name, tld); if (entry) { Tld *other_tld = entry->value; + if (other_tld->id == TldIdVar) { + ZigVar *var = reinterpret_cast(other_tld)->var; + if (var->var_type != nullptr && type_is_invalid(var->var_type)) { + return; // already reported compile error + } + } ErrorMsg *msg = add_node_error(g, tld->source_node, buf_sprintf("redefinition of '%s'", buf_ptr(tld->name))); add_error_note(g, msg, other_tld->source_node, buf_sprintf("previous definition is here")); return; @@ -3887,9 +3893,18 @@ ZigVar *add_variable(CodeGen *g, AstNode *source_node, Scope *parent_scope, Buf if (search_scope != nullptr) { Tld *tld = find_decl(g, search_scope, name); if (tld != nullptr && tld != src_tld) { - ErrorMsg *msg = add_node_error(g, source_node, - buf_sprintf("redefinition of '%s'", buf_ptr(name))); - add_error_note(g, msg, tld->source_node, buf_sprintf("previous definition is here")); + bool want_err_msg = true; + if (tld->id == TldIdVar) { + ZigVar *var = reinterpret_cast(tld)->var; + if (var->var_type != nullptr && type_is_invalid(var->var_type)) { + want_err_msg = false; + } + } + if (want_err_msg) { + ErrorMsg *msg = add_node_error(g, source_node, + buf_sprintf("redefinition of '%s'", buf_ptr(name))); + add_error_note(g, msg, tld->source_node, buf_sprintf("previous definition is here")); + } variable_entry->var_type = g->builtin_types.entry_invalid; } } diff --git a/src/ir.cpp b/src/ir.cpp index 20e83ecb9d..cb0b6223c7 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -5300,9 +5300,18 @@ static ZigVar *create_local_var(CodeGen *codegen, AstNode *node, Scope *parent_s } else { Tld *tld = find_decl(codegen, parent_scope, name); if (tld != nullptr) { - ErrorMsg *msg = add_node_error(codegen, node, - buf_sprintf("redefinition of '%s'", buf_ptr(name))); - add_error_note(codegen, msg, tld->source_node, buf_sprintf("previous definition is here")); + bool want_err_msg = true; + if (tld->id == TldIdVar) { + ZigVar *var = reinterpret_cast(tld)->var; + if (var->var_type != nullptr && type_is_invalid(var->var_type)) { + want_err_msg = false; + } + } + if (want_err_msg) { + ErrorMsg *msg = add_node_error(codegen, node, + buf_sprintf("redefinition of '%s'", buf_ptr(name))); + add_error_note(codegen, msg, tld->source_node, buf_sprintf("previous definition is here")); + } variable_entry->var_type = codegen->builtin_types.entry_invalid; } } From 3c5d581ce35b137a7b80ac1431c1d9132e281fef Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 20 May 2020 23:28:31 -0400 Subject: [PATCH 15/38] update translate-c to the new fn params AST node API --- src-self-hosted/translate_c.zig | 76 ++++++++++++++------------------- 1 file changed, 32 insertions(+), 44 deletions(-) diff --git a/src-self-hosted/translate_c.zig b/src-self-hosted/translate_c.zig index b55197c91f..a93973eb36 100644 --- a/src-self-hosted/translate_c.zig +++ b/src-self-hosted/translate_c.zig @@ -534,17 +534,13 @@ fn visitFnDecl(c: *Context, fn_decl: *const ZigClangFunctionDecl) Error!void { const block_node = try transCreateNodeBlock(rp.c, null); block_scope.setBlockNode(block_node); - var it = proto_node.params.first; var param_id: c_uint = 0; - var prev_node_link = &proto_node.params_first; - while (it) |p_node| : ({prev_node_link = &p_node.next; it = p_node.next;}) { - const p = p_node.data; - const param = @fieldParentPtr(ast.Node.ParamDecl, "base", p); + for (proto_node.params()) |param, i| { const param_name = if (param.name_token) |name_tok| tokenSlice(c, name_tok) else if (param.param_type == .var_args) { - assert(p_node.next == null); - prev_node_link.* = null; + assert(i + 1 == proto_node.params_len); + proto_node.params_len -= 1; break; } else return failDecl(c, fn_decl_loc, fn_name, "function {} parameter has no name", .{fn_name}); @@ -3843,7 +3839,7 @@ fn transCreateNodeFnCall(c: *Context, fn_expr: *ast.Node) !*ast.Node.SuffixOp { _ = try appendToken(c, .LParen, "("); const node = try c.arena.create(ast.Node.SuffixOp); node.* = .{ - .lhs = .{ .node = fn_expr }, + .lhs = fn_expr, .op = .{ .Call = .{ .params = ast.Node.SuffixOp.Op.Call.ParamList{}, @@ -4129,41 +4125,36 @@ fn transCreateNodeMacroFn(c: *Context, name: []const u8, ref: *ast.Node, proto_a const name_tok = try appendIdentifier(c, name); _ = try appendToken(c, .LParen, "("); - var fn_params = ast.Node.FnProto.ParamList{}; - var fn_params_list = &fn_params.first; - var it = proto_alias.params.first; - while (it) |pn_node| : (it = pn_node.next) { - const pn = pn_node.data; - if (pn_node != proto_alias.params.first.?) { + var fn_params = std.ArrayList(ast.Node.FnProto.ParamDecl).init(c.gpa); + defer fn_params.deinit(); + + for (proto_alias.params()) |param, i| { + if (i != 0) { _ = try appendToken(c, .Comma, ","); } - const param = pn.cast(ast.Node.ParamDecl).?; - const param_name_tok = param.name_token orelse try appendTokenFmt(c, .Identifier, "arg_{}", .{c.getMangle()}); _ = try appendToken(c, .Colon, ":"); - const param_node = try c.arena.create(ast.Node.ParamDecl); - param_node.* = .{ + (try fn_params.addOne()).* = .{ .doc_comments = null, .comptime_token = null, .noalias_token = param.noalias_token, .name_token = param_name_tok, .param_type = param.param_type, }; - fn_params_list = try c.llpush(*ast.Node, fn_params_list, ¶m_node.base); } _ = try appendToken(c, .RParen, ")"); - const fn_proto = try c.arena.create(ast.Node.FnProto); + const fn_proto = try ast.Node.FnProto.alloc(c.arena, fn_params.items.len); fn_proto.* = .{ .doc_comments = null, .visib_token = pub_tok, .fn_token = fn_tok, .name_token = name_tok, - .params = fn_params, + .params_len = fn_params.items.len, .return_type = proto_alias.return_type, .var_args_token = null, .extern_export_inline_token = inline_tok, @@ -4173,6 +4164,7 @@ fn transCreateNodeMacroFn(c: *Context, name: []const u8, ref: *ast.Node, proto_a .section_expr = null, .callconv_expr = null, }; + mem.copy(ast.Node.FnProto.ParamDecl, fn_proto.params(), fn_params.items); const block = try transCreateNodeBlock(c, null); var block_statements_it = &block.statements.first; @@ -4182,13 +4174,10 @@ fn transCreateNodeMacroFn(c: *Context, name: []const u8, ref: *ast.Node, proto_a const call_expr = try transCreateNodeFnCall(c, unwrap_expr); var call_params_it = &call_expr.op.Call.params.first; - it = fn_params.first; - while (it) |pn_node| : (it = pn_node.next) { - const pn = pn_node.data; - if (fn_params.first.? != pn_node) { + for (fn_params.items) |param, i| { + if (i != 0) { _ = try appendToken(c, .Comma, ","); } - const param = pn.cast(ast.Node.ParamDecl).?; call_params_it = try c.llpush(*ast.Node, call_params_it, try transCreateNodeIdentifier(c, tokenSlice(c, param.name_token.?)),); } call_expr.rtoken = try appendToken(c, .RParen, ")"); @@ -4207,7 +4196,7 @@ fn transCreateNodeUnwrapNull(c: *Context, wrapped: *ast.Node) !*ast.Node { const node = try c.arena.create(ast.Node.SuffixOp); node.* = .{ .op = .UnwrapOptional, - .lhs = .{ .node = wrapped }, + .lhs = wrapped, .rtoken = qm, }; return &node.base; @@ -4697,9 +4686,10 @@ fn finishTransFnProto( const name_tok = if (fn_decl_context) |ctx| try appendIdentifier(rp.c, ctx.fn_name) else null; const lparen_tok = try appendToken(rp.c, .LParen, "("); - var fn_params = ast.Node.FnProto.ParamList{}; - var fn_params_list = rp.c.llpusher(&fn_params); + var fn_params = std.ArrayList(ast.Node.FnProto.ParamDecl).init(rp.c.gpa); + defer fn_params.deinit(); const param_count: usize = if (fn_proto_ty != null) ZigClangFunctionProtoType_getNumParams(fn_proto_ty.?) else 0; + try fn_params.ensureCapacity(param_count + 1); // +1 for possible var args node var i: usize = 0; while (i < param_count) : (i += 1) { @@ -4723,15 +4713,13 @@ fn finishTransFnProto( const type_node = try transQualType(rp, param_qt, source_loc); - const param_node = try rp.c.arena.create(ast.Node.ParamDecl); - param_node.* = .{ + fn_params.addOneAssumeCapacity().* = .{ .doc_comments = null, .comptime_token = null, .noalias_token = noalias_tok, .name_token = param_name_tok, .param_type = .{ .type_expr = type_node }, }; - try fn_params_list.push(¶m_node.base); if (i + 1 < param_count) { _ = try appendToken(rp.c, .Comma, ","); @@ -4743,15 +4731,13 @@ fn finishTransFnProto( _ = try appendToken(rp.c, .Comma, ","); } - const var_arg_node = try rp.c.arena.create(ast.Node.ParamDecl); - var_arg_node.* = .{ + fn_params.addOneAssumeCapacity().* = .{ .doc_comments = null, .comptime_token = null, .noalias_token = null, .name_token = null, .param_type = .{ .var_args = try appendToken(rp.c, .Ellipsis3, "...") } }; - try fn_params_list.push(&var_arg_node.base); } const rparen_tok = try appendToken(rp.c, .RParen, ")"); @@ -4818,13 +4804,13 @@ fn finishTransFnProto( } }; - const fn_proto = try rp.c.arena.create(ast.Node.FnProto); + const fn_proto = try ast.Node.FnProto.alloc(rp.c.arena, fn_params.items.len); fn_proto.* = .{ .doc_comments = null, .visib_token = pub_tok, .fn_token = fn_tok, .name_token = name_tok, - .params = fn_params, + .params_len = fn_params.items.len, .return_type = .{ .Explicit = return_type_node }, .var_args_token = null, // TODO this field is broken in the AST data model .extern_export_inline_token = extern_export_inline_tok, @@ -4834,6 +4820,7 @@ fn finishTransFnProto( .section_expr = linksection_expr, .callconv_expr = callconv_expr, }; + mem.copy(ast.Node.FnProto.ParamDecl, fn_proto.params(), fn_params.items); return fn_proto; } @@ -5130,8 +5117,10 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, .{}, ); } - var fn_params = ast.Node.FnProto.ParamList{}; - var fn_params_it = &fn_params.first; + + var fn_params = std.ArrayList(ast.Node.FnProto.ParamDecl).init(c.gpa); + defer fn_params.deinit(); + while (true) { const param_tok = it.next().?; if (param_tok.id != .Identifier) { @@ -5154,15 +5143,13 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, .token = token_index, }; - const param_node = try c.arena.create(ast.Node.ParamDecl); - param_node.* = .{ + (try fn_params.addOne()).* = .{ .doc_comments = null, .comptime_token = null, .noalias_token = null, .name_token = param_name_tok, .param_type = .{ .type_expr = &identifier.base }, }; - fn_params_it = try c.llpush(*ast.Node, fn_params_it, ¶m_node.base); if (it.peek().?.id != .Comma) break; @@ -5186,13 +5173,13 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, type_of.rparen_token = try appendToken(c, .RParen, ")"); var type_of_params = c.llpusher(&type_of.params); - const fn_proto = try c.arena.create(ast.Node.FnProto); + const fn_proto = try ast.Node.FnProto.alloc(c.arena, fn_params.items.len); fn_proto.* = .{ .visib_token = pub_tok, .extern_export_inline_token = inline_tok, .fn_token = fn_tok, .name_token = name_tok, - .params = fn_params, + .params_len = fn_params.items.len, .return_type = .{ .Explicit = &type_of.base }, .doc_comments = null, .var_args_token = null, @@ -5202,6 +5189,7 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, .section_expr = null, .callconv_expr = null, }; + mem.copy(ast.Node.FnProto.ParamDecl, fn_proto.params(), fn_params.items); const block = try transCreateNodeBlock(c, null); var block_statements = c.llpusher(&block.statements); From d57d9448aa322c1818de110adeb3cc69ac5dbcd9 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 20 May 2020 23:47:04 -0400 Subject: [PATCH 16/38] stage2 parsing: rework block statements AST memory layout block statements are now directly following the Block AST node rather than a singly linked list. This had negligible impact on performance: throughput: 72.3 MiB/s => 72.7 MiB/s however it greatly improves the API since the statements are laid out in a flat array in memory. --- lib/std/zig/ast.zig | 43 +++++++++++++++++++++++++++++++++++------- lib/std/zig/parse.zig | 12 +++++++----- lib/std/zig/render.zig | 11 +++++------ 3 files changed, 48 insertions(+), 18 deletions(-) diff --git a/lib/std/zig/ast.zig b/lib/std/zig/ast.zig index e636710e08..70c8b3509c 100644 --- a/lib/std/zig/ast.zig +++ b/lib/std/zig/ast.zig @@ -804,6 +804,7 @@ pub const Node = struct { } }; + /// The fields and decls Node pointers directly follow this struct in memory. pub const ContainerDecl = struct { base: Node = Node{ .id = .ContainerDecl }, kind_token: TokenIndex, @@ -1188,23 +1189,37 @@ pub const Node = struct { } }; + /// The statements of the block follow Block directly in memory. pub const Block = struct { base: Node = Node{ .id = .Block }, - label: ?TokenIndex, + statements_len: NodeIndex, lbrace: TokenIndex, - statements: StatementList, rbrace: TokenIndex, + label: ?TokenIndex, - pub const StatementList = LinkedList(*Node); + /// After this the caller must initialize the statements list. + pub fn alloc(allocator: *mem.Allocator, statements_len: NodeIndex) !*Block { + const bytes = try allocator.alignedAlloc(u8, @alignOf(Block), sizeInBytes(statements_len)); + return @ptrCast(*Block, bytes.ptr); + } + + pub fn free(self: *Block, allocator: *mem.Allocator) void { + const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.statements_len)]; + allocator.free(bytes); + } pub fn iterate(self: *const Block) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = self.statements.first }; + return .{ .parent_node = &self.base, .index = 0, .node = null }; } pub fn iterateNext(self: *const Block, it: *Node.Iterator) ?*Node { - const child = it.node orelse return null; - it.node = child.next; - return child.data; + var i = it.index; + it.index += 1; + + if (i < self.statements_len) return self.statementsConst()[i]; + i -= self.statements_len; + + return null; } pub fn firstToken(self: *const Block) TokenIndex { @@ -1218,6 +1233,20 @@ pub const Node = struct { pub fn lastToken(self: *const Block) TokenIndex { return self.rbrace; } + + pub fn statements(self: *Block) []*Node { + const decls_start = @ptrCast([*]u8, self) + @sizeOf(Block); + return @ptrCast([*]*Node, decls_start)[0..self.statements_len]; + } + + pub fn statementsConst(self: *const Block) []const *Node { + const decls_start = @ptrCast([*]const u8, self) + @sizeOf(Block); + return @ptrCast([*]const *Node, decls_start)[0..self.statements_len]; + } + + fn sizeInBytes(statements_len: NodeIndex) usize { + return @sizeOf(Block) + @sizeOf(*Node) * @as(usize, statements_len); + } }; pub const Defer = struct { diff --git a/lib/std/zig/parse.zig b/lib/std/zig/parse.zig index 98224413b2..ab41f2df00 100644 --- a/lib/std/zig/parse.zig +++ b/lib/std/zig/parse.zig @@ -1178,8 +1178,9 @@ const Parser = struct { fn parseBlock(p: *Parser) !?*Node { const lbrace = p.eatToken(.LBrace) orelse return null; - var statements = Node.Block.StatementList{}; - var statements_it = &statements.first; + var statements = std.ArrayList(*Node).init(p.gpa); + defer statements.deinit(); + while (true) { const statement = (p.parseStatement() catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, @@ -1189,18 +1190,19 @@ const Parser = struct { continue; }, }) orelse break; - statements_it = try p.llpush(*Node, statements_it, statement); + try statements.append(statement); } const rbrace = try p.expectToken(.RBrace); - const block_node = try p.arena.allocator.create(Node.Block); + const block_node = try Node.Block.alloc(&p.arena.allocator, statements.items.len); block_node.* = .{ .label = null, .lbrace = lbrace, - .statements = statements, + .statements_len = statements.items.len, .rbrace = rbrace, }; + std.mem.copy(*Node, block_node.statements(), statements.items); return &block_node.base; } diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index 150f6f6016..0d5a840ef1 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -358,21 +358,20 @@ fn renderExpression( try renderToken(tree, stream, tree.nextToken(label), indent, start_col, Space.Space); } - if (block.statements.first == null) { + if (block.statements_len == 0) { try renderToken(tree, stream, block.lbrace, indent + indent_delta, start_col, Space.None); return renderToken(tree, stream, block.rbrace, indent, start_col, space); } else { const block_indent = indent + indent_delta; try renderToken(tree, stream, block.lbrace, block_indent, start_col, Space.Newline); - var it = block.statements.first; - while (it) |statement_node| : (it = statement_node.next) { - const statement = statement_node.data; + const block_statements = block.statements(); + for (block_statements) |statement, i| { try stream.writeByteNTimes(' ', block_indent); try renderStatement(allocator, stream, tree, block_indent, start_col, statement); - if (statement_node.next) |next_statement| { - try renderExtraNewline(tree, stream, start_col, next_statement.data); + if (i + 1 < block_statements.len) { + try renderExtraNewline(tree, stream, start_col, block_statements[i + 1]); } } From e694cd265ad762e1db037e191edfa2f0601126a9 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 21 May 2020 00:14:55 -0400 Subject: [PATCH 17/38] more progress on updating translate-c --- src-self-hosted/translate_c.zig | 34 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/src-self-hosted/translate_c.zig b/src-self-hosted/translate_c.zig index a93973eb36..e0c2b44f42 100644 --- a/src-self-hosted/translate_c.zig +++ b/src-self-hosted/translate_c.zig @@ -2054,7 +2054,8 @@ fn transInitListExprRecord( const ty_node = try transType(rp, ty, loc); const init_count = ZigClangInitListExpr_getNumInits(expr); - var init_node = try transCreateNodeStructInitializer(rp.c, ty_node); + var field_inits = std.ArrayList(*ast.Node).init(rp.c.gpa); + defer field_inits.deinit(); var init_i: c_uint = 0; var it = ZigClangRecordDecl_field_begin(record_def); @@ -2091,13 +2092,19 @@ fn transInitListExprRecord( .expr = try transExpr(rp, scope, elem_expr, .used, .r_value), }; - try init_node.op.StructInitializer.push(&field_init_node.base); + try field_inits.append(&field_init_node.base); _ = try appendToken(rp.c, .Comma, ","); } - init_node.rtoken = try appendToken(rp.c, .RBrace, "}"); + _ = try appendToken(c, .LBrace, "{"); - return &init_node.base; + const node = try rp.c.arena.create(ast.Node.StructInitializer); + node.* = .{ + .lhs = ty, + .rtoken = try appendToken(rp.c, .RBrace, "}"), + .list = try rp.c.arena.dupe(*ast.Node, field_inits.items), + }; + return &node.base; } fn transCreateNodeArrayType( @@ -4069,7 +4076,7 @@ fn transCreateNodeArrayInitializer(c: *Context, ty: *ast.Node) !*ast.Node.Suffix _ = try appendToken(c, .LBrace, "{"); const node = try c.arena.create(ast.Node.SuffixOp); node.* = .{ - .lhs = .{ .node = ty }, + .lhs = ty, .op = .{ .ArrayInitializer = ast.Node.SuffixOp.Op.InitList{}, }, @@ -4078,19 +4085,6 @@ fn transCreateNodeArrayInitializer(c: *Context, ty: *ast.Node) !*ast.Node.Suffix return node; } -fn transCreateNodeStructInitializer(c: *Context, ty: *ast.Node) !*ast.Node.SuffixOp { - _ = try appendToken(c, .LBrace, "{"); - const node = try c.arena.create(ast.Node.SuffixOp); - node.* = .{ - .lhs = .{ .node = ty }, - .op = .{ - .StructInitializer = ast.Node.SuffixOp.Op.InitList{}, - }, - .rtoken = undefined, // set after appending values - }; - return node; -} - fn transCreateNodeInt(c: *Context, int: var) !*ast.Node { const token = try appendTokenFmt(c, .IntegerLiteral, "{}", .{int}); const node = try c.arena.create(ast.Node.IntegerLiteral); @@ -4405,7 +4399,7 @@ fn transCreateNodeShiftOp( fn transCreateNodePtrDeref(c: *Context, lhs: *ast.Node) !*ast.Node { const node = try c.arena.create(ast.Node.SuffixOp); node.* = .{ - .lhs = .{ .node = lhs }, + .lhs = lhs, .op = .Deref, .rtoken = try appendToken(c, .PeriodAsterisk, ".*"), }; @@ -4416,7 +4410,7 @@ fn transCreateNodeArrayAccess(c: *Context, lhs: *ast.Node) !*ast.Node.SuffixOp { _ = try appendToken(c, .LBrace, "["); const node = try c.arena.create(ast.Node.SuffixOp); node.* = .{ - .lhs = .{ .node = lhs }, + .lhs = lhs, .op = .{ .ArrayAccess = undefined, }, From 32ecb416f3acc49b80268711562f2c4133a828b9 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 21 May 2020 00:30:08 -0400 Subject: [PATCH 18/38] fix compile errors when setting NodeIndex/TokenIndex to u32 --- lib/std/zig/parse.zig | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/lib/std/zig/parse.zig b/lib/std/zig/parse.zig index ab41f2df00..156d841b3b 100644 --- a/lib/std/zig/parse.zig +++ b/lib/std/zig/parse.zig @@ -6,6 +6,7 @@ const Node = ast.Node; const Tree = ast.Tree; const AstError = ast.Error; const TokenIndex = ast.TokenIndex; +const NodeIndex = ast.NodeIndex; const Token = std.zig.Token; pub const Error = error{ParseError} || Allocator.Error; @@ -70,7 +71,8 @@ const Parser = struct { // invalid tokens as it can so this can only be the EOF const eof_token = p.eatToken(.Eof).?; - const node = try Node.Root.create(&p.arena.allocator, decls.len, eof_token); + const decls_len = @intCast(NodeIndex, decls.len); + const node = try Node.Root.create(&p.arena.allocator, decls_len, eof_token); std.mem.copy(*Node, node.decls(), decls); return node; @@ -545,13 +547,15 @@ const Parser = struct { else R{ .Explicit = return_type_expr.? }; - const fn_proto_node = try Node.FnProto.alloc(&p.arena.allocator, params.len); + const params_len = @intCast(NodeIndex, params.len); + + const fn_proto_node = try Node.FnProto.alloc(&p.arena.allocator, params_len); fn_proto_node.* = .{ .doc_comments = null, .visib_token = null, .fn_token = fn_token, .name_token = name_token, - .params_len = params.len, + .params_len = params_len, .return_type = return_type, .var_args_token = var_args_token, .extern_export_inline_token = null, @@ -1195,11 +1199,13 @@ const Parser = struct { const rbrace = try p.expectToken(.RBrace); - const block_node = try Node.Block.alloc(&p.arena.allocator, statements.items.len); + const statements_len = @intCast(NodeIndex, statements.items.len); + + const block_node = try Node.Block.alloc(&p.arena.allocator, statements_len); block_node.* = .{ .label = null, .lbrace = lbrace, - .statements_len = statements.items.len, + .statements_len = statements_len, .rbrace = rbrace, }; std.mem.copy(*Node, block_node.statements(), statements.items); @@ -2844,12 +2850,13 @@ const Parser = struct { defer p.gpa.free(members); const rbrace = try p.expectToken(.RBrace); - const node = try Node.ContainerDecl.alloc(&p.arena.allocator, members.len); + const members_len = @intCast(NodeIndex, members.len); + const node = try Node.ContainerDecl.alloc(&p.arena.allocator, members_len); node.* = .{ .layout_token = null, .kind_token = container_decl_type.kind_token, .init_arg_expr = container_decl_type.init_arg_expr, - .fields_and_decls_len = members.len, + .fields_and_decls_len = members_len, .lbrace_token = lbrace, .rbrace_token = rbrace, }; From d37b81d43bc52daa94dd1ad1631018ea0cd11f77 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 21 May 2020 21:48:01 -0400 Subject: [PATCH 19/38] stage2 parser performance/API improvements * Extract Call ast node tag out of SuffixOp; parameters go in memory after Call. * Demote AsmInput and AsmOutput from AST nodes to structs inside the Asm node. * The following ast nodes get their sub-node lists directly following them in memory: - ErrorSetDecl - Switch - BuiltinCall * ast.Node.Asm gets slices for inputs, outputs, clobbers instead of singly linked lists Performance changes: throughput: 72.7 MiB/s => 74.0 MiB/s maxrss: 72 KB => 69 KB (nice) --- lib/std/zig/ast.zig | 415 ++++++++++++++++++++++++++--------------- lib/std/zig/parse.zig | 203 +++++++++----------- lib/std/zig/render.zig | 331 ++++++++++++++++---------------- 3 files changed, 516 insertions(+), 433 deletions(-) diff --git a/lib/std/zig/ast.zig b/lib/std/zig/ast.zig index 70c8b3509c..182422f179 100644 --- a/lib/std/zig/ast.zig +++ b/lib/std/zig/ast.zig @@ -323,7 +323,7 @@ pub const Error = union(enum) { node: *Node, pub fn render(self: *const ExpectedCall, tokens: []const Token, stream: var) !void { - return stream.print("expected " ++ @tagName(@TagType(Node.SuffixOp.Op).Call) ++ ", found {}", .{ + return stream.print("expected " ++ @tagName(Node.Id.Call) ++ ", found {}", .{ @tagName(self.node.id), }); } @@ -333,7 +333,7 @@ pub const Error = union(enum) { node: *Node, pub fn render(self: *const ExpectedCallOrFnProto, tokens: []const Token, stream: var) !void { - return stream.print("expected " ++ @tagName(@TagType(Node.SuffixOp.Op).Call) ++ " or " ++ + return stream.print("expected " ++ @tagName(Node.Id.Call) ++ " or " ++ @tagName(Node.Id.FnProto) ++ ", found {}", .{@tagName(self.node.id)}); } }; @@ -428,15 +428,19 @@ pub const Node = struct { // Operators InfixOp, PrefixOp, + /// Not all suffix operations are under this tag. To save memory, some + /// suffix operations have dedicated Node tags. SuffixOp, - /// This is a suffix operation but to save memory we have a dedicated Node id for it. + /// `T{a, b}` ArrayInitializer, /// ArrayInitializer but with `.` instead of a left-hand-side operand. ArrayInitializerDot, - /// This is a suffix operation but to save memory we have a dedicated Node id for it. + /// `T{.a = b}` StructInitializer, /// StructInitializer but with `.` instead of a left-hand-side operand. StructInitializerDot, + /// `foo()` + Call, // Control flow Switch, @@ -483,8 +487,6 @@ pub const Node = struct { PointerIndexPayload, ContainerField, ErrorTag, - AsmInput, - AsmOutput, FieldInitializer, }; @@ -780,13 +782,22 @@ pub const Node = struct { pub const ErrorSetDecl = struct { base: Node = Node{ .id = .ErrorSetDecl }, error_token: TokenIndex, - decls: DeclList, rbrace_token: TokenIndex, + decls_len: NodeIndex, - pub const DeclList = LinkedList(*Node); + /// After this the caller must initialize the decls list. + pub fn alloc(allocator: *mem.Allocator, decls_len: NodeIndex) !*ErrorSetDecl { + const bytes = try allocator.alignedAlloc(u8, @alignOf(ErrorSetDecl), sizeInBytes(decls_len)); + return @ptrCast(*ErrorSetDecl, bytes.ptr); + } + + pub fn free(self: *ErrorSetDecl, allocator: *mem.Allocator) void { + const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.decls_len)]; + allocator.free(bytes); + } pub fn iterate(self: *const ErrorSetDecl) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = self.decls.first }; + return .{ .parent_node = &self.base, .index = 0, .node = null }; } pub fn iterateNext(self: *const ErrorSetDecl, it: *Node.Iterator) ?*Node { @@ -802,6 +813,20 @@ pub const Node = struct { pub fn lastToken(self: *const ErrorSetDecl) TokenIndex { return self.rbrace_token; } + + pub fn decls(self: *ErrorSetDecl) []*Node { + const decls_start = @ptrCast([*]u8, self) + @sizeOf(ErrorSetDecl); + return @ptrCast([*]*Node, decls_start)[0..self.decls_len]; + } + + pub fn declsConst(self: *const ErrorSetDecl) []const *Node { + const decls_start = @ptrCast([*]const u8, self) + @sizeOf(ErrorSetDecl); + return @ptrCast([*]const *Node, decls_start)[0..self.decls_len]; + } + + fn sizeInBytes(decls_len: NodeIndex) usize { + return @sizeOf(ErrorSetDecl) + @sizeOf(*Node) * @as(usize, decls_len); + } }; /// The fields and decls Node pointers directly follow this struct in memory. @@ -1464,19 +1489,28 @@ pub const Node = struct { } }; + /// The cases node pointers are found in memory after Switch. + /// They must be SwitchCase or SwitchElse nodes. pub const Switch = struct { base: Node = Node{ .id = .Switch }, switch_token: TokenIndex, + rbrace: TokenIndex, + cases_len: NodeIndex, expr: *Node, - /// these must be SwitchCase nodes - cases: CaseList, - rbrace: TokenIndex, + /// After this the caller must initialize the fields_and_decls list. + pub fn alloc(allocator: *mem.Allocator, cases_len: NodeIndex) !*Switch { + const bytes = try allocator.alignedAlloc(u8, @alignOf(Switch), sizeInBytes(cases_len)); + return @ptrCast(*Switch, bytes.ptr); + } - pub const CaseList = LinkedList(*Node); + pub fn free(self: *Switch, allocator: *mem.Allocator) void { + const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.cases_len)]; + allocator.free(bytes); + } pub fn iterate(self: *const Switch) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = self.cases.first }; + return .{ .parent_node = &self.base, .index = 0, .node = null }; } pub fn iterateNext(self: *const Switch, it: *Node.Iterator) ?*Node { @@ -1502,6 +1536,20 @@ pub const Node = struct { pub fn lastToken(self: *const Switch) TokenIndex { return self.rbrace; } + + pub fn cases(self: *Switch) []*Node { + const decls_start = @ptrCast([*]u8, self) + @sizeOf(Switch); + return @ptrCast([*]*Node, decls_start)[0..self.cases_len]; + } + + pub fn casesConst(self: *const Switch) []const *Node { + const decls_start = @ptrCast([*]const u8, self) + @sizeOf(Switch); + return @ptrCast([*]const *Node, decls_start)[0..self.cases_len]; + } + + fn sizeInBytes(cases_len: NodeIndex) usize { + return @sizeOf(Switch) + @sizeOf(*Node) * @as(usize, cases_len); + } }; pub const SwitchCase = struct { @@ -2120,6 +2168,66 @@ pub const Node = struct { } }; + /// Parameter nodes directly follow Call in memory. + pub const Call = struct { + base: Node = Node{ .id = .Call }, + lhs: *Node, + rtoken: TokenIndex, + params_len: NodeIndex, + async_token: ?TokenIndex, + + /// After this the caller must initialize the fields_and_decls list. + pub fn alloc(allocator: *mem.Allocator, params_len: NodeIndex) !*Call { + const bytes = try allocator.alignedAlloc(u8, @alignOf(Call), sizeInBytes(params_len)); + return @ptrCast(*Call, bytes.ptr); + } + + pub fn free(self: *Call, allocator: *mem.Allocator) void { + const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.params_len)]; + allocator.free(bytes); + } + + pub fn iterate(self: *const Call) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null}; + } + + pub fn iterateNext(self: *const Call, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; + + if (i < 1) return self.lhs; + i -= 1; + + if (i < self.params_len) return self.paramsConst()[i]; + i -= self.params_len; + + return null; + } + + pub fn firstToken(self: *const Call) TokenIndex { + if (self.async_token) |async_token| return async_token; + return self.lhs.firstToken(); + } + + pub fn lastToken(self: *const Call) TokenIndex { + return self.rtoken; + } + + pub fn params(self: *Call) []*Node { + const decls_start = @ptrCast([*]u8, self) + @sizeOf(Call); + return @ptrCast([*]*Node, decls_start)[0..self.params_len]; + } + + pub fn paramsConst(self: *const Call) []const *Node { + const decls_start = @ptrCast([*]const u8, self) + @sizeOf(Call); + return @ptrCast([*]const *Node, decls_start)[0..self.params_len]; + } + + fn sizeInBytes(params_len: NodeIndex) usize { + return @sizeOf(Call) + @sizeOf(*Node) * @as(usize, params_len); + } + }; + pub const SuffixOp = struct { base: Node = Node{ .id = .SuffixOp }, op: Op, @@ -2127,19 +2235,11 @@ pub const Node = struct { rtoken: TokenIndex, pub const Op = union(enum) { - Call: Call, ArrayAccess: *Node, Slice: Slice, Deref, UnwrapOptional, - pub const Call = struct { - params: ParamList, - async_token: ?TokenIndex, - - pub const ParamList = LinkedList(*Node); - }; - pub const Slice = struct { start: *Node, end: ?*Node, @@ -2148,12 +2248,7 @@ pub const Node = struct { }; pub fn iterate(self: *const SuffixOp) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, - .node = switch(self.op) { - .Call => |call| call.params.first, - else => null, - }, - }; + return .{ .parent_node = &self.base, .index = 0, .node = null}; } pub fn iterateNext(self: *const SuffixOp, it: *Node.Iterator) ?*Node { @@ -2164,13 +2259,6 @@ pub const Node = struct { i -= 1; switch (self.op) { - .Call => |call_info| { - if (it.node) |child| { - it.index -= 1; - it.node = child.next; - return child.data; - } - }, .ArrayAccess => |index_expr| { if (i < 1) return index_expr; i -= 1; @@ -2197,10 +2285,6 @@ pub const Node = struct { } pub fn firstToken(self: *const SuffixOp) TokenIndex { - switch (self.op) { - .Call => |*call_info| if (call_info.async_token) |async_token| return async_token, - else => {}, - } return self.lhs.firstToken(); } @@ -2396,22 +2480,36 @@ pub const Node = struct { } }; + /// Parameters are in memory following BuiltinCall. pub const BuiltinCall = struct { base: Node = Node{ .id = .BuiltinCall }, + params_len: NodeIndex, builtin_token: TokenIndex, - params: ParamList, rparen_token: TokenIndex, - pub const ParamList = LinkedList(*Node); + /// After this the caller must initialize the fields_and_decls list. + pub fn alloc(allocator: *mem.Allocator, params_len: NodeIndex) !*BuiltinCall { + const bytes = try allocator.alignedAlloc(u8, @alignOf(BuiltinCall), sizeInBytes(params_len)); + return @ptrCast(*BuiltinCall, bytes.ptr); + } + + pub fn free(self: *BuiltinCall, allocator: *mem.Allocator) void { + const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.params_len)]; + allocator.free(bytes); + } pub fn iterate(self: *const BuiltinCall) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = self.params.first }; + return .{ .parent_node = &self.base, .index = 0, .node = null }; } pub fn iterateNext(self: *const BuiltinCall, it: *Node.Iterator) ?*Node { - const param = it.node orelse return null; - it.node = param.next; - return param.data; + var i = it.index; + it.index += 1; + + if (i < self.params_len) return self.paramsConst()[i]; + i -= self.params_len; + + return null; } pub fn firstToken(self: *const BuiltinCall) TokenIndex { @@ -2421,6 +2519,20 @@ pub const Node = struct { pub fn lastToken(self: *const BuiltinCall) TokenIndex { return self.rparen_token; } + + pub fn params(self: *BuiltinCall) []*Node { + const decls_start = @ptrCast([*]u8, self) + @sizeOf(BuiltinCall); + return @ptrCast([*]*Node, decls_start)[0..self.params_len]; + } + + pub fn paramsConst(self: *const BuiltinCall) []const *Node { + const decls_start = @ptrCast([*]const u8, self) + @sizeOf(BuiltinCall); + return @ptrCast([*]const *Node, decls_start)[0..self.params_len]; + } + + fn sizeInBytes(params_len: NodeIndex) usize { + return @sizeOf(BuiltinCall) + @sizeOf(*Node) * @as(usize, params_len); + } }; pub const StringLiteral = struct { @@ -2554,106 +2666,102 @@ pub const Node = struct { } }; - pub const AsmOutput = struct { - base: Node = Node{ .id = .AsmOutput }, - lbracket: TokenIndex, - symbolic_name: *Node, - constraint: *Node, - kind: Kind, - rparen: TokenIndex, - - pub const Kind = union(enum) { - Variable: *Identifier, - Return: *Node, - }; - - pub fn iterate(self: *const AsmOutput) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; - } - - pub fn iterateNext(self: *const AsmOutput, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; - - if (i < 1) return self.symbolic_name; - i -= 1; - - if (i < 1) return self.constraint; - i -= 1; - - switch (self.kind) { - .Variable => |variable_name| { - if (i < 1) return &variable_name.base; - i -= 1; - }, - .Return => |return_type| { - if (i < 1) return return_type; - i -= 1; - }, - } - - return null; - } - - pub fn firstToken(self: *const AsmOutput) TokenIndex { - return self.lbracket; - } - - pub fn lastToken(self: *const AsmOutput) TokenIndex { - return self.rparen; - } - }; - - pub const AsmInput = struct { - base: Node = Node{ .id = .AsmInput }, - lbracket: TokenIndex, - symbolic_name: *Node, - constraint: *Node, - expr: *Node, - rparen: TokenIndex, - - pub fn iterate(self: *const AsmInput) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; - } - - pub fn iterateNext(self: *const AsmInput, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; - - if (i < 1) return self.symbolic_name; - i -= 1; - - if (i < 1) return self.constraint; - i -= 1; - - if (i < 1) return self.expr; - i -= 1; - - return null; - } - - pub fn firstToken(self: *const AsmInput) TokenIndex { - return self.lbracket; - } - - pub fn lastToken(self: *const AsmInput) TokenIndex { - return self.rparen; - } - }; - pub const Asm = struct { base: Node = Node{ .id = .Asm }, asm_token: TokenIndex, + rparen: TokenIndex, volatile_token: ?TokenIndex, template: *Node, - outputs: OutputList, - inputs: InputList, - clobbers: ClobberList, - rparen: TokenIndex, + outputs: []Output, + inputs: []Input, + /// A clobber node must be a StringLiteral or MultilineStringLiteral. + clobbers: []*Node, + + pub const Output = struct { + lbracket: TokenIndex, + symbolic_name: *Node, + constraint: *Node, + kind: Kind, + rparen: TokenIndex, + + pub const Kind = union(enum) { + Variable: *Identifier, + Return: *Node, + }; + + pub fn iterate(self: *const Output) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const Output, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; + + if (i < 1) return self.symbolic_name; + i -= 1; + + if (i < 1) return self.constraint; + i -= 1; + + switch (self.kind) { + .Variable => |variable_name| { + if (i < 1) return &variable_name.base; + i -= 1; + }, + .Return => |return_type| { + if (i < 1) return return_type; + i -= 1; + }, + } + + return null; + } + + pub fn firstToken(self: *const Output) TokenIndex { + return self.lbracket; + } + + pub fn lastToken(self: *const Output) TokenIndex { + return self.rparen; + } + }; + + pub const Input = struct { + lbracket: TokenIndex, + symbolic_name: *Node, + constraint: *Node, + expr: *Node, + rparen: TokenIndex, + + pub fn iterate(self: *const Input) Node.Iterator { + return .{ .parent_node = &self.base, .index = 0, .node = null }; + } + + pub fn iterateNext(self: *const Input, it: *Node.Iterator) ?*Node { + var i = it.index; + it.index += 1; + + if (i < 1) return self.symbolic_name; + i -= 1; + + if (i < 1) return self.constraint; + i -= 1; + + if (i < 1) return self.expr; + i -= 1; + + return null; + } + + pub fn firstToken(self: *const Input) TokenIndex { + return self.lbracket; + } + + pub fn lastToken(self: *const Input) TokenIndex { + return self.rparen; + } + }; - pub const OutputList = LinkedList(*AsmOutput); - pub const InputList = LinkedList(*AsmInput); - pub const ClobberList = LinkedList(*Node); pub fn iterate(self: *const Asm) Node.Iterator { return .{ .parent_node = &self.base, .index = 0, .node = null}; @@ -2663,19 +2771,24 @@ pub const Node = struct { var i = it.index; it.index += 1; - var output: ?*LinkedList(*AsmOutput).Node = self.outputs.first; - while (output) |o| { - if (i < 1) return &o.data.base; - i -= 1; - output = o.next; - } + if (i < self.outputs.len * 3) switch (i % 3) { + 0 => return self.outputs[i / 3].symbolic_name, + 1 => return self.outputs[i / 3].constraint, + 2 => switch (self.outputs[i / 3].kind) { + .Variable => |variable_name| return &variable_name.base, + .Return => |return_type| return return_type, + }, + else => unreachable, + }; + i -= self.outputs.len * 3; - var input: ?*LinkedList(*AsmInput).Node = self.inputs.first; - while (input) |o| { - if (i < 1) return &o.data.base; - i -= 1; - input = o.next; - } + if (i < self.inputs.len * 3) switch (i % 3) { + 0 => return self.inputs[i / 3].symbolic_name, + 1 => return self.inputs[i / 3].constraint, + 2 => return self.inputs[i / 3].expr, + else => unreachable, + }; + i -= self.inputs.len * 3; return null; } diff --git a/lib/std/zig/parse.zig b/lib/std/zig/parse.zig index 156d841b3b..873d2cc252 100644 --- a/lib/std/zig/parse.zig +++ b/lib/std/zig/parse.zig @@ -520,10 +520,9 @@ const Parser = struct { p.putBackToken(token); return null; }; - var var_args_token: ?TokenIndex = null; const name_token = p.eatToken(.Identifier); const lparen = try p.expectToken(.LParen); - const params = try p.parseParamDeclList(&var_args_token); + const params = try p.parseParamDeclList(); defer p.gpa.free(params); const rparen = try p.expectToken(.RParen); const align_expr = try p.parseByteAlign(); @@ -547,15 +546,19 @@ const Parser = struct { else R{ .Explicit = return_type_expr.? }; - const params_len = @intCast(NodeIndex, params.len); + const var_args_token = if (params.len > 0) blk: { + const param_type = params[params.len - 1].param_type; + break :blk if (param_type == .var_args) param_type.var_args else null; + } else + null; - const fn_proto_node = try Node.FnProto.alloc(&p.arena.allocator, params_len); + const fn_proto_node = try Node.FnProto.alloc(&p.arena.allocator, params.len); fn_proto_node.* = .{ .doc_comments = null, .visib_token = null, .fn_token = fn_token, .name_token = name_token, - .params_len = params_len, + .params_len = params.len, .return_type = return_type, .var_args_token = var_args_token, .extern_export_inline_token = null, @@ -1455,17 +1458,15 @@ const Parser = struct { // ignore this, continue parsing return res; }; - const node = try p.arena.allocator.create(Node.SuffixOp); + defer p.gpa.free(params.list); + const node = try Node.Call.alloc(&p.arena.allocator, params.list.len); node.* = .{ .lhs = res, - .op = .{ - .Call = .{ - .params = params.list, - .async_token = async_token, - }, - }, + .params_len = params.list.len, + .async_token = async_token, .rtoken = params.rparen, }; + std.mem.copy(*Node, node.params(), params.list); return &node.base; } if (try p.parsePrimaryTypeExpr()) |expr| { @@ -1482,17 +1483,15 @@ const Parser = struct { continue; } if (try p.parseFnCallArguments()) |params| { - const call = try p.arena.allocator.create(Node.SuffixOp); + defer p.gpa.free(params.list); + const call = try Node.Call.alloc(&p.arena.allocator, params.list.len); call.* = .{ .lhs = res, - .op = .{ - .Call = .{ - .params = params.list, - .async_token = null, - }, - }, + .params_len = params.list.len, + .async_token = null, .rtoken = params.rparen, }; + std.mem.copy(*Node, call.params(), params.list); res = &call.base; continue; } @@ -1615,14 +1614,16 @@ const Parser = struct { return null; } const decls = try p.parseErrorTagList(); + defer p.gpa.free(decls); const rbrace = try p.expectToken(.RBrace); - const node = try p.arena.allocator.create(Node.ErrorSetDecl); + const node = try Node.ErrorSetDecl.alloc(&p.arena.allocator, decls.len); node.* = .{ .error_token = error_token, - .decls = decls, + .decls_len = decls.len, .rbrace_token = rbrace, }; + std.mem.copy(*Node, node.decls(), decls); return &node.base; } @@ -1769,19 +1770,25 @@ const Parser = struct { _ = try p.expectToken(.RParen); _ = try p.expectToken(.LBrace); const cases = try p.parseSwitchProngList(); + defer p.gpa.free(cases); const rbrace = try p.expectToken(.RBrace); - const node = try p.arena.allocator.create(Node.Switch); + const node = try Node.Switch.alloc(&p.arena.allocator, cases.len); node.* = .{ .switch_token = switch_token, .expr = expr_node, - .cases = cases, + .cases_len = cases.len, .rbrace = rbrace, }; + std.mem.copy(*Node, node.cases(), cases); return &node.base; } /// AsmExpr <- KEYWORD_asm KEYWORD_volatile? LPAREN Expr AsmOutput? RPAREN + /// AsmOutput <- COLON AsmOutputList AsmInput? + /// AsmInput <- COLON AsmInputList AsmClobbers? + /// AsmClobbers <- COLON StringList + /// StringList <- (STRINGLITERAL COMMA)* STRINGLITERAL? fn parseAsmExpr(p: *Parser) !?*Node { const asm_token = p.eatToken(.Keyword_asm) orelse return null; const volatile_token = p.eatToken(.Keyword_volatile); @@ -1790,19 +1797,39 @@ const Parser = struct { .ExpectedExpr = .{ .token = p.tok_i }, }); + var arena_outputs: []Node.Asm.Output = &[0]Node.Asm.Output{}; + var arena_inputs: []Node.Asm.Input = &[0]Node.Asm.Input{}; + var arena_clobbers: []*Node = &[0]*Node{}; + + if (p.eatToken(.Colon) != null) { + const outputs = try p.parseAsmOutputList(); + defer p.gpa.free(outputs); + arena_outputs = try p.arena.allocator.dupe(Node.Asm.Output, outputs); + + if (p.eatToken(.Colon) != null) { + const inputs = try p.parseAsmInputList(); + defer p.gpa.free(inputs); + arena_inputs = try p.arena.allocator.dupe(Node.Asm.Input, inputs); + + if (p.eatToken(.Colon) != null) { + const clobbers = try ListParseFn(*Node, parseStringLiteral)(p); + defer p.gpa.free(clobbers); + arena_clobbers = try p.arena.allocator.dupe(*Node, clobbers); + } + } + } + const node = try p.arena.allocator.create(Node.Asm); node.* = .{ .asm_token = asm_token, .volatile_token = volatile_token, .template = template, - .outputs = Node.Asm.OutputList{}, - .inputs = Node.Asm.InputList{}, - .clobbers = Node.Asm.ClobberList{}, - .rparen = undefined, + .outputs = arena_outputs, + .inputs = arena_inputs, + .clobbers = arena_clobbers, + .rparen = try p.expectToken(.RParen), }; - try p.parseAsmOutput(node); - node.rparen = try p.expectToken(.RParen); return &node.base; } @@ -1828,15 +1855,8 @@ const Parser = struct { return null; } - /// AsmOutput <- COLON AsmOutputList AsmInput? - fn parseAsmOutput(p: *Parser, asm_node: *Node.Asm) !void { - if (p.eatToken(.Colon) == null) return; - asm_node.outputs = try p.parseAsmOutputList(); - try p.parseAsmInput(asm_node); - } - /// AsmOutputItem <- LBRACKET IDENTIFIER RBRACKET STRINGLITERAL LPAREN (MINUSRARROW TypeExpr / IDENTIFIER) RPAREN - fn parseAsmOutputItem(p: *Parser) !?*Node.AsmOutput { + fn parseAsmOutputItem(p: *Parser) !?Node.Asm.Output { const lbracket = p.eatToken(.LBracket) orelse return null; const name = try p.expectNode(parseIdentifier, .{ .ExpectedIdentifier = .{ .token = p.tok_i }, @@ -1848,7 +1868,7 @@ const Parser = struct { }); _ = try p.expectToken(.LParen); - const kind: Node.AsmOutput.Kind = blk: { + const kind: Node.Asm.Output.Kind = blk: { if (p.eatToken(.Arrow) != null) { const return_ident = try p.expectNode(parseTypeExpr, .{ .ExpectedTypeExpr = .{ .token = p.tok_i }, @@ -1862,26 +1882,17 @@ const Parser = struct { }; const rparen = try p.expectToken(.RParen); - const node = try p.arena.allocator.create(Node.AsmOutput); - node.* = .{ + return Node.Asm.Output{ .lbracket = lbracket, .symbolic_name = name, .constraint = constraint, .kind = kind, .rparen = rparen, }; - return node; - } - - /// AsmInput <- COLON AsmInputList AsmClobbers? - fn parseAsmInput(p: *Parser, asm_node: *Node.Asm) !void { - if (p.eatToken(.Colon) == null) return; - asm_node.inputs = try p.parseAsmInputList(); - try p.parseAsmClobbers(asm_node); } /// AsmInputItem <- LBRACKET IDENTIFIER RBRACKET STRINGLITERAL LPAREN Expr RPAREN - fn parseAsmInputItem(p: *Parser) !?*Node.AsmInput { + fn parseAsmInputItem(p: *Parser) !?Node.Asm.Input { const lbracket = p.eatToken(.LBracket) orelse return null; const name = try p.expectNode(parseIdentifier, .{ .ExpectedIdentifier = .{ .token = p.tok_i }, @@ -1898,25 +1909,13 @@ const Parser = struct { }); const rparen = try p.expectToken(.RParen); - const node = try p.arena.allocator.create(Node.AsmInput); - node.* = .{ + return Node.Asm.Input{ .lbracket = lbracket, .symbolic_name = name, .constraint = constraint, .expr = expr, .rparen = rparen, }; - return node; - } - - /// AsmClobbers <- COLON StringList - /// StringList <- (STRINGLITERAL COMMA)* STRINGLITERAL? - fn parseAsmClobbers(p: *Parser, asm_node: *Node.Asm) !void { - if (p.eatToken(.Colon) == null) return; - asm_node.clobbers = try ListParseFn( - Node.Asm.ClobberList, - parseStringLiteral, - )(p); } /// BreakLabel <- COLON IDENTIFIER @@ -1999,7 +1998,7 @@ const Parser = struct { } /// ParamDecl <- (KEYWORD_noalias / KEYWORD_comptime)? (IDENTIFIER COLON)? ParamType - fn parseParamDecl(p: *Parser, list: *std.ArrayList(Node.FnProto.ParamDecl)) !bool { + fn parseParamDecl(p: *Parser) !?Node.FnProto.ParamDecl { const doc_comments = try p.parseDocComment(); const noalias_token = p.eatToken(.Keyword_noalias); const comptime_token = if (noalias_token == null) p.eatToken(.Keyword_comptime) else null; @@ -2014,21 +2013,23 @@ const Parser = struct { if (noalias_token == null and comptime_token == null and name_token == null and - doc_comments == null) return false; + doc_comments == null) + { + return null; + } try p.errors.append(p.gpa, .{ .ExpectedParamType = .{ .token = p.tok_i }, }); return error.ParseError; }; - (try list.addOne()).* = .{ + return Node.FnProto.ParamDecl{ .doc_comments = doc_comments, .comptime_token = comptime_token, .noalias_token = noalias_token, .name_token = name_token, .param_type = param_type, }; - return true; } /// ParamType @@ -2714,13 +2715,14 @@ const Parser = struct { /// ExprList <- (Expr COMMA)* Expr? fn parseFnCallArguments(p: *Parser) !?AnnotatedParamList { if (p.eatToken(.LParen) == null) return null; - const list = try ListParseFn(std.SinglyLinkedList(*Node), parseExpr)(p); + const list = try ListParseFn(*Node, parseExpr)(p); + errdefer p.gpa.free(list); const rparen = try p.expectToken(.RParen); return AnnotatedParamList{ .list = list, .rparen = rparen }; } const AnnotatedParamList = struct { - list: std.SinglyLinkedList(*Node), + list: []*Node, rparen: TokenIndex, }; @@ -2936,62 +2938,40 @@ const Parser = struct { /// IdentifierList <- (IDENTIFIER COMMA)* IDENTIFIER? /// Only ErrorSetDecl parses an IdentifierList - fn parseErrorTagList(p: *Parser) !Node.ErrorSetDecl.DeclList { - return ListParseFn(Node.ErrorSetDecl.DeclList, parseErrorTag)(p); + fn parseErrorTagList(p: *Parser) ![]*Node { + return ListParseFn(*Node, parseErrorTag)(p); } /// SwitchProngList <- (SwitchProng COMMA)* SwitchProng? - fn parseSwitchProngList(p: *Parser) !Node.Switch.CaseList { - return ListParseFn(Node.Switch.CaseList, parseSwitchProng)(p); + fn parseSwitchProngList(p: *Parser) ![]*Node { + return ListParseFn(*Node, parseSwitchProng)(p); } /// AsmOutputList <- (AsmOutputItem COMMA)* AsmOutputItem? - fn parseAsmOutputList(p: *Parser) Error!Node.Asm.OutputList { - return ListParseFn(Node.Asm.OutputList, parseAsmOutputItem)(p); + fn parseAsmOutputList(p: *Parser) Error![]Node.Asm.Output { + return ListParseFn(Node.Asm.Output, parseAsmOutputItem)(p); } /// AsmInputList <- (AsmInputItem COMMA)* AsmInputItem? - fn parseAsmInputList(p: *Parser) Error!Node.Asm.InputList { - return ListParseFn(Node.Asm.InputList, parseAsmInputItem)(p); + fn parseAsmInputList(p: *Parser) Error![]Node.Asm.Input { + return ListParseFn(Node.Asm.Input, parseAsmInputItem)(p); } /// ParamDeclList <- (ParamDecl COMMA)* ParamDecl? - fn parseParamDeclList(p: *Parser, var_args_token: *?TokenIndex) ![]Node.FnProto.ParamDecl { - var list = std.ArrayList(Node.FnProto.ParamDecl).init(p.gpa); - defer list.deinit(); - - while (try p.parseParamDecl(&list)) { - switch (p.tokens[p.tok_i].id) { - .Comma => _ = p.nextToken(), - // all possible delimiters - .Colon, .RParen, .RBrace, .RBracket => break, - else => { - // this is likely just a missing comma, - // continue parsing this list and give an error - try p.errors.append(p.gpa, .{ - .ExpectedToken = .{ .token = p.tok_i, .expected_id = .Comma }, - }); - }, - } - } - if (list.items.len != 0) { - const param_type = list.items[list.items.len - 1].param_type; - if (param_type == .var_args) { - var_args_token.* = param_type.var_args; - } - } - return list.toOwnedSlice(); + fn parseParamDeclList(p: *Parser) ![]Node.FnProto.ParamDecl { + return ListParseFn(Node.FnProto.ParamDecl, parseParamDecl)(p); } const NodeParseFn = fn (p: *Parser) Error!?*Node; - fn ListParseFn(comptime L: type, comptime nodeParseFn: var) ParseFn(L) { + fn ListParseFn(comptime E: type, comptime nodeParseFn: var) ParseFn([]E) { return struct { - pub fn parse(p: *Parser) !L { - var list = L{}; - var list_it = &list.first; - while (try nodeParseFn(p)) |node| { - list_it = try p.llpush(L.Node.Data, list_it, node); + pub fn parse(p: *Parser) ![]E { + var list = std.ArrayList(E).init(p.gpa); + defer list.deinit(); + + while (try nodeParseFn(p)) |item| { + try list.append(item); switch (p.tokens[p.tok_i].id) { .Comma => _ = p.nextToken(), @@ -3006,7 +2986,7 @@ const Parser = struct { }, } } - return list; + return list.toOwnedSlice(); } }.parse; } @@ -3053,12 +3033,15 @@ const Parser = struct { }; return &node.base; }; - const node = try p.arena.allocator.create(Node.BuiltinCall); + defer p.gpa.free(params.list); + + const node = try Node.BuiltinCall.alloc(&p.arena.allocator, params.list.len); node.* = .{ .builtin_token = token, - .params = params.list, + .params_len = params.list.len, .rparen_token = params.rparen, }; + std.mem.copy(*Node, node.params(), params.list); return &node.base; } diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index 0d5a840ef1..3241f55a38 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -187,7 +187,10 @@ fn renderRoot( } fn renderExtraNewline(tree: *ast.Tree, stream: var, start_col: *usize, node: *ast.Node) @TypeOf(stream).Error!void { - const first_token = node.firstToken(); + return renderExtraNewlineToken(tree, stream, start_col, node.firstToken()); +} + +fn renderExtraNewlineToken(tree: *ast.Tree, stream: var, start_col: *usize, first_token: ast.TokenIndex,) @TypeOf(stream).Error!void { var prev_token = first_token; if (prev_token == 0) return; var newline_threshold: usize = 2; @@ -902,74 +905,70 @@ fn renderExpression( return renderToken(tree, stream, rtoken, indent, start_col, space); }, + .Call => { + const call = @fieldParentPtr(ast.Node.Call, "base", base); + if (call.async_token) |async_token| { + try renderToken(tree, stream, async_token, indent, start_col, Space.Space); + } + + try renderExpression(allocator, stream, tree, indent, start_col, call.lhs, Space.None); + + const lparen = tree.nextToken(call.lhs.lastToken()); + + if (call.params_len == 0) { + try renderToken(tree, stream, lparen, indent, start_col, Space.None); + return renderToken(tree, stream, call.rtoken, indent, start_col, space); + } + + const src_has_trailing_comma = blk: { + const maybe_comma = tree.prevToken(call.rtoken); + break :blk tree.tokens[maybe_comma].id == .Comma; + }; + + if (src_has_trailing_comma) { + const new_indent = indent + indent_delta; + try renderToken(tree, stream, lparen, new_indent, start_col, Space.Newline); + + const params = call.params(); + for (params) |param_node, i| { + const param_node_new_indent = if (param_node.id == .MultilineStringLiteral) blk: { + break :blk indent; + } else blk: { + try stream.writeByteNTimes(' ', new_indent); + break :blk new_indent; + }; + + if (i + 1 < params.len) { + try renderExpression(allocator, stream, tree, param_node_new_indent, start_col, param_node, Space.None); + const comma = tree.nextToken(param_node.lastToken()); + try renderToken(tree, stream, comma, new_indent, start_col, Space.Newline); // , + try renderExtraNewline(tree, stream, start_col, params[i + 1]); + } else { + try renderExpression(allocator, stream, tree, param_node_new_indent, start_col, param_node, Space.Comma); + try stream.writeByteNTimes(' ', indent); + return renderToken(tree, stream, call.rtoken, indent, start_col, space); + } + } + } + + try renderToken(tree, stream, lparen, indent, start_col, Space.None); // ( + + const params = call.params(); + for (params) |param_node, i| { + try renderExpression(allocator, stream, tree, indent, start_col, param_node, Space.None); + + if (i + 1 < params.len) { + const comma = tree.nextToken(param_node.lastToken()); + try renderToken(tree, stream, comma, indent, start_col, Space.Space); + } + } + return renderToken(tree, stream, call.rtoken, indent, start_col, space); + }, + .SuffixOp => { const suffix_op = @fieldParentPtr(ast.Node.SuffixOp, "base", base); switch (suffix_op.op) { - .Call => |*call_info| { - if (call_info.async_token) |async_token| { - try renderToken(tree, stream, async_token, indent, start_col, Space.Space); - } - - try renderExpression(allocator, stream, tree, indent, start_col, suffix_op.lhs, Space.None); - - const lparen = tree.nextToken(suffix_op.lhs.lastToken()); - - if (call_info.params.first == null) { - try renderToken(tree, stream, lparen, indent, start_col, Space.None); - return renderToken(tree, stream, suffix_op.rtoken, indent, start_col, space); - } - - const src_has_trailing_comma = blk: { - const maybe_comma = tree.prevToken(suffix_op.rtoken); - break :blk tree.tokens[maybe_comma].id == .Comma; - }; - - if (src_has_trailing_comma) { - const new_indent = indent + indent_delta; - try renderToken(tree, stream, lparen, new_indent, start_col, Space.Newline); - - var it = call_info.params.first; - while (true) { - const param_node_node = it.?; - it = param_node_node.next; - const param_node = param_node_node.data; - - const param_node_new_indent = if (param_node.id == .MultilineStringLiteral) blk: { - break :blk indent; - } else blk: { - try stream.writeByteNTimes(' ', new_indent); - break :blk new_indent; - }; - - if (it) |next_node| { - try renderExpression(allocator, stream, tree, param_node_new_indent, start_col, param_node, Space.None); - const comma = tree.nextToken(param_node.lastToken()); - try renderToken(tree, stream, comma, new_indent, start_col, Space.Newline); // , - try renderExtraNewline(tree, stream, start_col, next_node.data); - } else { - try renderExpression(allocator, stream, tree, param_node_new_indent, start_col, param_node, Space.Comma); - try stream.writeByteNTimes(' ', indent); - return renderToken(tree, stream, suffix_op.rtoken, indent, start_col, space); - } - } - } - - try renderToken(tree, stream, lparen, indent, start_col, Space.None); // ( - - var it = call_info.params.first; - while (it) |param_node_node| : (it = param_node_node.next) { - const param_node = param_node_node.data; - try renderExpression(allocator, stream, tree, indent, start_col, param_node, Space.None); - - if (param_node_node.next != null) { - const comma = tree.nextToken(param_node.lastToken()); - try renderToken(tree, stream, comma, indent, start_col, Space.Space); - } - } - return renderToken(tree, stream, suffix_op.rtoken, indent, start_col, space); - }, - .ArrayAccess => |index_expr| { const lbracket = tree.nextToken(suffix_op.lhs.lastToken()); const rbracket = tree.nextToken(index_expr.lastToken()); @@ -1288,14 +1287,14 @@ fn renderExpression( const lbrace = tree.nextToken(err_set_decl.error_token); - if (err_set_decl.decls.first == null) { + if (err_set_decl.decls_len == 0) { try renderToken(tree, stream, err_set_decl.error_token, indent, start_col, Space.None); try renderToken(tree, stream, lbrace, indent, start_col, Space.None); return renderToken(tree, stream, err_set_decl.rbrace_token, indent, start_col, space); } - if (err_set_decl.decls.first.?.next == null) blk: { - const node = err_set_decl.decls.first.?.data; + if (err_set_decl.decls_len == 1) blk: { + const node = err_set_decl.decls()[0]; // if there are any doc comments or same line comments // don't try to put it all on one line @@ -1322,16 +1321,15 @@ fn renderExpression( try renderToken(tree, stream, lbrace, indent, start_col, Space.Newline); // { const new_indent = indent + indent_delta; - var it = err_set_decl.decls.first; - while (it) |node_node| : (it = node_node.next) { - const node = node_node.data; + const decls = err_set_decl.decls(); + for (decls) |node, i| { try stream.writeByteNTimes(' ', new_indent); - if (node_node.next) |next_node| { + if (i + 1 < decls.len) { try renderExpression(allocator, stream, tree, new_indent, start_col, node, Space.None); try renderToken(tree, stream, tree.nextToken(node.lastToken()), new_indent, start_col, Space.Newline); // , - try renderExtraNewline(tree, stream, start_col, next_node.data); + try renderExtraNewline(tree, stream, start_col, decls[i + 1]); } else { try renderExpression(allocator, stream, tree, new_indent, start_col, node, Space.Comma); } @@ -1342,16 +1340,15 @@ fn renderExpression( } else { try renderToken(tree, stream, lbrace, indent, start_col, Space.Space); // { - var it = err_set_decl.decls.first; - while (it) |node_node| : (it = node_node.next) { - const node = node_node.data; - if (node_node.next) |next_node| { + const decls = err_set_decl.decls(); + for (decls) |node, i| { + if (i + 1 < decls.len) { try renderExpression(allocator, stream, tree, indent, start_col, node, Space.None); const comma_token = tree.nextToken(node.lastToken()); assert(tree.tokens[comma_token].id == .Comma); try renderToken(tree, stream, comma_token, indent, start_col, Space.Space); // , - try renderExtraNewline(tree, stream, start_col, next_node.data); + try renderExtraNewline(tree, stream, start_col, decls[i + 1]); } else { try renderExpression(allocator, stream, tree, indent, start_col, node, Space.Space); } @@ -1401,12 +1398,8 @@ fn renderExpression( try renderToken(tree, stream, builtin_call.builtin_token, indent, start_col, Space.None); // @name const src_params_trailing_comma = blk: { - if (builtin_call.params.first == null or - builtin_call.params.first.?.next == null) - { - break :blk false; - } - const last_node = builtin_call.params.first.?.findLast().data; + if (builtin_call.params_len < 2) break :blk false; + const last_node = builtin_call.params()[builtin_call.params_len - 1]; const maybe_comma = tree.nextToken(last_node.lastToken()); break :blk tree.tokens[maybe_comma].id == .Comma; }; @@ -1417,12 +1410,11 @@ fn renderExpression( try renderToken(tree, stream, lparen, indent, start_col, Space.None); // ( // render all on one line, no trailing comma - var it = builtin_call.params.first; - while (it) |param_node_node| : (it = param_node_node.next) { - const param_node = param_node_node.data; + const params = builtin_call.params(); + for (params) |param_node, i| { try renderExpression(allocator, stream, tree, indent, start_col, param_node, Space.None); - if (param_node_node.next != null) { + if (i + 1 < params.len) { const comma_token = tree.nextToken(param_node.lastToken()); try renderToken(tree, stream, comma_token, indent, start_col, Space.Space); // , } @@ -1432,9 +1424,7 @@ fn renderExpression( const new_indent = indent + indent_delta; try renderToken(tree, stream, lparen, new_indent, start_col, Space.Newline); // ( - var it = builtin_call.params.first; - while (it) |param_node_node| : (it = param_node_node.next) { - const param_node = param_node_node.data; + for (builtin_call.params()) |param_node| { try stream.writeByteNTimes(' ', new_indent); try renderExpression(allocator, stream, tree, indent, start_col, param_node, Space.Comma); } @@ -1592,7 +1582,7 @@ fn renderExpression( const rparen = tree.nextToken(switch_node.expr.lastToken()); const lbrace = tree.nextToken(rparen); - if (switch_node.cases.first == null) { + if (switch_node.cases_len == 0) { try renderExpression(allocator, stream, tree, indent, start_col, switch_node.expr, Space.None); try renderToken(tree, stream, rparen, indent, start_col, Space.Space); // ) try renderToken(tree, stream, lbrace, indent, start_col, Space.None); // { @@ -1606,14 +1596,13 @@ fn renderExpression( try renderToken(tree, stream, rparen, indent, start_col, Space.Space); // ) try renderToken(tree, stream, lbrace, new_indent, start_col, Space.Newline); // { - var it = switch_node.cases.first; - while (it) |node_node| : (it = node_node.next) { - const node = node_node.data; + const cases = switch_node.cases(); + for (cases) |node, i| { try stream.writeByteNTimes(' ', new_indent); try renderExpression(allocator, stream, tree, new_indent, start_col, node, Space.Comma); - if (node_node.next) |next_node| { - try renderExtraNewline(tree, stream, start_col, next_node.data); + if (i + 1 < cases.len) { + try renderExtraNewline(tree, stream, start_col, cases[i + 1]); } } @@ -1929,7 +1918,7 @@ fn renderExpression( try renderToken(tree, stream, tree.nextToken(asm_node.asm_token), indent, start_col, Space.None); // ( } - if (asm_node.outputs.first == null and asm_node.inputs.first == null and asm_node.clobbers.first == null) { + if (asm_node.outputs.len == 0 and asm_node.inputs.len == 0 and asm_node.clobbers.len == 0) { try renderExpression(allocator, stream, tree, indent, start_col, asm_node.template, Space.None); return renderToken(tree, stream, asm_node.rparen, indent, start_col, space); } @@ -1949,7 +1938,7 @@ fn renderExpression( const colon1 = tree.nextToken(asm_node.template.lastToken()); const indent_extra = indent_once + 2; - const colon2 = if (asm_node.outputs.first == null) blk: { + const colon2 = if (asm_node.outputs.len == 0) blk: { try renderToken(tree, stream, colon1, indent, start_col, Space.Newline); // : try stream.writeByteNTimes(' ', indent_once); @@ -1957,39 +1946,34 @@ fn renderExpression( } else blk: { try renderToken(tree, stream, colon1, indent, start_col, Space.Space); // : - var it = asm_node.outputs.first; - while (true) { - const asm_output_node = it.?; - it = asm_output_node.next; - const asm_output = asm_output_node.data; - const node = &asm_output.base; + for (asm_node.outputs) |*asm_output, i| { + if (i + 1 < asm_node.outputs.len) { + const next_asm_output = asm_node.outputs[i + 1]; + try renderAsmOutput(allocator, stream, tree, indent_extra, start_col, asm_output, Space.None); - if (asm_output_node.next) |next_asm_output| { - try renderExpression(allocator, stream, tree, indent_extra, start_col, node, Space.None); - const next_node = &next_asm_output.data.base; - - const comma = tree.prevToken(next_asm_output.data.firstToken()); + const comma = tree.prevToken(next_asm_output.firstToken()); try renderToken(tree, stream, comma, indent_extra, start_col, Space.Newline); // , - try renderExtraNewline(tree, stream, start_col, next_node); + try renderExtraNewlineToken(tree, stream, start_col, next_asm_output.firstToken()); try stream.writeByteNTimes(' ', indent_extra); - } else if (asm_node.inputs.first == null and asm_node.clobbers.first == null) { - try renderExpression(allocator, stream, tree, indent_extra, start_col, node, Space.Newline); + } else if (asm_node.inputs.len == 0 and asm_node.clobbers.len == 0) { + try renderAsmOutput(allocator, stream, tree, indent_extra, start_col, asm_output, Space.Newline); try stream.writeByteNTimes(' ', indent); return renderToken(tree, stream, asm_node.rparen, indent, start_col, space); } else { - try renderExpression(allocator, stream, tree, indent_extra, start_col, node, Space.Newline); + try renderAsmOutput(allocator, stream, tree, indent_extra, start_col, asm_output, Space.Newline); try stream.writeByteNTimes(' ', indent_once); - const comma_or_colon = tree.nextToken(node.lastToken()); + const comma_or_colon = tree.nextToken(asm_output.lastToken()); break :blk switch (tree.tokens[comma_or_colon].id) { .Comma => tree.nextToken(comma_or_colon), else => comma_or_colon, }; } } + unreachable; }; - const colon3 = if (asm_node.inputs.first == null) blk: { + const colon3 = if (asm_node.inputs.len == 0) blk: { try renderToken(tree, stream, colon2, indent, start_col, Space.Newline); // : try stream.writeByteNTimes(' ', indent_once); @@ -1997,46 +1981,37 @@ fn renderExpression( } else blk: { try renderToken(tree, stream, colon2, indent, start_col, Space.Space); // : - var it = asm_node.inputs.first; - while (true) { - const asm_input_node = it.?; - it = asm_input_node.next; - const node = &asm_input_node.data.base; + for (asm_node.inputs) |*asm_input, i| { + if (i + 1 < asm_node.inputs.len) { + const next_asm_input = &asm_node.inputs[i + 1]; + try renderAsmInput(allocator, stream, tree, indent_extra, start_col, asm_input, Space.None); - if (it) |next_asm_input| { - try renderExpression(allocator, stream, tree, indent_extra, start_col, node, Space.None); - const next_node = &next_asm_input.data.base; - - const comma = tree.prevToken(next_asm_input.data.firstToken()); + const comma = tree.prevToken(next_asm_input.firstToken()); try renderToken(tree, stream, comma, indent_extra, start_col, Space.Newline); // , - try renderExtraNewline(tree, stream, start_col, next_node); + try renderExtraNewlineToken(tree, stream, start_col, next_asm_input.firstToken()); try stream.writeByteNTimes(' ', indent_extra); - } else if (asm_node.clobbers.first == null) { - try renderExpression(allocator, stream, tree, indent_extra, start_col, node, Space.Newline); + } else if (asm_node.clobbers.len == 0) { + try renderAsmInput(allocator, stream, tree, indent_extra, start_col, asm_input, Space.Newline); try stream.writeByteNTimes(' ', indent); return renderToken(tree, stream, asm_node.rparen, indent, start_col, space); // ) } else { - try renderExpression(allocator, stream, tree, indent_extra, start_col, node, Space.Newline); + try renderAsmInput(allocator, stream, tree, indent_extra, start_col, asm_input, Space.Newline); try stream.writeByteNTimes(' ', indent_once); - const comma_or_colon = tree.nextToken(node.lastToken()); + const comma_or_colon = tree.nextToken(asm_input.lastToken()); break :blk switch (tree.tokens[comma_or_colon].id) { .Comma => tree.nextToken(comma_or_colon), else => comma_or_colon, }; } } + unreachable; }; try renderToken(tree, stream, colon3, indent, start_col, Space.Space); // : - var it = asm_node.clobbers.first; - while (true) { - const clobber_node_node = it.?; - it = clobber_node_node.next; - const clobber_node = clobber_node_node.data; - - if (it == null) { + for (asm_node.clobbers) |clobber_node, i| { + if (i + 1 >= asm_node.clobbers.len) { try renderExpression(allocator, stream, tree, indent_extra, start_col, clobber_node, Space.Newline); try stream.writeByteNTimes(' ', indent); return renderToken(tree, stream, asm_node.rparen, indent, start_col, space); @@ -2048,40 +2023,6 @@ fn renderExpression( } }, - .AsmInput => { - const asm_input = @fieldParentPtr(ast.Node.AsmInput, "base", base); - - try stream.writeAll("["); - try renderExpression(allocator, stream, tree, indent, start_col, asm_input.symbolic_name, Space.None); - try stream.writeAll("] "); - try renderExpression(allocator, stream, tree, indent, start_col, asm_input.constraint, Space.None); - try stream.writeAll(" ("); - try renderExpression(allocator, stream, tree, indent, start_col, asm_input.expr, Space.None); - return renderToken(tree, stream, asm_input.lastToken(), indent, start_col, space); // ) - }, - - .AsmOutput => { - const asm_output = @fieldParentPtr(ast.Node.AsmOutput, "base", base); - - try stream.writeAll("["); - try renderExpression(allocator, stream, tree, indent, start_col, asm_output.symbolic_name, Space.None); - try stream.writeAll("] "); - try renderExpression(allocator, stream, tree, indent, start_col, asm_output.constraint, Space.None); - try stream.writeAll(" ("); - - switch (asm_output.kind) { - ast.Node.AsmOutput.Kind.Variable => |variable_name| { - try renderExpression(allocator, stream, tree, indent, start_col, &variable_name.base, Space.None); - }, - ast.Node.AsmOutput.Kind.Return => |return_type| { - try stream.writeAll("-> "); - try renderExpression(allocator, stream, tree, indent, start_col, return_type, Space.None); - }, - } - - return renderToken(tree, stream, asm_output.lastToken(), indent, start_col, space); // ) - }, - .EnumLiteral => { const enum_literal = @fieldParentPtr(ast.Node.EnumLiteral, "base", base); @@ -2098,6 +2039,52 @@ fn renderExpression( } } +fn renderAsmOutput( + allocator: *mem.Allocator, + stream: var, + tree: *ast.Tree, + indent: usize, + start_col: *usize, + asm_output: *const ast.Node.Asm.Output, + space: Space, +) (@TypeOf(stream).Error || Error)!void { + try stream.writeAll("["); + try renderExpression(allocator, stream, tree, indent, start_col, asm_output.symbolic_name, Space.None); + try stream.writeAll("] "); + try renderExpression(allocator, stream, tree, indent, start_col, asm_output.constraint, Space.None); + try stream.writeAll(" ("); + + switch (asm_output.kind) { + ast.Node.Asm.Output.Kind.Variable => |variable_name| { + try renderExpression(allocator, stream, tree, indent, start_col, &variable_name.base, Space.None); + }, + ast.Node.Asm.Output.Kind.Return => |return_type| { + try stream.writeAll("-> "); + try renderExpression(allocator, stream, tree, indent, start_col, return_type, Space.None); + }, + } + + return renderToken(tree, stream, asm_output.lastToken(), indent, start_col, space); // ) +} + +fn renderAsmInput( + allocator: *mem.Allocator, + stream: var, + tree: *ast.Tree, + indent: usize, + start_col: *usize, + asm_input: *const ast.Node.Asm.Input, + space: Space, +) (@TypeOf(stream).Error || Error)!void { + try stream.writeAll("["); + try renderExpression(allocator, stream, tree, indent, start_col, asm_input.symbolic_name, Space.None); + try stream.writeAll("] "); + try renderExpression(allocator, stream, tree, indent, start_col, asm_input.constraint, Space.None); + try stream.writeAll(" ("); + try renderExpression(allocator, stream, tree, indent, start_col, asm_input.expr, Space.None); + return renderToken(tree, stream, asm_input.lastToken(), indent, start_col, space); // ) +} + fn renderVarDecl( allocator: *mem.Allocator, stream: var, From 9377af934f5be92821f36b5d2255f086e5a84cda Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 21 May 2020 22:28:30 -0400 Subject: [PATCH 20/38] stage2 parser: SwitchCase uses intrusive array instead of linkedlist no perf impact, but the API is better --- lib/std/zig/ast.zig | 39 ++++++++++++++++++++++++++++++--------- lib/std/zig/parse.zig | 15 ++++++++------- lib/std/zig/render.zig | 24 +++++++++++------------- 3 files changed, 49 insertions(+), 29 deletions(-) diff --git a/lib/std/zig/ast.zig b/lib/std/zig/ast.zig index 182422f179..6e31555ca3 100644 --- a/lib/std/zig/ast.zig +++ b/lib/std/zig/ast.zig @@ -1552,28 +1552,35 @@ pub const Node = struct { } }; + /// Items sub-nodes appear in memory directly following SwitchCase. pub const SwitchCase = struct { base: Node = Node{ .id = .SwitchCase }, - items: ItemList, arrow_token: TokenIndex, payload: ?*Node, expr: *Node, + items_len: NodeIndex, - pub const ItemList = LinkedList(*Node); + /// After this the caller must initialize the fields_and_decls list. + pub fn alloc(allocator: *mem.Allocator, items_len: NodeIndex) !*SwitchCase { + const bytes = try allocator.alignedAlloc(u8, @alignOf(SwitchCase), sizeInBytes(items_len)); + return @ptrCast(*SwitchCase, bytes.ptr); + } + + pub fn free(self: *SwitchCase, allocator: *mem.Allocator) void { + const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.items_len)]; + allocator.free(bytes); + } pub fn iterate(self: *const SwitchCase) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = self.items.first }; + return .{ .parent_node = &self.base, .index = 0, .node = null }; } pub fn iterateNext(self: *const SwitchCase, it: *Node.Iterator) ?*Node { var i = it.index; it.index += 1; - if (it.node) |child| { - it.index -= 1; - it.node = child.next; - return child.data; - } + if (i < self.items_len) return self.itemsConst()[i]; + i -= self.items_len; if (self.payload) |payload| { if (i < 1) return payload; @@ -1587,12 +1594,26 @@ pub const Node = struct { } pub fn firstToken(self: *const SwitchCase) TokenIndex { - return self.items.first.?.data.firstToken(); + return self.itemsConst()[0].firstToken(); } pub fn lastToken(self: *const SwitchCase) TokenIndex { return self.expr.lastToken(); } + + pub fn items(self: *SwitchCase) []*Node { + const decls_start = @ptrCast([*]u8, self) + @sizeOf(SwitchCase); + return @ptrCast([*]*Node, decls_start)[0..self.items_len]; + } + + pub fn itemsConst(self: *const SwitchCase) []const *Node { + const decls_start = @ptrCast([*]const u8, self) + @sizeOf(SwitchCase); + return @ptrCast([*]const *Node, decls_start)[0..self.items_len]; + } + + fn sizeInBytes(items_len: NodeIndex) usize { + return @sizeOf(SwitchCase) + @sizeOf(*Node) * @as(usize, items_len); + } }; pub const SwitchElse = struct { diff --git a/lib/std/zig/parse.zig b/lib/std/zig/parse.zig index 873d2cc252..3c7f0cf06e 100644 --- a/lib/std/zig/parse.zig +++ b/lib/std/zig/parse.zig @@ -2205,30 +2205,31 @@ const Parser = struct { /// <- SwitchItem (COMMA SwitchItem)* COMMA? /// / KEYWORD_else fn parseSwitchCase(p: *Parser) !?*Node { - var list = Node.SwitchCase.ItemList{}; - var list_it = &list.first; + var list = std.ArrayList(*Node).init(p.gpa); + defer list.deinit(); if (try p.parseSwitchItem()) |first_item| { - list_it = try p.llpush(*Node, list_it, first_item); + try list.append(first_item); while (p.eatToken(.Comma) != null) { const next_item = (try p.parseSwitchItem()) orelse break; - list_it = try p.llpush(*Node, list_it, next_item); + try list.append(next_item); } } else if (p.eatToken(.Keyword_else)) |else_token| { const else_node = try p.arena.allocator.create(Node.SwitchElse); else_node.* = .{ .token = else_token, }; - list_it = try p.llpush(*Node, list_it, &else_node.base); + try list.append(&else_node.base); } else return null; - const node = try p.arena.allocator.create(Node.SwitchCase); + const node = try Node.SwitchCase.alloc(&p.arena.allocator, list.items.len); node.* = .{ - .items = list, + .items_len = list.items.len, .arrow_token = undefined, // set by caller .payload = null, .expr = undefined, // set by caller }; + std.mem.copy(*Node, node.items(), list.items); return &node.base; } diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index 3241f55a38..97ce9617c3 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -1613,37 +1613,35 @@ fn renderExpression( .SwitchCase => { const switch_case = @fieldParentPtr(ast.Node.SwitchCase, "base", base); - assert(switch_case.items.first != null); + assert(switch_case.items_len != 0); const src_has_trailing_comma = blk: { - const last_node = switch_case.items.first.?.findLast().data; + const last_node = switch_case.items()[switch_case.items_len - 1]; const maybe_comma = tree.nextToken(last_node.lastToken()); break :blk tree.tokens[maybe_comma].id == .Comma; }; - if (switch_case.items.first.?.next == null or !src_has_trailing_comma) { - var it = switch_case.items.first; - while (it) |node_node| : (it = node_node.next) { - const node = node_node.data; - if (node_node.next) |next_node| { + if (switch_case.items_len == 1 or !src_has_trailing_comma) { + const items = switch_case.items(); + for (items) |node, i| { + if (i + 1 < items.len) { try renderExpression(allocator, stream, tree, indent, start_col, node, Space.None); const comma_token = tree.nextToken(node.lastToken()); try renderToken(tree, stream, comma_token, indent, start_col, Space.Space); // , - try renderExtraNewline(tree, stream, start_col, next_node.data); + try renderExtraNewline(tree, stream, start_col, items[i + 1]); } else { try renderExpression(allocator, stream, tree, indent, start_col, node, Space.Space); } } } else { - var it = switch_case.items.first; - while (it) |node_node| : (it = node_node.next) { - const node = node_node.data; - if (node_node.next) |next_node| { + const items = switch_case.items(); + for (items) |node, i| { + if (i + 1 < items.len) { try renderExpression(allocator, stream, tree, indent, start_col, node, Space.None); const comma_token = tree.nextToken(node.lastToken()); try renderToken(tree, stream, comma_token, indent, start_col, Space.Newline); // , - try renderExtraNewline(tree, stream, start_col, next_node.data); + try renderExtraNewline(tree, stream, start_col, items[i + 1]); try stream.writeByteNTimes(' ', indent); } else { try renderExpression(allocator, stream, tree, indent, start_col, node, Space.Comma); From 19de25993654f3e089eaf46391612e8a7e406d7e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 21 May 2020 22:52:45 -0400 Subject: [PATCH 21/38] stage2 parser: arrays and structs directly in memory after the node Slightly reduced memory usage. Roughly the same API and perf. --- lib/std/zig/ast.zig | 128 +++++++++++++++++++++++++++++++++++++---- lib/std/zig/parse.zig | 24 ++++---- lib/std/zig/render.zig | 8 +-- 3 files changed, 134 insertions(+), 26 deletions(-) diff --git a/lib/std/zig/ast.zig b/lib/std/zig/ast.zig index 6e31555ca3..5a491efb79 100644 --- a/lib/std/zig/ast.zig +++ b/lib/std/zig/ast.zig @@ -2067,11 +2067,23 @@ pub const Node = struct { } }; + /// Elements occur directly in memory after ArrayInitializer. pub const ArrayInitializer = struct { base: Node = Node{ .id = .ArrayInitializer }, rtoken: TokenIndex, + list_len: NodeIndex, lhs: *Node, - list: []*Node, + + /// After this the caller must initialize the fields_and_decls list. + pub fn alloc(allocator: *mem.Allocator, list_len: NodeIndex) !*ArrayInitializer { + const bytes = try allocator.alignedAlloc(u8, @alignOf(ArrayInitializer), sizeInBytes(list_len)); + return @ptrCast(*ArrayInitializer, bytes.ptr); + } + + pub fn free(self: *ArrayInitializer, allocator: *mem.Allocator) void { + const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.list_len)]; + allocator.free(bytes); + } pub fn iterate(self: *const ArrayInitializer) Node.Iterator { return .{ .parent_node = &self.base, .index = 0, .node = null }; @@ -2084,8 +2096,8 @@ pub const Node = struct { if (i < 1) return self.lhs; i -= 1; - if (i < self.list.len) return self.list[i]; - i -= self.list.len; + if (i < self.list_len) return self.listConst()[i]; + i -= self.list_len; return null; } @@ -2097,13 +2109,39 @@ pub const Node = struct { pub fn lastToken(self: *const ArrayInitializer) TokenIndex { return self.rtoken; } + + pub fn list(self: *ArrayInitializer) []*Node { + const decls_start = @ptrCast([*]u8, self) + @sizeOf(ArrayInitializer); + return @ptrCast([*]*Node, decls_start)[0..self.list_len]; + } + + pub fn listConst(self: *const ArrayInitializer) []const *Node { + const decls_start = @ptrCast([*]const u8, self) + @sizeOf(ArrayInitializer); + return @ptrCast([*]const *Node, decls_start)[0..self.list_len]; + } + + fn sizeInBytes(list_len: NodeIndex) usize { + return @sizeOf(ArrayInitializer) + @sizeOf(*Node) * @as(usize, list_len); + } }; + /// Elements occur directly in memory after ArrayInitializerDot. pub const ArrayInitializerDot = struct { base: Node = Node{ .id = .ArrayInitializerDot }, dot: TokenIndex, rtoken: TokenIndex, - list: []*Node, + list_len: NodeIndex, + + /// After this the caller must initialize the fields_and_decls list. + pub fn alloc(allocator: *mem.Allocator, list_len: NodeIndex) !*ArrayInitializerDot { + const bytes = try allocator.alignedAlloc(u8, @alignOf(ArrayInitializerDot), sizeInBytes(list_len)); + return @ptrCast(*ArrayInitializerDot, bytes.ptr); + } + + pub fn free(self: *ArrayInitializerDot, allocator: *mem.Allocator) void { + const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.list_len)]; + allocator.free(bytes); + } pub fn iterate(self: *const ArrayInitializerDot) Node.Iterator { return .{ .parent_node = &self.base, .index = 0, .node = null }; @@ -2113,8 +2151,8 @@ pub const Node = struct { var i = it.index; it.index += 1; - if (i < self.list.len) return self.list[i]; - i -= self.list.len; + if (i < self.list_len) return self.listConst()[i]; + i -= self.list_len; return null; } @@ -2126,13 +2164,39 @@ pub const Node = struct { pub fn lastToken(self: *const ArrayInitializerDot) TokenIndex { return self.rtoken; } + + pub fn list(self: *ArrayInitializerDot) []*Node { + const decls_start = @ptrCast([*]u8, self) + @sizeOf(ArrayInitializerDot); + return @ptrCast([*]*Node, decls_start)[0..self.list_len]; + } + + pub fn listConst(self: *const ArrayInitializerDot) []const *Node { + const decls_start = @ptrCast([*]const u8, self) + @sizeOf(ArrayInitializerDot); + return @ptrCast([*]const *Node, decls_start)[0..self.list_len]; + } + + fn sizeInBytes(list_len: NodeIndex) usize { + return @sizeOf(ArrayInitializerDot) + @sizeOf(*Node) * @as(usize, list_len); + } }; + /// Elements occur directly in memory after StructInitializer. pub const StructInitializer = struct { base: Node = Node{ .id = .StructInitializer }, rtoken: TokenIndex, + list_len: NodeIndex, lhs: *Node, - list: []*Node, + + /// After this the caller must initialize the fields_and_decls list. + pub fn alloc(allocator: *mem.Allocator, list_len: NodeIndex) !*StructInitializer { + const bytes = try allocator.alignedAlloc(u8, @alignOf(StructInitializer), sizeInBytes(list_len)); + return @ptrCast(*StructInitializer, bytes.ptr); + } + + pub fn free(self: *StructInitializer, allocator: *mem.Allocator) void { + const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.list_len)]; + allocator.free(bytes); + } pub fn iterate(self: *const StructInitializer) Node.Iterator { return .{ .parent_node = &self.base, .index = 0, .node = null }; @@ -2145,8 +2209,8 @@ pub const Node = struct { if (i < 1) return self.lhs; i -= 1; - if (i < self.list.len) return self.list[i]; - i -= self.list.len; + if (i < self.list_len) return self.listConst()[i]; + i -= self.list_len; return null; } @@ -2158,13 +2222,39 @@ pub const Node = struct { pub fn lastToken(self: *const StructInitializer) TokenIndex { return self.rtoken; } + + pub fn list(self: *StructInitializer) []*Node { + const decls_start = @ptrCast([*]u8, self) + @sizeOf(StructInitializer); + return @ptrCast([*]*Node, decls_start)[0..self.list_len]; + } + + pub fn listConst(self: *const StructInitializer) []const *Node { + const decls_start = @ptrCast([*]const u8, self) + @sizeOf(StructInitializer); + return @ptrCast([*]const *Node, decls_start)[0..self.list_len]; + } + + fn sizeInBytes(list_len: NodeIndex) usize { + return @sizeOf(StructInitializer) + @sizeOf(*Node) * @as(usize, list_len); + } }; + /// Elements occur directly in memory after StructInitializerDot. pub const StructInitializerDot = struct { base: Node = Node{ .id = .StructInitializerDot }, dot: TokenIndex, rtoken: TokenIndex, - list: []*Node, + list_len: NodeIndex, + + /// After this the caller must initialize the fields_and_decls list. + pub fn alloc(allocator: *mem.Allocator, list_len: NodeIndex) !*StructInitializerDot { + const bytes = try allocator.alignedAlloc(u8, @alignOf(StructInitializerDot), sizeInBytes(list_len)); + return @ptrCast(*StructInitializerDot, bytes.ptr); + } + + pub fn free(self: *StructInitializerDot, allocator: *mem.Allocator) void { + const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.list_len)]; + allocator.free(bytes); + } pub fn iterate(self: *const StructInitializerDot) Node.Iterator { return .{ .parent_node = &self.base, .index = 0, .node = null }; @@ -2174,8 +2264,8 @@ pub const Node = struct { var i = it.index; it.index += 1; - if (i < self.list.len) return self.list[i]; - i -= self.list.len; + if (i < self.list_len) return self.listConst()[i]; + i -= self.list_len; return null; } @@ -2187,6 +2277,20 @@ pub const Node = struct { pub fn lastToken(self: *const StructInitializerDot) TokenIndex { return self.rtoken; } + + pub fn list(self: *StructInitializerDot) []*Node { + const decls_start = @ptrCast([*]u8, self) + @sizeOf(StructInitializerDot); + return @ptrCast([*]*Node, decls_start)[0..self.list_len]; + } + + pub fn listConst(self: *const StructInitializerDot) []const *Node { + const decls_start = @ptrCast([*]const u8, self) + @sizeOf(StructInitializerDot); + return @ptrCast([*]const *Node, decls_start)[0..self.list_len]; + } + + fn sizeInBytes(list_len: NodeIndex) usize { + return @sizeOf(StructInitializerDot) + @sizeOf(*Node) * @as(usize, list_len); + } }; /// Parameter nodes directly follow Call in memory. diff --git a/lib/std/zig/parse.zig b/lib/std/zig/parse.zig index 3c7f0cf06e..779c626002 100644 --- a/lib/std/zig/parse.zig +++ b/lib/std/zig/parse.zig @@ -1318,12 +1318,13 @@ const Parser = struct { const next = (try p.parseFieldInit()) orelse break; try init_list.append(next); } - const node = try p.arena.allocator.create(Node.StructInitializer); + const node = try Node.StructInitializer.alloc(&p.arena.allocator, init_list.items.len); node.* = .{ .lhs = lhs, .rtoken = try p.expectToken(.RBrace), - .list = try p.arena.allocator.dupe(*Node, init_list.items), + .list_len = init_list.items.len, }; + std.mem.copy(*Node, node.list(), init_list.items); return &node.base; } @@ -1333,12 +1334,13 @@ const Parser = struct { const next = (try p.parseExpr()) orelse break; try init_list.append(next); } - const node = try p.arena.allocator.create(Node.ArrayInitializer); + const node = try Node.ArrayInitializer.alloc(&p.arena.allocator, init_list.items.len); node.* = .{ .lhs = lhs, .rtoken = try p.expectToken(.RBrace), - .list = try p.arena.allocator.dupe(*Node, init_list.items), + .list_len = init_list.items.len, }; + std.mem.copy(*Node, node.list(), init_list.items); return &node.base; } @@ -1346,7 +1348,7 @@ const Parser = struct { node.* = .{ .lhs = lhs, .rtoken = try p.expectToken(.RBrace), - .list = &[0]*Node{}, + .list_len = 0, }; return &node.base; } @@ -1366,12 +1368,13 @@ const Parser = struct { const next = (try p.parseFieldInit()) orelse break; try init_list.append(next); } - const node = try p.arena.allocator.create(Node.StructInitializerDot); + const node = try Node.StructInitializerDot.alloc(&p.arena.allocator, init_list.items.len); node.* = .{ .dot = dot, .rtoken = try p.expectToken(.RBrace), - .list = try p.arena.allocator.dupe(*Node, init_list.items), + .list_len = init_list.items.len, }; + std.mem.copy(*Node, node.list(), init_list.items); return &node.base; } @@ -1381,12 +1384,13 @@ const Parser = struct { const next = (try p.parseExpr()) orelse break; try init_list.append(next); } - const node = try p.arena.allocator.create(Node.ArrayInitializerDot); + const node = try Node.ArrayInitializerDot.alloc(&p.arena.allocator, init_list.items.len); node.* = .{ .dot = dot, .rtoken = try p.expectToken(.RBrace), - .list = try p.arena.allocator.dupe(*Node, init_list.items), + .list_len = init_list.items.len, }; + std.mem.copy(*Node, node.list(), init_list.items); return &node.base; } @@ -1394,7 +1398,7 @@ const Parser = struct { node.* = .{ .dot = dot, .rtoken = try p.expectToken(.RBrace), - .list = &[0]*Node{}, + .list_len = 0, }; return &node.base; } diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index 97ce9617c3..f0688b44be 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -620,13 +620,13 @@ fn renderExpression( .ArrayInitializerDot => blk: { const casted = @fieldParentPtr(ast.Node.ArrayInitializerDot, "base", base); rtoken = casted.rtoken; - exprs = casted.list; + exprs = casted.list(); break :blk .{ .dot = casted.dot }; }, .ArrayInitializer => blk: { const casted = @fieldParentPtr(ast.Node.ArrayInitializer, "base", base); rtoken = casted.rtoken; - exprs = casted.list; + exprs = casted.list(); break :blk .{ .node = casted.lhs }; }, else => unreachable, @@ -784,13 +784,13 @@ fn renderExpression( .StructInitializerDot => blk: { const casted = @fieldParentPtr(ast.Node.StructInitializerDot, "base", base); rtoken = casted.rtoken; - field_inits = casted.list; + field_inits = casted.list(); break :blk .{ .dot = casted.dot }; }, .StructInitializer => blk: { const casted = @fieldParentPtr(ast.Node.StructInitializer, "base", base); rtoken = casted.rtoken; - field_inits = casted.list; + field_inits = casted.list(); break :blk .{ .node = casted.lhs }; }, else => unreachable, From 8252c8b9d698ca671413ad4e23c5228dcf505632 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 21 May 2020 23:24:31 -0400 Subject: [PATCH 22/38] stage2 parser: different multiline string literal parsing strategy and using flat memory rather than singly linked list roughly equivalent performance, slightly reduced memory usage, better API. --- lib/std/zig/ast.zig | 163 +++++++++++++++++++++++------------------ lib/std/zig/parse.zig | 35 +++++++-- lib/std/zig/render.zig | 4 +- 3 files changed, 120 insertions(+), 82 deletions(-) diff --git a/lib/std/zig/ast.zig b/lib/std/zig/ast.zig index 5a491efb79..eff461fdd8 100644 --- a/lib/std/zig/ast.zig +++ b/lib/std/zig/ast.zig @@ -401,7 +401,6 @@ pub const Node = struct { /// All the child Node types use this same Iterator state for their iteration. pub const Iterator = struct { parent_node: *const Node, - node: ?*LinkedList(*Node).Node, index: usize, pub fn next(it: *Iterator) ?*Node { @@ -648,7 +647,7 @@ pub const Node = struct { } pub fn iterate(self: *const Root) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const Root, it: *Node.Iterator) ?*Node { @@ -702,7 +701,7 @@ pub const Node = struct { semicolon_token: TokenIndex, pub fn iterate(self: *const VarDecl) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const VarDecl, it: *Node.Iterator) ?*Node { @@ -756,7 +755,7 @@ pub const Node = struct { semicolon_token: TokenIndex, pub fn iterate(self: *const Use) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const Use, it: *Node.Iterator) ?*Node { @@ -797,13 +796,17 @@ pub const Node = struct { } pub fn iterate(self: *const ErrorSetDecl) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const ErrorSetDecl, it: *Node.Iterator) ?*Node { - const decl = it.node orelse return null; - it.node = decl.next; - return decl.data; + var i = it.index; + it.index += 1; + + if (i < self.decls_len) return self.declsConst()[i]; + i -= self.decls_len; + + return null; } pub fn firstToken(self: *const ErrorSetDecl) TokenIndex { @@ -857,7 +860,7 @@ pub const Node = struct { } pub fn iterate(self: *const ContainerDecl) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const ContainerDecl, it: *Node.Iterator) ?*Node { @@ -914,7 +917,7 @@ pub const Node = struct { align_expr: ?*Node, pub fn iterate(self: *const ContainerField) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const ContainerField, it: *Node.Iterator) ?*Node { @@ -966,7 +969,7 @@ pub const Node = struct { name_token: TokenIndex, pub fn iterate(self: *const ErrorTag) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const ErrorTag, it: *Node.Iterator) ?*Node { @@ -995,7 +998,7 @@ pub const Node = struct { token: TokenIndex, pub fn iterate(self: *const Identifier) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const Identifier, it: *Node.Iterator) ?*Node { @@ -1050,7 +1053,7 @@ pub const Node = struct { }; pub fn iterate(self: *const ParamDecl) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const ParamDecl, it: *Node.Iterator) ?*Node { @@ -1098,7 +1101,7 @@ pub const Node = struct { } pub fn iterate(self: *const FnProto) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const FnProto, it: *Node.Iterator) ?*Node { @@ -1189,7 +1192,7 @@ pub const Node = struct { }; pub fn iterate(self: *const AnyFrameType) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const AnyFrameType, it: *Node.Iterator) ?*Node { @@ -1234,7 +1237,7 @@ pub const Node = struct { } pub fn iterate(self: *const Block) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const Block, it: *Node.Iterator) ?*Node { @@ -1281,7 +1284,7 @@ pub const Node = struct { expr: *Node, pub fn iterate(self: *const Defer) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const Defer, it: *Node.Iterator) ?*Node { @@ -1310,7 +1313,7 @@ pub const Node = struct { expr: *Node, pub fn iterate(self: *const Comptime) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const Comptime, it: *Node.Iterator) ?*Node { @@ -1338,7 +1341,7 @@ pub const Node = struct { expr: *Node, pub fn iterate(self: *const Nosuspend) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const Nosuspend, it: *Node.Iterator) ?*Node { @@ -1367,7 +1370,7 @@ pub const Node = struct { rpipe: TokenIndex, pub fn iterate(self: *const Payload) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const Payload, it: *Node.Iterator) ?*Node { @@ -1397,7 +1400,7 @@ pub const Node = struct { rpipe: TokenIndex, pub fn iterate(self: *const PointerPayload) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const PointerPayload, it: *Node.Iterator) ?*Node { @@ -1428,7 +1431,7 @@ pub const Node = struct { rpipe: TokenIndex, pub fn iterate(self: *const PointerIndexPayload) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const PointerIndexPayload, it: *Node.Iterator) ?*Node { @@ -1462,7 +1465,7 @@ pub const Node = struct { body: *Node, pub fn iterate(self: *const Else) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const Else, it: *Node.Iterator) ?*Node { @@ -1510,7 +1513,7 @@ pub const Node = struct { } pub fn iterate(self: *const Switch) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const Switch, it: *Node.Iterator) ?*Node { @@ -1520,11 +1523,8 @@ pub const Node = struct { if (i < 1) return self.expr; i -= 1; - if (it.node) |child| { - it.index -= 1; - it.node = child.next; - return child.data; - } + if (i < self.cases_len) return self.casesConst()[i]; + i -= self.cases_len; return null; } @@ -1572,7 +1572,7 @@ pub const Node = struct { } pub fn iterate(self: *const SwitchCase) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const SwitchCase, it: *Node.Iterator) ?*Node { @@ -1621,7 +1621,7 @@ pub const Node = struct { token: TokenIndex, pub fn iterate(self: *const SwitchElse) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const SwitchElse, it: *Node.Iterator) ?*Node { @@ -1649,7 +1649,7 @@ pub const Node = struct { @"else": ?*Else, pub fn iterate(self: *const While) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const While, it: *Node.Iterator) ?*Node { @@ -1712,7 +1712,7 @@ pub const Node = struct { @"else": ?*Else, pub fn iterate(self: *const For) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const For, it: *Node.Iterator) ?*Node { @@ -1766,7 +1766,7 @@ pub const Node = struct { @"else": ?*Else, pub fn iterate(self: *const If) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const If, it: *Node.Iterator) ?*Node { @@ -1859,7 +1859,7 @@ pub const Node = struct { }; pub fn iterate(self: *const InfixOp) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const InfixOp, it: *Node.Iterator) ?*Node { @@ -1982,7 +1982,7 @@ pub const Node = struct { }; pub fn iterate(self: *const PrefixOp) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const PrefixOp, it: *Node.Iterator) ?*Node { @@ -2045,7 +2045,7 @@ pub const Node = struct { expr: *Node, pub fn iterate(self: *const FieldInitializer) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const FieldInitializer, it: *Node.Iterator) ?*Node { @@ -2086,7 +2086,7 @@ pub const Node = struct { } pub fn iterate(self: *const ArrayInitializer) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const ArrayInitializer, it: *Node.Iterator) ?*Node { @@ -2144,7 +2144,7 @@ pub const Node = struct { } pub fn iterate(self: *const ArrayInitializerDot) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const ArrayInitializerDot, it: *Node.Iterator) ?*Node { @@ -2199,7 +2199,7 @@ pub const Node = struct { } pub fn iterate(self: *const StructInitializer) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const StructInitializer, it: *Node.Iterator) ?*Node { @@ -2257,7 +2257,7 @@ pub const Node = struct { } pub fn iterate(self: *const StructInitializerDot) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const StructInitializerDot, it: *Node.Iterator) ?*Node { @@ -2313,7 +2313,7 @@ pub const Node = struct { } pub fn iterate(self: *const Call) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null}; + return .{ .parent_node = &self.base, .index = 0}; } pub fn iterateNext(self: *const Call, it: *Node.Iterator) ?*Node { @@ -2373,7 +2373,7 @@ pub const Node = struct { }; pub fn iterate(self: *const SuffixOp) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null}; + return .{ .parent_node = &self.base, .index = 0}; } pub fn iterateNext(self: *const SuffixOp, it: *Node.Iterator) ?*Node { @@ -2425,7 +2425,7 @@ pub const Node = struct { rparen: TokenIndex, pub fn iterate(self: *const GroupedExpression) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const GroupedExpression, it: *Node.Iterator) ?*Node { @@ -2460,7 +2460,7 @@ pub const Node = struct { }; pub fn iterate(self: *const ControlFlowExpression) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const ControlFlowExpression, it: *Node.Iterator) ?*Node { @@ -2513,7 +2513,7 @@ pub const Node = struct { body: ?*Node, pub fn iterate(self: *const Suspend) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const Suspend, it: *Node.Iterator) ?*Node { @@ -2546,7 +2546,7 @@ pub const Node = struct { token: TokenIndex, pub fn iterate(self: *const IntegerLiteral) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const IntegerLiteral, it: *Node.Iterator) ?*Node { @@ -2568,7 +2568,7 @@ pub const Node = struct { name: TokenIndex, pub fn iterate(self: *const EnumLiteral) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const EnumLiteral, it: *Node.Iterator) ?*Node { @@ -2589,7 +2589,7 @@ pub const Node = struct { token: TokenIndex, pub fn iterate(self: *const FloatLiteral) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const FloatLiteral, it: *Node.Iterator) ?*Node { @@ -2624,7 +2624,7 @@ pub const Node = struct { } pub fn iterate(self: *const BuiltinCall) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const BuiltinCall, it: *Node.Iterator) ?*Node { @@ -2665,7 +2665,7 @@ pub const Node = struct { token: TokenIndex, pub fn iterate(self: *const StringLiteral) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const StringLiteral, it: *Node.Iterator) ?*Node { @@ -2681,14 +2681,24 @@ pub const Node = struct { } }; + /// The string literal tokens appear directly in memory after MultilineStringLiteral. pub const MultilineStringLiteral = struct { base: Node = Node{ .id = .MultilineStringLiteral }, - lines: LineList, + lines_len: TokenIndex, - pub const LineList = LinkedList(TokenIndex); + /// After this the caller must initialize the lines list. + pub fn alloc(allocator: *mem.Allocator, lines_len: NodeIndex) !*MultilineStringLiteral { + const bytes = try allocator.alignedAlloc(u8, @alignOf(MultilineStringLiteral), sizeInBytes(lines_len)); + return @ptrCast(*MultilineStringLiteral, bytes.ptr); + } + + pub fn free(self: *MultilineStringLiteral, allocator: *mem.Allocator) void { + const bytes = @ptrCast([*]u8, self)[0..sizeInBytes(self.lines_len)]; + allocator.free(bytes); + } pub fn iterate(self: *const MultilineStringLiteral) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const MultilineStringLiteral, it: *Node.Iterator) ?*Node { @@ -2696,14 +2706,25 @@ pub const Node = struct { } pub fn firstToken(self: *const MultilineStringLiteral) TokenIndex { - return self.lines.first.?.data; + return self.linesConst()[0]; } pub fn lastToken(self: *const MultilineStringLiteral) TokenIndex { - var node = self.lines.first.?; - while (true) { - node = node.next orelse return node.data; - } + return self.linesConst()[self.lines_len - 1]; + } + + pub fn lines(self: *MultilineStringLiteral) []TokenIndex { + const decls_start = @ptrCast([*]u8, self) + @sizeOf(MultilineStringLiteral); + return @ptrCast([*]TokenIndex, decls_start)[0..self.lines_len]; + } + + pub fn linesConst(self: *const MultilineStringLiteral) []const TokenIndex { + const decls_start = @ptrCast([*]const u8, self) + @sizeOf(MultilineStringLiteral); + return @ptrCast([*]const TokenIndex, decls_start)[0..self.lines_len]; + } + + fn sizeInBytes(lines_len: NodeIndex) usize { + return @sizeOf(MultilineStringLiteral) + @sizeOf(TokenIndex) * @as(usize, lines_len); } }; @@ -2712,7 +2733,7 @@ pub const Node = struct { token: TokenIndex, pub fn iterate(self: *const CharLiteral) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const CharLiteral, it: *Node.Iterator) ?*Node { @@ -2733,7 +2754,7 @@ pub const Node = struct { token: TokenIndex, pub fn iterate(self: *const BoolLiteral) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const BoolLiteral, it: *Node.Iterator) ?*Node { @@ -2754,7 +2775,7 @@ pub const Node = struct { token: TokenIndex, pub fn iterate(self: *const NullLiteral) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const NullLiteral, it: *Node.Iterator) ?*Node { @@ -2775,7 +2796,7 @@ pub const Node = struct { token: TokenIndex, pub fn iterate(self: *const UndefinedLiteral) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const UndefinedLiteral, it: *Node.Iterator) ?*Node { @@ -2815,7 +2836,7 @@ pub const Node = struct { }; pub fn iterate(self: *const Output) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const Output, it: *Node.Iterator) ?*Node { @@ -2859,7 +2880,7 @@ pub const Node = struct { rparen: TokenIndex, pub fn iterate(self: *const Input) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const Input, it: *Node.Iterator) ?*Node { @@ -2889,7 +2910,7 @@ pub const Node = struct { pub fn iterate(self: *const Asm) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null}; + return .{ .parent_node = &self.base, .index = 0}; } pub fn iterateNext(self: *const Asm, it: *Node.Iterator) ?*Node { @@ -2932,7 +2953,7 @@ pub const Node = struct { token: TokenIndex, pub fn iterate(self: *const Unreachable) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const Unreachable, it: *Node.Iterator) ?*Node { @@ -2953,7 +2974,7 @@ pub const Node = struct { token: TokenIndex, pub fn iterate(self: *const ErrorType) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const ErrorType, it: *Node.Iterator) ?*Node { @@ -2974,7 +2995,7 @@ pub const Node = struct { token: TokenIndex, pub fn iterate(self: *const VarType) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const VarType, it: *Node.Iterator) ?*Node { @@ -2997,7 +3018,7 @@ pub const Node = struct { pub const LineList = LinkedList(TokenIndex); pub fn iterate(self: *const DocComment) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const DocComment, it: *Node.Iterator) ?*Node { @@ -3024,7 +3045,7 @@ pub const Node = struct { body_node: *Node, pub fn iterate(self: *const TestDecl) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0, .node = null }; + return .{ .parent_node = &self.base, .index = 0 }; } pub fn iterateNext(self: *const TestDecl, it: *Node.Iterator) ?*Node { diff --git a/lib/std/zig/parse.zig b/lib/std/zig/parse.zig index 779c626002..818446efd6 100644 --- a/lib/std/zig/parse.zig +++ b/lib/std/zig/parse.zig @@ -3105,15 +3105,34 @@ const Parser = struct { if (try p.parseStringLiteralSingle()) |node| return node; if (p.eatToken(.MultilineStringLiteralLine)) |first_line| { - const node = try p.arena.allocator.create(Node.MultilineStringLiteral); - node.* = .{ - .lines = Node.MultilineStringLiteral.LineList{}, - }; - var lines_it = &node.lines.first; - lines_it = try p.llpush(TokenIndex, lines_it, first_line); - while (p.eatToken(.MultilineStringLiteralLine)) |line| - lines_it = try p.llpush(TokenIndex, lines_it, line); + const start_tok_i = p.tok_i; + var tok_i = start_tok_i; + var count: usize = 1; // including first_line + while (true) : (tok_i += 1) { + switch (p.tokens[tok_i].id) { + .LineComment => continue, + .MultilineStringLiteralLine => count += 1, + else => break, + } + } + const node = try Node.MultilineStringLiteral.alloc(&p.arena.allocator, count); + node.* = .{ .lines_len = count }; + const lines = node.lines(); + tok_i = start_tok_i; + lines[0] = first_line; + count = 1; + while (true) : (tok_i += 1) { + switch (p.tokens[tok_i].id) { + .LineComment => continue, + .MultilineStringLiteralLine => { + lines[count] = tok_i; + count += 1; + }, + else => break, + } + } + p.tok_i = tok_i; return &node.base; } diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index f0688b44be..236e645d43 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -1376,9 +1376,7 @@ fn renderExpression( skip_first_indent = false; } - var it = multiline_str_literal.lines.first; - while (it) |t_node| : (it = t_node.next) { - const t = t_node.data; + for (multiline_str_literal.lines()) |t| { if (!skip_first_indent) { try stream.writeByteNTimes(' ', indent + indent_delta); } From 1dac9e71b525cb14eb358eaf0bc45b1554f5bf55 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 22 May 2020 00:27:51 -0400 Subject: [PATCH 23/38] std.testing: fix a crash when printing diffs --- lib/std/testing.zig | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/lib/std/testing.zig b/lib/std/testing.zig index 34bebad043..2d136d56c9 100644 --- a/lib/std/testing.zig +++ b/lib/std/testing.zig @@ -320,10 +320,11 @@ fn printWithVisibleNewlines(source: []const u8) void { } fn printLine(line: []const u8) void { - switch (line[line.len - 1]) { + if (line.len != 0) switch (line[line.len - 1]) { ' ', '\t' => warn("{}⏎\n", .{line}), // Carriage return symbol, - else => warn("{}\n", .{line}), - } + else => {}, + }; + warn("{}\n", .{line}); } test "" { From 295bca9b5f397ff98e5a0162fcb2a9f5e0a3e35c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 22 May 2020 00:28:07 -0400 Subject: [PATCH 24/38] stage2 parser: don't append doc comments to the list The DocComment AST node now only points to the first doc comment token. API users are expected to iterate over the following tokens directly. After this commit there are no more linked lists in use in the self-hosted AST API. Performance impact is negligible. Memory usage slightly reduced. --- lib/std/zig/ast.zig | 17 +++++++------- lib/std/zig/parse.zig | 45 ++++++++++++------------------------ lib/std/zig/render.zig | 52 ++++++++++++++++++++++++++++++------------ 3 files changed, 60 insertions(+), 54 deletions(-) diff --git a/lib/std/zig/ast.zig b/lib/std/zig/ast.zig index eff461fdd8..eb22ac2330 100644 --- a/lib/std/zig/ast.zig +++ b/lib/std/zig/ast.zig @@ -1,7 +1,6 @@ const std = @import("../std.zig"); const assert = std.debug.assert; const testing = std.testing; -const LinkedList = std.SinglyLinkedList; const mem = std.mem; const Token = std.zig.Token; @@ -3013,9 +3012,10 @@ pub const Node = struct { pub const DocComment = struct { base: Node = Node{ .id = .DocComment }, - lines: LineList, - - pub const LineList = LinkedList(TokenIndex); + /// Points to the first doc comment token. API users are expected to iterate over the + /// tokens array, looking for more doc comments, ignoring line comments, and stopping + /// at the first other token. + first_line: TokenIndex, pub fn iterate(self: *const DocComment) Node.Iterator { return .{ .parent_node = &self.base, .index = 0 }; @@ -3026,14 +3026,13 @@ pub const Node = struct { } pub fn firstToken(self: *const DocComment) TokenIndex { - return self.lines.first.?.data; + return self.first_line; } + /// Returns the first doc comment line. Be careful, this may not be the desired behavior, + /// which would require the tokens array. pub fn lastToken(self: *const DocComment) TokenIndex { - var node = self.lines.first.?; - while (true) { - node = node.next orelse return node.data; - } + return self.first_line; } }; diff --git a/lib/std/zig/parse.zig b/lib/std/zig/parse.zig index 818446efd6..6d568cf5ec 100644 --- a/lib/std/zig/parse.zig +++ b/lib/std/zig/parse.zig @@ -58,6 +58,8 @@ const Parser = struct { arena: std.heap.ArenaAllocator, gpa: *Allocator, source: []const u8, + /// TODO: Optimization idea: have this be several arrays of the token fields rather + /// than an array of structs. tokens: []const Token, tok_i: TokenIndex, errors: std.ArrayListUnmanaged(AstError), @@ -367,20 +369,13 @@ const Parser = struct { /// Eat a multiline container doc comment fn parseContainerDocComments(p: *Parser) !?*Node { - var lines = Node.DocComment.LineList{}; - var lines_it: *?*Node.DocComment.LineList.Node = &lines.first; - - while (p.eatToken(.ContainerDocComment)) |line| { - lines_it = try p.llpush(TokenIndex, lines_it, line); + if (p.eatToken(.ContainerDocComment)) |first_line| { + while (p.eatToken(.ContainerDocComment)) |_| {} + const node = try p.arena.allocator.create(Node.DocComment); + node.* = .{ .first_line = first_line }; + return &node.base; } - - if (lines.first == null) return null; - - const node = try p.arena.allocator.create(Node.DocComment); - node.* = .{ - .lines = lines, - }; - return &node.base; + return null; } /// TestDecl <- KEYWORD_test STRINGLITERALSINGLE Block @@ -3210,20 +3205,13 @@ const Parser = struct { /// Eat a multiline doc comment fn parseDocComment(p: *Parser) !?*Node.DocComment { - var lines = Node.DocComment.LineList{}; - var lines_it = &lines.first; - - while (p.eatToken(.DocComment)) |line| { - lines_it = try p.llpush(TokenIndex, lines_it, line); + if (p.eatToken(.DocComment)) |first_line| { + while (p.eatToken(.DocComment)) |_| {} + const node = try p.arena.allocator.create(Node.DocComment); + node.* = .{ .first_line = first_line }; + return node; } - - if (lines.first == null) return null; - - const node = try p.arena.allocator.create(Node.DocComment); - node.* = .{ - .lines = lines, - }; - return node; + return null; } fn tokensOnSameLine(p: *Parser, token1: TokenIndex, token2: TokenIndex) bool { @@ -3234,11 +3222,8 @@ const Parser = struct { fn parseAppendedDocComment(p: *Parser, after_token: TokenIndex) !?*Node.DocComment { const comment_token = p.eatToken(.DocComment) orelse return null; if (p.tokensOnSameLine(after_token, comment_token)) { - var lines = Node.DocComment.LineList{}; - _ = try p.llpush(TokenIndex, &lines.first, comment_token); - const node = try p.arena.allocator.create(Node.DocComment); - node.* = .{ .lines = lines }; + node.* = .{ .first_line = comment_token }; return node; } p.putBackToken(comment_token); diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index 236e645d43..bc38cf4067 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -327,11 +327,18 @@ fn renderContainerDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree, .DocComment => { const comment = @fieldParentPtr(ast.Node.DocComment, "base", decl); - var it = comment.lines.first; - while (it) |node| : (it = node.next) { - try renderToken(tree, stream, node.data, indent, start_col, .Newline); - if (node.next != null) { + const kind = tree.tokens[comment.first_line].id; + try renderToken(tree, stream, comment.first_line, indent, start_col, .Newline); + var tok_i = comment.first_line + 1; + while (true) : (tok_i += 1) { + const tok_id = tree.tokens[tok_i].id; + if (tok_id == kind) { try stream.writeByteNTimes(' ', indent); + try renderToken(tree, stream, tok_i, indent, start_col, .Newline); + } else if (tok_id == .LineComment) { + continue; + } else { + break; } } }, @@ -2428,17 +2435,32 @@ fn renderDocComments( start_col: *usize, ) (@TypeOf(stream).Error || Error)!void { const comment = node.doc_comments orelse return; - var it = comment.lines.first; - const first_token = node.firstToken(); - while (it) |line_token_index_node| : (it = line_token_index_node.next) { - const line_token_index = line_token_index_node.data; - if (line_token_index < first_token) { - try renderToken(tree, stream, line_token_index, indent, start_col, Space.Newline); - try stream.writeByteNTimes(' ', indent); - } else { - try renderToken(tree, stream, line_token_index, indent, start_col, Space.NoComment); - try stream.writeAll("\n"); - try stream.writeByteNTimes(' ', indent); + return renderDocCommentsToken(tree, stream, comment, node.firstToken(), indent, start_col); +} + +fn renderDocCommentsToken( + tree: *ast.Tree, + stream: var, + comment: *ast.Node.DocComment, + first_token: ast.TokenIndex, + indent: usize, + start_col: *usize, +) (@TypeOf(stream).Error || Error)!void { + var tok_i = comment.first_line; + while (true) : (tok_i += 1) { + switch (tree.tokens[tok_i].id) { + .DocComment, .ContainerDocComment => { + if (comment.first_line < first_token) { + try renderToken(tree, stream, tok_i, indent, start_col, Space.Newline); + try stream.writeByteNTimes(' ', indent); + } else { + try renderToken(tree, stream, tok_i, indent, start_col, Space.NoComment); + try stream.writeAll("\n"); + try stream.writeByteNTimes(' ', indent); + } + }, + .LineComment => continue, + else => break, } } } From 8df0841d6ab964e2aec750a8ceeda51912ae448b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 22 May 2020 12:34:12 -0400 Subject: [PATCH 25/38] stage2 parser: token ids in their own array To prevent cache misses, token ids go in their own array, and the start/end offsets go in a different one. perf measurement before: 2,667,914 cache-misses:u 2,139,139,935 instructions:u 894,167,331 cycles:u perf measurement after: 1,757,723 cache-misses:u 2,069,932,298 instructions:u 858,105,570 cycles:u --- lib/std/zig/ast.zig | 50 +++++---- lib/std/zig/parse.zig | 168 +++++++++++++--------------- lib/std/zig/parser_test.zig | 2 +- lib/std/zig/render.zig | 188 +++++++++++++++++--------------- lib/std/zig/tokenizer.zig | 28 +++-- src-self-hosted/translate_c.zig | 21 ---- 6 files changed, 217 insertions(+), 240 deletions(-) diff --git a/lib/std/zig/ast.zig b/lib/std/zig/ast.zig index eb22ac2330..0b4d4f6d89 100644 --- a/lib/std/zig/ast.zig +++ b/lib/std/zig/ast.zig @@ -10,7 +10,8 @@ pub const NodeIndex = usize; pub const Tree = struct { /// Reference to externally-owned data. source: []const u8, - tokens: []const Token, + token_ids: []const Token.Id, + token_locs: []const Token.Loc, errors: []const Error, /// undefined on parse error (when errors field is not empty) root_node: *Node.Root, @@ -23,26 +24,27 @@ pub const Tree = struct { generated: bool = false, pub fn deinit(self: *Tree) void { - self.gpa.free(self.tokens); + self.gpa.free(self.token_ids); + self.gpa.free(self.token_locs); self.gpa.free(self.errors); self.arena.promote(self.gpa).deinit(); } pub fn renderError(self: *Tree, parse_error: *const Error, stream: var) !void { - return parse_error.render(self.tokens, stream); + return parse_error.render(self.token_ids, stream); } pub fn tokenSlice(self: *Tree, token_index: TokenIndex) []const u8 { - return self.tokenSlicePtr(self.tokens[token_index]); + return self.tokenSliceLoc(self.token_locs[token_index]); } - pub fn tokenSlicePtr(self: *Tree, token: Token) []const u8 { + pub fn tokenSliceLoc(self: *Tree, token: Token.Loc) []const u8 { return self.source[token.start..token.end]; } pub fn getNodeSource(self: *const Tree, node: *const Node) []const u8 { - const first_token = self.tokens[node.firstToken()]; - const last_token = self.tokens[node.lastToken()]; + const first_token = self.token_locs[node.firstToken()]; + const last_token = self.token_locs[node.lastToken()]; return self.source[first_token.start..last_token.end]; } @@ -54,7 +56,7 @@ pub const Tree = struct { }; /// Return the Location of the token relative to the offset specified by `start_index`. - pub fn tokenLocationPtr(self: *Tree, start_index: usize, token: Token) Location { + pub fn tokenLocationLoc(self: *Tree, start_index: usize, token: Token.Loc) Location { var loc = Location{ .line = 0, .column = 0, @@ -82,14 +84,14 @@ pub const Tree = struct { } pub fn tokenLocation(self: *Tree, start_index: usize, token_index: TokenIndex) Location { - return self.tokenLocationPtr(start_index, self.tokens[token_index]); + return self.tokenLocationLoc(start_index, self.token_locs[token_index]); } pub fn tokensOnSameLine(self: *Tree, token1_index: TokenIndex, token2_index: TokenIndex) bool { - return self.tokensOnSameLinePtr(self.tokens[token1_index], self.tokens[token2_index]); + return self.tokensOnSameLineLoc(self.token_locs[token1_index], self.token_locs[token2_index]); } - pub fn tokensOnSameLinePtr(self: *Tree, token1: Token, token2: Token) bool { + pub fn tokensOnSameLineLoc(self: *Tree, token1: Token.Loc, token2: Token.Loc) bool { return mem.indexOfScalar(u8, self.source[token1.end..token2.start], '\n') == null; } @@ -100,7 +102,7 @@ pub const Tree = struct { /// Skips over comments pub fn prevToken(self: *Tree, token_index: TokenIndex) TokenIndex { var index = token_index - 1; - while (self.tokens[index].id == Token.Id.LineComment) { + while (self.token_ids[index] == Token.Id.LineComment) { index -= 1; } return index; @@ -109,7 +111,7 @@ pub const Tree = struct { /// Skips over comments pub fn nextToken(self: *Tree, token_index: TokenIndex) TokenIndex { var index = token_index + 1; - while (self.tokens[index].id == Token.Id.LineComment) { + while (self.token_ids[index] == Token.Id.LineComment) { index += 1; } return index; @@ -166,7 +168,7 @@ pub const Error = union(enum) { DeclBetweenFields: DeclBetweenFields, InvalidAnd: InvalidAnd, - pub fn render(self: *const Error, tokens: []const Token, stream: var) !void { + pub fn render(self: *const Error, tokens: []const Token.Id, stream: var) !void { switch (self.*) { .InvalidToken => |*x| return x.render(tokens, stream), .ExpectedContainerMembers => |*x| return x.render(tokens, stream), @@ -321,7 +323,7 @@ pub const Error = union(enum) { pub const ExpectedCall = struct { node: *Node, - pub fn render(self: *const ExpectedCall, tokens: []const Token, stream: var) !void { + pub fn render(self: *const ExpectedCall, tokens: []const Token.Id, stream: var) !void { return stream.print("expected " ++ @tagName(Node.Id.Call) ++ ", found {}", .{ @tagName(self.node.id), }); @@ -331,7 +333,7 @@ pub const Error = union(enum) { pub const ExpectedCallOrFnProto = struct { node: *Node, - pub fn render(self: *const ExpectedCallOrFnProto, tokens: []const Token, stream: var) !void { + pub fn render(self: *const ExpectedCallOrFnProto, tokens: []const Token.Id, stream: var) !void { return stream.print("expected " ++ @tagName(Node.Id.Call) ++ " or " ++ @tagName(Node.Id.FnProto) ++ ", found {}", .{@tagName(self.node.id)}); } @@ -341,14 +343,14 @@ pub const Error = union(enum) { token: TokenIndex, expected_id: Token.Id, - pub fn render(self: *const ExpectedToken, tokens: []const Token, stream: var) !void { + pub fn render(self: *const ExpectedToken, tokens: []const Token.Id, stream: var) !void { const found_token = tokens[self.token]; - switch (found_token.id) { + switch (found_token) { .Invalid => { return stream.print("expected '{}', found invalid bytes", .{self.expected_id.symbol()}); }, else => { - const token_name = found_token.id.symbol(); + const token_name = found_token.symbol(); return stream.print("expected '{}', found '{}'", .{ self.expected_id.symbol(), token_name }); }, } @@ -359,11 +361,11 @@ pub const Error = union(enum) { token: TokenIndex, end_id: Token.Id, - pub fn render(self: *const ExpectedCommaOrEnd, tokens: []const Token, stream: var) !void { + pub fn render(self: *const ExpectedCommaOrEnd, tokens: []const Token.Id, stream: var) !void { const actual_token = tokens[self.token]; return stream.print("expected ',' or '{}', found '{}'", .{ self.end_id.symbol(), - actual_token.id.symbol(), + actual_token.symbol(), }); } }; @@ -374,9 +376,9 @@ pub const Error = union(enum) { token: TokenIndex, - pub fn render(self: *const ThisError, tokens: []const Token, stream: var) !void { + pub fn render(self: *const ThisError, tokens: []const Token.Id, stream: var) !void { const actual_token = tokens[self.token]; - return stream.print(msg, .{actual_token.id.symbol()}); + return stream.print(msg, .{actual_token.symbol()}); } }; } @@ -387,7 +389,7 @@ pub const Error = union(enum) { token: TokenIndex, - pub fn render(self: *const ThisError, tokens: []const Token, stream: var) !void { + pub fn render(self: *const ThisError, tokens: []const Token.Id, stream: var) !void { return stream.writeAll(msg); } }; diff --git a/lib/std/zig/parse.zig b/lib/std/zig/parse.zig index 6d568cf5ec..a8aadc5ef7 100644 --- a/lib/std/zig/parse.zig +++ b/lib/std/zig/parse.zig @@ -16,28 +16,32 @@ pub const Error = error{ParseError} || Allocator.Error; pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!*Tree { // TODO optimization idea: ensureCapacity on the tokens list and // then appendAssumeCapacity inside the loop. - var tokens = std.ArrayList(Token).init(gpa); - defer tokens.deinit(); + var token_ids = std.ArrayList(Token.Id).init(gpa); + defer token_ids.deinit(); + var token_locs = std.ArrayList(Token.Loc).init(gpa); + defer token_locs.deinit(); var tokenizer = std.zig.Tokenizer.init(source); while (true) { - const tree_token = try tokens.addOne(); - tree_token.* = tokenizer.next(); - if (tree_token.id == .Eof) break; + const token = tokenizer.next(); + try token_ids.append(token.id); + try token_locs.append(token.loc); + if (token.id == .Eof) break; } var parser: Parser = .{ .source = source, .arena = std.heap.ArenaAllocator.init(gpa), .gpa = gpa, - .tokens = tokens.items, + .token_ids = token_ids.items, + .token_locs = token_locs.items, .errors = .{}, .tok_i = 0, }; defer parser.errors.deinit(gpa); errdefer parser.arena.deinit(); - while (tokens.items[parser.tok_i].id == .LineComment) parser.tok_i += 1; + while (token_ids.items[parser.tok_i] == .LineComment) parser.tok_i += 1; const root_node = try parser.parseRoot(); @@ -45,7 +49,8 @@ pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!*Tree { tree.* = .{ .gpa = gpa, .source = source, - .tokens = tokens.toOwnedSlice(), + .token_ids = token_ids.toOwnedSlice(), + .token_locs = token_locs.toOwnedSlice(), .errors = parser.errors.toOwnedSlice(gpa), .root_node = root_node, .arena = parser.arena.state, @@ -58,9 +63,8 @@ const Parser = struct { arena: std.heap.ArenaAllocator, gpa: *Allocator, source: []const u8, - /// TODO: Optimization idea: have this be several arrays of the token fields rather - /// than an array of structs. - tokens: []const Token, + token_ids: []const Token.Id, + token_locs: []const Token.Loc, tok_i: TokenIndex, errors: std.ArrayListUnmanaged(AstError), @@ -80,19 +84,6 @@ const Parser = struct { return node; } - /// Helper function for appending elements to a singly linked list. - fn llpush( - p: *Parser, - comptime T: type, - it: *?*std.SinglyLinkedList(T).Node, - data: T, - ) !*?*std.SinglyLinkedList(T).Node { - const llnode = try p.arena.allocator.create(std.SinglyLinkedList(T).Node); - llnode.* = .{ .data = data }; - it.* = llnode; - return &llnode.next; - } - /// ContainerMembers /// <- TestDecl ContainerMembers /// / TopLevelComptime ContainerMembers @@ -228,7 +219,7 @@ const Parser = struct { // try to continue parsing const index = p.tok_i; p.findNextContainerMember(); - const next = p.tokens[p.tok_i].id; + const next = p.token_ids[p.tok_i]; switch (next) { .Eof => break, else => { @@ -257,7 +248,7 @@ const Parser = struct { }); } - const next = p.tokens[p.tok_i].id; + const next = p.token_ids[p.tok_i]; switch (next) { .Eof => break, .Keyword_comptime => { @@ -291,7 +282,7 @@ const Parser = struct { var level: u32 = 0; while (true) { const tok = p.nextToken(); - switch (tok.ptr.id) { + switch (p.token_ids[tok]) { // any of these can start a new top level declaration .Keyword_test, .Keyword_comptime, @@ -308,7 +299,7 @@ const Parser = struct { .Identifier, => { if (level == 0) { - p.putBackToken(tok.index); + p.putBackToken(tok); return; } }, @@ -325,13 +316,13 @@ const Parser = struct { .RBrace => { if (level == 0) { // end of container, exit - p.putBackToken(tok.index); + p.putBackToken(tok); return; } level -= 1; }, .Eof => { - p.putBackToken(tok.index); + p.putBackToken(tok); return; }, else => {}, @@ -344,11 +335,11 @@ const Parser = struct { var level: u32 = 0; while (true) { const tok = p.nextToken(); - switch (tok.ptr.id) { + switch (p.token_ids[tok]) { .LBrace => level += 1, .RBrace => { if (level == 0) { - p.putBackToken(tok.index); + p.putBackToken(tok); return; } level -= 1; @@ -359,7 +350,7 @@ const Parser = struct { } }, .Eof => { - p.putBackToken(tok.index); + p.putBackToken(tok); return; }, else => {}, @@ -454,8 +445,8 @@ const Parser = struct { } if (extern_export_inline_token) |token| { - if (p.tokens[token].id == .Keyword_inline or - p.tokens[token].id == .Keyword_noinline) + if (p.token_ids[token] == .Keyword_inline or + p.token_ids[token] == .Keyword_noinline) { try p.errors.append(p.gpa, .{ .ExpectedFn = .{ .token = p.tok_i }, @@ -722,7 +713,7 @@ const Parser = struct { const defer_token = p.eatToken(.Keyword_defer) orelse p.eatToken(.Keyword_errdefer); if (defer_token) |token| { - const payload = if (p.tokens[token].id == .Keyword_errdefer) + const payload = if (p.token_ids[token] == .Keyword_errdefer) try p.parsePayload() else null; @@ -2269,7 +2260,7 @@ const Parser = struct { /// / EQUAL fn parseAssignOp(p: *Parser) !?*Node { const token = p.nextToken(); - const op: Node.InfixOp.Op = switch (token.ptr.id) { + const op: Node.InfixOp.Op = switch (p.token_ids[token]) { .AsteriskEqual => .AssignMul, .SlashEqual => .AssignDiv, .PercentEqual => .AssignMod, @@ -2285,14 +2276,14 @@ const Parser = struct { .MinusPercentEqual => .AssignSubWrap, .Equal => .Assign, else => { - p.putBackToken(token.index); + p.putBackToken(token); return null; }, }; const node = try p.arena.allocator.create(Node.InfixOp); node.* = .{ - .op_token = token.index, + .op_token = token, .lhs = undefined, // set by caller .op = op, .rhs = undefined, // set by caller @@ -2309,7 +2300,7 @@ const Parser = struct { /// / RARROWEQUAL fn parseCompareOp(p: *Parser) !?*Node { const token = p.nextToken(); - const op: Node.InfixOp.Op = switch (token.ptr.id) { + const op: Node.InfixOp.Op = switch (p.token_ids[token]) { .EqualEqual => .EqualEqual, .BangEqual => .BangEqual, .AngleBracketLeft => .LessThan, @@ -2317,12 +2308,12 @@ const Parser = struct { .AngleBracketLeftEqual => .LessOrEqual, .AngleBracketRightEqual => .GreaterOrEqual, else => { - p.putBackToken(token.index); + p.putBackToken(token); return null; }, }; - return p.createInfixOp(token.index, op); + return p.createInfixOp(token, op); } /// BitwiseOp @@ -2333,19 +2324,19 @@ const Parser = struct { /// / KEYWORD_catch Payload? fn parseBitwiseOp(p: *Parser) !?*Node { const token = p.nextToken(); - const op: Node.InfixOp.Op = switch (token.ptr.id) { + const op: Node.InfixOp.Op = switch (p.token_ids[token]) { .Ampersand => .BitAnd, .Caret => .BitXor, .Pipe => .BitOr, .Keyword_orelse => .UnwrapOptional, .Keyword_catch => .{ .Catch = try p.parsePayload() }, else => { - p.putBackToken(token.index); + p.putBackToken(token); return null; }, }; - return p.createInfixOp(token.index, op); + return p.createInfixOp(token, op); } /// BitShiftOp @@ -2353,16 +2344,16 @@ const Parser = struct { /// / RARROW2 fn parseBitShiftOp(p: *Parser) !?*Node { const token = p.nextToken(); - const op: Node.InfixOp.Op = switch (token.ptr.id) { + const op: Node.InfixOp.Op = switch (p.token_ids[token]) { .AngleBracketAngleBracketLeft => .BitShiftLeft, .AngleBracketAngleBracketRight => .BitShiftRight, else => { - p.putBackToken(token.index); + p.putBackToken(token); return null; }, }; - return p.createInfixOp(token.index, op); + return p.createInfixOp(token, op); } /// AdditionOp @@ -2373,19 +2364,19 @@ const Parser = struct { /// / MINUSPERCENT fn parseAdditionOp(p: *Parser) !?*Node { const token = p.nextToken(); - const op: Node.InfixOp.Op = switch (token.ptr.id) { + const op: Node.InfixOp.Op = switch (p.token_ids[token]) { .Plus => .Add, .Minus => .Sub, .PlusPlus => .ArrayCat, .PlusPercent => .AddWrap, .MinusPercent => .SubWrap, else => { - p.putBackToken(token.index); + p.putBackToken(token); return null; }, }; - return p.createInfixOp(token.index, op); + return p.createInfixOp(token, op); } /// MultiplyOp @@ -2397,7 +2388,7 @@ const Parser = struct { /// / ASTERISKPERCENT fn parseMultiplyOp(p: *Parser) !?*Node { const token = p.nextToken(); - const op: Node.InfixOp.Op = switch (token.ptr.id) { + const op: Node.InfixOp.Op = switch (p.token_ids[token]) { .PipePipe => .MergeErrorSets, .Asterisk => .Mul, .Slash => .Div, @@ -2405,12 +2396,12 @@ const Parser = struct { .AsteriskAsterisk => .ArrayMult, .AsteriskPercent => .MulWrap, else => { - p.putBackToken(token.index); + p.putBackToken(token); return null; }, }; - return p.createInfixOp(token.index, op); + return p.createInfixOp(token, op); } /// PrefixOp @@ -2423,7 +2414,7 @@ const Parser = struct { /// / KEYWORD_await fn parsePrefixOp(p: *Parser) !?*Node { const token = p.nextToken(); - const op: Node.PrefixOp.Op = switch (token.ptr.id) { + const op: Node.PrefixOp.Op = switch (p.token_ids[token]) { .Bang => .BoolNot, .Minus => .Negation, .Tilde => .BitNot, @@ -2432,14 +2423,14 @@ const Parser = struct { .Keyword_try => .Try, .Keyword_await => .Await, else => { - p.putBackToken(token.index); + p.putBackToken(token); return null; }, }; const node = try p.arena.allocator.create(Node.PrefixOp); node.* = .{ - .op_token = token.index, + .op_token = token, .op = op, .rhs = undefined, // set by caller }; @@ -2493,7 +2484,7 @@ const Parser = struct { // If the token encountered was **, there will be two nodes instead of one. // The attributes should be applied to the rightmost operator. const prefix_op = node.cast(Node.PrefixOp).?; - var ptr_info = if (p.tokens[prefix_op.op_token].id == .AsteriskAsterisk) + var ptr_info = if (p.token_ids[prefix_op.op_token] == .AsteriskAsterisk) &prefix_op.rhs.cast(Node.PrefixOp).?.op.PtrType else &prefix_op.op.PtrType; @@ -2812,7 +2803,8 @@ const Parser = struct { return null; }; if (p.eatToken(.Identifier)) |ident| { - const token_slice = p.source[p.tokens[ident].start..p.tokens[ident].end]; + const token_loc = p.token_locs[ident]; + const token_slice = p.source[token_loc.start..token_loc.end]; if (!std.mem.eql(u8, token_slice, "c")) { p.putBackToken(ident); } else { @@ -2879,7 +2871,7 @@ const Parser = struct { fn parseContainerDeclType(p: *Parser) !?ContainerDeclType { const kind_token = p.nextToken(); - const init_arg_expr = switch (kind_token.ptr.id) { + const init_arg_expr = switch (p.token_ids[kind_token]) { .Keyword_struct => Node.ContainerDecl.InitArg{ .None = {} }, .Keyword_enum => blk: { if (p.eatToken(.LParen) != null) { @@ -2914,13 +2906,13 @@ const Parser = struct { break :blk Node.ContainerDecl.InitArg{ .None = {} }; }, else => { - p.putBackToken(kind_token.index); + p.putBackToken(kind_token); return null; }, }; return ContainerDeclType{ - .kind_token = kind_token.index, + .kind_token = kind_token, .init_arg_expr = init_arg_expr, }; } @@ -2973,7 +2965,7 @@ const Parser = struct { while (try nodeParseFn(p)) |item| { try list.append(item); - switch (p.tokens[p.tok_i].id) { + switch (p.token_ids[p.tok_i]) { .Comma => _ = p.nextToken(), // all possible delimiters .Colon, .RParen, .RBrace, .RBracket => break, @@ -2994,13 +2986,13 @@ const Parser = struct { fn SimpleBinOpParseFn(comptime token: Token.Id, comptime op: Node.InfixOp.Op) NodeParseFn { return struct { pub fn parse(p: *Parser) Error!?*Node { - const op_token = if (token == .Keyword_and) switch (p.tokens[p.tok_i].id) { - .Keyword_and => p.nextToken().index, + const op_token = if (token == .Keyword_and) switch (p.token_ids[p.tok_i]) { + .Keyword_and => p.nextToken(), .Invalid_ampersands => blk: { try p.errors.append(p.gpa, .{ .InvalidAnd = .{ .token = p.tok_i }, }); - break :blk p.nextToken().index; + break :blk p.nextToken(); }, else => return null, } else p.eatToken(token) orelse return null; @@ -3104,7 +3096,7 @@ const Parser = struct { var tok_i = start_tok_i; var count: usize = 1; // including first_line while (true) : (tok_i += 1) { - switch (p.tokens[tok_i].id) { + switch (p.token_ids[tok_i]) { .LineComment => continue, .MultilineStringLiteralLine => count += 1, else => break, @@ -3118,7 +3110,7 @@ const Parser = struct { lines[0] = first_line; count = 1; while (true) : (tok_i += 1) { - switch (p.tokens[tok_i].id) { + switch (p.token_ids[tok_i]) { .LineComment => continue, .MultilineStringLiteralLine => { lines[count] = tok_i; @@ -3215,7 +3207,7 @@ const Parser = struct { } fn tokensOnSameLine(p: *Parser, token1: TokenIndex, token2: TokenIndex) bool { - return std.mem.indexOfScalar(u8, p.source[p.tokens[token1].end..p.tokens[token2].start], '\n') == null; + return std.mem.indexOfScalar(u8, p.source[p.token_locs[token1].end..p.token_locs[token2].start], '\n') == null; } /// Eat a single-line doc comment on the same line as another node @@ -3239,7 +3231,7 @@ const Parser = struct { .PrefixOp => { var prefix_op = rightmost_op.cast(Node.PrefixOp).?; // If the token encountered was **, there will be two nodes - if (p.tokens[prefix_op.op_token].id == .AsteriskAsterisk) { + if (p.token_ids[prefix_op.op_token] == .AsteriskAsterisk) { rightmost_op = prefix_op.rhs; prefix_op = rightmost_op.cast(Node.PrefixOp).?; } @@ -3328,11 +3320,7 @@ const Parser = struct { } fn eatToken(p: *Parser, id: Token.Id) ?TokenIndex { - return if (p.eatAnnotatedToken(id)) |token| token.index else null; - } - - fn eatAnnotatedToken(p: *Parser, id: Token.Id) ?AnnotatedToken { - return if (p.tokens[p.tok_i].id == id) p.nextToken() else null; + return if (p.token_ids[p.tok_i] == id) p.nextToken() else null; } fn expectToken(p: *Parser, id: Token.Id) Error!TokenIndex { @@ -3341,29 +3329,25 @@ const Parser = struct { fn expectTokenRecoverable(p: *Parser, id: Token.Id) !?TokenIndex { const token = p.nextToken(); - if (token.ptr.id != id) { + if (p.token_ids[token] != id) { try p.errors.append(p.gpa, .{ - .ExpectedToken = .{ .token = token.index, .expected_id = id }, + .ExpectedToken = .{ .token = token, .expected_id = id }, }); // go back so that we can recover properly - p.putBackToken(token.index); + p.putBackToken(token); return null; } - return token.index; + return token; } - fn nextToken(p: *Parser) AnnotatedToken { - const result = AnnotatedToken{ - .index = p.tok_i, - .ptr = &p.tokens[p.tok_i], - }; + fn nextToken(p: *Parser) TokenIndex { + const result = p.tok_i; p.tok_i += 1; - assert(result.ptr.id != .LineComment); - if (p.tok_i >= p.tokens.len) return result; + assert(p.token_ids[result] != .LineComment); + if (p.tok_i >= p.token_ids.len) return result; while (true) { - const next_tok = p.tokens[p.tok_i]; - if (next_tok.id != .LineComment) return result; + if (p.token_ids[p.tok_i] != .LineComment) return result; p.tok_i += 1; } } @@ -3371,18 +3355,12 @@ const Parser = struct { fn putBackToken(p: *Parser, putting_back: TokenIndex) void { while (p.tok_i > 0) { p.tok_i -= 1; - const prev_tok = p.tokens[p.tok_i]; - if (prev_tok.id == .LineComment) continue; + if (p.token_ids[p.tok_i] == .LineComment) continue; assert(putting_back == p.tok_i); return; } } - const AnnotatedToken = struct { - index: TokenIndex, - ptr: *const Token, - }; - fn expectNode( p: *Parser, parseFn: NodeParseFn, diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index 32667a02fb..25fc48e46d 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -3181,7 +3181,7 @@ fn testParse(source: []const u8, allocator: *mem.Allocator, anything_changed: *b defer tree.deinit(); for (tree.errors) |*parse_error| { - const token = tree.tokens[parse_error.loc()]; + const token = tree.token_locs[parse_error.loc()]; const loc = tree.tokenLocation(0, parse_error.loc()); try stderr.print("(memory buffer):{}:{}: error: ", .{ loc.line + 1, loc.column + 1 }); try tree.renderError(parse_error, stderr); diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index bc38cf4067..690592bc17 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -68,11 +68,12 @@ fn renderRoot( tree: *ast.Tree, ) (@TypeOf(stream).Error || Error)!void { // render all the line comments at the beginning of the file - for (tree.tokens) |token, i| { - if (token.id != .LineComment) break; - try stream.print("{}\n", .{mem.trimRight(u8, tree.tokenSlicePtr(token), " ")}); - const next_token = &tree.tokens[i + 1]; - const loc = tree.tokenLocationPtr(token.end, next_token.*); + for (tree.token_ids) |token_id, i| { + if (token_id != .LineComment) break; + const token_loc = tree.token_locs[i]; + try stream.print("{}\n", .{mem.trimRight(u8, tree.tokenSliceLoc(token_loc), " ")}); + const next_token = tree.token_locs[i + 1]; + const loc = tree.tokenLocationLoc(token_loc.end, next_token); if (loc.line >= 2) { try stream.writeByte('\n'); } @@ -101,8 +102,8 @@ fn renderRoot( while (token_index != 0) { token_index -= 1; - const token = tree.tokens[token_index]; - switch (token.id) { + const token_id = tree.token_ids[token_index]; + switch (token_id) { .LineComment => {}, .DocComment => { copy_start_token_index = token_index; @@ -111,12 +112,13 @@ fn renderRoot( else => break, } - if (mem.eql(u8, mem.trim(u8, tree.tokenSlicePtr(token)[2..], " "), "zig fmt: off")) { + const token_loc = tree.token_locs[token_index]; + if (mem.eql(u8, mem.trim(u8, tree.tokenSliceLoc(token_loc)[2..], " "), "zig fmt: off")) { if (!found_fmt_directive) { fmt_active = false; found_fmt_directive = true; } - } else if (mem.eql(u8, mem.trim(u8, tree.tokenSlicePtr(token)[2..], " "), "zig fmt: on")) { + } else if (mem.eql(u8, mem.trim(u8, tree.tokenSliceLoc(token_loc)[2..], " "), "zig fmt: on")) { if (!found_fmt_directive) { fmt_active = true; found_fmt_directive = true; @@ -135,7 +137,7 @@ fn renderRoot( if (decl_i >= root_decls.len) { // If there's no next reformatted `decl`, just copy the // remaining input tokens and bail out. - const start = tree.tokens[copy_start_token_index].start; + const start = tree.token_locs[copy_start_token_index].start; try copyFixingWhitespace(stream, tree.source[start..]); return; } @@ -143,15 +145,16 @@ fn renderRoot( var decl_first_token_index = decl.firstToken(); while (token_index < decl_first_token_index) : (token_index += 1) { - const token = tree.tokens[token_index]; - switch (token.id) { + const token_id = tree.token_ids[token_index]; + switch (token_id) { .LineComment => {}, .Eof => unreachable, else => continue, } - if (mem.eql(u8, mem.trim(u8, tree.tokenSlicePtr(token)[2..], " "), "zig fmt: on")) { + const token_loc = tree.token_locs[token_index]; + if (mem.eql(u8, mem.trim(u8, tree.tokenSliceLoc(token_loc)[2..], " "), "zig fmt: on")) { fmt_active = true; - } else if (mem.eql(u8, mem.trim(u8, tree.tokenSlicePtr(token)[2..], " "), "zig fmt: off")) { + } else if (mem.eql(u8, mem.trim(u8, tree.tokenSliceLoc(token_loc)[2..], " "), "zig fmt: off")) { fmt_active = false; } } @@ -163,8 +166,8 @@ fn renderRoot( token_index = copy_end_token_index; while (token_index != 0) { token_index -= 1; - const token = tree.tokens[token_index]; - switch (token.id) { + const token_id = tree.token_ids[token_index]; + switch (token_id) { .LineComment => {}, .DocComment => { copy_end_token_index = token_index; @@ -174,8 +177,8 @@ fn renderRoot( } } - const start = tree.tokens[copy_start_token_index].start; - const end = tree.tokens[copy_end_token_index].start; + const start = tree.token_locs[copy_start_token_index].start; + const end = tree.token_locs[copy_end_token_index].start; try copyFixingWhitespace(stream, tree.source[start..end]); } @@ -194,13 +197,13 @@ fn renderExtraNewlineToken(tree: *ast.Tree, stream: var, start_col: *usize, firs var prev_token = first_token; if (prev_token == 0) return; var newline_threshold: usize = 2; - while (tree.tokens[prev_token - 1].id == .DocComment) { - if (tree.tokenLocation(tree.tokens[prev_token - 1].end, prev_token).line == 1) { + while (tree.token_ids[prev_token - 1] == .DocComment) { + if (tree.tokenLocation(tree.token_locs[prev_token - 1].end, prev_token).line == 1) { newline_threshold += 1; } prev_token -= 1; } - const prev_token_end = tree.tokens[prev_token - 1].end; + const prev_token_end = tree.token_locs[prev_token - 1].end; const loc = tree.tokenLocation(prev_token_end, first_token); if (loc.line >= newline_threshold) { try stream.writeByte('\n'); @@ -265,7 +268,7 @@ fn renderContainerDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree, const src_has_trailing_comma = blk: { const maybe_comma = tree.nextToken(field.lastToken()); - break :blk tree.tokens[maybe_comma].id == .Comma; + break :blk tree.token_ids[maybe_comma] == .Comma; }; // The trailing comma is emitted at the end, but if it's not present @@ -327,11 +330,11 @@ fn renderContainerDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree, .DocComment => { const comment = @fieldParentPtr(ast.Node.DocComment, "base", decl); - const kind = tree.tokens[comment.first_line].id; + const kind = tree.token_ids[comment.first_line]; try renderToken(tree, stream, comment.first_line, indent, start_col, .Newline); var tok_i = comment.first_line + 1; while (true) : (tok_i += 1) { - const tok_id = tree.tokens[tok_i].id; + const tok_id = tree.token_ids[tok_i]; if (tok_id == kind) { try stream.writeByteNTimes(' ', indent); try renderToken(tree, stream, tok_i, indent, start_col, .Newline); @@ -436,13 +439,13 @@ fn renderExpression( try renderExpression(allocator, stream, tree, indent, start_col, infix_op_node.lhs, op_space); const after_op_space = blk: { - const loc = tree.tokenLocation(tree.tokens[infix_op_node.op_token].end, tree.nextToken(infix_op_node.op_token)); + const loc = tree.tokenLocation(tree.token_locs[infix_op_node.op_token].end, tree.nextToken(infix_op_node.op_token)); break :blk if (loc.line == 0) op_space else Space.Newline; }; try renderToken(tree, stream, infix_op_node.op_token, indent, start_col, after_op_space); if (after_op_space == Space.Newline and - tree.tokens[tree.nextToken(infix_op_node.op_token)].id != .MultilineStringLiteralLine) + tree.token_ids[tree.nextToken(infix_op_node.op_token)] != .MultilineStringLiteralLine) { try stream.writeByteNTimes(' ', indent + indent_delta); start_col.* = indent + indent_delta; @@ -463,10 +466,10 @@ fn renderExpression( switch (prefix_op_node.op) { .PtrType => |ptr_info| { - const op_tok_id = tree.tokens[prefix_op_node.op_token].id; + const op_tok_id = tree.token_ids[prefix_op_node.op_token]; switch (op_tok_id) { .Asterisk, .AsteriskAsterisk => try stream.writeByte('*'), - .LBracket => if (tree.tokens[prefix_op_node.op_token + 2].id == .Identifier) + .LBracket => if (tree.token_ids[prefix_op_node.op_token + 2] == .Identifier) try stream.writeAll("[*c") else try stream.writeAll("[*"), @@ -578,8 +581,8 @@ fn renderExpression( try renderToken(tree, stream, lbracket, indent, start_col, Space.None); // [ - const starts_with_comment = tree.tokens[lbracket + 1].id == .LineComment; - const ends_with_comment = tree.tokens[rbracket - 1].id == .LineComment; + const starts_with_comment = tree.token_ids[lbracket + 1] == .LineComment; + const ends_with_comment = tree.token_ids[rbracket - 1] == .LineComment; const new_indent = if (ends_with_comment) indent + indent_delta else indent; const new_space = if (ends_with_comment) Space.Newline else Space.None; try renderExpression(allocator, stream, tree, new_indent, start_col, array_info.len_expr, new_space); @@ -653,7 +656,7 @@ fn renderExpression( return renderToken(tree, stream, rtoken, indent, start_col, space); } - if (exprs.len == 1 and tree.tokens[exprs[0].lastToken() + 1].id == .RBrace) { + if (exprs.len == 1 and tree.token_ids[exprs[0].lastToken() + 1] == .RBrace) { const expr = exprs[0]; switch (lhs) { .dot => |dot| try renderToken(tree, stream, dot, indent, start_col, Space.None), @@ -675,17 +678,17 @@ fn renderExpression( for (exprs) |expr, i| { if (i + 1 < exprs.len) { const expr_last_token = expr.lastToken() + 1; - const loc = tree.tokenLocation(tree.tokens[expr_last_token].end, exprs[i+1].firstToken()); + const loc = tree.tokenLocation(tree.token_locs[expr_last_token].end, exprs[i+1].firstToken()); if (loc.line != 0) break :blk count; count += 1; } else { const expr_last_token = expr.lastToken(); - const loc = tree.tokenLocation(tree.tokens[expr_last_token].end, rtoken); + const loc = tree.tokenLocation(tree.token_locs[expr_last_token].end, rtoken); if (loc.line == 0) { // all on one line const src_has_trailing_comma = trailblk: { const maybe_comma = tree.prevToken(rtoken); - break :trailblk tree.tokens[maybe_comma].id == .Comma; + break :trailblk tree.token_ids[maybe_comma] == .Comma; }; if (src_has_trailing_comma) { break :blk 1; // force row size 1 @@ -723,7 +726,7 @@ fn renderExpression( var new_indent = indent + indent_delta; - if (tree.tokens[tree.nextToken(lbrace)].id != .MultilineStringLiteralLine) { + if (tree.token_ids[tree.nextToken(lbrace)] != .MultilineStringLiteralLine) { try renderToken(tree, stream, lbrace, new_indent, start_col, Space.Newline); try stream.writeByteNTimes(' ', new_indent); } else { @@ -750,7 +753,7 @@ fn renderExpression( } col = 1; - if (tree.tokens[tree.nextToken(comma)].id != .MultilineStringLiteralLine) { + if (tree.token_ids[tree.nextToken(comma)] != .MultilineStringLiteralLine) { try renderToken(tree, stream, comma, new_indent, start_col, Space.Newline); // , } else { try renderToken(tree, stream, comma, new_indent, start_col, Space.None); // , @@ -819,11 +822,11 @@ fn renderExpression( const src_has_trailing_comma = blk: { const maybe_comma = tree.prevToken(rtoken); - break :blk tree.tokens[maybe_comma].id == .Comma; + break :blk tree.token_ids[maybe_comma] == .Comma; }; const src_same_line = blk: { - const loc = tree.tokenLocation(tree.tokens[lbrace].end, rtoken); + const loc = tree.tokenLocation(tree.token_locs[lbrace].end, rtoken); break :blk loc.line == 0; }; @@ -929,7 +932,7 @@ fn renderExpression( const src_has_trailing_comma = blk: { const maybe_comma = tree.prevToken(call.rtoken); - break :blk tree.tokens[maybe_comma].id == .Comma; + break :blk tree.token_ids[maybe_comma] == .Comma; }; if (src_has_trailing_comma) { @@ -983,8 +986,8 @@ fn renderExpression( try renderExpression(allocator, stream, tree, indent, start_col, suffix_op.lhs, Space.None); try renderToken(tree, stream, lbracket, indent, start_col, Space.None); // [ - const starts_with_comment = tree.tokens[lbracket + 1].id == .LineComment; - const ends_with_comment = tree.tokens[rbracket - 1].id == .LineComment; + const starts_with_comment = tree.token_ids[lbracket + 1] == .LineComment; + const ends_with_comment = tree.token_ids[rbracket - 1] == .LineComment; const new_indent = if (ends_with_comment) indent + indent_delta else indent; const new_space = if (ends_with_comment) Space.Newline else Space.None; try renderExpression(allocator, stream, tree, new_indent, start_col, index_expr, new_space); @@ -1226,9 +1229,9 @@ fn renderExpression( var maybe_comma = tree.prevToken(container_decl.lastToken()); // Doc comments for a field may also appear after the comma, eg. // field_name: T, // comment attached to field_name - if (tree.tokens[maybe_comma].id == .DocComment) + if (tree.token_ids[maybe_comma] == .DocComment) maybe_comma = tree.prevToken(maybe_comma); - break :blk tree.tokens[maybe_comma].id == .Comma; + break :blk tree.token_ids[maybe_comma] == .Comma; }; const fields_and_decls = container_decl.fieldsAndDecls(); @@ -1321,7 +1324,7 @@ fn renderExpression( const src_has_trailing_comma = blk: { const maybe_comma = tree.prevToken(err_set_decl.rbrace_token); - break :blk tree.tokens[maybe_comma].id == .Comma; + break :blk tree.token_ids[maybe_comma] == .Comma; }; if (src_has_trailing_comma) { @@ -1353,7 +1356,7 @@ fn renderExpression( try renderExpression(allocator, stream, tree, indent, start_col, node, Space.None); const comma_token = tree.nextToken(node.lastToken()); - assert(tree.tokens[comma_token].id == .Comma); + assert(tree.token_ids[comma_token] == .Comma); try renderToken(tree, stream, comma_token, indent, start_col, Space.Space); // , try renderExtraNewline(tree, stream, start_col, decls[i + 1]); } else { @@ -1378,7 +1381,7 @@ fn renderExpression( const multiline_str_literal = @fieldParentPtr(ast.Node.MultilineStringLiteral, "base", base); var skip_first_indent = true; - if (tree.tokens[multiline_str_literal.firstToken() - 1].id != .LineComment) { + if (tree.token_ids[multiline_str_literal.firstToken() - 1] != .LineComment) { try stream.print("\n", .{}); skip_first_indent = false; } @@ -1406,7 +1409,7 @@ fn renderExpression( if (builtin_call.params_len < 2) break :blk false; const last_node = builtin_call.params()[builtin_call.params_len - 1]; const maybe_comma = tree.nextToken(last_node.lastToken()); - break :blk tree.tokens[maybe_comma].id == .Comma; + break :blk tree.token_ids[maybe_comma] == .Comma; }; const lparen = tree.nextToken(builtin_call.builtin_token); @@ -1443,8 +1446,8 @@ fn renderExpression( const fn_proto = @fieldParentPtr(ast.Node.FnProto, "base", base); if (fn_proto.visib_token) |visib_token_index| { - const visib_token = tree.tokens[visib_token_index]; - assert(visib_token.id == .Keyword_pub or visib_token.id == .Keyword_export); + const visib_token = tree.token_ids[visib_token_index]; + assert(visib_token == .Keyword_pub or visib_token == .Keyword_export); try renderToken(tree, stream, visib_token_index, indent, start_col, Space.Space); // pub } @@ -1466,7 +1469,7 @@ fn renderExpression( try renderToken(tree, stream, fn_proto.fn_token, indent, start_col, Space.Space); // fn break :blk tree.nextToken(fn_proto.fn_token); }; - assert(tree.tokens[lparen].id == .LParen); + assert(tree.token_ids[lparen] == .LParen); const rparen = tree.prevToken( // the first token for the annotation expressions is the left @@ -1482,10 +1485,10 @@ fn renderExpression( .InferErrorSet => |node| tree.prevToken(node.firstToken()), .Invalid => unreachable, }); - assert(tree.tokens[rparen].id == .RParen); + assert(tree.token_ids[rparen] == .RParen); const src_params_trailing_comma = blk: { - const maybe_comma = tree.tokens[rparen - 1].id; + const maybe_comma = tree.token_ids[rparen - 1]; break :blk maybe_comma == .Comma or maybe_comma == .LineComment; }; @@ -1622,7 +1625,7 @@ fn renderExpression( const src_has_trailing_comma = blk: { const last_node = switch_case.items()[switch_case.items_len - 1]; const maybe_comma = tree.nextToken(last_node.lastToken()); - break :blk tree.tokens[maybe_comma].id == .Comma; + break :blk tree.token_ids[maybe_comma] == .Comma; }; if (switch_case.items_len == 1 or !src_has_trailing_comma) { @@ -1967,7 +1970,7 @@ fn renderExpression( try renderAsmOutput(allocator, stream, tree, indent_extra, start_col, asm_output, Space.Newline); try stream.writeByteNTimes(' ', indent_once); const comma_or_colon = tree.nextToken(asm_output.lastToken()); - break :blk switch (tree.tokens[comma_or_colon].id) { + break :blk switch (tree.token_ids[comma_or_colon]) { .Comma => tree.nextToken(comma_or_colon), else => comma_or_colon, }; @@ -2002,7 +2005,7 @@ fn renderExpression( try renderAsmInput(allocator, stream, tree, indent_extra, start_col, asm_input, Space.Newline); try stream.writeByteNTimes(' ', indent_once); const comma_or_colon = tree.nextToken(asm_input.lastToken()); - break :blk switch (tree.tokens[comma_or_colon].id) { + break :blk switch (tree.token_ids[comma_or_colon]) { .Comma => tree.nextToken(comma_or_colon), else => comma_or_colon, }; @@ -2205,7 +2208,7 @@ fn renderStatement( try renderExpression(allocator, stream, tree, indent, start_col, base, Space.None); const semicolon_index = tree.nextToken(base.lastToken()); - assert(tree.tokens[semicolon_index].id == .Semicolon); + assert(tree.token_ids[semicolon_index] == .Semicolon); try renderToken(tree, stream, semicolon_index, indent, start_col, Space.Newline); } else { try renderExpression(allocator, stream, tree, indent, start_col, base, Space.Newline); @@ -2243,22 +2246,25 @@ fn renderTokenOffset( return; } - var token = tree.tokens[token_index]; - try stream.writeAll(mem.trimRight(u8, tree.tokenSlicePtr(token)[token_skip_bytes..], " ")); + var token_loc = tree.token_locs[token_index]; + try stream.writeAll(mem.trimRight(u8, tree.tokenSliceLoc(token_loc)[token_skip_bytes..], " ")); if (space == Space.NoComment) return; - var next_token = tree.tokens[token_index + 1]; + var next_token_id = tree.token_ids[token_index + 1]; + var next_token_loc = tree.token_locs[token_index + 1]; - if (space == Space.Comma) switch (next_token.id) { + if (space == Space.Comma) switch (next_token_id) { .Comma => return renderToken(tree, stream, token_index + 1, indent, start_col, Space.Newline), .LineComment => { try stream.writeAll(", "); return renderToken(tree, stream, token_index + 1, indent, start_col, Space.Newline); }, else => { - if (token_index + 2 < tree.tokens.len and tree.tokens[token_index + 2].id == .MultilineStringLiteralLine) { + if (token_index + 2 < tree.token_ids.len and + tree.token_ids[token_index + 2] == .MultilineStringLiteralLine) + { try stream.writeAll(","); return; } else { @@ -2271,19 +2277,20 @@ fn renderTokenOffset( // Skip over same line doc comments var offset: usize = 1; - if (next_token.id == .DocComment) { - const loc = tree.tokenLocationPtr(token.end, next_token); + if (next_token_id == .DocComment) { + const loc = tree.tokenLocationLoc(token_loc.end, next_token_loc); if (loc.line == 0) { offset += 1; - next_token = tree.tokens[token_index + offset]; + next_token_id = tree.token_ids[token_index + offset]; + next_token_loc = tree.token_locs[token_index + offset]; } } - if (next_token.id != .LineComment) blk: { + if (next_token_id != .LineComment) blk: { switch (space) { Space.None, Space.NoNewline => return, Space.Newline => { - if (next_token.id == .MultilineStringLiteralLine) { + if (next_token_id == .MultilineStringLiteralLine) { return; } else { try stream.writeAll("\n"); @@ -2292,7 +2299,7 @@ fn renderTokenOffset( } }, Space.Space, Space.SpaceOrOutdent => { - if (next_token.id == .MultilineStringLiteralLine) + if (next_token_id == .MultilineStringLiteralLine) return; try stream.writeByte(' '); return; @@ -2302,14 +2309,15 @@ fn renderTokenOffset( } while (true) { - const comment_is_empty = mem.trimRight(u8, tree.tokenSlicePtr(next_token), " ").len == 2; + const comment_is_empty = mem.trimRight(u8, tree.tokenSliceLoc(next_token_loc), " ").len == 2; if (comment_is_empty) { switch (space) { Space.Newline => { offset += 1; - token = next_token; - next_token = tree.tokens[token_index + offset]; - if (next_token.id != .LineComment) { + token_loc = next_token_loc; + next_token_id = tree.token_ids[token_index + offset]; + next_token_loc = tree.token_locs[token_index + offset]; + if (next_token_id != .LineComment) { try stream.writeByte('\n'); start_col.* = 0; return; @@ -2322,18 +2330,19 @@ fn renderTokenOffset( } } - var loc = tree.tokenLocationPtr(token.end, next_token); + var loc = tree.tokenLocationLoc(token_loc.end, next_token_loc); if (loc.line == 0) { - try stream.print(" {}", .{mem.trimRight(u8, tree.tokenSlicePtr(next_token), " ")}); + try stream.print(" {}", .{mem.trimRight(u8, tree.tokenSliceLoc(next_token_loc), " ")}); offset = 2; - token = next_token; - next_token = tree.tokens[token_index + offset]; - if (next_token.id != .LineComment) { + token_loc = next_token_loc; + next_token_loc = tree.token_locs[token_index + offset]; + next_token_id = tree.token_ids[token_index + offset]; + if (next_token_id != .LineComment) { switch (space) { Space.None, Space.Space => { try stream.writeByte('\n'); - const after_comment_token = tree.tokens[token_index + offset]; - const next_line_indent = switch (after_comment_token.id) { + const after_comment_token = tree.token_ids[token_index + offset]; + const next_line_indent = switch (after_comment_token) { .RParen, .RBrace, .RBracket => indent, else => indent + indent_delta, }; @@ -2346,7 +2355,7 @@ fn renderTokenOffset( start_col.* = indent; }, Space.Newline => { - if (next_token.id == .MultilineStringLiteralLine) { + if (next_token_id == .MultilineStringLiteralLine) { return; } else { try stream.writeAll("\n"); @@ -2359,7 +2368,7 @@ fn renderTokenOffset( } return; } - loc = tree.tokenLocationPtr(token.end, next_token); + loc = tree.tokenLocationLoc(token_loc.end, next_token_loc); } while (true) { @@ -2369,15 +2378,16 @@ fn renderTokenOffset( const newline_count = if (loc.line <= 1) @as(u8, 1) else @as(u8, 2); try stream.writeByteNTimes('\n', newline_count); try stream.writeByteNTimes(' ', indent); - try stream.writeAll(mem.trimRight(u8, tree.tokenSlicePtr(next_token), " ")); + try stream.writeAll(mem.trimRight(u8, tree.tokenSliceLoc(next_token_loc), " ")); offset += 1; - token = next_token; - next_token = tree.tokens[token_index + offset]; - if (next_token.id != .LineComment) { + token_loc = next_token_loc; + next_token_loc = tree.token_locs[token_index + offset]; + next_token_id = tree.token_ids[token_index + offset]; + if (next_token_id != .LineComment) { switch (space) { Space.Newline => { - if (next_token.id == .MultilineStringLiteralLine) { + if (next_token_id == .MultilineStringLiteralLine) { return; } else { try stream.writeAll("\n"); @@ -2388,8 +2398,8 @@ fn renderTokenOffset( Space.None, Space.Space => { try stream.writeByte('\n'); - const after_comment_token = tree.tokens[token_index + offset]; - const next_line_indent = switch (after_comment_token.id) { + const after_comment_token = tree.token_ids[token_index + offset]; + const next_line_indent = switch (after_comment_token) { .RParen, .RBrace, .RBracket => blk: { if (indent > indent_delta) { break :blk indent - indent_delta; @@ -2412,7 +2422,7 @@ fn renderTokenOffset( } return; } - loc = tree.tokenLocationPtr(token.end, next_token); + loc = tree.tokenLocationLoc(token_loc.end, next_token_loc); } } @@ -2448,7 +2458,7 @@ fn renderDocCommentsToken( ) (@TypeOf(stream).Error || Error)!void { var tok_i = comment.first_line; while (true) : (tok_i += 1) { - switch (tree.tokens[tok_i].id) { + switch (tree.token_ids[tok_i]) { .DocComment, .ContainerDocComment => { if (comment.first_line < first_token) { try renderToken(tree, stream, tok_i, indent, start_col, Space.Newline); diff --git a/lib/std/zig/tokenizer.zig b/lib/std/zig/tokenizer.zig index 160530f459..3ed57f621f 100644 --- a/lib/std/zig/tokenizer.zig +++ b/lib/std/zig/tokenizer.zig @@ -3,8 +3,12 @@ const mem = std.mem; pub const Token = struct { id: Id, - start: usize, - end: usize, + loc: Loc, + + pub const Loc = struct { + start: usize, + end: usize, + }; pub const Keyword = struct { bytes: []const u8, @@ -426,8 +430,10 @@ pub const Tokenizer = struct { var state: State = .start; var result = Token{ .id = .Eof, - .start = self.index, - .end = undefined, + .loc = .{ + .start = self.index, + .end = undefined, + }, }; var seen_escape_digits: usize = undefined; var remaining_code_units: usize = undefined; @@ -436,7 +442,7 @@ pub const Tokenizer = struct { switch (state) { .start => switch (c) { ' ', '\n', '\t', '\r' => { - result.start = self.index + 1; + result.loc.start = self.index + 1; }, '"' => { state = .string_literal; @@ -686,7 +692,7 @@ pub const Tokenizer = struct { .identifier => switch (c) { 'a'...'z', 'A'...'Z', '_', '0'...'9' => {}, else => { - if (Token.getKeyword(self.buffer[result.start..self.index])) |id| { + if (Token.getKeyword(self.buffer[result.loc.start..self.index])) |id| { result.id = id; } break; @@ -1313,7 +1319,7 @@ pub const Tokenizer = struct { => {}, .identifier => { - if (Token.getKeyword(self.buffer[result.start..self.index])) |id| { + if (Token.getKeyword(self.buffer[result.loc.start..self.index])) |id| { result.id = id; } }, @@ -1420,7 +1426,7 @@ pub const Tokenizer = struct { } } - result.end = self.index; + result.loc.end = self.index; return result; } @@ -1430,8 +1436,10 @@ pub const Tokenizer = struct { if (invalid_length == 0) return; self.pending_invalid_token = .{ .id = .Invalid, - .start = self.index, - .end = self.index + invalid_length, + .loc = .{ + .start = self.index, + .end = self.index + invalid_length, + }, }; } diff --git a/src-self-hosted/translate_c.zig b/src-self-hosted/translate_c.zig index e0c2b44f42..fca959d536 100644 --- a/src-self-hosted/translate_c.zig +++ b/src-self-hosted/translate_c.zig @@ -247,27 +247,6 @@ pub const Context = struct { } }; - /// Helper function to append items to a singly linked list. - fn llpusher(c: *Context, list: *std.SinglyLinkedList(*ast.Node)) LinkedListPusher { - assert(list.first == null); - return .{ - .c = c, - .it = &list.first, - }; - } - - fn llpush( - c: *Context, - comptime T: type, - it: *?*std.SinglyLinkedList(T).Node, - data: T, - ) !*?*std.SinglyLinkedList(T).Node { - const llnode = try c.arena.create(std.SinglyLinkedList(T).Node); - llnode.* = .{ .data = data }; - it.* = llnode; - return &llnode.next; - } - fn getMangle(c: *Context) u32 { c.mangle_count += 1; return c.mangle_count; From e072692e1ffb6012d3d75051996bfcbbbd80e093 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2020 12:48:17 -0400 Subject: [PATCH 26/38] update translate-c to new AST API --- src-self-hosted/clang.zig | 13 +- src-self-hosted/translate_c.zig | 980 +++++++++++++++++--------------- 2 files changed, 527 insertions(+), 466 deletions(-) diff --git a/src-self-hosted/clang.zig b/src-self-hosted/clang.zig index b3d67aab25..72818d4ccc 100644 --- a/src-self-hosted/clang.zig +++ b/src-self-hosted/clang.zig @@ -63,7 +63,7 @@ pub const struct_ZigClangReturnStmt = @Type(.Opaque); pub const struct_ZigClangSkipFunctionBodiesScope = @Type(.Opaque); pub const struct_ZigClangSourceManager = @Type(.Opaque); pub const struct_ZigClangSourceRange = @Type(.Opaque); -pub const struct_ZigClangStmt = @Type(.Opaque); +pub const ZigClangStmt = @Type(.Opaque); pub const struct_ZigClangStringLiteral = @Type(.Opaque); pub const struct_ZigClangStringRef = @Type(.Opaque); pub const struct_ZigClangSwitchStmt = @Type(.Opaque); @@ -842,9 +842,9 @@ pub extern fn ZigClangType_getTypeClassName(self: *const struct_ZigClangType) [* pub extern fn ZigClangType_getAsArrayTypeUnsafe(self: *const ZigClangType) *const ZigClangArrayType; pub extern fn ZigClangType_getAsRecordType(self: *const ZigClangType) ?*const ZigClangRecordType; pub extern fn ZigClangType_getAsUnionType(self: *const ZigClangType) ?*const ZigClangRecordType; -pub extern fn ZigClangStmt_getBeginLoc(self: *const struct_ZigClangStmt) struct_ZigClangSourceLocation; -pub extern fn ZigClangStmt_getStmtClass(self: ?*const struct_ZigClangStmt) ZigClangStmtClass; -pub extern fn ZigClangStmt_classof_Expr(self: ?*const struct_ZigClangStmt) bool; +pub extern fn ZigClangStmt_getBeginLoc(self: *const ZigClangStmt) struct_ZigClangSourceLocation; +pub extern fn ZigClangStmt_getStmtClass(self: ?*const ZigClangStmt) ZigClangStmtClass; +pub extern fn ZigClangStmt_classof_Expr(self: ?*const ZigClangStmt) bool; pub extern fn ZigClangExpr_getStmtClass(self: *const struct_ZigClangExpr) ZigClangStmtClass; pub extern fn ZigClangExpr_getType(self: *const struct_ZigClangExpr) struct_ZigClangQualType; pub extern fn ZigClangExpr_getBeginLoc(self: *const struct_ZigClangExpr) struct_ZigClangSourceLocation; @@ -873,7 +873,7 @@ pub extern fn ZigClangFunctionDecl_getLocation(self: *const ZigClangFunctionDecl pub extern fn ZigClangFunctionDecl_hasBody(self: *const ZigClangFunctionDecl) bool; pub extern fn ZigClangFunctionDecl_getStorageClass(self: *const ZigClangFunctionDecl) ZigClangStorageClass; pub extern fn ZigClangFunctionDecl_getParamDecl(self: *const ZigClangFunctionDecl, i: c_uint) *const struct_ZigClangParmVarDecl; -pub extern fn ZigClangFunctionDecl_getBody(self: *const ZigClangFunctionDecl) *const struct_ZigClangStmt; +pub extern fn ZigClangFunctionDecl_getBody(self: *const ZigClangFunctionDecl) *const ZigClangStmt; pub extern fn ZigClangFunctionDecl_doesDeclarationForceExternallyVisibleDefinition(self: *const ZigClangFunctionDecl) bool; pub extern fn ZigClangFunctionDecl_isThisDeclarationADefinition(self: *const ZigClangFunctionDecl) bool; pub extern fn ZigClangFunctionDecl_doesThisDeclarationHaveABody(self: *const ZigClangFunctionDecl) bool; @@ -959,7 +959,6 @@ pub const ZigClangReturnStmt = struct_ZigClangReturnStmt; pub const ZigClangSkipFunctionBodiesScope = struct_ZigClangSkipFunctionBodiesScope; pub const ZigClangSourceManager = struct_ZigClangSourceManager; pub const ZigClangSourceRange = struct_ZigClangSourceRange; -pub const ZigClangStmt = struct_ZigClangStmt; pub const ZigClangStringLiteral = struct_ZigClangStringLiteral; pub const ZigClangStringRef = struct_ZigClangStringRef; pub const ZigClangSwitchStmt = struct_ZigClangSwitchStmt; @@ -1018,7 +1017,7 @@ pub extern fn ZigClangLoadFromCommandLine( pub extern fn ZigClangDecl_getKind(decl: *const ZigClangDecl) ZigClangDeclKind; pub extern fn ZigClangDecl_getDeclKindName(decl: *const struct_ZigClangDecl) [*:0]const u8; -pub const ZigClangCompoundStmt_const_body_iterator = [*]const *struct_ZigClangStmt; +pub const ZigClangCompoundStmt_const_body_iterator = [*]const *ZigClangStmt; pub extern fn ZigClangCompoundStmt_body_begin(self: *const ZigClangCompoundStmt) ZigClangCompoundStmt_const_body_iterator; pub extern fn ZigClangCompoundStmt_body_end(self: *const ZigClangCompoundStmt) ZigClangCompoundStmt_const_body_iterator; diff --git a/src-self-hosted/translate_c.zig b/src-self-hosted/translate_c.zig index fca959d536..17b6c1d9e0 100644 --- a/src-self-hosted/translate_c.zig +++ b/src-self-hosted/translate_c.zig @@ -54,44 +54,79 @@ const Scope = struct { Loop, }; + /// Represents an in-progress ast.Node.Switch. This struct is stack-allocated. + /// When it is deinitialized, it produces an ast.Node.Switch which is allocated + /// into the main arena. const Switch = struct { base: Scope, - pending_block: *ast.Node.Block, - cases: *ast.Node.Switch.CaseList, + pending_block: *Block, + cases: []*ast.Node, + case_index: usize, has_default: bool = false, }; + /// Used for the scope of condition expressions, for example `if (cond)`. + /// The block is lazily initialised because it is only needed for rare + /// cases of comma operators being used. + const Condition = struct { + base: Scope, + block: ?Block = null, + + fn getBlockScope(self: *Condition, c: *Context) !*Block { + if (self.block) |*b| return b; + self.block = try Block.init(c, &self.base, "blk"); + return &self.block.?; + } + + fn deinit(self: *Condition) void { + if (self.block) |*b| b.deinit(); + } + }; + + /// Represents an in-progress ast.Node.Block. This struct is stack-allocated. + /// When it is deinitialized, it produces an ast.Node.Block which is allocated + /// into the main arena. const Block = struct { base: Scope, - block_node: *ast.Node.Block, - statements_it: *?*std.SinglyLinkedList(*ast.Node), + statements: std.ArrayList(*ast.Node), variables: AliasList, - label: ?[]const u8, + label: ?ast.TokenIndex, mangle_count: u32 = 0, + lbrace: ast.TokenIndex, - /// Don't forget to set rbrace token and block_node later - fn init(c: *Context, parent: *Scope, label: ?[]const u8) !*Block { - const block = try c.arena.create(Block); - block.* = .{ + fn init(c: *Context, parent: *Scope, label: ?[]const u8) !Block { + return Block{ .base = .{ .id = .Block, .parent = parent, }, - .block_node = undefined, - .statements_it = undefined, - .variables = AliasList.init(c.arena), - .label = label, + .statements = std.ArrayList(*ast.Node).init(c.gpa), + .variables = AliasList.init(c.gpa), + .label = if (self.label) |l| blk: { + const ll = try appendIdentifier(c, l); + _ = try appendToken(c, .Colon, ":"); + break :blk ll; + } else null, + .lbrace = try appendToken(c, .LBrace, "{"), }; - return block; } - fn pushStatement(self: *Block, c: *Context, stmt: *ast.Node) !void { - self.statements_it = c.llpush(*ast.Node, self.statements_it, stmt); + fn deinit(self: *Block) void { + self.statements.deinit(); + self.variables.deinit(); + self.* = undefined; } - fn setBlockNode(self: *Block, block: *ast.Node.Block) void { - self.block_node = block; - self.statements_it = &block.statements.first; + fn complete(self: *Block, c: *Context) !*ast.Node.Block { + const node = try ast.Node.Block.alloc(c.arena, self.statements.items.len); + node.* = .{ + .statements_len = self.statements.items.len, + .lbrace = self.lbrace, + .rbrace = try appendToken(c, .RBrace, "}"), + .label = self.label, + }; + mem.copy(*ast.Node, node.statements(), self.statements.items); + return node; } /// Given the desired name, return a name that does not shadow anything from outer scopes. @@ -102,12 +137,13 @@ const Scope = struct { scope.mangle_count += 1; proposed_name = try std.fmt.allocPrint(c.arena, "{}_{}", .{ name, scope.mangle_count }); } - try scope.variables.append(.{ .name = name, .alias = proposed_name }); + try scope.variables.push(.{ .name = name, .alias = proposed_name }); return proposed_name; } fn getAlias(scope: *Block, name: []const u8) []const u8 { - for (scope.variables.items) |p| { + var it = scope.variables.iterator(0); + while (it.next()) |p| { if (mem.eql(u8, p.name, name)) return p.alias; } @@ -115,7 +151,8 @@ const Scope = struct { } fn localContains(scope: *Block, name: []const u8) bool { - for (scope.variables.items) |p| { + var it = scope.variables.iterator(0); + while (it.next()) |p| { if (mem.eql(u8, p.name, name)) return true; } @@ -167,10 +204,7 @@ const Scope = struct { switch (scope.id) { .Root => unreachable, .Block => return @fieldParentPtr(Block, "base", scope), - .Condition => { - // comma operator used - return try Block.init(c, scope, "blk"); - }, + .Condition => return @fieldParentPtr(Condition, "base", scope).getBlockScope(c), else => scope = scope.parent.?, } } @@ -219,7 +253,8 @@ const Scope = struct { pub const Context = struct { gpa: *mem.Allocator, arena: *mem.Allocator, - tokens: std.ArrayListUnmanaged(Token), + token_ids: std.ArrayListUnmanaged(Token.Id), + token_locs: std.ArrayListUnmanaged(Token.Loc), errors: std.ArrayListUnmanaged(ast.Error), source_buffer: *std.ArrayList(u8), err: Error, @@ -237,16 +272,6 @@ pub const Context = struct { /// up front in a pre-processing step. global_names: std.StringHashMap(void), - /// Helper type to append elements to a singly linked list. - const LinkedListPusher = struct { - c: *Context, - it: *?*std.SinglyLinkedList(*ast.Node).Node, - - fn push(self: *LinkedListPusher, element: *ast.Node) !void { - self.it = try self.c.llpush(*ast.Node, self.it, element); - } - }; - fn getMangle(c: *Context) u32 { c.mangle_count += 1; return c.mangle_count; @@ -267,6 +292,46 @@ pub const Context = struct { const column = ZigClangSourceManager_getSpellingColumnNumber(c.source_manager, spelling_loc); return std.fmt.allocPrint(c.arena, "{}:{}:{}", .{ filename, line, column }); } + + fn createCall(c: *Context, fn_expr: *ast.Node, params_len: ast.NodeIndex) !*ast.Node.Call { + _ = try appendToken(c, .LParen, "("); + const node = try ast.Node.Call.alloc(c.arena, params_len); + node.* = .{ + .lhs = fn_expr, + .params_len = params_len, + .async_token = null, + .rtoken = undefined, // set after appending args + }; + return node; + } + + fn createBuiltinCall(c: *Context, name: []const u8, params_len: ast.NodeIndex) !*ast.Node.BuiltinCall { + const builtin_token = try appendToken(c, .Builtin, name); + _ = try appendToken(c, .LParen, "("); + const node = try ast.Node.BuiltinCall.alloc(c.arena, params_len); + node.* = .{ + .builtin_token = builtin_token, + .params_len = params_len, + .rparen_token = undefined, // set after appending args + }; + return node; + } + + fn createBlock(c: *Context, label: ?[]const u8, statements_len: ast.NodeIndex) !*ast.Node.Block { + const label_node = if (label) |l| blk: { + const ll = try appendIdentifier(c, l); + _ = try appendToken(c, .Colon, ":"); + break :blk ll; + } else null; + const block_node = try ast.Node.Block.alloc(c.arena, statements_len); + block_node.* = .{ + .label = label_node, + .lbrace = try appendToken(c, .LBrace, "{"), + .statements_len = statements_len, + .rbrace = undefined, + }; + return block_node; + } }; pub fn translate( @@ -307,14 +372,16 @@ pub fn translate( .global_scope = try arena.allocator.create(Scope.Root), .clang_context = ZigClangASTUnit_getASTContext(ast_unit).?, .global_names = std.StringHashMap(void).init(gpa), - .tokens = .{}, + .token_ids = .{}, + .token_locs = .{}, .errors = .{}, .root_decls = .{}, }; context.global_scope.* = Scope.Root.init(&context); defer context.decl_table.deinit(); defer context.alias_list.deinit(); - defer context.tokens.deinit(gpa); + defer context.token_ids.deinit(gpa); + defer context.token_locs.deinit(gpa); defer context.errors.deinit(gpa); defer context.global_names.deinit(); defer context.root_decls.deinit(gpa); @@ -349,7 +416,8 @@ pub fn translate( tree.* = .{ .gpa = gpa, .source = try arena.allocator.dupe(u8, source_buffer.items), - .tokens = context.tokens.toOwnedSlice(gpa), + .token_ids = context.token_ids.toOwnedSlice(gpa), + .token_locs = context.token_locs.toOwnedSlice(gpa), .errors = context.errors.toOwnedSlice(gpa), .root_node = root_node, .arena = arena.state, @@ -508,10 +576,9 @@ fn visitFnDecl(c: *Context, fn_decl: *const ZigClangFunctionDecl) Error!void { // actual function definition with body const body_stmt = ZigClangFunctionDecl_getBody(fn_decl); - const block_scope = try Scope.Block.init(rp.c, &c.global_scope.base, null); + var block_scope = try Scope.Block.init(rp.c, &c.global_scope.base, null); + defer block_scope.deinit(); var scope = &block_scope.base; - const block_node = try transCreateNodeBlock(rp.c, null); - block_scope.setBlockNode(block_node); var param_id: c_uint = 0; for (proto_node.params()) |param, i| { @@ -541,7 +608,7 @@ fn visitFnDecl(c: *Context, fn_decl: *const ZigClangFunctionDecl) Error!void { node.eq_token = try appendToken(c, .Equal, "="); node.init_node = try transCreateNodeIdentifier(c, arg_name); node.semicolon_token = try appendToken(c, .Semicolon, ";"); - block_scope.block_statements_it = try c.llpush(*ast.Node, block_scope.block_statements_it, &node.base); + try block_scope.statements.append(&node.base); param.name_token = try appendIdentifier(c, arg_name); _ = try appendToken(c, .Colon, ":"); } @@ -549,14 +616,14 @@ fn visitFnDecl(c: *Context, fn_decl: *const ZigClangFunctionDecl) Error!void { param_id += 1; } - transCompoundStmtInline(rp, &block_scope.base, @ptrCast(*const ZigClangCompoundStmt, body_stmt), block_node) catch |err| switch (err) { + const casted_body = @ptrCast(*const ZigClangCompoundStmt, body_stmt); + transCompoundStmtInline(rp, &block_scope.base, casted_body, &block_scope) catch |err| switch (err) { error.OutOfMemory => |e| return e, error.UnsupportedTranslation, error.UnsupportedType, => return failDecl(c, fn_decl_loc, fn_name, "unable to translate function", .{}), }; - block_node.rbrace = try appendToken(rp.c, .RBrace, "}"); - proto_node.body_node = &block_node.base; + proto_node.body_node = try block_scope.complete(rp.c); return addTopLevelDecl(c, fn_name, &proto_node.base); } @@ -999,7 +1066,7 @@ fn transEnumDecl(c: *Context, enum_decl: *const ZigClangEnumDecl) Error!?*ast.No // At this point we can rely on the enum emitting successfully. const tld_node = try transCreateNodeVarDecl(c, true, true, enum_val_name); tld_node.eq_token = try appendToken(c, .Equal, "="); - const cast_node = try transCreateNodeBuiltinFnCall(rp.c, "@enumToInt"); + const cast_node = try rp.c.createBuiltinCall("@enumToInt", 1); const enum_ident = try transCreateNodeIdentifier(c, name); const period_tok = try appendToken(c, .Period, "."); const field_ident = try transCreateNodeIdentifier(c, field_name); @@ -1010,7 +1077,7 @@ fn transEnumDecl(c: *Context, enum_decl: *const ZigClangEnumDecl) Error!?*ast.No .op = .Period, .rhs = field_ident, }; - try cast_node.params.push(&field_access_node.base); + cast_node.params()[0] = &field_access_node.base; cast_node.rparen_token = try appendToken(rp.c, .RParen, ")"); tld_node.init_node = &cast_node.base; tld_node.semicolon_token = try appendToken(c, .Semicolon, ";"); @@ -1104,7 +1171,7 @@ fn transStmt( .WhileStmtClass => return transWhileLoop(rp, scope, @ptrCast(*const ZigClangWhileStmt, stmt)), .DoStmtClass => return transDoWhileLoop(rp, scope, @ptrCast(*const ZigClangDoStmt, stmt)), .NullStmtClass => { - const block = try transCreateNodeBlock(rp.c, null); + const block = try rp.c.createBlock(null, 0); block.rbrace = try appendToken(rp.c, .RBrace, "}"); return &block.base; }, @@ -1170,27 +1237,23 @@ fn transBinaryOperator( .Comma => { const block_scope = try scope.findBlockScope(rp.c); const expr = block_scope.base.parent == scope; - const lparen = if (expr) blk: { - const l = try appendToken(rp.c, .LParen, "("); - block_scope.setBlockNode(try transCreateNodeBlock(rp.c, block_scope.label)); - break :blk l; - } else undefined; + const lparen = if (expr) try appendToken(rp.c, .LParen, "(") else undefined; const lhs = try transExpr(rp, &block_scope.base, ZigClangBinaryOperator_getLHS(stmt), .unused, .r_value); - try block_scope.statements.push(lhs); + try block_scope.statements.append(lhs); const rhs = try transExpr(rp, &block_scope.base, ZigClangBinaryOperator_getRHS(stmt), .used, .r_value); if (expr) { _ = try appendToken(rp.c, .Semicolon, ";"); const break_node = try transCreateNodeBreak(rp.c, block_scope.label); break_node.rhs = rhs; - try block_scope.block_node.statements.push(&break_node.base); - block_scope.block_node.rbrace = try appendToken(rp.c, .RBrace, "}"); + try block_scope.statements.append(&break_node.base); + const block_node = try block_scope.complete(rp.c); const rparen = try appendToken(rp.c, .RParen, ")"); const grouped_expr = try rp.c.arena.create(ast.Node.GroupedExpression); grouped_expr.* = .{ .lparen = lparen, - .expr = &block_scope.block_node.base, + .expr = &block_node.base, .rparen = rparen, }; return maybeSuppressResult(rp, scope, result_used, &grouped_expr.base); @@ -1201,11 +1264,11 @@ fn transBinaryOperator( .Div => { if (cIsSignedInteger(qt)) { // signed integer division uses @divTrunc - const div_trunc_node = try transCreateNodeBuiltinFnCall(rp.c, "@divTrunc"); - try div_trunc_node.params.push(try transExpr(rp, scope, ZigClangBinaryOperator_getLHS(stmt), .used, .l_value)); + const div_trunc_node = try rp.c.createBuiltinCall("@divTrunc", 2); + div_trunc_node.params()[0] = try transExpr(rp, scope, ZigClangBinaryOperator_getLHS(stmt), .used, .l_value); _ = try appendToken(rp.c, .Comma, ","); const rhs = try transExpr(rp, scope, ZigClangBinaryOperator_getRHS(stmt), .used, .r_value); - try div_trunc_node.params.push(rhs); + div_trunc_node.params()[1] = rhs; div_trunc_node.rparen_token = try appendToken(rp.c, .RParen, ")"); return maybeSuppressResult(rp, scope, result_used, &div_trunc_node.base); } @@ -1213,11 +1276,11 @@ fn transBinaryOperator( .Rem => { if (cIsSignedInteger(qt)) { // signed integer division uses @rem - const rem_node = try transCreateNodeBuiltinFnCall(rp.c, "@rem"); - try rem_node.params.push(try transExpr(rp, scope, ZigClangBinaryOperator_getLHS(stmt), .used, .l_value)); + const rem_node = try rp.c.createBuiltinCall("@rem", 2); + rem_node.params()[0] = try transExpr(rp, scope, ZigClangBinaryOperator_getLHS(stmt), .used, .l_value); _ = try appendToken(rp.c, .Comma, ","); const rhs = try transExpr(rp, scope, ZigClangBinaryOperator_getRHS(stmt), .used, .r_value); - try rem_node.params.push(rhs); + rem_node.params()[1] = rhs; rem_node.rparen_token = try appendToken(rp.c, .RParen, ")"); return maybeSuppressResult(rp, scope, result_used, &rem_node.base); } @@ -1321,15 +1384,15 @@ fn transBinaryOperator( const rhs_node = try transExpr(rp, scope, ZigClangBinaryOperator_getRHS(stmt), .used, .r_value); const lhs = if (isBoolRes(lhs_node)) init: { - const cast_node = try transCreateNodeBuiltinFnCall(rp.c, "@boolToInt"); - try cast_node.params.push(lhs_node); + const cast_node = try rp.c.createBuiltinCall("@boolToInt", 1); + cast_node.params()[0] = lhs_node; cast_node.rparen_token = try appendToken(rp.c, .RParen, ")"); break :init &cast_node.base; } else lhs_node; const rhs = if (isBoolRes(rhs_node)) init: { - const cast_node = try transCreateNodeBuiltinFnCall(rp.c, "@boolToInt"); - try cast_node.params.push(rhs_node); + const cast_node = try rp.c.createBuiltinCall("@boolToInt", 1); + cast_node.params()[0] = rhs_node; cast_node.rparen_token = try appendToken(rp.c, .RParen, ")"); break :init &cast_node.base; } else rhs_node; @@ -1341,23 +1404,22 @@ fn transCompoundStmtInline( rp: RestorePoint, parent_scope: *Scope, stmt: *const ZigClangCompoundStmt, - block_node: *ast.Node.Block, + block: *Scope.Block, ) TransError!void { var it = ZigClangCompoundStmt_body_begin(stmt); const end_it = ZigClangCompoundStmt_body_end(stmt); while (it != end_it) : (it += 1) { const result = try transStmt(rp, parent_scope, it[0], .unused, .r_value); - if (result != &block_node.base) - try block_node.statements.push(result); + try block.statements.append(result); } } fn transCompoundStmt(rp: RestorePoint, scope: *Scope, stmt: *const ZigClangCompoundStmt) TransError!*ast.Node { - const block_scope = try Scope.Block.init(rp.c, scope, null); - block_scope.setBlockNode(try transCreateNodeBlock(rp.c, null)); - try transCompoundStmtInline(rp, &block_scope.base, stmt, block_scope.block_node); - block_scope.block_node.rbrace = try appendToken(rp.c, .RBrace, "}"); - return &block_scope.block_node.base; + var block_scope = try Scope.Block.init(rp.c, scope, null); + defer block_scope.deinit(); + try transCompoundStmtInline(rp, &block_scope.base, stmt, &block_scope); + const node = block_scope.complete(c); + return &node.base; } fn transCStyleCastExprClass( @@ -1411,14 +1473,14 @@ fn transDeclStmt(rp: RestorePoint, scope: *Scope, stmt: *const ZigClangDeclStmt) else try transCreateNodeUndefinedLiteral(c); if (!qualTypeIsBoolean(qual_type) and isBoolRes(init_node)) { - const builtin_node = try transCreateNodeBuiltinFnCall(rp.c, "@boolToInt"); - try builtin_node.params.push(init_node); + const builtin_node = try rp.c.createBuiltinCall("@boolToInt", 1); + builtin_node.params()[0] = init_node; builtin_node.rparen_token = try appendToken(rp.c, .RParen, ")"); init_node = &builtin_node.base; } node.init_node = init_node; node.semicolon_token = try appendToken(c, .Semicolon, ";"); - try block_scope.block_node.statements.push(&node.base); + try block_scope.statements.append(&node.base); }, .Typedef => { const typedef_decl = @ptrCast(*const ZigClangTypedefNameDecl, it[0]); @@ -1437,7 +1499,7 @@ fn transDeclStmt(rp: RestorePoint, scope: *Scope, stmt: *const ZigClangDeclStmt) }); } else { const node = (try transCreateNodeTypedef(rp, typedef_decl, false, mangled_name)) orelse return error.UnsupportedTranslation; - try block_scope.block_node.statements.push(&node.base); + try block_scope.statements.append(&node.base); } }, else => |kind| return revertAndWarn( @@ -1499,8 +1561,8 @@ fn transImplicitCastExpr( }, .PointerToBoolean => { // @ptrToInt(val) != 0 - const ptr_to_int = try transCreateNodeBuiltinFnCall(rp.c, "@ptrToInt"); - try ptr_to_int.params.push(try transExpr(rp, scope, sub_expr, .used, .r_value)); + const ptr_to_int = try rp.c.createBuiltinCall("@ptrToInt", 1); + ptr_to_int.params()[0] = try transExpr(rp, scope, sub_expr, .used, .r_value); ptr_to_int.rparen_token = try appendToken(rp.c, .RParen, ")"); const op_token = try appendToken(rp.c, .BangEqual, "!="); @@ -1724,13 +1786,11 @@ fn transIntegerLiteral( // @as(T, x) const expr_base = @ptrCast(*const ZigClangExpr, expr); - const as_node = try transCreateNodeBuiltinFnCall(rp.c, "@as"); + const as_node = try rp.c.createBuiltinCall("@as", 2); const ty_node = try transQualType(rp, ZigClangExpr_getType(expr_base), ZigClangExpr_getBeginLoc(expr_base)); - try as_node.params.push(ty_node); + as_node.params()[0] = ty_node; _ = try appendToken(rp.c, .Comma, ","); - - const int_lit_node = try transCreateNodeAPInt(rp.c, ZigClangAPValue_getInt(&eval_result.Val)); - try as_node.params.push(int_lit_node); + as_node.params()[1] = try transCreateNodeAPInt(rp.c, ZigClangAPValue_getInt(&eval_result.Val)); as_node.rparen_token = try appendToken(rp.c, .RParen, ")"); return maybeSuppressResult(rp, scope, result_used, &as_node.base); @@ -1840,35 +1900,35 @@ fn transCCast( // 2. Bit-cast to correct signed-ness // @bitCast(dest_type, intermediate_value) - const cast_node = try transCreateNodeBuiltinFnCall(rp.c, "@bitCast"); - try cast_node.params.push(try transQualType(rp, dst_type, loc)); + const cast_node = try rp.c.createBuiltinCall("@bitCast", 2); + cast_node.params()[0] = try transQualType(rp, dst_type, loc); _ = try appendToken(rp.c, .Comma, ","); switch (cIntTypeCmp(dst_type, src_type)) { .lt => { // @truncate(SameSignSmallerInt, src_type) - const trunc_node = try transCreateNodeBuiltinFnCall(rp.c, "@truncate"); + const trunc_node = try rp.c.createBuiltinCall("@truncate", 2); const ty_node = try transQualTypeIntWidthOf(rp.c, dst_type, cIsSignedInteger(src_type)); - try trunc_node.params.push(ty_node); + trunc_node.params()[0] = ty_node; _ = try appendToken(rp.c, .Comma, ","); - try trunc_node.params.push(expr); + trunc_node.params()[1] = expr; trunc_node.rparen_token = try appendToken(rp.c, .RParen, ")"); - try cast_node.params.push(&trunc_node.base); + cast_node.params()[1] = &trunc_node.base; }, .gt => { // @as(SameSignBiggerInt, src_type) - const as_node = try transCreateNodeBuiltinFnCall(rp.c, "@as"); + const as_node = try rp.c.createBuiltinCall("@as", 2); const ty_node = try transQualTypeIntWidthOf(rp.c, dst_type, cIsSignedInteger(src_type)); - try as_node.params.push(ty_node); + as_node.params()[0] = ty_node; _ = try appendToken(rp.c, .Comma, ","); - try as_node.params.push(expr); + as_node.params()[1] = expr; as_node.rparen_token = try appendToken(rp.c, .RParen, ")"); - try cast_node.params.push(&as_node.base); + cast_node.params()[1] = &as_node.base; }, .eq => { - try cast_node.params.push(expr); + cast_node.params()[1] = expr; }, } cast_node.rparen_token = try appendToken(rp.c, .RParen, ")"); @@ -1876,46 +1936,46 @@ fn transCCast( } if (cIsInteger(dst_type) and qualTypeIsPtr(src_type)) { // @intCast(dest_type, @ptrToInt(val)) - const cast_node = try transCreateNodeBuiltinFnCall(rp.c, "@intCast"); - try cast_node.params.push(try transQualType(rp, dst_type, loc)); + const cast_node = try rp.c.createBuiltinCall("@intCast", 2); + cast_node.params()[0] = try transQualType(rp, dst_type, loc); _ = try appendToken(rp.c, .Comma, ","); - const builtin_node = try transCreateNodeBuiltinFnCall(rp.c, "@ptrToInt"); - try builtin_node.params.push(expr); + const builtin_node = try rp.c.createBuiltinCall("@ptrToInt", 1); + builtin_node.params()[0] = expr; builtin_node.rparen_token = try appendToken(rp.c, .RParen, ")"); - try cast_node.params.push(&builtin_node.base); + cast_node.params()[1] = &builtin_node.base; cast_node.rparen_token = try appendToken(rp.c, .RParen, ")"); return &cast_node.base; } if (cIsInteger(src_type) and qualTypeIsPtr(dst_type)) { // @intToPtr(dest_type, val) - const builtin_node = try transCreateNodeBuiltinFnCall(rp.c, "@intToPtr"); - try builtin_node.params.push(try transQualType(rp, dst_type, loc)); + const builtin_node = try rp.c.createBuiltinCall("@intToPtr", 2); + builtin_node.params()[0] = try transQualType(rp, dst_type, loc); _ = try appendToken(rp.c, .Comma, ","); - try builtin_node.params.push(expr); + builtin_node.params()[1] = expr; builtin_node.rparen_token = try appendToken(rp.c, .RParen, ")"); return &builtin_node.base; } if (cIsFloating(src_type) and cIsFloating(dst_type)) { - const builtin_node = try transCreateNodeBuiltinFnCall(rp.c, "@floatCast"); - try builtin_node.params.push(try transQualType(rp, dst_type, loc)); + const builtin_node = try rp.c.createBuiltinCall("@floatCast", 2); + builtin_node.params()[0] = try transQualType(rp, dst_type, loc); _ = try appendToken(rp.c, .Comma, ","); - try builtin_node.params.push(expr); + builtin_node.params()[1] = expr; builtin_node.rparen_token = try appendToken(rp.c, .RParen, ")"); return &builtin_node.base; } if (cIsFloating(src_type) and !cIsFloating(dst_type)) { - const builtin_node = try transCreateNodeBuiltinFnCall(rp.c, "@floatToInt"); - try builtin_node.params.push(try transQualType(rp, dst_type, loc)); + const builtin_node = try rp.c.createBuiltinCall("@floatToInt", 2); + builtin_node.params()[0] = try transQualType(rp, dst_type, loc); _ = try appendToken(rp.c, .Comma, ","); - try builtin_node.params.push(expr); + builtin_node.params()[1] = expr; builtin_node.rparen_token = try appendToken(rp.c, .RParen, ")"); return &builtin_node.base; } if (!cIsFloating(src_type) and cIsFloating(dst_type)) { - const builtin_node = try transCreateNodeBuiltinFnCall(rp.c, "@intToFloat"); - try builtin_node.params.push(try transQualType(rp, dst_type, loc)); + const builtin_node = try rp.c.createBuiltinCall("@intToFloat", 2); + builtin_node.params()[0] = try transQualType(rp, dst_type, loc); _ = try appendToken(rp.c, .Comma, ","); - try builtin_node.params.push(expr); + builtin_node.params()[1] = expr; builtin_node.rparen_token = try appendToken(rp.c, .RParen, ")"); return &builtin_node.base; } @@ -1923,54 +1983,54 @@ fn transCCast( !ZigClangType_isBooleanType(qualTypeCanon(dst_type))) { // @boolToInt returns either a comptime_int or a u1 - const builtin_node = try transCreateNodeBuiltinFnCall(rp.c, "@boolToInt"); - try builtin_node.params.push(expr); + const builtin_node = try rp.c.createBuiltinCall("@boolToInt", 1); + builtin_node.params()[0] = expr; builtin_node.rparen_token = try appendToken(rp.c, .RParen, ")"); - const inner_cast_node = try transCreateNodeBuiltinFnCall(rp.c, "@intCast"); - try inner_cast_node.params.push(try transCreateNodeIdentifier(rp.c, "u1")); + const inner_cast_node = try rp.c.createBuiltinCall("@intCast", 2); + inner_cast_node.params()[0] = try transCreateNodeIdentifier(rp.c, "u1"); _ = try appendToken(rp.c, .Comma, ","); - try inner_cast_node.params.push(&builtin_node.base); + inner_cast_node.params()[1] = &builtin_node.base; inner_cast_node.rparen_token = try appendToken(rp.c, .RParen, ")"); - const cast_node = try transCreateNodeBuiltinFnCall(rp.c, "@intCast"); - try cast_node.params.push(try transQualType(rp, dst_type, loc)); + const cast_node = try rp.c.createBuiltinCall("@intCast", 2); + cast_node.params()[0] = try transQualType(rp, dst_type, loc); _ = try appendToken(rp.c, .Comma, ","); if (cIsSignedInteger(dst_type)) { - const bitcast_node = try transCreateNodeBuiltinFnCall(rp.c, "@bitCast"); - try bitcast_node.params.push(try transCreateNodeIdentifier(rp.c, "i1")); + const bitcast_node = try rp.c.createBuiltinCall("@bitCast", 2); + bitcast_node.params()[0] = try transCreateNodeIdentifier(rp.c, "i1"); _ = try appendToken(rp.c, .Comma, ","); - try bitcast_node.params.push(&inner_cast_node.base); + bitcast_node.params()[1] = &inner_cast_node.base; bitcast_node.rparen_token = try appendToken(rp.c, .RParen, ")"); - try cast_node.params.push(&bitcast_node.base); + cast_node.params()[1] = &bitcast_node.base; } else { - try cast_node.params.push(&inner_cast_node.base); + cast_node.params()[1] = &inner_cast_node.base; } cast_node.rparen_token = try appendToken(rp.c, .RParen, ")"); return &cast_node.base; } if (ZigClangQualType_getTypeClass(ZigClangQualType_getCanonicalType(dst_type)) == .Enum) { - const builtin_node = try transCreateNodeBuiltinFnCall(rp.c, "@intToEnum"); - try builtin_node.params.push(try transQualType(rp, dst_type, loc)); + const builtin_node = try rp.c.createBuiltinCall("@intToEnum", 2); + builtin_node.params()[0] = try transQualType(rp, dst_type, loc); _ = try appendToken(rp.c, .Comma, ","); - try builtin_node.params.push(expr); + builtin_node.params()[1] = expr; builtin_node.rparen_token = try appendToken(rp.c, .RParen, ")"); return &builtin_node.base; } if (ZigClangQualType_getTypeClass(ZigClangQualType_getCanonicalType(src_type)) == .Enum and ZigClangQualType_getTypeClass(ZigClangQualType_getCanonicalType(dst_type)) != .Enum) { - const builtin_node = try transCreateNodeBuiltinFnCall(rp.c, "@enumToInt"); - try builtin_node.params.push(expr); + const builtin_node = try rp.c.createBuiltinCall("@enumToInt", 1); + builtin_node.params()[0] = expr; builtin_node.rparen_token = try appendToken(rp.c, .RParen, ")"); return &builtin_node.base; } - const cast_node = try transCreateNodeBuiltinFnCall(rp.c, "@as"); - try cast_node.params.push(try transQualType(rp, dst_type, loc)); + const cast_node = try rp.c.createBuiltinCall("@as", 2); + cast_node.params()[0] = try transQualType(rp, dst_type, loc); _ = try appendToken(rp.c, .Comma, ","); - try cast_node.params.push(expr); + cast_node.params()[1] = expr; cast_node.rparen_token = try appendToken(rp.c, .RParen, ")"); return &cast_node.base; } @@ -2077,12 +2137,13 @@ fn transInitListExprRecord( _ = try appendToken(c, .LBrace, "{"); - const node = try rp.c.arena.create(ast.Node.StructInitializer); + const node = try ast.Node.StructInitializer.alloc(rp.c.arena, field_inits.items.len); node.* = .{ .lhs = ty, .rtoken = try appendToken(rp.c, .RBrace, "}"), - .list = try rp.c.arena.dupe(*ast.Node, field_inits.items), + .list_len = field_inits.items.len, }; + mem.copy(*ast.Node, node.list(), field_inits.items); return &node.base; } @@ -2292,11 +2353,15 @@ fn transIfStmt( // if (c) t else e const if_node = try transCreateNodeIf(rp.c); - var cond_scope = Scope{ - .parent = scope, - .id = .Condition, + var cond_scope = Scope.Condition{ + .base = .{ + .parent = scope, + .id = .Condition, + }, }; - if_node.condition = try transBoolExpr(rp, &cond_scope, @ptrCast(*const ZigClangExpr, ZigClangIfStmt_getCond(stmt)), .used, .r_value, false); + defer cond_scope.deinit(); + const cond_expr = @ptrCast(*const ZigClangExpr, ZigClangIfStmt_getCond(stmt)); + if_node.condition = try transBoolExpr(rp, &cond_scope.base, cond_expr, .used, .r_value, false); _ = try appendToken(rp.c, .RParen, ")"); if_node.body = try transStmt(rp, scope, ZigClangIfStmt_getThen(stmt), .unused, .r_value); @@ -2316,11 +2381,15 @@ fn transWhileLoop( ) TransError!*ast.Node { const while_node = try transCreateNodeWhile(rp.c); - var cond_scope = Scope{ - .parent = scope, - .id = .Condition, + var cond_scope = Scope.Condition{ + .base = .{ + .parent = scope, + .id = .Condition, + }, }; - while_node.condition = try transBoolExpr(rp, &cond_scope, @ptrCast(*const ZigClangExpr, ZigClangWhileStmt_getCond(stmt)), .used, .r_value, false); + defer cond_scope.deinit(); + const cond_expr = @ptrCast(*const ZigClangExpr, ZigClangWhileStmt_getCond(stmt)); + while_node.condition = try transBoolExpr(rp, &cond_scope.base, cond_expr, .used, .r_value, false); _ = try appendToken(rp.c, .RParen, ")"); var loop_scope = Scope{ @@ -2349,12 +2418,15 @@ fn transDoWhileLoop( // if (!cond) break; const if_node = try transCreateNodeIf(rp.c); - var cond_scope = Scope{ - .parent = scope, - .id = .Condition, + var cond_scope = Scope.Condition{ + .base = .{ + .parent = scope, + .id = .Condition, + }, }; + defer cond_scope.deinit(); const prefix_op = try transCreateNodePrefixOp(rp.c, .BoolNot, .Bang, "!"); - prefix_op.rhs = try transBoolExpr(rp, &cond_scope, @ptrCast(*const ZigClangExpr, ZigClangDoStmt_getCond(stmt)), .used, .r_value, true); + prefix_op.rhs = try transBoolExpr(rp, &cond_scope.base, @ptrCast(*const ZigClangExpr, ZigClangDoStmt_getCond(stmt)), .used, .r_value, true); _ = try appendToken(rp.c, .RParen, ")"); if_node.condition = &prefix_op.base; if_node.body = &(try transCreateNodeBreak(rp.c, null)).base; @@ -2404,24 +2476,26 @@ fn transForLoop( .id = .Loop, }; - var block_scope: ?*Scope.Block = null; + var block_scope: ?Scope.Block = null; + defer if (block_scope) |*bs| bs.deinit(); + if (ZigClangForStmt_getInit(stmt)) |init| { block_scope = try Scope.Block.init(rp.c, scope, null); - const block = try transCreateNodeBlock(rp.c, null); - block_scope.?.setBlockNode(block); loop_scope.parent = &block_scope.?.base; const result = try transStmt(rp, &block_scope.?.base, init, .unused, .r_value); - if (result != &block.base) - try block.statements.push(result); + try block_scope.statements.push(result); } - var cond_scope = Scope{ - .parent = scope, - .id = .Condition, + var cond_scope = Scope.Condition{ + .base = .{ + .parent = scope, + .id = .Condition, + }, }; + defer cond_scope.deinit(); const while_node = try transCreateNodeWhile(rp.c); while_node.condition = if (ZigClangForStmt_getCond(stmt)) |cond| - try transBoolExpr(rp, &cond_scope, cond, .used, .r_value, false) + try transBoolExpr(rp, &cond_scope.base, cond, .used, .r_value, false) else try transCreateNodeBoolLiteral(rp.c, true); _ = try appendToken(rp.c, .RParen, ")"); @@ -2429,62 +2503,90 @@ fn transForLoop( if (ZigClangForStmt_getInc(stmt)) |incr| { _ = try appendToken(rp.c, .Colon, ":"); _ = try appendToken(rp.c, .LParen, "("); - while_node.continue_expr = try transExpr(rp, &cond_scope, incr, .unused, .r_value); + while_node.continue_expr = try transExpr(rp, &cond_scope.base, incr, .unused, .r_value); _ = try appendToken(rp.c, .RParen, ")"); } while_node.body = try transStmt(rp, &loop_scope, ZigClangForStmt_getBody(stmt), .unused, .r_value); - if (block_scope != null) { - try block_scope.?.block_node.statements.push(&while_node.base); - block_scope.?.block_node.rbrace = try appendToken(rp.c, .RBrace, "}"); - return &block_scope.?.block_node.base; + if (block_scope) |*bs| { + try bs.statements.push(&while_node.base); + const node = bs.complete(rp.c); + return &node.base; } else { _ = try appendToken(rp.c, .Semicolon, ";"); return &while_node.base; } } +fn getSwitchCaseCount(stmt: *const ZigClangSwitchStmt) usize { + const stmt = ZigClangSwitchStmt_getBody(stmt); + const comp = ZigClangStmtExpr_getSubStmt(stmt); + return ZigClangCompoundStmt_body_end(comp) - ZigClangCompoundStmt_body_begin(comp); +} + fn transSwitch( rp: RestorePoint, scope: *Scope, stmt: *const ZigClangSwitchStmt, ) TransError!*ast.Node { - const switch_node = try transCreateNodeSwitch(rp.c); + const switch_tok = try appendToken(rp.c, .Keyword_switch, "switch"); + _ = try appendToken(rp.c, .LParen, "("); + + const cases_len = getSwitchCaseCount(ZigClangSwitchStmt_getBody(stmt)); + + var cond_scope = Scope.Condition{ + .base = .{ + .parent = scope, + .id = .Condition, + }, + }; + defer cond_scope.deinit(); + const switch_expr = try transExpr(rp, &cond_scope.base, ZigClangSwitchStmt_getCond(stmt), .used, .r_value); + _ = try appendToken(rp.c, .RParen, ")"); + _ = try appendToken(rp.c, .LBrace, "{"); + // reserve +1 case in case there is no default case + const switch_node = try ast.Node.Switch.alloc(rp.c.arena, cases_len + 1); + switch_node.* = .{ + .switch_token = switch_tok, + .expr = switch_expr, + .cases_len = cases_len + 1, + .rbrace = try appendToken(rp.c, .RBrace, "}"), + }; + var switch_scope = Scope.Switch{ .base = .{ .id = .Switch, .parent = scope, }, - .cases = &switch_node.cases, + .cases = switch_node.cases(), + .case_index = 0, .pending_block = undefined, }; - var cond_scope = Scope{ - .parent = scope, - .id = .Condition, - }; - switch_node.expr = try transExpr(rp, &cond_scope, ZigClangSwitchStmt_getCond(stmt), .used, .r_value); - _ = try appendToken(rp.c, .RParen, ")"); - _ = try appendToken(rp.c, .LBrace, "{"); - switch_node.rbrace = try appendToken(rp.c, .RBrace, "}"); - - const block_scope = try Scope.Block.init(rp.c, &switch_scope.base, null); // tmp block that all statements will go before being picked up by a case or default - const block = try transCreateNodeBlock(rp.c, null); - block_scope.setBlockNode(block); + var block_scope = try Scope.Block.init(rp.c, &switch_scope.base, null); + defer block_scope.deinit(); - const switch_block = try transCreateNodeBlock(rp.c, null); - try switch_block.statements.push(&switch_node.base); - switch_scope.pending_block = switch_block; + // Note that we do not defer a deinit here; the switch_scope.pending_block field + // has its own memory management. This resource is freed inside `transCase` and + // then the final pending_block is freed at the bottom of this function with + // pending_block.deinit(). + switch_scope.pending_block = try Scope.Block.init(rp.c, scope, null); + try switch_scope.pending_block.statements.append(&switch_node.base); - const last = try transStmt(rp, &block_scope.base, ZigClangSwitchStmt_getBody(stmt), .unused, .r_value); + _ = try transStmt(rp, &block_scope.base, ZigClangSwitchStmt_getBody(stmt), .unused, .r_value); _ = try appendToken(rp.c, .Semicolon, ";"); // take all pending statements - var it = last.cast(ast.Node.Block).?.statements.iterator(0); - while (it.next()) |n| { - try switch_scope.pending_block.statements.push(n.*); - } + try switch_scope.pending_block.statements.appendSlice(block_scope.statements.items); + // TODO delete the following commented out code + //const last_block_stmts = last.cast(ast.Node.Block).?.statements(); + //switch_scope.pending_block.statements.ensureCapacity( + // switch_scope.pending_block.statements.items.len + last_block_stmts.len, + //); + //for (last_block_stmts) |n| { + // switch_scope.pending_block.statements.appendAssumeCapacity(n); + //} switch_scope.pending_block.label = try appendIdentifier(rp.c, "__switch"); _ = try appendToken(rp.c, .Colon, ":"); @@ -2492,10 +2594,16 @@ fn transSwitch( const else_prong = try transCreateNodeSwitchCase(rp.c, try transCreateNodeSwitchElse(rp.c)); else_prong.expr = &(try transCreateNodeBreak(rp.c, "__switch")).base; _ = try appendToken(rp.c, .Comma, ","); - try switch_node.cases.push(&else_prong.base); + switch_node.cases[switch_node.case_index] = &else_prong.base; + switch_node.case_index += 1; } - switch_scope.pending_block.rbrace = try appendToken(rp.c, .RBrace, "}"); - return &switch_scope.pending_block.base; + // We overallocated in case there was no default, so now we correct + // the number of cases in the AST node. + switch_node.cases_len = switch_scope.case_index; + + const result_node = try switch_scope.pending_block.complete(rp.c); + switch_scope.pending_block.deinit(); + return &result_node.base; } fn transCase( @@ -2527,22 +2635,21 @@ fn transCase( const switch_prong = try transCreateNodeSwitchCase(rp.c, expr); switch_prong.expr = &(try transCreateNodeBreak(rp.c, label)).base; _ = try appendToken(rp.c, .Comma, ","); - try switch_scope.cases.push(&switch_prong.base); + switch_scope.cases[switch_scope.case_index] = &switch_prong.base; + switch_scope.case_index += 1; - const block = try transCreateNodeBlock(rp.c, null); switch_scope.pending_block.label = try appendIdentifier(rp.c, label); _ = try appendToken(rp.c, .Colon, ":"); - switch_scope.pending_block.rbrace = try appendToken(rp.c, .RBrace, "}"); - try block.statements.push(&switch_scope.pending_block.base); // take all pending statements - var it = block_scope.block_node.statements.iterator(0); - while (it.next()) |n| { - try switch_scope.pending_block.statements.push(n.*); - } - block_scope.block_node.statements.shrink(0); + try switch_scope.pending_block.statements.appendSlice(block_scope.statements.items); + block_scope.statements.shrink(0); - switch_scope.pending_block = block; + const pending_node = try switch_scope.pending_block.complete(rp.c); + switch_scope.pending_block.deinit(); + switch_scope.pending_block = try Scope.Block.init(rp.c, scope, null); + + try switch_scope.pending_block.statements.append(&pending_node.base); return transStmt(rp, scope, ZigClangCaseStmt_getSubStmt(stmt), .unused, .r_value); } @@ -2561,22 +2668,21 @@ fn transDefault( const else_prong = try transCreateNodeSwitchCase(rp.c, try transCreateNodeSwitchElse(rp.c)); else_prong.expr = &(try transCreateNodeBreak(rp.c, label)).base; _ = try appendToken(rp.c, .Comma, ","); - try switch_scope.cases.push(&else_prong.base); + switch_scope.cases[switch_scope.case_index] = &else_prong.base; + switch_scope.case_index += 1; - const block = try transCreateNodeBlock(rp.c, null); switch_scope.pending_block.label = try appendIdentifier(rp.c, label); _ = try appendToken(rp.c, .Colon, ":"); - switch_scope.pending_block.rbrace = try appendToken(rp.c, .RBrace, "}"); - try block.statements.push(&switch_scope.pending_block.base); // take all pending statements - var it = block_scope.block_node.statements.iterator(0); - while (it.next()) |n| { - try switch_scope.pending_block.statements.push(n.*); - } - block_scope.block_node.statements.shrink(0); + try switch_scope.pending_block.statements.appendSlice(block_scope.statements.items); + block_scope.statements.shrink(0); + + const pending_node = try switch_scope.pending_block.complete(rp.c); + switch_scope.pending_block.deinit(); + switch_scope.pending_block = try Scope.Block.init(rp.c, scope, null); + try switch_scope.pending_block.statements.append(&pending_node.base); - switch_scope.pending_block = block; return transStmt(rp, scope, ZigClangDefaultStmt_getSubStmt(stmt), .unused, .r_value); } @@ -2591,13 +2697,13 @@ fn transConstantExpr(rp: RestorePoint, scope: *Scope, expr: *const ZigClangExpr, // See comment in `transIntegerLiteral` for why this code is here. // @as(T, x) const expr_base = @ptrCast(*const ZigClangExpr, expr); - const as_node = try transCreateNodeBuiltinFnCall(rp.c, "@as"); + const as_node = try rp.c.createBuiltinCall("@as", 2); const ty_node = try transQualType(rp, ZigClangExpr_getType(expr_base), ZigClangExpr_getBeginLoc(expr_base)); - try as_node.params.push(ty_node); + as_node.params()[0] = ty_node; _ = try appendToken(rp.c, .Comma, ","); const int_lit_node = try transCreateNodeAPInt(rp.c, ZigClangAPValue_getInt(&result.Val)); - try as_node.params.push(int_lit_node); + as_node.params()[1] = int_lit_node; as_node.rparen_token = try appendToken(rp.c, .RParen, ")"); @@ -2653,12 +2759,11 @@ fn transCharLiteral( // See comment in `transIntegerLiteral` for why this code is here. // @as(T, x) const expr_base = @ptrCast(*const ZigClangExpr, stmt); - const as_node = try transCreateNodeBuiltinFnCall(rp.c, "@as"); + const as_node = try rp.c.createBuiltinCall("@as", 2); const ty_node = try transQualType(rp, ZigClangExpr_getType(expr_base), ZigClangExpr_getBeginLoc(expr_base)); - try as_node.params.push(ty_node); + as_node.params()[0] = ty_node; _ = try appendToken(rp.c, .Comma, ","); - - try as_node.params.push(int_lit_node); + as_node.params()[1] = int_lit_node; as_node.rparen_token = try appendToken(rp.c, .RParen, ")"); return maybeSuppressResult(rp, scope, result_used, &as_node.base); @@ -2671,26 +2776,24 @@ fn transStmtExpr(rp: RestorePoint, scope: *Scope, stmt: *const ZigClangStmtExpr, } const lparen = try appendToken(rp.c, .LParen, "("); const block_scope = try Scope.Block.init(rp.c, scope, "blk"); - const block = try transCreateNodeBlock(rp.c, "blk"); - block_scope.setBlockNode(block); + defer block_scope.deinit(); var it = ZigClangCompoundStmt_body_begin(comp); const end_it = ZigClangCompoundStmt_body_end(comp); while (it != end_it - 1) : (it += 1) { const result = try transStmt(rp, &block_scope.base, it[0], .unused, .r_value); - if (result != &block.base) - try block.statements.push(result); + try block_scope.statements.append(result); } const break_node = try transCreateNodeBreak(rp.c, "blk"); break_node.rhs = try transStmt(rp, &block_scope.base, it[0], .used, .r_value); _ = try appendToken(rp.c, .Semicolon, ";"); - try block.statements.push(&break_node.base); - block.rbrace = try appendToken(rp.c, .RBrace, "}"); + try block_scope.statements.append(&break_node.base); + const block_node = try block_scope.complete(rp.c); const rparen = try appendToken(rp.c, .RParen, ")"); const grouped_expr = try rp.c.arena.create(ast.Node.GroupedExpression); grouped_expr.* = .{ .lparen = lparen, - .expr = &block.base, + .expr = &block_node.base, .rparen = rparen, }; return maybeSuppressResult(rp, scope, used, &grouped_expr.base); @@ -2746,12 +2849,12 @@ fn transArrayAccess(rp: RestorePoint, scope: *Scope, stmt: *const ZigClangArrayS const is_signed = cIsSignedInteger(qt); if (is_longlong or is_signed) { - const cast_node = try transCreateNodeBuiltinFnCall(rp.c, "@intCast"); + const cast_node = try rp.c.createBuiltinCall("@intCast", 2); // check if long long first so that signed long long doesn't just become unsigned long long var typeid_node = if (is_longlong) try transCreateNodeIdentifier(rp.c, "usize") else try transQualTypeIntWidthOf(rp.c, qt, false); - try cast_node.params.push(typeid_node); + cast_node.params()[0] = typeid_node; _ = try appendToken(rp.c, .Comma, ","); - try cast_node.params.push(try transExpr(rp, scope, subscr_expr, .used, .r_value)); + cast_node.params()[1] = try transExpr(rp, scope, subscr_expr, .used, .r_value); cast_node.rparen_token = try appendToken(rp.c, .RParen, ")"); node.rtoken = try appendToken(rp.c, .RBrace, "]"); node.op.ArrayAccess = &cast_node.base; @@ -2787,17 +2890,18 @@ fn transCallExpr(rp: RestorePoint, scope: *Scope, stmt: *const ZigClangCallExpr, break :blk try transCreateNodeUnwrapNull(rp.c, raw_fn_expr); } else raw_fn_expr; - const node = try transCreateNodeFnCall(rp.c, fn_expr); const num_args = ZigClangCallExpr_getNumArgs(stmt); + const node = try c.createCall(fn_expr, num_args); + const call_params = node.params(); + const args = ZigClangCallExpr_getArgs(stmt); var i: usize = 0; while (i < num_args) : (i += 1) { if (i != 0) { _ = try appendToken(rp.c, .Comma, ","); } - const arg = try transExpr(rp, scope, args[i], .used, .r_value); - try node.op.Call.params.push(arg); + call_params[i] = try transExpr(rp, scope, args[i], .used, .r_value); } node.rtoken = try appendToken(rp.c, .RParen, ")"); @@ -2856,8 +2960,8 @@ fn transUnaryExprOrTypeTraitExpr( ZigClangUnaryExprOrTypeTraitExpr_getBeginLoc(stmt), ); - const builtin_node = try transCreateNodeBuiltinFnCall(rp.c, "@sizeOf"); - try builtin_node.params.push(type_node); + const builtin_node = try rp.c.createBuiltinCall("@sizeOf", 1); + builtin_node.params()[0] = type_node; builtin_node.rparen_token = try appendToken(rp.c, .RParen, ")"); return maybeSuppressResult(rp, scope, result_used, &builtin_node.base); } @@ -2966,8 +3070,8 @@ fn transCreatePreCrement( // zig: _ref.* += 1; // zig: break :blk _ref.* // zig: }) - const block_scope = try Scope.Block.init(rp.c, scope, "blk"); - block_scope.setBlockNode(try transCreateNodeBlock(rp.c, block_scope.label)); + var block_scope = try Scope.Block.init(rp.c, scope, "blk"); + defer block_scope.deinit(); const ref = try block_scope.makeMangledName(rp.c, "ref"); const node = try transCreateNodeVarDecl(rp.c, false, true, ref); @@ -2976,7 +3080,7 @@ fn transCreatePreCrement( rhs_node.rhs = try transExpr(rp, scope, op_expr, .used, .r_value); node.init_node = &rhs_node.base; node.semicolon_token = try appendToken(rp.c, .Semicolon, ";"); - try block_scope.block_node.statements.push(&node.base); + try block_scope.statements.append(&node.base); const lhs_node = try transCreateNodeIdentifier(rp.c, ref); const ref_node = try transCreateNodePtrDeref(rp.c, lhs_node); @@ -2985,18 +3089,18 @@ fn transCreatePreCrement( const one = try transCreateNodeInt(rp.c, 1); _ = try appendToken(rp.c, .Semicolon, ";"); const assign = try transCreateNodeInfixOp(rp, scope, ref_node, op, token, one, .used, false); - try block_scope.block_node.statements.push(assign); + try block_scope.statements.append(assign); const break_node = try transCreateNodeBreak(rp.c, block_scope.label); break_node.rhs = ref_node; - try block_scope.block_node.statements.push(&break_node.base); - block_scope.block_node.rbrace = try appendToken(rp.c, .RBrace, "}"); + try block_scope.statements.push(&break_node.base); + const block_node = try block_scope.complete(rp.c); // semicolon must immediately follow rbrace because it is the last token in a block _ = try appendToken(rp.c, .Semicolon, ";"); const grouped_expr = try rp.c.arena.create(ast.Node.GroupedExpression); grouped_expr.* = .{ .lparen = try appendToken(rp.c, .LParen, "("), - .expr = &block_scope.block_node.base, + .expr = &block_node.base, .rparen = try appendToken(rp.c, .RParen, ")"), }; return &grouped_expr.base; @@ -3032,8 +3136,8 @@ fn transCreatePostCrement( // zig: _ref.* += 1; // zig: break :blk _tmp // zig: }) - const block_scope = try Scope.Block.init(rp.c, scope, "blk"); - block_scope.setBlockNode(try transCreateNodeBlock(rp.c, block_scope.label)); + var block_scope = try Scope.Block.init(rp.c, scope, "blk"); + defer block_scope.deinit(); const ref = try block_scope.makeMangledName(rp.c, "ref"); const node = try transCreateNodeVarDecl(rp.c, false, true, ref); @@ -3042,7 +3146,7 @@ fn transCreatePostCrement( rhs_node.rhs = try transExpr(rp, scope, op_expr, .used, .r_value); node.init_node = &rhs_node.base; node.semicolon_token = try appendToken(rp.c, .Semicolon, ";"); - try block_scope.block_node.statements.push(&node.base); + try block_scope.statements.push(&node.base); const lhs_node = try transCreateNodeIdentifier(rp.c, ref); const ref_node = try transCreateNodePtrDeref(rp.c, lhs_node); @@ -3053,23 +3157,23 @@ fn transCreatePostCrement( tmp_node.eq_token = try appendToken(rp.c, .Equal, "="); tmp_node.init_node = ref_node; tmp_node.semicolon_token = try appendToken(rp.c, .Semicolon, ";"); - try block_scope.block_node.statements.push(&tmp_node.base); + try block_scope.statements.push(&tmp_node.base); const token = try appendToken(rp.c, op_tok_id, bytes); const one = try transCreateNodeInt(rp.c, 1); _ = try appendToken(rp.c, .Semicolon, ";"); const assign = try transCreateNodeInfixOp(rp, scope, ref_node, op, token, one, .used, false); - try block_scope.block_node.statements.push(assign); + try block_scope.statements.push(assign); const break_node = try transCreateNodeBreak(rp.c, block_scope.label); break_node.rhs = try transCreateNodeIdentifier(rp.c, tmp); - try block_scope.block_node.statements.push(&break_node.base); + try block_scope.statements.push(&break_node.base); _ = try appendToken(rp.c, .Semicolon, ";"); - block_scope.block_node.rbrace = try appendToken(rp.c, .RBrace, "}"); + const block_node = try block_scope.complete(rp.c); const grouped_expr = try rp.c.arena.create(ast.Node.GroupedExpression); grouped_expr.* = .{ .lparen = try appendToken(rp.c, .LParen, "("), - .expr = &block_scope.block_node.base, + .expr = &block_node.base, .rparen = try appendToken(rp.c, .RParen, ")"), }; return &grouped_expr.base; @@ -3134,11 +3238,11 @@ fn transCreateCompoundAssign( const op_token = try appendToken(rp.c, .Equal, "="); const op_node = try rp.c.arena.create(ast.Node.InfixOp); const builtin = if (is_mod) "@rem" else "@divTrunc"; - const builtin_node = try transCreateNodeBuiltinFnCall(rp.c, builtin); + const builtin_node = try rp.c.createBuiltinCall(builtin, 2); const lhs_node = try transExpr(rp, scope, lhs, .used, .l_value); - try builtin_node.params.push(lhs_node); + builtin_node.params()[0] = lhs_node; _ = try appendToken(rp.c, .Comma, ","); - try builtin_node.params.push(try transExpr(rp, scope, rhs, .used, .r_value)); + builtin_node.params()[1] = try transExpr(rp, scope, rhs, .used, .r_value); builtin_node.rparen_token = try appendToken(rp.c, .RParen, ")"); op_node.* = .{ .op_token = op_token, @@ -3158,11 +3262,11 @@ fn transCreateCompoundAssign( try transExpr(rp, scope, rhs, .used, .r_value); if (is_shift) { - const cast_node = try transCreateNodeBuiltinFnCall(rp.c, "@intCast"); + const cast_node = try rp.c.createBuiltinCall("@intCast", 2); const rhs_type = try qualTypeToLog2IntRef(rp, getExprQualType(rp.c, rhs), loc); - try cast_node.params.push(rhs_type); + cast_node.params()[0] = rhs_type; _ = try appendToken(rp.c, .Comma, ","); - try cast_node.params.push(rhs_node); + cast_node.params()[1] = rhs_node; cast_node.rparen_token = try appendToken(rp.c, .RParen, ")"); rhs_node = &cast_node.base; } @@ -3177,8 +3281,8 @@ fn transCreateCompoundAssign( // zig: _ref.* = _ref.* + rhs; // zig: break :blk _ref.* // zig: }) - const block_scope = try Scope.Block.init(rp.c, scope, "blk"); - block_scope.setBlockNode(try transCreateNodeBlock(rp.c, block_scope.label)); + var block_scope = try Scope.Block.init(rp.c, scope, "blk"); + defer block_scope.deinit(); const ref = try block_scope.makeMangledName(rp.c, "ref"); const node = try transCreateNodeVarDecl(rp.c, false, true, ref); @@ -3187,7 +3291,7 @@ fn transCreateCompoundAssign( addr_node.rhs = try transExpr(rp, scope, lhs, .used, .l_value); node.init_node = &addr_node.base; node.semicolon_token = try appendToken(rp.c, .Semicolon, ";"); - try block_scope.block_node.statements.push(&node.base); + try block_scope.statements.append(&node.base); const lhs_node = try transCreateNodeIdentifier(rp.c, ref); const ref_node = try transCreateNodePtrDeref(rp.c, lhs_node); @@ -3197,10 +3301,10 @@ fn transCreateCompoundAssign( const op_token = try appendToken(rp.c, .Equal, "="); const op_node = try rp.c.arena.create(ast.Node.InfixOp); const builtin = if (is_mod) "@rem" else "@divTrunc"; - const builtin_node = try transCreateNodeBuiltinFnCall(rp.c, builtin); - try builtin_node.params.push(try transCreateNodePtrDeref(rp.c, lhs_node)); + const builtin_node = try rp.c.createBuiltinCall(builtin, 2); + builtin_node.params()[0] = try transCreateNodePtrDeref(rp.c, lhs_node); _ = try appendToken(rp.c, .Comma, ","); - try builtin_node.params.push(try transExpr(rp, scope, rhs, .used, .r_value)); + builtin_node.params()[1] = try transExpr(rp, scope, rhs, .used, .r_value); builtin_node.rparen_token = try appendToken(rp.c, .RParen, ")"); _ = try appendToken(rp.c, .Semicolon, ";"); op_node.* = .{ @@ -3210,17 +3314,17 @@ fn transCreateCompoundAssign( .rhs = &builtin_node.base, }; _ = try appendToken(rp.c, .Semicolon, ";"); - try block_scope.block_node.statements.push(&op_node.base); + try block_scope.statements.append(&op_node.base); } else { const bin_token = try appendToken(rp.c, bin_tok_id, bin_bytes); var rhs_node = try transExpr(rp, scope, rhs, .used, .r_value); if (is_shift) { - const cast_node = try transCreateNodeBuiltinFnCall(rp.c, "@intCast"); + const cast_node = try rp.c.createBuiltinCall("@intCast", 2); const rhs_type = try qualTypeToLog2IntRef(rp, getExprQualType(rp.c, rhs), loc); - try cast_node.params.push(rhs_type); + cast_node.params()[0] = rhs_type; _ = try appendToken(rp.c, .Comma, ","); - try cast_node.params.push(rhs_node); + cast_node.params()[1] = rhs_node; cast_node.rparen_token = try appendToken(rp.c, .RParen, ")"); rhs_node = &cast_node.base; } @@ -3230,17 +3334,17 @@ fn transCreateCompoundAssign( const eq_token = try appendToken(rp.c, .Equal, "="); const assign = try transCreateNodeInfixOp(rp, scope, ref_node, .Assign, eq_token, rhs_bin, .used, false); - try block_scope.block_node.statements.push(assign); + try block_scope.statements.append(assign); } const break_node = try transCreateNodeBreak(rp.c, block_scope.label); break_node.rhs = ref_node; - try block_scope.block_node.statements.push(&break_node.base); - block_scope.block_node.rbrace = try appendToken(rp.c, .RBrace, "}"); + try block_scope.statements.append(&break_node.base); + const block_node = try block_scope.complete(rp.c); const grouped_expr = try rp.c.arena.create(ast.Node.GroupedExpression); grouped_expr.* = .{ .lparen = try appendToken(rp.c, .LParen, "("), - .expr = &block_scope.block_node.base, + .expr = &block_node.base, .rparen = try appendToken(rp.c, .RParen, ")"), }; return &grouped_expr.base; @@ -3264,43 +3368,43 @@ fn transCPtrCast( !ZigClangQualType_isVolatileQualified(child_type))) { // Casting away const or volatile requires us to use @intToPtr - const inttoptr_node = try transCreateNodeBuiltinFnCall(rp.c, "@intToPtr"); + const inttoptr_node = try rp.c.createBuiltinCall("@intToPtr", 2); const dst_type_node = try transType(rp, ty, loc); - try inttoptr_node.params.push(dst_type_node); + inttoptr_node.params()[0] = dst_type_node; _ = try appendToken(rp.c, .Comma, ","); - const ptrtoint_node = try transCreateNodeBuiltinFnCall(rp.c, "@ptrToInt"); - try ptrtoint_node.params.push(expr); + const ptrtoint_node = try rp.c.createBuiltinCall("@ptrToInt", 1); + ptrtoint_node.params()[0] = expr; ptrtoint_node.rparen_token = try appendToken(rp.c, .RParen, ")"); - try inttoptr_node.params.push(&ptrtoint_node.base); + inttoptr_node.params()[1] = &ptrtoint_node.base; inttoptr_node.rparen_token = try appendToken(rp.c, .RParen, ")"); return &inttoptr_node.base; } else { // Implicit downcasting from higher to lower alignment values is forbidden, // use @alignCast to side-step this problem - const ptrcast_node = try transCreateNodeBuiltinFnCall(rp.c, "@ptrCast"); + const ptrcast_node = try rp.c.createBuiltinCall("@ptrCast", 2); const dst_type_node = try transType(rp, ty, loc); - try ptrcast_node.params.push(dst_type_node); + ptrcast_node.params()[0] = dst_type_node; _ = try appendToken(rp.c, .Comma, ","); if (ZigClangType_isVoidType(qualTypeCanon(child_type))) { // void has 1-byte alignment, so @alignCast is not needed - try ptrcast_node.params.push(expr); + ptrcast_node.params()[1] = expr; } else if (typeIsOpaque(rp.c, qualTypeCanon(child_type), loc)) { // For opaque types a ptrCast is enough - try ptrcast_node.params.push(expr); + ptrcast_node.params()[1] = expr; } else { - const aligncast_node = try transCreateNodeBuiltinFnCall(rp.c, "@alignCast"); - const alignof_node = try transCreateNodeBuiltinFnCall(rp.c, "@alignOf"); + const aligncast_node = try rp.c.createBuiltinCall("@alignCast", 2); + const alignof_node = try rp.c.createBuiltinCall("@alignOf", 1); const child_type_node = try transQualType(rp, child_type, loc); - try alignof_node.params.push(child_type_node); + alignof_node.params()[0] = child_type_node; alignof_node.rparen_token = try appendToken(rp.c, .RParen, ")"); - try aligncast_node.params.push(&alignof_node.base); + aligncast_node.params()[0] = &alignof_node.base; _ = try appendToken(rp.c, .Comma, ","); - try aligncast_node.params.push(expr); + aligncast_node.params()[1] = expr; aligncast_node.rparen_token = try appendToken(rp.c, .RParen, ")"); - try ptrcast_node.params.push(&aligncast_node.base); + ptrcast_node.params()[1] = &aligncast_node.base; } ptrcast_node.rparen_token = try appendToken(rp.c, .RParen, ")"); @@ -3343,27 +3447,30 @@ fn transBinaryConditionalOperator(rp: RestorePoint, scope: *Scope, stmt: *const // }) const lparen = try appendToken(rp.c, .LParen, "("); - const block_scope = try Scope.Block.init(rp.c, scope, "blk"); - block_scope.setBlockNode(try transCreateNodeBlock(rp.c, block_scope.label)); + var block_scope = try Scope.Block.init(rp.c, scope, "blk"); + defer block_scope.deinit(); const mangled_name = try block_scope.makeMangledName(rp.c, "cond_temp"); const tmp_var = try transCreateNodeVarDecl(rp.c, false, true, mangled_name); tmp_var.eq_token = try appendToken(rp.c, .Equal, "="); tmp_var.init_node = try transExpr(rp, &block_scope.base, cond_expr, .used, .r_value); tmp_var.semicolon_token = try appendToken(rp.c, .Semicolon, ";"); - try block_scope.block_node.statements.push(&tmp_var.base); + try block_scope.statements.append(&tmp_var.base); const break_node = try transCreateNodeBreak(rp.c, block_scope.label); const if_node = try transCreateNodeIf(rp.c); - var cond_scope = Scope{ - .parent = &block_scope.base, - .id = .Condition, + var cond_scope = Scope.Condition{ + .base = .{ + .parent = &block_scope.base, + .id = .Condition, + }, }; + defer cond_scope.deinit(); const tmp_var_node = try transCreateNodeIdentifier(rp.c, mangled_name); const ty = ZigClangQualType_getTypePtr(getExprQualType(rp.c, cond_expr)); - const cond_node = try finishBoolExpr(rp, &block_scope.base, ZigClangExpr_getBeginLoc(cond_expr), ty, tmp_var_node, used); + const cond_node = try finishBoolExpr(rp, &cond_scope.base, ZigClangExpr_getBeginLoc(cond_expr), ty, tmp_var_node, used); if_node.condition = cond_node; _ = try appendToken(rp.c, .RParen, ")"); @@ -3374,13 +3481,13 @@ fn transBinaryConditionalOperator(rp: RestorePoint, scope: *Scope, stmt: *const break_node.rhs = &if_node.base; _ = try appendToken(rp.c, .Semicolon, ";"); - try block_scope.block_node.statements.push(&break_node.base); - block_scope.block_node.rbrace = try appendToken(rp.c, .RBrace, "}"); + try block_scope.statements.append(&break_node.base); + const block_node = block_scope.complete(rp.c); const grouped_expr = try rp.c.arena.create(ast.Node.GroupedExpression); grouped_expr.* = .{ .lparen = lparen, - .expr = &block_scope.block_node.base, + .expr = &block_node.base, .rparen = try appendToken(rp.c, .RParen, ")"), }; return maybeSuppressResult(rp, scope, used, &grouped_expr.base); @@ -3390,17 +3497,20 @@ fn transConditionalOperator(rp: RestorePoint, scope: *Scope, stmt: *const ZigCla const grouped = scope.id == .Condition; const lparen = if (grouped) try appendToken(rp.c, .LParen, "(") else undefined; const if_node = try transCreateNodeIf(rp.c); - var cond_scope = Scope{ - .parent = scope, - .id = .Condition, + var cond_scope = Scope.Condition{ + .base = .{ + .parent = scope, + .id = .Condition, + }, }; + defer cond_scope.deinit(); const casted_stmt = @ptrCast(*const ZigClangAbstractConditionalOperator, stmt); const cond_expr = ZigClangAbstractConditionalOperator_getCond(casted_stmt); const true_expr = ZigClangAbstractConditionalOperator_getTrueExpr(casted_stmt); const false_expr = ZigClangAbstractConditionalOperator_getFalseExpr(casted_stmt); - if_node.condition = try transBoolExpr(rp, &cond_scope, cond_expr, .used, .r_value, false); + if_node.condition = try transBoolExpr(rp, &cond_scope.base, cond_expr, .used, .r_value, false); _ = try appendToken(rp.c, .RParen, ")"); if_node.body = try transExpr(rp, scope, true_expr, .used, .r_value); @@ -3571,19 +3681,19 @@ fn qualTypeToLog2IntRef(rp: RestorePoint, qt: ZigClangQualType, source_loc: ZigC // Symbol "Log2Int" // Symbol (var from above) - const import_fn_call = try transCreateNodeBuiltinFnCall(rp.c, "@import"); + const import_fn_call = try rp.c.createBuiltinCall("@import", 1); const std_token = try appendToken(rp.c, .StringLiteral, "\"std\""); const std_node = try rp.c.arena.create(ast.Node.StringLiteral); std_node.* = .{ .token = std_token, }; - try import_fn_call.params.push(&std_node.base); + import_fn_call.params()[0] = &std_node.base; import_fn_call.rparen_token = try appendToken(rp.c, .RParen, ")"); const inner_field_access = try transCreateNodeFieldAccess(rp.c, &import_fn_call.base, "math"); const outer_field_access = try transCreateNodeFieldAccess(rp.c, inner_field_access, "Log2Int"); - const log2int_fn_call = try transCreateNodeFnCall(rp.c, outer_field_access); - try @fieldParentPtr(ast.Node.SuffixOp, "base", &log2int_fn_call.base).op.Call.params.push(zig_type_node); + const log2int_fn_call = try rp.c.createCall(outer_field_access, 1); + log2int_fn_call.params()[0] = zig_type_node; log2int_fn_call.rtoken = try appendToken(rp.c, .RParen, ")"); return &log2int_fn_call.base; @@ -3757,8 +3867,8 @@ fn transCreateNodeAssign( const eq_token = try appendToken(rp.c, .Equal, "="); var rhs_node = try transExprCoercing(rp, scope, rhs, .used, .r_value); if (!exprIsBooleanType(lhs) and isBoolRes(rhs_node)) { - const builtin_node = try transCreateNodeBuiltinFnCall(rp.c, "@boolToInt"); - try builtin_node.params.push(rhs_node); + const builtin_node = try rp.c.createBuiltinCall("@boolToInt", 1); + builtin_node.params()[0] = rhs_node; builtin_node.rparen_token = try appendToken(rp.c, .RParen, ")"); rhs_node = &builtin_node.base; } @@ -3774,22 +3884,23 @@ fn transCreateNodeAssign( // zig: lhs = _tmp; // zig: break :blk _tmp // zig: }) - const block_scope = try Scope.Block.init(rp.c, scope, "blk"); - block_scope.setBlockNode(try transCreateNodeBlock(rp.c, block_scope.label)); + var block_scope = try Scope.Block.init(rp.c, scope, "blk"); + defer block_scope.deinit(); + const tmp = try block_scope.makeMangledName(rp.c, "tmp"); const node = try transCreateNodeVarDecl(rp.c, false, true, tmp); node.eq_token = try appendToken(rp.c, .Equal, "="); var rhs_node = try transExpr(rp, &block_scope.base, rhs, .used, .r_value); if (!exprIsBooleanType(lhs) and isBoolRes(rhs_node)) { - const builtin_node = try transCreateNodeBuiltinFnCall(rp.c, "@boolToInt"); - try builtin_node.params.push(rhs_node); + const builtin_node = try rp.c.createBuiltinCall("@boolToInt", 1); + builtin_node.params()[0] = rhs_node; builtin_node.rparen_token = try appendToken(rp.c, .RParen, ")"); rhs_node = &builtin_node.base; } node.init_node = rhs_node; node.semicolon_token = try appendToken(rp.c, .Semicolon, ";"); - try block_scope.block_node.statements.push(&node.base); + try block_scope.statements.append(&node.base); const lhs_node = try transExpr(rp, &block_scope.base, lhs, .used, .l_value); const eq_token = try appendToken(rp.c, .Equal, "="); @@ -3797,44 +3908,16 @@ fn transCreateNodeAssign( _ = try appendToken(rp.c, .Semicolon, ";"); const assign = try transCreateNodeInfixOp(rp, &block_scope.base, lhs_node, .Assign, eq_token, ident, .used, false); - try block_scope.block_node.statements.push(assign); + try block_scope.statements.append(assign); const break_node = try transCreateNodeBreak(rp.c, block_scope.label); break_node.rhs = try transCreateNodeIdentifier(rp.c, tmp); _ = try appendToken(rp.c, .Semicolon, ";"); - try block_scope.block_node.statements.push(&break_node.base); - block_scope.block_node.rbrace = try appendToken(rp.c, .RBrace, "}"); + try block_scope.statements.append(&break_node.base); + const block_node = try block_scope.complete(rp.c); // semicolon must immediately follow rbrace because it is the last token in a block _ = try appendToken(rp.c, .Semicolon, ";"); - return &block_scope.block_node.base; -} - -fn transCreateNodeBuiltinFnCall(c: *Context, name: []const u8) !*ast.Node.BuiltinCall { - const builtin_token = try appendToken(c, .Builtin, name); - _ = try appendToken(c, .LParen, "("); - const node = try c.arena.create(ast.Node.BuiltinCall); - node.* = .{ - .builtin_token = builtin_token, - .params = ast.Node.BuiltinCall.ParamList{}, - .rparen_token = undefined, // set after appending args - }; - return node; -} - -fn transCreateNodeFnCall(c: *Context, fn_expr: *ast.Node) !*ast.Node.SuffixOp { - _ = try appendToken(c, .LParen, "("); - const node = try c.arena.create(ast.Node.SuffixOp); - node.* = .{ - .lhs = fn_expr, - .op = .{ - .Call = .{ - .params = ast.Node.SuffixOp.Op.Call.ParamList{}, - .async_token = null, - }, - }, - .rtoken = undefined, // set after appending args - }; - return node; + return &block_node.base; } fn transCreateNodeFieldAccess(c: *Context, container: *ast.Node, field_name: []const u8) !*ast.Node { @@ -4083,8 +4166,8 @@ fn transCreateNodeFloat(c: *Context, int: var) !*ast.Node { } fn transCreateNodeOpaqueType(c: *Context) !*ast.Node { - const call_node = try transCreateNodeBuiltinFnCall(c, "@Type"); - try call_node.params.push(try transCreateNodeEnumLiteral(c, "Opaque")); + const call_node = try c.createBuiltinCall("@Type", 1); + call_node.params()[0] = try transCreateNodeEnumLiteral(c, "Opaque"); call_node.rparen_token = try appendToken(c, .RParen, ")"); return &call_node.base; } @@ -4139,26 +4222,33 @@ fn transCreateNodeMacroFn(c: *Context, name: []const u8, ref: *ast.Node, proto_a }; mem.copy(ast.Node.FnProto.ParamDecl, fn_proto.params(), fn_params.items); - const block = try transCreateNodeBlock(c, null); - var block_statements_it = &block.statements.first; + const block_lbrace = try appendToken(c, .LBrace, "{"); const return_expr = try transCreateNodeReturnExpr(c); const unwrap_expr = try transCreateNodeUnwrapNull(c, ref.cast(ast.Node.VarDecl).?.init_node.?); - const call_expr = try transCreateNodeFnCall(c, unwrap_expr); - var call_params_it = &call_expr.op.Call.params.first; + + const call_expr = try c.createCall(unwrap_expr, fn_params.items.len); + const call_params = call_expr.params(); for (fn_params.items) |param, i| { if (i != 0) { _ = try appendToken(c, .Comma, ","); } - call_params_it = try c.llpush(*ast.Node, call_params_it, try transCreateNodeIdentifier(c, tokenSlice(c, param.name_token.?)),); + call_params[i] = try transCreateNodeIdentifier(c, tokenSlice(c, param.name_token.?)); } call_expr.rtoken = try appendToken(c, .RParen, ")"); + return_expr.rhs = &call_expr.base; _ = try appendToken(c, .Semicolon, ";"); - block.rbrace = try appendToken(c, .RBrace, "}"); - block_statements_it = try c.llpush(*ast.Node, block_statements_it, &return_expr.base); + const block = try ast.Node.Block.alloc(c.arena, 1); + block.* = .{ + .label = null, + .lbrace = block_lbrace, + .statements_len = 1, + .rbrace = try appendToken(c, .RBrace, "}"), + }; + block.statements()[0] = &return_expr.base; fn_proto.body_node = &block.base; return &fn_proto.base; } @@ -4216,22 +4306,6 @@ fn transCreateNodeElse(c: *Context) !*ast.Node.Else { return node; } -fn transCreateNodeBlock(c: *Context, label: ?[]const u8) !*ast.Node.Block { - const label_node = if (label) |l| blk: { - const ll = try appendIdentifier(c, l); - _ = try appendToken(c, .Colon, ":"); - break :blk ll; - } else null; - const block_node = try c.arena.create(ast.Node.Block); - block_node.* = .{ - .label = label_node, - .lbrace = try appendToken(c, .LBrace, "{"), - .statements = ast.Node.Block.StatementList{}, - .rbrace = undefined, - }; - return block_node; -} - fn transCreateNodeBreak(c: *Context, label: ?[]const u8) !*ast.Node.ControlFlowExpression { const ltoken = try appendToken(c, .Keyword_break, "break"); const label_node = if (label) |l| blk: { @@ -4302,20 +4376,6 @@ fn transCreateNodeContinue(c: *Context) !*ast.Node { return &node.base; } -fn transCreateNodeSwitch(c: *Context) !*ast.Node.Switch { - const switch_tok = try appendToken(c, .Keyword_switch, "switch"); - _ = try appendToken(c, .LParen, "("); - - const node = try c.arena.create(ast.Node.Switch); - node.* = .{ - .switch_token = switch_tok, - .expr = undefined, - .cases = ast.Node.Switch.CaseList{}, - .rbrace = undefined, - }; - return node; -} - fn transCreateNodeSwitchCase(c: *Context, lhs: *ast.Node) !*ast.Node.SwitchCase { const arrow_tok = try appendToken(c, .EqualAngleBracketRight, "=>"); @@ -4356,12 +4416,12 @@ fn transCreateNodeShiftOp( const lhs = try transExpr(rp, scope, lhs_expr, .used, .l_value); const op_token = try appendToken(rp.c, op_tok_id, bytes); - const cast_node = try transCreateNodeBuiltinFnCall(rp.c, "@intCast"); + const cast_node = try rp.c.createBuiltinCall("@intCast", 2); const rhs_type = try qualTypeToLog2IntRef(rp, ZigClangBinaryOperator_getType(stmt), rhs_location); - try cast_node.params.push(rhs_type); + cast_node.params()[0] = rhs_type; _ = try appendToken(rp.c, .Comma, ","); const rhs = try transExprCoercing(rp, scope, rhs_expr, .used, .r_value); - try cast_node.params.push(rhs); + cast_node.params()[1] = rhs; cast_node.rparen_token = try appendToken(rp.c, .RParen, ")"); const node = try rp.c.arena.create(ast.Node.InfixOp); @@ -4832,14 +4892,13 @@ pub fn failDecl(c: *Context, loc: ZigClangSourceLocation, name: []const u8, comp .token = msg_tok, }; - const call_node = try c.arena.create(ast.Node.BuiltinCall); + const call_node = try ast.Node.BuiltinCall.alloc(c.arena, 1); call_node.* = .{ .builtin_token = builtin_tok, - .params = ast.Node.BuiltinCall.ParamList{}, + .params_len = 1, .rparen_token = rparen_tok, }; - var call_params_it = &call_node.params.first; - call_params_it = try c.llpush(*ast.Node, call_params_it, &msg_node.base); + call_node.params()[0] = &msg_node.base; const var_decl_node = try c.arena.create(ast.Node.VarDecl); var_decl_node.* = .{ @@ -4868,23 +4927,20 @@ fn appendToken(c: *Context, token_id: Token.Id, bytes: []const u8) !ast.TokenInd fn appendTokenFmt(c: *Context, token_id: Token.Id, comptime format: []const u8, args: var) !ast.TokenIndex { assert(token_id != .Invalid); + + try c.token_ids.ensureCapacity(c.gpa, c.token_ids.items.len + 1); + try c.token_locs.ensureCapacity(c.gpa, c.token_locs.items.len + 1); + const start_index = c.source_buffer.items.len; - errdefer c.source_buffer.shrink(start_index); + try c.source_buffer.outStream().print(format ++ " ", args); - try c.source_buffer.outStream().print(format, args); - const end_index = c.source_buffer.items.len; - const token_index = c.tokens.items.len; - const new_token = try c.tokens.addOne(c.gpa); - errdefer c.tokens.shrink(c.gpa, token_index); - - new_token.* = .{ - .id = token_id, + c.token_ids.appendAssumeCapacity(token_id); + c.token_locs.appendAssumeCapacity(.{ .start = start_index, - .end = end_index, - }; - try c.source_buffer.append(' '); + .end = c.source_buffer.items.len - 1, // back up before the space + }); - return token_index; + return c.token_ids.items.len - 1; } // TODO hook up with codegen @@ -5072,7 +5128,8 @@ fn transMacroDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, n } fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, name: []const u8, source_loc: ZigClangSourceLocation) ParseError!void { - const block_scope = try Scope.Block.init(c, &c.global_scope.base, null); + var block_scope = try Scope.Block.init(c, &c.global_scope.base, null); + defer block_scope.deinit(); const scope = &block_scope.base; const pub_tok = try appendToken(c, .Keyword_pub, "pub"); @@ -5142,9 +5199,7 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, _ = try appendToken(c, .RParen, ")"); - const type_of = try transCreateNodeBuiltinFnCall(c, "@TypeOf"); - type_of.rparen_token = try appendToken(c, .RParen, ")"); - var type_of_params = c.llpusher(&type_of.params); + const type_of = try c.createBuiltinCall("@TypeOf", 1); const fn_proto = try ast.Node.FnProto.alloc(c.arena, fn_params.items.len); fn_proto.* = .{ @@ -5164,9 +5219,6 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, }; mem.copy(ast.Node.FnProto.ParamDecl, fn_proto.params(), fn_params.items); - const block = try transCreateNodeBlock(c, null); - var block_statements = c.llpusher(&block.statements); - const return_expr = try transCreateNodeReturnExpr(c); const expr = try parseCExpr(c, it, source, source_loc, scope); const last = it.next().?; @@ -5186,12 +5238,13 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, const br = @fieldParentPtr(ast.Node.ControlFlowExpression, "base", blk_last); break :blk br.rhs.?; }; - try type_of_params.push(type_of_arg); + type_of.params()[0] = type_of_arg; + type_of.rparen_token = try appendToken(c, .RParen, ")"); return_expr.rhs = expr; - block.rbrace = try appendToken(c, .RBrace, "}"); - try block_statements.push(&return_expr.base); - fn_proto.body_node = &block.base; + try block_scope.statements.append(&return_expr.base); + const block_node = try block_scope.complete(c); + fn_proto.body_node = &block_node.base; _ = try c.global_scope.macro_table.put(name, &fn_proto.base); } @@ -5223,8 +5276,8 @@ fn parseCExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, source_ }, .Comma => { _ = try appendToken(c, .Semicolon, ";"); - const block_scope = try Scope.Block.init(c, scope, "blk"); - block_scope.setBlockNode(try transCreateNodeBlock(c, block_scope.label)); + var block_scope = try Scope.Block.init(c, scope, "blk"); + defer block_scope.deinit(); var last = node; while (true) { @@ -5238,7 +5291,7 @@ fn parseCExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, source_ .op = .Assign, .rhs = last, }; - try block_scope.block_node.statements.push(&op_node.base); + try block_scope.statements.append(&op_node.base); last = try parseCPrefixOpExpr(c, it, source, source_loc, scope); _ = try appendToken(c, .Semicolon, ";"); @@ -5250,9 +5303,9 @@ fn parseCExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, source_ const break_node = try transCreateNodeBreak(c, block_scope.label); break_node.rhs = last; - try block_scope.block_node.statements.push(&break_node.base); - block_scope.block_node.rbrace = try appendToken(c, .RBrace, "}"); - return &block_scope.block_node.base; + try block_scope.statements.append(&break_node.base); + const block_node = try block_scope.complete(c); + return &block_node.base; }, else => { _ = it.prev(); @@ -5283,15 +5336,15 @@ fn parseCNumLit(c: *Context, tok: *CToken, source: []const u8, source_loc: ZigCl return transCreateNodeInt(c, lit_bytes); } - const cast_node = try transCreateNodeBuiltinFnCall(c, "@as"); - try cast_node.params.push(try transCreateNodeIdentifier(c, switch (tok.id.IntegerLiteral) { + const cast_node = try c.createBuiltinCall("@as", 2); + cast_node.params()[0] = try transCreateNodeIdentifier(c, switch (tok.id.IntegerLiteral) { .U => "c_uint", .L => "c_long", .LU => "c_ulong", .LL => "c_longlong", .LLU => "c_ulonglong", else => unreachable, - })); + }); lit_bytes = lit_bytes[0 .. lit_bytes.len - switch (tok.id.IntegerLiteral) { .U, .L => @as(u8, 1), .LU, .LL => 2, @@ -5299,7 +5352,7 @@ fn parseCNumLit(c: *Context, tok: *CToken, source: []const u8, source_loc: ZigCl else => unreachable, }]; _ = try appendToken(c, .Comma, ","); - try cast_node.params.push(try transCreateNodeInt(c, lit_bytes)); + cast_node.params()[1] = try transCreateNodeInt(c, lit_bytes); cast_node.rparen_token = try appendToken(c, .RParen, ")"); return &cast_node.base; } else if (tok.id == .FloatLiteral) { @@ -5308,14 +5361,14 @@ fn parseCNumLit(c: *Context, tok: *CToken, source: []const u8, source_loc: ZigCl if (tok.id.FloatLiteral == .None) { return transCreateNodeFloat(c, lit_bytes); } - const cast_node = try transCreateNodeBuiltinFnCall(c, "@as"); - try cast_node.params.push(try transCreateNodeIdentifier(c, switch (tok.id.FloatLiteral) { + const cast_node = try c.createBuiltinCall("@as", 2); + cast_node.params()[0] = try transCreateNodeIdentifier(c, switch (tok.id.FloatLiteral) { .F => "f32", .L => "c_longdouble", else => unreachable, - })); + }); _ = try appendToken(c, .Comma, ","); - try cast_node.params.push(try transCreateNodeFloat(c, lit_bytes[0 .. lit_bytes.len - 1])); + cast_node.params()[1] = try transCreateNodeFloat(c, lit_bytes[0 .. lit_bytes.len - 1]); cast_node.rparen_token = try appendToken(c, .RParen, ")"); return &cast_node.base; } else unreachable; @@ -5585,8 +5638,8 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, //else // @as(dest, x) ) const if_node = try transCreateNodeIf(c); - const type_info_node = try transCreateNodeBuiltinFnCall(c, "@typeInfo"); - try type_info_node.params.push(inner_node); + const type_info_node = try rp.c.createBuiltinCall("@typeInfo", 1); + type_info_node.params()[0] = inner_node; type_info_node.rparen_token = try appendToken(c, .LParen, ")"); const cmp_node = try c.arena.create(ast.Node.InfixOp); cmp_node.* = .{ @@ -5598,18 +5651,18 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, if_node.condition = &cmp_node.base; _ = try appendToken(c, .RParen, ")"); - const int_to_ptr = try transCreateNodeBuiltinFnCall(c, "@intToPtr"); - try int_to_ptr.params.push(inner_node); - try int_to_ptr.params.push(node_to_cast); + const int_to_ptr = try c.createBuiltinCall("@intToPtr", 2); + int_to_ptr.params()[0] = inner_node; + int_to_ptr.params()[1] = node_to_cast; int_to_ptr.rparen_token = try appendToken(c, .RParen, ")"); if_node.body = &int_to_ptr.base; const else_node = try transCreateNodeElse(c); if_node.@"else" = else_node; - const as_node = try transCreateNodeBuiltinFnCall(c, "@as"); - try as_node.params.push(inner_node); - try as_node.params.push(node_to_cast); + const as_node = try c.createBuiltinCall("@as", 2); + as_node.params()[0] = inner_node; + as_node.params()[1] = node_to_cast; as_node.rparen_token = try appendToken(c, .RParen, ")"); else_node.body = &as_node.base; @@ -5630,10 +5683,10 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, // @as(dest, x) ) const if_1 = try transCreateNodeIf(c); - const type_info_1 = try transCreateNodeBuiltinFnCall(c, "@typeInfo"); - const type_of_1 = try transCreateNodeBuiltinFnCall(c, "@TypeOf"); - try type_info_1.params.push(&type_of_1.base); - try type_of_1.params.push(node_to_cast); + const type_info_1 = try c.createBuiltinCall("@typeInfo", 1); + const type_of_1 = try c.createBuiltinCall("@TypeOf", 1); + type_info_1.params()[0] = &type_of_1.base; + type_of_1.params()[0] = node_to_cast; type_of_1.rparen_token = try appendToken(c, .RParen, ")"); type_info_1.rparen_token = try appendToken(c, .RParen, ")"); @@ -5657,20 +5710,20 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, .rhs = child_ident, }; - const align_of = try transCreateNodeBuiltinFnCall(c, "@alignOf"); - try align_of.params.push(&inner_node_child.base); + const align_of = try rp.c.createBuiltinCall("@alignOf", 1); + align_of.params()[0] = &inner_node_child.base; align_of.rparen_token = try appendToken(c, .RParen, ")"); // hack to get zig fmt to render a comma in builtin calls _ = try appendToken(c, .Comma, ","); - const align_cast = try transCreateNodeBuiltinFnCall(c, "@alignCast"); - try align_cast.params.push(&align_of.base); - try align_cast.params.push(node_to_cast); + const align_cast = try c.createBuiltinCall("@alignCast", 2); + align_cast.params()[0] = &align_of.base; + align_cast.params()[1] = node_to_cast; align_cast.rparen_token = try appendToken(c, .RParen, ")"); - const ptr_cast = try transCreateNodeBuiltinFnCall(c, "@ptrCast"); - try ptr_cast.params.push(inner_node); - try ptr_cast.params.push(&align_cast.base); + const ptr_cast = try c.createBuiltinCall("@ptrCast", 2); + ptr_cast.params()[0] = inner_node; + ptr_cast.params()[1] = &align_cast.base; ptr_cast.rparen_token = try appendToken(c, .RParen, ")"); if_1.body = &ptr_cast.base; @@ -5678,10 +5731,10 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, if_1.@"else" = else_1; const if_2 = try transCreateNodeIf(c); - const type_info_2 = try transCreateNodeBuiltinFnCall(c, "@typeInfo"); - const type_of_2 = try transCreateNodeBuiltinFnCall(c, "@TypeOf"); - try type_info_2.params.push(&type_of_2.base); - try type_of_2.params.push(node_to_cast); + const type_info_2 = try c.createBuiltinCall("@typeInfo", 1); + const type_of_2 = try c.createBuiltinCall("@TypeOf", 1); + type_info_2.params()[0] = &type_of_2.base; + type_of_2.params()[0] = node_to_cast; type_of_2.rparen_token = try appendToken(c, .RParen, ")"); type_info_2.rparen_token = try appendToken(c, .RParen, ")"); @@ -5700,8 +5753,8 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, .op = .BoolAnd, .rhs = undefined, }; - const type_info_3 = try transCreateNodeBuiltinFnCall(c, "@typeInfo"); - try type_info_3.params.push(inner_node); + const type_info_3 = try c.createBuiltinCall("@typeInfo", 1); + type_info_3.params()[0] = inner_node; type_info_3.rparen_token = try appendToken(c, .LParen, ")"); const cmp_3 = try c.arena.create(ast.Node.InfixOp); cmp_3.* = .{ @@ -5715,18 +5768,18 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, else_1.body = &if_2.base; _ = try appendToken(c, .RParen, ")"); - const int_to_ptr = try transCreateNodeBuiltinFnCall(c, "@intToPtr"); - try int_to_ptr.params.push(inner_node); - try int_to_ptr.params.push(node_to_cast); + const int_to_ptr = try c.createBuiltinCall("@intToPtr", 2); + int_to_ptr.params()[0] = inner_node; + int_to_ptr.params()[1] = node_to_cast; int_to_ptr.rparen_token = try appendToken(c, .RParen, ")"); if_2.body = &int_to_ptr.base; const else_2 = try transCreateNodeElse(c); if_2.@"else" = else_2; - const as = try transCreateNodeBuiltinFnCall(c, "@as"); - try as.params.push(inner_node); - try as.params.push(node_to_cast); + const as = try c.createBuiltinCall("@as", 2); + as.params()[0] = inner_node; + as.params()[1] = node_to_cast; as.rparen_token = try appendToken(c, .RParen, ")"); else_2.body = &as.base; @@ -5765,8 +5818,8 @@ fn macroBoolToInt(c: *Context, node: *ast.Node) !*ast.Node { return &group_node.base; } - const builtin_node = try transCreateNodeBuiltinFnCall(c, "@boolToInt"); - try builtin_node.params.push(node); + const builtin_node = try rp.c.createBuiltinCall("@boolToInt", 1); + builtin_node.params()[0] = node; builtin_node.rparen_token = try appendToken(c, .RParen, ")"); return &builtin_node.base; } @@ -5944,10 +5997,12 @@ fn parseCSuffixOpExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, continue; }, .LParen => { - const call_node = try transCreateNodeFnCall(c, node); + _ = try appendToken(c, .LParen, "("); + var call_params = std.ArrayList(*ast.Node).init(c.gpa); + defer call_params.deinit(); while (true) { const arg = try parseCPrefixOpExpr(c, it, source, source_loc, scope); - try call_node.op.Call.params.push(arg); + try call_params.append(arg); const next = it.next().?; if (next.id == .Comma) _ = try appendToken(c, .Comma, ",") @@ -5965,7 +6020,14 @@ fn parseCSuffixOpExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, return error.ParseError; } } - call_node.rtoken = try appendToken(c, .RParen, ")"); + const call_node = try ast.Node.Call.alloc(c.arena, call_params.items.len); + call_node.* = .{ + .lhs = node, + .params_len = call_params.items.len, + .async_token = null, + .rtoken = try appendToken(c, .RParen, ")"), + }; + mem.copy(*ast.Node, call_node.params(), call_params.items); node = &call_node.base; continue; }, From 1a90a5e63adfd585836aee2658275c02856a4427 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2020 16:24:03 -0400 Subject: [PATCH 27/38] translate-c is building again, but I have 1 `@panic` in there --- src-self-hosted/main.zig | 10 +- src-self-hosted/translate_c.zig | 179 ++++++++++++++++++-------------- 2 files changed, 106 insertions(+), 83 deletions(-) diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 295d7b1337..40d3068d3f 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -743,15 +743,15 @@ fn printErrMsgToFile( const span_first = lok_token; const span_last = lok_token; - const first_token = tree.tokens[span_first]; - const last_token = tree.tokens[span_last]; - const start_loc = tree.tokenLocationPtr(0, first_token); - const end_loc = tree.tokenLocationPtr(first_token.end, last_token); + const first_token = tree.token_locs[span_first]; + const last_token = tree.token_locs[span_last]; + const start_loc = tree.tokenLocationLoc(0, first_token); + const end_loc = tree.tokenLocationLoc(first_token.end, last_token); var text_buf = std.ArrayList(u8).init(gpa); defer text_buf.deinit(); const out_stream = text_buf.outStream(); - try parse_error.render(tree.tokens, out_stream); + try parse_error.render(tree.token_ids, out_stream); const text = text_buf.span(); const stream = file.outStream(); diff --git a/src-self-hosted/translate_c.zig b/src-self-hosted/translate_c.zig index 17b6c1d9e0..9efbe282e2 100644 --- a/src-self-hosted/translate_c.zig +++ b/src-self-hosted/translate_c.zig @@ -59,7 +59,7 @@ const Scope = struct { /// into the main arena. const Switch = struct { base: Scope, - pending_block: *Block, + pending_block: Block, cases: []*ast.Node, case_index: usize, has_default: bool = false, @@ -102,7 +102,7 @@ const Scope = struct { }, .statements = std.ArrayList(*ast.Node).init(c.gpa), .variables = AliasList.init(c.gpa), - .label = if (self.label) |l| blk: { + .label = if (label) |l| blk: { const ll = try appendIdentifier(c, l); _ = try appendToken(c, .Colon, ":"); break :blk ll; @@ -118,7 +118,10 @@ const Scope = struct { } fn complete(self: *Block, c: *Context) !*ast.Node.Block { - const node = try ast.Node.Block.alloc(c.arena, self.statements.items.len); + // We reserve 1 extra statement if the parent is a Loop. This is in case of + // do while, we want to put `if (cond) break;` at the end. + const alloc_len = self.statements.items.len + @boolToInt(self.base.parent.?.id == .Loop); + const node = try ast.Node.Block.alloc(c.arena, alloc_len); node.* = .{ .statements_len = self.statements.items.len, .lbrace = self.lbrace, @@ -137,13 +140,12 @@ const Scope = struct { scope.mangle_count += 1; proposed_name = try std.fmt.allocPrint(c.arena, "{}_{}", .{ name, scope.mangle_count }); } - try scope.variables.push(.{ .name = name, .alias = proposed_name }); + try scope.variables.append(.{ .name = name, .alias = proposed_name }); return proposed_name; } fn getAlias(scope: *Block, name: []const u8) []const u8 { - var it = scope.variables.iterator(0); - while (it.next()) |p| { + for (scope.variables.items) |p| { if (mem.eql(u8, p.name, name)) return p.alias; } @@ -151,8 +153,7 @@ const Scope = struct { } fn localContains(scope: *Block, name: []const u8) bool { - var it = scope.variables.iterator(0); - while (it.next()) |p| { + for (scope.variables.items) |p| { if (mem.eql(u8, p.name, name)) return true; } @@ -407,7 +408,7 @@ pub fn translate( if (false) { std.debug.warn("debug source:\n{}\n==EOF==\ntokens:\n", .{source_buffer.items}); - for (context.tokens.items) |token| { + for (context.token_ids.items) |token| { std.debug.warn("{}\n", .{token}); } } @@ -581,7 +582,7 @@ fn visitFnDecl(c: *Context, fn_decl: *const ZigClangFunctionDecl) Error!void { var scope = &block_scope.base; var param_id: c_uint = 0; - for (proto_node.params()) |param, i| { + for (proto_node.params()) |*param, i| { const param_name = if (param.name_token) |name_tok| tokenSlice(c, name_tok) else if (param.param_type == .var_args) { @@ -623,7 +624,8 @@ fn visitFnDecl(c: *Context, fn_decl: *const ZigClangFunctionDecl) Error!void { error.UnsupportedType, => return failDecl(c, fn_decl_loc, fn_name, "unable to translate function", .{}), }; - proto_node.body_node = try block_scope.complete(rp.c); + const body_node = try block_scope.complete(rp.c); + proto_node.body_node = &body_node.base; return addTopLevelDecl(c, fn_name, &proto_node.base); } @@ -964,7 +966,7 @@ fn transRecordDecl(c: *Context, record_decl: *const ZigClangRecordDecl) Error!?* try addTopLevelDecl(c, name, &node.base); if (!is_unnamed) - try c.alias_list.push(.{ .alias = bare_name, .name = name }); + try c.alias_list.append(.{ .alias = bare_name, .name = name }); return transCreateNodeIdentifier(c, name); } @@ -1114,7 +1116,7 @@ fn transEnumDecl(c: *Context, enum_decl: *const ZigClangEnumDecl) Error!?*ast.No try addTopLevelDecl(c, name, &node.base); if (!is_unnamed) - try c.alias_list.push(.{ .alias = bare_name, .name = name }); + try c.alias_list.append(.{ .alias = bare_name, .name = name }); return transCreateNodeIdentifier(c, name); } @@ -1245,7 +1247,7 @@ fn transBinaryOperator( const rhs = try transExpr(rp, &block_scope.base, ZigClangBinaryOperator_getRHS(stmt), .used, .r_value); if (expr) { _ = try appendToken(rp.c, .Semicolon, ";"); - const break_node = try transCreateNodeBreak(rp.c, block_scope.label); + const break_node = try transCreateNodeBreakToken(rp.c, block_scope.label); break_node.rhs = rhs; try block_scope.statements.append(&break_node.base); const block_node = try block_scope.complete(rp.c); @@ -1418,7 +1420,7 @@ fn transCompoundStmt(rp: RestorePoint, scope: *Scope, stmt: *const ZigClangCompo var block_scope = try Scope.Block.init(rp.c, scope, null); defer block_scope.deinit(); try transCompoundStmtInline(rp, &block_scope.base, stmt, &block_scope); - const node = block_scope.complete(c); + const node = try block_scope.complete(rp.c); return &node.base; } @@ -1493,7 +1495,7 @@ fn transDeclStmt(rp: RestorePoint, scope: *Scope, stmt: *const ZigClangDeclStmt) const mangled_name = try block_scope.makeMangledName(c, name); if (checkForBuiltinTypedef(name)) |builtin| { - try block_scope.variables.push(.{ + try block_scope.variables.append(.{ .alias = builtin, .name = mangled_name, }); @@ -1511,7 +1513,8 @@ fn transDeclStmt(rp: RestorePoint, scope: *Scope, stmt: *const ZigClangDeclStmt) ), } } - return &block_scope.block_node.base; + + @panic("previously the code was returning the containing block scope node here and I do not understand why it would do that"); } fn transDeclRefExpr( @@ -2135,11 +2138,11 @@ fn transInitListExprRecord( _ = try appendToken(rp.c, .Comma, ","); } - _ = try appendToken(c, .LBrace, "{"); + _ = try appendToken(rp.c, .LBrace, "{"); const node = try ast.Node.StructInitializer.alloc(rp.c.arena, field_inits.items.len); node.* = .{ - .lhs = ty, + .lhs = ty_node, .rtoken = try appendToken(rp.c, .RBrace, "}"), .list_len = field_inits.items.len, }; @@ -2187,7 +2190,7 @@ fn transInitListExprArray( const all_count = ZigClangAPInt_getLimitedValue(size_ap_int, math.maxInt(usize)); const leftover_count = all_count - init_count; - var init_node: *ast.Node.SuffixOp = undefined; + var init_node: *ast.Node.ArrayInitializer = undefined; var cat_tok: ast.TokenIndex = undefined; if (init_count != 0) { const ty_node = try transCreateNodeArrayType( @@ -2196,11 +2199,19 @@ fn transInitListExprArray( ZigClangQualType_getTypePtr(child_qt), init_count, ); - init_node = try transCreateNodeArrayInitializer(rp.c, ty_node); + _ = try appendToken(rp.c, .LBrace, "{"); + init_node = try ast.Node.ArrayInitializer.alloc(rp.c.arena, init_count); + init_node.* = .{ + .lhs = ty_node, + .rtoken = undefined, + .list_len = init_count, + }; + const init_list = init_node.list(); + var i: c_uint = 0; while (i < init_count) : (i += 1) { const elem_expr = ZigClangInitListExpr_getInit(expr, i); - try init_node.op.ArrayInitializer.push(try transExpr(rp, scope, elem_expr, .used, .r_value)); + init_list[i] = try transExpr(rp, scope, elem_expr, .used, .r_value); _ = try appendToken(rp.c, .Comma, ","); } init_node.rtoken = try appendToken(rp.c, .RBrace, "}"); @@ -2211,9 +2222,14 @@ fn transInitListExprArray( } const ty_node = try transCreateNodeArrayType(rp, loc, ZigClangQualType_getTypePtr(child_qt), 1); - var filler_init_node = try transCreateNodeArrayInitializer(rp.c, ty_node); + const filler_init_node = try ast.Node.ArrayInitializer.alloc(rp.c.arena, 1); + filler_init_node.* = .{ + .lhs = ty_node, + .rtoken = undefined, + .list_len = 1, + }; const filler_val_expr = ZigClangInitListExpr_getArrayFiller(expr); - try filler_init_node.op.ArrayInitializer.push(try transExpr(rp, scope, filler_val_expr, .used, .r_value)); + filler_init_node.list()[0] = try transExpr(rp, scope, filler_val_expr, .used, .r_value); filler_init_node.rtoken = try appendToken(rp.c, .RBrace, "}"); const rhs_node = if (leftover_count == 1) @@ -2443,7 +2459,8 @@ fn transDoWhileLoop( // zig: b; // zig: if (!cond) break; // zig: } - break :blk (try transStmt(rp, &loop_scope, ZigClangDoStmt_getBody(stmt), .unused, .r_value)).cast(ast.Node.Block).?; + const node = try transStmt(rp, &loop_scope, ZigClangDoStmt_getBody(stmt), .unused, .r_value); + break :blk node.cast(ast.Node.Block).?; } else blk: { // the C statement is without a block, so we need to create a block to contain it. // c: do @@ -2454,12 +2471,15 @@ fn transDoWhileLoop( // zig: if (!cond) break; // zig: } new = true; - const block = try transCreateNodeBlock(rp.c, null); - try block.statements.push(try transStmt(rp, &loop_scope, ZigClangDoStmt_getBody(stmt), .unused, .r_value)); + const block = try rp.c.createBlock(null, 2); + block.statements_len = 1; // over-allocated so we can add another below + block.statements()[0] = try transStmt(rp, &loop_scope, ZigClangDoStmt_getBody(stmt), .unused, .r_value); break :blk block; }; - try body_node.statements.push(&if_node.base); + // In both cases above, we reserved 1 extra statement. + body_node.statements()[body_node.statements_len] = &if_node.base; + body_node.statements_len += 1; if (new) body_node.rbrace = try appendToken(rp.c, .RBrace, "}"); while_node.body = &body_node.base; @@ -2482,8 +2502,8 @@ fn transForLoop( if (ZigClangForStmt_getInit(stmt)) |init| { block_scope = try Scope.Block.init(rp.c, scope, null); loop_scope.parent = &block_scope.?.base; - const result = try transStmt(rp, &block_scope.?.base, init, .unused, .r_value); - try block_scope.statements.push(result); + const init_node = try transStmt(rp, &block_scope.?.base, init, .unused, .r_value); + try block_scope.?.statements.append(init_node); } var cond_scope = Scope.Condition{ .base = .{ @@ -2509,8 +2529,8 @@ fn transForLoop( while_node.body = try transStmt(rp, &loop_scope, ZigClangForStmt_getBody(stmt), .unused, .r_value); if (block_scope) |*bs| { - try bs.statements.push(&while_node.base); - const node = bs.complete(rp.c); + try bs.statements.append(&while_node.base); + const node = try bs.complete(rp.c); return &node.base; } else { _ = try appendToken(rp.c, .Semicolon, ";"); @@ -2519,9 +2539,15 @@ fn transForLoop( } fn getSwitchCaseCount(stmt: *const ZigClangSwitchStmt) usize { - const stmt = ZigClangSwitchStmt_getBody(stmt); - const comp = ZigClangStmtExpr_getSubStmt(stmt); - return ZigClangCompoundStmt_body_end(comp) - ZigClangCompoundStmt_body_begin(comp); + const body = ZigClangSwitchStmt_getBody(stmt); + assert(ZigClangStmt_getStmtClass(body) == .StmtExprClass); + const stmt_expr = @ptrCast(*const ZigClangStmtExpr, body); + const comp = ZigClangStmtExpr_getSubStmt(stmt_expr); + // TODO https://github.com/ziglang/zig/issues/1738 + // return ZigClangCompoundStmt_body_end(comp) - ZigClangCompoundStmt_body_begin(comp); + const start_addr = @ptrToInt(ZigClangCompoundStmt_body_begin(comp)); + const end_addr = @ptrToInt(ZigClangCompoundStmt_body_end(comp)); + return (end_addr - start_addr) / @sizeOf(*ZigClangStmt); } fn transSwitch( @@ -2532,7 +2558,7 @@ fn transSwitch( const switch_tok = try appendToken(rp.c, .Keyword_switch, "switch"); _ = try appendToken(rp.c, .LParen, "("); - const cases_len = getSwitchCaseCount(ZigClangSwitchStmt_getBody(stmt)); + const cases_len = getSwitchCaseCount(stmt); var cond_scope = Scope.Condition{ .base = .{ @@ -2594,8 +2620,8 @@ fn transSwitch( const else_prong = try transCreateNodeSwitchCase(rp.c, try transCreateNodeSwitchElse(rp.c)); else_prong.expr = &(try transCreateNodeBreak(rp.c, "__switch")).base; _ = try appendToken(rp.c, .Comma, ","); - switch_node.cases[switch_node.case_index] = &else_prong.base; - switch_node.case_index += 1; + switch_scope.cases[switch_scope.case_index] = &else_prong.base; + switch_scope.case_index += 1; } // We overallocated in case there was no default, so now we correct // the number of cases in the AST node. @@ -2613,7 +2639,7 @@ fn transCase( ) TransError!*ast.Node { const block_scope = scope.findBlockScope(rp.c) catch unreachable; const switch_scope = scope.getSwitch(); - const label = try std.fmt.allocPrint(rp.c.arena, "__case_{}", .{switch_scope.cases.len() - @boolToInt(switch_scope.has_default)}); + const label = try std.fmt.allocPrint(rp.c.arena, "__case_{}", .{switch_scope.case_index - @boolToInt(switch_scope.has_default)}); _ = try appendToken(rp.c, .Semicolon, ";"); const expr = if (ZigClangCaseStmt_getRHS(stmt)) |rhs| blk: { @@ -2775,7 +2801,7 @@ fn transStmtExpr(rp: RestorePoint, scope: *Scope, stmt: *const ZigClangStmtExpr, return transCompoundStmt(rp, scope, comp); } const lparen = try appendToken(rp.c, .LParen, "("); - const block_scope = try Scope.Block.init(rp.c, scope, "blk"); + var block_scope = try Scope.Block.init(rp.c, scope, "blk"); defer block_scope.deinit(); var it = ZigClangCompoundStmt_body_begin(comp); @@ -2892,7 +2918,7 @@ fn transCallExpr(rp: RestorePoint, scope: *Scope, stmt: *const ZigClangCallExpr, raw_fn_expr; const num_args = ZigClangCallExpr_getNumArgs(stmt); - const node = try c.createCall(fn_expr, num_args); + const node = try rp.c.createCall(fn_expr, num_args); const call_params = node.params(); const args = ZigClangCallExpr_getArgs(stmt); @@ -3091,9 +3117,9 @@ fn transCreatePreCrement( const assign = try transCreateNodeInfixOp(rp, scope, ref_node, op, token, one, .used, false); try block_scope.statements.append(assign); - const break_node = try transCreateNodeBreak(rp.c, block_scope.label); + const break_node = try transCreateNodeBreakToken(rp.c, block_scope.label); break_node.rhs = ref_node; - try block_scope.statements.push(&break_node.base); + try block_scope.statements.append(&break_node.base); const block_node = try block_scope.complete(rp.c); // semicolon must immediately follow rbrace because it is the last token in a block _ = try appendToken(rp.c, .Semicolon, ";"); @@ -3146,7 +3172,7 @@ fn transCreatePostCrement( rhs_node.rhs = try transExpr(rp, scope, op_expr, .used, .r_value); node.init_node = &rhs_node.base; node.semicolon_token = try appendToken(rp.c, .Semicolon, ";"); - try block_scope.statements.push(&node.base); + try block_scope.statements.append(&node.base); const lhs_node = try transCreateNodeIdentifier(rp.c, ref); const ref_node = try transCreateNodePtrDeref(rp.c, lhs_node); @@ -3157,17 +3183,17 @@ fn transCreatePostCrement( tmp_node.eq_token = try appendToken(rp.c, .Equal, "="); tmp_node.init_node = ref_node; tmp_node.semicolon_token = try appendToken(rp.c, .Semicolon, ";"); - try block_scope.statements.push(&tmp_node.base); + try block_scope.statements.append(&tmp_node.base); const token = try appendToken(rp.c, op_tok_id, bytes); const one = try transCreateNodeInt(rp.c, 1); _ = try appendToken(rp.c, .Semicolon, ";"); const assign = try transCreateNodeInfixOp(rp, scope, ref_node, op, token, one, .used, false); - try block_scope.statements.push(assign); + try block_scope.statements.append(assign); - const break_node = try transCreateNodeBreak(rp.c, block_scope.label); + const break_node = try transCreateNodeBreakToken(rp.c, block_scope.label); break_node.rhs = try transCreateNodeIdentifier(rp.c, tmp); - try block_scope.statements.push(&break_node.base); + try block_scope.statements.append(&break_node.base); _ = try appendToken(rp.c, .Semicolon, ";"); const block_node = try block_scope.complete(rp.c); const grouped_expr = try rp.c.arena.create(ast.Node.GroupedExpression); @@ -3337,7 +3363,7 @@ fn transCreateCompoundAssign( try block_scope.statements.append(assign); } - const break_node = try transCreateNodeBreak(rp.c, block_scope.label); + const break_node = try transCreateNodeBreakToken(rp.c, block_scope.label); break_node.rhs = ref_node; try block_scope.statements.append(&break_node.base); const block_node = try block_scope.complete(rp.c); @@ -3457,7 +3483,7 @@ fn transBinaryConditionalOperator(rp: RestorePoint, scope: *Scope, stmt: *const tmp_var.semicolon_token = try appendToken(rp.c, .Semicolon, ";"); try block_scope.statements.append(&tmp_var.base); - const break_node = try transCreateNodeBreak(rp.c, block_scope.label); + const break_node = try transCreateNodeBreakToken(rp.c, block_scope.label); const if_node = try transCreateNodeIf(rp.c); var cond_scope = Scope.Condition{ @@ -3482,7 +3508,7 @@ fn transBinaryConditionalOperator(rp: RestorePoint, scope: *Scope, stmt: *const break_node.rhs = &if_node.base; _ = try appendToken(rp.c, .Semicolon, ";"); try block_scope.statements.append(&break_node.base); - const block_node = block_scope.complete(rp.c); + const block_node = try block_scope.complete(rp.c); const grouped_expr = try rp.c.arena.create(ast.Node.GroupedExpression); grouped_expr.* = .{ @@ -3884,7 +3910,8 @@ fn transCreateNodeAssign( // zig: lhs = _tmp; // zig: break :blk _tmp // zig: }) - var block_scope = try Scope.Block.init(rp.c, scope, "blk"); + const label_name = "blk"; + var block_scope = try Scope.Block.init(rp.c, scope, label_name); defer block_scope.deinit(); const tmp = try block_scope.makeMangledName(rp.c, "tmp"); @@ -3910,7 +3937,7 @@ fn transCreateNodeAssign( const assign = try transCreateNodeInfixOp(rp, &block_scope.base, lhs_node, .Assign, eq_token, ident, .used, false); try block_scope.statements.append(assign); - const break_node = try transCreateNodeBreak(rp.c, block_scope.label); + const break_node = try transCreateNodeBreak(rp.c, label_name); break_node.rhs = try transCreateNodeIdentifier(rp.c, tmp); _ = try appendToken(rp.c, .Semicolon, ";"); try block_scope.statements.append(&break_node.base); @@ -4134,19 +4161,6 @@ fn transCreateNodeBoolLiteral(c: *Context, value: bool) !*ast.Node { return &node.base; } -fn transCreateNodeArrayInitializer(c: *Context, ty: *ast.Node) !*ast.Node.SuffixOp { - _ = try appendToken(c, .LBrace, "{"); - const node = try c.arena.create(ast.Node.SuffixOp); - node.* = .{ - .lhs = ty, - .op = .{ - .ArrayInitializer = ast.Node.SuffixOp.Op.InitList{}, - }, - .rtoken = undefined, // set after appending values - }; - return node; -} - fn transCreateNodeInt(c: *Context, int: var) !*ast.Node { const token = try appendTokenFmt(c, .IntegerLiteral, "{}", .{int}); const node = try c.arena.create(ast.Node.IntegerLiteral); @@ -4306,6 +4320,13 @@ fn transCreateNodeElse(c: *Context) !*ast.Node.Else { return node; } +fn transCreateNodeBreakToken(c: *Context, label: ?ast.TokenIndex) !*ast.Node.ControlFlowExpression { + const other_token = label orelse return transCreateNodeBreak(c, null); + const loc = c.token_locs.items[other_token]; + const label_name = c.source_buffer.items[loc.start..loc.end]; + return transCreateNodeBreak(c, label_name); +} + fn transCreateNodeBreak(c: *Context, label: ?[]const u8) !*ast.Node.ControlFlowExpression { const ltoken = try appendToken(c, .Keyword_break, "break"); const label_node = if (label) |l| blk: { @@ -4379,14 +4400,14 @@ fn transCreateNodeContinue(c: *Context) !*ast.Node { fn transCreateNodeSwitchCase(c: *Context, lhs: *ast.Node) !*ast.Node.SwitchCase { const arrow_tok = try appendToken(c, .EqualAngleBracketRight, "=>"); - const node = try c.arena.create(ast.Node.SwitchCase); + const node = try ast.Node.SwitchCase.alloc(c.arena, 1); node.* = .{ - .items = ast.Node.SwitchCase.ItemList{}, + .items_len = 1, .arrow_token = arrow_tok, .payload = null, .expr = undefined, }; - try node.items.push(lhs); + node.items()[0] = lhs; return node; } @@ -4464,7 +4485,8 @@ const RestorePoint = struct { src_buf_index: usize, fn activate(self: RestorePoint) void { - self.c.tokens.shrink(self.c.gpa, self.token_index); + self.c.token_ids.shrink(self.c.gpa, self.token_index); + self.c.token_locs.shrink(self.c.gpa, self.token_index); self.c.source_buffer.shrink(self.src_buf_index); } }; @@ -4472,7 +4494,7 @@ const RestorePoint = struct { fn makeRestorePoint(c: *Context) RestorePoint { return RestorePoint{ .c = c, - .token_index = c.tokens.items.len, + .token_index = c.token_ids.items.len, .src_buf_index = c.source_buffer.items.len, }; } @@ -5233,7 +5255,7 @@ fn transMacroFnDefine(c: *Context, it: *CTokenList.Iterator, source: []const u8, _ = try appendToken(c, .Semicolon, ";"); const type_of_arg = if (expr.id != .Block) expr else blk: { const blk = @fieldParentPtr(ast.Node.Block, "base", expr); - const blk_last = blk.statements.first.?.findLast().data; + const blk_last = blk.statements()[blk.statements_len - 1]; std.debug.assert(blk_last.id == .ControlFlowExpression); const br = @fieldParentPtr(ast.Node.ControlFlowExpression, "base", blk_last); break :blk br.rhs.?; @@ -5276,7 +5298,8 @@ fn parseCExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, source_ }, .Comma => { _ = try appendToken(c, .Semicolon, ";"); - var block_scope = try Scope.Block.init(c, scope, "blk"); + const label_name = "blk"; + var block_scope = try Scope.Block.init(c, scope, label_name); defer block_scope.deinit(); var last = node; @@ -5301,7 +5324,7 @@ fn parseCExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, source_ } } - const break_node = try transCreateNodeBreak(c, block_scope.label); + const break_node = try transCreateNodeBreak(c, label_name); break_node.rhs = last; try block_scope.statements.append(&break_node.base); const block_node = try block_scope.complete(c); @@ -5638,7 +5661,7 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, //else // @as(dest, x) ) const if_node = try transCreateNodeIf(c); - const type_info_node = try rp.c.createBuiltinCall("@typeInfo", 1); + const type_info_node = try c.createBuiltinCall("@typeInfo", 1); type_info_node.params()[0] = inner_node; type_info_node.rparen_token = try appendToken(c, .LParen, ")"); const cmp_node = try c.arena.create(ast.Node.InfixOp); @@ -5710,7 +5733,7 @@ fn parseCPrimaryExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, .rhs = child_ident, }; - const align_of = try rp.c.createBuiltinCall("@alignOf", 1); + const align_of = try c.createBuiltinCall("@alignOf", 1); align_of.params()[0] = &inner_node_child.base; align_of.rparen_token = try appendToken(c, .RParen, ")"); // hack to get zig fmt to render a comma in builtin calls @@ -5818,7 +5841,7 @@ fn macroBoolToInt(c: *Context, node: *ast.Node) !*ast.Node { return &group_node.base; } - const builtin_node = try rp.c.createBuiltinCall("@boolToInt", 1); + const builtin_node = try c.createBuiltinCall("@boolToInt", 1); builtin_node.params()[0] = node; builtin_node.rparen_token = try appendToken(c, .RParen, ")"); return &builtin_node.base; @@ -6115,7 +6138,7 @@ fn parseCPrefixOpExpr(c: *Context, it: *CTokenList.Iterator, source: []const u8, } fn tokenSlice(c: *Context, token: ast.TokenIndex) []u8 { - const tok = c.tokens.items[token]; + const tok = c.token_locs.items[token]; const slice = c.source_buffer.span()[tok.start..tok.end]; return if (mem.startsWith(u8, slice, "@\"")) slice[2 .. slice.len - 1] From 8f6d7b32086c70164926e5bb95886e5bdf737bfd Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2020 16:38:43 -0400 Subject: [PATCH 28/38] std: update singly linked list tests to new API --- lib/std/linked_list.zig | 33 +++++++++++++-------------------- 1 file changed, 13 insertions(+), 20 deletions(-) diff --git a/lib/std/linked_list.zig b/lib/std/linked_list.zig index c4c56df181..03520b0640 100644 --- a/lib/std/linked_list.zig +++ b/lib/std/linked_list.zig @@ -121,27 +121,20 @@ pub fn SinglyLinkedList(comptime T: type) type { } test "basic SinglyLinkedList test" { - const allocator = testing.allocator; - var list = SinglyLinkedList(u32).init(); + const L = SinglyLinkedList(u32); + var list = L{}; - var one = try list.createNode(1, allocator); - var two = try list.createNode(2, allocator); - var three = try list.createNode(3, allocator); - var four = try list.createNode(4, allocator); - var five = try list.createNode(5, allocator); - defer { - list.destroyNode(one, allocator); - list.destroyNode(two, allocator); - list.destroyNode(three, allocator); - list.destroyNode(four, allocator); - list.destroyNode(five, allocator); - } + var one = L.Node{.data = 1}; + var two = L.Node{.data = 2}; + var three = L.Node{.data = 3}; + var four = L.Node{.data = 4}; + var five = L.Node{.data = 5}; - list.prepend(two); // {2} - list.insertAfter(two, five); // {2, 5} - list.prepend(one); // {1, 2, 5} - list.insertAfter(two, three); // {1, 2, 3, 5} - list.insertAfter(three, four); // {1, 2, 3, 4, 5} + list.prepend(&two); // {2} + two.insertAfter(&five); // {2, 5} + list.prepend(&one); // {1, 2, 5} + two.insertAfter(&three); // {1, 2, 3, 5} + three.insertAfter(&four); // {1, 2, 3, 4, 5} // Traverse forwards. { @@ -154,7 +147,7 @@ test "basic SinglyLinkedList test" { } _ = list.popFirst(); // {2, 3, 4, 5} - _ = list.remove(five); // {2, 3, 4} + _ = list.remove(&five); // {2, 3, 4} _ = two.removeNext(); // {2, 4} testing.expect(list.first.?.data == 2); From 46f50ee76c1b97bdddc56e594e24d946c4f4b621 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2020 17:25:13 -0400 Subject: [PATCH 29/38] translate-c: fix regression in do-while loop --- src-self-hosted/translate_c.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-self-hosted/translate_c.zig b/src-self-hosted/translate_c.zig index 9efbe282e2..428b8e11c1 100644 --- a/src-self-hosted/translate_c.zig +++ b/src-self-hosted/translate_c.zig @@ -2478,8 +2478,8 @@ fn transDoWhileLoop( }; // In both cases above, we reserved 1 extra statement. - body_node.statements()[body_node.statements_len] = &if_node.base; body_node.statements_len += 1; + body_node.statements()[body_node.statements_len - 1] = &if_node.base; if (new) body_node.rbrace = try appendToken(rp.c, .RBrace, "}"); while_node.body = &body_node.base; From 32c27daae413b3348b5e25faf174651775e8bf4a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2020 17:50:35 -0400 Subject: [PATCH 30/38] translate-c: fix decl statement regression --- src-self-hosted/clang.zig | 11 ++- src-self-hosted/translate_c.zig | 147 +++++++++++++++++--------------- 2 files changed, 85 insertions(+), 73 deletions(-) diff --git a/src-self-hosted/clang.zig b/src-self-hosted/clang.zig index 72818d4ccc..624dbe1dad 100644 --- a/src-self-hosted/clang.zig +++ b/src-self-hosted/clang.zig @@ -22,7 +22,7 @@ pub const struct_ZigClangCompoundStmt = @Type(.Opaque); pub const struct_ZigClangConstantArrayType = @Type(.Opaque); pub const struct_ZigClangContinueStmt = @Type(.Opaque); pub const struct_ZigClangDecayedType = @Type(.Opaque); -pub const struct_ZigClangDecl = @Type(.Opaque); +pub const ZigClangDecl = @Type(.Opaque); pub const struct_ZigClangDeclRefExpr = @Type(.Opaque); pub const struct_ZigClangDeclStmt = @Type(.Opaque); pub const struct_ZigClangDefaultStmt = @Type(.Opaque); @@ -781,7 +781,7 @@ pub extern fn ZigClangSourceManager_getCharacterData(self: ?*const struct_ZigCla pub extern fn ZigClangASTContext_getPointerType(self: ?*const struct_ZigClangASTContext, T: struct_ZigClangQualType) struct_ZigClangQualType; pub extern fn ZigClangASTUnit_getASTContext(self: ?*struct_ZigClangASTUnit) ?*struct_ZigClangASTContext; pub extern fn ZigClangASTUnit_getSourceManager(self: *struct_ZigClangASTUnit) *struct_ZigClangSourceManager; -pub extern fn ZigClangASTUnit_visitLocalTopLevelDecls(self: *struct_ZigClangASTUnit, context: ?*c_void, Fn: ?fn (?*c_void, *const struct_ZigClangDecl) callconv(.C) bool) bool; +pub extern fn ZigClangASTUnit_visitLocalTopLevelDecls(self: *struct_ZigClangASTUnit, context: ?*c_void, Fn: ?fn (?*c_void, *const ZigClangDecl) callconv(.C) bool) bool; pub extern fn ZigClangRecordType_getDecl(record_ty: ?*const struct_ZigClangRecordType) *const struct_ZigClangRecordDecl; pub extern fn ZigClangTagDecl_isThisDeclarationADefinition(self: *const ZigClangTagDecl) bool; pub extern fn ZigClangEnumType_getDecl(record_ty: ?*const struct_ZigClangEnumType) *const struct_ZigClangEnumDecl; @@ -817,7 +817,7 @@ pub extern fn ZigClangEnumDecl_enumerator_end(*const ZigClangEnumDecl) ZigClangE pub extern fn ZigClangEnumDecl_enumerator_iterator_next(ZigClangEnumDecl_enumerator_iterator) ZigClangEnumDecl_enumerator_iterator; pub extern fn ZigClangEnumDecl_enumerator_iterator_deref(ZigClangEnumDecl_enumerator_iterator) *const ZigClangEnumConstantDecl; pub extern fn ZigClangEnumDecl_enumerator_iterator_neq(ZigClangEnumDecl_enumerator_iterator, ZigClangEnumDecl_enumerator_iterator) bool; -pub extern fn ZigClangDecl_castToNamedDecl(decl: *const struct_ZigClangDecl) ?*const ZigClangNamedDecl; +pub extern fn ZigClangDecl_castToNamedDecl(decl: *const ZigClangDecl) ?*const ZigClangNamedDecl; pub extern fn ZigClangNamedDecl_getName_bytes_begin(decl: ?*const struct_ZigClangNamedDecl) [*:0]const u8; pub extern fn ZigClangSourceLocation_eq(a: struct_ZigClangSourceLocation, b: struct_ZigClangSourceLocation) bool; pub extern fn ZigClangTypedefType_getDecl(self: ?*const struct_ZigClangTypedefType) *const struct_ZigClangTypedefNameDecl; @@ -918,7 +918,6 @@ pub const ZigClangCompoundStmt = struct_ZigClangCompoundStmt; pub const ZigClangConstantArrayType = struct_ZigClangConstantArrayType; pub const ZigClangContinueStmt = struct_ZigClangContinueStmt; pub const ZigClangDecayedType = struct_ZigClangDecayedType; -pub const ZigClangDecl = struct_ZigClangDecl; pub const ZigClangDeclRefExpr = struct_ZigClangDeclRefExpr; pub const ZigClangDeclStmt = struct_ZigClangDeclStmt; pub const ZigClangDefaultStmt = struct_ZigClangDefaultStmt; @@ -1015,14 +1014,14 @@ pub extern fn ZigClangLoadFromCommandLine( ) ?*ZigClangASTUnit; pub extern fn ZigClangDecl_getKind(decl: *const ZigClangDecl) ZigClangDeclKind; -pub extern fn ZigClangDecl_getDeclKindName(decl: *const struct_ZigClangDecl) [*:0]const u8; +pub extern fn ZigClangDecl_getDeclKindName(decl: *const ZigClangDecl) [*:0]const u8; pub const ZigClangCompoundStmt_const_body_iterator = [*]const *ZigClangStmt; pub extern fn ZigClangCompoundStmt_body_begin(self: *const ZigClangCompoundStmt) ZigClangCompoundStmt_const_body_iterator; pub extern fn ZigClangCompoundStmt_body_end(self: *const ZigClangCompoundStmt) ZigClangCompoundStmt_const_body_iterator; -pub const ZigClangDeclStmt_const_decl_iterator = [*]const *struct_ZigClangDecl; +pub const ZigClangDeclStmt_const_decl_iterator = [*]const *ZigClangDecl; pub extern fn ZigClangDeclStmt_decl_begin(self: *const ZigClangDeclStmt) ZigClangDeclStmt_const_decl_iterator; pub extern fn ZigClangDeclStmt_decl_end(self: *const ZigClangDeclStmt) ZigClangDeclStmt_const_decl_iterator; diff --git a/src-self-hosted/translate_c.zig b/src-self-hosted/translate_c.zig index 428b8e11c1..365295264b 100644 --- a/src-self-hosted/translate_c.zig +++ b/src-self-hosted/translate_c.zig @@ -1443,78 +1443,91 @@ fn transCStyleCastExprClass( return maybeSuppressResult(rp, scope, result_used, cast_node); } -fn transDeclStmt(rp: RestorePoint, scope: *Scope, stmt: *const ZigClangDeclStmt) TransError!*ast.Node { +fn transDeclStmtOne(rp: RestorePoint, scope: *Scope, decl: *const ZigClangDecl, block_scope: *Scope.Block,) TransError!*ast.Node { const c = rp.c; - const block_scope = scope.findBlockScope(c) catch unreachable; + + switch (ZigClangDecl_getKind(decl)) { + .Var => { + const var_decl = @ptrCast(*const ZigClangVarDecl, decl); + + const thread_local_token = if (ZigClangVarDecl_getTLSKind(var_decl) == .None) + null + else + try appendToken(c, .Keyword_threadlocal, "threadlocal"); + const qual_type = ZigClangVarDecl_getTypeSourceInfo_getType(var_decl); + const name = try c.str(ZigClangNamedDecl_getName_bytes_begin( + @ptrCast(*const ZigClangNamedDecl, var_decl), + )); + const mangled_name = try block_scope.makeMangledName(c, name); + const node = try transCreateNodeVarDecl(c, false, ZigClangQualType_isConstQualified(qual_type), mangled_name); + + _ = try appendToken(c, .Colon, ":"); + const loc = ZigClangDecl_getLocation(decl); + node.type_node = try transQualType(rp, qual_type, loc); + + node.eq_token = try appendToken(c, .Equal, "="); + var init_node = if (ZigClangVarDecl_getInit(var_decl)) |expr| + try transExprCoercing(rp, scope, expr, .used, .r_value) + else + try transCreateNodeUndefinedLiteral(c); + if (!qualTypeIsBoolean(qual_type) and isBoolRes(init_node)) { + const builtin_node = try rp.c.createBuiltinCall("@boolToInt", 1); + builtin_node.params()[0] = init_node; + builtin_node.rparen_token = try appendToken(rp.c, .RParen, ")"); + init_node = &builtin_node.base; + } + node.init_node = init_node; + node.semicolon_token = try appendToken(c, .Semicolon, ";"); + return &node.base; + }, + .Typedef => { + const typedef_decl = @ptrCast(*const ZigClangTypedefNameDecl, decl); + const name = try c.str(ZigClangNamedDecl_getName_bytes_begin( + @ptrCast(*const ZigClangNamedDecl, typedef_decl), + )); + + const underlying_qual = ZigClangTypedefNameDecl_getUnderlyingType(typedef_decl); + const underlying_type = ZigClangQualType_getTypePtr(underlying_qual); + + const mangled_name = try block_scope.makeMangledName(c, name); + if (checkForBuiltinTypedef(name)) |builtin| { + try block_scope.variables.append(.{ + .alias = builtin, + .name = mangled_name, + }); + @panic("what are we supposed to return here?"); + } else { + const node = (try transCreateNodeTypedef(rp, typedef_decl, false, mangled_name)) orelse + return error.UnsupportedTranslation; + return &node.base; + } + }, + else => |kind| return revertAndWarn( + rp, + error.UnsupportedTranslation, + ZigClangDecl_getLocation(decl), + "TODO implement translation of DeclStmt kind {}", + .{@tagName(kind)}, + ), + } +} + +fn transDeclStmt(rp: RestorePoint, scope: *Scope, stmt: *const ZigClangDeclStmt) TransError!*ast.Node { + const block_scope = scope.findBlockScope(rp.c) catch unreachable; var it = ZigClangDeclStmt_decl_begin(stmt); const end_it = ZigClangDeclStmt_decl_end(stmt); - while (it != end_it) : (it += 1) { - switch (ZigClangDecl_getKind(it[0])) { - .Var => { - const var_decl = @ptrCast(*const ZigClangVarDecl, it[0]); + assert(it != end_it); + while (true) : (it += 1) { + const node = try transDeclStmtOne(rp, scope, it[0], block_scope); - const thread_local_token = if (ZigClangVarDecl_getTLSKind(var_decl) == .None) - null - else - try appendToken(c, .Keyword_threadlocal, "threadlocal"); - const qual_type = ZigClangVarDecl_getTypeSourceInfo_getType(var_decl); - const name = try c.str(ZigClangNamedDecl_getName_bytes_begin( - @ptrCast(*const ZigClangNamedDecl, var_decl), - )); - const mangled_name = try block_scope.makeMangledName(c, name); - const node = try transCreateNodeVarDecl(c, false, ZigClangQualType_isConstQualified(qual_type), mangled_name); - - _ = try appendToken(c, .Colon, ":"); - const loc = ZigClangStmt_getBeginLoc(@ptrCast(*const ZigClangStmt, stmt)); - node.type_node = try transQualType(rp, qual_type, loc); - - node.eq_token = try appendToken(c, .Equal, "="); - var init_node = if (ZigClangVarDecl_getInit(var_decl)) |expr| - try transExprCoercing(rp, scope, expr, .used, .r_value) - else - try transCreateNodeUndefinedLiteral(c); - if (!qualTypeIsBoolean(qual_type) and isBoolRes(init_node)) { - const builtin_node = try rp.c.createBuiltinCall("@boolToInt", 1); - builtin_node.params()[0] = init_node; - builtin_node.rparen_token = try appendToken(rp.c, .RParen, ")"); - init_node = &builtin_node.base; - } - node.init_node = init_node; - node.semicolon_token = try appendToken(c, .Semicolon, ";"); - try block_scope.statements.append(&node.base); - }, - .Typedef => { - const typedef_decl = @ptrCast(*const ZigClangTypedefNameDecl, it[0]); - const name = try c.str(ZigClangNamedDecl_getName_bytes_begin( - @ptrCast(*const ZigClangNamedDecl, typedef_decl), - )); - - const underlying_qual = ZigClangTypedefNameDecl_getUnderlyingType(typedef_decl); - const underlying_type = ZigClangQualType_getTypePtr(underlying_qual); - - const mangled_name = try block_scope.makeMangledName(c, name); - if (checkForBuiltinTypedef(name)) |builtin| { - try block_scope.variables.append(.{ - .alias = builtin, - .name = mangled_name, - }); - } else { - const node = (try transCreateNodeTypedef(rp, typedef_decl, false, mangled_name)) orelse return error.UnsupportedTranslation; - try block_scope.statements.append(&node.base); - } - }, - else => |kind| return revertAndWarn( - rp, - error.UnsupportedTranslation, - ZigClangStmt_getBeginLoc(@ptrCast(*const ZigClangStmt, stmt)), - "TODO implement translation of DeclStmt kind {}", - .{@tagName(kind)}, - ), + if (it + 1 == end_it) { + return node; + } else { + try block_scope.statements.append(node); } } - - @panic("previously the code was returning the containing block scope node here and I do not understand why it would do that"); + unreachable; } fn transDeclRefExpr( @@ -2099,6 +2112,8 @@ fn transInitListExprRecord( var field_inits = std.ArrayList(*ast.Node).init(rp.c.gpa); defer field_inits.deinit(); + _ = try appendToken(rp.c, .LBrace, "{"); + var init_i: c_uint = 0; var it = ZigClangRecordDecl_field_begin(record_def); const end_it = ZigClangRecordDecl_field_end(record_def); @@ -2138,8 +2153,6 @@ fn transInitListExprRecord( _ = try appendToken(rp.c, .Comma, ","); } - _ = try appendToken(rp.c, .LBrace, "{"); - const node = try ast.Node.StructInitializer.alloc(rp.c.arena, field_inits.items.len); node.* = .{ .lhs = ty_node, From f67b8c68a0e8ddc60bcfc928d0781005d129526a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2020 17:52:46 -0400 Subject: [PATCH 31/38] translate-c: fix regression on array initialization --- src-self-hosted/translate_c.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/src-self-hosted/translate_c.zig b/src-self-hosted/translate_c.zig index 365295264b..d80449d87a 100644 --- a/src-self-hosted/translate_c.zig +++ b/src-self-hosted/translate_c.zig @@ -2235,6 +2235,7 @@ fn transInitListExprArray( } const ty_node = try transCreateNodeArrayType(rp, loc, ZigClangQualType_getTypePtr(child_qt), 1); + _ = try appendToken(rp.c, .LBrace, "{"); const filler_init_node = try ast.Node.ArrayInitializer.alloc(rp.c.arena, 1); filler_init_node.* = .{ .lhs = ty_node, From 88f5e3a60d6ea528f27935c63aaf3312d736cdbe Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2020 17:58:35 -0400 Subject: [PATCH 32/38] translate-c: fix regression in switch statement --- src-self-hosted/translate_c.zig | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/src-self-hosted/translate_c.zig b/src-self-hosted/translate_c.zig index d80449d87a..a4e16c83ff 100644 --- a/src-self-hosted/translate_c.zig +++ b/src-self-hosted/translate_c.zig @@ -2554,9 +2554,8 @@ fn transForLoop( fn getSwitchCaseCount(stmt: *const ZigClangSwitchStmt) usize { const body = ZigClangSwitchStmt_getBody(stmt); - assert(ZigClangStmt_getStmtClass(body) == .StmtExprClass); - const stmt_expr = @ptrCast(*const ZigClangStmtExpr, body); - const comp = ZigClangStmtExpr_getSubStmt(stmt_expr); + assert(ZigClangStmt_getStmtClass(body) == .CompoundStmtClass); + const comp = @ptrCast(*const ZigClangCompoundStmt, body); // TODO https://github.com/ziglang/zig/issues/1738 // return ZigClangCompoundStmt_body_end(comp) - ZigClangCompoundStmt_body_begin(comp); const start_addr = @ptrToInt(ZigClangCompoundStmt_body_begin(comp)); @@ -2614,19 +2613,17 @@ fn transSwitch( switch_scope.pending_block = try Scope.Block.init(rp.c, scope, null); try switch_scope.pending_block.statements.append(&switch_node.base); - _ = try transStmt(rp, &block_scope.base, ZigClangSwitchStmt_getBody(stmt), .unused, .r_value); + const last = try transStmt(rp, &block_scope.base, ZigClangSwitchStmt_getBody(stmt), .unused, .r_value); _ = try appendToken(rp.c, .Semicolon, ";"); // take all pending statements - try switch_scope.pending_block.statements.appendSlice(block_scope.statements.items); - // TODO delete the following commented out code - //const last_block_stmts = last.cast(ast.Node.Block).?.statements(); - //switch_scope.pending_block.statements.ensureCapacity( - // switch_scope.pending_block.statements.items.len + last_block_stmts.len, - //); - //for (last_block_stmts) |n| { - // switch_scope.pending_block.statements.appendAssumeCapacity(n); - //} + const last_block_stmts = last.cast(ast.Node.Block).?.statements(); + try switch_scope.pending_block.statements.ensureCapacity( + switch_scope.pending_block.statements.items.len + last_block_stmts.len, + ); + for (last_block_stmts) |n| { + switch_scope.pending_block.statements.appendAssumeCapacity(n); + } switch_scope.pending_block.label = try appendIdentifier(rp.c, "__switch"); _ = try appendToken(rp.c, .Colon, ":"); From 395786fd50897ec4e07d9fa0e666251715fa712b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2020 19:28:14 -0400 Subject: [PATCH 33/38] translate-c: fix regression in param name mangling --- src-self-hosted/translate_c.zig | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/src-self-hosted/translate_c.zig b/src-self-hosted/translate_c.zig index a4e16c83ff..2cb503e2d8 100644 --- a/src-self-hosted/translate_c.zig +++ b/src-self-hosted/translate_c.zig @@ -592,19 +592,15 @@ fn visitFnDecl(c: *Context, fn_decl: *const ZigClangFunctionDecl) Error!void { } else return failDecl(c, fn_decl_loc, fn_name, "function {} parameter has no name", .{fn_name}); - const mangled_param_name = try block_scope.makeMangledName(c, param_name); - const c_param = ZigClangFunctionDecl_getParamDecl(fn_decl, param_id); const qual_type = ZigClangParmVarDecl_getOriginalType(c_param); const is_const = ZigClangQualType_isConstQualified(qual_type); - const arg_name = blk: { - const param_prefix = if (is_const) "" else "arg_"; - const bare_arg_name = try std.fmt.allocPrint(c.arena, "{}{}", .{ param_prefix, mangled_param_name }); - break :blk try block_scope.makeMangledName(c, bare_arg_name); - }; + const mangled_param_name = try block_scope.makeMangledName(c, param_name); if (!is_const) { + const bare_arg_name = try std.fmt.allocPrint(c.arena, "arg_{}", .{ mangled_param_name }); + const arg_name = try block_scope.makeMangledName(c, bare_arg_name); const node = try transCreateNodeVarDecl(c, false, false, mangled_param_name); node.eq_token = try appendToken(c, .Equal, "="); node.init_node = try transCreateNodeIdentifier(c, arg_name); From c78a2e2e8dc50a6e57ad4d390184e564dbe7be67 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2020 19:59:26 -0400 Subject: [PATCH 34/38] translate-c: emit local typedefs --- src-self-hosted/translate_c.zig | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/src-self-hosted/translate_c.zig b/src-self-hosted/translate_c.zig index 2cb503e2d8..22b3386427 100644 --- a/src-self-hosted/translate_c.zig +++ b/src-self-hosted/translate_c.zig @@ -1486,17 +1486,9 @@ fn transDeclStmtOne(rp: RestorePoint, scope: *Scope, decl: *const ZigClangDecl, const underlying_type = ZigClangQualType_getTypePtr(underlying_qual); const mangled_name = try block_scope.makeMangledName(c, name); - if (checkForBuiltinTypedef(name)) |builtin| { - try block_scope.variables.append(.{ - .alias = builtin, - .name = mangled_name, - }); - @panic("what are we supposed to return here?"); - } else { - const node = (try transCreateNodeTypedef(rp, typedef_decl, false, mangled_name)) orelse - return error.UnsupportedTranslation; - return &node.base; - } + const node = (try transCreateNodeTypedef(rp, typedef_decl, false, mangled_name)) orelse + return error.UnsupportedTranslation; + return &node.base; }, else => |kind| return revertAndWarn( rp, From f771545a7ecfa77ad18745b11d80d0f5238ecb65 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2020 20:14:38 -0400 Subject: [PATCH 35/38] revert std.zig.ast iterator changes back to master branch API --- lib/std/zig/ast.zig | 486 ++++++++++---------------------------------- 1 file changed, 105 insertions(+), 381 deletions(-) diff --git a/lib/std/zig/ast.zig b/lib/std/zig/ast.zig index 0b4d4f6d89..a257b9e1c3 100644 --- a/lib/std/zig/ast.zig +++ b/lib/std/zig/ast.zig @@ -399,22 +399,6 @@ pub const Error = union(enum) { pub const Node = struct { id: Id, - /// All the child Node types use this same Iterator state for their iteration. - pub const Iterator = struct { - parent_node: *const Node, - index: usize, - - pub fn next(it: *Iterator) ?*Node { - inline for (@typeInfo(Id).Enum.fields) |f| { - if (it.parent_node.id == @field(Id, f.name)) { - const T = @field(Node, f.name); - return @fieldParentPtr(T, "base", it.parent_node).iterateNext(it); - } - } - unreachable; - } - }; - pub const Id = enum { // Top level Root, @@ -497,11 +481,11 @@ pub const Node = struct { return null; } - pub fn iterate(base: *Node) Iterator { + pub fn iterate(base: *Node, index: usize) ?*Node { inline for (@typeInfo(Id).Enum.fields) |f| { if (base.id == @field(Id, f.name)) { const T = @field(Node, f.name); - return @fieldParentPtr(T, "base", base).iterate(); + return @fieldParentPtr(T, "base", base).iterate(index); } } unreachable; @@ -647,13 +631,8 @@ pub const Node = struct { allocator.free(bytes); } - pub fn iterate(self: *const Root) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const Root, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const Root, index: usize) ?*Node { + var i = index; if (i < self.decls_len) return self.declsConst()[i]; return null; @@ -701,13 +680,8 @@ pub const Node = struct { init_node: ?*Node, semicolon_token: TokenIndex, - pub fn iterate(self: *const VarDecl) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const VarDecl, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const VarDecl, index: usize) ?*Node { + var i = index; if (self.type_node) |type_node| { if (i < 1) return type_node; @@ -755,13 +729,8 @@ pub const Node = struct { expr: *Node, semicolon_token: TokenIndex, - pub fn iterate(self: *const Use) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const Use, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const Use, index: usize) ?*Node { + var i = index; if (i < 1) return self.expr; i -= 1; @@ -796,13 +765,8 @@ pub const Node = struct { allocator.free(bytes); } - pub fn iterate(self: *const ErrorSetDecl) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const ErrorSetDecl, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const ErrorSetDecl, index: usize) ?*Node { + var i = index; if (i < self.decls_len) return self.declsConst()[i]; i -= self.decls_len; @@ -860,13 +824,8 @@ pub const Node = struct { allocator.free(bytes); } - pub fn iterate(self: *const ContainerDecl) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const ContainerDecl, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const ContainerDecl, index: usize) ?*Node { + var i = index; switch (self.init_arg_expr) { .Type => |t| { @@ -917,13 +876,8 @@ pub const Node = struct { value_expr: ?*Node, align_expr: ?*Node, - pub fn iterate(self: *const ContainerField) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const ContainerField, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const ContainerField, index: usize) ?*Node { + var i = index; if (self.type_expr) |type_expr| { if (i < 1) return type_expr; @@ -969,13 +923,8 @@ pub const Node = struct { doc_comments: ?*DocComment, name_token: TokenIndex, - pub fn iterate(self: *const ErrorTag) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const ErrorTag, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const ErrorTag, index: usize) ?*Node { + var i = index; if (self.doc_comments) |comments| { if (i < 1) return &comments.base; @@ -998,11 +947,7 @@ pub const Node = struct { base: Node = Node{ .id = .Identifier }, token: TokenIndex, - pub fn iterate(self: *const Identifier) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const Identifier, it: *Node.Iterator) ?*Node { + pub fn iterate(self: *const Identifier, index: usize) ?*Node { return null; } @@ -1053,13 +998,8 @@ pub const Node = struct { type_expr: *Node, }; - pub fn iterate(self: *const ParamDecl) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const ParamDecl, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const ParamDecl, index: usize) ?*Node { + var i = index; if (i < 1) { switch (self.param_type) { @@ -1101,30 +1041,26 @@ pub const Node = struct { allocator.free(bytes); } - pub fn iterate(self: *const FnProto) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const FnProto, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const FnProto, index: usize) ?*Node { + var i = index; if (self.lib_name) |lib_name| { if (i < 1) return lib_name; i -= 1; } - if (i < self.params_len) { + const params_len = switch (self.paramsConst()[self.params_len - 1].param_type) { + .var_type, .type_expr => self.params_len, + .var_args => self.params_len - 1, + }; + if (i < params_len) { switch (self.paramsConst()[i].param_type) { .var_type => |n| return n, - .var_args => { - i += 1; - it.index += 1; - }, + .var_args => unreachable, .type_expr => |n| return n, } } - i -= self.params_len; + i -= params_len; if (self.align_expr) |align_expr| { if (i < 1) return align_expr; @@ -1192,13 +1128,8 @@ pub const Node = struct { return_type: *Node, }; - pub fn iterate(self: *const AnyFrameType) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const AnyFrameType, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const AnyFrameType, index: usize) ?*Node { + var i = index; if (self.result) |result| { if (i < 1) return result.return_type; @@ -1237,13 +1168,8 @@ pub const Node = struct { allocator.free(bytes); } - pub fn iterate(self: *const Block) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const Block, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const Block, index: usize) ?*Node { + var i = index; if (i < self.statements_len) return self.statementsConst()[i]; i -= self.statements_len; @@ -1284,13 +1210,8 @@ pub const Node = struct { payload: ?*Node, expr: *Node, - pub fn iterate(self: *const Defer) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const Defer, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const Defer, index: usize) ?*Node { + var i = index; if (i < 1) return self.expr; i -= 1; @@ -1313,13 +1234,8 @@ pub const Node = struct { comptime_token: TokenIndex, expr: *Node, - pub fn iterate(self: *const Comptime) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const Comptime, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const Comptime, index: usize) ?*Node { + var i = index; if (i < 1) return self.expr; i -= 1; @@ -1341,13 +1257,8 @@ pub const Node = struct { nosuspend_token: TokenIndex, expr: *Node, - pub fn iterate(self: *const Nosuspend) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const Nosuspend, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const Nosuspend, index: usize) ?*Node { + var i = index; if (i < 1) return self.expr; i -= 1; @@ -1370,13 +1281,8 @@ pub const Node = struct { error_symbol: *Node, rpipe: TokenIndex, - pub fn iterate(self: *const Payload) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const Payload, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const Payload, index: usize) ?*Node { + var i = index; if (i < 1) return self.error_symbol; i -= 1; @@ -1400,13 +1306,8 @@ pub const Node = struct { value_symbol: *Node, rpipe: TokenIndex, - pub fn iterate(self: *const PointerPayload) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const PointerPayload, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const PointerPayload, index: usize) ?*Node { + var i = index; if (i < 1) return self.value_symbol; i -= 1; @@ -1431,13 +1332,8 @@ pub const Node = struct { index_symbol: ?*Node, rpipe: TokenIndex, - pub fn iterate(self: *const PointerIndexPayload) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const PointerIndexPayload, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const PointerIndexPayload, index: usize) ?*Node { + var i = index; if (i < 1) return self.value_symbol; i -= 1; @@ -1465,13 +1361,8 @@ pub const Node = struct { payload: ?*Node, body: *Node, - pub fn iterate(self: *const Else) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const Else, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const Else, index: usize) ?*Node { + var i = index; if (self.payload) |payload| { if (i < 1) return payload; @@ -1513,13 +1404,8 @@ pub const Node = struct { allocator.free(bytes); } - pub fn iterate(self: *const Switch) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const Switch, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const Switch, index: usize) ?*Node { + var i = index; if (i < 1) return self.expr; i -= 1; @@ -1572,13 +1458,8 @@ pub const Node = struct { allocator.free(bytes); } - pub fn iterate(self: *const SwitchCase) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const SwitchCase, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const SwitchCase, index: usize) ?*Node { + var i = index; if (i < self.items_len) return self.itemsConst()[i]; i -= self.items_len; @@ -1621,11 +1502,7 @@ pub const Node = struct { base: Node = Node{ .id = .SwitchElse }, token: TokenIndex, - pub fn iterate(self: *const SwitchElse) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const SwitchElse, it: *Node.Iterator) ?*Node { + pub fn iterate(self: *const SwitchElse, index: usize) ?*Node { return null; } @@ -1649,13 +1526,8 @@ pub const Node = struct { body: *Node, @"else": ?*Else, - pub fn iterate(self: *const While) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const While, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const While, index: usize) ?*Node { + var i = index; if (i < 1) return self.condition; i -= 1; @@ -1712,13 +1584,8 @@ pub const Node = struct { body: *Node, @"else": ?*Else, - pub fn iterate(self: *const For) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const For, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const For, index: usize) ?*Node { + var i = index; if (i < 1) return self.array_expr; i -= 1; @@ -1766,13 +1633,8 @@ pub const Node = struct { body: *Node, @"else": ?*Else, - pub fn iterate(self: *const If) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const If, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const If, index: usize) ?*Node { + var i = index; if (i < 1) return self.condition; i -= 1; @@ -1859,13 +1721,8 @@ pub const Node = struct { UnwrapOptional, }; - pub fn iterate(self: *const InfixOp) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const InfixOp, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const InfixOp, index: usize) ?*Node { + var i = index; if (i < 1) return self.lhs; i -= 1; @@ -1982,13 +1839,8 @@ pub const Node = struct { }; }; - pub fn iterate(self: *const PrefixOp) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const PrefixOp, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const PrefixOp, index: usize) ?*Node { + var i = index; switch (self.op) { .PtrType, .SliceType => |addr_of_info| { @@ -2045,13 +1897,8 @@ pub const Node = struct { name_token: TokenIndex, expr: *Node, - pub fn iterate(self: *const FieldInitializer) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const FieldInitializer, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const FieldInitializer, index: usize) ?*Node { + var i = index; if (i < 1) return self.expr; i -= 1; @@ -2086,13 +1933,8 @@ pub const Node = struct { allocator.free(bytes); } - pub fn iterate(self: *const ArrayInitializer) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const ArrayInitializer, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const ArrayInitializer, index: usize) ?*Node { + var i = index; if (i < 1) return self.lhs; i -= 1; @@ -2144,13 +1986,8 @@ pub const Node = struct { allocator.free(bytes); } - pub fn iterate(self: *const ArrayInitializerDot) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const ArrayInitializerDot, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const ArrayInitializerDot, index: usize) ?*Node { + var i = index; if (i < self.list_len) return self.listConst()[i]; i -= self.list_len; @@ -2199,13 +2036,8 @@ pub const Node = struct { allocator.free(bytes); } - pub fn iterate(self: *const StructInitializer) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const StructInitializer, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const StructInitializer, index: usize) ?*Node { + var i = index; if (i < 1) return self.lhs; i -= 1; @@ -2257,13 +2089,8 @@ pub const Node = struct { allocator.free(bytes); } - pub fn iterate(self: *const StructInitializerDot) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const StructInitializerDot, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const StructInitializerDot, index: usize) ?*Node { + var i = index; if (i < self.list_len) return self.listConst()[i]; i -= self.list_len; @@ -2313,13 +2140,8 @@ pub const Node = struct { allocator.free(bytes); } - pub fn iterate(self: *const Call) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0}; - } - - pub fn iterateNext(self: *const Call, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const Call, index: usize) ?*Node { + var i = index; if (i < 1) return self.lhs; i -= 1; @@ -2373,13 +2195,8 @@ pub const Node = struct { }; }; - pub fn iterate(self: *const SuffixOp) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0}; - } - - pub fn iterateNext(self: *const SuffixOp, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const SuffixOp, index: usize) ?*Node { + var i = index; if (i == 0) return self.lhs; i -= 1; @@ -2425,13 +2242,8 @@ pub const Node = struct { expr: *Node, rparen: TokenIndex, - pub fn iterate(self: *const GroupedExpression) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const GroupedExpression, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const GroupedExpression, index: usize) ?*Node { + var i = index; if (i < 1) return self.expr; i -= 1; @@ -2460,13 +2272,8 @@ pub const Node = struct { Return, }; - pub fn iterate(self: *const ControlFlowExpression) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const ControlFlowExpression, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const ControlFlowExpression, index: usize) ?*Node { + var i = index; switch (self.kind) { .Break, .Continue => |maybe_label| { @@ -2513,13 +2320,8 @@ pub const Node = struct { suspend_token: TokenIndex, body: ?*Node, - pub fn iterate(self: *const Suspend) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const Suspend, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const Suspend, index: usize) ?*Node { + var i = index; if (self.body) |body| { if (i < 1) return body; @@ -2546,11 +2348,7 @@ pub const Node = struct { base: Node = Node{ .id = .IntegerLiteral }, token: TokenIndex, - pub fn iterate(self: *const IntegerLiteral) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const IntegerLiteral, it: *Node.Iterator) ?*Node { + pub fn iterate(self: *const IntegerLiteral, index: usize) ?*Node { return null; } @@ -2568,11 +2366,7 @@ pub const Node = struct { dot: TokenIndex, name: TokenIndex, - pub fn iterate(self: *const EnumLiteral) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const EnumLiteral, it: *Node.Iterator) ?*Node { + pub fn iterate(self: *const EnumLiteral, index: usize) ?*Node { return null; } @@ -2589,11 +2383,7 @@ pub const Node = struct { base: Node = Node{ .id = .FloatLiteral }, token: TokenIndex, - pub fn iterate(self: *const FloatLiteral) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const FloatLiteral, it: *Node.Iterator) ?*Node { + pub fn iterate(self: *const FloatLiteral, index: usize) ?*Node { return null; } @@ -2624,13 +2414,8 @@ pub const Node = struct { allocator.free(bytes); } - pub fn iterate(self: *const BuiltinCall) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const BuiltinCall, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const BuiltinCall, index: usize) ?*Node { + var i = index; if (i < self.params_len) return self.paramsConst()[i]; i -= self.params_len; @@ -2665,11 +2450,7 @@ pub const Node = struct { base: Node = Node{ .id = .StringLiteral }, token: TokenIndex, - pub fn iterate(self: *const StringLiteral) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const StringLiteral, it: *Node.Iterator) ?*Node { + pub fn iterate(self: *const StringLiteral, index: usize) ?*Node { return null; } @@ -2698,11 +2479,7 @@ pub const Node = struct { allocator.free(bytes); } - pub fn iterate(self: *const MultilineStringLiteral) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const MultilineStringLiteral, it: *Node.Iterator) ?*Node { + pub fn iterate(self: *const MultilineStringLiteral, index: usize) ?*Node { return null; } @@ -2733,11 +2510,7 @@ pub const Node = struct { base: Node = Node{ .id = .CharLiteral }, token: TokenIndex, - pub fn iterate(self: *const CharLiteral) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const CharLiteral, it: *Node.Iterator) ?*Node { + pub fn iterate(self: *const CharLiteral, index: usize) ?*Node { return null; } @@ -2754,11 +2527,7 @@ pub const Node = struct { base: Node = Node{ .id = .BoolLiteral }, token: TokenIndex, - pub fn iterate(self: *const BoolLiteral) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const BoolLiteral, it: *Node.Iterator) ?*Node { + pub fn iterate(self: *const BoolLiteral, index: usize) ?*Node { return null; } @@ -2775,11 +2544,7 @@ pub const Node = struct { base: Node = Node{ .id = .NullLiteral }, token: TokenIndex, - pub fn iterate(self: *const NullLiteral) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const NullLiteral, it: *Node.Iterator) ?*Node { + pub fn iterate(self: *const NullLiteral, index: usize) ?*Node { return null; } @@ -2796,11 +2561,7 @@ pub const Node = struct { base: Node = Node{ .id = .UndefinedLiteral }, token: TokenIndex, - pub fn iterate(self: *const UndefinedLiteral) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const UndefinedLiteral, it: *Node.Iterator) ?*Node { + pub fn iterate(self: *const UndefinedLiteral, index: usize) ?*Node { return null; } @@ -2836,13 +2597,8 @@ pub const Node = struct { Return: *Node, }; - pub fn iterate(self: *const Output) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const Output, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const Output, index: usize) ?*Node { + var i = index; if (i < 1) return self.symbolic_name; i -= 1; @@ -2880,13 +2636,8 @@ pub const Node = struct { expr: *Node, rparen: TokenIndex, - pub fn iterate(self: *const Input) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const Input, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const Input, index: usize) ?*Node { + var i = index; if (i < 1) return self.symbolic_name; i -= 1; @@ -2910,13 +2661,8 @@ pub const Node = struct { }; - pub fn iterate(self: *const Asm) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0}; - } - - pub fn iterateNext(self: *const Asm, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const Asm, index: usize) ?*Node { + var i = index; if (i < self.outputs.len * 3) switch (i % 3) { 0 => return self.outputs[i / 3].symbolic_name, @@ -2953,11 +2699,7 @@ pub const Node = struct { base: Node = Node{ .id = .Unreachable }, token: TokenIndex, - pub fn iterate(self: *const Unreachable) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const Unreachable, it: *Node.Iterator) ?*Node { + pub fn iterate(self: *const Unreachable, index: usize) ?*Node { return null; } @@ -2974,11 +2716,7 @@ pub const Node = struct { base: Node = Node{ .id = .ErrorType }, token: TokenIndex, - pub fn iterate(self: *const ErrorType) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const ErrorType, it: *Node.Iterator) ?*Node { + pub fn iterate(self: *const ErrorType, index: usize) ?*Node { return null; } @@ -2995,11 +2733,7 @@ pub const Node = struct { base: Node = Node{ .id = .VarType }, token: TokenIndex, - pub fn iterate(self: *const VarType) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const VarType, it: *Node.Iterator) ?*Node { + pub fn iterate(self: *const VarType, index: usize) ?*Node { return null; } @@ -3019,11 +2753,7 @@ pub const Node = struct { /// at the first other token. first_line: TokenIndex, - pub fn iterate(self: *const DocComment) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const DocComment, it: *Node.Iterator) ?*Node { + pub fn iterate(self: *const DocComment, index: usize) ?*Node { return null; } @@ -3045,13 +2775,8 @@ pub const Node = struct { name: *Node, body_node: *Node, - pub fn iterate(self: *const TestDecl) Node.Iterator { - return .{ .parent_node = &self.base, .index = 0 }; - } - - pub fn iterateNext(self: *const TestDecl, it: *Node.Iterator) ?*Node { - var i = it.index; - it.index += 1; + pub fn iterate(self: *const TestDecl, index: usize) ?*Node { + var i = index; if (i < 1) return self.body_node; i -= 1; @@ -3076,6 +2801,5 @@ test "iterate" { .eof_token = 0, }; var base = &root.base; - var it = base.iterate(); - testing.expect(it.next() == null); + testing.expect(base.iterate(0) == null); } From c432811d96de1e8ac019ce16457c3992f039cb8c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2020 20:27:09 -0400 Subject: [PATCH 36/38] fix regression in compile errors --- src/analyze.cpp | 4 ++-- src/ir.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/analyze.cpp b/src/analyze.cpp index 0fddc568a5..88f967240a 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -3614,7 +3614,7 @@ static void add_top_level_decl(CodeGen *g, ScopeDecls *decls_scope, Tld *tld) { Tld *other_tld = entry->value; if (other_tld->id == TldIdVar) { ZigVar *var = reinterpret_cast(other_tld)->var; - if (var->var_type != nullptr && type_is_invalid(var->var_type)) { + if (var != nullptr && var->var_type != nullptr && type_is_invalid(var->var_type)) { return; // already reported compile error } } @@ -3896,7 +3896,7 @@ ZigVar *add_variable(CodeGen *g, AstNode *source_node, Scope *parent_scope, Buf bool want_err_msg = true; if (tld->id == TldIdVar) { ZigVar *var = reinterpret_cast(tld)->var; - if (var->var_type != nullptr && type_is_invalid(var->var_type)) { + if (var != nullptr && var->var_type != nullptr && type_is_invalid(var->var_type)) { want_err_msg = false; } } diff --git a/src/ir.cpp b/src/ir.cpp index cb0b6223c7..48c4162a77 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -5303,7 +5303,7 @@ static ZigVar *create_local_var(CodeGen *codegen, AstNode *node, Scope *parent_s bool want_err_msg = true; if (tld->id == TldIdVar) { ZigVar *var = reinterpret_cast(tld)->var; - if (var->var_type != nullptr && type_is_invalid(var->var_type)) { + if (var != nullptr && var->var_type != nullptr && type_is_invalid(var->var_type)) { want_err_msg = false; } } From 2952604d5d18078ed768e12010f4463176620e9a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2020 20:40:09 -0400 Subject: [PATCH 37/38] update docgen to new ast API --- doc/docgen.zig | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/doc/docgen.zig b/doc/docgen.zig index 92a7353fa1..7886c7cc90 100644 --- a/doc/docgen.zig +++ b/doc/docgen.zig @@ -776,7 +776,7 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Tok next_tok_is_fn = false; const token = tokenizer.next(); - try writeEscaped(out, src[index..token.start]); + try writeEscaped(out, src[index..token.loc.start]); switch (token.id) { .Eof => break, @@ -827,13 +827,13 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Tok .Keyword_while, => { try out.writeAll(""); - try writeEscaped(out, src[token.start..token.end]); + try writeEscaped(out, src[token.loc.start..token.loc.end]); try out.writeAll(""); }, .Keyword_fn => { try out.writeAll(""); - try writeEscaped(out, src[token.start..token.end]); + try writeEscaped(out, src[token.loc.start..token.loc.end]); try out.writeAll(""); next_tok_is_fn = true; }, @@ -844,7 +844,7 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Tok .Keyword_false, => { try out.writeAll(""); - try writeEscaped(out, src[token.start..token.end]); + try writeEscaped(out, src[token.loc.start..token.loc.end]); try out.writeAll(""); }, @@ -853,13 +853,13 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Tok .CharLiteral, => { try out.writeAll(""); - try writeEscaped(out, src[token.start..token.end]); + try writeEscaped(out, src[token.loc.start..token.loc.end]); try out.writeAll(""); }, .Builtin => { try out.writeAll(""); - try writeEscaped(out, src[token.start..token.end]); + try writeEscaped(out, src[token.loc.start..token.loc.end]); try out.writeAll(""); }, @@ -869,34 +869,34 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Tok .ShebangLine, => { try out.writeAll(""); - try writeEscaped(out, src[token.start..token.end]); + try writeEscaped(out, src[token.loc.start..token.loc.end]); try out.writeAll(""); }, .Identifier => { if (prev_tok_was_fn) { try out.writeAll(""); - try writeEscaped(out, src[token.start..token.end]); + try writeEscaped(out, src[token.loc.start..token.loc.end]); try out.writeAll(""); } else { const is_int = blk: { - if (src[token.start] != 'i' and src[token.start] != 'u') + if (src[token.loc.start] != 'i' and src[token.loc.start] != 'u') break :blk false; - var i = token.start + 1; - if (i == token.end) + var i = token.loc.start + 1; + if (i == token.loc.end) break :blk false; - while (i != token.end) : (i += 1) { + while (i != token.loc.end) : (i += 1) { if (src[i] < '0' or src[i] > '9') break :blk false; } break :blk true; }; - if (is_int or isType(src[token.start..token.end])) { + if (is_int or isType(src[token.loc.start..token.loc.end])) { try out.writeAll(""); - try writeEscaped(out, src[token.start..token.end]); + try writeEscaped(out, src[token.loc.start..token.loc.end]); try out.writeAll(""); } else { - try writeEscaped(out, src[token.start..token.end]); + try writeEscaped(out, src[token.loc.start..token.loc.end]); } } }, @@ -905,7 +905,7 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Tok .FloatLiteral, => { try out.writeAll(""); - try writeEscaped(out, src[token.start..token.end]); + try writeEscaped(out, src[token.loc.start..token.loc.end]); try out.writeAll(""); }, @@ -963,7 +963,7 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Tok .AngleBracketAngleBracketRight, .AngleBracketAngleBracketRightEqual, .Tilde, - => try writeEscaped(out, src[token.start..token.end]), + => try writeEscaped(out, src[token.loc.start..token.loc.end]), .Invalid, .Invalid_ampersands => return parseError( docgen_tokenizer, @@ -972,7 +972,7 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Tok .{}, ), } - index = token.end; + index = token.loc.end; } try out.writeAll(""); } From 8a3cd82b854164758eef17440d86105472e9e7a9 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 23 May 2020 23:15:58 -0400 Subject: [PATCH 38/38] translate-c: fix a use-after-free bug --- src-self-hosted/translate_c.zig | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src-self-hosted/translate_c.zig b/src-self-hosted/translate_c.zig index 22b3386427..94122f465e 100644 --- a/src-self-hosted/translate_c.zig +++ b/src-self-hosted/translate_c.zig @@ -135,12 +135,13 @@ const Scope = struct { /// Given the desired name, return a name that does not shadow anything from outer scopes. /// Inserts the returned name into the scope. fn makeMangledName(scope: *Block, c: *Context, name: []const u8) ![]const u8 { - var proposed_name = name; + const name_copy = try c.arena.dupe(u8, name); + var proposed_name = name_copy; while (scope.contains(proposed_name)) { scope.mangle_count += 1; proposed_name = try std.fmt.allocPrint(c.arena, "{}_{}", .{ name, scope.mangle_count }); } - try scope.variables.append(.{ .name = name, .alias = proposed_name }); + try scope.variables.append(.{ .name = name_copy, .alias = proposed_name }); return proposed_name; }