From 8df0841d6ab964e2aec750a8ceeda51912ae448b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 22 May 2020 12:34:12 -0400 Subject: [PATCH] stage2 parser: token ids in their own array To prevent cache misses, token ids go in their own array, and the start/end offsets go in a different one. perf measurement before: 2,667,914 cache-misses:u 2,139,139,935 instructions:u 894,167,331 cycles:u perf measurement after: 1,757,723 cache-misses:u 2,069,932,298 instructions:u 858,105,570 cycles:u --- lib/std/zig/ast.zig | 50 +++++---- lib/std/zig/parse.zig | 168 +++++++++++++--------------- lib/std/zig/parser_test.zig | 2 +- lib/std/zig/render.zig | 188 +++++++++++++++++--------------- lib/std/zig/tokenizer.zig | 28 +++-- src-self-hosted/translate_c.zig | 21 ---- 6 files changed, 217 insertions(+), 240 deletions(-) diff --git a/lib/std/zig/ast.zig b/lib/std/zig/ast.zig index eb22ac2330..0b4d4f6d89 100644 --- a/lib/std/zig/ast.zig +++ b/lib/std/zig/ast.zig @@ -10,7 +10,8 @@ pub const NodeIndex = usize; pub const Tree = struct { /// Reference to externally-owned data. source: []const u8, - tokens: []const Token, + token_ids: []const Token.Id, + token_locs: []const Token.Loc, errors: []const Error, /// undefined on parse error (when errors field is not empty) root_node: *Node.Root, @@ -23,26 +24,27 @@ pub const Tree = struct { generated: bool = false, pub fn deinit(self: *Tree) void { - self.gpa.free(self.tokens); + self.gpa.free(self.token_ids); + self.gpa.free(self.token_locs); self.gpa.free(self.errors); self.arena.promote(self.gpa).deinit(); } pub fn renderError(self: *Tree, parse_error: *const Error, stream: var) !void { - return parse_error.render(self.tokens, stream); + return parse_error.render(self.token_ids, stream); } pub fn tokenSlice(self: *Tree, token_index: TokenIndex) []const u8 { - return self.tokenSlicePtr(self.tokens[token_index]); + return self.tokenSliceLoc(self.token_locs[token_index]); } - pub fn tokenSlicePtr(self: *Tree, token: Token) []const u8 { + pub fn tokenSliceLoc(self: *Tree, token: Token.Loc) []const u8 { return self.source[token.start..token.end]; } pub fn getNodeSource(self: *const Tree, node: *const Node) []const u8 { - const first_token = self.tokens[node.firstToken()]; - const last_token = self.tokens[node.lastToken()]; + const first_token = self.token_locs[node.firstToken()]; + const last_token = self.token_locs[node.lastToken()]; return self.source[first_token.start..last_token.end]; } @@ -54,7 +56,7 @@ pub const Tree = struct { }; /// Return the Location of the token relative to the offset specified by `start_index`. - pub fn tokenLocationPtr(self: *Tree, start_index: usize, token: Token) Location { + pub fn tokenLocationLoc(self: *Tree, start_index: usize, token: Token.Loc) Location { var loc = Location{ .line = 0, .column = 0, @@ -82,14 +84,14 @@ pub const Tree = struct { } pub fn tokenLocation(self: *Tree, start_index: usize, token_index: TokenIndex) Location { - return self.tokenLocationPtr(start_index, self.tokens[token_index]); + return self.tokenLocationLoc(start_index, self.token_locs[token_index]); } pub fn tokensOnSameLine(self: *Tree, token1_index: TokenIndex, token2_index: TokenIndex) bool { - return self.tokensOnSameLinePtr(self.tokens[token1_index], self.tokens[token2_index]); + return self.tokensOnSameLineLoc(self.token_locs[token1_index], self.token_locs[token2_index]); } - pub fn tokensOnSameLinePtr(self: *Tree, token1: Token, token2: Token) bool { + pub fn tokensOnSameLineLoc(self: *Tree, token1: Token.Loc, token2: Token.Loc) bool { return mem.indexOfScalar(u8, self.source[token1.end..token2.start], '\n') == null; } @@ -100,7 +102,7 @@ pub const Tree = struct { /// Skips over comments pub fn prevToken(self: *Tree, token_index: TokenIndex) TokenIndex { var index = token_index - 1; - while (self.tokens[index].id == Token.Id.LineComment) { + while (self.token_ids[index] == Token.Id.LineComment) { index -= 1; } return index; @@ -109,7 +111,7 @@ pub const Tree = struct { /// Skips over comments pub fn nextToken(self: *Tree, token_index: TokenIndex) TokenIndex { var index = token_index + 1; - while (self.tokens[index].id == Token.Id.LineComment) { + while (self.token_ids[index] == Token.Id.LineComment) { index += 1; } return index; @@ -166,7 +168,7 @@ pub const Error = union(enum) { DeclBetweenFields: DeclBetweenFields, InvalidAnd: InvalidAnd, - pub fn render(self: *const Error, tokens: []const Token, stream: var) !void { + pub fn render(self: *const Error, tokens: []const Token.Id, stream: var) !void { switch (self.*) { .InvalidToken => |*x| return x.render(tokens, stream), .ExpectedContainerMembers => |*x| return x.render(tokens, stream), @@ -321,7 +323,7 @@ pub const Error = union(enum) { pub const ExpectedCall = struct { node: *Node, - pub fn render(self: *const ExpectedCall, tokens: []const Token, stream: var) !void { + pub fn render(self: *const ExpectedCall, tokens: []const Token.Id, stream: var) !void { return stream.print("expected " ++ @tagName(Node.Id.Call) ++ ", found {}", .{ @tagName(self.node.id), }); @@ -331,7 +333,7 @@ pub const Error = union(enum) { pub const ExpectedCallOrFnProto = struct { node: *Node, - pub fn render(self: *const ExpectedCallOrFnProto, tokens: []const Token, stream: var) !void { + pub fn render(self: *const ExpectedCallOrFnProto, tokens: []const Token.Id, stream: var) !void { return stream.print("expected " ++ @tagName(Node.Id.Call) ++ " or " ++ @tagName(Node.Id.FnProto) ++ ", found {}", .{@tagName(self.node.id)}); } @@ -341,14 +343,14 @@ pub const Error = union(enum) { token: TokenIndex, expected_id: Token.Id, - pub fn render(self: *const ExpectedToken, tokens: []const Token, stream: var) !void { + pub fn render(self: *const ExpectedToken, tokens: []const Token.Id, stream: var) !void { const found_token = tokens[self.token]; - switch (found_token.id) { + switch (found_token) { .Invalid => { return stream.print("expected '{}', found invalid bytes", .{self.expected_id.symbol()}); }, else => { - const token_name = found_token.id.symbol(); + const token_name = found_token.symbol(); return stream.print("expected '{}', found '{}'", .{ self.expected_id.symbol(), token_name }); }, } @@ -359,11 +361,11 @@ pub const Error = union(enum) { token: TokenIndex, end_id: Token.Id, - pub fn render(self: *const ExpectedCommaOrEnd, tokens: []const Token, stream: var) !void { + pub fn render(self: *const ExpectedCommaOrEnd, tokens: []const Token.Id, stream: var) !void { const actual_token = tokens[self.token]; return stream.print("expected ',' or '{}', found '{}'", .{ self.end_id.symbol(), - actual_token.id.symbol(), + actual_token.symbol(), }); } }; @@ -374,9 +376,9 @@ pub const Error = union(enum) { token: TokenIndex, - pub fn render(self: *const ThisError, tokens: []const Token, stream: var) !void { + pub fn render(self: *const ThisError, tokens: []const Token.Id, stream: var) !void { const actual_token = tokens[self.token]; - return stream.print(msg, .{actual_token.id.symbol()}); + return stream.print(msg, .{actual_token.symbol()}); } }; } @@ -387,7 +389,7 @@ pub const Error = union(enum) { token: TokenIndex, - pub fn render(self: *const ThisError, tokens: []const Token, stream: var) !void { + pub fn render(self: *const ThisError, tokens: []const Token.Id, stream: var) !void { return stream.writeAll(msg); } }; diff --git a/lib/std/zig/parse.zig b/lib/std/zig/parse.zig index 6d568cf5ec..a8aadc5ef7 100644 --- a/lib/std/zig/parse.zig +++ b/lib/std/zig/parse.zig @@ -16,28 +16,32 @@ pub const Error = error{ParseError} || Allocator.Error; pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!*Tree { // TODO optimization idea: ensureCapacity on the tokens list and // then appendAssumeCapacity inside the loop. - var tokens = std.ArrayList(Token).init(gpa); - defer tokens.deinit(); + var token_ids = std.ArrayList(Token.Id).init(gpa); + defer token_ids.deinit(); + var token_locs = std.ArrayList(Token.Loc).init(gpa); + defer token_locs.deinit(); var tokenizer = std.zig.Tokenizer.init(source); while (true) { - const tree_token = try tokens.addOne(); - tree_token.* = tokenizer.next(); - if (tree_token.id == .Eof) break; + const token = tokenizer.next(); + try token_ids.append(token.id); + try token_locs.append(token.loc); + if (token.id == .Eof) break; } var parser: Parser = .{ .source = source, .arena = std.heap.ArenaAllocator.init(gpa), .gpa = gpa, - .tokens = tokens.items, + .token_ids = token_ids.items, + .token_locs = token_locs.items, .errors = .{}, .tok_i = 0, }; defer parser.errors.deinit(gpa); errdefer parser.arena.deinit(); - while (tokens.items[parser.tok_i].id == .LineComment) parser.tok_i += 1; + while (token_ids.items[parser.tok_i] == .LineComment) parser.tok_i += 1; const root_node = try parser.parseRoot(); @@ -45,7 +49,8 @@ pub fn parse(gpa: *Allocator, source: []const u8) Allocator.Error!*Tree { tree.* = .{ .gpa = gpa, .source = source, - .tokens = tokens.toOwnedSlice(), + .token_ids = token_ids.toOwnedSlice(), + .token_locs = token_locs.toOwnedSlice(), .errors = parser.errors.toOwnedSlice(gpa), .root_node = root_node, .arena = parser.arena.state, @@ -58,9 +63,8 @@ const Parser = struct { arena: std.heap.ArenaAllocator, gpa: *Allocator, source: []const u8, - /// TODO: Optimization idea: have this be several arrays of the token fields rather - /// than an array of structs. - tokens: []const Token, + token_ids: []const Token.Id, + token_locs: []const Token.Loc, tok_i: TokenIndex, errors: std.ArrayListUnmanaged(AstError), @@ -80,19 +84,6 @@ const Parser = struct { return node; } - /// Helper function for appending elements to a singly linked list. - fn llpush( - p: *Parser, - comptime T: type, - it: *?*std.SinglyLinkedList(T).Node, - data: T, - ) !*?*std.SinglyLinkedList(T).Node { - const llnode = try p.arena.allocator.create(std.SinglyLinkedList(T).Node); - llnode.* = .{ .data = data }; - it.* = llnode; - return &llnode.next; - } - /// ContainerMembers /// <- TestDecl ContainerMembers /// / TopLevelComptime ContainerMembers @@ -228,7 +219,7 @@ const Parser = struct { // try to continue parsing const index = p.tok_i; p.findNextContainerMember(); - const next = p.tokens[p.tok_i].id; + const next = p.token_ids[p.tok_i]; switch (next) { .Eof => break, else => { @@ -257,7 +248,7 @@ const Parser = struct { }); } - const next = p.tokens[p.tok_i].id; + const next = p.token_ids[p.tok_i]; switch (next) { .Eof => break, .Keyword_comptime => { @@ -291,7 +282,7 @@ const Parser = struct { var level: u32 = 0; while (true) { const tok = p.nextToken(); - switch (tok.ptr.id) { + switch (p.token_ids[tok]) { // any of these can start a new top level declaration .Keyword_test, .Keyword_comptime, @@ -308,7 +299,7 @@ const Parser = struct { .Identifier, => { if (level == 0) { - p.putBackToken(tok.index); + p.putBackToken(tok); return; } }, @@ -325,13 +316,13 @@ const Parser = struct { .RBrace => { if (level == 0) { // end of container, exit - p.putBackToken(tok.index); + p.putBackToken(tok); return; } level -= 1; }, .Eof => { - p.putBackToken(tok.index); + p.putBackToken(tok); return; }, else => {}, @@ -344,11 +335,11 @@ const Parser = struct { var level: u32 = 0; while (true) { const tok = p.nextToken(); - switch (tok.ptr.id) { + switch (p.token_ids[tok]) { .LBrace => level += 1, .RBrace => { if (level == 0) { - p.putBackToken(tok.index); + p.putBackToken(tok); return; } level -= 1; @@ -359,7 +350,7 @@ const Parser = struct { } }, .Eof => { - p.putBackToken(tok.index); + p.putBackToken(tok); return; }, else => {}, @@ -454,8 +445,8 @@ const Parser = struct { } if (extern_export_inline_token) |token| { - if (p.tokens[token].id == .Keyword_inline or - p.tokens[token].id == .Keyword_noinline) + if (p.token_ids[token] == .Keyword_inline or + p.token_ids[token] == .Keyword_noinline) { try p.errors.append(p.gpa, .{ .ExpectedFn = .{ .token = p.tok_i }, @@ -722,7 +713,7 @@ const Parser = struct { const defer_token = p.eatToken(.Keyword_defer) orelse p.eatToken(.Keyword_errdefer); if (defer_token) |token| { - const payload = if (p.tokens[token].id == .Keyword_errdefer) + const payload = if (p.token_ids[token] == .Keyword_errdefer) try p.parsePayload() else null; @@ -2269,7 +2260,7 @@ const Parser = struct { /// / EQUAL fn parseAssignOp(p: *Parser) !?*Node { const token = p.nextToken(); - const op: Node.InfixOp.Op = switch (token.ptr.id) { + const op: Node.InfixOp.Op = switch (p.token_ids[token]) { .AsteriskEqual => .AssignMul, .SlashEqual => .AssignDiv, .PercentEqual => .AssignMod, @@ -2285,14 +2276,14 @@ const Parser = struct { .MinusPercentEqual => .AssignSubWrap, .Equal => .Assign, else => { - p.putBackToken(token.index); + p.putBackToken(token); return null; }, }; const node = try p.arena.allocator.create(Node.InfixOp); node.* = .{ - .op_token = token.index, + .op_token = token, .lhs = undefined, // set by caller .op = op, .rhs = undefined, // set by caller @@ -2309,7 +2300,7 @@ const Parser = struct { /// / RARROWEQUAL fn parseCompareOp(p: *Parser) !?*Node { const token = p.nextToken(); - const op: Node.InfixOp.Op = switch (token.ptr.id) { + const op: Node.InfixOp.Op = switch (p.token_ids[token]) { .EqualEqual => .EqualEqual, .BangEqual => .BangEqual, .AngleBracketLeft => .LessThan, @@ -2317,12 +2308,12 @@ const Parser = struct { .AngleBracketLeftEqual => .LessOrEqual, .AngleBracketRightEqual => .GreaterOrEqual, else => { - p.putBackToken(token.index); + p.putBackToken(token); return null; }, }; - return p.createInfixOp(token.index, op); + return p.createInfixOp(token, op); } /// BitwiseOp @@ -2333,19 +2324,19 @@ const Parser = struct { /// / KEYWORD_catch Payload? fn parseBitwiseOp(p: *Parser) !?*Node { const token = p.nextToken(); - const op: Node.InfixOp.Op = switch (token.ptr.id) { + const op: Node.InfixOp.Op = switch (p.token_ids[token]) { .Ampersand => .BitAnd, .Caret => .BitXor, .Pipe => .BitOr, .Keyword_orelse => .UnwrapOptional, .Keyword_catch => .{ .Catch = try p.parsePayload() }, else => { - p.putBackToken(token.index); + p.putBackToken(token); return null; }, }; - return p.createInfixOp(token.index, op); + return p.createInfixOp(token, op); } /// BitShiftOp @@ -2353,16 +2344,16 @@ const Parser = struct { /// / RARROW2 fn parseBitShiftOp(p: *Parser) !?*Node { const token = p.nextToken(); - const op: Node.InfixOp.Op = switch (token.ptr.id) { + const op: Node.InfixOp.Op = switch (p.token_ids[token]) { .AngleBracketAngleBracketLeft => .BitShiftLeft, .AngleBracketAngleBracketRight => .BitShiftRight, else => { - p.putBackToken(token.index); + p.putBackToken(token); return null; }, }; - return p.createInfixOp(token.index, op); + return p.createInfixOp(token, op); } /// AdditionOp @@ -2373,19 +2364,19 @@ const Parser = struct { /// / MINUSPERCENT fn parseAdditionOp(p: *Parser) !?*Node { const token = p.nextToken(); - const op: Node.InfixOp.Op = switch (token.ptr.id) { + const op: Node.InfixOp.Op = switch (p.token_ids[token]) { .Plus => .Add, .Minus => .Sub, .PlusPlus => .ArrayCat, .PlusPercent => .AddWrap, .MinusPercent => .SubWrap, else => { - p.putBackToken(token.index); + p.putBackToken(token); return null; }, }; - return p.createInfixOp(token.index, op); + return p.createInfixOp(token, op); } /// MultiplyOp @@ -2397,7 +2388,7 @@ const Parser = struct { /// / ASTERISKPERCENT fn parseMultiplyOp(p: *Parser) !?*Node { const token = p.nextToken(); - const op: Node.InfixOp.Op = switch (token.ptr.id) { + const op: Node.InfixOp.Op = switch (p.token_ids[token]) { .PipePipe => .MergeErrorSets, .Asterisk => .Mul, .Slash => .Div, @@ -2405,12 +2396,12 @@ const Parser = struct { .AsteriskAsterisk => .ArrayMult, .AsteriskPercent => .MulWrap, else => { - p.putBackToken(token.index); + p.putBackToken(token); return null; }, }; - return p.createInfixOp(token.index, op); + return p.createInfixOp(token, op); } /// PrefixOp @@ -2423,7 +2414,7 @@ const Parser = struct { /// / KEYWORD_await fn parsePrefixOp(p: *Parser) !?*Node { const token = p.nextToken(); - const op: Node.PrefixOp.Op = switch (token.ptr.id) { + const op: Node.PrefixOp.Op = switch (p.token_ids[token]) { .Bang => .BoolNot, .Minus => .Negation, .Tilde => .BitNot, @@ -2432,14 +2423,14 @@ const Parser = struct { .Keyword_try => .Try, .Keyword_await => .Await, else => { - p.putBackToken(token.index); + p.putBackToken(token); return null; }, }; const node = try p.arena.allocator.create(Node.PrefixOp); node.* = .{ - .op_token = token.index, + .op_token = token, .op = op, .rhs = undefined, // set by caller }; @@ -2493,7 +2484,7 @@ const Parser = struct { // If the token encountered was **, there will be two nodes instead of one. // The attributes should be applied to the rightmost operator. const prefix_op = node.cast(Node.PrefixOp).?; - var ptr_info = if (p.tokens[prefix_op.op_token].id == .AsteriskAsterisk) + var ptr_info = if (p.token_ids[prefix_op.op_token] == .AsteriskAsterisk) &prefix_op.rhs.cast(Node.PrefixOp).?.op.PtrType else &prefix_op.op.PtrType; @@ -2812,7 +2803,8 @@ const Parser = struct { return null; }; if (p.eatToken(.Identifier)) |ident| { - const token_slice = p.source[p.tokens[ident].start..p.tokens[ident].end]; + const token_loc = p.token_locs[ident]; + const token_slice = p.source[token_loc.start..token_loc.end]; if (!std.mem.eql(u8, token_slice, "c")) { p.putBackToken(ident); } else { @@ -2879,7 +2871,7 @@ const Parser = struct { fn parseContainerDeclType(p: *Parser) !?ContainerDeclType { const kind_token = p.nextToken(); - const init_arg_expr = switch (kind_token.ptr.id) { + const init_arg_expr = switch (p.token_ids[kind_token]) { .Keyword_struct => Node.ContainerDecl.InitArg{ .None = {} }, .Keyword_enum => blk: { if (p.eatToken(.LParen) != null) { @@ -2914,13 +2906,13 @@ const Parser = struct { break :blk Node.ContainerDecl.InitArg{ .None = {} }; }, else => { - p.putBackToken(kind_token.index); + p.putBackToken(kind_token); return null; }, }; return ContainerDeclType{ - .kind_token = kind_token.index, + .kind_token = kind_token, .init_arg_expr = init_arg_expr, }; } @@ -2973,7 +2965,7 @@ const Parser = struct { while (try nodeParseFn(p)) |item| { try list.append(item); - switch (p.tokens[p.tok_i].id) { + switch (p.token_ids[p.tok_i]) { .Comma => _ = p.nextToken(), // all possible delimiters .Colon, .RParen, .RBrace, .RBracket => break, @@ -2994,13 +2986,13 @@ const Parser = struct { fn SimpleBinOpParseFn(comptime token: Token.Id, comptime op: Node.InfixOp.Op) NodeParseFn { return struct { pub fn parse(p: *Parser) Error!?*Node { - const op_token = if (token == .Keyword_and) switch (p.tokens[p.tok_i].id) { - .Keyword_and => p.nextToken().index, + const op_token = if (token == .Keyword_and) switch (p.token_ids[p.tok_i]) { + .Keyword_and => p.nextToken(), .Invalid_ampersands => blk: { try p.errors.append(p.gpa, .{ .InvalidAnd = .{ .token = p.tok_i }, }); - break :blk p.nextToken().index; + break :blk p.nextToken(); }, else => return null, } else p.eatToken(token) orelse return null; @@ -3104,7 +3096,7 @@ const Parser = struct { var tok_i = start_tok_i; var count: usize = 1; // including first_line while (true) : (tok_i += 1) { - switch (p.tokens[tok_i].id) { + switch (p.token_ids[tok_i]) { .LineComment => continue, .MultilineStringLiteralLine => count += 1, else => break, @@ -3118,7 +3110,7 @@ const Parser = struct { lines[0] = first_line; count = 1; while (true) : (tok_i += 1) { - switch (p.tokens[tok_i].id) { + switch (p.token_ids[tok_i]) { .LineComment => continue, .MultilineStringLiteralLine => { lines[count] = tok_i; @@ -3215,7 +3207,7 @@ const Parser = struct { } fn tokensOnSameLine(p: *Parser, token1: TokenIndex, token2: TokenIndex) bool { - return std.mem.indexOfScalar(u8, p.source[p.tokens[token1].end..p.tokens[token2].start], '\n') == null; + return std.mem.indexOfScalar(u8, p.source[p.token_locs[token1].end..p.token_locs[token2].start], '\n') == null; } /// Eat a single-line doc comment on the same line as another node @@ -3239,7 +3231,7 @@ const Parser = struct { .PrefixOp => { var prefix_op = rightmost_op.cast(Node.PrefixOp).?; // If the token encountered was **, there will be two nodes - if (p.tokens[prefix_op.op_token].id == .AsteriskAsterisk) { + if (p.token_ids[prefix_op.op_token] == .AsteriskAsterisk) { rightmost_op = prefix_op.rhs; prefix_op = rightmost_op.cast(Node.PrefixOp).?; } @@ -3328,11 +3320,7 @@ const Parser = struct { } fn eatToken(p: *Parser, id: Token.Id) ?TokenIndex { - return if (p.eatAnnotatedToken(id)) |token| token.index else null; - } - - fn eatAnnotatedToken(p: *Parser, id: Token.Id) ?AnnotatedToken { - return if (p.tokens[p.tok_i].id == id) p.nextToken() else null; + return if (p.token_ids[p.tok_i] == id) p.nextToken() else null; } fn expectToken(p: *Parser, id: Token.Id) Error!TokenIndex { @@ -3341,29 +3329,25 @@ const Parser = struct { fn expectTokenRecoverable(p: *Parser, id: Token.Id) !?TokenIndex { const token = p.nextToken(); - if (token.ptr.id != id) { + if (p.token_ids[token] != id) { try p.errors.append(p.gpa, .{ - .ExpectedToken = .{ .token = token.index, .expected_id = id }, + .ExpectedToken = .{ .token = token, .expected_id = id }, }); // go back so that we can recover properly - p.putBackToken(token.index); + p.putBackToken(token); return null; } - return token.index; + return token; } - fn nextToken(p: *Parser) AnnotatedToken { - const result = AnnotatedToken{ - .index = p.tok_i, - .ptr = &p.tokens[p.tok_i], - }; + fn nextToken(p: *Parser) TokenIndex { + const result = p.tok_i; p.tok_i += 1; - assert(result.ptr.id != .LineComment); - if (p.tok_i >= p.tokens.len) return result; + assert(p.token_ids[result] != .LineComment); + if (p.tok_i >= p.token_ids.len) return result; while (true) { - const next_tok = p.tokens[p.tok_i]; - if (next_tok.id != .LineComment) return result; + if (p.token_ids[p.tok_i] != .LineComment) return result; p.tok_i += 1; } } @@ -3371,18 +3355,12 @@ const Parser = struct { fn putBackToken(p: *Parser, putting_back: TokenIndex) void { while (p.tok_i > 0) { p.tok_i -= 1; - const prev_tok = p.tokens[p.tok_i]; - if (prev_tok.id == .LineComment) continue; + if (p.token_ids[p.tok_i] == .LineComment) continue; assert(putting_back == p.tok_i); return; } } - const AnnotatedToken = struct { - index: TokenIndex, - ptr: *const Token, - }; - fn expectNode( p: *Parser, parseFn: NodeParseFn, diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index 32667a02fb..25fc48e46d 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -3181,7 +3181,7 @@ fn testParse(source: []const u8, allocator: *mem.Allocator, anything_changed: *b defer tree.deinit(); for (tree.errors) |*parse_error| { - const token = tree.tokens[parse_error.loc()]; + const token = tree.token_locs[parse_error.loc()]; const loc = tree.tokenLocation(0, parse_error.loc()); try stderr.print("(memory buffer):{}:{}: error: ", .{ loc.line + 1, loc.column + 1 }); try tree.renderError(parse_error, stderr); diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index bc38cf4067..690592bc17 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -68,11 +68,12 @@ fn renderRoot( tree: *ast.Tree, ) (@TypeOf(stream).Error || Error)!void { // render all the line comments at the beginning of the file - for (tree.tokens) |token, i| { - if (token.id != .LineComment) break; - try stream.print("{}\n", .{mem.trimRight(u8, tree.tokenSlicePtr(token), " ")}); - const next_token = &tree.tokens[i + 1]; - const loc = tree.tokenLocationPtr(token.end, next_token.*); + for (tree.token_ids) |token_id, i| { + if (token_id != .LineComment) break; + const token_loc = tree.token_locs[i]; + try stream.print("{}\n", .{mem.trimRight(u8, tree.tokenSliceLoc(token_loc), " ")}); + const next_token = tree.token_locs[i + 1]; + const loc = tree.tokenLocationLoc(token_loc.end, next_token); if (loc.line >= 2) { try stream.writeByte('\n'); } @@ -101,8 +102,8 @@ fn renderRoot( while (token_index != 0) { token_index -= 1; - const token = tree.tokens[token_index]; - switch (token.id) { + const token_id = tree.token_ids[token_index]; + switch (token_id) { .LineComment => {}, .DocComment => { copy_start_token_index = token_index; @@ -111,12 +112,13 @@ fn renderRoot( else => break, } - if (mem.eql(u8, mem.trim(u8, tree.tokenSlicePtr(token)[2..], " "), "zig fmt: off")) { + const token_loc = tree.token_locs[token_index]; + if (mem.eql(u8, mem.trim(u8, tree.tokenSliceLoc(token_loc)[2..], " "), "zig fmt: off")) { if (!found_fmt_directive) { fmt_active = false; found_fmt_directive = true; } - } else if (mem.eql(u8, mem.trim(u8, tree.tokenSlicePtr(token)[2..], " "), "zig fmt: on")) { + } else if (mem.eql(u8, mem.trim(u8, tree.tokenSliceLoc(token_loc)[2..], " "), "zig fmt: on")) { if (!found_fmt_directive) { fmt_active = true; found_fmt_directive = true; @@ -135,7 +137,7 @@ fn renderRoot( if (decl_i >= root_decls.len) { // If there's no next reformatted `decl`, just copy the // remaining input tokens and bail out. - const start = tree.tokens[copy_start_token_index].start; + const start = tree.token_locs[copy_start_token_index].start; try copyFixingWhitespace(stream, tree.source[start..]); return; } @@ -143,15 +145,16 @@ fn renderRoot( var decl_first_token_index = decl.firstToken(); while (token_index < decl_first_token_index) : (token_index += 1) { - const token = tree.tokens[token_index]; - switch (token.id) { + const token_id = tree.token_ids[token_index]; + switch (token_id) { .LineComment => {}, .Eof => unreachable, else => continue, } - if (mem.eql(u8, mem.trim(u8, tree.tokenSlicePtr(token)[2..], " "), "zig fmt: on")) { + const token_loc = tree.token_locs[token_index]; + if (mem.eql(u8, mem.trim(u8, tree.tokenSliceLoc(token_loc)[2..], " "), "zig fmt: on")) { fmt_active = true; - } else if (mem.eql(u8, mem.trim(u8, tree.tokenSlicePtr(token)[2..], " "), "zig fmt: off")) { + } else if (mem.eql(u8, mem.trim(u8, tree.tokenSliceLoc(token_loc)[2..], " "), "zig fmt: off")) { fmt_active = false; } } @@ -163,8 +166,8 @@ fn renderRoot( token_index = copy_end_token_index; while (token_index != 0) { token_index -= 1; - const token = tree.tokens[token_index]; - switch (token.id) { + const token_id = tree.token_ids[token_index]; + switch (token_id) { .LineComment => {}, .DocComment => { copy_end_token_index = token_index; @@ -174,8 +177,8 @@ fn renderRoot( } } - const start = tree.tokens[copy_start_token_index].start; - const end = tree.tokens[copy_end_token_index].start; + const start = tree.token_locs[copy_start_token_index].start; + const end = tree.token_locs[copy_end_token_index].start; try copyFixingWhitespace(stream, tree.source[start..end]); } @@ -194,13 +197,13 @@ fn renderExtraNewlineToken(tree: *ast.Tree, stream: var, start_col: *usize, firs var prev_token = first_token; if (prev_token == 0) return; var newline_threshold: usize = 2; - while (tree.tokens[prev_token - 1].id == .DocComment) { - if (tree.tokenLocation(tree.tokens[prev_token - 1].end, prev_token).line == 1) { + while (tree.token_ids[prev_token - 1] == .DocComment) { + if (tree.tokenLocation(tree.token_locs[prev_token - 1].end, prev_token).line == 1) { newline_threshold += 1; } prev_token -= 1; } - const prev_token_end = tree.tokens[prev_token - 1].end; + const prev_token_end = tree.token_locs[prev_token - 1].end; const loc = tree.tokenLocation(prev_token_end, first_token); if (loc.line >= newline_threshold) { try stream.writeByte('\n'); @@ -265,7 +268,7 @@ fn renderContainerDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree, const src_has_trailing_comma = blk: { const maybe_comma = tree.nextToken(field.lastToken()); - break :blk tree.tokens[maybe_comma].id == .Comma; + break :blk tree.token_ids[maybe_comma] == .Comma; }; // The trailing comma is emitted at the end, but if it's not present @@ -327,11 +330,11 @@ fn renderContainerDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree, .DocComment => { const comment = @fieldParentPtr(ast.Node.DocComment, "base", decl); - const kind = tree.tokens[comment.first_line].id; + const kind = tree.token_ids[comment.first_line]; try renderToken(tree, stream, comment.first_line, indent, start_col, .Newline); var tok_i = comment.first_line + 1; while (true) : (tok_i += 1) { - const tok_id = tree.tokens[tok_i].id; + const tok_id = tree.token_ids[tok_i]; if (tok_id == kind) { try stream.writeByteNTimes(' ', indent); try renderToken(tree, stream, tok_i, indent, start_col, .Newline); @@ -436,13 +439,13 @@ fn renderExpression( try renderExpression(allocator, stream, tree, indent, start_col, infix_op_node.lhs, op_space); const after_op_space = blk: { - const loc = tree.tokenLocation(tree.tokens[infix_op_node.op_token].end, tree.nextToken(infix_op_node.op_token)); + const loc = tree.tokenLocation(tree.token_locs[infix_op_node.op_token].end, tree.nextToken(infix_op_node.op_token)); break :blk if (loc.line == 0) op_space else Space.Newline; }; try renderToken(tree, stream, infix_op_node.op_token, indent, start_col, after_op_space); if (after_op_space == Space.Newline and - tree.tokens[tree.nextToken(infix_op_node.op_token)].id != .MultilineStringLiteralLine) + tree.token_ids[tree.nextToken(infix_op_node.op_token)] != .MultilineStringLiteralLine) { try stream.writeByteNTimes(' ', indent + indent_delta); start_col.* = indent + indent_delta; @@ -463,10 +466,10 @@ fn renderExpression( switch (prefix_op_node.op) { .PtrType => |ptr_info| { - const op_tok_id = tree.tokens[prefix_op_node.op_token].id; + const op_tok_id = tree.token_ids[prefix_op_node.op_token]; switch (op_tok_id) { .Asterisk, .AsteriskAsterisk => try stream.writeByte('*'), - .LBracket => if (tree.tokens[prefix_op_node.op_token + 2].id == .Identifier) + .LBracket => if (tree.token_ids[prefix_op_node.op_token + 2] == .Identifier) try stream.writeAll("[*c") else try stream.writeAll("[*"), @@ -578,8 +581,8 @@ fn renderExpression( try renderToken(tree, stream, lbracket, indent, start_col, Space.None); // [ - const starts_with_comment = tree.tokens[lbracket + 1].id == .LineComment; - const ends_with_comment = tree.tokens[rbracket - 1].id == .LineComment; + const starts_with_comment = tree.token_ids[lbracket + 1] == .LineComment; + const ends_with_comment = tree.token_ids[rbracket - 1] == .LineComment; const new_indent = if (ends_with_comment) indent + indent_delta else indent; const new_space = if (ends_with_comment) Space.Newline else Space.None; try renderExpression(allocator, stream, tree, new_indent, start_col, array_info.len_expr, new_space); @@ -653,7 +656,7 @@ fn renderExpression( return renderToken(tree, stream, rtoken, indent, start_col, space); } - if (exprs.len == 1 and tree.tokens[exprs[0].lastToken() + 1].id == .RBrace) { + if (exprs.len == 1 and tree.token_ids[exprs[0].lastToken() + 1] == .RBrace) { const expr = exprs[0]; switch (lhs) { .dot => |dot| try renderToken(tree, stream, dot, indent, start_col, Space.None), @@ -675,17 +678,17 @@ fn renderExpression( for (exprs) |expr, i| { if (i + 1 < exprs.len) { const expr_last_token = expr.lastToken() + 1; - const loc = tree.tokenLocation(tree.tokens[expr_last_token].end, exprs[i+1].firstToken()); + const loc = tree.tokenLocation(tree.token_locs[expr_last_token].end, exprs[i+1].firstToken()); if (loc.line != 0) break :blk count; count += 1; } else { const expr_last_token = expr.lastToken(); - const loc = tree.tokenLocation(tree.tokens[expr_last_token].end, rtoken); + const loc = tree.tokenLocation(tree.token_locs[expr_last_token].end, rtoken); if (loc.line == 0) { // all on one line const src_has_trailing_comma = trailblk: { const maybe_comma = tree.prevToken(rtoken); - break :trailblk tree.tokens[maybe_comma].id == .Comma; + break :trailblk tree.token_ids[maybe_comma] == .Comma; }; if (src_has_trailing_comma) { break :blk 1; // force row size 1 @@ -723,7 +726,7 @@ fn renderExpression( var new_indent = indent + indent_delta; - if (tree.tokens[tree.nextToken(lbrace)].id != .MultilineStringLiteralLine) { + if (tree.token_ids[tree.nextToken(lbrace)] != .MultilineStringLiteralLine) { try renderToken(tree, stream, lbrace, new_indent, start_col, Space.Newline); try stream.writeByteNTimes(' ', new_indent); } else { @@ -750,7 +753,7 @@ fn renderExpression( } col = 1; - if (tree.tokens[tree.nextToken(comma)].id != .MultilineStringLiteralLine) { + if (tree.token_ids[tree.nextToken(comma)] != .MultilineStringLiteralLine) { try renderToken(tree, stream, comma, new_indent, start_col, Space.Newline); // , } else { try renderToken(tree, stream, comma, new_indent, start_col, Space.None); // , @@ -819,11 +822,11 @@ fn renderExpression( const src_has_trailing_comma = blk: { const maybe_comma = tree.prevToken(rtoken); - break :blk tree.tokens[maybe_comma].id == .Comma; + break :blk tree.token_ids[maybe_comma] == .Comma; }; const src_same_line = blk: { - const loc = tree.tokenLocation(tree.tokens[lbrace].end, rtoken); + const loc = tree.tokenLocation(tree.token_locs[lbrace].end, rtoken); break :blk loc.line == 0; }; @@ -929,7 +932,7 @@ fn renderExpression( const src_has_trailing_comma = blk: { const maybe_comma = tree.prevToken(call.rtoken); - break :blk tree.tokens[maybe_comma].id == .Comma; + break :blk tree.token_ids[maybe_comma] == .Comma; }; if (src_has_trailing_comma) { @@ -983,8 +986,8 @@ fn renderExpression( try renderExpression(allocator, stream, tree, indent, start_col, suffix_op.lhs, Space.None); try renderToken(tree, stream, lbracket, indent, start_col, Space.None); // [ - const starts_with_comment = tree.tokens[lbracket + 1].id == .LineComment; - const ends_with_comment = tree.tokens[rbracket - 1].id == .LineComment; + const starts_with_comment = tree.token_ids[lbracket + 1] == .LineComment; + const ends_with_comment = tree.token_ids[rbracket - 1] == .LineComment; const new_indent = if (ends_with_comment) indent + indent_delta else indent; const new_space = if (ends_with_comment) Space.Newline else Space.None; try renderExpression(allocator, stream, tree, new_indent, start_col, index_expr, new_space); @@ -1226,9 +1229,9 @@ fn renderExpression( var maybe_comma = tree.prevToken(container_decl.lastToken()); // Doc comments for a field may also appear after the comma, eg. // field_name: T, // comment attached to field_name - if (tree.tokens[maybe_comma].id == .DocComment) + if (tree.token_ids[maybe_comma] == .DocComment) maybe_comma = tree.prevToken(maybe_comma); - break :blk tree.tokens[maybe_comma].id == .Comma; + break :blk tree.token_ids[maybe_comma] == .Comma; }; const fields_and_decls = container_decl.fieldsAndDecls(); @@ -1321,7 +1324,7 @@ fn renderExpression( const src_has_trailing_comma = blk: { const maybe_comma = tree.prevToken(err_set_decl.rbrace_token); - break :blk tree.tokens[maybe_comma].id == .Comma; + break :blk tree.token_ids[maybe_comma] == .Comma; }; if (src_has_trailing_comma) { @@ -1353,7 +1356,7 @@ fn renderExpression( try renderExpression(allocator, stream, tree, indent, start_col, node, Space.None); const comma_token = tree.nextToken(node.lastToken()); - assert(tree.tokens[comma_token].id == .Comma); + assert(tree.token_ids[comma_token] == .Comma); try renderToken(tree, stream, comma_token, indent, start_col, Space.Space); // , try renderExtraNewline(tree, stream, start_col, decls[i + 1]); } else { @@ -1378,7 +1381,7 @@ fn renderExpression( const multiline_str_literal = @fieldParentPtr(ast.Node.MultilineStringLiteral, "base", base); var skip_first_indent = true; - if (tree.tokens[multiline_str_literal.firstToken() - 1].id != .LineComment) { + if (tree.token_ids[multiline_str_literal.firstToken() - 1] != .LineComment) { try stream.print("\n", .{}); skip_first_indent = false; } @@ -1406,7 +1409,7 @@ fn renderExpression( if (builtin_call.params_len < 2) break :blk false; const last_node = builtin_call.params()[builtin_call.params_len - 1]; const maybe_comma = tree.nextToken(last_node.lastToken()); - break :blk tree.tokens[maybe_comma].id == .Comma; + break :blk tree.token_ids[maybe_comma] == .Comma; }; const lparen = tree.nextToken(builtin_call.builtin_token); @@ -1443,8 +1446,8 @@ fn renderExpression( const fn_proto = @fieldParentPtr(ast.Node.FnProto, "base", base); if (fn_proto.visib_token) |visib_token_index| { - const visib_token = tree.tokens[visib_token_index]; - assert(visib_token.id == .Keyword_pub or visib_token.id == .Keyword_export); + const visib_token = tree.token_ids[visib_token_index]; + assert(visib_token == .Keyword_pub or visib_token == .Keyword_export); try renderToken(tree, stream, visib_token_index, indent, start_col, Space.Space); // pub } @@ -1466,7 +1469,7 @@ fn renderExpression( try renderToken(tree, stream, fn_proto.fn_token, indent, start_col, Space.Space); // fn break :blk tree.nextToken(fn_proto.fn_token); }; - assert(tree.tokens[lparen].id == .LParen); + assert(tree.token_ids[lparen] == .LParen); const rparen = tree.prevToken( // the first token for the annotation expressions is the left @@ -1482,10 +1485,10 @@ fn renderExpression( .InferErrorSet => |node| tree.prevToken(node.firstToken()), .Invalid => unreachable, }); - assert(tree.tokens[rparen].id == .RParen); + assert(tree.token_ids[rparen] == .RParen); const src_params_trailing_comma = blk: { - const maybe_comma = tree.tokens[rparen - 1].id; + const maybe_comma = tree.token_ids[rparen - 1]; break :blk maybe_comma == .Comma or maybe_comma == .LineComment; }; @@ -1622,7 +1625,7 @@ fn renderExpression( const src_has_trailing_comma = blk: { const last_node = switch_case.items()[switch_case.items_len - 1]; const maybe_comma = tree.nextToken(last_node.lastToken()); - break :blk tree.tokens[maybe_comma].id == .Comma; + break :blk tree.token_ids[maybe_comma] == .Comma; }; if (switch_case.items_len == 1 or !src_has_trailing_comma) { @@ -1967,7 +1970,7 @@ fn renderExpression( try renderAsmOutput(allocator, stream, tree, indent_extra, start_col, asm_output, Space.Newline); try stream.writeByteNTimes(' ', indent_once); const comma_or_colon = tree.nextToken(asm_output.lastToken()); - break :blk switch (tree.tokens[comma_or_colon].id) { + break :blk switch (tree.token_ids[comma_or_colon]) { .Comma => tree.nextToken(comma_or_colon), else => comma_or_colon, }; @@ -2002,7 +2005,7 @@ fn renderExpression( try renderAsmInput(allocator, stream, tree, indent_extra, start_col, asm_input, Space.Newline); try stream.writeByteNTimes(' ', indent_once); const comma_or_colon = tree.nextToken(asm_input.lastToken()); - break :blk switch (tree.tokens[comma_or_colon].id) { + break :blk switch (tree.token_ids[comma_or_colon]) { .Comma => tree.nextToken(comma_or_colon), else => comma_or_colon, }; @@ -2205,7 +2208,7 @@ fn renderStatement( try renderExpression(allocator, stream, tree, indent, start_col, base, Space.None); const semicolon_index = tree.nextToken(base.lastToken()); - assert(tree.tokens[semicolon_index].id == .Semicolon); + assert(tree.token_ids[semicolon_index] == .Semicolon); try renderToken(tree, stream, semicolon_index, indent, start_col, Space.Newline); } else { try renderExpression(allocator, stream, tree, indent, start_col, base, Space.Newline); @@ -2243,22 +2246,25 @@ fn renderTokenOffset( return; } - var token = tree.tokens[token_index]; - try stream.writeAll(mem.trimRight(u8, tree.tokenSlicePtr(token)[token_skip_bytes..], " ")); + var token_loc = tree.token_locs[token_index]; + try stream.writeAll(mem.trimRight(u8, tree.tokenSliceLoc(token_loc)[token_skip_bytes..], " ")); if (space == Space.NoComment) return; - var next_token = tree.tokens[token_index + 1]; + var next_token_id = tree.token_ids[token_index + 1]; + var next_token_loc = tree.token_locs[token_index + 1]; - if (space == Space.Comma) switch (next_token.id) { + if (space == Space.Comma) switch (next_token_id) { .Comma => return renderToken(tree, stream, token_index + 1, indent, start_col, Space.Newline), .LineComment => { try stream.writeAll(", "); return renderToken(tree, stream, token_index + 1, indent, start_col, Space.Newline); }, else => { - if (token_index + 2 < tree.tokens.len and tree.tokens[token_index + 2].id == .MultilineStringLiteralLine) { + if (token_index + 2 < tree.token_ids.len and + tree.token_ids[token_index + 2] == .MultilineStringLiteralLine) + { try stream.writeAll(","); return; } else { @@ -2271,19 +2277,20 @@ fn renderTokenOffset( // Skip over same line doc comments var offset: usize = 1; - if (next_token.id == .DocComment) { - const loc = tree.tokenLocationPtr(token.end, next_token); + if (next_token_id == .DocComment) { + const loc = tree.tokenLocationLoc(token_loc.end, next_token_loc); if (loc.line == 0) { offset += 1; - next_token = tree.tokens[token_index + offset]; + next_token_id = tree.token_ids[token_index + offset]; + next_token_loc = tree.token_locs[token_index + offset]; } } - if (next_token.id != .LineComment) blk: { + if (next_token_id != .LineComment) blk: { switch (space) { Space.None, Space.NoNewline => return, Space.Newline => { - if (next_token.id == .MultilineStringLiteralLine) { + if (next_token_id == .MultilineStringLiteralLine) { return; } else { try stream.writeAll("\n"); @@ -2292,7 +2299,7 @@ fn renderTokenOffset( } }, Space.Space, Space.SpaceOrOutdent => { - if (next_token.id == .MultilineStringLiteralLine) + if (next_token_id == .MultilineStringLiteralLine) return; try stream.writeByte(' '); return; @@ -2302,14 +2309,15 @@ fn renderTokenOffset( } while (true) { - const comment_is_empty = mem.trimRight(u8, tree.tokenSlicePtr(next_token), " ").len == 2; + const comment_is_empty = mem.trimRight(u8, tree.tokenSliceLoc(next_token_loc), " ").len == 2; if (comment_is_empty) { switch (space) { Space.Newline => { offset += 1; - token = next_token; - next_token = tree.tokens[token_index + offset]; - if (next_token.id != .LineComment) { + token_loc = next_token_loc; + next_token_id = tree.token_ids[token_index + offset]; + next_token_loc = tree.token_locs[token_index + offset]; + if (next_token_id != .LineComment) { try stream.writeByte('\n'); start_col.* = 0; return; @@ -2322,18 +2330,19 @@ fn renderTokenOffset( } } - var loc = tree.tokenLocationPtr(token.end, next_token); + var loc = tree.tokenLocationLoc(token_loc.end, next_token_loc); if (loc.line == 0) { - try stream.print(" {}", .{mem.trimRight(u8, tree.tokenSlicePtr(next_token), " ")}); + try stream.print(" {}", .{mem.trimRight(u8, tree.tokenSliceLoc(next_token_loc), " ")}); offset = 2; - token = next_token; - next_token = tree.tokens[token_index + offset]; - if (next_token.id != .LineComment) { + token_loc = next_token_loc; + next_token_loc = tree.token_locs[token_index + offset]; + next_token_id = tree.token_ids[token_index + offset]; + if (next_token_id != .LineComment) { switch (space) { Space.None, Space.Space => { try stream.writeByte('\n'); - const after_comment_token = tree.tokens[token_index + offset]; - const next_line_indent = switch (after_comment_token.id) { + const after_comment_token = tree.token_ids[token_index + offset]; + const next_line_indent = switch (after_comment_token) { .RParen, .RBrace, .RBracket => indent, else => indent + indent_delta, }; @@ -2346,7 +2355,7 @@ fn renderTokenOffset( start_col.* = indent; }, Space.Newline => { - if (next_token.id == .MultilineStringLiteralLine) { + if (next_token_id == .MultilineStringLiteralLine) { return; } else { try stream.writeAll("\n"); @@ -2359,7 +2368,7 @@ fn renderTokenOffset( } return; } - loc = tree.tokenLocationPtr(token.end, next_token); + loc = tree.tokenLocationLoc(token_loc.end, next_token_loc); } while (true) { @@ -2369,15 +2378,16 @@ fn renderTokenOffset( const newline_count = if (loc.line <= 1) @as(u8, 1) else @as(u8, 2); try stream.writeByteNTimes('\n', newline_count); try stream.writeByteNTimes(' ', indent); - try stream.writeAll(mem.trimRight(u8, tree.tokenSlicePtr(next_token), " ")); + try stream.writeAll(mem.trimRight(u8, tree.tokenSliceLoc(next_token_loc), " ")); offset += 1; - token = next_token; - next_token = tree.tokens[token_index + offset]; - if (next_token.id != .LineComment) { + token_loc = next_token_loc; + next_token_loc = tree.token_locs[token_index + offset]; + next_token_id = tree.token_ids[token_index + offset]; + if (next_token_id != .LineComment) { switch (space) { Space.Newline => { - if (next_token.id == .MultilineStringLiteralLine) { + if (next_token_id == .MultilineStringLiteralLine) { return; } else { try stream.writeAll("\n"); @@ -2388,8 +2398,8 @@ fn renderTokenOffset( Space.None, Space.Space => { try stream.writeByte('\n'); - const after_comment_token = tree.tokens[token_index + offset]; - const next_line_indent = switch (after_comment_token.id) { + const after_comment_token = tree.token_ids[token_index + offset]; + const next_line_indent = switch (after_comment_token) { .RParen, .RBrace, .RBracket => blk: { if (indent > indent_delta) { break :blk indent - indent_delta; @@ -2412,7 +2422,7 @@ fn renderTokenOffset( } return; } - loc = tree.tokenLocationPtr(token.end, next_token); + loc = tree.tokenLocationLoc(token_loc.end, next_token_loc); } } @@ -2448,7 +2458,7 @@ fn renderDocCommentsToken( ) (@TypeOf(stream).Error || Error)!void { var tok_i = comment.first_line; while (true) : (tok_i += 1) { - switch (tree.tokens[tok_i].id) { + switch (tree.token_ids[tok_i]) { .DocComment, .ContainerDocComment => { if (comment.first_line < first_token) { try renderToken(tree, stream, tok_i, indent, start_col, Space.Newline); diff --git a/lib/std/zig/tokenizer.zig b/lib/std/zig/tokenizer.zig index 160530f459..3ed57f621f 100644 --- a/lib/std/zig/tokenizer.zig +++ b/lib/std/zig/tokenizer.zig @@ -3,8 +3,12 @@ const mem = std.mem; pub const Token = struct { id: Id, - start: usize, - end: usize, + loc: Loc, + + pub const Loc = struct { + start: usize, + end: usize, + }; pub const Keyword = struct { bytes: []const u8, @@ -426,8 +430,10 @@ pub const Tokenizer = struct { var state: State = .start; var result = Token{ .id = .Eof, - .start = self.index, - .end = undefined, + .loc = .{ + .start = self.index, + .end = undefined, + }, }; var seen_escape_digits: usize = undefined; var remaining_code_units: usize = undefined; @@ -436,7 +442,7 @@ pub const Tokenizer = struct { switch (state) { .start => switch (c) { ' ', '\n', '\t', '\r' => { - result.start = self.index + 1; + result.loc.start = self.index + 1; }, '"' => { state = .string_literal; @@ -686,7 +692,7 @@ pub const Tokenizer = struct { .identifier => switch (c) { 'a'...'z', 'A'...'Z', '_', '0'...'9' => {}, else => { - if (Token.getKeyword(self.buffer[result.start..self.index])) |id| { + if (Token.getKeyword(self.buffer[result.loc.start..self.index])) |id| { result.id = id; } break; @@ -1313,7 +1319,7 @@ pub const Tokenizer = struct { => {}, .identifier => { - if (Token.getKeyword(self.buffer[result.start..self.index])) |id| { + if (Token.getKeyword(self.buffer[result.loc.start..self.index])) |id| { result.id = id; } }, @@ -1420,7 +1426,7 @@ pub const Tokenizer = struct { } } - result.end = self.index; + result.loc.end = self.index; return result; } @@ -1430,8 +1436,10 @@ pub const Tokenizer = struct { if (invalid_length == 0) return; self.pending_invalid_token = .{ .id = .Invalid, - .start = self.index, - .end = self.index + invalid_length, + .loc = .{ + .start = self.index, + .end = self.index + invalid_length, + }, }; } diff --git a/src-self-hosted/translate_c.zig b/src-self-hosted/translate_c.zig index e0c2b44f42..fca959d536 100644 --- a/src-self-hosted/translate_c.zig +++ b/src-self-hosted/translate_c.zig @@ -247,27 +247,6 @@ pub const Context = struct { } }; - /// Helper function to append items to a singly linked list. - fn llpusher(c: *Context, list: *std.SinglyLinkedList(*ast.Node)) LinkedListPusher { - assert(list.first == null); - return .{ - .c = c, - .it = &list.first, - }; - } - - fn llpush( - c: *Context, - comptime T: type, - it: *?*std.SinglyLinkedList(T).Node, - data: T, - ) !*?*std.SinglyLinkedList(T).Node { - const llnode = try c.arena.create(std.SinglyLinkedList(T).Node); - llnode.* = .{ .data = data }; - it.* = llnode; - return &llnode.next; - } - fn getMangle(c: *Context) u32 { c.mangle_count += 1; return c.mangle_count;