From ca597e2bfb3c39aecdf3dea2718e84deb749ed07 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 14 Feb 2018 23:00:53 -0500 Subject: [PATCH] std.zig.parser understands try. zig fmt respects a double line break. --- src/tokenizer.cpp | 2 - src/tokenizer.hpp | 1 - std/debug/index.zig | 4 +- std/zig/ast.zig | 164 ++++++++++++++++++++++++++++++++++++++++-- std/zig/parser.zig | 111 ++++++++++++++++++++++++---- std/zig/tokenizer.zig | 54 +++++++++----- 6 files changed, 296 insertions(+), 40 deletions(-) diff --git a/src/tokenizer.cpp b/src/tokenizer.cpp index 44d838a723..dd60815b7f 100644 --- a/src/tokenizer.cpp +++ b/src/tokenizer.cpp @@ -125,7 +125,6 @@ static const struct ZigKeyword zig_keywords[] = { {"false", TokenIdKeywordFalse}, {"fn", TokenIdKeywordFn}, {"for", TokenIdKeywordFor}, - {"goto", TokenIdKeywordGoto}, {"if", TokenIdKeywordIf}, {"inline", TokenIdKeywordInline}, {"nakedcc", TokenIdKeywordNakedCC}, @@ -1542,7 +1541,6 @@ const char * token_name(TokenId id) { case TokenIdKeywordFalse: return "false"; case TokenIdKeywordFn: return "fn"; case TokenIdKeywordFor: return "for"; - case TokenIdKeywordGoto: return "goto"; case TokenIdKeywordIf: return "if"; case TokenIdKeywordInline: return "inline"; case TokenIdKeywordNakedCC: return "nakedcc"; diff --git a/src/tokenizer.hpp b/src/tokenizer.hpp index 92a3b8de0d..225b75d844 100644 --- a/src/tokenizer.hpp +++ b/src/tokenizer.hpp @@ -66,7 +66,6 @@ enum TokenId { TokenIdKeywordFalse, TokenIdKeywordFn, TokenIdKeywordFor, - TokenIdKeywordGoto, TokenIdKeywordIf, TokenIdKeywordInline, TokenIdKeywordNakedCC, diff --git a/std/debug/index.zig b/std/debug/index.zig index df1a1c5041..cc4832b1ea 100644 --- a/std/debug/index.zig +++ b/std/debug/index.zig @@ -47,7 +47,7 @@ pub fn getSelfDebugInfo() !&ElfStackTrace { pub fn dumpCurrentStackTrace() void { const stderr = getStderrStream() catch return; const debug_info = getSelfDebugInfo() catch |err| { - stderr.print("Unable to open debug info: {}\n", @errorName(err)) catch return; + stderr.print("Unable to dump stack trace: Unable to open debug info: {}\n", @errorName(err)) catch return; return; }; defer debug_info.close(); @@ -61,7 +61,7 @@ pub fn dumpCurrentStackTrace() void { pub fn dumpStackTrace(stack_trace: &const builtin.StackTrace) void { const stderr = getStderrStream() catch return; const debug_info = getSelfDebugInfo() catch |err| { - stderr.print("Unable to open debug info: {}\n", @errorName(err)) catch return; + stderr.print("Unable to dump stack trace: Unable to open debug info: {}\n", @errorName(err)) catch return; return; }; defer debug_info.close(); diff --git a/std/zig/ast.zig b/std/zig/ast.zig index 3892812882..77a2b8cd6a 100644 --- a/std/zig/ast.zig +++ b/std/zig/ast.zig @@ -38,11 +38,46 @@ pub const Node = struct { Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).iterate(index), }; } + + pub fn firstToken(base: &Node) Token { + return switch (base.id) { + Id.Root => @fieldParentPtr(NodeRoot, "base", base).firstToken(), + Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).firstToken(), + Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).firstToken(), + Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).firstToken(), + Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).firstToken(), + Id.Block => @fieldParentPtr(NodeBlock, "base", base).firstToken(), + Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).firstToken(), + Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).firstToken(), + Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).firstToken(), + Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).firstToken(), + Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).firstToken(), + Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).firstToken(), + }; + } + + pub fn lastToken(base: &Node) Token { + return switch (base.id) { + Id.Root => @fieldParentPtr(NodeRoot, "base", base).lastToken(), + Id.VarDecl => @fieldParentPtr(NodeVarDecl, "base", base).lastToken(), + Id.Identifier => @fieldParentPtr(NodeIdentifier, "base", base).lastToken(), + Id.FnProto => @fieldParentPtr(NodeFnProto, "base", base).lastToken(), + Id.ParamDecl => @fieldParentPtr(NodeParamDecl, "base", base).lastToken(), + Id.Block => @fieldParentPtr(NodeBlock, "base", base).lastToken(), + Id.InfixOp => @fieldParentPtr(NodeInfixOp, "base", base).lastToken(), + Id.PrefixOp => @fieldParentPtr(NodePrefixOp, "base", base).lastToken(), + Id.IntegerLiteral => @fieldParentPtr(NodeIntegerLiteral, "base", base).lastToken(), + Id.FloatLiteral => @fieldParentPtr(NodeFloatLiteral, "base", base).lastToken(), + Id.StringLiteral => @fieldParentPtr(NodeStringLiteral, "base", base).lastToken(), + Id.BuiltinCall => @fieldParentPtr(NodeBuiltinCall, "base", base).lastToken(), + }; + } }; pub const NodeRoot = struct { base: Node, decls: ArrayList(&Node), + eof_token: Token, pub fn iterate(self: &NodeRoot, index: usize) ?&Node { if (index < self.decls.len) { @@ -50,6 +85,14 @@ pub const NodeRoot = struct { } return null; } + + pub fn firstToken(self: &NodeRoot) Token { + return if (self.decls.len == 0) self.eof_token else self.decls.at(0).firstToken(); + } + + pub fn lastToken(self: &NodeRoot) Token { + return if (self.decls.len == 0) self.eof_token else self.decls.at(self.decls.len - 1).lastToken(); + } }; pub const NodeVarDecl = struct { @@ -64,6 +107,7 @@ pub const NodeVarDecl = struct { type_node: ?&Node, align_node: ?&Node, init_node: ?&Node, + semicolon_token: Token, pub fn iterate(self: &NodeVarDecl, index: usize) ?&Node { var i = index; @@ -85,6 +129,18 @@ pub const NodeVarDecl = struct { return null; } + + pub fn firstToken(self: &NodeVarDecl) Token { + if (self.visib_token) |visib_token| return visib_token; + if (self.comptime_token) |comptime_token| return comptime_token; + if (self.extern_token) |extern_token| return extern_token; + assert(self.lib_name == null); + return self.mut_token; + } + + pub fn lastToken(self: &NodeVarDecl) Token { + return self.semicolon_token; + } }; pub const NodeIdentifier = struct { @@ -94,6 +150,14 @@ pub const NodeIdentifier = struct { pub fn iterate(self: &NodeIdentifier, index: usize) ?&Node { return null; } + + pub fn firstToken(self: &NodeIdentifier) Token { + return self.name_token; + } + + pub fn lastToken(self: &NodeIdentifier) Token { + return self.name_token; + } }; pub const NodeFnProto = struct { @@ -113,7 +177,7 @@ pub const NodeFnProto = struct { pub const ReturnType = union(enum) { Explicit: &Node, - Infer, + Infer: Token, InferErrorSet: &Node, }; @@ -153,6 +217,25 @@ pub const NodeFnProto = struct { return null; } + + pub fn firstToken(self: &NodeFnProto) Token { + if (self.visib_token) |visib_token| return visib_token; + if (self.extern_token) |extern_token| return extern_token; + assert(self.lib_name == null); + if (self.inline_token) |inline_token| return inline_token; + if (self.cc_token) |cc_token| return cc_token; + return self.fn_token; + } + + pub fn lastToken(self: &NodeFnProto) Token { + if (self.body_node) |body_node| return body_node.lastToken(); + switch (self.return_type) { + // TODO allow this and next prong to share bodies since the types are the same + ReturnType.Explicit => |node| return node.lastToken(), + ReturnType.InferErrorSet => |node| return node.lastToken(), + ReturnType.Infer => |token| return token, + } + } }; pub const NodeParamDecl = struct { @@ -171,6 +254,18 @@ pub const NodeParamDecl = struct { return null; } + + pub fn firstToken(self: &NodeParamDecl) Token { + if (self.comptime_token) |comptime_token| return comptime_token; + if (self.noalias_token) |noalias_token| return noalias_token; + if (self.name_token) |name_token| return name_token; + return self.type_node.firstToken(); + } + + pub fn lastToken(self: &NodeParamDecl) Token { + if (self.var_args_token) |var_args_token| return var_args_token; + return self.type_node.lastToken(); + } }; pub const NodeBlock = struct { @@ -187,6 +282,14 @@ pub const NodeBlock = struct { return null; } + + pub fn firstToken(self: &NodeBlock) Token { + return self.begin_token; + } + + pub fn lastToken(self: &NodeBlock) Token { + return self.end_token; + } }; pub const NodeInfixOp = struct { @@ -199,6 +302,7 @@ pub const NodeInfixOp = struct { const InfixOp = enum { EqualEqual, BangEqual, + Period, }; pub fn iterate(self: &NodeInfixOp, index: usize) ?&Node { @@ -208,8 +312,9 @@ pub const NodeInfixOp = struct { i -= 1; switch (self.op) { - InfixOp.EqualEqual => {}, - InfixOp.BangEqual => {}, + InfixOp.EqualEqual, + InfixOp.BangEqual, + InfixOp.Period => {}, } if (i < 1) return self.rhs; @@ -217,6 +322,14 @@ pub const NodeInfixOp = struct { return null; } + + pub fn firstToken(self: &NodeInfixOp) Token { + return self.lhs.firstToken(); + } + + pub fn lastToken(self: &NodeInfixOp) Token { + return self.rhs.lastToken(); + } }; pub const NodePrefixOp = struct { @@ -227,6 +340,7 @@ pub const NodePrefixOp = struct { const PrefixOp = union(enum) { Return, + Try, AddrOf: AddrOfInfo, }; const AddrOfInfo = struct { @@ -241,7 +355,8 @@ pub const NodePrefixOp = struct { var i = index; switch (self.op) { - PrefixOp.Return => {}, + PrefixOp.Return, + PrefixOp.Try => {}, PrefixOp.AddrOf => |addr_of_info| { if (addr_of_info.align_expr) |align_expr| { if (i < 1) return align_expr; @@ -255,6 +370,14 @@ pub const NodePrefixOp = struct { return null; } + + pub fn firstToken(self: &NodePrefixOp) Token { + return self.op_token; + } + + pub fn lastToken(self: &NodePrefixOp) Token { + return self.rhs.lastToken(); + } }; pub const NodeIntegerLiteral = struct { @@ -264,6 +387,14 @@ pub const NodeIntegerLiteral = struct { pub fn iterate(self: &NodeIntegerLiteral, index: usize) ?&Node { return null; } + + pub fn firstToken(self: &NodeIntegerLiteral) Token { + return self.token; + } + + pub fn lastToken(self: &NodeIntegerLiteral) Token { + return self.token; + } }; pub const NodeFloatLiteral = struct { @@ -273,12 +404,21 @@ pub const NodeFloatLiteral = struct { pub fn iterate(self: &NodeFloatLiteral, index: usize) ?&Node { return null; } + + pub fn firstToken(self: &NodeFloatLiteral) Token { + return self.token; + } + + pub fn lastToken(self: &NodeFloatLiteral) Token { + return self.token; + } }; pub const NodeBuiltinCall = struct { base: Node, builtin_token: Token, params: ArrayList(&Node), + rparen_token: Token, pub fn iterate(self: &NodeBuiltinCall, index: usize) ?&Node { var i = index; @@ -288,6 +428,14 @@ pub const NodeBuiltinCall = struct { return null; } + + pub fn firstToken(self: &NodeBuiltinCall) Token { + return self.builtin_token; + } + + pub fn lastToken(self: &NodeBuiltinCall) Token { + return self.rparen_token; + } }; pub const NodeStringLiteral = struct { @@ -297,4 +445,12 @@ pub const NodeStringLiteral = struct { pub fn iterate(self: &NodeStringLiteral, index: usize) ?&Node { return null; } + + pub fn firstToken(self: &NodeStringLiteral) Token { + return self.token; + } + + pub fn lastToken(self: &NodeStringLiteral) Token { + return self.token; + } }; diff --git a/std/zig/parser.zig b/std/zig/parser.zig index 0c24fe1410..35bba378d4 100644 --- a/std/zig/parser.zig +++ b/std/zig/parser.zig @@ -69,6 +69,11 @@ pub const Parser = struct { } }; + const ExpectTokenSave = struct { + id: Token.Id, + ptr: &Token, + }; + const State = union(enum) { TopLevel, TopLevelExtern: ?Token, @@ -85,6 +90,7 @@ pub const Parser = struct { VarDeclAlign: &ast.NodeVarDecl, VarDeclEq: &ast.NodeVarDecl, ExpectToken: @TagType(Token.Id), + ExpectTokenSave: ExpectTokenSave, FnProto: &ast.NodeFnProto, FnProtoAlign: &ast.NodeFnProto, FnProtoReturnType: &ast.NodeFnProto, @@ -136,7 +142,10 @@ pub const Parser = struct { stack.append(State { .TopLevelExtern = token }) catch unreachable; continue; }, - Token.Id.Eof => return Tree {.root_node = root_node, .arena_allocator = arena_allocator}, + Token.Id.Eof => { + root_node.eof_token = token; + return Tree {.root_node = root_node, .arena_allocator = arena_allocator}; + }, else => { self.putBackToken(token); stack.append(State { .TopLevelExtern = null }) catch unreachable; @@ -231,13 +240,19 @@ pub const Parser = struct { const token = self.getNextToken(); if (token.id == Token.Id.Equal) { var_decl.eq_token = token; - stack.append(State { .ExpectToken = Token.Id.Semicolon }) catch unreachable; + stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.Semicolon, + .ptr = &var_decl.semicolon_token, + }, + }) catch unreachable; try stack.append(State { .Expression = DestPtr {.NullableField = &var_decl.init_node}, }); continue; } if (token.id == Token.Id.Semicolon) { + var_decl.semicolon_token = token; continue; } return self.parseError(token, "expected '=' or ';', found {}", @tagName(token.id)); @@ -247,6 +262,11 @@ pub const Parser = struct { continue; }, + State.ExpectTokenSave => |expect_token_save| { + *expect_token_save.ptr = try self.eatToken(expect_token_save.id); + continue; + }, + State.Expression => |dest_ptr| { // save the dest_ptr for later stack.append(state) catch unreachable; @@ -264,6 +284,12 @@ pub const Parser = struct { try stack.append(State.ExpectOperand); continue; }, + Token.Id.Keyword_try => { + try stack.append(State { .PrefixOp = try self.createPrefixOp(arena, token, + ast.NodePrefixOp.PrefixOp.Try) }); + try stack.append(State.ExpectOperand); + continue; + }, Token.Id.Ampersand => { const prefix_op = try self.createPrefixOp(arena, token, ast.NodePrefixOp.PrefixOp{ .AddrOf = ast.NodePrefixOp.AddrOfInfo { @@ -306,13 +332,19 @@ pub const Parser = struct { .base = ast.Node {.id = ast.Node.Id.BuiltinCall}, .builtin_token = token, .params = ArrayList(&ast.Node).init(arena), + .rparen_token = undefined, }; try stack.append(State { .Operand = &node.base }); try stack.append(State.AfterOperand); try stack.append(State {.ExprListItemOrEnd = &node.params }); - try stack.append(State {.ExpectToken = Token.Id.LParen }); + try stack.append(State { + .ExpectTokenSave = ExpectTokenSave { + .id = Token.Id.LParen, + .ptr = &node.rparen_token, + }, + }); continue; }, Token.Id.StringLiteral => { @@ -351,6 +383,13 @@ pub const Parser = struct { try stack.append(State.ExpectOperand); continue; }, + Token.Id.Period => { + try stack.append(State { + .InfixOp = try self.createInfixOp(arena, token, ast.NodeInfixOp.InfixOp.Period) + }); + try stack.append(State.ExpectOperand); + continue; + }, else => { // no postfix/infix operator after this operand. self.putBackToken(token); @@ -476,7 +515,7 @@ pub const Parser = struct { const token = self.getNextToken(); switch (token.id) { Token.Id.Keyword_var => { - fn_proto.return_type = ast.NodeFnProto.ReturnType.Infer; + fn_proto.return_type = ast.NodeFnProto.ReturnType { .Infer = token }; }, Token.Id.Bang => { fn_proto.return_type = ast.NodeFnProto.ReturnType { .InferErrorSet = undefined }; @@ -627,6 +666,8 @@ pub const Parser = struct { *node = ast.NodeRoot { .base = ast.Node {.id = ast.Node.Id.Root}, .decls = ArrayList(&ast.Node).init(arena), + // initialized when we get the eof token + .eof_token = undefined, }; return node; } @@ -649,6 +690,7 @@ pub const Parser = struct { // initialized later .name_token = undefined, .eq_token = undefined, + .semicolon_token = undefined, }; return node; } @@ -789,11 +831,11 @@ pub const Parser = struct { fn parseError(self: &Parser, token: &const Token, comptime fmt: []const u8, args: ...) (error{ParseError}) { const loc = self.tokenizer.getTokenLocation(token); - warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, loc.line + 1, loc.column + 1, args); + warn("{}:{}:{}: error: " ++ fmt ++ "\n", self.source_file_name, token.line + 1, token.column + 1, args); warn("{}\n", self.tokenizer.buffer[loc.line_start..loc.line_end]); { var i: usize = 0; - while (i < loc.column) : (i += 1) { + while (i < token.column) : (i += 1) { warn(" "); } } @@ -885,11 +927,26 @@ pub const Parser = struct { defer self.deinitUtilityArrayList(stack); { + try stack.append(RenderState { .Text = "\n"}); + var i = root_node.decls.len; while (i != 0) { i -= 1; const decl = root_node.decls.items[i]; try stack.append(RenderState {.TopLevelDecl = decl}); + if (i != 0) { + try stack.append(RenderState { + .Text = blk: { + const prev_node = root_node.decls.at(i - 1); + const prev_line_index = prev_node.lastToken().line; + const this_line_index = decl.firstToken().line; + if (this_line_index - prev_line_index >= 2) { + break :blk "\n\n"; + } + break :blk "\n"; + }, + }); + } } } @@ -919,7 +976,6 @@ pub const Parser = struct { try stream.print("("); - try stack.append(RenderState { .Text = "\n" }); if (fn_proto.body_node == null) { try stack.append(RenderState { .Text = ";" }); } @@ -937,7 +993,6 @@ pub const Parser = struct { }, ast.Node.Id.VarDecl => { const var_decl = @fieldParentPtr(ast.NodeVarDecl, "base", decl); - try stack.append(RenderState { .Text = "\n"}); try stack.append(RenderState { .VarDecl = var_decl}); }, @@ -1019,7 +1074,19 @@ pub const Parser = struct { try stack.append(RenderState { .Statement = statement_node}); try stack.append(RenderState.PrintIndent); try stack.append(RenderState { .Indent = indent + indent_delta}); - try stack.append(RenderState { .Text = "\n" }); + try stack.append(RenderState { + .Text = blk: { + if (i != 0) { + const prev_statement_node = block.statements.items[i - 1]; + const prev_line_index = prev_statement_node.lastToken().line; + const this_line_index = statement_node.firstToken().line; + if (this_line_index - prev_line_index >= 2) { + break :blk "\n\n"; + } + } + break :blk "\n"; + }, + }); } } }, @@ -1033,7 +1100,9 @@ pub const Parser = struct { ast.NodeInfixOp.InfixOp.BangEqual => { try stack.append(RenderState { .Text = " != "}); }, - else => unreachable, + ast.NodeInfixOp.InfixOp.Period => { + try stack.append(RenderState { .Text = "."}); + }, } try stack.append(RenderState { .Expression = prefix_op_node.lhs }); }, @@ -1044,6 +1113,9 @@ pub const Parser = struct { ast.NodePrefixOp.PrefixOp.Return => { try stream.write("return "); }, + ast.NodePrefixOp.PrefixOp.Try => { + try stream.write("try "); + }, ast.NodePrefixOp.PrefixOp.AddrOf => |addr_of_info| { try stream.write("&"); if (addr_of_info.volatile_token != null) { @@ -1058,7 +1130,6 @@ pub const Parser = struct { try stack.append(RenderState { .Expression = align_expr}); } }, - else => unreachable, } }, ast.Node.Id.IntegerLiteral => { @@ -1153,10 +1224,7 @@ pub const Parser = struct { var fixed_buffer_mem: [100 * 1024]u8 = undefined; fn testParse(source: []const u8, allocator: &mem.Allocator) ![]u8 { - var padded_source: [0x100]u8 = undefined; - std.mem.copy(u8, padded_source[0..source.len], source); - - var tokenizer = Tokenizer.init(padded_source[0..source.len]); + var tokenizer = Tokenizer.init(source); var parser = Parser.init(&tokenizer, allocator, "(memory buffer)"); defer parser.deinit(); @@ -1211,6 +1279,19 @@ fn testCanonical(source: []const u8) !void { } test "zig fmt" { + try testCanonical( + \\const std = @import("std"); + \\ + \\pub fn main() !void { + \\ var stdout_file = try std.io.getStdOut; + \\ var stdout_file = try std.io.getStdOut; + \\ + \\ var stdout_file = try std.io.getStdOut; + \\ var stdout_file = try std.io.getStdOut; + \\} + \\ + ); + try testCanonical( \\pub fn main() !void {} \\pub fn main() var {} diff --git a/std/zig/tokenizer.zig b/std/zig/tokenizer.zig index de1263ac55..bdd82e4a44 100644 --- a/std/zig/tokenizer.zig +++ b/std/zig/tokenizer.zig @@ -5,6 +5,8 @@ pub const Token = struct { id: Id, start: usize, end: usize, + line: usize, + column: usize, const KeywordId = struct { bytes: []const u8, @@ -16,6 +18,7 @@ pub const Token = struct { KeywordId{.bytes="and", .id = Id.Keyword_and}, KeywordId{.bytes="asm", .id = Id.Keyword_asm}, KeywordId{.bytes="break", .id = Id.Keyword_break}, + KeywordId{.bytes="catch", .id = Id.Keyword_catch}, KeywordId{.bytes="comptime", .id = Id.Keyword_comptime}, KeywordId{.bytes="const", .id = Id.Keyword_const}, KeywordId{.bytes="continue", .id = Id.Keyword_continue}, @@ -28,7 +31,6 @@ pub const Token = struct { KeywordId{.bytes="false", .id = Id.Keyword_false}, KeywordId{.bytes="fn", .id = Id.Keyword_fn}, KeywordId{.bytes="for", .id = Id.Keyword_for}, - KeywordId{.bytes="goto", .id = Id.Keyword_goto}, KeywordId{.bytes="if", .id = Id.Keyword_if}, KeywordId{.bytes="inline", .id = Id.Keyword_inline}, KeywordId{.bytes="nakedcc", .id = Id.Keyword_nakedcc}, @@ -38,12 +40,14 @@ pub const Token = struct { KeywordId{.bytes="packed", .id = Id.Keyword_packed}, KeywordId{.bytes="pub", .id = Id.Keyword_pub}, KeywordId{.bytes="return", .id = Id.Keyword_return}, + KeywordId{.bytes="section", .id = Id.Keyword_section}, KeywordId{.bytes="stdcallcc", .id = Id.Keyword_stdcallcc}, KeywordId{.bytes="struct", .id = Id.Keyword_struct}, KeywordId{.bytes="switch", .id = Id.Keyword_switch}, KeywordId{.bytes="test", .id = Id.Keyword_test}, KeywordId{.bytes="this", .id = Id.Keyword_this}, KeywordId{.bytes="true", .id = Id.Keyword_true}, + KeywordId{.bytes="try", .id = Id.Keyword_try}, KeywordId{.bytes="undefined", .id = Id.Keyword_undefined}, KeywordId{.bytes="union", .id = Id.Keyword_union}, KeywordId{.bytes="unreachable", .id = Id.Keyword_unreachable}, @@ -99,6 +103,7 @@ pub const Token = struct { Keyword_and, Keyword_asm, Keyword_break, + Keyword_catch, Keyword_comptime, Keyword_const, Keyword_continue, @@ -111,7 +116,6 @@ pub const Token = struct { Keyword_false, Keyword_fn, Keyword_for, - Keyword_goto, Keyword_if, Keyword_inline, Keyword_nakedcc, @@ -121,12 +125,14 @@ pub const Token = struct { Keyword_packed, Keyword_pub, Keyword_return, + Keyword_section, Keyword_stdcallcc, Keyword_struct, Keyword_switch, Keyword_test, Keyword_this, Keyword_true, + Keyword_try, Keyword_undefined, Keyword_union, Keyword_unreachable, @@ -140,21 +146,19 @@ pub const Token = struct { pub const Tokenizer = struct { buffer: []const u8, index: usize, + line: usize, + column: usize, pending_invalid_token: ?Token, - pub const Location = struct { - line: usize, - column: usize, + pub const LineLocation = struct { line_start: usize, line_end: usize, }; - pub fn getTokenLocation(self: &Tokenizer, token: &const Token) Location { - var loc = Location { - .line = 0, - .column = 0, + pub fn getTokenLocation(self: &Tokenizer, token: &const Token) LineLocation { + var loc = LineLocation { .line_start = 0, - .line_end = 0, + .line_end = self.buffer.len, }; for (self.buffer) |c, i| { if (i == token.start) { @@ -163,11 +167,7 @@ pub const Tokenizer = struct { return loc; } if (c == '\n') { - loc.line += 1; - loc.column = 0; loc.line_start = i + 1; - } else { - loc.column += 1; } } return loc; @@ -182,6 +182,8 @@ pub const Tokenizer = struct { return Tokenizer { .buffer = buffer, .index = 0, + .line = 0, + .column = 0, .pending_invalid_token = null, }; } @@ -222,13 +224,21 @@ pub const Tokenizer = struct { .id = Token.Id.Eof, .start = self.index, .end = undefined, + .line = self.line, + .column = self.column, }; - while (self.index < self.buffer.len) : (self.index += 1) { + while (self.index < self.buffer.len) { const c = self.buffer[self.index]; switch (state) { State.Start => switch (c) { - ' ', '\n' => { + ' ' => { result.start = self.index + 1; + result.column += 1; + }, + '\n' => { + result.start = self.index + 1; + result.line += 1; + result.column = 0; }, 'c' => { state = State.C; @@ -474,6 +484,8 @@ pub const Tokenizer = struct { result = Token { .id = Token.Id.Eof, .start = self.index + 1, + .column = 0, + .line = self.line + 1, .end = undefined, }; }, @@ -543,6 +555,14 @@ pub const Tokenizer = struct { else => break, }, } + + self.index += 1; + if (c == '\n') { + self.line += 1; + self.column = 0; + } else { + self.column += 1; + } } else if (self.index == self.buffer.len) { switch (state) { State.Start, @@ -622,6 +642,8 @@ pub const Tokenizer = struct { .id = Token.Id.Invalid, .start = self.index, .end = self.index + invalid_length, + .line = self.line, + .column = self.column, }; }