Merge branch 'rework-parser'

CMakeLists.txt

@@ -576,7 +576,8 @@ set(ZIG_STD_FILES
     "unicode.zig"
     "zig/ast.zig"
     "zig/index.zig"
-    "zig/parser.zig"
+    "zig/parse.zig"
+    "zig/render.zig"
     "zig/tokenizer.zig"
 )
 

src-self-hosted/main.zig

@@ -671,34 +671,45 @@ fn cmdFmt(allocator: &Allocator, args: []const []const u8) !void {
         };
         defer allocator.free(source_code);
 
-        var tokenizer = std.zig.Tokenizer.init(source_code);
-        var parser = std.zig.Parser.init(&tokenizer, allocator, file_path);
-        defer parser.deinit();
-
-        var tree = parser.parse() catch |err| {
+        var tree = std.zig.parse(allocator, source_code) catch |err| {
             try stderr.print("error parsing file '{}': {}\n", file_path, err);
             continue;
         };
         defer tree.deinit();
 
-        var original_file_backup = try Buffer.init(allocator, file_path);
-        defer original_file_backup.deinit();
-        try original_file_backup.append(".backup");
 
-        try os.rename(allocator, file_path, original_file_backup.toSliceConst());
+        var error_it = tree.errors.iterator(0);
+        while (error_it.next()) |parse_error| {
+            const token = tree.tokens.at(parse_error.loc());
+            const loc = tree.tokenLocation(0, parse_error.loc());
+            try stderr.print("{}:{}:{}: error: ", file_path, loc.line + 1, loc.column + 1);
+            try tree.renderError(parse_error, stderr);
+            try stderr.print("\n{}\n", source_code[loc.line_start..loc.line_end]);
+            {
+                var i: usize = 0;
+                while (i < loc.column) : (i += 1) {
+                    try stderr.write(" ");
+                }
+            }
+            {
+                const caret_count = token.end - token.start;
+                var i: usize = 0;
+                while (i < caret_count) : (i += 1) {
+                    try stderr.write("~");
+                }
+            }
+            try stderr.write("\n");
+        }
+        if (tree.errors.len != 0) {
+            continue;
+        }
 
         try stderr.print("{}\n", file_path);
 
-        // TODO: BufferedAtomicFile has some access problems.
-        var out_file = try os.File.openWrite(allocator, file_path);
-        defer out_file.close();
+        const baf = try io.BufferedAtomicFile.create(allocator, file_path);
+        defer baf.destroy();
 
-        var out_file_stream = io.FileOutStream.init(&out_file);
-        try parser.renderSource(out_file_stream.stream, tree.root_node);
-
-        if (!flags.present("keep-backups")) {
-            try os.deleteFile(allocator, original_file_backup.toSliceConst());
-        }
+        try std.zig.render(allocator, baf.stream(), &tree);
     }
 }
 

src-self-hosted/module.zig

@@ -8,9 +8,7 @@ const c = @import("c.zig");
 const builtin = @import("builtin");
 const Target = @import("target.zig").Target;
 const warn = std.debug.warn;
-const Tokenizer = std.zig.Tokenizer;
 const Token = std.zig.Token;
-const Parser = std.zig.Parser;
 const ArrayList = std.ArrayList;
 
 pub const Module = struct {
@@ -246,34 +244,17 @@ pub const Module = struct {
 
         warn("{}", source_code);
 
-        warn("====tokenization:====\n");
-        {
-            var tokenizer = Tokenizer.init(source_code);
-            while (true) {
-                const token = tokenizer.next();
-                tokenizer.dump(token);
-                if (token.id == Token.Id.Eof) {
-                    break;
-                }
-            }
-        }
-
         warn("====parse:====\n");
 
-        var tokenizer = Tokenizer.init(source_code);
-        var parser = Parser.init(&tokenizer, self.allocator, root_src_real_path);
-        defer parser.deinit();
-
-        var tree = try parser.parse();
+        var tree = try std.zig.parse(self.allocator, source_code);
         defer tree.deinit();
 
         var stderr_file = try std.io.getStdErr();
         var stderr_file_out_stream = std.io.FileOutStream.init(&stderr_file);
         const out_stream = &stderr_file_out_stream.stream;
-        try parser.renderAst(out_stream, tree.root_node);
 
         warn("====fmt:====\n");
-        try parser.renderSource(out_stream, tree.root_node);
+        try std.zig.render(self.allocator, out_stream, &tree);
 
         warn("====ir:====\n");
         warn("TODO\n\n");

src/ir.cpp

@@ -14709,7 +14709,7 @@ static IrInstruction *ir_analyze_union_tag(IrAnalyze *ira, IrInstruction *source
     }
 
     if (value->value.type->id != TypeTableEntryIdUnion) {
-        ir_add_error(ira, source_instr,
+        ir_add_error(ira, value,
             buf_sprintf("expected enum or union type, found '%s'", buf_ptr(&value->value.type->name)));
         return ira->codegen->invalid_instruction;
     }

std/segmented_list.zig

@@ -91,6 +91,8 @@ pub fn SegmentedList(comptime T: type, comptime prealloc_item_count: usize) type
         allocator: &Allocator,
         len: usize,
 
+        pub const prealloc_count = prealloc_item_count;
+
         /// Deinitialize with `deinit`
         pub fn init(allocator: &Allocator) Self {
             return Self {
@@ -287,6 +289,15 @@ pub fn SegmentedList(comptime T: type, comptime prealloc_item_count: usize) type
 
                 return &it.list.dynamic_segments[it.shelf_index][it.box_index];
             }
+
+            pub fn peek(it: &Iterator) ?&T {
+                if (it.index >= it.list.len)
+                    return null;
+                if (it.index < prealloc_item_count)
+                    return &it.list.prealloc_segment[it.index];
+
+                return &it.list.dynamic_segments[it.shelf_index][it.box_index];
+            }
         };
 
         pub fn iterator(self: &Self, start_index: usize) Iterator {

std/zig/index.zig

@@ -1,11 +1,13 @@
 const tokenizer = @import("tokenizer.zig");
 pub const Token = tokenizer.Token;
 pub const Tokenizer = tokenizer.Tokenizer;
-pub const Parser = @import("parser.zig").Parser;
+pub const parse = @import("parse.zig").parse;
+pub const render = @import("render.zig").render;
 pub const ast = @import("ast.zig");
 
 test "std.zig tests" {
-    _ = @import("tokenizer.zig");
-    _ = @import("parser.zig");
     _ = @import("ast.zig");
+    _ = @import("parse.zig");
+    _ = @import("render.zig");
+    _ = @import("tokenizer.zig");
 }

std/zig/parser_test.zig

@@ -1,19 +1,53 @@
-test "zig fmt: same-line comment after non-block if expression" {
+test "zig fmt: same-line comment after a statement" {
     try testCanonical(
-        \\comptime {
-        \\    if (sr > n_uword_bits - 1) {
-        \\        // d > r
-        \\        return 0;
+        \\test "" {
+        \\    a = b;
+        \\    debug.assert(H.digest_size <= H.block_size); // HMAC makes this assumption
+        \\    a = b;
+        \\}
+        \\
+    );
+}
+
+test "zig fmt: same-line comment after var decl in struct" {
+    try testCanonical(
+        \\pub const vfs_cap_data = extern struct {
+        \\    const Data = struct {}; // when on disk.
+        \\};
+        \\
+    );
+}
+
+test "zig fmt: same-line comment after field decl" {
+    try testCanonical(
+        \\pub const dirent = extern struct {
+        \\    d_name: u8,
+        \\    d_name: u8, // comment 1
+        \\    d_name: u8,
+        \\    d_name: u8, // comment 2
+        \\    d_name: u8,
+        \\};
+        \\
+    );
+}
+
+test "zig fmt: same-line comment after switch prong" {
+    try testCanonical(
+        \\test "" {
+        \\    switch (err) {
+        \\        error.PathAlreadyExists => {}, // comment 2
+        \\        else => return err, // comment 1
         \\    }
         \\}
         \\
     );
 }
 
-test "zig fmt: switch with empty body" {
+test "zig fmt: same-line comment after non-block if expression" {
     try testCanonical(
-        \\test "" {
-        \\    foo() catch |err| switch (err) {};
+        \\comptime {
+        \\    if (sr > n_uword_bits - 1) // d > r
+        \\        return 0;
         \\}
         \\
     );
@@ -28,6 +62,15 @@ test "zig fmt: same-line comment on comptime expression" {
     );
 }
 
+test "zig fmt: switch with empty body" {
+    try testCanonical(
+        \\test "" {
+        \\    foo() catch |err| switch (err) {};
+        \\}
+        \\
+    );
+}
+
 test "zig fmt: float literal with exponent" {
     try testCanonical(
         \\pub const f64_true_min = 4.94065645841246544177e-324;
@@ -154,18 +197,6 @@ test "zig fmt: comments before switch prong" {
     );
 }
 
-test "zig fmt: same-line comment after switch prong" {
-    try testCanonical(
-        \\test "" {
-        \\    switch (err) {
-        \\        error.PathAlreadyExists => {}, // comment 2
-        \\        else => return err, // comment 1
-        \\    }
-        \\}
-        \\
-    );
-}
-
 test "zig fmt: comments before var decl in struct" {
     try testCanonical(
         \\pub const vfs_cap_data = extern struct {
@@ -191,28 +222,6 @@ test "zig fmt: comments before var decl in struct" {
     );
 }
 
-test "zig fmt: same-line comment after var decl in struct" {
-    try testCanonical(
-        \\pub const vfs_cap_data = extern struct {
-        \\    const Data = struct {}; // when on disk.
-        \\};
-        \\
-    );
-}
-
-test "zig fmt: same-line comment after field decl" {
-    try testCanonical(
-        \\pub const dirent = extern struct {
-        \\    d_name: u8,
-        \\    d_name: u8, // comment 1
-        \\    d_name: u8,
-        \\    d_name: u8, // comment 2
-        \\    d_name: u8,
-        \\};
-        \\
-    );
-}
-
 test "zig fmt: array literal with 1 item on 1 line" {
     try testCanonical(
         \\var s = []const u64{0} ** 25;
@@ -220,17 +229,6 @@ test "zig fmt: array literal with 1 item on 1 line" {
     );
 }
 
-test "zig fmt: same-line comment after a statement" {
-    try testCanonical(
-        \\test "" {
-        \\    a = b;
-        \\    debug.assert(H.digest_size <= H.block_size); // HMAC makes this assumption
-        \\    a = b;
-        \\}
-        \\
-    );
-}
-
 test "zig fmt: comments before global variables" {
     try testCanonical(
         \\/// Foo copies keys and values before they go into the map, and
@@ -1094,25 +1092,48 @@ test "zig fmt: error return" {
 const std = @import("std");
 const mem = std.mem;
 const warn = std.debug.warn;
-const Tokenizer = std.zig.Tokenizer;
-const Parser = std.zig.Parser;
 const io = std.io;
 
 var fixed_buffer_mem: [100 * 1024]u8 = undefined;
 
 fn testParse(source: []const u8, allocator: &mem.Allocator) ![]u8 {
-    var tokenizer = Tokenizer.init(source);
-    var parser = Parser.init(&tokenizer, allocator, "(memory buffer)");
-    defer parser.deinit();
+    var stderr_file = try io.getStdErr();
+    var stderr = &io.FileOutStream.init(&stderr_file).stream;
 
-    var tree = try parser.parse();
+    var tree = try std.zig.parse(allocator, source);
     defer tree.deinit();
 
+    var error_it = tree.errors.iterator(0);
+    while (error_it.next()) |parse_error| {
+        const token = tree.tokens.at(parse_error.loc());
+        const loc = tree.tokenLocation(0, parse_error.loc());
+        try stderr.print("(memory buffer):{}:{}: error: ", loc.line + 1, loc.column + 1);
+        try tree.renderError(parse_error, stderr);
+        try stderr.print("\n{}\n", source[loc.line_start..loc.line_end]);
+        {
+            var i: usize = 0;
+            while (i < loc.column) : (i += 1) {
+                try stderr.write(" ");
+            }
+        }
+        {
+            const caret_count = token.end - token.start;
+            var i: usize = 0;
+            while (i < caret_count) : (i += 1) {
+                try stderr.write("~");
+            }
+        }
+        try stderr.write("\n");
+    }
+    if (tree.errors.len != 0) {
+        return error.ParseError;
+    }
+
     var buffer = try std.Buffer.initSize(allocator, 0);
     errdefer buffer.deinit();
 
     var buffer_out_stream = io.BufferOutStream.init(&buffer);
-    try parser.renderSource(&buffer_out_stream.stream, tree.root_node);
+    try std.zig.render(allocator, &buffer_out_stream.stream, &tree);
     return buffer.toOwnedSlice();
 }
 
@@ -1151,6 +1172,7 @@ fn testTransform(source: []const u8, expected_source: []const u8) !void {
                 }
             },
             error.ParseError => @panic("test failed"),
+            else => @panic("test failed"),
         }
     }
 }

std/zig/tokenizer.zig

@@ -195,37 +195,6 @@ pub const Tokenizer = struct {
     index: usize,
     pending_invalid_token: ?Token,
 
-    pub const Location = struct {
-        line: usize,
-        column: usize,
-        line_start: usize,
-        line_end: usize,
-    };
-
-    pub fn getTokenLocation(self: &Tokenizer, start_index: usize, token: &const Token) Location {
-        var loc = Location {
-            .line = 0,
-            .column = 0,
-            .line_start = start_index,
-            .line_end = self.buffer.len,
-        };
-        for (self.buffer[start_index..]) |c, i| {
-            if (i + start_index == token.start) {
-                loc.line_end = i + start_index;
-                while (loc.line_end < self.buffer.len and self.buffer[loc.line_end] != '\n') : (loc.line_end += 1) {}
-                return loc;
-            }
-            if (c == '\n') {
-                loc.line += 1;
-                loc.column = 0;
-                loc.line_start = i + 1;
-            } else {
-                loc.column += 1;
-            }
-        }
-        return loc;
-    }
-
     /// For debugging purposes
     pub fn dump(self: &Tokenizer, token: &const Token) void {
         std.debug.warn("{} \"{}\"\n", @tagName(token.id), self.buffer[token.start..token.end]);
@@ -1047,10 +1016,6 @@ pub const Tokenizer = struct {
         return result;
     }
 
-    pub fn getTokenSlice(self: &const Tokenizer, token: &const Token) []const u8 {
-        return self.buffer[token.start..token.end];
-    }
-
     fn checkLiteralCharacter(self: &Tokenizer) void {
         if (self.pending_invalid_token != null) return;
         const invalid_length = self.getInvalidCharacterLength();