From 9d00f69be58e7b807e26ba8a38050dd486d487f4 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 19 Jul 2024 17:38:15 -0700 Subject: [PATCH 1/7] move std.zig.fatal to std.process.fatal --- lib/std/process.zig | 6 ++++++ lib/std/zig.zig | 6 ++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/std/process.zig b/lib/std/process.zig index 86654e4b5a..eca3a26c29 100644 --- a/lib/std/process.zig +++ b/lib/std/process.zig @@ -2032,3 +2032,9 @@ pub fn createWindowsEnvBlock(allocator: mem.Allocator, env_map: *const EnvMap) ! i += 1; return try allocator.realloc(result, i); } + +/// Logs an error and then terminates the process with exit code 1. +pub fn fatal(comptime format: []const u8, format_arguments: anytype) noreturn { + std.log.err(format, format_arguments); + exit(1); +} diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 230dc45ddc..01d3aafa75 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -667,10 +667,8 @@ pub fn parseTargetQueryOrReportFatalError( }; } -pub fn fatal(comptime format: []const u8, args: anytype) noreturn { - std.log.err(format, args); - std.process.exit(1); -} +/// Deprecated; see `std.process.fatal`. +pub const fatal = std.process.fatal; /// Collects all the environment variables that Zig could possibly inspect, so /// that we can do reflection on this and print them with `zig env`. From eb4028bf30bca7036c6bdc65feb321b163e84ecd Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 19 Jul 2024 17:39:55 -0700 Subject: [PATCH 2/7] add std.fmt.hex converts an unsigned integer into an array --- lib/std/Build.zig | 15 +++------------ lib/std/Build/Step/Options.zig | 2 +- lib/std/Build/Step/Run.zig | 2 +- lib/std/fmt.zig | 35 ++++++++++++++++++++++++++++++++++ src/Compilation.zig | 4 ++-- src/Package/Fetch.zig | 2 +- src/Package/Manifest.zig | 35 +++++++++------------------------- src/link.zig | 2 +- src/main.zig | 5 ++--- 9 files changed, 55 insertions(+), 47 deletions(-) diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 06de7aa6f4..c68170b2c1 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -2522,7 +2522,7 @@ pub const InstallDir = union(enum) { /// function. pub fn makeTempPath(b: *Build) []const u8 { const rand_int = std.crypto.random.int(u64); - const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ hex64(rand_int); + const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int); const result_path = b.cache_root.join(b.allocator, &.{tmp_dir_sub_path}) catch @panic("OOM"); b.cache_root.handle.makePath(tmp_dir_sub_path) catch |err| { std.debug.print("unable to make tmp path '{s}': {s}\n", .{ @@ -2532,18 +2532,9 @@ pub fn makeTempPath(b: *Build) []const u8 { return result_path; } -/// There are a few copies of this function in miscellaneous places. Would be nice to find -/// a home for them. +/// Deprecated; use `std.fmt.hex` instead. pub fn hex64(x: u64) [16]u8 { - const hex_charset = "0123456789abcdef"; - var result: [16]u8 = undefined; - var i: usize = 0; - while (i < 8) : (i += 1) { - const byte: u8 = @truncate(x >> @as(u6, @intCast(8 * i))); - result[i * 2 + 0] = hex_charset[byte >> 4]; - result[i * 2 + 1] = hex_charset[byte & 15]; - } - return result; + return std.fmt.hex(x); } /// A pair of target query and fully resolved target. diff --git a/lib/std/Build/Step/Options.zig b/lib/std/Build/Step/Options.zig index 7c872db170..6131912b1b 100644 --- a/lib/std/Build/Step/Options.zig +++ b/lib/std/Build/Step/Options.zig @@ -457,7 +457,7 @@ fn make(step: *Step, make_options: Step.MakeOptions) !void { const rand_int = std.crypto.random.int(u64); const tmp_sub_path = "tmp" ++ fs.path.sep_str ++ - std.Build.hex64(rand_int) ++ fs.path.sep_str ++ + std.fmt.hex(rand_int) ++ fs.path.sep_str ++ basename; const tmp_sub_path_dirname = fs.path.dirname(tmp_sub_path).?; diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index d79d4647b1..0712ce8083 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -743,7 +743,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void { // We do not know the final output paths yet, use temp paths to run the command. const rand_int = std.crypto.random.int(u64); - const tmp_dir_path = "tmp" ++ fs.path.sep_str ++ std.Build.hex64(rand_int); + const tmp_dir_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int); for (output_placeholders.items) |placeholder| { const output_components = .{ tmp_dir_path, placeholder.output.basename }; diff --git a/lib/std/fmt.zig b/lib/std/fmt.zig index a4407a8cd2..6283c9453e 100644 --- a/lib/std/fmt.zig +++ b/lib/std/fmt.zig @@ -2718,3 +2718,38 @@ test "recursive format function" { var r = R{ .Leaf = 1 }; try expectFmt("Leaf(1)\n", "{}\n", .{&r}); } + +pub const hex_charset = "0123456789abcdef"; + +/// Converts an unsigned integer of any multiple of u8 to an array of lowercase +/// hex bytes. +pub fn hex(x: anytype) [@sizeOf(@TypeOf(x)) * 2]u8 { + comptime assert(@typeInfo(@TypeOf(x)).Int.signedness == .unsigned); + var result: [@sizeOf(@TypeOf(x)) * 2]u8 = undefined; + var i: usize = 0; + while (i < result.len / 2) : (i += 1) { + const byte: u8 = @truncate(x >> @intCast(8 * i)); + result[i * 2 + 0] = hex_charset[byte >> 4]; + result[i * 2 + 1] = hex_charset[byte & 15]; + } + return result; +} + +test hex { + { + const x = hex(@as(u32, 0xdeadbeef)); + try std.testing.expect(x.len == 8); + switch (builtin.cpu.arch.endian()) { + .little => try std.testing.expectEqualStrings("efbeadde", &x), + .big => try std.testing.expectEqualStrings("deadbeef", &x), + } + } + { + const s = "[" ++ hex(@as(u64, 0x12345678_abcdef00)) ++ "]"; + try std.testing.expect(s.len == 18); + switch (builtin.cpu.arch.endian()) { + .little => try std.testing.expectEqualStrings("[00efcdab78563412]", s), + .big => try std.testing.expectEqualStrings("[12345678abcdef00]", s), + } + } +} diff --git a/src/Compilation.zig b/src/Compilation.zig index b1c1851a6c..a36ab2a47e 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -2106,7 +2106,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void { const tmp_artifact_directory = d: { const s = std.fs.path.sep_str; tmp_dir_rand_int = std.crypto.random.int(u64); - const tmp_dir_sub_path = "tmp" ++ s ++ Package.Manifest.hex64(tmp_dir_rand_int); + const tmp_dir_sub_path = "tmp" ++ s ++ std.fmt.hex(tmp_dir_rand_int); const path = try comp.local_cache_directory.join(gpa, &.{tmp_dir_sub_path}); errdefer gpa.free(path); @@ -2298,7 +2298,7 @@ pub fn update(comp: *Compilation, main_progress_node: std.Progress.Node) !void { } else unreachable; const s = std.fs.path.sep_str; - const tmp_dir_sub_path = "tmp" ++ s ++ Package.Manifest.hex64(tmp_dir_rand_int); + const tmp_dir_sub_path = "tmp" ++ s ++ std.fmt.hex(tmp_dir_rand_int); const o_sub_path = "o" ++ s ++ digest; // Work around windows `AccessDenied` if any files within this diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index f6e5c2e222..28030e3879 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -445,7 +445,7 @@ fn runResource( const s = fs.path.sep_str; const cache_root = f.job_queue.global_cache; const rand_int = std.crypto.random.int(u64); - const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int); + const tmp_dir_sub_path = "tmp" ++ s ++ std.fmt.hex(rand_int); const package_sub_path = blk: { const tmp_directory_path = try cache_root.join(arena, &.{tmp_dir_sub_path}); diff --git a/src/Package/Manifest.zig b/src/Package/Manifest.zig index 3bcb4a7958..d9c39f5ab3 100644 --- a/src/Package/Manifest.zig +++ b/src/Package/Manifest.zig @@ -1,3 +1,12 @@ +const Manifest = @This(); +const std = @import("std"); +const mem = std.mem; +const Allocator = std.mem.Allocator; +const assert = std.debug.assert; +const Ast = std.zig.Ast; +const testing = std.testing; +const hex_charset = std.fmt.hex_charset; + pub const max_bytes = 10 * 1024 * 1024; pub const basename = "build.zig.zon"; pub const Hash = std.crypto.hash.sha2.Sha256; @@ -153,24 +162,6 @@ pub fn copyErrorsIntoBundle( } } -const hex_charset = "0123456789abcdef"; - -pub fn hex64(x: u64) [16]u8 { - var result: [16]u8 = undefined; - var i: usize = 0; - while (i < 8) : (i += 1) { - const byte = @as(u8, @truncate(x >> @as(u6, @intCast(8 * i)))); - result[i * 2 + 0] = hex_charset[byte >> 4]; - result[i * 2 + 1] = hex_charset[byte & 15]; - } - return result; -} - -test hex64 { - const s = "[" ++ hex64(0x12345678_abcdef00) ++ "]"; - try std.testing.expectEqualStrings("[00efcdab78563412]", s); -} - pub fn hexDigest(digest: Digest) MultiHashHexDigest { var result: MultiHashHexDigest = undefined; @@ -590,14 +581,6 @@ const Parse = struct { } }; -const Manifest = @This(); -const std = @import("std"); -const mem = std.mem; -const Allocator = std.mem.Allocator; -const assert = std.debug.assert; -const Ast = std.zig.Ast; -const testing = std.testing; - test "basic" { const gpa = testing.allocator; diff --git a/src/link.zig b/src/link.zig index cf780c76fa..fd74f078ac 100644 --- a/src/link.zig +++ b/src/link.zig @@ -1031,7 +1031,7 @@ pub fn spawnLld( error.NameTooLong => err: { const s = fs.path.sep_str; const rand_int = std.crypto.random.int(u64); - const rsp_path = "tmp" ++ s ++ Package.Manifest.hex64(rand_int) ++ ".rsp"; + const rsp_path = "tmp" ++ s ++ std.fmt.hex(rand_int) ++ ".rsp"; const rsp_file = try comp.local_cache_directory.handle.createFileZ(rsp_path, .{}); defer comp.local_cache_directory.handle.deleteFileZ(rsp_path) catch |err| diff --git a/src/main.zig b/src/main.zig index 64cbbdffe4..561dd2f28e 100644 --- a/src/main.zig +++ b/src/main.zig @@ -4746,7 +4746,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { // the strategy is to choose a temporary file name ahead of time, and then // read this file in the parent to obtain the results, in the case the child // exits with code 3. - const results_tmp_file_nonce = Package.Manifest.hex64(std.crypto.random.int(u64)); + const results_tmp_file_nonce = std.fmt.hex(std.crypto.random.int(u64)); try child_argv.append("-Z" ++ results_tmp_file_nonce); var color: Color = .auto; @@ -7196,8 +7196,7 @@ fn createDependenciesModule( // Atomically create the file in a directory named after the hash of its contents. const basename = "dependencies.zig"; const rand_int = std.crypto.random.int(u64); - const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ - Package.Manifest.hex64(rand_int); + const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ std.fmt.hex(rand_int); { var tmp_dir = try local_cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{}); defer tmp_dir.close(); From 397c9174cb8683ae894beafc97eac23e8aa5a760 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 20 Jul 2024 01:05:59 -0700 Subject: [PATCH 3/7] fix std.fmt.hex --- lib/std/fmt.zig | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/lib/std/fmt.zig b/lib/std/fmt.zig index 6283c9453e..a7136d99bb 100644 --- a/lib/std/fmt.zig +++ b/lib/std/fmt.zig @@ -2722,7 +2722,7 @@ test "recursive format function" { pub const hex_charset = "0123456789abcdef"; /// Converts an unsigned integer of any multiple of u8 to an array of lowercase -/// hex bytes. +/// hex bytes, little endian. pub fn hex(x: anytype) [@sizeOf(@TypeOf(x)) * 2]u8 { comptime assert(@typeInfo(@TypeOf(x)).Int.signedness == .unsigned); var result: [@sizeOf(@TypeOf(x)) * 2]u8 = undefined; @@ -2739,17 +2739,11 @@ test hex { { const x = hex(@as(u32, 0xdeadbeef)); try std.testing.expect(x.len == 8); - switch (builtin.cpu.arch.endian()) { - .little => try std.testing.expectEqualStrings("efbeadde", &x), - .big => try std.testing.expectEqualStrings("deadbeef", &x), - } + try std.testing.expectEqualStrings("efbeadde", &x); } { const s = "[" ++ hex(@as(u64, 0x12345678_abcdef00)) ++ "]"; try std.testing.expect(s.len == 18); - switch (builtin.cpu.arch.endian()) { - .little => try std.testing.expectEqualStrings("[00efcdab78563412]", s), - .big => try std.testing.expectEqualStrings("[12345678abcdef00]", s), - } + try std.testing.expectEqualStrings("[00efcdab78563412]", s); } } From 7b1d3e72761e646083774e11ccf6a122f7798d35 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 19 Jul 2024 21:38:08 -0700 Subject: [PATCH 4/7] compiler server: detect when parent process pipe is broken closes #18340 --- lib/std/zig/Server.zig | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/std/zig/Server.zig b/lib/std/zig/Server.zig index 2d66ee628d..4046fe4014 100644 --- a/lib/std/zig/Server.zig +++ b/lib/std/zig/Server.zig @@ -109,6 +109,7 @@ pub fn deinit(s: *Server) void { pub fn receiveMessage(s: *Server) !InMessage.Header { const Header = InMessage.Header; const fifo = &s.receive_fifo; + var last_amt_zero = false; while (true) { const buf = fifo.readableSlice(0); @@ -136,6 +137,10 @@ pub fn receiveMessage(s: *Server) !InMessage.Header { const write_buffer = try fifo.writableWithSize(256); const amt = try s.in.read(write_buffer); fifo.update(amt); + if (amt == 0) { + if (last_amt_zero) return error.BrokenPipe; + last_amt_zero = true; + } } } From ea2c45227af0000353fe3626293bc22e1f04d1f7 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 19 Jul 2024 17:41:04 -0700 Subject: [PATCH 5/7] init incremental compilation check tool --- test/incremental/hello | 13 +++ tools/incr-check.zig | 215 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 228 insertions(+) create mode 100644 test/incremental/hello create mode 100644 tools/incr-check.zig diff --git a/test/incremental/hello b/test/incremental/hello new file mode 100644 index 0000000000..c6f4b5ace6 --- /dev/null +++ b/test/incremental/hello @@ -0,0 +1,13 @@ +#target=x86_64-linux +#update=initial version +#file=main.zig +pub fn main() !void { + try std.io.getStdOut().writeAll("good morning\n"); +} +#expect_stdout="good morning\n" +#update=change the string +#file=main.zig +pub fn main() !void { + try std.io.getStdOut().writeAll("おはようございます\n"); +} +#expect_stdout="おはようございます\n" diff --git a/tools/incr-check.zig b/tools/incr-check.zig new file mode 100644 index 0000000000..9f638bd6ff --- /dev/null +++ b/tools/incr-check.zig @@ -0,0 +1,215 @@ +const std = @import("std"); +const fatal = std.process.fatal; +const Allocator = std.mem.Allocator; + +pub fn main() !void { + var arena_instance = std.heap.ArenaAllocator.init(std.heap.page_allocator); + defer arena_instance.deinit(); + const arena = arena_instance.allocator(); + + const args = try std.process.argsAlloc(arena); + const zig_exe = args[1]; + const input_file_name = args[2]; + + const input_file_bytes = try std.fs.cwd().readFileAlloc(arena, input_file_name, std.math.maxInt(u32)); + const case = try Case.parse(arena, input_file_bytes); + + const prog_node = std.Progress.start(.{}); + defer prog_node.end(); + + const rand_int = std.crypto.random.int(u64); + const tmp_dir_path = "tmp_" ++ std.fmt.hex(rand_int); + const local_cache_path = tmp_dir_path ++ std.fs.path.sep_str ++ ".local-cache"; + const global_cache_path = tmp_dir_path ++ std.fs.path.sep_str ++ ".global-cache"; + const tmp_dir = try std.fs.cwd().makeOpenPath(tmp_dir_path, .{}); + + const child_prog_node = prog_node.start("zig build-exe", 0); + defer child_prog_node.end(); + + var child = std.process.Child.init(&.{ + zig_exe, + "build-exe", + case.root_source_file, + "-fno-llvm", + "-fno-lld", + "-fincremental", + "--listen=-", + "-target", + case.target_query, + "--cache-dir", + local_cache_path, + "--global-cache-dir", + global_cache_path, + }, arena); + + child.stdin_behavior = .Pipe; + child.stdout_behavior = .Pipe; + child.stderr_behavior = .Pipe; + child.progress_node = child_prog_node; + + var eval: Eval = .{ + .case = case, + .tmp_dir = tmp_dir, + .child = &child, + }; + + eval.write(case.updates[0]); + + try child.spawn(); + + var poller = std.io.poll(arena, enum { stdout, stderr }, .{ + .stdout = child.stdout.?, + .stderr = child.stderr.?, + }); + defer poller.deinit(); + + try eval.check(case.updates[0]); + + for (case.updates[1..]) |update| { + eval.write(update); + try eval.requestIncrementalUpdate(); + try eval.check(update); + } +} + +const Eval = struct { + case: Case, + tmp_dir: std.fs.Dir, + child: *std.process.Child, + + /// Currently this function assumes the previous updates have already been written. + fn write(eval: *Eval, update: Case.Update) void { + for (update.changes) |full_contents| { + eval.tmp_dir.writeFile(.{ + .sub_path = full_contents.name, + .data = full_contents.bytes, + }) catch |err| { + fatal("failed to update '{s}': {s}", .{ full_contents.name, @errorName(err) }); + }; + } + for (update.deletes) |doomed_name| { + eval.tmp_dir.deleteFile(doomed_name) catch |err| { + fatal("failed to delete '{s}': {s}", .{ doomed_name, @errorName(err) }); + }; + } + } + + fn check(eval: *Eval, update: Case.Update) !void { + _ = eval; + _ = update; + @panic("TODO: read messages from the compiler"); + } + + fn requestIncrementalUpdate(eval: *Eval) !void { + _ = eval; + @panic("TODO: send update request to the compiler"); + } +}; + +const Case = struct { + updates: []Update, + root_source_file: []const u8, + target_query: []const u8, + + const Update = struct { + name: []const u8, + outcome: Outcome, + changes: []const FullContents = &.{}, + deletes: []const []const u8 = &.{}, + }; + + const FullContents = struct { + name: []const u8, + bytes: []const u8, + }; + + const Outcome = union(enum) { + unknown, + compile_errors: []const ExpectedError, + stdout: []const u8, + exit_code: u8, + }; + + const ExpectedError = struct { + file_name: ?[]const u8 = null, + line: ?u32 = null, + column: ?u32 = null, + msg_exact: ?[]const u8 = null, + msg_substring: ?[]const u8 = null, + }; + + fn parse(arena: Allocator, bytes: []const u8) !Case { + var updates: std.ArrayListUnmanaged(Update) = .{}; + var changes: std.ArrayListUnmanaged(FullContents) = .{}; + var target_query: ?[]const u8 = null; + var it = std.mem.splitScalar(u8, bytes, '\n'); + var line_n: usize = 1; + var root_source_file: ?[]const u8 = null; + while (it.next()) |line| : (line_n += 1) { + if (std.mem.startsWith(u8, line, "#")) { + var line_it = std.mem.splitScalar(u8, line, '='); + const key = line_it.first()[1..]; + const val = line_it.rest(); + if (val.len == 0) { + fatal("line {d}: missing value", .{line_n}); + } else if (std.mem.eql(u8, key, "target")) { + if (target_query != null) fatal("line {d}: duplicate target", .{line_n}); + target_query = val; + } else if (std.mem.eql(u8, key, "update")) { + if (updates.items.len > 0) { + const last_update = &updates.items[updates.items.len - 1]; + last_update.changes = try changes.toOwnedSlice(arena); + } + try updates.append(arena, .{ + .name = val, + .outcome = .unknown, + }); + } else if (std.mem.eql(u8, key, "file")) { + if (updates.items.len == 0) fatal("line {d}: expect directive before update", .{line_n}); + + if (root_source_file == null) + root_source_file = val; + + const start_index = it.index.?; + const src = while (true) : (line_n += 1) { + const old = it; + const next_line = it.next() orelse fatal("line {d}: unexpected EOF", .{line_n}); + if (std.mem.startsWith(u8, next_line, "#")) { + const end_index = old.index.?; + const src = bytes[start_index..end_index]; + it = old; + break src; + } + }; + + try changes.append(arena, .{ + .name = val, + .bytes = src, + }); + } else if (std.mem.eql(u8, key, "expect_stdout")) { + if (updates.items.len == 0) fatal("line {d}: expect directive before update", .{line_n}); + const last_update = &updates.items[updates.items.len - 1]; + if (last_update.outcome != .unknown) fatal("line {d}: conflicting expect directive", .{line_n}); + last_update.outcome = .{ + .stdout = std.zig.string_literal.parseAlloc(arena, val) catch |err| { + fatal("line {d}: bad string literal: {s}", .{ line_n, @errorName(err) }); + }, + }; + } else { + fatal("line {d}: unrecognized key '{s}'", .{ line_n, key }); + } + } + } + + if (changes.items.len > 0) { + const last_update = &updates.items[updates.items.len - 1]; + last_update.changes = try changes.toOwnedSlice(arena); + } + + return .{ + .updates = updates.items, + .root_source_file = root_source_file orelse fatal("missing root source file", .{}), + .target_query = target_query orelse fatal("missing target", .{}), + }; + } +}; From 9f112ce8689213b34f10709a337f713b3df5f581 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 19 Jul 2024 21:40:06 -0700 Subject: [PATCH 6/7] incr-test: running an update --- test/incremental/hello | 2 + tools/incr-check.zig | 194 ++++++++++++++++++++++++++++++++++++----- 2 files changed, 176 insertions(+), 20 deletions(-) diff --git a/test/incremental/hello b/test/incremental/hello index c6f4b5ace6..cad14d6e3d 100644 --- a/test/incremental/hello +++ b/test/incremental/hello @@ -1,12 +1,14 @@ #target=x86_64-linux #update=initial version #file=main.zig +const std = @import("std"); pub fn main() !void { try std.io.getStdOut().writeAll("good morning\n"); } #expect_stdout="good morning\n" #update=change the string #file=main.zig +const std = @import("std"); pub fn main() !void { try std.io.getStdOut().writeAll("おはようございます\n"); } diff --git a/tools/incr-check.zig b/tools/incr-check.zig index 9f638bd6ff..8bfd2079be 100644 --- a/tools/incr-check.zig +++ b/tools/incr-check.zig @@ -19,64 +19,70 @@ pub fn main() !void { const rand_int = std.crypto.random.int(u64); const tmp_dir_path = "tmp_" ++ std.fmt.hex(rand_int); - const local_cache_path = tmp_dir_path ++ std.fs.path.sep_str ++ ".local-cache"; - const global_cache_path = tmp_dir_path ++ std.fs.path.sep_str ++ ".global-cache"; const tmp_dir = try std.fs.cwd().makeOpenPath(tmp_dir_path, .{}); const child_prog_node = prog_node.start("zig build-exe", 0); defer child_prog_node.end(); var child = std.process.Child.init(&.{ - zig_exe, + // Convert incr-check-relative path to subprocess-relative path. + try std.fs.path.relative(arena, tmp_dir_path, zig_exe), "build-exe", case.root_source_file, "-fno-llvm", "-fno-lld", "-fincremental", - "--listen=-", "-target", case.target_query, "--cache-dir", - local_cache_path, + ".local-cache", "--global-cache-dir", - global_cache_path, + ".global_cache", + "--listen=-", }, arena); child.stdin_behavior = .Pipe; child.stdout_behavior = .Pipe; child.stderr_behavior = .Pipe; child.progress_node = child_prog_node; + child.cwd_dir = tmp_dir; + child.cwd = tmp_dir_path; var eval: Eval = .{ + .arena = arena, .case = case, .tmp_dir = tmp_dir, .child = &child, }; - eval.write(case.updates[0]); - try child.spawn(); - var poller = std.io.poll(arena, enum { stdout, stderr }, .{ + var poller = std.io.poll(arena, Eval.StreamEnum, .{ .stdout = child.stdout.?, .stderr = child.stderr.?, }); defer poller.deinit(); - try eval.check(case.updates[0]); - - for (case.updates[1..]) |update| { + for (case.updates) |update| { eval.write(update); - try eval.requestIncrementalUpdate(); - try eval.check(update); + try eval.requestUpdate(); + try eval.check(&poller, update); } + + try eval.end(&poller); + + waitChild(&child); } const Eval = struct { + arena: Allocator, case: Case, tmp_dir: std.fs.Dir, child: *std.process.Child, + const StreamEnum = enum { stdout, stderr }; + const Poller = std.io.Poller(StreamEnum); + /// Currently this function assumes the previous updates have already been written. fn write(eval: *Eval, update: Case.Update) void { for (update.changes) |full_contents| { @@ -94,15 +100,137 @@ const Eval = struct { } } - fn check(eval: *Eval, update: Case.Update) !void { - _ = eval; - _ = update; - @panic("TODO: read messages from the compiler"); + fn check(eval: *Eval, poller: *Poller, update: Case.Update) !void { + const arena = eval.arena; + const Header = std.zig.Server.Message.Header; + const stdout = poller.fifo(.stdout); + const stderr = poller.fifo(.stderr); + + poll: while (true) { + while (stdout.readableLength() < @sizeOf(Header)) { + if (!(try poller.poll())) break :poll; + } + const header = stdout.reader().readStruct(Header) catch unreachable; + while (stdout.readableLength() < header.bytes_len) { + if (!(try poller.poll())) break :poll; + } + const body = stdout.readableSliceOfLen(header.bytes_len); + std.log.debug("received message: {s}", .{@tagName(header.tag)}); + + switch (header.tag) { + .error_bundle => { + const EbHdr = std.zig.Server.Message.ErrorBundle; + const eb_hdr = @as(*align(1) const EbHdr, @ptrCast(body)); + const extra_bytes = + body[@sizeOf(EbHdr)..][0 .. @sizeOf(u32) * eb_hdr.extra_len]; + const string_bytes = + body[@sizeOf(EbHdr) + extra_bytes.len ..][0..eb_hdr.string_bytes_len]; + // TODO: use @ptrCast when the compiler supports it + const unaligned_extra = std.mem.bytesAsSlice(u32, extra_bytes); + const extra_array = try arena.alloc(u32, unaligned_extra.len); + @memcpy(extra_array, unaligned_extra); + const result_error_bundle: std.zig.ErrorBundle = .{ + .string_bytes = try arena.dupe(u8, string_bytes), + .extra = extra_array, + }; + if (stderr.readableLength() > 0) { + const stderr_data = try stderr.toOwnedSlice(); + fatal("error_bundle included unexpected stderr:\n{s}", .{stderr_data}); + } + try eval.checkErrorOutcome(update, result_error_bundle); + // This message indicates the end of the update. + stdout.discard(body.len); + return; + }, + .emit_bin_path => { + const EbpHdr = std.zig.Server.Message.EmitBinPath; + const ebp_hdr = @as(*align(1) const EbpHdr, @ptrCast(body)); + _ = ebp_hdr; + const result_binary = try arena.dupe(u8, body[@sizeOf(EbpHdr)..]); + if (stderr.readableLength() > 0) { + const stderr_data = try stderr.toOwnedSlice(); + fatal("emit_bin_path included unexpected stderr:\n{s}", .{stderr_data}); + } + try eval.checkSuccessOutcome(update, result_binary); + // This message indicates the end of the update. + stdout.discard(body.len); + return; + }, + else => { + // Ignore other messages. + stdout.discard(body.len); + }, + } + } + + if (stderr.readableLength() > 0) { + const stderr_data = try stderr.toOwnedSlice(); + fatal("update '{s}' failed:\n{s}", .{ update.name, stderr_data }); + } + + waitChild(eval.child); + fatal("update '{s}': compiler failed to send error_bundle or emit_bin_path", .{update.name}); } - fn requestIncrementalUpdate(eval: *Eval) !void { + fn checkErrorOutcome(eval: *Eval, update: Case.Update, error_bundle: std.zig.ErrorBundle) !void { _ = eval; - @panic("TODO: send update request to the compiler"); + switch (update.outcome) { + .unknown => return, + .compile_errors => |expected_errors| { + for (expected_errors) |expected_error| { + _ = expected_error; + @panic("TODO check if the expected error matches the compile errors"); + } + }, + .stdout, .exit_code => { + const color: std.zig.Color = .auto; + error_bundle.renderToStdErr(color.renderOptions()); + fatal("update '{s}': unexpected compile errors", .{update.name}); + }, + } + } + + fn checkSuccessOutcome(eval: *Eval, update: Case.Update, binary_path: []const u8) !void { + _ = eval; + switch (update.outcome) { + .unknown => return, + .compile_errors => fatal("expected compile errors but compilation incorrectly succeeded", .{}), + .stdout, .exit_code => {}, + } + fatal("TODO: run this binary: '{s}'", .{binary_path}); + } + + fn requestUpdate(eval: *Eval) !void { + const header: std.zig.Client.Message.Header = .{ + .tag = .update, + .bytes_len = 0, + }; + try eval.child.stdin.?.writeAll(std.mem.asBytes(&header)); + } + + fn end(eval: *Eval, poller: *Poller) !void { + requestExit(eval.child); + + const Header = std.zig.Server.Message.Header; + const stdout = poller.fifo(.stdout); + const stderr = poller.fifo(.stderr); + + poll: while (true) { + while (stdout.readableLength() < @sizeOf(Header)) { + if (!(try poller.poll())) break :poll; + } + const header = stdout.reader().readStruct(Header) catch unreachable; + while (stdout.readableLength() < header.bytes_len) { + if (!(try poller.poll())) break :poll; + } + const body = stdout.readableSliceOfLen(header.bytes_len); + stdout.discard(body.len); + } + + if (stderr.readableLength() > 0) { + const stderr_data = try stderr.toOwnedSlice(); + fatal("unexpected stderr:\n{s}", .{stderr_data}); + } } }; @@ -213,3 +341,29 @@ const Case = struct { }; } }; + +fn requestExit(child: *std.process.Child) void { + if (child.stdin == null) return; + + const header: std.zig.Client.Message.Header = .{ + .tag = .exit, + .bytes_len = 0, + }; + child.stdin.?.writeAll(std.mem.asBytes(&header)) catch |err| switch (err) { + error.BrokenPipe => {}, + else => fatal("failed to send exit: {s}", .{@errorName(err)}), + }; + + // Send EOF to stdin. + child.stdin.?.close(); + child.stdin = null; +} + +fn waitChild(child: *std.process.Child) void { + requestExit(child); + const term = child.wait() catch |err| fatal("child process failed: {s}", .{@errorName(err)}); + switch (term) { + .Exited => |code| if (code != 0) fatal("compiler failed with code {d}", .{code}), + .Signal, .Stopped, .Unknown => fatal("compiler terminated unexpectedly", .{}), + } +} From 645ad1ef72a09785fe5d7b33f1f9f2394bf52f57 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 19 Jul 2024 21:59:12 -0700 Subject: [PATCH 7/7] incr-check: support running the generated binary --- tools/incr-check.zig | 40 +++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/tools/incr-check.zig b/tools/incr-check.zig index 8bfd2079be..56858b88a3 100644 --- a/tools/incr-check.zig +++ b/tools/incr-check.zig @@ -52,6 +52,7 @@ pub fn main() !void { .arena = arena, .case = case, .tmp_dir = tmp_dir, + .tmp_dir_path = tmp_dir_path, .child = &child, }; @@ -78,6 +79,7 @@ const Eval = struct { arena: Allocator, case: Case, tmp_dir: std.fs.Dir, + tmp_dir_path: []const u8, child: *std.process.Child, const StreamEnum = enum { stdout, stderr }; @@ -115,7 +117,6 @@ const Eval = struct { if (!(try poller.poll())) break :poll; } const body = stdout.readableSliceOfLen(header.bytes_len); - std.log.debug("received message: {s}", .{@tagName(header.tag)}); switch (header.tag) { .error_bundle => { @@ -191,13 +192,46 @@ const Eval = struct { } fn checkSuccessOutcome(eval: *Eval, update: Case.Update, binary_path: []const u8) !void { - _ = eval; switch (update.outcome) { .unknown => return, .compile_errors => fatal("expected compile errors but compilation incorrectly succeeded", .{}), .stdout, .exit_code => {}, } - fatal("TODO: run this binary: '{s}'", .{binary_path}); + const result = std.process.Child.run(.{ + .allocator = eval.arena, + .argv = &.{binary_path}, + .cwd_dir = eval.tmp_dir, + .cwd = eval.tmp_dir_path, + }) catch |err| { + fatal("update '{s}': failed to run the generated executable '{s}': {s}", .{ + update.name, binary_path, @errorName(err), + }); + }; + if (result.stderr.len != 0) { + std.log.err("update '{s}': generated executable '{s}' had unexpected stderr:\n{s}", .{ + update.name, binary_path, result.stderr, + }); + } + switch (result.term) { + .Exited => |code| switch (update.outcome) { + .unknown, .compile_errors => unreachable, + .stdout => |expected_stdout| { + if (code != 0) { + fatal("update '{s}': generated executable '{s}' failed with code {d}", .{ + update.name, binary_path, code, + }); + } + try std.testing.expectEqualStrings(expected_stdout, result.stdout); + }, + .exit_code => |expected_code| try std.testing.expectEqual(expected_code, result.term.Exited), + }, + .Signal, .Stopped, .Unknown => { + fatal("update '{s}': generated executable '{s}' terminated unexpectedly", .{ + update.name, binary_path, + }); + }, + } + if (result.stderr.len != 0) std.process.exit(1); } fn requestUpdate(eval: *Eval) !void {