From 0e078790feaf49964d7a0da3042117ebd10de13b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 28 Feb 2023 23:58:13 -0700 Subject: [PATCH] multiplex compiler progress messages into the build runner --- lib/build_runner.zig | 2 +- lib/std/Build.zig | 16 ++++-- lib/std/Build/CheckFileStep.zig | 3 +- lib/std/Build/CheckObjectStep.zig | 3 +- lib/std/Build/CompileStep.zig | 6 +-- lib/std/Build/ConfigHeaderStep.zig | 3 +- lib/std/Build/EmulatableRunStep.zig | 3 +- lib/std/Build/FmtStep.zig | 3 +- lib/std/Build/InstallArtifactStep.zig | 3 +- lib/std/Build/InstallDirStep.zig | 3 +- lib/std/Build/InstallFileStep.zig | 3 +- lib/std/Build/LogStep.zig | 3 +- lib/std/Build/ObjCopyStep.zig | 3 +- lib/std/Build/OptionsStep.zig | 3 +- lib/std/Build/RemoveDirStep.zig | 3 +- lib/std/Build/RunStep.zig | 3 +- lib/std/Build/Step.zig | 13 +++-- lib/std/Build/TranslateCStep.zig | 4 +- lib/std/Build/WriteFileStep.zig | 3 +- src/main.zig | 73 ++++++++++++++++++++++++++- test/tests.zig | 6 ++- 21 files changed, 131 insertions(+), 31 deletions(-) diff --git a/lib/build_runner.zig b/lib/build_runner.zig index 9a8f138979..ea4703bfbb 100644 --- a/lib/build_runner.zig +++ b/lib/build_runner.zig @@ -571,7 +571,7 @@ fn workerMakeOneStep( // For example, CompileStep does some sus things with modifying the saved // *Build object in install header steps that might be able to be removed // by passing the *Build object through the make() functions. - const make_result = s.make(); + const make_result = s.make(&sub_prog_node); // No matter the result, we want to display error/warning messages. if (s.result_error_msgs.items.len > 0) { diff --git a/lib/std/Build.zig b/lib/std/Build.zig index a9d7c31733..ca7ddb591c 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -718,7 +718,8 @@ pub fn getUninstallStep(self: *Build) *Step { return &self.uninstall_tls.step; } -fn makeUninstall(uninstall_step: *Step) anyerror!void { +fn makeUninstall(uninstall_step: *Step, prog_node: *std.Progress.Node) anyerror!void { + _ = prog_node; const uninstall_tls = @fieldParentPtr(TopLevelStep, "step", uninstall_step); const self = @fieldParentPtr(Build, "uninstall_tls", uninstall_tls); @@ -1404,7 +1405,7 @@ pub fn execAllowFail( /// This function is used exclusively for spawning and communicating with the zig compiler. /// TODO: move to build_runner.zig -pub fn execFromStep(b: *Build, argv: []const []const u8, s: *Step) ![]const u8 { +pub fn execFromStep(b: *Build, argv: []const []const u8, s: *Step, prog_node: *std.Progress.Node) ![]const u8 { assert(argv.len != 0); if (b.verbose) { @@ -1439,6 +1440,11 @@ pub fn execFromStep(b: *Build, argv: []const []const u8, s: *Step) ![]const u8 { const Header = std.zig.Server.Message.Header; var result: ?[]const u8 = null; + var node_name: std.ArrayListUnmanaged(u8) = .{}; + defer node_name.deinit(b.allocator); + var sub_prog_node: ?std.Progress.Node = null; + defer if (sub_prog_node) |*n| n.end(); + while (try poller.poll()) { const stdout = poller.fifo(.stdout); const buf = stdout.readableSlice(0); @@ -1478,7 +1484,11 @@ pub fn execFromStep(b: *Build, argv: []const []const u8, s: *Step) ![]const u8 { }; }, .progress => { - @panic("TODO handle progress message"); + if (sub_prog_node) |*n| n.end(); + node_name.clearRetainingCapacity(); + try node_name.appendSlice(b.allocator, body); + sub_prog_node = prog_node.start(node_name.items, 0); + sub_prog_node.?.activate(); }, .emit_bin_path => { result = try b.allocator.dupe(u8, body); diff --git a/lib/std/Build/CheckFileStep.zig b/lib/std/Build/CheckFileStep.zig index 5b2e5b4e5b..9fee947386 100644 --- a/lib/std/Build/CheckFileStep.zig +++ b/lib/std/Build/CheckFileStep.zig @@ -33,7 +33,8 @@ pub fn create( return self; } -fn make(step: *Step) !void { +fn make(step: *Step, prog_node: *std.Progress.Node) !void { + _ = prog_node; const self = @fieldParentPtr(CheckFileStep, "step", step); const src_path = self.source.getPath(self.builder); diff --git a/lib/std/Build/CheckObjectStep.zig b/lib/std/Build/CheckObjectStep.zig index 1775a5d39a..eccbbd1696 100644 --- a/lib/std/Build/CheckObjectStep.zig +++ b/lib/std/Build/CheckObjectStep.zig @@ -300,7 +300,8 @@ pub fn checkComputeCompare( self.checks.append(new_check) catch @panic("OOM"); } -fn make(step: *Step) !void { +fn make(step: *Step, prog_node: *std.Progress.Node) !void { + _ = prog_node; const self = @fieldParentPtr(CheckObjectStep, "step", step); const gpa = self.builder.allocator; diff --git a/lib/std/Build/CompileStep.zig b/lib/std/Build/CompileStep.zig index 491c8ac8e4..711092c762 100644 --- a/lib/std/Build/CompileStep.zig +++ b/lib/std/Build/CompileStep.zig @@ -1160,7 +1160,7 @@ fn constructDepString( } } -fn make(step: *Step) !void { +fn make(step: *Step, prog_node: *std.Progress.Node) !void { const self = @fieldParentPtr(CompileStep, "step", step); const builder = self.builder; @@ -1718,7 +1718,7 @@ fn make(step: *Step) !void { } if (other.installed_headers.items.len > 0) { for (other.installed_headers.items) |install_step| { - try install_step.make(); + try install_step.make(prog_node); } try zig_args.append("-I"); try zig_args.append(builder.pathJoin(&.{ @@ -1894,7 +1894,7 @@ fn make(step: *Step) !void { try zig_args.append(resolved_args_file); } - const output_bin_path = try builder.execFromStep(zig_args.items, &self.step); + const output_bin_path = try builder.execFromStep(zig_args.items, &self.step, prog_node); const build_output_dir = fs.path.dirname(output_bin_path).?; if (self.output_dir) |output_dir| { diff --git a/lib/std/Build/ConfigHeaderStep.zig b/lib/std/Build/ConfigHeaderStep.zig index fb8d92e4e2..c74c03718e 100644 --- a/lib/std/Build/ConfigHeaderStep.zig +++ b/lib/std/Build/ConfigHeaderStep.zig @@ -152,7 +152,8 @@ fn putValue(self: *ConfigHeaderStep, field_name: []const u8, comptime T: type, v } } -fn make(step: *Step) !void { +fn make(step: *Step, prog_node: *std.Progress.Node) !void { + _ = prog_node; const self = @fieldParentPtr(ConfigHeaderStep, "step", step); const gpa = self.builder.allocator; diff --git a/lib/std/Build/EmulatableRunStep.zig b/lib/std/Build/EmulatableRunStep.zig index 6556f4b8c0..44387c36f6 100644 --- a/lib/std/Build/EmulatableRunStep.zig +++ b/lib/std/Build/EmulatableRunStep.zig @@ -71,7 +71,8 @@ pub fn create(builder: *std.Build, name: []const u8, artifact: *CompileStep) *Em return self; } -fn make(step: *Step) !void { +fn make(step: *Step, prog_node: *std.Progress.Node) !void { + _ = prog_node; const self = @fieldParentPtr(EmulatableRunStep, "step", step); const host_info = self.builder.host; diff --git a/lib/std/Build/FmtStep.zig b/lib/std/Build/FmtStep.zig index 04b8ccb3e4..5efada7507 100644 --- a/lib/std/Build/FmtStep.zig +++ b/lib/std/Build/FmtStep.zig @@ -29,7 +29,8 @@ pub fn create(builder: *std.Build, paths: []const []const u8) *FmtStep { return self; } -fn make(step: *Step) !void { +fn make(step: *Step, prog_node: *std.Progress.Node) !void { + _ = prog_node; const self = @fieldParentPtr(FmtStep, "step", step); return self.builder.spawnChild(self.argv); diff --git a/lib/std/Build/InstallArtifactStep.zig b/lib/std/Build/InstallArtifactStep.zig index b652ade38f..d8907eb59f 100644 --- a/lib/std/Build/InstallArtifactStep.zig +++ b/lib/std/Build/InstallArtifactStep.zig @@ -64,7 +64,8 @@ pub fn create(builder: *std.Build, artifact: *CompileStep) *InstallArtifactStep return self; } -fn make(step: *Step) !void { +fn make(step: *Step, prog_node: *std.Progress.Node) !void { + _ = prog_node; const self = @fieldParentPtr(InstallArtifactStep, "step", step); const builder = self.builder; diff --git a/lib/std/Build/InstallDirStep.zig b/lib/std/Build/InstallDirStep.zig index 54a37af1d4..bf89d9e7c7 100644 --- a/lib/std/Build/InstallDirStep.zig +++ b/lib/std/Build/InstallDirStep.zig @@ -56,7 +56,8 @@ pub fn init( }; } -fn make(step: *Step) !void { +fn make(step: *Step, prog_node: *std.Progress.Node) !void { + _ = prog_node; const self = @fieldParentPtr(InstallDirStep, "step", step); const dest_prefix = self.builder.getInstallPath(self.options.install_dir, self.options.install_subdir); const src_builder = self.override_source_builder orelse self.builder; diff --git a/lib/std/Build/InstallFileStep.zig b/lib/std/Build/InstallFileStep.zig index e6c57a8050..f77b22c112 100644 --- a/lib/std/Build/InstallFileStep.zig +++ b/lib/std/Build/InstallFileStep.zig @@ -35,7 +35,8 @@ pub fn init( }; } -fn make(step: *Step) !void { +fn make(step: *Step, prog_node: *std.Progress.Node) !void { + _ = prog_node; const self = @fieldParentPtr(InstallFileStep, "step", step); const src_builder = self.override_source_builder orelse self.builder; const full_src_path = self.source.getPath2(src_builder, step); diff --git a/lib/std/Build/LogStep.zig b/lib/std/Build/LogStep.zig index 008ed6e8bb..25bba747bf 100644 --- a/lib/std/Build/LogStep.zig +++ b/lib/std/Build/LogStep.zig @@ -21,7 +21,8 @@ pub fn init(builder: *std.Build, data: []const u8) LogStep { }; } -fn make(step: *Step) anyerror!void { +fn make(step: *Step, prog_node: *std.Progress.Node) anyerror!void { + _ = prog_node; const self = @fieldParentPtr(LogStep, "step", step); log.info("{s}", .{self.data}); } diff --git a/lib/std/Build/ObjCopyStep.zig b/lib/std/Build/ObjCopyStep.zig index 549bb1ed00..839d95903c 100644 --- a/lib/std/Build/ObjCopyStep.zig +++ b/lib/std/Build/ObjCopyStep.zig @@ -66,7 +66,8 @@ pub fn getOutputSource(self: *const ObjCopyStep) std.Build.FileSource { return .{ .generated = &self.output_file }; } -fn make(step: *Step) !void { +fn make(step: *Step, prog_node: *std.Progress.Node) !void { + _ = prog_node; const self = @fieldParentPtr(ObjCopyStep, "step", step); const b = self.builder; diff --git a/lib/std/Build/OptionsStep.zig b/lib/std/Build/OptionsStep.zig index 8e86578e30..2cb3bb13be 100644 --- a/lib/std/Build/OptionsStep.zig +++ b/lib/std/Build/OptionsStep.zig @@ -219,7 +219,8 @@ pub fn getSource(self: *OptionsStep) FileSource { return .{ .generated = &self.generated_file }; } -fn make(step: *Step) !void { +fn make(step: *Step, prog_node: *std.Progress.Node) !void { + _ = prog_node; const self = @fieldParentPtr(OptionsStep, "step", step); for (self.artifact_args.items) |item| { diff --git a/lib/std/Build/RemoveDirStep.zig b/lib/std/Build/RemoveDirStep.zig index acde60a745..4fc8e6d338 100644 --- a/lib/std/Build/RemoveDirStep.zig +++ b/lib/std/Build/RemoveDirStep.zig @@ -22,7 +22,8 @@ pub fn init(builder: *std.Build, dir_path: []const u8) RemoveDirStep { }; } -fn make(step: *Step) !void { +fn make(step: *Step, prog_node: *std.Progress.Node) !void { + _ = prog_node; const self = @fieldParentPtr(RemoveDirStep, "step", step); const full_path = self.builder.pathFromRoot(self.dir_path); diff --git a/lib/std/Build/RunStep.zig b/lib/std/Build/RunStep.zig index 904ef0935f..7ec60b6d22 100644 --- a/lib/std/Build/RunStep.zig +++ b/lib/std/Build/RunStep.zig @@ -206,7 +206,8 @@ fn needOutputCheck(self: RunStep) bool { return false; } -fn make(step: *Step) !void { +fn make(step: *Step, prog_node: *std.Progress.Node) !void { + _ = prog_node; const self = @fieldParentPtr(RunStep, "step", step); const need_output_check = self.needOutputCheck(); diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index 4edece8038..53861683ac 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -1,6 +1,6 @@ id: Id, name: []const u8, -makeFn: *const fn (self: *Step) anyerror!void, +makeFn: MakeFn, dependencies: std.ArrayList(*Step), /// This field is empty during execution of the user's build script, and /// then populated during dependency loop checking in the build runner. @@ -13,6 +13,8 @@ debug_stack_trace: [n_debug_stack_frames]usize, result_error_msgs: std.ArrayListUnmanaged([]const u8), result_error_bundle: std.zig.ErrorBundle, +pub const MakeFn = *const fn (self: *Step, prog_node: *std.Progress.Node) anyerror!void; + const n_debug_stack_frames = 4; pub const State = enum { @@ -72,7 +74,7 @@ pub const Id = enum { pub const Options = struct { id: Id, name: []const u8, - makeFn: *const fn (self: *Step) anyerror!void = makeNoOp, + makeFn: MakeFn = makeNoOp, first_ret_addr: ?usize = null, }; @@ -101,8 +103,8 @@ pub fn init(allocator: Allocator, options: Options) Step { /// If the Step's `make` function reports `error.MakeFailed`, it indicates they /// have already reported the error. Otherwise, we add a simple error report /// here. -pub fn make(s: *Step) error{MakeFailed}!void { - return s.makeFn(s) catch |err| { +pub fn make(s: *Step, prog_node: *std.Progress.Node) error{MakeFailed}!void { + return s.makeFn(s, prog_node) catch |err| { if (err != error.MakeFailed) { const gpa = s.dependencies.allocator; s.result_error_msgs.append(gpa, std.fmt.allocPrint(gpa, "{s} failed: {s}", .{ @@ -129,8 +131,9 @@ pub fn getStackTrace(s: *Step) std.builtin.StackTrace { }; } -fn makeNoOp(self: *Step) anyerror!void { +fn makeNoOp(self: *Step, prog_node: *std.Progress.Node) anyerror!void { _ = self; + _ = prog_node; } pub fn cast(step: *Step, comptime T: type) ?*T { diff --git a/lib/std/Build/TranslateCStep.zig b/lib/std/Build/TranslateCStep.zig index 8c3a254d6a..d19e598d93 100644 --- a/lib/std/Build/TranslateCStep.zig +++ b/lib/std/Build/TranslateCStep.zig @@ -88,7 +88,7 @@ pub fn defineCMacroRaw(self: *TranslateCStep, name_and_value: []const u8) void { self.c_macros.append(self.builder.dupe(name_and_value)) catch @panic("OOM"); } -fn make(step: *Step) !void { +fn make(step: *Step, prog_node: *std.Progress.Node) !void { const self = @fieldParentPtr(TranslateCStep, "step", step); var argv_list = std.ArrayList([]const u8).init(self.builder.allocator); @@ -120,7 +120,7 @@ fn make(step: *Step) !void { try argv_list.append(self.source.getPath(self.builder)); - const output_path_nl = try self.builder.execFromStep(argv_list.items, &self.step); + const output_path_nl = try self.builder.execFromStep(argv_list.items, &self.step, prog_node); const output_path = mem.trimRight(u8, output_path_nl, "\r\n"); self.out_basename = fs.path.basename(output_path); diff --git a/lib/std/Build/WriteFileStep.zig b/lib/std/Build/WriteFileStep.zig index 4f5e768237..62acd6e8ee 100644 --- a/lib/std/Build/WriteFileStep.zig +++ b/lib/std/Build/WriteFileStep.zig @@ -99,7 +99,8 @@ pub fn getFileSource(wf: *WriteFileStep, sub_path: []const u8) ?std.Build.FileSo return null; } -fn make(step: *Step) !void { +fn make(step: *Step, prog_node: *std.Progress.Node) !void { + _ = prog_node; const wf = @fieldParentPtr(WriteFileStep, "step", step); // Writing to source files is kind of an extra capability of this diff --git a/src/main.zig b/src/main.zig index 495cc40ed5..1d5cf57388 100644 --- a/src/main.zig +++ b/src/main.zig @@ -3573,7 +3573,21 @@ fn serve( if (comp.bin_file.options.output_mode == .Exe) { try comp.makeBinFileWritable(); } - try comp.update(main_progress_node); + + { + var reset: std.Thread.ResetEvent = .{}; + + var progress_thread = try std.Thread.spawn(.{}, progressThread, .{ + &progress, out, &reset, + }); + defer { + reset.set(); + progress_thread.join(); + } + + try comp.update(main_progress_node); + } + try comp.makeBinFileExecutable(); try serveUpdateResults(out, comp); }, @@ -3629,6 +3643,63 @@ fn serve( } } +fn progressThread(progress: *std.Progress, out: fs.File, reset: *std.Thread.ResetEvent) void { + while (true) { + if (reset.timedWait(500 * std.time.ns_per_ms)) |_| { + // The Compilation update has completed. + return; + } else |err| switch (err) { + error.Timeout => {}, + } + + var buf: std.BoundedArray(u8, 160) = .{}; + + { + progress.update_mutex.lock(); + defer progress.update_mutex.unlock(); + + var need_ellipse = false; + var maybe_node: ?*std.Progress.Node = &progress.root; + while (maybe_node) |node| { + if (need_ellipse) { + buf.appendSlice("... ") catch {}; + } + need_ellipse = false; + const eti = @atomicLoad(usize, &node.unprotected_estimated_total_items, .Monotonic); + const completed_items = @atomicLoad(usize, &node.unprotected_completed_items, .Monotonic); + const current_item = completed_items + 1; + if (node.name.len != 0 or eti > 0) { + if (node.name.len != 0) { + buf.appendSlice(node.name) catch {}; + need_ellipse = true; + } + if (eti > 0) { + if (need_ellipse) buf.appendSlice(" ") catch {}; + buf.writer().print("[{d}/{d}] ", .{ current_item, eti }) catch {}; + need_ellipse = false; + } else if (completed_items != 0) { + if (need_ellipse) buf.appendSlice(" ") catch {}; + buf.writer().print("[{d}] ", .{current_item}) catch {}; + need_ellipse = false; + } + } + maybe_node = @atomicLoad(?*std.Progress.Node, &node.recently_updated_child, .Acquire); + } + } + + const progress_string = buf.slice(); + + serveMessage(out, .{ + .tag = .progress, + .bytes_len = @intCast(u32, progress_string.len), + }, &.{ + progress_string, + }) catch |err| { + fatal("unable to write to client: {s}", .{@errorName(err)}); + }; + } +} + fn serveMessage( out: fs.File, header: std.zig.Server.Message.Header, diff --git a/test/tests.zig b/test/tests.zig index fe2efbc06e..fceaa173d1 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -875,7 +875,8 @@ pub const StackTracesContext = struct { return ptr; } - fn make(step: *Step) !void { + fn make(step: *Step, prog_node: *std.Progress.Node) !void { + _ = prog_node; const self = @fieldParentPtr(RunAndCompareStep, "step", step); const b = self.context.b; @@ -1218,7 +1219,8 @@ pub const GenHContext = struct { return ptr; } - fn make(step: *Step) !void { + fn make(step: *Step, prog_node: *std.Progress.Node) !void { + _ = prog_node; const self = @fieldParentPtr(GenHCmpOutputStep, "step", step); const b = self.context.b;