diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig index f06ae7d4b9..5f7c12fe92 100644 --- a/lib/compiler/build_runner.zig +++ b/lib/compiler/build_runner.zig @@ -10,7 +10,8 @@ const File = std.fs.File; const Step = std.Build.Step; const Watch = std.Build.Watch; const Allocator = std.mem.Allocator; -const fatal = std.zig.fatal; +const fatal = std.process.fatal; +const runner = @This(); pub const root = @import("@build"); pub const dependencies = @import("@dependencies"); @@ -102,6 +103,7 @@ pub fn main() !void { var steps_menu = false; var output_tmp_nonce: ?[16]u8 = null; var watch = false; + var fuzz = false; var debounce_interval_ms: u16 = 50; while (nextArg(args, &arg_idx)) |arg| { @@ -234,6 +236,8 @@ pub fn main() !void { prominent_compile_errors = true; } else if (mem.eql(u8, arg, "--watch")) { watch = true; + } else if (mem.eql(u8, arg, "--fuzz")) { + fuzz = true; } else if (mem.eql(u8, arg, "-fincremental")) { graph.incremental = true; } else if (mem.eql(u8, arg, "-fno-incremental")) { @@ -353,6 +357,7 @@ pub fn main() !void { .max_rss_mutex = .{}, .skip_oom_steps = skip_oom_steps, .watch = watch, + .fuzz = fuzz, .memory_blocked_steps = std.ArrayList(*Step).init(arena), .step_stack = .{}, .prominent_compile_errors = prominent_compile_errors, @@ -394,6 +399,10 @@ pub fn main() !void { }, else => return err, }; + if (fuzz) { + startFuzzing(&run.thread_pool, run.step_stack.keys(), main_progress_node); + } + if (!watch) return cleanExit(); switch (builtin.os.tag) { @@ -430,6 +439,43 @@ pub fn main() !void { } } +fn startFuzzing(thread_pool: *std.Thread.Pool, all_steps: []const *Step, prog_node: std.Progress.Node) void { + { + const rebuild_node = prog_node.start("Rebuilding Unit Tests", 0); + defer rebuild_node.end(); + var count: usize = 0; + var wait_group: std.Thread.WaitGroup = .{}; + defer wait_group.wait(); + for (all_steps) |step| { + const run = step.cast(Step.Run) orelse continue; + if (run.fuzz_tests.items.len > 0 and run.producer != null) { + thread_pool.spawnWg(&wait_group, rebuildTestsWorkerRun, .{ run, prog_node }); + count += 1; + } + } + if (count == 0) { + std.debug.lockStdErr(); + std.debug.print("no fuzz tests found\n", .{}); + process.exit(2); + } + rebuild_node.setEstimatedTotalItems(count); + } + @panic("TODO do something with the rebuilt unit tests"); +} + +fn rebuildTestsWorkerRun(run: *Step.Run, parent_prog_node: std.Progress.Node) void { + const compile_step = run.producer.?; + const prog_node = parent_prog_node.start(compile_step.step.name, 0); + defer prog_node.end(); + const rebuilt_bin_path = compile_step.rebuildInFuzzMode(prog_node) catch |err| { + std.debug.print("failed to rebuild {s} in fuzz mode: {s}", .{ + compile_step.step.name, @errorName(err), + }); + return; + }; + std.debug.print("rebuilt binary: '{s}'\n", .{rebuilt_bin_path}); +} + fn markFailedStepsDirty(gpa: Allocator, all_steps: []const *Step) void { for (all_steps) |step| switch (step.state) { .dependency_failure, .failure, .skipped => step.recursiveReset(gpa), @@ -457,6 +503,7 @@ const Run = struct { max_rss_mutex: std.Thread.Mutex, skip_oom_steps: bool, watch: bool, + fuzz: bool, memory_blocked_steps: std.ArrayList(*Step), step_stack: std.AutoArrayHashMapUnmanaged(*Step, void), prominent_compile_errors: bool, @@ -466,6 +513,11 @@ const Run = struct { summary: Summary, ttyconf: std.io.tty.Config, stderr: File, + + fn cleanExit(run: Run) void { + if (run.watch or run.fuzz) return; + return runner.cleanExit(); + } }; fn prepare( @@ -614,8 +666,7 @@ fn runStepNames( else => false, }; if (failure_count == 0 and failures_only) { - if (!run.watch) cleanExit(); - return; + return run.cleanExit(); } const ttyconf = run.ttyconf; @@ -672,8 +723,7 @@ fn runStepNames( } if (failure_count == 0) { - if (!run.watch) cleanExit(); - return; + return run.cleanExit(); } // Finally, render compile errors at the bottom of the terminal. @@ -1226,6 +1276,7 @@ fn usage(b: *std.Build, out_stream: anytype) !void { \\ --skip-oom-steps Instead of failing, skip steps that would exceed --maxrss \\ --fetch Exit after fetching dependency tree \\ --watch Continuously rebuild when source files are modified + \\ --fuzz Continuously search for unit test failures \\ --debounce Delay before rebuilding after changed file detected \\ -fincremental Enable incremental compilation \\ -fno-incremental Disable incremental compilation diff --git a/lib/compiler/test_runner.zig b/lib/compiler/test_runner.zig index 15905ca7f9..81eb156e73 100644 --- a/lib/compiler/test_runner.zig +++ b/lib/compiler/test_runner.zig @@ -143,6 +143,7 @@ fn mainTerminal() void { var ok_count: usize = 0; var skip_count: usize = 0; var fail_count: usize = 0; + var fuzz_count: usize = 0; const root_node = std.Progress.start(.{ .root_name = "Test", .estimated_total_items = test_fn_list.len, @@ -168,7 +169,7 @@ fn mainTerminal() void { if (!have_tty) { std.debug.print("{d}/{d} {s}...", .{ i + 1, test_fn_list.len, test_fn.name }); } - // Track in a global variable so that `fuzzInput` can see it. + is_fuzz_test = false; if (test_fn.func()) |_| { ok_count += 1; test_node.end(); @@ -198,6 +199,7 @@ fn mainTerminal() void { test_node.end(); }, } + fuzz_count += @intFromBool(is_fuzz_test); } root_node.end(); if (ok_count == test_fn_list.len) { @@ -211,6 +213,9 @@ fn mainTerminal() void { if (leaks != 0) { std.debug.print("{d} tests leaked memory.\n", .{leaks}); } + if (fuzz_count != 0) { + std.debug.print("{d} fuzz tests found.\n", .{fuzz_count}); + } if (leaks != 0 or log_err_count != 0 or fail_count != 0) { std.process.exit(1); } diff --git a/lib/std/Build.zig b/lib/std/Build.zig index f76e3263cb..1ad2e0ee51 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -977,6 +977,7 @@ pub fn addRunArtifact(b: *Build, exe: *Step.Compile) *Step.Run { // Consider that this is declarative; the run step may not be run unless a user // option is supplied. const run_step = Step.Run.create(b, b.fmt("run {s}", .{exe.name})); + run_step.producer = exe; if (exe.kind == .@"test") { if (exe.exec_cmd_args) |exec_cmd_args| { for (exec_cmd_args) |cmd_arg| { diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig index 4f504151c0..ffb2337ac5 100644 --- a/lib/std/Build/Step/Compile.zig +++ b/lib/std/Build/Step/Compile.zig @@ -1004,7 +1004,7 @@ fn getGeneratedFilePath(compile: *Compile, comptime tag_name: []const u8, asking return path; } -fn getZigArgs(compile: *Compile) ![][]const u8 { +fn getZigArgs(compile: *Compile, fuzz: bool) ![][]const u8 { const step = &compile.step; const b = step.owner; const arena = b.allocator; @@ -1055,6 +1055,10 @@ fn getZigArgs(compile: *Compile) ![][]const u8 { try zig_args.append(try std.fmt.allocPrint(arena, "{}", .{stack_size})); } + if (fuzz) { + try zig_args.append("-ffuzz"); + } + { // Stores system libraries that have already been seen for at least one // module, along with any arguments that need to be passed to the @@ -1757,7 +1761,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void { const b = step.owner; const compile: *Compile = @fieldParentPtr("step", step); - const zig_args = try getZigArgs(compile); + const zig_args = try getZigArgs(compile, false); const maybe_output_bin_path = step.evalZigProcess( zig_args, @@ -1835,6 +1839,12 @@ fn make(step: *Step, options: Step.MakeOptions) !void { } } +pub fn rebuildInFuzzMode(c: *Compile, progress_node: std.Progress.Node) ![]const u8 { + const zig_args = try getZigArgs(c, true); + const maybe_output_bin_path = try c.step.evalZigProcess(zig_args, progress_node, false); + return maybe_output_bin_path.?; +} + pub fn doAtomicSymLinks( step: *Step, output_path: []const u8, @@ -1861,10 +1871,10 @@ pub fn doAtomicSymLinks( }; } -fn execPkgConfigList(compile: *std.Build, out_code: *u8) (PkgConfigError || RunError)![]const PkgConfigPkg { - const pkg_config_exe = compile.graph.env_map.get("PKG_CONFIG") orelse "pkg-config"; - const stdout = try compile.runAllowFail(&[_][]const u8{ pkg_config_exe, "--list-all" }, out_code, .Ignore); - var list = ArrayList(PkgConfigPkg).init(compile.allocator); +fn execPkgConfigList(b: *std.Build, out_code: *u8) (PkgConfigError || RunError)![]const PkgConfigPkg { + const pkg_config_exe = b.graph.env_map.get("PKG_CONFIG") orelse "pkg-config"; + const stdout = try b.runAllowFail(&[_][]const u8{ pkg_config_exe, "--list-all" }, out_code, .Ignore); + var list = ArrayList(PkgConfigPkg).init(b.allocator); errdefer list.deinit(); var line_it = mem.tokenizeAny(u8, stdout, "\r\n"); while (line_it.next()) |line| { @@ -1878,13 +1888,13 @@ fn execPkgConfigList(compile: *std.Build, out_code: *u8) (PkgConfigError || RunE return list.toOwnedSlice(); } -fn getPkgConfigList(compile: *std.Build) ![]const PkgConfigPkg { - if (compile.pkg_config_pkg_list) |res| { +fn getPkgConfigList(b: *std.Build) ![]const PkgConfigPkg { + if (b.pkg_config_pkg_list) |res| { return res; } var code: u8 = undefined; - if (execPkgConfigList(compile, &code)) |list| { - compile.pkg_config_pkg_list = list; + if (execPkgConfigList(b, &code)) |list| { + b.pkg_config_pkg_list = list; return list; } else |err| { const result = switch (err) { @@ -1896,7 +1906,7 @@ fn getPkgConfigList(compile: *std.Build) ![]const PkgConfigPkg { error.PkgConfigInvalidOutput => error.PkgConfigInvalidOutput, else => return err, }; - compile.pkg_config_pkg_list = result; + b.pkg_config_pkg_list = result; return result; } } diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index dbb865047b..7927ac9479 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -86,6 +86,13 @@ dep_output_file: ?*Output, has_side_effects: bool, +/// If this is a Zig unit test binary, this tracks the indexes of the unit +/// tests that are also fuzz tests. +fuzz_tests: std.ArrayListUnmanaged(u32), + +/// If this Run step was produced by a Compile step, it is tracked here. +producer: ?*Step.Compile, + pub const StdIn = union(enum) { none, bytes: []const u8, @@ -175,6 +182,8 @@ pub fn create(owner: *std.Build, name: []const u8) *Run { .captured_stderr = null, .dep_output_file = null, .has_side_effects = false, + .fuzz_tests = .{}, + .producer = null, }; return run; } @@ -1347,6 +1356,8 @@ fn evalZigTest( var sub_prog_node: ?std.Progress.Node = null; defer if (sub_prog_node) |n| n.end(); + run.fuzz_tests.clearRetainingCapacity(); + poll: while (true) { while (stdout.readableLength() < @sizeOf(Header)) { if (!(try poller.poll())) break :poll; @@ -1404,6 +1415,8 @@ fn evalZigTest( leak_count +|= @intFromBool(tr_hdr.flags.leak); log_err_count +|= tr_hdr.flags.log_err_count; + if (tr_hdr.flags.fuzz) try run.fuzz_tests.append(gpa, tr_hdr.index); + if (tr_hdr.flags.fail or tr_hdr.flags.leak or tr_hdr.flags.log_err_count > 0) { const name = std.mem.sliceTo(md.string_bytes[md.names[tr_hdr.index]..], 0); const orig_msg = stderr.readableSlice(0);