Merge pull request #25342 from ziglang/fuzz-limit

fuzzing: implement limited fuzzing

lib/compiler/build_runner.zig

@@ -112,7 +112,7 @@ pub fn main() !void {
     var steps_menu = false;
     var output_tmp_nonce: ?[16]u8 = null;
     var watch = false;
-    var fuzz = false;
+    var fuzz: ?std.Build.Fuzz.Mode = null;
     var debounce_interval_ms: u16 = 50;
     var webui_listen: ?std.net.Address = null;
 
@@ -274,10 +274,44 @@ pub fn main() !void {
                     webui_listen = std.net.Address.parseIp("::1", 0) catch unreachable;
                 }
             } else if (mem.eql(u8, arg, "--fuzz")) {
-                fuzz = true;
+                fuzz = .{ .forever = undefined };
                 if (webui_listen == null) {
                     webui_listen = std.net.Address.parseIp("::1", 0) catch unreachable;
                 }
+            } else if (mem.startsWith(u8, arg, "--fuzz=")) {
+                const value = arg["--fuzz=".len..];
+                if (value.len == 0) fatal("missing argument to --fuzz", .{});
+
+                const unit: u8 = value[value.len - 1];
+                const digits = switch (unit) {
+                    '0'...'9' => value,
+                    'K', 'M', 'G' => value[0 .. value.len - 1],
+                    else => fatal(
+                        "invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]",
+                        .{},
+                    ),
+                };
+
+                const amount = std.fmt.parseInt(u64, digits, 10) catch {
+                    fatal(
+                        "invalid argument to --fuzz, expected a positive number optionally suffixed by one of: [KMG]",
+                        .{},
+                    );
+                };
+
+                const normalized_amount = std.math.mul(u64, amount, switch (unit) {
+                    else => unreachable,
+                    '0'...'9' => 1,
+                    'K' => 1000,
+                    'M' => 1_000_000,
+                    'G' => 1_000_000_000,
+                }) catch fatal("fuzzing limit amount overflows u64", .{});
+
+                fuzz = .{
+                    .limit = .{
+                        .amount = normalized_amount,
+                    },
+                };
             } else if (mem.eql(u8, arg, "-fincremental")) {
                 graph.incremental = true;
             } else if (mem.eql(u8, arg, "-fno-incremental")) {
@@ -476,6 +510,7 @@ pub fn main() !void {
             targets.items,
             main_progress_node,
             &run,
+            fuzz,
         ) catch |err| switch (err) {
             error.UncleanExit => {
                 assert(!run.watch and run.web_server == null);
@@ -485,7 +520,12 @@ pub fn main() !void {
         };
 
         if (run.web_server) |*web_server| {
-            web_server.finishBuild(.{ .fuzz = fuzz });
+            if (fuzz) |mode| if (mode != .forever) fatal(
+                "error: limited fuzzing is not implemented yet for --webui",
+                .{},
+            );
+
+            web_server.finishBuild(.{ .fuzz = fuzz != null });
         }
 
         if (!watch and run.web_server == null) {
@@ -651,6 +691,7 @@ fn runStepNames(
     step_names: []const []const u8,
     parent_prog_node: std.Progress.Node,
     run: *Run,
+    fuzz: ?std.Build.Fuzz.Mode,
 ) !void {
     const gpa = run.gpa;
     const step_stack = &run.step_stack;
@@ -676,6 +717,7 @@ fn runStepNames(
             });
         }
     }
+
     assert(run.memory_blocked_steps.items.len == 0);
 
     var test_skip_count: usize = 0;
@@ -724,6 +766,45 @@ fn runStepNames(
         }
     }
 
+    const ttyconf = run.ttyconf;
+
+    if (fuzz) |mode| blk: {
+        switch (builtin.os.tag) {
+            // Current implementation depends on two things that need to be ported to Windows:
+            // * Memory-mapping to share data between the fuzzer and build runner.
+            // * COFF/PE support added to `std.debug.Info` (it needs a batching API for resolving
+            //   many addresses to source locations).
+            .windows => fatal("--fuzz not yet implemented for {s}", .{@tagName(builtin.os.tag)}),
+            else => {},
+        }
+        if (@bitSizeOf(usize) != 64) {
+            // Current implementation depends on posix.mmap()'s second parameter, `length: usize`,
+            // being compatible with `std.fs.getEndPos() u64`'s return value. This is not the case
+            // on 32-bit platforms.
+            // Affects or affected by issues #5185, #22523, and #22464.
+            fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)});
+        }
+
+        switch (mode) {
+            .forever => break :blk,
+            .limit => {},
+        }
+
+        assert(mode == .limit);
+        var f = std.Build.Fuzz.init(
+            gpa,
+            thread_pool,
+            step_stack.keys(),
+            parent_prog_node,
+            ttyconf,
+            mode,
+        ) catch |err| fatal("failed to start fuzzer: {s}", .{@errorName(err)});
+        defer f.deinit();
+
+        f.start();
+        f.waitAndPrintReport();
+    }
+
     // A proper command line application defaults to silently succeeding.
     // The user may request verbose mode if they have a different preference.
     const failures_only = switch (run.summary) {
@@ -737,8 +818,6 @@ fn runStepNames(
         std.Progress.setStatus(.failure);
     }
 
-    const ttyconf = run.ttyconf;
-
     if (run.summary != .none) {
         const w = std.debug.lockStderrWriter(&stdio_buffer_allocation);
         defer std.debug.unlockStderrWriter();
@@ -1366,7 +1445,10 @@ fn printUsage(b: *std.Build, w: *Writer) !void {
         \\  --watch                      Continuously rebuild when source files are modified
         \\  --debounce <ms>              Delay before rebuilding after changed file detected
         \\  --webui[=ip]                 Enable the web interface on the given IP address
-        \\  --fuzz                       Continuously search for unit test failures (implies '--webui')
+        \\  --fuzz[=limit]               Continuously search for unit test failures with an optional 
+        \\                               limit to the max number of iterations. The argument supports
+        \\                               an optional 'K', 'M', or 'G' suffix (e.g. '10K'). Implies
+        \\                               '--webui' when no limit is specified.
         \\  --time-report                Force full rebuild and provide detailed information on
         \\                               compilation time of Zig source code (implies '--webui')
         \\     -fincremental             Enable incremental compilation

lib/compiler/test_runner.zig

@@ -2,6 +2,7 @@
 const builtin = @import("builtin");
 
 const std = @import("std");
+const fatal = std.process.fatal;
 const testing = std.testing;
 const assert = std.debug.assert;
 const fuzz_abi = std.Build.abi.fuzz;
@@ -55,12 +56,13 @@ pub fn main() void {
         }
     }
 
-    fba.reset();
     if (builtin.fuzz) {
         const cache_dir = opt_cache_dir orelse @panic("missing --cache-dir=[path] argument");
         fuzz_abi.fuzzer_init(.fromSlice(cache_dir));
     }
 
+    fba.reset();
+
     if (listen) {
         return mainServer() catch @panic("internal test runner failure");
     } else {
@@ -79,8 +81,13 @@ fn mainServer() !void {
     });
 
     if (builtin.fuzz) {
-        const coverage_id = fuzz_abi.fuzzer_coverage_id();
+        const coverage = fuzz_abi.fuzzer_coverage();
-        try server.serveU64Message(.coverage_id, coverage_id);
+        try server.serveCoverageIdMessage(
+            coverage.id,
+            coverage.runs,
+            coverage.unique,
+            coverage.seen,
+        );
     }
 
     while (true) {
@@ -158,6 +165,9 @@ fn mainServer() !void {
                 if (!builtin.fuzz) unreachable;
 
                 const index = try server.receiveBody_u32();
+                const mode: fuzz_abi.LimitKind = @enumFromInt(try server.receiveBody_u8());
+                const amount_or_instance = try server.receiveBody_u64();
+
                 const test_fn = builtin.test_functions[index];
                 const entry_addr = @intFromPtr(test_fn.func);
 
@@ -165,6 +175,8 @@ fn mainServer() !void {
                 defer if (testing.allocator_instance.deinit() == .leak) std.process.exit(1);
                 is_fuzz_test = false;
                 fuzz_test_index = index;
+                fuzz_mode = mode;
+                fuzz_amount_or_instance = amount_or_instance;
 
                 test_fn.func() catch |err| switch (err) {
                     error.SkipZigTest => return,
@@ -172,12 +184,14 @@ fn mainServer() !void {
                         if (@errorReturnTrace()) |trace| {
                             std.debug.dumpStackTrace(trace.*);
                         }
-                        std.debug.print("failed with error.{s}\n", .{@errorName(err)});
+                        std.debug.print("failed with error.{t}\n", .{err});
                         std.process.exit(1);
                     },
                 };
                 if (!is_fuzz_test) @panic("missed call to std.testing.fuzz");
                 if (log_err_count != 0) @panic("error logs detected");
+                assert(mode != .forever);
+                std.process.exit(0);
             },
 
             else => {
@@ -240,11 +254,11 @@ fn mainTerminal() void {
             else => {
                 fail_count += 1;
                 if (have_tty) {
-                    std.debug.print("{d}/{d} {s}...FAIL ({s})\n", .{
+                    std.debug.print("{d}/{d} {s}...FAIL ({t})\n", .{
-                        i + 1, test_fn_list.len, test_fn.name, @errorName(err),
+                        i + 1, test_fn_list.len, test_fn.name, err,
                     });
                 } else {
-                    std.debug.print("FAIL ({s})\n", .{@errorName(err)});
+                    std.debug.print("FAIL ({t})\n", .{err});
                 }
                 if (@errorReturnTrace()) |trace| {
                     std.debug.dumpStackTrace(trace.*);
@@ -343,6 +357,8 @@ pub fn mainSimple() anyerror!void {
 
 var is_fuzz_test: bool = undefined;
 var fuzz_test_index: u32 = undefined;
+var fuzz_mode: fuzz_abi.LimitKind = undefined;
+var fuzz_amount_or_instance: u64 = undefined;
 
 pub fn fuzz(
     context: anytype,
@@ -383,7 +399,7 @@ pub fn fuzz(
                 else => {
                     std.debug.lockStdErr();
                     if (@errorReturnTrace()) |trace| std.debug.dumpStackTrace(trace.*);
-                    std.debug.print("failed with error.{s}\n", .{@errorName(err)});
+                    std.debug.print("failed with error.{t}\n", .{err});
                     std.process.exit(1);
                 },
             };
@@ -401,9 +417,11 @@ pub fn fuzz(
 
         global.ctx = context;
         fuzz_abi.fuzzer_init_test(&global.test_one, .fromSlice(builtin.test_functions[fuzz_test_index].name));
+
         for (options.corpus) |elem|
             fuzz_abi.fuzzer_new_input(.fromSlice(elem));
-        fuzz_abi.fuzzer_main();
+
+        fuzz_abi.fuzzer_main(fuzz_mode, fuzz_amount_or_instance);
         return;
     }
 

lib/fuzzer.zig

@@ -1,5 +1,6 @@
 const builtin = @import("builtin");
 const std = @import("std");
+const fatal = std.process.fatal;
 const mem = std.mem;
 const math = std.math;
 const Allocator = mem.Allocator;
@@ -105,6 +106,7 @@ const Executable = struct {
         const coverage_file_len = @sizeOf(abi.SeenPcsHeader) +
             pc_bitset_usizes * @sizeOf(usize) +
             pcs.len * @sizeOf(usize);
+
         if (populate) {
             defer coverage_file.lock(.shared) catch |e| panic(
                 "failed to demote lock for coverage file '{s}': {t}",
@@ -510,7 +512,7 @@ const Fuzzer = struct {
             self.corpus_pos = 0;
 
         const rng = self.rng.random();
-        while (true) {
+        const m = while (true) {
             const m = self.mutations.items[rng.uintLessThanBiased(usize, self.mutations.items.len)];
             if (!m.mutate(
                 rng,
@@ -522,53 +524,53 @@ const Fuzzer = struct {
                 inst.const_vals8.items,
                 inst.const_vals16.items,
             )) continue;
+            break m;
+        };
 
-            self.run();
+        self.run();
-            if (inst.isFresh()) {
-                @branchHint(.unlikely);
 
-                const header = mem.bytesAsValue(
+        if (inst.isFresh()) {
-                    abi.SeenPcsHeader,
+            @branchHint(.unlikely);
-                    exec.shared_seen_pcs.items[0..@sizeOf(abi.SeenPcsHeader)],
-                );
-                _ = @atomicRmw(usize, &header.unique_runs, .Add, 1, .monotonic);
 
-                inst.setFresh();
+            const header = mem.bytesAsValue(
-                self.minimizeInput();
+                abi.SeenPcsHeader,
-                inst.updateSeen();
+                exec.shared_seen_pcs.items[0..@sizeOf(abi.SeenPcsHeader)],
+            );
+            _ = @atomicRmw(usize, &header.unique_runs, .Add, 1, .monotonic);
 
-                // An empty-input has always been tried, so if an empty input is fresh then the
+            inst.setFresh();
-                // test has to be non-deterministic. This has to be checked as duplicate empty
+            self.minimizeInput();
-                // entries are not allowed.
+            inst.updateSeen();
-                if (self.input.items.len - 8 == 0) {
-                    std.log.warn("non-deterministic test (empty input produces different hits)", .{});
-                    _ = @atomicRmw(usize, &header.unique_runs, .Sub, 1, .monotonic);
-                    return;
-                }
 
-                const arena = self.arena_ctx.allocator();
+            // An empty-input has always been tried, so if an empty input is fresh then the
-                const bytes = arena.dupe(u8, @volatileCast(self.input.items[8..])) catch @panic("OOM");
+            // test has to be non-deterministic. This has to be checked as duplicate empty
-
+            // entries are not allowed.
-                self.corpus.append(gpa, bytes) catch @panic("OOM");
+            if (self.input.items.len - 8 == 0) {
-                self.mutations.appendNTimes(gpa, m, 6) catch @panic("OOM");
+                std.log.warn("non-deterministic test (empty input produces different hits)", .{});
-
+                _ = @atomicRmw(usize, &header.unique_runs, .Sub, 1, .monotonic);
-                // Write new corpus to cache
+                return;
-                var name_buf: [@sizeOf(usize) * 2]u8 = undefined;
-                self.corpus_dir.writeFile(.{
-                    .sub_path = std.fmt.bufPrint(
-                        &name_buf,
-                        "{x}",
-                        .{self.corpus_dir_idx},
-                    ) catch unreachable,
-                    .data = bytes,
-                }) catch |e| panic(
-                    "failed to write corpus file '{x}': {t}",
-                    .{ self.corpus_dir_idx, e },
-                );
-                self.corpus_dir_idx += 1;
             }
 
-            break;
+            const arena = self.arena_ctx.allocator();
+            const bytes = arena.dupe(u8, @volatileCast(self.input.items[8..])) catch @panic("OOM");
+
+            self.corpus.append(gpa, bytes) catch @panic("OOM");
+            self.mutations.appendNTimes(gpa, m, 6) catch @panic("OOM");
+
+            // Write new corpus to cache
+            var name_buf: [@sizeOf(usize) * 2]u8 = undefined;
+            self.corpus_dir.writeFile(.{
+                .sub_path = std.fmt.bufPrint(
+                    &name_buf,
+                    "{x}",
+                    .{self.corpus_dir_idx},
+                ) catch unreachable,
+                .data = bytes,
+            }) catch |e| panic(
+                "failed to write corpus file '{x}': {t}",
+                .{ self.corpus_dir_idx, e },
+            );
+            self.corpus_dir_idx += 1;
         }
     }
 };
@@ -581,8 +583,21 @@ export fn fuzzer_init(cache_dir_path: abi.Slice) void {
 }
 
 /// Invalid until `fuzzer_init` is called.
-export fn fuzzer_coverage_id() u64 {
+export fn fuzzer_coverage() abi.Coverage {
-    return exec.pc_digest;
+    const coverage_id = exec.pc_digest;
+    const header: *const abi.SeenPcsHeader = @ptrCast(@volatileCast(exec.shared_seen_pcs.items.ptr));
+
+    var seen_count: usize = 0;
+    for (header.seenBits()) |chunk| {
+        seen_count += @popCount(chunk);
+    }
+
+    return .{
+        .id = coverage_id,
+        .runs = header.n_runs,
+        .unique = header.unique_runs,
+        .seen = seen_count,
+    };
 }
 
 /// fuzzer_init must be called beforehand
@@ -600,9 +615,10 @@ export fn fuzzer_new_input(bytes: abi.Slice) void {
 }
 
 /// fuzzer_init_test must be called first
-export fn fuzzer_main() void {
+export fn fuzzer_main(limit_kind: abi.LimitKind, amount: u64) void {
-    while (true) {
+    switch (limit_kind) {
-        fuzzer.cycle();
+        .forever => while (true) fuzzer.cycle(),
+        .iterations => for (0..amount) |_| fuzzer.cycle(),
     }
 }
 

lib/std/Build/Fuzz.zig

@@ -8,17 +8,22 @@ const Allocator = std.mem.Allocator;
 const log = std.log;
 const Coverage = std.debug.Coverage;
 const abi = Build.abi.fuzz;
+const tty = std.Io.tty;
 
 const Fuzz = @This();
 const build_runner = @import("root");
 
-ws: *Build.WebServer,
+gpa: Allocator,
+mode: Mode,
 
-/// Allocated into `ws.gpa`.
+/// Allocated into `gpa`.
 run_steps: []const *Step.Run,
 
 wait_group: std.Thread.WaitGroup,
+root_prog_node: std.Progress.Node,
 prog_node: std.Progress.Node,
+thread_pool: *std.Thread.Pool,
+ttyconf: tty.Config,
 
 /// Protects `coverage_files`.
 coverage_mutex: std.Thread.Mutex,
@@ -28,9 +33,23 @@ queue_mutex: std.Thread.Mutex,
 queue_cond: std.Thread.Condition,
 msg_queue: std.ArrayListUnmanaged(Msg),
 
+pub const Mode = union(enum) {
+    forever: struct { ws: *Build.WebServer },
+    limit: Limited,
+
+    pub const Limited = struct {
+        amount: u64,
+    };
+};
+
 const Msg = union(enum) {
     coverage: struct {
         id: u64,
+        cumulative: struct {
+            runs: u64,
+            unique: u64,
+            coverage: u64,
+        },
         run: *Step.Run,
     },
     entry_point: struct {
@@ -54,23 +73,28 @@ const CoverageMap = struct {
     }
 };
 
-pub fn init(ws: *Build.WebServer) Allocator.Error!Fuzz {
+pub fn init(
-    const gpa = ws.gpa;
+    gpa: Allocator,
-
+    thread_pool: *std.Thread.Pool,
+    all_steps: []const *Build.Step,
+    root_prog_node: std.Progress.Node,
+    ttyconf: tty.Config,
+    mode: Mode,
+) Allocator.Error!Fuzz {
     const run_steps: []const *Step.Run = steps: {
         var steps: std.ArrayListUnmanaged(*Step.Run) = .empty;
         defer steps.deinit(gpa);
-        const rebuild_node = ws.root_prog_node.start("Rebuilding Unit Tests", 0);
+        const rebuild_node = root_prog_node.start("Rebuilding Unit Tests", 0);
         defer rebuild_node.end();
         var rebuild_wg: std.Thread.WaitGroup = .{};
         defer rebuild_wg.wait();
 
-        for (ws.all_steps) |step| {
+        for (all_steps) |step| {
             const run = step.cast(Step.Run) orelse continue;
             if (run.producer == null) continue;
             if (run.fuzz_tests.items.len == 0) continue;
             try steps.append(gpa, run);
-            ws.thread_pool.spawnWg(&rebuild_wg, rebuildTestsWorkerRun, .{ run, gpa, ws.ttyconf, rebuild_node });
+            thread_pool.spawnWg(&rebuild_wg, rebuildTestsWorkerRun, .{ run, gpa, ttyconf, rebuild_node });
         }
 
         if (steps.items.len == 0) fatal("no fuzz tests found", .{});
@@ -86,9 +110,13 @@ pub fn init(ws: *Build.WebServer) Allocator.Error!Fuzz {
     }
 
     return .{
-        .ws = ws,
+        .gpa = gpa,
+        .mode = mode,
         .run_steps = run_steps,
         .wait_group = .{},
+        .thread_pool = thread_pool,
+        .ttyconf = ttyconf,
+        .root_prog_node = root_prog_node,
         .prog_node = .none,
         .coverage_files = .empty,
         .coverage_mutex = .{},
@@ -99,32 +127,31 @@ pub fn init(ws: *Build.WebServer) Allocator.Error!Fuzz {
 }
 
 pub fn start(fuzz: *Fuzz) void {
-    const ws = fuzz.ws;
+    fuzz.prog_node = fuzz.root_prog_node.start("Fuzzing", fuzz.run_steps.len);
-    fuzz.prog_node = ws.root_prog_node.start("Fuzzing", fuzz.run_steps.len);
 
-    // For polling messages and sending updates to subscribers.
+    if (fuzz.mode == .forever) {
-    fuzz.wait_group.start();
+        // For polling messages and sending updates to subscribers.
-    _ = std.Thread.spawn(.{}, coverageRun, .{fuzz}) catch |err| {
+        fuzz.wait_group.start();
-        fuzz.wait_group.finish();
+        _ = std.Thread.spawn(.{}, coverageRun, .{fuzz}) catch |err| {
-        fatal("unable to spawn coverage thread: {s}", .{@errorName(err)});
+            fuzz.wait_group.finish();
-    };
+            fatal("unable to spawn coverage thread: {s}", .{@errorName(err)});
+        };
+    }
 
     for (fuzz.run_steps) |run| {
         for (run.fuzz_tests.items) |unit_test_index| {
             assert(run.rebuilt_executable != null);
-            ws.thread_pool.spawnWg(&fuzz.wait_group, fuzzWorkerRun, .{
+            fuzz.thread_pool.spawnWg(&fuzz.wait_group, fuzzWorkerRun, .{
                 fuzz, run, unit_test_index,
             });
         }
     }
 }
-pub fn deinit(fuzz: *Fuzz) void {
-    if (true) @panic("TODO: terminate the fuzzer processes");
-    fuzz.wait_group.wait();
-    fuzz.prog_node.end();
 
-    const gpa = fuzz.ws.gpa;
+pub fn deinit(fuzz: *Fuzz) void {
-    gpa.free(fuzz.run_steps);
+    if (!fuzz.wait_group.isDone()) @panic("TODO: terminate the fuzzer processes");
+    fuzz.prog_node.end();
+    fuzz.gpa.free(fuzz.run_steps);
 }
 
 fn rebuildTestsWorkerRun(run: *Step.Run, gpa: Allocator, ttyconf: std.Io.tty.Config, parent_prog_node: std.Progress.Node) void {
@@ -177,7 +204,7 @@ fn fuzzWorkerRun(
             var buf: [256]u8 = undefined;
             const w = std.debug.lockStderrWriter(&buf);
             defer std.debug.unlockStderrWriter();
-            build_runner.printErrorMessages(gpa, &run.step, .{ .ttyconf = fuzz.ws.ttyconf }, w, false) catch {};
+            build_runner.printErrorMessages(gpa, &run.step, .{ .ttyconf = fuzz.ttyconf }, w, false) catch {};
             return;
         },
         else => {
@@ -190,20 +217,20 @@ fn fuzzWorkerRun(
 }
 
 pub fn serveSourcesTar(fuzz: *Fuzz, req: *std.http.Server.Request) !void {
-    const gpa = fuzz.ws.gpa;
+    assert(fuzz.mode == .forever);
 
-    var arena_state: std.heap.ArenaAllocator = .init(gpa);
+    var arena_state: std.heap.ArenaAllocator = .init(fuzz.gpa);
     defer arena_state.deinit();
     const arena = arena_state.allocator();
 
     const DedupTable = std.ArrayHashMapUnmanaged(Build.Cache.Path, void, Build.Cache.Path.TableAdapter, false);
     var dedup_table: DedupTable = .empty;
-    defer dedup_table.deinit(gpa);
+    defer dedup_table.deinit(fuzz.gpa);
 
     for (fuzz.run_steps) |run_step| {
         const compile_inputs = run_step.producer.?.step.inputs.table;
         for (compile_inputs.keys(), compile_inputs.values()) |dir_path, *file_list| {
-            try dedup_table.ensureUnusedCapacity(gpa, file_list.items.len);
+            try dedup_table.ensureUnusedCapacity(fuzz.gpa, file_list.items.len);
             for (file_list.items) |sub_path| {
                 if (!std.mem.endsWith(u8, sub_path, ".zig")) continue;
                 const joined_path = try dir_path.join(arena, sub_path);
@@ -224,13 +251,18 @@ pub fn serveSourcesTar(fuzz: *Fuzz, req: *std.http.Server.Request) !void {
         }
     };
     std.mem.sortUnstable(Build.Cache.Path, deduped_paths, SortContext{}, SortContext.lessThan);
-    return fuzz.ws.serveTarFile(req, deduped_paths);
+    return fuzz.mode.forever.ws.serveTarFile(req, deduped_paths);
 }
 
 pub const Previous = struct {
     unique_runs: usize,
     entry_points: usize,
-    pub const init: Previous = .{ .unique_runs = 0, .entry_points = 0 };
+    sent_source_index: bool,
+    pub const init: Previous = .{
+        .unique_runs = 0,
+        .entry_points = 0,
+        .sent_source_index = false,
+    };
 };
 pub fn sendUpdate(
     fuzz: *Fuzz,
@@ -253,7 +285,8 @@ pub fn sendUpdate(
     const n_runs = @atomicLoad(usize, &cov_header.n_runs, .monotonic);
     const unique_runs = @atomicLoad(usize, &cov_header.unique_runs, .monotonic);
     {
-        if (unique_runs != 0 and prev.unique_runs == 0) {
+        if (!prev.sent_source_index) {
+            prev.sent_source_index = true;
             // We need to send initial context.
             const header: abi.SourceIndexHeader = .{
                 .directories_len = @intCast(coverage_map.coverage.directories.entries.len),
@@ -319,13 +352,13 @@ fn coverageRun(fuzz: *Fuzz) void {
     }
 }
 fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutOfMemory, AlreadyReported }!void {
-    const ws = fuzz.ws;
+    assert(fuzz.mode == .forever);
-    const gpa = ws.gpa;
+    const ws = fuzz.mode.forever.ws;
 
     fuzz.coverage_mutex.lock();
     defer fuzz.coverage_mutex.unlock();
 
-    const gop = try fuzz.coverage_files.getOrPut(gpa, coverage_id);
+    const gop = try fuzz.coverage_files.getOrPut(fuzz.gpa, coverage_id);
     if (gop.found_existing) {
         // We are fuzzing the same executable with multiple threads.
         // Perhaps the same unit test; perhaps a different one. In any
@@ -343,16 +376,16 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO
         .entry_points = .{},
         .start_timestamp = ws.now(),
     };
-    errdefer gop.value_ptr.coverage.deinit(gpa);
+    errdefer gop.value_ptr.coverage.deinit(fuzz.gpa);
 
     const rebuilt_exe_path = run_step.rebuilt_executable.?;
-    var debug_info = std.debug.Info.load(gpa, rebuilt_exe_path, &gop.value_ptr.coverage) catch |err| {
+    var debug_info = std.debug.Info.load(fuzz.gpa, rebuilt_exe_path, &gop.value_ptr.coverage) catch |err| {
         log.err("step '{s}': failed to load debug information for '{f}': {s}", .{
             run_step.step.name, rebuilt_exe_path, @errorName(err),
         });
         return error.AlreadyReported;
     };
-    defer debug_info.deinit(gpa);
+    defer debug_info.deinit(fuzz.gpa);
 
     const coverage_file_path: Build.Cache.Path = .{
         .root_dir = run_step.step.owner.cache_root,
@@ -386,14 +419,14 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO
 
     const header: *const abi.SeenPcsHeader = @ptrCast(mapped_memory[0..@sizeOf(abi.SeenPcsHeader)]);
     const pcs = header.pcAddrs();
-    const source_locations = try gpa.alloc(Coverage.SourceLocation, pcs.len);
+    const source_locations = try fuzz.gpa.alloc(Coverage.SourceLocation, pcs.len);
-    errdefer gpa.free(source_locations);
+    errdefer fuzz.gpa.free(source_locations);
 
     // Unfortunately the PCs array that LLVM gives us from the 8-bit PC
     // counters feature is not sorted.
     var sorted_pcs: std.MultiArrayList(struct { pc: u64, index: u32, sl: Coverage.SourceLocation }) = .{};
-    defer sorted_pcs.deinit(gpa);
+    defer sorted_pcs.deinit(fuzz.gpa);
-    try sorted_pcs.resize(gpa, pcs.len);
+    try sorted_pcs.resize(fuzz.gpa, pcs.len);
     @memcpy(sorted_pcs.items(.pc), pcs);
     for (sorted_pcs.items(.index), 0..) |*v, i| v.* = @intCast(i);
     sorted_pcs.sortUnstable(struct {
@@ -404,7 +437,7 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO
         }
     }{ .addrs = sorted_pcs.items(.pc) });
 
-    debug_info.resolveAddresses(gpa, sorted_pcs.items(.pc), sorted_pcs.items(.sl)) catch |err| {
+    debug_info.resolveAddresses(fuzz.gpa, sorted_pcs.items(.pc), sorted_pcs.items(.sl)) catch |err| {
         log.err("failed to resolve addresses to source locations: {s}", .{@errorName(err)});
         return error.AlreadyReported;
     };
@@ -414,6 +447,7 @@ fn prepareTables(fuzz: *Fuzz, run_step: *Step.Run, coverage_id: u64) error{ OutO
 
     ws.notifyUpdate();
 }
+
 fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReported, OutOfMemory }!void {
     fuzz.coverage_mutex.lock();
     defer fuzz.coverage_mutex.unlock();
@@ -445,5 +479,89 @@ fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReporte
             addr, file_name, sl.line, sl.column, index, pcs[index - 1], pcs[index + 1],
         });
     }
-    try coverage_map.entry_points.append(fuzz.ws.gpa, @intCast(index));
+    try coverage_map.entry_points.append(fuzz.gpa, @intCast(index));
+}
+
+pub fn waitAndPrintReport(fuzz: *Fuzz) void {
+    assert(fuzz.mode == .limit);
+
+    fuzz.wait_group.wait();
+    fuzz.wait_group.reset();
+
+    std.debug.print("======= FUZZING REPORT =======\n", .{});
+    for (fuzz.msg_queue.items) |msg| {
+        if (msg != .coverage) continue;
+
+        const cov = msg.coverage;
+        const coverage_file_path: std.Build.Cache.Path = .{
+            .root_dir = cov.run.step.owner.cache_root,
+            .sub_path = "v/" ++ std.fmt.hex(cov.id),
+        };
+        var coverage_file = coverage_file_path.root_dir.handle.openFile(coverage_file_path.sub_path, .{}) catch |err| {
+            fatal("step '{s}': failed to load coverage file '{f}': {s}", .{
+                cov.run.step.name, coverage_file_path, @errorName(err),
+            });
+        };
+        defer coverage_file.close();
+
+        const fuzz_abi = std.Build.abi.fuzz;
+        var rbuf: [0x1000]u8 = undefined;
+        var r = coverage_file.reader(&rbuf);
+
+        var header: fuzz_abi.SeenPcsHeader = undefined;
+        r.interface.readSliceAll(std.mem.asBytes(&header)) catch |err| {
+            fatal("step '{s}': failed to read from coverage file '{f}': {s}", .{
+                cov.run.step.name, coverage_file_path, @errorName(err),
+            });
+        };
+
+        if (header.pcs_len == 0) {
+            fatal("step '{s}': corrupted coverage file '{f}': pcs_len was zero", .{
+                cov.run.step.name, coverage_file_path,
+            });
+        }
+
+        var seen_count: usize = 0;
+        const chunk_count = fuzz_abi.SeenPcsHeader.seenElemsLen(header.pcs_len);
+        for (0..chunk_count) |_| {
+            const seen = r.interface.takeInt(usize, .little) catch |err| {
+                fatal("step '{s}': failed to read from coverage file '{f}': {s}", .{
+                    cov.run.step.name, coverage_file_path, @errorName(err),
+                });
+            };
+            seen_count += @popCount(seen);
+        }
+
+        const seen_f: f64 = @floatFromInt(seen_count);
+        const total_f: f64 = @floatFromInt(header.pcs_len);
+        const ratio = seen_f / total_f;
+        std.debug.print(
+            \\Step: {s}
+            \\Fuzz test: "{s}" ({x})
+            \\Runs: {} -> {}
+            \\Unique runs: {} -> {}
+            \\Coverage: {}/{} -> {}/{} ({:.02}%)
+            \\
+        , .{
+            cov.run.step.name,
+            cov.run.cached_test_metadata.?.testName(cov.run.fuzz_tests.items[0]),
+            cov.id,
+            cov.cumulative.runs,
+            header.n_runs,
+            cov.cumulative.unique,
+            header.unique_runs,
+            cov.cumulative.coverage,
+            header.pcs_len,
+            seen_count,
+            header.pcs_len,
+            ratio * 100,
+        });
+
+        std.debug.print("------------------------------\n", .{});
+    }
+    std.debug.print(
+        \\Values are accumulated across multiple runs when preserving the cache.
+        \\==============================
+        \\
+    , .{});
 }

lib/std/Build/Step/Run.zig

@@ -1662,12 +1662,24 @@ fn evalZigTest(
     // If this is `true`, we avoid ever entering the polling loop below, because the stdin pipe has
     // somehow already closed; instead, we go straight to capturing stderr in case it has anything
     // useful.
-    const first_write_failed = if (fuzz_context) |fuzz| failed: {
+    const first_write_failed = if (fuzz_context) |fctx| failed: {
-        sendRunTestMessage(child.stdin.?, .start_fuzzing, fuzz.unit_test_index) catch |err| {
+        switch (fctx.fuzz.mode) {
-            try run.step.addError("unable to write stdin: {s}", .{@errorName(err)});
+            .forever => {
-            break :failed true;
+                const instance_id = 0; // will be used by mutiprocess forever fuzzing
-        };
+                sendRunFuzzTestMessage(child.stdin.?, fctx.unit_test_index, .forever, instance_id) catch |err| {
-        break :failed false;
+                    try run.step.addError("unable to write stdin: {s}", .{@errorName(err)});
+                    break :failed true;
+                };
+                break :failed false;
+            },
+            .limit => |limit| {
+                sendRunFuzzTestMessage(child.stdin.?, fctx.unit_test_index, .iterations, limit.amount) catch |err| {
+                    try run.step.addError("unable to write stdin: {s}", .{@errorName(err)});
+                    break :failed true;
+                };
+                break :failed false;
+            },
+        }
     } else failed: {
         run.fuzz_tests.clearRetainingCapacity();
         sendMessage(child.stdin.?, .query_test_metadata) catch |err| {
@@ -1778,13 +1790,18 @@ fn evalZigTest(
             },
             .coverage_id => {
                 const fuzz = fuzz_context.?.fuzz;
-                const msg_ptr: *align(1) const u64 = @ptrCast(body);
+                const msg_ptr: *align(1) const [4]u64 = @ptrCast(body);
-                coverage_id = msg_ptr.*;
+                coverage_id = msg_ptr[0];
                 {
                     fuzz.queue_mutex.lock();
                     defer fuzz.queue_mutex.unlock();
-                    try fuzz.msg_queue.append(fuzz.ws.gpa, .{ .coverage = .{
+                    try fuzz.msg_queue.append(fuzz.gpa, .{ .coverage = .{
                         .id = coverage_id.?,
+                        .cumulative = .{
+                            .runs = msg_ptr[1],
+                            .unique = msg_ptr[2],
+                            .coverage = msg_ptr[3],
+                        },
                         .run = run,
                     } });
                     fuzz.queue_cond.signal();
@@ -1797,7 +1814,7 @@ fn evalZigTest(
                 {
                     fuzz.queue_mutex.lock();
                     defer fuzz.queue_mutex.unlock();
-                    try fuzz.msg_queue.append(fuzz.ws.gpa, .{ .entry_point = .{
+                    try fuzz.msg_queue.append(fuzz.gpa, .{ .entry_point = .{
                         .addr = addr,
                         .coverage_id = coverage_id.?,
                     } });
@@ -1900,6 +1917,22 @@ fn sendRunTestMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag, index:
     try file.writeAll(full_msg);
 }
 
+fn sendRunFuzzTestMessage(
+    file: std.fs.File,
+    index: u32,
+    kind: std.Build.abi.fuzz.LimitKind,
+    amount_or_instance: u64,
+) !void {
+    const header: std.zig.Client.Message.Header = .{
+        .tag = .start_fuzzing,
+        .bytes_len = 4 + 1 + 8,
+    };
+    const full_msg = std.mem.asBytes(&header) ++ std.mem.asBytes(&index) ++
+        std.mem.asBytes(&kind) ++ std.mem.asBytes(&amount_or_instance);
+
+    try file.writeAll(full_msg);
+}
+
 fn evalGeneric(run: *Run, child: *std.process.Child) !StdIoResult {
     const b = run.step.owner;
     const arena = b.allocator;

lib/std/Build/WebServer.zig

@@ -219,12 +219,20 @@ pub fn finishBuild(ws: *WebServer, opts: struct {
             // Affects or affected by issues #5185, #22523, and #22464.
             std.process.fatal("--fuzz not yet implemented on {d}-bit platforms", .{@bitSizeOf(usize)});
         }
+
         assert(ws.fuzz == null);
 
         ws.build_status.store(.fuzz_init, .monotonic);
         ws.notifyUpdate();
 
-        ws.fuzz = Fuzz.init(ws) catch |err| std.process.fatal("failed to start fuzzer: {s}", .{@errorName(err)});
+        ws.fuzz = Fuzz.init(
+            ws.gpa,
+            ws.thread_pool,
+            ws.all_steps,
+            ws.root_prog_node,
+            ws.ttyconf,
+            .{ .forever = .{ .ws = ws } },
+        ) catch |err| std.process.fatal("failed to start fuzzer: {s}", .{@errorName(err)});
         ws.fuzz.?.start();
     }
 

lib/std/Build/abi.zig

@@ -140,10 +140,10 @@ pub const Rebuild = extern struct {
 pub const fuzz = struct {
     pub const TestOne = *const fn (Slice) callconv(.c) void;
     pub extern fn fuzzer_init(cache_dir_path: Slice) void;
-    pub extern fn fuzzer_coverage_id() u64;
+    pub extern fn fuzzer_coverage() Coverage;
     pub extern fn fuzzer_init_test(test_one: TestOne, unit_test_name: Slice) void;
     pub extern fn fuzzer_new_input(bytes: Slice) void;
-    pub extern fn fuzzer_main() void;
+    pub extern fn fuzzer_main(limit_kind: LimitKind, amount: u64) void;
 
     pub const Slice = extern struct {
         ptr: [*]const u8,
@@ -158,6 +158,8 @@ pub const fuzz = struct {
         }
     };
 
+    pub const LimitKind = enum(u8) { forever, iterations };
+
     /// libfuzzer uses this and its usize is the one that counts. To match the ABI,
     /// make the ints be the size of the target used with libfuzzer.
     ///
@@ -251,6 +253,16 @@ pub const fuzz = struct {
             return .{ .locs_len_raw = @bitCast(locs_len) };
         }
     };
+
+    /// Sent by lib/fuzzer to test_runner to obtain information about the
+    /// active memory mapped input file and cumulative stats about previous
+    /// fuzzing runs.
+    pub const Coverage = extern struct {
+        id: u64,
+        runs: u64,
+        unique: u64,
+        seen: u64,
+    };
 };
 
 /// ABI bits specifically relating to the time report interface.

lib/std/zig/Client.zig

@@ -33,10 +33,18 @@ pub const Message = struct {
         /// Ask the test runner to run a particular test.
         /// The message body is a u32 test index.
         run_test,
-        /// Ask the test runner to start fuzzing a particular test.
+        /// Ask the test runner to start fuzzing a particular test forever or for a given amount of time/iterations.
-        /// The message body is a u32 test index.
+        /// The message body is:
+        /// - a u32 test index.
+        /// - a u8 test limit kind (std.Build.api.fuzz.LimitKind)
+        /// - a u64 value whose meaning depends on FuzzLimitKind (either a limit amount or an instance id)
         start_fuzzing,
 
         _,
     };
+
+    comptime {
+        const std = @import("std");
+        std.debug.assert(@sizeOf(std.Build.abi.fuzz.LimitKind) == 1);
+    }
 };

lib/std/zig/Server.zig

@@ -42,9 +42,13 @@ pub const Message = struct {
         /// The remaining bytes is the file path relative to that prefix.
         /// The prefixes are hard-coded in Compilation.create (cwd, zig lib dir, local cache dir)
         file_system_inputs,
-        /// Body is a u64le that indicates the file path within the cache used
+        /// Body is:
-        /// to store coverage information. The integer is a hash of the PCs
+        /// - a u64le that indicates the file path within the cache used
-        /// stored within that file.
+        ///   to store coverage information. The integer is a hash of the PCs
+        ///   stored within that file.
+        /// - u64le of total runs accumulated
+        /// - u64le of unique runs accumulated
+        /// - u64le of coverage accumulated
         coverage_id,
         /// Body is a u64le that indicates the function pointer virtual memory
         /// address of the fuzz unit test. This is used to provide a starting
@@ -141,9 +145,15 @@ pub fn receiveMessage(s: *Server) !InMessage.Header {
     return s.in.takeStruct(InMessage.Header, .little);
 }
 
+pub fn receiveBody_u8(s: *Server) !u8 {
+    return s.in.takeInt(u8, .little);
+}
 pub fn receiveBody_u32(s: *Server) !u32 {
     return s.in.takeInt(u32, .little);
 }
+pub fn receiveBody_u64(s: *Server) !u64 {
+    return s.in.takeInt(u64, .little);
+}
 
 pub fn serveStringMessage(s: *Server, tag: OutMessage.Tag, msg: []const u8) !void {
     try s.serveMessageHeader(.{
@@ -160,6 +170,7 @@ pub fn serveMessageHeader(s: *const Server, header: OutMessage.Header) !void {
 }
 
 pub fn serveU64Message(s: *const Server, tag: OutMessage.Tag, int: u64) !void {
+    assert(tag != .coverage_id);
     try serveMessageHeader(s, .{
         .tag = tag,
         .bytes_len = @sizeOf(u64),
@@ -168,6 +179,18 @@ pub fn serveU64Message(s: *const Server, tag: OutMessage.Tag, int: u64) !void {
     try s.out.flush();
 }
 
+pub fn serveCoverageIdMessage(s: *const Server, id: u64, runs: u64, unique: u64, cov: u64) !void {
+    try serveMessageHeader(s, .{
+        .tag = .coverage_id,
+        .bytes_len = @sizeOf(u64) + @sizeOf(u64) + @sizeOf(u64) + @sizeOf(u64),
+    });
+    try s.out.writeInt(u64, id, .little);
+    try s.out.writeInt(u64, runs, .little);
+    try s.out.writeInt(u64, unique, .little);
+    try s.out.writeInt(u64, cov, .little);
+    try s.out.flush();
+}
+
 pub fn serveEmitDigest(
     s: *Server,
     digest: *const [Cache.bin_digest_len]u8,

src/Compilation.zig

@@ -8120,7 +8120,7 @@ pub fn addLinkLib(comp: *Compilation, lib_name: []const u8) !void {
 /// compiler-rt, libcxx, libc, libunwind, etc.
 pub fn compilerRtOptMode(comp: Compilation) std.builtin.OptimizeMode {
     if (comp.debug_compiler_runtime_libs) {
-        return comp.root_mod.optimize_mode;
+        return .Debug;
     }
     const target = &comp.root_mod.resolved_target.result;
     switch (comp.root_mod.optimize_mode) {

test/standalone/libfuzzer/main.zig

@@ -24,7 +24,7 @@ pub fn main() !void {
     abi.fuzzer_new_input(.fromSlice(""));
     abi.fuzzer_new_input(.fromSlice("hello"));
 
-    const pc_digest = abi.fuzzer_coverage_id();
+    const pc_digest = abi.fuzzer_coverage().id;
     const coverage_file_path = "v/" ++ std.fmt.hex(pc_digest);
     const coverage_file = try cache_dir.openFile(coverage_file_path, .{});
     defer coverage_file.close();