diff --git a/lib/compiler/test_runner.zig b/lib/compiler/test_runner.zig index ac9629a57d..2468c12645 100644 --- a/lib/compiler/test_runner.zig +++ b/lib/compiler/test_runner.zig @@ -145,31 +145,27 @@ fn mainServer() !void { .start_fuzzing => { if (!builtin.fuzz) unreachable; const index = try server.receiveBody_u32(); - var first = true; const test_fn = builtin.test_functions[index]; - while (true) { - testing.allocator_instance = .{}; - defer if (testing.allocator_instance.deinit() == .leak) std.process.exit(1); - log_err_count = 0; - is_fuzz_test = false; - test_fn.func() catch |err| switch (err) { - error.SkipZigTest => continue, - else => { - if (@errorReturnTrace()) |trace| { - std.debug.dumpStackTrace(trace.*); - } - std.debug.print("failed with error.{s}\n", .{@errorName(err)}); - std.process.exit(1); - }, - }; - if (!is_fuzz_test) @panic("missed call to std.testing.fuzzInput"); - if (log_err_count != 0) @panic("error logs detected"); - if (first) { - first = false; - const entry_addr = @intFromPtr(test_fn.func); - try server.serveU64Message(.fuzz_start_addr, entry_addr); - } + const entry_addr = @intFromPtr(test_fn.func); + try server.serveU64Message(.fuzz_start_addr, entry_addr); + const prev_allocator_state = testing.allocator_instance; + defer { + testing.allocator_instance = prev_allocator_state; + if (testing.allocator_instance.deinit() == .leak) std.process.exit(1); } + is_fuzz_test = false; + test_fn.func() catch |err| switch (err) { + error.SkipZigTest => return, + else => { + if (@errorReturnTrace()) |trace| { + std.debug.dumpStackTrace(trace.*); + } + std.debug.print("failed with error.{s}\n", .{@errorName(err)}); + std.process.exit(1); + }, + }; + if (!is_fuzz_test) @panic("missed call to std.testing.fuzz"); + if (log_err_count != 0) @panic("error logs detected"); }, else => { @@ -349,19 +345,67 @@ const FuzzerSlice = extern struct { var is_fuzz_test: bool = undefined; -extern fn fuzzer_next() FuzzerSlice; +extern fn fuzzer_start() void; extern fn fuzzer_init(cache_dir: FuzzerSlice) void; extern fn fuzzer_coverage_id() u64; -pub fn fuzzInput(options: testing.FuzzInputOptions) []const u8 { +pub fn fuzz( + comptime testOne: fn ([]const u8) anyerror!void, + options: testing.FuzzInputOptions, +) anyerror!void { + // Prevent this function from confusing the fuzzer by omitting its own code + // coverage from being considered. @disableInstrumentation(); - if (crippled) return ""; + + // Some compiler backends are not capable of handling fuzz testing yet but + // we still want CI test coverage enabled. + if (crippled) return; + + // Smoke test to ensure the test did not use conditional compilation to + // contradict itself by making it not actually be a fuzz test when the test + // is built in fuzz mode. is_fuzz_test = true; + + // Ensure no test failure occurred before starting fuzzing. + if (log_err_count != 0) @panic("error logs detected"); + + // libfuzzer is in a separate compilation unit so that its own code can be + // excluded from code coverage instrumentation. It needs a function pointer + // it can call for checking exactly one input. Inside this function we do + // our standard unit test checks such as memory leaks, and interaction with + // error logs. + const global = struct { + fn fuzzer_one(input_ptr: [*]const u8, input_len: usize) callconv(.C) void { + @disableInstrumentation(); + testing.allocator_instance = .{}; + defer if (testing.allocator_instance.deinit() == .leak) std.process.exit(1); + log_err_count = 0; + testOne(input_ptr[0..input_len]) catch |err| switch (err) { + error.SkipZigTest => return, + else => { + if (@errorReturnTrace()) |trace| { + std.debug.dumpStackTrace(trace.*); + } + std.debug.print("failed with error.{s}\n", .{@errorName(err)}); + std.process.exit(1); + }, + }; + if (log_err_count != 0) @panic("error logs detected"); + } + }; if (builtin.fuzz) { - return fuzzer_next().toSlice(); + @export(&global.fuzzer_one, .{ .name = "fuzzer_one" }); + fuzzer_start(); + return; } - if (options.corpus.len == 0) return ""; - var prng = std.Random.DefaultPrng.init(testing.random_seed); - const random = prng.random(); - return options.corpus[random.uintLessThan(usize, options.corpus.len)]; + + // When the unit test executable is not built in fuzz mode, only run the + // provided corpus. + for (options.corpus) |input| { + try testOne(input); + } + + // In case there is no provided corpus, also use an empty + // string as a smoke test. + try testOne(""); } diff --git a/lib/fuzzer.zig b/lib/fuzzer.zig index 9c67756a6d..2aa9744275 100644 --- a/lib/fuzzer.zig +++ b/lib/fuzzer.zig @@ -235,22 +235,41 @@ const Fuzzer = struct { }; } - fn next(f: *Fuzzer) ![]const u8 { + fn start(f: *Fuzzer) !void { const gpa = f.gpa; const rng = fuzzer.rng.random(); - if (f.recent_cases.entries.len == 0) { - // Prepare initial input. - try f.recent_cases.ensureUnusedCapacity(gpa, 100); - const len = rng.uintLessThanBiased(usize, 80); - try f.input.resize(gpa, len); - rng.bytes(f.input.items); - f.recent_cases.putAssumeCapacity(.{ - .id = 0, - .input = try gpa.dupe(u8, f.input.items), - .score = 0, - }, {}); - } else { + // Prepare initial input. + assert(f.recent_cases.entries.len == 0); + assert(f.n_runs == 0); + try f.recent_cases.ensureUnusedCapacity(gpa, 100); + const len = rng.uintLessThanBiased(usize, 80); + try f.input.resize(gpa, len); + rng.bytes(f.input.items); + f.recent_cases.putAssumeCapacity(.{ + .id = 0, + .input = try gpa.dupe(u8, f.input.items), + .score = 0, + }, {}); + + const header: *volatile SeenPcsHeader = @ptrCast(f.seen_pcs.items[0..@sizeOf(SeenPcsHeader)]); + + while (true) { + const chosen_index = rng.uintLessThanBiased(usize, f.recent_cases.entries.len); + const run = &f.recent_cases.keys()[chosen_index]; + f.input.clearRetainingCapacity(); + f.input.appendSliceAssumeCapacity(run.input); + try f.mutate(); + + _ = @atomicRmw(usize, &header.lowest_stack, .Min, __sancov_lowest_stack, .monotonic); + @memset(f.pc_counters, 0); + f.coverage.reset(); + + fuzzer_one(f.input.items.ptr, f.input.items.len); + + f.n_runs += 1; + _ = @atomicRmw(usize, &header.n_runs, .Add, 1, .monotonic); + if (f.n_runs % 10000 == 0) f.dumpStats(); const analysis = f.analyzeLastRun(); @@ -301,7 +320,6 @@ const Fuzzer = struct { } } - const header: *volatile SeenPcsHeader = @ptrCast(f.seen_pcs.items[0..@sizeOf(SeenPcsHeader)]); _ = @atomicRmw(usize, &header.unique_runs, .Add, 1, .monotonic); } @@ -317,26 +335,12 @@ const Fuzzer = struct { // This has to be done before deinitializing the deleted items. const doomed_runs = f.recent_cases.keys()[cap..]; f.recent_cases.shrinkRetainingCapacity(cap); - for (doomed_runs) |*run| { - std.log.info("culling score={d} id={d}", .{ run.score, run.id }); - run.deinit(gpa); + for (doomed_runs) |*doomed_run| { + std.log.info("culling score={d} id={d}", .{ doomed_run.score, doomed_run.id }); + doomed_run.deinit(gpa); } } } - - const chosen_index = rng.uintLessThanBiased(usize, f.recent_cases.entries.len); - const run = &f.recent_cases.keys()[chosen_index]; - f.input.clearRetainingCapacity(); - f.input.appendSliceAssumeCapacity(run.input); - try f.mutate(); - - f.n_runs += 1; - const header: *volatile SeenPcsHeader = @ptrCast(f.seen_pcs.items[0..@sizeOf(SeenPcsHeader)]); - _ = @atomicRmw(usize, &header.n_runs, .Add, 1, .monotonic); - _ = @atomicRmw(usize, &header.lowest_stack, .Min, __sancov_lowest_stack, .monotonic); - @memset(f.pc_counters, 0); - f.coverage.reset(); - return f.input.items; } fn visitPc(f: *Fuzzer, pc: usize) void { @@ -419,10 +423,12 @@ export fn fuzzer_coverage_id() u64 { return fuzzer.coverage_id; } -export fn fuzzer_next() Fuzzer.Slice { - return Fuzzer.Slice.fromZig(fuzzer.next() catch |err| switch (err) { - error.OutOfMemory => @panic("out of memory"), - }); +extern fn fuzzer_one(input_ptr: [*]const u8, input_len: usize) callconv(.C) void; + +export fn fuzzer_start() void { + fuzzer.start() catch |err| switch (err) { + error.OutOfMemory => fatal("out of memory", .{}), + }; } export fn fuzzer_init(cache_dir_struct: Fuzzer.Slice) void { @@ -432,24 +438,24 @@ export fn fuzzer_init(cache_dir_struct: Fuzzer.Slice) void { const pc_counters_start = @extern([*]u8, .{ .name = "__start___sancov_cntrs", .linkage = .weak, - }) orelse fatal("missing __start___sancov_cntrs symbol"); + }) orelse fatal("missing __start___sancov_cntrs symbol", .{}); const pc_counters_end = @extern([*]u8, .{ .name = "__stop___sancov_cntrs", .linkage = .weak, - }) orelse fatal("missing __stop___sancov_cntrs symbol"); + }) orelse fatal("missing __stop___sancov_cntrs symbol", .{}); const pc_counters = pc_counters_start[0 .. pc_counters_end - pc_counters_start]; const pcs_start = @extern([*]usize, .{ .name = "__start___sancov_pcs1", .linkage = .weak, - }) orelse fatal("missing __start___sancov_pcs1 symbol"); + }) orelse fatal("missing __start___sancov_pcs1 symbol", .{}); const pcs_end = @extern([*]usize, .{ .name = "__stop___sancov_pcs1", .linkage = .weak, - }) orelse fatal("missing __stop___sancov_pcs1 symbol"); + }) orelse fatal("missing __stop___sancov_pcs1 symbol", .{}); const pcs = pcs_start[0 .. pcs_end - pcs_start]; diff --git a/lib/std/testing.zig b/lib/std/testing.zig index 35bb13bf0d..2cc38749eb 100644 --- a/lib/std/testing.zig +++ b/lib/std/testing.zig @@ -1141,6 +1141,10 @@ pub const FuzzInputOptions = struct { corpus: []const []const u8 = &.{}, }; -pub inline fn fuzzInput(options: FuzzInputOptions) []const u8 { - return @import("root").fuzzInput(options); +/// Inline to avoid coverage instrumentation. +pub inline fn fuzz( + comptime testOne: fn (input: []const u8) anyerror!void, + options: FuzzInputOptions, +) anyerror!void { + return @import("root").fuzz(testOne, options); } diff --git a/lib/std/zig/tokenizer.zig b/lib/std/zig/tokenizer.zig index 06c6b859ac..db69693a93 100644 --- a/lib/std/zig/tokenizer.zig +++ b/lib/std/zig/tokenizer.zig @@ -1708,6 +1708,10 @@ test "invalid tabs and carriage returns" { try testTokenize("\rpub\rswitch\r", &.{ .keyword_pub, .keyword_switch }); } +test "fuzzable properties upheld" { + return std.testing.fuzz(testPropertiesUpheld, .{}); +} + fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !void { var tokenizer = Tokenizer.init(source); for (expected_token_tags) |expected_token_tag| { @@ -1723,8 +1727,7 @@ fn testTokenize(source: [:0]const u8, expected_token_tags: []const Token.Tag) !v try std.testing.expectEqual(source.len, last_token.loc.end); } -test "fuzzable properties upheld" { - const source = std.testing.fuzzInput(.{}); +fn testPropertiesUpheld(source: []const u8) anyerror!void { const source0 = try std.testing.allocator.dupeZ(u8, source); defer std.testing.allocator.free(source0); var tokenizer = Tokenizer.init(source0);