//! Default test runner for unit tests. const builtin = @import("builtin"); const std = @import("std"); const io = std.io; const testing = std.testing; pub const std_options = .{ .logFn = log, }; var log_err_count: usize = 0; var fba_buffer: [8192]u8 = undefined; var fba = std.heap.FixedBufferAllocator.init(&fba_buffer); const crippled = switch (builtin.zig_backend) { .stage2_riscv64 => true, else => false, }; pub fn main() void { @disableInstrumentation(); if (crippled) { return mainSimple() catch @panic("test failure\n"); } const args = std.process.argsAlloc(fba.allocator()) catch @panic("unable to parse command line args"); var listen = false; for (args[1..]) |arg| { if (std.mem.eql(u8, arg, "--listen=-")) { listen = true; } else if (std.mem.startsWith(u8, arg, "--seed=")) { testing.random_seed = std.fmt.parseUnsigned(u32, arg["--seed=".len..], 0) catch @panic("unable to parse --seed command line argument"); } else { @panic("unrecognized command line argument"); } } fba.reset(); if (listen) { return mainServer() catch @panic("internal test runner failure"); } else { return mainTerminal(); } } fn mainServer() !void { @disableInstrumentation(); var server = try std.zig.Server.init(.{ .gpa = fba.allocator(), .in = std.io.getStdIn(), .out = std.io.getStdOut(), .zig_version = builtin.zig_version_string, }); defer server.deinit(); while (true) { const hdr = try server.receiveMessage(); switch (hdr.tag) { .exit => { return std.process.exit(0); }, .query_test_metadata => { testing.allocator_instance = .{}; defer if (testing.allocator_instance.deinit() == .leak) { @panic("internal test runner memory leak"); }; var string_bytes: std.ArrayListUnmanaged(u8) = .{}; defer string_bytes.deinit(testing.allocator); try string_bytes.append(testing.allocator, 0); // Reserve 0 for null. const test_fns = builtin.test_functions; const names = try testing.allocator.alloc(u32, test_fns.len); defer testing.allocator.free(names); const expected_panic_msgs = try testing.allocator.alloc(u32, test_fns.len); defer testing.allocator.free(expected_panic_msgs); for (test_fns, names, expected_panic_msgs) |test_fn, *name, *expected_panic_msg| { name.* = @as(u32, @intCast(string_bytes.items.len)); try string_bytes.ensureUnusedCapacity(testing.allocator, test_fn.name.len + 1); string_bytes.appendSliceAssumeCapacity(test_fn.name); string_bytes.appendAssumeCapacity(0); expected_panic_msg.* = 0; } try server.serveTestMetadata(.{ .names = names, .expected_panic_msgs = expected_panic_msgs, .string_bytes = string_bytes.items, }); }, .run_test => { testing.allocator_instance = .{}; log_err_count = 0; const index = try server.receiveBody_u32(); const test_fn = builtin.test_functions[index]; var fail = false; var skip = false; is_fuzz_test = false; test_fn.func() catch |err| switch (err) { error.SkipZigTest => skip = true, else => { fail = true; if (@errorReturnTrace()) |trace| { std.debug.dumpStackTrace(trace.*); } }, }; const leak = testing.allocator_instance.deinit() == .leak; try server.serveTestResults(.{ .index = index, .flags = .{ .fail = fail, .skip = skip, .leak = leak, .fuzz = is_fuzz_test, .log_err_count = std.math.lossyCast( @TypeOf(@as(std.zig.Server.Message.TestResults.Flags, undefined).log_err_count), log_err_count, ), }, }); }, .start_fuzzing => { const index = try server.receiveBody_u32(); const test_fn = builtin.test_functions[index]; while (true) { testing.allocator_instance = .{}; defer if (testing.allocator_instance.deinit() == .leak) std.process.exit(1); log_err_count = 0; is_fuzz_test = false; test_fn.func() catch |err| switch (err) { error.SkipZigTest => continue, else => { if (@errorReturnTrace()) |trace| { std.debug.dumpStackTrace(trace.*); } std.debug.print("failed with error.{s}\n", .{@errorName(err)}); std.process.exit(1); }, }; if (!is_fuzz_test) @panic("missed call to std.testing.fuzzInput"); if (log_err_count != 0) @panic("error logs detected"); } }, else => { std.debug.print("unsupported message: {x}\n", .{@intFromEnum(hdr.tag)}); std.process.exit(1); }, } } } fn mainTerminal() void { @disableInstrumentation(); const test_fn_list = builtin.test_functions; var ok_count: usize = 0; var skip_count: usize = 0; var fail_count: usize = 0; var fuzz_count: usize = 0; const root_node = std.Progress.start(.{ .root_name = "Test", .estimated_total_items = test_fn_list.len, }); const have_tty = std.io.getStdErr().isTty(); var async_frame_buffer: []align(builtin.target.stackAlignment()) u8 = undefined; // TODO this is on the next line (using `undefined` above) because otherwise zig incorrectly // ignores the alignment of the slice. async_frame_buffer = &[_]u8{}; var leaks: usize = 0; for (test_fn_list, 0..) |test_fn, i| { testing.allocator_instance = .{}; defer { if (testing.allocator_instance.deinit() == .leak) { leaks += 1; } } testing.log_level = .warn; const test_node = root_node.start(test_fn.name, 0); if (!have_tty) { std.debug.print("{d}/{d} {s}...", .{ i + 1, test_fn_list.len, test_fn.name }); } is_fuzz_test = false; if (test_fn.func()) |_| { ok_count += 1; test_node.end(); if (!have_tty) std.debug.print("OK\n", .{}); } else |err| switch (err) { error.SkipZigTest => { skip_count += 1; if (have_tty) { std.debug.print("{d}/{d} {s}...SKIP\n", .{ i + 1, test_fn_list.len, test_fn.name }); } else { std.debug.print("SKIP\n", .{}); } test_node.end(); }, else => { fail_count += 1; if (have_tty) { std.debug.print("{d}/{d} {s}...FAIL ({s})\n", .{ i + 1, test_fn_list.len, test_fn.name, @errorName(err), }); } else { std.debug.print("FAIL ({s})\n", .{@errorName(err)}); } if (@errorReturnTrace()) |trace| { std.debug.dumpStackTrace(trace.*); } test_node.end(); }, } fuzz_count += @intFromBool(is_fuzz_test); } root_node.end(); if (ok_count == test_fn_list.len) { std.debug.print("All {d} tests passed.\n", .{ok_count}); } else { std.debug.print("{d} passed; {d} skipped; {d} failed.\n", .{ ok_count, skip_count, fail_count }); } if (log_err_count != 0) { std.debug.print("{d} errors were logged.\n", .{log_err_count}); } if (leaks != 0) { std.debug.print("{d} tests leaked memory.\n", .{leaks}); } if (fuzz_count != 0) { std.debug.print("{d} fuzz tests found.\n", .{fuzz_count}); } if (leaks != 0 or log_err_count != 0 or fail_count != 0) { std.process.exit(1); } } pub fn log( comptime message_level: std.log.Level, comptime scope: @Type(.EnumLiteral), comptime format: []const u8, args: anytype, ) void { @disableInstrumentation(); if (@intFromEnum(message_level) <= @intFromEnum(std.log.Level.err)) { log_err_count +|= 1; } if (@intFromEnum(message_level) <= @intFromEnum(testing.log_level)) { std.debug.print( "[" ++ @tagName(scope) ++ "] (" ++ @tagName(message_level) ++ "): " ++ format ++ "\n", args, ); } } /// Simpler main(), exercising fewer language features, so that /// work-in-progress backends can handle it. pub fn mainSimple() anyerror!void { @disableInstrumentation(); // is the backend capable of printing to stderr? const enable_print = switch (builtin.zig_backend) { else => false, }; // is the backend capable of using std.fmt.format to print a summary at the end? const print_summary = switch (builtin.zig_backend) { .stage2_riscv64 => true, else => false, }; var passed: u64 = 0; var skipped: u64 = 0; var failed: u64 = 0; // we don't want to bring in File and Writer if the backend doesn't support it const stderr = if (comptime enable_print) std.io.getStdErr() else {}; for (builtin.test_functions) |test_fn| { if (test_fn.func()) |_| { if (enable_print) { stderr.writeAll(test_fn.name) catch {}; stderr.writeAll("... ") catch {}; stderr.writeAll("PASS\n") catch {}; } } else |err| if (enable_print) { if (enable_print) { stderr.writeAll(test_fn.name) catch {}; stderr.writeAll("... ") catch {}; } if (err != error.SkipZigTest) { if (enable_print) stderr.writeAll("FAIL\n") catch {}; failed += 1; if (!enable_print) return err; continue; } if (enable_print) stderr.writeAll("SKIP\n") catch {}; skipped += 1; continue; } passed += 1; } if (enable_print and print_summary) { stderr.writer().print("{} passed, {} skipped, {} failed\n", .{ passed, skipped, failed }) catch {}; } if (failed != 0) std.process.exit(1); } const FuzzerSlice = extern struct { ptr: [*]const u8, len: usize, inline fn toSlice(s: FuzzerSlice) []const u8 { return s.ptr[0..s.len]; } }; var is_fuzz_test: bool = undefined; extern fn fuzzer_next() FuzzerSlice; pub fn fuzzInput(options: testing.FuzzInputOptions) []const u8 { @disableInstrumentation(); if (crippled) return ""; is_fuzz_test = true; if (builtin.fuzz) return fuzzer_next().toSlice(); if (options.corpus.len == 0) return ""; var prng = std.Random.DefaultPrng.init(testing.random_seed); const random = prng.random(); return options.corpus[random.uintLessThan(usize, options.corpus.len)]; }