From ede5dcffea5a3a5fc9fd14e4e180464633402fae Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 12 Mar 2023 00:39:21 -0700 Subject: [PATCH] make the build runner and test runner talk to each other std.Build.addTest creates a CompileStep as before, however, this kind of step no longer actually runs the unit tests. Instead it only compiles it, and one must additionally create a RunStep from the CompileStep in order to actually run the tests. RunStep gains integration with the default test runner, which now supports the standard --listen=- argument in order to communicate over stdin and stdout. It also reports test statistics; how many passed, failed, and leaked, as well as directly associating the relevant stderr with the particular test name that failed. This separation of CompileStep and RunStep means that `CompileStep.Kind.test_exe` is no longer needed, and therefore has been removed in this commit. * build runner: show unit test statistics in build summary * added Step.writeManifest since many steps want to treat it as a warning and emit the same message if it fails. * RunStep: fixed error message that prints the failed command printing the original argv and not the adjusted argv in case an interpreter was used. * RunStep: fixed not passing the command line arguments to the interpreter. * move src/Server.zig to std.zig.Server so that the default test runner can use it. * the simpler test runner function which is used by work-in-progress backends now no longer prints to stderr, which is necessary in order for the build runner to not print the stderr as a warning message. --- CMakeLists.txt | 2 +- lib/build_runner.zig | 59 ++- lib/std/Build.zig | 10 +- lib/std/Build/CompileStep.zig | 93 +---- lib/std/Build/InstallArtifactStep.zig | 5 +- lib/std/Build/RunStep.zig | 355 +++++++++++++++--- lib/std/Build/Step.zig | 33 +- lib/std/Build/WriteFileStep.zig | 2 +- lib/std/zig/Client.zig | 7 + lib/std/zig/Server.zig | 200 ++++++++++ lib/test_runner.zig | 172 ++++++--- src/Server.zig | 113 ------ src/main.zig | 21 +- src/objcopy.zig | 9 +- test/link/common_symbols/build.zig | 2 +- test/link/common_symbols_alignment/build.zig | 2 +- .../interdependent_static_c_libs/build.zig | 2 +- test/link/macho/tls/build.zig | 5 +- test/src/Cases.zig | 11 +- test/standalone/emit_asm_and_bin/build.zig | 2 +- test/standalone/global_linkage/build.zig | 2 +- test/standalone/issue_13970/build.zig | 6 +- test/standalone/main_pkg_path/build.zig | 2 +- test/standalone/options/build.zig | 2 +- test/standalone/pie/build.zig | 2 +- test/standalone/static_c_lib/build.zig | 2 +- .../test_runner_module_imports/build.zig | 2 +- test/standalone/test_runner_path/build.zig | 1 - test/standalone/use_alias/build.zig | 2 +- test/tests.zig | 27 +- 30 files changed, 780 insertions(+), 373 deletions(-) delete mode 100644 src/Server.zig diff --git a/CMakeLists.txt b/CMakeLists.txt index 7ff8249f4d..b0867c220b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -518,6 +518,7 @@ set(ZIG_STAGE2_SOURCES "${CMAKE_SOURCE_DIR}/lib/std/zig/c_builtins.zig" "${CMAKE_SOURCE_DIR}/lib/std/zig/Parse.zig" "${CMAKE_SOURCE_DIR}/lib/std/zig/render.zig" + "${CMAKE_SOURCE_DIR}/lib/std/zig/Server.zig" "${CMAKE_SOURCE_DIR}/lib/std/zig/string_literal.zig" "${CMAKE_SOURCE_DIR}/lib/std/zig/system.zig" "${CMAKE_SOURCE_DIR}/lib/std/zig/system/NativePaths.zig" @@ -623,7 +624,6 @@ set(ZIG_STAGE2_SOURCES "${CMAKE_SOURCE_DIR}/src/print_targets.zig" "${CMAKE_SOURCE_DIR}/src/print_zir.zig" "${CMAKE_SOURCE_DIR}/src/register_manager.zig" - "${CMAKE_SOURCE_DIR}/src/Server.zig" "${CMAKE_SOURCE_DIR}/src/target.zig" "${CMAKE_SOURCE_DIR}/src/tracy.zig" "${CMAKE_SOURCE_DIR}/src/translate_c.zig" diff --git a/lib/build_runner.zig b/lib/build_runner.zig index e28be8274d..bb341574a7 100644 --- a/lib/build_runner.zig +++ b/lib/build_runner.zig @@ -416,6 +416,12 @@ fn runStepNames( } assert(run.memory_blocked_steps.items.len == 0); + var test_skip_count: usize = 0; + var test_fail_count: usize = 0; + var test_pass_count: usize = 0; + var test_leak_count: usize = 0; + var test_count: usize = 0; + var success_count: usize = 0; var skipped_count: usize = 0; var failure_count: usize = 0; @@ -425,6 +431,12 @@ fn runStepNames( defer compile_error_steps.deinit(gpa); for (step_stack.keys()) |s| { + test_fail_count += s.test_results.fail_count; + test_skip_count += s.test_results.skip_count; + test_leak_count += s.test_results.leak_count; + test_pass_count += s.test_results.passCount(); + test_count += s.test_results.test_count; + switch (s.state) { .precheck_unstarted => unreachable, .precheck_started => unreachable, @@ -468,6 +480,11 @@ fn runStepNames( if (skipped_count > 0) stderr.writer().print("; {d} skipped", .{skipped_count}) catch {}; if (failure_count > 0) stderr.writer().print("; {d} failed", .{failure_count}) catch {}; + if (test_count > 0) stderr.writer().print("; {d}/{d} tests passed", .{ test_pass_count, test_count }) catch {}; + if (test_skip_count > 0) stderr.writer().print("; {d} skipped", .{test_skip_count}) catch {}; + if (test_fail_count > 0) stderr.writer().print("; {d} failed", .{test_fail_count}) catch {}; + if (test_leak_count > 0) stderr.writer().print("; {d} leaked", .{test_leak_count}) catch {}; + if (run.enable_summary == null) { ttyconf.setColor(stderr, .Dim) catch {}; stderr.writeAll(" (disable with -fno-summary)") catch {}; @@ -566,6 +583,13 @@ fn printTreeStep( try ttyconf.setColor(stderr, .Green); if (s.result_cached) { try stderr.writeAll(" cached"); + } else if (s.test_results.test_count > 0) { + const pass_count = s.test_results.passCount(); + try stderr.writer().print(" {d} passed", .{pass_count}); + if (s.test_results.skip_count > 0) { + try ttyconf.setColor(stderr, .Yellow); + try stderr.writer().print(" {d} skipped", .{s.test_results.skip_count}); + } } else { try stderr.writeAll(" success"); } @@ -609,15 +633,46 @@ fn printTreeStep( }, .failure => { - try ttyconf.setColor(stderr, .Red); if (s.result_error_bundle.errorMessageCount() > 0) { + try ttyconf.setColor(stderr, .Red); try stderr.writer().print(" {d} errors\n", .{ s.result_error_bundle.errorMessageCount(), }); + try ttyconf.setColor(stderr, .Reset); + } else if (!s.test_results.isSuccess()) { + try stderr.writer().print(" {d}/{d} passed", .{ + s.test_results.passCount(), s.test_results.test_count, + }); + if (s.test_results.fail_count > 0) { + try stderr.writeAll(", "); + try ttyconf.setColor(stderr, .Red); + try stderr.writer().print("{d} failed", .{ + s.test_results.fail_count, + }); + try ttyconf.setColor(stderr, .Reset); + } + if (s.test_results.skip_count > 0) { + try stderr.writeAll(", "); + try ttyconf.setColor(stderr, .Yellow); + try stderr.writer().print("{d} skipped", .{ + s.test_results.skip_count, + }); + try ttyconf.setColor(stderr, .Reset); + } + if (s.test_results.leak_count > 0) { + try stderr.writeAll(", "); + try ttyconf.setColor(stderr, .Red); + try stderr.writer().print("{d} leaked", .{ + s.test_results.leak_count, + }); + try ttyconf.setColor(stderr, .Reset); + } + try stderr.writeAll("\n"); } else { + try ttyconf.setColor(stderr, .Red); try stderr.writeAll(" failure\n"); + try ttyconf.setColor(stderr, .Reset); } - try ttyconf.setColor(stderr, .Reset); }, } diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 5f74af3db6..279dd765b5 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -531,7 +531,6 @@ pub fn addStaticLibrary(b: *Build, options: StaticLibraryOptions) *CompileStep { pub const TestOptions = struct { name: []const u8 = "test", - kind: CompileStep.Kind = .@"test", root_source_file: FileSource, target: CrossTarget = .{}, optimize: std.builtin.Mode = .Debug, @@ -542,7 +541,7 @@ pub const TestOptions = struct { pub fn addTest(b: *Build, options: TestOptions) *CompileStep { return CompileStep.create(b, .{ .name = options.name, - .kind = options.kind, + .kind = .@"test", .root_source_file = options.root_source_file, .target = options.target, .optimize = options.optimize, @@ -626,16 +625,15 @@ pub fn addSystemCommand(self: *Build, argv: []const []const u8) *RunStep { /// Creates a `RunStep` with an executable built with `addExecutable`. /// Add command line arguments with methods of `RunStep`. pub fn addRunArtifact(b: *Build, exe: *CompileStep) *RunStep { - assert(exe.kind == .exe or exe.kind == .test_exe); - // It doesn't have to be native. We catch that if you actually try to run it. // Consider that this is declarative; the run step may not be run unless a user // option is supplied. const run_step = RunStep.create(b, b.fmt("run {s}", .{exe.name})); run_step.addArtifactArg(exe); - if (exe.kind == .test_exe) { - run_step.addArg(b.zig_exe); + if (exe.kind == .@"test") { + run_step.stdio = .zig_test; + run_step.addArgs(&.{"--listen=-"}); } if (exe.vcpkg_bin_path) |path| { diff --git a/lib/std/Build/CompileStep.zig b/lib/std/Build/CompileStep.zig index 5c44ab82d3..5753641966 100644 --- a/lib/std/Build/CompileStep.zig +++ b/lib/std/Build/CompileStep.zig @@ -289,7 +289,6 @@ pub const Kind = enum { lib, obj, @"test", - test_exe, }; pub const Linkage = enum { dynamic, static }; @@ -328,7 +327,7 @@ pub fn create(owner: *std.Build, options: Options) *CompileStep { .exe => "zig build-exe", .lib => "zig build-lib", .obj => "zig build-obj", - .test_exe, .@"test" => "zig test", + .@"test" => "zig test", }, name_adjusted, @tagName(options.optimize), @@ -410,7 +409,7 @@ fn computeOutFileNames(self: *CompileStep) void { .output_mode = switch (self.kind) { .lib => .Lib, .obj => .Obj, - .exe, .@"test", .test_exe => .Exe, + .exe, .@"test" => .Exe, }, .link_mode = if (self.linkage) |some| @as(std.builtin.LinkMode, switch (some) { .dynamic => .Dynamic, @@ -621,7 +620,7 @@ pub fn producesPdbFile(self: *CompileStep) bool { if (!self.target.isWindows() and !self.target.isUefi()) return false; if (self.target.getObjectFormat() == .c) return false; if (self.strip == true) return false; - return self.isDynamicLibrary() or self.kind == .exe or self.kind == .test_exe; + return self.isDynamicLibrary() or self.kind == .exe or self.kind == .@"test"; } pub fn linkLibC(self: *CompileStep) void { @@ -850,19 +849,19 @@ fn linkSystemLibraryInner(self: *CompileStep, name: []const u8, opts: struct { pub fn setNamePrefix(self: *CompileStep, text: []const u8) void { const b = self.step.owner; - assert(self.kind == .@"test" or self.kind == .test_exe); + assert(self.kind == .@"test"); self.name_prefix = b.dupe(text); } pub fn setFilter(self: *CompileStep, text: ?[]const u8) void { const b = self.step.owner; - assert(self.kind == .@"test" or self.kind == .test_exe); + assert(self.kind == .@"test"); self.filter = if (text) |t| b.dupe(t) else null; } pub fn setTestRunner(self: *CompileStep, path: ?[]const u8) void { const b = self.step.owner; - assert(self.kind == .@"test" or self.kind == .test_exe); + assert(self.kind == .@"test"); self.test_runner = if (path) |p| b.dupePath(p) else null; } @@ -938,7 +937,7 @@ pub fn getOutputLibSource(self: *CompileStep) FileSource { /// Returns the generated header file. /// This function can only be called for libraries or object files which have `emit_h` set. pub fn getOutputHSource(self: *CompileStep) FileSource { - assert(self.kind != .exe and self.kind != .test_exe and self.kind != .@"test"); + assert(self.kind != .exe and self.kind != .@"test"); assert(self.emit_h); return .{ .generated = &self.output_h_path_source }; } @@ -1243,7 +1242,6 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { .exe => "build-exe", .obj => "build-obj", .@"test" => "test", - .test_exe => "test", }; try zig_args.append(cmd); @@ -1293,7 +1291,6 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { .other_step => |other| switch (other.kind) { .exe => @panic("Cannot link with an executable build artifact"), - .test_exe => @panic("Cannot link with an executable build artifact"), .@"test" => @panic("Cannot link with a test"), .obj => { try zig_args.append(other.getOutputSource().getPath(b)); @@ -1661,83 +1658,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { try zig_args.append("--test-cmd-bin"); } } - } else { - const need_cross_glibc = self.target.isGnuLibC() and transitive_deps.is_linking_libc; - - switch (b.host.getExternalExecutor(self.target_info, .{ - .qemu_fixes_dl = need_cross_glibc and b.glibc_runtimes_dir != null, - .link_libc = transitive_deps.is_linking_libc, - })) { - .native => {}, - .bad_dl, .bad_os_or_cpu => { - try zig_args.append("--test-no-exec"); - }, - .rosetta => if (b.enable_rosetta) { - try zig_args.append("--test-cmd-bin"); - } else { - try zig_args.append("--test-no-exec"); - }, - .qemu => |bin_name| ok: { - if (b.enable_qemu) qemu: { - const glibc_dir_arg = if (need_cross_glibc) - b.glibc_runtimes_dir orelse break :qemu - else - null; - try zig_args.append("--test-cmd"); - try zig_args.append(bin_name); - if (glibc_dir_arg) |dir| { - // TODO look into making this a call to `linuxTriple`. This - // needs the directory to be called "i686" rather than - // "x86" which is why we do it manually here. - const fmt_str = "{s}" ++ fs.path.sep_str ++ "{s}-{s}-{s}"; - const cpu_arch = self.target.getCpuArch(); - const os_tag = self.target.getOsTag(); - const abi = self.target.getAbi(); - const cpu_arch_name: []const u8 = if (cpu_arch == .x86) - "i686" - else - @tagName(cpu_arch); - const full_dir = try std.fmt.allocPrint(b.allocator, fmt_str, .{ - dir, cpu_arch_name, @tagName(os_tag), @tagName(abi), - }); - - try zig_args.append("--test-cmd"); - try zig_args.append("-L"); - try zig_args.append("--test-cmd"); - try zig_args.append(full_dir); - } - try zig_args.append("--test-cmd-bin"); - break :ok; - } - try zig_args.append("--test-no-exec"); - }, - .wine => |bin_name| if (b.enable_wine) { - try zig_args.append("--test-cmd"); - try zig_args.append(bin_name); - try zig_args.append("--test-cmd-bin"); - } else { - try zig_args.append("--test-no-exec"); - }, - .wasmtime => |bin_name| if (b.enable_wasmtime) { - try zig_args.append("--test-cmd"); - try zig_args.append(bin_name); - try zig_args.append("--test-cmd"); - try zig_args.append("--dir=."); - try zig_args.append("--test-cmd-bin"); - } else { - try zig_args.append("--test-no-exec"); - }, - .darling => |bin_name| if (b.enable_darling) { - try zig_args.append("--test-cmd"); - try zig_args.append(bin_name); - try zig_args.append("--test-cmd-bin"); - } else { - try zig_args.append("--test-no-exec"); - }, - } } - } else if (self.kind == .test_exe) { - try zig_args.append("--test-no-exec"); } try self.appendModuleArgs(&zig_args); diff --git a/lib/std/Build/InstallArtifactStep.zig b/lib/std/Build/InstallArtifactStep.zig index 803998a619..445f1e8ea8 100644 --- a/lib/std/Build/InstallArtifactStep.zig +++ b/lib/std/Build/InstallArtifactStep.zig @@ -32,12 +32,11 @@ pub fn create(owner: *std.Build, artifact: *CompileStep) *InstallArtifactStep { .artifact = artifact, .dest_dir = artifact.override_dest_dir orelse switch (artifact.kind) { .obj => @panic("Cannot install a .obj build artifact."), - .@"test" => @panic("Cannot install a .test build artifact, use .test_exe instead."), - .exe, .test_exe => InstallDir{ .bin = {} }, + .exe, .@"test" => InstallDir{ .bin = {} }, .lib => InstallDir{ .lib = {} }, }, .pdb_dir = if (artifact.producesPdbFile()) blk: { - if (artifact.kind == .exe or artifact.kind == .test_exe) { + if (artifact.kind == .exe or artifact.kind == .@"test") { break :blk InstallDir{ .bin = {} }; } else { break :blk InstallDir{ .lib = {} }; diff --git a/lib/std/Build/RunStep.zig b/lib/std/Build/RunStep.zig index 9a1c887d7d..484600bf9b 100644 --- a/lib/std/Build/RunStep.zig +++ b/lib/std/Build/RunStep.zig @@ -92,6 +92,9 @@ pub const StdIo = union(enum) { /// Note that an explicit check for exit code 0 needs to be added to this /// list if such a check is desireable. check: std.ArrayList(Check), + /// This RunStep is running a zig unit test binary and will communicate + /// extra metadata over the IPC protocol. + zig_test, pub const Check = union(enum) { expect_stderr_exact: []const u8, @@ -324,6 +327,7 @@ fn hasSideEffects(self: RunStep) bool { .infer_from_args => !self.hasAnyOutputArgs(), .inherit => true, .check => false, + .zig_test => false, }; } @@ -366,11 +370,6 @@ fn checksContainStderr(checks: []const StdIo.Check) bool { } fn make(step: *Step, prog_node: *std.Progress.Node) !void { - // Unfortunately we have no way to collect progress from arbitrary programs. - // Perhaps in the future Zig could offer some kind of opt-in IPC mechanism that - // processes could use to supply progress updates. - _ = prog_node; - const b = step.owner; const arena = b.allocator; const self = @fieldParentPtr(RunStep, "step", step); @@ -439,7 +438,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { hashStdIo(&man.hash, self.stdio); if (has_side_effects) { - try runCommand(self, argv_list.items, has_side_effects, null); + try runCommand(self, argv_list.items, has_side_effects, null, prog_node); return; } @@ -492,8 +491,9 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { argv_list.items[placeholder.index] = cli_arg; } - try runCommand(self, argv_list.items, has_side_effects, &digest); - try man.writeManifest(); + try runCommand(self, argv_list.items, has_side_effects, &digest, prog_node); + + try step.writeManifest(&man); } fn formatTerm( @@ -546,6 +546,7 @@ fn runCommand( argv: []const []const u8, has_side_effects: bool, digest: ?*const [std.Build.Cache.hex_digest_len]u8, + prog_node: *std.Progress.Node, ) !void { const step = &self.step; const b = step.owner; @@ -554,7 +555,15 @@ fn runCommand( try step.handleChildProcUnsupported(self.cwd, argv); try Step.handleVerbose(step.owner, self.cwd, argv); - const result = spawnChildAndCollect(self, argv, has_side_effects) catch |err| term: { + const allow_skip = switch (self.stdio) { + .check, .zig_test => self.skip_foreign_checks, + else => false, + }; + + var interp_argv = std.ArrayList([]const u8).init(b.allocator); + defer interp_argv.deinit(); + + const result = spawnChildAndCollect(self, argv, has_side_effects, prog_node) catch |err| term: { // InvalidExe: cpu arch mismatch // FileNotFound: can happen with a wrong dynamic linker path if (err == error.InvalidExe or err == error.FileNotFound) interpret: { @@ -566,10 +575,10 @@ fn runCommand( .artifact => |exe| exe, else => break :interpret, }; - if (exe.kind != .exe) break :interpret; - - var interp_argv = std.ArrayList([]const u8).init(b.allocator); - defer interp_argv.deinit(); + switch (exe.kind) { + .exe, .@"test" => {}, + else => break :interpret, + } const need_cross_glibc = exe.target.isGnuLibC() and exe.is_linking_libc; switch (b.host.getExternalExecutor(exe.target_info, .{ @@ -577,14 +586,13 @@ fn runCommand( .link_libc = exe.is_linking_libc, })) { .native, .rosetta => { - if (self.stdio == .check and self.skip_foreign_checks) - return error.MakeSkipped; - + if (allow_skip) return error.MakeSkipped; break :interpret; }, .wine => |bin_name| { if (b.enable_wine) { try interp_argv.append(bin_name); + try interp_argv.appendSlice(argv); } else { return failForeign(self, "-fwine", argv[0], exe); } @@ -617,6 +625,8 @@ fn runCommand( try interp_argv.append("-L"); try interp_argv.append(full_dir); } + + try interp_argv.appendSlice(argv); } else { return failForeign(self, "-fqemu", argv[0], exe); } @@ -624,6 +634,7 @@ fn runCommand( .darling => |bin_name| { if (b.enable_darling) { try interp_argv.append(bin_name); + try interp_argv.appendSlice(argv); } else { return failForeign(self, "-fdarling", argv[0], exe); } @@ -632,13 +643,15 @@ fn runCommand( if (b.enable_wasmtime) { try interp_argv.append(bin_name); try interp_argv.append("--dir=."); + try interp_argv.append(argv[0]); + try interp_argv.append("--"); + try interp_argv.appendSlice(argv[1..]); } else { return failForeign(self, "-fwasmtime", argv[0], exe); } }, .bad_dl => |foreign_dl| { - if (self.stdio == .check and self.skip_foreign_checks) - return error.MakeSkipped; + if (allow_skip) return error.MakeSkipped; const host_dl = b.host.dynamic_linker.get() orelse "(none)"; @@ -650,8 +663,7 @@ fn runCommand( , .{ host_dl, foreign_dl }); }, .bad_os_or_cpu => { - if (self.stdio == .check and self.skip_foreign_checks) - return error.MakeSkipped; + if (allow_skip) return error.MakeSkipped; const host_name = try b.host.target.zigTriple(b.allocator); const foreign_name = try exe.target.zigTriple(b.allocator); @@ -667,11 +679,9 @@ fn runCommand( RunStep.addPathForDynLibsInternal(&self.step, b, exe); } - try interp_argv.append(argv[0]); - try Step.handleVerbose(step.owner, self.cwd, interp_argv.items); - break :term spawnChildAndCollect(self, interp_argv.items, has_side_effects) catch |e| { + break :term spawnChildAndCollect(self, interp_argv.items, has_side_effects, prog_node) catch |e| { return step.fail("unable to spawn {s}: {s}", .{ interp_argv.items[0], @errorName(e), }); @@ -683,6 +693,7 @@ fn runCommand( step.result_duration_ns = result.elapsed_ns; step.result_peak_rss = result.peak_rss; + step.test_results = result.stdio.test_results; // Capture stdout and stderr to GeneratedFile objects. const Stream = struct { @@ -693,13 +704,13 @@ fn runCommand( for ([_]Stream{ .{ .captured = self.captured_stdout, - .is_null = result.stdout_null, - .bytes = result.stdout, + .is_null = result.stdio.stdout_null, + .bytes = result.stdio.stdout, }, .{ .captured = self.captured_stderr, - .is_null = result.stderr_null, - .bytes = result.stderr, + .is_null = result.stdio.stderr_null, + .bytes = result.stdio.stderr, }, }) |stream| { if (stream.captured) |output| { @@ -724,11 +735,13 @@ fn runCommand( } } + const final_argv = if (interp_argv.items.len == 0) argv else interp_argv.items; + switch (self.stdio) { .check => |checks| for (checks.items) |check| switch (check) { .expect_stderr_exact => |expected_bytes| { - assert(!result.stderr_null); - if (!mem.eql(u8, expected_bytes, result.stderr)) { + assert(!result.stdio.stderr_null); + if (!mem.eql(u8, expected_bytes, result.stdio.stderr)) { return step.fail( \\ \\========= expected this stderr: ========= @@ -739,14 +752,14 @@ fn runCommand( \\{s} , .{ expected_bytes, - result.stderr, - try Step.allocPrintCmd(arena, self.cwd, argv), + result.stdio.stderr, + try Step.allocPrintCmd(arena, self.cwd, final_argv), }); } }, .expect_stderr_match => |match| { - assert(!result.stderr_null); - if (mem.indexOf(u8, result.stderr, match) == null) { + assert(!result.stdio.stderr_null); + if (mem.indexOf(u8, result.stdio.stderr, match) == null) { return step.fail( \\ \\========= expected to find in stderr: ========= @@ -757,14 +770,14 @@ fn runCommand( \\{s} , .{ match, - result.stderr, - try Step.allocPrintCmd(arena, self.cwd, argv), + result.stdio.stderr, + try Step.allocPrintCmd(arena, self.cwd, final_argv), }); } }, .expect_stdout_exact => |expected_bytes| { - assert(!result.stdout_null); - if (!mem.eql(u8, expected_bytes, result.stdout)) { + assert(!result.stdio.stdout_null); + if (!mem.eql(u8, expected_bytes, result.stdio.stdout)) { return step.fail( \\ \\========= expected this stdout: ========= @@ -775,14 +788,14 @@ fn runCommand( \\{s} , .{ expected_bytes, - result.stdout, - try Step.allocPrintCmd(arena, self.cwd, argv), + result.stdio.stdout, + try Step.allocPrintCmd(arena, self.cwd, final_argv), }); } }, .expect_stdout_match => |match| { - assert(!result.stdout_null); - if (mem.indexOf(u8, result.stdout, match) == null) { + assert(!result.stdio.stdout_null); + if (mem.indexOf(u8, result.stdio.stdout, match) == null) { return step.fail( \\ \\========= expected to find in stdout: ========= @@ -793,8 +806,8 @@ fn runCommand( \\{s} , .{ match, - result.stdout, - try Step.allocPrintCmd(arena, self.cwd, argv), + result.stdio.stdout, + try Step.allocPrintCmd(arena, self.cwd, final_argv), }); } }, @@ -803,33 +816,46 @@ fn runCommand( return step.fail("the following command {} (expected {}):\n{s}", .{ fmtTerm(result.term), fmtTerm(expected_term), - try Step.allocPrintCmd(arena, self.cwd, argv), + try Step.allocPrintCmd(arena, self.cwd, final_argv), }); } }, }, + .zig_test => { + const expected_term: std.process.Child.Term = .{ .Exited = 0 }; + if (!termMatches(expected_term, result.term)) { + return step.fail("the following command {} (expected {}):\n{s}", .{ + fmtTerm(result.term), + fmtTerm(expected_term), + try Step.allocPrintCmd(arena, self.cwd, final_argv), + }); + } + if (!result.stdio.test_results.isSuccess()) { + return step.fail( + "the following test command failed:\n{s}", + .{try Step.allocPrintCmd(arena, self.cwd, final_argv)}, + ); + } + }, else => { - try step.handleChildProcessTerm(result.term, self.cwd, argv); + try step.handleChildProcessTerm(result.term, self.cwd, final_argv); }, } } const ChildProcResult = struct { - // These use boolean flags instead of optionals as a workaround for - // https://github.com/ziglang/zig/issues/14783 - stdout: []const u8, - stderr: []const u8, - stdout_null: bool, - stderr_null: bool, term: std.process.Child.Term, elapsed_ns: u64, peak_rss: usize, + + stdio: StdIoResult, }; fn spawnChildAndCollect( self: *RunStep, argv: []const []const u8, has_side_effects: bool, + prog_node: *std.Progress.Node, ) !ChildProcResult { const b = self.step.owner; const arena = b.allocator; @@ -848,16 +874,19 @@ fn spawnChildAndCollect( .infer_from_args => if (has_side_effects) .Inherit else .Close, .inherit => .Inherit, .check => .Close, + .zig_test => .Pipe, }; child.stdout_behavior = switch (self.stdio) { .infer_from_args => if (has_side_effects) .Inherit else .Ignore, .inherit => .Inherit, .check => |checks| if (checksContainStdout(checks.items)) .Pipe else .Ignore, + .zig_test => .Pipe, }; child.stderr_behavior = switch (self.stdio) { .infer_from_args => if (has_side_effects) .Inherit else .Pipe, .inherit => .Inherit, .check => .Pipe, + .zig_test => .Pipe, }; if (self.captured_stdout != null) child.stdout_behavior = .Pipe; if (self.captured_stderr != null) child.stderr_behavior = .Pipe; @@ -871,6 +900,219 @@ fn spawnChildAndCollect( }); var timer = try std.time.Timer.start(); + const result = if (self.stdio == .zig_test) + evalZigTest(self, &child, prog_node) + else + evalGeneric(self, &child); + + const term = try child.wait(); + const elapsed_ns = timer.read(); + + return .{ + .stdio = try result, + .term = term, + .elapsed_ns = elapsed_ns, + .peak_rss = child.resource_usage_statistics.getMaxRss() orelse 0, + }; +} + +const StdIoResult = struct { + // These use boolean flags instead of optionals as a workaround for + // https://github.com/ziglang/zig/issues/14783 + stdout: []const u8, + stderr: []const u8, + stdout_null: bool, + stderr_null: bool, + test_results: Step.TestResults, +}; + +fn evalZigTest( + self: *RunStep, + child: *std.process.Child, + prog_node: *std.Progress.Node, +) !StdIoResult { + const gpa = self.step.owner.allocator; + const arena = self.step.owner.allocator; + + var poller = std.io.poll(gpa, enum { stdout, stderr }, .{ + .stdout = child.stdout.?, + .stderr = child.stderr.?, + }); + defer poller.deinit(); + + try sendMessage(child.stdin.?, .query_test_metadata); + + const Header = std.zig.Server.Message.Header; + + const stdout = poller.fifo(.stdout); + const stderr = poller.fifo(.stderr); + + var fail_count: u32 = 0; + var skip_count: u32 = 0; + var leak_count: u32 = 0; + var test_count: u32 = 0; + + var metadata: ?TestMetadata = null; + + var sub_prog_node: ?std.Progress.Node = null; + defer if (sub_prog_node) |*n| n.end(); + + poll: while (try poller.poll()) { + while (true) { + const buf = stdout.readableSlice(0); + assert(stdout.readableLength() == buf.len); + if (buf.len < @sizeOf(Header)) continue :poll; + const header = @ptrCast(*align(1) const Header, buf[0..@sizeOf(Header)]); + const header_and_msg_len = header.bytes_len + @sizeOf(Header); + if (buf.len < header_and_msg_len) continue :poll; + const body = buf[@sizeOf(Header)..][0..header.bytes_len]; + switch (header.tag) { + .zig_version => { + if (!std.mem.eql(u8, builtin.zig_version_string, body)) { + return self.step.fail( + "zig version mismatch build runner vs compiler: '{s}' vs '{s}'", + .{ builtin.zig_version_string, body }, + ); + } + }, + .test_metadata => { + const TmHdr = std.zig.Server.Message.TestMetadata; + const tm_hdr = @ptrCast(*align(1) const TmHdr, body); + test_count = tm_hdr.tests_len; + + const names_bytes = body[@sizeOf(TmHdr)..][0 .. test_count * @sizeOf(u32)]; + const async_frame_lens_bytes = body[@sizeOf(TmHdr) + names_bytes.len ..][0 .. test_count * @sizeOf(u32)]; + const expected_panic_msgs_bytes = body[@sizeOf(TmHdr) + names_bytes.len + async_frame_lens_bytes.len ..][0 .. test_count * @sizeOf(u32)]; + const string_bytes = body[@sizeOf(TmHdr) + names_bytes.len + async_frame_lens_bytes.len + expected_panic_msgs_bytes.len ..][0..tm_hdr.string_bytes_len]; + + const names = std.mem.bytesAsSlice(u32, names_bytes); + const async_frame_lens = std.mem.bytesAsSlice(u32, async_frame_lens_bytes); + const expected_panic_msgs = std.mem.bytesAsSlice(u32, expected_panic_msgs_bytes); + const names_aligned = try arena.alloc(u32, names.len); + for (names_aligned, names) |*dest, src| dest.* = src; + + const async_frame_lens_aligned = try arena.alloc(u32, async_frame_lens.len); + for (async_frame_lens_aligned, async_frame_lens) |*dest, src| dest.* = src; + + const expected_panic_msgs_aligned = try arena.alloc(u32, expected_panic_msgs.len); + for (expected_panic_msgs_aligned, expected_panic_msgs) |*dest, src| dest.* = src; + + prog_node.setEstimatedTotalItems(names.len); + metadata = .{ + .string_bytes = try arena.dupe(u8, string_bytes), + .names = names_aligned, + .async_frame_lens = async_frame_lens_aligned, + .expected_panic_msgs = expected_panic_msgs_aligned, + .next_index = 0, + .prog_node = prog_node, + }; + + try requestNextTest(child.stdin.?, &metadata.?, &sub_prog_node); + }, + .test_results => { + const md = metadata.?; + + const TrHdr = std.zig.Server.Message.TestResults; + const tr_hdr = @ptrCast(*align(1) const TrHdr, body); + fail_count += @boolToInt(tr_hdr.flags.fail); + skip_count += @boolToInt(tr_hdr.flags.skip); + leak_count += @boolToInt(tr_hdr.flags.leak); + + if (tr_hdr.flags.fail or tr_hdr.flags.leak) { + const name = std.mem.sliceTo(md.string_bytes[md.names[tr_hdr.index]..], 0); + const msg = std.mem.trim(u8, stderr.readableSlice(0), "\n"); + const label = if (tr_hdr.flags.fail) "failed" else "leaked"; + if (msg.len > 0) { + try self.step.addError("'{s}' {s}: {s}", .{ name, label, msg }); + } else { + try self.step.addError("'{s}' {s}", .{ name, label }); + } + stderr.discard(msg.len); + } + + try requestNextTest(child.stdin.?, &metadata.?, &sub_prog_node); + }, + else => {}, // ignore other messages + } + stdout.discard(header_and_msg_len); + } + } + + if (stderr.readableLength() > 0) { + const msg = std.mem.trim(u8, try stderr.toOwnedSlice(), "\n"); + if (msg.len > 0) try self.step.result_error_msgs.append(arena, msg); + } + + // Send EOF to stdin. + child.stdin.?.close(); + child.stdin = null; + + return .{ + .stdout = &.{}, + .stderr = &.{}, + .stdout_null = true, + .stderr_null = true, + .test_results = .{ + .test_count = test_count, + .fail_count = fail_count, + .skip_count = skip_count, + .leak_count = leak_count, + }, + }; +} + +const TestMetadata = struct { + names: []const u32, + async_frame_lens: []const u32, + expected_panic_msgs: []const u32, + string_bytes: []const u8, + next_index: u32, + prog_node: *std.Progress.Node, + + fn testName(tm: TestMetadata, index: u32) []const u8 { + return std.mem.sliceTo(tm.string_bytes[tm.names[index]..], 0); + } +}; + +fn requestNextTest(in: fs.File, metadata: *TestMetadata, sub_prog_node: *?std.Progress.Node) !void { + while (metadata.next_index < metadata.names.len) { + const i = metadata.next_index; + metadata.next_index += 1; + + if (metadata.async_frame_lens[i] != 0) continue; + if (metadata.expected_panic_msgs[i] != 0) continue; + + const name = metadata.testName(i); + if (sub_prog_node.*) |*n| n.end(); + sub_prog_node.* = metadata.prog_node.start(name, 0); + + try sendRunTestMessage(in, i); + return; + } else { + try sendMessage(in, .exit); + } +} + +fn sendMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag) !void { + const header: std.zig.Client.Message.Header = .{ + .tag = tag, + .bytes_len = 0, + }; + try file.writeAll(std.mem.asBytes(&header)); +} + +fn sendRunTestMessage(file: std.fs.File, index: u32) !void { + const header: std.zig.Client.Message.Header = .{ + .tag = .run_test, + .bytes_len = 4, + }; + const full_msg = std.mem.asBytes(&header) ++ std.mem.asBytes(&index); + try file.writeAll(full_msg); +} + +fn evalGeneric(self: *RunStep, child: *std.process.Child) !StdIoResult { + const arena = self.step.owner.allocator; + if (self.stdin) |stdin| { child.stdin.?.writeAll(stdin) catch |err| { return self.step.fail("unable to write stdin: {s}", .{@errorName(err)}); @@ -925,17 +1167,12 @@ fn spawnChildAndCollect( } } - const term = try child.wait(); - const elapsed_ns = timer.read(); - return .{ .stdout = stdout_bytes, .stderr = stderr_bytes, .stdout_null = stdout_null, .stderr_null = stderr_null, - .term = term, - .elapsed_ns = elapsed_ns, - .peak_rss = child.resource_usage_statistics.getMaxRss() orelse 0, + .test_results = .{}, }; } @@ -966,7 +1203,7 @@ fn failForeign( exe: *CompileStep, ) error{ MakeFailed, MakeSkipped, OutOfMemory } { switch (self.stdio) { - .check => { + .check, .zig_test => { if (self.skip_foreign_checks) return error.MakeSkipped; @@ -987,7 +1224,7 @@ fn failForeign( fn hashStdIo(hh: *std.Build.Cache.HashHelper, stdio: StdIo) void { switch (stdio) { - .infer_from_args, .inherit => {}, + .infer_from_args, .inherit, .zig_test => {}, .check => |checks| for (checks.items) |check| { hh.add(@as(std.meta.Tag(StdIo.Check), check)); switch (check) { diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index 45aa635972..05c4faa52d 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -35,11 +35,27 @@ result_cached: bool, result_duration_ns: ?u64, /// 0 means unavailable or not reported. result_peak_rss: usize, +test_results: TestResults, /// The return addresss associated with creation of this step that can be useful /// to print along with debugging messages. debug_stack_trace: [n_debug_stack_frames]usize, +pub const TestResults = struct { + fail_count: u32 = 0, + skip_count: u32 = 0, + leak_count: u32 = 0, + test_count: u32 = 0, + + pub fn isSuccess(tr: TestResults) bool { + return tr.fail_count == 0 and tr.leak_count == 0; + } + + pub fn passCount(tr: TestResults) u32 { + return tr.test_count - tr.fail_count - tr.skip_count; + } +}; + pub const MakeFn = *const fn (self: *Step, prog_node: *std.Progress.Node) anyerror!void; const n_debug_stack_frames = 4; @@ -134,6 +150,7 @@ pub fn init(options: Options) Step { .result_cached = false, .result_duration_ns = null, .result_peak_rss = 0, + .test_results = .{}, }; } @@ -152,6 +169,10 @@ pub fn make(s: *Step, prog_node: *std.Progress.Node) error{ MakeFailed, MakeSkip }, }; + if (!s.test_results.isSuccess()) { + return error.MakeFailed; + } + if (s.max_rss != 0 and s.result_peak_rss > s.max_rss) { const msg = std.fmt.allocPrint(arena, "memory usage peaked at {d} bytes, exceeding the declared upper bound of {d}", .{ s.result_peak_rss, s.max_rss, @@ -346,9 +367,7 @@ pub fn evalZigProcess( s.result_cached = ebp_hdr.flags.cache_hit; result = try arena.dupe(u8, body[@sizeOf(EbpHdr)..]); }, - _ => { - // Unrecognized message. - }, + else => {}, // ignore other messages } stdout.discard(header_and_msg_len); } @@ -475,3 +494,11 @@ fn failWithCacheError(s: *Step, man: *const std.Build.Cache.Manifest, err: anyer const prefix = man.cache.prefixes()[pp.prefix].path orelse ""; return s.fail("{s}: {s}/{s}", .{ @errorName(err), prefix, pp.sub_path }); } + +pub fn writeManifest(s: *Step, man: *std.Build.Cache.Manifest) !void { + if (s.test_results.isSuccess()) { + man.writeManifest() catch |err| { + try s.addError("unable to write cache manifest: {s}", .{@errorName(err)}); + }; + } +} diff --git a/lib/std/Build/WriteFileStep.zig b/lib/std/Build/WriteFileStep.zig index 9b033e5ae2..dee79af5be 100644 --- a/lib/std/Build/WriteFileStep.zig +++ b/lib/std/Build/WriteFileStep.zig @@ -282,7 +282,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { }); } - try man.writeManifest(); + try step.writeManifest(&man); } const std = @import("../std.zig"); diff --git a/lib/std/zig/Client.zig b/lib/std/zig/Client.zig index a68c189e57..af4c29d37d 100644 --- a/lib/std/zig/Client.zig +++ b/lib/std/zig/Client.zig @@ -26,6 +26,13 @@ pub const Message = struct { /// swap. /// No body. hot_update, + /// Ask the test runner for metadata about all the unit tests that can + /// be run. Server will respond with a `test_metadata` message. + /// No body. + query_test_metadata, + /// Ask the test runner to run a particular test. + /// The message body is a u32 test index. + run_test, _, }; diff --git a/lib/std/zig/Server.zig b/lib/std/zig/Server.zig index d34b2193e9..3238f22043 100644 --- a/lib/std/zig/Server.zig +++ b/lib/std/zig/Server.zig @@ -1,3 +1,7 @@ +in: std.fs.File, +out: std.fs.File, +receive_fifo: std.fifo.LinearFifo(u8, .Dynamic), + pub const Message = struct { pub const Header = extern struct { tag: Tag, @@ -14,6 +18,11 @@ pub const Message = struct { progress, /// Body is a EmitBinPath. emit_bin_path, + /// Body is a TestMetadata + test_metadata, + /// Body is a TestResults + test_results, + _, }; @@ -26,6 +35,33 @@ pub const Message = struct { string_bytes_len: u32, }; + /// Trailing: + /// * name: [tests_len]u32 + /// - null-terminated string_bytes index + /// * async_frame_len: [tests_len]u32, + /// - 0 means not async + /// * expected_panic_msg: [tests_len]u32, + /// - null-terminated string_bytes index + /// - 0 means does not expect pani + /// * string_bytes: [string_bytes_len]u8, + pub const TestMetadata = extern struct { + string_bytes_len: u32, + tests_len: u32, + }; + + pub const TestResults = extern struct { + index: u32, + flags: Flags, + + pub const Flags = packed struct(u8) { + fail: bool, + skip: bool, + leak: bool, + + reserved: u5 = 0, + }; + }; + /// Trailing: /// * the file system path the emitted binary can be found pub const EmitBinPath = extern struct { @@ -37,3 +73,167 @@ pub const Message = struct { }; }; }; + +pub const Options = struct { + gpa: Allocator, + in: std.fs.File, + out: std.fs.File, + zig_version: []const u8, +}; + +pub fn init(options: Options) !Server { + var s: Server = .{ + .in = options.in, + .out = options.out, + .receive_fifo = std.fifo.LinearFifo(u8, .Dynamic).init(options.gpa), + }; + try s.serveStringMessage(.zig_version, options.zig_version); + return s; +} + +pub fn deinit(s: *Server) void { + s.receive_fifo.deinit(); + s.* = undefined; +} + +pub fn receiveMessage(s: *Server) !InMessage.Header { + const Header = InMessage.Header; + const fifo = &s.receive_fifo; + + while (true) { + const buf = fifo.readableSlice(0); + assert(fifo.readableLength() == buf.len); + if (buf.len >= @sizeOf(Header)) { + const header = @ptrCast(*align(1) const Header, buf[0..@sizeOf(Header)]); + + if (buf.len - @sizeOf(Header) >= header.bytes_len) { + const result = header.*; + fifo.discard(@sizeOf(Header)); + return result; + } else { + const needed = header.bytes_len - (buf.len - @sizeOf(Header)); + const write_buffer = try fifo.writableWithSize(needed); + const amt = try s.in.read(write_buffer); + fifo.update(amt); + continue; + } + } + + const write_buffer = try fifo.writableWithSize(256); + const amt = try s.in.read(write_buffer); + fifo.update(amt); + } +} + +pub fn receiveBody_u32(s: *Server) !u32 { + const fifo = &s.receive_fifo; + const buf = fifo.readableSlice(0); + const result = @ptrCast(*align(1) const u32, buf[0..4]).*; + fifo.discard(4); + return result; +} + +pub fn serveStringMessage(s: *Server, tag: OutMessage.Tag, msg: []const u8) !void { + return s.serveMessage(.{ + .tag = tag, + .bytes_len = @intCast(u32, msg.len), + }, &.{msg}); +} + +pub fn serveMessage( + s: *const Server, + header: OutMessage.Header, + bufs: []const []const u8, +) !void { + var iovecs: [10]std.os.iovec_const = undefined; + iovecs[0] = .{ + .iov_base = @ptrCast([*]const u8, &header), + .iov_len = @sizeOf(OutMessage.Header), + }; + for (bufs, iovecs[1 .. bufs.len + 1]) |buf, *iovec| { + iovec.* = .{ + .iov_base = buf.ptr, + .iov_len = buf.len, + }; + } + try s.out.writevAll(iovecs[0 .. bufs.len + 1]); +} + +pub fn serveEmitBinPath( + s: *Server, + fs_path: []const u8, + header: OutMessage.EmitBinPath, +) !void { + try s.serveMessage(.{ + .tag = .emit_bin_path, + .bytes_len = @intCast(u32, fs_path.len + @sizeOf(OutMessage.EmitBinPath)), + }, &.{ + std.mem.asBytes(&header), + fs_path, + }); +} + +pub fn serveTestResults( + s: *Server, + msg: OutMessage.TestResults, +) !void { + try s.serveMessage(.{ + .tag = .test_results, + .bytes_len = @intCast(u32, @sizeOf(OutMessage.TestResults)), + }, &.{ + std.mem.asBytes(&msg), + }); +} + +pub fn serveErrorBundle(s: *Server, error_bundle: std.zig.ErrorBundle) !void { + const eb_hdr: OutMessage.ErrorBundle = .{ + .extra_len = @intCast(u32, error_bundle.extra.len), + .string_bytes_len = @intCast(u32, error_bundle.string_bytes.len), + }; + const bytes_len = @sizeOf(OutMessage.ErrorBundle) + + 4 * error_bundle.extra.len + error_bundle.string_bytes.len; + try s.serveMessage(.{ + .tag = .error_bundle, + .bytes_len = @intCast(u32, bytes_len), + }, &.{ + std.mem.asBytes(&eb_hdr), + // TODO: implement @ptrCast between slices changing the length + std.mem.sliceAsBytes(error_bundle.extra), + error_bundle.string_bytes, + }); +} + +pub const TestMetadata = struct { + names: []const u32, + async_frame_sizes: []const u32, + expected_panic_msgs: []const u32, + string_bytes: []const u8, +}; + +pub fn serveTestMetadata(s: *Server, test_metadata: TestMetadata) !void { + const header: OutMessage.TestMetadata = .{ + .tests_len = @intCast(u32, test_metadata.names.len), + .string_bytes_len = @intCast(u32, test_metadata.string_bytes.len), + }; + const bytes_len = @sizeOf(OutMessage.TestMetadata) + + 3 * 4 * test_metadata.names.len + test_metadata.string_bytes.len; + return s.serveMessage(.{ + .tag = .test_metadata, + .bytes_len = @intCast(u32, bytes_len), + }, &.{ + std.mem.asBytes(&header), + // TODO: implement @ptrCast between slices changing the length + std.mem.sliceAsBytes(test_metadata.names), + std.mem.sliceAsBytes(test_metadata.async_frame_sizes), + std.mem.sliceAsBytes(test_metadata.expected_panic_msgs), + test_metadata.string_bytes, + }); +} + +const OutMessage = std.zig.Server.Message; +const InMessage = std.zig.Client.Message; + +const Server = @This(); +const std = @import("std"); +const Allocator = std.mem.Allocator; +const assert = std.debug.assert; diff --git a/lib/test_runner.zig b/lib/test_runner.zig index 5968fdaa54..47ababbc2c 100644 --- a/lib/test_runner.zig +++ b/lib/test_runner.zig @@ -8,14 +8,130 @@ pub const std_options = struct { }; var log_err_count: usize = 0; +var cmdline_buffer: [4096]u8 = undefined; +var fba = std.heap.FixedBufferAllocator.init(&cmdline_buffer); pub fn main() void { - if (builtin.zig_backend != .stage1 and - builtin.zig_backend != .stage2_llvm and - builtin.zig_backend != .stage2_c) + if (builtin.zig_backend == .stage2_wasm or + builtin.zig_backend == .stage2_x86_64 or + builtin.zig_backend == .stage2_aarch64) { - return main2() catch @panic("test failure"); + return mainSimple() catch @panic("test failure"); } + + const args = std.process.argsAlloc(fba.allocator()) catch + @panic("unable to parse command line args"); + + var listen = false; + + for (args[1..]) |arg| { + if (std.mem.eql(u8, arg, "--listen=-")) { + listen = true; + } else { + @panic("unrecognized command line argument"); + } + } + + if (listen) { + return mainServer(); + } else { + return mainTerminal(); + } +} + +fn mainServer() void { + return mainServerFallible() catch @panic("internal test runner failure"); +} + +fn mainServerFallible() !void { + var server = try std.zig.Server.init(.{ + .gpa = fba.allocator(), + .in = std.io.getStdIn(), + .out = std.io.getStdOut(), + .zig_version = builtin.zig_version_string, + }); + defer server.deinit(); + + while (true) { + const hdr = try server.receiveMessage(); + switch (hdr.tag) { + .exit => { + return std.process.exit(0); + }, + .query_test_metadata => { + std.testing.allocator_instance = .{}; + defer if (std.testing.allocator_instance.deinit()) { + @panic("internal test runner memory leak"); + }; + + var string_bytes: std.ArrayListUnmanaged(u8) = .{}; + defer string_bytes.deinit(std.testing.allocator); + try string_bytes.append(std.testing.allocator, 0); // Reserve 0 for null. + + const test_fns = builtin.test_functions; + const names = try std.testing.allocator.alloc(u32, test_fns.len); + defer std.testing.allocator.free(names); + const async_frame_sizes = try std.testing.allocator.alloc(u32, test_fns.len); + defer std.testing.allocator.free(async_frame_sizes); + const expected_panic_msgs = try std.testing.allocator.alloc(u32, test_fns.len); + defer std.testing.allocator.free(expected_panic_msgs); + + for (test_fns, names, async_frame_sizes, expected_panic_msgs) |test_fn, *name, *async_frame_size, *expected_panic_msg| { + name.* = @intCast(u32, string_bytes.items.len); + try string_bytes.ensureUnusedCapacity(std.testing.allocator, test_fn.name.len + 1); + string_bytes.appendSliceAssumeCapacity(test_fn.name); + string_bytes.appendAssumeCapacity(0); + + async_frame_size.* = @intCast(u32, test_fn.async_frame_size orelse 0); + expected_panic_msg.* = 0; + } + + try server.serveTestMetadata(.{ + .names = names, + .async_frame_sizes = async_frame_sizes, + .expected_panic_msgs = expected_panic_msgs, + .string_bytes = string_bytes.items, + }); + }, + + .run_test => { + std.testing.allocator_instance = .{}; + const index = try server.receiveBody_u32(); + const test_fn = builtin.test_functions[index]; + if (test_fn.async_frame_size != null) + @panic("TODO test runner implement async tests"); + var fail = false; + var skip = false; + var leak = false; + test_fn.func() catch |err| switch (err) { + error.SkipZigTest => skip = true, + else => { + fail = true; + if (@errorReturnTrace()) |trace| { + std.debug.dumpStackTrace(trace.*); + } + }, + }; + leak = std.testing.allocator_instance.deinit(); + try server.serveTestResults(.{ + .index = index, + .flags = .{ + .fail = fail, + .skip = skip, + .leak = leak, + }, + }); + }, + + else => { + std.debug.print("unsupported message: {x}", .{@enumToInt(hdr.tag)}); + std.process.exit(1); + }, + } + } +} + +fn mainTerminal() void { const test_fn_list = builtin.test_functions; var ok_count: usize = 0; var skip_count: usize = 0; @@ -118,51 +234,17 @@ pub fn log( } } -pub fn main2() anyerror!void { - var skipped: usize = 0; - var failed: usize = 0; - // Simpler main(), exercising fewer language features, so that stage2 can handle it. +/// Simpler main(), exercising fewer language features, so that +/// work-in-progress backends can handle it. +pub fn mainSimple() anyerror!void { + //const stderr = std.io.getStdErr(); for (builtin.test_functions) |test_fn| { test_fn.func() catch |err| { if (err != error.SkipZigTest) { - failed += 1; - } else { - skipped += 1; + //stderr.writeAll(test_fn.name) catch {}; + //stderr.writeAll("\n") catch {}; + return err; } }; } - if (builtin.zig_backend == .stage2_wasm or - builtin.zig_backend == .stage2_x86_64 or - builtin.zig_backend == .stage2_aarch64 or - builtin.zig_backend == .stage2_llvm or - builtin.zig_backend == .stage2_c) - { - const passed = builtin.test_functions.len - skipped - failed; - const stderr = std.io.getStdErr(); - writeInt(stderr, passed) catch {}; - stderr.writeAll(" passed; ") catch {}; - writeInt(stderr, skipped) catch {}; - stderr.writeAll(" skipped; ") catch {}; - writeInt(stderr, failed) catch {}; - stderr.writeAll(" failed.\n") catch {}; - } - if (failed != 0) { - return error.TestsFailed; - } -} - -fn writeInt(stderr: std.fs.File, int: usize) anyerror!void { - const base = 10; - var buf: [100]u8 = undefined; - var a: usize = int; - var index: usize = buf.len; - while (true) { - const digit = a % base; - index -= 1; - buf[index] = std.fmt.digitToChar(@intCast(u8, digit), .lower); - a /= base; - if (a == 0) break; - } - const slice = buf[index..]; - try stderr.writeAll(slice); } diff --git a/src/Server.zig b/src/Server.zig deleted file mode 100644 index a25dc93857..0000000000 --- a/src/Server.zig +++ /dev/null @@ -1,113 +0,0 @@ -in: std.fs.File, -out: std.fs.File, -receive_fifo: std.fifo.LinearFifo(u8, .Dynamic), - -pub const Options = struct { - gpa: Allocator, - in: std.fs.File, - out: std.fs.File, -}; - -pub fn init(options: Options) !Server { - var s: Server = .{ - .in = options.in, - .out = options.out, - .receive_fifo = std.fifo.LinearFifo(u8, .Dynamic).init(options.gpa), - }; - try s.serveStringMessage(.zig_version, build_options.version); - return s; -} - -pub fn deinit(s: *Server) void { - s.receive_fifo.deinit(); - s.* = undefined; -} - -pub fn receiveMessage(s: *Server) !InMessage.Header { - const Header = InMessage.Header; - const fifo = &s.receive_fifo; - - while (true) { - const buf = fifo.readableSlice(0); - assert(fifo.readableLength() == buf.len); - if (buf.len >= @sizeOf(Header)) { - const header = @ptrCast(*align(1) const Header, buf[0..@sizeOf(Header)]); - if (header.bytes_len != 0) - return error.InvalidClientMessage; - const result = header.*; - fifo.discard(@sizeOf(Header)); - return result; - } - - const write_buffer = try fifo.writableWithSize(256); - const amt = try s.in.read(write_buffer); - fifo.update(amt); - } -} - -pub fn serveStringMessage(s: *Server, tag: OutMessage.Tag, msg: []const u8) !void { - return s.serveMessage(.{ - .tag = tag, - .bytes_len = @intCast(u32, msg.len), - }, &.{msg}); -} - -pub fn serveMessage( - s: *const Server, - header: OutMessage.Header, - bufs: []const []const u8, -) !void { - var iovecs: [10]std.os.iovec_const = undefined; - iovecs[0] = .{ - .iov_base = @ptrCast([*]const u8, &header), - .iov_len = @sizeOf(OutMessage.Header), - }; - for (bufs, iovecs[1 .. bufs.len + 1]) |buf, *iovec| { - iovec.* = .{ - .iov_base = buf.ptr, - .iov_len = buf.len, - }; - } - try s.out.writevAll(iovecs[0 .. bufs.len + 1]); -} - -pub fn serveEmitBinPath( - s: *Server, - fs_path: []const u8, - header: std.zig.Server.Message.EmitBinPath, -) !void { - try s.serveMessage(.{ - .tag = .emit_bin_path, - .bytes_len = @intCast(u32, fs_path.len + @sizeOf(std.zig.Server.Message.EmitBinPath)), - }, &.{ - std.mem.asBytes(&header), - fs_path, - }); -} - -pub fn serveErrorBundle(s: *Server, error_bundle: std.zig.ErrorBundle) !void { - const eb_hdr: std.zig.Server.Message.ErrorBundle = .{ - .extra_len = @intCast(u32, error_bundle.extra.len), - .string_bytes_len = @intCast(u32, error_bundle.string_bytes.len), - }; - const bytes_len = @sizeOf(std.zig.Server.Message.ErrorBundle) + - 4 * error_bundle.extra.len + error_bundle.string_bytes.len; - try s.serveMessage(.{ - .tag = .error_bundle, - .bytes_len = @intCast(u32, bytes_len), - }, &.{ - std.mem.asBytes(&eb_hdr), - // TODO: implement @ptrCast between slices changing the length - std.mem.sliceAsBytes(error_bundle.extra), - error_bundle.string_bytes, - }); -} - -const OutMessage = std.zig.Server.Message; -const InMessage = std.zig.Client.Message; - -const Server = @This(); -const std = @import("std"); -const build_options = @import("build_options"); -const Allocator = std.mem.Allocator; -const assert = std.debug.assert; diff --git a/src/main.zig b/src/main.zig index e0283143d0..9602a5cd31 100644 --- a/src/main.zig +++ b/src/main.zig @@ -10,6 +10,7 @@ const ArrayList = std.ArrayList; const Ast = std.zig.Ast; const warn = std.log.warn; const ThreadPool = std.Thread.Pool; +const cleanExit = std.process.cleanExit; const tracy = @import("tracy.zig"); const Compilation = @import("Compilation.zig"); @@ -26,7 +27,7 @@ const target_util = @import("target.zig"); const crash_report = @import("crash_report.zig"); const Module = @import("Module.zig"); const AstGen = @import("AstGen.zig"); -const Server = @import("Server.zig"); +const Server = std.zig.Server; pub const std_options = struct { pub const wasiCwd = wasi_cwd; @@ -3545,6 +3546,7 @@ fn serve( .gpa = gpa, .in = in, .out = out, + .zig_version = build_options.version, }); defer server.deinit(); @@ -3656,8 +3658,8 @@ fn serve( ); } }, - _ => { - @panic("TODO unrecognized message from client"); + else => { + fatal("unrecognized message from client: 0x{x}", .{@enumToInt(hdr.tag)}); }, } } @@ -5624,19 +5626,6 @@ fn detectNativeTargetInfo(cross_target: std.zig.CrossTarget) !std.zig.system.Nat return std.zig.system.NativeTargetInfo.detect(cross_target); } -/// Indicate that we are now terminating with a successful exit code. -/// In debug builds, this is a no-op, so that the calling code's -/// cleanup mechanisms are tested and so that external tools that -/// check for resource leaks can be accurate. In release builds, this -/// calls exit(0), and does not return. -pub fn cleanExit() void { - if (builtin.mode == .Debug) { - return; - } else { - process.exit(0); - } -} - const usage_ast_check = \\Usage: zig ast-check [file] \\ diff --git a/src/objcopy.zig b/src/objcopy.zig index e821a94b59..c3305e8c04 100644 --- a/src/objcopy.zig +++ b/src/objcopy.zig @@ -8,8 +8,8 @@ const assert = std.debug.assert; const main = @import("main.zig"); const fatal = main.fatal; -const cleanExit = main.cleanExit; -const Server = @import("Server.zig"); +const Server = std.zig.Server; +const build_options = @import("build_options"); pub fn cmdObjCopy( gpa: Allocator, @@ -116,6 +116,7 @@ pub fn cmdObjCopy( .gpa = gpa, .in = std.io.getStdIn(), .out = std.io.getStdOut(), + .zig_version = build_options.version, }); defer server.deinit(); @@ -124,7 +125,7 @@ pub fn cmdObjCopy( const hdr = try server.receiveMessage(); switch (hdr.tag) { .exit => { - return cleanExit(); + return std.process.cleanExit(); }, .update => { if (seen_update) { @@ -144,7 +145,7 @@ pub fn cmdObjCopy( } } } - return cleanExit(); + return std.process.cleanExit(); } const usage = diff --git a/test/link/common_symbols/build.zig b/test/link/common_symbols/build.zig index e3c302f0f7..a8c276d1f3 100644 --- a/test/link/common_symbols/build.zig +++ b/test/link/common_symbols/build.zig @@ -24,5 +24,5 @@ fn add(b: *std.Build, test_step: *std.Build.Step, optimize: std.builtin.Optimize }); test_exe.linkLibrary(lib_a); - test_step.dependOn(&test_exe.step); + test_step.dependOn(&test_exe.run().step); } diff --git a/test/link/common_symbols_alignment/build.zig b/test/link/common_symbols_alignment/build.zig index 7d1d813447..83548f2d8a 100644 --- a/test/link/common_symbols_alignment/build.zig +++ b/test/link/common_symbols_alignment/build.zig @@ -24,5 +24,5 @@ fn add(b: *std.Build, test_step: *std.Build.Step, optimize: std.builtin.Optimize }); test_exe.linkLibrary(lib_a); - test_step.dependOn(&test_exe.step); + test_step.dependOn(&test_exe.run().step); } diff --git a/test/link/interdependent_static_c_libs/build.zig b/test/link/interdependent_static_c_libs/build.zig index c4118c1ca4..0d06410a79 100644 --- a/test/link/interdependent_static_c_libs/build.zig +++ b/test/link/interdependent_static_c_libs/build.zig @@ -35,5 +35,5 @@ fn add(b: *std.Build, test_step: *std.Build.Step, optimize: std.builtin.Optimize test_exe.linkLibrary(lib_b); test_exe.addIncludePath("."); - test_step.dependOn(&test_exe.step); + test_step.dependOn(&test_exe.run().step); } diff --git a/test/link/macho/tls/build.zig b/test/link/macho/tls/build.zig index e91f40bd59..5981fea194 100644 --- a/test/link/macho/tls/build.zig +++ b/test/link/macho/tls/build.zig @@ -32,5 +32,8 @@ fn add(b: *std.Build, test_step: *std.Build.Step, optimize: std.builtin.Optimize test_exe.linkLibrary(lib); test_exe.linkLibC(); - test_step.dependOn(&test_exe.step); + const run = test_exe.run(); + run.skip_foreign_checks = true; + + test_step.dependOn(&run.step); } diff --git a/test/src/Cases.zig b/test/src/Cases.zig index 5007939b14..6affe968d1 100644 --- a/test/src/Cases.zig +++ b/test/src/Cases.zig @@ -547,15 +547,12 @@ pub fn lowerToBuildSteps( parent_step.dependOn(&artifact.step); }, .Execution => |expected_stdout| { - if (case.is_test) { - parent_step.dependOn(&artifact.step); - } else { - const run = b.addRunArtifact(artifact); - run.skip_foreign_checks = true; + const run = b.addRunArtifact(artifact); + run.skip_foreign_checks = true; + if (!case.is_test) { run.expectStdOutEqual(expected_stdout); - - parent_step.dependOn(&run.step); } + parent_step.dependOn(&run.step); }, .Header => @panic("TODO"), } diff --git a/test/standalone/emit_asm_and_bin/build.zig b/test/standalone/emit_asm_and_bin/build.zig index 9bdfadc33d..594bf6552e 100644 --- a/test/standalone/emit_asm_and_bin/build.zig +++ b/test/standalone/emit_asm_and_bin/build.zig @@ -11,5 +11,5 @@ pub fn build(b: *std.Build) void { main.emit_asm = .{ .emit_to = b.pathFromRoot("main.s") }; main.emit_bin = .{ .emit_to = b.pathFromRoot("main") }; - test_step.dependOn(&main.step); + test_step.dependOn(&main.run().step); } diff --git a/test/standalone/global_linkage/build.zig b/test/standalone/global_linkage/build.zig index 2cf1b248a5..ddcddd612a 100644 --- a/test/standalone/global_linkage/build.zig +++ b/test/standalone/global_linkage/build.zig @@ -28,5 +28,5 @@ pub fn build(b: *std.Build) void { main.linkLibrary(obj1); main.linkLibrary(obj2); - test_step.dependOn(&main.step); + test_step.dependOn(&main.run().step); } diff --git a/test/standalone/issue_13970/build.zig b/test/standalone/issue_13970/build.zig index bbaaac5886..1eb8a5a121 100644 --- a/test/standalone/issue_13970/build.zig +++ b/test/standalone/issue_13970/build.zig @@ -17,7 +17,7 @@ pub fn build(b: *std.Build) void { test2.setTestRunner("src/main.zig"); test3.setTestRunner("src/main.zig"); - test_step.dependOn(&test1.step); - test_step.dependOn(&test2.step); - test_step.dependOn(&test3.step); + test_step.dependOn(&test1.run().step); + test_step.dependOn(&test2.run().step); + test_step.dependOn(&test3.run().step); } diff --git a/test/standalone/main_pkg_path/build.zig b/test/standalone/main_pkg_path/build.zig index cd49573692..a4dd301c43 100644 --- a/test/standalone/main_pkg_path/build.zig +++ b/test/standalone/main_pkg_path/build.zig @@ -9,5 +9,5 @@ pub fn build(b: *std.Build) void { }); test_exe.setMainPkgPath("."); - test_step.dependOn(&test_exe.step); + test_step.dependOn(&test_exe.run().step); } diff --git a/test/standalone/options/build.zig b/test/standalone/options/build.zig index 3f1e823359..5e894102a7 100644 --- a/test/standalone/options/build.zig +++ b/test/standalone/options/build.zig @@ -20,5 +20,5 @@ pub fn build(b: *std.Build) void { options.addOption([]const u8, "string", b.option([]const u8, "string", "s").?); const test_step = b.step("test", "Run unit tests"); - test_step.dependOn(&main.step); + test_step.dependOn(&main.run().step); } diff --git a/test/standalone/pie/build.zig b/test/standalone/pie/build.zig index 615111b6c2..546b4a922f 100644 --- a/test/standalone/pie/build.zig +++ b/test/standalone/pie/build.zig @@ -17,5 +17,5 @@ pub fn build(b: *std.Build) void { }); main.pie = true; - test_step.dependOn(&main.step); + test_step.dependOn(&main.run().step); } diff --git a/test/standalone/static_c_lib/build.zig b/test/standalone/static_c_lib/build.zig index 5996c978d8..794b813b75 100644 --- a/test/standalone/static_c_lib/build.zig +++ b/test/standalone/static_c_lib/build.zig @@ -21,5 +21,5 @@ pub fn build(b: *std.Build) void { test_exe.linkLibrary(foo); test_exe.addIncludePath("."); - test_step.dependOn(&test_exe.step); + test_step.dependOn(&test_exe.run().step); } diff --git a/test/standalone/test_runner_module_imports/build.zig b/test/standalone/test_runner_module_imports/build.zig index 973365e495..73c5536dc6 100644 --- a/test/standalone/test_runner_module_imports/build.zig +++ b/test/standalone/test_runner_module_imports/build.zig @@ -15,5 +15,5 @@ pub fn build(b: *std.Build) void { t.addModule("module2", module2); const test_step = b.step("test", "Run unit tests"); - test_step.dependOn(&t.step); + test_step.dependOn(&t.run().step); } diff --git a/test/standalone/test_runner_path/build.zig b/test/standalone/test_runner_path/build.zig index 40aad42b21..ce5b668054 100644 --- a/test/standalone/test_runner_path/build.zig +++ b/test/standalone/test_runner_path/build.zig @@ -8,7 +8,6 @@ pub fn build(b: *std.Build) void { const test_exe = b.addTest(.{ .root_source_file = .{ .path = "test.zig" }, - .kind = .test_exe, }); test_exe.test_runner = "test_runner.zig"; diff --git a/test/standalone/use_alias/build.zig b/test/standalone/use_alias/build.zig index 947db0828d..db47fe6692 100644 --- a/test/standalone/use_alias/build.zig +++ b/test/standalone/use_alias/build.zig @@ -12,5 +12,5 @@ pub fn build(b: *std.Build) void { }); main.addIncludePath("."); - test_step.dependOn(&main.step); + test_step.dependOn(&main.run().step); } diff --git a/test/tests.zig b/test/tests.zig index 9ba9639e2a..76ea537bb2 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -596,7 +596,7 @@ pub fn addStandaloneTests( }); if (case.link_libc) exe.linkLibC(); - step.dependOn(&exe.step); + step.dependOn(&exe.run().step); } } } @@ -981,14 +981,6 @@ pub fn addModuleTests(b: *std.Build, options: ModuleTestOptions) *Step { }); const single_threaded_txt = if (test_target.single_threaded) "single" else "multi"; const backend_txt = if (test_target.backend) |backend| @tagName(backend) else "default"; - these_tests.setNamePrefix(b.fmt("{s}-{s}-{s}-{s}-{s}-{s} ", .{ - options.name, - triple_prefix, - @tagName(test_target.optimize_mode), - libc_prefix, - single_threaded_txt, - backend_txt, - })); these_tests.single_threaded = test_target.single_threaded; these_tests.setFilter(options.test_filter); if (test_target.link_libc) { @@ -1014,7 +1006,18 @@ pub fn addModuleTests(b: *std.Build, options: ModuleTestOptions) *Step { }, }; - step.dependOn(&these_tests.step); + const run = these_tests.run(); + run.skip_foreign_checks = true; + run.setName(b.fmt("run test {s}-{s}-{s}-{s}-{s}-{s}", .{ + options.name, + triple_prefix, + @tagName(test_target.optimize_mode), + libc_prefix, + single_threaded_txt, + backend_txt, + })); + + step.dependOn(&run.step); } return step; } @@ -1053,7 +1056,9 @@ pub fn addCAbiTests(b: *std.Build, skip_non_native: bool, skip_release: bool) *S @tagName(optimize_mode), })); - step.dependOn(&test_step.step); + const run = test_step.run(); + run.skip_foreign_checks = true; + step.dependOn(&run.step); } } return step;