From 55c91fc42d048a18c724bdaffe3709bef8aa143e Mon Sep 17 00:00:00 2001 From: dweiller <4678790+dweiller@users.noreplay.github.com> Date: Tue, 25 Oct 2022 16:52:29 +1100 Subject: [PATCH 1/4] stage2: add test_runner_path for user provided test runner --- src/Compilation.zig | 16 ++++++++++------ src/main.zig | 5 +++++ 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index 958aac5e1b..e8e15b75ad 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -994,6 +994,7 @@ pub const InitOptions = struct { reference_trace: ?u32 = null, test_filter: ?[]const u8 = null, test_name_prefix: ?[]const u8 = null, + test_runner_path: ?[]const u8 = null, subsystem: ?std.Target.SubSystem = null, /// WASI-only. Type of WASI execution model ("command" or "reactor"). wasi_exec_model: ?std.builtin.WasiExecModel = null, @@ -1578,12 +1579,15 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { errdefer std_pkg.destroy(gpa); const root_pkg = if (options.is_test) root_pkg: { - const test_pkg = try Package.createWithDir( - gpa, - options.zig_lib_directory, - null, - "test_runner.zig", - ); + const test_pkg = if (options.test_runner_path) |test_runner| + try Package.create(gpa, null, test_runner) + else + try Package.createWithDir( + gpa, + options.zig_lib_directory, + null, + "test_runner.zig", + ); errdefer test_pkg.destroy(gpa); break :root_pkg test_pkg; diff --git a/src/main.zig b/src/main.zig index d0edbbed6d..4d861851c2 100644 --- a/src/main.zig +++ b/src/main.zig @@ -503,6 +503,7 @@ const usage_build_generic = \\ --test-cmd-bin Appends test binary path to test cmd args \\ --test-evented-io Runs the test in evented I/O mode \\ --test-no-exec Compiles test binary without running it + \\ --test-runner [path] Specify a custom test runner \\ \\Debug Options (Zig Compiler Development): \\ -ftime-report Print timing diagnostics @@ -726,6 +727,7 @@ fn buildOutputType( var runtime_args_start: ?usize = null; var test_filter: ?[]const u8 = null; var test_name_prefix: ?[]const u8 = null; + var test_runner_path: ?[]const u8 = null; var override_local_cache_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_LOCAL_CACHE_DIR"); var override_global_cache_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_GLOBAL_CACHE_DIR"); var override_lib_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_LIB_DIR"); @@ -1043,6 +1045,8 @@ fn buildOutputType( test_filter = args_iter.nextOrFatal(); } else if (mem.eql(u8, arg, "--test-name-prefix")) { test_name_prefix = args_iter.nextOrFatal(); + } else if (mem.eql(u8, arg, "--test-runner")) { + test_runner_path = args_iter.nextOrFatal(); } else if (mem.eql(u8, arg, "--test-cmd")) { try test_exec_args.append(args_iter.nextOrFatal()); } else if (mem.eql(u8, arg, "--cache-dir")) { @@ -2943,6 +2947,7 @@ fn buildOutputType( .test_evented_io = test_evented_io, .test_filter = test_filter, .test_name_prefix = test_name_prefix, + .test_runner_path = test_runner_path, .disable_lld_caching = !have_enable_cache, .subsystem = subsystem, .wasi_exec_model = wasi_exec_model, From 33a401dd603ec76805ebb5c933fe41f1dfeabd5b Mon Sep 17 00:00:00 2001 From: dweiller <4678790+dweiller@users.noreplay.github.com> Date: Tue, 1 Nov 2022 11:57:43 +1100 Subject: [PATCH 2/4] std.build: add support for custom test runner --- lib/std/build.zig | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/std/build.zig b/lib/std/build.zig index 3f903a0b12..3def982017 100644 --- a/lib/std/build.zig +++ b/lib/std/build.zig @@ -1509,6 +1509,7 @@ pub const LibExeObjStep = struct { name_prefix: []const u8, filter: ?[]const u8, test_evented_io: bool = false, + test_runner: ?[]const u8, code_model: std.builtin.CodeModel = .default, wasi_exec_model: ?std.builtin.WasiExecModel = null, /// Symbols to be exported when compiling to wasm @@ -1772,6 +1773,7 @@ pub const LibExeObjStep = struct { .exec_cmd_args = null, .name_prefix = "", .filter = null, + .test_runner = null, .disable_stack_probing = false, .disable_sanitize_c = false, .sanitize_thread = false, @@ -2670,6 +2672,11 @@ pub const LibExeObjStep = struct { try zig_args.append(self.name_prefix); } + if (self.test_runner) |test_runner| { + try zig_args.append("--test-runner"); + try zig_args.append(builder.pathFromRoot(test_runner)); + } + for (builder.debug_log_scopes) |log_scope| { try zig_args.append("--debug-log"); try zig_args.append(log_scope); From ae0f12885bd0f6f436269a51634f56c3fe7fddc8 Mon Sep 17 00:00:00 2001 From: dweiller <4678790+dweiller@users.noreplay.github.com> Date: Wed, 2 Nov 2022 22:41:20 +1100 Subject: [PATCH 3/4] test: add test_runner_path test --- test/standalone.zig | 1 + test/standalone/test_runner_path/build.zig | 11 ++++ test/standalone/test_runner_path/test.zig | 9 ++++ .../test_runner_path/test_runner.zig | 52 +++++++++++++++++++ 4 files changed, 73 insertions(+) create mode 100644 test/standalone/test_runner_path/build.zig create mode 100644 test/standalone/test_runner_path/test.zig create mode 100644 test/standalone/test_runner_path/test_runner.zig diff --git a/test/standalone.zig b/test/standalone.zig index c945fe6bb6..dc90eed011 100644 --- a/test/standalone.zig +++ b/test/standalone.zig @@ -15,6 +15,7 @@ pub fn addCases(cases: *tests.StandaloneContext) void { cases.add("test/standalone/main_return_error/error_u8_non_zero.zig"); cases.add("test/standalone/noreturn_call/inline.zig"); cases.add("test/standalone/noreturn_call/as_arg.zig"); + cases.addBuildFile("test/standalone/test_runner_path/build.zig", .{ .requires_stage2 = true }); cases.addBuildFile("test/standalone/main_pkg_path/build.zig", .{}); cases.addBuildFile("test/standalone/shared_library/build.zig", .{}); cases.addBuildFile("test/standalone/mix_o_files/build.zig", .{}); diff --git a/test/standalone/test_runner_path/build.zig b/test/standalone/test_runner_path/build.zig new file mode 100644 index 0000000000..738cac9783 --- /dev/null +++ b/test/standalone/test_runner_path/build.zig @@ -0,0 +1,11 @@ +const Builder = @import("std").build.Builder; + +pub fn build(b: *Builder) void { + const test_exe = b.addTestExe("test", "test.zig"); + test_exe.test_runner = "test_runner.zig"; + + const test_run = test_exe.run(); + + const test_step = b.step("test", "Test the program"); + test_step.dependOn(&test_run.step); +} diff --git a/test/standalone/test_runner_path/test.zig b/test/standalone/test_runner_path/test.zig new file mode 100644 index 0000000000..52f3c693bc --- /dev/null +++ b/test/standalone/test_runner_path/test.zig @@ -0,0 +1,9 @@ +test "test runner path pass" {} + +test "test runner path fail" { + return error.Fail; +} + +test "test runner path skip" { + return error.SkipZigTest; +} diff --git a/test/standalone/test_runner_path/test_runner.zig b/test/standalone/test_runner_path/test_runner.zig new file mode 100644 index 0000000000..f49ff55cae --- /dev/null +++ b/test/standalone/test_runner_path/test_runner.zig @@ -0,0 +1,52 @@ +const std = @import("std"); +const io = std.io; +const builtin = @import("builtin"); + +pub const io_mode: io.Mode = builtin.test_io_mode; + +pub fn main() void { + const test_fn_list = builtin.test_functions; + var ok_count: usize = 0; + var skip_count: usize = 0; + var fail_count: usize = 0; + + var async_frame_buffer: []align(std.Target.stack_align) u8 = undefined; + // TODO this is on the next line (using `undefined` above) because otherwise zig incorrectly + // ignores the alignment of the slice. + async_frame_buffer = &[_]u8{}; + + for (test_fn_list) |test_fn| { + const result = if (test_fn.async_frame_size) |size| switch (io_mode) { + .evented => blk: { + if (async_frame_buffer.len < size) { + std.heap.page_allocator.free(async_frame_buffer); + async_frame_buffer = std.heap.page_allocator.alignedAlloc(u8, std.Target.stack_align, size) catch @panic("out of memory"); + } + const casted_fn = @ptrCast(fn () callconv(.Async) anyerror!void, test_fn.func); + break :blk await @asyncCall(async_frame_buffer, {}, casted_fn, .{}); + }, + .blocking => { + skip_count += 1; + continue; + }, + } else test_fn.func(); + if (result) |_| { + ok_count += 1; + } else |err| switch (err) { + error.SkipZigTest => { + skip_count += 1; + }, + else => { + fail_count += 1; + }, + } + } + if (ok_count == test_fn_list.len) { + std.debug.print("All {d} tests passed.\n", .{ok_count}); + } else { + std.debug.print("{d} passed; {d} skipped; {d} failed.\n", .{ ok_count, skip_count, fail_count }); + } + if (ok_count != 1 or skip_count != 1 or fail_count != 1) { + std.process.exit(1); + } +} From a1b123bccbc2ae50442fcc1544b0da0595401038 Mon Sep 17 00:00:00 2001 From: dweiller <4678790+dweiller@users.noreplay.github.com> Date: Sun, 13 Nov 2022 13:47:42 +1100 Subject: [PATCH 4/4] std.build: add setter for LibObjExeStep test runner path --- lib/std/build.zig | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/std/build.zig b/lib/std/build.zig index 3def982017..67e3cf1203 100644 --- a/lib/std/build.zig +++ b/lib/std/build.zig @@ -2207,6 +2207,11 @@ pub const LibExeObjStep = struct { self.filter = if (text) |t| self.builder.dupe(t) else null; } + pub fn setTestRunner(self: *LibExeObjStep, path: ?[]const u8) void { + assert(self.kind == .@"test" or self.kind == .test_exe); + self.test_runner = if (path) |p| self.builder.dupePath(p) else null; + } + /// Handy when you have many C/C++ source files and want them all to have the same flags. pub fn addCSourceFiles(self: *LibExeObjStep, files: []const []const u8, flags: []const []const u8) void { const c_source_files = self.builder.allocator.create(CSourceFiles) catch unreachable;