tests: avoid skipping native tests

Make the test targets use options that match the actual options of
CompileStep. This makes the code more straightforward, and ends up
making fewer tests incorrectly skipped. For example, now the CI runner
on Windows will no longer skip self-hosted x86_64 backend tests.

build.zig

@@ -85,9 +85,7 @@ pub fn build(b: *std.Build) !void {
     const skip_cross_glibc = b.option(bool, "skip-cross-glibc", "Main test suite skips builds that require cross glibc") orelse false;
     const skip_libc = b.option(bool, "skip-libc", "Main test suite skips tests that link libc") orelse false;
     const skip_single_threaded = b.option(bool, "skip-single-threaded", "Main test suite skips tests that are single-threaded") orelse false;
-    const skip_stage1 = b.option(bool, "skip-stage1", "Main test suite skips stage1 compile error tests") orelse false;
     const skip_run_translated_c = b.option(bool, "skip-run-translated-c", "Main test suite skips run-translated-c tests") orelse false;
-    const skip_stage2_tests = b.option(bool, "skip-stage2-tests", "Main test suite skips self-hosted compiler tests") orelse false;
 
     const only_install_lib_files = b.option(bool, "lib-files-only", "Only install library files") orelse false;
 
@@ -187,9 +185,7 @@ pub fn build(b: *std.Build) !void {
     const compile_step = b.step("compile", "Build the self-hosted compiler");
     compile_step.dependOn(&exe.step);
 
-    if (!skip_stage2_tests) {
+    test_step.dependOn(&exe.step);
-        test_step.dependOn(&exe.step);
-    }
 
     exe.single_threaded = single_threaded;
 
@@ -360,7 +356,6 @@ pub fn build(b: *std.Build) !void {
     test_cases_options.addOption(bool, "enable_link_snapshots", enable_link_snapshots);
     test_cases_options.addOption(bool, "skip_non_native", skip_non_native);
     test_cases_options.addOption(bool, "skip_cross_glibc", skip_cross_glibc);
-    test_cases_options.addOption(bool, "skip_stage1", skip_stage1);
     test_cases_options.addOption(bool, "have_llvm", enable_llvm);
     test_cases_options.addOption(bool, "llvm_has_m68k", llvm_has_m68k);
     test_cases_options.addOption(bool, "llvm_has_csky", llvm_has_csky);
@@ -416,7 +411,7 @@ pub fn build(b: *std.Build) !void {
 
     const test_cases_step = b.step("test-cases", "Run the main compiler test cases");
     try tests.addCases(b, test_cases_step, test_filter, check_case_exe);
-    if (!skip_stage2_tests) test_step.dependOn(test_cases_step);
+    test_step.dependOn(test_cases_step);
 
     test_step.dependOn(tests.addModuleTests(b, .{
         .test_filter = test_filter,
@@ -428,8 +423,6 @@ pub fn build(b: *std.Build) !void {
         .skip_non_native = skip_non_native,
         .skip_cross_glibc = skip_cross_glibc,
         .skip_libc = skip_libc,
-        .skip_stage1 = skip_stage1,
-        .skip_stage2 = skip_stage2_tests,
         .max_rss = 1 * 1024 * 1024 * 1024,
     }));
 
@@ -443,8 +436,6 @@ pub fn build(b: *std.Build) !void {
         .skip_non_native = skip_non_native,
         .skip_cross_glibc = skip_cross_glibc,
         .skip_libc = true,
-        .skip_stage1 = skip_stage1,
-        .skip_stage2 = true, // TODO get all these passing
     }));
 
     test_step.dependOn(tests.addModuleTests(b, .{
@@ -457,8 +448,6 @@ pub fn build(b: *std.Build) !void {
         .skip_non_native = skip_non_native,
         .skip_cross_glibc = skip_cross_glibc,
         .skip_libc = true,
-        .skip_stage1 = skip_stage1,
-        .skip_stage2 = true, // TODO get all these passing
     }));
 
     test_step.dependOn(tests.addCompareOutputTests(b, test_filter, optimization_modes));
@@ -466,11 +455,11 @@ pub fn build(b: *std.Build) !void {
         b,
         optimization_modes,
         enable_macos_sdk,
-        skip_stage2_tests,
+        false,
         enable_symlinks_windows,
     ));
     test_step.dependOn(tests.addCAbiTests(b, skip_non_native, skip_release));
-    test_step.dependOn(tests.addLinkTests(b, enable_macos_sdk, skip_stage2_tests, enable_symlinks_windows));
+    test_step.dependOn(tests.addLinkTests(b, enable_macos_sdk, false, enable_symlinks_windows));
     test_step.dependOn(tests.addStackTraceTests(b, test_filter, optimization_modes));
     test_step.dependOn(tests.addCliTests(b));
     test_step.dependOn(tests.addAssembleAndLinkTests(b, test_filter, optimization_modes));
@@ -489,8 +478,6 @@ pub fn build(b: *std.Build) !void {
         .skip_non_native = skip_non_native,
         .skip_cross_glibc = skip_cross_glibc,
         .skip_libc = skip_libc,
-        .skip_stage1 = skip_stage1,
-        .skip_stage2 = true, // TODO get all these passing
         // I observed a value of 3398275072 on my M1, and multiplied by 1.1 to
         // get this amount:
         .max_rss = 3738102579,

test/tests.zig

@@ -22,12 +22,12 @@ pub const CompareOutputContext = @import("src/CompareOutput.zig");
 pub const StackTracesContext = @import("src/StackTrace.zig");
 
 const TestTarget = struct {
-    target: CrossTarget = @as(CrossTarget, .{}),
+    target: CrossTarget = .{},
     optimize_mode: std.builtin.OptimizeMode = .Debug,
-    link_libc: bool = false,
+    link_libc: ?bool = null,
-    single_threaded: bool = false,
+    single_threaded: ?bool = null,
-    disable_native: bool = false,
+    use_llvm: ?bool = null,
-    backend: ?std.builtin.CompilerBackend = null,
+    use_lld: ?bool = null,
 };
 
 const test_targets = blk: {
@@ -43,13 +43,47 @@ const test_targets = blk: {
         .{
             .single_threaded = true,
         },
+        .{
+            .optimize_mode = .ReleaseFast,
+        },
+        .{
+            .link_libc = true,
+            .optimize_mode = .ReleaseFast,
+        },
+        .{
+            .optimize_mode = .ReleaseFast,
+            .single_threaded = true,
+        },
+
+        .{
+            .optimize_mode = .ReleaseSafe,
+        },
+        .{
+            .link_libc = true,
+            .optimize_mode = .ReleaseSafe,
+        },
+        .{
+            .optimize_mode = .ReleaseSafe,
+            .single_threaded = true,
+        },
+
+        .{
+            .optimize_mode = .ReleaseSmall,
+        },
+        .{
+            .link_libc = true,
+            .optimize_mode = .ReleaseSmall,
+        },
+        .{
+            .optimize_mode = .ReleaseSmall,
+            .single_threaded = true,
+        },
 
         .{
             .target = .{
                 .ofmt = .c,
             },
             .link_libc = true,
-            .backend = .stage2_c,
         },
         .{
             .target = .{
@@ -57,22 +91,24 @@ const test_targets = blk: {
                 .os_tag = .linux,
                 .abi = .none,
             },
-            .backend = .stage2_x86_64,
+            .use_llvm = false,
+            .use_lld = false,
         },
         .{
             .target = .{
                 .cpu_arch = .aarch64,
                 .os_tag = .linux,
             },
-            .backend = .stage2_aarch64,
+            .use_llvm = false,
+            .use_lld = false,
         },
         .{
             .target = .{
                 .cpu_arch = .wasm32,
                 .os_tag = .wasi,
             },
-            .single_threaded = true,
+            .use_llvm = false,
-            .backend = .stage2_wasm,
+            .use_lld = false,
         },
         // https://github.com/ziglang/zig/issues/13623
         //.{
@@ -80,7 +116,8 @@ const test_targets = blk: {
         //        .cpu_arch = .arm,
         //        .os_tag = .linux,
         //    },
-        //    .backend = .stage2_arm,
+        //    .use_llvm = false,
+        //    .use_lld = false,
         //},
         // https://github.com/ziglang/zig/issues/13623
         //.{
@@ -88,7 +125,8 @@ const test_targets = blk: {
         //        .arch_os_abi = "arm-linux-none",
         //        .cpu_features = "generic+v8a",
         //    }) catch unreachable,
-        //    .backend = .stage2_arm,
+        //    .use_llvm = false,
+        //    .use_lld = false,
         //},
         .{
             .target = .{
@@ -96,7 +134,8 @@ const test_targets = blk: {
                 .os_tag = .macos,
                 .abi = .none,
             },
-            .backend = .stage2_aarch64,
+            .use_llvm = false,
+            .use_lld = false,
         },
         .{
             .target = .{
@@ -104,7 +143,8 @@ const test_targets = blk: {
                 .os_tag = .macos,
                 .abi = .none,
             },
-            .backend = .stage2_x86_64,
+            .use_llvm = false,
+            .use_lld = false,
         },
         .{
             .target = .{
@@ -112,7 +152,8 @@ const test_targets = blk: {
                 .os_tag = .windows,
                 .abi = .gnu,
             },
-            .backend = .stage2_x86_64,
+            .use_llvm = false,
+            .use_lld = false,
         },
 
         .{
@@ -121,7 +162,6 @@ const test_targets = blk: {
                 .os_tag = .wasi,
             },
             .link_libc = false,
-            .single_threaded = true,
         },
         .{
             .target = .{
@@ -129,7 +169,6 @@ const test_targets = blk: {
                 .os_tag = .wasi,
             },
             .link_libc = true,
-            .single_threaded = true,
         },
 
         .{
@@ -413,43 +452,6 @@ const test_targets = blk: {
             },
             .link_libc = true,
         },
-
-        // Do the release tests last because they take a long time
-        .{
-            .optimize_mode = .ReleaseFast,
-        },
-        .{
-            .link_libc = true,
-            .optimize_mode = .ReleaseFast,
-        },
-        .{
-            .optimize_mode = .ReleaseFast,
-            .single_threaded = true,
-        },
-
-        .{
-            .optimize_mode = .ReleaseSafe,
-        },
-        .{
-            .link_libc = true,
-            .optimize_mode = .ReleaseSafe,
-        },
-        .{
-            .optimize_mode = .ReleaseSafe,
-            .single_threaded = true,
-        },
-
-        .{
-            .optimize_mode = .ReleaseSmall,
-        },
-        .{
-            .link_libc = true,
-            .optimize_mode = .ReleaseSmall,
-        },
-        .{
-            .optimize_mode = .ReleaseSmall,
-            .single_threaded = true,
-        },
     };
 };
 
@@ -913,8 +915,6 @@ const ModuleTestOptions = struct {
     skip_non_native: bool,
     skip_cross_glibc: bool,
     skip_libc: bool,
-    skip_stage1: bool,
-    skip_stage2: bool,
     max_rss: usize = 0,
 };
 
@@ -922,49 +922,41 @@ pub fn addModuleTests(b: *std.Build, options: ModuleTestOptions) *Step {
     const step = b.step(b.fmt("test-{s}", .{options.name}), options.desc);
 
     for (test_targets) |test_target| {
-        if (options.skip_non_native and !test_target.target.isNative())
+        const is_native = test_target.target.isNative() or
+            (test_target.target.getOsTag() == builtin.os.tag and
+            test_target.target.getCpuArch() == builtin.cpu.arch);
+
+        if (options.skip_non_native and !is_native)
             continue;
 
-        if (options.skip_cross_glibc and test_target.target.isGnuLibC() and test_target.link_libc)
+        if (options.skip_cross_glibc and test_target.target.isGnuLibC() and test_target.link_libc == true)
             continue;
 
-        if (options.skip_libc and test_target.link_libc)
+        if (options.skip_libc and test_target.link_libc == true)
             continue;
 
-        if (test_target.link_libc and test_target.target.getOs().requiresLibC()) {
+        if (options.skip_single_threaded and test_target.single_threaded == true)
-            // This would be a redundant test.
-            continue;
-        }
-
-        if (options.skip_single_threaded and test_target.single_threaded)
             continue;
 
-        if (test_target.disable_native and
+        // TODO get compiler-rt tests passing for self-hosted backends.
-            test_target.target.getOsTag() == builtin.os.tag and
+        if (test_target.use_llvm == false and mem.eql(u8, options.name, "compiler-rt"))
-            test_target.target.getCpuArch() == builtin.cpu.arch)
-        {
             continue;
-        }
 
-        if (test_target.backend) |backend| switch (backend) {
+        // TODO get universal-libc tests passing for self-hosted backends.
-            .stage1 => if (options.skip_stage1) continue,
+        if (test_target.use_llvm == false and mem.eql(u8, options.name, "universal-libc"))
-            .stage2_llvm => {},
+            continue;
-            else => if (options.skip_stage2) continue,
+
-        };
+        // TODO get std lib tests passing for self-hosted backends.
+        if (test_target.use_llvm == false and mem.eql(u8, options.name, "std"))
+            continue;
 
         const want_this_mode = for (options.optimize_modes) |m| {
             if (m == test_target.optimize_mode) break true;
         } else false;
         if (!want_this_mode) continue;
 
-        const libc_prefix = if (test_target.target.getOs().requiresLibC())
+        const libc_suffix = if (test_target.link_libc == true) "-libc" else "";
-            ""
+        const triple_txt = test_target.target.zigTriple(b.allocator) catch @panic("OOM");
-        else if (test_target.link_libc)
-            "c"
-        else
-            "bare";
-
-        const triple_prefix = test_target.target.zigTriple(b.allocator) catch @panic("OOM");
 
         // wasm32-wasi builds need more RAM, idk why
         const max_rss = if (test_target.target.getOs().tag == .wasi)
@@ -978,42 +970,33 @@ pub fn addModuleTests(b: *std.Build, options: ModuleTestOptions) *Step {
             .target = test_target.target,
             .max_rss = max_rss,
             .filter = options.test_filter,
+            .link_libc = test_target.link_libc,
+            .single_threaded = test_target.single_threaded,
+            .use_llvm = test_target.use_llvm,
+            .use_lld = test_target.use_lld,
         });
-        const single_threaded_txt = if (test_target.single_threaded) "single" else "multi";
+        const single_threaded_suffix = if (test_target.single_threaded == true) "-single" else "";
-        const backend_txt = if (test_target.backend) |backend| @tagName(backend) else "default";
+        const backend_suffix = if (test_target.use_llvm == true)
-        these_tests.single_threaded = test_target.single_threaded;
+            "-llvm"
-        if (test_target.link_libc) {
+        else if (test_target.target.ofmt == std.Target.ObjectFormat.c)
-            these_tests.linkSystemLibrary("c");
+            "-cbe"
-        }
+        else if (test_target.use_llvm == false)
+            "-selfhosted"
+        else
+            "";
+
         these_tests.overrideZigLibDir("lib");
         these_tests.addIncludePath("test");
-        if (test_target.backend) |backend| switch (backend) {
-            .stage1 => {
-                @panic("stage1 testing requested");
-            },
-            .stage2_llvm => {
-                these_tests.use_llvm = true;
-            },
-            .stage2_c => {
-                these_tests.use_llvm = false;
-            },
-            else => {
-                these_tests.use_llvm = false;
-                // TODO: force self-hosted linkers to avoid LLD creeping in
-                // until the auto-select mechanism deems them worthy
-                these_tests.use_lld = false;
-            },
-        };
 
         const run = b.addRunArtifact(these_tests);
         run.skip_foreign_checks = true;
-        run.setName(b.fmt("run test {s}-{s}-{s}-{s}-{s}-{s}", .{
+        run.setName(b.fmt("run test {s}-{s}-{s}{s}{s}{s}", .{
             options.name,
-            triple_prefix,
+            triple_txt,
             @tagName(test_target.optimize_mode),
-            libc_prefix,
+            libc_suffix,
-            single_threaded_txt,
+            single_threaded_suffix,
-            backend_txt,
+            backend_suffix,
         }));
 
         step.dependOn(&run.step);