re-enable CLI tests

CLI tests are now ported over to the new std.Build API and thus work
properly with concurrency.

 * add `std.Build.addCheckFile` for creating a
   `std.Build.CheckFileStep`.
 * add `std.Build.makeTempPath`. This function is intended to be called
   in the `configure` phase only. It returns an absolute directory path,
   which is potentially going to be a source of API breakage in the
   future, so keep that in mind when using this function.
 * add `std.Build.CheckFileStep.setName`.
 * `std.Build.CheckFileStep`: better error message when reading the
   input file fails.
 * `std.Build.RunStep`: add a `has_side_effects` flag for when you need
   to override the autodetection.
 * `std.Build.RunStep`: add the ability to obtain a FileSource for the
   directory that contains the written files.
 * `std.Build.WriteFileStep`: add a way to write bytes to an arbitrary
   path - absolute or relative to the package root. Be careful with this
   because it updates source files. This should not be used as part of
   the normal build process, but as a utility occasionally run by a
   developer with intent to modify source files and then commit those
   changes to version control. A file added this way is not available
   with `getFileSource`.

build.zig

@@ -463,7 +463,7 @@ pub fn build(b: *std.Build) !void {
     //test_step.dependOn(tests.addCAbiTests(b, skip_non_native, skip_release));
     //test_step.dependOn(tests.addLinkTests(b, test_filter, optimization_modes, enable_macos_sdk, skip_stage2_tests, enable_symlinks_windows));
     test_step.dependOn(tests.addStackTraceTests(b, test_filter, optimization_modes));
-    //test_step.dependOn(tests.addCliTests(b, test_filter, optimization_modes));
+    test_step.dependOn(tests.addCliTests(b, test_filter, optimization_modes));
     //test_step.dependOn(tests.addAssembleAndLinkTests(b, test_filter, optimization_modes));
     test_step.dependOn(tests.addTranslateCTests(b, test_filter));
     if (!skip_run_translated_c) {

lib/std/Build.zig

@@ -699,10 +699,8 @@ pub fn addWriteFile(self: *Build, file_path: []const u8, data: []const u8) *Writ
     return write_file_step;
 }
 
-pub fn addWriteFiles(self: *Build) *WriteFileStep {
+pub fn addWriteFiles(b: *Build) *WriteFileStep {
-    const write_file_step = self.allocator.create(WriteFileStep) catch @panic("OOM");
+    return WriteFileStep.create(b);
-    write_file_step.* = WriteFileStep.init(self);
-    return write_file_step;
 }
 
 pub fn addRemoveDirTree(self: *Build, dir_path: []const u8) *RemoveDirStep {
@@ -1239,6 +1237,14 @@ pub fn addInstallDirectory(self: *Build, options: InstallDirectoryOptions) *Inst
     return install_step;
 }
 
+pub fn addCheckFile(
+    b: *Build,
+    file_source: FileSource,
+    options: CheckFileStep.Options,
+) *CheckFileStep {
+    return CheckFileStep.create(b, file_source, options);
+}
+
 pub fn pushInstalledFile(self: *Build, dir: InstallDir, dest_rel_path: []const u8) void {
     const file = InstalledFile{
         .dir = dir,
@@ -1713,6 +1719,36 @@ pub fn serializeCpu(allocator: Allocator, cpu: std.Target.Cpu) ![]const u8 {
     }
 }
 
+/// This function is intended to be called in the `configure` phase only.
+/// It returns an absolute directory path, which is potentially going to be a
+/// source of API breakage in the future, so keep that in mind when using this
+/// function.
+pub fn makeTempPath(b: *Build) []const u8 {
+    const rand_int = std.crypto.random.int(u64);
+    const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ hex64(rand_int);
+    const result_path = b.cache_root.join(b.allocator, &.{tmp_dir_sub_path}) catch @panic("OOM");
+    fs.cwd().makePath(result_path) catch |err| {
+        std.debug.print("unable to make tmp path '{s}': {s}\n", .{
+            result_path, @errorName(err),
+        });
+    };
+    return result_path;
+}
+
+/// There are a few copies of this function in miscellaneous places. Would be nice to find
+/// a home for them.
+fn hex64(x: u64) [16]u8 {
+    const hex_charset = "0123456789abcdef";
+    var result: [16]u8 = undefined;
+    var i: usize = 0;
+    while (i < 8) : (i += 1) {
+        const byte = @truncate(u8, x >> @intCast(u6, 8 * i));
+        result[i * 2 + 0] = hex_charset[byte >> 4];
+        result[i * 2 + 1] = hex_charset[byte & 15];
+    }
+    return result;
+}
+
 test {
     _ = CheckFileStep;
     _ = CheckObjectStep;

lib/std/Build/CheckFileStep.zig

@@ -12,13 +12,17 @@ expected_matches: []const []const u8,
 source: std.Build.FileSource,
 max_bytes: usize = 20 * 1024 * 1024,
 
+pub const Options = struct {
+    expected_matches: []const []const u8,
+};
+
 pub fn create(
     owner: *std.Build,
     source: std.Build.FileSource,
-    expected_matches: []const []const u8,
+    options: Options,
 ) *CheckFileStep {
     const self = owner.allocator.create(CheckFileStep) catch @panic("OOM");
-    self.* = CheckFileStep{
+    self.* = .{
         .step = Step.init(.{
             .id = .check_file,
             .name = "CheckFile",
@@ -26,19 +30,27 @@ pub fn create(
             .makeFn = make,
         }),
         .source = source.dupe(owner),
-        .expected_matches = owner.dupeStrings(expected_matches),
+        .expected_matches = owner.dupeStrings(options.expected_matches),
     };
     self.source.addStepDependencies(&self.step);
     return self;
 }
 
+pub fn setName(self: *CheckFileStep, name: []const u8) void {
+    self.step.name = name;
+}
+
 fn make(step: *Step, prog_node: *std.Progress.Node) !void {
     _ = prog_node;
     const b = step.owner;
     const self = @fieldParentPtr(CheckFileStep, "step", step);
 
     const src_path = self.source.getPath(b);
-    const contents = try fs.cwd().readFileAlloc(b.allocator, src_path, self.max_bytes);
+    const contents = fs.cwd().readFileAlloc(b.allocator, src_path, self.max_bytes) catch |err| {
+        return step.fail("unable to read '{s}': {s}", .{
+            src_path, @errorName(err),
+        });
+    };
 
     for (self.expected_matches) |expected_match| {
         if (mem.indexOf(u8, contents, expected_match) == null) {

lib/std/Build/RunStep.zig

@@ -70,6 +70,8 @@ max_stdio_size: usize = 10 * 1024 * 1024,
 captured_stdout: ?*Output = null,
 captured_stderr: ?*Output = null,
 
+has_side_effects: bool = false,
+
 pub const StdIo = union(enum) {
     /// Whether the RunStep has side-effects will be determined by whether or not one
     /// of the args is an output file (added with `addOutputFileArg`).
@@ -103,12 +105,14 @@ pub const StdIo = union(enum) {
 pub const Arg = union(enum) {
     artifact: *CompileStep,
     file_source: std.Build.FileSource,
+    directory_source: std.Build.FileSource,
     bytes: []u8,
     output: *Output,
 };
 
 pub const Output = struct {
     generated_file: std.Build.GeneratedFile,
+    prefix: []const u8,
     basename: []const u8,
 };
 
@@ -142,10 +146,19 @@ pub fn addArtifactArg(self: *RunStep, artifact: *CompileStep) void {
 /// run, and returns a FileSource which can be used as inputs to other APIs
 /// throughout the build system.
 pub fn addOutputFileArg(rs: *RunStep, basename: []const u8) std.Build.FileSource {
+    return addPrefixedOutputFileArg(rs, "", basename);
+}
+
+pub fn addPrefixedOutputFileArg(
+    rs: *RunStep,
+    prefix: []const u8,
+    basename: []const u8,
+) std.Build.FileSource {
     const b = rs.step.owner;
 
     const output = b.allocator.create(Output) catch @panic("OOM");
     output.* = .{
+        .prefix = prefix,
         .basename = basename,
         .generated_file = .{ .step = &rs.step },
     };
@@ -159,14 +172,21 @@ pub fn addOutputFileArg(rs: *RunStep, basename: []const u8) std.Build.FileSource
 }
 
 pub fn addFileSourceArg(self: *RunStep, file_source: std.Build.FileSource) void {
-    self.argv.append(Arg{
+    self.argv.append(.{
         .file_source = file_source.dupe(self.step.owner),
     }) catch @panic("OOM");
     file_source.addStepDependencies(&self.step);
 }
 
+pub fn addDirectorySourceArg(self: *RunStep, directory_source: std.Build.FileSource) void {
+    self.argv.append(.{
+        .directory_source = directory_source.dupe(self.step.owner),
+    }) catch @panic("OOM");
+    directory_source.addStepDependencies(&self.step);
+}
+
 pub fn addArg(self: *RunStep, arg: []const u8) void {
-    self.argv.append(Arg{ .bytes = self.step.owner.dupe(arg) }) catch @panic("OOM");
+    self.argv.append(.{ .bytes = self.step.owner.dupe(arg) }) catch @panic("OOM");
 }
 
 pub fn addArgs(self: *RunStep, args: []const []const u8) void {
@@ -274,6 +294,7 @@ pub fn captureStdErr(self: *RunStep) std.Build.FileSource {
 
     const output = self.step.owner.allocator.create(Output) catch @panic("OOM");
     output.* = .{
+        .prefix = "",
         .basename = "stderr",
         .generated_file = .{ .step = &self.step },
     };
@@ -288,6 +309,7 @@ pub fn captureStdOut(self: *RunStep) *std.Build.GeneratedFile {
 
     const output = self.step.owner.allocator.create(Output) catch @panic("OOM");
     output.* = .{
+        .prefix = "",
         .basename = "stdout",
         .generated_file = .{ .step = &self.step },
     };
@@ -297,6 +319,7 @@ pub fn captureStdOut(self: *RunStep) *std.Build.GeneratedFile {
 
 /// Returns whether the RunStep has side effects *other than* updating the output arguments.
 fn hasSideEffects(self: RunStep) bool {
+    if (self.has_side_effects) return true;
     return switch (self.stdio) {
         .infer_from_args => !self.hasAnyOutputArgs(),
         .inherit => true,
@@ -373,6 +396,11 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {
                 try argv_list.append(file_path);
                 _ = try man.addFile(file_path, null);
             },
+            .directory_source => |file| {
+                const file_path = file.getPath(b);
+                try argv_list.append(file_path);
+                man.hash.addBytes(file_path);
+            },
             .artifact => |artifact| {
                 if (artifact.target.isWindows()) {
                     // On Windows we don't have rpaths so we have to add .dll search paths to PATH
@@ -386,6 +414,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {
                 _ = try man.addFile(file_path, null);
             },
             .output => |output| {
+                man.hash.addBytes(output.prefix);
                 man.hash.addBytes(output.basename);
                 // Add a placeholder into the argument list because we need the
                 // manifest hash to be updated with all arguments before the
@@ -456,7 +485,11 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {
         };
         const output_path = try b.cache_root.join(arena, &output_components);
         placeholder.output.generated_file.path = output_path;
-        argv_list.items[placeholder.index] = output_path;
+        const cli_arg = if (placeholder.output.prefix.len == 0)
+            output_path
+        else
+            b.fmt("{s}{s}", .{ placeholder.output.prefix, output_path });
+        argv_list.items[placeholder.index] = cli_arg;
     }
 
     try runCommand(self, argv_list.items, has_side_effects, &digest);

lib/std/Build/TranslateCStep.zig

@@ -72,7 +72,11 @@ pub fn addIncludeDir(self: *TranslateCStep, include_dir: []const u8) void {
 }
 
 pub fn addCheckFile(self: *TranslateCStep, expected_matches: []const []const u8) *CheckFileStep {
-    return CheckFileStep.create(self.step.owner, .{ .generated = &self.output_file }, self.step.owner.dupeStrings(expected_matches));
+    return CheckFileStep.create(
+        self.step.owner,
+        .{ .generated = &self.output_file },
+        .{ .expected_matches = expected_matches },
+    );
 }
 
 /// If the value is omitted, it is set to 1.

lib/std/Build/WriteFileStep.zig

@@ -14,6 +14,7 @@ step: Step,
 /// GeneratedFile field.
 files: std.ArrayListUnmanaged(*File),
 output_source_files: std.ArrayListUnmanaged(OutputSourceFile),
+generated_directory: std.Build.GeneratedFile,
 
 pub const base_id = .write_file;
 
@@ -33,8 +34,9 @@ pub const Contents = union(enum) {
     copy: std.Build.FileSource,
 };
 
-pub fn init(owner: *std.Build) WriteFileStep {
+pub fn create(owner: *std.Build) *WriteFileStep {
-    return .{
+    const wf = owner.allocator.create(WriteFileStep) catch @panic("OOM");
+    wf.* = .{
         .step = Step.init(.{
             .id = .write_file,
             .name = "WriteFile",
@@ -43,7 +45,9 @@ pub fn init(owner: *std.Build) WriteFileStep {
         }),
         .files = .{},
         .output_source_files = .{},
+        .generated_directory = .{ .step = &wf.step },
     };
+    return wf;
 }
 
 pub fn add(wf: *WriteFileStep, sub_path: []const u8, bytes: []const u8) void {
@@ -95,6 +99,20 @@ pub fn addCopyFileToSource(wf: *WriteFileStep, source: std.Build.FileSource, sub
     }) catch @panic("OOM");
 }
 
+/// A path relative to the package root.
+/// Be careful with this because it updates source files. This should not be
+/// used as part of the normal build process, but as a utility occasionally
+/// run by a developer with intent to modify source files and then commit
+/// those changes to version control.
+/// A file added this way is not available with `getFileSource`.
+pub fn addBytesToSource(wf: *WriteFileStep, bytes: []const u8, sub_path: []const u8) void {
+    const b = wf.step.owner;
+    wf.output_source_files.append(b.allocator, .{
+        .contents = .{ .bytes = bytes },
+        .sub_path = sub_path,
+    }) catch @panic("OOM");
+}
+
 /// Gets a file source for the given sub_path. If the file does not exist, returns `null`.
 pub fn getFileSource(wf: *WriteFileStep, sub_path: []const u8) ?std.Build.FileSource {
     for (wf.files.items) |file| {
@@ -105,6 +123,12 @@ pub fn getFileSource(wf: *WriteFileStep, sub_path: []const u8) ?std.Build.FileSo
     return null;
 }
 
+/// Returns a `FileSource` representing the base directory that contains all the
+/// files from this `WriteFileStep`.
+pub fn getDirectorySource(wf: *WriteFileStep) std.Build.FileSource {
+    return .{ .generated = &wf.generated_directory };
+}
+
 fn maybeUpdateName(wf: *WriteFileStep) void {
     if (wf.files.items.len == 1) {
         // First time adding a file; update name.
@@ -193,12 +217,15 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void {
                 "o", &digest, file.sub_path,
             });
         }
+        wf.generated_directory.path = try b.cache_root.join(b.allocator, &.{ "o", &digest });
         return;
     }
 
     const digest = man.final();
     const cache_path = "o" ++ fs.path.sep_str ++ digest;
 
+    wf.generated_directory.path = try b.cache_root.join(b.allocator, &.{ "o", &digest });
+
     var cache_dir = b.cache_root.handle.makeOpenPath(cache_path, .{}) catch |err| {
         return step.fail("unable to make path '{}{s}': {s}", .{
             b.cache_root, cache_path, @errorName(err),

test/cli.zig

@@ -1,195 +0,0 @@
-const std = @import("std");
-const builtin = @import("builtin");
-const testing = std.testing;
-const process = std.process;
-const fs = std.fs;
-const ChildProcess = std.ChildProcess;
-
-var a: std.mem.Allocator = undefined;
-
-pub fn main() !void {
-    var gpa = std.heap.GeneralPurposeAllocator(.{}){};
-    defer _ = gpa.deinit();
-    var arena = std.heap.ArenaAllocator.init(gpa.allocator());
-    defer arena.deinit();
-
-    a = arena.allocator();
-    var arg_it = try process.argsWithAllocator(a);
-
-    // skip my own exe name
-    _ = arg_it.skip();
-
-    const zig_exe_rel = arg_it.next() orelse {
-        std.debug.print("Expected first argument to be path to zig compiler\n", .{});
-        return error.InvalidArgs;
-    };
-    const cache_root = arg_it.next() orelse {
-        std.debug.print("Expected second argument to be cache root directory path\n", .{});
-        return error.InvalidArgs;
-    };
-    const zig_exe = try fs.path.resolve(a, &[_][]const u8{zig_exe_rel});
-
-    const dir_path = try fs.path.join(a, &[_][]const u8{ cache_root, "clitest" });
-    defer fs.cwd().deleteTree(dir_path) catch {};
-
-    const TestFn = fn ([]const u8, []const u8) anyerror!void;
-    const Test = struct {
-        func: TestFn,
-        name: []const u8,
-    };
-    const tests = [_]Test{
-        .{ .func = testZigInitLib, .name = "zig init-lib" },
-        .{ .func = testZigInitExe, .name = "zig init-exe" },
-        .{ .func = testGodboltApi, .name = "godbolt API" },
-        .{ .func = testMissingOutputPath, .name = "missing output path" },
-        .{ .func = testZigFmt, .name = "zig fmt" },
-    };
-    inline for (tests) |t| {
-        try fs.cwd().deleteTree(dir_path);
-        try fs.cwd().makeDir(dir_path);
-        t.func(zig_exe, dir_path) catch |err| {
-            std.debug.print("test '{s}' failed: {s}\n", .{
-                t.name, @errorName(err),
-            });
-            return err;
-        };
-    }
-}
-
-fn printCmd(cwd: []const u8, argv: []const []const u8) void {
-    std.debug.print("cd {s} && ", .{cwd});
-    for (argv) |arg| {
-        std.debug.print("{s} ", .{arg});
-    }
-    std.debug.print("\n", .{});
-}
-
-fn exec(cwd: []const u8, expect_0: bool, argv: []const []const u8) !ChildProcess.ExecResult {
-    const max_output_size = 100 * 1024;
-    const result = ChildProcess.exec(.{
-        .allocator = a,
-        .argv = argv,
-        .cwd = cwd,
-        .max_output_bytes = max_output_size,
-    }) catch |err| {
-        std.debug.print("The following command failed:\n", .{});
-        printCmd(cwd, argv);
-        return err;
-    };
-    switch (result.term) {
-        .Exited => |code| {
-            if ((code != 0) == expect_0) {
-                std.debug.print("The following command exited with error code {}:\n", .{code});
-                printCmd(cwd, argv);
-                std.debug.print("stderr:\n{s}\n", .{result.stderr});
-                return error.CommandFailed;
-            }
-        },
-        else => {
-            std.debug.print("The following command terminated unexpectedly:\n", .{});
-            printCmd(cwd, argv);
-            std.debug.print("stderr:\n{s}\n", .{result.stderr});
-            return error.CommandFailed;
-        },
-    }
-    return result;
-}
-
-fn testZigInitLib(zig_exe: []const u8, dir_path: []const u8) !void {
-    _ = try exec(dir_path, true, &[_][]const u8{ zig_exe, "init-lib" });
-    const test_result = try exec(dir_path, true, &[_][]const u8{ zig_exe, "build", "test" });
-    try testing.expectStringEndsWith(test_result.stderr, "All 1 tests passed.\n");
-}
-
-fn testZigInitExe(zig_exe: []const u8, dir_path: []const u8) !void {
-    _ = try exec(dir_path, true, &[_][]const u8{ zig_exe, "init-exe" });
-    const run_result = try exec(dir_path, true, &[_][]const u8{ zig_exe, "build", "run" });
-    try testing.expectEqualStrings("All your codebase are belong to us.\n", run_result.stderr);
-    try testing.expectEqualStrings("Run `zig build test` to run the tests.\n", run_result.stdout);
-}
-
-fn testGodboltApi(zig_exe: []const u8, dir_path: []const u8) anyerror!void {
-    if (builtin.os.tag != .linux or builtin.cpu.arch != .x86_64) return;
-
-    const example_zig_path = try fs.path.join(a, &[_][]const u8{ dir_path, "example.zig" });
-    const example_s_path = try fs.path.join(a, &[_][]const u8{ dir_path, "example.s" });
-
-    try fs.cwd().writeFile(example_zig_path,
-        \\// Type your code here, or load an example.
-        \\export fn square(num: i32) i32 {
-        \\    return num * num;
-        \\}
-        \\extern fn zig_panic() noreturn;
-        \\pub fn panic(msg: []const u8, error_return_trace: ?*@import("std").builtin.StackTrace, _: ?usize) noreturn {
-        \\    _ = msg;
-        \\    _ = error_return_trace;
-        \\    zig_panic();
-        \\}
-    );
-
-    var args = std.ArrayList([]const u8).init(a);
-    try args.appendSlice(&[_][]const u8{
-        zig_exe,          "build-obj",
-        "--cache-dir",    dir_path,
-        "--name",         "example",
-        "-fno-emit-bin",  "-fno-emit-h",
-        "-fstrip",        "-OReleaseFast",
-        example_zig_path,
-    });
-
-    const emit_asm_arg = try std.fmt.allocPrint(a, "-femit-asm={s}", .{example_s_path});
-    try args.append(emit_asm_arg);
-
-    _ = try exec(dir_path, true, args.items);
-
-    const out_asm = try std.fs.cwd().readFileAlloc(a, example_s_path, std.math.maxInt(usize));
-    try testing.expect(std.mem.indexOf(u8, out_asm, "square:") != null);
-    try testing.expect(std.mem.indexOf(u8, out_asm, "mov\teax, edi") != null);
-    try testing.expect(std.mem.indexOf(u8, out_asm, "imul\teax, edi") != null);
-}
-
-fn testMissingOutputPath(zig_exe: []const u8, dir_path: []const u8) !void {
-    _ = try exec(dir_path, true, &[_][]const u8{ zig_exe, "init-exe" });
-    const output_path = try fs.path.join(a, &[_][]const u8{ "does", "not", "exist", "foo.exe" });
-    const output_arg = try std.fmt.allocPrint(a, "-femit-bin={s}", .{output_path});
-    const source_path = try fs.path.join(a, &[_][]const u8{ "src", "main.zig" });
-    const result = try exec(dir_path, false, &[_][]const u8{ zig_exe, "build-exe", source_path, output_arg });
-    const s = std.fs.path.sep_str;
-    const expected: []const u8 = "error: unable to open output directory 'does" ++ s ++ "not" ++ s ++ "exist': FileNotFound\n";
-    try testing.expectEqualStrings(expected, result.stderr);
-}
-
-fn testZigFmt(zig_exe: []const u8, dir_path: []const u8) !void {
-    _ = try exec(dir_path, true, &[_][]const u8{ zig_exe, "init-exe" });
-
-    const unformatted_code = "    // no reason for indent";
-
-    const fmt1_zig_path = try fs.path.join(a, &[_][]const u8{ dir_path, "fmt1.zig" });
-    try fs.cwd().writeFile(fmt1_zig_path, unformatted_code);
-
-    const run_result1 = try exec(dir_path, true, &[_][]const u8{ zig_exe, "fmt", fmt1_zig_path });
-    // stderr should be file path + \n
-    try testing.expect(std.mem.startsWith(u8, run_result1.stdout, fmt1_zig_path));
-    try testing.expect(run_result1.stdout.len == fmt1_zig_path.len + 1 and run_result1.stdout[run_result1.stdout.len - 1] == '\n');
-
-    const fmt2_zig_path = try fs.path.join(a, &[_][]const u8{ dir_path, "fmt2.zig" });
-    try fs.cwd().writeFile(fmt2_zig_path, unformatted_code);
-
-    const run_result2 = try exec(dir_path, true, &[_][]const u8{ zig_exe, "fmt", dir_path });
-    // running it on the dir, only the new file should be changed
-    try testing.expect(std.mem.startsWith(u8, run_result2.stdout, fmt2_zig_path));
-    try testing.expect(run_result2.stdout.len == fmt2_zig_path.len + 1 and run_result2.stdout[run_result2.stdout.len - 1] == '\n');
-
-    const run_result3 = try exec(dir_path, true, &[_][]const u8{ zig_exe, "fmt", dir_path });
-    // both files have been formatted, nothing should change now
-    try testing.expect(run_result3.stdout.len == 0);
-
-    // Check UTF-16 decoding
-    const fmt4_zig_path = try fs.path.join(a, &[_][]const u8{ dir_path, "fmt4.zig" });
-    var unformatted_code_utf16 = "\xff\xfe \x00 \x00 \x00 \x00/\x00/\x00 \x00n\x00o\x00 \x00r\x00e\x00a\x00s\x00o\x00n\x00";
-    try fs.cwd().writeFile(fmt4_zig_path, unformatted_code_utf16);
-
-    const run_result4 = try exec(dir_path, true, &[_][]const u8{ zig_exe, "fmt", dir_path });
-    try testing.expect(std.mem.startsWith(u8, run_result4.stdout, fmt4_zig_path));
-    try testing.expect(run_result4.stdout.len == fmt4_zig_path.len + 1 and run_result4.stdout[run_result4.stdout.len - 1] == '\n');
-}

test/tests.zig

@@ -635,19 +635,189 @@ pub fn addCliTests(b: *std.Build, test_filter: ?[]const u8, optimize_modes: []co
     _ = optimize_modes;
     const step = b.step("test-cli", "Test the command line interface");
 
-    const exe = b.addExecutable(.{
+    {
-        .name = "test-cli",
+        // Test `zig init-lib`.
-        .root_source_file = .{ .path = "test/cli.zig" },
+        const tmp_path = b.makeTempPath();
-        .target = .{},
+        const init_lib = b.addSystemCommand(&.{ b.zig_exe, "init-lib" });
-        .optimize = .Debug,
+        init_lib.cwd = tmp_path;
-    });
+        init_lib.setName("zig init-lib");
-    const run_cmd = exe.run();
+        init_lib.expectStdOutEqual("");
-    run_cmd.addArgs(&[_][]const u8{
+        init_lib.expectStdErrEqual(
-        fs.realpathAlloc(b.allocator, b.zig_exe) catch @panic("OOM"),
+            \\info: Created build.zig
-        b.pathFromRoot(b.cache_root.path orelse "."),
+            \\info: Created src/main.zig
-    });
+            \\info: Next, try `zig build --help` or `zig build test`
+            \\
+        );
+
+        const run_test = b.addSystemCommand(&.{ b.zig_exe, "build", "test" });
+        run_test.cwd = tmp_path;
+        run_test.setName("zig build test");
+        run_test.expectStdOutEqual("");
+        run_test.step.dependOn(&init_lib.step);
+
+        const cleanup = b.addRemoveDirTree(tmp_path);
+        cleanup.step.dependOn(&run_test.step);
+
+        step.dependOn(&cleanup.step);
+    }
+
+    {
+        // Test `zig init-exe`.
+        const tmp_path = b.makeTempPath();
+        const init_exe = b.addSystemCommand(&.{ b.zig_exe, "init-exe" });
+        init_exe.cwd = tmp_path;
+        init_exe.setName("zig init-exe");
+        init_exe.expectStdOutEqual("");
+        init_exe.expectStdErrEqual(
+            \\info: Created build.zig
+            \\info: Created src/main.zig
+            \\info: Next, try `zig build --help` or `zig build run`
+            \\
+        );
+
+        // Test missing output path.
+        const s = std.fs.path.sep_str;
+        const bad_out_arg = "-femit-bin=does" ++ s ++ "not" ++ s ++ "exist" ++ s ++ "foo.exe";
+        const ok_src_arg = "src" ++ s ++ "main.zig";
+        const expected = "error: unable to open output directory 'does" ++ s ++ "not" ++ s ++ "exist': FileNotFound\n";
+        const run_bad = b.addSystemCommand(&.{ b.zig_exe, "build-exe", ok_src_arg, bad_out_arg });
+        run_bad.setName("zig build-exe error message for bad -femit-bin arg");
+        run_bad.expectExitCode(1);
+        run_bad.expectStdErrEqual(expected);
+        run_bad.expectStdOutEqual("");
+        run_bad.step.dependOn(&init_exe.step);
+
+        const run_test = b.addSystemCommand(&.{ b.zig_exe, "build", "test" });
+        run_test.cwd = tmp_path;
+        run_test.setName("zig build test");
+        run_test.expectStdOutEqual("");
+        run_test.step.dependOn(&init_exe.step);
+
+        const run_run = b.addSystemCommand(&.{ b.zig_exe, "build", "run" });
+        run_run.cwd = tmp_path;
+        run_run.setName("zig build run");
+        run_run.expectStdOutEqual("Run `zig build test` to run the tests.\n");
+        run_run.expectStdErrEqual("All your codebase are belong to us.\n");
+        run_run.step.dependOn(&init_exe.step);
+
+        const cleanup = b.addRemoveDirTree(tmp_path);
+        cleanup.step.dependOn(&run_test.step);
+        cleanup.step.dependOn(&run_run.step);
+        cleanup.step.dependOn(&run_bad.step);
+
+        step.dependOn(&cleanup.step);
+    }
+
+    // Test Godbolt API
+    if (builtin.os.tag == .linux and builtin.cpu.arch == .x86_64) {
+        const tmp_path = b.makeTempPath();
+
+        const writefile = b.addWriteFile("example.zig",
+            \\// Type your code here, or load an example.
+            \\export fn square(num: i32) i32 {
+            \\    return num * num;
+            \\}
+            \\extern fn zig_panic() noreturn;
+            \\pub fn panic(msg: []const u8, error_return_trace: ?*@import("std").builtin.StackTrace, _: ?usize) noreturn {
+            \\    _ = msg;
+            \\    _ = error_return_trace;
+            \\    zig_panic();
+            \\}
+        );
+
+        // This is intended to be the exact CLI usage used by godbolt.org.
+        const run = b.addSystemCommand(&.{
+            b.zig_exe,       "build-obj",
+            "--cache-dir",   tmp_path,
+            "--name",        "example",
+            "-fno-emit-bin", "-fno-emit-h",
+            "-fstrip",       "-OReleaseFast",
+        });
+        run.addFileSourceArg(writefile.getFileSource("example.zig").?);
+        const example_s = run.addPrefixedOutputFileArg("-femit-asm=", "example.s");
+
+        const checkfile = b.addCheckFile(example_s, .{
+            .expected_matches = &.{
+                "square:",
+                "mov\teax, edi",
+                "imul\teax, edi",
+            },
+        });
+        checkfile.setName("check godbolt.org CLI usage generating valid asm");
+
+        const cleanup = b.addRemoveDirTree(tmp_path);
+        cleanup.step.dependOn(&checkfile.step);
+
+        step.dependOn(&cleanup.step);
+    }
+
+    {
+        // Test `zig fmt`.
+        // This test must use a temporary directory rather than a cache
+        // directory because this test will be mutating the files. The cache
+        // system relies on cache directories being mutated only by their
+        // owners.
+        const tmp_path = b.makeTempPath();
+        const unformatted_code = "    // no reason for indent";
+        const s = std.fs.path.sep_str;
+
+        var dir = fs.cwd().openDir(tmp_path, .{}) catch @panic("unhandled");
+        defer dir.close();
+        dir.writeFile("fmt1.zig", unformatted_code) catch @panic("unhandled");
+        dir.writeFile("fmt2.zig", unformatted_code) catch @panic("unhandled");
+
+        // Test zig fmt affecting only the appropriate files.
+        const run1 = b.addSystemCommand(&.{ b.zig_exe, "fmt", "fmt1.zig" });
+        run1.setName("run zig fmt one file");
+        run1.cwd = tmp_path;
+        run1.has_side_effects = true;
+        // stdout should be file path + \n
+        run1.expectStdOutEqual("fmt1.zig\n");
+
+        // running it on the dir, only the new file should be changed
+        const run2 = b.addSystemCommand(&.{ b.zig_exe, "fmt", "." });
+        run2.setName("run zig fmt the directory");
+        run2.cwd = tmp_path;
+        run2.has_side_effects = true;
+        run2.expectStdOutEqual("." ++ s ++ "fmt2.zig\n");
+        run2.step.dependOn(&run1.step);
+
+        // both files have been formatted, nothing should change now
+        const run3 = b.addSystemCommand(&.{ b.zig_exe, "fmt", "." });
+        run3.setName("run zig fmt with nothing to do");
+        run3.cwd = tmp_path;
+        run3.has_side_effects = true;
+        run3.expectStdOutEqual("");
+        run3.step.dependOn(&run2.step);
+
+        const unformatted_code_utf16 = "\xff\xfe \x00 \x00 \x00 \x00/\x00/\x00 \x00n\x00o\x00 \x00r\x00e\x00a\x00s\x00o\x00n\x00";
+        const fmt4_path = fs.path.join(b.allocator, &.{ tmp_path, "fmt4.zig" }) catch @panic("OOM");
+        const write4 = b.addWriteFiles();
+        write4.addBytesToSource(unformatted_code_utf16, fmt4_path);
+        write4.step.dependOn(&run3.step);
+
+        // Test `zig fmt` handling UTF-16 decoding.
+        const run4 = b.addSystemCommand(&.{ b.zig_exe, "fmt", "." });
+        run4.setName("run zig fmt convert UTF-16 to UTF-8");
+        run4.cwd = tmp_path;
+        run4.has_side_effects = true;
+        run4.expectStdOutEqual("." ++ s ++ "fmt4.zig\n");
+        run4.step.dependOn(&write4.step);
+
+        // TODO change this to an exact match
+        const check4 = b.addCheckFile(.{ .path = fmt4_path }, .{
+            .expected_matches = &.{
+                "// no reason",
+            },
+        });
+        check4.step.dependOn(&run4.step);
+
+        const cleanup = b.addRemoveDirTree(tmp_path);
+        cleanup.step.dependOn(&check4.step);
+
+        step.dependOn(&cleanup.step);
+    }
 
-    step.dependOn(&run_cmd.step);
     return step;
 }