From 26e27f5f644437526b5ed5cad6559db8250bb545 Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Mon, 11 Dec 2023 23:08:03 +0100 Subject: [PATCH 1/2] std.Build.Cache: add HexDigest type --- lib/std/Build/Cache.zig | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index 283a3d1f5f..5cdb335c1b 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -179,6 +179,7 @@ fn getPrefixSubpath(allocator: Allocator, prefix: []const u8, path: []u8) ![]u8 pub const bin_digest_len = 16; pub const hex_digest_len = bin_digest_len * 2; pub const BinDigest = [bin_digest_len]u8; +pub const HexDigest = [hex_digest_len]u8; /// This is currently just an arbitrary non-empty string that can't match another manifest line. const manifest_header = "0"; @@ -300,11 +301,11 @@ pub const HashHelper = struct { } /// Returns a hex encoded hash of the inputs, mutating the state of the hasher. - pub fn final(hh: *HashHelper) [hex_digest_len]u8 { + pub fn final(hh: *HashHelper) HexDigest { var bin_digest: BinDigest = undefined; hh.hasher.final(&bin_digest); - var out_digest: [hex_digest_len]u8 = undefined; + var out_digest: HexDigest = undefined; _ = fmt.bufPrint( &out_digest, "{s}", @@ -346,7 +347,7 @@ pub const Manifest = struct { // will then use the same timestamp, to avoid unnecessary filesystem writes. want_refresh_timestamp: bool = true, files: std.ArrayListUnmanaged(File) = .{}, - hex_digest: [hex_digest_len]u8, + hex_digest: HexDigest, /// Populated when hit() returns an error because of one /// of the files listed in the manifest. failed_file_index: ?usize = null, @@ -829,7 +830,7 @@ pub const Manifest = struct { } /// Returns a hex encoded hash of the inputs. - pub fn final(self: *Manifest) [hex_digest_len]u8 { + pub fn final(self: *Manifest) HexDigest { assert(self.manifest_file != null); // We don't close the manifest file yet, because we want to @@ -841,7 +842,7 @@ pub const Manifest = struct { var bin_digest: BinDigest = undefined; self.hash.hasher.final(&bin_digest); - var out_digest: [hex_digest_len]u8 = undefined; + var out_digest: HexDigest = undefined; _ = fmt.bufPrint( &out_digest, "{s}", @@ -1021,8 +1022,8 @@ test "cache file and then recall it" { std.time.sleep(1); } - var digest1: [hex_digest_len]u8 = undefined; - var digest2: [hex_digest_len]u8 = undefined; + var digest1: HexDigest = undefined; + var digest2: HexDigest = undefined; { var cache = Cache{ @@ -1089,8 +1090,8 @@ test "check that changing a file makes cache fail" { std.time.sleep(1); } - var digest1: [hex_digest_len]u8 = undefined; - var digest2: [hex_digest_len]u8 = undefined; + var digest1: HexDigest = undefined; + var digest2: HexDigest = undefined; { var cache = Cache{ @@ -1152,8 +1153,8 @@ test "no file inputs" { const temp_manifest_dir = "no_file_inputs_manifest_dir"; - var digest1: [hex_digest_len]u8 = undefined; - var digest2: [hex_digest_len]u8 = undefined; + var digest1: HexDigest = undefined; + var digest2: HexDigest = undefined; var cache = Cache{ .gpa = testing.allocator, @@ -1211,9 +1212,9 @@ test "Manifest with files added after initial hash work" { std.time.sleep(1); } - var digest1: [hex_digest_len]u8 = undefined; - var digest2: [hex_digest_len]u8 = undefined; - var digest3: [hex_digest_len]u8 = undefined; + var digest1: HexDigest = undefined; + var digest2: HexDigest = undefined; + var digest3: HexDigest = undefined; { var cache = Cache{ From 65878c16ee3e6677912343328e39c87c8f89c784 Mon Sep 17 00:00:00 2001 From: Loris Cro Date: Mon, 11 Dec 2023 23:08:22 +0100 Subject: [PATCH 2/2] std.Build.Step.Run: fix depfile support --- lib/std/Build/Step/Run.zig | 119 +++++++++++++++++++++++++++++-------- 1 file changed, 93 insertions(+), 26 deletions(-) diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index 5555b03e94..83d94f00c6 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -249,7 +249,7 @@ pub fn addDepFileOutputArg(self: *Run, basename: []const u8) std.Build.LazyPath /// Add a prefixed path argument to a dep file (.d) for the child process to /// write its discovered additional dependencies. /// Only one dep file argument is allowed by instance. -pub fn addPrefixedDepFileOutputArg(self: *Run, prefix: []const u8, basename: []const u8) void { +pub fn addPrefixedDepFileOutputArg(self: *Run, prefix: []const u8, basename: []const u8) std.Build.LazyPath { assert(self.dep_output_file == null); const b = self.step.owner; @@ -264,6 +264,8 @@ pub fn addPrefixedDepFileOutputArg(self: *Run, prefix: []const u8, basename: []c self.dep_output_file = dep_file; self.argv.append(.{ .output = dep_file }) catch @panic("OOM"); + + return .{ .generated = &dep_file.generated_file }; } pub fn addArg(self: *Run, arg: []const u8) void { @@ -454,6 +456,10 @@ fn checksContainStderr(checks: []const StdIo.Check) bool { return false; } +const IndexedOutput = struct { + index: usize, + output: *Output, +}; fn make(step: *Step, prog_node: *std.Progress.Node) !void { const b = step.owner; const arena = b.allocator; @@ -461,10 +467,7 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { const has_side_effects = self.hasSideEffects(); var argv_list = ArrayList([]const u8).init(arena); - var output_placeholders = ArrayList(struct { - index: usize, - output: *Output, - }).init(arena); + var output_placeholders = ArrayList(IndexedOutput).init(arena); var man = b.cache.obtain(); defer man.deinit(); @@ -546,32 +549,25 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { if (try step.cacheHit(&man)) { // cache hit, skip running command const digest = man.final(); - for (output_placeholders.items) |placeholder| { - placeholder.output.generated_file.path = try b.cache_root.join(arena, &.{ - "o", &digest, placeholder.output.basename, - }); - } - if (self.captured_stdout) |output| { - output.generated_file.path = try b.cache_root.join(arena, &.{ - "o", &digest, output.basename, - }); - } - - if (self.captured_stderr) |output| { - output.generated_file.path = try b.cache_root.join(arena, &.{ - "o", &digest, output.basename, - }); - } + try populateGeneratedPaths( + arena, + output_placeholders.items, + self.captured_stdout, + self.captured_stderr, + b.cache_root, + &digest, + ); step.result_cached = true; return; } - const digest = man.final(); + const rand_int = std.crypto.random.int(u64); + const tmp_dir_path = "tmp" ++ fs.path.sep_str ++ std.Build.hex64(rand_int); for (output_placeholders.items) |placeholder| { - const output_components = .{ "o", &digest, placeholder.output.basename }; + const output_components = .{ tmp_dir_path, placeholder.output.basename }; const output_sub_path = try fs.path.join(arena, &output_components); const output_sub_dir_path = fs.path.dirname(output_sub_path).?; b.cache_root.handle.makePath(output_sub_dir_path) catch |err| { @@ -588,12 +584,83 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { argv_list.items[placeholder.index] = cli_arg; } - try runCommand(self, argv_list.items, has_side_effects, &digest, prog_node); + try runCommand(self, argv_list.items, has_side_effects, tmp_dir_path, prog_node); if (self.dep_output_file) |dep_output_file| try man.addDepFilePost(std.fs.cwd(), dep_output_file.generated_file.getPath()); + const digest = man.final(); + + const any_output = output_placeholders.items.len > 0 or + self.captured_stdout != null or self.captured_stderr != null; + + // Rename into place + if (any_output) { + const o_sub_path = "o" ++ fs.path.sep_str ++ &digest; + + b.cache_root.handle.rename(tmp_dir_path, o_sub_path) catch |err| { + if (err == error.PathAlreadyExists) { + b.cache_root.handle.deleteTree(o_sub_path) catch |del_err| { + return step.fail("unable to remove dir '{}'{s}: {s}", .{ + b.cache_root, + tmp_dir_path, + @errorName(del_err), + }); + }; + b.cache_root.handle.rename(tmp_dir_path, o_sub_path) catch |retry_err| { + return step.fail("unable to rename dir '{}{s}' to '{}{s}': {s}", .{ + b.cache_root, tmp_dir_path, + b.cache_root, o_sub_path, + @errorName(retry_err), + }); + }; + } else { + return step.fail("unable to rename dir '{}{s}' to '{}{s}': {s}", .{ + b.cache_root, tmp_dir_path, + b.cache_root, o_sub_path, + @errorName(err), + }); + } + }; + } + try step.writeManifest(&man); + + try populateGeneratedPaths( + arena, + output_placeholders.items, + self.captured_stdout, + self.captured_stderr, + b.cache_root, + &digest, + ); +} + +fn populateGeneratedPaths( + arena: std.mem.Allocator, + output_placeholders: []const IndexedOutput, + captured_stdout: ?*Output, + captured_stderr: ?*Output, + cache_root: Build.Cache.Directory, + digest: *const Build.Cache.HexDigest, +) !void { + for (output_placeholders) |placeholder| { + placeholder.output.generated_file.path = try cache_root.join(arena, &.{ + "o", digest, placeholder.output.basename, + }); + } + + if (captured_stdout) |output| { + output.generated_file.path = try cache_root.join(arena, &.{ + "o", digest, output.basename, + }); + } + + if (captured_stderr) |output| { + output.generated_file.path = try cache_root.join(arena, &.{ + "o", digest, output.basename, + }); + } } fn formatTerm( @@ -645,7 +712,7 @@ fn runCommand( self: *Run, argv: []const []const u8, has_side_effects: bool, - digest: ?*const [std.Build.Cache.hex_digest_len]u8, + tmp_dir_path: ?[]const u8, prog_node: *std.Progress.Node, ) !void { const step = &self.step; @@ -816,7 +883,7 @@ fn runCommand( }, }) |stream| { if (stream.captured) |output| { - const output_components = .{ "o", digest.?, output.basename }; + const output_components = .{ tmp_dir_path.?, output.basename }; const output_path = try b.cache_root.join(arena, &output_components); output.generated_file.path = output_path;