From 27317eaff08453792ae2d3bd0c96f81be8a26bfc Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 5 Feb 2023 10:49:57 -0700 Subject: [PATCH 01/11] std.Build.ConfigHeaderStep: support sentinel-terminated strings --- lib/std/Build/ConfigHeaderStep.zig | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/lib/std/Build/ConfigHeaderStep.zig b/lib/std/Build/ConfigHeaderStep.zig index ca4d69dfa9..125b562bb5 100644 --- a/lib/std/Build/ConfigHeaderStep.zig +++ b/lib/std/Build/ConfigHeaderStep.zig @@ -125,6 +125,12 @@ fn putValue(self: *ConfigHeaderStep, field_name: []const u8, comptime T: type, v return; } }, + .Int => { + if (ptr.size == .Slice and ptr.child == u8) { + try self.values.put(field_name, .{ .string = v }); + return; + } + }, else => {}, } From dad6039092d02460d5e993855b129da9949e9c5c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 5 Feb 2023 16:44:03 -0700 Subject: [PATCH 02/11] std.Build: support running build artifacts from packages Deprecate CompileStep.run. The problem with this function is that it does the RunStep with the same build.zig context as the CompileStep, but this is not desirable when running an executable that is provided by a dependency package. Instead, users should use `b.addRunArtifact`. This has the additional benefit of conforming to the existing naming conventions. Additionally, support enum literals in config header options values. --- lib/std/Build.zig | 24 +++++++++++++++++++++++- lib/std/Build/CompileStep.zig | 23 ++++------------------- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 817073f867..a67ded03e8 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -348,7 +348,7 @@ fn applyArgs(b: *Build, args: anytype) !void { .used = false, }); }, - .Enum => { + .Enum, .EnumLiteral => { try b.user_input_options.put(field.name, .{ .name = field.name, .value = .{ .scalar = @tagName(v) }, @@ -599,6 +599,28 @@ pub fn addSystemCommand(self: *Build, argv: []const []const u8) *RunStep { return run_step; } +/// Creates a `RunStep` with an executable built with `addExecutable`. +/// Add command line arguments with methods of `RunStep`. +pub fn addRunArtifact(b: *Build, exe: *CompileStep) *RunStep { + assert(exe.kind == .exe or exe.kind == .test_exe); + + // It doesn't have to be native. We catch that if you actually try to run it. + // Consider that this is declarative; the run step may not be run unless a user + // option is supplied. + const run_step = RunStep.create(b, b.fmt("run {s}", .{exe.step.name})); + run_step.addArtifactArg(exe); + + if (exe.kind == .test_exe) { + run_step.addArg(b.zig_exe); + } + + if (exe.vcpkg_bin_path) |path| { + run_step.addPathDir(path); + } + + return run_step; +} + /// Using the `values` provided, produces a C header file, possibly based on a /// template input file (e.g. config.h.in). /// When an input template file is provided, this function will fail the build diff --git a/lib/std/Build/CompileStep.zig b/lib/std/Build/CompileStep.zig index e0d90add3c..d9d133c5e1 100644 --- a/lib/std/Build/CompileStep.zig +++ b/lib/std/Build/CompileStep.zig @@ -506,26 +506,11 @@ pub fn installLibraryHeaders(a: *CompileStep, l: *CompileStep) void { a.installed_headers.appendSlice(l.installed_headers.items) catch @panic("OOM"); } -/// Creates a `RunStep` with an executable built with `addExecutable`. -/// Add command line arguments with `addArg`. +/// Deprecated: use `std.Build.addRunArtifact` +/// This function will run in the context of the package that created the executable, +/// which is undesirable when running an executable provided by a dependency package. pub fn run(exe: *CompileStep) *RunStep { - assert(exe.kind == .exe or exe.kind == .test_exe); - - // It doesn't have to be native. We catch that if you actually try to run it. - // Consider that this is declarative; the run step may not be run unless a user - // option is supplied. - const run_step = RunStep.create(exe.builder, exe.builder.fmt("run {s}", .{exe.step.name})); - run_step.addArtifactArg(exe); - - if (exe.kind == .test_exe) { - run_step.addArg(exe.builder.zig_exe); - } - - if (exe.vcpkg_bin_path) |path| { - run_step.addPathDir(path); - } - - return run_step; + return exe.builder.addRunArtifact(exe); } /// Creates an `EmulatableRunStep` with an executable built with `addExecutable`. From b061cc9e3ff197d4af1408c8e3a8504dda208e10 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 5 Feb 2023 16:45:26 -0700 Subject: [PATCH 03/11] std.Build.ConfigHeaderStep: support outputting assembly config files --- lib/std/Build/ConfigHeaderStep.zig | 64 +++++++++++++++++++++++++++--- 1 file changed, 58 insertions(+), 6 deletions(-) diff --git a/lib/std/Build/ConfigHeaderStep.zig b/lib/std/Build/ConfigHeaderStep.zig index 125b562bb5..a22618f34a 100644 --- a/lib/std/Build/ConfigHeaderStep.zig +++ b/lib/std/Build/ConfigHeaderStep.zig @@ -13,11 +13,13 @@ pub const Style = union(enum) { cmake: std.Build.FileSource, /// Instead of starting with an input file, start with nothing. blank, + /// Start with nothing, like blank, and output a nasm .asm file. + nasm, pub fn getFileSource(style: Style) ?std.Build.FileSource { switch (style) { .autoconf, .cmake => |s| return s, - .blank => return null, + .blank, .nasm => return null, } } }; @@ -84,6 +86,10 @@ pub fn addValues(self: *ConfigHeaderStep, values: anytype) void { return addValuesInner(self, values) catch @panic("OOM"); } +pub fn getFileSource(self: *ConfigHeaderStep) std.Build.FileSource { + return .{ .generated = &self.output_file }; +} + fn addValuesInner(self: *ConfigHeaderStep, values: anytype) !void { inline for (@typeInfo(@TypeOf(values)).Struct.fields) |field| { try putValue(self, field.name, field.type, @field(values, field.name)); @@ -164,22 +170,31 @@ fn make(step: *Step) !void { var output = std.ArrayList(u8).init(gpa); defer output.deinit(); - try output.appendSlice("/* This file was generated by ConfigHeaderStep using the Zig Build System. */\n"); + const header_text = "This file was generated by ConfigHeaderStep using the Zig Build System."; + const c_generated_line = "/* " ++ header_text ++ " */\n"; + const asm_generated_line = "; " ++ header_text ++ "\n"; switch (self.style) { .autoconf => |file_source| { + try output.appendSlice(c_generated_line); const src_path = file_source.getPath(self.builder); const contents = try std.fs.cwd().readFileAlloc(gpa, src_path, self.max_bytes); try render_autoconf(contents, &output, self.values, src_path); }, .cmake => |file_source| { + try output.appendSlice(c_generated_line); const src_path = file_source.getPath(self.builder); const contents = try std.fs.cwd().readFileAlloc(gpa, src_path, self.max_bytes); try render_cmake(contents, &output, self.values, src_path); }, .blank => { + try output.appendSlice(c_generated_line); try render_blank(&output, self.values, self.include_path); }, + .nasm => { + try output.appendSlice(asm_generated_line); + try render_nasm(&output, self.values); + }, } hash.update(output.items); @@ -253,7 +268,7 @@ fn render_autoconf( any_errors = true; continue; }; - try renderValue(output, name, kv.value); + try renderValueC(output, name, kv.value); } for (values_copy.keys()) |name| { @@ -304,7 +319,7 @@ fn render_cmake( any_errors = true; continue; }; - try renderValue(output, name, kv.value); + try renderValueC(output, name, kv.value); } for (values_copy.keys()) |name| { @@ -338,7 +353,7 @@ fn render_blank( const values = defines.values(); for (defines.keys()) |name, i| { - try renderValue(output, name, values[i]); + try renderValueC(output, name, values[i]); } try output.appendSlice("#endif /* "); @@ -346,7 +361,14 @@ fn render_blank( try output.appendSlice(" */\n"); } -fn renderValue(output: *std.ArrayList(u8), name: []const u8, value: Value) !void { +fn render_nasm(output: *std.ArrayList(u8), defines: std.StringArrayHashMap(Value)) !void { + const values = defines.values(); + for (defines.keys()) |name, i| { + try renderValueNasm(output, name, values[i]); + } +} + +fn renderValueC(output: *std.ArrayList(u8), name: []const u8, value: Value) !void { switch (value) { .undef => { try output.appendSlice("/* #undef "); @@ -376,3 +398,33 @@ fn renderValue(output: *std.ArrayList(u8), name: []const u8, value: Value) !void }, } } + +fn renderValueNasm(output: *std.ArrayList(u8), name: []const u8, value: Value) !void { + switch (value) { + .undef => { + try output.appendSlice("; %undef "); + try output.appendSlice(name); + try output.appendSlice("\n"); + }, + .defined => { + try output.appendSlice("%define "); + try output.appendSlice(name); + try output.appendSlice("\n"); + }, + .boolean => |b| { + try output.appendSlice("%define "); + try output.appendSlice(name); + try output.appendSlice(if (b) " 1\n" else " 0\n"); + }, + .int => |i| { + try output.writer().print("%define {s} {d}\n", .{ name, i }); + }, + .ident => |ident| { + try output.writer().print("%define {s} {s}\n", .{ name, ident }); + }, + .string => |string| { + // TODO: use nasm-specific escaping instead of zig string literals + try output.writer().print("%define {s} \"{}\"\n", .{ name, std.zig.fmtEscapes(string) }); + }, + } +} From 2654d0c66860d32714e33404554482cbc0cbabf5 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 5 Feb 2023 16:45:51 -0700 Subject: [PATCH 04/11] std.Build.RunStep: introduce addOutputFileArg API This provides file path as a command line argument to the command being run, and returns a FileSource which can be used as inputs to other APIs throughout the build system. Unfortunately, it is implemented by pooping a ton of temporary files into zig-cache/tmp for the time being. I think one of the very next improvements to the build system should be moving the compiler's cache system to the standard library and using it in the build system. I had a look at the dependencies and it is already pretty untangled. --- lib/std/Build/RunStep.zig | 65 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 64 insertions(+), 1 deletion(-) diff --git a/lib/std/Build/RunStep.zig b/lib/std/Build/RunStep.zig index 07f2363623..da35bb26a9 100644 --- a/lib/std/Build/RunStep.zig +++ b/lib/std/Build/RunStep.zig @@ -39,6 +39,10 @@ expected_exit_code: ?u8 = 0, /// Print the command before running it print: bool, +/// Controls whether execution is skipped if the output file is up-to-date. +/// The default is to always run if there is no output file, and to skip +/// running if all output files are up-to-date. +condition: enum { output_outdated, always } = .output_outdated, pub const StdIoAction = union(enum) { inherit, @@ -51,6 +55,12 @@ pub const Arg = union(enum) { artifact: *CompileStep, file_source: std.Build.FileSource, bytes: []u8, + output: Output, + + pub const Output = struct { + generated_file: *std.Build.GeneratedFile, + basename: []const u8, + }; }; pub fn create(builder: *std.Build, name: []const u8) *RunStep { @@ -71,6 +81,20 @@ pub fn addArtifactArg(self: *RunStep, artifact: *CompileStep) void { self.step.dependOn(&artifact.step); } +/// This provides file path as a command line argument to the command being +/// run, and returns a FileSource which can be used as inputs to other APIs +/// throughout the build system. +pub fn addOutputFileArg(rs: *RunStep, basename: []const u8) std.Build.FileSource { + const generated_file = rs.builder.allocator.create(std.Build.GeneratedFile) catch @panic("OOM"); + generated_file.* = .{ .step = &rs.step }; + rs.argv.append(.{ .output = .{ + .generated_file = generated_file, + .basename = rs.builder.dupe(basename), + } }) catch @panic("OOM"); + + return .{ .generated = generated_file }; +} + pub fn addFileSourceArg(self: *RunStep, file_source: std.Build.FileSource) void { self.argv.append(Arg{ .file_source = file_source.dupe(self.builder), @@ -159,10 +183,24 @@ fn stdIoActionToBehavior(action: StdIoAction) std.ChildProcess.StdIo { }; } +fn needOutputCheck(self: RunStep) bool { + switch (self.condition) { + .always => return false, + .output_outdated => { + for (self.argv.items) |arg| switch (arg) { + .output => return true, + else => continue, + }; + return false; + }, + } +} + fn make(step: *Step) !void { const self = @fieldParentPtr(RunStep, "step", step); var argv_list = ArrayList([]const u8).init(self.builder.allocator); + for (self.argv.items) |arg| { switch (arg) { .bytes => |bytes| try argv_list.append(bytes), @@ -172,9 +210,34 @@ fn make(step: *Step) !void { // On Windows we don't have rpaths so we have to add .dll search paths to PATH self.addPathForDynLibs(artifact); } - const executable_path = artifact.installed_path orelse artifact.getOutputSource().getPath(self.builder); + const executable_path = artifact.installed_path orelse + artifact.getOutputSource().getPath(self.builder); try argv_list.append(executable_path); }, + .output => |output| { + // TODO: until the cache system is brought into the build system, + // we use a temporary directory here for each run. + var digest: [16]u8 = undefined; + std.crypto.random.bytes(&digest); + var hash_basename: [digest.len * 2]u8 = undefined; + _ = std.fmt.bufPrint( + &hash_basename, + "{s}", + .{std.fmt.fmtSliceHexLower(&digest)}, + ) catch unreachable; + + const output_path = try fs.path.join(self.builder.allocator, &[_][]const u8{ + self.builder.cache_root, "tmp", &hash_basename, output.basename, + }); + const output_dir = fs.path.dirname(output_path).?; + fs.cwd().makePath(output_dir) catch |err| { + std.debug.print("unable to make path {s}: {s}\n", .{ output_dir, @errorName(err) }); + return err; + }; + + output.generated_file.path = output_path; + try argv_list.append(output_path); + }, } } From 9cb52ca6ce7043ba0ce08d5650ac542075f10685 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 5 Feb 2023 19:39:04 -0700 Subject: [PATCH 05/11] move the cache system from compiler to std lib --- CMakeLists.txt | 5 +- lib/std/Build.zig | 2 + {src => lib/std/Build}/Cache.zig | 79 +++++++++------- {src => lib/std/Build/Cache}/DepTokenizer.zig | 0 src/Compilation.zig | 92 +++++++++---------- src/Module.zig | 2 +- src/Package.zig | 2 +- src/glibc.zig | 2 +- src/link.zig | 2 +- src/link/Coff/lld.zig | 2 +- src/link/Elf.zig | 2 +- src/link/MachO.zig | 2 +- src/link/MachO/zld.zig | 2 +- src/link/Wasm.zig | 2 +- src/main.zig | 4 +- src/mingw.zig | 2 +- 16 files changed, 104 insertions(+), 98 deletions(-) rename {src => lib/std/Build}/Cache.zig (96%) rename {src => lib/std/Build/Cache}/DepTokenizer.zig (100%) diff --git a/CMakeLists.txt b/CMakeLists.txt index f8029fdcde..448fb400b6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -216,6 +216,9 @@ set(ZIG_STAGE2_SOURCES "${CMAKE_SOURCE_DIR}/lib/std/atomic/stack.zig" "${CMAKE_SOURCE_DIR}/lib/std/base64.zig" "${CMAKE_SOURCE_DIR}/lib/std/buf_map.zig" + "${CMAKE_SOURCE_DIR}/lib/std/Build.zig" + "${CMAKE_SOURCE_DIR}/lib/std/Build/Cache.zig" + "${CMAKE_SOURCE_DIR}/lib/std/Build/Cache/DepTokenizer.zig" "${CMAKE_SOURCE_DIR}/lib/std/builtin.zig" "${CMAKE_SOURCE_DIR}/lib/std/c.zig" "${CMAKE_SOURCE_DIR}/lib/std/c/linux.zig" @@ -523,9 +526,7 @@ set(ZIG_STAGE2_SOURCES "${CMAKE_SOURCE_DIR}/lib/std/zig/tokenizer.zig" "${CMAKE_SOURCE_DIR}/src/Air.zig" "${CMAKE_SOURCE_DIR}/src/AstGen.zig" - "${CMAKE_SOURCE_DIR}/src/Cache.zig" "${CMAKE_SOURCE_DIR}/src/Compilation.zig" - "${CMAKE_SOURCE_DIR}/src/DepTokenizer.zig" "${CMAKE_SOURCE_DIR}/src/Liveness.zig" "${CMAKE_SOURCE_DIR}/src/Module.zig" "${CMAKE_SOURCE_DIR}/src/Package.zig" diff --git a/lib/std/Build.zig b/lib/std/Build.zig index a67ded03e8..428a11948d 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -19,6 +19,8 @@ const NativeTargetInfo = std.zig.system.NativeTargetInfo; const Sha256 = std.crypto.hash.sha2.Sha256; const Build = @This(); +pub const Cache = @import("Build/Cache.zig"); + /// deprecated: use `CompileStep`. pub const LibExeObjStep = CompileStep; /// deprecated: use `Build`. diff --git a/src/Cache.zig b/lib/std/Build/Cache.zig similarity index 96% rename from src/Cache.zig rename to lib/std/Build/Cache.zig index 3020f8e8c6..e316322c68 100644 --- a/src/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -2,6 +2,45 @@ //! This is not a general-purpose cache. It is designed to be fast and simple, //! not to withstand attacks using specially-crafted input. +pub const Directory = struct { + /// This field is redundant for operations that can act on the open directory handle + /// directly, but it is needed when passing the directory to a child process. + /// `null` means cwd. + path: ?[]const u8, + handle: std.fs.Dir, + + pub fn join(self: Directory, allocator: Allocator, paths: []const []const u8) ![]u8 { + if (self.path) |p| { + // TODO clean way to do this with only 1 allocation + const part2 = try std.fs.path.join(allocator, paths); + defer allocator.free(part2); + return std.fs.path.join(allocator, &[_][]const u8{ p, part2 }); + } else { + return std.fs.path.join(allocator, paths); + } + } + + pub fn joinZ(self: Directory, allocator: Allocator, paths: []const []const u8) ![:0]u8 { + if (self.path) |p| { + // TODO clean way to do this with only 1 allocation + const part2 = try std.fs.path.join(allocator, paths); + defer allocator.free(part2); + return std.fs.path.joinZ(allocator, &[_][]const u8{ p, part2 }); + } else { + return std.fs.path.joinZ(allocator, paths); + } + } + + /// Whether or not the handle should be closed, or the path should be freed + /// is determined by usage, however this function is provided for convenience + /// if it happens to be what the caller needs. + pub fn closeAndFree(self: *Directory, gpa: Allocator) void { + self.handle.close(); + if (self.path) |p| gpa.free(p); + self.* = undefined; + } +}; + gpa: Allocator, manifest_dir: fs.Dir, hash: HashHelper = .{}, @@ -14,9 +53,11 @@ mutex: std.Thread.Mutex = .{}, /// are replaced with single-character indicators. This is not to save /// space but to eliminate absolute file paths. This improves portability /// and usefulness of the cache for advanced use cases. -prefixes_buffer: [3]Compilation.Directory = undefined, +prefixes_buffer: [3]Directory = undefined, prefixes_len: usize = 0, +pub const DepTokenizer = @import("Cache/DepTokenizer.zig"); + const Cache = @This(); const std = @import("std"); const builtin = @import("builtin"); @@ -27,10 +68,9 @@ const testing = std.testing; const mem = std.mem; const fmt = std.fmt; const Allocator = std.mem.Allocator; -const Compilation = @import("Compilation.zig"); const log = std.log.scoped(.cache); -pub fn addPrefix(cache: *Cache, directory: Compilation.Directory) void { +pub fn addPrefix(cache: *Cache, directory: Directory) void { if (directory.path) |p| { log.debug("Cache.addPrefix {d} {s}", .{ cache.prefixes_len, p }); } @@ -49,7 +89,7 @@ pub fn obtain(cache: *Cache) Manifest { }; } -pub fn prefixes(cache: *const Cache) []const Compilation.Directory { +pub fn prefixes(cache: *const Cache) []const Directory { return cache.prefixes_buffer[0..cache.prefixes_len]; } @@ -135,8 +175,6 @@ pub const File = struct { pub const HashHelper = struct { hasher: Hasher = hasher_init, - const EmitLoc = Compilation.EmitLoc; - /// Record a slice of bytes as an dependency of the process being cached pub fn addBytes(hh: *HashHelper, bytes: []const u8) void { hh.hasher.update(mem.asBytes(&bytes.len)); @@ -148,15 +186,6 @@ pub const HashHelper = struct { hh.addBytes(optional_bytes orelse return); } - pub fn addEmitLoc(hh: *HashHelper, emit_loc: EmitLoc) void { - hh.addBytes(emit_loc.basename); - } - - pub fn addOptionalEmitLoc(hh: *HashHelper, optional_emit_loc: ?EmitLoc) void { - hh.add(optional_emit_loc != null); - hh.addEmitLoc(optional_emit_loc orelse return); - } - pub fn addListOfBytes(hh: *HashHelper, list_of_bytes: []const []const u8) void { hh.add(list_of_bytes.len); for (list_of_bytes) |bytes| hh.addBytes(bytes); @@ -308,24 +337,6 @@ pub const Manifest = struct { return self.files.items.len - 1; } - pub fn hashCSource(self: *Manifest, c_source: Compilation.CSourceFile) !void { - _ = try self.addFile(c_source.src_path, null); - // Hash the extra flags, with special care to call addFile for file parameters. - // TODO this logic can likely be improved by utilizing clang_options_data.zig. - const file_args = [_][]const u8{"-include"}; - var arg_i: usize = 0; - while (arg_i < c_source.extra_flags.len) : (arg_i += 1) { - const arg = c_source.extra_flags[arg_i]; - self.hash.addBytes(arg); - for (file_args) |file_arg| { - if (mem.eql(u8, file_arg, arg) and arg_i + 1 < c_source.extra_flags.len) { - arg_i += 1; - _ = try self.addFile(c_source.extra_flags[arg_i], null); - } - } - } - } - pub fn addOptionalFile(self: *Manifest, optional_file_path: ?[]const u8) !void { self.hash.add(optional_file_path != null); const file_path = optional_file_path orelse return; @@ -778,7 +789,7 @@ pub const Manifest = struct { var error_buf = std.ArrayList(u8).init(self.cache.gpa); defer error_buf.deinit(); - var it: @import("DepTokenizer.zig") = .{ .bytes = dep_file_contents }; + var it: DepTokenizer = .{ .bytes = dep_file_contents }; // Skip first token: target. switch (it.next() orelse return) { // Empty dep file OK. diff --git a/src/DepTokenizer.zig b/lib/std/Build/Cache/DepTokenizer.zig similarity index 100% rename from src/DepTokenizer.zig rename to lib/std/Build/Cache/DepTokenizer.zig diff --git a/src/Compilation.zig b/src/Compilation.zig index 18d0e46892..ea83d82109 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -26,7 +26,7 @@ const wasi_libc = @import("wasi_libc.zig"); const fatal = @import("main.zig").fatal; const clangMain = @import("main.zig").clangMain; const Module = @import("Module.zig"); -const Cache = @import("Cache.zig"); +const Cache = std.Build.Cache; const translate_c = @import("translate_c.zig"); const clang = @import("clang.zig"); const c_codegen = @import("codegen/c.zig"); @@ -807,44 +807,7 @@ pub const AllErrors = struct { } }; -pub const Directory = struct { - /// This field is redundant for operations that can act on the open directory handle - /// directly, but it is needed when passing the directory to a child process. - /// `null` means cwd. - path: ?[]const u8, - handle: std.fs.Dir, - - pub fn join(self: Directory, allocator: Allocator, paths: []const []const u8) ![]u8 { - if (self.path) |p| { - // TODO clean way to do this with only 1 allocation - const part2 = try std.fs.path.join(allocator, paths); - defer allocator.free(part2); - return std.fs.path.join(allocator, &[_][]const u8{ p, part2 }); - } else { - return std.fs.path.join(allocator, paths); - } - } - - pub fn joinZ(self: Directory, allocator: Allocator, paths: []const []const u8) ![:0]u8 { - if (self.path) |p| { - // TODO clean way to do this with only 1 allocation - const part2 = try std.fs.path.join(allocator, paths); - defer allocator.free(part2); - return std.fs.path.joinZ(allocator, &[_][]const u8{ p, part2 }); - } else { - return std.fs.path.joinZ(allocator, paths); - } - } - - /// Whether or not the handle should be closed, or the path should be freed - /// is determined by usage, however this function is provided for convenience - /// if it happens to be what the caller needs. - pub fn closeAndFree(self: *Directory, gpa: Allocator) void { - self.handle.close(); - if (self.path) |p| gpa.free(p); - self.* = undefined; - } -}; +pub const Directory = Cache.Directory; pub const EmitLoc = struct { /// If this is `null` it means the file will be output to the cache directory. @@ -854,6 +817,35 @@ pub const EmitLoc = struct { basename: []const u8, }; +pub const cache_helpers = struct { + pub fn addEmitLoc(hh: *Cache.HashHelper, emit_loc: EmitLoc) void { + hh.addBytes(emit_loc.basename); + } + + pub fn addOptionalEmitLoc(hh: *Cache.HashHelper, optional_emit_loc: ?EmitLoc) void { + hh.add(optional_emit_loc != null); + addEmitLoc(hh, optional_emit_loc orelse return); + } + + pub fn hashCSource(self: *Cache.Manifest, c_source: Compilation.CSourceFile) !void { + _ = try self.addFile(c_source.src_path, null); + // Hash the extra flags, with special care to call addFile for file parameters. + // TODO this logic can likely be improved by utilizing clang_options_data.zig. + const file_args = [_][]const u8{"-include"}; + var arg_i: usize = 0; + while (arg_i < c_source.extra_flags.len) : (arg_i += 1) { + const arg = c_source.extra_flags[arg_i]; + self.hash.addBytes(arg); + for (file_args) |file_arg| { + if (mem.eql(u8, file_arg, arg) and arg_i + 1 < c_source.extra_flags.len) { + arg_i += 1; + _ = try self.addFile(c_source.extra_flags[arg_i], null); + } + } + } + } +}; + pub const ClangPreprocessorMode = enum { no, /// This means we are doing `zig cc -E -o `. @@ -1522,8 +1514,8 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { cache.hash.add(link_libunwind); cache.hash.add(options.output_mode); cache.hash.add(options.machine_code_model); - cache.hash.addOptionalEmitLoc(options.emit_bin); - cache.hash.addOptionalEmitLoc(options.emit_implib); + cache_helpers.addOptionalEmitLoc(&cache.hash, options.emit_bin); + cache_helpers.addOptionalEmitLoc(&cache.hash, options.emit_implib); cache.hash.addBytes(options.root_name); if (options.target.os.tag == .wasi) cache.hash.add(wasi_exec_model); // TODO audit this and make sure everything is in it @@ -2636,11 +2628,11 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes man.hash.addListOfBytes(key.src.extra_flags); } - man.hash.addOptionalEmitLoc(comp.emit_asm); - man.hash.addOptionalEmitLoc(comp.emit_llvm_ir); - man.hash.addOptionalEmitLoc(comp.emit_llvm_bc); - man.hash.addOptionalEmitLoc(comp.emit_analysis); - man.hash.addOptionalEmitLoc(comp.emit_docs); + cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_asm); + cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_llvm_ir); + cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_llvm_bc); + cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_analysis); + cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_docs); man.hash.addListOfBytes(comp.clang_argv); @@ -3959,11 +3951,11 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: *std.P defer man.deinit(); man.hash.add(comp.clang_preprocessor_mode); - man.hash.addOptionalEmitLoc(comp.emit_asm); - man.hash.addOptionalEmitLoc(comp.emit_llvm_ir); - man.hash.addOptionalEmitLoc(comp.emit_llvm_bc); + cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_asm); + cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_llvm_ir); + cache_helpers.addOptionalEmitLoc(&man.hash, comp.emit_llvm_bc); - try man.hashCSource(c_object.src); + try cache_helpers.hashCSource(&man, c_object.src); var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa); defer arena_allocator.deinit(); diff --git a/src/Module.zig b/src/Module.zig index e4cf0189cc..a129cb0cb6 100644 --- a/src/Module.zig +++ b/src/Module.zig @@ -16,7 +16,7 @@ const Ast = std.zig.Ast; const Module = @This(); const Compilation = @import("Compilation.zig"); -const Cache = @import("Cache.zig"); +const Cache = std.Build.Cache; const Value = @import("value.zig").Value; const Type = @import("type.zig").Type; const TypedValue = @import("TypedValue.zig"); diff --git a/src/Package.zig b/src/Package.zig index 401eef2121..a3afe21009 100644 --- a/src/Package.zig +++ b/src/Package.zig @@ -13,7 +13,7 @@ const Compilation = @import("Compilation.zig"); const Module = @import("Module.zig"); const ThreadPool = @import("ThreadPool.zig"); const WaitGroup = @import("WaitGroup.zig"); -const Cache = @import("Cache.zig"); +const Cache = std.Build.Cache; const build_options = @import("build_options"); const Manifest = @import("Manifest.zig"); diff --git a/src/glibc.zig b/src/glibc.zig index 8dce1c5132..2a2887c334 100644 --- a/src/glibc.zig +++ b/src/glibc.zig @@ -11,7 +11,7 @@ const target_util = @import("target.zig"); const Compilation = @import("Compilation.zig"); const build_options = @import("build_options"); const trace = @import("tracy.zig").trace; -const Cache = @import("Cache.zig"); +const Cache = std.Build.Cache; const Package = @import("Package.zig"); pub const Lib = struct { diff --git a/src/link.zig b/src/link.zig index 2b3ce51667..5650e0679a 100644 --- a/src/link.zig +++ b/src/link.zig @@ -10,7 +10,7 @@ const wasi_libc = @import("wasi_libc.zig"); const Air = @import("Air.zig"); const Allocator = std.mem.Allocator; -const Cache = @import("Cache.zig"); +const Cache = std.Build.Cache; const Compilation = @import("Compilation.zig"); const LibCInstallation = @import("libc_installation.zig").LibCInstallation; const Liveness = @import("Liveness.zig"); diff --git a/src/link/Coff/lld.zig b/src/link/Coff/lld.zig index d705f62f5c..c308ff5989 100644 --- a/src/link/Coff/lld.zig +++ b/src/link/Coff/lld.zig @@ -5,6 +5,7 @@ const assert = std.debug.assert; const fs = std.fs; const log = std.log.scoped(.link); const mem = std.mem; +const Cache = std.Build.Cache; const mingw = @import("../../mingw.zig"); const link = @import("../../link.zig"); @@ -13,7 +14,6 @@ const trace = @import("../../tracy.zig").trace; const Allocator = mem.Allocator; -const Cache = @import("../../Cache.zig"); const Coff = @import("../Coff.zig"); const Compilation = @import("../../Compilation.zig"); diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 45952da6c0..37ebfdc0dc 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -21,7 +21,7 @@ const trace = @import("../tracy.zig").trace; const Air = @import("../Air.zig"); const Allocator = std.mem.Allocator; pub const Atom = @import("Elf/Atom.zig"); -const Cache = @import("../Cache.zig"); +const Cache = std.Build.Cache; const Compilation = @import("../Compilation.zig"); const Dwarf = @import("Dwarf.zig"); const File = link.File; diff --git a/src/link/MachO.zig b/src/link/MachO.zig index 24ef275c5b..35f5f1b562 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -28,7 +28,7 @@ const Air = @import("../Air.zig"); const Allocator = mem.Allocator; const Archive = @import("MachO/Archive.zig"); pub const Atom = @import("MachO/Atom.zig"); -const Cache = @import("../Cache.zig"); +const Cache = std.Build.Cache; const CodeSignature = @import("MachO/CodeSignature.zig"); const Compilation = @import("../Compilation.zig"); const Dwarf = File.Dwarf; diff --git a/src/link/MachO/zld.zig b/src/link/MachO/zld.zig index 095ac9b5ce..785fa71445 100644 --- a/src/link/MachO/zld.zig +++ b/src/link/MachO/zld.zig @@ -20,7 +20,7 @@ const trace = @import("../../tracy.zig").trace; const Allocator = mem.Allocator; const Archive = @import("Archive.zig"); const Atom = @import("ZldAtom.zig"); -const Cache = @import("../../Cache.zig"); +const Cache = std.Build.Cache; const CodeSignature = @import("CodeSignature.zig"); const Compilation = @import("../../Compilation.zig"); const DwarfInfo = @import("DwarfInfo.zig"); diff --git a/src/link/Wasm.zig b/src/link/Wasm.zig index 9d20412788..e62a2050d7 100644 --- a/src/link/Wasm.zig +++ b/src/link/Wasm.zig @@ -20,7 +20,7 @@ const lldMain = @import("../main.zig").lldMain; const trace = @import("../tracy.zig").trace; const build_options = @import("build_options"); const wasi_libc = @import("../wasi_libc.zig"); -const Cache = @import("../Cache.zig"); +const Cache = std.Build.Cache; const Type = @import("../type.zig").Type; const TypedValue = @import("../TypedValue.zig"); const LlvmObject = @import("../codegen/llvm.zig").Object; diff --git a/src/main.zig b/src/main.zig index 00a7b126c8..2add2f9165 100644 --- a/src/main.zig +++ b/src/main.zig @@ -20,7 +20,7 @@ const LibCInstallation = @import("libc_installation.zig").LibCInstallation; const wasi_libc = @import("wasi_libc.zig"); const translate_c = @import("translate_c.zig"); const clang = @import("clang.zig"); -const Cache = @import("Cache.zig"); +const Cache = std.Build.Cache; const target_util = @import("target.zig"); const ThreadPool = @import("ThreadPool.zig"); const crash_report = @import("crash_report.zig"); @@ -3607,7 +3607,7 @@ fn cmdTranslateC(comp: *Compilation, arena: Allocator, enable_cache: bool) !void defer if (enable_cache) man.deinit(); man.hash.add(@as(u16, 0xb945)); // Random number to distinguish translate-c from compiling C objects - man.hashCSource(c_source_file) catch |err| { + Compilation.cache_helpers.hashCSource(&man, c_source_file) catch |err| { fatal("unable to process '{s}': {s}", .{ c_source_file.src_path, @errorName(err) }); }; diff --git a/src/mingw.zig b/src/mingw.zig index 06880743c6..4f94e26a98 100644 --- a/src/mingw.zig +++ b/src/mingw.zig @@ -8,7 +8,7 @@ const log = std.log.scoped(.mingw); const builtin = @import("builtin"); const Compilation = @import("Compilation.zig"); const build_options = @import("build_options"); -const Cache = @import("Cache.zig"); +const Cache = std.Build.Cache; pub const CRTFile = enum { crt2_o, From 066632261492ee7624117ad09269f57526aca4c0 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 9 Feb 2023 10:00:25 -0700 Subject: [PATCH 06/11] std.Build.Cache: remove debug log statements Now that this API is used by the build system, these debug logs are problematic because build scripts run in debug mode, making these logs noisy output. --- lib/std/Build/Cache.zig | 25 +------------------------ 1 file changed, 1 insertion(+), 24 deletions(-) diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index e316322c68..c459fca633 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -53,7 +53,7 @@ mutex: std.Thread.Mutex = .{}, /// are replaced with single-character indicators. This is not to save /// space but to eliminate absolute file paths. This improves portability /// and usefulness of the cache for advanced use cases. -prefixes_buffer: [3]Directory = undefined, +prefixes_buffer: [4]Directory = undefined, prefixes_len: usize = 0, pub const DepTokenizer = @import("Cache/DepTokenizer.zig"); @@ -71,9 +71,6 @@ const Allocator = std.mem.Allocator; const log = std.log.scoped(.cache); pub fn addPrefix(cache: *Cache, directory: Directory) void { - if (directory.path) |p| { - log.debug("Cache.addPrefix {d} {s}", .{ cache.prefixes_len, p }); - } cache.prefixes_buffer[cache.prefixes_len] = directory; cache.prefixes_len += 1; } @@ -120,8 +117,6 @@ fn findPrefixResolved(cache: *const Cache, resolved_path: []u8) !PrefixedPath { .prefix = @intCast(u8, i), .sub_path = sub_path, }; - } else { - log.debug("'{s}' does not start with '{s}'", .{ resolved_path, p }); } } @@ -319,10 +314,6 @@ pub const Manifest = struct { const prefixed_path = try self.cache.findPrefix(file_path); errdefer gpa.free(prefixed_path.sub_path); - log.debug("Manifest.addFile {s} -> {d} {s}", .{ - file_path, prefixed_path.prefix, prefixed_path.sub_path, - }); - self.files.addOneAssumeCapacity().* = .{ .prefixed_path = prefixed_path, .contents = null, @@ -687,10 +678,6 @@ pub const Manifest = struct { const prefixed_path = try self.cache.findPrefix(file_path); errdefer gpa.free(prefixed_path.sub_path); - log.debug("Manifest.addFilePostFetch {s} -> {d} {s}", .{ - file_path, prefixed_path.prefix, prefixed_path.sub_path, - }); - const new_ch_file = try self.files.addOne(gpa); new_ch_file.* = .{ .prefixed_path = prefixed_path, @@ -717,10 +704,6 @@ pub const Manifest = struct { const prefixed_path = try self.cache.findPrefix(file_path); errdefer gpa.free(prefixed_path.sub_path); - log.debug("Manifest.addFilePost {s} -> {d} {s}", .{ - file_path, prefixed_path.prefix, prefixed_path.sub_path, - }); - const new_ch_file = try self.files.addOne(gpa); new_ch_file.* = .{ .prefixed_path = prefixed_path, @@ -748,15 +731,9 @@ pub const Manifest = struct { const ch_file = try self.files.addOne(gpa); errdefer self.files.shrinkRetainingCapacity(self.files.items.len - 1); - log.debug("Manifest.addFilePostContents resolved_path={s}", .{resolved_path}); - const prefixed_path = try self.cache.findPrefixResolved(resolved_path); errdefer gpa.free(prefixed_path.sub_path); - log.debug("Manifest.addFilePostContents -> {d} {s}", .{ - prefixed_path.prefix, prefixed_path.sub_path, - }); - ch_file.* = .{ .prefixed_path = prefixed_path, .max_file_size = null, From d97042ad2e41b173334ec542eb4b07e81864d10e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Thu, 9 Feb 2023 10:01:01 -0700 Subject: [PATCH 07/11] std.Build: start using the cache system with RunStep * Use std.Build.Cache.Directory instead of a string for storing the cache roots and build roots. * Set up a std.Build.Cache in build_runner.zig and use it in std.Build.RunStep for avoiding redundant work. --- build.zig | 9 +- lib/build_runner.zig | 35 +++++++- lib/std/Build.zig | 58 ++++++------ lib/std/Build/CompileStep.zig | 41 ++++----- lib/std/Build/ConfigHeaderStep.zig | 4 +- lib/std/Build/OptionsStep.zig | 22 ++--- lib/std/Build/RunStep.zig | 138 +++++++++++++++++++++-------- lib/std/Build/WriteFileStep.zig | 4 +- test/tests.zig | 4 +- 9 files changed, 198 insertions(+), 117 deletions(-) diff --git a/build.zig b/build.zig index 2e5c3ddd96..e121850221 100644 --- a/build.zig +++ b/build.zig @@ -40,11 +40,8 @@ pub fn build(b: *std.Build) !void { }); docgen_exe.single_threaded = single_threaded; - const rel_zig_exe = try fs.path.relative(b.allocator, b.build_root, b.zig_exe); - const langref_out_path = fs.path.join( - b.allocator, - &[_][]const u8{ b.cache_root, "langref.html" }, - ) catch unreachable; + const rel_zig_exe = try b.build_root.join(b.allocator, &.{b.zig_exe}); + const langref_out_path = try b.cache_root.join(b.allocator, &.{"langref.html"}); const docgen_cmd = docgen_exe.run(); docgen_cmd.addArgs(&[_][]const u8{ "--zig", @@ -215,7 +212,7 @@ pub fn build(b: *std.Build) !void { var code: u8 = undefined; const git_describe_untrimmed = b.execAllowFail(&[_][]const u8{ - "git", "-C", b.build_root, "describe", "--match", "*.*.*", "--tags", + "git", "-C", b.build_root.path orelse ".", "describe", "--match", "*.*.*", "--tags", }, &code, .Ignore) catch { break :v version_string; }; diff --git a/lib/build_runner.zig b/lib/build_runner.zig index aeefb57bfc..53f439802e 100644 --- a/lib/build_runner.zig +++ b/lib/build_runner.zig @@ -43,13 +43,40 @@ pub fn main() !void { const host = try std.zig.system.NativeTargetInfo.detect(.{}); + const build_root_directory: std.Build.Cache.Directory = .{ + .path = build_root, + .handle = try std.fs.cwd().openDir(build_root, .{}), + }; + + const local_cache_directory: std.Build.Cache.Directory = .{ + .path = try std.fs.path.relative(allocator, build_root, cache_root), + .handle = try std.fs.cwd().makeOpenPath(cache_root, .{}), + }; + + const global_cache_directory: std.Build.Cache.Directory = .{ + .path = try std.fs.path.relative(allocator, build_root, global_cache_root), + .handle = try std.fs.cwd().makeOpenPath(global_cache_root, .{}), + }; + + var cache: std.Build.Cache = .{ + .gpa = allocator, + .manifest_dir = try local_cache_directory.handle.makeOpenPath("h", .{}), + }; + cache.addPrefix(.{ .path = null, .handle = std.fs.cwd() }); + cache.addPrefix(build_root_directory); + cache.addPrefix(local_cache_directory); + cache.addPrefix(global_cache_directory); + + //cache.hash.addBytes(builtin.zig_version); + const builder = try std.Build.create( allocator, zig_exe, - build_root, - cache_root, - global_cache_root, + build_root_directory, + local_cache_directory, + global_cache_directory, host, + &cache, ); defer builder.destroy(); @@ -138,7 +165,7 @@ pub fn main() !void { return usageAndErr(builder, false, stderr_stream); }; } else if (mem.eql(u8, arg, "--zig-lib-dir")) { - builder.override_lib_dir = nextArg(args, &arg_idx) orelse { + builder.zig_lib_dir = nextArg(args, &arg_idx) orelse { std.debug.print("Expected argument after --zig-lib-dir\n\n", .{}); return usageAndErr(builder, false, stderr_stream); }; diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 428a11948d..cc4ac43916 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -79,11 +79,12 @@ search_prefixes: ArrayList([]const u8), libc_file: ?[]const u8 = null, installed_files: ArrayList(InstalledFile), /// Path to the directory containing build.zig. -build_root: []const u8, -cache_root: []const u8, -global_cache_root: []const u8, -/// zig lib dir -override_lib_dir: ?[]const u8, +build_root: Cache.Directory, +cache_root: Cache.Directory, +global_cache_root: Cache.Directory, +cache: *Cache, +/// If non-null, overrides the default zig lib dir. +zig_lib_dir: ?[]const u8, vcpkg_root: VcpkgRoot = .unattempted, pkg_config_pkg_list: ?(PkgConfigError![]const PkgConfigPkg) = null, args: ?[][]const u8 = null, @@ -187,10 +188,11 @@ pub const DirList = struct { pub fn create( allocator: Allocator, zig_exe: []const u8, - build_root: []const u8, - cache_root: []const u8, - global_cache_root: []const u8, + build_root: Cache.Directory, + cache_root: Cache.Directory, + global_cache_root: Cache.Directory, host: NativeTargetInfo, + cache: *Cache, ) !*Build { const env_map = try allocator.create(EnvMap); env_map.* = try process.getEnvMap(allocator); @@ -199,8 +201,9 @@ pub fn create( self.* = Build{ .zig_exe = zig_exe, .build_root = build_root, - .cache_root = try fs.path.relative(allocator, build_root, cache_root), + .cache_root = cache_root, .global_cache_root = global_cache_root, + .cache = cache, .verbose = false, .verbose_link = false, .verbose_cc = false, @@ -232,7 +235,7 @@ pub fn create( .step = Step.init(.top_level, "uninstall", allocator, makeUninstall), .description = "Remove build artifacts from prefix path", }, - .override_lib_dir = null, + .zig_lib_dir = null, .install_path = undefined, .args = null, .host = host, @@ -247,7 +250,7 @@ pub fn create( fn createChild( parent: *Build, dep_name: []const u8, - build_root: []const u8, + build_root: Cache.Directory, args: anytype, ) !*Build { const child = try createChildOnly(parent, dep_name, build_root); @@ -255,7 +258,7 @@ fn createChild( return child; } -fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: []const u8) !*Build { +fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: Cache.Directory) !*Build { const allocator = parent.allocator; const child = try allocator.create(Build); child.* = .{ @@ -299,7 +302,8 @@ fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: []const u8) .build_root = build_root, .cache_root = parent.cache_root, .global_cache_root = parent.global_cache_root, - .override_lib_dir = parent.override_lib_dir, + .cache = parent.cache, + .zig_lib_dir = parent.zig_lib_dir, .debug_log_scopes = parent.debug_log_scopes, .debug_compile_errors = parent.debug_compile_errors, .enable_darling = parent.enable_darling, @@ -381,7 +385,7 @@ fn applyArgs(b: *Build, args: anytype) !void { _ = std.fmt.bufPrint(&hash_basename, "{s}", .{std.fmt.fmtSliceHexLower(&digest)}) catch unreachable; - const install_prefix = b.pathJoin(&.{ b.cache_root, "i", &hash_basename }); + const install_prefix = try b.cache_root.join(b.allocator, &.{ "i", &hash_basename }); b.resolveInstallPrefix(install_prefix, .{}); } @@ -398,7 +402,7 @@ pub fn resolveInstallPrefix(self: *Build, install_prefix: ?[]const u8, dir_list: self.install_path = self.pathJoin(&.{ dest_dir, self.install_prefix }); } else { self.install_prefix = install_prefix orelse - (self.pathJoin(&.{ self.build_root, "zig-out" })); + (self.build_root.join(self.allocator, &.{"zig-out"}) catch @panic("unhandled error")); self.install_path = self.install_prefix; } @@ -698,8 +702,6 @@ pub fn addTranslateC(self: *Build, options: TranslateCStep.Options) *TranslateCS } pub fn make(self: *Build, step_names: []const []const u8) !void { - try self.makePath(self.cache_root); - var wanted_steps = ArrayList(*Step).init(self.allocator); defer wanted_steps.deinit(); @@ -1225,13 +1227,6 @@ pub fn spawnChildEnvMap(self: *Build, cwd: ?[]const u8, env_map: *const EnvMap, } } -pub fn makePath(self: *Build, path: []const u8) !void { - fs.cwd().makePath(self.pathFromRoot(path)) catch |err| { - log.err("Unable to create path {s}: {s}", .{ path, @errorName(err) }); - return err; - }; -} - pub fn installArtifact(self: *Build, artifact: *CompileStep) void { self.getInstallStep().dependOn(&self.addInstallArtifact(artifact).step); } @@ -1346,8 +1341,8 @@ pub fn truncateFile(self: *Build, dest_path: []const u8) !void { src_file.close(); } -pub fn pathFromRoot(self: *Build, rel_path: []const u8) []u8 { - return fs.path.resolve(self.allocator, &[_][]const u8{ self.build_root, rel_path }) catch @panic("OOM"); +pub fn pathFromRoot(b: *Build, p: []const u8) []u8 { + return fs.path.resolve(b.allocator, &.{ b.build_root.path orelse ".", p }) catch @panic("OOM"); } pub fn pathJoin(self: *Build, paths: []const []const u8) []u8 { @@ -1568,10 +1563,19 @@ pub fn dependency(b: *Build, name: []const u8, args: anytype) *Dependency { fn dependencyInner( b: *Build, name: []const u8, - build_root: []const u8, + build_root_string: []const u8, comptime build_zig: type, args: anytype, ) *Dependency { + const build_root: std.Build.Cache.Directory = .{ + .path = build_root_string, + .handle = std.fs.cwd().openDir(build_root_string, .{}) catch |err| { + std.debug.print("unable to open '{s}': {s}\n", .{ + build_root_string, @errorName(err), + }); + std.process.exit(1); + }, + }; const sub_builder = b.createChild(name, build_root, args) catch @panic("unhandled error"); sub_builder.runBuild(build_zig) catch @panic("unhandled error"); diff --git a/lib/std/Build/CompileStep.zig b/lib/std/Build/CompileStep.zig index d9d133c5e1..70ed4a5463 100644 --- a/lib/std/Build/CompileStep.zig +++ b/lib/std/Build/CompileStep.zig @@ -83,7 +83,7 @@ max_memory: ?u64 = null, shared_memory: bool = false, global_base: ?u64 = null, c_std: std.Build.CStd, -override_lib_dir: ?[]const u8, +zig_lib_dir: ?[]const u8, main_pkg_path: ?[]const u8, exec_cmd_args: ?[]const ?[]const u8, name_prefix: []const u8, @@ -344,7 +344,7 @@ pub fn create(builder: *std.Build, options: Options) *CompileStep { .installed_headers = ArrayList(*Step).init(builder.allocator), .object_src = undefined, .c_std = std.Build.CStd.C99, - .override_lib_dir = null, + .zig_lib_dir = null, .main_pkg_path = null, .exec_cmd_args = null, .name_prefix = "", @@ -857,7 +857,7 @@ pub fn setVerboseCC(self: *CompileStep, value: bool) void { } pub fn overrideZigLibDir(self: *CompileStep, dir_path: []const u8) void { - self.override_lib_dir = self.builder.dupePath(dir_path); + self.zig_lib_dir = self.builder.dupePath(dir_path); } pub fn setMainPkgPath(self: *CompileStep, dir_path: []const u8) void { @@ -1350,10 +1350,10 @@ fn make(step: *Step) !void { } try zig_args.append("--cache-dir"); - try zig_args.append(builder.pathFromRoot(builder.cache_root)); + try zig_args.append(builder.pathFromRoot(builder.cache_root.path orelse ".")); try zig_args.append("--global-cache-dir"); - try zig_args.append(builder.pathFromRoot(builder.global_cache_root)); + try zig_args.append(builder.pathFromRoot(builder.global_cache_root.path orelse ".")); try zig_args.append("--name"); try zig_args.append(self.name); @@ -1703,12 +1703,12 @@ fn make(step: *Step) !void { try addFlag(&zig_args, "each-lib-rpath", self.each_lib_rpath); try addFlag(&zig_args, "build-id", self.build_id); - if (self.override_lib_dir) |dir| { + if (self.zig_lib_dir) |dir| { try zig_args.append("--zig-lib-dir"); try zig_args.append(builder.pathFromRoot(dir)); - } else if (builder.override_lib_dir) |dir| { + } else if (builder.zig_lib_dir) |dir| { try zig_args.append("--zig-lib-dir"); - try zig_args.append(builder.pathFromRoot(dir)); + try zig_args.append(dir); } if (self.main_pkg_path) |dir| { @@ -1745,23 +1745,15 @@ fn make(step: *Step) !void { args_length += arg.len + 1; // +1 to account for null terminator } if (args_length >= 30 * 1024) { - const args_dir = try fs.path.join( - builder.allocator, - &[_][]const u8{ builder.pathFromRoot("zig-cache"), "args" }, - ); - try std.fs.cwd().makePath(args_dir); - - var args_arena = std.heap.ArenaAllocator.init(builder.allocator); - defer args_arena.deinit(); + try builder.cache_root.handle.makePath("args"); const args_to_escape = zig_args.items[2..]; - var escaped_args = try ArrayList([]const u8).initCapacity(args_arena.allocator(), args_to_escape.len); - + var escaped_args = try ArrayList([]const u8).initCapacity(builder.allocator, args_to_escape.len); arg_blk: for (args_to_escape) |arg| { for (arg) |c, arg_idx| { if (c == '\\' or c == '"') { // Slow path for arguments that need to be escaped. We'll need to allocate and copy - var escaped = try ArrayList(u8).initCapacity(args_arena.allocator(), arg.len + 1); + var escaped = try ArrayList(u8).initCapacity(builder.allocator, arg.len + 1); const writer = escaped.writer(); try writer.writeAll(arg[0..arg_idx]); for (arg[arg_idx..]) |to_escape| { @@ -1789,11 +1781,16 @@ fn make(step: *Step) !void { .{std.fmt.fmtSliceHexLower(&args_hash)}, ); - const args_file = try fs.path.join(builder.allocator, &[_][]const u8{ args_dir, args_hex_hash[0..] }); - try std.fs.cwd().writeFile(args_file, args); + const args_file = "args" ++ fs.path.sep_str ++ args_hex_hash; + try builder.cache_root.handle.writeFile(args_file, args); + + const resolved_args_file = try mem.concat(builder.allocator, u8, &.{ + "@", + builder.pathFromRoot(try builder.cache_root.join(builder.allocator, &.{args_file})), + }); zig_args.shrinkRetainingCapacity(2); - try zig_args.append(try std.mem.concat(builder.allocator, u8, &[_][]const u8{ "@", args_file })); + try zig_args.append(resolved_args_file); } const output_dir_nl = try builder.execFromStep(zig_args.items, &self.step); diff --git a/lib/std/Build/ConfigHeaderStep.zig b/lib/std/Build/ConfigHeaderStep.zig index a22618f34a..f8d6f7bd57 100644 --- a/lib/std/Build/ConfigHeaderStep.zig +++ b/lib/std/Build/ConfigHeaderStep.zig @@ -208,9 +208,7 @@ fn make(step: *Step) !void { .{std.fmt.fmtSliceHexLower(&digest)}, ) catch unreachable; - const output_dir = try std.fs.path.join(gpa, &[_][]const u8{ - self.builder.cache_root, "o", &hash_basename, - }); + const output_dir = try self.builder.cache_root.join(gpa, &.{ "o", &hash_basename }); // If output_path has directory parts, deal with them. Example: // output_dir is zig-cache/o/HASH diff --git a/lib/std/Build/OptionsStep.zig b/lib/std/Build/OptionsStep.zig index 8a50456539..485c8a36f5 100644 --- a/lib/std/Build/OptionsStep.zig +++ b/lib/std/Build/OptionsStep.zig @@ -234,26 +234,20 @@ fn make(step: *Step) !void { ); } - const options_directory = self.builder.pathFromRoot( - try fs.path.join( - self.builder.allocator, - &[_][]const u8{ self.builder.cache_root, "options" }, - ), - ); + var options_dir = try self.builder.cache_root.handle.makeOpenPath("options", .{}); + defer options_dir.close(); - try fs.cwd().makePath(options_directory); + const basename = self.hashContentsToFileName(); - const options_file = try fs.path.join( - self.builder.allocator, - &[_][]const u8{ options_directory, &self.hashContentsToFileName() }, - ); + try options_dir.writeFile(&basename, self.contents.items); - try fs.cwd().writeFile(options_file, self.contents.items); - - self.generated_file.path = options_file; + self.generated_file.path = try self.builder.cache_root.join(self.builder.allocator, &.{ + "options", &basename, + }); } fn hashContentsToFileName(self: *OptionsStep) [64]u8 { + // TODO update to use the cache system instead of this // This implementation is copied from `WriteFileStep.make` var hash = std.crypto.hash.blake2.Blake2b384.init(.{}); diff --git a/lib/std/Build/RunStep.zig b/lib/std/Build/RunStep.zig index da35bb26a9..5bc271409a 100644 --- a/lib/std/Build/RunStep.zig +++ b/lib/std/Build/RunStep.zig @@ -44,6 +44,10 @@ print: bool, /// running if all output files are up-to-date. condition: enum { output_outdated, always } = .output_outdated, +/// Additional file paths relative to build.zig that, when modified, indicate +/// that the RunStep should be re-executed. +extra_file_dependencies: []const []const u8 = &.{}, + pub const StdIoAction = union(enum) { inherit, ignore, @@ -184,63 +188,104 @@ fn stdIoActionToBehavior(action: StdIoAction) std.ChildProcess.StdIo { } fn needOutputCheck(self: RunStep) bool { - switch (self.condition) { - .always => return false, - .output_outdated => { - for (self.argv.items) |arg| switch (arg) { - .output => return true, - else => continue, - }; - return false; - }, - } + if (self.extra_file_dependencies.len > 0) return true; + + for (self.argv.items) |arg| switch (arg) { + .output => return true, + else => continue, + }; + + return switch (self.condition) { + .always => false, + .output_outdated => true, + }; } fn make(step: *Step) !void { const self = @fieldParentPtr(RunStep, "step", step); + const need_output_check = self.needOutputCheck(); var argv_list = ArrayList([]const u8).init(self.builder.allocator); + var output_placeholders = ArrayList(struct { + index: usize, + output: Arg.Output, + }).init(self.builder.allocator); + + var man = self.builder.cache.obtain(); + defer man.deinit(); for (self.argv.items) |arg| { switch (arg) { - .bytes => |bytes| try argv_list.append(bytes), - .file_source => |file| try argv_list.append(file.getPath(self.builder)), + .bytes => |bytes| { + try argv_list.append(bytes); + man.hash.addBytes(bytes); + }, + .file_source => |file| { + const file_path = file.getPath(self.builder); + try argv_list.append(file_path); + _ = try man.addFile(file_path, null); + }, .artifact => |artifact| { if (artifact.target.isWindows()) { // On Windows we don't have rpaths so we have to add .dll search paths to PATH self.addPathForDynLibs(artifact); } - const executable_path = artifact.installed_path orelse + const file_path = artifact.installed_path orelse artifact.getOutputSource().getPath(self.builder); - try argv_list.append(executable_path); + + try argv_list.append(file_path); + + _ = try man.addFile(file_path, null); }, .output => |output| { - // TODO: until the cache system is brought into the build system, - // we use a temporary directory here for each run. - var digest: [16]u8 = undefined; - std.crypto.random.bytes(&digest); - var hash_basename: [digest.len * 2]u8 = undefined; - _ = std.fmt.bufPrint( - &hash_basename, - "{s}", - .{std.fmt.fmtSliceHexLower(&digest)}, - ) catch unreachable; - - const output_path = try fs.path.join(self.builder.allocator, &[_][]const u8{ - self.builder.cache_root, "tmp", &hash_basename, output.basename, + man.hash.addBytes(output.basename); + // Add a placeholder into the argument list because we need the + // manifest hash to be updated with all arguments before the + // object directory is computed. + try argv_list.append(""); + try output_placeholders.append(.{ + .index = argv_list.items.len - 1, + .output = output, }); - const output_dir = fs.path.dirname(output_path).?; - fs.cwd().makePath(output_dir) catch |err| { - std.debug.print("unable to make path {s}: {s}\n", .{ output_dir, @errorName(err) }); - return err; - }; - - output.generated_file.path = output_path; - try argv_list.append(output_path); }, } } + if (need_output_check) { + for (self.extra_file_dependencies) |file_path| { + _ = try man.addFile(self.builder.pathFromRoot(file_path), null); + } + + if (man.hit() catch |err| failWithCacheError(man, err)) { + // cache hit, skip running command + const digest = man.final(); + for (output_placeholders.items) |placeholder| { + placeholder.output.generated_file.path = try self.builder.cache_root.join( + self.builder.allocator, + &.{ "o", &digest, placeholder.output.basename }, + ); + } + return; + } + + const digest = man.final(); + + for (output_placeholders.items) |placeholder| { + const output_path = try self.builder.cache_root.join( + self.builder.allocator, + &.{ "o", &digest, placeholder.output.basename }, + ); + const output_dir = fs.path.dirname(output_path).?; + fs.cwd().makePath(output_dir) catch |err| { + std.debug.print("unable to make path {s}: {s}\n", .{ output_dir, @errorName(err) }); + return err; + }; + + placeholder.output.generated_file.path = output_path; + argv_list.items[placeholder.index] = output_path; + } + } + try runCommand( argv_list.items, self.builder, @@ -252,6 +297,10 @@ fn make(step: *Step) !void { self.cwd, self.print, ); + + if (need_output_check) { + try man.writeManifest(); + } } pub fn runCommand( @@ -265,11 +314,13 @@ pub fn runCommand( maybe_cwd: ?[]const u8, print: bool, ) !void { - const cwd = if (maybe_cwd) |cwd| builder.pathFromRoot(cwd) else builder.build_root; + const cwd = if (maybe_cwd) |cwd| builder.pathFromRoot(cwd) else builder.build_root.path; if (!std.process.can_spawn) { const cmd = try std.mem.join(builder.allocator, " ", argv); - std.debug.print("the following command cannot be executed ({s} does not support spawning a child process):\n{s}", .{ @tagName(builtin.os.tag), cmd }); + std.debug.print("the following command cannot be executed ({s} does not support spawning a child process):\n{s}", .{ + @tagName(builtin.os.tag), cmd, + }); builder.allocator.free(cmd); return ExecError.ExecNotSupported; } @@ -410,6 +461,19 @@ pub fn runCommand( } } +fn failWithCacheError(man: std.Build.Cache.Manifest, err: anyerror) noreturn { + const i = man.failed_file_index orelse failWithSimpleError(err); + const pp = man.files.items[i].prefixed_path orelse failWithSimpleError(err); + const prefix = man.cache.prefixes()[pp.prefix].path orelse ""; + std.debug.print("{s}: {s}/{s}\n", .{ @errorName(err), prefix, pp.sub_path }); + std.process.exit(1); +} + +fn failWithSimpleError(err: anyerror) noreturn { + std.debug.print("{s}\n", .{@errorName(err)}); + std.process.exit(1); +} + fn printCmd(cwd: ?[]const u8, argv: []const []const u8) void { if (cwd) |yes_cwd| std.debug.print("cd {s} && ", .{yes_cwd}); for (argv) |arg| { diff --git a/lib/std/Build/WriteFileStep.zig b/lib/std/Build/WriteFileStep.zig index 3cd447e4b8..1621295ad8 100644 --- a/lib/std/Build/WriteFileStep.zig +++ b/lib/std/Build/WriteFileStep.zig @@ -85,8 +85,8 @@ fn make(step: *Step) !void { .{std.fmt.fmtSliceHexLower(&digest)}, ) catch unreachable; - const output_dir = try fs.path.join(self.builder.allocator, &[_][]const u8{ - self.builder.cache_root, "o", &hash_basename, + const output_dir = try self.builder.cache_root.join(self.builder.allocator, &.{ + "o", &hash_basename, }); var dir = fs.cwd().makeOpenPath(output_dir, .{}) catch |err| { std.debug.print("unable to make path {s}: {s}\n", .{ output_dir, @errorName(err) }); diff --git a/test/tests.zig b/test/tests.zig index 94030ce851..d3ebe5a046 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -570,7 +570,7 @@ pub fn addCliTests(b: *std.Build, test_filter: ?[]const u8, optimize_modes: []co const run_cmd = exe.run(); run_cmd.addArgs(&[_][]const u8{ fs.realpathAlloc(b.allocator, b.zig_exe) catch unreachable, - b.pathFromRoot(b.cache_root), + b.pathFromRoot(b.cache_root.path orelse "."), }); step.dependOn(&run_cmd.step); @@ -1059,7 +1059,7 @@ pub const StandaloneContext = struct { } var zig_args = ArrayList([]const u8).init(b.allocator); - const rel_zig_exe = fs.path.relative(b.allocator, b.build_root, b.zig_exe) catch unreachable; + const rel_zig_exe = fs.path.relative(b.allocator, b.build_root.path orelse ".", b.zig_exe) catch unreachable; zig_args.append(rel_zig_exe) catch unreachable; zig_args.append("build") catch unreachable; From 3f8f63b132566683ffc1d3ec51e7f49346d88f2e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 12 Feb 2023 08:17:49 -0700 Subject: [PATCH 08/11] std.Build: make cache_root and global_cache_root relative to cwd This makes it so that when there is a tree of std.Build objects, only one zig-cache is used (the top-level application) instead of polluting package directories with zig-cache folders. --- lib/build_runner.zig | 4 ++-- lib/std/Build.zig | 1 - lib/std/Build/CompileStep.zig | 6 +++--- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/lib/build_runner.zig b/lib/build_runner.zig index 53f439802e..ca78ce713f 100644 --- a/lib/build_runner.zig +++ b/lib/build_runner.zig @@ -49,12 +49,12 @@ pub fn main() !void { }; const local_cache_directory: std.Build.Cache.Directory = .{ - .path = try std.fs.path.relative(allocator, build_root, cache_root), + .path = cache_root, .handle = try std.fs.cwd().makeOpenPath(cache_root, .{}), }; const global_cache_directory: std.Build.Cache.Directory = .{ - .path = try std.fs.path.relative(allocator, build_root, global_cache_root), + .path = global_cache_root, .handle = try std.fs.cwd().makeOpenPath(global_cache_root, .{}), }; diff --git a/lib/std/Build.zig b/lib/std/Build.zig index cc4ac43916..835b083608 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -1644,7 +1644,6 @@ pub const GeneratedFile = struct { }; /// A file source is a reference to an existing or future file. -/// pub const FileSource = union(enum) { /// A plain file path, relative to build root or absolute. path: []const u8, diff --git a/lib/std/Build/CompileStep.zig b/lib/std/Build/CompileStep.zig index 70ed4a5463..1f145f8171 100644 --- a/lib/std/Build/CompileStep.zig +++ b/lib/std/Build/CompileStep.zig @@ -1350,10 +1350,10 @@ fn make(step: *Step) !void { } try zig_args.append("--cache-dir"); - try zig_args.append(builder.pathFromRoot(builder.cache_root.path orelse ".")); + try zig_args.append(builder.cache_root.path orelse "."); try zig_args.append("--global-cache-dir"); - try zig_args.append(builder.pathFromRoot(builder.global_cache_root.path orelse ".")); + try zig_args.append(builder.global_cache_root.path orelse "."); try zig_args.append("--name"); try zig_args.append(self.name); @@ -1786,7 +1786,7 @@ fn make(step: *Step) !void { const resolved_args_file = try mem.concat(builder.allocator, u8, &.{ "@", - builder.pathFromRoot(try builder.cache_root.join(builder.allocator, &.{args_file})), + try builder.cache_root.join(builder.allocator, &.{args_file}), }); zig_args.shrinkRetainingCapacity(2); From 5c1f7288d93fe3071800762f2c258bc18c07d173 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 13 Feb 2023 00:02:42 -0700 Subject: [PATCH 09/11] std.Build: delete test that doesn't test anything --- lib/std/Build.zig | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 835b083608..8adca32600 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -1596,26 +1596,6 @@ pub fn runBuild(b: *Build, build_zig: anytype) anyerror!void { } } -test "builder.findProgram compiles" { - if (builtin.os.tag == .wasi) return error.SkipZigTest; - - var arena = std.heap.ArenaAllocator.init(std.heap.page_allocator); - defer arena.deinit(); - - const host = try NativeTargetInfo.detect(.{}); - - const builder = try Build.create( - arena.allocator(), - "zig", - "zig-cache", - "zig-cache", - "zig-cache", - host, - ); - defer builder.destroy(); - _ = builder.findProgram(&[_][]const u8{}, &[_][]const u8{}) catch null; -} - pub const Module = struct { builder: *Build, /// This could either be a generated file, in which case the module From e9c7e539e486c28144b03e934fc96a20c004fbdc Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 13 Feb 2023 06:41:53 -0700 Subject: [PATCH 10/11] std.Build.OptionsStep: update test case --- lib/std/Build/OptionsStep.zig | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/lib/std/Build/OptionsStep.zig b/lib/std/Build/OptionsStep.zig index 485c8a36f5..e5c3e23821 100644 --- a/lib/std/Build/OptionsStep.zig +++ b/lib/std/Build/OptionsStep.zig @@ -283,13 +283,19 @@ test "OptionsStep" { const host = try std.zig.system.NativeTargetInfo.detect(.{}); + var cache: std.Build.Cache = .{ + .gpa = arena.allocator(), + .manifest_dir = std.fs.cwd(), + }; + var builder = try std.Build.create( arena.allocator(), "test", - "test", - "test", - "test", + .{ .path = "test", .handle = std.fs.cwd() }, + .{ .path = "test", .handle = std.fs.cwd() }, + .{ .path = "test", .handle = std.fs.cwd() }, host, + &cache, ); defer builder.destroy(); From 35bb823131a93d7407e79897d4cd20ae2a98ca54 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 13 Feb 2023 09:13:36 -0700 Subject: [PATCH 11/11] build.zig: builder.zig_exe is not relative --- build.zig | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/build.zig b/build.zig index e121850221..faf14cc405 100644 --- a/build.zig +++ b/build.zig @@ -40,12 +40,11 @@ pub fn build(b: *std.Build) !void { }); docgen_exe.single_threaded = single_threaded; - const rel_zig_exe = try b.build_root.join(b.allocator, &.{b.zig_exe}); const langref_out_path = try b.cache_root.join(b.allocator, &.{"langref.html"}); const docgen_cmd = docgen_exe.run(); docgen_cmd.addArgs(&[_][]const u8{ "--zig", - rel_zig_exe, + b.zig_exe, "doc" ++ fs.path.sep_str ++ "langref.html.in", langref_out_path, });