mirror/zig
1
0
Fork 0
mirror of https://github.com/ziglang/zig.git synced 2025-12-06 06:13:07 +00:00
Code Issues Packages Projects Releases Wiki Activity

std.Build: start using the cache system with RunStep

Browse Source 
* Use std.Build.Cache.Directory instead of a string for storing the
   cache roots and build roots.
 * Set up a std.Build.Cache in build_runner.zig and use it in
   std.Build.RunStep for avoiding redundant work.
This commit is contained in:
Andrew Kelley 2023-02-09 10:01:01 -07:00
parent 0666322614
commit d97042ad2e
9 changed files with 198 additions and 117 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
build.zig
View File

@ -40,11 +40,8 @@ pub fn build(b: *std.Build) !void {
});
docgen_exe.single_threaded = single_threaded;
const rel_zig_exe = try fs.path.relative(b.allocator, b.build_root, b.zig_exe);
const langref_out_path = fs.path.join(
b.allocator,
&[_][]const u8{ b.cache_root, "langref.html" },
) catch unreachable;
const rel_zig_exe = try b.build_root.join(b.allocator, &.{b.zig_exe});
const langref_out_path = try b.cache_root.join(b.allocator, &.{"langref.html"});
const docgen_cmd = docgen_exe.run();
docgen_cmd.addArgs(&[_][]const u8{
"--zig",
@ -215,7 +212,7 @@ pub fn build(b: *std.Build) !void {
var code: u8 = undefined;
const git_describe_untrimmed = b.execAllowFail(&[_][]const u8{
"git", "-C", b.build_root, "describe", "--match", "*.*.*", "--tags",
"git", "-C", b.build_root.path orelse ".", "describe", "--match", "*.*.*", "--tags",
}, &code, .Ignore) catch {
break :v version_string;
};

35
lib/build_runner.zig
View File

@ -43,13 +43,40 @@ pub fn main() !void {
const host = try std.zig.system.NativeTargetInfo.detect(.{});
const build_root_directory: std.Build.Cache.Directory = .{
.path = build_root,
.handle = try std.fs.cwd().openDir(build_root, .{}),
};
const local_cache_directory: std.Build.Cache.Directory = .{
.path = try std.fs.path.relative(allocator, build_root, cache_root),
.handle = try std.fs.cwd().makeOpenPath(cache_root, .{}),
};
const global_cache_directory: std.Build.Cache.Directory = .{
.path = try std.fs.path.relative(allocator, build_root, global_cache_root),
.handle = try std.fs.cwd().makeOpenPath(global_cache_root, .{}),
};
var cache: std.Build.Cache = .{
.gpa = allocator,
.manifest_dir = try local_cache_directory.handle.makeOpenPath("h", .{}),
};
cache.addPrefix(.{ .path = null, .handle = std.fs.cwd() });
cache.addPrefix(build_root_directory);
cache.addPrefix(local_cache_directory);
cache.addPrefix(global_cache_directory);
//cache.hash.addBytes(builtin.zig_version);
const builder = try std.Build.create(
allocator,
zig_exe,
build_root,
cache_root,
global_cache_root,
build_root_directory,
local_cache_directory,
global_cache_directory,
host,
&cache,
);
defer builder.destroy();
@ -138,7 +165,7 @@ pub fn main() !void {
return usageAndErr(builder, false, stderr_stream);
};
} else if (mem.eql(u8, arg, "--zig-lib-dir")) {
builder.override_lib_dir = nextArg(args, &arg_idx) orelse {
builder.zig_lib_dir = nextArg(args, &arg_idx) orelse {
std.debug.print("Expected argument after --zig-lib-dir\n\n", .{});
return usageAndErr(builder, false, stderr_stream);
};

58
lib/std/Build.zig
View File

@ -79,11 +79,12 @@ search_prefixes: ArrayList([]const u8),
libc_file: ?[]const u8 = null,
installed_files: ArrayList(InstalledFile),
/// Path to the directory containing build.zig.
build_root: []const u8,
cache_root: []const u8,
global_cache_root: []const u8,
/// zig lib dir
override_lib_dir: ?[]const u8,
build_root: Cache.Directory,
cache_root: Cache.Directory,
global_cache_root: Cache.Directory,
cache: *Cache,
/// If non-null, overrides the default zig lib dir.
zig_lib_dir: ?[]const u8,
vcpkg_root: VcpkgRoot = .unattempted,
pkg_config_pkg_list: ?(PkgConfigError![]const PkgConfigPkg) = null,
args: ?[][]const u8 = null,
@ -187,10 +188,11 @@ pub const DirList = struct {
pub fn create(
allocator: Allocator,
zig_exe: []const u8,
build_root: []const u8,
cache_root: []const u8,
global_cache_root: []const u8,
build_root: Cache.Directory,
cache_root: Cache.Directory,
global_cache_root: Cache.Directory,
host: NativeTargetInfo,
cache: *Cache,
) !*Build {
const env_map = try allocator.create(EnvMap);
env_map.* = try process.getEnvMap(allocator);
@ -199,8 +201,9 @@ pub fn create(
self.* = Build{
.zig_exe = zig_exe,
.build_root = build_root,
.cache_root = try fs.path.relative(allocator, build_root, cache_root),
.cache_root = cache_root,
.global_cache_root = global_cache_root,
.cache = cache,
.verbose = false,
.verbose_link = false,
.verbose_cc = false,
@ -232,7 +235,7 @@ pub fn create(
.step = Step.init(.top_level, "uninstall", allocator, makeUninstall),
.description = "Remove build artifacts from prefix path",
},
.override_lib_dir = null,
.zig_lib_dir = null,
.install_path = undefined,
.args = null,
.host = host,
@ -247,7 +250,7 @@ pub fn create(
fn createChild(
parent: *Build,
dep_name: []const u8,
build_root: []const u8,
build_root: Cache.Directory,
args: anytype,
) !*Build {
const child = try createChildOnly(parent, dep_name, build_root);
@ -255,7 +258,7 @@ fn createChild(
return child;
}
fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: []const u8) !*Build {
fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: Cache.Directory) !*Build {
const allocator = parent.allocator;
const child = try allocator.create(Build);
child.* = .{
@ -299,7 +302,8 @@ fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: []const u8)
.build_root = build_root,
.cache_root = parent.cache_root,
.global_cache_root = parent.global_cache_root,
.override_lib_dir = parent.override_lib_dir,
.cache = parent.cache,
.zig_lib_dir = parent.zig_lib_dir,
.debug_log_scopes = parent.debug_log_scopes,
.debug_compile_errors = parent.debug_compile_errors,
.enable_darling = parent.enable_darling,
@ -381,7 +385,7 @@ fn applyArgs(b: *Build, args: anytype) !void {
_ = std.fmt.bufPrint(&hash_basename, "{s}", .{std.fmt.fmtSliceHexLower(&digest)}) catch
unreachable;
const install_prefix = b.pathJoin(&.{ b.cache_root, "i", &hash_basename });
const install_prefix = try b.cache_root.join(b.allocator, &.{ "i", &hash_basename });
b.resolveInstallPrefix(install_prefix, .{});
}
@ -398,7 +402,7 @@ pub fn resolveInstallPrefix(self: *Build, install_prefix: ?[]const u8, dir_list:
self.install_path = self.pathJoin(&.{ dest_dir, self.install_prefix });
} else {
self.install_prefix = install_prefix orelse
(self.pathJoin(&.{ self.build_root, "zig-out" }));
(self.build_root.join(self.allocator, &.{"zig-out"}) catch @panic("unhandled error"));
self.install_path = self.install_prefix;
}
@ -698,8 +702,6 @@ pub fn addTranslateC(self: *Build, options: TranslateCStep.Options) *TranslateCS
}
pub fn make(self: *Build, step_names: []const []const u8) !void {
try self.makePath(self.cache_root);
var wanted_steps = ArrayList(*Step).init(self.allocator);
defer wanted_steps.deinit();
@ -1225,13 +1227,6 @@ pub fn spawnChildEnvMap(self: *Build, cwd: ?[]const u8, env_map: *const EnvMap,
}
}
pub fn makePath(self: *Build, path: []const u8) !void {
fs.cwd().makePath(self.pathFromRoot(path)) catch |err| {
log.err("Unable to create path {s}: {s}", .{ path, @errorName(err) });
return err;
};
}
pub fn installArtifact(self: *Build, artifact: *CompileStep) void {
self.getInstallStep().dependOn(&self.addInstallArtifact(artifact).step);
}
@ -1346,8 +1341,8 @@ pub fn truncateFile(self: *Build, dest_path: []const u8) !void {
src_file.close();
}
pub fn pathFromRoot(self: *Build, rel_path: []const u8) []u8 {
return fs.path.resolve(self.allocator, &[_][]const u8{ self.build_root, rel_path }) catch @panic("OOM");
pub fn pathFromRoot(b: *Build, p: []const u8) []u8 {
return fs.path.resolve(b.allocator, &.{ b.build_root.path orelse ".", p }) catch @panic("OOM");
}
pub fn pathJoin(self: *Build, paths: []const []const u8) []u8 {
@ -1568,10 +1563,19 @@ pub fn dependency(b: *Build, name: []const u8, args: anytype) *Dependency {
fn dependencyInner(
b: *Build,
name: []const u8,
build_root: []const u8,
build_root_string: []const u8,
comptime build_zig: type,
args: anytype,
) *Dependency {
const build_root: std.Build.Cache.Directory = .{
.path = build_root_string,
.handle = std.fs.cwd().openDir(build_root_string, .{}) catch |err| {
std.debug.print("unable to open '{s}': {s}\n", .{
build_root_string, @errorName(err),
});
std.process.exit(1);
},
};
const sub_builder = b.createChild(name, build_root, args) catch @panic("unhandled error");
sub_builder.runBuild(build_zig) catch @panic("unhandled error");

41
lib/std/Build/CompileStep.zig
View File

@ -83,7 +83,7 @@ max_memory: ?u64 = null,
shared_memory: bool = false,
global_base: ?u64 = null,
c_std: std.Build.CStd,
override_lib_dir: ?[]const u8,
zig_lib_dir: ?[]const u8,
main_pkg_path: ?[]const u8,
exec_cmd_args: ?[]const ?[]const u8,
name_prefix: []const u8,
@ -344,7 +344,7 @@ pub fn create(builder: *std.Build, options: Options) *CompileStep {
.installed_headers = ArrayList(*Step).init(builder.allocator),
.object_src = undefined,
.c_std = std.Build.CStd.C99,
.override_lib_dir = null,
.zig_lib_dir = null,
.main_pkg_path = null,
.exec_cmd_args = null,
.name_prefix = "",
@ -857,7 +857,7 @@ pub fn setVerboseCC(self: *CompileStep, value: bool) void {
}
pub fn overrideZigLibDir(self: *CompileStep, dir_path: []const u8) void {
self.override_lib_dir = self.builder.dupePath(dir_path);
self.zig_lib_dir = self.builder.dupePath(dir_path);
}
pub fn setMainPkgPath(self: *CompileStep, dir_path: []const u8) void {
@ -1350,10 +1350,10 @@ fn make(step: *Step) !void {
}
try zig_args.append("--cache-dir");
try zig_args.append(builder.pathFromRoot(builder.cache_root));
try zig_args.append(builder.pathFromRoot(builder.cache_root.path orelse "."));
try zig_args.append("--global-cache-dir");
try zig_args.append(builder.pathFromRoot(builder.global_cache_root));
try zig_args.append(builder.pathFromRoot(builder.global_cache_root.path orelse "."));
try zig_args.append("--name");
try zig_args.append(self.name);
@ -1703,12 +1703,12 @@ fn make(step: *Step) !void {
try addFlag(&zig_args, "each-lib-rpath", self.each_lib_rpath);
try addFlag(&zig_args, "build-id", self.build_id);
if (self.override_lib_dir) |dir| {
if (self.zig_lib_dir) |dir| {
try zig_args.append("--zig-lib-dir");
try zig_args.append(builder.pathFromRoot(dir));
} else if (builder.override_lib_dir) |dir| {
} else if (builder.zig_lib_dir) |dir| {
try zig_args.append("--zig-lib-dir");
try zig_args.append(builder.pathFromRoot(dir));
try zig_args.append(dir);
}
if (self.main_pkg_path) |dir| {
@ -1745,23 +1745,15 @@ fn make(step: *Step) !void {
args_length += arg.len + 1; // +1 to account for null terminator
}
if (args_length >= 30 * 1024) {
const args_dir = try fs.path.join(
builder.allocator,
&[_][]const u8{ builder.pathFromRoot("zig-cache"), "args" },
);
try std.fs.cwd().makePath(args_dir);
var args_arena = std.heap.ArenaAllocator.init(builder.allocator);
defer args_arena.deinit();
try builder.cache_root.handle.makePath("args");
const args_to_escape = zig_args.items[2..];
var escaped_args = try ArrayList([]const u8).initCapacity(args_arena.allocator(), args_to_escape.len);
var escaped_args = try ArrayList([]const u8).initCapacity(builder.allocator, args_to_escape.len);
arg_blk: for (args_to_escape) |arg| {
for (arg) |c, arg_idx| {
if (c == '\\' or c == '"') {
// Slow path for arguments that need to be escaped. We'll need to allocate and copy
var escaped = try ArrayList(u8).initCapacity(args_arena.allocator(), arg.len + 1);
var escaped = try ArrayList(u8).initCapacity(builder.allocator, arg.len + 1);
const writer = escaped.writer();
try writer.writeAll(arg[0..arg_idx]);
for (arg[arg_idx..]) |to_escape| {
@ -1789,11 +1781,16 @@ fn make(step: *Step) !void {
.{std.fmt.fmtSliceHexLower(&args_hash)},
);
const args_file = try fs.path.join(builder.allocator, &[_][]const u8{ args_dir, args_hex_hash[0..] });
try std.fs.cwd().writeFile(args_file, args);
const args_file = "args" ++ fs.path.sep_str ++ args_hex_hash;
try builder.cache_root.handle.writeFile(args_file, args);
const resolved_args_file = try mem.concat(builder.allocator, u8, &.{
"@",
builder.pathFromRoot(try builder.cache_root.join(builder.allocator, &.{args_file})),
});
zig_args.shrinkRetainingCapacity(2);
try zig_args.append(try std.mem.concat(builder.allocator, u8, &[_][]const u8{ "@", args_file }));
try zig_args.append(resolved_args_file);
}
const output_dir_nl = try builder.execFromStep(zig_args.items, &self.step);

4
lib/std/Build/ConfigHeaderStep.zig
View File

@ -208,9 +208,7 @@ fn make(step: *Step) !void {
.{std.fmt.fmtSliceHexLower(&digest)},
) catch unreachable;
const output_dir = try std.fs.path.join(gpa, &[_][]const u8{
self.builder.cache_root, "o", &hash_basename,
});
const output_dir = try self.builder.cache_root.join(gpa, &.{ "o", &hash_basename });
// If output_path has directory parts, deal with them. Example:
// output_dir is zig-cache/o/HASH

22
lib/std/Build/OptionsStep.zig
View File

@ -234,26 +234,20 @@ fn make(step: *Step) !void {
);
}
const options_directory = self.builder.pathFromRoot(
try fs.path.join(
self.builder.allocator,
&[_][]const u8{ self.builder.cache_root, "options" },
),
);
var options_dir = try self.builder.cache_root.handle.makeOpenPath("options", .{});
defer options_dir.close();
try fs.cwd().makePath(options_directory);
const basename = self.hashContentsToFileName();
const options_file = try fs.path.join(
self.builder.allocator,
&[_][]const u8{ options_directory, &self.hashContentsToFileName() },
);
try options_dir.writeFile(&basename, self.contents.items);
try fs.cwd().writeFile(options_file, self.contents.items);
self.generated_file.path = options_file;
self.generated_file.path = try self.builder.cache_root.join(self.builder.allocator, &.{
"options", &basename,
});
}
fn hashContentsToFileName(self: *OptionsStep) [64]u8 {
// TODO update to use the cache system instead of this
// This implementation is copied from `WriteFileStep.make`
var hash = std.crypto.hash.blake2.Blake2b384.init(.{});

120
lib/std/Build/RunStep.zig
View File

@ -44,6 +44,10 @@ print: bool,
/// running if all output files are up-to-date.
condition: enum { output_outdated, always } = .output_outdated,
/// Additional file paths relative to build.zig that, when modified, indicate
/// that the RunStep should be re-executed.
extra_file_dependencies: []const []const u8 = &.{},
pub const StdIoAction = union(enum) {
inherit,
ignore,
@ -184,60 +188,101 @@ fn stdIoActionToBehavior(action: StdIoAction) std.ChildProcess.StdIo {
}
fn needOutputCheck(self: RunStep) bool {
switch (self.condition) {
.always => return false,
.output_outdated => {
if (self.extra_file_dependencies.len > 0) return true;
for (self.argv.items) |arg| switch (arg) {
.output => return true,
else => continue,
};
return false;
},
}
return switch (self.condition) {
.always => false,
.output_outdated => true,
};
}
fn make(step: *Step) !void {
const self = @fieldParentPtr(RunStep, "step", step);
const need_output_check = self.needOutputCheck();
var argv_list = ArrayList([]const u8).init(self.builder.allocator);
var output_placeholders = ArrayList(struct {
index: usize,
output: Arg.Output,
}).init(self.builder.allocator);
var man = self.builder.cache.obtain();
defer man.deinit();
for (self.argv.items) |arg| {
switch (arg) {
.bytes => |bytes| try argv_list.append(bytes),
.file_source => |file| try argv_list.append(file.getPath(self.builder)),
.bytes => |bytes| {
try argv_list.append(bytes);
man.hash.addBytes(bytes);
},
.file_source => |file| {
const file_path = file.getPath(self.builder);
try argv_list.append(file_path);
_ = try man.addFile(file_path, null);
},
.artifact => |artifact| {
if (artifact.target.isWindows()) {
// On Windows we don't have rpaths so we have to add .dll search paths to PATH
self.addPathForDynLibs(artifact);
}
const executable_path = artifact.installed_path orelse
const file_path = artifact.installed_path orelse
artifact.getOutputSource().getPath(self.builder);
try argv_list.append(executable_path);
try argv_list.append(file_path);
_ = try man.addFile(file_path, null);
},
.output => |output| {
// TODO: until the cache system is brought into the build system,
// we use a temporary directory here for each run.
var digest: [16]u8 = undefined;
std.crypto.random.bytes(&digest);
var hash_basename: [digest.len * 2]u8 = undefined;
_ = std.fmt.bufPrint(
&hash_basename,
"{s}",
.{std.fmt.fmtSliceHexLower(&digest)},
) catch unreachable;
const output_path = try fs.path.join(self.builder.allocator, &[_][]const u8{
self.builder.cache_root, "tmp", &hash_basename, output.basename,
man.hash.addBytes(output.basename);
// Add a placeholder into the argument list because we need the
// manifest hash to be updated with all arguments before the
// object directory is computed.
try argv_list.append("");
try output_placeholders.append(.{
.index = argv_list.items.len - 1,
.output = output,
});
},
}
}
if (need_output_check) {
for (self.extra_file_dependencies) |file_path| {
_ = try man.addFile(self.builder.pathFromRoot(file_path), null);
}
if (man.hit() catch |err| failWithCacheError(man, err)) {
// cache hit, skip running command
const digest = man.final();
for (output_placeholders.items) |placeholder| {
placeholder.output.generated_file.path = try self.builder.cache_root.join(
self.builder.allocator,
&.{ "o", &digest, placeholder.output.basename },
);
}
return;
}
const digest = man.final();
for (output_placeholders.items) |placeholder| {
const output_path = try self.builder.cache_root.join(
self.builder.allocator,
&.{ "o", &digest, placeholder.output.basename },
);
const output_dir = fs.path.dirname(output_path).?;
fs.cwd().makePath(output_dir) catch |err| {
std.debug.print("unable to make path {s}: {s}\n", .{ output_dir, @errorName(err) });
return err;
};
output.generated_file.path = output_path;
try argv_list.append(output_path);
},
placeholder.output.generated_file.path = output_path;
argv_list.items[placeholder.index] = output_path;
}
}
@ -252,6 +297,10 @@ fn make(step: *Step) !void {
self.cwd,
self.print,
);
if (need_output_check) {
try man.writeManifest();
}
}
pub fn runCommand(
@ -265,11 +314,13 @@ pub fn runCommand(
maybe_cwd: ?[]const u8,
print: bool,
) !void {
const cwd = if (maybe_cwd) |cwd| builder.pathFromRoot(cwd) else builder.build_root;
const cwd = if (maybe_cwd) |cwd| builder.pathFromRoot(cwd) else builder.build_root.path;
if (!std.process.can_spawn) {
const cmd = try std.mem.join(builder.allocator, " ", argv);
std.debug.print("the following command cannot be executed ({s} does not support spawning a child process):\n{s}", .{ @tagName(builtin.os.tag), cmd });
std.debug.print("the following command cannot be executed ({s} does not support spawning a child process):\n{s}", .{
@tagName(builtin.os.tag), cmd,
});
builder.allocator.free(cmd);
return ExecError.ExecNotSupported;
}
@ -410,6 +461,19 @@ pub fn runCommand(
}
}
fn failWithCacheError(man: std.Build.Cache.Manifest, err: anyerror) noreturn {
const i = man.failed_file_index orelse failWithSimpleError(err);
const pp = man.files.items[i].prefixed_path orelse failWithSimpleError(err);
const prefix = man.cache.prefixes()[pp.prefix].path orelse "";
std.debug.print("{s}: {s}/{s}\n", .{ @errorName(err), prefix, pp.sub_path });
std.process.exit(1);
}
fn failWithSimpleError(err: anyerror) noreturn {
std.debug.print("{s}\n", .{@errorName(err)});
std.process.exit(1);
}
fn printCmd(cwd: ?[]const u8, argv: []const []const u8) void {
if (cwd) |yes_cwd| std.debug.print("cd {s} && ", .{yes_cwd});
for (argv) |arg| {

4
lib/std/Build/WriteFileStep.zig
View File

@ -85,8 +85,8 @@ fn make(step: *Step) !void {
.{std.fmt.fmtSliceHexLower(&digest)},
) catch unreachable;
const output_dir = try fs.path.join(self.builder.allocator, &[_][]const u8{
self.builder.cache_root, "o", &hash_basename,
const output_dir = try self.builder.cache_root.join(self.builder.allocator, &.{
"o", &hash_basename,
});
var dir = fs.cwd().makeOpenPath(output_dir, .{}) catch |err| {
std.debug.print("unable to make path {s}: {s}\n", .{ output_dir, @errorName(err) });

4
test/tests.zig
View File

@ -570,7 +570,7 @@ pub fn addCliTests(b: *std.Build, test_filter: ?[]const u8, optimize_modes: []co
const run_cmd = exe.run();
run_cmd.addArgs(&[_][]const u8{
fs.realpathAlloc(b.allocator, b.zig_exe) catch unreachable,
b.pathFromRoot(b.cache_root),
b.pathFromRoot(b.cache_root.path orelse "."),
});
step.dependOn(&run_cmd.step);
@ -1059,7 +1059,7 @@ pub const StandaloneContext = struct {
}
var zig_args = ArrayList([]const u8).init(b.allocator);
const rel_zig_exe = fs.path.relative(b.allocator, b.build_root, b.zig_exe) catch unreachable;
const rel_zig_exe = fs.path.relative(b.allocator, b.build_root.path orelse ".", b.zig_exe) catch unreachable;
zig_args.append(rel_zig_exe) catch unreachable;
zig_args.append("build") catch unreachable;