From 9959319d5395a668867c90f63b5f82eb03f0ac9f Mon Sep 17 00:00:00 2001 From: Matt Knight Date: Thu, 10 Aug 2023 15:32:55 -0700 Subject: [PATCH] Compare user input for multiple dependency build variants (#16600) --- lib/std/Build.zig | 247 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 216 insertions(+), 31 deletions(-) diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 74aea80931..463e2c8492 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -127,7 +127,42 @@ dep_prefix: []const u8 = "", modules: std.StringArrayHashMap(*Module), /// A map from build root dirs to the corresponding `*Dependency`. This is shared with all child /// `Build`s. -initialized_deps: *std.StringHashMap(*Dependency), +initialized_deps: *InitializedDepMap, + +const InitializedDepMap = std.HashMap(InitializedDepKey, *Dependency, InitializedDepContext, std.hash_map.default_max_load_percentage); +const InitializedDepKey = struct { + build_root_string: []const u8, + user_input_options: UserInputOptionsMap, +}; + +const InitializedDepContext = struct { + allocator: Allocator, + + pub fn hash(self: @This(), k: InitializedDepKey) u64 { + var hasher = std.hash.Wyhash.init(0); + hasher.update(k.build_root_string); + hashUserInputOptionsMap(self.allocator, k.user_input_options, &hasher); + return hasher.final(); + } + + pub fn eql(self: @This(), lhs: InitializedDepKey, rhs: InitializedDepKey) bool { + _ = self; + if (!std.mem.eql(u8, lhs.build_root_string, rhs.build_root_string)) + return false; + + if (lhs.user_input_options.count() != rhs.user_input_options.count()) + return false; + + var it = lhs.user_input_options.iterator(); + while (it.next()) |lhs_entry| { + const rhs_value = rhs.user_input_options.get(lhs_entry.key_ptr.*) orelse return false; + if (!userValuesAreSame(lhs_entry.value_ptr.*.value, rhs_value.value)) + return false; + } + + return true; + } +}; pub const ExecError = error{ ReadFailure, @@ -213,8 +248,8 @@ pub fn create( const env_map = try allocator.create(EnvMap); env_map.* = try process.getEnvMap(allocator); - const initialized_deps = try allocator.create(std.StringHashMap(*Dependency)); - initialized_deps.* = std.StringHashMap(*Dependency).init(allocator); + const initialized_deps = try allocator.create(InitializedDepMap); + initialized_deps.* = InitializedDepMap.initContext(allocator, .{ .allocator = allocator }); const self = try allocator.create(Build); self.* = .{ @@ -280,14 +315,14 @@ fn createChild( parent: *Build, dep_name: []const u8, build_root: Cache.Directory, - args: anytype, + user_input_options: UserInputOptionsMap, ) !*Build { - const child = try createChildOnly(parent, dep_name, build_root); - try applyArgs(child, args); + const child = try createChildOnly(parent, dep_name, build_root, user_input_options); + try determineAndApplyInstallPrefix(child); return child; } -fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: Cache.Directory) !*Build { +fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: Cache.Directory, user_input_options: UserInputOptionsMap) !*Build { const allocator = parent.allocator; const child = try allocator.create(Build); child.* = .{ @@ -309,7 +344,7 @@ fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: Cache.Direc }), .description = "Remove build artifacts from prefix path", }, - .user_input_options = UserInputOptionsMap.init(allocator), + .user_input_options = user_input_options, .available_options_map = AvailableOptionsMap.init(allocator), .available_options_list = ArrayList(AvailableOption).init(allocator), .verbose = parent.verbose, @@ -361,57 +396,152 @@ fn createChildOnly(parent: *Build, dep_name: []const u8, build_root: Cache.Direc return child; } -fn applyArgs(b: *Build, args: anytype) !void { +fn userInputOptionsFromArgs(allocator: Allocator, args: anytype) UserInputOptionsMap { + var user_input_options = UserInputOptionsMap.init(allocator); inline for (@typeInfo(@TypeOf(args)).Struct.fields) |field| { const v = @field(args, field.name); const T = @TypeOf(v); switch (T) { CrossTarget => { - try b.user_input_options.put(field.name, .{ + user_input_options.put(field.name, .{ .name = field.name, - .value = .{ .scalar = try v.zigTriple(b.allocator) }, + .value = .{ .scalar = v.zigTriple(allocator) catch @panic("OOM") }, .used = false, - }); - try b.user_input_options.put("cpu", .{ + }) catch @panic("OOM"); + user_input_options.put("cpu", .{ .name = "cpu", - .value = .{ .scalar = try serializeCpu(b.allocator, v.getCpu()) }, + .value = .{ .scalar = serializeCpu(allocator, v.getCpu()) catch unreachable }, .used = false, - }); + }) catch @panic("OOM"); }, []const u8 => { - try b.user_input_options.put(field.name, .{ + user_input_options.put(field.name, .{ .name = field.name, .value = .{ .scalar = v }, .used = false, - }); + }) catch @panic("OOM"); }, else => switch (@typeInfo(T)) { .Bool => { - try b.user_input_options.put(field.name, .{ + user_input_options.put(field.name, .{ .name = field.name, .value = .{ .scalar = if (v) "true" else "false" }, .used = false, - }); + }) catch @panic("OOM"); }, .Enum, .EnumLiteral => { - try b.user_input_options.put(field.name, .{ + user_input_options.put(field.name, .{ .name = field.name, .value = .{ .scalar = @tagName(v) }, .used = false, - }); + }) catch @panic("OOM"); }, .Int => { - try b.user_input_options.put(field.name, .{ + user_input_options.put(field.name, .{ .name = field.name, - .value = .{ .scalar = try std.fmt.allocPrint(b.allocator, "{d}", .{v}) }, + .value = .{ .scalar = std.fmt.allocPrint(allocator, "{d}", .{v}) catch @panic("OOM") }, .used = false, - }); + }) catch @panic("OOM"); }, else => @compileError("option '" ++ field.name ++ "' has unsupported type: " ++ @typeName(T)), }, } } + return user_input_options; +} + +const OrderedUserValue = union(enum) { + flag: void, + scalar: []const u8, + list: ArrayList([]const u8), + map: ArrayList(Pair), + + const Pair = struct { + name: []const u8, + value: OrderedUserValue, + fn lessThan(_: void, lhs: Pair, rhs: Pair) bool { + return std.ascii.lessThanIgnoreCase(lhs.name, rhs.name); + } + }; + + fn hash(self: OrderedUserValue, hasher: *std.hash.Wyhash) void { + switch (self) { + .flag => {}, + .scalar => |scalar| hasher.update(scalar), + // lists are already ordered + .list => |list| for (list.items) |list_entry| + hasher.update(list_entry), + .map => |map| for (map.items) |map_entry| { + hasher.update(map_entry.name); + map_entry.value.hash(hasher); + }, + } + } + + fn mapFromUnordered(allocator: Allocator, unordered: std.StringHashMap(*const UserValue)) ArrayList(Pair) { + var ordered = ArrayList(Pair).init(allocator); + var it = unordered.iterator(); + while (it.next()) |entry| { + ordered.append(.{ + .name = entry.key_ptr.*, + .value = OrderedUserValue.fromUnordered(allocator, entry.value_ptr.*.*), + }) catch @panic("OOM"); + } + + std.mem.sortUnstable(Pair, ordered.items, {}, Pair.lessThan); + return ordered; + } + + fn fromUnordered(allocator: Allocator, unordered: UserValue) OrderedUserValue { + return switch (unordered) { + .flag => .{ .flag = {} }, + .scalar => |scalar| .{ .scalar = scalar }, + .list => |list| .{ .list = list }, + .map => |map| .{ .map = OrderedUserValue.mapFromUnordered(allocator, map) }, + }; + } +}; + +const OrderedUserInputOption = struct { + name: []const u8, + value: OrderedUserValue, + used: bool, + + fn hash(self: OrderedUserInputOption, hasher: *std.hash.Wyhash) void { + hasher.update(self.name); + self.value.hash(hasher); + } + + fn fromUnordered(allocator: Allocator, user_input_option: UserInputOption) OrderedUserInputOption { + return OrderedUserInputOption{ + .name = user_input_option.name, + .used = user_input_option.used, + .value = OrderedUserValue.fromUnordered(allocator, user_input_option.value), + }; + } + + fn lessThan(_: void, lhs: OrderedUserInputOption, rhs: OrderedUserInputOption) bool { + return std.ascii.lessThanIgnoreCase(lhs.name, rhs.name); + } +}; + +// The hash should be consistent with the same values given a different order. +// This function takes a user input map, orders it, then hashes the contents. +fn hashUserInputOptionsMap(allocator: Allocator, user_input_options: UserInputOptionsMap, hasher: *std.hash.Wyhash) void { + var ordered = ArrayList(OrderedUserInputOption).init(allocator); + var it = user_input_options.iterator(); + while (it.next()) |entry| + ordered.append(OrderedUserInputOption.fromUnordered(allocator, entry.value_ptr.*)) catch @panic("OOM"); + + std.mem.sortUnstable(OrderedUserInputOption, ordered.items, {}, OrderedUserInputOption.lessThan); + + // juice it + for (ordered.items) |user_option| + user_option.hash(hasher); +} + +fn determineAndApplyInstallPrefix(b: *Build) !void { // Create an installation directory local to this package. This will be used when // dependant packages require a standard prefix, such as include directories for C headers. var hash = b.cache.hash; @@ -419,7 +549,11 @@ fn applyArgs(b: *Build, args: anytype) !void { // implementation is modified in a non-backwards-compatible way. hash.add(@as(u32, 0xd8cb0055)); hash.addBytes(b.dep_prefix); - // TODO additionally update the hash with `args`. + + var wyhash = std.hash.Wyhash.init(0); + hashUserInputOptionsMap(b.allocator, b.user_input_options, &wyhash); + hash.add(wyhash.final()); + const digest = hash.final(); const install_prefix = try b.cache_root.join(b.allocator, &.{ "i", &digest }); b.resolveInstallPrefix(install_prefix, .{}); @@ -1597,6 +1731,53 @@ pub fn anonymousDependency( return dependencyInner(b, name, build_root, build_zig, args); } +fn userValuesAreSame(lhs: UserValue, rhs: UserValue) bool { + switch (lhs) { + .flag => {}, + .scalar => |lhs_scalar| { + const rhs_scalar = switch (rhs) { + .scalar => |scalar| scalar, + else => return false, + }; + + if (!std.mem.eql(u8, lhs_scalar, rhs_scalar)) + return false; + }, + .list => |lhs_list| { + const rhs_list = switch (rhs) { + .list => |list| list, + else => return false, + }; + + if (lhs_list.items.len != rhs_list.items.len) + return false; + + for (lhs_list.items, rhs_list.items) |lhs_list_entry, rhs_list_entry| { + if (!std.mem.eql(u8, lhs_list_entry, rhs_list_entry)) + return false; + } + }, + .map => |lhs_map| { + const rhs_map = switch (rhs) { + .map => |map| map, + else => return false, + }; + + if (lhs_map.count() != rhs_map.count()) + return false; + + var lhs_it = lhs_map.iterator(); + while (lhs_it.next()) |lhs_entry| { + const rhs_value = rhs_map.get(lhs_entry.key_ptr.*) orelse return false; + if (!userValuesAreSame(lhs_entry.value_ptr.*.*, rhs_value.*)) + return false; + } + }, + } + + return true; +} + pub fn dependencyInner( b: *Build, name: []const u8, @@ -1604,10 +1785,12 @@ pub fn dependencyInner( comptime build_zig: type, args: anytype, ) *Dependency { - if (b.initialized_deps.get(build_root_string)) |dep| { - // TODO: check args are the same + const user_input_options = userInputOptionsFromArgs(b.allocator, args); + if (b.initialized_deps.get(.{ + .build_root_string = build_root_string, + .user_input_options = user_input_options, + })) |dep| return dep; - } const build_root: std.Build.Cache.Directory = .{ .path = build_root_string, @@ -1618,7 +1801,7 @@ pub fn dependencyInner( process.exit(1); }, }; - const sub_builder = b.createChild(name, build_root, args) catch @panic("unhandled error"); + const sub_builder = b.createChild(name, build_root, user_input_options) catch @panic("unhandled error"); sub_builder.runBuild(build_zig) catch @panic("unhandled error"); if (sub_builder.validateUserInputDidItFail()) { @@ -1628,8 +1811,10 @@ pub fn dependencyInner( const dep = b.allocator.create(Dependency) catch @panic("OOM"); dep.* = .{ .builder = sub_builder }; - b.initialized_deps.put(build_root_string, dep) catch @panic("OOM"); - + b.initialized_deps.put(.{ + .build_root_string = build_root_string, + .user_input_options = user_input_options, + }, dep) catch @panic("OOM"); return dep; }