From e579c88f4ca6deaff53f0d1d226fdfc8715f85ac Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 7 Jan 2023 18:06:26 -0700 Subject: [PATCH 1/4] std.crypto.siphash: add finalResult() and peek() Useful for avoiding mutable state when using this API. --- lib/std/crypto/siphash.zig | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/lib/std/crypto/siphash.zig b/lib/std/crypto/siphash.zig index e960a3476b..e527e4558a 100644 --- a/lib/std/crypto/siphash.zig +++ b/lib/std/crypto/siphash.zig @@ -87,6 +87,11 @@ fn SipHashStateless(comptime T: type, comptime c_rounds: usize, comptime d_round self.msg_len +%= @truncate(u8, b.len); } + pub fn peek(self: Self) [digest_length]u8 { + var copy = self; + return copy.finalResult(); + } + pub fn final(self: *Self, b: []const u8) T { std.debug.assert(b.len < 8); @@ -124,6 +129,12 @@ fn SipHashStateless(comptime T: type, comptime c_rounds: usize, comptime d_round return (@as(u128, b2) << 64) | b1; } + pub fn finalResult(self: *Self) [digest_length]u8 { + var result: [digest_length]u8 = undefined; + self.final(&result); + return result; + } + fn round(self: *Self, b: [8]u8) void { const m = mem.readIntLittle(u64, b[0..8]); self.v3 ^= m; @@ -205,12 +216,23 @@ fn SipHash(comptime T: type, comptime c_rounds: usize, comptime d_rounds: usize) self.buf_len += @intCast(u8, b[off + aligned_len ..].len); } + pub fn peek(self: Self) [mac_length]u8 { + var copy = self; + return copy.finalResult(); + } + /// Return an authentication tag for the current state /// Assumes `out` is less than or equal to `mac_length`. pub fn final(self: *Self, out: *[mac_length]u8) void { mem.writeIntLittle(T, out, self.state.final(self.buf[0..self.buf_len])); } + pub fn finalResult(self: *Self) [mac_length]u8 { + var result: [mac_length]u8 = undefined; + self.final(&result); + return result; + } + /// Return an authentication tag for a message and a key pub fn create(out: *[mac_length]u8, msg: []const u8, key: *const [key_length]u8) void { var ctx = Self.init(key); From ea792011d188cff29872c4dd8e732af60765852c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 7 Jan 2023 18:07:10 -0700 Subject: [PATCH 2/4] add std.build.ConfigHeaderStep This API converts a config.h.in file into config.h. This is useful when introducing a build.zig file to an existing C/C++ project that is configured with autotools or cmake. The cmake syntax is not implemented yet. --- lib/std/build.zig | 13 ++ lib/std/build/ConfigHeaderStep.zig | 221 +++++++++++++++++++++++++++++ lib/std/build/LibExeObjStep.zig | 11 ++ lib/std/build/RunStep.zig | 4 +- lib/std/build/WriteFileStep.zig | 23 +-- 5 files changed, 260 insertions(+), 12 deletions(-) create mode 100644 lib/std/build/ConfigHeaderStep.zig diff --git a/lib/std/build.zig b/lib/std/build.zig index 4d2b5de1f1..7ce8ae2d10 100644 --- a/lib/std/build.zig +++ b/lib/std/build.zig @@ -21,6 +21,7 @@ const ThisModule = @This(); pub const CheckFileStep = @import("build/CheckFileStep.zig"); pub const CheckObjectStep = @import("build/CheckObjectStep.zig"); +pub const ConfigHeaderStep = @import("build/ConfigHeaderStep.zig"); pub const EmulatableRunStep = @import("build/EmulatableRunStep.zig"); pub const FmtStep = @import("build/FmtStep.zig"); pub const InstallArtifactStep = @import("build/InstallArtifactStep.zig"); @@ -364,6 +365,17 @@ pub const Builder = struct { return run_step; } + pub fn addConfigHeader( + b: *Builder, + source: FileSource, + style: ConfigHeaderStep.Style, + values: anytype, + ) *ConfigHeaderStep { + const config_header_step = ConfigHeaderStep.create(b, source, style); + config_header_step.addValues(values); + return config_header_step; + } + /// Allocator.dupe without the need to handle out of memory. pub fn dupe(self: *Builder, bytes: []const u8) []u8 { return self.allocator.dupe(u8, bytes) catch unreachable; @@ -1427,6 +1439,7 @@ pub const Step = struct { emulatable_run, check_file, check_object, + config_header, install_raw, options, custom, diff --git a/lib/std/build/ConfigHeaderStep.zig b/lib/std/build/ConfigHeaderStep.zig new file mode 100644 index 0000000000..fcfd876b53 --- /dev/null +++ b/lib/std/build/ConfigHeaderStep.zig @@ -0,0 +1,221 @@ +const std = @import("../std.zig"); +const ConfigHeaderStep = @This(); +const Step = std.build.Step; +const Builder = std.build.Builder; + +pub const base_id: Step.Id = .config_header; + +pub const Style = enum { + /// The configure format supported by autotools. It uses `#undef foo` to + /// mark lines that can be substituted with different values. + autoconf, + /// The configure format supported by CMake. It uses `@@FOO@@` and + /// `#cmakedefine` for template substitution. + cmake, +}; + +pub const Value = union(enum) { + undef, + defined, + boolean: bool, + int: i64, + ident: []const u8, + string: []const u8, +}; + +step: Step, +builder: *Builder, +source: std.build.FileSource, +style: Style, +values: std.StringHashMap(Value), +max_bytes: usize = 2 * 1024 * 1024, +output_dir: []const u8, +output_basename: []const u8, + +pub fn create(builder: *Builder, source: std.build.FileSource, style: Style) *ConfigHeaderStep { + const self = builder.allocator.create(ConfigHeaderStep) catch @panic("OOM"); + const name = builder.fmt("configure header {s}", .{source.getDisplayName()}); + self.* = .{ + .builder = builder, + .step = Step.init(base_id, name, builder.allocator, make), + .source = source, + .style = style, + .values = std.StringHashMap(Value).init(builder.allocator), + .output_dir = undefined, + .output_basename = "config.h", + }; + switch (source) { + .path => |p| { + const basename = std.fs.path.basename(p); + if (std.mem.endsWith(u8, basename, ".h.in")) { + self.output_basename = basename[0 .. basename.len - 3]; + } + }, + else => {}, + } + return self; +} + +pub fn addValues(self: *ConfigHeaderStep, values: anytype) void { + return addValuesInner(self, values) catch @panic("OOM"); +} + +fn addValuesInner(self: *ConfigHeaderStep, values: anytype) !void { + inline for (@typeInfo(@TypeOf(values)).Struct.fields) |field| { + switch (@typeInfo(field.type)) { + .Null => { + try self.values.put(field.name, .undef); + }, + .Void => { + try self.values.put(field.name, .defined); + }, + .Bool => { + try self.values.put(field.name, .{ .boolean = @field(values, field.name) }); + }, + .ComptimeInt => { + try self.values.put(field.name, .{ .int = @field(values, field.name) }); + }, + .EnumLiteral => { + try self.values.put(field.name, .{ .ident = @tagName(@field(values, field.name)) }); + }, + .Pointer => |ptr| { + switch (@typeInfo(ptr.child)) { + .Array => |array| { + if (ptr.size == .One and array.child == u8) { + try self.values.put(field.name, .{ .string = @field(values, field.name) }); + continue; + } + }, + else => {}, + } + + @compileError("unsupported ConfigHeaderStep value type: " ++ + @typeName(field.type)); + }, + else => @compileError("unsupported ConfigHeaderStep value type: " ++ + @typeName(field.type)), + } + } +} + +fn make(step: *Step) !void { + const self = @fieldParentPtr(ConfigHeaderStep, "step", step); + const gpa = self.builder.allocator; + const src_path = self.source.getPath(self.builder); + const contents = try std.fs.cwd().readFileAlloc(gpa, src_path, self.max_bytes); + + // The cache is used here not really as a way to speed things up - because writing + // the data to a file would probably be very fast - but as a way to find a canonical + // location to put build artifacts. + + // If, for example, a hard-coded path was used as the location to put ConfigHeaderStep + // files, then two ConfigHeaderStep executing in parallel might clobber each other. + + // TODO port the cache system from the compiler to zig std lib. Until then + // we construct the path directly, and no "cache hit" detection happens; + // the files are always written. + // Note there is very similar code over in WriteFileStep + const Hasher = std.crypto.auth.siphash.SipHash128(1, 3); + // Random bytes to make ConfigHeaderStep unique. Refresh this with new + // random bytes when ConfigHeaderStep implementation is modified in a + // non-backwards-compatible way. + var hash = Hasher.init("X1pQzdDt91Zlh7Eh"); + hash.update(self.source.getDisplayName()); + hash.update(contents); + + var digest: [16]u8 = undefined; + hash.final(&digest); + var hash_basename: [digest.len * 2]u8 = undefined; + _ = std.fmt.bufPrint( + &hash_basename, + "{s}", + .{std.fmt.fmtSliceHexLower(&digest)}, + ) catch unreachable; + + self.output_dir = try std.fs.path.join(gpa, &[_][]const u8{ + self.builder.cache_root, "o", &hash_basename, + }); + + var dir = std.fs.cwd().makeOpenPath(self.output_dir, .{}) catch |err| { + std.debug.print("unable to make path {s}: {s}\n", .{ self.output_dir, @errorName(err) }); + return err; + }; + defer dir.close(); + + var values_copy = try self.values.clone(); + defer values_copy.deinit(); + + var output = std.ArrayList(u8).init(gpa); + defer output.deinit(); + try output.ensureTotalCapacity(contents.len); + + try output.appendSlice("/* This file was generated by ConfigHeaderStep using the Zig Build System. */\n"); + + var any_errors = false; + var line_index: u32 = 0; + var line_it = std.mem.split(u8, contents, "\n"); + while (line_it.next()) |line| : (line_index += 1) { + if (!std.mem.startsWith(u8, line, "#")) { + try output.appendSlice(line); + try output.appendSlice("\n"); + continue; + } + var it = std.mem.tokenize(u8, line[1..], " \t\r"); + const undef = it.next().?; + if (!std.mem.eql(u8, undef, "undef")) { + try output.appendSlice(line); + try output.appendSlice("\n"); + continue; + } + const name = it.rest(); + const kv = values_copy.fetchRemove(name) orelse { + std.debug.print("{s}:{d}: error: unspecified config header value: '{s}'\n", .{ + src_path, line_index + 1, name, + }); + any_errors = true; + continue; + }; + switch (kv.value) { + .undef => { + try output.appendSlice("/* #undef "); + try output.appendSlice(name); + try output.appendSlice(" */\n"); + }, + .defined => { + try output.appendSlice("#define "); + try output.appendSlice(name); + try output.appendSlice("\n"); + }, + .boolean => |b| { + try output.appendSlice("#define "); + try output.appendSlice(name); + try output.appendSlice(" "); + try output.appendSlice(if (b) "true\n" else "false\n"); + }, + .int => |i| { + try output.writer().print("#define {s} {d}\n", .{ name, i }); + }, + .ident => |ident| { + try output.writer().print("#define {s} {s}\n", .{ name, ident }); + }, + .string => |string| { + // TODO: use C-specific escaping instead of zig string literals + try output.writer().print("#define {s} \"{}\"\n", .{ name, std.zig.fmtEscapes(string) }); + }, + } + } + + { + var it = values_copy.iterator(); + while (it.next()) |entry| { + const name = entry.key_ptr.*; + std.debug.print("{s}: error: config header value unused: '{s}'\n", .{ src_path, name }); + } + } + + if (any_errors) { + return error.HeaderConfigFailed; + } + + try dir.writeFile(self.output_basename, output.items); +} diff --git a/lib/std/build/LibExeObjStep.zig b/lib/std/build/LibExeObjStep.zig index 5f18be5ad3..4795ec6222 100644 --- a/lib/std/build/LibExeObjStep.zig +++ b/lib/std/build/LibExeObjStep.zig @@ -28,6 +28,7 @@ const EmulatableRunStep = std.build.EmulatableRunStep; const CheckObjectStep = std.build.CheckObjectStep; const RunStep = std.build.RunStep; const OptionsStep = std.build.OptionsStep; +const ConfigHeaderStep = std.build.ConfigHeaderStep; const LibExeObjStep = @This(); pub const base_id = .lib_exe_obj; @@ -266,6 +267,7 @@ pub const IncludeDir = union(enum) { raw_path: []const u8, raw_path_system: []const u8, other_step: *LibExeObjStep, + config_header_step: *ConfigHeaderStep, }; pub const Kind = enum { @@ -932,6 +934,11 @@ pub fn addIncludePath(self: *LibExeObjStep, path: []const u8) void { self.include_dirs.append(IncludeDir{ .raw_path = self.builder.dupe(path) }) catch unreachable; } +pub fn addConfigHeader(self: *LibExeObjStep, config_header: *ConfigHeaderStep) void { + self.step.dependOn(&config_header.step); + self.include_dirs.append(.{ .config_header_step = config_header }) catch @panic("OOM"); +} + pub fn addLibraryPath(self: *LibExeObjStep, path: []const u8) void { self.lib_paths.append(self.builder.dupe(path)) catch unreachable; } @@ -1684,6 +1691,10 @@ fn make(step: *Step) !void { try zig_args.append("-isystem"); try zig_args.append(fs.path.dirname(h_path).?); }, + .config_header_step => |config_header| { + try zig_args.append("-I"); + try zig_args.append(config_header.output_dir); + }, } } diff --git a/lib/std/build/RunStep.zig b/lib/std/build/RunStep.zig index 3422b83a6d..5183a328cd 100644 --- a/lib/std/build/RunStep.zig +++ b/lib/std/build/RunStep.zig @@ -17,7 +17,7 @@ const max_stdout_size = 1 * 1024 * 1024; // 1 MiB const RunStep = @This(); -pub const base_id = .run; +pub const base_id: Step.Id = .run; step: Step, builder: *Builder, @@ -59,7 +59,7 @@ pub fn create(builder: *Builder, name: []const u8) *RunStep { const self = builder.allocator.create(RunStep) catch unreachable; self.* = RunStep{ .builder = builder, - .step = Step.init(.run, name, builder.allocator, make), + .step = Step.init(base_id, name, builder.allocator, make), .argv = ArrayList(Arg).init(builder.allocator), .cwd = null, .env_map = null, diff --git a/lib/std/build/WriteFileStep.zig b/lib/std/build/WriteFileStep.zig index e608f1bd9c..cb3f005927 100644 --- a/lib/std/build/WriteFileStep.zig +++ b/lib/std/build/WriteFileStep.zig @@ -63,14 +63,15 @@ fn make(step: *Step) !void { // files, then two WriteFileSteps executing in parallel might clobber each other. // TODO port the cache system from the compiler to zig std lib. Until then - // we use blake2b directly and construct the path, and no "cache hit" - // detection happens; the files are always written. - var hash = std.crypto.hash.blake2.Blake2b384.init(.{}); - + // we directly construct the path, and no "cache hit" detection happens; + // the files are always written. + // Note there is similar code over in ConfigHeaderStep. + const Hasher = std.crypto.auth.siphash.SipHash128(1, 3); // Random bytes to make WriteFileStep unique. Refresh this with // new random bytes when WriteFileStep implementation is modified // in a non-backwards-compatible way. - hash.update("eagVR1dYXoE7ARDP"); + var hash = Hasher.init("eagVR1dYXoE7ARDP"); + { var it = self.files.first; while (it) |node| : (it = node.next) { @@ -79,21 +80,23 @@ fn make(step: *Step) !void { hash.update("|"); } } - var digest: [48]u8 = undefined; + var digest: [16]u8 = undefined; hash.final(&digest); var hash_basename: [64]u8 = undefined; - _ = fs.base64_encoder.encode(&hash_basename, &digest); + _ = std.fmt.bufPrint( + &hash_basename, + "{s}", + .{std.fmt.fmtSliceHexLower(&digest)}, + ) catch unreachable; self.output_dir = try fs.path.join(self.builder.allocator, &[_][]const u8{ self.builder.cache_root, "o", &hash_basename, }); - // TODO replace with something like fs.makePathAndOpenDir - fs.cwd().makePath(self.output_dir) catch |err| { + var dir = fs.cwd().makeOpenPath(self.output_dir, .{}) catch |err| { std.debug.print("unable to make path {s}: {s}\n", .{ self.output_dir, @errorName(err) }); return err; }; - var dir = try fs.cwd().openDir(self.output_dir, .{}); defer dir.close(); { var it = self.files.first; From 01e34c1cd9a1cb56563e6c611c19d864c81de3c3 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 7 Jan 2023 18:12:20 -0700 Subject: [PATCH 3/4] std.build.ConfigHeaderStep: stub out cmake style --- lib/std/build/ConfigHeaderStep.zig | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/lib/std/build/ConfigHeaderStep.zig b/lib/std/build/ConfigHeaderStep.zig index fcfd876b53..23a21a1481 100644 --- a/lib/std/build/ConfigHeaderStep.zig +++ b/lib/std/build/ConfigHeaderStep.zig @@ -151,6 +151,20 @@ fn make(step: *Step) !void { try output.appendSlice("/* This file was generated by ConfigHeaderStep using the Zig Build System. */\n"); + switch (self.style) { + .autoconf => try render_autoconf(contents, &output, &values_copy, src_path), + .cmake => try render_cmake(contents, &output, &values_copy, src_path), + } + + try dir.writeFile(self.output_basename, output.items); +} + +fn render_autoconf( + contents: []const u8, + output: *std.ArrayList(u8), + values_copy: *std.StringHashMap(Value), + src_path: []const u8, +) !void { var any_errors = false; var line_index: u32 = 0; var line_it = std.mem.split(u8, contents, "\n"); @@ -216,6 +230,17 @@ fn make(step: *Step) !void { if (any_errors) { return error.HeaderConfigFailed; } - - try dir.writeFile(self.output_basename, output.items); +} + +fn render_cmake( + contents: []const u8, + output: *std.ArrayList(u8), + values_copy: *std.StringHashMap(Value), + src_path: []const u8, +) !void { + _ = contents; + _ = output; + _ = values_copy; + _ = src_path; + @panic("TODO: render_cmake is not implemented yet"); } From c53a556a61fabbfa664d0bc433d0edf8092ee0db Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 7 Jan 2023 19:07:24 -0700 Subject: [PATCH 4/4] std.build.WriteFileStep: fix regression in this branch This branch makes WriteFileStep use the same hashing algorithm as our cache system, reducing the divergence between zig build and the cache system. --- lib/std/build/ConfigHeaderStep.zig | 1 - lib/std/build/WriteFileStep.zig | 7 +++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/lib/std/build/ConfigHeaderStep.zig b/lib/std/build/ConfigHeaderStep.zig index 23a21a1481..4df4b827e7 100644 --- a/lib/std/build/ConfigHeaderStep.zig +++ b/lib/std/build/ConfigHeaderStep.zig @@ -135,7 +135,6 @@ fn make(step: *Step) !void { self.output_dir = try std.fs.path.join(gpa, &[_][]const u8{ self.builder.cache_root, "o", &hash_basename, }); - var dir = std.fs.cwd().makeOpenPath(self.output_dir, .{}) catch |err| { std.debug.print("unable to make path {s}: {s}\n", .{ self.output_dir, @errorName(err) }); return err; diff --git a/lib/std/build/WriteFileStep.zig b/lib/std/build/WriteFileStep.zig index cb3f005927..4faae8f74e 100644 --- a/lib/std/build/WriteFileStep.zig +++ b/lib/std/build/WriteFileStep.zig @@ -82,16 +82,15 @@ fn make(step: *Step) !void { } var digest: [16]u8 = undefined; hash.final(&digest); - var hash_basename: [64]u8 = undefined; + var hash_basename: [digest.len * 2]u8 = undefined; _ = std.fmt.bufPrint( &hash_basename, "{s}", .{std.fmt.fmtSliceHexLower(&digest)}, ) catch unreachable; + self.output_dir = try fs.path.join(self.builder.allocator, &[_][]const u8{ - self.builder.cache_root, - "o", - &hash_basename, + self.builder.cache_root, "o", &hash_basename, }); var dir = fs.cwd().makeOpenPath(self.output_dir, .{}) catch |err| { std.debug.print("unable to make path {s}: {s}\n", .{ self.output_dir, @errorName(err) });