From f91503e5773ddf4ce3989533ed586ace81e41e90 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Sat, 25 Jun 2022 00:33:29 +0200 Subject: [PATCH 1/4] link-tests: defer parsing of the RPN program until running the action --- lib/std/build/CheckObjectStep.zig | 217 +++++++++++++++++++----------- test/link/macho/entry/build.zig | 2 +- 2 files changed, 137 insertions(+), 82 deletions(-) diff --git a/lib/std/build/CheckObjectStep.zig b/lib/std/build/CheckObjectStep.zig index 65a57f8832..375e183231 100644 --- a/lib/std/build/CheckObjectStep.zig +++ b/lib/std/build/CheckObjectStep.zig @@ -3,6 +3,7 @@ const assert = std.debug.assert; const build = std.build; const fs = std.fs; const macho = std.macho; +const math = std.math; const mem = std.mem; const testing = std.testing; @@ -36,20 +37,24 @@ pub fn create(builder: *Builder, source: build.FileSource, obj_format: std.Targe return self; } -const Action = union(enum) { - match: MatchAction, - compute_eq: ComputeEqAction, -}; - -/// MatchAction is the main building block of standard matchers with optional eat-all token `{*}` +/// There two types of actions currently suported: +/// * `.match` - is the main building block of standard matchers with optional eat-all token `{*}` /// and extractors by name such as `{n_value}`. Please note this action is very simplistic in nature /// i.e., it won't really handle edge cases/nontrivial examples. But given that we do want to use /// it mainly to test the output of our object format parser-dumpers when testing the linkers, etc. /// it should be plenty useful in its current form. -const MatchAction = struct { - needle: []const u8, +/// * `.compute_cmp` - can be used to perform an operation on the extracted global variables +/// using the MatchAction. It currently only supports an addition. The operation is required +/// to be specified in Reverse Polish Notation to ease in operator-precedence parsing (well, +/// to avoid any parsing really). +/// For example, if the two extracted values were saved as `vmaddr` and `entryoff` respectively +/// they could then be added with this simple program `vmaddr entryoff +`. +const Action = struct { + tag: enum { match, compute_cmp }, + phrase: []const u8, + expected: ?ComputeCompareExpected = null, - /// Will return true if the `needle` was found in the `haystack`. + /// Will return true if the `phrase` was found in the `haystack`. /// Some examples include: /// /// LC 0 => will match in its entirety @@ -57,9 +62,11 @@ const MatchAction = struct { /// and save under `vmaddr` global name (see `global_vars` param) /// name {*}libobjc{*}.dylib => will match `name` followed by a token which contains `libobjc` and `.dylib` /// in that order with other letters in between - fn match(act: MatchAction, haystack: []const u8, global_vars: anytype) !bool { + fn match(act: Action, haystack: []const u8, global_vars: anytype) !bool { + assert(act.tag == .match); + var hay_it = mem.tokenize(u8, mem.trim(u8, haystack, " "), " "); - var needle_it = mem.tokenize(u8, mem.trim(u8, act.needle, " "), " "); + var needle_it = mem.tokenize(u8, mem.trim(u8, act.phrase, " "), " "); while (needle_it.next()) |needle_tok| { const hay_tok = hay_it.next() orelse return false; @@ -93,22 +100,80 @@ const MatchAction = struct { return true; } + + /// Will return true if the `phrase` is correctly parsed into an RPN program and + /// its reduced, computed value compares using `op` with the expected value, either + /// a literal or another extracted variable. + fn computeCmp(act: Action, gpa: Allocator, global_vars: anytype) !bool { + var op_stack = std.ArrayList(enum { add }).init(gpa); + var values = std.ArrayList(u64).init(gpa); + + var it = mem.tokenize(u8, act.phrase, " "); + while (it.next()) |next| { + if (mem.eql(u8, next, "+")) { + try op_stack.append(.add); + } else { + const val = global_vars.get(next) orelse { + std.debug.print( + \\ + \\========= Variable was not extracted: =========== + \\{s} + \\ + , .{next}); + return error.UnknownVariable; + }; + try values.append(val); + } + } + + var op_i: usize = 1; + var reduced: u64 = values.items[0]; + for (op_stack.items) |op| { + const other = values.items[op_i]; + switch (op) { + .add => { + reduced += other; + }, + } + } + + const exp_value = switch (act.expected.?.value) { + .variable => |name| global_vars.get(name) orelse { + std.debug.print( + \\ + \\========= Variable was not extracted: =========== + \\{s} + \\ + , .{name}); + return error.UnknownVariable; + }, + .literal => |x| x, + }; + return math.compare(reduced, act.expected.?.op, exp_value); + } }; -/// ComputeEqAction can be used to perform an operation on the extracted global variables -/// using the MatchAction. It currently only supports an addition. The operation is required -/// to be specified in Reverse Polish Notation to ease in operator-precedence parsing (well, -/// to avoid any parsing really). -/// For example, if the two extracted values were saved as `vmaddr` and `entryoff` respectively -/// they could then be added with this simple program `vmaddr entryoff +`. -const ComputeEqAction = struct { - expected: []const u8, - var_stack: std.ArrayList([]const u8), - op_stack: std.ArrayList(Op), +const ComputeCompareExpected = struct { + op: math.CompareOperator, + value: union(enum) { + variable: []const u8, + literal: u64, + }, - const Op = enum { - add, - }; + pub fn format( + value: @This(), + comptime fmt: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) !void { + _ = fmt; + _ = options; + try writer.print("{s} ", .{@tagName(value.op)}); + switch (value.value) { + .variable => |name| try writer.writeAll(name), + .literal => |x| try writer.print("{x}", .{x}), + } + } }; const Check = struct { @@ -122,15 +187,18 @@ const Check = struct { }; } - fn match(self: *Check, needle: []const u8) void { + fn match(self: *Check, phrase: []const u8) void { self.actions.append(.{ - .match = .{ .needle = self.builder.dupe(needle) }, + .tag = .match, + .phrase = self.builder.dupe(phrase), }) catch unreachable; } - fn computeEq(self: *Check, act: ComputeEqAction) void { + fn computeCmp(self: *Check, phrase: []const u8, expected: ComputeCompareExpected) void { self.actions.append(.{ - .compute_eq = act, + .tag = .compute_cmp, + .phrase = self.builder.dupe(phrase), + .expected = expected, }) catch unreachable; } }; @@ -165,25 +233,13 @@ pub fn checkInSymtab(self: *CheckObjectStep) void { /// Creates a new standalone, singular check which allows running simple binary operations /// on the extracted variables. It will then compare the reduced program with the value of /// the expected variable. -pub fn checkComputeEq(self: *CheckObjectStep, program: []const u8, expected: []const u8) void { - const gpa = self.builder.allocator; - var ca = ComputeEqAction{ - .expected = expected, - .var_stack = std.ArrayList([]const u8).init(gpa), - .op_stack = std.ArrayList(ComputeEqAction.Op).init(gpa), - }; - - var it = mem.tokenize(u8, program, " "); - while (it.next()) |next| { - if (mem.eql(u8, next, "+")) { - ca.op_stack.append(.add) catch unreachable; - } else { - ca.var_stack.append(self.builder.dupe(next)) catch unreachable; - } - } - +pub fn checkComputeCompare( + self: *CheckObjectStep, + program: []const u8, + expected: ComputeCompareExpected, +) void { var new_check = Check.create(self.builder); - new_check.computeEq(ca); + new_check.computeCmp(program, expected); self.checks.append(new_check) catch unreachable; } @@ -210,10 +266,10 @@ fn make(step: *Step) !void { for (self.checks.items) |chk| { var it = mem.tokenize(u8, output, "\r\n"); for (chk.actions.items) |act| { - switch (act) { - .match => |match_act| { + switch (act.tag) { + .match => { while (it.next()) |line| { - if (try match_act.match(line, &vars)) break; + if (try act.match(line, &vars)) break; } else { std.debug.print( \\ @@ -222,51 +278,33 @@ fn make(step: *Step) !void { \\========= But parsed file does not contain it: ======= \\{s} \\ - , .{ match_act.needle, output }); + , .{ act.phrase, output }); return error.TestFailed; } }, - .compute_eq => |c_eq| { - var values = std.ArrayList(u64).init(gpa); - try values.ensureTotalCapacity(c_eq.var_stack.items.len); - for (c_eq.var_stack.items) |vv| { - const val = vars.get(vv) orelse { + .compute_cmp => { + const res = act.computeCmp(gpa, vars) catch |err| switch (err) { + error.UnknownVariable => { std.debug.print( - \\ - \\========= Variable was not extracted: =========== - \\{s} \\========= From parsed file: ===================== \\{s} \\ - , .{ vv, output }); + , .{output}); return error.TestFailed; - }; - values.appendAssumeCapacity(val); - } - - var op_i: usize = 1; - var reduced: u64 = values.items[0]; - for (c_eq.op_stack.items) |op| { - const other = values.items[op_i]; - switch (op) { - .add => { - reduced += other; - }, - } - } - - const expected = vars.get(c_eq.expected) orelse { + }, + else => |e| return e, + }; + if (!res) { std.debug.print( \\ - \\========= Variable was not extracted: =========== - \\{s} - \\========= From parsed file: ===================== + \\========= Comparison failed for action: =========== + \\{s} {s} + \\========= From parsed file: ======================= \\{s} \\ - , .{ c_eq.expected, output }); + , .{ act.phrase, act.expected.?, output }); return error.TestFailed; - }; - try testing.expectEqual(reduced, expected); + } }, } } @@ -349,6 +387,23 @@ const MachODumper = struct { seg.fileoff, seg.filesize, }); + + for (lc.segment.sections.items) |sect| { + try writer.writeByte('\n'); + try writer.print( + \\sectname {s} + \\addr {x} + \\size {x} + \\offset {x} + \\align {x} + , .{ + sect.sectName(), + sect.addr, + sect.size, + sect.offset, + sect.@"align", + }); + } }, .ID_DYLIB, diff --git a/test/link/macho/entry/build.zig b/test/link/macho/entry/build.zig index f0ac084e6a..1cd9099985 100644 --- a/test/link/macho/entry/build.zig +++ b/test/link/macho/entry/build.zig @@ -24,7 +24,7 @@ pub fn build(b: *Builder) void { check_exe.checkInSymtab(); check_exe.checkNext("_non_main {n_value}"); - check_exe.checkComputeEq("vmaddr entryoff +", "n_value"); + check_exe.checkComputeCompare("vmaddr entryoff +", .{ .op = .eq, .value = .{ .variable = "n_value" } }); test_step.dependOn(&check_exe.step); From 8c1feef4cd3af223ed8278bd55e8191936b526f0 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Sat, 25 Jun 2022 00:34:16 +0200 Subject: [PATCH 2/4] macho: implement -headerpad_size option Includes both traditiona and incremental codepaths with one caveat that in incremental case, the requested size cannot be smaller than the default padding size due to prealloc required due to incremental nature of linking. Also parse `-headerpad_max_install_names`, however, not actionable just yet - missing implementation. --- lib/std/build.zig | 15 +++++++++++++++ src/Compilation.zig | 6 ++++++ src/link.zig | 6 ++++++ src/link/MachO.zig | 35 ++++++++++++++++++++++++++++------- src/main.zig | 26 ++++++++++++++++++++++++++ 5 files changed, 81 insertions(+), 7 deletions(-) diff --git a/lib/std/build.zig b/lib/std/build.zig index 0b49087747..20f59040c9 100644 --- a/lib/std/build.zig +++ b/lib/std/build.zig @@ -1593,6 +1593,14 @@ pub const LibExeObjStep = struct { /// search strategy. search_strategy: ?enum { paths_first, dylibs_first } = null, + /// (Darwin) Set size of the padding between the end of load commands + /// and start of `__TEXT,__text` section. + headerpad_size: ?u64 = null, + + /// (Darwin) Automatically Set size of the padding between the end of load commands + /// and start of `__TEXT,__text` section to a value fitting all paths expanded to MAXPATHLEN. + headerpad_max_install_names: bool = false, + /// Position Independent Code force_pic: ?bool = null, @@ -2661,6 +2669,13 @@ pub const LibExeObjStep = struct { .paths_first => try zig_args.append("-search_paths_first"), .dylibs_first => try zig_args.append("-search_dylibs_first"), }; + if (self.headerpad_size) |headerpad_size| { + const size = try std.fmt.allocPrint(builder.allocator, "{x}", .{headerpad_size}); + try zig_args.appendSlice(&[_][]const u8{ "-headerpad_size", size }); + } + if (self.headerpad_max_install_names) { + try zig_args.append("-headerpad_max_install_names"); + } if (self.bundle_compiler_rt) |x| { if (x) { diff --git a/src/Compilation.zig b/src/Compilation.zig index ccaad7c8c4..30603b91c5 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -907,6 +907,10 @@ pub const InitOptions = struct { pagezero_size: ?u64 = null, /// (Darwin) search strategy for system libraries search_strategy: ?link.File.MachO.SearchStrategy = null, + /// (Darwin) set minimum space for future expansion of the load commands + headerpad_size: ?u64 = null, + /// (Darwin) set enough space as if all paths were MATPATHLEN + headerpad_max_install_names: bool = false, }; fn addPackageTableToCacheHash( @@ -1748,6 +1752,8 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { .entitlements = options.entitlements, .pagezero_size = options.pagezero_size, .search_strategy = options.search_strategy, + .headerpad_size = options.headerpad_size, + .headerpad_max_install_names = options.headerpad_max_install_names, }); errdefer bin_file.destroy(); comp.* = .{ diff --git a/src/link.zig b/src/link.zig index cfcb112e9b..96ec3ef8cd 100644 --- a/src/link.zig +++ b/src/link.zig @@ -193,6 +193,12 @@ pub const Options = struct { /// (Darwin) search strategy for system libraries search_strategy: ?File.MachO.SearchStrategy = null, + /// (Darwin) set minimum space for future expansion of the load commands + headerpad_size: ?u64 = null, + + /// (Darwin) set enough space as if all paths were MATPATHLEN + headerpad_max_install_names: bool = false, + pub fn effectiveOutputMode(options: Options) std.builtin.OutputMode { return if (options.use_lld) .Obj else options.output_mode; } diff --git a/src/link/MachO.zig b/src/link/MachO.zig index 2bbd2e7a72..dc6fe6e194 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -69,10 +69,8 @@ page_size: u16, /// and potentially stage2 release builds in the future. needs_prealloc: bool = true, -/// We commit 0x1000 = 4096 bytes of space to the header and -/// the table of load commands. This should be plenty for any -/// potential future extensions. -header_pad: u16 = 0x1000, +/// Size of the padding between the end of load commands and start of the '__TEXT,__text' section. +headerpad_size: u64, /// The absolute address of the entry point. entry_addr: ?u64 = null, @@ -295,6 +293,11 @@ pub const min_text_capacity = padToIdeal(minimum_text_block_size); /// start of __TEXT segment. const default_pagezero_vmsize: u64 = 0x100000000; +/// We commit 0x1000 = 4096 bytes of space to the header and +/// the table of load commands. This should be plenty for any +/// potential future extensions. +const default_headerpad_size: u64 = 0x1000; + pub const Export = struct { sym_index: ?u32 = null, }; @@ -400,6 +403,12 @@ pub fn createEmpty(gpa: Allocator, options: link.Options) !*MachO { const use_llvm = build_options.have_llvm and options.use_llvm; const use_stage1 = build_options.is_stage1 and options.use_stage1; const needs_prealloc = !(use_stage1 or use_llvm or options.cache_mode == .whole); + // TODO handle `headerpad_max_install_names` in incremental context + const explicit_headerpad_size = options.headerpad_size orelse 0; + const headerpad_size = if (needs_prealloc) + @maximum(explicit_headerpad_size, default_headerpad_size) + else + explicit_headerpad_size; const self = try gpa.create(MachO); errdefer gpa.destroy(self); @@ -412,6 +421,7 @@ pub fn createEmpty(gpa: Allocator, options: link.Options) !*MachO { .file = null, }, .page_size = page_size, + .headerpad_size = headerpad_size, .code_signature = if (requires_adhoc_codesig) CodeSignature.init(page_size) else null, .needs_prealloc = needs_prealloc, }; @@ -976,6 +986,15 @@ pub fn flushModule(self: *MachO, comp: *Compilation, prog_node: *std.Progress.No .dylibs_first => try argv.append("-search_dylibs_first"), }; + if (self.base.options.headerpad_size) |headerpad_size| { + try argv.append("-headerpad_size"); + try argv.append(try std.fmt.allocPrint(arena, "0x{x}", .{headerpad_size})); + } + + if (self.base.options.headerpad_max_install_names) { + try argv.append("-headerpad_max_install_names"); + } + if (self.base.options.entry) |entry| { try argv.append("-e"); try argv.append(entry); @@ -4453,7 +4472,7 @@ fn populateMissingMetadata(self: *MachO) !void { const needed_size = if (self.needs_prealloc) blk: { const program_code_size_hint = self.base.options.program_code_size_hint; const got_size_hint = @sizeOf(u64) * self.base.options.symbol_count_hint; - const ideal_size = self.header_pad + program_code_size_hint + got_size_hint; + const ideal_size = self.headerpad_size + program_code_size_hint + got_size_hint; const needed_size = mem.alignForwardGeneric(u64, padToIdeal(ideal_size), self.page_size); log.debug("found __TEXT segment free space 0x{x} to 0x{x}", .{ 0, needed_size }); break :blk needed_size; @@ -4961,7 +4980,9 @@ fn allocateTextSegment(self: *MachO) !void { sizeofcmds += lc.cmdsize(); } - try self.allocateSegment(self.text_segment_cmd_index.?, @sizeOf(macho.mach_header_64) + sizeofcmds); + // TODO verify if `headerpad_max_install_names` leads to larger padding size + const offset = @sizeOf(macho.mach_header_64) + sizeofcmds + self.headerpad_size; + try self.allocateSegment(self.text_segment_cmd_index.?, offset); // Shift all sections to the back to minimize jump size between __TEXT and __DATA segments. var min_alignment: u32 = 0; @@ -5088,7 +5109,7 @@ fn initSection( if (self.needs_prealloc) { const alignment_pow_2 = try math.powi(u32, 2, alignment); - const padding: ?u64 = if (segment_id == self.text_segment_cmd_index.?) self.header_pad else null; + const padding: ?u64 = if (segment_id == self.text_segment_cmd_index.?) self.headerpad_size else null; const off = self.findFreeSpace(segment_id, alignment_pow_2, padding); log.debug("allocating {s},{s} section from 0x{x} to 0x{x}", .{ sect.segName(), diff --git a/src/main.zig b/src/main.zig index 3d65875975..87b0e6fc8a 100644 --- a/src/main.zig +++ b/src/main.zig @@ -450,6 +450,8 @@ const usage_build_generic = \\ -pagezero_size [value] (Darwin) size of the __PAGEZERO segment in hexadecimal notation \\ -search_paths_first (Darwin) search each dir in library search paths for `libx.dylib` then `libx.a` \\ -search_dylibs_first (Darwin) search `libx.dylib` in each dir in library search paths, then `libx.a` + \\ -headerpad_size [value] (Darwin) set minimum space for future expansion of the load commands in hexadecimal notation + \\ -headerpad_max_install_names (Darwin) set enough space as if all paths were MAXPATHLEN \\ --import-memory (WebAssembly) import memory from the environment \\ --import-table (WebAssembly) import function table from the host environment \\ --export-table (WebAssembly) export function table to the host environment @@ -699,6 +701,8 @@ fn buildOutputType( var entitlements: ?[]const u8 = null; var pagezero_size: ?u64 = null; var search_strategy: ?link.File.MachO.SearchStrategy = null; + var headerpad_size: ?u64 = null; + var headerpad_max_install_names: bool = false; // e.g. -m3dnow or -mno-outline-atomics. They correspond to std.Target llvm cpu feature names. // This array is populated by zig cc frontend and then has to be converted to zig-style @@ -924,6 +928,15 @@ fn buildOutputType( search_strategy = .paths_first; } else if (mem.eql(u8, arg, "-search_dylibs_first")) { search_strategy = .dylibs_first; + } else if (mem.eql(u8, arg, "-headerpad_size")) { + const next_arg = args_iter.next() orelse { + fatal("expected parameter after {s}", .{arg}); + }; + headerpad_size = std.fmt.parseUnsigned(u64, eatIntPrefix(next_arg, 16), 16) catch |err| { + fatal("unable to parser '{s}': {s}", .{ arg, @errorName(err) }); + }; + } else if (mem.eql(u8, arg, "-headerpad_max_install_names")) { + headerpad_max_install_names = true; } else if (mem.eql(u8, arg, "-T") or mem.eql(u8, arg, "--script")) { linker_script = args_iter.next() orelse { fatal("expected parameter after {s}", .{arg}); @@ -1676,6 +1689,17 @@ fn buildOutputType( pagezero_size = std.fmt.parseUnsigned(u64, eatIntPrefix(next_arg, 16), 16) catch |err| { fatal("unable to parse '{s}': {s}", .{ arg, @errorName(err) }); }; + } else if (mem.eql(u8, arg, "-headerpad_size")) { + i += 1; + if (i >= linker_args.items.len) { + fatal("expected linker arg after '{s}'", .{arg}); + } + const next_arg = linker_args.items[i]; + headerpad_size = std.fmt.parseUnsigned(u64, eatIntPrefix(next_arg, 16), 16) catch |err| { + fatal("unable to parse '{s}': {s}", .{ arg, @errorName(err) }); + }; + } else if (mem.eql(u8, arg, "-headerpad_max_install_names")) { + headerpad_max_install_names = true; } else if (mem.eql(u8, arg, "--gc-sections")) { linker_gc_sections = true; } else if (mem.eql(u8, arg, "--no-gc-sections")) { @@ -2795,6 +2819,8 @@ fn buildOutputType( .entitlements = entitlements, .pagezero_size = pagezero_size, .search_strategy = search_strategy, + .headerpad_size = headerpad_size, + .headerpad_max_install_names = headerpad_max_install_names, }) catch |err| switch (err) { error.LibCUnavailable => { const target = target_info.target; From a6fbdfabb9b60262594da012b8d877a43d8d8e99 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Sat, 25 Jun 2022 00:36:42 +0200 Subject: [PATCH 3/4] link-tests: add -headerpad_size test scenario --- test/link/macho/headerpad_size/build.zig | 25 ++++++++++++++++++++++++ test/link/macho/headerpad_size/main.c | 3 +++ 2 files changed, 28 insertions(+) create mode 100644 test/link/macho/headerpad_size/build.zig create mode 100644 test/link/macho/headerpad_size/main.c diff --git a/test/link/macho/headerpad_size/build.zig b/test/link/macho/headerpad_size/build.zig new file mode 100644 index 0000000000..0a02405e79 --- /dev/null +++ b/test/link/macho/headerpad_size/build.zig @@ -0,0 +1,25 @@ +const std = @import("std"); +const Builder = std.build.Builder; + +pub fn build(b: *Builder) void { + const mode = b.standardReleaseOptions(); + + const test_step = b.step("test", "Test"); + test_step.dependOn(b.getInstallStep()); + + const exe = b.addExecutable("main", null); + exe.setBuildMode(mode); + exe.addCSourceFile("main.c", &.{}); + exe.linkLibC(); + exe.headerpad_size = 0x10000; + + const check = exe.checkObject(.macho); + check.checkStart("sectname __text"); + check.checkNext("offset {offset}"); + check.checkComputeCompare("offset", .{ .op = .gte, .value = .{ .literal = 0x10000 } }); + + test_step.dependOn(&check.step); + + const run = exe.run(); + test_step.dependOn(&run.step); +} diff --git a/test/link/macho/headerpad_size/main.c b/test/link/macho/headerpad_size/main.c new file mode 100644 index 0000000000..ca68d24cc7 --- /dev/null +++ b/test/link/macho/headerpad_size/main.c @@ -0,0 +1,3 @@ +int main(int argc, char* argv[]) { + return 0; +} From 589bf67635a9fe9c3e6df4b63c09e0cc4954d29a Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Sat, 25 Jun 2022 10:39:53 +0200 Subject: [PATCH 4/4] macho: implement -headerpad_max_install_names --- lib/std/build.zig | 4 +- src/Compilation.zig | 8 +- src/link.zig | 2 +- src/link/Coff.zig | 2 +- src/link/Elf.zig | 2 +- src/link/MachO.zig | 53 +++++--- src/link/Wasm.zig | 2 +- src/main.zig | 12 +- test/link.zig | 5 + test/link/macho/headerpad/build.zig | 120 ++++++++++++++++++ .../{headerpad_size => headerpad}/main.c | 0 test/link/macho/headerpad_size/build.zig | 25 ---- 12 files changed, 177 insertions(+), 58 deletions(-) create mode 100644 test/link/macho/headerpad/build.zig rename test/link/macho/{headerpad_size => headerpad}/main.c (100%) delete mode 100644 test/link/macho/headerpad_size/build.zig diff --git a/lib/std/build.zig b/lib/std/build.zig index 20f59040c9..968ee043bf 100644 --- a/lib/std/build.zig +++ b/lib/std/build.zig @@ -1595,7 +1595,7 @@ pub const LibExeObjStep = struct { /// (Darwin) Set size of the padding between the end of load commands /// and start of `__TEXT,__text` section. - headerpad_size: ?u64 = null, + headerpad_size: ?u32 = null, /// (Darwin) Automatically Set size of the padding between the end of load commands /// and start of `__TEXT,__text` section to a value fitting all paths expanded to MAXPATHLEN. @@ -2671,7 +2671,7 @@ pub const LibExeObjStep = struct { }; if (self.headerpad_size) |headerpad_size| { const size = try std.fmt.allocPrint(builder.allocator, "{x}", .{headerpad_size}); - try zig_args.appendSlice(&[_][]const u8{ "-headerpad_size", size }); + try zig_args.appendSlice(&[_][]const u8{ "-headerpad", size }); } if (self.headerpad_max_install_names) { try zig_args.append("-headerpad_max_install_names"); diff --git a/src/Compilation.zig b/src/Compilation.zig index 30603b91c5..652938d741 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -908,7 +908,7 @@ pub const InitOptions = struct { /// (Darwin) search strategy for system libraries search_strategy: ?link.File.MachO.SearchStrategy = null, /// (Darwin) set minimum space for future expansion of the load commands - headerpad_size: ?u64 = null, + headerpad_size: ?u32 = null, /// (Darwin) set enough space as if all paths were MATPATHLEN headerpad_max_install_names: bool = false, }; @@ -2369,7 +2369,7 @@ fn prepareWholeEmitSubPath(arena: Allocator, opt_emit: ?EmitLoc) error{OutOfMemo /// to remind the programmer to update multiple related pieces of code that /// are in different locations. Bump this number when adding or deleting /// anything from the link cache manifest. -pub const link_hash_implementation_version = 5; +pub const link_hash_implementation_version = 6; fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifest) !void { const gpa = comp.gpa; @@ -2379,7 +2379,7 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes defer arena_allocator.deinit(); const arena = arena_allocator.allocator(); - comptime assert(link_hash_implementation_version == 5); + comptime assert(link_hash_implementation_version == 6); if (comp.bin_file.options.module) |mod| { const main_zig_file = try mod.main_pkg.root_src_directory.join(arena, &[_][]const u8{ @@ -2486,6 +2486,8 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes try man.addOptionalFile(comp.bin_file.options.entitlements); man.hash.addOptional(comp.bin_file.options.pagezero_size); man.hash.addOptional(comp.bin_file.options.search_strategy); + man.hash.addOptional(comp.bin_file.options.headerpad_size); + man.hash.add(comp.bin_file.options.headerpad_max_install_names); // COFF specific stuff man.hash.addOptional(comp.bin_file.options.subsystem); diff --git a/src/link.zig b/src/link.zig index 96ec3ef8cd..21d54d531c 100644 --- a/src/link.zig +++ b/src/link.zig @@ -194,7 +194,7 @@ pub const Options = struct { search_strategy: ?File.MachO.SearchStrategy = null, /// (Darwin) set minimum space for future expansion of the load commands - headerpad_size: ?u64 = null, + headerpad_size: ?u32 = null, /// (Darwin) set enough space as if all paths were MATPATHLEN headerpad_max_install_names: bool = false, diff --git a/src/link/Coff.zig b/src/link/Coff.zig index 3418a206df..77059a7fd9 100644 --- a/src/link/Coff.zig +++ b/src/link/Coff.zig @@ -969,7 +969,7 @@ fn linkWithLLD(self: *Coff, comp: *Compilation, prog_node: *std.Progress.Node) ! man = comp.cache_parent.obtain(); self.base.releaseLock(); - comptime assert(Compilation.link_hash_implementation_version == 5); + comptime assert(Compilation.link_hash_implementation_version == 6); for (self.base.options.objects) |obj| { _ = try man.addFile(obj.path, null); diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 1051c0eb78..79545d1e1a 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -1298,7 +1298,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node) !v // We are about to obtain this lock, so here we give other processes a chance first. self.base.releaseLock(); - comptime assert(Compilation.link_hash_implementation_version == 5); + comptime assert(Compilation.link_hash_implementation_version == 6); try man.addOptionalFile(self.base.options.linker_script); try man.addOptionalFile(self.base.options.version_script); diff --git a/src/link/MachO.zig b/src/link/MachO.zig index dc6fe6e194..4ddee493b8 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -69,9 +69,6 @@ page_size: u16, /// and potentially stage2 release builds in the future. needs_prealloc: bool = true, -/// Size of the padding between the end of load commands and start of the '__TEXT,__text' section. -headerpad_size: u64, - /// The absolute address of the entry point. entry_addr: ?u64 = null, @@ -296,7 +293,7 @@ const default_pagezero_vmsize: u64 = 0x100000000; /// We commit 0x1000 = 4096 bytes of space to the header and /// the table of load commands. This should be plenty for any /// potential future extensions. -const default_headerpad_size: u64 = 0x1000; +const default_headerpad_size: u32 = 0x1000; pub const Export = struct { sym_index: ?u32 = null, @@ -403,12 +400,6 @@ pub fn createEmpty(gpa: Allocator, options: link.Options) !*MachO { const use_llvm = build_options.have_llvm and options.use_llvm; const use_stage1 = build_options.is_stage1 and options.use_stage1; const needs_prealloc = !(use_stage1 or use_llvm or options.cache_mode == .whole); - // TODO handle `headerpad_max_install_names` in incremental context - const explicit_headerpad_size = options.headerpad_size orelse 0; - const headerpad_size = if (needs_prealloc) - @maximum(explicit_headerpad_size, default_headerpad_size) - else - explicit_headerpad_size; const self = try gpa.create(MachO); errdefer gpa.destroy(self); @@ -421,7 +412,6 @@ pub fn createEmpty(gpa: Allocator, options: link.Options) !*MachO { .file = null, }, .page_size = page_size, - .headerpad_size = headerpad_size, .code_signature = if (requires_adhoc_codesig) CodeSignature.init(page_size) else null, .needs_prealloc = needs_prealloc, }; @@ -551,7 +541,7 @@ pub fn flushModule(self: *MachO, comp: *Compilation, prog_node: *std.Progress.No // We are about to obtain this lock, so here we give other processes a chance first. self.base.releaseLock(); - comptime assert(Compilation.link_hash_implementation_version == 5); + comptime assert(Compilation.link_hash_implementation_version == 6); for (self.base.options.objects) |obj| { _ = try man.addFile(obj.path, null); @@ -566,6 +556,8 @@ pub fn flushModule(self: *MachO, comp: *Compilation, prog_node: *std.Progress.No man.hash.add(stack_size); man.hash.addOptional(self.base.options.pagezero_size); man.hash.addOptional(self.base.options.search_strategy); + man.hash.addOptional(self.base.options.headerpad_size); + man.hash.add(self.base.options.headerpad_max_install_names); man.hash.addListOfBytes(self.base.options.lib_dirs); man.hash.addListOfBytes(self.base.options.framework_dirs); man.hash.addListOfBytes(self.base.options.frameworks); @@ -4470,9 +4462,10 @@ fn populateMissingMetadata(self: *MachO) !void { if (self.text_segment_cmd_index == null) { self.text_segment_cmd_index = @intCast(u16, self.load_commands.items.len); const needed_size = if (self.needs_prealloc) blk: { + const headerpad_size = @maximum(self.base.options.headerpad_size orelse 0, default_headerpad_size); const program_code_size_hint = self.base.options.program_code_size_hint; const got_size_hint = @sizeOf(u64) * self.base.options.symbol_count_hint; - const ideal_size = self.headerpad_size + program_code_size_hint + got_size_hint; + const ideal_size = headerpad_size + program_code_size_hint + got_size_hint; const needed_size = mem.alignForwardGeneric(u64, padToIdeal(ideal_size), self.page_size); log.debug("found __TEXT segment free space 0x{x} to 0x{x}", .{ 0, needed_size }); break :blk needed_size; @@ -4975,13 +4968,34 @@ fn allocateTextSegment(self: *MachO) !void { seg.inner.fileoff = 0; seg.inner.vmaddr = base_vmaddr; - var sizeofcmds: u64 = 0; + var sizeofcmds: u32 = 0; for (self.load_commands.items) |lc| { sizeofcmds += lc.cmdsize(); } - // TODO verify if `headerpad_max_install_names` leads to larger padding size - const offset = @sizeOf(macho.mach_header_64) + sizeofcmds + self.headerpad_size; + var padding: u32 = sizeofcmds + (self.base.options.headerpad_size orelse 0); + log.debug("minimum requested headerpad size 0x{x}", .{padding + @sizeOf(macho.mach_header_64)}); + + if (self.base.options.headerpad_max_install_names) { + var min_headerpad_size: u32 = 0; + for (self.load_commands.items) |lc| switch (lc.cmd()) { + .ID_DYLIB, + .LOAD_WEAK_DYLIB, + .LOAD_DYLIB, + .REEXPORT_DYLIB, + => { + min_headerpad_size += @sizeOf(macho.dylib_command) + std.os.PATH_MAX + 1; + }, + + else => {}, + }; + log.debug("headerpad_max_install_names minimum headerpad size 0x{x}", .{ + min_headerpad_size + @sizeOf(macho.mach_header_64), + }); + padding = @maximum(padding, min_headerpad_size); + } + const offset = @sizeOf(macho.mach_header_64) + padding; + log.debug("actual headerpad size 0x{x}", .{offset}); try self.allocateSegment(self.text_segment_cmd_index.?, offset); // Shift all sections to the back to minimize jump size between __TEXT and __DATA segments. @@ -5109,7 +5123,10 @@ fn initSection( if (self.needs_prealloc) { const alignment_pow_2 = try math.powi(u32, 2, alignment); - const padding: ?u64 = if (segment_id == self.text_segment_cmd_index.?) self.headerpad_size else null; + const padding: ?u32 = if (segment_id == self.text_segment_cmd_index.?) + @maximum(self.base.options.headerpad_size orelse 0, default_headerpad_size) + else + null; const off = self.findFreeSpace(segment_id, alignment_pow_2, padding); log.debug("allocating {s},{s} section from 0x{x} to 0x{x}", .{ sect.segName(), @@ -5148,7 +5165,7 @@ fn initSection( return index; } -fn findFreeSpace(self: MachO, segment_id: u16, alignment: u64, start: ?u64) u64 { +fn findFreeSpace(self: MachO, segment_id: u16, alignment: u64, start: ?u32) u64 { const seg = self.load_commands.items[segment_id].segment; if (seg.sections.items.len == 0) { return if (start) |v| v else seg.inner.fileoff; diff --git a/src/link/Wasm.zig b/src/link/Wasm.zig index c654fbe719..bc7e70b7da 100644 --- a/src/link/Wasm.zig +++ b/src/link/Wasm.zig @@ -2481,7 +2481,7 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation, prog_node: *std.Progress.Node) ! // We are about to obtain this lock, so here we give other processes a chance first. self.base.releaseLock(); - comptime assert(Compilation.link_hash_implementation_version == 5); + comptime assert(Compilation.link_hash_implementation_version == 6); for (self.base.options.objects) |obj| { _ = try man.addFile(obj.path, null); diff --git a/src/main.zig b/src/main.zig index 87b0e6fc8a..d63e235360 100644 --- a/src/main.zig +++ b/src/main.zig @@ -450,7 +450,7 @@ const usage_build_generic = \\ -pagezero_size [value] (Darwin) size of the __PAGEZERO segment in hexadecimal notation \\ -search_paths_first (Darwin) search each dir in library search paths for `libx.dylib` then `libx.a` \\ -search_dylibs_first (Darwin) search `libx.dylib` in each dir in library search paths, then `libx.a` - \\ -headerpad_size [value] (Darwin) set minimum space for future expansion of the load commands in hexadecimal notation + \\ -headerpad [value] (Darwin) set minimum space for future expansion of the load commands in hexadecimal notation \\ -headerpad_max_install_names (Darwin) set enough space as if all paths were MAXPATHLEN \\ --import-memory (WebAssembly) import memory from the environment \\ --import-table (WebAssembly) import function table from the host environment @@ -701,7 +701,7 @@ fn buildOutputType( var entitlements: ?[]const u8 = null; var pagezero_size: ?u64 = null; var search_strategy: ?link.File.MachO.SearchStrategy = null; - var headerpad_size: ?u64 = null; + var headerpad_size: ?u32 = null; var headerpad_max_install_names: bool = false; // e.g. -m3dnow or -mno-outline-atomics. They correspond to std.Target llvm cpu feature names. @@ -928,11 +928,11 @@ fn buildOutputType( search_strategy = .paths_first; } else if (mem.eql(u8, arg, "-search_dylibs_first")) { search_strategy = .dylibs_first; - } else if (mem.eql(u8, arg, "-headerpad_size")) { + } else if (mem.eql(u8, arg, "-headerpad")) { const next_arg = args_iter.next() orelse { fatal("expected parameter after {s}", .{arg}); }; - headerpad_size = std.fmt.parseUnsigned(u64, eatIntPrefix(next_arg, 16), 16) catch |err| { + headerpad_size = std.fmt.parseUnsigned(u32, eatIntPrefix(next_arg, 16), 16) catch |err| { fatal("unable to parser '{s}': {s}", .{ arg, @errorName(err) }); }; } else if (mem.eql(u8, arg, "-headerpad_max_install_names")) { @@ -1689,13 +1689,13 @@ fn buildOutputType( pagezero_size = std.fmt.parseUnsigned(u64, eatIntPrefix(next_arg, 16), 16) catch |err| { fatal("unable to parse '{s}': {s}", .{ arg, @errorName(err) }); }; - } else if (mem.eql(u8, arg, "-headerpad_size")) { + } else if (mem.eql(u8, arg, "-headerpad")) { i += 1; if (i >= linker_args.items.len) { fatal("expected linker arg after '{s}'", .{arg}); } const next_arg = linker_args.items[i]; - headerpad_size = std.fmt.parseUnsigned(u64, eatIntPrefix(next_arg, 16), 16) catch |err| { + headerpad_size = std.fmt.parseUnsigned(u32, eatIntPrefix(next_arg, 16), 16) catch |err| { fatal("unable to parse '{s}': {s}", .{ arg, @errorName(err) }); }; } else if (mem.eql(u8, arg, "-headerpad_max_install_names")) { diff --git a/test/link.zig b/test/link.zig index 62bdcff4b0..0c301d6bcb 100644 --- a/test/link.zig +++ b/test/link.zig @@ -64,5 +64,10 @@ pub fn addCases(cases: *tests.StandaloneContext) void { cases.addBuildFile("test/link/macho/search_strategy/build.zig", .{ .build_modes = true, }); + + cases.addBuildFile("test/link/macho/headerpad/build.zig", .{ + .build_modes = true, + .requires_macos_sdk = true, + }); } } diff --git a/test/link/macho/headerpad/build.zig b/test/link/macho/headerpad/build.zig new file mode 100644 index 0000000000..0730a01d44 --- /dev/null +++ b/test/link/macho/headerpad/build.zig @@ -0,0 +1,120 @@ +const std = @import("std"); +const builtin = @import("builtin"); +const Builder = std.build.Builder; +const LibExeObjectStep = std.build.LibExeObjStep; + +pub fn build(b: *Builder) void { + const mode = b.standardReleaseOptions(); + + const test_step = b.step("test", "Test"); + test_step.dependOn(b.getInstallStep()); + + { + // Test -headerpad_max_install_names + const exe = simpleExe(b, mode); + exe.headerpad_max_install_names = true; + + const check = exe.checkObject(.macho); + check.checkStart("sectname __text"); + check.checkNext("offset {offset}"); + + switch (builtin.cpu.arch) { + .aarch64 => { + check.checkComputeCompare("offset", .{ .op = .gte, .value = .{ .literal = 0x4000 } }); + }, + .x86_64 => { + check.checkComputeCompare("offset", .{ .op = .gte, .value = .{ .literal = 0x1000 } }); + }, + else => unreachable, + } + + test_step.dependOn(&check.step); + + const run = exe.run(); + test_step.dependOn(&run.step); + } + + { + // Test -headerpad + const exe = simpleExe(b, mode); + exe.headerpad_size = 0x10000; + + const check = exe.checkObject(.macho); + check.checkStart("sectname __text"); + check.checkNext("offset {offset}"); + check.checkComputeCompare("offset", .{ .op = .gte, .value = .{ .literal = 0x10000 } }); + + test_step.dependOn(&check.step); + + const run = exe.run(); + test_step.dependOn(&run.step); + } + + { + // Test both flags with -headerpad overriding -headerpad_max_install_names + const exe = simpleExe(b, mode); + exe.headerpad_max_install_names = true; + exe.headerpad_size = 0x10000; + + const check = exe.checkObject(.macho); + check.checkStart("sectname __text"); + check.checkNext("offset {offset}"); + check.checkComputeCompare("offset", .{ .op = .gte, .value = .{ .literal = 0x10000 } }); + + test_step.dependOn(&check.step); + + const run = exe.run(); + test_step.dependOn(&run.step); + } + + { + // Test both flags with -headerpad_max_install_names overriding -headerpad + const exe = simpleExe(b, mode); + exe.headerpad_size = 0x1000; + exe.headerpad_max_install_names = true; + + const check = exe.checkObject(.macho); + check.checkStart("sectname __text"); + check.checkNext("offset {offset}"); + + switch (builtin.cpu.arch) { + .aarch64 => { + check.checkComputeCompare("offset", .{ .op = .gte, .value = .{ .literal = 0x4000 } }); + }, + .x86_64 => { + check.checkComputeCompare("offset", .{ .op = .gte, .value = .{ .literal = 0x1000 } }); + }, + else => unreachable, + } + + test_step.dependOn(&check.step); + + const run = exe.run(); + test_step.dependOn(&run.step); + } +} + +fn simpleExe(b: *Builder, mode: std.builtin.Mode) *LibExeObjectStep { + const exe = b.addExecutable("main", null); + exe.setBuildMode(mode); + exe.addCSourceFile("main.c", &.{}); + exe.linkLibC(); + exe.linkFramework("CoreFoundation"); + exe.linkFramework("Foundation"); + exe.linkFramework("Cocoa"); + exe.linkFramework("CoreGraphics"); + exe.linkFramework("CoreHaptics"); + exe.linkFramework("CoreAudio"); + exe.linkFramework("AVFoundation"); + exe.linkFramework("CoreImage"); + exe.linkFramework("CoreLocation"); + exe.linkFramework("CoreML"); + exe.linkFramework("CoreVideo"); + exe.linkFramework("CoreText"); + exe.linkFramework("CryptoKit"); + exe.linkFramework("GameKit"); + exe.linkFramework("SwiftUI"); + exe.linkFramework("StoreKit"); + exe.linkFramework("SpriteKit"); + return exe; +} diff --git a/test/link/macho/headerpad_size/main.c b/test/link/macho/headerpad/main.c similarity index 100% rename from test/link/macho/headerpad_size/main.c rename to test/link/macho/headerpad/main.c diff --git a/test/link/macho/headerpad_size/build.zig b/test/link/macho/headerpad_size/build.zig deleted file mode 100644 index 0a02405e79..0000000000 --- a/test/link/macho/headerpad_size/build.zig +++ /dev/null @@ -1,25 +0,0 @@ -const std = @import("std"); -const Builder = std.build.Builder; - -pub fn build(b: *Builder) void { - const mode = b.standardReleaseOptions(); - - const test_step = b.step("test", "Test"); - test_step.dependOn(b.getInstallStep()); - - const exe = b.addExecutable("main", null); - exe.setBuildMode(mode); - exe.addCSourceFile("main.c", &.{}); - exe.linkLibC(); - exe.headerpad_size = 0x10000; - - const check = exe.checkObject(.macho); - check.checkStart("sectname __text"); - check.checkNext("offset {offset}"); - check.checkComputeCompare("offset", .{ .op = .gte, .value = .{ .literal = 0x10000 } }); - - test_step.dependOn(&check.step); - - const run = exe.run(); - test_step.dependOn(&run.step); -}