From 5154ece8b9fd64f276fb6cb6f88fdfda1fc0f935 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Sun, 9 Apr 2023 13:25:40 +0200 Subject: [PATCH 01/26] macho: fix incorrect DWARF subprogram generation --- src/link/Dwarf.zig | 15 ++++++++++++--- src/link/MachO.zig | 24 +++++++++++++++++------- 2 files changed, 29 insertions(+), 10 deletions(-) diff --git a/src/link/Dwarf.zig b/src/link/Dwarf.zig index 3fb6de7b73..0f2dfbda0e 100644 --- a/src/link/Dwarf.zig +++ b/src/link/Dwarf.zig @@ -150,7 +150,7 @@ pub const DeclState = struct { .type = ty, .offset = undefined, }); - log.debug("%{d}: {}", .{ sym_index, ty.fmtDebug() }); + log.debug("%{d}: {}", .{ sym_index, ty.fmt(self.mod) }); try self.abbrev_resolver.putNoClobberContext(self.gpa, ty, sym_index, .{ .mod = self.mod, }); @@ -570,7 +570,7 @@ pub const DeclState = struct { try dbg_info_buffer.append(0); }, else => { - log.debug("TODO implement .debug_info for type '{}'", .{ty.fmtDebug()}); + log.debug("TODO implement .debug_info for type '{}'", .{ty.fmt(self.mod)}); try dbg_info_buffer.append(@enumToInt(AbbrevKind.pad1)); }, } @@ -1055,6 +1055,10 @@ pub fn commitDeclState( }, } { + log.debug("relocating subprogram high PC value: {x} => {x}", .{ + self.getRelocDbgInfoSubprogramHighPC(), + sym_size, + }); const ptr = dbg_info_buffer.items[self.getRelocDbgInfoSubprogramHighPC()..][0..4]; mem.writeInt(u32, ptr, @intCast(u32, sym_size), target_endian); } @@ -1263,7 +1267,12 @@ pub fn commitDeclState( } else { const atom = self.getAtom(.di_atom, symbol.atom_index); const value = atom.off + symbol.offset + reloc.addend; - log.debug("{x}: [() => {x}] (%{d}, '{}')", .{ reloc.offset, value, target, ty.fmtDebug() }); + log.debug("{x}: [() => {x}] (%{d}, '{}')", .{ + reloc.offset, + value, + target, + ty.fmt(module), + }); mem.writeInt( u32, dbg_info_buffer.items[reloc.offset..][0..@sizeOf(u32)], diff --git a/src/link/MachO.zig b/src/link/MachO.zig index e8fab08912..5ec9c9c356 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -2070,8 +2070,6 @@ pub fn updateFunc(self: *MachO, module: *Module, func: *Module.Fn, air: Air, liv self.freeUnnamedConsts(decl_index); Atom.freeRelocations(self, atom_index); - const atom = self.getAtom(atom_index); - var code_buffer = std.ArrayList(u8).init(self.base.allocator); defer code_buffer.deinit(); @@ -2100,7 +2098,13 @@ pub fn updateFunc(self: *MachO, module: *Module, func: *Module.Fn, air: Air, liv const addr = try self.updateDeclCode(decl_index, code); if (decl_state) |*ds| { - try self.d_sym.?.dwarf.commitDeclState(module, decl_index, addr, atom.size, ds); + try self.d_sym.?.dwarf.commitDeclState( + module, + decl_index, + addr, + self.getAtom(atom_index).size, + ds, + ); } // Since we updated the vaddr and the size, each corresponding export symbol also @@ -2196,8 +2200,8 @@ pub fn updateDecl(self: *MachO, module: *Module, decl_index: Module.Decl.Index) } const atom_index = try self.getOrCreateAtomForDecl(decl_index); + const sym_index = self.getAtom(atom_index).getSymbolIndex().?; Atom.freeRelocations(self, atom_index); - const atom = self.getAtom(atom_index); var code_buffer = std.ArrayList(u8).init(self.base.allocator); defer code_buffer.deinit(); @@ -2216,14 +2220,14 @@ pub fn updateDecl(self: *MachO, module: *Module, decl_index: Module.Decl.Index) }, &code_buffer, .{ .dwarf = ds, }, .{ - .parent_atom_index = atom.getSymbolIndex().?, + .parent_atom_index = sym_index, }) else try codegen.generateSymbol(&self.base, decl.srcLoc(), .{ .ty = decl.ty, .val = decl_val, }, &code_buffer, .none, .{ - .parent_atom_index = atom.getSymbolIndex().?, + .parent_atom_index = sym_index, }); var code = switch (res) { @@ -2237,7 +2241,13 @@ pub fn updateDecl(self: *MachO, module: *Module, decl_index: Module.Decl.Index) const addr = try self.updateDeclCode(decl_index, code); if (decl_state) |*ds| { - try self.d_sym.?.dwarf.commitDeclState(module, decl_index, addr, atom.size, ds); + try self.d_sym.?.dwarf.commitDeclState( + module, + decl_index, + addr, + self.getAtom(atom_index).size, + ds, + ); } // Since we updated the vaddr and the size, each corresponding export symbol also From 382de7bf1de8314e97eb67426b693b966f2c4e1b Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 5 Apr 2023 09:16:33 +0200 Subject: [PATCH 02/26] codegen: use non-debug Type/Value formatting --- src/codegen.zig | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/codegen.zig b/src/codegen.zig index b322d336cd..4156dd853b 100644 --- a/src/codegen.zig +++ b/src/codegen.zig @@ -493,7 +493,7 @@ pub fn generateSymbol( bin_file.allocator, src_loc, "TODO implement generateSymbol for big int enums ('{}')", - .{typed_value.ty.fmtDebug()}, + .{typed_value.ty.fmt(mod)}, ), }; } @@ -957,13 +957,13 @@ fn genDeclRef( tv: TypedValue, decl_index: Module.Decl.Index, ) CodeGenError!GenResult { - log.debug("genDeclRef: ty = {}, val = {}", .{ tv.ty.fmtDebug(), tv.val.fmtDebug() }); + const module = bin_file.options.module.?; + log.debug("genDeclRef: ty = {}, val = {}", .{ tv.ty.fmt(module), tv.val.fmtValue(tv.ty, module) }); const target = bin_file.options.target; const ptr_bits = target.cpu.arch.ptrBitWidth(); const ptr_bytes: u64 = @divExact(ptr_bits, 8); - const module = bin_file.options.module.?; const decl = module.declPtr(decl_index); if (!decl.ty.isFnOrHasRuntimeBitsIgnoreComptime()) { @@ -1025,7 +1025,8 @@ fn genUnnamedConst( tv: TypedValue, owner_decl_index: Module.Decl.Index, ) CodeGenError!GenResult { - log.debug("genUnnamedConst: ty = {}, val = {}", .{ tv.ty.fmtDebug(), tv.val.fmtDebug() }); + const mod = bin_file.options.module.?; + log.debug("genUnnamedConst: ty = {}, val = {}", .{ tv.ty.fmt(mod), tv.val.fmtValue(tv.ty, mod) }); const target = bin_file.options.target; const local_sym_index = bin_file.lowerUnnamedConst(tv, owner_decl_index) catch |err| { @@ -1065,7 +1066,11 @@ pub fn genTypedValue( typed_value.val = rt.data; } - log.debug("genTypedValue: ty = {}, val = {}", .{ typed_value.ty.fmtDebug(), typed_value.val.fmtDebug() }); + const mod = bin_file.options.module.?; + log.debug("genTypedValue: ty = {}, val = {}", .{ + typed_value.ty.fmt(mod), + typed_value.val.fmtValue(typed_value.ty, mod), + }); if (typed_value.val.isUndef()) return GenResult.mcv(.undef); From ee6b4fad4714d596255f034c0b000438ee6ca943 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 11 Apr 2023 22:17:04 +0200 Subject: [PATCH 03/26] x86_64: remove loadMemPtrIntoRegister in genSetReg Add two emit helpers for linker reloc based `lea` and `mov` instructions: `asmMovLinker` and `asmLeaLinker`. --- src/arch/x86_64/CodeGen.zig | 96 +++++++++++++++++++++++++++++-------- src/arch/x86_64/Emit.zig | 2 +- src/arch/x86_64/Lower.zig | 10 ++++ src/arch/x86_64/Mir.zig | 2 + 4 files changed, 90 insertions(+), 20 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 7344af9673..374029c64c 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -731,6 +731,40 @@ fn asmMemoryRegisterImmediate( }); } +fn asmMovLinker(self: *Self, reg: Register, atom_index: u32, linker_load: codegen.LinkerLoad) !void { + const ops: Mir.Inst.Ops = switch (linker_load.type) { + .got => .got_reloc, + .direct => .direct_reloc, + .import => .import_reloc, + }; + _ = try self.addInst(.{ + .tag = .mov_linker, + .ops = ops, + .data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{ + .reg = @enumToInt(reg), + .atom_index = atom_index, + .sym_index = linker_load.sym_index, + }) }, + }); +} + +fn asmLeaLinker(self: *Self, reg: Register, atom_index: u32, linker_load: codegen.LinkerLoad) !void { + const ops: Mir.Inst.Ops = switch (linker_load.type) { + .got => .got_reloc, + .direct => .direct_reloc, + .import => .import_reloc, + }; + _ = try self.addInst(.{ + .tag = .lea_linker, + .ops = ops, + .data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{ + .reg = @enumToInt(reg), + .atom_index = atom_index, + .sym_index = linker_load.sym_index, + }) }, + }); +} + fn gen(self: *Self) InnerError!void { const cc = self.fn_type.fnCallingConvention(); if (cc != .Naked) { @@ -7454,10 +7488,10 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void try self.asmRegisterRegister(.mov, registerAlias(reg, abi_size), registerAlias(src_reg, abi_size)); }, - .memory, .linker_load => switch (ty.zigTypeTag()) { + .memory => |addr| switch (ty.zigTypeTag()) { .Float => { - const base_reg = try self.register_manager.allocReg(null, gp); - try self.loadMemPtrIntoRegister(base_reg, Type.usize, mcv); + const base_reg = (try self.register_manager.allocReg(null, gp)).to64(); + try self.genSetReg(Type.usize, base_reg, .{ .immediate = addr }); if (intrinsicsAllowed(self.target.*, ty)) { return self.asmRegisterMemory( @@ -7469,29 +7503,20 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void }), }, reg.to128(), - Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base_reg.to64() }), + Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base_reg }), ); } return self.fail("TODO genSetReg from memory for float with no intrinsics", .{}); }, - else => switch (mcv) { - else => unreachable, - .linker_load => { - try self.loadMemPtrIntoRegister(reg, Type.usize, mcv); - try self.asmRegisterMemory( - .mov, - registerAlias(reg, abi_size), - Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }), - ); - }, - .memory => |x| if (x <= math.maxInt(i32)) { + else => { + if (addr <= math.maxInt(i32)) { try self.asmRegisterMemory( .mov, registerAlias(reg, abi_size), Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = .ds, - .disp = @intCast(i32, x), + .disp = @intCast(i32, addr), }), ); } else { @@ -7501,20 +7526,53 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void _ = try self.addInst(.{ .tag = .mov_moffs, .ops = .rax_moffs, - .data = .{ .payload = try self.addExtra(Mir.MemoryMoffs.encode(.ds, x)) }, + .data = .{ .payload = try self.addExtra(Mir.MemoryMoffs.encode(.ds, addr)) }, }); } else { // Rather than duplicate the logic used for the move, we just use a self-call with a new MCValue. - try self.genSetReg(ty, reg, MCValue{ .immediate = x }); + try self.genSetReg(Type.usize, reg, MCValue{ .immediate = addr }); try self.asmRegisterMemory( .mov, registerAlias(reg, abi_size), Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }), ); } - }, + } }, }, + .linker_load => |load_struct| { + const atom_index = if (self.bin_file.cast(link.File.MachO)) |macho_file| blk: { + const atom = try macho_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk macho_file.getAtom(atom).getSymbolIndex().?; + } else if (self.bin_file.cast(link.File.Coff)) |coff_file| blk: { + const atom = try coff_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk coff_file.getAtom(atom).getSymbolIndex().?; + } else unreachable; + + switch (ty.zigTypeTag()) { + .Float => { + const base_reg = (try self.register_manager.allocReg(null, gp)).to64(); + try self.asmLeaLinker(base_reg, atom_index, load_struct); + + if (intrinsicsAllowed(self.target.*, ty)) { + return self.asmRegisterMemory( + switch (ty.tag()) { + .f32 => .movss, + .f64 => .movsd, + else => return self.fail("TODO genSetReg from memory for {}", .{ + ty.fmt(self.bin_file.options.module.?), + }), + }, + reg.to128(), + Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = base_reg.to64() }), + ); + } + + return self.fail("TODO genSetReg from memory for float with no intrinsics", .{}); + }, + else => try self.asmMovLinker(registerAlias(reg, abi_size), atom_index, load_struct), + } + }, .stack_offset => |off| { switch (ty.zigTypeTag()) { .Int => switch (ty.intInfo(self.target.*).signedness) { diff --git a/src/arch/x86_64/Emit.zig b/src/arch/x86_64/Emit.zig index 052139a4e5..30d63b6adf 100644 --- a/src/arch/x86_64/Emit.zig +++ b/src/arch/x86_64/Emit.zig @@ -65,7 +65,7 @@ pub fn emitMir(emit: *Emit) Error!void { }); } else return emit.fail("TODO implement {} for {}", .{ inst.tag, emit.bin_file.tag }), - .lea_linker => if (emit.bin_file.cast(link.File.MachO)) |macho_file| { + .mov_linker, .lea_linker => if (emit.bin_file.cast(link.File.MachO)) |macho_file| { const metadata = emit.lower.mir.extraData(Mir.LeaRegisterReloc, inst.data.payload).data; const reloc_type = switch (inst.ops) { diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig index f4d9db57e8..e88484c9da 100644 --- a/src/arch/x86_64/Lower.zig +++ b/src/arch/x86_64/Lower.zig @@ -127,6 +127,7 @@ pub fn lowerMir(lower: *Lower, inst: Mir.Inst) Error![]const Instruction { .call_extern => try lower.emit(.none, .call, &.{.{ .imm = Immediate.s(0) }}), .lea_linker => try lower.mirLeaLinker(inst), + .mov_linker => try lower.mirMovLinker(inst), .mov_moffs => try lower.mirMovMoffs(inst), @@ -444,6 +445,15 @@ fn mirLeaLinker(lower: *Lower, inst: Mir.Inst) Error!void { }); } +fn mirMovLinker(lower: *Lower, inst: Mir.Inst) Error!void { + const metadata = lower.mir.extraData(Mir.LeaRegisterReloc, inst.data.payload).data; + const reg = @intToEnum(Register, metadata.reg); + try lower.emit(.none, .mov, &.{ + .{ .reg = reg }, + .{ .mem = Memory.rip(Memory.PtrSize.fromBitSize(reg.bitSize()), 0) }, + }); +} + const abi = @import("abi.zig"); const assert = std.debug.assert; const bits = @import("bits.zig"); diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig index 0ceee6bac1..835e9bfdcb 100644 --- a/src/arch/x86_64/Mir.zig +++ b/src/arch/x86_64/Mir.zig @@ -233,6 +233,8 @@ pub const Inst = struct { /// Load effective address of a symbol not yet allocated in VM. lea_linker, + /// Move address of a symbol not yet allocated in VM. + mov_linker, /// End of prologue dbg_prologue_end, From 397dd77697934a808b3545c5691399d8d69b841d Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 11 Apr 2023 22:45:50 +0200 Subject: [PATCH 04/26] x86_64: simplify store() logic when value is in memory --- src/arch/x86_64/CodeGen.zig | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 374029c64c..f7992969e3 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -3809,11 +3809,29 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty); } else { - const addr_reg = try self.register_manager.allocReg(null, gp); + const addr_reg = (try self.register_manager.allocReg(null, gp)).to64(); const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg); defer self.register_manager.unlockReg(addr_lock); - try self.loadMemPtrIntoRegister(addr_reg, Type.usize, value); + switch (value) { + .memory => |addr| try self.genSetReg(Type.usize, addr_reg, .{ .immediate = addr }), + .linker_load => |load_struct| { + const atom_index = if (self.bin_file.cast(link.File.MachO)) |macho_file| blk: { + const atom = try macho_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk macho_file.getAtom(atom).getSymbolIndex().?; + } else if (self.bin_file.cast(link.File.Coff)) |coff_file| blk: { + const atom = try coff_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk coff_file.getAtom(atom).getSymbolIndex().?; + } else unreachable; + switch (load_struct.type) { + .import => unreachable, + .got => try self.asmMovLinker(addr_reg, atom_index, load_struct), + .direct => try self.asmLeaLinker(addr_reg, atom_index, load_struct), + } + }, + else => unreachable, + } + try self.genInlineMemcpy( ptr, .{ .register = addr_reg }, From dbdf8086bd964cec22c44d41702d35643cce412a Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 11 Apr 2023 23:00:39 +0200 Subject: [PATCH 05/26] x86_64: simplify store() logic when ptr is in memory --- src/arch/x86_64/CodeGen.zig | 32 +++++++++++++++++++++++--------- 1 file changed, 23 insertions(+), 9 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index f7992969e3..4f68fff0de 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -3867,19 +3867,33 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type }; defer if (value_lock) |lock| self.register_manager.unlockReg(lock); - const addr_reg = try self.register_manager.allocReg(null, gp); + const addr_reg = (try self.register_manager.allocReg(null, gp)).to64(); const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg); defer self.register_manager.unlockReg(addr_reg_lock); - try self.loadMemPtrIntoRegister(addr_reg, ptr_ty, ptr); - // Load the pointer, which is stored in memory - try self.asmRegisterMemory( - .mov, - addr_reg.to64(), - Memory.sib(.qword, .{ .base = addr_reg.to64() }), - ); + switch (ptr) { + .memory => |addr| { + try self.genSetReg(ptr_ty, addr_reg, .{ .immediate = addr }); + // Load the pointer, which is stored in memory + try self.asmRegisterMemory(.mov, addr_reg, Memory.sib(.qword, .{ .base = addr_reg })); + }, + .linker_load => |load_struct| { + const atom_index = if (self.bin_file.cast(link.File.MachO)) |macho_file| blk: { + const atom = try macho_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk macho_file.getAtom(atom).getSymbolIndex().?; + } else if (self.bin_file.cast(link.File.Coff)) |coff_file| blk: { + const atom = try coff_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk coff_file.getAtom(atom).getSymbolIndex().?; + } else unreachable; + switch (load_struct.type) { + .import => unreachable, + .got, .direct => try self.asmMovLinker(addr_reg, atom_index, load_struct), + } + }, + else => unreachable, + } - const new_ptr = MCValue{ .register = addr_reg.to64() }; + const new_ptr = MCValue{ .register = addr_reg }; try self.store(new_ptr, value, ptr_ty, value_ty); }, } From 38d46a6764a63bfa17560b70c5e82831bd4aeba1 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 11 Apr 2023 23:07:49 +0200 Subject: [PATCH 06/26] x86_64: simplify genUnOpMir() when modifying value in memory --- src/arch/x86_64/CodeGen.zig | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 4f68fff0de..c48172f39f 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -4228,7 +4228,25 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg); defer self.register_manager.unlockReg(addr_reg_lock); - try self.loadMemPtrIntoRegister(addr_reg, Type.usize, dst_mcv); + switch (dst_mcv) { + .memory => |addr| try self.genSetReg(Type.usize, addr_reg, .{ .immediate = addr }), + .linker_load => |load_struct| { + const atom_index = if (self.bin_file.cast(link.File.MachO)) |macho_file| blk: { + const atom = try macho_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk macho_file.getAtom(atom).getSymbolIndex().?; + } else if (self.bin_file.cast(link.File.Coff)) |coff_file| blk: { + const atom = try coff_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk coff_file.getAtom(atom).getSymbolIndex().?; + } else unreachable; + + switch (load_struct.type) { + .import => unreachable, + .got => try self.asmMovLinker(addr_reg, atom_index, load_struct), + .direct => try self.asmLeaLinker(addr_reg, atom_index, load_struct), + } + }, + else => unreachable, + } try self.asmMemory( mir_tag, Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = addr_reg }), From c8c5951e23082734750619edd411cae16058db6d Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 11 Apr 2023 23:22:19 +0200 Subject: [PATCH 07/26] x86_64: simplify genBinOp() and genBinOpMir() when modifying value in memory --- src/arch/x86_64/CodeGen.zig | 67 ++++++++++++++++++++++++++++++++++--- 1 file changed, 62 insertions(+), 5 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index c48172f39f..15cc5bd302 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -4903,7 +4903,26 @@ fn genBinOp( const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg); defer self.register_manager.unlockReg(addr_reg_lock); - try self.loadMemPtrIntoRegister(addr_reg, Type.usize, mat_src_mcv); + switch (mat_src_mcv) { + .memory => |addr| try self.genSetReg(Type.usize, addr_reg, .{ .immediate = addr }), + .linker_load => |load_struct| { + const atom_index = if (self.bin_file.cast(link.File.MachO)) |macho_file| blk: { + const atom = try macho_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk macho_file.getAtom(atom).getSymbolIndex().?; + } else if (self.bin_file.cast(link.File.Coff)) |coff_file| blk: { + const atom = try coff_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk coff_file.getAtom(atom).getSymbolIndex().?; + } else unreachable; + + switch (load_struct.type) { + .import => unreachable, + .got => try self.asmMovLinker(addr_reg, atom_index, load_struct), + .direct => try self.asmLeaLinker(addr_reg, atom_index, load_struct), + } + }, + else => unreachable, + } + try self.asmCmovccRegisterMemory( registerAlias(tmp_reg, cmov_abi_size), Memory.sib(Memory.PtrSize.fromSize(cmov_abi_size), .{ .base = addr_reg }), @@ -5026,11 +5045,30 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s } = switch (dst_mcv) { else => unreachable, .memory, .linker_load => dst: { - const dst_addr_reg = try self.register_manager.allocReg(null, gp); + const dst_addr_reg = (try self.register_manager.allocReg(null, gp)).to64(); const dst_addr_lock = self.register_manager.lockRegAssumeUnused(dst_addr_reg); errdefer self.register_manager.unlockReg(dst_addr_lock); - try self.loadMemPtrIntoRegister(dst_addr_reg, Type.usize, dst_mcv); + switch (dst_mcv) { + .memory => |addr| try self.genSetReg(Type.usize, dst_addr_reg, .{ .immediate = addr }), + .linker_load => |load_struct| { + const atom_index = if (self.bin_file.cast(link.File.MachO)) |macho_file| blk: { + const atom = try macho_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk macho_file.getAtom(atom).getSymbolIndex().?; + } else if (self.bin_file.cast(link.File.Coff)) |coff_file| blk: { + const atom = try coff_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk coff_file.getAtom(atom).getSymbolIndex().?; + } else unreachable; + + switch (load_struct.type) { + .import => unreachable, + .got => try self.asmMovLinker(dst_addr_reg, atom_index, load_struct), + .direct => try self.asmLeaLinker(dst_addr_reg, atom_index, load_struct), + } + }, + else => unreachable, + } + break :dst .{ .addr_reg = dst_addr_reg, .addr_lock = dst_addr_lock, @@ -5052,11 +5090,30 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s const src_limb_lock = self.register_manager.lockRegAssumeUnused(src_limb_reg); errdefer self.register_manager.unlockReg(src_limb_lock); - const src_addr_reg = try self.register_manager.allocReg(null, gp); + const src_addr_reg = (try self.register_manager.allocReg(null, gp)).to64(); const src_addr_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg); errdefer self.register_manager.unlockReg(src_addr_lock); - try self.loadMemPtrIntoRegister(src_addr_reg, Type.usize, src_mcv); + switch (src_mcv) { + .memory => |addr| try self.genSetReg(Type.usize, src_addr_reg, .{ .immediate = addr }), + .linker_load => |load_struct| { + const atom_index = if (self.bin_file.cast(link.File.MachO)) |macho_file| blk: { + const atom = try macho_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk macho_file.getAtom(atom).getSymbolIndex().?; + } else if (self.bin_file.cast(link.File.Coff)) |coff_file| blk: { + const atom = try coff_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk coff_file.getAtom(atom).getSymbolIndex().?; + } else unreachable; + + switch (load_struct.type) { + .import => unreachable, + .got => try self.asmMovLinker(src_addr_reg, atom_index, load_struct), + .direct => try self.asmLeaLinker(src_addr_reg, atom_index, load_struct), + } + }, + else => unreachable, + } + break :src .{ .addr_reg = src_addr_reg, .addr_lock = src_addr_lock, From e05a51cb6264d073cc0d8d0d8c8f18f47084f170 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 12 Apr 2023 10:02:38 +0200 Subject: [PATCH 08/26] x86_64: simplify airArrayElemVal() when pulling array from memory --- src/arch/x86_64/CodeGen.zig | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 15cc5bd302..df97408dee 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -2897,7 +2897,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void { const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg); defer self.register_manager.unlockReg(offset_reg_lock); - const addr_reg = try self.register_manager.allocReg(null, gp); + const addr_reg = (try self.register_manager.allocReg(null, gp)).to64(); switch (array) { .register => { const off = @intCast(i32, try self.allocMem( @@ -2906,19 +2906,32 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void { array_ty.abiAlignment(self.target.*), )); try self.genSetStack(array_ty, off, array, .{}); - try self.asmRegisterMemory(.lea, addr_reg.to64(), Memory.sib(.qword, .{ + try self.asmRegisterMemory(.lea, addr_reg, Memory.sib(.qword, .{ .base = .rbp, .disp = -off, })); }, .stack_offset => |off| { - try self.asmRegisterMemory(.lea, addr_reg.to64(), Memory.sib(.qword, .{ + try self.asmRegisterMemory(.lea, addr_reg, Memory.sib(.qword, .{ .base = .rbp, .disp = -off, })); }, - .memory, .linker_load => { - try self.loadMemPtrIntoRegister(addr_reg, Type.usize, array); + .memory => |addr| try self.genSetReg(Type.usize, addr_reg, .{ .immediate = addr }), + .linker_load => |load_struct| { + const atom_index = if (self.bin_file.cast(link.File.MachO)) |macho_file| blk: { + const atom = try macho_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk macho_file.getAtom(atom).getSymbolIndex().?; + } else if (self.bin_file.cast(link.File.Coff)) |coff_file| blk: { + const atom = try coff_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk coff_file.getAtom(atom).getSymbolIndex().?; + } else unreachable; + + switch (load_struct.type) { + .import => unreachable, + .got => try self.asmMovLinker(addr_reg, atom_index, load_struct), + .direct => try self.asmLeaLinker(addr_reg, atom_index, load_struct), + } }, else => return self.fail("TODO implement array_elem_val when array is {}", .{array}), } From 059ed062bc8d98be338c294d5ef443b11e0a180b Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 12 Apr 2023 10:10:13 +0200 Subject: [PATCH 09/26] x86_64: simplify isNull() when loading optional from memory --- src/arch/x86_64/CodeGen.zig | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index df97408dee..f3e5b8b38f 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -6144,7 +6144,25 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg); defer self.register_manager.unlockReg(addr_reg_lock); - try self.loadMemPtrIntoRegister(addr_reg, Type.usize, opt_mcv); + switch (opt_mcv) { + .memory => |addr| try self.genSetReg(Type.usize, addr_reg, .{ .immediate = addr }), + .linker_load => |load_struct| { + const atom_index = if (self.bin_file.cast(link.File.MachO)) |macho_file| blk: { + const atom = try macho_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk macho_file.getAtom(atom).getSymbolIndex().?; + } else if (self.bin_file.cast(link.File.Coff)) |coff_file| blk: { + const atom = try coff_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk coff_file.getAtom(atom).getSymbolIndex().?; + } else unreachable; + + switch (load_struct.type) { + .import => unreachable, + .got => try self.asmMovLinker(addr_reg, atom_index, load_struct), + .direct => try self.asmLeaLinker(addr_reg, atom_index, load_struct), + } + }, + else => unreachable, + } const some_abi_size = @intCast(u32, some_info.ty.abiSize(self.target.*)); try self.asmMemoryImmediate(.cmp, Memory.sib( From 0abaf5f88a687d29dea8556a3c4e79b4a61815e3 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 12 Apr 2023 10:19:43 +0200 Subject: [PATCH 10/26] x86_64: simplify genSetStackArg() when storing value from memory --- src/arch/x86_64/CodeGen.zig | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index f3e5b8b38f..7d3ba96b84 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -7097,11 +7097,30 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg }); } - const addr_reg = try self.register_manager.allocReg(null, gp); + const addr_reg = (try self.register_manager.allocReg(null, gp)).to64(); const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg); defer self.register_manager.unlockReg(addr_lock); - try self.loadMemPtrIntoRegister(addr_reg, Type.usize, mcv); + switch (mcv) { + .memory => |addr| try self.genSetReg(Type.usize, addr_reg, .{ .immediate = addr }), + .linker_load => |load_struct| { + const atom_index = if (self.bin_file.cast(link.File.MachO)) |macho_file| blk: { + const atom = try macho_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk macho_file.getAtom(atom).getSymbolIndex().?; + } else if (self.bin_file.cast(link.File.Coff)) |coff_file| blk: { + const atom = try coff_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk coff_file.getAtom(atom).getSymbolIndex().?; + } else unreachable; + + switch (load_struct.type) { + .import => unreachable, + .got => try self.asmMovLinker(addr_reg, atom_index, load_struct), + .direct => try self.asmLeaLinker(addr_reg, atom_index, load_struct), + } + }, + else => unreachable, + } + try self.genInlineMemcpy( .{ .ptr_stack_offset = stack_offset }, .{ .register = addr_reg }, From aab0039b42ba40fe666782b4f0bd08d0e5d3c460 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 12 Apr 2023 10:26:13 +0200 Subject: [PATCH 11/26] x86_64: simplify genSetStack() when storing value from memory --- src/arch/x86_64/CodeGen.zig | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 7d3ba96b84..bcd30013aa 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -7324,11 +7324,30 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl const reg = try self.copyToTmpRegister(ty, mcv); return self.genSetStack(ty, stack_offset, MCValue{ .register = reg }, opts); } else { - const addr_reg = try self.register_manager.allocReg(null, gp); + const addr_reg = (try self.register_manager.allocReg(null, gp)).to64(); const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg); defer self.register_manager.unlockReg(addr_lock); - try self.loadMemPtrIntoRegister(addr_reg, Type.usize, mcv); + switch (mcv) { + .memory => |addr| try self.genSetReg(Type.usize, addr_reg, .{ .immediate = addr }), + .linker_load => |load_struct| { + const atom_index = if (self.bin_file.cast(link.File.MachO)) |macho_file| blk: { + const atom = try macho_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk macho_file.getAtom(atom).getSymbolIndex().?; + } else if (self.bin_file.cast(link.File.Coff)) |coff_file| blk: { + const atom = try coff_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk coff_file.getAtom(atom).getSymbolIndex().?; + } else unreachable; + + switch (load_struct.type) { + .import => unreachable, + .got => try self.asmMovLinker(addr_reg, atom_index, load_struct), + .direct => try self.asmLeaLinker(addr_reg, atom_index, load_struct), + } + }, + else => unreachable, + } + try self.genInlineMemcpy( .{ .ptr_stack_offset = stack_offset }, .{ .register = addr_reg }, From 4e3573955fd912a6b2eee982f90743a939e8cbb1 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 12 Apr 2023 10:32:08 +0200 Subject: [PATCH 12/26] x86_64: simplify genInlineMemcpy() when copying from memory --- src/arch/x86_64/CodeGen.zig | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index bcd30013aa..96350cb164 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -7457,11 +7457,25 @@ fn genInlineMemcpy( try self.spillRegisters(&.{ .rdi, .rsi, .rcx }); switch (dst_ptr) { - .memory, .linker_load => { - try self.loadMemPtrIntoRegister(.rdi, Type.usize, dst_ptr); + .memory => |addr| { + try self.genSetReg(Type.usize, .rdi, .{ .immediate = addr }); // Load the pointer, which is stored in memory try self.asmRegisterMemory(.mov, .rdi, Memory.sib(.qword, .{ .base = .rdi })); }, + .linker_load => |load_struct| { + const atom_index = if (self.bin_file.cast(link.File.MachO)) |macho_file| blk: { + const atom = try macho_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk macho_file.getAtom(atom).getSymbolIndex().?; + } else if (self.bin_file.cast(link.File.Coff)) |coff_file| blk: { + const atom = try coff_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk coff_file.getAtom(atom).getSymbolIndex().?; + } else unreachable; + + switch (load_struct.type) { + .import => unreachable, + .got, .direct => try self.asmMovLinker(.rdi, atom_index, load_struct), + } + }, .stack_offset, .ptr_stack_offset => |off| { try self.asmRegisterMemory(switch (dst_ptr) { .stack_offset => .mov, @@ -7485,11 +7499,25 @@ fn genInlineMemcpy( } switch (src_ptr) { - .memory, .linker_load => { - try self.loadMemPtrIntoRegister(.rsi, Type.usize, src_ptr); + .memory => |addr| { + try self.genSetReg(Type.usize, .rsi, .{ .immediate = addr }); // Load the pointer, which is stored in memory try self.asmRegisterMemory(.mov, .rsi, Memory.sib(.qword, .{ .base = .rsi })); }, + .linker_load => |load_struct| { + const atom_index = if (self.bin_file.cast(link.File.MachO)) |macho_file| blk: { + const atom = try macho_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk macho_file.getAtom(atom).getSymbolIndex().?; + } else if (self.bin_file.cast(link.File.Coff)) |coff_file| blk: { + const atom = try coff_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk coff_file.getAtom(atom).getSymbolIndex().?; + } else unreachable; + + switch (load_struct.type) { + .import => unreachable, + .got, .direct => try self.asmMovLinker(.rsi, atom_index, load_struct), + } + }, .stack_offset, .ptr_stack_offset => |off| { try self.asmRegisterMemory(switch (src_ptr) { .stack_offset => .mov, From e07120d704fe931dd8f6952d20739dc1e1b194aa Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 12 Apr 2023 10:36:00 +0200 Subject: [PATCH 13/26] x86_64: simplify genInlineMemset() when setting value in memory --- src/arch/x86_64/CodeGen.zig | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 96350cb164..1bf99c4e1f 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -7564,11 +7564,25 @@ fn genInlineMemset( try self.spillRegisters(&.{ .rdi, .al, .rcx }); switch (dst_ptr) { - .memory, .linker_load => { - try self.loadMemPtrIntoRegister(.rdi, Type.usize, dst_ptr); + .memory => |addr| { + try self.genSetReg(Type.usize, .rdi, .{ .immediate = addr }); // Load the pointer, which is stored in memory try self.asmRegisterMemory(.mov, .rdi, Memory.sib(.qword, .{ .base = .rdi })); }, + .linker_load => |load_struct| { + const atom_index = if (self.bin_file.cast(link.File.MachO)) |macho_file| blk: { + const atom = try macho_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk macho_file.getAtom(atom).getSymbolIndex().?; + } else if (self.bin_file.cast(link.File.Coff)) |coff_file| blk: { + const atom = try coff_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk coff_file.getAtom(atom).getSymbolIndex().?; + } else unreachable; + + switch (load_struct.type) { + .import => unreachable, + .got, .direct => try self.asmMovLinker(.rdi, atom_index, load_struct), + } + }, .stack_offset, .ptr_stack_offset => |off| { try self.asmRegisterMemory(switch (dst_ptr) { .stack_offset => .mov, From cf9ba2965eb90de994dd648950ad81041f90b755 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 12 Apr 2023 10:36:28 +0200 Subject: [PATCH 14/26] x86_64: remove loadMemPtrIntoRegister() now redundant --- src/arch/x86_64/CodeGen.zig | 35 ----------------------------------- 1 file changed, 35 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 1bf99c4e1f..491144283c 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -3677,41 +3677,6 @@ fn airLoad(self: *Self, inst: Air.Inst.Index) !void { return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); } -fn loadMemPtrIntoRegister(self: *Self, reg: Register, ptr_ty: Type, ptr: MCValue) InnerError!void { - switch (ptr) { - .linker_load => |load_struct| { - const abi_size = @intCast(u32, ptr_ty.abiSize(self.target.*)); - const atom_index = if (self.bin_file.cast(link.File.MachO)) |macho_file| blk: { - const atom = try macho_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); - break :blk macho_file.getAtom(atom).getSymbolIndex().?; - } else if (self.bin_file.cast(link.File.Coff)) |coff_file| blk: { - const atom = try coff_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); - break :blk coff_file.getAtom(atom).getSymbolIndex().?; - } else unreachable; - const ops: Mir.Inst.Ops = switch (load_struct.type) { - .got => .got_reloc, - .direct => .direct_reloc, - .import => .import_reloc, - }; - _ = try self.addInst(.{ - .tag = .lea_linker, - .ops = ops, - .data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{ - .reg = @enumToInt(registerAlias(reg, abi_size)), - .atom_index = atom_index, - .sym_index = load_struct.sym_index, - }) }, - }); - }, - .memory => |addr| { - // TODO: in case the address fits in an imm32 we can use [ds:imm32] - // instead of wasting an instruction copying the address to a register - try self.genSetReg(ptr_ty, reg, .{ .immediate = addr }); - }, - else => unreachable, - } -} - fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type) InnerError!void { const abi_size = @intCast(u32, value_ty.abiSize(self.target.*)); switch (ptr) { From 1795b8eb4e8db95cb639349c6c1bc4b0b82741bb Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 4 Apr 2023 08:48:48 +0200 Subject: [PATCH 15/26] macho: emit TLS sections --- src/link/MachO.zig | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/src/link/MachO.zig b/src/link/MachO.zig index 5ec9c9c356..ad0dd8ac1d 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -138,6 +138,8 @@ got_section_index: ?u8 = null, data_const_section_index: ?u8 = null, la_symbol_ptr_section_index: ?u8 = null, data_section_index: ?u8 = null, +tls_vars_section_index: ?u8 = null, +tls_data_section_index: ?u8 = null, locals: std.ArrayListUnmanaged(macho.nlist_64) = .{}, globals: std.ArrayListUnmanaged(SymbolWithLoc) = .{}, @@ -2366,6 +2368,7 @@ fn getDeclOutputSection(self: *MachO, decl_index: Module.Decl.Index) u8 { const val = decl.val; const zig_ty = ty.zigTypeTag(); const mode = self.base.options.optimize_mode; + const single_threaded = self.base.options.single_threaded; const sect_id: u8 = blk: { // TODO finish and audit this function if (val.isUndefDeep()) { @@ -2376,7 +2379,10 @@ fn getDeclOutputSection(self: *MachO, decl_index: Module.Decl.Index) u8 { } } - if (val.castTag(.variable)) |_| { + if (val.castTag(.variable)) |variable| { + if (variable.data.is_threadlocal and !single_threaded) { + break :blk self.tls_data_section_index.?; + } break :blk self.data_section_index.?; } @@ -2800,6 +2806,28 @@ fn populateMissingMetadata(self: *MachO) !void { self.segment_table_dirty = true; } + if (!self.base.options.single_threaded) { + if (self.tls_vars_section_index == null) { + self.tls_vars_section_index = try self.allocateSection("__DATA2", "__thread_vars", .{ + .size = @sizeOf(u64) * 3, + .alignment = @alignOf(u64), + .flags = macho.S_THREAD_LOCAL_VARIABLES, + .prot = macho.PROT.READ | macho.PROT.WRITE, + }); + self.segment_table_dirty = true; + } + + if (self.tls_data_section_index == null) { + self.tls_data_section_index = try self.allocateSection("__DATA3", "__thread_data", .{ + .size = @sizeOf(u64), + .alignment = @alignOf(u64), + .flags = macho.S_THREAD_LOCAL_REGULAR, + .prot = macho.PROT.READ | macho.PROT.WRITE, + }); + self.segment_table_dirty = true; + } + } + if (self.linkedit_segment_cmd_index == null) { self.linkedit_segment_cmd_index = @intCast(u8, self.segments.items.len); From eba280ce20d308ec0abb17719ab4cc43a27901eb Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 5 Apr 2023 08:23:46 +0200 Subject: [PATCH 16/26] macho: refactor relocation type in incremental linker --- src/arch/aarch64/Emit.zig | 29 ++---- src/arch/x86_64/Emit.zig | 13 ++- src/link/MachO.zig | 68 ++++--------- src/link/MachO/Atom.zig | 16 +-- src/link/MachO/Relocation.zig | 182 +++++++++++++++------------------- 5 files changed, 118 insertions(+), 190 deletions(-) diff --git a/src/arch/aarch64/Emit.zig b/src/arch/aarch64/Emit.zig index b2e23c6278..3903229a21 100644 --- a/src/arch/aarch64/Emit.zig +++ b/src/arch/aarch64/Emit.zig @@ -673,7 +673,7 @@ fn mirCallExtern(emit: *Emit, inst: Mir.Inst.Index) !void { const atom_index = macho_file.getAtomIndexForSymbol(.{ .sym_index = relocation.atom_index, .file = null }).?; const target = macho_file.getGlobalByIndex(relocation.sym_index); try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{ - .type = @enumToInt(std.macho.reloc_type_arm64.ARM64_RELOC_BRANCH26), + .type = .branch, .target = target, .offset = offset, .addend = 0, @@ -883,41 +883,32 @@ fn mirLoadMemoryPie(emit: *Emit, inst: Mir.Inst.Index) !void { } if (emit.bin_file.cast(link.File.MachO)) |macho_file| { + const Atom = link.File.MachO.Atom; + const Relocation = Atom.Relocation; const atom_index = macho_file.getAtomIndexForSymbol(.{ .sym_index = data.atom_index, .file = null }).?; - // TODO this causes segfault in stage1 - // try atom.addRelocations(macho_file, 2, .{ - try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{ + try Atom.addRelocations(macho_file, atom_index, &[_]Relocation{ .{ .target = .{ .sym_index = data.sym_index, .file = null }, .offset = offset, .addend = 0, .pcrel = true, .length = 2, .type = switch (tag) { - .load_memory_got, - .load_memory_ptr_got, - => @enumToInt(std.macho.reloc_type_arm64.ARM64_RELOC_GOT_LOAD_PAGE21), - .load_memory_direct, - .load_memory_ptr_direct, - => @enumToInt(std.macho.reloc_type_arm64.ARM64_RELOC_PAGE21), + .load_memory_got, .load_memory_ptr_got => Relocation.Type.got_page, + .load_memory_direct, .load_memory_ptr_direct => Relocation.Type.page, else => unreachable, }, - }); - try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{ + }, .{ .target = .{ .sym_index = data.sym_index, .file = null }, .offset = offset + 4, .addend = 0, .pcrel = false, .length = 2, .type = switch (tag) { - .load_memory_got, - .load_memory_ptr_got, - => @enumToInt(std.macho.reloc_type_arm64.ARM64_RELOC_GOT_LOAD_PAGEOFF12), - .load_memory_direct, - .load_memory_ptr_direct, - => @enumToInt(std.macho.reloc_type_arm64.ARM64_RELOC_PAGEOFF12), + .load_memory_got, .load_memory_ptr_got => Relocation.Type.got_pageoff, + .load_memory_direct, .load_memory_ptr_direct => Relocation.Type.pageoff, else => unreachable, }, - }); + } }); } else if (emit.bin_file.cast(link.File.Coff)) |coff_file| { const atom_index = coff_file.getAtomIndexForSymbol(.{ .sym_index = data.atom_index, .file = null }).?; const target = switch (tag) { diff --git a/src/arch/x86_64/Emit.zig b/src/arch/x86_64/Emit.zig index 30d63b6adf..16fb8a0183 100644 --- a/src/arch/x86_64/Emit.zig +++ b/src/arch/x86_64/Emit.zig @@ -42,7 +42,7 @@ pub fn emitMir(emit: *Emit) Error!void { ).?; const target = macho_file.getGlobalByIndex(inst.data.relocation.sym_index); try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{ - .type = @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_BRANCH), + .type = .branch, .target = target, .offset = end_offset - 4, .addend = 0, @@ -68,17 +68,16 @@ pub fn emitMir(emit: *Emit) Error!void { .mov_linker, .lea_linker => if (emit.bin_file.cast(link.File.MachO)) |macho_file| { const metadata = emit.lower.mir.extraData(Mir.LeaRegisterReloc, inst.data.payload).data; - const reloc_type = switch (inst.ops) { - .got_reloc => @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_GOT), - .direct_reloc => @enumToInt(std.macho.reloc_type_x86_64.X86_64_RELOC_SIGNED), - else => unreachable, - }; const atom_index = macho_file.getAtomIndexForSymbol(.{ .sym_index = metadata.atom_index, .file = null, }).?; try link.File.MachO.Atom.addRelocation(macho_file, atom_index, .{ - .type = reloc_type, + .type = switch (inst.ops) { + .got_reloc => .got, + .direct_reloc => .signed, + else => unreachable, + }, .target = .{ .sym_index = metadata.sym_index, .file = null }, .offset = @intCast(u32, end_offset - 4), .addend = 0, diff --git a/src/link/MachO.zig b/src/link/MachO.zig index ad0dd8ac1d..cc5b9ecdc6 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -1250,11 +1250,7 @@ pub fn createGotAtom(self: *MachO, target: SymbolWithLoc) !Atom.Index { log.debug("allocated GOT atom at 0x{x}", .{sym.n_value}); try Atom.addRelocation(self, atom_index, .{ - .type = switch (self.base.options.target.cpu.arch) { - .aarch64 => @enumToInt(macho.reloc_type_arm64.ARM64_RELOC_UNSIGNED), - .x86_64 => @enumToInt(macho.reloc_type_x86_64.X86_64_RELOC_UNSIGNED), - else => unreachable, - }, + .type = .unsigned, .target = target, .offset = 0, .addend = 0, @@ -1336,15 +1332,15 @@ fn createStubHelperPreambleAtom(self: *MachO) !void { code[9] = 0xff; code[10] = 0x25; - try Atom.addRelocations(self, atom_index, 2, .{ .{ - .type = @enumToInt(macho.reloc_type_x86_64.X86_64_RELOC_SIGNED), + try Atom.addRelocations(self, atom_index, &[_]Relocation{ .{ + .type = .signed, .target = dyld_private, .offset = 3, .addend = 0, .pcrel = true, .length = 2, }, .{ - .type = @enumToInt(macho.reloc_type_x86_64.X86_64_RELOC_GOT), + .type = .got, .target = dyld_stub_binder, .offset = 11, .addend = 0, @@ -1376,29 +1372,29 @@ fn createStubHelperPreambleAtom(self: *MachO) !void { // br x16 mem.writeIntLittle(u32, code[20..][0..4], aarch64.Instruction.br(.x16).toU32()); - try Atom.addRelocations(self, atom_index, 4, .{ .{ - .type = @enumToInt(macho.reloc_type_arm64.ARM64_RELOC_PAGE21), + try Atom.addRelocations(self, atom_index, &[_]Relocation{ .{ + .type = .page, .target = dyld_private, .offset = 0, .addend = 0, .pcrel = true, .length = 2, }, .{ - .type = @enumToInt(macho.reloc_type_arm64.ARM64_RELOC_PAGEOFF12), + .type = .pageoff, .target = dyld_private, .offset = 4, .addend = 0, .pcrel = false, .length = 2, }, .{ - .type = @enumToInt(macho.reloc_type_arm64.ARM64_RELOC_GOT_LOAD_PAGE21), + .type = .got_page, .target = dyld_stub_binder, .offset = 12, .addend = 0, .pcrel = true, .length = 2, }, .{ - .type = @enumToInt(macho.reloc_type_arm64.ARM64_RELOC_GOT_LOAD_PAGEOFF12), + .type = .got_pageoff, .target = dyld_stub_binder, .offset = 16, .addend = 0, @@ -1456,7 +1452,7 @@ fn createStubHelperAtom(self: *MachO) !Atom.Index { code[5] = 0xe9; try Atom.addRelocation(self, atom_index, .{ - .type = @enumToInt(macho.reloc_type_x86_64.X86_64_RELOC_BRANCH), + .type = .branch, .target = .{ .sym_index = stub_helper_preamble_atom_sym_index, .file = null }, .offset = 6, .addend = 0, @@ -1479,7 +1475,7 @@ fn createStubHelperAtom(self: *MachO) !Atom.Index { // Next 4 bytes 8..12 are just a placeholder populated in `populateLazyBindOffsetsInStubHelper`. try Atom.addRelocation(self, atom_index, .{ - .type = @enumToInt(macho.reloc_type_arm64.ARM64_RELOC_BRANCH26), + .type = .branch, .target = .{ .sym_index = stub_helper_preamble_atom_sym_index, .file = null }, .offset = 4, .addend = 0, @@ -1507,11 +1503,7 @@ fn createLazyPointerAtom(self: *MachO, stub_sym_index: u32, target: SymbolWithLo sym.n_sect = self.la_symbol_ptr_section_index.? + 1; try Atom.addRelocation(self, atom_index, .{ - .type = switch (self.base.options.target.cpu.arch) { - .aarch64 => @enumToInt(macho.reloc_type_arm64.ARM64_RELOC_UNSIGNED), - .x86_64 => @enumToInt(macho.reloc_type_x86_64.X86_64_RELOC_UNSIGNED), - else => unreachable, - }, + .type = .unsigned, .target = .{ .sym_index = stub_sym_index, .file = null }, .offset = 0, .addend = 0, @@ -1565,7 +1557,7 @@ fn createStubAtom(self: *MachO, laptr_sym_index: u32) !Atom.Index { code[1] = 0x25; try Atom.addRelocation(self, atom_index, .{ - .type = @enumToInt(macho.reloc_type_x86_64.X86_64_RELOC_BRANCH), + .type = .branch, .target = .{ .sym_index = laptr_sym_index, .file = null }, .offset = 2, .addend = 0, @@ -1585,9 +1577,9 @@ fn createStubAtom(self: *MachO, laptr_sym_index: u32) !Atom.Index { // br x16 mem.writeIntLittle(u32, code[8..12], aarch64.Instruction.br(.x16).toU32()); - try Atom.addRelocations(self, atom_index, 2, .{ + try Atom.addRelocations(self, atom_index, &[_]Relocation{ .{ - .type = @enumToInt(macho.reloc_type_arm64.ARM64_RELOC_PAGE21), + .type = .page, .target = .{ .sym_index = laptr_sym_index, .file = null }, .offset = 0, .addend = 0, @@ -1595,7 +1587,7 @@ fn createStubAtom(self: *MachO, laptr_sym_index: u32) !Atom.Index { .length = 2, }, .{ - .type = @enumToInt(macho.reloc_type_arm64.ARM64_RELOC_PAGEOFF12), + .type = .pageoff, .target = .{ .sym_index = laptr_sym_index, .file = null }, .offset = 4, .addend = 0, @@ -2663,11 +2655,7 @@ pub fn getDeclVAddr(self: *MachO, decl_index: Module.Decl.Index, reloc_info: Fil const sym_index = self.getAtom(this_atom_index).getSymbolIndex().?; const atom_index = self.getAtomIndexForSymbol(.{ .sym_index = reloc_info.parent_atom_index, .file = null }).?; try Atom.addRelocation(self, atom_index, .{ - .type = switch (self.base.options.target.cpu.arch) { - .aarch64 => @enumToInt(macho.reloc_type_arm64.ARM64_RELOC_UNSIGNED), - .x86_64 => @enumToInt(macho.reloc_type_x86_64.X86_64_RELOC_UNSIGNED), - else => unreachable, - }, + .type = .unsigned, .target = .{ .sym_index = sym_index, .file = null }, .offset = @intCast(u32, reloc_info.offset), .addend = reloc_info.addend, @@ -3115,28 +3103,6 @@ fn allocateAtom(self: *MachO, atom_index: Atom.Index, new_atom_size: u64, alignm return vaddr; } -fn getSectionPrecedence(header: macho.section_64) u4 { - if (header.isCode()) { - if (mem.eql(u8, "__text", header.sectName())) return 0x0; - if (header.type() == macho.S_SYMBOL_STUBS) return 0x1; - return 0x2; - } - switch (header.type()) { - macho.S_NON_LAZY_SYMBOL_POINTERS, - macho.S_LAZY_SYMBOL_POINTERS, - => return 0x0, - macho.S_MOD_INIT_FUNC_POINTERS => return 0x1, - macho.S_MOD_TERM_FUNC_POINTERS => return 0x2, - macho.S_ZEROFILL => return 0xf, - macho.S_THREAD_LOCAL_REGULAR => return 0xd, - macho.S_THREAD_LOCAL_ZEROFILL => return 0xe, - else => if (mem.eql(u8, "__eh_frame", header.sectName())) - return 0xf - else - return 0x3, - } -} - pub fn getGlobalSymbol(self: *MachO, name: []const u8, lib_name: ?[]const u8) !u32 { _ = lib_name; const gpa = self.base.allocator; diff --git a/src/link/MachO/Atom.zig b/src/link/MachO/Atom.zig index bd1a21a04f..5b17dc689d 100644 --- a/src/link/MachO/Atom.zig +++ b/src/link/MachO/Atom.zig @@ -14,7 +14,7 @@ const trace = @import("../../tracy.zig").trace; const Allocator = mem.Allocator; const Arch = std.Target.Cpu.Arch; const MachO = @import("../MachO.zig"); -const Relocation = @import("Relocation.zig"); +pub const Relocation = @import("Relocation.zig"); const SymbolWithLoc = MachO.SymbolWithLoc; /// Each decl always gets a local symbol with the fully qualified name. @@ -113,25 +113,19 @@ pub fn freeListEligible(self: Atom, macho_file: *MachO) bool { } pub fn addRelocation(macho_file: *MachO, atom_index: Index, reloc: Relocation) !void { - return addRelocations(macho_file, atom_index, 1, .{reloc}); + return addRelocations(macho_file, atom_index, &[_]Relocation{reloc}); } -pub fn addRelocations( - macho_file: *MachO, - atom_index: Index, - comptime count: comptime_int, - relocs: [count]Relocation, -) !void { +pub fn addRelocations(macho_file: *MachO, atom_index: Index, relocs: []Relocation) !void { const gpa = macho_file.base.allocator; - const target = macho_file.base.options.target; const gop = try macho_file.relocs.getOrPut(gpa, atom_index); if (!gop.found_existing) { gop.value_ptr.* = .{}; } - try gop.value_ptr.ensureUnusedCapacity(gpa, count); + try gop.value_ptr.ensureUnusedCapacity(gpa, relocs.len); for (relocs) |reloc| { log.debug(" (adding reloc of type {s} to target %{d})", .{ - reloc.fmtType(target), + @tagName(reloc.type), reloc.target.sym_index, }); gop.value_ptr.appendAssumeCapacity(reloc); diff --git a/src/link/MachO/Relocation.zig b/src/link/MachO/Relocation.zig index 32f24f243d..1d1d91279c 100644 --- a/src/link/MachO/Relocation.zig +++ b/src/link/MachO/Relocation.zig @@ -1,19 +1,7 @@ -const Relocation = @This(); +//! Relocation used by the self-hosted backends to instruct the linker where and how to +//! fixup the values when flushing the contents to file and/or memory. -const std = @import("std"); -const aarch64 = @import("../../arch/aarch64/bits.zig"); -const assert = std.debug.assert; -const log = std.log.scoped(.link); -const macho = std.macho; -const math = std.math; -const mem = std.mem; -const meta = std.meta; - -const Atom = @import("Atom.zig"); -const MachO = @import("../MachO.zig"); -const SymbolWithLoc = MachO.SymbolWithLoc; - -type: u4, +type: Type, target: SymbolWithLoc, offset: u32, addend: i64, @@ -21,36 +9,46 @@ pcrel: bool, length: u2, dirty: bool = true, +pub const Type = enum { + // x86, x86_64 + /// RIP-relative displacement to a GOT pointer + got, + /// RIP-relative displacement + signed, + /// RIP-relative displacemen to threadlocal variable descriptor + tlv, + + // aarch64 + /// PC-relative distance to target page in GOT section + got_page, + /// Offset to a GOT pointer relative to the start of a page in GOT section + got_pageoff, + /// PC-relative distance to target page in a section + page, + /// Offset to a pointer relative to the start of a page in a section + pageoff, + /// PC-relative distance to target page in TLV section + tlv_page, + /// Offset to a pointer relative to the start of a page in TLV section + tlv_pageoff, + + // common + /// PC/RIP-relative displacement B/BL/CALL + branch, + /// Absolute pointer value + unsigned, +}; + /// Returns true if and only if the reloc is dirty AND the target address is available. pub fn isResolvable(self: Relocation, macho_file: *MachO) bool { _ = self.getTargetAtomIndex(macho_file) orelse return false; return self.dirty; } -pub fn fmtType(self: Relocation, target: std.Target) []const u8 { - switch (target.cpu.arch) { - .aarch64 => return @tagName(@intToEnum(macho.reloc_type_arm64, self.type)), - .x86_64 => return @tagName(@intToEnum(macho.reloc_type_x86_64, self.type)), - else => unreachable, - } -} - pub fn getTargetAtomIndex(self: Relocation, macho_file: *MachO) ?Atom.Index { - switch (macho_file.base.options.target.cpu.arch) { - .aarch64 => switch (@intToEnum(macho.reloc_type_arm64, self.type)) { - .ARM64_RELOC_GOT_LOAD_PAGE21, - .ARM64_RELOC_GOT_LOAD_PAGEOFF12, - .ARM64_RELOC_POINTER_TO_GOT, - => return macho_file.getGotAtomIndexForSymbol(self.target), - else => {}, - }, - .x86_64 => switch (@intToEnum(macho.reloc_type_x86_64, self.type)) { - .X86_64_RELOC_GOT, - .X86_64_RELOC_GOT_LOAD, - => return macho_file.getGotAtomIndexForSymbol(self.target), - else => {}, - }, - else => unreachable, + switch (self.type) { + .got, .got_page, .got_pageoff => return macho_file.getGotAtomIndexForSymbol(self.target), + else => {}, } if (macho_file.getStubsAtomIndexForSymbol(self.target)) |stubs_atom| return stubs_atom; return macho_file.getAtomIndexForSymbol(self.target); @@ -70,7 +68,7 @@ pub fn resolve(self: Relocation, macho_file: *MachO, atom_index: Atom.Index, cod source_addr, target_addr, macho_file.getSymbolName(self.target), - self.fmtType(macho_file.base.options.target), + @tagName(self.type), }); switch (arch) { @@ -81,18 +79,9 @@ pub fn resolve(self: Relocation, macho_file: *MachO, atom_index: Atom.Index, cod } fn resolveAarch64(self: Relocation, source_addr: u64, target_addr: i64, code: []u8) void { - const rel_type = @intToEnum(macho.reloc_type_arm64, self.type); - if (rel_type == .ARM64_RELOC_UNSIGNED) { - return switch (self.length) { - 2 => mem.writeIntLittle(u32, code[self.offset..][0..4], @truncate(u32, @bitCast(u64, target_addr))), - 3 => mem.writeIntLittle(u64, code[self.offset..][0..8], @bitCast(u64, target_addr)), - else => unreachable, - }; - } - - var buffer = code[self.offset..][0..4]; - switch (rel_type) { - .ARM64_RELOC_BRANCH26 => { + var buffer = code[self.offset..]; + switch (self.type) { + .branch => { const displacement = math.cast( i28, @intCast(i64, target_addr) - @intCast(i64, source_addr), @@ -101,15 +90,12 @@ fn resolveAarch64(self: Relocation, source_addr: u64, target_addr: i64, code: [] .unconditional_branch_immediate = mem.bytesToValue(meta.TagPayload( aarch64.Instruction, aarch64.Instruction.unconditional_branch_immediate, - ), buffer), + ), buffer[0..4]), }; inst.unconditional_branch_immediate.imm26 = @truncate(u26, @bitCast(u28, displacement >> 2)); - mem.writeIntLittle(u32, buffer, inst.toU32()); + mem.writeIntLittle(u32, buffer[0..4], inst.toU32()); }, - .ARM64_RELOC_PAGE21, - .ARM64_RELOC_GOT_LOAD_PAGE21, - .ARM64_RELOC_TLVP_LOAD_PAGE21, - => { + .page, .got_page, .tlv_page => { const source_page = @intCast(i32, source_addr >> 12); const target_page = @intCast(i32, target_addr >> 12); const pages = @bitCast(u21, @intCast(i21, target_page - source_page)); @@ -117,31 +103,29 @@ fn resolveAarch64(self: Relocation, source_addr: u64, target_addr: i64, code: [] .pc_relative_address = mem.bytesToValue(meta.TagPayload( aarch64.Instruction, aarch64.Instruction.pc_relative_address, - ), buffer), + ), buffer[0..4]), }; inst.pc_relative_address.immhi = @truncate(u19, pages >> 2); inst.pc_relative_address.immlo = @truncate(u2, pages); - mem.writeIntLittle(u32, buffer, inst.toU32()); + mem.writeIntLittle(u32, buffer[0..4], inst.toU32()); }, - .ARM64_RELOC_PAGEOFF12, - .ARM64_RELOC_GOT_LOAD_PAGEOFF12, - => { + .pageoff, .got_pageoff => { const narrowed = @truncate(u12, @intCast(u64, target_addr)); - if (isArithmeticOp(buffer)) { + if (isArithmeticOp(buffer[0..4])) { var inst = aarch64.Instruction{ .add_subtract_immediate = mem.bytesToValue(meta.TagPayload( aarch64.Instruction, aarch64.Instruction.add_subtract_immediate, - ), buffer), + ), buffer[0..4]), }; inst.add_subtract_immediate.imm12 = narrowed; - mem.writeIntLittle(u32, buffer, inst.toU32()); + mem.writeIntLittle(u32, buffer[0..4], inst.toU32()); } else { var inst = aarch64.Instruction{ .load_store_register = mem.bytesToValue(meta.TagPayload( aarch64.Instruction, aarch64.Instruction.load_store_register, - ), buffer), + ), buffer[0..4]), }; const offset: u12 = blk: { if (inst.load_store_register.size == 0) { @@ -157,21 +141,21 @@ fn resolveAarch64(self: Relocation, source_addr: u64, target_addr: i64, code: [] } }; inst.load_store_register.offset = offset; - mem.writeIntLittle(u32, buffer, inst.toU32()); + mem.writeIntLittle(u32, buffer[0..4], inst.toU32()); } }, - .ARM64_RELOC_TLVP_LOAD_PAGEOFF12 => { + .tlv_pageoff => { const RegInfo = struct { rd: u5, rn: u5, size: u2, }; const reg_info: RegInfo = blk: { - if (isArithmeticOp(buffer)) { + if (isArithmeticOp(buffer[0..4])) { const inst = mem.bytesToValue(meta.TagPayload( aarch64.Instruction, aarch64.Instruction.add_subtract_immediate, - ), buffer); + ), buffer[0..4]); break :blk .{ .rd = inst.rd, .rn = inst.rn, @@ -181,7 +165,7 @@ fn resolveAarch64(self: Relocation, source_addr: u64, target_addr: i64, code: [] const inst = mem.bytesToValue(meta.TagPayload( aarch64.Instruction, aarch64.Instruction.load_store_register, - ), buffer); + ), buffer[0..4]); break :blk .{ .rd = inst.rt, .rn = inst.rn, @@ -201,45 +185,24 @@ fn resolveAarch64(self: Relocation, source_addr: u64, target_addr: i64, code: [] .sf = @truncate(u1, reg_info.size), }, }; - mem.writeIntLittle(u32, buffer, inst.toU32()); + mem.writeIntLittle(u32, buffer[0..4], inst.toU32()); }, - .ARM64_RELOC_POINTER_TO_GOT => { - const result = @intCast(i32, @intCast(i64, target_addr) - @intCast(i64, source_addr)); - mem.writeIntLittle(i32, buffer, result); + .unsigned => switch (self.length) { + 2 => mem.writeIntLittle(u32, buffer[0..4], @truncate(u32, @bitCast(u64, target_addr))), + 3 => mem.writeIntLittle(u64, buffer[0..8], @bitCast(u64, target_addr)), + else => unreachable, }, - .ARM64_RELOC_SUBTRACTOR => unreachable, - .ARM64_RELOC_ADDEND => unreachable, - .ARM64_RELOC_UNSIGNED => unreachable, + .got, .signed, .tlv => unreachable, // Invalid target architecture. } } fn resolveX8664(self: Relocation, source_addr: u64, target_addr: i64, code: []u8) void { - const rel_type = @intToEnum(macho.reloc_type_x86_64, self.type); - switch (rel_type) { - .X86_64_RELOC_BRANCH, - .X86_64_RELOC_GOT, - .X86_64_RELOC_GOT_LOAD, - .X86_64_RELOC_TLV, - => { + switch (self.type) { + .branch, .got, .tlv, .signed => { const displacement = @intCast(i32, @intCast(i64, target_addr) - @intCast(i64, source_addr) - 4); mem.writeIntLittle(u32, code[self.offset..][0..4], @bitCast(u32, displacement)); }, - .X86_64_RELOC_SIGNED, - .X86_64_RELOC_SIGNED_1, - .X86_64_RELOC_SIGNED_2, - .X86_64_RELOC_SIGNED_4, - => { - const correction: u3 = switch (rel_type) { - .X86_64_RELOC_SIGNED => 0, - .X86_64_RELOC_SIGNED_1 => 1, - .X86_64_RELOC_SIGNED_2 => 2, - .X86_64_RELOC_SIGNED_4 => 4, - else => unreachable, - }; - const displacement = @intCast(i32, target_addr - @intCast(i64, source_addr + correction + 4)); - mem.writeIntLittle(u32, code[self.offset..][0..4], @bitCast(u32, displacement)); - }, - .X86_64_RELOC_UNSIGNED => { + .unsigned => { switch (self.length) { 2 => { mem.writeIntLittle(u32, code[self.offset..][0..4], @truncate(u32, @bitCast(u64, target_addr))); @@ -250,7 +213,7 @@ fn resolveX8664(self: Relocation, source_addr: u64, target_addr: i64, code: []u8 else => unreachable, } }, - .X86_64_RELOC_SUBTRACTOR => unreachable, + .got_page, .got_pageoff, .page, .pageoff, .tlv_page, .tlv_pageoff => unreachable, // Invalid target architecture. } } @@ -258,3 +221,18 @@ inline fn isArithmeticOp(inst: *const [4]u8) bool { const group_decode = @truncate(u5, inst[3]); return ((group_decode >> 2) == 4); } + +const Relocation = @This(); + +const std = @import("std"); +const aarch64 = @import("../../arch/aarch64/bits.zig"); +const assert = std.debug.assert; +const log = std.log.scoped(.link); +const macho = std.macho; +const math = std.math; +const mem = std.mem; +const meta = std.meta; + +const Atom = @import("Atom.zig"); +const MachO = @import("../MachO.zig"); +const SymbolWithLoc = MachO.SymbolWithLoc; From 38ecaf3ab6c71aae213edbd38b8f661a03035b3a Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 12 Apr 2023 10:40:29 +0200 Subject: [PATCH 17/26] macho: add machinery for emitting TLV refs --- src/link/MachO.zig | 3 +++ src/link/MachO/Relocation.zig | 1 + 2 files changed, 4 insertions(+) diff --git a/src/link/MachO.zig b/src/link/MachO.zig index cc5b9ecdc6..9e205417e3 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -613,6 +613,9 @@ pub fn flushModule(self: *MachO, comp: *Compilation, prog_node: *std.Progress.No if (self.dyld_stub_binder_index == null) { self.dyld_stub_binder_index = try self.addUndefined("dyld_stub_binder", .add_got); } + if (!self.base.options.single_threaded) { + _ = try self.addUndefined("_tlv_bootstrap", .none); + } try self.createMhExecuteHeaderSymbol(); diff --git a/src/link/MachO/Relocation.zig b/src/link/MachO/Relocation.zig index 1d1d91279c..cb62ff98f0 100644 --- a/src/link/MachO/Relocation.zig +++ b/src/link/MachO/Relocation.zig @@ -48,6 +48,7 @@ pub fn isResolvable(self: Relocation, macho_file: *MachO) bool { pub fn getTargetAtomIndex(self: Relocation, macho_file: *MachO) ?Atom.Index { switch (self.type) { .got, .got_page, .got_pageoff => return macho_file.getGotAtomIndexForSymbol(self.target), + .tlv, .tlv_page, .tlv_pageoff => return macho_file.getTlvpAtomIndexForSymbol(self.target), else => {}, } if (macho_file.getStubsAtomIndexForSymbol(self.target)) |stubs_atom| return stubs_atom; From ef645ab1754d6117d06cf2296de689605546f5f7 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 5 Apr 2023 12:43:31 +0200 Subject: [PATCH 18/26] macho: refactor common logic between synthetic tables --- src/link/MachO.zig | 274 +++++++++++++++----------------- src/link/MachO/DebugSymbols.zig | 18 +-- src/link/MachO/Relocation.zig | 16 +- 3 files changed, 147 insertions(+), 161 deletions(-) diff --git a/src/link/MachO.zig b/src/link/MachO.zig index 9e205417e3..9dbdeaaad1 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -5,7 +5,6 @@ const build_options = @import("build_options"); const builtin = @import("builtin"); const assert = std.debug.assert; const dwarf = std.dwarf; -const fmt = std.fmt; const fs = std.fs; const log = std.log.scoped(.link); const macho = std.macho; @@ -155,13 +154,9 @@ stub_helper_preamble_atom_index: ?Atom.Index = null, strtab: StringTable(.strtab) = .{}, -got_entries: std.ArrayListUnmanaged(Entry) = .{}, -got_entries_free_list: std.ArrayListUnmanaged(u32) = .{}, -got_entries_table: std.AutoHashMapUnmanaged(SymbolWithLoc, u32) = .{}, - -stubs: std.ArrayListUnmanaged(Entry) = .{}, -stubs_free_list: std.ArrayListUnmanaged(u32) = .{}, -stubs_table: std.AutoHashMapUnmanaged(SymbolWithLoc, u32) = .{}, +got_table: SectionTable = .{}, +stubs_table: SectionTable = .{}, +tlvp_table: SectionTable = .{}, error_flags: File.ErrorFlags = File.ErrorFlags{}, @@ -270,26 +265,120 @@ const DeclMetadata = struct { } }; -const Entry = struct { - target: SymbolWithLoc, - // Index into the synthetic symbol table (i.e., file == null). - sym_index: u32, +const SectionTable = struct { + entries: std.ArrayListUnmanaged(Entry) = .{}, + free_list: std.ArrayListUnmanaged(u32) = .{}, + lookup: std.AutoHashMapUnmanaged(SymbolWithLoc, u32) = .{}, - pub fn getSymbol(entry: Entry, macho_file: *MachO) macho.nlist_64 { - return macho_file.getSymbol(.{ .sym_index = entry.sym_index, .file = null }); + pub fn deinit(st: *ST, allocator: Allocator) void { + st.entries.deinit(allocator); + st.free_list.deinit(allocator); + st.lookup.deinit(allocator); } - pub fn getSymbolPtr(entry: Entry, macho_file: *MachO) *macho.nlist_64 { - return macho_file.getSymbolPtr(.{ .sym_index = entry.sym_index, .file = null }); + pub fn allocateEntry(st: *ST, allocator: Allocator, target: SymbolWithLoc) !u32 { + try st.entries.ensureUnusedCapacity(allocator, 1); + const index = blk: { + if (st.free_list.popOrNull()) |index| { + log.debug(" (reusing entry index {d})", .{index}); + break :blk index; + } else { + log.debug(" (allocating entry at index {d})", .{st.entries.items.len}); + const index = @intCast(u32, st.entries.items.len); + _ = st.entries.addOneAssumeCapacity(); + break :blk index; + } + }; + st.entries.items[index] = .{ .target = target, .sym_index = 0 }; + try st.lookup.putNoClobber(allocator, target, index); + return index; } - pub fn getAtomIndex(entry: Entry, macho_file: *MachO) ?Atom.Index { - return macho_file.getAtomIndexForSymbol(.{ .sym_index = entry.sym_index, .file = null }); + pub fn freeEntry(st: *ST, allocator: Allocator, target: SymbolWithLoc) void { + const index = st.lookup.get(target) orelse return; + st.free_list.append(allocator, index) catch {}; + st.entries.items[index] = .{ + .target = .{ .sym_index = 0 }, + .sym_index = 0, + }; + _ = st.lookup.remove(target); } - pub fn getName(entry: Entry, macho_file: *MachO) []const u8 { - return macho_file.getSymbolName(.{ .sym_index = entry.sym_index, .file = null }); + pub fn getAtomIndex(st: *const ST, macho_file: *MachO, target: SymbolWithLoc) ?Atom.Index { + const index = st.lookup.get(target) orelse return null; + return st.entries.items[index].getAtomIndex(macho_file); } + + const FormatContext = struct { + macho_file: *MachO, + st: *const ST, + }; + + fn fmt( + ctx: FormatContext, + comptime unused_format_string: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) @TypeOf(writer).Error!void { + _ = options; + comptime assert(unused_format_string.len == 0); + try writer.writeAll("SectionTable:\n"); + for (ctx.st.entries.items, 0..) |entry, i| { + const atom_sym = entry.getSymbol(ctx.macho_file); + const target_sym = ctx.macho_file.getSymbol(entry.target); + try writer.print(" {d}@{x} => ", .{ i, atom_sym.n_value }); + if (target_sym.undf()) { + try writer.print("import('{s}')", .{ + ctx.macho_file.getSymbolName(entry.target), + }); + } else { + try writer.print("local(%{d}) in object({?d})", .{ + entry.target.sym_index, + entry.target.file, + }); + } + try writer.writeByte('\n'); + } + } + + fn format(st: *const ST, comptime unused_format_string: []const u8, options: std.fmt.FormatOptions, writer: anytype) !void { + _ = st; + _ = unused_format_string; + _ = options; + _ = writer; + @compileError("do not format SectionTable directly; use st.fmtDebug()"); + } + + pub fn fmtDebug(st: *const ST, macho_file: *MachO) std.fmt.Formatter(fmt) { + return .{ .data = .{ + .macho_file = macho_file, + .st = st, + } }; + } + + const ST = @This(); + + const Entry = struct { + target: SymbolWithLoc, + // Index into the synthetic symbol table (i.e., file == null). + sym_index: u32, + + pub fn getSymbol(entry: Entry, macho_file: *MachO) macho.nlist_64 { + return macho_file.getSymbol(.{ .sym_index = entry.sym_index }); + } + + pub fn getSymbolPtr(entry: Entry, macho_file: *MachO) *macho.nlist_64 { + return macho_file.getSymbolPtr(.{ .sym_index = entry.sym_index }); + } + + pub fn getAtomIndex(entry: Entry, macho_file: *MachO) ?Atom.Index { + return macho_file.getAtomIndexForSymbol(.{ .sym_index = entry.sym_index }); + } + + pub fn getName(entry: Entry, macho_file: *MachO) []const u8 { + return macho_file.getSymbolName(.{ .sym_index = entry.sym_index }); + } + }; }; const BindingTable = std.AutoArrayHashMapUnmanaged(Atom.Index, std.ArrayListUnmanaged(Atom.Binding)); @@ -399,7 +488,7 @@ pub fn openPath(allocator: Allocator, options: link.Options) !*MachO { // Create dSYM bundle. log.debug("creating {s}.dSYM bundle", .{sub_path}); - const d_sym_path = try fmt.allocPrint( + const d_sym_path = try std.fmt.allocPrint( allocator, "{s}.dSYM" ++ fs.path.sep_str ++ "Contents" ++ fs.path.sep_str ++ "Resources" ++ fs.path.sep_str ++ "DWARF", .{sub_path}, @@ -613,9 +702,9 @@ pub fn flushModule(self: *MachO, comp: *Compilation, prog_node: *std.Progress.No if (self.dyld_stub_binder_index == null) { self.dyld_stub_binder_index = try self.addUndefined("dyld_stub_binder", .add_got); } - if (!self.base.options.single_threaded) { - _ = try self.addUndefined("_tlv_bootstrap", .none); - } + // if (!self.base.options.single_threaded) { + // _ = try self.addUndefined("_tlv_bootstrap", .none); + // } try self.createMhExecuteHeaderSymbol(); @@ -1757,12 +1846,9 @@ pub fn deinit(self: *MachO) void { d_sym.deinit(); } - self.got_entries.deinit(gpa); - self.got_entries_free_list.deinit(gpa); - self.got_entries_table.deinit(gpa); - self.stubs.deinit(gpa); - self.stubs_free_list.deinit(gpa); + self.got_table.deinit(gpa); self.stubs_table.deinit(gpa); + self.tlvp_table.deinit(gpa); self.strtab.deinit(gpa); self.locals.deinit(gpa); @@ -1895,20 +1981,10 @@ fn freeAtom(self: *MachO, atom_index: Atom.Index) void { self.locals_free_list.append(gpa, sym_index) catch {}; // Try freeing GOT atom if this decl had one - const got_target = SymbolWithLoc{ .sym_index = sym_index, .file = null }; - if (self.got_entries_table.get(got_target)) |got_index| { - self.got_entries_free_list.append(gpa, @intCast(u32, got_index)) catch {}; - self.got_entries.items[got_index] = .{ - .target = .{ .sym_index = 0, .file = null }, - .sym_index = 0, - }; - _ = self.got_entries_table.remove(got_target); + self.got_table.freeEntry(gpa, .{ .sym_index = sym_index }); - if (self.d_sym) |*d_sym| { - d_sym.swapRemoveRelocs(sym_index); - } - - log.debug(" adding GOT index {d} to free list (target local@{d})", .{ got_index, sym_index }); + if (self.d_sym) |*d_sym| { + d_sym.swapRemoveRelocs(sym_index); } self.locals.items[sym_index].n_type = 0; @@ -1983,70 +2059,25 @@ fn allocateGlobal(self: *MachO) !u32 { return index; } -fn allocateGotEntry(self: *MachO, target: SymbolWithLoc) !u32 { - const gpa = self.base.allocator; - try self.got_entries.ensureUnusedCapacity(gpa, 1); - - const index = blk: { - if (self.got_entries_free_list.popOrNull()) |index| { - log.debug(" (reusing GOT entry index {d})", .{index}); - break :blk index; - } else { - log.debug(" (allocating GOT entry at index {d})", .{self.got_entries.items.len}); - const index = @intCast(u32, self.got_entries.items.len); - _ = self.got_entries.addOneAssumeCapacity(); - break :blk index; - } - }; - - self.got_entries.items[index] = .{ .target = target, .sym_index = 0 }; - try self.got_entries_table.putNoClobber(gpa, target, index); - - return index; -} - fn addGotEntry(self: *MachO, target: SymbolWithLoc) !void { - if (self.got_entries_table.contains(target)) return; - - const got_index = try self.allocateGotEntry(target); + if (self.got_table.lookup.contains(target)) return; + const got_index = try self.got_table.allocateEntry(self.base.allocator, target); const got_atom_index = try self.createGotAtom(target); const got_atom = self.getAtom(got_atom_index); - self.got_entries.items[got_index].sym_index = got_atom.getSymbolIndex().?; + self.got_table.entries.items[got_index].sym_index = got_atom.getSymbolIndex().?; try self.writePtrWidthAtom(got_atom_index); } -fn allocateStubEntry(self: *MachO, target: SymbolWithLoc) !u32 { - try self.stubs.ensureUnusedCapacity(self.base.allocator, 1); - - const index = blk: { - if (self.stubs_free_list.popOrNull()) |index| { - log.debug(" (reusing stub entry index {d})", .{index}); - break :blk index; - } else { - log.debug(" (allocating stub entry at index {d})", .{self.stubs.items.len}); - const index = @intCast(u32, self.stubs.items.len); - _ = self.stubs.addOneAssumeCapacity(); - break :blk index; - } - }; - - self.stubs.items[index] = .{ .target = target, .sym_index = 0 }; - try self.stubs_table.putNoClobber(self.base.allocator, target, index); - - return index; -} - fn addStubEntry(self: *MachO, target: SymbolWithLoc) !void { - if (self.stubs_table.contains(target)) return; - - const stub_index = try self.allocateStubEntry(target); + if (self.stubs_table.lookup.contains(target)) return; + const stub_index = try self.stubs_table.allocateEntry(self.base.allocator, target); const stub_helper_atom_index = try self.createStubHelperAtom(); const stub_helper_atom = self.getAtom(stub_helper_atom_index); const laptr_atom_index = try self.createLazyPointerAtom(stub_helper_atom.getSymbolIndex().?, target); const laptr_atom = self.getAtom(laptr_atom_index); const stub_atom_index = try self.createStubAtom(laptr_atom.getSymbolIndex().?); const stub_atom = self.getAtom(stub_atom_index); - self.stubs.items[stub_index].sym_index = stub_atom.getSymbolIndex().?; + self.stubs_table.entries.items[stub_index].sym_index = stub_atom.getSymbolIndex().?; self.markRelocsDirtyByTarget(target); } @@ -2431,7 +2462,7 @@ fn updateDeclCode(self: *MachO, decl_index: Module.Decl.Index, code: []u8) !u64 sym.n_value = vaddr; log.debug(" (updating GOT entry)", .{}); const got_target = SymbolWithLoc{ .sym_index = sym_index, .file = null }; - const got_atom_index = self.getGotAtomIndexForSymbol(got_target).?; + const got_atom_index = self.got_table.getAtomIndex(self, got_target).?; self.markRelocsDirtyByTarget(got_target); try self.writePtrWidthAtom(got_atom_index); } @@ -3481,8 +3512,8 @@ const SymtabCtx = struct { fn writeDysymtab(self: *MachO, ctx: SymtabCtx) !void { const gpa = self.base.allocator; - const nstubs = @intCast(u32, self.stubs_table.count()); - const ngot_entries = @intCast(u32, self.got_entries_table.count()); + const nstubs = @intCast(u32, self.stubs_table.lookup.count()); + const ngot_entries = @intCast(u32, self.got_table.lookup.count()); const nindirectsyms = nstubs * 2 + ngot_entries; const iextdefsym = ctx.nlocalsym; const iundefsym = iextdefsym + ctx.nextdefsym; @@ -3504,7 +3535,7 @@ fn writeDysymtab(self: *MachO, ctx: SymtabCtx) !void { if (self.stubs_section_index) |sect_id| { const stubs = &self.sections.items(.header)[sect_id]; stubs.reserved1 = 0; - for (self.stubs.items) |entry| { + for (self.stubs_table.entries.items) |entry| { if (entry.sym_index == 0) continue; const target_sym = self.getSymbol(entry.target); assert(target_sym.undf()); @@ -3515,7 +3546,7 @@ fn writeDysymtab(self: *MachO, ctx: SymtabCtx) !void { if (self.got_section_index) |sect_id| { const got = &self.sections.items(.header)[sect_id]; got.reserved1 = nstubs; - for (self.got_entries.items) |entry| { + for (self.got_table.entries.items) |entry| { if (entry.sym_index == 0) continue; const target_sym = self.getSymbol(entry.target); if (target_sym.undf()) { @@ -3529,7 +3560,7 @@ fn writeDysymtab(self: *MachO, ctx: SymtabCtx) !void { if (self.la_symbol_ptr_section_index) |sect_id| { const la_symbol_ptr = &self.sections.items(.header)[sect_id]; la_symbol_ptr.reserved1 = nstubs + ngot_entries; - for (self.stubs.items) |entry| { + for (self.stubs_table.entries.items) |entry| { if (entry.sym_index == 0) continue; const target_sym = self.getSymbol(entry.target); assert(target_sym.undf()); @@ -3874,20 +3905,6 @@ pub fn getAtomIndexForSymbol(self: *MachO, sym_with_loc: SymbolWithLoc) ?Atom.In return self.atom_by_index_table.get(sym_with_loc.sym_index); } -/// Returns GOT atom that references `sym_with_loc` if one exists. -/// Returns null otherwise. -pub fn getGotAtomIndexForSymbol(self: *MachO, sym_with_loc: SymbolWithLoc) ?Atom.Index { - const got_index = self.got_entries_table.get(sym_with_loc) orelse return null; - return self.got_entries.items[got_index].getAtomIndex(self); -} - -/// Returns stubs atom that references `sym_with_loc` if one exists. -/// Returns null otherwise. -pub fn getStubsAtomIndexForSymbol(self: *MachO, sym_with_loc: SymbolWithLoc) ?Atom.Index { - const stubs_index = self.stubs_table.get(sym_with_loc) orelse return null; - return self.stubs.items[stubs_index].getAtomIndex(self); -} - /// Returns symbol location corresponding to the set entrypoint. /// Asserts output mode is executable. pub fn getEntryPoint(self: MachO) error{MissingMainEntrypoint}!SymbolWithLoc { @@ -4234,37 +4251,10 @@ pub fn logSymtab(self: *MachO) void { } log.debug("GOT entries:", .{}); - for (self.got_entries.items, 0..) |entry, i| { - const atom_sym = entry.getSymbol(self); - const target_sym = self.getSymbol(entry.target); - if (target_sym.undf()) { - log.debug(" {d}@{x} => import('{s}')", .{ - i, - atom_sym.n_value, - self.getSymbolName(entry.target), - }); - } else { - log.debug(" {d}@{x} => local(%{d}) in object({?d}) {s}", .{ - i, - atom_sym.n_value, - entry.target.sym_index, - entry.target.file, - logSymAttributes(target_sym, &buf), - }); - } - } + log.debug("{}", .{self.got_table.fmtDebug(self)}); log.debug("stubs entries:", .{}); - for (self.stubs.items, 0..) |entry, i| { - const target_sym = self.getSymbol(entry.target); - const atom_sym = entry.getSymbol(self); - assert(target_sym.undf()); - log.debug(" {d}@{x} => import('{s}')", .{ - i, - atom_sym.n_value, - self.getSymbolName(entry.target), - }); - } + log.debug("{}", .{self.stubs_table.fmtDebug(self)}); } pub fn logAtoms(self: *MachO) void { diff --git a/src/link/MachO/DebugSymbols.zig b/src/link/MachO/DebugSymbols.zig index 1f41fc1cb8..98abf2b1cc 100644 --- a/src/link/MachO/DebugSymbols.zig +++ b/src/link/MachO/DebugSymbols.zig @@ -226,26 +226,20 @@ pub fn flushModule(self: *DebugSymbols, macho_file: *MachO) !void { for (self.relocs.items) |*reloc| { const sym = switch (reloc.type) { - .direct_load => macho_file.getSymbol(.{ .sym_index = reloc.target, .file = null }), + .direct_load => macho_file.getSymbol(.{ .sym_index = reloc.target }), .got_load => blk: { - const got_index = macho_file.got_entries_table.get(.{ - .sym_index = reloc.target, - .file = null, - }).?; - const got_entry = macho_file.got_entries.items[got_index]; + const got_index = macho_file.got_table.lookup.get(.{ .sym_index = reloc.target }).?; + const got_entry = macho_file.got_table.entries.items[got_index]; break :blk got_entry.getSymbol(macho_file); }, }; if (sym.n_value == reloc.prev_vaddr) continue; const sym_name = switch (reloc.type) { - .direct_load => macho_file.getSymbolName(.{ .sym_index = reloc.target, .file = null }), + .direct_load => macho_file.getSymbolName(.{ .sym_index = reloc.target }), .got_load => blk: { - const got_index = macho_file.got_entries_table.get(.{ - .sym_index = reloc.target, - .file = null, - }).?; - const got_entry = macho_file.got_entries.items[got_index]; + const got_index = macho_file.got_table.lookup.get(.{ .sym_index = reloc.target }).?; + const got_entry = macho_file.got_table.entries.items[got_index]; break :blk got_entry.getName(macho_file); }, }; diff --git a/src/link/MachO/Relocation.zig b/src/link/MachO/Relocation.zig index cb62ff98f0..d77acd76f0 100644 --- a/src/link/MachO/Relocation.zig +++ b/src/link/MachO/Relocation.zig @@ -46,13 +46,15 @@ pub fn isResolvable(self: Relocation, macho_file: *MachO) bool { } pub fn getTargetAtomIndex(self: Relocation, macho_file: *MachO) ?Atom.Index { - switch (self.type) { - .got, .got_page, .got_pageoff => return macho_file.getGotAtomIndexForSymbol(self.target), - .tlv, .tlv_page, .tlv_pageoff => return macho_file.getTlvpAtomIndexForSymbol(self.target), - else => {}, - } - if (macho_file.getStubsAtomIndexForSymbol(self.target)) |stubs_atom| return stubs_atom; - return macho_file.getAtomIndexForSymbol(self.target); + return switch (self.type) { + .got, .got_page, .got_pageoff => macho_file.got_table.getAtomIndex(macho_file, self.target), + .tlv, .tlv_page, .tlv_pageoff => macho_file.tlvp_table.getAtomIndex(macho_file, self.target), + .branch => if (macho_file.stubs_table.getAtomIndex(macho_file, self.target)) |index| + index + else + macho_file.getAtomIndexForSymbol(self.target), + else => macho_file.getAtomIndexForSymbol(self.target), + }; } pub fn resolve(self: Relocation, macho_file: *MachO, atom_index: Atom.Index, code: []u8) void { From 57f9304275b7704fc2bd2879cdbb344c9f82e75c Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 5 Apr 2023 18:31:47 +0200 Subject: [PATCH 19/26] macho: improve reporting of missing symbols Fix a typo in TLV bootstrap symbol: __tlv_bootstrap. --- src/link/MachO.zig | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/src/link/MachO.zig b/src/link/MachO.zig index 9dbdeaaad1..990ad16452 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -702,9 +702,9 @@ pub fn flushModule(self: *MachO, comp: *Compilation, prog_node: *std.Progress.No if (self.dyld_stub_binder_index == null) { self.dyld_stub_binder_index = try self.addUndefined("dyld_stub_binder", .add_got); } - // if (!self.base.options.single_threaded) { - // _ = try self.addUndefined("_tlv_bootstrap", .none); - // } + if (!self.base.options.single_threaded) { + _ = try self.addUndefined("__tlv_bootstrap", .none); + } try self.createMhExecuteHeaderSymbol(); @@ -713,6 +713,12 @@ pub fn flushModule(self: *MachO, comp: *Compilation, prog_node: *std.Progress.No try self.resolveSymbolsInDylibs(&actions); if (self.unresolved.count() > 0) { + for (self.unresolved.keys()) |index| { + // TODO: convert into compiler errors. + const global = self.globals.items[index]; + const sym_name = self.getSymbolName(global); + log.err("undefined symbol reference '{s}'", .{sym_name}); + } return error.UndefinedSymbolReference; } From 09b6bd86dea690aeb1fe1a832bb56b44b414120b Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 6 Apr 2023 07:59:48 +0200 Subject: [PATCH 20/26] macho: cleanup dirtying and writing GOT atoms --- src/link/MachO.zig | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/src/link/MachO.zig b/src/link/MachO.zig index 990ad16452..694d115cdd 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -137,8 +137,9 @@ got_section_index: ?u8 = null, data_const_section_index: ?u8 = null, la_symbol_ptr_section_index: ?u8 = null, data_section_index: ?u8 = null, -tls_vars_section_index: ?u8 = null, -tls_data_section_index: ?u8 = null, +thread_ptr_section_index: ?u8 = null, +thread_vars_section_index: ?u8 = null, +thread_data_section_index: ?u8 = null, locals: std.ArrayListUnmanaged(macho.nlist_64) = .{}, globals: std.ArrayListUnmanaged(SymbolWithLoc) = .{}, @@ -1365,6 +1366,7 @@ pub fn createGotAtom(self: *MachO, target: SymbolWithLoc) !Atom.Index { } else { try Atom.addRebase(self, atom_index, 0); } + try self.writePtrWidthAtom(atom_index); return atom_index; } @@ -2071,7 +2073,7 @@ fn addGotEntry(self: *MachO, target: SymbolWithLoc) !void { const got_atom_index = try self.createGotAtom(target); const got_atom = self.getAtom(got_atom_index); self.got_table.entries.items[got_index].sym_index = got_atom.getSymbolIndex().?; - try self.writePtrWidthAtom(got_atom_index); + self.markRelocsDirtyByTarget(target); } fn addStubEntry(self: *MachO, target: SymbolWithLoc) !void { @@ -2087,6 +2089,10 @@ fn addStubEntry(self: *MachO, target: SymbolWithLoc) !void { self.markRelocsDirtyByTarget(target); } +fn addTlvEntry(self: *MachO, target: SymbolWithLoc) !void { + if (self.tlvp_table.lookup.contains(target)) return; +} + pub fn updateFunc(self: *MachO, module: *Module, func: *Module.Fn, air: Air, liveness: Liveness) !void { if (build_options.skip_non_native and builtin.object_format != .macho) { @panic("Attempted to compile for object format that was disabled by build configuration"); @@ -2366,7 +2372,6 @@ fn updateLazySymbolAtom( symbol.n_value = vaddr; try self.addGotEntry(.{ .sym_index = local_sym_index }); - self.markRelocsDirtyByTarget(atom.getSymbolWithLoc()); try self.writeAtom(atom_index, code); } @@ -2413,7 +2418,7 @@ fn getDeclOutputSection(self: *MachO, decl_index: Module.Decl.Index) u8 { if (val.castTag(.variable)) |variable| { if (variable.data.is_threadlocal and !single_threaded) { - break :blk self.tls_data_section_index.?; + break :blk self.thread_data_section_index.?; } break :blk self.data_section_index.?; } @@ -2501,7 +2506,6 @@ fn updateDeclCode(self: *MachO, decl_index: Module.Decl.Index, code: []u8) !u64 try self.addGotEntry(.{ .sym_index = sym_index }); } - self.markRelocsDirtyByTarget(atom.getSymbolWithLoc()); try self.writeAtom(atom_index, code); return atom.getSymbol(self).n_value; @@ -2835,8 +2839,17 @@ fn populateMissingMetadata(self: *MachO) !void { } if (!self.base.options.single_threaded) { - if (self.tls_vars_section_index == null) { - self.tls_vars_section_index = try self.allocateSection("__DATA2", "__thread_vars", .{ + if (self.thread_ptr_section_index == null) { + self.thread_ptr_section_index = try self.allocateSection("__DATA2", "__thread_ptrs", .{ + .size = @sizeOf(u64), + .alignment = @alignOf(u64), + .flags = macho.S_THREAD_LOCAL_VARIABLE_POINTERS, + .prot = macho.PROT.READ | macho.PROT.WRITE, + }); + self.segment_table_dirty = true; + } + if (self.thread_vars_section_index == null) { + self.thread_vars_section_index = try self.allocateSection("__DATA3", "__thread_vars", .{ .size = @sizeOf(u64) * 3, .alignment = @alignOf(u64), .flags = macho.S_THREAD_LOCAL_VARIABLES, @@ -2845,8 +2858,8 @@ fn populateMissingMetadata(self: *MachO) !void { self.segment_table_dirty = true; } - if (self.tls_data_section_index == null) { - self.tls_data_section_index = try self.allocateSection("__DATA3", "__thread_data", .{ + if (self.thread_data_section_index == null) { + self.thread_data_section_index = try self.allocateSection("__DATA4", "__thread_data", .{ .size = @sizeOf(u64), .alignment = @alignOf(u64), .flags = macho.S_THREAD_LOCAL_REGULAR, From 094ff60252cf57b2ae7bb69abb3e8e696873a3be Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Fri, 7 Apr 2023 07:37:35 +0200 Subject: [PATCH 21/26] macho: emit TLV pointers and variables --- src/link.zig | 4 +- src/link/MachO.zig | 150 +++++++++++++++++++++++++++------- src/link/MachO/Relocation.zig | 17 +++- 3 files changed, 135 insertions(+), 36 deletions(-) diff --git a/src/link.zig b/src/link.zig index 18002f64a6..c8999cee47 100644 --- a/src/link.zig +++ b/src/link.zig @@ -540,7 +540,7 @@ pub const File = struct { /// May be called before or after updateDeclExports for any given Decl. pub fn updateDecl(base: *File, module: *Module, decl_index: Module.Decl.Index) UpdateDeclError!void { const decl = module.declPtr(decl_index); - log.debug("updateDecl {*} ({s}), type={}", .{ decl, decl.name, decl.ty.fmtDebug() }); + log.debug("updateDecl {*} ({s}), type={}", .{ decl, decl.name, decl.ty.fmt(module) }); assert(decl.has_tv); if (build_options.only_c) { assert(base.tag == .c); @@ -564,7 +564,7 @@ pub const File = struct { pub fn updateFunc(base: *File, module: *Module, func: *Module.Fn, air: Air, liveness: Liveness) UpdateDeclError!void { const owner_decl = module.declPtr(func.owner_decl); log.debug("updateFunc {*} ({s}), type={}", .{ - owner_decl, owner_decl.name, owner_decl.ty.fmtDebug(), + owner_decl, owner_decl.name, owner_decl.ty.fmt(module), }); if (build_options.only_c) { assert(base.tag == .c); diff --git a/src/link/MachO.zig b/src/link/MachO.zig index 694d115cdd..b958e64e58 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -1338,13 +1338,12 @@ pub fn createAtom(self: *MachO) !Atom.Index { pub fn createGotAtom(self: *MachO, target: SymbolWithLoc) !Atom.Index { const atom_index = try self.createAtom(); - const atom = self.getAtomPtr(atom_index); - atom.size = @sizeOf(u64); + self.getAtomPtr(atom_index).size = @sizeOf(u64); - const sym = atom.getSymbolPtr(self); + const sym = self.getAtom(atom_index).getSymbolPtr(self); sym.n_type = macho.N_SECT; sym.n_sect = self.got_section_index.? + 1; - sym.n_value = try self.allocateAtom(atom_index, atom.size, @alignOf(u64)); + sym.n_value = try self.allocateAtom(atom_index, @sizeOf(u64), @alignOf(u64)); log.debug("allocated GOT atom at 0x{x}", .{sym.n_value}); @@ -1553,7 +1552,7 @@ fn createStubHelperAtom(self: *MachO) !Atom.Index { try Atom.addRelocation(self, atom_index, .{ .type = .branch, - .target = .{ .sym_index = stub_helper_preamble_atom_sym_index, .file = null }, + .target = .{ .sym_index = stub_helper_preamble_atom_sym_index }, .offset = 6, .addend = 0, .pcrel = true, @@ -1576,7 +1575,7 @@ fn createStubHelperAtom(self: *MachO) !Atom.Index { try Atom.addRelocation(self, atom_index, .{ .type = .branch, - .target = .{ .sym_index = stub_helper_preamble_atom_sym_index, .file = null }, + .target = .{ .sym_index = stub_helper_preamble_atom_sym_index }, .offset = 4, .addend = 0, .pcrel = true, @@ -1604,7 +1603,7 @@ fn createLazyPointerAtom(self: *MachO, stub_sym_index: u32, target: SymbolWithLo try Atom.addRelocation(self, atom_index, .{ .type = .unsigned, - .target = .{ .sym_index = stub_sym_index, .file = null }, + .target = .{ .sym_index = stub_sym_index }, .offset = 0, .addend = 0, .pcrel = false, @@ -1658,7 +1657,7 @@ fn createStubAtom(self: *MachO, laptr_sym_index: u32) !Atom.Index { try Atom.addRelocation(self, atom_index, .{ .type = .branch, - .target = .{ .sym_index = laptr_sym_index, .file = null }, + .target = .{ .sym_index = laptr_sym_index }, .offset = 2, .addend = 0, .pcrel = true, @@ -1680,7 +1679,7 @@ fn createStubAtom(self: *MachO, laptr_sym_index: u32) !Atom.Index { try Atom.addRelocations(self, atom_index, &[_]Relocation{ .{ .type = .page, - .target = .{ .sym_index = laptr_sym_index, .file = null }, + .target = .{ .sym_index = laptr_sym_index }, .offset = 0, .addend = 0, .pcrel = true, @@ -1688,7 +1687,7 @@ fn createStubAtom(self: *MachO, laptr_sym_index: u32) !Atom.Index { }, .{ .type = .pageoff, - .target = .{ .sym_index = laptr_sym_index, .file = null }, + .target = .{ .sym_index = laptr_sym_index }, .offset = 4, .addend = 0, .pcrel = false, @@ -1706,6 +1705,67 @@ fn createStubAtom(self: *MachO, laptr_sym_index: u32) !Atom.Index { return atom_index; } +fn createThreadLocalDescriptorAtom(self: *MachO, target: SymbolWithLoc) !Atom.Index { + const gpa = self.base.allocator; + const size = 3 * @sizeOf(u64); + const required_alignment: u32 = 1; + const atom_index = try self.createAtom(); + self.getAtomPtr(atom_index).size = size; + + const target_sym_name = self.getSymbolName(target); + const name_delimiter = mem.indexOf(u8, target_sym_name, "$").?; + const sym_name = try gpa.dupe(u8, target_sym_name[0..name_delimiter]); + defer gpa.free(sym_name); + + const sym = self.getAtom(atom_index).getSymbolPtr(self); + sym.n_type = macho.N_SECT; + sym.n_sect = self.thread_vars_section_index.? + 1; + sym.n_strx = try self.strtab.insert(gpa, sym_name); + sym.n_value = try self.allocateAtom(atom_index, size, required_alignment); + + log.debug("allocated threadlocal descriptor atom '{s}' at 0x{x}", .{ sym_name, sym.n_value }); + + try Atom.addRelocation(self, atom_index, .{ + .type = .tlv_initializer, + .target = target, + .offset = 0x10, + .addend = 0, + .pcrel = false, + .length = 3, + }); + + var code: [size]u8 = undefined; + mem.set(u8, &code, 0); + try self.writeAtom(atom_index, &code); + + return atom_index; +} + +fn createThreadLocalPointerAtom(self: *MachO, tlv_desc_sym_index: u32) !Atom.Index { + const atom_index = try self.createAtom(); + self.getAtomPtr(atom_index).size = @sizeOf(u64); + + const sym = self.getAtom(atom_index).getSymbolPtr(self); + sym.n_type = macho.N_SECT; + sym.n_sect = self.thread_ptr_section_index.? + 1; + sym.n_value = try self.allocateAtom(atom_index, @sizeOf(u64), @alignOf(u64)); + + log.debug("allocated threadlocal pointer atom at 0x{x}", .{sym.n_value}); + + try Atom.addRelocation(self, atom_index, .{ + .type = .unsigned, + .target = .{ .sym_index = tlv_desc_sym_index }, + .offset = 0, + .addend = 0, + .pcrel = false, + .length = 3, + }); + try Atom.addRebase(self, atom_index, 0); + try self.writePtrWidthAtom(atom_index); + + return atom_index; +} + fn createMhExecuteHeaderSymbol(self: *MachO) !void { if (self.base.options.output_mode != .Exe) return; if (self.getGlobal("__mh_execute_header")) |global| { @@ -2091,6 +2151,13 @@ fn addStubEntry(self: *MachO, target: SymbolWithLoc) !void { fn addTlvEntry(self: *MachO, target: SymbolWithLoc) !void { if (self.tlvp_table.lookup.contains(target)) return; + const tlvp_index = try self.tlvp_table.allocateEntry(self.base.allocator, target); + const tlv_desc_atom_index = try self.createThreadLocalDescriptorAtom(target); + const tlv_desc_atom = self.getAtom(tlv_desc_atom_index); + const tlv_ptr_atom_index = try self.createThreadLocalPointerAtom(tlv_desc_atom.getSymbolIndex().?); + const tlv_ptr_atom = self.getAtom(tlv_ptr_atom_index); + self.tlvp_table.entries.items[tlvp_index].sym_index = tlv_ptr_atom.getSymbolIndex().?; + self.markRelocsDirtyByTarget(target); } pub fn updateFunc(self: *MachO, module: *Module, func: *Module.Fn, air: Air, liveness: Liveness) !void { @@ -2444,16 +2511,28 @@ fn updateDeclCode(self: *MachO, decl_index: Module.Decl.Index, code: []u8) !u64 const required_alignment = decl.getAlignment(self.base.options.target); - const sym_name = try decl.getFullyQualifiedName(mod); - defer self.base.allocator.free(sym_name); + const decl_name = try decl.getFullyQualifiedName(mod); + defer gpa.free(decl_name); const decl_metadata = self.decls.get(decl_index).?; const atom_index = decl_metadata.atom; const atom = self.getAtom(atom_index); const sym_index = atom.getSymbolIndex().?; const sect_id = decl_metadata.section; + const header = &self.sections.items(.header)[sect_id]; + const segment = self.getSegment(sect_id); + const is_threadlocal = if (!self.base.options.single_threaded) + header.flags == macho.S_THREAD_LOCAL_REGULAR or header.flags == macho.S_THREAD_LOCAL_ZEROFILL + else + false; const code_len = code.len; + const sym_name = if (is_threadlocal) + try std.fmt.allocPrint(gpa, "{s}$tlv$init", .{decl_name}) + else + decl_name; + defer if (is_threadlocal) gpa.free(sym_name); + if (atom.size != 0) { const sym = atom.getSymbolPtr(self); sym.n_strx = try self.strtab.insert(gpa, sym_name); @@ -2471,25 +2550,29 @@ fn updateDeclCode(self: *MachO, decl_index: Module.Decl.Index, code: []u8) !u64 if (vaddr != sym.n_value) { sym.n_value = vaddr; - log.debug(" (updating GOT entry)", .{}); - const got_target = SymbolWithLoc{ .sym_index = sym_index, .file = null }; - const got_atom_index = self.got_table.getAtomIndex(self, got_target).?; - self.markRelocsDirtyByTarget(got_target); - try self.writePtrWidthAtom(got_atom_index); + const target = SymbolWithLoc{ .sym_index = sym_index }; + self.markRelocsDirtyByTarget(target); + if (is_threadlocal) { + @panic("TODO update the threadlocal variable's name also"); + // log.debug(" (updating threadlocal pointer entry)", .{}); + // const tlvp_atom_index = self.tlvp_table.getAtomIndex(self, target).?; + // try self.writePtrWidthAtom(tlvp_atom_index); + } else { + log.debug(" (updating GOT entry)", .{}); + const got_atom_index = self.got_table.getAtomIndex(self, target).?; + try self.writePtrWidthAtom(got_atom_index); + } } } else if (code_len < atom.size) { self.shrinkAtom(atom_index, code_len); } else if (atom.next_index == null) { - const header = &self.sections.items(.header)[sect_id]; - const segment = self.getSegment(sect_id); const needed_size = (sym.n_value + code_len) - segment.vmaddr; header.size = needed_size; } self.getAtomPtr(atom_index).size = code_len; } else { - const name_str_index = try self.strtab.insert(gpa, sym_name); const sym = atom.getSymbolPtr(self); - sym.n_strx = name_str_index; + sym.n_strx = try self.strtab.insert(gpa, sym_name); sym.n_type = macho.N_SECT; sym.n_sect = sect_id + 1; sym.n_desc = 0; @@ -2503,7 +2586,12 @@ fn updateDeclCode(self: *MachO, decl_index: Module.Decl.Index, code: []u8) !u64 self.getAtomPtr(atom_index).size = code_len; sym.n_value = vaddr; - try self.addGotEntry(.{ .sym_index = sym_index }); + const target: SymbolWithLoc = .{ .sym_index = sym_index }; + if (is_threadlocal) { + try self.addTlvEntry(target); + } else { + try self.addGotEntry(target); + } } try self.writeAtom(atom_index, code); @@ -2851,7 +2939,7 @@ fn populateMissingMetadata(self: *MachO) !void { if (self.thread_vars_section_index == null) { self.thread_vars_section_index = try self.allocateSection("__DATA3", "__thread_vars", .{ .size = @sizeOf(u64) * 3, - .alignment = @alignOf(u64), + .alignment = @sizeOf(u64), .flags = macho.S_THREAD_LOCAL_VARIABLES, .prot = macho.PROT.READ | macho.PROT.WRITE, }); @@ -3650,6 +3738,10 @@ fn writeHeader(self: *MachO, ncmds: u32, sizeofcmds: u32) !void { var header: macho.mach_header_64 = .{}; header.flags = macho.MH_NOUNDEFS | macho.MH_DYLDLINK | macho.MH_PIE | macho.MH_TWOLEVEL; + if (!self.base.options.single_threaded) { + header.flags |= macho.MH_HAS_TLV_DESCRIPTORS; + } + switch (self.base.options.target.cpu.arch) { .aarch64 => { header.cputype = macho.CPU_TYPE_ARM64; @@ -3674,12 +3766,6 @@ fn writeHeader(self: *MachO, ncmds: u32, sizeofcmds: u32) !void { else => unreachable, } - if (self.getSectionByName("__DATA", "__thread_vars")) |sect_id| { - if (self.sections.items(.header)[sect_id].size > 0) { - header.flags |= macho.MH_HAS_TLV_DESCRIPTORS; - } - } - header.ncmds = ncmds; header.sizeofcmds = sizeofcmds; @@ -3859,8 +3945,7 @@ pub fn getSymbol(self: *const MachO, sym_with_loc: SymbolWithLoc) macho.nlist_64 /// Returns name of the symbol described by `sym_with_loc` descriptor. pub fn getSymbolName(self: *const MachO, sym_with_loc: SymbolWithLoc) []const u8 { - assert(sym_with_loc.file == null); - const sym = self.locals.items[sym_with_loc.sym_index]; + const sym = self.getSymbol(sym_with_loc); return self.strtab.get(sym.n_strx).?; } @@ -4274,6 +4359,9 @@ pub fn logSymtab(self: *MachO) void { log.debug("stubs entries:", .{}); log.debug("{}", .{self.stubs_table.fmtDebug(self)}); + + log.debug("threadlocal entries:", .{}); + log.debug("{}", .{self.tlvp_table.fmtDebug(self)}); } pub fn logAtoms(self: *MachO) void { diff --git a/src/link/MachO/Relocation.zig b/src/link/MachO/Relocation.zig index d77acd76f0..06a255ad59 100644 --- a/src/link/MachO/Relocation.zig +++ b/src/link/MachO/Relocation.zig @@ -37,6 +37,8 @@ pub const Type = enum { branch, /// Absolute pointer value unsigned, + /// Relative offset to TLV initializer + tlv_initializer, }; /// Returns true if and only if the reloc is dirty AND the target address is available. @@ -65,7 +67,16 @@ pub fn resolve(self: Relocation, macho_file: *MachO, atom_index: Atom.Index, cod const target_atom_index = self.getTargetAtomIndex(macho_file).?; // Oops, you didn't check if the relocation can be resolved with isResolvable(). const target_atom = macho_file.getAtom(target_atom_index); - const target_addr = @intCast(i64, target_atom.getSymbol(macho_file).n_value) + self.addend; + + const target_addr: i64 = switch (self.type) { + .tlv_initializer => blk: { + assert(self.addend == 0); // Addend here makes no sense. + const header = macho_file.sections.items(.header)[macho_file.thread_data_section_index.?]; + const target_sym = target_atom.getSymbol(macho_file); + break :blk @intCast(i64, target_sym.n_value - header.addr); + }, + else => @intCast(i64, target_atom.getSymbol(macho_file).n_value) + self.addend, + }; log.debug(" ({x}: [() => 0x{x} ({s})) ({s})", .{ source_addr, @@ -190,7 +201,7 @@ fn resolveAarch64(self: Relocation, source_addr: u64, target_addr: i64, code: [] }; mem.writeIntLittle(u32, buffer[0..4], inst.toU32()); }, - .unsigned => switch (self.length) { + .tlv_initializer, .unsigned => switch (self.length) { 2 => mem.writeIntLittle(u32, buffer[0..4], @truncate(u32, @bitCast(u64, target_addr))), 3 => mem.writeIntLittle(u64, buffer[0..8], @bitCast(u64, target_addr)), else => unreachable, @@ -205,7 +216,7 @@ fn resolveX8664(self: Relocation, source_addr: u64, target_addr: i64, code: []u8 const displacement = @intCast(i32, @intCast(i64, target_addr) - @intCast(i64, source_addr) - 4); mem.writeIntLittle(u32, code[self.offset..][0..4], @bitCast(u32, displacement)); }, - .unsigned => { + .tlv_initializer, .unsigned => { switch (self.length) { 2 => { mem.writeIntLittle(u32, code[self.offset..][0..4], @truncate(u32, @bitCast(u64, target_addr))); From 1f6165f6211885d3f5392cdfcc15f990284159de Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 12 Apr 2023 10:43:42 +0200 Subject: [PATCH 22/26] macho: reference TLV thunks via GOT table --- src/link/MachO.zig | 90 +++++++++++------------------------ src/link/MachO/Relocation.zig | 60 ++++------------------- 2 files changed, 37 insertions(+), 113 deletions(-) diff --git a/src/link/MachO.zig b/src/link/MachO.zig index b958e64e58..0ffb72f087 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -137,7 +137,6 @@ got_section_index: ?u8 = null, data_const_section_index: ?u8 = null, la_symbol_ptr_section_index: ?u8 = null, data_section_index: ?u8 = null, -thread_ptr_section_index: ?u8 = null, thread_vars_section_index: ?u8 = null, thread_data_section_index: ?u8 = null, @@ -157,7 +156,7 @@ strtab: StringTable(.strtab) = .{}, got_table: SectionTable = .{}, stubs_table: SectionTable = .{}, -tlvp_table: SectionTable = .{}, +tlv_table: SectionTable = .{}, error_flags: File.ErrorFlags = File.ErrorFlags{}, @@ -1741,31 +1740,6 @@ fn createThreadLocalDescriptorAtom(self: *MachO, target: SymbolWithLoc) !Atom.In return atom_index; } -fn createThreadLocalPointerAtom(self: *MachO, tlv_desc_sym_index: u32) !Atom.Index { - const atom_index = try self.createAtom(); - self.getAtomPtr(atom_index).size = @sizeOf(u64); - - const sym = self.getAtom(atom_index).getSymbolPtr(self); - sym.n_type = macho.N_SECT; - sym.n_sect = self.thread_ptr_section_index.? + 1; - sym.n_value = try self.allocateAtom(atom_index, @sizeOf(u64), @alignOf(u64)); - - log.debug("allocated threadlocal pointer atom at 0x{x}", .{sym.n_value}); - - try Atom.addRelocation(self, atom_index, .{ - .type = .unsigned, - .target = .{ .sym_index = tlv_desc_sym_index }, - .offset = 0, - .addend = 0, - .pcrel = false, - .length = 3, - }); - try Atom.addRebase(self, atom_index, 0); - try self.writePtrWidthAtom(atom_index); - - return atom_index; -} - fn createMhExecuteHeaderSymbol(self: *MachO) !void { if (self.base.options.output_mode != .Exe) return; if (self.getGlobal("__mh_execute_header")) |global| { @@ -1916,7 +1890,7 @@ pub fn deinit(self: *MachO) void { self.got_table.deinit(gpa); self.stubs_table.deinit(gpa); - self.tlvp_table.deinit(gpa); + self.tlv_table.deinit(gpa); self.strtab.deinit(gpa); self.locals.deinit(gpa); @@ -2150,13 +2124,11 @@ fn addStubEntry(self: *MachO, target: SymbolWithLoc) !void { } fn addTlvEntry(self: *MachO, target: SymbolWithLoc) !void { - if (self.tlvp_table.lookup.contains(target)) return; - const tlvp_index = try self.tlvp_table.allocateEntry(self.base.allocator, target); - const tlv_desc_atom_index = try self.createThreadLocalDescriptorAtom(target); - const tlv_desc_atom = self.getAtom(tlv_desc_atom_index); - const tlv_ptr_atom_index = try self.createThreadLocalPointerAtom(tlv_desc_atom.getSymbolIndex().?); - const tlv_ptr_atom = self.getAtom(tlv_ptr_atom_index); - self.tlvp_table.entries.items[tlvp_index].sym_index = tlv_ptr_atom.getSymbolIndex().?; + if (self.tlv_table.lookup.contains(target)) return; + const tlv_index = try self.tlv_table.allocateEntry(self.base.allocator, target); + const tlv_atom_index = try self.createThreadLocalDescriptorAtom(target); + const tlv_atom = self.getAtom(tlv_atom_index); + self.tlv_table.entries.items[tlv_index].sym_index = tlv_atom.getSymbolIndex().?; self.markRelocsDirtyByTarget(target); } @@ -2550,18 +2522,18 @@ fn updateDeclCode(self: *MachO, decl_index: Module.Decl.Index, code: []u8) !u64 if (vaddr != sym.n_value) { sym.n_value = vaddr; - const target = SymbolWithLoc{ .sym_index = sym_index }; + // TODO: I think we should update the offset to the initializer here too. + const target: SymbolWithLoc = if (is_threadlocal) blk: { + const tlv_atom_index = self.tlv_table.getAtomIndex(self, .{ + .sym_index = sym_index, + }).?; + const tlv_atom = self.getAtom(tlv_atom_index); + break :blk tlv_atom.getSymbolWithLoc(); + } else .{ .sym_index = sym_index }; self.markRelocsDirtyByTarget(target); - if (is_threadlocal) { - @panic("TODO update the threadlocal variable's name also"); - // log.debug(" (updating threadlocal pointer entry)", .{}); - // const tlvp_atom_index = self.tlvp_table.getAtomIndex(self, target).?; - // try self.writePtrWidthAtom(tlvp_atom_index); - } else { - log.debug(" (updating GOT entry)", .{}); - const got_atom_index = self.got_table.getAtomIndex(self, target).?; - try self.writePtrWidthAtom(got_atom_index); - } + log.debug(" (updating GOT entry)", .{}); + const got_atom_index = self.got_table.getAtomIndex(self, target).?; + try self.writePtrWidthAtom(got_atom_index); } } else if (code_len < atom.size) { self.shrinkAtom(atom_index, code_len); @@ -2586,12 +2558,15 @@ fn updateDeclCode(self: *MachO, decl_index: Module.Decl.Index, code: []u8) !u64 self.getAtomPtr(atom_index).size = code_len; sym.n_value = vaddr; - const target: SymbolWithLoc = .{ .sym_index = sym_index }; if (is_threadlocal) { - try self.addTlvEntry(target); - } else { - try self.addGotEntry(target); + try self.addTlvEntry(.{ .sym_index = sym_index }); } + const target: SymbolWithLoc = if (is_threadlocal) blk: { + const tlv_atom_index = self.tlv_table.getAtomIndex(self, .{ .sym_index = sym_index }).?; + const tlv_atom = self.getAtom(tlv_atom_index); + break :blk tlv_atom.getSymbolWithLoc(); + } else .{ .sym_index = sym_index }; + try self.addGotEntry(target); } try self.writeAtom(atom_index, code); @@ -2927,17 +2902,8 @@ fn populateMissingMetadata(self: *MachO) !void { } if (!self.base.options.single_threaded) { - if (self.thread_ptr_section_index == null) { - self.thread_ptr_section_index = try self.allocateSection("__DATA2", "__thread_ptrs", .{ - .size = @sizeOf(u64), - .alignment = @alignOf(u64), - .flags = macho.S_THREAD_LOCAL_VARIABLE_POINTERS, - .prot = macho.PROT.READ | macho.PROT.WRITE, - }); - self.segment_table_dirty = true; - } if (self.thread_vars_section_index == null) { - self.thread_vars_section_index = try self.allocateSection("__DATA3", "__thread_vars", .{ + self.thread_vars_section_index = try self.allocateSection("__DATA2", "__thread_vars", .{ .size = @sizeOf(u64) * 3, .alignment = @sizeOf(u64), .flags = macho.S_THREAD_LOCAL_VARIABLES, @@ -2947,7 +2913,7 @@ fn populateMissingMetadata(self: *MachO) !void { } if (self.thread_data_section_index == null) { - self.thread_data_section_index = try self.allocateSection("__DATA4", "__thread_data", .{ + self.thread_data_section_index = try self.allocateSection("__DATA3", "__thread_data", .{ .size = @sizeOf(u64), .alignment = @alignOf(u64), .flags = macho.S_THREAD_LOCAL_REGULAR, @@ -4361,7 +4327,7 @@ pub fn logSymtab(self: *MachO) void { log.debug("{}", .{self.stubs_table.fmtDebug(self)}); log.debug("threadlocal entries:", .{}); - log.debug("{}", .{self.tlvp_table.fmtDebug(self)}); + log.debug("{}", .{self.tlv_table.fmtDebug(self)}); } pub fn logAtoms(self: *MachO) void { diff --git a/src/link/MachO/Relocation.zig b/src/link/MachO/Relocation.zig index 06a255ad59..81340b1120 100644 --- a/src/link/MachO/Relocation.zig +++ b/src/link/MachO/Relocation.zig @@ -15,7 +15,7 @@ pub const Type = enum { got, /// RIP-relative displacement signed, - /// RIP-relative displacemen to threadlocal variable descriptor + /// RIP-relative displacement to GOT pointer to TLV thunk tlv, // aarch64 @@ -27,10 +27,6 @@ pub const Type = enum { page, /// Offset to a pointer relative to the start of a page in a section pageoff, - /// PC-relative distance to target page in TLV section - tlv_page, - /// Offset to a pointer relative to the start of a page in TLV section - tlv_pageoff, // common /// PC/RIP-relative displacement B/BL/CALL @@ -50,7 +46,12 @@ pub fn isResolvable(self: Relocation, macho_file: *MachO) bool { pub fn getTargetAtomIndex(self: Relocation, macho_file: *MachO) ?Atom.Index { return switch (self.type) { .got, .got_page, .got_pageoff => macho_file.got_table.getAtomIndex(macho_file, self.target), - .tlv, .tlv_page, .tlv_pageoff => macho_file.tlvp_table.getAtomIndex(macho_file, self.target), + .tlv => { + const thunk_atom_index = macho_file.tlv_table.getAtomIndex(macho_file, self.target) orelse + return null; + const thunk_atom = macho_file.getAtom(thunk_atom_index); + return macho_file.got_table.getAtomIndex(macho_file, thunk_atom.getSymbolWithLoc()); + }, .branch => if (macho_file.stubs_table.getAtomIndex(macho_file, self.target)) |index| index else @@ -109,7 +110,7 @@ fn resolveAarch64(self: Relocation, source_addr: u64, target_addr: i64, code: [] inst.unconditional_branch_immediate.imm26 = @truncate(u26, @bitCast(u28, displacement >> 2)); mem.writeIntLittle(u32, buffer[0..4], inst.toU32()); }, - .page, .got_page, .tlv_page => { + .page, .got_page => { const source_page = @intCast(i32, source_addr >> 12); const target_page = @intCast(i32, target_addr >> 12); const pages = @bitCast(u21, @intCast(i21, target_page - source_page)); @@ -158,49 +159,6 @@ fn resolveAarch64(self: Relocation, source_addr: u64, target_addr: i64, code: [] mem.writeIntLittle(u32, buffer[0..4], inst.toU32()); } }, - .tlv_pageoff => { - const RegInfo = struct { - rd: u5, - rn: u5, - size: u2, - }; - const reg_info: RegInfo = blk: { - if (isArithmeticOp(buffer[0..4])) { - const inst = mem.bytesToValue(meta.TagPayload( - aarch64.Instruction, - aarch64.Instruction.add_subtract_immediate, - ), buffer[0..4]); - break :blk .{ - .rd = inst.rd, - .rn = inst.rn, - .size = inst.sf, - }; - } else { - const inst = mem.bytesToValue(meta.TagPayload( - aarch64.Instruction, - aarch64.Instruction.load_store_register, - ), buffer[0..4]); - break :blk .{ - .rd = inst.rt, - .rn = inst.rn, - .size = inst.size, - }; - } - }; - const narrowed = @truncate(u12, @intCast(u64, target_addr)); - var inst = aarch64.Instruction{ - .add_subtract_immediate = .{ - .rd = reg_info.rd, - .rn = reg_info.rn, - .imm12 = narrowed, - .sh = 0, - .s = 0, - .op = 0, - .sf = @truncate(u1, reg_info.size), - }, - }; - mem.writeIntLittle(u32, buffer[0..4], inst.toU32()); - }, .tlv_initializer, .unsigned => switch (self.length) { 2 => mem.writeIntLittle(u32, buffer[0..4], @truncate(u32, @bitCast(u64, target_addr))), 3 => mem.writeIntLittle(u64, buffer[0..8], @bitCast(u64, target_addr)), @@ -227,7 +185,7 @@ fn resolveX8664(self: Relocation, source_addr: u64, target_addr: i64, code: []u8 else => unreachable, } }, - .got_page, .got_pageoff, .page, .pageoff, .tlv_page, .tlv_pageoff => unreachable, // Invalid target architecture. + .got_page, .got_pageoff, .page, .pageoff => unreachable, // Invalid target architecture. } } From fd52d4537a86fc312e4491206228ab3aeabe0f2c Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 12 Apr 2023 12:06:49 +0200 Subject: [PATCH 23/26] x86_64: emit pointer to TLV for macho --- src/arch/aarch64/CodeGen.zig | 4 +++ src/arch/x86_64/CodeGen.zig | 57 ++++++++++++++++++++++++++++++++++-- src/arch/x86_64/Emit.zig | 1 + src/arch/x86_64/Mir.zig | 3 ++ src/codegen.zig | 6 +++- 5 files changed, 68 insertions(+), 3 deletions(-) diff --git a/src/arch/aarch64/CodeGen.zig b/src/arch/aarch64/CodeGen.zig index 3a70144407..a85ca21016 100644 --- a/src/arch/aarch64/CodeGen.zig +++ b/src/arch/aarch64/CodeGen.zig @@ -3982,6 +3982,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type .got => .load_memory_ptr_got, .direct => .load_memory_ptr_direct, .import => unreachable, + .tlv => @panic("TODO TLV support"), }; const atom_index = switch (self.bin_file.tag) { .macho => blk: { @@ -5510,6 +5511,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro .got => .load_memory_ptr_got, .direct => .load_memory_ptr_direct, .import => unreachable, + .tlv => @panic("TODO TLV support"), }; const atom_index = switch (self.bin_file.tag) { .macho => blk: { @@ -5630,6 +5632,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void .got => .load_memory_got, .direct => .load_memory_direct, .import => .load_memory_import, + .tlv => @panic("TODO TLV support"), }; const atom_index = switch (self.bin_file.tag) { .macho => blk: { @@ -5830,6 +5833,7 @@ fn genSetStackArgument(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) I .got => .load_memory_ptr_got, .direct => .load_memory_ptr_direct, .import => unreachable, + .tlv => @panic("TODO TLV support"), }; const atom_index = switch (self.bin_file.tag) { .macho => blk: { diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 491144283c..16c85c622e 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -736,6 +736,7 @@ fn asmMovLinker(self: *Self, reg: Register, atom_index: u32, linker_load: codege .got => .got_reloc, .direct => .direct_reloc, .import => .import_reloc, + .tlv => .tlv_reloc, }; _ = try self.addInst(.{ .tag = .mov_linker, @@ -753,6 +754,7 @@ fn asmLeaLinker(self: *Self, reg: Register, atom_index: u32, linker_load: codege .got => .got_reloc, .direct => .direct_reloc, .import => .import_reloc, + .tlv => .tlv_reloc, }; _ = try self.addInst(.{ .tag = .lea_linker, @@ -765,6 +767,15 @@ fn asmLeaLinker(self: *Self, reg: Register, atom_index: u32, linker_load: codege }); } +fn genTlvPtr(self: *Self, reg: Register, atom_index: u32, linker_load: codegen.LinkerLoad) !void { + assert(linker_load.type == .tlv); + if (self.bin_file.cast(link.File.MachO)) |_| { + try self.asmMovLinker(.rdi, atom_index, linker_load); + try self.asmMemory(.call, Memory.sib(.qword, .{ .base = .rdi })); + try self.genSetReg(Type.usize, reg, .{ .register = .rax }); + } else return self.fail("TODO emit ptr-to-TLV sequence on {s}", .{@tagName(self.bin_file.tag)}); +} + fn gen(self: *Self) InnerError!void { const cc = self.fn_type.fnCallingConvention(); if (cc != .Naked) { @@ -2931,6 +2942,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void { .import => unreachable, .got => try self.asmMovLinker(addr_reg, atom_index, load_struct), .direct => try self.asmLeaLinker(addr_reg, atom_index, load_struct), + .tlv => try self.genTlvPtr(addr_reg, atom_index, load_struct), } }, else => return self.fail("TODO implement array_elem_val when array is {}", .{array}), @@ -3805,6 +3817,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type .import => unreachable, .got => try self.asmMovLinker(addr_reg, atom_index, load_struct), .direct => try self.asmLeaLinker(addr_reg, atom_index, load_struct), + .tlv => try self.genTlvPtr(addr_reg, atom_index, load_struct), } }, else => unreachable, @@ -3866,6 +3879,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type switch (load_struct.type) { .import => unreachable, .got, .direct => try self.asmMovLinker(addr_reg, atom_index, load_struct), + .tlv => { + try self.genTlvPtr(addr_reg, atom_index, load_struct); + // Load the pointer, which is stored in memory + try self.asmRegisterMemory(.mov, addr_reg, Memory.sib(.qword, .{ .base = addr_reg })); + }, } }, else => unreachable, @@ -4221,6 +4239,7 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue .import => unreachable, .got => try self.asmMovLinker(addr_reg, atom_index, load_struct), .direct => try self.asmLeaLinker(addr_reg, atom_index, load_struct), + .tlv => try self.genTlvPtr(addr_reg, atom_index, load_struct), } }, else => unreachable, @@ -4896,6 +4915,7 @@ fn genBinOp( .import => unreachable, .got => try self.asmMovLinker(addr_reg, atom_index, load_struct), .direct => try self.asmLeaLinker(addr_reg, atom_index, load_struct), + .tlv => try self.genTlvPtr(addr_reg, atom_index, load_struct), } }, else => unreachable, @@ -5042,6 +5062,7 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s .import => unreachable, .got => try self.asmMovLinker(dst_addr_reg, atom_index, load_struct), .direct => try self.asmLeaLinker(dst_addr_reg, atom_index, load_struct), + .tlv => try self.genTlvPtr(dst_addr_reg, atom_index, load_struct), } }, else => unreachable, @@ -5087,6 +5108,7 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s .import => unreachable, .got => try self.asmMovLinker(src_addr_reg, atom_index, load_struct), .direct => try self.asmLeaLinker(src_addr_reg, atom_index, load_struct), + .tlv => try self.genTlvPtr(src_addr_reg, atom_index, load_struct), } }, else => unreachable, @@ -6124,6 +6146,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC .import => unreachable, .got => try self.asmMovLinker(addr_reg, atom_index, load_struct), .direct => try self.asmLeaLinker(addr_reg, atom_index, load_struct), + .tlv => try self.genTlvPtr(addr_reg, atom_index, load_struct), } }, else => unreachable, @@ -7081,6 +7104,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE .import => unreachable, .got => try self.asmMovLinker(addr_reg, atom_index, load_struct), .direct => try self.asmLeaLinker(addr_reg, atom_index, load_struct), + .tlv => try self.genTlvPtr(addr_reg, atom_index, load_struct), } }, else => unreachable, @@ -7308,6 +7332,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl .import => unreachable, .got => try self.asmMovLinker(addr_reg, atom_index, load_struct), .direct => try self.asmLeaLinker(addr_reg, atom_index, load_struct), + .tlv => try self.genTlvPtr(addr_reg, atom_index, load_struct), } }, else => unreachable, @@ -7439,6 +7464,11 @@ fn genInlineMemcpy( switch (load_struct.type) { .import => unreachable, .got, .direct => try self.asmMovLinker(.rdi, atom_index, load_struct), + .tlv => { + try self.genTlvPtr(.rdi, atom_index, load_struct); + // Load the pointer, which is stored in memory + try self.asmRegisterMemory(.mov, .rdi, Memory.sib(.qword, .{ .base = .rdi })); + }, } }, .stack_offset, .ptr_stack_offset => |off| { @@ -7481,6 +7511,11 @@ fn genInlineMemcpy( switch (load_struct.type) { .import => unreachable, .got, .direct => try self.asmMovLinker(.rsi, atom_index, load_struct), + .tlv => { + try self.genTlvPtr(.rsi, atom_index, load_struct); + // Load the pointer, which is stored in memory + try self.asmRegisterMemory(.mov, .rsi, Memory.sib(.qword, .{ .base = .rsi })); + }, } }, .stack_offset, .ptr_stack_offset => |off| { @@ -7546,6 +7581,11 @@ fn genInlineMemset( switch (load_struct.type) { .import => unreachable, .got, .direct => try self.asmMovLinker(.rdi, atom_index, load_struct), + .tlv => { + try self.genTlvPtr(.rdi, atom_index, load_struct); + // Load the pointer, which is stored in memory + try self.asmRegisterMemory(.mov, .rdi, Memory.sib(.qword, .{ .base = .rdi })); + }, } }, .stack_offset, .ptr_stack_offset => |off| { @@ -7735,7 +7775,10 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void switch (ty.zigTypeTag()) { .Float => { const base_reg = (try self.register_manager.allocReg(null, gp)).to64(); - try self.asmLeaLinker(base_reg, atom_index, load_struct); + switch (load_struct.type) { + .tlv => try self.genTlvPtr(base_reg, atom_index, load_struct), + else => try self.asmLeaLinker(base_reg, atom_index, load_struct), + } if (intrinsicsAllowed(self.target.*, ty)) { return self.asmRegisterMemory( @@ -7753,7 +7796,17 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void return self.fail("TODO genSetReg from memory for float with no intrinsics", .{}); }, - else => try self.asmMovLinker(registerAlias(reg, abi_size), atom_index, load_struct), + else => switch (load_struct.type) { + .tlv => { + try self.genTlvPtr(reg.to64(), atom_index, load_struct); + try self.asmRegisterMemory( + .mov, + registerAlias(reg, abi_size), + Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }), + ); + }, + else => try self.asmMovLinker(registerAlias(reg, abi_size), atom_index, load_struct), + }, } }, .stack_offset => |off| { diff --git a/src/arch/x86_64/Emit.zig b/src/arch/x86_64/Emit.zig index 16fb8a0183..a81840be47 100644 --- a/src/arch/x86_64/Emit.zig +++ b/src/arch/x86_64/Emit.zig @@ -76,6 +76,7 @@ pub fn emitMir(emit: *Emit) Error!void { .type = switch (inst.ops) { .got_reloc => .got, .direct_reloc => .signed, + .tlv_reloc => .tlv, else => unreachable, }, .target = .{ .sym_index = metadata.sym_index, .file = null }, diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig index 835e9bfdcb..8772b835dd 100644 --- a/src/arch/x86_64/Mir.zig +++ b/src/arch/x86_64/Mir.zig @@ -404,6 +404,9 @@ pub const Inst = struct { /// Linker relocation - imports table indirection (binding). /// Uses `payload` payload with extra data of type `LeaRegisterReloc`. import_reloc, + /// Linker relocation - threadlocal variable via GOT indirection. + /// Uses `payload` payload with extra data of type `LeaRegisterReloc`. + tlv_reloc, }; pub const Data = union { diff --git a/src/codegen.zig b/src/codegen.zig index 4156dd853b..e00141db97 100644 --- a/src/codegen.zig +++ b/src/codegen.zig @@ -912,11 +912,13 @@ fn lowerDeclRef( /// * got - the value is referenced indirectly via GOT entry index (the linker emits a got-type reloc) /// * direct - the value is referenced directly via symbol index index (the linker emits a displacement reloc) /// * import - the value is referenced indirectly via import entry index (the linker emits an import-type reloc) +/// * tlv - the value is a threadlocal variable referenced indirectly via GOT (the linker emits a got-type reloc) pub const LinkerLoad = struct { type: enum { got, direct, import, + tlv, }, sym_index: u32, }; @@ -991,6 +993,8 @@ fn genDeclRef( module.markDeclAlive(decl); + const is_threadlocal = tv.val.isPtrToThreadLocal(module) and !bin_file.options.single_threaded; + if (bin_file.cast(link.File.Elf)) |elf_file| { const atom_index = try elf_file.getOrCreateAtomForDecl(decl_index); const atom = elf_file.getAtom(atom_index); @@ -999,7 +1003,7 @@ fn genDeclRef( const atom_index = try macho_file.getOrCreateAtomForDecl(decl_index); const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?; return GenResult.mcv(.{ .linker_load = .{ - .type = .got, + .type = if (is_threadlocal) .tlv else .got, .sym_index = sym_index, } }); } else if (bin_file.cast(link.File.Coff)) |coff_file| { From 887da399eb266b166d1a109bdfe631f4533ead64 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 12 Apr 2023 13:07:19 +0200 Subject: [PATCH 24/26] x86_64: fix load() and store() to accommodate TLV --- src/arch/x86_64/CodeGen.zig | 29 +++++++++++++++++++++++------ 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 16c85c622e..c49955fcc0 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -3656,10 +3656,31 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo else => return self.fail("TODO implement loading from register into {}", .{dst_mcv}), } }, - .memory, .linker_load => { + .memory => { const reg = try self.copyToTmpRegister(ptr_ty, ptr); try self.load(dst_mcv, .{ .register = reg }, ptr_ty); }, + .linker_load => |load_struct| { + const addr_reg = (try self.register_manager.allocReg(null, gp)).to64(); + const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg); + defer self.register_manager.unlockReg(addr_reg_lock); + + const atom_index = if (self.bin_file.cast(link.File.MachO)) |macho_file| blk: { + const atom = try macho_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk macho_file.getAtom(atom).getSymbolIndex().?; + } else if (self.bin_file.cast(link.File.Coff)) |coff_file| blk: { + const atom = try coff_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk coff_file.getAtom(atom).getSymbolIndex().?; + } else unreachable; + + switch (load_struct.type) { + .import => unreachable, + .got, .direct => try self.asmMovLinker(addr_reg, atom_index, load_struct), + .tlv => try self.genTlvPtr(addr_reg, atom_index, load_struct), + } + + try self.load(dst_mcv, .{ .register = addr_reg }, ptr_ty); + }, } } @@ -3879,11 +3900,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type switch (load_struct.type) { .import => unreachable, .got, .direct => try self.asmMovLinker(addr_reg, atom_index, load_struct), - .tlv => { - try self.genTlvPtr(addr_reg, atom_index, load_struct); - // Load the pointer, which is stored in memory - try self.asmRegisterMemory(.mov, addr_reg, Memory.sib(.qword, .{ .base = addr_reg })); - }, + .tlv => try self.genTlvPtr(addr_reg, atom_index, load_struct), } }, else => unreachable, From 3f35d6984fa7e90a86cae3d6769cc2f1036bbdde Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 13 Apr 2023 16:09:47 +0200 Subject: [PATCH 25/26] x86_64: make TLV a separate MCValue --- src/arch/aarch64/CodeGen.zig | 5 +- src/arch/arm/CodeGen.zig | 2 +- src/arch/riscv64/CodeGen.zig | 2 +- src/arch/sparc64/CodeGen.zig | 2 +- src/arch/x86_64/CodeGen.zig | 119 ++++++++++++++++++----------------- src/codegen.zig | 11 +++- 6 files changed, 72 insertions(+), 69 deletions(-) diff --git a/src/arch/aarch64/CodeGen.zig b/src/arch/aarch64/CodeGen.zig index a85ca21016..acaddedce7 100644 --- a/src/arch/aarch64/CodeGen.zig +++ b/src/arch/aarch64/CodeGen.zig @@ -3982,7 +3982,6 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type .got => .load_memory_ptr_got, .direct => .load_memory_ptr_direct, .import => unreachable, - .tlv => @panic("TODO TLV support"), }; const atom_index = switch (self.bin_file.tag) { .macho => blk: { @@ -5511,7 +5510,6 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro .got => .load_memory_ptr_got, .direct => .load_memory_ptr_direct, .import => unreachable, - .tlv => @panic("TODO TLV support"), }; const atom_index = switch (self.bin_file.tag) { .macho => blk: { @@ -5632,7 +5630,6 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void .got => .load_memory_got, .direct => .load_memory_direct, .import => .load_memory_import, - .tlv => @panic("TODO TLV support"), }; const atom_index = switch (self.bin_file.tag) { .macho => blk: { @@ -5833,7 +5830,6 @@ fn genSetStackArgument(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) I .got => .load_memory_ptr_got, .direct => .load_memory_ptr_direct, .import => unreachable, - .tlv => @panic("TODO TLV support"), }; const atom_index = switch (self.bin_file.tag) { .macho => blk: { @@ -6175,6 +6171,7 @@ fn genTypedValue(self: *Self, arg_tv: TypedValue) InnerError!MCValue { .linker_load => |ll| .{ .linker_load = ll }, .immediate => |imm| .{ .immediate = imm }, .memory => |addr| .{ .memory = addr }, + .tlv_reloc => unreachable, // TODO }, .fail => |msg| { self.err_msg = msg; diff --git a/src/arch/arm/CodeGen.zig b/src/arch/arm/CodeGen.zig index ed94b895ed..69a71a2c7e 100644 --- a/src/arch/arm/CodeGen.zig +++ b/src/arch/arm/CodeGen.zig @@ -6114,7 +6114,7 @@ fn genTypedValue(self: *Self, arg_tv: TypedValue) InnerError!MCValue { .mcv => |mcv| switch (mcv) { .none => .none, .undef => .undef, - .linker_load => unreachable, // TODO + .tlv_reloc, .linker_load => unreachable, // TODO .immediate => |imm| .{ .immediate = @truncate(u32, imm) }, .memory => |addr| .{ .memory = addr }, }, diff --git a/src/arch/riscv64/CodeGen.zig b/src/arch/riscv64/CodeGen.zig index 11dbb2cb08..49c6ff183c 100644 --- a/src/arch/riscv64/CodeGen.zig +++ b/src/arch/riscv64/CodeGen.zig @@ -2572,7 +2572,7 @@ fn genTypedValue(self: *Self, typed_value: TypedValue) InnerError!MCValue { .mcv => |mcv| switch (mcv) { .none => .none, .undef => .undef, - .linker_load => unreachable, // TODO + .tlv_reloc, .linker_load => unreachable, // TODO .immediate => |imm| .{ .immediate = imm }, .memory => |addr| .{ .memory = addr }, }, diff --git a/src/arch/sparc64/CodeGen.zig b/src/arch/sparc64/CodeGen.zig index 3816b63ab6..52f8ba085f 100644 --- a/src/arch/sparc64/CodeGen.zig +++ b/src/arch/sparc64/CodeGen.zig @@ -3931,7 +3931,7 @@ fn genTypedValue(self: *Self, typed_value: TypedValue) InnerError!MCValue { .mcv => |mcv| switch (mcv) { .none => .none, .undef => .undef, - .linker_load => unreachable, // TODO + .tlv_reloc, .linker_load => unreachable, // TODO .immediate => |imm| .{ .immediate = imm }, .memory => |addr| .{ .memory = addr }, }, diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index c49955fcc0..cbf8a48d0f 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -132,6 +132,10 @@ pub const MCValue = union(enum) { memory: u64, /// The value is in memory but requires a linker relocation fixup. linker_load: codegen.LinkerLoad, + /// Pointer to a threadlocal variable. + /// The address resolution will be deferred until the linker allocates everything in virtual memory. + /// Payload is a symbol index. + tlv_reloc: u32, /// The value is one of the stack variables. /// If the type is a pointer, it means the pointer address is in the stack at this offset. stack_offset: i32, @@ -146,6 +150,7 @@ pub const MCValue = union(enum) { .stack_offset, .ptr_stack_offset, .linker_load, + .tlv_reloc, => true, else => false, }; @@ -736,7 +741,6 @@ fn asmMovLinker(self: *Self, reg: Register, atom_index: u32, linker_load: codege .got => .got_reloc, .direct => .direct_reloc, .import => .import_reloc, - .tlv => .tlv_reloc, }; _ = try self.addInst(.{ .tag = .mov_linker, @@ -754,7 +758,6 @@ fn asmLeaLinker(self: *Self, reg: Register, atom_index: u32, linker_load: codege .got => .got_reloc, .direct => .direct_reloc, .import => .import_reloc, - .tlv => .tlv_reloc, }; _ = try self.addInst(.{ .tag = .lea_linker, @@ -767,15 +770,6 @@ fn asmLeaLinker(self: *Self, reg: Register, atom_index: u32, linker_load: codege }); } -fn genTlvPtr(self: *Self, reg: Register, atom_index: u32, linker_load: codegen.LinkerLoad) !void { - assert(linker_load.type == .tlv); - if (self.bin_file.cast(link.File.MachO)) |_| { - try self.asmMovLinker(.rdi, atom_index, linker_load); - try self.asmMemory(.call, Memory.sib(.qword, .{ .base = .rdi })); - try self.genSetReg(Type.usize, reg, .{ .register = .rax }); - } else return self.fail("TODO emit ptr-to-TLV sequence on {s}", .{@tagName(self.bin_file.tag)}); -} - fn gen(self: *Self) InnerError!void { const cc = self.fn_type.fnCallingConvention(); if (cc != .Naked) { @@ -2929,6 +2923,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void { })); }, .memory => |addr| try self.genSetReg(Type.usize, addr_reg, .{ .immediate = addr }), + .tlv_reloc => try self.genSetReg(array_ty, addr_reg, array), .linker_load => |load_struct| { const atom_index = if (self.bin_file.cast(link.File.MachO)) |macho_file| blk: { const atom = try macho_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); @@ -2942,7 +2937,6 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void { .import => unreachable, .got => try self.asmMovLinker(addr_reg, atom_index, load_struct), .direct => try self.asmLeaLinker(addr_reg, atom_index, load_struct), - .tlv => try self.genTlvPtr(addr_reg, atom_index, load_struct), } }, else => return self.fail("TODO implement array_elem_val when array is {}", .{array}), @@ -3656,7 +3650,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo else => return self.fail("TODO implement loading from register into {}", .{dst_mcv}), } }, - .memory => { + .memory, .tlv_reloc => { const reg = try self.copyToTmpRegister(ptr_ty, ptr); try self.load(dst_mcv, .{ .register = reg }, ptr_ty); }, @@ -3676,7 +3670,6 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo switch (load_struct.type) { .import => unreachable, .got, .direct => try self.asmMovLinker(addr_reg, atom_index, load_struct), - .tlv => try self.genTlvPtr(addr_reg, atom_index, load_struct), } try self.load(dst_mcv, .{ .register = addr_reg }, ptr_ty); @@ -3838,7 +3831,6 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type .import => unreachable, .got => try self.asmMovLinker(addr_reg, atom_index, load_struct), .direct => try self.asmLeaLinker(addr_reg, atom_index, load_struct), - .tlv => try self.genTlvPtr(addr_reg, atom_index, load_struct), } }, else => unreachable, @@ -3863,7 +3855,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type .{ .immediate = abi_size }, .{}, ), - .ptr_stack_offset => { + .ptr_stack_offset, .tlv_reloc => { const tmp_reg = try self.copyToTmpRegister(value_ty, value); const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); defer self.register_manager.unlockReg(tmp_lock); @@ -3900,12 +3892,19 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type switch (load_struct.type) { .import => unreachable, .got, .direct => try self.asmMovLinker(addr_reg, atom_index, load_struct), - .tlv => try self.genTlvPtr(addr_reg, atom_index, load_struct), } }, else => unreachable, } + const new_ptr = MCValue{ .register = addr_reg }; + try self.store(new_ptr, value, ptr_ty, value_ty); + }, + .tlv_reloc => { + const addr_reg = try self.copyToTmpRegister(Type.usize, ptr); + const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg); + defer self.register_manager.unlockReg(addr_reg_lock); + const new_ptr = MCValue{ .register = addr_reg }; try self.store(new_ptr, value, ptr_ty, value_ty); }, @@ -3954,7 +3953,7 @@ fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32 const dst_mcv: MCValue = result: { switch (mcv) { - .stack_offset => { + .stack_offset, .tlv_reloc => { const offset_reg = try self.copyToTmpRegister(ptr_ty, .{ .immediate = field_offset, }); @@ -4236,6 +4235,7 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue })); }, .ptr_stack_offset => unreachable, + .tlv_reloc => unreachable, .memory, .linker_load => { const addr_reg = (try self.register_manager.allocReg(null, gp)).to64(); const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg); @@ -4256,7 +4256,6 @@ fn genUnOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue .import => unreachable, .got => try self.asmMovLinker(addr_reg, atom_index, load_struct), .direct => try self.asmLeaLinker(addr_reg, atom_index, load_struct), - .tlv => try self.genTlvPtr(addr_reg, atom_index, load_struct), } }, else => unreachable, @@ -4898,6 +4897,7 @@ fn genBinOp( .eflags, .register_overflow, .ptr_stack_offset, + .tlv_reloc, => unreachable, .register => |src_reg| try self.asmCmovccRegisterRegister( registerAlias(tmp_reg, cmov_abi_size), @@ -4932,7 +4932,6 @@ fn genBinOp( .import => unreachable, .got => try self.asmMovLinker(addr_reg, atom_index, load_struct), .direct => try self.asmLeaLinker(addr_reg, atom_index, load_struct), - .tlv => try self.genTlvPtr(addr_reg, atom_index, load_struct), } }, else => unreachable, @@ -4983,7 +4982,7 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s .undef => unreachable, .dead, .unreach => unreachable, .register_overflow => unreachable, - .ptr_stack_offset => { + .ptr_stack_offset, .tlv_reloc => { const dst_reg_lock = self.register_manager.lockReg(dst_reg); defer if (dst_reg_lock) |lock| self.register_manager.unlockReg(lock); @@ -5079,7 +5078,6 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s .import => unreachable, .got => try self.asmMovLinker(dst_addr_reg, atom_index, load_struct), .direct => try self.asmLeaLinker(dst_addr_reg, atom_index, load_struct), - .tlv => try self.genTlvPtr(dst_addr_reg, atom_index, load_struct), } }, else => unreachable, @@ -5125,7 +5123,6 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s .import => unreachable, .got => try self.asmMovLinker(src_addr_reg, atom_index, load_struct), .direct => try self.asmLeaLinker(src_addr_reg, atom_index, load_struct), - .tlv => try self.genTlvPtr(src_addr_reg, atom_index, load_struct), } }, else => unreachable, @@ -5236,7 +5233,7 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s else => unreachable, } }, - .memory, .linker_load => { + .memory, .linker_load, .tlv_reloc => { try self.asmRegisterMemory( .mov, registerAlias(src.?.limb_reg, limb_abi_size), @@ -5275,6 +5272,7 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, ty: Type, dst_mcv: MCValue, s } }, .ptr_stack_offset => unreachable, + .tlv_reloc => unreachable, } } @@ -5288,6 +5286,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M .dead, .unreach, .immediate => unreachable, .eflags => unreachable, .ptr_stack_offset => unreachable, + .tlv_reloc => unreachable, .register_overflow => unreachable, .register => |dst_reg| { const dst_alias = registerAlias(dst_reg, abi_size); @@ -5299,6 +5298,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M .undef => try self.genSetReg(dst_ty, dst_reg, .undef), .dead, .unreach => unreachable, .ptr_stack_offset => unreachable, + .tlv_reloc => unreachable, .register_overflow => unreachable, .register => |src_reg| try self.asmRegisterRegister( .imul, @@ -5342,6 +5342,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M .undef => return self.genSetStack(dst_ty, off, .undef, .{}), .dead, .unreach => unreachable, .ptr_stack_offset => unreachable, + .tlv_reloc => unreachable, .register_overflow => unreachable, .register => |src_reg| { // copy dst to a register @@ -5564,6 +5565,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier .linker_load => unreachable, .eflags => unreachable, .register_overflow => unreachable, + .tlv_reloc => unreachable, } } @@ -5602,6 +5604,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier .linker_load => unreachable, .eflags => unreachable, .register_overflow => unreachable, + .tlv_reloc => unreachable, } } @@ -6123,6 +6126,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC .register_overflow, .ptr_stack_offset, .eflags, + .tlv_reloc, => unreachable, .register => |opt_reg| { @@ -6163,7 +6167,6 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC .import => unreachable, .got => try self.asmMovLinker(addr_reg, atom_index, load_struct), .direct => try self.asmLeaLinker(addr_reg, atom_index, load_struct), - .tlv => try self.genTlvPtr(addr_reg, atom_index, load_struct), } }, else => unreachable, @@ -7121,7 +7124,6 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE .import => unreachable, .got => try self.asmMovLinker(addr_reg, atom_index, load_struct), .direct => try self.asmLeaLinker(addr_reg, atom_index, load_struct), - .tlv => try self.genTlvPtr(addr_reg, atom_index, load_struct), } }, else => unreachable, @@ -7168,7 +7170,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE }, } }, - .ptr_stack_offset => { + .ptr_stack_offset, .tlv_reloc => { const reg = try self.copyToTmpRegister(ty, mcv); return self.genSetStackArg(ty, stack_offset, MCValue{ .register = reg }); }, @@ -7349,7 +7351,6 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl .import => unreachable, .got => try self.asmMovLinker(addr_reg, atom_index, load_struct), .direct => try self.asmLeaLinker(addr_reg, atom_index, load_struct), - .tlv => try self.genTlvPtr(addr_reg, atom_index, load_struct), } }, else => unreachable, @@ -7374,7 +7375,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl .{ .immediate = abi_size }, .{}, ), - .ptr_stack_offset => { + .ptr_stack_offset, .tlv_reloc => { const tmp_reg = try self.copyToTmpRegister(ty, mcv); const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); defer self.register_manager.unlockReg(tmp_lock); @@ -7481,13 +7482,9 @@ fn genInlineMemcpy( switch (load_struct.type) { .import => unreachable, .got, .direct => try self.asmMovLinker(.rdi, atom_index, load_struct), - .tlv => { - try self.genTlvPtr(.rdi, atom_index, load_struct); - // Load the pointer, which is stored in memory - try self.asmRegisterMemory(.mov, .rdi, Memory.sib(.qword, .{ .base = .rdi })); - }, } }, + .tlv_reloc => try self.genSetReg(Type.usize, .rdi, dst_ptr), .stack_offset, .ptr_stack_offset => |off| { try self.asmRegisterMemory(switch (dst_ptr) { .stack_offset => .mov, @@ -7528,13 +7525,9 @@ fn genInlineMemcpy( switch (load_struct.type) { .import => unreachable, .got, .direct => try self.asmMovLinker(.rsi, atom_index, load_struct), - .tlv => { - try self.genTlvPtr(.rsi, atom_index, load_struct); - // Load the pointer, which is stored in memory - try self.asmRegisterMemory(.mov, .rsi, Memory.sib(.qword, .{ .base = .rsi })); - }, } }, + .tlv_reloc => try self.genSetReg(Type.usize, .rsi, src_ptr), .stack_offset, .ptr_stack_offset => |off| { try self.asmRegisterMemory(switch (src_ptr) { .stack_offset => .mov, @@ -7598,13 +7591,9 @@ fn genInlineMemset( switch (load_struct.type) { .import => unreachable, .got, .direct => try self.asmMovLinker(.rdi, atom_index, load_struct), - .tlv => { - try self.genTlvPtr(.rdi, atom_index, load_struct); - // Load the pointer, which is stored in memory - try self.asmRegisterMemory(.mov, .rdi, Memory.sib(.qword, .{ .base = .rdi })); - }, } }, + .tlv_reloc => try self.genSetReg(Type.usize, .rdi, dst_ptr), .stack_offset, .ptr_stack_offset => |off| { try self.asmRegisterMemory(switch (dst_ptr) { .stack_offset => .mov, @@ -7780,6 +7769,30 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void } }, }, + .tlv_reloc => |sym_index| { + const atom_index = if (self.bin_file.cast(link.File.MachO)) |macho_file| blk: { + const atom = try macho_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk macho_file.getAtom(atom).getSymbolIndex().?; + } else if (self.bin_file.cast(link.File.Coff)) |coff_file| blk: { + const atom = try coff_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); + break :blk coff_file.getAtom(atom).getSymbolIndex().?; + } else unreachable; + + if (self.bin_file.cast(link.File.MachO)) |_| { + _ = try self.addInst(.{ + .tag = .mov_linker, + .ops = .tlv_reloc, + .data = .{ .payload = try self.addExtra(Mir.LeaRegisterReloc{ + .reg = @enumToInt(Register.rdi), + .atom_index = atom_index, + .sym_index = sym_index, + }) }, + }); + // TODO: spill registers before calling + try self.asmMemory(.call, Memory.sib(.qword, .{ .base = .rdi })); + try self.genSetReg(Type.usize, reg, .{ .register = .rax }); + } else return self.fail("TODO emit ptr to TLV sequence on {s}", .{@tagName(self.bin_file.tag)}); + }, .linker_load => |load_struct| { const atom_index = if (self.bin_file.cast(link.File.MachO)) |macho_file| blk: { const atom = try macho_file.getOrCreateAtomForDecl(self.mod_fn.owner_decl); @@ -7792,10 +7805,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void switch (ty.zigTypeTag()) { .Float => { const base_reg = (try self.register_manager.allocReg(null, gp)).to64(); - switch (load_struct.type) { - .tlv => try self.genTlvPtr(base_reg, atom_index, load_struct), - else => try self.asmLeaLinker(base_reg, atom_index, load_struct), - } + try self.asmLeaLinker(base_reg, atom_index, load_struct); if (intrinsicsAllowed(self.target.*, ty)) { return self.asmRegisterMemory( @@ -7813,17 +7823,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void return self.fail("TODO genSetReg from memory for float with no intrinsics", .{}); }, - else => switch (load_struct.type) { - .tlv => { - try self.genTlvPtr(reg.to64(), atom_index, load_struct); - try self.asmRegisterMemory( - .mov, - registerAlias(reg, abi_size), - Memory.sib(Memory.PtrSize.fromSize(abi_size), .{ .base = reg.to64() }), - ); - }, - else => try self.asmMovLinker(registerAlias(reg, abi_size), atom_index, load_struct), - }, + else => try self.asmMovLinker(registerAlias(reg, abi_size), atom_index, load_struct), } }, .stack_offset => |off| { @@ -8779,6 +8779,7 @@ fn genTypedValue(self: *Self, arg_tv: TypedValue) InnerError!MCValue { .linker_load => |ll| .{ .linker_load = ll }, .immediate => |imm| .{ .immediate = imm }, .memory => |addr| .{ .memory = addr }, + .tlv_reloc => |sym_index| .{ .tlv_reloc = sym_index }, }, .fail => |msg| { self.err_msg = msg; diff --git a/src/codegen.zig b/src/codegen.zig index e00141db97..6939b750fd 100644 --- a/src/codegen.zig +++ b/src/codegen.zig @@ -912,13 +912,11 @@ fn lowerDeclRef( /// * got - the value is referenced indirectly via GOT entry index (the linker emits a got-type reloc) /// * direct - the value is referenced directly via symbol index index (the linker emits a displacement reloc) /// * import - the value is referenced indirectly via import entry index (the linker emits an import-type reloc) -/// * tlv - the value is a threadlocal variable referenced indirectly via GOT (the linker emits a got-type reloc) pub const LinkerLoad = struct { type: enum { got, direct, import, - tlv, }, sym_index: u32, }; @@ -934,6 +932,10 @@ pub const GenResult = union(enum) { /// such as ARM, the immediate will never exceed 32-bits. immediate: u64, linker_load: LinkerLoad, + /// Pointer to a threadlocal variable. + /// The address resolution will be deferred until the linker allocates everything in virtual memory. + /// Payload is a symbol index. + tlv_reloc: u32, /// Direct by-address reference to memory location. memory: u64, }; @@ -1002,8 +1004,11 @@ fn genDeclRef( } else if (bin_file.cast(link.File.MachO)) |macho_file| { const atom_index = try macho_file.getOrCreateAtomForDecl(decl_index); const sym_index = macho_file.getAtom(atom_index).getSymbolIndex().?; + if (is_threadlocal) { + return GenResult.mcv(.{ .tlv_reloc = sym_index }); + } return GenResult.mcv(.{ .linker_load = .{ - .type = if (is_threadlocal) .tlv else .got, + .type = .got, .sym_index = sym_index, } }); } else if (bin_file.cast(link.File.Coff)) |coff_file| { From a34752c941c81e600c56670abb612fc86d0a2b73 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 13 Apr 2023 16:44:22 +0200 Subject: [PATCH 26/26] ci: let's not skip non native tests on macos! --- ci/x86_64-macos-debug.sh | 1 - ci/x86_64-macos-release.sh | 1 - 2 files changed, 2 deletions(-) diff --git a/ci/x86_64-macos-debug.sh b/ci/x86_64-macos-debug.sh index d1e86123ed..cbfece9dd9 100755 --- a/ci/x86_64-macos-debug.sh +++ b/ci/x86_64-macos-debug.sh @@ -51,7 +51,6 @@ stage3/bin/zig build test docs \ --zig-lib-dir "$(pwd)/../lib" \ -Denable-macos-sdk \ -Dstatic-llvm \ - -Dskip-non-native \ --search-prefix "$PREFIX" # Produce the experimental std lib documentation. diff --git a/ci/x86_64-macos-release.sh b/ci/x86_64-macos-release.sh index 1dd17943c3..17a5e43bb0 100755 --- a/ci/x86_64-macos-release.sh +++ b/ci/x86_64-macos-release.sh @@ -51,7 +51,6 @@ stage3/bin/zig build test docs \ --zig-lib-dir "$(pwd)/../lib" \ -Denable-macos-sdk \ -Dstatic-llvm \ - -Dskip-non-native \ --search-prefix "$PREFIX" # Produce the experimental std lib documentation.