From 27e1e636710ca72fb4508e2ea731dcb39cedbc95 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 8 Aug 2024 07:45:05 +0200 Subject: [PATCH 01/39] elf: introduce OffsetTable in ZigObject for funcs only --- src/link/Elf.zig | 6 +++- src/link/Elf/ZigObject.zig | 58 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 103c69202b..41c4f202d7 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -5657,10 +5657,14 @@ fn fmtDumpState( if (self.zigObjectPtr()) |zig_object| { try writer.print("zig_object({d}) : {s}\n", .{ zig_object.index, zig_object.path }); - try writer.print("{}{}\n", .{ + try writer.print("{}{}", .{ zig_object.fmtAtoms(self), zig_object.fmtSymtab(self), }); + if (zig_object.func_offset_table) |ot| { + try writer.print("{}", .{ot.fmt(zig_object, self)}); + } + try writer.writeByte('\n'); } for (self.objects.items) |index| { diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index ef3e2ed77c..671dc285e3 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -55,6 +55,8 @@ debug_str_section_zig_size: u64 = 0, debug_aranges_section_zig_size: u64 = 0, debug_line_section_zig_size: u64 = 0, +func_offset_table: ?OffsetTable = null, + pub const global_symbol_bit: u32 = 0x80000000; pub const symbol_mask: u32 = 0x7fffffff; pub const SHN_ATOM: u16 = 0x100; @@ -128,6 +130,10 @@ pub fn deinit(self: *ZigObject, allocator: Allocator) void { if (self.dwarf) |*dw| { dw.deinit(); } + + if (self.func_offset_table) |*ot| { + ot.deinit(allocator); + } } pub fn flushModule(self: *ZigObject, elf_file: *Elf, tid: Zcu.PerThread.Id) !void { @@ -1687,6 +1693,58 @@ const UavTable = std.AutoHashMapUnmanaged(InternPool.Index, AvMetadata); const LazySymbolTable = std.AutoArrayHashMapUnmanaged(InternPool.Index, LazySymbolMetadata); const TlsTable = std.AutoArrayHashMapUnmanaged(Atom.Index, TlsVariable); +pub const OffsetTable = struct { + atom_index: Atom.Index, + entries: std.ArrayListUnmanaged(Symbol.Index) = .{}, + + pub fn deinit(ot: *OffsetTable, allocator: Allocator) void { + ot.entries.deinit(allocator); + } + + pub fn size(ot: OffsetTable, elf_file: *Elf) usize { + return ot.entries.items.len * switch (elf_file.ptr_width) { + .p32 => @as(usize, 4), + .p64 => 8, + }; + } + + const OffsetTableFormatContext = struct { OffsetTable, *ZigObject, *Elf }; + + pub fn format( + ot: OffsetTable, + comptime unused_fmt_string: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) !void { + _ = ot; + _ = unused_fmt_string; + _ = options; + _ = writer; + @compileError("do not format OffsetTable directly"); + } + + pub fn fmt(ot: OffsetTable, zo: *ZigObject, elf_file: *Elf) std.fmt.Formatter(format2) { + return .{ .data = .{ ot, zo, elf_file } }; + } + + fn format2( + ctx: OffsetTableFormatContext, + comptime unused_fmt_string: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) !void { + _ = options; + _ = unused_fmt_string; + const ot, const zo, const ef = ctx; + const atom_ptr = zo.atom(ot.atom_index).?; + try writer.print("@{x} : size({x})\n", .{ atom_ptr.address(ef), ot.size(ef) }); + for (ot.entries.items) |sym_index| { + const sym = zo.symbol(sym_index); + try writer.print(" %{d} : {s} : @{x}\n", .{ sym_index, sym.name(ef), sym.value }); + } + } +}; + const assert = std.debug.assert; const builtin = @import("builtin"); const codegen = @import("../../codegen.zig"); From 67e703dc71a30989f9caaa3fa13e141d1e9584c3 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 8 Aug 2024 08:30:56 +0200 Subject: [PATCH 02/39] elf: allocate new offset table via Atom.allocate mechanism --- src/link/Elf.zig | 2 +- src/link/Elf/Symbol.zig | 5 +++ src/link/Elf/ZigObject.zig | 79 +++++++++++++++++++++++++++++++++----- 3 files changed, 75 insertions(+), 11 deletions(-) diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 41c4f202d7..ffba511e87 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -5661,7 +5661,7 @@ fn fmtDumpState( zig_object.fmtAtoms(self), zig_object.fmtSymtab(self), }); - if (zig_object.func_offset_table) |ot| { + if (zig_object.offset_table) |ot| { try writer.print("{}", .{ot.fmt(zig_object, self)}); } try writer.writeByte('\n'); diff --git a/src/link/Elf/Symbol.zig b/src/link/Elf/Symbol.zig index ce6b94a185..f023ad5d91 100644 --- a/src/link/Elf/Symbol.zig +++ b/src/link/Elf/Symbol.zig @@ -260,6 +260,7 @@ const AddExtraOpts = struct { gottp: ?u32 = null, tlsdesc: ?u32 = null, zig_got: ?u32 = null, + zig_offset_table: ?u32 = null, }; pub fn addExtra(symbol: *Symbol, opts: AddExtraOpts, elf_file: *Elf) void { @@ -467,6 +468,9 @@ pub const Flags = packed struct { /// Whether the symbol is a merge subsection. merge_subsection: bool = false, + + /// Whether the symbol has __zig_offset_table indirection. + zig_offset_table: bool = false, }; pub const Extra = struct { @@ -481,6 +485,7 @@ pub const Extra = struct { tlsdesc: u32 = 0, zig_got: u32 = 0, merge_section: u32 = 0, + zig_offset_table: u32 = 0, }; pub const Index = u32; diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index 671dc285e3..e652afe727 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -55,7 +55,10 @@ debug_str_section_zig_size: u64 = 0, debug_aranges_section_zig_size: u64 = 0, debug_line_section_zig_size: u64 = 0, -func_offset_table: ?OffsetTable = null, +/// Function offset table containing pointers to Zig generated functions. +/// The table is used for Zig's incremental compilation and is embedded with +/// the machine code section. +offset_table: ?OffsetTable = null, pub const global_symbol_bit: u32 = 0x80000000; pub const symbol_mask: u32 = 0x7fffffff; @@ -131,7 +134,7 @@ pub fn deinit(self: *ZigObject, allocator: Allocator) void { dw.deinit(); } - if (self.func_offset_table) |*ot| { + if (self.offset_table) |*ot| { ot.deinit(allocator); } } @@ -1040,12 +1043,24 @@ pub fn updateFunc( const ip = &zcu.intern_pool; const gpa = elf_file.base.comp.gpa; const func = zcu.funcInfo(func_index); + const offset_table = self.offsetTablePtr() orelse try self.initOffsetTable(gpa, elf_file); log.debug("updateFunc {}({d})", .{ ip.getNav(func.owner_nav).fqn.fmt(ip), func.owner_nav }); const sym_index = try self.getOrCreateMetadataForNav(elf_file, func.owner_nav); self.symbol(sym_index).atom(elf_file).?.freeRelocs(elf_file); + { + const sym = self.symbol(sym_index); + if (!sym.flags.zig_offset_table) { + const index = try offset_table.addSymbol(gpa, sym_index); + sym.flags.zig_offset_table = true; + sym.addExtra(.{ .zig_offset_table = index }, elf_file); + try offset_table.updateSize(self, elf_file); + try self.symbol(offset_table.sym_index).atom(elf_file).?.allocate(elf_file); + } + } + var code_buffer = std.ArrayList(u8).init(gpa); defer code_buffer.deinit(); @@ -1446,6 +1461,27 @@ pub fn getGlobalSymbol(self: *ZigObject, elf_file: *Elf, name: []const u8, lib_n return lookup_gop.value_ptr.*; } +fn offsetTablePtr(self: *ZigObject) ?*OffsetTable { + return if (self.offset_table) |*ot| ot else null; +} + +fn initOffsetTable(self: *ZigObject, allocator: Allocator, elf_file: *Elf) error{OutOfMemory}!*OffsetTable { + const name_off = try self.addString(allocator, "__zig_offset_table"); + const sym_index = try self.newSymbolWithAtom(allocator, name_off); + const sym = self.symbol(sym_index); + const esym = &self.symtab.items(.elf_sym)[sym.esym_index]; + esym.st_info |= elf.STT_OBJECT; + const atom_ptr = sym.atom(elf_file).?; + atom_ptr.alive = true; + atom_ptr.alignment = Atom.Alignment.fromNonzeroByteUnits(switch (elf_file.ptr_width) { + .p32 => 4, + .p64 => 8, + }); + atom_ptr.output_section_index = elf_file.zig_text_section_index.?; + self.offset_table = OffsetTable{ .sym_index = sym_index }; + return &(self.offset_table.?); +} + pub fn asFile(self: *ZigObject) File { return .{ .zig_object = self }; } @@ -1694,18 +1730,39 @@ const LazySymbolTable = std.AutoArrayHashMapUnmanaged(InternPool.Index, LazySymb const TlsTable = std.AutoArrayHashMapUnmanaged(Atom.Index, TlsVariable); pub const OffsetTable = struct { - atom_index: Atom.Index, + sym_index: Symbol.Index, entries: std.ArrayListUnmanaged(Symbol.Index) = .{}, pub fn deinit(ot: *OffsetTable, allocator: Allocator) void { ot.entries.deinit(allocator); } - pub fn size(ot: OffsetTable, elf_file: *Elf) usize { - return ot.entries.items.len * switch (elf_file.ptr_width) { - .p32 => @as(usize, 4), + pub fn addSymbol(ot: *OffsetTable, allocator: Allocator, sym_index: Symbol.Index) !Index { + const index: Index = @intCast(ot.entries.items.len); + try ot.entries.append(allocator, sym_index); + return index; + } + + pub fn address(ot: OffsetTable, zo: *ZigObject, elf_file: *Elf) i64 { + const sym = zo.symbol(ot.sym_index); + return sym.address(.{}, elf_file); + } + + pub fn size(ot: OffsetTable, zo: *ZigObject, elf_file: *Elf) u64 { + const sym = zo.symbol(ot.sym_index); + return sym.atom(elf_file).?.size; + } + + pub fn updateSize(ot: OffsetTable, zo: *ZigObject, elf_file: *Elf) !void { + const ot_size: u64 = @intCast(ot.entries.items.len * switch (elf_file.ptr_width) { + .p32 => @as(u64, 4), .p64 => 8, - }; + }); + const sym = zo.symbol(ot.sym_index); + const esym = &zo.symtab.items(.elf_sym)[sym.esym_index]; + esym.st_size = ot_size; + const atom_ptr = sym.atom(elf_file).?; + atom_ptr.size = ot_size; } const OffsetTableFormatContext = struct { OffsetTable, *ZigObject, *Elf }; @@ -1736,13 +1793,15 @@ pub const OffsetTable = struct { _ = options; _ = unused_fmt_string; const ot, const zo, const ef = ctx; - const atom_ptr = zo.atom(ot.atom_index).?; - try writer.print("@{x} : size({x})\n", .{ atom_ptr.address(ef), ot.size(ef) }); + try writer.writeAll("offset table\n"); + try writer.print(" @{x} : size({x})\n", .{ ot.address(zo, ef), ot.size(zo, ef) }); for (ot.entries.items) |sym_index| { const sym = zo.symbol(sym_index); - try writer.print(" %{d} : {s} : @{x}\n", .{ sym_index, sym.name(ef), sym.value }); + try writer.print(" %{d} : {s} : @{x}\n", .{ sym_index, sym.name(ef), sym.address(.{}, ef) }); } } + + pub const Index = u32; }; const assert = std.debug.assert; From 24b915c9f218d48fa32bf0e8c0a02d274be5ca21 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 8 Aug 2024 21:40:17 +0200 Subject: [PATCH 03/39] elf: write offset table entry if dirty --- src/link/Elf/ZigObject.zig | 97 ++++++++++++++++++++++++++++++++++---- 1 file changed, 88 insertions(+), 9 deletions(-) diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index e652afe727..17e926417e 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -1101,6 +1101,17 @@ pub fn updateFunc( } // Exports will be updated by `Zcu.processExports` after the update. + + { + const sym = self.symbol(sym_index); + const ot_index = sym.extra(elf_file).zig_offset_table; + var ot_entry = offset_table.entries.get(ot_index); + if (ot_entry.dirty) { + try offset_table.writeEntry(ot_index, self, elf_file); + ot_entry.dirty = false; + } + offset_table.entries.set(ot_index, ot_entry); + } } pub fn updateNav( @@ -1731,15 +1742,15 @@ const TlsTable = std.AutoArrayHashMapUnmanaged(Atom.Index, TlsVariable); pub const OffsetTable = struct { sym_index: Symbol.Index, - entries: std.ArrayListUnmanaged(Symbol.Index) = .{}, + entries: std.MultiArrayList(Entry) = .{}, pub fn deinit(ot: *OffsetTable, allocator: Allocator) void { ot.entries.deinit(allocator); } pub fn addSymbol(ot: *OffsetTable, allocator: Allocator, sym_index: Symbol.Index) !Index { - const index: Index = @intCast(ot.entries.items.len); - try ot.entries.append(allocator, sym_index); + const index: Index = @intCast(try ot.entries.addOne(allocator)); + ot.entries.set(index, .{ .sym_index = sym_index }); return index; } @@ -1753,11 +1764,72 @@ pub const OffsetTable = struct { return sym.atom(elf_file).?.size; } + pub fn entryAddress(ot: OffsetTable, index: Index, zo: *ZigObject, elf_file: *Elf) i64 { + return ot.address(zo, elf_file) + index * elf_file.archPtrWidthBytes(); + } + + pub fn entryOffset(ot: OffsetTable, index: Index, zo: *ZigObject, elf_file: *Elf) u64 { + const sym = zo.symbol(ot.sym_index); + const atom_ptr = sym.atom(elf_file).?; + const shdr = elf_file.shdrs.items[atom_ptr.output_section_index]; + return shdr.sh_offset + @as(u64, @intCast(atom_ptr.value)) + index * elf_file.archPtrWidthBytes(); + } + + pub fn targetAddress(ot: OffsetTable, index: Index, zo: *ZigObject, elf_file: *Elf) i64 { + const sym_index = ot.entries.items(.sym_index)[index]; + return zo.symbol(sym_index).address(.{}, elf_file); + } + + pub fn writeEntry(ot: OffsetTable, index: Index, zo: *ZigObject, elf_file: *Elf) !void { + const entry_size: u16 = elf_file.archPtrWidthBytes(); + const target = elf_file.getTarget(); + const endian = target.cpu.arch.endian(); + const fileoff = ot.entryOffset(index, zo, elf_file); + const vaddr: u64 = @intCast(ot.entryAddress(index, zo, elf_file)); + const value = ot.targetAddress(index, zo, elf_file); + switch (entry_size) { + 2 => { + var buf: [2]u8 = undefined; + std.mem.writeInt(u16, &buf, @intCast(value), endian); + try elf_file.base.file.?.pwriteAll(&buf, fileoff); + }, + 4 => { + var buf: [4]u8 = undefined; + std.mem.writeInt(u32, &buf, @intCast(value), endian); + try elf_file.base.file.?.pwriteAll(&buf, fileoff); + }, + 8 => { + var buf: [8]u8 = undefined; + std.mem.writeInt(u64, &buf, @intCast(value), endian); + try elf_file.base.file.?.pwriteAll(&buf, fileoff); + + if (elf_file.base.child_pid) |pid| { + switch (builtin.os.tag) { + .linux => { + var local_vec: [1]std.posix.iovec_const = .{.{ + .base = &buf, + .len = buf.len, + }}; + var remote_vec: [1]std.posix.iovec_const = .{.{ + .base = @as([*]u8, @ptrFromInt(@as(usize, @intCast(vaddr)))), + .len = buf.len, + }}; + const rc = std.os.linux.process_vm_writev(pid, &local_vec, &remote_vec, 0); + switch (std.os.linux.E.init(rc)) { + .SUCCESS => assert(rc == buf.len), + else => |errno| log.warn("process_vm_writev failure: {s}", .{@tagName(errno)}), + } + }, + else => return error.HotSwapUnavailableOnHostOperatingSystem, + } + } + }, + else => unreachable, + } + } + pub fn updateSize(ot: OffsetTable, zo: *ZigObject, elf_file: *Elf) !void { - const ot_size: u64 = @intCast(ot.entries.items.len * switch (elf_file.ptr_width) { - .p32 => @as(u64, 4), - .p64 => 8, - }); + const ot_size: u64 = @intCast(ot.entries.items(.sym_index).len * elf_file.archPtrWidthBytes()); const sym = zo.symbol(ot.sym_index); const esym = &zo.symtab.items(.elf_sym)[sym.esym_index]; esym.st_size = ot_size; @@ -1795,12 +1867,19 @@ pub const OffsetTable = struct { const ot, const zo, const ef = ctx; try writer.writeAll("offset table\n"); try writer.print(" @{x} : size({x})\n", .{ ot.address(zo, ef), ot.size(zo, ef) }); - for (ot.entries.items) |sym_index| { + for (ot.entries.items(.sym_index), ot.entries.items(.dirty)) |sym_index, dirty| { const sym = zo.symbol(sym_index); - try writer.print(" %{d} : {s} : @{x}\n", .{ sym_index, sym.name(ef), sym.address(.{}, ef) }); + try writer.print(" %{d} : {s} : @{x}", .{ sym_index, sym.name(ef), sym.address(.{}, ef) }); + if (dirty) try writer.writeAll(" : [!]"); + try writer.writeByte('\n'); } } + const Entry = struct { + sym_index: Symbol.Index, + dirty: bool = true, + }; + pub const Index = u32; }; From 97a65ea0d5275602fac39bb08ab23f2908f8845e Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 8 Aug 2024 21:46:19 +0200 Subject: [PATCH 04/39] elf: dirty offset table entry on moving Atom in off/addr space --- src/link/Elf/ZigObject.zig | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index 17e926417e..75a0fd2cbe 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -924,6 +924,10 @@ fn updateNavCode( assert(sym.flags.has_zig_got); const extra = sym.extra(elf_file); try elf_file.zig_got.writeOne(elf_file, extra.zig_got); + if (stt_bits == elf.STT_FUNC) { + const offset_table = self.offsetTablePtr().?; + offset_table.entries.items(.dirty)[extra.zig_offset_table] = true; + } } } } else if (code.len < old_size) { From 7556b32840eabb027945d9a30b57bb9cb46cf0bb Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 8 Aug 2024 23:21:52 +0200 Subject: [PATCH 05/39] elf: indirect via offset table in the linker away from backend --- src/link/Elf/Atom.zig | 12 +++++--- src/link/Elf/Symbol.zig | 8 ++++++ src/link/Elf/ZigObject.zig | 56 ++++++++++++++++++++++++++------------ 3 files changed, 54 insertions(+), 22 deletions(-) diff --git a/src/link/Elf/Atom.zig b/src/link/Elf/Atom.zig index f2757f570b..6fa5d700ef 100644 --- a/src/link/Elf/Atom.zig +++ b/src/link/Elf/Atom.zig @@ -759,13 +759,14 @@ pub fn resolveRelocsAlloc(self: Atom, elf_file: *Elf, code: []u8) RelocError!voi // Address of the dynamic thread pointer. const DTP = elf_file.dtpAddress(); - relocs_log.debug(" {s}: {x}: [{x} => {x}] G({x}) ZG({x}) ({s})", .{ + relocs_log.debug(" {s}: {x}: [{x} => {x}] G({x}) ZG({x}) ZG2({x}) ({s})", .{ relocation.fmtRelocType(rel.r_type(), cpu_arch), r_offset, P, S + A, G + GOT + A, ZIG_GOT + A, + target.zigOffsetTableAddress(elf_file) + A, target.name(elf_file), }); @@ -1224,9 +1225,12 @@ const x86_64 = struct { ); }, - .PLT32, - .PC32, - => try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little), + .PLT32 => try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little), + + .PC32 => { + const S_ = if (target.flags.zig_offset_table) target.zigOffsetTableAddress(elf_file) else S; + try cwriter.writeInt(i32, @as(i32, @intCast(S_ + A - P)), .little); + }, .GOTPCREL => try cwriter.writeInt(i32, @as(i32, @intCast(G + GOT + A - P)), .little), .GOTPC32 => try cwriter.writeInt(i32, @as(i32, @intCast(GOT + A - P)), .little), diff --git a/src/link/Elf/Symbol.zig b/src/link/Elf/Symbol.zig index f023ad5d91..cf285b12d0 100644 --- a/src/link/Elf/Symbol.zig +++ b/src/link/Elf/Symbol.zig @@ -236,6 +236,14 @@ pub fn zigGotAddress(symbol: Symbol, elf_file: *Elf) i64 { return elf_file.zig_got.entryAddress(extras.zig_got, elf_file); } +pub fn zigOffsetTableAddress(symbol: Symbol, elf_file: *Elf) i64 { + if (!symbol.flags.zig_offset_table) return 0; + const zo = elf_file.zigObjectPtr().?; + const offset_table = zo.offsetTablePtr().?; + const ot_index = symbol.extra(elf_file).zig_offset_table; + return offset_table.entryAddress(ot_index, zo, elf_file); +} + pub fn dsoAlignment(symbol: Symbol, elf_file: *Elf) !u64 { const file_ptr = symbol.file(elf_file) orelse return 0; assert(file_ptr == .shared_object); diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index 75a0fd2cbe..f426e43c0a 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -760,7 +760,9 @@ pub fn getOrCreateMetadataForLazySymbol( const gpa = elf_file.base.comp.gpa; const symbol_index = try self.newSymbolWithAtom(gpa, 0); const sym = self.symbol(symbol_index); - sym.flags.needs_zig_got = true; + if (lazy_sym.kind != .code) { + sym.flags.needs_zig_got = true; + } symbol_index_ptr.* = symbol_index; }, .pending_flush => return symbol_index_ptr.*, @@ -816,7 +818,7 @@ pub fn getOrCreateMetadataForNav( sym.flags.is_tls = true; } } - if (!sym.flags.is_tls) { + if (!sym.flags.is_tls and nav_val.typeOf(zcu).zigTypeTag(zcu) != .Fn) { sym.flags.needs_zig_got = true; } gop.value_ptr.* = .{ .symbol_index = symbol_index }; @@ -919,16 +921,19 @@ fn updateNavCode( sym.value = 0; esym.st_value = 0; - if (!elf_file.base.isRelocatable()) { - log.debug(" (writing new offset table entry)", .{}); - assert(sym.flags.has_zig_got); - const extra = sym.extra(elf_file); - try elf_file.zig_got.writeOne(elf_file, extra.zig_got); - if (stt_bits == elf.STT_FUNC) { - const offset_table = self.offsetTablePtr().?; - offset_table.entries.items(.dirty)[extra.zig_offset_table] = true; + if (stt_bits != elf.STT_FUNC) { + if (!elf_file.base.isRelocatable()) { + log.debug(" (writing new offset table entry)", .{}); + assert(sym.flags.has_zig_got); + const extra = sym.extra(elf_file); + try elf_file.zig_got.writeOne(elf_file, extra.zig_got); } } + if (stt_bits == elf.STT_FUNC) { + const extra = sym.extra(elf_file); + const offset_table = self.offsetTablePtr().?; + offset_table.entries.items(.dirty)[extra.zig_offset_table] = true; + } } } else if (code.len < old_size) { atom_ptr.shrink(elf_file); @@ -938,12 +943,13 @@ fn updateNavCode( errdefer self.freeNavMetadata(elf_file, sym_index); sym.value = 0; - sym.flags.needs_zig_got = true; esym.st_value = 0; - - if (!elf_file.base.isRelocatable()) { - const gop = try sym.getOrCreateZigGotEntry(sym_index, elf_file); - try elf_file.zig_got.writeOne(elf_file, gop.index); + if (stt_bits != elf.STT_FUNC) { + sym.flags.needs_zig_got = true; + if (!elf_file.base.isRelocatable()) { + const gop = try sym.getOrCreateZigGotEntry(sym_index, elf_file); + try elf_file.zig_got.writeOne(elf_file, gop.index); + } } } @@ -1061,7 +1067,12 @@ pub fn updateFunc( sym.flags.zig_offset_table = true; sym.addExtra(.{ .zig_offset_table = index }, elf_file); try offset_table.updateSize(self, elf_file); + const old_vaddr = offset_table.address(self, elf_file); try self.symbol(offset_table.sym_index).atom(elf_file).?.allocate(elf_file); + const new_vaddr = offset_table.address(self, elf_file); + if (old_vaddr != new_vaddr) { + offset_table.dirty = true; + } } } @@ -1106,7 +1117,13 @@ pub fn updateFunc( // Exports will be updated by `Zcu.processExports` after the update. - { + if (offset_table.dirty) { + // TODO write in bulk + for (offset_table.entries.items(.dirty), 0..) |*dirty, i| { + try offset_table.writeEntry(@intCast(i), self, elf_file); + dirty.* = false; + } + } else { const sym = self.symbol(sym_index); const ot_index = sym.extra(elf_file).zig_offset_table; var ot_entry = offset_table.entries.get(ot_index); @@ -1261,7 +1278,9 @@ fn updateLazySymbol( errdefer self.freeNavMetadata(elf_file, symbol_index); local_sym.value = 0; - local_sym.flags.needs_zig_got = true; + if (sym.kind != .code) { + local_sym.flags.needs_zig_got = true; + } local_esym.st_value = 0; if (!elf_file.base.isRelocatable()) { @@ -1476,7 +1495,7 @@ pub fn getGlobalSymbol(self: *ZigObject, elf_file: *Elf, name: []const u8, lib_n return lookup_gop.value_ptr.*; } -fn offsetTablePtr(self: *ZigObject) ?*OffsetTable { +pub fn offsetTablePtr(self: *ZigObject) ?*OffsetTable { return if (self.offset_table) |*ot| ot else null; } @@ -1747,6 +1766,7 @@ const TlsTable = std.AutoArrayHashMapUnmanaged(Atom.Index, TlsVariable); pub const OffsetTable = struct { sym_index: Symbol.Index, entries: std.MultiArrayList(Entry) = .{}, + dirty: bool = false, pub fn deinit(ot: *OffsetTable, allocator: Allocator) void { ot.entries.deinit(allocator); From 4c2b34e8abd1fc2091eeb10798658bd9bb3910f5 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Fri, 9 Aug 2024 08:35:46 +0200 Subject: [PATCH 06/39] elf: relax R_X86_64_32 into jump table indirection if zig_offset_table set --- src/link/Elf/Atom.zig | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/link/Elf/Atom.zig b/src/link/Elf/Atom.zig index 6fa5d700ef..b5db642c40 100644 --- a/src/link/Elf/Atom.zig +++ b/src/link/Elf/Atom.zig @@ -1254,7 +1254,10 @@ const x86_64 = struct { try cwriter.writeInt(i32, @as(i32, @intCast(G + GOT + A - P)), .little); }, - .@"32" => try cwriter.writeInt(u32, @as(u32, @truncate(@as(u64, @intCast(S + A)))), .little), + .@"32" => { + const S_ = if (target.flags.zig_offset_table) target.zigOffsetTableAddress(elf_file) else S; + try cwriter.writeInt(u32, @as(u32, @truncate(@as(u64, @intCast(S_ + A)))), .little); + }, .@"32S" => try cwriter.writeInt(i32, @as(i32, @truncate(S + A)), .little), .TPOFF32 => try cwriter.writeInt(i32, @as(i32, @truncate(S + A - TP)), .little), From d328140858ab7f0f3eeb5d53b50bae32ab331686 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Fri, 9 Aug 2024 09:10:59 +0200 Subject: [PATCH 07/39] elf: nuke ZigGotSection from existence --- src/arch/x86_64/Emit.zig | 45 ++---- src/arch/x86_64/Lower.zig | 22 +-- src/link/Elf.zig | 78 +---------- src/link/Elf/Atom.zig | 49 ++----- src/link/Elf/Symbol.zig | 25 ---- src/link/Elf/ZigObject.zig | 36 +---- src/link/Elf/relocation.zig | 4 - src/link/Elf/synthetic_sections.zig | 209 ---------------------------- 8 files changed, 30 insertions(+), 438 deletions(-) diff --git a/src/arch/x86_64/Emit.zig b/src/arch/x86_64/Emit.zig index b6c1fbb968..b7d0a29c2d 100644 --- a/src/arch/x86_64/Emit.zig +++ b/src/arch/x86_64/Emit.zig @@ -110,21 +110,11 @@ pub fn emitMir(emit: *Emit) Error!void { }); }, .linker_reloc => |data| if (emit.lower.bin_file.cast(.elf)) |elf_file| { - const is_obj_or_static_lib = switch (emit.lower.output_mode) { - .Exe => false, - .Obj => true, - .Lib => emit.lower.link_mode == .static, - }; const zo = elf_file.zigObjectPtr().?; const atom = zo.symbol(data.atom_index).atom(elf_file).?; const sym = zo.symbol(data.sym_index); - if (sym.flags.needs_zig_got and !is_obj_or_static_lib) { - _ = try sym.getOrCreateZigGotEntry(data.sym_index, elf_file); - } if (emit.lower.pic) { - const r_type: u32 = if (sym.flags.needs_zig_got and !is_obj_or_static_lib) - link.File.Elf.R_ZIG_GOTPCREL - else if (sym.flags.needs_got) + const r_type: u32 = if (sym.flags.needs_got) @intFromEnum(std.elf.R_X86_64.GOTPCREL) else @intFromEnum(std.elf.R_X86_64.PC32); @@ -134,28 +124,17 @@ pub fn emitMir(emit: *Emit) Error!void { .r_addend = -4, }); } else { - if (lowered_inst.encoding.mnemonic == .call and sym.flags.needs_zig_got and is_obj_or_static_lib) { - const r_type = @intFromEnum(std.elf.R_X86_64.PC32); - try atom.addReloc(elf_file, .{ - .r_offset = end_offset - 4, - .r_info = (@as(u64, @intCast(data.sym_index)) << 32) | r_type, - .r_addend = -4, - }); - } else { - const r_type: u32 = if (sym.flags.needs_zig_got and !is_obj_or_static_lib) - link.File.Elf.R_ZIG_GOT32 - else if (sym.flags.needs_got) - @intFromEnum(std.elf.R_X86_64.GOT32) - else if (sym.flags.is_tls) - @intFromEnum(std.elf.R_X86_64.TPOFF32) - else - @intFromEnum(std.elf.R_X86_64.@"32"); - try atom.addReloc(elf_file, .{ - .r_offset = end_offset - 4, - .r_info = (@as(u64, @intCast(data.sym_index)) << 32) | r_type, - .r_addend = 0, - }); - } + const r_type: u32 = if (sym.flags.needs_got) + @intFromEnum(std.elf.R_X86_64.GOT32) + else if (sym.flags.is_tls) + @intFromEnum(std.elf.R_X86_64.TPOFF32) + else + @intFromEnum(std.elf.R_X86_64.@"32"); + try atom.addReloc(elf_file, .{ + .r_offset = end_offset - 4, + .r_info = (@as(u64, @intCast(data.sym_index)) << 32) | r_type, + .r_addend = 0, + }); } } else if (emit.lower.bin_file.cast(.macho)) |macho_file| { const is_obj_or_static_lib = switch (emit.lower.output_mode) { diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig index 11fe279dd9..9ad1579ecc 100644 --- a/src/arch/x86_64/Lower.zig +++ b/src/arch/x86_64/Lower.zig @@ -398,30 +398,20 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) _ = lower.reloc(.{ .linker_reloc = sym }); break :op if (lower.pic) switch (mnemonic) { - .lea => { - break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }; - }, - .mov => { - if (is_obj_or_static_lib and elf_sym.flags.needs_zig_got) emit_mnemonic = .lea; - break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }; - }, + .lea => break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }, + .mov => break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }, else => unreachable, } else switch (mnemonic) { - .call => break :op if (is_obj_or_static_lib and elf_sym.flags.needs_zig_got) .{ - .imm = Immediate.s(0), - } else .{ .mem = Memory.sib(mem_op.sib.ptr_size, .{ + .call => break :op .{ .mem = Memory.sib(mem_op.sib.ptr_size, .{ .base = .{ .reg = .ds }, }) }, .lea => { emit_mnemonic = .mov; break :op .{ .imm = Immediate.s(0) }; }, - .mov => { - if (is_obj_or_static_lib and elf_sym.flags.needs_zig_got) emit_mnemonic = .lea; - break :op .{ .mem = Memory.sib(mem_op.sib.ptr_size, .{ - .base = .{ .reg = .ds }, - }) }; - }, + .mov => break :op .{ .mem = Memory.sib(mem_op.sib.ptr_size, .{ + .base = .{ .reg = .ds }, + }) }, else => unreachable, }; } else if (lower.bin_file.cast(.macho)) |macho_file| { diff --git a/src/link/Elf.zig b/src/link/Elf.zig index ffba511e87..ebafd16d6d 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -64,9 +64,6 @@ phdrs: std.ArrayListUnmanaged(elf.Elf64_Phdr) = .{}, /// Tracked loadable segments during incremental linking. /// The index into the program headers of a PT_LOAD program header with Read and Execute flags phdr_zig_load_re_index: ?u16 = null, -/// The index into the program headers of the global offset table. -/// It needs PT_LOAD and Read flags. -phdr_zig_got_index: ?u16 = null, /// The index into the program headers of a PT_LOAD program header with Read flag phdr_zig_load_ro_index: ?u16 = null, /// The index into the program headers of a PT_LOAD program header with Write flag @@ -130,8 +127,6 @@ plt_got: PltGotSection = .{}, copy_rel: CopyRelSection = .{}, /// .rela.plt section rela_plt: std.ArrayListUnmanaged(elf.Elf64_Rela) = .{}, -/// .got.zig section -zig_got: ZigGotSection = .{}, /// SHT_GROUP sections /// Applies only to a relocatable. comdat_group_sections: std.ArrayListUnmanaged(ComdatGroupSection) = .{}, @@ -142,7 +137,6 @@ zig_text_section_index: ?u32 = null, zig_data_rel_ro_section_index: ?u32 = null, zig_data_section_index: ?u32 = null, zig_bss_section_index: ?u32 = null, -zig_got_section_index: ?u32 = null, debug_info_section_index: ?u32 = null, debug_abbrev_section_index: ?u32 = null, @@ -474,7 +468,6 @@ pub fn deinit(self: *Elf) void { self.copy_rel.deinit(gpa); self.rela_dyn.deinit(gpa); self.rela_plt.deinit(gpa); - self.zig_got.deinit(gpa); self.comdat_group_sections.deinit(gpa); } @@ -618,21 +611,6 @@ pub fn initMetadata(self: *Elf, options: InitMetadataOptions) !void { }); } - if (self.phdr_zig_got_index == null) { - const alignment = self.page_size; - const filesz = @as(u64, ptr_size) * options.symbol_count_hint; - const off = self.findFreeSpace(filesz, alignment); - self.phdr_zig_got_index = try self.addPhdr(.{ - .type = elf.PT_LOAD, - .offset = off, - .filesz = filesz, - .addr = if (ptr_bit_width >= 32) 0x4000000 else 0x4000, - .memsz = filesz, - .@"align" = alignment, - .flags = elf.PF_R | elf.PF_W, - }); - } - if (self.phdr_zig_load_ro_index == null) { const alignment = self.page_size; const filesz: u64 = 1024; @@ -701,27 +679,6 @@ pub fn initMetadata(self: *Elf, options: InitMetadataOptions) !void { try self.last_atom_and_free_list_table.putNoClobber(gpa, self.zig_text_section_index.?, .{}); } - if (self.zig_got_section_index == null and !self.base.isRelocatable()) { - self.zig_got_section_index = try self.addSection(.{ - .name = try self.insertShString(".got.zig"), - .type = elf.SHT_PROGBITS, - .addralign = ptr_size, - .flags = elf.SHF_ALLOC | elf.SHF_WRITE, - .offset = std.math.maxInt(u64), - }); - const shdr = &self.shdrs.items[self.zig_got_section_index.?]; - const phndx = self.phdr_zig_got_index.?; - const phdr = self.phdrs.items[phndx]; - shdr.sh_addr = phdr.p_vaddr; - shdr.sh_offset = phdr.p_offset; - shdr.sh_size = phdr.p_memsz; - try self.phdr_to_shdr_table.putNoClobber( - gpa, - self.zig_got_section_index.?, - self.phdr_zig_got_index.?, - ); - } - if (self.zig_data_rel_ro_section_index == null) { self.zig_data_rel_ro_section_index = try self.addSection(.{ .name = try self.insertShString(".data.rel.ro.zig"), @@ -3156,8 +3113,8 @@ fn initSyntheticSections(self: *Elf) !void { }); const needs_rela_dyn = blk: { - if (self.got.flags.needs_rela or self.got.flags.needs_tlsld or - self.zig_got.flags.needs_rela or self.copy_rel.symbols.items.len > 0) break :blk true; + if (self.got.flags.needs_rela or self.got.flags.needs_tlsld or self.copy_rel.symbols.items.len > 0) + break :blk true; if (self.zigObjectPtr()) |zig_object| { if (zig_object.num_dynrelocs > 0) break :blk true; } @@ -3562,7 +3519,6 @@ fn sortPhdrs(self: *Elf) error{OutOfMemory}!void { for (&[_]*?u16{ &self.phdr_zig_load_re_index, - &self.phdr_zig_got_index, &self.phdr_zig_load_ro_index, &self.phdr_zig_load_zerofill_index, &self.phdr_table_index, @@ -3694,7 +3650,6 @@ fn resetShdrIndexes(self: *Elf, backlinks: []const u32) !void { &self.versym_section_index, &self.verneed_section_index, &self.zig_text_section_index, - &self.zig_got_section_index, &self.zig_data_rel_ro_section_index, &self.zig_data_section_index, &self.zig_bss_section_index, @@ -3893,7 +3848,7 @@ fn updateSectionSizes(self: *Elf) !void { } if (self.rela_dyn_section_index) |shndx| { - var num = self.got.numRela(self) + self.copy_rel.numRela() + self.zig_got.numRela(); + var num = self.got.numRela(self) + self.copy_rel.numRela(); if (self.zigObjectPtr()) |zig_object| { num += zig_object.num_dynrelocs; } @@ -4431,15 +4386,6 @@ pub fn updateSymtabSize(self: *Elf) !void { strsize += ctx.strsize; } - if (self.zigObjectPtr()) |_| { - if (self.zig_got_section_index) |_| { - self.zig_got.output_symtab_ctx.ilocal = nlocals + 1; - self.zig_got.updateSymtabSize(self); - nlocals += self.zig_got.output_symtab_ctx.nlocals; - strsize += self.zig_got.output_symtab_ctx.strsize; - } - } - if (self.got_section_index) |_| { self.got.output_symtab_ctx.ilocal = nlocals + 1; self.got.updateSymtabSize(self); @@ -4576,9 +4522,6 @@ fn writeSyntheticSections(self: *Elf) !void { const shdr = self.shdrs.items[shndx]; try self.got.addRela(self); try self.copy_rel.addRela(self); - if (self.zigObjectPtr()) |_| { - try self.zig_got.addRela(self); - } self.sortRelaDyn(); try self.base.file.?.pwriteAll(mem.sliceAsBytes(self.rela_dyn.items), shdr.sh_offset); } @@ -4674,10 +4617,6 @@ pub fn writeSymtab(self: *Elf) !void { obj.asFile().writeSymtab(self); } - if (self.zig_got_section_index) |_| { - self.zig_got.writeSymtab(self); - } - if (self.got_section_index) |_| { self.got.writeSymtab(self); } @@ -5085,7 +5024,6 @@ pub fn isZigSection(self: Elf, shndx: u32) bool { self.zig_data_rel_ro_section_index, self.zig_data_section_index, self.zig_bss_section_index, - self.zig_got_section_index, }) |maybe_index| { if (maybe_index) |index| { if (index == shndx) return true; @@ -5704,7 +5642,6 @@ fn fmtDumpState( } } - try writer.print("{}\n", .{self.zig_got.fmt(self)}); try writer.print("{}\n", .{self.got.fmt(self)}); try writer.print("{}\n", .{self.plt.fmt(self)}); @@ -5995,20 +5932,12 @@ const RelaSection = struct { }; const RelaSectionTable = std.AutoArrayHashMapUnmanaged(u32, RelaSection); -pub const R_ZIG_GOT32: u32 = 0xff00; -pub const R_ZIG_GOTPCREL: u32 = 0xff01; -pub const R_ZIG_GOT_HI20: u32 = 0xff02; -pub const R_ZIG_GOT_LO12: u32 = 0xff03; pub const R_GOT_HI20_STATIC: u32 = 0xff04; pub const R_GOT_LO12_I_STATIC: u32 = 0xff05; // Comptime asserts that no Zig relocs overlap with another ISA's reloc number comptime { const zig_relocs = .{ - R_ZIG_GOT32, - R_ZIG_GOT_HI20, - R_ZIG_GOT_LO12, - R_ZIG_GOTPCREL, R_GOT_HI20_STATIC, R_GOT_LO12_I_STATIC, }; @@ -6099,6 +6028,5 @@ const StringTable = @import("StringTable.zig"); const Thunk = thunks.Thunk; const Value = @import("../Value.zig"); const VerneedSection = synthetic_sections.VerneedSection; -const ZigGotSection = synthetic_sections.ZigGotSection; const ZigObject = @import("Elf/ZigObject.zig"); const riscv = @import("riscv.zig"); diff --git a/src/link/Elf/Atom.zig b/src/link/Elf/Atom.zig index b5db642c40..cb505b19e3 100644 --- a/src/link/Elf/Atom.zig +++ b/src/link/Elf/Atom.zig @@ -750,8 +750,8 @@ pub fn resolveRelocsAlloc(self: Atom, elf_file: *Elf, code: []u8) RelocError!voi const S = target.address(.{}, elf_file); // Address of the global offset table. const GOT = elf_file.gotAddress(); - // Address of the .zig.got table entry if any. - const ZIG_GOT = target.zigGotAddress(elf_file); + // Address of the offset table entry if any. + const ZIG_GOT = target.zigOffsetTableAddress(elf_file); // Relative offset to the start of the global offset table. const G = target.gotAddress(elf_file) - GOT; // // Address of the thread pointer. @@ -759,14 +759,13 @@ pub fn resolveRelocsAlloc(self: Atom, elf_file: *Elf, code: []u8) RelocError!voi // Address of the dynamic thread pointer. const DTP = elf_file.dtpAddress(); - relocs_log.debug(" {s}: {x}: [{x} => {x}] G({x}) ZG({x}) ZG2({x}) ({s})", .{ + relocs_log.debug(" {s}: {x}: [{x} => {x}] G({x}) ZG({x}) ({s})", .{ relocation.fmtRelocType(rel.r_type(), cpu_arch), r_offset, P, S + A, G + GOT + A, ZIG_GOT + A, - target.zigOffsetTableAddress(elf_file) + A, target.name(elf_file), }); @@ -1181,16 +1180,7 @@ const x86_64 = struct { .TLSDESC_CALL, => {}, - else => |x| switch (@intFromEnum(x)) { - // Zig custom relocations - Elf.R_ZIG_GOT32, - Elf.R_ZIG_GOTPCREL, - => { - assert(symbol.flags.has_zig_got); - }, - - else => try atom.reportUnhandledRelocError(rel, elf_file), - }, + else => try atom.reportUnhandledRelocError(rel, elf_file), } } @@ -1228,7 +1218,7 @@ const x86_64 = struct { .PLT32 => try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little), .PC32 => { - const S_ = if (target.flags.zig_offset_table) target.zigOffsetTableAddress(elf_file) else S; + const S_ = if (target.flags.zig_offset_table) ZIG_GOT else S; try cwriter.writeInt(i32, @as(i32, @intCast(S_ + A - P)), .little); }, @@ -1255,7 +1245,7 @@ const x86_64 = struct { }, .@"32" => { - const S_ = if (target.flags.zig_offset_table) target.zigOffsetTableAddress(elf_file) else S; + const S_ = if (target.flags.zig_offset_table) ZIG_GOT else S; try cwriter.writeInt(u32, @as(u32, @truncate(@as(u64, @intCast(S_ + A)))), .little); }, .@"32S" => try cwriter.writeInt(i32, @as(i32, @truncate(S + A)), .little), @@ -1336,13 +1326,7 @@ const x86_64 = struct { .GOT32 => try cwriter.writeInt(i32, @as(i32, @intCast(G + GOT + A)), .little), - else => |x| switch (@intFromEnum(x)) { - // Zig custom relocations - Elf.R_ZIG_GOT32 => try cwriter.writeInt(u32, @as(u32, @intCast(ZIG_GOT + A)), .little), - Elf.R_ZIG_GOTPCREL => try cwriter.writeInt(i32, @as(i32, @intCast(ZIG_GOT + A - P)), .little), - - else => try atom.reportUnhandledRelocError(rel, elf_file), - }, + else => try atom.reportUnhandledRelocError(rel, elf_file), } } @@ -2006,12 +1990,6 @@ const riscv = struct { => {}, else => |x| switch (@intFromEnum(x)) { - Elf.R_ZIG_GOT_HI20, - Elf.R_ZIG_GOT_LO12, - => { - assert(symbol.flags.has_zig_got); - }, - Elf.R_GOT_HI20_STATIC, Elf.R_GOT_LO12_I_STATIC, => symbol.flags.needs_got = true, @@ -2038,6 +2016,7 @@ const riscv = struct { const P, const A, const S, const GOT, const G, const TP, const DTP, const ZIG_GOT = args; _ = TP; _ = DTP; + _ = ZIG_GOT; switch (r_type) { .NONE => unreachable, @@ -2156,18 +2135,6 @@ const riscv = struct { else => |x| switch (@intFromEnum(x)) { // Zig custom relocations - Elf.R_ZIG_GOT_HI20 => { - assert(target.flags.has_zig_got); - const disp: u32 = @bitCast(math.cast(i32, ZIG_GOT + A) orelse return error.Overflow); - riscv_util.writeInstU(code[r_offset..][0..4], disp); - }, - - Elf.R_ZIG_GOT_LO12 => { - assert(target.flags.has_zig_got); - const value: u32 = @bitCast(math.cast(i32, ZIG_GOT + A) orelse return error.Overflow); - riscv_util.writeInstI(code[r_offset..][0..4], value); - }, - Elf.R_GOT_HI20_STATIC => { assert(target.flags.has_got); const disp: u32 = @bitCast(math.cast(i32, G + GOT + A) orelse return error.Overflow); diff --git a/src/link/Elf/Symbol.zig b/src/link/Elf/Symbol.zig index cf285b12d0..20697a0426 100644 --- a/src/link/Elf/Symbol.zig +++ b/src/link/Elf/Symbol.zig @@ -217,25 +217,6 @@ pub fn tlsDescAddress(symbol: Symbol, elf_file: *Elf) i64 { return entry.address(elf_file); } -const GetOrCreateZigGotEntryResult = struct { - found_existing: bool, - index: ZigGotSection.Index, -}; - -pub fn getOrCreateZigGotEntry(symbol: *Symbol, symbol_index: Index, elf_file: *Elf) !GetOrCreateZigGotEntryResult { - assert(!elf_file.base.isRelocatable()); - assert(symbol.flags.needs_zig_got); - if (symbol.flags.has_zig_got) return .{ .found_existing = true, .index = symbol.extra(elf_file).zig_got }; - const index = try elf_file.zig_got.addSymbol(symbol_index, elf_file); - return .{ .found_existing = false, .index = index }; -} - -pub fn zigGotAddress(symbol: Symbol, elf_file: *Elf) i64 { - if (!symbol.flags.has_zig_got) return 0; - const extras = symbol.extra(elf_file); - return elf_file.zig_got.entryAddress(extras.zig_got, elf_file); -} - pub fn zigOffsetTableAddress(symbol: Symbol, elf_file: *Elf) i64 { if (!symbol.flags.zig_offset_table) return 0; const zo = elf_file.zigObjectPtr().?; @@ -267,7 +248,6 @@ const AddExtraOpts = struct { tlsgd: ?u32 = null, gottp: ?u32 = null, tlsdesc: ?u32 = null, - zig_got: ?u32 = null, zig_offset_table: ?u32 = null, }; @@ -465,10 +445,6 @@ pub const Flags = packed struct { needs_tlsdesc: bool = false, has_tlsdesc: bool = false, - /// Whether the symbol contains .zig.got indirection. - needs_zig_got: bool = false, - has_zig_got: bool = false, - /// Whether the symbol is a TLS variable. /// TODO this is really not needed if only we operated on esyms between /// codegen and ZigObject. @@ -491,7 +467,6 @@ pub const Extra = struct { tlsgd: u32 = 0, gottp: u32 = 0, tlsdesc: u32 = 0, - zig_got: u32 = 0, merge_section: u32 = 0, zig_offset_table: u32 = 0, }; diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index f426e43c0a..107ab78385 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -756,15 +756,7 @@ pub fn getOrCreateMetadataForLazySymbol( .const_data => .{ &gop.value_ptr.rodata_symbol_index, &gop.value_ptr.rodata_state }, }; switch (state_ptr.*) { - .unused => { - const gpa = elf_file.base.comp.gpa; - const symbol_index = try self.newSymbolWithAtom(gpa, 0); - const sym = self.symbol(symbol_index); - if (lazy_sym.kind != .code) { - sym.flags.needs_zig_got = true; - } - symbol_index_ptr.* = symbol_index; - }, + .unused => symbol_index_ptr.* = try self.newSymbolWithAtom(pt.zcu.gpa, 0), .pending_flush => return symbol_index_ptr.*, .flushed => {}, } @@ -818,9 +810,6 @@ pub fn getOrCreateMetadataForNav( sym.flags.is_tls = true; } } - if (!sym.flags.is_tls and nav_val.typeOf(zcu).zigTypeTag(zcu) != .Fn) { - sym.flags.needs_zig_got = true; - } gop.value_ptr.* = .{ .symbol_index = symbol_index }; } return gop.value_ptr.symbol_index; @@ -921,14 +910,6 @@ fn updateNavCode( sym.value = 0; esym.st_value = 0; - if (stt_bits != elf.STT_FUNC) { - if (!elf_file.base.isRelocatable()) { - log.debug(" (writing new offset table entry)", .{}); - assert(sym.flags.has_zig_got); - const extra = sym.extra(elf_file); - try elf_file.zig_got.writeOne(elf_file, extra.zig_got); - } - } if (stt_bits == elf.STT_FUNC) { const extra = sym.extra(elf_file); const offset_table = self.offsetTablePtr().?; @@ -944,13 +925,6 @@ fn updateNavCode( sym.value = 0; esym.st_value = 0; - if (stt_bits != elf.STT_FUNC) { - sym.flags.needs_zig_got = true; - if (!elf_file.base.isRelocatable()) { - const gop = try sym.getOrCreateZigGotEntry(sym_index, elf_file); - try elf_file.zig_got.writeOne(elf_file, gop.index); - } - } } if (elf_file.base.child_pid) |pid| { @@ -1278,16 +1252,8 @@ fn updateLazySymbol( errdefer self.freeNavMetadata(elf_file, symbol_index); local_sym.value = 0; - if (sym.kind != .code) { - local_sym.flags.needs_zig_got = true; - } local_esym.st_value = 0; - if (!elf_file.base.isRelocatable()) { - const gop = try local_sym.getOrCreateZigGotEntry(symbol_index, elf_file); - try elf_file.zig_got.writeOne(elf_file, gop.index); - } - const shdr = elf_file.shdrs.items[output_section_index]; const file_offset = shdr.sh_offset + @as(u64, @intCast(atom_ptr.value)); try elf_file.base.file.?.pwriteAll(code, file_offset); diff --git a/src/link/Elf/relocation.zig b/src/link/Elf/relocation.zig index 5f6810d6f9..887aece8bc 100644 --- a/src/link/Elf/relocation.zig +++ b/src/link/Elf/relocation.zig @@ -113,10 +113,6 @@ fn formatRelocType( _ = options; const r_type = ctx.r_type; switch (r_type) { - Elf.R_ZIG_GOT32 => try writer.writeAll("R_ZIG_GOT32"), - Elf.R_ZIG_GOTPCREL => try writer.writeAll("R_ZIG_GOTPCREL"), - Elf.R_ZIG_GOT_HI20 => try writer.writeAll("R_ZIG_GOT_HI20"), - Elf.R_ZIG_GOT_LO12 => try writer.writeAll("R_ZIG_GOT_LO12"), Elf.R_GOT_HI20_STATIC => try writer.writeAll("R_GOT_HI20_STATIC"), Elf.R_GOT_LO12_I_STATIC => try writer.writeAll("R_GOT_LO12_I_STATIC"), else => switch (ctx.cpu_arch) { diff --git a/src/link/Elf/synthetic_sections.zig b/src/link/Elf/synthetic_sections.zig index e1ec90139e..40cb37b967 100644 --- a/src/link/Elf/synthetic_sections.zig +++ b/src/link/Elf/synthetic_sections.zig @@ -223,215 +223,6 @@ pub const DynamicSection = struct { } }; -pub const ZigGotSection = struct { - entries: std.ArrayListUnmanaged(Symbol.Index) = .{}, - output_symtab_ctx: Elf.SymtabCtx = .{}, - flags: Flags = .{}, - - const Flags = packed struct { - needs_rela: bool = false, - dirty: bool = false, - }; - - pub const Index = u32; - - pub fn deinit(zig_got: *ZigGotSection, allocator: Allocator) void { - zig_got.entries.deinit(allocator); - } - - fn allocateEntry(zig_got: *ZigGotSection, allocator: Allocator) !Index { - try zig_got.entries.ensureUnusedCapacity(allocator, 1); - // TODO add free list - const index = @as(Index, @intCast(zig_got.entries.items.len)); - _ = zig_got.entries.addOneAssumeCapacity(); - zig_got.flags.dirty = true; - return index; - } - - pub fn addSymbol(zig_got: *ZigGotSection, sym_index: Symbol.Index, elf_file: *Elf) !Index { - const comp = elf_file.base.comp; - const gpa = comp.gpa; - const zo = elf_file.zigObjectPtr().?; - const index = try zig_got.allocateEntry(gpa); - const entry = &zig_got.entries.items[index]; - entry.* = sym_index; - const symbol = zo.symbol(sym_index); - symbol.flags.has_zig_got = true; - if (elf_file.isEffectivelyDynLib() or (elf_file.base.isExe() and comp.config.pie)) { - zig_got.flags.needs_rela = true; - } - symbol.addExtra(.{ .zig_got = index }, elf_file); - return index; - } - - pub fn entryOffset(zig_got: ZigGotSection, index: Index, elf_file: *Elf) u64 { - _ = zig_got; - const entry_size = elf_file.archPtrWidthBytes(); - const shdr = elf_file.shdrs.items[elf_file.zig_got_section_index.?]; - return shdr.sh_offset + @as(u64, entry_size) * index; - } - - pub fn entryAddress(zig_got: ZigGotSection, index: Index, elf_file: *Elf) i64 { - _ = zig_got; - const entry_size = elf_file.archPtrWidthBytes(); - const shdr = elf_file.shdrs.items[elf_file.zig_got_section_index.?]; - return @as(i64, @intCast(shdr.sh_addr)) + entry_size * index; - } - - pub fn size(zig_got: ZigGotSection, elf_file: *Elf) usize { - return elf_file.archPtrWidthBytes() * zig_got.entries.items.len; - } - - pub fn writeOne(zig_got: *ZigGotSection, elf_file: *Elf, index: Index) !void { - const zo = elf_file.zigObjectPtr().?; - if (zig_got.flags.dirty) { - const needed_size = zig_got.size(elf_file); - try elf_file.growAllocSection(elf_file.zig_got_section_index.?, needed_size); - zig_got.flags.dirty = false; - } - const entry_size: u16 = elf_file.archPtrWidthBytes(); - const target = elf_file.getTarget(); - const endian = target.cpu.arch.endian(); - const off = zig_got.entryOffset(index, elf_file); - const vaddr: u64 = @intCast(zig_got.entryAddress(index, elf_file)); - const entry = zig_got.entries.items[index]; - const value = zo.symbol(entry).address(.{}, elf_file); - switch (entry_size) { - 2 => { - var buf: [2]u8 = undefined; - std.mem.writeInt(u16, &buf, @intCast(value), endian); - try elf_file.base.file.?.pwriteAll(&buf, off); - }, - 4 => { - var buf: [4]u8 = undefined; - std.mem.writeInt(u32, &buf, @intCast(value), endian); - try elf_file.base.file.?.pwriteAll(&buf, off); - }, - 8 => { - var buf: [8]u8 = undefined; - std.mem.writeInt(u64, &buf, @intCast(value), endian); - try elf_file.base.file.?.pwriteAll(&buf, off); - - if (elf_file.base.child_pid) |pid| { - switch (builtin.os.tag) { - .linux => { - var local_vec: [1]std.posix.iovec_const = .{.{ - .base = &buf, - .len = buf.len, - }}; - var remote_vec: [1]std.posix.iovec_const = .{.{ - .base = @as([*]u8, @ptrFromInt(@as(usize, @intCast(vaddr)))), - .len = buf.len, - }}; - const rc = std.os.linux.process_vm_writev(pid, &local_vec, &remote_vec, 0); - switch (std.os.linux.E.init(rc)) { - .SUCCESS => assert(rc == buf.len), - else => |errno| log.warn("process_vm_writev failure: {s}", .{@tagName(errno)}), - } - }, - else => return error.HotSwapUnavailableOnHostOperatingSystem, - } - } - }, - else => unreachable, - } - } - - pub fn writeAll(zig_got: ZigGotSection, elf_file: *Elf, writer: anytype) !void { - const zo = elf_file.zigObjectPtr().?; - for (zig_got.entries.items) |entry| { - const symbol = zo.symbol(entry); - const value = symbol.address(.{ .plt = false }, elf_file); - try writeInt(value, elf_file, writer); - } - } - - pub fn numRela(zig_got: ZigGotSection) usize { - return zig_got.entries.items.len; - } - - pub fn addRela(zig_got: ZigGotSection, elf_file: *Elf) !void { - const comp = elf_file.base.comp; - const gpa = comp.gpa; - const cpu_arch = elf_file.getTarget().cpu.arch; - const zo = elf_file.zigObjectPtr().?; - try elf_file.rela_dyn.ensureUnusedCapacity(gpa, zig_got.numRela()); - for (zig_got.entries.items) |entry| { - const symbol = zo.symbol(entry); - const offset = symbol.zigGotAddress(elf_file); - elf_file.addRelaDynAssumeCapacity(.{ - .offset = @intCast(offset), - .type = relocation.encode(.rel, cpu_arch), - .addend = symbol.address(.{ .plt = false }, elf_file), - }); - } - } - - pub fn updateSymtabSize(zig_got: *ZigGotSection, elf_file: *Elf) void { - const zo = elf_file.zigObjectPtr().?; - zig_got.output_symtab_ctx.nlocals = @as(u32, @intCast(zig_got.entries.items.len)); - for (zig_got.entries.items) |entry| { - const name = zo.symbol(entry).name(elf_file); - zig_got.output_symtab_ctx.strsize += @as(u32, @intCast(name.len + "$ziggot".len)) + 1; - } - } - - pub fn writeSymtab(zig_got: ZigGotSection, elf_file: *Elf) void { - const zo = elf_file.zigObjectPtr().?; - for (zig_got.entries.items, zig_got.output_symtab_ctx.ilocal.., 0..) |entry, ilocal, index| { - const symbol = zo.symbol(entry); - const symbol_name = symbol.name(elf_file); - const st_name = @as(u32, @intCast(elf_file.strtab.items.len)); - elf_file.strtab.appendSliceAssumeCapacity(symbol_name); - elf_file.strtab.appendSliceAssumeCapacity("$ziggot"); - elf_file.strtab.appendAssumeCapacity(0); - const st_value = zig_got.entryAddress(@intCast(index), elf_file); - const st_size = elf_file.archPtrWidthBytes(); - elf_file.symtab.items[ilocal] = .{ - .st_name = st_name, - .st_info = elf.STT_OBJECT, - .st_other = 0, - .st_shndx = @intCast(elf_file.zig_got_section_index.?), - .st_value = @intCast(st_value), - .st_size = st_size, - }; - } - } - - const FormatCtx = struct { - zig_got: ZigGotSection, - elf_file: *Elf, - }; - - pub fn fmt(zig_got: ZigGotSection, elf_file: *Elf) std.fmt.Formatter(format2) { - return .{ .data = .{ .zig_got = zig_got, .elf_file = elf_file } }; - } - - pub fn format2( - ctx: FormatCtx, - comptime unused_fmt_string: []const u8, - options: std.fmt.FormatOptions, - writer: anytype, - ) !void { - _ = options; - _ = unused_fmt_string; - const zig_got = ctx.zig_got; - const elf_file = ctx.elf_file; - try writer.writeAll(".zig.got\n"); - for (zig_got.entries.items, 0..) |entry, index| { - const zo = elf_file.zigObjectPtr().?; - const symbol = zo.symbol(entry); - try writer.print(" {d}@0x{x} => {d}@0x{x} ({s})\n", .{ - index, - zig_got.entryAddress(@intCast(index), elf_file), - entry, - symbol.address(.{}, elf_file), - symbol.name(elf_file), - }); - } - } -}; - pub const GotSection = struct { entries: std.ArrayListUnmanaged(Entry) = .{}, output_symtab_ctx: Elf.SymtabCtx = .{}, From d7c5fbce92b0da1bb302db4c4adab21775f3d98b Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Mon, 12 Aug 2024 12:34:06 +0200 Subject: [PATCH 08/39] elf: emit a jump table in place of offset table for functions --- src/link/Elf/ZigObject.zig | 113 +++++++++++++++++++++---------------- 1 file changed, 63 insertions(+), 50 deletions(-) diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index 107ab78385..f289e0ad19 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -1473,10 +1473,7 @@ fn initOffsetTable(self: *ZigObject, allocator: Allocator, elf_file: *Elf) error esym.st_info |= elf.STT_OBJECT; const atom_ptr = sym.atom(elf_file).?; atom_ptr.alive = true; - atom_ptr.alignment = Atom.Alignment.fromNonzeroByteUnits(switch (elf_file.ptr_width) { - .p32 => 4, - .p64 => 8, - }); + atom_ptr.alignment = Atom.Alignment.fromNonzeroByteUnits(OffsetTable.alignment(elf_file.getTarget().cpu.arch)); atom_ptr.output_section_index = elf_file.zig_text_section_index.?; self.offset_table = OffsetTable{ .sym_index = sym_index }; return &(self.offset_table.?); @@ -1754,15 +1751,31 @@ pub const OffsetTable = struct { return sym.atom(elf_file).?.size; } + pub fn alignment(cpu_arch: std.Target.Cpu.Arch) u64 { + return switch (cpu_arch) { + .x86_64 => 1, + else => @panic("TODO implement alignment for this CPU arch"), + }; + } + pub fn entryAddress(ot: OffsetTable, index: Index, zo: *ZigObject, elf_file: *Elf) i64 { - return ot.address(zo, elf_file) + index * elf_file.archPtrWidthBytes(); + return ot.address(zo, elf_file) + @as(i64, @intCast(index * entrySize(elf_file.getTarget().cpu.arch))); } pub fn entryOffset(ot: OffsetTable, index: Index, zo: *ZigObject, elf_file: *Elf) u64 { const sym = zo.symbol(ot.sym_index); const atom_ptr = sym.atom(elf_file).?; const shdr = elf_file.shdrs.items[atom_ptr.output_section_index]; - return shdr.sh_offset + @as(u64, @intCast(atom_ptr.value)) + index * elf_file.archPtrWidthBytes(); + return shdr.sh_offset + @as(u64, @intCast(atom_ptr.value)) + index * entrySize(elf_file.getTarget().cpu.arch); + } + + pub fn entrySize(cpu_arch: std.Target.Cpu.Arch) u64 { + const seq_len = switch (cpu_arch) { + .x86_64 => 5, // jmp rel32 + else => @panic("TODO implement entry size for this CPU arch"), + }; + comptime assert(seq_len <= max_jump_seq_len); + return seq_len; } pub fn targetAddress(ot: OffsetTable, index: Index, zo: *ZigObject, elf_file: *Elf) i64 { @@ -1770,56 +1783,43 @@ pub const OffsetTable = struct { return zo.symbol(sym_index).address(.{}, elf_file); } - pub fn writeEntry(ot: OffsetTable, index: Index, zo: *ZigObject, elf_file: *Elf) !void { - const entry_size: u16 = elf_file.archPtrWidthBytes(); - const target = elf_file.getTarget(); - const endian = target.cpu.arch.endian(); - const fileoff = ot.entryOffset(index, zo, elf_file); - const vaddr: u64 = @intCast(ot.entryAddress(index, zo, elf_file)); - const value = ot.targetAddress(index, zo, elf_file); - switch (entry_size) { - 2 => { - var buf: [2]u8 = undefined; - std.mem.writeInt(u16, &buf, @intCast(value), endian); - try elf_file.base.file.?.pwriteAll(&buf, fileoff); - }, - 4 => { - var buf: [4]u8 = undefined; - std.mem.writeInt(u32, &buf, @intCast(value), endian); - try elf_file.base.file.?.pwriteAll(&buf, fileoff); - }, - 8 => { - var buf: [8]u8 = undefined; - std.mem.writeInt(u64, &buf, @intCast(value), endian); - try elf_file.base.file.?.pwriteAll(&buf, fileoff); + const max_jump_seq_len = 12; - if (elf_file.base.child_pid) |pid| { - switch (builtin.os.tag) { - .linux => { - var local_vec: [1]std.posix.iovec_const = .{.{ - .base = &buf, - .len = buf.len, - }}; - var remote_vec: [1]std.posix.iovec_const = .{.{ - .base = @as([*]u8, @ptrFromInt(@as(usize, @intCast(vaddr)))), - .len = buf.len, - }}; - const rc = std.os.linux.process_vm_writev(pid, &local_vec, &remote_vec, 0); - switch (std.os.linux.E.init(rc)) { - .SUCCESS => assert(rc == buf.len), - else => |errno| log.warn("process_vm_writev failure: {s}", .{@tagName(errno)}), - } - }, - else => return error.HotSwapUnavailableOnHostOperatingSystem, + pub fn writeEntry(ot: OffsetTable, index: Index, zo: *ZigObject, elf_file: *Elf) !void { + const fileoff = ot.entryOffset(index, zo, elf_file); + const source_addr = ot.entryAddress(index, zo, elf_file); + const target_addr = @as(i64, @intCast(ot.targetAddress(index, zo, elf_file))); + var buf: [max_jump_seq_len]u8 = undefined; + const out = switch (elf_file.getTarget().cpu.arch) { + .x86_64 => try x86_64.writeEntry(source_addr, target_addr, &buf), + else => @panic("TODO implement write entry for this CPU arch"), + }; + try elf_file.base.file.?.pwriteAll(out, fileoff); + + if (elf_file.base.child_pid) |pid| { + switch (builtin.os.tag) { + .linux => { + var local_vec: [1]std.posix.iovec_const = .{.{ + .base = out.ptr, + .len = out.len, + }}; + var remote_vec: [1]std.posix.iovec_const = .{.{ + .base = @as([*]u8, @ptrFromInt(@as(usize, @intCast(source_addr)))), + .len = out.len, + }}; + const rc = std.os.linux.process_vm_writev(pid, &local_vec, &remote_vec, 0); + switch (std.os.linux.E.init(rc)) { + .SUCCESS => assert(rc == out.len), + else => |errno| log.warn("process_vm_writev failure: {s}", .{@tagName(errno)}), } - } - }, - else => unreachable, + }, + else => return error.HotSwapUnavailableOnHostOperatingSystem, + } } } pub fn updateSize(ot: OffsetTable, zo: *ZigObject, elf_file: *Elf) !void { - const ot_size: u64 = @intCast(ot.entries.items(.sym_index).len * elf_file.archPtrWidthBytes()); + const ot_size: u64 = @intCast(ot.entries.items(.sym_index).len * entrySize(elf_file.getTarget().cpu.arch)); const sym = zo.symbol(ot.sym_index); const esym = &zo.symtab.items(.elf_sym)[sym.esym_index]; esym.st_size = ot_size; @@ -1871,6 +1871,19 @@ pub const OffsetTable = struct { }; pub const Index = u32; + + const x86_64 = struct { + fn writeEntry(source_addr: i64, target_addr: i64, buf: *[max_jump_seq_len]u8) ![]u8 { + const disp = @as(i64, @intCast(target_addr)) - source_addr - 4; + var bytes = [_]u8{ + 0xe8, 0x00, 0x00, 0x00, 0x00, // jmp rel32 + }; + assert(bytes.len == entrySize(.x86_64)); + mem.writeInt(i32, bytes[1..][0..4], @intCast(disp), .little); + @memcpy(buf[0..bytes.len], &bytes); + return buf[0..bytes.len]; + } + }; }; const assert = std.debug.assert; From 5fd53dc36fb019a0930a9be0700411bb04c688a1 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Mon, 12 Aug 2024 17:40:41 +0200 Subject: [PATCH 09/39] x86_64: start converting away from .got.zig knowledge --- src/arch/x86_64/CodeGen.zig | 48 +++++++++++++++++++++---------------- src/arch/x86_64/Lower.zig | 1 + 2 files changed, 29 insertions(+), 20 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index a36bd981bc..cfeca4c390 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -12311,25 +12311,33 @@ fn genCall(self: *Self, info: union(enum) { const zo = elf_file.zigObjectPtr().?; const sym_index = try zo.getOrCreateMetadataForNav(elf_file, func.owner_nav); if (self.mod.pic) { - const callee_reg: Register = switch (resolved_cc) { - .SysV => callee: { - if (!fn_info.is_var_args) break :callee .rax; - const param_regs = abi.getCAbiIntParamRegs(resolved_cc); - break :callee if (call_info.gp_count < param_regs.len) - param_regs[call_info.gp_count] - else - .r10; - }, - .Win64 => .rax, - else => unreachable, - }; - try self.genSetReg( - callee_reg, - Type.usize, - .{ .load_symbol = .{ .sym = sym_index } }, - .{}, - ); - try self.asmRegister(.{ ._, .call }, callee_reg); + // const callee_reg: Register = switch (resolved_cc) { + // .SysV => callee: { + // if (!fn_info.is_var_args) break :callee .rax; + // const param_regs = abi.getCAbiIntParamRegs(resolved_cc); + // break :callee if (call_info.gp_count < param_regs.len) + // param_regs[call_info.gp_count] + // else + // .r10; + // }, + // .Win64 => .rax, + // else => unreachable, + // }; + // TODO convert to near jump + try self.asmMemory(.{ ._, .call }, .{ + .base = .{ .reloc = .{ + .atom_index = try self.owner.getSymbolIndex(self), + .sym_index = sym_index, + } }, + .mod = .{ .rm = .{ .size = .qword } }, + }); + // try self.genSetReg( + // callee_reg, + // Type.usize, + // .{ .load_symbol = .{ .sym = sym_index } }, + // .{}, + // ); + // try self.asmRegister(.{ ._, .call }, callee_reg); } else try self.asmMemory(.{ ._, .call }, .{ .base = .{ .reloc = .{ .atom_index = try self.owner.getSymbolIndex(self), @@ -14097,7 +14105,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { else return self.fail("invalid modifier: '{s}'", .{modifier}), .load_symbol => |sym_off| if (mem.eql(u8, modifier, "P")) - .{ .reg = try self.copyToTmpRegister(Type.usize, .{ .load_symbol = sym_off }) } + .{ .reg = try self.copyToTmpRegister(Type.usize, .{ .lea_symbol = sym_off }) } else return self.fail("invalid modifier: '{s}'", .{modifier}), else => return self.fail("invalid constraint: '{s}'", .{op_str}), diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig index 9ad1579ecc..2cb874f003 100644 --- a/src/arch/x86_64/Lower.zig +++ b/src/arch/x86_64/Lower.zig @@ -398,6 +398,7 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) _ = lower.reloc(.{ .linker_reloc = sym }); break :op if (lower.pic) switch (mnemonic) { + .call => break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }, .lea => break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }, .mov => break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }, else => unreachable, From f968dd0cb1fc9197fc6483ce235b037f6c86c8ce Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Mon, 12 Aug 2024 21:57:14 +0200 Subject: [PATCH 10/39] comp: actually report codegen errors --- src/Compilation.zig | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/Compilation.zig b/src/Compilation.zig index d7dabd5164..0beb960d02 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -3092,6 +3092,10 @@ pub fn totalErrorCount(comp: *Compilation) u32 { if (zcu.intern_pool.global_error_set.getNamesFromMainThread().len > zcu.error_limit) { total += 1; } + + for (zcu.failed_codegen.keys()) |_| { + total += 1; + } } // The "no entry point found" error only counts if there are no semantic analysis errors. @@ -3237,6 +3241,9 @@ pub fn getAllErrorsAlloc(comp: *Compilation) !ErrorBundle { } } } + for (zcu.failed_codegen.values()) |error_msg| { + try addModuleErrorMsg(zcu, &bundle, error_msg.*, &all_references); + } for (zcu.failed_exports.values()) |value| { try addModuleErrorMsg(zcu, &bundle, value.*, &all_references); } From 16abf51ceef2ba2ebb38f27270af676387435f05 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Mon, 12 Aug 2024 22:04:18 +0200 Subject: [PATCH 11/39] x86_64: handle lea_symbol returned by genNavRef --- src/arch/x86_64/CodeGen.zig | 3 ++- src/codegen.zig | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index cfeca4c390..dede09fc71 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -14104,7 +14104,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { .{ .reg = try self.copyToTmpRegister(Type.usize, .{ .lea_got = sym_index }) } else return self.fail("invalid modifier: '{s}'", .{modifier}), - .load_symbol => |sym_off| if (mem.eql(u8, modifier, "P")) + .lea_symbol => |sym_off| if (mem.eql(u8, modifier, "P")) .{ .reg = try self.copyToTmpRegister(Type.usize, .{ .lea_symbol = sym_off }) } else return self.fail("invalid modifier: '{s}'", .{modifier}), @@ -18798,6 +18798,7 @@ fn genTypedValue(self: *Self, val: Value) InnerError!MCValue { .immediate => |imm| .{ .immediate = imm }, .memory => |addr| .{ .memory = addr }, .load_symbol => |sym_index| .{ .load_symbol = .{ .sym = sym_index } }, + .lea_symbol => |sym_index| .{ .lea_symbol = .{ .sym = sym_index } }, .load_direct => |sym_index| .{ .load_direct = sym_index }, .load_got => |sym_index| .{ .lea_got = sym_index }, .load_tlv => |sym_index| .{ .lea_tlv = sym_index }, diff --git a/src/codegen.zig b/src/codegen.zig index f2fa60fdf8..679ee03c27 100644 --- a/src/codegen.zig +++ b/src/codegen.zig @@ -828,6 +828,9 @@ pub const GenResult = union(enum) { /// Reference to memory location but deferred until linker allocated the Decl in memory. /// Traditionally, this corresponds to emitting a relocation in a relocatable object file. load_symbol: u32, + /// Reference to memory location but deferred until linker allocated the Decl in memory. + /// Traditionally, this corresponds to emitting a relocation in a relocatable object file. + lea_symbol: u32, }; fn mcv(val: MCValue) GenResult { @@ -904,7 +907,7 @@ fn genNavRef( if (!single_threaded and is_threadlocal) { return GenResult.mcv(.{ .load_tlv = sym_index }); } - return GenResult.mcv(.{ .load_symbol = sym_index }); + return GenResult.mcv(.{ .lea_symbol = sym_index }); } else if (lf.cast(.macho)) |macho_file| { const zo = macho_file.getZigObject().?; if (is_extern) { From e3f6ebaea94bb11c2e990e1fa17b48e51e598b5c Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Mon, 12 Aug 2024 22:51:33 +0200 Subject: [PATCH 12/39] x86_64+elf: fix jump table indirection for functions --- src/arch/x86_64/CodeGen.zig | 46 +++++++++++++++---------------------- src/link/Elf/ZigObject.zig | 26 ++++++++------------- 2 files changed, 29 insertions(+), 43 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index dede09fc71..f6fbc67ff1 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -12311,33 +12311,25 @@ fn genCall(self: *Self, info: union(enum) { const zo = elf_file.zigObjectPtr().?; const sym_index = try zo.getOrCreateMetadataForNav(elf_file, func.owner_nav); if (self.mod.pic) { - // const callee_reg: Register = switch (resolved_cc) { - // .SysV => callee: { - // if (!fn_info.is_var_args) break :callee .rax; - // const param_regs = abi.getCAbiIntParamRegs(resolved_cc); - // break :callee if (call_info.gp_count < param_regs.len) - // param_regs[call_info.gp_count] - // else - // .r10; - // }, - // .Win64 => .rax, - // else => unreachable, - // }; - // TODO convert to near jump - try self.asmMemory(.{ ._, .call }, .{ - .base = .{ .reloc = .{ - .atom_index = try self.owner.getSymbolIndex(self), - .sym_index = sym_index, - } }, - .mod = .{ .rm = .{ .size = .qword } }, - }); - // try self.genSetReg( - // callee_reg, - // Type.usize, - // .{ .load_symbol = .{ .sym = sym_index } }, - // .{}, - // ); - // try self.asmRegister(.{ ._, .call }, callee_reg); + const callee_reg: Register = switch (resolved_cc) { + .SysV => callee: { + if (!fn_info.is_var_args) break :callee .rax; + const param_regs = abi.getCAbiIntParamRegs(resolved_cc); + break :callee if (call_info.gp_count < param_regs.len) + param_regs[call_info.gp_count] + else + .r10; + }, + .Win64 => .rax, + else => unreachable, + }; + try self.genSetReg( + callee_reg, + Type.usize, + .{ .lea_symbol = .{ .sym = sym_index } }, + .{}, + ); + try self.asmRegister(.{ ._, .call }, callee_reg); } else try self.asmMemory(.{ ._, .call }, .{ .base = .{ .reloc = .{ .atom_index = try self.owner.getSymbolIndex(self), diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index f289e0ad19..0cb47d7473 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -107,23 +107,17 @@ pub fn deinit(self: *ZigObject, allocator: Allocator) void { } self.relocs.deinit(allocator); - { - var it = self.navs.iterator(); - while (it.next()) |entry| { - entry.value_ptr.exports.deinit(allocator); - } - self.navs.deinit(allocator); + for (self.navs.values()) |*meta| { + meta.exports.deinit(allocator); } + self.navs.deinit(allocator); self.lazy_syms.deinit(allocator); - { - var it = self.uavs.iterator(); - while (it.next()) |entry| { - entry.value_ptr.exports.deinit(allocator); - } - self.uavs.deinit(allocator); + for (self.uavs.values()) |*meta| { + meta.exports.deinit(allocator); } + self.uavs.deinit(allocator); for (self.tls_variables.values()) |*tlv| { tlv.deinit(allocator); @@ -1721,8 +1715,8 @@ const TlsVariable = struct { }; const AtomList = std.ArrayListUnmanaged(Atom.Index); -const NavTable = std.AutoHashMapUnmanaged(InternPool.Nav.Index, AvMetadata); -const UavTable = std.AutoHashMapUnmanaged(InternPool.Index, AvMetadata); +const NavTable = std.AutoArrayHashMapUnmanaged(InternPool.Nav.Index, AvMetadata); +const UavTable = std.AutoArrayHashMapUnmanaged(InternPool.Index, AvMetadata); const LazySymbolTable = std.AutoArrayHashMapUnmanaged(InternPool.Index, LazySymbolMetadata); const TlsTable = std.AutoArrayHashMapUnmanaged(Atom.Index, TlsVariable); @@ -1874,9 +1868,9 @@ pub const OffsetTable = struct { const x86_64 = struct { fn writeEntry(source_addr: i64, target_addr: i64, buf: *[max_jump_seq_len]u8) ![]u8 { - const disp = @as(i64, @intCast(target_addr)) - source_addr - 4; + const disp = @as(i64, @intCast(target_addr)) - source_addr - 5; var bytes = [_]u8{ - 0xe8, 0x00, 0x00, 0x00, 0x00, // jmp rel32 + 0xe9, 0x00, 0x00, 0x00, 0x00, // jmp rel32 }; assert(bytes.len == entrySize(.x86_64)); mem.writeInt(i32, bytes[1..][0..4], @intCast(disp), .little); From e1ce9a7065e92e3e9b61229c7903f82c3684f489 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Mon, 12 Aug 2024 22:52:11 +0200 Subject: [PATCH 13/39] elf: add poorman's reporting tool for unallocated NAVs/UAVs --- src/link/Elf/ZigObject.zig | 46 +++++++++++++++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index 0cb47d7473..4415180597 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -164,6 +164,16 @@ pub fn flushModule(self: *ZigObject, elf_file: *Elf, tid: Zcu.PerThread.Id) !voi if (metadata.rodata_state != .unused) metadata.rodata_state = .flushed; } + if (build_options.enable_logging) { + const pt: Zcu.PerThread = .{ .zcu = elf_file.base.comp.module.?, .tid = tid }; + for (self.navs.keys(), self.navs.values()) |nav_index, meta| { + checkNavAllocated(pt, nav_index, meta); + } + for (self.uavs.keys(), self.uavs.values()) |uav_index, meta| { + checkUavAllocated(pt, uav_index, meta); + } + } + if (self.dwarf) |*dw| { const pt: Zcu.PerThread = .{ .zcu = elf_file.base.comp.module.?, .tid = tid }; try dw.flushModule(pt); @@ -701,6 +711,7 @@ pub fn lowerUav( else => explicit_alignment, }; if (self.uavs.get(uav)) |metadata| { + assert(metadata.allocated); const sym = self.symbol(metadata.symbol_index); const existing_alignment = sym.atom(elf_file).?.alignment; if (uav_alignment.order(existing_alignment).compare(.lte)) @@ -732,7 +743,7 @@ pub fn lowerUav( .ok => |sym_index| sym_index, .fail => |em| return .{ .fail = em }, }; - try self.uavs.put(gpa, uav, .{ .symbol_index = sym_index }); + try self.uavs.put(gpa, uav, .{ .symbol_index = sym_index, .allocated = true }); return .{ .mcv = .{ .load_symbol = sym_index } }; } @@ -921,6 +932,8 @@ fn updateNavCode( esym.st_value = 0; } + self.navs.getPtr(nav_index).?.allocated = true; + if (elf_file.base.child_pid) |pid| { switch (builtin.os.tag) { .linux => { @@ -988,6 +1001,8 @@ fn updateTlv( atom_ptr.alignment = required_alignment; atom_ptr.size = code.len; + self.navs.getPtr(nav_index).?.allocated = true; + { const gop = try self.tls_variables.getOrPut(gpa, atom_ptr.atom_index); assert(!gop.found_existing); // TODO incremental updates @@ -1695,6 +1710,8 @@ const AvMetadata = struct { symbol_index: Symbol.Index, /// A list of all exports aliases of this Av. exports: std.ArrayListUnmanaged(Symbol.Index) = .{}, + /// Set to true if the AV has been initialized and allocated. + allocated: bool = false, fn @"export"(m: AvMetadata, zig_object: *ZigObject, name: []const u8) ?*u32 { for (m.exports.items) |*exp| { @@ -1705,6 +1722,32 @@ const AvMetadata = struct { } }; +fn checkNavAllocated(pt: Zcu.PerThread, index: InternPool.Nav.Index, meta: AvMetadata) void { + if (!meta.allocated) { + const zcu = pt.zcu; + const ip = &zcu.intern_pool; + const nav = ip.getNav(index); + log.err("NAV {}({d}) assigned symbol {d} but not allocated!", .{ + nav.fqn.fmt(ip), + index, + meta.symbol_index, + }); + } +} + +fn checkUavAllocated(pt: Zcu.PerThread, index: InternPool.Index, meta: AvMetadata) void { + if (!meta.allocated) { + const zcu = pt.zcu; + const uav = Value.fromInterned(index); + const ty = uav.typeOf(zcu); + log.err("UAV {}({d}) assigned symbol {d} but not allocated!", .{ + ty.fmt(pt), + index, + meta.symbol_index, + }); + } +} + const TlsVariable = struct { symbol_index: Symbol.Index, code: []const u8 = &[0]u8{}, @@ -1881,6 +1924,7 @@ pub const OffsetTable = struct { }; const assert = std.debug.assert; +const build_options = @import("build_options"); const builtin = @import("builtin"); const codegen = @import("../../codegen.zig"); const elf = std.elf; From 78b1c736026443c937145ba85e1de52c11c9a44e Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Mon, 12 Aug 2024 23:23:39 +0200 Subject: [PATCH 14/39] elf: rename OffsetTable to JumpTable --- src/link/Elf.zig | 4 +- src/link/Elf/Atom.zig | 24 +++---- src/link/Elf/Symbol.zig | 18 ++--- src/link/Elf/ZigObject.zig | 136 ++++++++++++++++++------------------- 4 files changed, 91 insertions(+), 91 deletions(-) diff --git a/src/link/Elf.zig b/src/link/Elf.zig index ebafd16d6d..f4003ff38f 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -5599,8 +5599,8 @@ fn fmtDumpState( zig_object.fmtAtoms(self), zig_object.fmtSymtab(self), }); - if (zig_object.offset_table) |ot| { - try writer.print("{}", .{ot.fmt(zig_object, self)}); + if (zig_object.jump_table) |jt| { + try writer.print("{}", .{jt.fmt(zig_object, self)}); } try writer.writeByte('\n'); } diff --git a/src/link/Elf/Atom.zig b/src/link/Elf/Atom.zig index cb505b19e3..31ba0d642e 100644 --- a/src/link/Elf/Atom.zig +++ b/src/link/Elf/Atom.zig @@ -750,8 +750,8 @@ pub fn resolveRelocsAlloc(self: Atom, elf_file: *Elf, code: []u8) RelocError!voi const S = target.address(.{}, elf_file); // Address of the global offset table. const GOT = elf_file.gotAddress(); - // Address of the offset table entry if any. - const ZIG_GOT = target.zigOffsetTableAddress(elf_file); + // Address of the zig jump table entry if any. + const ZJT = target.zigJumpTableAddress(elf_file); // Relative offset to the start of the global offset table. const G = target.gotAddress(elf_file) - GOT; // // Address of the thread pointer. @@ -759,19 +759,19 @@ pub fn resolveRelocsAlloc(self: Atom, elf_file: *Elf, code: []u8) RelocError!voi // Address of the dynamic thread pointer. const DTP = elf_file.dtpAddress(); - relocs_log.debug(" {s}: {x}: [{x} => {x}] G({x}) ZG({x}) ({s})", .{ + relocs_log.debug(" {s}: {x}: [{x} => {x}] GOT({x}) ZJT({x}) ({s})", .{ relocation.fmtRelocType(rel.r_type(), cpu_arch), r_offset, P, S + A, G + GOT + A, - ZIG_GOT + A, + ZJT + A, target.name(elf_file), }); try stream.seekTo(r_offset); - const args = ResolveArgs{ P, A, S, GOT, G, TP, DTP, ZIG_GOT }; + const args = ResolveArgs{ P, A, S, GOT, G, TP, DTP, ZJT }; switch (cpu_arch) { .x86_64 => x86_64.resolveRelocAlloc(self, elf_file, rel, target, args, &it, code, &stream) catch |err| switch (err) { @@ -1200,7 +1200,7 @@ const x86_64 = struct { const cwriter = stream.writer(); - const P, const A, const S, const GOT, const G, const TP, const DTP, const ZIG_GOT = args; + const P, const A, const S, const GOT, const G, const TP, const DTP, const ZJT = args; switch (r_type) { .NONE => unreachable, @@ -1218,7 +1218,7 @@ const x86_64 = struct { .PLT32 => try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little), .PC32 => { - const S_ = if (target.flags.zig_offset_table) ZIG_GOT else S; + const S_ = if (target.flags.zig_jump_table) ZJT else S; try cwriter.writeInt(i32, @as(i32, @intCast(S_ + A - P)), .little); }, @@ -1245,7 +1245,7 @@ const x86_64 = struct { }, .@"32" => { - const S_ = if (target.flags.zig_offset_table) ZIG_GOT else S; + const S_ = if (target.flags.zig_jump_table) ZJT else S; try cwriter.writeInt(u32, @as(u32, @truncate(@as(u64, @intCast(S_ + A)))), .little); }, .@"32S" => try cwriter.writeInt(i32, @as(i32, @truncate(S + A)), .little), @@ -1729,9 +1729,9 @@ const aarch64 = struct { const code = code_buffer[r_offset..][0..4]; const file_ptr = atom.file(elf_file).?; - const P, const A, const S, const GOT, const G, const TP, const DTP, const ZIG_GOT = args; + const P, const A, const S, const GOT, const G, const TP, const DTP, const ZJT = args; _ = DTP; - _ = ZIG_GOT; + _ = ZJT; switch (r_type) { .NONE => unreachable, @@ -2013,10 +2013,10 @@ const riscv = struct { const r_offset = std.math.cast(usize, rel.r_offset) orelse return error.Overflow; const cwriter = stream.writer(); - const P, const A, const S, const GOT, const G, const TP, const DTP, const ZIG_GOT = args; + const P, const A, const S, const GOT, const G, const TP, const DTP, const ZJT = args; _ = TP; _ = DTP; - _ = ZIG_GOT; + _ = ZJT; switch (r_type) { .NONE => unreachable, diff --git a/src/link/Elf/Symbol.zig b/src/link/Elf/Symbol.zig index 20697a0426..38ef6bb8f7 100644 --- a/src/link/Elf/Symbol.zig +++ b/src/link/Elf/Symbol.zig @@ -217,12 +217,12 @@ pub fn tlsDescAddress(symbol: Symbol, elf_file: *Elf) i64 { return entry.address(elf_file); } -pub fn zigOffsetTableAddress(symbol: Symbol, elf_file: *Elf) i64 { - if (!symbol.flags.zig_offset_table) return 0; +pub fn zigJumpTableAddress(symbol: Symbol, elf_file: *Elf) i64 { + if (!symbol.flags.zig_jump_table) return 0; const zo = elf_file.zigObjectPtr().?; - const offset_table = zo.offsetTablePtr().?; - const ot_index = symbol.extra(elf_file).zig_offset_table; - return offset_table.entryAddress(ot_index, zo, elf_file); + const jump_table = zo.jumpTablePtr().?; + const jt_index = symbol.extra(elf_file).zig_jump_table; + return jump_table.entryAddress(jt_index, zo, elf_file); } pub fn dsoAlignment(symbol: Symbol, elf_file: *Elf) !u64 { @@ -248,7 +248,7 @@ const AddExtraOpts = struct { tlsgd: ?u32 = null, gottp: ?u32 = null, tlsdesc: ?u32 = null, - zig_offset_table: ?u32 = null, + zig_jump_table: ?u32 = null, }; pub fn addExtra(symbol: *Symbol, opts: AddExtraOpts, elf_file: *Elf) void { @@ -453,8 +453,8 @@ pub const Flags = packed struct { /// Whether the symbol is a merge subsection. merge_subsection: bool = false, - /// Whether the symbol has __zig_offset_table indirection. - zig_offset_table: bool = false, + /// Whether the symbol has __zig_jump_table indirection. + zig_jump_table: bool = false, }; pub const Extra = struct { @@ -468,7 +468,7 @@ pub const Extra = struct { gottp: u32 = 0, tlsdesc: u32 = 0, merge_section: u32 = 0, - zig_offset_table: u32 = 0, + zig_jump_table: u32 = 0, }; pub const Index = u32; diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index 4415180597..596ed781ca 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -55,10 +55,10 @@ debug_str_section_zig_size: u64 = 0, debug_aranges_section_zig_size: u64 = 0, debug_line_section_zig_size: u64 = 0, -/// Function offset table containing pointers to Zig generated functions. +/// Function jump table containing trampolines to Zcu functions. /// The table is used for Zig's incremental compilation and is embedded with /// the machine code section. -offset_table: ?OffsetTable = null, +jump_table: ?JumpTable = null, pub const global_symbol_bit: u32 = 0x80000000; pub const symbol_mask: u32 = 0x7fffffff; @@ -128,8 +128,8 @@ pub fn deinit(self: *ZigObject, allocator: Allocator) void { dw.deinit(); } - if (self.offset_table) |*ot| { - ot.deinit(allocator); + if (self.jump_table) |*jt| { + jt.deinit(allocator); } } @@ -917,8 +917,8 @@ fn updateNavCode( if (stt_bits == elf.STT_FUNC) { const extra = sym.extra(elf_file); - const offset_table = self.offsetTablePtr().?; - offset_table.entries.items(.dirty)[extra.zig_offset_table] = true; + const jump_table = self.jumpTablePtr().?; + jump_table.entries.items(.dirty)[extra.zig_jump_table] = true; } } } else if (code.len < old_size) { @@ -1036,7 +1036,7 @@ pub fn updateFunc( const ip = &zcu.intern_pool; const gpa = elf_file.base.comp.gpa; const func = zcu.funcInfo(func_index); - const offset_table = self.offsetTablePtr() orelse try self.initOffsetTable(gpa, elf_file); + const jump_table = self.jumpTablePtr() orelse try self.initJumpTable(gpa, elf_file); log.debug("updateFunc {}({d})", .{ ip.getNav(func.owner_nav).fqn.fmt(ip), func.owner_nav }); @@ -1045,16 +1045,16 @@ pub fn updateFunc( { const sym = self.symbol(sym_index); - if (!sym.flags.zig_offset_table) { - const index = try offset_table.addSymbol(gpa, sym_index); - sym.flags.zig_offset_table = true; - sym.addExtra(.{ .zig_offset_table = index }, elf_file); - try offset_table.updateSize(self, elf_file); - const old_vaddr = offset_table.address(self, elf_file); - try self.symbol(offset_table.sym_index).atom(elf_file).?.allocate(elf_file); - const new_vaddr = offset_table.address(self, elf_file); + if (!sym.flags.zig_jump_table) { + const index = try jump_table.addSymbol(gpa, sym_index); + sym.flags.zig_jump_table = true; + sym.addExtra(.{ .zig_jump_table = index }, elf_file); + try jump_table.updateSize(self, elf_file); + const old_vaddr = jump_table.address(self, elf_file); + try self.symbol(jump_table.sym_index).atom(elf_file).?.allocate(elf_file); + const new_vaddr = jump_table.address(self, elf_file); if (old_vaddr != new_vaddr) { - offset_table.dirty = true; + jump_table.dirty = true; } } } @@ -1100,21 +1100,21 @@ pub fn updateFunc( // Exports will be updated by `Zcu.processExports` after the update. - if (offset_table.dirty) { + if (jump_table.dirty) { // TODO write in bulk - for (offset_table.entries.items(.dirty), 0..) |*dirty, i| { - try offset_table.writeEntry(@intCast(i), self, elf_file); + for (jump_table.entries.items(.dirty), 0..) |*dirty, i| { + try jump_table.writeEntry(@intCast(i), self, elf_file); dirty.* = false; } } else { const sym = self.symbol(sym_index); - const ot_index = sym.extra(elf_file).zig_offset_table; - var ot_entry = offset_table.entries.get(ot_index); - if (ot_entry.dirty) { - try offset_table.writeEntry(ot_index, self, elf_file); - ot_entry.dirty = false; + const jt_index = sym.extra(elf_file).zig_jump_table; + var jt_entry = jump_table.entries.get(jt_index); + if (jt_entry.dirty) { + try jump_table.writeEntry(jt_index, self, elf_file); + jt_entry.dirty = false; } - offset_table.entries.set(ot_index, ot_entry); + jump_table.entries.set(jt_index, jt_entry); } } @@ -1470,22 +1470,22 @@ pub fn getGlobalSymbol(self: *ZigObject, elf_file: *Elf, name: []const u8, lib_n return lookup_gop.value_ptr.*; } -pub fn offsetTablePtr(self: *ZigObject) ?*OffsetTable { - return if (self.offset_table) |*ot| ot else null; +pub fn jumpTablePtr(self: *ZigObject) ?*JumpTable { + return if (self.jump_table) |*jt| jt else null; } -fn initOffsetTable(self: *ZigObject, allocator: Allocator, elf_file: *Elf) error{OutOfMemory}!*OffsetTable { - const name_off = try self.addString(allocator, "__zig_offset_table"); +fn initJumpTable(self: *ZigObject, allocator: Allocator, elf_file: *Elf) error{OutOfMemory}!*JumpTable { + const name_off = try self.addString(allocator, "__zig_jump_table"); const sym_index = try self.newSymbolWithAtom(allocator, name_off); const sym = self.symbol(sym_index); const esym = &self.symtab.items(.elf_sym)[sym.esym_index]; esym.st_info |= elf.STT_OBJECT; const atom_ptr = sym.atom(elf_file).?; atom_ptr.alive = true; - atom_ptr.alignment = Atom.Alignment.fromNonzeroByteUnits(OffsetTable.alignment(elf_file.getTarget().cpu.arch)); + atom_ptr.alignment = Atom.Alignment.fromNonzeroByteUnits(JumpTable.alignment(elf_file.getTarget().cpu.arch)); atom_ptr.output_section_index = elf_file.zig_text_section_index.?; - self.offset_table = OffsetTable{ .sym_index = sym_index }; - return &(self.offset_table.?); + self.jump_table = JumpTable{ .sym_index = sym_index }; + return &(self.jump_table.?); } pub fn asFile(self: *ZigObject) File { @@ -1763,28 +1763,28 @@ const UavTable = std.AutoArrayHashMapUnmanaged(InternPool.Index, AvMetadata); const LazySymbolTable = std.AutoArrayHashMapUnmanaged(InternPool.Index, LazySymbolMetadata); const TlsTable = std.AutoArrayHashMapUnmanaged(Atom.Index, TlsVariable); -pub const OffsetTable = struct { +pub const JumpTable = struct { sym_index: Symbol.Index, entries: std.MultiArrayList(Entry) = .{}, dirty: bool = false, - pub fn deinit(ot: *OffsetTable, allocator: Allocator) void { - ot.entries.deinit(allocator); + pub fn deinit(jt: *JumpTable, allocator: Allocator) void { + jt.entries.deinit(allocator); } - pub fn addSymbol(ot: *OffsetTable, allocator: Allocator, sym_index: Symbol.Index) !Index { - const index: Index = @intCast(try ot.entries.addOne(allocator)); - ot.entries.set(index, .{ .sym_index = sym_index }); + pub fn addSymbol(jt: *JumpTable, allocator: Allocator, sym_index: Symbol.Index) !Index { + const index: Index = @intCast(try jt.entries.addOne(allocator)); + jt.entries.set(index, .{ .sym_index = sym_index }); return index; } - pub fn address(ot: OffsetTable, zo: *ZigObject, elf_file: *Elf) i64 { - const sym = zo.symbol(ot.sym_index); + pub fn address(jt: JumpTable, zo: *ZigObject, elf_file: *Elf) i64 { + const sym = zo.symbol(jt.sym_index); return sym.address(.{}, elf_file); } - pub fn size(ot: OffsetTable, zo: *ZigObject, elf_file: *Elf) u64 { - const sym = zo.symbol(ot.sym_index); + pub fn size(jt: JumpTable, zo: *ZigObject, elf_file: *Elf) u64 { + const sym = zo.symbol(jt.sym_index); return sym.atom(elf_file).?.size; } @@ -1795,12 +1795,12 @@ pub const OffsetTable = struct { }; } - pub fn entryAddress(ot: OffsetTable, index: Index, zo: *ZigObject, elf_file: *Elf) i64 { - return ot.address(zo, elf_file) + @as(i64, @intCast(index * entrySize(elf_file.getTarget().cpu.arch))); + pub fn entryAddress(jt: JumpTable, index: Index, zo: *ZigObject, elf_file: *Elf) i64 { + return jt.address(zo, elf_file) + @as(i64, @intCast(index * entrySize(elf_file.getTarget().cpu.arch))); } - pub fn entryOffset(ot: OffsetTable, index: Index, zo: *ZigObject, elf_file: *Elf) u64 { - const sym = zo.symbol(ot.sym_index); + pub fn entryOffset(jt: JumpTable, index: Index, zo: *ZigObject, elf_file: *Elf) u64 { + const sym = zo.symbol(jt.sym_index); const atom_ptr = sym.atom(elf_file).?; const shdr = elf_file.shdrs.items[atom_ptr.output_section_index]; return shdr.sh_offset + @as(u64, @intCast(atom_ptr.value)) + index * entrySize(elf_file.getTarget().cpu.arch); @@ -1815,17 +1815,17 @@ pub const OffsetTable = struct { return seq_len; } - pub fn targetAddress(ot: OffsetTable, index: Index, zo: *ZigObject, elf_file: *Elf) i64 { - const sym_index = ot.entries.items(.sym_index)[index]; + pub fn targetAddress(jt: JumpTable, index: Index, zo: *ZigObject, elf_file: *Elf) i64 { + const sym_index = jt.entries.items(.sym_index)[index]; return zo.symbol(sym_index).address(.{}, elf_file); } const max_jump_seq_len = 12; - pub fn writeEntry(ot: OffsetTable, index: Index, zo: *ZigObject, elf_file: *Elf) !void { - const fileoff = ot.entryOffset(index, zo, elf_file); - const source_addr = ot.entryAddress(index, zo, elf_file); - const target_addr = @as(i64, @intCast(ot.targetAddress(index, zo, elf_file))); + pub fn writeEntry(jt: JumpTable, index: Index, zo: *ZigObject, elf_file: *Elf) !void { + const fileoff = jt.entryOffset(index, zo, elf_file); + const source_addr = jt.entryAddress(index, zo, elf_file); + const target_addr = @as(i64, @intCast(jt.targetAddress(index, zo, elf_file))); var buf: [max_jump_seq_len]u8 = undefined; const out = switch (elf_file.getTarget().cpu.arch) { .x86_64 => try x86_64.writeEntry(source_addr, target_addr, &buf), @@ -1855,46 +1855,46 @@ pub const OffsetTable = struct { } } - pub fn updateSize(ot: OffsetTable, zo: *ZigObject, elf_file: *Elf) !void { - const ot_size: u64 = @intCast(ot.entries.items(.sym_index).len * entrySize(elf_file.getTarget().cpu.arch)); - const sym = zo.symbol(ot.sym_index); + pub fn updateSize(jt: JumpTable, zo: *ZigObject, elf_file: *Elf) !void { + const jt_size: u64 = @intCast(jt.entries.items(.sym_index).len * entrySize(elf_file.getTarget().cpu.arch)); + const sym = zo.symbol(jt.sym_index); const esym = &zo.symtab.items(.elf_sym)[sym.esym_index]; - esym.st_size = ot_size; + esym.st_size = jt_size; const atom_ptr = sym.atom(elf_file).?; - atom_ptr.size = ot_size; + atom_ptr.size = jt_size; } - const OffsetTableFormatContext = struct { OffsetTable, *ZigObject, *Elf }; + const JumpTableFormatContext = struct { JumpTable, *ZigObject, *Elf }; pub fn format( - ot: OffsetTable, + jt: JumpTable, comptime unused_fmt_string: []const u8, options: std.fmt.FormatOptions, writer: anytype, ) !void { - _ = ot; + _ = jt; _ = unused_fmt_string; _ = options; _ = writer; - @compileError("do not format OffsetTable directly"); + @compileError("do not format JumpTable directly"); } - pub fn fmt(ot: OffsetTable, zo: *ZigObject, elf_file: *Elf) std.fmt.Formatter(format2) { - return .{ .data = .{ ot, zo, elf_file } }; + pub fn fmt(jt: JumpTable, zo: *ZigObject, elf_file: *Elf) std.fmt.Formatter(format2) { + return .{ .data = .{ jt, zo, elf_file } }; } fn format2( - ctx: OffsetTableFormatContext, + ctx: JumpTableFormatContext, comptime unused_fmt_string: []const u8, options: std.fmt.FormatOptions, writer: anytype, ) !void { _ = options; _ = unused_fmt_string; - const ot, const zo, const ef = ctx; - try writer.writeAll("offset table\n"); - try writer.print(" @{x} : size({x})\n", .{ ot.address(zo, ef), ot.size(zo, ef) }); - for (ot.entries.items(.sym_index), ot.entries.items(.dirty)) |sym_index, dirty| { + const jt, const zo, const ef = ctx; + try writer.writeAll("__zig_jump_table\n"); + try writer.print(" @{x} : size({x})\n", .{ jt.address(zo, ef), jt.size(zo, ef) }); + for (jt.entries.items(.sym_index), jt.entries.items(.dirty)) |sym_index, dirty| { const sym = zo.symbol(sym_index); try writer.print(" %{d} : {s} : @{x}", .{ sym_index, sym.name(ef), sym.address(.{}, ef) }); if (dirty) try writer.writeAll(" : [!]"); From ffcf0478fe646569465ac509b3e60faadd9b3697 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Mon, 12 Aug 2024 23:42:37 +0200 Subject: [PATCH 15/39] x86_64: remove handling of .call since it's unused for now --- src/arch/x86_64/Lower.zig | 1 - 1 file changed, 1 deletion(-) diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig index 2cb874f003..9ad1579ecc 100644 --- a/src/arch/x86_64/Lower.zig +++ b/src/arch/x86_64/Lower.zig @@ -398,7 +398,6 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) _ = lower.reloc(.{ .linker_reloc = sym }); break :op if (lower.pic) switch (mnemonic) { - .call => break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }, .lea => break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }, .mov => break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }, else => unreachable, From d25c93a868585e4e2f5941fcff15fa49c90510e9 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 13 Aug 2024 09:56:17 +0200 Subject: [PATCH 16/39] x86_64: emit call rel32 for near calls with linker reloc --- src/arch/x86_64/CodeGen.zig | 60 ++++++-------- src/arch/x86_64/Disassembler.zig | 2 +- src/arch/x86_64/Lower.zig | 6 +- src/arch/x86_64/Mir.zig | 2 +- src/arch/x86_64/bits.zig | 45 +++-------- src/arch/x86_64/encoder.zig | 134 +++++++++++++++++++++---------- src/link/Elf/Atom.zig | 7 +- 7 files changed, 138 insertions(+), 118 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index f6fbc67ff1..ea4c7ae358 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -1379,14 +1379,19 @@ fn asmImmediate(self: *Self, tag: Mir.Inst.FixedTag, imm: Immediate) !void { .ops = switch (imm) { .signed => .i_s, .unsigned => .i_u, + .reloc => .rel, + }, + .data = switch (imm) { + .reloc => |x| .{ .reloc = x }, + .signed, .unsigned => .{ .i = .{ + .fixes = tag[0], + .i = switch (imm) { + .signed => |s| @bitCast(s), + .unsigned => |u| @intCast(u), + .reloc => unreachable, + }, + } }, }, - .data = .{ .i = .{ - .fixes = tag[0], - .i = switch (imm) { - .signed => |s| @bitCast(s), - .unsigned => |u| @intCast(u), - }, - } }, }); } @@ -1406,6 +1411,7 @@ fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, imm: const ops: Mir.Inst.Ops = switch (imm) { .signed => .ri_s, .unsigned => |u| if (math.cast(u32, u)) |_| .ri_u else .ri64, + .reloc => unreachable, }; _ = try self.addInst(.{ .tag = tag[1], @@ -1417,6 +1423,7 @@ fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, imm: .i = switch (imm) { .signed => |s| @bitCast(s), .unsigned => |u| @intCast(u), + .reloc => unreachable, }, } }, .ri64 => .{ .rx = .{ @@ -1488,6 +1495,7 @@ fn asmRegisterRegisterRegisterImmediate( .i = switch (imm) { .signed => |s| @bitCast(@as(i8, @intCast(s))), .unsigned => |u| @intCast(u), + .reloc => unreachable, }, } }, }); @@ -1505,6 +1513,7 @@ fn asmRegisterRegisterImmediate( .ops = switch (imm) { .signed => .rri_s, .unsigned => .rri_u, + .reloc => unreachable, }, .data = .{ .rri = .{ .fixes = tag[0], @@ -1513,6 +1522,7 @@ fn asmRegisterRegisterImmediate( .i = switch (imm) { .signed => |s| @bitCast(s), .unsigned => |u| @intCast(u), + .reloc => unreachable, }, } }, }); @@ -1610,6 +1620,7 @@ fn asmRegisterMemoryImmediate( if (switch (imm) { .signed => |s| if (math.cast(i16, s)) |x| @as(u16, @bitCast(x)) else null, .unsigned => |u| math.cast(u16, u), + .reloc => unreachable, }) |small_imm| { _ = try self.addInst(.{ .tag = tag[1], @@ -1625,6 +1636,7 @@ fn asmRegisterMemoryImmediate( const payload = try self.addExtra(Mir.Imm32{ .imm = switch (imm) { .signed => |s| @bitCast(s), .unsigned => unreachable, + .reloc => unreachable, } }); assert(payload + 1 == try self.addExtra(Mir.Memory.encode(m))); _ = try self.addInst(.{ @@ -1632,6 +1644,7 @@ fn asmRegisterMemoryImmediate( .ops = switch (imm) { .signed => .rmi_s, .unsigned => .rmi_u, + .reloc => unreachable, }, .data = .{ .rx = .{ .fixes = tag[0], @@ -1679,6 +1692,7 @@ fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, imm: Immed const payload = try self.addExtra(Mir.Imm32{ .imm = switch (imm) { .signed => |s| @bitCast(s), .unsigned => |u| @intCast(u), + .reloc => unreachable, } }); assert(payload + 1 == try self.addExtra(Mir.Memory.encode(m))); _ = try self.addInst(.{ @@ -1686,6 +1700,7 @@ fn asmMemoryImmediate(self: *Self, tag: Mir.Inst.FixedTag, m: Memory, imm: Immed .ops = switch (imm) { .signed => .mi_s, .unsigned => .mi_u, + .reloc => unreachable, }, .data = .{ .x = .{ .fixes = tag[0], @@ -12310,33 +12325,10 @@ fn genCall(self: *Self, info: union(enum) { if (self.bin_file.cast(.elf)) |elf_file| { const zo = elf_file.zigObjectPtr().?; const sym_index = try zo.getOrCreateMetadataForNav(elf_file, func.owner_nav); - if (self.mod.pic) { - const callee_reg: Register = switch (resolved_cc) { - .SysV => callee: { - if (!fn_info.is_var_args) break :callee .rax; - const param_regs = abi.getCAbiIntParamRegs(resolved_cc); - break :callee if (call_info.gp_count < param_regs.len) - param_regs[call_info.gp_count] - else - .r10; - }, - .Win64 => .rax, - else => unreachable, - }; - try self.genSetReg( - callee_reg, - Type.usize, - .{ .lea_symbol = .{ .sym = sym_index } }, - .{}, - ); - try self.asmRegister(.{ ._, .call }, callee_reg); - } else try self.asmMemory(.{ ._, .call }, .{ - .base = .{ .reloc = .{ - .atom_index = try self.owner.getSymbolIndex(self), - .sym_index = sym_index, - } }, - .mod = .{ .rm = .{ .size = .qword } }, - }); + try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ + .atom_index = try self.owner.getSymbolIndex(self), + .sym_index = sym_index, + })); } else if (self.bin_file.cast(.coff)) |coff_file| { const atom = try coff_file.getOrCreateAtomForNav(func.owner_nav); const sym_index = coff_file.getAtom(atom).getSymbolIndex().?; diff --git a/src/arch/x86_64/Disassembler.zig b/src/arch/x86_64/Disassembler.zig index e0117fa17b..d21cec28ab 100644 --- a/src/arch/x86_64/Disassembler.zig +++ b/src/arch/x86_64/Disassembler.zig @@ -8,7 +8,7 @@ const bits = @import("bits.zig"); const encoder = @import("encoder.zig"); const Encoding = @import("Encoding.zig"); -const Immediate = bits.Immediate; +const Immediate = Instruction.Immediate; const Instruction = encoder.Instruction; const LegacyPrefixes = encoder.LegacyPrefixes; const Memory = Instruction.Memory; diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig index 9ad1579ecc..56cd2669d6 100644 --- a/src/arch/x86_64/Lower.zig +++ b/src/arch/x86_64/Lower.zig @@ -475,7 +475,7 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void { .rrmi => inst.data.rrix.fixes, .mi_u, .mi_s => inst.data.x.fixes, .m => inst.data.x.fixes, - .extern_fn_reloc, .got_reloc, .direct_reloc, .import_reloc, .tlv_reloc => ._, + .extern_fn_reloc, .got_reloc, .direct_reloc, .import_reloc, .tlv_reloc, .rel => ._, else => return lower.fail("TODO lower .{s}", .{@tagName(inst.ops)}), }; try lower.emit(switch (fixes) { @@ -607,7 +607,7 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void { .{ .mem = lower.mem(inst.data.rrix.payload) }, .{ .imm = lower.imm(inst.ops, inst.data.rrix.i) }, }, - .extern_fn_reloc => &.{ + .extern_fn_reloc, .rel => &.{ .{ .imm = lower.reloc(.{ .linker_extern_fn = inst.data.reloc }) }, }, .got_reloc, .direct_reloc, .import_reloc => ops: { @@ -650,7 +650,7 @@ const std = @import("std"); const Air = @import("../../Air.zig"); const Allocator = std.mem.Allocator; const ErrorMsg = Zcu.ErrorMsg; -const Immediate = bits.Immediate; +const Immediate = Instruction.Immediate; const Instruction = encoder.Instruction; const Lower = @This(); const Memory = Instruction.Memory; diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig index d2dd6237a5..08f2cc3b25 100644 --- a/src/arch/x86_64/Mir.zig +++ b/src/arch/x86_64/Mir.zig @@ -769,7 +769,7 @@ pub const Inst = struct { /// Uses `imm` payload. i_u, /// Relative displacement operand. - /// Uses `imm` payload. + /// Uses `reloc` payload. rel, /// Register, memory operands. /// Uses `rx` payload with extra data of type `Memory`. diff --git a/src/arch/x86_64/bits.zig b/src/arch/x86_64/bits.zig index 6896d6c2e7..79f9b20924 100644 --- a/src/arch/x86_64/bits.zig +++ b/src/arch/x86_64/bits.zig @@ -569,6 +569,7 @@ pub const Memory = struct { pub const Immediate = union(enum) { signed: i32, unsigned: u64, + reloc: Symbol, pub fn u(x: u64) Immediate { return .{ .unsigned = x }; @@ -578,39 +579,19 @@ pub const Immediate = union(enum) { return .{ .signed = x }; } - pub fn asSigned(imm: Immediate, bit_size: u64) i64 { - return switch (imm) { - .signed => |x| switch (bit_size) { - 1, 8 => @as(i8, @intCast(x)), - 16 => @as(i16, @intCast(x)), - 32, 64 => x, - else => unreachable, - }, - .unsigned => |x| switch (bit_size) { - 1, 8 => @as(i8, @bitCast(@as(u8, @intCast(x)))), - 16 => @as(i16, @bitCast(@as(u16, @intCast(x)))), - 32 => @as(i32, @bitCast(@as(u32, @intCast(x)))), - 64 => @bitCast(x), - else => unreachable, - }, - }; + pub fn rel(symbol: Symbol) Immediate { + return .{ .reloc = symbol }; } - pub fn asUnsigned(imm: Immediate, bit_size: u64) u64 { - return switch (imm) { - .signed => |x| switch (bit_size) { - 1, 8 => @as(u8, @bitCast(@as(i8, @intCast(x)))), - 16 => @as(u16, @bitCast(@as(i16, @intCast(x)))), - 32, 64 => @as(u32, @bitCast(x)), - else => unreachable, - }, - .unsigned => |x| switch (bit_size) { - 1, 8 => @as(u8, @intCast(x)), - 16 => @as(u16, @intCast(x)), - 32 => @as(u32, @intCast(x)), - 64 => x, - else => unreachable, - }, - }; + pub fn format( + imm: Immediate, + comptime fmt: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) @TypeOf(writer).Error!void { + switch (imm) { + .reloc => |x| try std.fmt.formatType(x, fmt, options, writer, 0), + inline else => |x| try writer.print("{d}", .{x}), + } } }; diff --git a/src/arch/x86_64/encoder.zig b/src/arch/x86_64/encoder.zig index 5f49443934..b525b0e11e 100644 --- a/src/arch/x86_64/encoder.zig +++ b/src/arch/x86_64/encoder.zig @@ -7,7 +7,6 @@ const testing = std.testing; const bits = @import("bits.zig"); const Encoding = @import("Encoding.zig"); const FrameIndex = bits.FrameIndex; -const Immediate = bits.Immediate; const Register = bits.Register; const Symbol = bits.Symbol; @@ -28,6 +27,55 @@ pub const Instruction = struct { repnz, }; + pub const Immediate = union(enum) { + signed: i32, + unsigned: u64, + + pub fn u(x: u64) Immediate { + return .{ .unsigned = x }; + } + + pub fn s(x: i32) Immediate { + return .{ .signed = x }; + } + + pub fn asSigned(imm: Immediate, bit_size: u64) i64 { + return switch (imm) { + .signed => |x| switch (bit_size) { + 1, 8 => @as(i8, @intCast(x)), + 16 => @as(i16, @intCast(x)), + 32, 64 => x, + else => unreachable, + }, + .unsigned => |x| switch (bit_size) { + 1, 8 => @as(i8, @bitCast(@as(u8, @intCast(x)))), + 16 => @as(i16, @bitCast(@as(u16, @intCast(x)))), + 32 => @as(i32, @bitCast(@as(u32, @intCast(x)))), + 64 => @bitCast(x), + else => unreachable, + }, + }; + } + + pub fn asUnsigned(imm: Immediate, bit_size: u64) u64 { + return switch (imm) { + .signed => |x| switch (bit_size) { + 1, 8 => @as(u8, @bitCast(@as(i8, @intCast(x)))), + 16 => @as(u16, @bitCast(@as(i16, @intCast(x)))), + 32, 64 => @as(u32, @bitCast(x)), + else => unreachable, + }, + .unsigned => |x| switch (bit_size) { + 1, 8 => @as(u8, @intCast(x)), + 16 => @as(u16, @intCast(x)), + 32 => @as(u32, @intCast(x)), + 64 => x, + else => unreachable, + }, + }; + } + }; + pub const Memory = union(enum) { sib: Sib, rip: Rip, @@ -1119,7 +1167,7 @@ test "encode" { const inst = try Instruction.new(.none, .mov, &.{ .{ .reg = .rbx }, - .{ .imm = Immediate.u(4) }, + .{ .imm = Instruction.Immediate.u(4) }, }); try inst.encode(buf.writer(), .{}); try testing.expectEqualSlices(u8, &.{ 0x48, 0xc7, 0xc3, 0x4, 0x0, 0x0, 0x0 }, buf.items); @@ -1129,47 +1177,47 @@ test "lower I encoding" { var enc = TestEncode{}; try enc.encode(.push, &.{ - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\x6A\x10", enc.code(), "push 0x10"); try enc.encode(.push, &.{ - .{ .imm = Immediate.u(0x1000) }, + .{ .imm = Instruction.Immediate.u(0x1000) }, }); try expectEqualHexStrings("\x66\x68\x00\x10", enc.code(), "push 0x1000"); try enc.encode(.push, &.{ - .{ .imm = Immediate.u(0x10000000) }, + .{ .imm = Instruction.Immediate.u(0x10000000) }, }); try expectEqualHexStrings("\x68\x00\x00\x00\x10", enc.code(), "push 0x10000000"); try enc.encode(.adc, &.{ .{ .reg = .rax }, - .{ .imm = Immediate.u(0x10000000) }, + .{ .imm = Instruction.Immediate.u(0x10000000) }, }); try expectEqualHexStrings("\x48\x15\x00\x00\x00\x10", enc.code(), "adc rax, 0x10000000"); try enc.encode(.add, &.{ .{ .reg = .al }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\x04\x10", enc.code(), "add al, 0x10"); try enc.encode(.add, &.{ .{ .reg = .rax }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\x48\x83\xC0\x10", enc.code(), "add rax, 0x10"); try enc.encode(.sbb, &.{ .{ .reg = .ax }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\x66\x1D\x10\x00", enc.code(), "sbb ax, 0x10"); try enc.encode(.xor, &.{ .{ .reg = .al }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\x34\x10", enc.code(), "xor al, 0x10"); } @@ -1179,43 +1227,43 @@ test "lower MI encoding" { try enc.encode(.mov, &.{ .{ .reg = .r12 }, - .{ .imm = Immediate.u(0x1000) }, + .{ .imm = Instruction.Immediate.u(0x1000) }, }); try expectEqualHexStrings("\x49\xC7\xC4\x00\x10\x00\x00", enc.code(), "mov r12, 0x1000"); try enc.encode(.mov, &.{ .{ .mem = Instruction.Memory.sib(.byte, .{ .base = .{ .reg = .r12 } }) }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\x41\xC6\x04\x24\x10", enc.code(), "mov BYTE PTR [r12], 0x10"); try enc.encode(.mov, &.{ .{ .reg = .r12 }, - .{ .imm = Immediate.u(0x1000) }, + .{ .imm = Instruction.Immediate.u(0x1000) }, }); try expectEqualHexStrings("\x49\xC7\xC4\x00\x10\x00\x00", enc.code(), "mov r12, 0x1000"); try enc.encode(.mov, &.{ .{ .reg = .r12 }, - .{ .imm = Immediate.u(0x1000) }, + .{ .imm = Instruction.Immediate.u(0x1000) }, }); try expectEqualHexStrings("\x49\xC7\xC4\x00\x10\x00\x00", enc.code(), "mov r12, 0x1000"); try enc.encode(.mov, &.{ .{ .reg = .rax }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\x48\xc7\xc0\x10\x00\x00\x00", enc.code(), "mov rax, 0x10"); try enc.encode(.mov, &.{ .{ .mem = Instruction.Memory.sib(.dword, .{ .base = .{ .reg = .r11 } }) }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\x41\xc7\x03\x10\x00\x00\x00", enc.code(), "mov DWORD PTR [r11], 0x10"); try enc.encode(.mov, &.{ .{ .mem = Instruction.Memory.rip(.qword, 0x10) }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings( "\x48\xC7\x05\x10\x00\x00\x00\x10\x00\x00\x00", @@ -1225,19 +1273,19 @@ test "lower MI encoding" { try enc.encode(.mov, &.{ .{ .mem = Instruction.Memory.sib(.qword, .{ .base = .{ .reg = .rbp }, .disp = -8 }) }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\x48\xc7\x45\xf8\x10\x00\x00\x00", enc.code(), "mov QWORD PTR [rbp - 8], 0x10"); try enc.encode(.mov, &.{ .{ .mem = Instruction.Memory.sib(.word, .{ .base = .{ .reg = .rbp }, .disp = -2 }) }, - .{ .imm = Immediate.s(-16) }, + .{ .imm = Instruction.Immediate.s(-16) }, }); try expectEqualHexStrings("\x66\xC7\x45\xFE\xF0\xFF", enc.code(), "mov WORD PTR [rbp - 2], -16"); try enc.encode(.mov, &.{ .{ .mem = Instruction.Memory.sib(.byte, .{ .base = .{ .reg = .rbp }, .disp = -1 }) }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\xC6\x45\xFF\x10", enc.code(), "mov BYTE PTR [rbp - 1], 0x10"); @@ -1247,7 +1295,7 @@ test "lower MI encoding" { .disp = 0x10000000, .scale_index = .{ .scale = 2, .index = .rcx }, }) }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings( "\x48\xC7\x04\x4D\x00\x00\x00\x10\x10\x00\x00\x00", @@ -1257,43 +1305,43 @@ test "lower MI encoding" { try enc.encode(.adc, &.{ .{ .mem = Instruction.Memory.sib(.byte, .{ .base = .{ .reg = .rbp }, .disp = -0x10 }) }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\x80\x55\xF0\x10", enc.code(), "adc BYTE PTR [rbp - 0x10], 0x10"); try enc.encode(.adc, &.{ .{ .mem = Instruction.Memory.rip(.qword, 0) }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\x48\x83\x15\x00\x00\x00\x00\x10", enc.code(), "adc QWORD PTR [rip], 0x10"); try enc.encode(.adc, &.{ .{ .reg = .rax }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\x48\x83\xD0\x10", enc.code(), "adc rax, 0x10"); try enc.encode(.add, &.{ .{ .mem = Instruction.Memory.sib(.dword, .{ .base = .{ .reg = .rdx }, .disp = -8 }) }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\x83\x42\xF8\x10", enc.code(), "add DWORD PTR [rdx - 8], 0x10"); try enc.encode(.add, &.{ .{ .reg = .rax }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\x48\x83\xC0\x10", enc.code(), "add rax, 0x10"); try enc.encode(.add, &.{ .{ .mem = Instruction.Memory.sib(.qword, .{ .base = .{ .reg = .rbp }, .disp = -0x10 }) }, - .{ .imm = Immediate.s(-0x10) }, + .{ .imm = Instruction.Immediate.s(-0x10) }, }); try expectEqualHexStrings("\x48\x83\x45\xF0\xF0", enc.code(), "add QWORD PTR [rbp - 0x10], -0x10"); try enc.encode(.@"and", &.{ .{ .mem = Instruction.Memory.sib(.dword, .{ .base = .{ .reg = .ds }, .disp = 0x10000000 }) }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings( "\x83\x24\x25\x00\x00\x00\x10\x10", @@ -1303,7 +1351,7 @@ test "lower MI encoding" { try enc.encode(.@"and", &.{ .{ .mem = Instruction.Memory.sib(.dword, .{ .base = .{ .reg = .es }, .disp = 0x10000000 }) }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings( "\x26\x83\x24\x25\x00\x00\x00\x10\x10", @@ -1313,7 +1361,7 @@ test "lower MI encoding" { try enc.encode(.@"and", &.{ .{ .mem = Instruction.Memory.sib(.dword, .{ .base = .{ .reg = .r12 }, .disp = 0x10000000 }) }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings( "\x41\x83\xA4\x24\x00\x00\x00\x10\x10", @@ -1323,7 +1371,7 @@ test "lower MI encoding" { try enc.encode(.sub, &.{ .{ .mem = Instruction.Memory.sib(.dword, .{ .base = .{ .reg = .r11 }, .disp = 0x10000000 }) }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings( "\x41\x83\xAB\x00\x00\x00\x10\x10", @@ -1542,14 +1590,14 @@ test "lower RMI encoding" { try enc.encode(.imul, &.{ .{ .reg = .r11 }, .{ .reg = .r12 }, - .{ .imm = Immediate.s(-2) }, + .{ .imm = Instruction.Immediate.s(-2) }, }); try expectEqualHexStrings("\x4D\x6B\xDC\xFE", enc.code(), "imul r11, r12, -2"); try enc.encode(.imul, &.{ .{ .reg = .r11 }, .{ .mem = Instruction.Memory.rip(.qword, -16) }, - .{ .imm = Immediate.s(-1024) }, + .{ .imm = Instruction.Immediate.s(-1024) }, }); try expectEqualHexStrings( "\x4C\x69\x1D\xF0\xFF\xFF\xFF\x00\xFC\xFF\xFF", @@ -1560,7 +1608,7 @@ test "lower RMI encoding" { try enc.encode(.imul, &.{ .{ .reg = .bx }, .{ .mem = Instruction.Memory.sib(.word, .{ .base = .{ .reg = .rbp }, .disp = -16 }) }, - .{ .imm = Immediate.s(-1024) }, + .{ .imm = Instruction.Immediate.s(-1024) }, }); try expectEqualHexStrings( "\x66\x69\x5D\xF0\x00\xFC", @@ -1571,7 +1619,7 @@ test "lower RMI encoding" { try enc.encode(.imul, &.{ .{ .reg = .bx }, .{ .mem = Instruction.Memory.sib(.word, .{ .base = .{ .reg = .rbp }, .disp = -16 }) }, - .{ .imm = Immediate.u(1024) }, + .{ .imm = Instruction.Immediate.u(1024) }, }); try expectEqualHexStrings( "\x66\x69\x5D\xF0\x00\x04", @@ -1687,7 +1735,7 @@ test "lower M encoding" { try expectEqualHexStrings("\x65\xFF\x14\x25\x00\x00\x00\x00", enc.code(), "call gs:0x0"); try enc.encode(.call, &.{ - .{ .imm = Immediate.s(0) }, + .{ .imm = Instruction.Immediate.s(0) }, }); try expectEqualHexStrings("\xE8\x00\x00\x00\x00", enc.code(), "call 0x0"); @@ -1746,7 +1794,7 @@ test "lower OI encoding" { try enc.encode(.mov, &.{ .{ .reg = .rax }, - .{ .imm = Immediate.u(0x1000000000000000) }, + .{ .imm = Instruction.Immediate.u(0x1000000000000000) }, }); try expectEqualHexStrings( "\x48\xB8\x00\x00\x00\x00\x00\x00\x00\x10", @@ -1756,7 +1804,7 @@ test "lower OI encoding" { try enc.encode(.mov, &.{ .{ .reg = .r11 }, - .{ .imm = Immediate.u(0x1000000000000000) }, + .{ .imm = Instruction.Immediate.u(0x1000000000000000) }, }); try expectEqualHexStrings( "\x49\xBB\x00\x00\x00\x00\x00\x00\x00\x10", @@ -1766,19 +1814,19 @@ test "lower OI encoding" { try enc.encode(.mov, &.{ .{ .reg = .r11d }, - .{ .imm = Immediate.u(0x10000000) }, + .{ .imm = Instruction.Immediate.u(0x10000000) }, }); try expectEqualHexStrings("\x41\xBB\x00\x00\x00\x10", enc.code(), "mov r11d, 0x10000000"); try enc.encode(.mov, &.{ .{ .reg = .r11w }, - .{ .imm = Immediate.u(0x1000) }, + .{ .imm = Instruction.Immediate.u(0x1000) }, }); try expectEqualHexStrings("\x66\x41\xBB\x00\x10", enc.code(), "mov r11w, 0x1000"); try enc.encode(.mov, &.{ .{ .reg = .r11b }, - .{ .imm = Immediate.u(0x10) }, + .{ .imm = Instruction.Immediate.u(0x10) }, }); try expectEqualHexStrings("\x41\xB3\x10", enc.code(), "mov r11b, 0x10"); } @@ -1900,7 +1948,7 @@ test "invalid instruction" { .{ .reg = .r12d }, }); try invalidInstruction(.push, &.{ - .{ .imm = Immediate.u(0x1000000000000000) }, + .{ .imm = Instruction.Immediate.u(0x1000000000000000) }, }); } @@ -2213,7 +2261,7 @@ const Assembler = struct { .immediate => { const is_neg = if (as.expect(.minus)) |_| true else |_| false; const imm_tok = try as.expect(.numeral); - const imm: Immediate = if (is_neg) blk: { + const imm: Instruction.Immediate = if (is_neg) blk: { const imm = try std.fmt.parseInt(i32, as.source(imm_tok), 0); break :blk .{ .signed = imm * -1 }; } else .{ .unsigned = try std.fmt.parseInt(u64, as.source(imm_tok), 0) }; diff --git a/src/link/Elf/Atom.zig b/src/link/Elf/Atom.zig index 31ba0d642e..32583b39d7 100644 --- a/src/link/Elf/Atom.zig +++ b/src/link/Elf/Atom.zig @@ -1215,12 +1215,11 @@ const x86_64 = struct { ); }, - .PLT32 => try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little), - - .PC32 => { + .PLT32 => { const S_ = if (target.flags.zig_jump_table) ZJT else S; try cwriter.writeInt(i32, @as(i32, @intCast(S_ + A - P)), .little); }, + .PC32 => try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little), .GOTPCREL => try cwriter.writeInt(i32, @as(i32, @intCast(G + GOT + A - P)), .little), .GOTPC32 => try cwriter.writeInt(i32, @as(i32, @intCast(GOT + A - P)), .little), @@ -1620,7 +1619,7 @@ const x86_64 = struct { const bits = @import("../../arch/x86_64/bits.zig"); const encoder = @import("../../arch/x86_64/encoder.zig"); const Disassembler = @import("../../arch/x86_64/Disassembler.zig"); - const Immediate = bits.Immediate; + const Immediate = Instruction.Immediate; const Instruction = encoder.Instruction; }; From ce8886d57d1593a7eb4909e518fc49e1c95360dd Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 13 Aug 2024 10:28:01 +0200 Subject: [PATCH 17/39] elf: make zig jump table indirection implicit via Symbol.address --- src/link/Elf/Atom.zig | 35 ++++++++++++----------------------- src/link/Elf/Symbol.zig | 21 ++++++++++++--------- src/link/Elf/ZigObject.zig | 17 +++++++++++------ 3 files changed, 35 insertions(+), 38 deletions(-) diff --git a/src/link/Elf/Atom.zig b/src/link/Elf/Atom.zig index 32583b39d7..e067e53c10 100644 --- a/src/link/Elf/Atom.zig +++ b/src/link/Elf/Atom.zig @@ -750,8 +750,6 @@ pub fn resolveRelocsAlloc(self: Atom, elf_file: *Elf, code: []u8) RelocError!voi const S = target.address(.{}, elf_file); // Address of the global offset table. const GOT = elf_file.gotAddress(); - // Address of the zig jump table entry if any. - const ZJT = target.zigJumpTableAddress(elf_file); // Relative offset to the start of the global offset table. const G = target.gotAddress(elf_file) - GOT; // // Address of the thread pointer. @@ -759,19 +757,18 @@ pub fn resolveRelocsAlloc(self: Atom, elf_file: *Elf, code: []u8) RelocError!voi // Address of the dynamic thread pointer. const DTP = elf_file.dtpAddress(); - relocs_log.debug(" {s}: {x}: [{x} => {x}] GOT({x}) ZJT({x}) ({s})", .{ + relocs_log.debug(" {s}: {x}: [{x} => {x}] GOT({x}) ({s})", .{ relocation.fmtRelocType(rel.r_type(), cpu_arch), r_offset, P, S + A, G + GOT + A, - ZJT + A, target.name(elf_file), }); try stream.seekTo(r_offset); - const args = ResolveArgs{ P, A, S, GOT, G, TP, DTP, ZJT }; + const args = ResolveArgs{ P, A, S, GOT, G, TP, DTP }; switch (cpu_arch) { .x86_64 => x86_64.resolveRelocAlloc(self, elf_file, rel, target, args, &it, code, &stream) catch |err| switch (err) { @@ -956,7 +953,7 @@ pub fn resolveRelocsNonAlloc(self: Atom, elf_file: *Elf, code: []u8, undefs: any // Address of the dynamic thread pointer. const DTP = elf_file.dtpAddress(); - const args = ResolveArgs{ P, A, S, GOT, 0, 0, DTP, 0 }; + const args = ResolveArgs{ P, A, S, GOT, 0, 0, DTP }; relocs_log.debug(" {}: {x}: [{x} => {x}] ({s})", .{ relocation.fmtRelocType(rel.r_type(), cpu_arch), @@ -1200,7 +1197,7 @@ const x86_64 = struct { const cwriter = stream.writer(); - const P, const A, const S, const GOT, const G, const TP, const DTP, const ZJT = args; + const P, const A, const S, const GOT, const G, const TP, const DTP = args; switch (r_type) { .NONE => unreachable, @@ -1215,10 +1212,7 @@ const x86_64 = struct { ); }, - .PLT32 => { - const S_ = if (target.flags.zig_jump_table) ZJT else S; - try cwriter.writeInt(i32, @as(i32, @intCast(S_ + A - P)), .little); - }, + .PLT32 => try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little), .PC32 => try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little), .GOTPCREL => try cwriter.writeInt(i32, @as(i32, @intCast(G + GOT + A - P)), .little), @@ -1243,10 +1237,7 @@ const x86_64 = struct { try cwriter.writeInt(i32, @as(i32, @intCast(G + GOT + A - P)), .little); }, - .@"32" => { - const S_ = if (target.flags.zig_jump_table) ZJT else S; - try cwriter.writeInt(u32, @as(u32, @truncate(@as(u64, @intCast(S_ + A)))), .little); - }, + .@"32" => try cwriter.writeInt(u32, @as(u32, @truncate(@as(u64, @intCast(S + A)))), .little), .@"32S" => try cwriter.writeInt(i32, @as(i32, @truncate(S + A)), .little), .TPOFF32 => try cwriter.writeInt(i32, @as(i32, @truncate(S + A - TP)), .little), @@ -1345,7 +1336,7 @@ const x86_64 = struct { const r_type: elf.R_X86_64 = @enumFromInt(rel.r_type()); const cwriter = stream.writer(); - _, const A, const S, const GOT, _, _, const DTP, _ = args; + _, const A, const S, const GOT, _, _, const DTP = args; switch (r_type) { .NONE => unreachable, @@ -1728,9 +1719,8 @@ const aarch64 = struct { const code = code_buffer[r_offset..][0..4]; const file_ptr = atom.file(elf_file).?; - const P, const A, const S, const GOT, const G, const TP, const DTP, const ZJT = args; + const P, const A, const S, const GOT, const G, const TP, const DTP = args; _ = DTP; - _ = ZJT; switch (r_type) { .NONE => unreachable, @@ -1932,7 +1922,7 @@ const aarch64 = struct { const r_type: elf.R_AARCH64 = @enumFromInt(rel.r_type()); const cwriter = stream.writer(); - _, const A, const S, _, _, _, _, _ = args; + _, const A, const S, _, _, _, _ = args; switch (r_type) { .NONE => unreachable, @@ -2012,10 +2002,9 @@ const riscv = struct { const r_offset = std.math.cast(usize, rel.r_offset) orelse return error.Overflow; const cwriter = stream.writer(); - const P, const A, const S, const GOT, const G, const TP, const DTP, const ZJT = args; + const P, const A, const S, const GOT, const G, const TP, const DTP = args; _ = TP; _ = DTP; - _ = ZJT; switch (r_type) { .NONE => unreachable, @@ -2167,7 +2156,7 @@ const riscv = struct { const r_offset = std.math.cast(usize, rel.r_offset) orelse return error.Overflow; const cwriter = stream.writer(); - _, const A, const S, const GOT, _, _, const DTP, _ = args; + _, const A, const S, const GOT, _, _, const DTP = args; _ = GOT; _ = DTP; @@ -2203,7 +2192,7 @@ const riscv = struct { const riscv_util = @import("../riscv.zig"); }; -const ResolveArgs = struct { i64, i64, i64, i64, i64, i64, i64, i64 }; +const ResolveArgs = struct { i64, i64, i64, i64, i64, i64, i64 }; const RelocError = error{ Overflow, diff --git a/src/link/Elf/Symbol.zig b/src/link/Elf/Symbol.zig index 38ef6bb8f7..8b30f0ab03 100644 --- a/src/link/Elf/Symbol.zig +++ b/src/link/Elf/Symbol.zig @@ -101,7 +101,7 @@ pub fn symbolRank(symbol: Symbol, elf_file: *Elf) u32 { return file_ptr.symbolRank(sym, in_archive); } -pub fn address(symbol: Symbol, opts: struct { plt: bool = true }, elf_file: *Elf) i64 { +pub fn address(symbol: Symbol, opts: struct { plt: bool = true, zjt: bool = true }, elf_file: *Elf) i64 { if (symbol.mergeSubsection(elf_file)) |msub| { if (!msub.alive) return 0; return msub.address(elf_file) + symbol.value; @@ -109,6 +109,9 @@ pub fn address(symbol: Symbol, opts: struct { plt: bool = true }, elf_file: *Elf if (symbol.flags.has_copy_rel) { return symbol.copyRelAddress(elf_file); } + if (symbol.flags.has_zjt and opts.zjt) { + return symbol.zjtAddress(elf_file); + } if (symbol.flags.has_plt and opts.plt) { if (!symbol.flags.is_canonical and symbol.flags.has_got) { // We have a non-lazy bound function pointer, use that! @@ -217,12 +220,12 @@ pub fn tlsDescAddress(symbol: Symbol, elf_file: *Elf) i64 { return entry.address(elf_file); } -pub fn zigJumpTableAddress(symbol: Symbol, elf_file: *Elf) i64 { - if (!symbol.flags.zig_jump_table) return 0; +pub fn zjtAddress(symbol: Symbol, elf_file: *Elf) i64 { + if (!symbol.flags.has_zjt) return 0; const zo = elf_file.zigObjectPtr().?; const jump_table = zo.jumpTablePtr().?; - const jt_index = symbol.extra(elf_file).zig_jump_table; - return jump_table.entryAddress(jt_index, zo, elf_file); + const index = symbol.extra(elf_file).zjt; + return jump_table.entryAddress(index, zo, elf_file); } pub fn dsoAlignment(symbol: Symbol, elf_file: *Elf) !u64 { @@ -248,7 +251,7 @@ const AddExtraOpts = struct { tlsgd: ?u32 = null, gottp: ?u32 = null, tlsdesc: ?u32 = null, - zig_jump_table: ?u32 = null, + zjt: ?u32 = null, }; pub fn addExtra(symbol: *Symbol, opts: AddExtraOpts, elf_file: *Elf) void { @@ -377,7 +380,7 @@ fn format2( try writer.print("%{d} : {s} : @{x}", .{ symbol.esym_index, symbol.fmtName(elf_file), - symbol.address(.{}, elf_file), + symbol.address(.{ .plt = false, .zjt = false }, elf_file), }); if (symbol.file(elf_file)) |file_ptr| { if (symbol.isAbs(elf_file)) { @@ -454,7 +457,7 @@ pub const Flags = packed struct { merge_subsection: bool = false, /// Whether the symbol has __zig_jump_table indirection. - zig_jump_table: bool = false, + has_zjt: bool = false, }; pub const Extra = struct { @@ -468,7 +471,7 @@ pub const Extra = struct { gottp: u32 = 0, tlsdesc: u32 = 0, merge_section: u32 = 0, - zig_jump_table: u32 = 0, + zjt: u32 = 0, }; pub const Index = u32; diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index 596ed781ca..9069e180a3 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -918,7 +918,7 @@ fn updateNavCode( if (stt_bits == elf.STT_FUNC) { const extra = sym.extra(elf_file); const jump_table = self.jumpTablePtr().?; - jump_table.entries.items(.dirty)[extra.zig_jump_table] = true; + jump_table.entries.items(.dirty)[extra.zjt] = true; } } } else if (code.len < old_size) { @@ -1045,10 +1045,10 @@ pub fn updateFunc( { const sym = self.symbol(sym_index); - if (!sym.flags.zig_jump_table) { + if (!sym.flags.has_zjt) { const index = try jump_table.addSymbol(gpa, sym_index); - sym.flags.zig_jump_table = true; - sym.addExtra(.{ .zig_jump_table = index }, elf_file); + sym.flags.has_zjt = true; + sym.addExtra(.{ .zjt = index }, elf_file); try jump_table.updateSize(self, elf_file); const old_vaddr = jump_table.address(self, elf_file); try self.symbol(jump_table.sym_index).atom(elf_file).?.allocate(elf_file); @@ -1108,7 +1108,7 @@ pub fn updateFunc( } } else { const sym = self.symbol(sym_index); - const jt_index = sym.extra(elf_file).zig_jump_table; + const jt_index = sym.extra(elf_file).zjt; var jt_entry = jump_table.entries.get(jt_index); if (jt_entry.dirty) { try jump_table.writeEntry(jt_index, self, elf_file); @@ -1896,7 +1896,12 @@ pub const JumpTable = struct { try writer.print(" @{x} : size({x})\n", .{ jt.address(zo, ef), jt.size(zo, ef) }); for (jt.entries.items(.sym_index), jt.entries.items(.dirty)) |sym_index, dirty| { const sym = zo.symbol(sym_index); - try writer.print(" %{d} : {s} : @{x}", .{ sym_index, sym.name(ef), sym.address(.{}, ef) }); + try writer.print(" {x} => {x} : %{d} : {s}", .{ + sym.address(.{}, ef), + sym.address(.{ .zjt = false }, ef), + sym_index, + sym.name(ef), + }); if (dirty) try writer.writeAll(" : [!]"); try writer.writeByte('\n'); } From 57f72095082a5fadd6419d8176475f296a65bdbc Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 13 Aug 2024 10:39:49 +0200 Subject: [PATCH 18/39] elf: replace use of linker_extern_fn with more generic Immediate.reloc --- src/arch/x86_64/CodeGen.zig | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index ea4c7ae358..116911c409 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -12357,7 +12357,16 @@ fn genCall(self: *Self, info: union(enum) { }); } else unreachable; }, - .@"extern" => |@"extern"| try self.genExternSymbolRef( + .@"extern" => |@"extern"| if (self.bin_file.cast(.elf)) |elf_file| { + const target_sym_index = try elf_file.getGlobalSymbol( + @"extern".name.toSlice(ip), + @"extern".lib_name.toSlice(ip), + ); + try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ + .atom_index = try self.owner.getSymbolIndex(self), + .sym_index = target_sym_index, + })); + } else try self.genExternSymbolRef( .call, @"extern".lib_name.toSlice(ip), @"extern".name.toSlice(ip), @@ -12369,7 +12378,13 @@ fn genCall(self: *Self, info: union(enum) { try self.genSetReg(.rax, Type.usize, .{ .air_ref = callee }, .{}); try self.asmRegister(.{ ._, .call }, .rax); }, - .lib => |lib| try self.genExternSymbolRef(.call, lib.lib, lib.callee), + .lib => |lib| if (self.bin_file.cast(.elf)) |elf_file| { + const target_sym_index = try elf_file.getGlobalSymbol(lib.callee, lib.lib); + try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ + .atom_index = try self.owner.getSymbolIndex(self), + .sym_index = target_sym_index, + })); + } else try self.genExternSymbolRef(.call, lib.lib, lib.callee), } return call_info.return_value.short; } @@ -15245,16 +15260,7 @@ fn genExternSymbolRef( callee: []const u8, ) InnerError!void { const atom_index = try self.owner.getSymbolIndex(self); - if (self.bin_file.cast(.elf)) |elf_file| { - _ = try self.addInst(.{ - .tag = tag, - .ops = .extern_fn_reloc, - .data = .{ .reloc = .{ - .atom_index = atom_index, - .sym_index = try elf_file.getGlobalSymbol(callee, lib), - } }, - }); - } else if (self.bin_file.cast(.coff)) |coff_file| { + if (self.bin_file.cast(.coff)) |coff_file| { const global_index = try coff_file.getGlobalSymbol(callee, lib); _ = try self.addInst(.{ .tag = .mov, From edd72c75eb4164f5799d56128ce6e17668efb0e2 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 13 Aug 2024 10:41:57 +0200 Subject: [PATCH 19/39] elf: fix circular dep loop in zig jump table --- src/link/Elf/ZigObject.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index 9069e180a3..70f6a2500d 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -1817,7 +1817,7 @@ pub const JumpTable = struct { pub fn targetAddress(jt: JumpTable, index: Index, zo: *ZigObject, elf_file: *Elf) i64 { const sym_index = jt.entries.items(.sym_index)[index]; - return zo.symbol(sym_index).address(.{}, elf_file); + return zo.symbol(sym_index).address(.{ .zjt = false }, elf_file); } const max_jump_seq_len = 12; From afaec5c3e477a66695fdfca30bcc662fd1c73cd4 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 13 Aug 2024 11:11:16 +0200 Subject: [PATCH 20/39] x86_64: fix generating lazy symbol refs --- src/arch/x86_64/CodeGen.zig | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 116911c409..1edf208d4c 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -15304,7 +15304,7 @@ fn genLazySymbolRef( if (self.mod.pic) { switch (tag) { .lea, .call => try self.genSetReg(reg, Type.usize, .{ - .load_symbol = .{ .sym = sym_index }, + .lea_symbol = .{ .sym = sym_index }, }, .{}), .mov => try self.genSetReg(reg, Type.usize, .{ .load_symbol = .{ .sym = sym_index }, @@ -15322,14 +15322,11 @@ fn genLazySymbolRef( .sym_index = sym_index, }; switch (tag) { - .lea, .mov => try self.asmRegisterMemory(.{ ._, .mov }, reg.to64(), .{ - .base = .{ .reloc = reloc }, - .mod = .{ .rm = .{ .size = .qword } }, - }), - .call => try self.asmMemory(.{ ._, .call }, .{ + .lea, .mov => try self.asmRegisterMemory(.{ ._, tag }, reg.to64(), .{ .base = .{ .reloc = reloc }, .mod = .{ .rm = .{ .size = .qword } }, }), + .call => try self.asmImmediate(.{ ._, .call }, Immediate.rel(reloc)), else => unreachable, } } From 39ee346681eac850f2f1f9bcb8ca6114dbad581a Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 13 Aug 2024 11:11:54 +0200 Subject: [PATCH 21/39] elf: make .text.zig bigger now that jump table is part of it --- src/link/Elf.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/link/Elf.zig b/src/link/Elf.zig index f4003ff38f..dce5d45c80 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -604,7 +604,7 @@ pub fn initMetadata(self: *Elf, options: InitMetadataOptions) !void { .type = elf.PT_LOAD, .offset = off, .filesz = filesz, - .addr = if (ptr_bit_width >= 32) 0x8000000 else 0x8000, + .addr = if (ptr_bit_width >= 32) 0x4000000 else 0x4000, .memsz = filesz, .@"align" = self.page_size, .flags = elf.PF_X | elf.PF_R | elf.PF_W, From df80ccf7600818faafd485b4c01bd4fe0f26a8e5 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 13 Aug 2024 12:01:56 +0200 Subject: [PATCH 22/39] elf: pretty print alingment when pretty printing atoms --- src/link/Elf/Atom.zig | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/link/Elf/Atom.zig b/src/link/Elf/Atom.zig index e067e53c10..6e68b84952 100644 --- a/src/link/Elf/Atom.zig +++ b/src/link/Elf/Atom.zig @@ -1022,7 +1022,7 @@ pub fn format( _ = unused_fmt_string; _ = options; _ = writer; - @compileError("do not format symbols directly"); + @compileError("do not format Atom directly"); } pub fn fmt(atom: Atom, elf_file: *Elf) std.fmt.Formatter(format2) { @@ -1048,8 +1048,8 @@ fn format2( const atom = ctx.atom; const elf_file = ctx.elf_file; try writer.print("atom({d}) : {s} : @{x} : shdr({d}) : align({x}) : size({x})", .{ - atom.atom_index, atom.name(elf_file), atom.address(elf_file), - atom.output_section_index, atom.alignment, atom.size, + atom.atom_index, atom.name(elf_file), atom.address(elf_file), + atom.output_section_index, atom.alignment.toByteUnits() orelse 0, atom.size, }); if (atom.fdes(elf_file).len > 0) { try writer.writeAll(" : fdes{ "); From 9daf5e81c4bdc413a3bb11c1bab207c19e6ba547 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 13 Aug 2024 12:08:21 +0200 Subject: [PATCH 23/39] elf: commit non-indirected symbol address to symtab --- src/link/Elf/Symbol.zig | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/link/Elf/Symbol.zig b/src/link/Elf/Symbol.zig index 8b30f0ab03..1f81b3ff98 100644 --- a/src/link/Elf/Symbol.zig +++ b/src/link/Elf/Symbol.zig @@ -300,11 +300,11 @@ pub fn setOutputSym(symbol: Symbol, elf_file: *Elf, out: *elf.Elf64_Sym) void { if (symbol.flags.is_canonical) break :blk symbol.address(.{}, elf_file); break :blk 0; } - if (st_shndx == elf.SHN_ABS or st_shndx == elf.SHN_COMMON) break :blk symbol.address(.{ .plt = false }, elf_file); + if (st_shndx == elf.SHN_ABS or st_shndx == elf.SHN_COMMON) break :blk symbol.address(.{ .plt = false, .zjt = false }, elf_file); const shdr = elf_file.shdrs.items[st_shndx]; if (shdr.sh_flags & elf.SHF_TLS != 0 and file_ptr != .linker_defined) - break :blk symbol.address(.{ .plt = false }, elf_file) - elf_file.tlsAddress(); - break :blk symbol.address(.{ .plt = false }, elf_file); + break :blk symbol.address(.{ .plt = false, .zjt = false }, elf_file) - elf_file.tlsAddress(); + break :blk symbol.address(.{ .plt = false, .zjt = false }, elf_file); }; out.st_info = (st_bind << 4) | st_type; out.st_other = esym.st_other; @@ -323,7 +323,7 @@ pub fn format( _ = unused_fmt_string; _ = options; _ = writer; - @compileError("do not format symbols directly"); + @compileError("do not format Symbol directly"); } const FormatContext = struct { From 49d78cc793f6dd987d2a51c7a96333713264691f Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 13 Aug 2024 12:50:53 +0200 Subject: [PATCH 24/39] elf: only apply zig jump table indirection to function calls (PLT32) --- src/link/Elf.zig | 2 +- src/link/Elf/Atom.zig | 7 +++++-- src/link/Elf/Symbol.zig | 10 +++++----- src/link/Elf/ZigObject.zig | 12 ++++++------ 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/src/link/Elf.zig b/src/link/Elf.zig index dce5d45c80..50dfeffa17 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -2798,7 +2798,7 @@ pub fn writeElfHeader(self: *Elf) !void { const e_entry: u64 = if (self.linkerDefinedPtr()) |obj| blk: { const entry_sym = obj.entrySymbol(self) orelse break :blk 0; - break :blk @intCast(entry_sym.address(.{}, self)); + break :blk @intCast(entry_sym.address(.{ .zjt = true }, self)); } else 0; const phdr_table_offset = if (self.phdr_table_index) |phndx| self.phdrs.items[phndx].p_offset else 0; switch (self.ptr_width) { diff --git a/src/link/Elf/Atom.zig b/src/link/Elf/Atom.zig index 6e68b84952..7338927ebd 100644 --- a/src/link/Elf/Atom.zig +++ b/src/link/Elf/Atom.zig @@ -747,7 +747,7 @@ pub fn resolveRelocsAlloc(self: Atom, elf_file: *Elf, code: []u8) RelocError!voi // Addend from the relocation. const A = rel.r_addend; // Address of the target symbol - can be address of the symbol within an atom or address of PLT stub. - const S = target.address(.{}, elf_file); + const S = target.address(.{ .zjt = false }, elf_file); // Address of the global offset table. const GOT = elf_file.gotAddress(); // Relative offset to the start of the global offset table. @@ -1212,7 +1212,10 @@ const x86_64 = struct { ); }, - .PLT32 => try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little), + .PLT32 => { + const S_ = if (target.flags.has_zjt) target.address(.{ .zjt = true }, elf_file) else S; + try cwriter.writeInt(i32, @as(i32, @intCast(S_ + A - P)), .little); + }, .PC32 => try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little), .GOTPCREL => try cwriter.writeInt(i32, @as(i32, @intCast(G + GOT + A - P)), .little), diff --git a/src/link/Elf/Symbol.zig b/src/link/Elf/Symbol.zig index 1f81b3ff98..d946e95511 100644 --- a/src/link/Elf/Symbol.zig +++ b/src/link/Elf/Symbol.zig @@ -101,7 +101,7 @@ pub fn symbolRank(symbol: Symbol, elf_file: *Elf) u32 { return file_ptr.symbolRank(sym, in_archive); } -pub fn address(symbol: Symbol, opts: struct { plt: bool = true, zjt: bool = true }, elf_file: *Elf) i64 { +pub fn address(symbol: Symbol, opts: struct { plt: bool = true, zjt: bool = false }, elf_file: *Elf) i64 { if (symbol.mergeSubsection(elf_file)) |msub| { if (!msub.alive) return 0; return msub.address(elf_file) + symbol.value; @@ -300,11 +300,11 @@ pub fn setOutputSym(symbol: Symbol, elf_file: *Elf, out: *elf.Elf64_Sym) void { if (symbol.flags.is_canonical) break :blk symbol.address(.{}, elf_file); break :blk 0; } - if (st_shndx == elf.SHN_ABS or st_shndx == elf.SHN_COMMON) break :blk symbol.address(.{ .plt = false, .zjt = false }, elf_file); + if (st_shndx == elf.SHN_ABS or st_shndx == elf.SHN_COMMON) break :blk symbol.address(.{ .plt = false }, elf_file); const shdr = elf_file.shdrs.items[st_shndx]; if (shdr.sh_flags & elf.SHF_TLS != 0 and file_ptr != .linker_defined) - break :blk symbol.address(.{ .plt = false, .zjt = false }, elf_file) - elf_file.tlsAddress(); - break :blk symbol.address(.{ .plt = false, .zjt = false }, elf_file); + break :blk symbol.address(.{ .plt = false }, elf_file) - elf_file.tlsAddress(); + break :blk symbol.address(.{ .plt = false }, elf_file); }; out.st_info = (st_bind << 4) | st_type; out.st_other = esym.st_other; @@ -380,7 +380,7 @@ fn format2( try writer.print("%{d} : {s} : @{x}", .{ symbol.esym_index, symbol.fmtName(elf_file), - symbol.address(.{ .plt = false, .zjt = false }, elf_file), + symbol.address(.{ .plt = false }, elf_file), }); if (symbol.file(elf_file)) |file_ptr| { if (symbol.isAbs(elf_file)) { diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index 70f6a2500d..865f4118d3 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -665,7 +665,7 @@ pub fn getNavVAddr( else => try self.getOrCreateMetadataForNav(elf_file, nav_index), }; const this_sym = self.symbol(this_sym_index); - const vaddr = this_sym.address(.{}, elf_file); + const vaddr = this_sym.address(.{ .zjt = true }, elf_file); const parent_atom = self.symbol(reloc_info.parent_atom_index).atom(elf_file).?; const r_type = relocation.encode(.abs, elf_file.getTarget().cpu.arch); try parent_atom.addReloc(elf_file, .{ @@ -942,7 +942,7 @@ fn updateNavCode( .len = code.len, }}; var remote_vec: [1]std.posix.iovec_const = .{.{ - .base = @as([*]u8, @ptrFromInt(@as(usize, @intCast(sym.address(.{}, elf_file))))), + .base = @as([*]u8, @ptrFromInt(@as(usize, @intCast(sym.address(.{ .zjt = true }, elf_file))))), .len = code.len, }}; const rc = std.os.linux.process_vm_writev(pid, &code_vec, &remote_vec, 0); @@ -1092,7 +1092,7 @@ pub fn updateFunc( try self.dwarf.?.commitNavState( pt, func.owner_nav, - @intCast(sym.address(.{}, elf_file)), + @intCast(sym.address(.{ .zjt = true }, elf_file)), sym.atom(elf_file).?.size, ds, ); @@ -1189,7 +1189,7 @@ pub fn updateNav( try self.dwarf.?.commitNavState( pt, nav_index, - @intCast(sym.address(.{}, elf_file)), + @intCast(sym.address(.{ .zjt = true }, elf_file)), sym.atom(elf_file).?.size, ns, ); @@ -1817,7 +1817,7 @@ pub const JumpTable = struct { pub fn targetAddress(jt: JumpTable, index: Index, zo: *ZigObject, elf_file: *Elf) i64 { const sym_index = jt.entries.items(.sym_index)[index]; - return zo.symbol(sym_index).address(.{ .zjt = false }, elf_file); + return zo.symbol(sym_index).address(.{}, elf_file); } const max_jump_seq_len = 12; @@ -1897,8 +1897,8 @@ pub const JumpTable = struct { for (jt.entries.items(.sym_index), jt.entries.items(.dirty)) |sym_index, dirty| { const sym = zo.symbol(sym_index); try writer.print(" {x} => {x} : %{d} : {s}", .{ + sym.address(.{ .zjt = true }, ef), sym.address(.{}, ef), - sym.address(.{ .zjt = false }, ef), sym_index, sym.name(ef), }); From 97ab420dcf9bdfe954d6bff730a7492abaeb90bb Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 13 Aug 2024 12:51:17 +0200 Subject: [PATCH 25/39] elf: do not emit zig jump table in relocatables --- src/link/Elf/ZigObject.zig | 39 ++++++++++++++++++++------------------ 1 file changed, 21 insertions(+), 18 deletions(-) diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index 865f4118d3..4a3bdd6049 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -1036,14 +1036,16 @@ pub fn updateFunc( const ip = &zcu.intern_pool; const gpa = elf_file.base.comp.gpa; const func = zcu.funcInfo(func_index); - const jump_table = self.jumpTablePtr() orelse try self.initJumpTable(gpa, elf_file); + if (elf_file.base.isRelocatable() and self.jumpTablePtr() == null) { + try self.initJumpTable(gpa, elf_file); + } log.debug("updateFunc {}({d})", .{ ip.getNav(func.owner_nav).fqn.fmt(ip), func.owner_nav }); const sym_index = try self.getOrCreateMetadataForNav(elf_file, func.owner_nav); self.symbol(sym_index).atom(elf_file).?.freeRelocs(elf_file); - { + if (self.jumpTablePtr()) |jump_table| { const sym = self.symbol(sym_index); if (!sym.flags.has_zjt) { const index = try jump_table.addSymbol(gpa, sym_index); @@ -1100,21 +1102,23 @@ pub fn updateFunc( // Exports will be updated by `Zcu.processExports` after the update. - if (jump_table.dirty) { - // TODO write in bulk - for (jump_table.entries.items(.dirty), 0..) |*dirty, i| { - try jump_table.writeEntry(@intCast(i), self, elf_file); - dirty.* = false; + if (self.jumpTablePtr()) |jump_table| { + if (jump_table.dirty) { + // TODO write in bulk + for (jump_table.entries.items(.dirty), 0..) |*dirty, i| { + try jump_table.writeEntry(@intCast(i), self, elf_file); + dirty.* = false; + } + } else { + const sym = self.symbol(sym_index); + const jt_index = sym.extra(elf_file).zjt; + var jt_entry = jump_table.entries.get(jt_index); + if (jt_entry.dirty) { + try jump_table.writeEntry(jt_index, self, elf_file); + jt_entry.dirty = false; + } + jump_table.entries.set(jt_index, jt_entry); } - } else { - const sym = self.symbol(sym_index); - const jt_index = sym.extra(elf_file).zjt; - var jt_entry = jump_table.entries.get(jt_index); - if (jt_entry.dirty) { - try jump_table.writeEntry(jt_index, self, elf_file); - jt_entry.dirty = false; - } - jump_table.entries.set(jt_index, jt_entry); } } @@ -1474,7 +1478,7 @@ pub fn jumpTablePtr(self: *ZigObject) ?*JumpTable { return if (self.jump_table) |*jt| jt else null; } -fn initJumpTable(self: *ZigObject, allocator: Allocator, elf_file: *Elf) error{OutOfMemory}!*JumpTable { +fn initJumpTable(self: *ZigObject, allocator: Allocator, elf_file: *Elf) error{OutOfMemory}!void { const name_off = try self.addString(allocator, "__zig_jump_table"); const sym_index = try self.newSymbolWithAtom(allocator, name_off); const sym = self.symbol(sym_index); @@ -1485,7 +1489,6 @@ fn initJumpTable(self: *ZigObject, allocator: Allocator, elf_file: *Elf) error{O atom_ptr.alignment = Atom.Alignment.fromNonzeroByteUnits(JumpTable.alignment(elf_file.getTarget().cpu.arch)); atom_ptr.output_section_index = elf_file.zig_text_section_index.?; self.jump_table = JumpTable{ .sym_index = sym_index }; - return &(self.jump_table.?); } pub fn asFile(self: *ZigObject) File { From 1bd54a55fa40db9f91d8a0c1cf5244ff51be6081 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 13 Aug 2024 21:52:40 +0200 Subject: [PATCH 26/39] fix compile errors in other codegen backends --- src/arch/aarch64/CodeGen.zig | 41 ++++++------------------------------ src/arch/arm/CodeGen.zig | 20 +++--------------- src/arch/riscv64/CodeGen.zig | 1 + src/arch/riscv64/Emit.zig | 13 +----------- src/arch/sparc64/CodeGen.zig | 32 +++------------------------- src/link/Elf/ZigObject.zig | 2 +- 6 files changed, 16 insertions(+), 93 deletions(-) diff --git a/src/arch/aarch64/CodeGen.zig b/src/arch/aarch64/CodeGen.zig index 2810b6b521..231db8e98c 100644 --- a/src/arch/aarch64/CodeGen.zig +++ b/src/arch/aarch64/CodeGen.zig @@ -4352,24 +4352,10 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier // on linking. if (try self.air.value(callee, pt)) |func_value| switch (ip.indexToKey(func_value.toIntern())) { .func => |func| { - if (self.bin_file.cast(.elf)) |elf_file| { - const zo = elf_file.zigObjectPtr().?; - const sym_index = try zo.getOrCreateMetadataForNav(elf_file, func.owner_nav); - const sym = zo.symbol(sym_index); - _ = try sym.getOrCreateZigGotEntry(sym_index, elf_file); - const got_addr = @as(u32, @intCast(sym.zigGotAddress(elf_file))); - try self.genSetReg(Type.usize, .x30, .{ .memory = got_addr }); - } else if (self.bin_file.cast(.macho)) |macho_file| { - _ = macho_file; - @panic("TODO airCall"); - // const atom = try macho_file.getOrCreateAtomForNav(func.owner_nav); - // const sym_index = macho_file.getAtom(atom).getSymbolIndex().?; - // try self.genSetReg(Type.u64, .x30, .{ - // .linker_load = .{ - // .type = .got, - // .sym_index = sym_index, - // }, - // }); + if (self.bin_file.cast(.elf)) |_| { + return self.fail("TODO implement calling functions for Elf", .{}); + } else if (self.bin_file.cast(.macho)) |_| { + return self.fail("TODO implement calling functions for MachO", .{}); } else if (self.bin_file.cast(.coff)) |coff_file| { const atom = try coff_file.getOrCreateAtomForNav(func.owner_nav); const sym_index = coff_file.getAtom(atom).getSymbolIndex().?; @@ -4393,21 +4379,8 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier .@"extern" => |@"extern"| { const nav_name = ip.getNav(@"extern".owner_nav).name.toSlice(ip); const lib_name = @"extern".lib_name.toSlice(ip); - if (self.bin_file.cast(.macho)) |macho_file| { - _ = macho_file; - @panic("TODO airCall"); - // const sym_index = try macho_file.getGlobalSymbol(nav_name, lib_name); - // const atom = try macho_file.getOrCreateAtomForNav(self.owner_nav); - // const atom_index = macho_file.getAtom(atom).getSymbolIndex().?; - // _ = try self.addInst(.{ - // .tag = .call_extern, - // .data = .{ - // .relocation = .{ - // .atom_index = atom_index, - // .sym_index = sym_index, - // }, - // }, - // }); + if (self.bin_file.cast(.macho)) |_| { + return self.fail("TODO implement calling extern functions for MachO", .{}); } else if (self.bin_file.cast(.coff)) |coff_file| { const sym_index = try coff_file.getGlobalSymbol(nav_name, lib_name); try self.genSetReg(Type.u64, .x30, .{ @@ -6234,7 +6207,7 @@ fn genTypedValue(self: *Self, val: Value) InnerError!MCValue { .memory => |addr| .{ .memory = addr }, .load_got => |sym_index| .{ .linker_load = .{ .type = .got, .sym_index = sym_index } }, .load_direct => |sym_index| .{ .linker_load = .{ .type = .direct, .sym_index = sym_index } }, - .load_symbol, .load_tlv => unreachable, // TODO + .load_symbol, .load_tlv, .lea_symbol => unreachable, // TODO }, .fail => |msg| { self.err_msg = msg; diff --git a/src/arch/arm/CodeGen.zig b/src/arch/arm/CodeGen.zig index e4d106921e..1c03699552 100644 --- a/src/arch/arm/CodeGen.zig +++ b/src/arch/arm/CodeGen.zig @@ -4333,22 +4333,8 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier // Due to incremental compilation, how function calls are generated depends // on linking. if (try self.air.value(callee, pt)) |func_value| switch (ip.indexToKey(func_value.toIntern())) { - .func => |func| { - if (self.bin_file.cast(.elf)) |elf_file| { - const zo = elf_file.zigObjectPtr().?; - const sym_index = try zo.getOrCreateMetadataForNav(elf_file, func.owner_nav); - const sym = zo.symbol(sym_index); - _ = try sym.getOrCreateZigGotEntry(sym_index, elf_file); - const got_addr: u32 = @intCast(sym.zigGotAddress(elf_file)); - try self.genSetReg(Type.usize, .lr, .{ .memory = got_addr }); - } else if (self.bin_file.cast(.macho)) |_| { - unreachable; // unsupported architecture for MachO - } else { - return self.fail("TODO implement call on {s} for {s}", .{ - @tagName(self.bin_file.tag), - @tagName(self.target.cpu.arch), - }); - } + .func => { + return self.fail("TODO implement calling functions", .{}); }, .@"extern" => { return self.fail("TODO implement calling extern functions", .{}); @@ -6184,7 +6170,7 @@ fn genTypedValue(self: *Self, val: Value) InnerError!MCValue { .mcv => |mcv| switch (mcv) { .none => .none, .undef => .undef, - .load_got, .load_symbol, .load_direct, .load_tlv => unreachable, // TODO + .load_got, .load_symbol, .load_direct, .load_tlv, .lea_symbol => unreachable, // TODO .immediate => |imm| .{ .immediate = @truncate(imm) }, .memory => |addr| .{ .memory = addr }, }, diff --git a/src/arch/riscv64/CodeGen.zig b/src/arch/riscv64/CodeGen.zig index b5384542e7..e9f593485f 100644 --- a/src/arch/riscv64/CodeGen.zig +++ b/src/arch/riscv64/CodeGen.zig @@ -8026,6 +8026,7 @@ fn genTypedValue(func: *Func, val: Value) InnerError!MCValue { .mcv => |mcv| switch (mcv) { .none => .none, .undef => unreachable, + .lea_symbol => |sym_index| .{ .lea_symbol = .{ .sym = sym_index } }, .load_symbol => |sym_index| .{ .load_symbol = .{ .sym = sym_index } }, .load_tlv => |sym_index| .{ .lea_tlv = sym_index }, .immediate => |imm| .{ .immediate = imm }, diff --git a/src/arch/riscv64/Emit.zig b/src/arch/riscv64/Emit.zig index 137bc39572..c1e7565c0b 100644 --- a/src/arch/riscv64/Emit.zig +++ b/src/arch/riscv64/Emit.zig @@ -43,12 +43,6 @@ pub fn emitMir(emit: *Emit) Error!void { .fmt = std.meta.activeTag(lowered_inst), }), .load_symbol_reloc => |symbol| { - const is_obj_or_static_lib = switch (emit.lower.output_mode) { - .Exe => false, - .Obj => true, - .Lib => emit.lower.link_mode == .static, - }; - const elf_file = emit.bin_file.cast(.elf).?; const zo = elf_file.zigObjectPtr().?; @@ -58,12 +52,7 @@ pub fn emitMir(emit: *Emit) Error!void { var hi_r_type: u32 = @intFromEnum(std.elf.R_RISCV.HI20); var lo_r_type: u32 = @intFromEnum(std.elf.R_RISCV.LO12_I); - if (sym.flags.needs_zig_got and !is_obj_or_static_lib) { - _ = try sym.getOrCreateZigGotEntry(symbol.sym_index, elf_file); - - hi_r_type = Elf.R_ZIG_GOT_HI20; - lo_r_type = Elf.R_ZIG_GOT_LO12; - } else if (sym.flags.needs_got) { + if (sym.flags.needs_got) { hi_r_type = Elf.R_GOT_HI20_STATIC; // TODO: rework this #20887 lo_r_type = Elf.R_GOT_LO12_I_STATIC; // TODO: rework this #20887 } diff --git a/src/arch/sparc64/CodeGen.zig b/src/arch/sparc64/CodeGen.zig index bba8cdec34..d0b0eedc57 100644 --- a/src/arch/sparc64/CodeGen.zig +++ b/src/arch/sparc64/CodeGen.zig @@ -1349,34 +1349,8 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallModifier // Due to incremental compilation, how function calls are generated depends // on linking. if (try self.air.value(callee, pt)) |func_value| switch (ip.indexToKey(func_value.toIntern())) { - .func => |func| { - const got_addr = if (self.bin_file.cast(.elf)) |elf_file| blk: { - const zo = elf_file.zigObjectPtr().?; - const sym_index = try zo.getOrCreateMetadataForNav(elf_file, func.owner_nav); - const sym = zo.symbol(sym_index); - _ = try sym.getOrCreateZigGotEntry(sym_index, elf_file); - break :blk @as(u32, @intCast(sym.zigGotAddress(elf_file))); - } else @panic("TODO SPARCv9 currently does not support non-ELF binaries"); - - try self.genSetReg(Type.usize, .o7, .{ .memory = got_addr }); - - _ = try self.addInst(.{ - .tag = .jmpl, - .data = .{ - .arithmetic_3op = .{ - .is_imm = false, - .rd = .o7, - .rs1 = .o7, - .rs2_or_imm = .{ .rs2 = .g0 }, - }, - }, - }); - - // TODO Find a way to fill this delay slot - _ = try self.addInst(.{ - .tag = .nop, - .data = .{ .nop = {} }, - }); + .func => { + return self.fail("TODO implement calling functions", .{}); }, .@"extern" => { return self.fail("TODO implement calling extern functions", .{}); @@ -4153,7 +4127,7 @@ fn genTypedValue(self: *Self, val: Value) InnerError!MCValue { .mcv => |mcv| switch (mcv) { .none => .none, .undef => .undef, - .load_got, .load_symbol, .load_direct, .load_tlv => unreachable, // TODO + .load_got, .load_symbol, .load_direct, .load_tlv, .lea_symbol => unreachable, // TODO .immediate => |imm| .{ .immediate = imm }, .memory => |addr| .{ .memory = addr }, }, diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index 4a3bdd6049..2adf2532c7 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -1036,7 +1036,7 @@ pub fn updateFunc( const ip = &zcu.intern_pool; const gpa = elf_file.base.comp.gpa; const func = zcu.funcInfo(func_index); - if (elf_file.base.isRelocatable() and self.jumpTablePtr() == null) { + if (!elf_file.base.isRelocatable() and self.jumpTablePtr() == null) { try self.initJumpTable(gpa, elf_file); } From f26573fddfcebe44d4d69ea0e01d345fc1b15835 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 14 Aug 2024 12:10:28 +0200 Subject: [PATCH 27/39] elf: re-use old atom slot for a trampoline to that atom This is the initial implementation of Jacob Young's idea of re-using old function slots as trampolines for new function's location. This way the trampoline is guaranteed to be aligned to the function's alignment. The only edge case is if an incremental update further overaligns the function in which case we skip/delete the trampoline and re-evaluate all references. --- src/link/Elf.zig | 5 +- src/link/Elf/Atom.zig | 9 +- src/link/Elf/Symbol.zig | 27 ++-- src/link/Elf/ZigObject.zig | 323 +++++++++++-------------------------- 4 files changed, 112 insertions(+), 252 deletions(-) diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 50dfeffa17..1c2f90d14c 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -2798,7 +2798,7 @@ pub fn writeElfHeader(self: *Elf) !void { const e_entry: u64 = if (self.linkerDefinedPtr()) |obj| blk: { const entry_sym = obj.entrySymbol(self) orelse break :blk 0; - break :blk @intCast(entry_sym.address(.{ .zjt = true }, self)); + break :blk @intCast(entry_sym.address(.{}, self)); } else 0; const phdr_table_offset = if (self.phdr_table_index) |phndx| self.phdrs.items[phndx].p_offset else 0; switch (self.ptr_width) { @@ -5599,9 +5599,6 @@ fn fmtDumpState( zig_object.fmtAtoms(self), zig_object.fmtSymtab(self), }); - if (zig_object.jump_table) |jt| { - try writer.print("{}", .{jt.fmt(zig_object, self)}); - } try writer.writeByte('\n'); } diff --git a/src/link/Elf/Atom.zig b/src/link/Elf/Atom.zig index 7338927ebd..47bb216080 100644 --- a/src/link/Elf/Atom.zig +++ b/src/link/Elf/Atom.zig @@ -746,8 +746,8 @@ pub fn resolveRelocsAlloc(self: Atom, elf_file: *Elf, code: []u8) RelocError!voi const P = self.address(elf_file) + @as(i64, @intCast(rel.r_offset)); // Addend from the relocation. const A = rel.r_addend; - // Address of the target symbol - can be address of the symbol within an atom or address of PLT stub. - const S = target.address(.{ .zjt = false }, elf_file); + // Address of the target symbol - can be address of the symbol within an atom or address of PLT stub, or address of a Zig trampoline. + const S = target.address(.{}, elf_file); // Address of the global offset table. const GOT = elf_file.gotAddress(); // Relative offset to the start of the global offset table. @@ -1212,10 +1212,7 @@ const x86_64 = struct { ); }, - .PLT32 => { - const S_ = if (target.flags.has_zjt) target.address(.{ .zjt = true }, elf_file) else S; - try cwriter.writeInt(i32, @as(i32, @intCast(S_ + A - P)), .little); - }, + .PLT32 => try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little), .PC32 => try cwriter.writeInt(i32, @as(i32, @intCast(S + A - P)), .little), .GOTPCREL => try cwriter.writeInt(i32, @as(i32, @intCast(G + GOT + A - P)), .little), diff --git a/src/link/Elf/Symbol.zig b/src/link/Elf/Symbol.zig index d946e95511..4beb6a0918 100644 --- a/src/link/Elf/Symbol.zig +++ b/src/link/Elf/Symbol.zig @@ -101,7 +101,7 @@ pub fn symbolRank(symbol: Symbol, elf_file: *Elf) u32 { return file_ptr.symbolRank(sym, in_archive); } -pub fn address(symbol: Symbol, opts: struct { plt: bool = true, zjt: bool = false }, elf_file: *Elf) i64 { +pub fn address(symbol: Symbol, opts: struct { plt: bool = true, trampoline: bool = true }, elf_file: *Elf) i64 { if (symbol.mergeSubsection(elf_file)) |msub| { if (!msub.alive) return 0; return msub.address(elf_file) + symbol.value; @@ -109,8 +109,8 @@ pub fn address(symbol: Symbol, opts: struct { plt: bool = true, zjt: bool = fals if (symbol.flags.has_copy_rel) { return symbol.copyRelAddress(elf_file); } - if (symbol.flags.has_zjt and opts.zjt) { - return symbol.zjtAddress(elf_file); + if (symbol.flags.has_trampoline and opts.trampoline) { + return symbol.trampolineAddress(elf_file); } if (symbol.flags.has_plt and opts.plt) { if (!symbol.flags.is_canonical and symbol.flags.has_got) { @@ -220,12 +220,11 @@ pub fn tlsDescAddress(symbol: Symbol, elf_file: *Elf) i64 { return entry.address(elf_file); } -pub fn zjtAddress(symbol: Symbol, elf_file: *Elf) i64 { - if (!symbol.flags.has_zjt) return 0; +pub fn trampolineAddress(symbol: Symbol, elf_file: *Elf) i64 { + if (!symbol.flags.has_trampoline) return 0; const zo = elf_file.zigObjectPtr().?; - const jump_table = zo.jumpTablePtr().?; - const index = symbol.extra(elf_file).zjt; - return jump_table.entryAddress(index, zo, elf_file); + const index = symbol.extra(elf_file).trampoline; + return zo.symbol(index).address(.{}, elf_file); } pub fn dsoAlignment(symbol: Symbol, elf_file: *Elf) !u64 { @@ -251,7 +250,7 @@ const AddExtraOpts = struct { tlsgd: ?u32 = null, gottp: ?u32 = null, tlsdesc: ?u32 = null, - zjt: ?u32 = null, + trampoline: ?u32 = null, }; pub fn addExtra(symbol: *Symbol, opts: AddExtraOpts, elf_file: *Elf) void { @@ -304,7 +303,7 @@ pub fn setOutputSym(symbol: Symbol, elf_file: *Elf, out: *elf.Elf64_Sym) void { const shdr = elf_file.shdrs.items[st_shndx]; if (shdr.sh_flags & elf.SHF_TLS != 0 and file_ptr != .linker_defined) break :blk symbol.address(.{ .plt = false }, elf_file) - elf_file.tlsAddress(); - break :blk symbol.address(.{ .plt = false }, elf_file); + break :blk symbol.address(.{ .plt = false, .trampoline = false }, elf_file); }; out.st_info = (st_bind << 4) | st_type; out.st_other = esym.st_other; @@ -380,7 +379,7 @@ fn format2( try writer.print("%{d} : {s} : @{x}", .{ symbol.esym_index, symbol.fmtName(elf_file), - symbol.address(.{ .plt = false }, elf_file), + symbol.address(.{ .plt = false, .trampoline = false }, elf_file), }); if (symbol.file(elf_file)) |file_ptr| { if (symbol.isAbs(elf_file)) { @@ -456,8 +455,8 @@ pub const Flags = packed struct { /// Whether the symbol is a merge subsection. merge_subsection: bool = false, - /// Whether the symbol has __zig_jump_table indirection. - has_zjt: bool = false, + /// Whether the symbol has a trampoline. + has_trampoline: bool = false, }; pub const Extra = struct { @@ -471,7 +470,7 @@ pub const Extra = struct { gottp: u32 = 0, tlsdesc: u32 = 0, merge_section: u32 = 0, - zjt: u32 = 0, + trampoline: u32 = 0, }; pub const Index = u32; diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index 2adf2532c7..a472c6fad0 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -55,11 +55,6 @@ debug_str_section_zig_size: u64 = 0, debug_aranges_section_zig_size: u64 = 0, debug_line_section_zig_size: u64 = 0, -/// Function jump table containing trampolines to Zcu functions. -/// The table is used for Zig's incremental compilation and is embedded with -/// the machine code section. -jump_table: ?JumpTable = null, - pub const global_symbol_bit: u32 = 0x80000000; pub const symbol_mask: u32 = 0x7fffffff; pub const SHN_ATOM: u16 = 0x100; @@ -127,10 +122,6 @@ pub fn deinit(self: *ZigObject, allocator: Allocator) void { if (self.dwarf) |*dw| { dw.deinit(); } - - if (self.jump_table) |*jt| { - jt.deinit(allocator); - } } pub fn flushModule(self: *ZigObject, elf_file: *Elf, tid: Zcu.PerThread.Id) !void { @@ -665,7 +656,7 @@ pub fn getNavVAddr( else => try self.getOrCreateMetadataForNav(elf_file, nav_index), }; const this_sym = self.symbol(this_sym_index); - const vaddr = this_sym.address(.{ .zjt = true }, elf_file); + const vaddr = this_sym.address(.{}, elf_file); const parent_atom = self.symbol(reloc_info.parent_atom_index).atom(elf_file).?; const r_type = relocation.encode(.abs, elf_file.getTarget().cpu.arch); try parent_atom.addReloc(elf_file, .{ @@ -914,12 +905,6 @@ fn updateNavCode( if (old_vaddr != atom_ptr.value) { sym.value = 0; esym.st_value = 0; - - if (stt_bits == elf.STT_FUNC) { - const extra = sym.extra(elf_file); - const jump_table = self.jumpTablePtr().?; - jump_table.entries.items(.dirty)[extra.zjt] = true; - } } } else if (code.len < old_size) { atom_ptr.shrink(elf_file); @@ -942,7 +927,7 @@ fn updateNavCode( .len = code.len, }}; var remote_vec: [1]std.posix.iovec_const = .{.{ - .base = @as([*]u8, @ptrFromInt(@as(usize, @intCast(sym.address(.{ .zjt = true }, elf_file))))), + .base = @as([*]u8, @ptrFromInt(@as(usize, @intCast(sym.address(.{}, elf_file))))), .len = code.len, }}; const rc = std.os.linux.process_vm_writev(pid, &code_vec, &remote_vec, 0); @@ -1036,31 +1021,12 @@ pub fn updateFunc( const ip = &zcu.intern_pool; const gpa = elf_file.base.comp.gpa; const func = zcu.funcInfo(func_index); - if (!elf_file.base.isRelocatable() and self.jumpTablePtr() == null) { - try self.initJumpTable(gpa, elf_file); - } log.debug("updateFunc {}({d})", .{ ip.getNav(func.owner_nav).fqn.fmt(ip), func.owner_nav }); const sym_index = try self.getOrCreateMetadataForNav(elf_file, func.owner_nav); self.symbol(sym_index).atom(elf_file).?.freeRelocs(elf_file); - if (self.jumpTablePtr()) |jump_table| { - const sym = self.symbol(sym_index); - if (!sym.flags.has_zjt) { - const index = try jump_table.addSymbol(gpa, sym_index); - sym.flags.has_zjt = true; - sym.addExtra(.{ .zjt = index }, elf_file); - try jump_table.updateSize(self, elf_file); - const old_vaddr = jump_table.address(self, elf_file); - try self.symbol(jump_table.sym_index).atom(elf_file).?.allocate(elf_file); - const new_vaddr = jump_table.address(self, elf_file); - if (old_vaddr != new_vaddr) { - jump_table.dirty = true; - } - } - } - var code_buffer = std.ArrayList(u8).init(gpa); defer code_buffer.deinit(); @@ -1087,14 +1053,22 @@ pub fn updateFunc( }; const shndx = try self.getNavShdrIndex(elf_file, zcu, func.owner_nav, code); + const old_rva, const old_alignment = blk: { + const atom_ptr = self.symbol(sym_index).atom(elf_file).?; + break :blk .{ atom_ptr.value, atom_ptr.alignment }; + }; try self.updateNavCode(elf_file, pt, func.owner_nav, sym_index, shndx, code, elf.STT_FUNC); + const new_rva, const new_alignment = blk: { + const atom_ptr = self.symbol(sym_index).atom(elf_file).?; + break :blk .{ atom_ptr.value, atom_ptr.alignment }; + }; if (dwarf_state) |*ds| { const sym = self.symbol(sym_index); try self.dwarf.?.commitNavState( pt, func.owner_nav, - @intCast(sym.address(.{ .zjt = true }, elf_file)), + @intCast(sym.address(.{}, elf_file)), sym.atom(elf_file).?.size, ds, ); @@ -1102,23 +1076,39 @@ pub fn updateFunc( // Exports will be updated by `Zcu.processExports` after the update. - if (self.jumpTablePtr()) |jump_table| { - if (jump_table.dirty) { - // TODO write in bulk - for (jump_table.entries.items(.dirty), 0..) |*dirty, i| { - try jump_table.writeEntry(@intCast(i), self, elf_file); - dirty.* = false; - } - } else { - const sym = self.symbol(sym_index); - const jt_index = sym.extra(elf_file).zjt; - var jt_entry = jump_table.entries.get(jt_index); - if (jt_entry.dirty) { - try jump_table.writeEntry(jt_index, self, elf_file); - jt_entry.dirty = false; - } - jump_table.entries.set(jt_index, jt_entry); + if (old_rva != new_rva and old_rva > 0) { + // If we had to reallocate the function, we re-use the existing slot for a trampoline. + // In the rare case that the function has been further overaligned we skip creating a + // trampoline and update all symbols referring this function. + if (old_alignment.order(new_alignment) == .lt) { + @panic("TODO update all symbols referring this function"); } + + // Create a trampoline to the new location at `old_rva`. + if (!self.symbol(sym_index).flags.has_trampoline) { + const name = try std.fmt.allocPrint(gpa, "{s}$trampoline", .{ + self.symbol(sym_index).name(elf_file), + }); + defer gpa.free(name); + const name_off = try self.addString(gpa, name); + const tr_size = trampolineSize(elf_file.getTarget().cpu.arch); + const tr_sym_index = try self.newSymbolWithAtom(gpa, name_off); + const tr_sym = self.symbol(tr_sym_index); + const tr_esym = &self.symtab.items(.elf_sym)[tr_sym.esym_index]; + tr_esym.st_info |= elf.STT_OBJECT; + tr_esym.st_size = tr_size; + const tr_atom_ptr = tr_sym.atom(elf_file).?; + tr_atom_ptr.value = old_rva; + tr_atom_ptr.alive = true; + tr_atom_ptr.alignment = old_alignment; + tr_atom_ptr.output_section_index = elf_file.zig_text_section_index.?; + tr_atom_ptr.size = tr_size; + const target_sym = self.symbol(sym_index); + target_sym.addExtra(.{ .trampoline = tr_sym_index }, elf_file); + target_sym.flags.has_trampoline = true; + } + const target_sym = self.symbol(sym_index); + try writeTrampoline(self.symbol(target_sym.extra(elf_file).trampoline).*, target_sym.*, elf_file); } } @@ -1193,7 +1183,7 @@ pub fn updateNav( try self.dwarf.?.commitNavState( pt, nav_index, - @intCast(sym.address(.{ .zjt = true }, elf_file)), + @intCast(sym.address(.{}, elf_file)), sym.atom(elf_file).?.size, ns, ); @@ -1474,21 +1464,50 @@ pub fn getGlobalSymbol(self: *ZigObject, elf_file: *Elf, name: []const u8, lib_n return lookup_gop.value_ptr.*; } -pub fn jumpTablePtr(self: *ZigObject) ?*JumpTable { - return if (self.jump_table) |*jt| jt else null; +const max_trampoline_len = 12; + +fn trampolineSize(cpu_arch: std.Target.Cpu.Arch) u64 { + const len = switch (cpu_arch) { + .x86_64 => 5, // jmp rel32 + else => @panic("TODO implement trampoline size for this CPU arch"), + }; + comptime assert(len <= max_trampoline_len); + return len; } -fn initJumpTable(self: *ZigObject, allocator: Allocator, elf_file: *Elf) error{OutOfMemory}!void { - const name_off = try self.addString(allocator, "__zig_jump_table"); - const sym_index = try self.newSymbolWithAtom(allocator, name_off); - const sym = self.symbol(sym_index); - const esym = &self.symtab.items(.elf_sym)[sym.esym_index]; - esym.st_info |= elf.STT_OBJECT; - const atom_ptr = sym.atom(elf_file).?; - atom_ptr.alive = true; - atom_ptr.alignment = Atom.Alignment.fromNonzeroByteUnits(JumpTable.alignment(elf_file.getTarget().cpu.arch)); - atom_ptr.output_section_index = elf_file.zig_text_section_index.?; - self.jump_table = JumpTable{ .sym_index = sym_index }; +fn writeTrampoline(tr_sym: Symbol, target: Symbol, elf_file: *Elf) !void { + const atom_ptr = tr_sym.atom(elf_file).?; + const shdr = elf_file.shdrs.items[atom_ptr.output_section_index]; + const fileoff = shdr.sh_offset + @as(u64, @intCast(atom_ptr.value)); + const source_addr = tr_sym.address(.{}, elf_file); + const target_addr = target.address(.{ .trampoline = false }, elf_file); + var buf: [max_trampoline_len]u8 = undefined; + const out = switch (elf_file.getTarget().cpu.arch) { + .x86_64 => try x86_64.writeTrampolineCode(source_addr, target_addr, &buf), + else => @panic("TODO implement write trampoline for this CPU arch"), + }; + try elf_file.base.file.?.pwriteAll(out, fileoff); + + if (elf_file.base.child_pid) |pid| { + switch (builtin.os.tag) { + .linux => { + var local_vec: [1]std.posix.iovec_const = .{.{ + .base = out.ptr, + .len = out.len, + }}; + var remote_vec: [1]std.posix.iovec_const = .{.{ + .base = @as([*]u8, @ptrFromInt(@as(usize, @intCast(source_addr)))), + .len = out.len, + }}; + const rc = std.os.linux.process_vm_writev(pid, &local_vec, &remote_vec, 0); + switch (std.os.linux.E.init(rc)) { + .SUCCESS => assert(rc == out.len), + else => |errno| log.warn("process_vm_writev failure: {s}", .{@tagName(errno)}), + } + }, + else => return error.HotSwapUnavailableOnHostOperatingSystem, + } + } } pub fn asFile(self: *ZigObject) File { @@ -1766,169 +1785,17 @@ const UavTable = std.AutoArrayHashMapUnmanaged(InternPool.Index, AvMetadata); const LazySymbolTable = std.AutoArrayHashMapUnmanaged(InternPool.Index, LazySymbolMetadata); const TlsTable = std.AutoArrayHashMapUnmanaged(Atom.Index, TlsVariable); -pub const JumpTable = struct { - sym_index: Symbol.Index, - entries: std.MultiArrayList(Entry) = .{}, - dirty: bool = false, - - pub fn deinit(jt: *JumpTable, allocator: Allocator) void { - jt.entries.deinit(allocator); - } - - pub fn addSymbol(jt: *JumpTable, allocator: Allocator, sym_index: Symbol.Index) !Index { - const index: Index = @intCast(try jt.entries.addOne(allocator)); - jt.entries.set(index, .{ .sym_index = sym_index }); - return index; - } - - pub fn address(jt: JumpTable, zo: *ZigObject, elf_file: *Elf) i64 { - const sym = zo.symbol(jt.sym_index); - return sym.address(.{}, elf_file); - } - - pub fn size(jt: JumpTable, zo: *ZigObject, elf_file: *Elf) u64 { - const sym = zo.symbol(jt.sym_index); - return sym.atom(elf_file).?.size; - } - - pub fn alignment(cpu_arch: std.Target.Cpu.Arch) u64 { - return switch (cpu_arch) { - .x86_64 => 1, - else => @panic("TODO implement alignment for this CPU arch"), +const x86_64 = struct { + fn writeTrampolineCode(source_addr: i64, target_addr: i64, buf: *[max_trampoline_len]u8) ![]u8 { + const disp = @as(i64, @intCast(target_addr)) - source_addr - 5; + var bytes = [_]u8{ + 0xe9, 0x00, 0x00, 0x00, 0x00, // jmp rel32 }; + assert(bytes.len == trampolineSize(.x86_64)); + mem.writeInt(i32, bytes[1..][0..4], @intCast(disp), .little); + @memcpy(buf[0..bytes.len], &bytes); + return buf[0..bytes.len]; } - - pub fn entryAddress(jt: JumpTable, index: Index, zo: *ZigObject, elf_file: *Elf) i64 { - return jt.address(zo, elf_file) + @as(i64, @intCast(index * entrySize(elf_file.getTarget().cpu.arch))); - } - - pub fn entryOffset(jt: JumpTable, index: Index, zo: *ZigObject, elf_file: *Elf) u64 { - const sym = zo.symbol(jt.sym_index); - const atom_ptr = sym.atom(elf_file).?; - const shdr = elf_file.shdrs.items[atom_ptr.output_section_index]; - return shdr.sh_offset + @as(u64, @intCast(atom_ptr.value)) + index * entrySize(elf_file.getTarget().cpu.arch); - } - - pub fn entrySize(cpu_arch: std.Target.Cpu.Arch) u64 { - const seq_len = switch (cpu_arch) { - .x86_64 => 5, // jmp rel32 - else => @panic("TODO implement entry size for this CPU arch"), - }; - comptime assert(seq_len <= max_jump_seq_len); - return seq_len; - } - - pub fn targetAddress(jt: JumpTable, index: Index, zo: *ZigObject, elf_file: *Elf) i64 { - const sym_index = jt.entries.items(.sym_index)[index]; - return zo.symbol(sym_index).address(.{}, elf_file); - } - - const max_jump_seq_len = 12; - - pub fn writeEntry(jt: JumpTable, index: Index, zo: *ZigObject, elf_file: *Elf) !void { - const fileoff = jt.entryOffset(index, zo, elf_file); - const source_addr = jt.entryAddress(index, zo, elf_file); - const target_addr = @as(i64, @intCast(jt.targetAddress(index, zo, elf_file))); - var buf: [max_jump_seq_len]u8 = undefined; - const out = switch (elf_file.getTarget().cpu.arch) { - .x86_64 => try x86_64.writeEntry(source_addr, target_addr, &buf), - else => @panic("TODO implement write entry for this CPU arch"), - }; - try elf_file.base.file.?.pwriteAll(out, fileoff); - - if (elf_file.base.child_pid) |pid| { - switch (builtin.os.tag) { - .linux => { - var local_vec: [1]std.posix.iovec_const = .{.{ - .base = out.ptr, - .len = out.len, - }}; - var remote_vec: [1]std.posix.iovec_const = .{.{ - .base = @as([*]u8, @ptrFromInt(@as(usize, @intCast(source_addr)))), - .len = out.len, - }}; - const rc = std.os.linux.process_vm_writev(pid, &local_vec, &remote_vec, 0); - switch (std.os.linux.E.init(rc)) { - .SUCCESS => assert(rc == out.len), - else => |errno| log.warn("process_vm_writev failure: {s}", .{@tagName(errno)}), - } - }, - else => return error.HotSwapUnavailableOnHostOperatingSystem, - } - } - } - - pub fn updateSize(jt: JumpTable, zo: *ZigObject, elf_file: *Elf) !void { - const jt_size: u64 = @intCast(jt.entries.items(.sym_index).len * entrySize(elf_file.getTarget().cpu.arch)); - const sym = zo.symbol(jt.sym_index); - const esym = &zo.symtab.items(.elf_sym)[sym.esym_index]; - esym.st_size = jt_size; - const atom_ptr = sym.atom(elf_file).?; - atom_ptr.size = jt_size; - } - - const JumpTableFormatContext = struct { JumpTable, *ZigObject, *Elf }; - - pub fn format( - jt: JumpTable, - comptime unused_fmt_string: []const u8, - options: std.fmt.FormatOptions, - writer: anytype, - ) !void { - _ = jt; - _ = unused_fmt_string; - _ = options; - _ = writer; - @compileError("do not format JumpTable directly"); - } - - pub fn fmt(jt: JumpTable, zo: *ZigObject, elf_file: *Elf) std.fmt.Formatter(format2) { - return .{ .data = .{ jt, zo, elf_file } }; - } - - fn format2( - ctx: JumpTableFormatContext, - comptime unused_fmt_string: []const u8, - options: std.fmt.FormatOptions, - writer: anytype, - ) !void { - _ = options; - _ = unused_fmt_string; - const jt, const zo, const ef = ctx; - try writer.writeAll("__zig_jump_table\n"); - try writer.print(" @{x} : size({x})\n", .{ jt.address(zo, ef), jt.size(zo, ef) }); - for (jt.entries.items(.sym_index), jt.entries.items(.dirty)) |sym_index, dirty| { - const sym = zo.symbol(sym_index); - try writer.print(" {x} => {x} : %{d} : {s}", .{ - sym.address(.{ .zjt = true }, ef), - sym.address(.{}, ef), - sym_index, - sym.name(ef), - }); - if (dirty) try writer.writeAll(" : [!]"); - try writer.writeByte('\n'); - } - } - - const Entry = struct { - sym_index: Symbol.Index, - dirty: bool = true, - }; - - pub const Index = u32; - - const x86_64 = struct { - fn writeEntry(source_addr: i64, target_addr: i64, buf: *[max_jump_seq_len]u8) ![]u8 { - const disp = @as(i64, @intCast(target_addr)) - source_addr - 5; - var bytes = [_]u8{ - 0xe9, 0x00, 0x00, 0x00, 0x00, // jmp rel32 - }; - assert(bytes.len == entrySize(.x86_64)); - mem.writeInt(i32, bytes[1..][0..4], @intCast(disp), .little); - @memcpy(buf[0..bytes.len], &bytes); - return buf[0..bytes.len]; - } - }; }; const assert = std.debug.assert; From b8203fac1b272d4f13f668797cf356d995b7967f Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 15 Aug 2024 08:37:13 +0200 Subject: [PATCH 28/39] elf: check for relocs before deciding on shndx in getNavShdrIndex --- src/link/Elf.zig | 5 +++++ src/link/Elf/ZigObject.zig | 32 +++++++++++++++++++++++--------- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 1c2f90d14c..3dcfb0bc6a 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -857,6 +857,11 @@ pub fn growAllocSection(self: *Elf, shdr_index: u32, needed_size: u64) !void { const shdr = &self.shdrs.items[shdr_index]; const maybe_phdr = if (self.phdr_to_shdr_table.get(shdr_index)) |phndx| &self.phdrs.items[phndx] else null; const is_zerofill = shdr.sh_type == elf.SHT_NOBITS; + log.debug("allocated size {x} of {s}, needed size {x}", .{ + self.allocatedSize(shdr.sh_offset), + self.getShString(shdr.sh_name), + needed_size, + }); if (needed_size > self.allocatedSize(shdr.sh_offset) and !is_zerofill) { const existing_size = shdr.sh_size; diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index a472c6fad0..4c05c0b985 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -816,9 +816,9 @@ fn getNavShdrIndex( elf_file: *Elf, zcu: *Zcu, nav_index: InternPool.Nav.Index, + sym_index: Symbol.Index, code: []const u8, ) error{OutOfMemory}!u32 { - _ = self; const ip = &zcu.intern_pool; const any_non_single_threaded = elf_file.base.comp.config.any_non_single_threaded; const nav_val = zcu.navValue(nav_index); @@ -828,10 +828,12 @@ fn getNavShdrIndex( .@"extern" => |@"extern"| .{ @"extern".is_const, @"extern".is_threadlocal, .none }, else => .{ true, false, nav_val.toIntern() }, }; + const has_relocs = self.symbol(sym_index).atom(elf_file).?.relocs(elf_file).len > 0; if (any_non_single_threaded and is_threadlocal) { - for (code) |byte| { - if (byte != 0) break; - } else return elf_file.sectionByName(".tbss") orelse try elf_file.addSection(.{ + const is_bss = !has_relocs and for (code) |byte| { + if (byte != 0) break false; + } else true; + if (is_bss) return elf_file.sectionByName(".tbss") orelse try elf_file.addSection(.{ .type = elf.SHT_NOBITS, .flags = elf.SHF_ALLOC | elf.SHF_WRITE | elf.SHF_TLS, .name = try elf_file.insertShString(".tbss"), @@ -850,9 +852,10 @@ fn getNavShdrIndex( .Debug, .ReleaseSafe => elf_file.zig_data_section_index.?, .ReleaseFast, .ReleaseSmall => elf_file.zig_bss_section_index.?, }; - for (code) |byte| { - if (byte != 0) break; - } else return elf_file.zig_bss_section_index.?; + const is_bss = !has_relocs and for (code) |byte| { + if (byte != 0) break false; + } else true; + if (is_bss) return elf_file.zig_bss_section_index.?; return elf_file.zig_data_section_index.?; } @@ -944,6 +947,7 @@ fn updateNavCode( if (shdr.sh_type != elf.SHT_NOBITS) { const file_offset = shdr.sh_offset + @as(u64, @intCast(atom_ptr.value)); try elf_file.base.file.?.pwriteAll(code, file_offset); + log.debug("writing {} from 0x{x} to 0x{x}", .{ nav.fqn.fmt(ip), file_offset, file_offset + code.len }); } } @@ -1052,7 +1056,12 @@ pub fn updateFunc( }, }; - const shndx = try self.getNavShdrIndex(elf_file, zcu, func.owner_nav, code); + const shndx = try self.getNavShdrIndex(elf_file, zcu, func.owner_nav, sym_index, code); + log.debug("setting shdr({x},{s}) for {}", .{ + shndx, + elf_file.getShString(elf_file.shdrs.items[shndx].sh_name), + ip.getNav(func.owner_nav).fqn.fmt(ip), + }); const old_rva, const old_alignment = blk: { const atom_ptr = self.symbol(sym_index).atom(elf_file).?; break :blk .{ atom_ptr.value, atom_ptr.alignment }; @@ -1172,7 +1181,12 @@ pub fn updateNav( }, }; - const shndx = try self.getNavShdrIndex(elf_file, zcu, nav_index, code); + const shndx = try self.getNavShdrIndex(elf_file, zcu, nav_index, sym_index, code); + log.debug("setting shdr({x},{s}) for {}", .{ + shndx, + elf_file.getShString(elf_file.shdrs.items[shndx].sh_name), + nav.fqn.fmt(ip), + }); if (elf_file.shdrs.items[shndx].sh_flags & elf.SHF_TLS != 0) try self.updateTlv(elf_file, pt, nav_index, sym_index, shndx, code) else From c9db8878006fe53eb438bc6a6f2d2416cc5ef412 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 15 Aug 2024 08:38:08 +0200 Subject: [PATCH 29/39] builtin: init test_functions to empty slice to avoid coercing to undefined --- src/Builtin.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Builtin.zig b/src/Builtin.zig index 6e573d843f..58abcb7a4b 100644 --- a/src/Builtin.zig +++ b/src/Builtin.zig @@ -218,7 +218,7 @@ pub fn append(opts: @This(), buffer: *std.ArrayList(u8)) Allocator.Error!void { if (opts.is_test) { try buffer.appendSlice( - \\pub var test_functions: []const std.builtin.TestFn = undefined; // overwritten later + \\pub var test_functions: []const std.builtin.TestFn = &.{}; // overwritten later \\ ); } From 2e8351cc9e30c28a9415821ed99d664b76eb2062 Mon Sep 17 00:00:00 2001 From: David Rubin Date: Wed, 14 Aug 2024 17:17:13 -0700 Subject: [PATCH 30/39] elf: fix up riscv for `.got.zig` rewrite --- src/arch/riscv64/CodeGen.zig | 8 ++++---- test/behavior/basic.zig | 1 - test/behavior/export_builtin.zig | 2 -- test/behavior/extern.zig | 1 - test/behavior/fn.zig | 1 - test/behavior/pointers.zig | 1 - 6 files changed, 4 insertions(+), 10 deletions(-) diff --git a/src/arch/riscv64/CodeGen.zig b/src/arch/riscv64/CodeGen.zig index e9f593485f..6531bf34d9 100644 --- a/src/arch/riscv64/CodeGen.zig +++ b/src/arch/riscv64/CodeGen.zig @@ -4937,7 +4937,7 @@ fn genCall( if (func.mod.pic) { return func.fail("TODO: genCall pic", .{}); } else { - try func.genSetReg(Type.u64, .ra, .{ .load_symbol = .{ .sym = sym_index } }); + try func.genSetReg(Type.u64, .ra, .{ .lea_symbol = .{ .sym = sym_index } }); _ = try func.addInst(.{ .tag = .jalr, .data = .{ .i_type = .{ @@ -6120,7 +6120,7 @@ fn airAsm(func: *Func, inst: Air.Inst.Index) !void { arg_map.get(op_str["%[".len .. mod_index orelse op_str.len - "]".len]) orelse return func.fail("no matching constraint: '{s}'", .{op_str}) ]) { - .load_symbol => |sym_off| if (mem.eql(u8, modifier, "plt")) blk: { + .lea_symbol => |sym_off| if (mem.eql(u8, modifier, "plt")) blk: { assert(sym_off.off == 0); break :blk .{ .sym = sym_off }; } else return func.fail("invalid modifier: '{s}'", .{modifier}), @@ -6388,7 +6388,7 @@ fn genCopy(func: *Func, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void { ty, src_mcv, ), - .load_tlv => { + .load_symbol, .load_tlv => { const addr_reg, const addr_lock = try func.allocReg(.int); defer func.register_manager.unlockReg(addr_lock); @@ -6433,7 +6433,7 @@ fn genCopy(func: *Func, ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void { part_disp += @intCast(dst_ty.abiSize(func.pt)); } }, - else => return func.fail("TODO: genCopy to {s} from {s}", .{ @tagName(dst_mcv), @tagName(src_mcv) }), + else => return std.debug.panic("TODO: genCopy to {s} from {s}", .{ @tagName(dst_mcv), @tagName(src_mcv) }), } } diff --git a/test/behavior/basic.zig b/test/behavior/basic.zig index 90d12e6858..9d34742604 100644 --- a/test/behavior/basic.zig +++ b/test/behavior/basic.zig @@ -733,7 +733,6 @@ test "extern variable with non-pointer opaque type" { if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest; if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; @export(var_to_export, .{ .name = "opaque_extern_var" }); try expect(@as(*align(1) u32, @ptrCast(&opaque_extern_var)).* == 42); diff --git a/test/behavior/export_builtin.zig b/test/behavior/export_builtin.zig index 547a9b990a..25b6e2527e 100644 --- a/test/behavior/export_builtin.zig +++ b/test/behavior/export_builtin.zig @@ -48,7 +48,6 @@ test "exporting using field access" { test "exporting comptime-known value" { if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; - if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; if (builtin.zig_backend == .stage2_x86_64 and (builtin.target.ofmt != .elf and builtin.target.ofmt != .macho and @@ -68,7 +67,6 @@ test "exporting comptime-known value" { test "exporting comptime var" { if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; - if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; if (builtin.zig_backend == .stage2_x86_64 and (builtin.target.ofmt != .elf and builtin.target.ofmt != .macho and diff --git a/test/behavior/extern.zig b/test/behavior/extern.zig index cd80c545ce..0ef3e49353 100644 --- a/test/behavior/extern.zig +++ b/test/behavior/extern.zig @@ -7,7 +7,6 @@ test "anyopaque extern symbol" { if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest; if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest; - if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; const a = @extern(*anyopaque, .{ .name = "a_mystery_symbol" }); const b: *i32 = @alignCast(@ptrCast(a)); diff --git a/test/behavior/fn.zig b/test/behavior/fn.zig index 1e4039c1bb..ab7aca6ed6 100644 --- a/test/behavior/fn.zig +++ b/test/behavior/fn.zig @@ -429,7 +429,6 @@ test "implicit cast function to function ptr" { if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_spirv64) return error.SkipZigTest; if (builtin.zig_backend == .stage2_x86_64 and builtin.target.ofmt != .elf and builtin.target.ofmt != .macho) return error.SkipZigTest; - if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; const S1 = struct { export fn someFunctionThatReturnsAValue() c_int { diff --git a/test/behavior/pointers.zig b/test/behavior/pointers.zig index 42e3ea0ae9..f8260d1f0d 100644 --- a/test/behavior/pointers.zig +++ b/test/behavior/pointers.zig @@ -45,7 +45,6 @@ test "pointer-integer arithmetic" { test "pointer subtraction" { if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_riscv64) return error.SkipZigTest; { const a: *u8 = @ptrFromInt(100); From 8a0cb7002e6b8f7b50fdf7ee40311ce93fbef009 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 15 Aug 2024 10:05:41 +0200 Subject: [PATCH 31/39] elf: introduce Symbol.flags.is_extern_ptr for refs potentially needing GOT --- src/arch/x86_64/Emit.zig | 4 ++-- src/codegen.zig | 3 +-- src/link/Elf/Symbol.zig | 12 +++++++----- src/link/Elf/ZigObject.zig | 3 +-- 4 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/arch/x86_64/Emit.zig b/src/arch/x86_64/Emit.zig index b7d0a29c2d..fd1389eb3b 100644 --- a/src/arch/x86_64/Emit.zig +++ b/src/arch/x86_64/Emit.zig @@ -114,7 +114,7 @@ pub fn emitMir(emit: *Emit) Error!void { const atom = zo.symbol(data.atom_index).atom(elf_file).?; const sym = zo.symbol(data.sym_index); if (emit.lower.pic) { - const r_type: u32 = if (sym.flags.needs_got) + const r_type: u32 = if (sym.flags.is_extern_ptr) @intFromEnum(std.elf.R_X86_64.GOTPCREL) else @intFromEnum(std.elf.R_X86_64.PC32); @@ -124,7 +124,7 @@ pub fn emitMir(emit: *Emit) Error!void { .r_addend = -4, }); } else { - const r_type: u32 = if (sym.flags.needs_got) + const r_type: u32 = if (sym.flags.is_extern_ptr) @intFromEnum(std.elf.R_X86_64.GOT32) else if (sym.flags.is_tls) @intFromEnum(std.elf.R_X86_64.TPOFF32) diff --git a/src/codegen.zig b/src/codegen.zig index 679ee03c27..3df7ce37ab 100644 --- a/src/codegen.zig +++ b/src/codegen.zig @@ -898,9 +898,8 @@ fn genNavRef( if (lf.cast(.elf)) |elf_file| { const zo = elf_file.zigObjectPtr().?; if (is_extern) { - // TODO audit this const sym_index = try elf_file.getGlobalSymbol(name.toSlice(ip), lib_name.toSlice(ip)); - zo.symbol(sym_index).flags.needs_got = true; + zo.symbol(sym_index).flags.is_extern_ptr = true; return GenResult.mcv(.{ .load_symbol = sym_index }); } const sym_index = try zo.getOrCreateMetadataForNav(elf_file, nav_index); diff --git a/src/link/Elf/Symbol.zig b/src/link/Elf/Symbol.zig index 4beb6a0918..4f6c2b8c7e 100644 --- a/src/link/Elf/Symbol.zig +++ b/src/link/Elf/Symbol.zig @@ -447,16 +447,18 @@ pub const Flags = packed struct { needs_tlsdesc: bool = false, has_tlsdesc: bool = false, - /// Whether the symbol is a TLS variable. - /// TODO this is really not needed if only we operated on esyms between - /// codegen and ZigObject. - is_tls: bool = false, - /// Whether the symbol is a merge subsection. merge_subsection: bool = false, + /// ZigObject specific flags /// Whether the symbol has a trampoline. has_trampoline: bool = false, + + /// Whether the symbol is a TLS variable. + is_tls: bool = false, + + /// Whether the symbol is an extern pointer (as opposed to function). + is_extern_ptr: bool = false, }; pub const Extra = struct { diff --git a/src/link/Elf/ZigObject.zig b/src/link/Elf/ZigObject.zig index 4c05c0b985..b72946c0f9 100644 --- a/src/link/Elf/ZigObject.zig +++ b/src/link/Elf/ZigObject.zig @@ -1141,13 +1141,12 @@ pub fn updateNav( .variable => |variable| Value.fromInterned(variable.init), .@"extern" => |@"extern"| { if (ip.isFunctionType(@"extern".ty)) return; - // Extern variable gets a .got entry only. const sym_index = try self.getGlobalSymbol( elf_file, nav.name.toSlice(ip), @"extern".lib_name.toSlice(ip), ); - self.symbol(sym_index).flags.needs_got = true; + self.symbol(sym_index).flags.is_extern_ptr = true; return; }, else => nav_val, From 0fd0b11bc43a6053c68f649e1e1a1e23b995a266 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 15 Aug 2024 10:21:20 +0200 Subject: [PATCH 32/39] riscv: do not emit GOT relocations for special linker symbols --- src/arch/riscv64/Emit.zig | 15 +++++++++------ src/codegen.zig | 2 +- src/link/Elf.zig | 27 --------------------------- src/link/Elf/Atom.zig | 25 ++----------------------- src/link/Elf/relocation.zig | 14 +++++--------- 5 files changed, 17 insertions(+), 66 deletions(-) diff --git a/src/arch/riscv64/Emit.zig b/src/arch/riscv64/Emit.zig index c1e7565c0b..8b32f1e580 100644 --- a/src/arch/riscv64/Emit.zig +++ b/src/arch/riscv64/Emit.zig @@ -49,14 +49,17 @@ pub fn emitMir(emit: *Emit) Error!void { const atom_ptr = zo.symbol(symbol.atom_index).atom(elf_file).?; const sym = zo.symbol(symbol.sym_index); - var hi_r_type: u32 = @intFromEnum(std.elf.R_RISCV.HI20); - var lo_r_type: u32 = @intFromEnum(std.elf.R_RISCV.LO12_I); - - if (sym.flags.needs_got) { - hi_r_type = Elf.R_GOT_HI20_STATIC; // TODO: rework this #20887 - lo_r_type = Elf.R_GOT_LO12_I_STATIC; // TODO: rework this #20887 + if (sym.flags.is_extern_ptr) blk: { + const name = sym.name(elf_file); + if (mem.eql(u8, "__init_array_start", name) or mem.eql(u8, "__init_array_end", name) or + mem.eql(u8, "__fini_array_start", name) or mem.eql(u8, "__fini_array_end", name)) + break :blk; + return emit.fail("emit GOT relocation for symbol '{s}'", .{name}); } + const hi_r_type: u32 = @intFromEnum(std.elf.R_RISCV.HI20); + const lo_r_type: u32 = @intFromEnum(std.elf.R_RISCV.LO12_I); + try atom_ptr.addReloc(elf_file, .{ .r_offset = start_offset, .r_info = (@as(u64, @intCast(symbol.sym_index)) << 32) | hi_r_type, diff --git a/src/codegen.zig b/src/codegen.zig index 3df7ce37ab..89b4a8a291 100644 --- a/src/codegen.zig +++ b/src/codegen.zig @@ -900,7 +900,7 @@ fn genNavRef( if (is_extern) { const sym_index = try elf_file.getGlobalSymbol(name.toSlice(ip), lib_name.toSlice(ip)); zo.symbol(sym_index).flags.is_extern_ptr = true; - return GenResult.mcv(.{ .load_symbol = sym_index }); + return GenResult.mcv(.{ .lea_symbol = sym_index }); } const sym_index = try zo.getOrCreateMetadataForNav(elf_file, nav_index); if (!single_threaded and is_threadlocal) { diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 3dcfb0bc6a..0ef1e9862d 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -5934,33 +5934,6 @@ const RelaSection = struct { }; const RelaSectionTable = std.AutoArrayHashMapUnmanaged(u32, RelaSection); -pub const R_GOT_HI20_STATIC: u32 = 0xff04; -pub const R_GOT_LO12_I_STATIC: u32 = 0xff05; - -// Comptime asserts that no Zig relocs overlap with another ISA's reloc number -comptime { - const zig_relocs = .{ - R_GOT_HI20_STATIC, - R_GOT_LO12_I_STATIC, - }; - - const other_relocs = .{ - elf.R_X86_64, - elf.R_AARCH64, - elf.R_RISCV, - elf.R_PPC64, - }; - - @setEvalBranchQuota(@min(other_relocs.len * zig_relocs.len * 256, 6200)); - for (other_relocs) |relocs| { - for (@typeInfo(relocs).Enum.fields) |reloc| { - for (zig_relocs) |zig_reloc| { - assert(reloc.value != zig_reloc); - } - } - } -} - fn defaultEntrySymbolName(cpu_arch: std.Target.Cpu.Arch) []const u8 { return switch (cpu_arch) { .mips, .mipsel, .mips64, .mips64el => "__start", diff --git a/src/link/Elf/Atom.zig b/src/link/Elf/Atom.zig index 47bb216080..c214b07914 100644 --- a/src/link/Elf/Atom.zig +++ b/src/link/Elf/Atom.zig @@ -1978,13 +1978,7 @@ const riscv = struct { .SUB32, => {}, - else => |x| switch (@intFromEnum(x)) { - Elf.R_GOT_HI20_STATIC, - Elf.R_GOT_LO12_I_STATIC, - => symbol.flags.needs_got = true, - - else => try atom.reportUnhandledRelocError(rel, elf_file), - }, + else => try atom.reportUnhandledRelocError(rel, elf_file), } } @@ -2121,22 +2115,7 @@ const riscv = struct { // TODO: annotates an ADD instruction that can be removed when TPREL is relaxed }, - else => |x| switch (@intFromEnum(x)) { - // Zig custom relocations - Elf.R_GOT_HI20_STATIC => { - assert(target.flags.has_got); - const disp: u32 = @bitCast(math.cast(i32, G + GOT + A) orelse return error.Overflow); - riscv_util.writeInstU(code[r_offset..][0..4], disp); - }, - - Elf.R_GOT_LO12_I_STATIC => { - assert(target.flags.has_got); - const disp: u32 = @bitCast(math.cast(i32, G + GOT + A) orelse return error.Overflow); - riscv_util.writeInstI(code[r_offset..][0..4], disp); - }, - - else => try atom.reportUnhandledRelocError(rel, elf_file), - }, + else => try atom.reportUnhandledRelocError(rel, elf_file), } } diff --git a/src/link/Elf/relocation.zig b/src/link/Elf/relocation.zig index 887aece8bc..e1a317f67d 100644 --- a/src/link/Elf/relocation.zig +++ b/src/link/Elf/relocation.zig @@ -112,15 +112,11 @@ fn formatRelocType( _ = unused_fmt_string; _ = options; const r_type = ctx.r_type; - switch (r_type) { - Elf.R_GOT_HI20_STATIC => try writer.writeAll("R_GOT_HI20_STATIC"), - Elf.R_GOT_LO12_I_STATIC => try writer.writeAll("R_GOT_LO12_I_STATIC"), - else => switch (ctx.cpu_arch) { - .x86_64 => try writer.print("R_X86_64_{s}", .{@tagName(@as(elf.R_X86_64, @enumFromInt(r_type)))}), - .aarch64 => try writer.print("R_AARCH64_{s}", .{@tagName(@as(elf.R_AARCH64, @enumFromInt(r_type)))}), - .riscv64 => try writer.print("R_RISCV_{s}", .{@tagName(@as(elf.R_RISCV, @enumFromInt(r_type)))}), - else => unreachable, - }, + switch (ctx.cpu_arch) { + .x86_64 => try writer.print("R_X86_64_{s}", .{@tagName(@as(elf.R_X86_64, @enumFromInt(r_type)))}), + .aarch64 => try writer.print("R_AARCH64_{s}", .{@tagName(@as(elf.R_AARCH64, @enumFromInt(r_type)))}), + .riscv64 => try writer.print("R_RISCV_{s}", .{@tagName(@as(elf.R_RISCV, @enumFromInt(r_type)))}), + else => unreachable, } } From f0df0acd70637eddbec559df20e8ea11429c26ed Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 15 Aug 2024 10:33:30 +0200 Subject: [PATCH 33/39] x86_64: fix handling on externs in lower/emit --- src/arch/x86_64/Emit.zig | 4 +--- src/arch/x86_64/Lower.zig | 9 ++++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/src/arch/x86_64/Emit.zig b/src/arch/x86_64/Emit.zig index fd1389eb3b..693dca1da6 100644 --- a/src/arch/x86_64/Emit.zig +++ b/src/arch/x86_64/Emit.zig @@ -124,9 +124,7 @@ pub fn emitMir(emit: *Emit) Error!void { .r_addend = -4, }); } else { - const r_type: u32 = if (sym.flags.is_extern_ptr) - @intFromEnum(std.elf.R_X86_64.GOT32) - else if (sym.flags.is_tls) + const r_type: u32 = if (sym.flags.is_tls) @intFromEnum(std.elf.R_X86_64.TPOFF32) else @intFromEnum(std.elf.R_X86_64.@"32"); diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig index 56cd2669d6..4fab1b806d 100644 --- a/src/arch/x86_64/Lower.zig +++ b/src/arch/x86_64/Lower.zig @@ -397,8 +397,11 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) } _ = lower.reloc(.{ .linker_reloc = sym }); - break :op if (lower.pic) switch (mnemonic) { - .lea => break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }, + if (lower.pic) switch (mnemonic) { + .lea => { + if (elf_sym.flags.is_extern_ptr) emit_mnemonic = .mov; + break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }; + }, .mov => break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }, else => unreachable, } else switch (mnemonic) { @@ -413,7 +416,7 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) .base = .{ .reg = .ds }, }) }, else => unreachable, - }; + } } else if (lower.bin_file.cast(.macho)) |macho_file| { const zo = macho_file.getZigObject().?; const macho_sym = zo.symbols.items[sym.sym_index]; From c79290467f4908c1fc4833d82d397829e92cf838 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 15 Aug 2024 10:36:37 +0200 Subject: [PATCH 34/39] elf: fix GOT32 reloc resolution logic --- src/link/Elf/Atom.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/link/Elf/Atom.zig b/src/link/Elf/Atom.zig index c214b07914..fe31712f30 100644 --- a/src/link/Elf/Atom.zig +++ b/src/link/Elf/Atom.zig @@ -1314,7 +1314,7 @@ const x86_64 = struct { } }, - .GOT32 => try cwriter.writeInt(i32, @as(i32, @intCast(G + GOT + A)), .little), + .GOT32 => try cwriter.writeInt(i32, @as(i32, @intCast(G + A)), .little), else => try atom.reportUnhandledRelocError(rel, elf_file), } From 79418fa0abe4b19a2fd2fb10cf3c1144b927d10f Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 15 Aug 2024 10:52:06 +0200 Subject: [PATCH 35/39] riscv: remove redundant by-symbol-name check; just check for PIC and extern ptr --- src/arch/riscv64/Emit.zig | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/src/arch/riscv64/Emit.zig b/src/arch/riscv64/Emit.zig index 8b32f1e580..a4fadad526 100644 --- a/src/arch/riscv64/Emit.zig +++ b/src/arch/riscv64/Emit.zig @@ -49,12 +49,8 @@ pub fn emitMir(emit: *Emit) Error!void { const atom_ptr = zo.symbol(symbol.atom_index).atom(elf_file).?; const sym = zo.symbol(symbol.sym_index); - if (sym.flags.is_extern_ptr) blk: { - const name = sym.name(elf_file); - if (mem.eql(u8, "__init_array_start", name) or mem.eql(u8, "__init_array_end", name) or - mem.eql(u8, "__fini_array_start", name) or mem.eql(u8, "__fini_array_end", name)) - break :blk; - return emit.fail("emit GOT relocation for symbol '{s}'", .{name}); + if (sym.flags.is_extern_ptr and emit.lower.pic) { + return emit.fail("emit GOT relocation for symbol '{s}'", .{sym.name(elf_file)}); } const hi_r_type: u32 = @intFromEnum(std.elf.R_RISCV.HI20); From 4d5bf0f09a29eeb0eb7f3602825f1a0095b8427e Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 15 Aug 2024 14:23:36 +0200 Subject: [PATCH 36/39] x86_64: deref GOT pointer when requesting var value --- src/arch/x86_64/Lower.zig | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/src/arch/x86_64/Lower.zig b/src/arch/x86_64/Lower.zig index 4fab1b806d..15322ce4f6 100644 --- a/src/arch/x86_64/Lower.zig +++ b/src/arch/x86_64/Lower.zig @@ -402,7 +402,21 @@ fn emit(lower: *Lower, prefix: Prefix, mnemonic: Mnemonic, ops: []const Operand) if (elf_sym.flags.is_extern_ptr) emit_mnemonic = .mov; break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }; }, - .mov => break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }, + .mov => { + if (elf_sym.flags.is_extern_ptr) { + const reg = ops[0].reg; + lower.result_insts[lower.result_insts_len] = + try Instruction.new(.none, .mov, &[_]Operand{ + .{ .reg = reg.to64() }, + .{ .mem = Memory.rip(.qword, 0) }, + }); + lower.result_insts_len += 1; + break :op .{ .mem = Memory.sib(mem_op.sib.ptr_size, .{ .base = .{ + .reg = reg.to64(), + } }) }; + } + break :op .{ .mem = Memory.rip(mem_op.sib.ptr_size, 0) }; + }, else => unreachable, } else switch (mnemonic) { .call => break :op .{ .mem = Memory.sib(mem_op.sib.ptr_size, .{ From 9473d764498fbb4eef3e2bec42e44a3f93a5a56b Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 15 Aug 2024 21:47:34 +0200 Subject: [PATCH 37/39] test/elf: enhance testImportingDataDynamic --- test/link/elf.zig | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/test/link/elf.zig b/test/link/elf.zig index 71b722a9f9..5539638ba7 100644 --- a/test/link/elf.zig +++ b/test/link/elf.zig @@ -1773,25 +1773,41 @@ fn testImportingDataDynamic(b: *Build, opts: Options) *Step { .use_llvm = true, }, .{ .name = "a", - .c_source_bytes = "int foo = 42;", + .c_source_bytes = + \\#include + \\int foo = 42; + \\void printFoo() { fprintf(stderr, "lib foo=%d\n", foo); } + , }); + dso.linkLibC(); const main = addExecutable(b, opts, .{ .name = "main", .zig_source_bytes = + \\const std = @import("std"); \\extern var foo: i32; + \\extern fn printFoo() void; \\pub fn main() void { - \\ @import("std").debug.print("{d}\n", .{foo}); + \\ std.debug.print("exe foo={d}\n", .{foo}); + \\ printFoo(); + \\ foo += 1; + \\ std.debug.print("exe foo={d}\n", .{foo}); + \\ printFoo(); \\} , .strip = true, // TODO temp hack }); main.pie = true; main.linkLibrary(dso); - main.linkLibC(); const run = addRunArtifact(main); - run.expectStdErrEqual("42\n"); + run.expectStdErrEqual( + \\exe foo=42 + \\lib foo=42 + \\exe foo=43 + \\lib foo=43 + \\ + ); test_step.dependOn(&run.step); return test_step; From 624016e8f39f291e92bb2e0319b2cdf2ce4b15c9 Mon Sep 17 00:00:00 2001 From: Jacob Young Date: Fri, 16 Aug 2024 05:42:50 -0400 Subject: [PATCH 38/39] riscv64: fix incorrect branch target --- src/arch/riscv64/CodeGen.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/arch/riscv64/CodeGen.zig b/src/arch/riscv64/CodeGen.zig index 6531bf34d9..942c562496 100644 --- a/src/arch/riscv64/CodeGen.zig +++ b/src/arch/riscv64/CodeGen.zig @@ -6594,7 +6594,7 @@ fn genInlineMemset( .tag = .beq, .data = .{ .b_type = .{ - .inst = @intCast(func.mir_instructions.len + 4), // points after the last inst + .inst = @intCast(func.mir_instructions.len + 3), // points after the last inst .rs1 = count, .rs2 = .zero, }, From 73f385eec57d4fc28069b60f059fa8553d3a0c1b Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Fri, 16 Aug 2024 11:49:23 +0200 Subject: [PATCH 39/39] Update src/arch/x86_64/CodeGen.zig Co-authored-by: Jacob Young --- src/arch/x86_64/CodeGen.zig | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 1edf208d4c..bfcc29348c 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -1382,7 +1382,10 @@ fn asmImmediate(self: *Self, tag: Mir.Inst.FixedTag, imm: Immediate) !void { .reloc => .rel, }, .data = switch (imm) { - .reloc => |x| .{ .reloc = x }, + .reloc => |x| reloc: { + assert(tag[0] == ._); + break :reloc .{ .reloc = x }; + }, .signed, .unsigned => .{ .i = .{ .fixes = tag[0], .i = switch (imm) {