diff --git a/src/link/Elf/Atom.zig b/src/link/Elf/Atom.zig index 71f9d79fac..9a896c1396 100644 --- a/src/link/Elf/Atom.zig +++ b/src/link/Elf/Atom.zig @@ -1689,7 +1689,7 @@ const aarch64 = struct { }); return; }; - try aarch64_util.writeBranchImm(disp, code[r_offset..][0..4]); + aarch64_util.writeBranchImm(disp, code[r_offset..][0..4]); }, .ADR_PREL_PG_HI21 => { @@ -1697,14 +1697,14 @@ const aarch64 = struct { const saddr = @as(u64, @intCast(P)); const taddr = @as(u64, @intCast(S + A)); const pages = @as(u21, @bitCast(try aarch64_util.calcNumberOfPages(saddr, taddr))); - try aarch64_util.writePages(pages, code[r_offset..][0..4]); + aarch64_util.writeAdrpInst(pages, code[r_offset..][0..4]); }, .ADR_GOT_PAGE => if (target.flags.has_got) { const saddr = @as(u64, @intCast(P)); const taddr = @as(u64, @intCast(G + GOT + A)); const pages = @as(u21, @bitCast(try aarch64_util.calcNumberOfPages(saddr, taddr))); - try aarch64_util.writePages(pages, code[r_offset..][0..4]); + aarch64_util.writeAdrpInst(pages, code[r_offset..][0..4]); } else { // TODO: relax var err = try elf_file.addErrorWithNotes(1); @@ -1719,10 +1719,14 @@ const aarch64 = struct { .LD64_GOT_LO12_NC => { assert(target.flags.has_got); const taddr = @as(u64, @intCast(G + GOT + A)); - try aarch64_util.writePageOffset(.load_store_64, taddr, code[r_offset..][0..4]); + aarch64_util.writeLoadStoreRegInst(@divExact(@as(u12, @truncate(taddr)), 8), code[rel.r_offset..][0..4]); + }, + + .ADD_ABS_LO12_NC => { + const taddr = @as(u64, @intCast(S + A)); + aarch64_util.writeAddImmInst(@truncate(taddr), code[rel.r_offset..][0..4]); }, - .ADD_ABS_LO12_NC, .LDST8_ABS_LO12_NC, .LDST16_ABS_LO12_NC, .LDST32_ABS_LO12_NC, @@ -1731,27 +1735,26 @@ const aarch64 = struct { => { // TODO: NC means no overflow check const taddr = @as(u64, @intCast(S + A)); - const kind: aarch64_util.PageOffsetInstKind = switch (r_type) { - .ADD_ABS_LO12_NC => .arithmetic, - .LDST8_ABS_LO12_NC => .load_store_8, - .LDST16_ABS_LO12_NC => .load_store_16, - .LDST32_ABS_LO12_NC => .load_store_32, - .LDST64_ABS_LO12_NC => .load_store_64, - .LDST128_ABS_LO12_NC => .load_store_128, + const offset: u12 = switch (r_type) { + .LDST8_ABS_LO12_NC => @truncate(taddr), + .LDST16_ABS_LO12_NC => @divExact(@as(u12, @truncate(taddr)), 2), + .LDST32_ABS_LO12_NC => @divExact(@as(u12, @truncate(taddr)), 4), + .LDST64_ABS_LO12_NC => @divExact(@as(u12, @truncate(taddr)), 8), + .LDST128_ABS_LO12_NC => @divExact(@as(u12, @truncate(taddr)), 16), else => unreachable, }; - try aarch64_util.writePageOffset(kind, taddr, code[r_offset..][0..4]); + aarch64_util.writeLoadStoreRegInst(offset, code[rel.r_offset..][0..4]); }, .TLSLE_ADD_TPREL_HI12 => { const value = math.cast(i12, (S + A - TP) >> 12) orelse return error.Overflow; - try aarch64_util.writeAddInst(@bitCast(value), code[rel.r_offset..][0..4]); + aarch64_util.writeAddImmInst(@bitCast(value), code[rel.r_offset..][0..4]); }, .TLSLE_ADD_TPREL_LO12_NC => { const value: i12 = @truncate(S + A - TP); - try aarch64_util.writeAddInst(@bitCast(value), code[rel.r_offset..][0..4]); + aarch64_util.writeAddImmInst(@bitCast(value), code[rel.r_offset..][0..4]); }, else => try atom.reportUnhandledRelocError(rel, elf_file), diff --git a/src/link/Elf/synthetic_sections.zig b/src/link/Elf/synthetic_sections.zig index 289c22d61f..04a3793561 100644 --- a/src/link/Elf/synthetic_sections.zig +++ b/src/link/Elf/synthetic_sections.zig @@ -1028,8 +1028,8 @@ pub const PltSection = struct { // TODO: relax if possible // .got.plt[2] const pages = try aarch64_util.calcNumberOfPages(plt_addr + 4, got_plt_addr + 16); - const ldr_off = try aarch64_util.calcPageOffset(.load_store_64, got_plt_addr + 16); - const add_off = try aarch64_util.calcPageOffset(.arithmetic, got_plt_addr + 16); + const ldr_off = try math.divExact(u12, @truncate(got_plt_addr + 16), 8); + const add_off: u12 = @truncate(got_plt_addr + 16); const preamble = &[_]Instruction{ Instruction.stp( @@ -1057,8 +1057,8 @@ pub const PltSection = struct { const target_addr = sym.gotPltAddress(elf_file); const source_addr = sym.pltAddress(elf_file); const pages = try aarch64_util.calcNumberOfPages(source_addr, target_addr); - const ldr_off = try aarch64_util.calcPageOffset(.load_store_64, target_addr); - const add_off = try aarch64_util.calcPageOffset(.arithmetic, target_addr); + const ldr_off = try math.divExact(u12, @truncate(target_addr), 8); + const add_off: u12 = @truncate(target_addr); const insts = &[_]Instruction{ Instruction.adrp(.x16, pages), Instruction.ldr(.x17, .x16, Instruction.LoadStoreOffset.imm(ldr_off)), @@ -1202,7 +1202,7 @@ pub const PltGotSection = struct { const target_addr = sym.gotAddress(elf_file); const source_addr = sym.pltGotAddress(elf_file); const pages = try aarch64_util.calcNumberOfPages(source_addr, target_addr); - const off = try aarch64_util.calcPageOffset(.load_store_64, target_addr); + const off = try math.divExact(u12, @truncate(target_addr), 8); const insts = &[_]Instruction{ Instruction.adrp(.x16, pages), Instruction.ldr(.x17, .x16, Instruction.LoadStoreOffset.imm(off)), @@ -1758,6 +1758,7 @@ fn writeInt(value: anytype, elf_file: *Elf, writer: anytype) !void { const assert = std.debug.assert; const builtin = @import("builtin"); const elf = std.elf; +const math = std.math; const mem = std.mem; const log = std.log.scoped(.link); const relocation = @import("relocation.zig"); diff --git a/src/link/MachO/Atom.zig b/src/link/MachO/Atom.zig index d40712046e..b4600176ce 100644 --- a/src/link/MachO/Atom.zig +++ b/src/link/MachO/Atom.zig @@ -700,7 +700,7 @@ fn resolveRelocInner( const S_: i64 = @intCast(thunk.getTargetAddress(rel.target, macho_file)); break :blk math.cast(i28, S_ + A - P) orelse return error.Overflow; }; - try aarch64.writeBranchImm(disp, code[rel_offset..][0..4]); + aarch64.writeBranchImm(disp, code[rel_offset..][0..4]); }, else => unreachable, } @@ -771,7 +771,7 @@ fn resolveRelocInner( break :target math.cast(u64, target) orelse return error.Overflow; }; const pages = @as(u21, @bitCast(try aarch64.calcNumberOfPages(source, target))); - try aarch64.writePages(pages, code[rel_offset..][0..4]); + aarch64.writeAdrpInst(pages, code[rel_offset..][0..4]); }, .pageoff => { @@ -780,8 +780,26 @@ fn resolveRelocInner( assert(!rel.meta.pcrel); const target = math.cast(u64, S + A) orelse return error.Overflow; const inst_code = code[rel_offset..][0..4]; - const kind = aarch64.classifyInst(inst_code); - try aarch64.writePageOffset(kind, target, inst_code); + if (aarch64.isArithmeticOp(inst_code)) { + aarch64.writeAddImmInst(@truncate(target), inst_code); + } else { + var inst = aarch64.Instruction{ + .load_store_register = mem.bytesToValue(std.meta.TagPayload( + aarch64.Instruction, + aarch64.Instruction.load_store_register, + ), inst_code), + }; + inst.load_store_register.offset = switch (inst.load_store_register.size) { + 0 => if (inst.load_store_register.v == 1) + try math.divExact(u12, @truncate(target), 16) + else + @truncate(target), + 1 => try math.divExact(u12, @truncate(target), 2), + 2 => try math.divExact(u12, @truncate(target), 4), + 3 => try math.divExact(u12, @truncate(target), 8), + }; + try writer.writeInt(u32, inst.toU32(), .little); + } }, .got_load_pageoff => { @@ -789,7 +807,7 @@ fn resolveRelocInner( assert(rel.meta.length == 2); assert(!rel.meta.pcrel); const target = math.cast(u64, G + A) orelse return error.Overflow; - try aarch64.writePageOffset(.load_store_64, target, code[rel_offset..][0..4]); + aarch64.writeLoadStoreRegInst(try math.divExact(u12, @truncate(target), 8), code[rel_offset..][0..4]); }, .tlvp_pageoff => { @@ -841,7 +859,7 @@ fn resolveRelocInner( .load_store_register = .{ .rt = reg_info.rd, .rn = reg_info.rn, - .offset = try aarch64.calcPageOffset(.load_store_64, target), + .offset = try math.divExact(u12, @truncate(target), 8), .opc = 0b01, .op1 = 0b01, .v = 0, @@ -851,7 +869,7 @@ fn resolveRelocInner( .add_subtract_immediate = .{ .rd = reg_info.rd, .rn = reg_info.rn, - .imm12 = try aarch64.calcPageOffset(.arithmetic, target), + .imm12 = @truncate(target), .sh = 0, .s = 0, .op = 0, diff --git a/src/link/MachO/synthetic.zig b/src/link/MachO/synthetic.zig index 0536026633..1afbd3ea6d 100644 --- a/src/link/MachO/synthetic.zig +++ b/src/link/MachO/synthetic.zig @@ -269,7 +269,7 @@ pub const StubsSection = struct { // TODO relax if possible const pages = try aarch64.calcNumberOfPages(source, target); try writer.writeInt(u32, aarch64.Instruction.adrp(.x16, pages).toU32(), .little); - const off = try aarch64.calcPageOffset(.load_store_64, target); + const off = try math.divExact(u12, @truncate(target), 8); try writer.writeInt( u32, aarch64.Instruction.ldr(.x16, .x16, aarch64.Instruction.LoadStoreOffset.imm(off)).toU32(), @@ -413,7 +413,7 @@ pub const StubsHelperSection = struct { // TODO relax if possible const pages = try aarch64.calcNumberOfPages(sect.addr, dyld_private_addr); try writer.writeInt(u32, aarch64.Instruction.adrp(.x17, pages).toU32(), .little); - const off = try aarch64.calcPageOffset(.arithmetic, dyld_private_addr); + const off: u12 = @truncate(dyld_private_addr); try writer.writeInt(u32, aarch64.Instruction.add(.x17, .x17, off, false).toU32(), .little); } try writer.writeInt(u32, aarch64.Instruction.stp( @@ -426,7 +426,7 @@ pub const StubsHelperSection = struct { // TODO relax if possible const pages = try aarch64.calcNumberOfPages(sect.addr + 12, dyld_stub_binder_addr); try writer.writeInt(u32, aarch64.Instruction.adrp(.x16, pages).toU32(), .little); - const off = try aarch64.calcPageOffset(.load_store_64, dyld_stub_binder_addr); + const off = try math.divExact(u12, @truncate(dyld_stub_binder_addr), 8); try writer.writeInt(u32, aarch64.Instruction.ldr( .x16, .x16, @@ -681,7 +681,7 @@ pub const ObjcStubsSection = struct { const source = addr; const pages = try aarch64.calcNumberOfPages(source, target); try writer.writeInt(u32, aarch64.Instruction.adrp(.x1, pages).toU32(), .little); - const off = try aarch64.calcPageOffset(.load_store_64, target); + const off = try math.divExact(u12, @truncate(target), 8); try writer.writeInt( u32, aarch64.Instruction.ldr(.x1, .x1, aarch64.Instruction.LoadStoreOffset.imm(off)).toU32(), @@ -694,7 +694,7 @@ pub const ObjcStubsSection = struct { const source = addr + 2 * @sizeOf(u32); const pages = try aarch64.calcNumberOfPages(source, target); try writer.writeInt(u32, aarch64.Instruction.adrp(.x16, pages).toU32(), .little); - const off = try aarch64.calcPageOffset(.load_store_64, target); + const off = try math.divExact(u12, @truncate(target), 8); try writer.writeInt( u32, aarch64.Instruction.ldr(.x16, .x16, aarch64.Instruction.LoadStoreOffset.imm(off)).toU32(), diff --git a/src/link/MachO/thunks.zig b/src/link/MachO/thunks.zig index e3f98c655c..02dacc1aec 100644 --- a/src/link/MachO/thunks.zig +++ b/src/link/MachO/thunks.zig @@ -101,7 +101,7 @@ pub const Thunk = struct { const taddr = sym.getAddress(.{}, macho_file); const pages = try aarch64.calcNumberOfPages(saddr, taddr); try writer.writeInt(u32, aarch64.Instruction.adrp(.x16, pages).toU32(), .little); - const off = try aarch64.calcPageOffset(.arithmetic, taddr); + const off: u12 = @truncate(taddr); try writer.writeInt(u32, aarch64.Instruction.add(.x16, .x16, off, false).toU32(), .little); try writer.writeInt(u32, aarch64.Instruction.br(.x16).toU32(), .little); } diff --git a/src/link/aarch64.zig b/src/link/aarch64.zig index 273dee7493..576249e988 100644 --- a/src/link/aarch64.zig +++ b/src/link/aarch64.zig @@ -3,66 +3,26 @@ pub inline fn isArithmeticOp(inst: *const [4]u8) bool { return ((group_decode >> 2) == 4); } -pub const PageOffsetInstKind = enum { - arithmetic, - load_store_8, - load_store_16, - load_store_32, - load_store_64, - load_store_128, -}; +pub fn writeAddImmInst(value: u12, code: *[4]u8) void { + var inst = Instruction{ + .add_subtract_immediate = mem.bytesToValue(std.meta.TagPayload( + Instruction, + Instruction.add_subtract_immediate, + ), code), + }; + inst.add_subtract_immediate.imm12 = value; + mem.writeInt(u32, code, inst.toU32(), .little); +} -pub fn classifyInst(code: *const [4]u8) PageOffsetInstKind { - if (isArithmeticOp(code)) return .arithmetic; - const inst = Instruction{ +pub fn writeLoadStoreRegInst(value: u12, code: *[4]u8) void { + var inst: Instruction = .{ .load_store_register = mem.bytesToValue(std.meta.TagPayload( Instruction, Instruction.load_store_register, ), code), }; - return switch (inst.load_store_register.size) { - 0 => if (inst.load_store_register.v == 1) .load_store_128 else .load_store_8, - 1 => .load_store_16, - 2 => .load_store_32, - 3 => .load_store_64, - }; -} - -pub fn calcPageOffset(kind: PageOffsetInstKind, taddr: u64) !u12 { - const narrowed = @as(u12, @truncate(taddr)); - return switch (kind) { - .arithmetic, .load_store_8 => narrowed, - .load_store_16 => try math.divExact(u12, narrowed, 2), - .load_store_32 => try math.divExact(u12, narrowed, 4), - .load_store_64 => try math.divExact(u12, narrowed, 8), - .load_store_128 => try math.divExact(u12, narrowed, 16), - }; -} - -pub fn writePageOffset(kind: PageOffsetInstKind, taddr: u64, code: *[4]u8) !void { - const value = try calcPageOffset(kind, taddr); - switch (kind) { - .arithmetic => { - var inst = Instruction{ - .add_subtract_immediate = mem.bytesToValue(std.meta.TagPayload( - Instruction, - Instruction.add_subtract_immediate, - ), code), - }; - inst.add_subtract_immediate.imm12 = value; - mem.writeInt(u32, code, inst.toU32(), .little); - }, - else => { - var inst: Instruction = .{ - .load_store_register = mem.bytesToValue(std.meta.TagPayload( - Instruction, - Instruction.load_store_register, - ), code), - }; - inst.load_store_register.offset = value; - mem.writeInt(u32, code, inst.toU32(), .little); - }, - } + inst.load_store_register.offset = value; + mem.writeInt(u32, code, inst.toU32(), .little); } pub fn calcNumberOfPages(saddr: u64, taddr: u64) error{Overflow}!i21 { @@ -72,7 +32,7 @@ pub fn calcNumberOfPages(saddr: u64, taddr: u64) error{Overflow}!i21 { return pages; } -pub fn writePages(pages: u21, code: *[4]u8) !void { +pub fn writeAdrpInst(pages: u21, code: *[4]u8) void { var inst = Instruction{ .pc_relative_address = mem.bytesToValue(std.meta.TagPayload( Instruction, @@ -84,7 +44,7 @@ pub fn writePages(pages: u21, code: *[4]u8) !void { mem.writeInt(u32, code, inst.toU32(), .little); } -pub fn writeBranchImm(disp: i28, code: *[4]u8) !void { +pub fn writeBranchImm(disp: i28, code: *[4]u8) void { var inst = Instruction{ .unconditional_branch_immediate = mem.bytesToValue(std.meta.TagPayload( Instruction, @@ -95,17 +55,6 @@ pub fn writeBranchImm(disp: i28, code: *[4]u8) !void { mem.writeInt(u32, code, inst.toU32(), .little); } -pub fn writeAddInst(value: u12, code: *[4]u8) !void { - var inst = Instruction{ - .add_subtract_immediate = mem.bytesToValue(std.meta.TagPayload( - Instruction, - Instruction.add_subtract_immediate, - ), code), - }; - inst.add_subtract_immediate.imm12 = value; - mem.writeInt(u32, code, inst.toU32(), .little); -} - const assert = std.debug.assert; const bits = @import("../arch/aarch64/bits.zig"); const builtin = @import("builtin");