From 70d809e0bbc51efa78b54838cc64158b4d4a2dd2 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 11 May 2022 23:18:01 +0200 Subject: [PATCH 01/19] x64: add AVX registers and Vex prefix sub-encoder --- src/arch/x86_64/bits.zig | 211 ++++++++++++++++++++++++++++++++++++++- 1 file changed, 209 insertions(+), 2 deletions(-) diff --git a/src/arch/x86_64/bits.zig b/src/arch/x86_64/bits.zig index 02f032ab72..df056fab2c 100644 --- a/src/arch/x86_64/bits.zig +++ b/src/arch/x86_64/bits.zig @@ -8,7 +8,7 @@ const DW = std.dwarf; // zig fmt: off -/// Definitions of all of the x64 registers. The order is semantically meaningful. +/// Definitions of all of the general purpose x64 registers. The order is semantically meaningful. /// The registers are defined such that IDs go in descending order of 64-bit, /// 32-bit, 16-bit, and then 8-bit, and each set contains exactly sixteen /// registers. This results in some useful properties: @@ -126,6 +126,52 @@ pub const Register = enum(u7) { } }; +/// AVX registers. +/// TODO missing dwarfLocOp implementation. +/// TODO add support for AVX-512 +pub const AvxRegister = enum(u6) { + // 256-bit registers + ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7, + ymm8, ymm9, ymm10, ymm11, ymm12, ymm13, ymm14, ymm15, + + // 128-bit registers + xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, + xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, + + // Pseudo, used only for MIR to signify that the + // operand is not a register but an immediate, etc. + none, + + /// Returns the bit-width of the register. + pub fn size(self: AvxRegister) u4 { + return switch (@enumToInt(self)) { + 0...15 => 256, + 16...31 => 128, + else => unreachable, + }; + } + + /// This returns the 4-bit register ID. + pub fn id(self: AvxRegister) u4 { + return @truncate(u4, @enumToInt(self)); + } + + /// Like id, but only returns the lower 3 bits. + pub fn lowId(self: AvxRegister) u3 { + return @truncate(u3, @enumToInt(self)); + } + + /// Convert from any register to its 256 bit alias. + pub fn to256(self: AvxRegister) AvxRegister { + return @intToEnum(AvxRegister, self.id()); + } + + /// Convert from any register to its 128 bit alias. + pub fn to128(self: AvxRegister) AvxRegister { + return @intToEnum(AvxRegister, @as(u8, self.id()) + 16); + } +}; + // zig fmt: on /// Encoding helper functions for x86_64 instructions @@ -251,6 +297,98 @@ pub const Encoder = struct { self.code.appendAssumeCapacity(0x66); } + pub fn Vex(comptime count: comptime_int) type { + if (count < 2 or count > 3) { + @compileError("VEX prefix can either be 2- or 3-byte long"); + } + + return struct { + bytes: [count]u8 = switch (count) { + 2 => .{ 0xc5, 0xf8 }, + 3 => .{ 0xc4, 0xe1, 0xf8 }, + else => unreachable, + }, + + pub fn rex(self: *@This(), prefix: Rex) void { + const byte = &self.bytes[1]; + if (prefix.w) switch (count) { + 3 => self.bytes[2] &= 0b0111_1111, + else => unreachable, + }; + if (prefix.r) byte.* &= 0b0111_1111; + if (prefix.x) switch (count) { + 3 => byte.* &= 0b1011_1111, + else => unreachable, + }; + if (prefix.b) switch (count) { + 3 => byte.* &= 0b1101_1111, + else => unreachable, + }; + } + + pub fn leading_opcode_0f(self: *@This()) void { + switch (count) { + 3 => self.bytes[1] |= 0b0_0001, + else => {}, + } + } + + pub fn leading_opcode_0f_38(self: *@This()) void { + switch (count) { + 3 => self.bytes[1] |= 0b0_0010, + else => unreachable, + } + } + + pub fn leading_opcode_0f_3a(self: *@This()) void { + switch (count) { + 3 => self.bytes[1] |= 0b0_0011, + else => unreachable, + } + } + + pub fn reg(self: *@This(), register: u4) void { + const byte = &self.bytes[count - 1]; + const mask = 0b1_0000_111; + byte.* &= mask; + byte.* |= @intCast(u7, ~register) << 3; + } + + pub fn len_128(self: *@This()) void { + const byte = &self.bytes[count - 1]; + byte.* &= 0b0_11; + } + + pub fn len_256(self: *@This()) void { + const byte = &self.bytes[count - 1]; + byte.* |= 0b1_00; + } + + pub fn simd_prefix_66(self: *@This()) void { + const byte = &self.bytes[count - 1]; + byte.* |= 0b01; + } + + pub fn simd_prefix_f2(self: *@This()) void { + const byte = &self.bytes[count - 1]; + byte.* |= 0b11; + } + + pub fn simd_prefix_f3(self: *@This()) void { + const byte = &self.bytes[count - 1]; + byte.* |= 0b10; + } + }; + } + + pub fn vex_2byte(self: Self, prefix: Vex(2)) void { + self.code.appendSliceAssumeCapacity(&prefix.bytes); + } + + pub fn vex_3byte(self: Self, prefix: Vex(3)) void { + self.code.appendSliceAssumeCapacity(&prefix.bytes); + } + /// From section 2.2.1.2 of the manual, REX is encoded as b0100WRXB pub const Rex = struct { /// Wide, enables 64-bit operation @@ -543,7 +681,7 @@ pub const Encoder = struct { } }; -test "x86_64 Encoder helpers" { +test "Encoder helpers - general purpose registers" { var code = ArrayList(u8).init(testing.allocator); defer code.deinit(); @@ -615,6 +753,75 @@ test "x86_64 Encoder helpers" { } } +test "Encoder helpers - Vex prefix" { + { + var vex_prefix = Encoder.Vex(2){}; + vex_prefix.rex(.{ + .r = true, + }); + try testing.expectEqualSlices(u8, &[_]u8{ 0xc5, 0x78 }, &vex_prefix.bytes); + } + + { + var vex_prefix = Encoder.Vex(2){}; + vex_prefix.reg(AvxRegister.xmm15.id()); + try testing.expectEqualSlices(u8, &[_]u8{ 0xc5, 0x80 }, &vex_prefix.bytes); + } + + { + var vex_prefix = Encoder.Vex(3){}; + vex_prefix.rex(.{ + .w = true, + .x = true, + }); + try testing.expectEqualSlices(u8, &[_]u8{ 0xc4, 0b101_0_0001, 0b0_1111_0_00 }, &vex_prefix.bytes); + } + + { + var vex_prefix = Encoder.Vex(3){}; + vex_prefix.rex(.{ + .w = true, + .r = true, + }); + vex_prefix.len_256(); + vex_prefix.leading_opcode_0f(); + vex_prefix.simd_prefix_66(); + try testing.expectEqualSlices(u8, &[_]u8{ 0xc4, 0b011_0_0001, 0b0_1111_1_01 }, &vex_prefix.bytes); + } + + var code = ArrayList(u8).init(testing.allocator); + defer code.deinit(); + + { + // vmovapd xmm1, xmm2 + const encoder = try Encoder.init(&code, 4); + var vex = Encoder.Vex(2){}; + vex.simd_prefix_66(); + encoder.vex_2byte(vex); // use 64 bit operation + encoder.opcode_1byte(0x28); + encoder.modRm_direct(0, AvxRegister.xmm1.lowId()); + try testing.expectEqualSlices(u8, &[_]u8{ 0xC5, 0xF9, 0x28, 0xC1 }, code.items); + } + + { + try code.resize(0); + + // vmovhpd xmm13, xmm1, qword ptr [rip] + const encoder = try Encoder.init(&code, 9); + var vex = Encoder.Vex(2){}; + vex.len_128(); + vex.simd_prefix_66(); + vex.leading_opcode_0f(); + vex.rex(.{ .r = true }); + vex.reg(AvxRegister.xmm1.id()); + encoder.vex_2byte(vex); + encoder.opcode_1byte(0x16); + encoder.modRm_RIPDisp32(AvxRegister.xmm13.lowId()); + encoder.disp32(0); + try testing.expectEqualSlices(u8, &[_]u8{ 0xC5, 0x71, 0x16, 0x2D, 0x00, 0x00, 0x00, 0x00 }, code.items); + } +} + // TODO add these registers to the enum and populate dwarfLocOp // // Return Address register. This is stored in `0(%rsp, "")` and is not a physical register. // RA = (16, "RA"), From 875a16030c2e58f9f8b9a2b66b43b565e3932cf5 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 12 May 2022 17:40:54 +0200 Subject: [PATCH 02/19] x64: extend Emit to allow for AVX registers --- src/arch/x86_64/Emit.zig | 167 +++++++++++++++++++++++++++------------ src/arch/x86_64/bits.zig | 5 ++ 2 files changed, 122 insertions(+), 50 deletions(-) diff --git a/src/arch/x86_64/Emit.zig b/src/arch/x86_64/Emit.zig index 57100abc0f..52183a542b 100644 --- a/src/arch/x86_64/Emit.zig +++ b/src/arch/x86_64/Emit.zig @@ -25,7 +25,8 @@ const MCValue = @import("CodeGen.zig").MCValue; const Mir = @import("Mir.zig"); const Module = @import("../../Module.zig"); const Instruction = bits.Instruction; -const Register = bits.Register; +const GpRegister = bits.Register; +const AvxRegister = bits.Register; const Type = @import("../../type.zig").Type; mir: Mir, @@ -248,7 +249,7 @@ fn mirPushPop(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { switch (ops.flags) { 0b00 => { // PUSH/POP reg - return lowerToOEnc(tag, ops.reg1, emit.code); + return lowerToOEnc(tag, .{ .register = ops.reg1 }, emit.code); }, 0b01 => { // PUSH/POP r/m64 @@ -271,6 +272,7 @@ fn mirPushPop(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { 0b11 => unreachable, } } + fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); const payload = emit.mir.instructions.items(.data)[inst].payload; @@ -283,9 +285,9 @@ fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Tag, inst: Mir.Inst.I try lowerToMrEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{ .disp = @bitCast(u32, -@intCast(i32, disp)), .base = ops.reg1, - }), reg.to64(), emit.code); + }), .{ .register = reg.to64() }, emit.code); } else { - try lowerToRmEnc(.mov, reg.to64(), RegisterOrMemory.mem(.qword_ptr, .{ + try lowerToRmEnc(.mov, .{ .register = reg.to64() }, RegisterOrMemory.mem(.qword_ptr, .{ .disp = @bitCast(u32, -@intCast(i32, disp)), .base = ops.reg1, }), emit.code); @@ -319,7 +321,7 @@ fn mirJmpCall(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { return lowerToMEnc(tag, RegisterOrMemory.mem(ptr_size, .{ .disp = imm }), emit.code); } // JMP/CALL reg - return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code); + return lowerToMEnc(tag, RegisterOrMemory.reg(.{ .register = ops.reg1 }), emit.code); }, 0b10 => { // JMP/CALL r/m64 @@ -392,13 +394,13 @@ fn mirCondSetByte(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) Inne }, else => unreachable, }; - return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1.to8()), emit.code); + return lowerToMEnc(tag, RegisterOrMemory.reg(.{ .register = ops.reg1.to8() }), emit.code); } fn mirCondMov(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); if (ops.flags == 0b00) { - return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code); + return lowerToRmEnc(tag, .{ .register = ops.reg1 }, RegisterOrMemory.reg(.{ .register = ops.reg2 }), emit.code); } const imm = emit.mir.instructions.items(.data)[inst].imm; const ptr_size: Memory.PtrSize = switch (ops.flags) { @@ -407,7 +409,7 @@ fn mirCondMov(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { 0b10 => .dword_ptr, 0b11 => .qword_ptr, }; - return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(ptr_size, .{ + return lowerToRmEnc(tag, .{ .register = ops.reg1 }, RegisterOrMemory.mem(ptr_size, .{ .disp = imm, .base = ops.reg2, }), emit.code); @@ -428,10 +430,15 @@ fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { // I return lowerToIEnc(.@"test", imm, emit.code); } - return lowerToMiEnc(.@"test", RegisterOrMemory.reg(ops.reg1), imm, emit.code); + return lowerToMiEnc(.@"test", RegisterOrMemory.reg(.{ .register = ops.reg1 }), imm, emit.code); } // TEST r/m64, r64 - return lowerToMrEnc(.@"test", RegisterOrMemory.reg(ops.reg1), ops.reg2, emit.code); + return lowerToMrEnc( + .@"test", + RegisterOrMemory.reg(.{ .register = ops.reg1 }), + .{ .register = ops.reg2 }, + emit.code, + ); }, else => return emit.fail("TODO more TEST alternatives", .{}), } @@ -471,18 +478,18 @@ fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { // mov reg1, imm32 // MI const imm = emit.mir.instructions.items(.data)[inst].imm; - return lowerToMiEnc(tag, RegisterOrMemory.reg(ops.reg1), imm, emit.code); + return lowerToMiEnc(tag, RegisterOrMemory.reg(.{ .register = ops.reg1 }), imm, emit.code); } // mov reg1, reg2 // RM - return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code); + return lowerToRmEnc(tag, .{ .register = ops.reg1 }, RegisterOrMemory.reg(.{ .register = ops.reg2 }), emit.code); }, 0b01 => { // mov reg1, [reg2 + imm32] // RM const imm = emit.mir.instructions.items(.data)[inst].imm; - const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2; - return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{ + const src_reg: ?GpRegister = if (ops.reg2 == .none) null else ops.reg2; + return lowerToRmEnc(tag, .{ .register = ops.reg1 }, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{ .disp = imm, .base = src_reg, }), emit.code); @@ -497,7 +504,7 @@ fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg2.size()), .{ .disp = imm, .base = ops.reg1, - }), ops.reg2, emit.code); + }), .{ .register = ops.reg2 }, emit.code); }, 0b11 => { return emit.fail("TODO unused variant: mov reg1, reg2, 0b11", .{}); @@ -523,11 +530,16 @@ fn mirArithMemImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { } inline fn setRexWRegister(reg: Register) bool { - if (reg.size() == 64) return true; - return switch (reg) { - .ah, .bh, .ch, .dh => true, - else => false, - }; + switch (reg) { + .avx_register => return false, + .register => |r| { + if (r.size() == 64) return true; + return switch (r) { + .ah, .bh, .ch, .dh => true, + else => false, + }; + }, + } } inline fn immOpSize(u_imm: u32) u8 { @@ -550,7 +562,7 @@ fn mirArithScaleSrc(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void .scale = scale, .index = .rcx, }; - return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{ + return lowerToRmEnc(tag, .{ .register = ops.reg1 }, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{ .disp = imm, .base = ops.reg2, .scale_index = scale_index, @@ -578,7 +590,7 @@ fn mirArithScaleDst(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void .disp = imm, .base = ops.reg1, .scale_index = scale_index, - }), ops.reg2, emit.code); + }), .{ .register = ops.reg2 }, emit.code); } fn mirArithScaleImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { @@ -629,22 +641,27 @@ fn mirMovSignExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { switch (ops.flags) { 0b00 => { const tag: Tag = if (ops.reg2.size() == 32) .movsxd else .movsx; - return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code); + return lowerToRmEnc( + tag, + .{ .register = ops.reg1 }, + RegisterOrMemory.reg(.{ .register = ops.reg2 }), + emit.code, + ); }, 0b01 => { - return lowerToRmEnc(.movsx, ops.reg1, RegisterOrMemory.mem(.byte_ptr, .{ + return lowerToRmEnc(.movsx, .{ .register = ops.reg1 }, RegisterOrMemory.mem(.byte_ptr, .{ .disp = imm, .base = ops.reg2, }), emit.code); }, 0b10 => { - return lowerToRmEnc(.movsx, ops.reg1, RegisterOrMemory.mem(.word_ptr, .{ + return lowerToRmEnc(.movsx, .{ .register = ops.reg1 }, RegisterOrMemory.mem(.word_ptr, .{ .disp = imm, .base = ops.reg2, }), emit.code); }, 0b11 => { - return lowerToRmEnc(.movsxd, ops.reg1, RegisterOrMemory.mem(.dword_ptr, .{ + return lowerToRmEnc(.movsxd, .{ .register = ops.reg1 }, RegisterOrMemory.mem(.dword_ptr, .{ .disp = imm, .base = ops.reg2, }), emit.code); @@ -659,16 +676,21 @@ fn mirMovZeroExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const imm = if (ops.flags != 0b00) emit.mir.instructions.items(.data)[inst].imm else undefined; switch (ops.flags) { 0b00 => { - return lowerToRmEnc(.movzx, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code); + return lowerToRmEnc( + .movzx, + .{ .register = ops.reg1 }, + RegisterOrMemory.reg(.{ .register = ops.reg2 }), + emit.code, + ); }, 0b01 => { - return lowerToRmEnc(.movzx, ops.reg1, RegisterOrMemory.mem(.byte_ptr, .{ + return lowerToRmEnc(.movzx, .{ .register = ops.reg1 }, RegisterOrMemory.mem(.byte_ptr, .{ .disp = imm, .base = ops.reg2, }), emit.code); }, 0b10 => { - return lowerToRmEnc(.movzx, ops.reg1, RegisterOrMemory.mem(.word_ptr, .{ + return lowerToRmEnc(.movzx, .{ .register = ops.reg1 }, RegisterOrMemory.mem(.word_ptr, .{ .disp = imm, .base = ops.reg2, }), emit.code); @@ -691,16 +713,16 @@ fn mirMovabs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { if (ops.flags == 0b00) { // movabs reg, imm64 // OI - return lowerToOiEnc(.mov, ops.reg1, imm, emit.code); + return lowerToOiEnc(.mov, .{ .register = ops.reg1 }, imm, emit.code); } if (ops.reg1 == .none) { // movabs moffs64, rax // TD - return lowerToTdEnc(.mov, imm, ops.reg2, emit.code); + return lowerToTdEnc(.mov, imm, .{ .register = ops.reg2 }, emit.code); } // movabs rax, moffs64 // FD - return lowerToFdEnc(.mov, ops.reg1, imm, emit.code); + return lowerToFdEnc(.mov, .{ .register = ops.reg1 }, imm, emit.code); } fn mirFisttp(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { @@ -751,18 +773,18 @@ fn mirShift(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { 0b00 => { // sal reg1, 1 // M1 - return lowerToM1Enc(tag, RegisterOrMemory.reg(ops.reg1), emit.code); + return lowerToM1Enc(tag, RegisterOrMemory.reg(.{ .register = ops.reg1 }), emit.code); }, 0b01 => { // sal reg1, .cl // MC - return lowerToMcEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code); + return lowerToMcEnc(tag, RegisterOrMemory.reg(.{ .register = ops.reg1 }), emit.code); }, 0b10 => { // sal reg1, imm8 // MI const imm = @truncate(u8, emit.mir.instructions.items(.data)[inst].imm); - return lowerToMiImm8Enc(tag, RegisterOrMemory.reg(ops.reg1), imm, emit.code); + return lowerToMiImm8Enc(tag, RegisterOrMemory.reg(.{ .register = ops.reg1 }), imm, emit.code); }, 0b11 => { return emit.fail("TODO unused variant: SHIFT reg1, 0b11", .{}); @@ -774,7 +796,7 @@ fn mirMulDiv(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); if (ops.reg1 != .none) { assert(ops.reg2 == .none); - return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code); + return lowerToMEnc(tag, RegisterOrMemory.reg(.{ .register = ops.reg1 }), emit.code); } assert(ops.reg1 == .none); assert(ops.reg2 != .none); @@ -797,24 +819,35 @@ fn mirIMulComplex(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); switch (ops.flags) { 0b00 => { - return lowerToRmEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code); + return lowerToRmEnc( + .imul, + .{ .register = ops.reg1 }, + RegisterOrMemory.reg(.{ .register = ops.reg2 }), + emit.code, + ); }, 0b01 => { const imm = emit.mir.instructions.items(.data)[inst].imm; - const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2; - return lowerToRmEnc(.imul, ops.reg1, RegisterOrMemory.mem(.qword_ptr, .{ + const src_reg: ?GpRegister = if (ops.reg2 == .none) null else ops.reg2; + return lowerToRmEnc(.imul, .{ .register = ops.reg1 }, RegisterOrMemory.mem(.qword_ptr, .{ .disp = imm, .base = src_reg, }), emit.code); }, 0b10 => { const imm = emit.mir.instructions.items(.data)[inst].imm; - return lowerToRmiEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), imm, emit.code); + return lowerToRmiEnc( + .imul, + .{ .register = ops.reg1 }, + RegisterOrMemory.reg(.{ .register = ops.reg2 }), + imm, + emit.code, + ); }, 0b11 => { const payload = emit.mir.instructions.items(.data)[inst].payload; const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data; - return lowerToRmiEnc(.imul, ops.reg1, RegisterOrMemory.mem(.qword_ptr, .{ + return lowerToRmiEnc(.imul, .{ .register = ops.reg1 }, RegisterOrMemory.mem(.qword_ptr, .{ .disp = imm_pair.dest_off, .base = ops.reg2, }), imm_pair.operand, emit.code); @@ -842,10 +875,10 @@ fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { // lea reg1, [reg2 + imm32] // RM const imm = emit.mir.instructions.items(.data)[inst].imm; - const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2; + const src_reg: ?GpRegister = if (ops.reg2 == .none) null else ops.reg2; return lowerToRmEnc( .lea, - ops.reg1, + .{ .register = ops.reg1 }, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{ .disp = imm, .base = src_reg, @@ -859,7 +892,7 @@ fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const start_offset = emit.code.items.len; try lowerToRmEnc( .lea, - ops.reg1, + .{ .register = ops.reg1 }, RegisterOrMemory.rip(Memory.PtrSize.fromBits(ops.reg1.size()), 0), emit.code, ); @@ -873,14 +906,14 @@ fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { 0b10 => { // lea reg, [rbp + rcx + imm32] const imm = emit.mir.instructions.items(.data)[inst].imm; - const src_reg: ?Register = if (ops.reg2 == .none) null else ops.reg2; + const src_reg: ?GpRegister = if (ops.reg2 == .none) null else ops.reg2; const scale_index = ScaleIndex{ .scale = 0, .index = .rcx, }; return lowerToRmEnc( .lea, - ops.reg1, + .{ .register = ops.reg1 }, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{ .disp = imm, .base = src_reg, @@ -903,7 +936,7 @@ fn mirLeaPie(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { // RM try lowerToRmEnc( .lea, - ops.reg1, + .{ .register = ops.reg1 }, RegisterOrMemory.rip(Memory.PtrSize.fromBits(ops.reg1.size()), 0), emit.code, ); @@ -1489,11 +1522,11 @@ inline fn getModRmExt(tag: Tag) ?u3 { const ScaleIndex = struct { scale: u2, - index: Register, + index: GpRegister, }; const Memory = struct { - base: ?Register, + base: ?GpRegister, rip: bool = false, disp: u32, ptr_size: PtrSize, @@ -1595,6 +1628,40 @@ fn encodeImm(encoder: Encoder, imm: u32, size: u64) void { } } +const Register = union(enum) { + register: GpRegister, + avx_register: AvxRegister, + + fn reg(register: GpRegister) Register { + return .{ .register = register }; + } + + fn avxReg(register: AvxRegister) Register { + return .{ .avx_register = register }; + } + + fn lowId(register: Register) u3 { + return switch (register) { + .register => |r| r.lowId(), + .avx_register => |r| r.lowId(), + }; + } + + fn size(register: Register) u64 { + return switch (register) { + .register => |r| r.size(), + .avx_register => |r| r.size(), + }; + } + + fn isExtended(register: Register) bool { + return switch (register) { + .register => |r| r.isExtended(), + .avx_register => |r| r.isExtended(), + }; + } +}; + const RegisterOrMemory = union(enum) { register: Register, memory: Memory, @@ -1605,7 +1672,7 @@ const RegisterOrMemory = union(enum) { fn mem(ptr_size: Memory.PtrSize, args: struct { disp: u32, - base: ?Register = null, + base: ?GpRegister = null, scale_index: ?ScaleIndex = null, }) RegisterOrMemory { return .{ diff --git a/src/arch/x86_64/bits.zig b/src/arch/x86_64/bits.zig index df056fab2c..43d5b92eb2 100644 --- a/src/arch/x86_64/bits.zig +++ b/src/arch/x86_64/bits.zig @@ -151,6 +151,11 @@ pub const AvxRegister = enum(u6) { }; } + /// Returns whether the register is *extended*. + pub fn isExtended(self: Register) bool { + return @enumToInt(self) & 0x08 != 0; + } + /// This returns the 4-bit register ID. pub fn id(self: AvxRegister) u4 { return @truncate(u4, @enumToInt(self)); From 019cc94ec7291c652b3e18980194e028f8df9ec1 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 12 May 2022 20:52:52 +0200 Subject: [PATCH 03/19] x64: clean up populating VEX prefix --- src/arch/x86_64/Emit.zig | 217 +++++++++++++++++++------------------- src/arch/x86_64/bits.zig | 220 ++++++++++++++++++++++----------------- 2 files changed, 229 insertions(+), 208 deletions(-) diff --git a/src/arch/x86_64/Emit.zig b/src/arch/x86_64/Emit.zig index 52183a542b..848371f4ba 100644 --- a/src/arch/x86_64/Emit.zig +++ b/src/arch/x86_64/Emit.zig @@ -249,7 +249,7 @@ fn mirPushPop(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { switch (ops.flags) { 0b00 => { // PUSH/POP reg - return lowerToOEnc(tag, .{ .register = ops.reg1 }, emit.code); + return lowerToOEnc(tag, Register.reg(ops.reg1), emit.code); }, 0b01 => { // PUSH/POP r/m64 @@ -285,9 +285,9 @@ fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Tag, inst: Mir.Inst.I try lowerToMrEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{ .disp = @bitCast(u32, -@intCast(i32, disp)), .base = ops.reg1, - }), .{ .register = reg.to64() }, emit.code); + }), Register.reg(reg.to64()), emit.code); } else { - try lowerToRmEnc(.mov, .{ .register = reg.to64() }, RegisterOrMemory.mem(.qword_ptr, .{ + try lowerToRmEnc(.mov, Register.reg(reg.to64()), RegisterOrMemory.mem(.qword_ptr, .{ .disp = @bitCast(u32, -@intCast(i32, disp)), .base = ops.reg1, }), emit.code); @@ -321,7 +321,7 @@ fn mirJmpCall(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { return lowerToMEnc(tag, RegisterOrMemory.mem(ptr_size, .{ .disp = imm }), emit.code); } // JMP/CALL reg - return lowerToMEnc(tag, RegisterOrMemory.reg(.{ .register = ops.reg1 }), emit.code); + return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code); }, 0b10 => { // JMP/CALL r/m64 @@ -394,13 +394,13 @@ fn mirCondSetByte(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) Inne }, else => unreachable, }; - return lowerToMEnc(tag, RegisterOrMemory.reg(.{ .register = ops.reg1.to8() }), emit.code); + return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1.to8()), emit.code); } fn mirCondMov(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); if (ops.flags == 0b00) { - return lowerToRmEnc(tag, .{ .register = ops.reg1 }, RegisterOrMemory.reg(.{ .register = ops.reg2 }), emit.code); + return lowerToRmEnc(tag, Register.reg(ops.reg1), RegisterOrMemory.reg(ops.reg2), emit.code); } const imm = emit.mir.instructions.items(.data)[inst].imm; const ptr_size: Memory.PtrSize = switch (ops.flags) { @@ -409,7 +409,7 @@ fn mirCondMov(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { 0b10 => .dword_ptr, 0b11 => .qword_ptr, }; - return lowerToRmEnc(tag, .{ .register = ops.reg1 }, RegisterOrMemory.mem(ptr_size, .{ + return lowerToRmEnc(tag, Register.reg(ops.reg1), RegisterOrMemory.mem(ptr_size, .{ .disp = imm, .base = ops.reg2, }), emit.code); @@ -430,13 +430,13 @@ fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { // I return lowerToIEnc(.@"test", imm, emit.code); } - return lowerToMiEnc(.@"test", RegisterOrMemory.reg(.{ .register = ops.reg1 }), imm, emit.code); + return lowerToMiEnc(.@"test", RegisterOrMemory.reg(ops.reg1), imm, emit.code); } // TEST r/m64, r64 return lowerToMrEnc( .@"test", - RegisterOrMemory.reg(.{ .register = ops.reg1 }), - .{ .register = ops.reg2 }, + RegisterOrMemory.reg(ops.reg1), + Register.reg(ops.reg2), emit.code, ); }, @@ -478,18 +478,18 @@ fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { // mov reg1, imm32 // MI const imm = emit.mir.instructions.items(.data)[inst].imm; - return lowerToMiEnc(tag, RegisterOrMemory.reg(.{ .register = ops.reg1 }), imm, emit.code); + return lowerToMiEnc(tag, RegisterOrMemory.reg(ops.reg1), imm, emit.code); } // mov reg1, reg2 // RM - return lowerToRmEnc(tag, .{ .register = ops.reg1 }, RegisterOrMemory.reg(.{ .register = ops.reg2 }), emit.code); + return lowerToRmEnc(tag, Register.reg(ops.reg1), RegisterOrMemory.reg(ops.reg2), emit.code); }, 0b01 => { // mov reg1, [reg2 + imm32] // RM const imm = emit.mir.instructions.items(.data)[inst].imm; const src_reg: ?GpRegister = if (ops.reg2 == .none) null else ops.reg2; - return lowerToRmEnc(tag, .{ .register = ops.reg1 }, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{ + return lowerToRmEnc(tag, Register.reg(ops.reg1), RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{ .disp = imm, .base = src_reg, }), emit.code); @@ -504,7 +504,7 @@ fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg2.size()), .{ .disp = imm, .base = ops.reg1, - }), .{ .register = ops.reg2 }, emit.code); + }), Register.reg(ops.reg2), emit.code); }, 0b11 => { return emit.fail("TODO unused variant: mov reg1, reg2, 0b11", .{}); @@ -562,7 +562,7 @@ fn mirArithScaleSrc(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void .scale = scale, .index = .rcx, }; - return lowerToRmEnc(tag, .{ .register = ops.reg1 }, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{ + return lowerToRmEnc(tag, Register.reg(ops.reg1), RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{ .disp = imm, .base = ops.reg2, .scale_index = scale_index, @@ -590,7 +590,7 @@ fn mirArithScaleDst(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void .disp = imm, .base = ops.reg1, .scale_index = scale_index, - }), .{ .register = ops.reg2 }, emit.code); + }), Register.reg(ops.reg2), emit.code); } fn mirArithScaleImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { @@ -641,27 +641,22 @@ fn mirMovSignExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { switch (ops.flags) { 0b00 => { const tag: Tag = if (ops.reg2.size() == 32) .movsxd else .movsx; - return lowerToRmEnc( - tag, - .{ .register = ops.reg1 }, - RegisterOrMemory.reg(.{ .register = ops.reg2 }), - emit.code, - ); + return lowerToRmEnc(tag, Register.reg(ops.reg1), RegisterOrMemory.reg(ops.reg2), emit.code); }, 0b01 => { - return lowerToRmEnc(.movsx, .{ .register = ops.reg1 }, RegisterOrMemory.mem(.byte_ptr, .{ + return lowerToRmEnc(.movsx, Register.reg(ops.reg1), RegisterOrMemory.mem(.byte_ptr, .{ .disp = imm, .base = ops.reg2, }), emit.code); }, 0b10 => { - return lowerToRmEnc(.movsx, .{ .register = ops.reg1 }, RegisterOrMemory.mem(.word_ptr, .{ + return lowerToRmEnc(.movsx, Register.reg(ops.reg1), RegisterOrMemory.mem(.word_ptr, .{ .disp = imm, .base = ops.reg2, }), emit.code); }, 0b11 => { - return lowerToRmEnc(.movsxd, .{ .register = ops.reg1 }, RegisterOrMemory.mem(.dword_ptr, .{ + return lowerToRmEnc(.movsxd, Register.reg(ops.reg1), RegisterOrMemory.mem(.dword_ptr, .{ .disp = imm, .base = ops.reg2, }), emit.code); @@ -676,21 +671,16 @@ fn mirMovZeroExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const imm = if (ops.flags != 0b00) emit.mir.instructions.items(.data)[inst].imm else undefined; switch (ops.flags) { 0b00 => { - return lowerToRmEnc( - .movzx, - .{ .register = ops.reg1 }, - RegisterOrMemory.reg(.{ .register = ops.reg2 }), - emit.code, - ); + return lowerToRmEnc(.movzx, Register.reg(ops.reg1), RegisterOrMemory.reg(ops.reg2), emit.code); }, 0b01 => { - return lowerToRmEnc(.movzx, .{ .register = ops.reg1 }, RegisterOrMemory.mem(.byte_ptr, .{ + return lowerToRmEnc(.movzx, Register.reg(ops.reg1), RegisterOrMemory.mem(.byte_ptr, .{ .disp = imm, .base = ops.reg2, }), emit.code); }, 0b10 => { - return lowerToRmEnc(.movzx, .{ .register = ops.reg1 }, RegisterOrMemory.mem(.word_ptr, .{ + return lowerToRmEnc(.movzx, Register.reg(ops.reg1), RegisterOrMemory.mem(.word_ptr, .{ .disp = imm, .base = ops.reg2, }), emit.code); @@ -713,16 +703,16 @@ fn mirMovabs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { if (ops.flags == 0b00) { // movabs reg, imm64 // OI - return lowerToOiEnc(.mov, .{ .register = ops.reg1 }, imm, emit.code); + return lowerToOiEnc(.mov, Register.reg(ops.reg1), imm, emit.code); } if (ops.reg1 == .none) { // movabs moffs64, rax // TD - return lowerToTdEnc(.mov, imm, .{ .register = ops.reg2 }, emit.code); + return lowerToTdEnc(.mov, imm, Register.reg(ops.reg2), emit.code); } // movabs rax, moffs64 // FD - return lowerToFdEnc(.mov, .{ .register = ops.reg1 }, imm, emit.code); + return lowerToFdEnc(.mov, Register.reg(ops.reg1), imm, emit.code); } fn mirFisttp(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { @@ -773,18 +763,18 @@ fn mirShift(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { 0b00 => { // sal reg1, 1 // M1 - return lowerToM1Enc(tag, RegisterOrMemory.reg(.{ .register = ops.reg1 }), emit.code); + return lowerToM1Enc(tag, RegisterOrMemory.reg(ops.reg1), emit.code); }, 0b01 => { // sal reg1, .cl // MC - return lowerToMcEnc(tag, RegisterOrMemory.reg(.{ .register = ops.reg1 }), emit.code); + return lowerToMcEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code); }, 0b10 => { // sal reg1, imm8 // MI const imm = @truncate(u8, emit.mir.instructions.items(.data)[inst].imm); - return lowerToMiImm8Enc(tag, RegisterOrMemory.reg(.{ .register = ops.reg1 }), imm, emit.code); + return lowerToMiImm8Enc(tag, RegisterOrMemory.reg(ops.reg1), imm, emit.code); }, 0b11 => { return emit.fail("TODO unused variant: SHIFT reg1, 0b11", .{}); @@ -796,7 +786,7 @@ fn mirMulDiv(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); if (ops.reg1 != .none) { assert(ops.reg2 == .none); - return lowerToMEnc(tag, RegisterOrMemory.reg(.{ .register = ops.reg1 }), emit.code); + return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code); } assert(ops.reg1 == .none); assert(ops.reg2 != .none); @@ -819,35 +809,24 @@ fn mirIMulComplex(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); switch (ops.flags) { 0b00 => { - return lowerToRmEnc( - .imul, - .{ .register = ops.reg1 }, - RegisterOrMemory.reg(.{ .register = ops.reg2 }), - emit.code, - ); + return lowerToRmEnc(.imul, Register.reg(ops.reg1), RegisterOrMemory.reg(ops.reg2), emit.code); }, 0b01 => { const imm = emit.mir.instructions.items(.data)[inst].imm; const src_reg: ?GpRegister = if (ops.reg2 == .none) null else ops.reg2; - return lowerToRmEnc(.imul, .{ .register = ops.reg1 }, RegisterOrMemory.mem(.qword_ptr, .{ + return lowerToRmEnc(.imul, Register.reg(ops.reg1), RegisterOrMemory.mem(.qword_ptr, .{ .disp = imm, .base = src_reg, }), emit.code); }, 0b10 => { const imm = emit.mir.instructions.items(.data)[inst].imm; - return lowerToRmiEnc( - .imul, - .{ .register = ops.reg1 }, - RegisterOrMemory.reg(.{ .register = ops.reg2 }), - imm, - emit.code, - ); + return lowerToRmiEnc(.imul, Register.reg(ops.reg1), RegisterOrMemory.reg(ops.reg2), imm, emit.code); }, 0b11 => { const payload = emit.mir.instructions.items(.data)[inst].payload; const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data; - return lowerToRmiEnc(.imul, .{ .register = ops.reg1 }, RegisterOrMemory.mem(.qword_ptr, .{ + return lowerToRmiEnc(.imul, Register.reg(ops.reg1), RegisterOrMemory.mem(.qword_ptr, .{ .disp = imm_pair.dest_off, .base = ops.reg2, }), imm_pair.operand, emit.code); @@ -878,7 +857,7 @@ fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const src_reg: ?GpRegister = if (ops.reg2 == .none) null else ops.reg2; return lowerToRmEnc( .lea, - .{ .register = ops.reg1 }, + Register.reg(ops.reg1), RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{ .disp = imm, .base = src_reg, @@ -892,7 +871,7 @@ fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const start_offset = emit.code.items.len; try lowerToRmEnc( .lea, - .{ .register = ops.reg1 }, + Register.reg(ops.reg1), RegisterOrMemory.rip(Memory.PtrSize.fromBits(ops.reg1.size()), 0), emit.code, ); @@ -913,7 +892,7 @@ fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { }; return lowerToRmEnc( .lea, - .{ .register = ops.reg1 }, + Register.reg(ops.reg1), RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{ .disp = imm, .base = src_reg, @@ -936,7 +915,7 @@ fn mirLeaPie(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { // RM try lowerToRmEnc( .lea, - .{ .register = ops.reg1 }, + Register.reg(ops.reg1), RegisterOrMemory.rip(Memory.PtrSize.fromBits(ops.reg1.size()), 0), emit.code, ); @@ -1666,8 +1645,12 @@ const RegisterOrMemory = union(enum) { register: Register, memory: Memory, - fn reg(register: Register) RegisterOrMemory { - return .{ .register = register }; + fn reg(register: GpRegister) RegisterOrMemory { + return .{ .register = Register.reg(register) }; + } + + fn avxReg(register: AvxRegister) RegisterOrMemory { + return .{ .register = Register.avxReg(register) }; } fn mem(ptr_size: Memory.PtrSize, args: struct { @@ -1976,22 +1959,32 @@ fn lowerToMrEnc( encoder.modRm_direct(reg.lowId(), dst_reg.lowId()); }, .memory => |dst_mem| { - const encoder = try Encoder.init(code, 9); - if (reg.size() == 16) { - encoder.prefix16BitMode(); - } - if (dst_mem.base) |base| { - encoder.rex(.{ - .w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg), - .r = reg.isExtended(), - .b = base.isExtended(), - }); - } else { - encoder.rex(.{ - .w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg), - .r = reg.isExtended(), - }); - } + const encoder = blk: { + switch (reg) { + .register => { + const encoder = try Encoder.init(code, 9); + if (reg.size() == 16) { + encoder.prefix16BitMode(); + } + if (dst_mem.base) |base| { + encoder.rex(.{ + .w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg), + .r = reg.isExtended(), + .b = base.isExtended(), + }); + } else { + encoder.rex(.{ + .w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg), + .r = reg.isExtended(), + }); + } + break :blk encoder; + }, + .avx_register => { + unreachable; + }, + } + }; opc.encode(encoder); dst_mem.encode(encoder, reg.lowId()); }, @@ -2168,23 +2161,23 @@ test "lower MI encoding" { test "lower RM encoding" { var emit = TestEmit.init(); defer emit.deinit(); - try lowerToRmEnc(.mov, .rax, RegisterOrMemory.reg(.rbx), emit.code()); + try lowerToRmEnc(.mov, Register.reg(.rax), RegisterOrMemory.reg(.rbx), emit.code()); try expectEqualHexStrings("\x48\x8b\xc3", emit.lowered(), "mov rax, rbx"); - try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0, .base = .r11 }), emit.code()); + try lowerToRmEnc(.mov, Register.reg(.rax), RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0, .base = .r11 }), emit.code()); try expectEqualHexStrings("\x49\x8b\x03", emit.lowered(), "mov rax, qword ptr [r11 + 0]"); - try lowerToRmEnc(.add, .r11, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10000000 }), emit.code()); + try lowerToRmEnc(.add, Register.reg(.r11), RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10000000 }), emit.code()); try expectEqualHexStrings( "\x4C\x03\x1C\x25\x00\x00\x00\x10", emit.lowered(), "add r11, qword ptr [ds:0x10000000]", ); - try lowerToRmEnc(.add, .r12b, RegisterOrMemory.mem(.byte_ptr, .{ .disp = 0x10000000 }), emit.code()); + try lowerToRmEnc(.add, Register.reg(.r12b), RegisterOrMemory.mem(.byte_ptr, .{ .disp = 0x10000000 }), emit.code()); try expectEqualHexStrings( "\x44\x02\x24\x25\x00\x00\x00\x10", emit.lowered(), "add r11b, byte ptr [ds:0x10000000]", ); - try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.qword_ptr, .{ + try lowerToRmEnc(.sub, Register.reg(.r11), RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10000000, .base = .r13, }), emit.code()); @@ -2193,7 +2186,7 @@ test "lower RM encoding" { emit.lowered(), "sub r11, qword ptr [r13 + 0x10000000]", ); - try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.qword_ptr, .{ + try lowerToRmEnc(.sub, Register.reg(.r11), RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10000000, .base = .r12, }), emit.code()); @@ -2202,14 +2195,14 @@ test "lower RM encoding" { emit.lowered(), "sub r11, qword ptr [r12 + 0x10000000]", ); - try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{ + try lowerToRmEnc(.mov, Register.reg(.rax), RegisterOrMemory.mem(.qword_ptr, .{ .disp = @bitCast(u32, @as(i32, -4)), .base = .rbp, }), emit.code()); try expectEqualHexStrings("\x48\x8B\x45\xFC", emit.lowered(), "mov rax, qword ptr [rbp - 4]"); - try lowerToRmEnc(.lea, .rax, RegisterOrMemory.rip(.qword_ptr, 0x10), emit.code()); + try lowerToRmEnc(.lea, Register.reg(.rax), RegisterOrMemory.rip(.qword_ptr, 0x10), emit.code()); try expectEqualHexStrings("\x48\x8D\x05\x10\x00\x00\x00", emit.lowered(), "lea rax, [rip + 0x10]"); - try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{ + try lowerToRmEnc(.mov, Register.reg(.rax), RegisterOrMemory.mem(.qword_ptr, .{ .disp = @bitCast(u32, @as(i32, -8)), .base = .rbp, .scale_index = .{ @@ -2218,7 +2211,7 @@ test "lower RM encoding" { }, }), emit.code()); try expectEqualHexStrings("\x48\x8B\x44\x0D\xF8", emit.lowered(), "mov rax, qword ptr [rbp + rcx*1 - 8]"); - try lowerToRmEnc(.mov, .eax, RegisterOrMemory.mem(.dword_ptr, .{ + try lowerToRmEnc(.mov, Register.reg(.eax), RegisterOrMemory.mem(.dword_ptr, .{ .disp = @bitCast(u32, @as(i32, -4)), .base = .rbp, .scale_index = .{ @@ -2227,7 +2220,7 @@ test "lower RM encoding" { }, }), emit.code()); try expectEqualHexStrings("\x8B\x44\x95\xFC", emit.lowered(), "mov eax, dword ptr [rbp + rdx*4 - 4]"); - try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{ + try lowerToRmEnc(.mov, Register.reg(.rax), RegisterOrMemory.mem(.qword_ptr, .{ .disp = @bitCast(u32, @as(i32, -8)), .base = .rbp, .scale_index = .{ @@ -2236,7 +2229,7 @@ test "lower RM encoding" { }, }), emit.code()); try expectEqualHexStrings("\x48\x8B\x44\xCD\xF8", emit.lowered(), "mov rax, qword ptr [rbp + rcx*8 - 8]"); - try lowerToRmEnc(.mov, .r8b, RegisterOrMemory.mem(.byte_ptr, .{ + try lowerToRmEnc(.mov, Register.reg(.r8b), RegisterOrMemory.mem(.byte_ptr, .{ .disp = @bitCast(u32, @as(i32, -24)), .base = .rsi, .scale_index = .{ @@ -2245,7 +2238,7 @@ test "lower RM encoding" { }, }), emit.code()); try expectEqualHexStrings("\x44\x8A\x44\x0E\xE8", emit.lowered(), "mov r8b, byte ptr [rsi + rcx*1 - 24]"); - try lowerToRmEnc(.lea, .rsi, RegisterOrMemory.mem(.qword_ptr, .{ + try lowerToRmEnc(.lea, Register.reg(.rsi), RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0, .base = .rbp, .scale_index = .{ @@ -2259,20 +2252,20 @@ test "lower RM encoding" { test "lower MR encoding" { var emit = TestEmit.init(); defer emit.deinit(); - try lowerToMrEnc(.mov, RegisterOrMemory.reg(.rax), .rbx, emit.code()); + try lowerToMrEnc(.mov, RegisterOrMemory.reg(.rax), Register.reg(.rbx), emit.code()); try expectEqualHexStrings("\x48\x89\xd8", emit.lowered(), "mov rax, rbx"); try lowerToMrEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{ .disp = @bitCast(u32, @as(i32, -4)), .base = .rbp, - }), .r11, emit.code()); + }), Register.reg(.r11), emit.code()); try expectEqualHexStrings("\x4c\x89\x5d\xfc", emit.lowered(), "mov qword ptr [rbp - 4], r11"); - try lowerToMrEnc(.add, RegisterOrMemory.mem(.byte_ptr, .{ .disp = 0x10000000 }), .r12b, emit.code()); + try lowerToMrEnc(.add, RegisterOrMemory.mem(.byte_ptr, .{ .disp = 0x10000000 }), Register.reg(.r12b), emit.code()); try expectEqualHexStrings( "\x44\x00\x24\x25\x00\x00\x00\x10", emit.lowered(), "add byte ptr [ds:0x10000000], r12b", ); - try lowerToMrEnc(.add, RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0x10000000 }), .r12d, emit.code()); + try lowerToMrEnc(.add, RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0x10000000 }), Register.reg(.r12d), emit.code()); try expectEqualHexStrings( "\x44\x01\x24\x25\x00\x00\x00\x10", emit.lowered(), @@ -2281,53 +2274,53 @@ test "lower MR encoding" { try lowerToMrEnc(.sub, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10000000, .base = .r11, - }), .r12, emit.code()); + }), Register.reg(.r12), emit.code()); try expectEqualHexStrings( "\x4D\x29\xA3\x00\x00\x00\x10", emit.lowered(), "sub qword ptr [r11 + 0x10000000], r12", ); - try lowerToMrEnc(.mov, RegisterOrMemory.rip(.qword_ptr, 0x10), .r12, emit.code()); + try lowerToMrEnc(.mov, RegisterOrMemory.rip(.qword_ptr, 0x10), Register.reg(.r12), emit.code()); try expectEqualHexStrings("\x4C\x89\x25\x10\x00\x00\x00", emit.lowered(), "mov qword ptr [rip + 0x10], r12"); } test "lower OI encoding" { var emit = TestEmit.init(); defer emit.deinit(); - try lowerToOiEnc(.mov, .rax, 0x1000000000000000, emit.code()); + try lowerToOiEnc(.mov, Register.reg(.rax), 0x1000000000000000, emit.code()); try expectEqualHexStrings( "\x48\xB8\x00\x00\x00\x00\x00\x00\x00\x10", emit.lowered(), "movabs rax, 0x1000000000000000", ); - try lowerToOiEnc(.mov, .r11, 0x1000000000000000, emit.code()); + try lowerToOiEnc(.mov, Register.reg(.r11), 0x1000000000000000, emit.code()); try expectEqualHexStrings( "\x49\xBB\x00\x00\x00\x00\x00\x00\x00\x10", emit.lowered(), "movabs r11, 0x1000000000000000", ); - try lowerToOiEnc(.mov, .r11d, 0x10000000, emit.code()); + try lowerToOiEnc(.mov, Register.reg(.r11d), 0x10000000, emit.code()); try expectEqualHexStrings("\x41\xBB\x00\x00\x00\x10", emit.lowered(), "mov r11d, 0x10000000"); - try lowerToOiEnc(.mov, .r11w, 0x1000, emit.code()); + try lowerToOiEnc(.mov, Register.reg(.r11w), 0x1000, emit.code()); try expectEqualHexStrings("\x66\x41\xBB\x00\x10", emit.lowered(), "mov r11w, 0x1000"); - try lowerToOiEnc(.mov, .r11b, 0x10, emit.code()); + try lowerToOiEnc(.mov, Register.reg(.r11b), 0x10, emit.code()); try expectEqualHexStrings("\x41\xB3\x10", emit.lowered(), "mov r11b, 0x10"); } test "lower FD/TD encoding" { var emit = TestEmit.init(); defer emit.deinit(); - try lowerToFdEnc(.mov, .rax, 0x1000000000000000, emit.code()); + try lowerToFdEnc(.mov, Register.reg(.rax), 0x1000000000000000, emit.code()); try expectEqualHexStrings( "\x48\xa1\x00\x00\x00\x00\x00\x00\x00\x10", emit.lowered(), "mov rax, ds:0x1000000000000000", ); - try lowerToFdEnc(.mov, .eax, 0x10000000, emit.code()); + try lowerToFdEnc(.mov, Register.reg(.eax), 0x10000000, emit.code()); try expectEqualHexStrings("\xa1\x00\x00\x00\x10", emit.lowered(), "mov eax, ds:0x10000000"); - try lowerToFdEnc(.mov, .ax, 0x1000, emit.code()); + try lowerToFdEnc(.mov, Register.reg(.ax), 0x1000, emit.code()); try expectEqualHexStrings("\x66\xa1\x00\x10", emit.lowered(), "mov ax, ds:0x1000"); - try lowerToFdEnc(.mov, .al, 0x10, emit.code()); + try lowerToFdEnc(.mov, Register.reg(.al), 0x10, emit.code()); try expectEqualHexStrings("\xa0\x10", emit.lowered(), "mov al, ds:0x10"); } @@ -2403,16 +2396,16 @@ test "lower M1 and MC encodings" { test "lower O encoding" { var emit = TestEmit.init(); defer emit.deinit(); - try lowerToOEnc(.pop, .r12, emit.code()); + try lowerToOEnc(.pop, Register.reg(.r12), emit.code()); try expectEqualHexStrings("\x41\x5c", emit.lowered(), "pop r12"); - try lowerToOEnc(.push, .r12w, emit.code()); + try lowerToOEnc(.push, Register.reg(.r12w), emit.code()); try expectEqualHexStrings("\x66\x41\x54", emit.lowered(), "push r12w"); } test "lower RMI encoding" { var emit = TestEmit.init(); defer emit.deinit(); - try lowerToRmiEnc(.imul, .rax, RegisterOrMemory.mem(.qword_ptr, .{ + try lowerToRmiEnc(.imul, Register.reg(.rax), RegisterOrMemory.mem(.qword_ptr, .{ .disp = @bitCast(u32, @as(i32, -8)), .base = .rbp, }), 0x10, emit.code()); @@ -2421,18 +2414,18 @@ test "lower RMI encoding" { emit.lowered(), "imul rax, qword ptr [rbp - 8], 0x10", ); - try lowerToRmiEnc(.imul, .eax, RegisterOrMemory.mem(.dword_ptr, .{ + try lowerToRmiEnc(.imul, Register.reg(.eax), RegisterOrMemory.mem(.dword_ptr, .{ .disp = @bitCast(u32, @as(i32, -4)), .base = .rbp, }), 0x10, emit.code()); try expectEqualHexStrings("\x69\x45\xFC\x10\x00\x00\x00", emit.lowered(), "imul eax, dword ptr [rbp - 4], 0x10"); - try lowerToRmiEnc(.imul, .ax, RegisterOrMemory.mem(.word_ptr, .{ + try lowerToRmiEnc(.imul, Register.reg(.ax), RegisterOrMemory.mem(.word_ptr, .{ .disp = @bitCast(u32, @as(i32, -2)), .base = .rbp, }), 0x10, emit.code()); try expectEqualHexStrings("\x66\x69\x45\xFE\x10\x00", emit.lowered(), "imul ax, word ptr [rbp - 2], 0x10"); - try lowerToRmiEnc(.imul, .r12, RegisterOrMemory.reg(.r12), 0x10, emit.code()); + try lowerToRmiEnc(.imul, Register.reg(.r12), RegisterOrMemory.reg(.r12), 0x10, emit.code()); try expectEqualHexStrings("\x4D\x69\xE4\x10\x00\x00\x00", emit.lowered(), "imul r12, r12, 0x10"); - try lowerToRmiEnc(.imul, .r12w, RegisterOrMemory.reg(.r12w), 0x10, emit.code()); + try lowerToRmiEnc(.imul, Register.reg(.r12w), RegisterOrMemory.reg(.r12w), 0x10, emit.code()); try expectEqualHexStrings("\x66\x45\x69\xE4\x10\x00", emit.lowered(), "imul r12w, r12w, 0x10"); } diff --git a/src/arch/x86_64/bits.zig b/src/arch/x86_64/bits.zig index 43d5b92eb2..9a91311e09 100644 --- a/src/arch/x86_64/bits.zig +++ b/src/arch/x86_64/bits.zig @@ -302,96 +302,113 @@ pub const Encoder = struct { self.code.appendAssumeCapacity(0x66); } - pub fn Vex(comptime count: comptime_int) type { - if (count < 2 or count > 3) { - @compileError("VEX prefix can either be 2- or 3-byte long"); + pub const Vex = struct { + rex_prefix: Rex = .{}, + lead_opc: u5 = 0b0_0001, + register: u4 = 0b1111, + length: u1 = 0b0, + simd_prefix: u2 = 0b00, + wig: bool = false, + lig: bool = false, + lz: bool = false, + + pub fn rex(self: *Vex, r: Rex) void { + self.rex_prefix = r; } - return struct { - bytes: [count]u8 = switch (count) { - 2 => .{ 0xc5, 0xf8 }, - 3 => .{ 0xc4, 0xe1, 0xf8 }, - else => unreachable, - }, + pub fn lead_opc_0f(self: *Vex) void { + self.lead_opc = 0b0_0001; + } - pub fn rex(self: *@This(), prefix: Rex) void { - const byte = &self.bytes[1]; - if (prefix.w) switch (count) { - 3 => self.bytes[2] &= 0b0111_1111, - else => unreachable, - }; - if (prefix.r) byte.* &= 0b0111_1111; - if (prefix.x) switch (count) { - 3 => byte.* &= 0b1011_1111, - else => unreachable, - }; - if (prefix.b) switch (count) { - 3 => byte.* &= 0b1101_1111, - else => unreachable, - }; + pub fn lead_opc_0f_38(self: *Vex) void { + self.lead_opc = 0b0_0010; + } + + pub fn lead_opc_0f_3a(self: *Vex) void { + self.lead_opc = 0b0_0011; + } + + pub fn reg(self: *Vex, register: u4) void { + self.register = ~register; + } + + pub fn len_128(self: *Vex) void { + self.length = 0; + } + + pub fn len_256(self: *Vex) void { + assert(!self.lz); + self.length = 1; + } + + pub fn simd_prefix_66(self: *Vex) void { + self.simd_prefix = 0b01; + } + + pub fn simd_prefix_f3(self: *Vex) void { + self.simd_prefix = 0b10; + } + + pub fn simd_prefix_f2(self: *Vex) void { + self.simd_prefix = 0b11; + } + + pub fn wig(self: *Vex) void { + self.wig = true; + } + + pub fn lig(self: *Vex) void { + self.lig = true; + } + + pub fn lz(self: *Vex) void { + self.lz = true; + } + + pub fn write(self: Vex, writer: anytype) usize { + var buf: [3]u8 = .{0} ** 3; + const form_3byte: bool = blk: { + if (self.rex_prefix.w and !self.wig) break :blk true; + if (self.rex_prefix.x or self.rex_prefix.b) break :blk true; + break :blk self.lead_opc != 0b0_0001; + }; + + if (self.lz) { + assert(self.length == 0); } - pub fn leading_opcode_0f(self: *@This()) void { - switch (count) { - 3 => self.bytes[1] |= 0b0_0001, - else => {}, - } + if (form_3byte) { + // First byte + buf[0] = 0xc4; + // Second byte + const rxb_mask: u3 = @intCast(u3, @boolToInt(!self.rex_prefix.r)) << 2 | + @intCast(u2, @boolToInt(!self.rex_prefix.x)) << 1 | + @boolToInt(!self.rex_prefix.b); + buf[1] |= @intCast(u8, rxb_mask) << 5; + buf[1] |= self.lead_opc; + // Third byte + buf[2] |= @intCast(u8, @boolToInt(!self.rex_prefix.w)) << 7; + buf[2] |= @intCast(u7, self.register) << 3; + buf[2] |= @intCast(u3, self.length) << 2; + buf[2] |= self.simd_prefix; + } else { + // First byte + buf[0] = 0xc5; + // Second byte + buf[1] |= @intCast(u8, @boolToInt(!self.rex_prefix.r)) << 7; + buf[1] |= @intCast(u7, self.register) << 3; + buf[1] |= @intCast(u3, self.length) << 2; + buf[1] |= self.simd_prefix; } - pub fn leading_opcode_0f_38(self: *@This()) void { - switch (count) { - 3 => self.bytes[1] |= 0b0_0010, - else => unreachable, - } - } + const count: usize = if (form_3byte) 3 else 2; + _ = writer.writeAll(buf[0..count]) catch unreachable; + return count; + } + }; - pub fn leading_opcode_0f_3a(self: *@This()) void { - switch (count) { - 3 => self.bytes[1] |= 0b0_0011, - else => unreachable, - } - } - - pub fn reg(self: *@This(), register: u4) void { - const byte = &self.bytes[count - 1]; - const mask = 0b1_0000_111; - byte.* &= mask; - byte.* |= @intCast(u7, ~register) << 3; - } - - pub fn len_128(self: *@This()) void { - const byte = &self.bytes[count - 1]; - byte.* &= 0b0_11; - } - - pub fn len_256(self: *@This()) void { - const byte = &self.bytes[count - 1]; - byte.* |= 0b1_00; - } - - pub fn simd_prefix_66(self: *@This()) void { - const byte = &self.bytes[count - 1]; - byte.* |= 0b01; - } - - pub fn simd_prefix_f2(self: *@This()) void { - const byte = &self.bytes[count - 1]; - byte.* |= 0b11; - } - - pub fn simd_prefix_f3(self: *@This()) void { - const byte = &self.bytes[count - 1]; - byte.* |= 0b10; - } - }; - } - - pub fn vex_2byte(self: Self, prefix: Vex(2)) void { - self.code.appendSliceAssumeCapacity(&prefix.bytes); - } - - pub fn vex_3byte(self: Self, prefix: Vex(3)) void { - self.code.appendSliceAssumeCapacity(&prefix.bytes); + pub fn vex(self: Self, prefix: Vex) void { + _ = prefix.write(self.code.writer()); } /// From section 2.2.1.2 of the manual, REX is encoded as b0100WRXB @@ -759,39 +776,50 @@ test "Encoder helpers - general purpose registers" { } test "Encoder helpers - Vex prefix" { + var buf: [3]u8 = undefined; + var stream = std.io.fixedBufferStream(&buf); + const writer = stream.writer(); + { - var vex_prefix = Encoder.Vex(2){}; + var vex_prefix = Encoder.Vex{}; vex_prefix.rex(.{ .r = true, }); - try testing.expectEqualSlices(u8, &[_]u8{ 0xc5, 0x78 }, &vex_prefix.bytes); + const nwritten = vex_prefix.write(writer); + try testing.expectEqualSlices(u8, &[_]u8{ 0xc5, 0x78 }, buf[0..nwritten]); } { - var vex_prefix = Encoder.Vex(2){}; + stream.reset(); + var vex_prefix = Encoder.Vex{}; vex_prefix.reg(AvxRegister.xmm15.id()); - try testing.expectEqualSlices(u8, &[_]u8{ 0xc5, 0x80 }, &vex_prefix.bytes); + const nwritten = vex_prefix.write(writer); + try testing.expectEqualSlices(u8, &[_]u8{ 0xc5, 0x80 }, buf[0..nwritten]); } { - var vex_prefix = Encoder.Vex(3){}; + stream.reset(); + var vex_prefix = Encoder.Vex{}; vex_prefix.rex(.{ .w = true, .x = true, }); - try testing.expectEqualSlices(u8, &[_]u8{ 0xc4, 0b101_0_0001, 0b0_1111_0_00 }, &vex_prefix.bytes); + const nwritten = vex_prefix.write(writer); + try testing.expectEqualSlices(u8, &[_]u8{ 0xc4, 0b101_0_0001, 0b0_1111_0_00 }, buf[0..nwritten]); } { - var vex_prefix = Encoder.Vex(3){}; + stream.reset(); + var vex_prefix = Encoder.Vex{}; vex_prefix.rex(.{ .w = true, .r = true, }); vex_prefix.len_256(); - vex_prefix.leading_opcode_0f(); + vex_prefix.lead_opc_0f(); vex_prefix.simd_prefix_66(); - try testing.expectEqualSlices(u8, &[_]u8{ 0xc4, 0b011_0_0001, 0b0_1111_1_01 }, &vex_prefix.bytes); + const nwritten = vex_prefix.write(writer); + try testing.expectEqualSlices(u8, &[_]u8{ 0xc4, 0b011_0_0001, 0b0_1111_1_01 }, buf[0..nwritten]); } var code = ArrayList(u8).init(testing.allocator); @@ -800,9 +828,9 @@ test "Encoder helpers - Vex prefix" { { // vmovapd xmm1, xmm2 const encoder = try Encoder.init(&code, 4); - var vex = Encoder.Vex(2){}; + var vex = Encoder.Vex{}; vex.simd_prefix_66(); - encoder.vex_2byte(vex); // use 64 bit operation + encoder.vex(vex); // use 64 bit operation encoder.opcode_1byte(0x28); encoder.modRm_direct(0, AvxRegister.xmm1.lowId()); try testing.expectEqualSlices(u8, &[_]u8{ 0xC5, 0xF9, 0x28, 0xC1 }, code.items); @@ -813,13 +841,13 @@ test "Encoder helpers - Vex prefix" { // vmovhpd xmm13, xmm1, qword ptr [rip] const encoder = try Encoder.init(&code, 9); - var vex = Encoder.Vex(2){}; + var vex = Encoder.Vex{}; vex.len_128(); vex.simd_prefix_66(); - vex.leading_opcode_0f(); + vex.lead_opc_0f(); vex.rex(.{ .r = true }); vex.reg(AvxRegister.xmm1.id()); - encoder.vex_2byte(vex); + encoder.vex(vex); encoder.opcode_1byte(0x16); encoder.modRm_RIPDisp32(AvxRegister.xmm13.lowId()); encoder.disp32(0); From 0835486249f5f3d187db61c57d4d075d65e10177 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 12 May 2022 21:43:44 +0200 Subject: [PATCH 04/19] x64: add vmovsd RM and MR lowerings (first draft) --- src/arch/x86_64/Emit.zig | 175 ++++++++++++++++++++++++++++++++++----- src/arch/x86_64/bits.zig | 22 ++--- 2 files changed, 165 insertions(+), 32 deletions(-) diff --git a/src/arch/x86_64/Emit.zig b/src/arch/x86_64/Emit.zig index 848371f4ba..9015a805fb 100644 --- a/src/arch/x86_64/Emit.zig +++ b/src/arch/x86_64/Emit.zig @@ -26,7 +26,7 @@ const Mir = @import("Mir.zig"); const Module = @import("../../Module.zig"); const Instruction = bits.Instruction; const GpRegister = bits.Register; -const AvxRegister = bits.Register; +const AvxRegister = bits.AvxRegister; const Type = @import("../../type.zig").Type; mir: Mir, @@ -1180,6 +1180,7 @@ const Tag = enum { cmovng, cmovb, cmovnae, + vmovsd, fn isSetCC(tag: Tag) bool { return switch (tag) { @@ -1393,6 +1394,7 @@ inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode { .cmp => OpCode.oneByte(if (is_one_byte) 0x38 else 0x39), .mov => OpCode.oneByte(if (is_one_byte) 0x88 else 0x89), .@"test" => OpCode.oneByte(if (is_one_byte) 0x84 else 0x85), + .vmovsd => OpCode.oneByte(0x11), else => null, }, .rm => return switch (tag) { @@ -1413,6 +1415,7 @@ inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode { .cmove, .cmovz => OpCode.twoByte(0x0f, 0x44), .cmovb, .cmovnae => OpCode.twoByte(0x0f, 0x42), .cmovl, .cmovng => OpCode.twoByte(0x0f, 0x4c), + .vmovsd => OpCode.oneByte(0x10), else => null, }, .oi => return switch (tag) { @@ -1499,6 +1502,80 @@ inline fn getModRmExt(tag: Tag) ?u3 { }; } +const VexPrefix = struct { + prefix: Encoder.Vex, + reg: ?enum { + ndd, + nds, + dds, + }, +}; + +inline fn getVexPrefix(tag: Tag, enc: Encoding) ?VexPrefix { + const desc: struct { + reg: enum { + none, + ndd, + nds, + dds, + } = .none, + len_256: bool = false, + wig: bool = false, + lig: bool = false, + lz: bool = false, + lead_opc: enum { + l_0f, + l_0f_3a, + l_0f_38, + } = .l_0f, + simd_prefix: enum { + none, + p_66, + p_f2, + p_f3, + } = .none, + } = blk: { + switch (enc) { + .mr => switch (tag) { + .vmovsd => break :blk .{ .lig = true, .simd_prefix = .p_f2, .wig = true }, + else => return null, + }, + .rm => switch (tag) { + .vmovsd => break :blk .{ .lig = true, .simd_prefix = .p_f2, .wig = true }, + else => return null, + }, + else => unreachable, + } + }; + + var vex: Encoder.Vex = .{}; + + if (desc.len_256) vex.len_256(); + if (desc.wig) vex.wig(); + if (desc.lig) vex.lig(); + if (desc.lz) vex.lz(); + + switch (desc.lead_opc) { + .l_0f => {}, + .l_0f_3a => vex.lead_opc_0f_3a(), + .l_0f_38 => vex.lead_opc_0f_38(), + } + + switch (desc.simd_prefix) { + .none => {}, + .p_66 => vex.simd_prefix_66(), + .p_f2 => vex.simd_prefix_f2(), + .p_f3 => vex.simd_prefix_f3(), + } + + return VexPrefix{ .prefix = vex, .reg = switch (desc.reg) { + .none => null, + .nds => .nds, + .dds => .dds, + .ndd => .ndd, + } }; +} + const ScaleIndex = struct { scale: u2, index: GpRegister, @@ -1913,24 +1990,48 @@ fn lowerToRmEnc( encoder.modRm_direct(reg.lowId(), src_reg.lowId()); }, .memory => |src_mem| { - const encoder = try Encoder.init(code, 9); - if (reg.size() == 16) { - encoder.prefix16BitMode(); - } - if (src_mem.base) |base| { - // TODO handle 32-bit base register - requires prefix 0x67 - // Intel Manual, Vol 1, chapter 3.6 and 3.6.1 - encoder.rex(.{ - .w = setRexWRegister(reg), - .r = reg.isExtended(), - .b = base.isExtended(), - }); - } else { - encoder.rex(.{ - .w = setRexWRegister(reg), - .r = reg.isExtended(), - }); - } + const encoder: Encoder = blk: { + switch (reg) { + .register => { + const encoder = try Encoder.init(code, 9); + if (reg.size() == 16) { + encoder.prefix16BitMode(); + } + if (src_mem.base) |base| { + // TODO handle 32-bit base register - requires prefix 0x67 + // Intel Manual, Vol 1, chapter 3.6 and 3.6.1 + encoder.rex(.{ + .w = setRexWRegister(reg), + .r = reg.isExtended(), + .b = base.isExtended(), + }); + } else { + encoder.rex(.{ + .w = setRexWRegister(reg), + .r = reg.isExtended(), + }); + } + break :blk encoder; + }, + .avx_register => { + const encoder = try Encoder.init(code, 10); + var vex_prefix = getVexPrefix(tag, .rm).?; + const vex = &vex_prefix.prefix; + if (src_mem.base) |base| { + vex.rex(.{ + .r = reg.isExtended(), + .b = base.isExtended(), + }); + } else { + vex.rex(.{ + .r = reg.isExtended(), + }); + } + encoder.vex(vex_prefix.prefix); + break :blk encoder; + }, + } + }; opc.encode(encoder); src_mem.encode(encoder, reg.lowId()); }, @@ -1959,7 +2060,7 @@ fn lowerToMrEnc( encoder.modRm_direct(reg.lowId(), dst_reg.lowId()); }, .memory => |dst_mem| { - const encoder = blk: { + const encoder: Encoder = blk: { switch (reg) { .register => { const encoder = try Encoder.init(code, 9); @@ -1981,7 +2082,23 @@ fn lowerToMrEnc( break :blk encoder; }, .avx_register => { - unreachable; + const encoder = try Encoder.init(code, 10); + var vex_prefix = getVexPrefix(tag, .mr).?; + const vex = &vex_prefix.prefix; + if (dst_mem.base) |base| { + vex.rex(.{ + .w = dst_mem.ptr_size == .qword_ptr, + .r = reg.isExtended(), + .b = base.isExtended(), + }); + } else { + vex.rex(.{ + .w = dst_mem.ptr_size == .qword_ptr, + .r = reg.isExtended(), + }); + } + encoder.vex(vex_prefix.prefix); + break :blk encoder; }, } }; @@ -2247,6 +2364,14 @@ test "lower RM encoding" { }, }), emit.code()); try expectEqualHexStrings("\x48\x8D\x74\x0D\x00", emit.lowered(), "lea rsi, qword ptr [rbp + rcx*1 + 0]"); + + // AVX extension tests + try lowerToRmEnc(.vmovsd, Register.avxReg(.xmm1), RegisterOrMemory.rip(.qword_ptr, 0x10), emit.code()); + try expectEqualHexStrings( + "\xC5\xFB\x10\x0D\x10\x00\x00\x00", + emit.lowered(), + "vmovsd xmm1, qword ptr [rip + 0x10]", + ); } test "lower MR encoding" { @@ -2282,6 +2407,14 @@ test "lower MR encoding" { ); try lowerToMrEnc(.mov, RegisterOrMemory.rip(.qword_ptr, 0x10), Register.reg(.r12), emit.code()); try expectEqualHexStrings("\x4C\x89\x25\x10\x00\x00\x00", emit.lowered(), "mov qword ptr [rip + 0x10], r12"); + + // AVX extension tests + try lowerToMrEnc(.vmovsd, RegisterOrMemory.rip(.qword_ptr, 0x10), Register.avxReg(.xmm1), emit.code()); + try expectEqualHexStrings( + "\xC5\xFB\x11\x0D\x10\x00\x00\x00", + emit.lowered(), + "vmovsd qword ptr [rip + 0x10], xmm1", + ); } test "lower OI encoding" { diff --git a/src/arch/x86_64/bits.zig b/src/arch/x86_64/bits.zig index 9a91311e09..0bba295c20 100644 --- a/src/arch/x86_64/bits.zig +++ b/src/arch/x86_64/bits.zig @@ -143,7 +143,7 @@ pub const AvxRegister = enum(u6) { none, /// Returns the bit-width of the register. - pub fn size(self: AvxRegister) u4 { + pub fn size(self: AvxRegister) u9 { return switch (@enumToInt(self)) { 0...15 => 256, 16...31 => 128, @@ -152,7 +152,7 @@ pub const AvxRegister = enum(u6) { } /// Returns whether the register is *extended*. - pub fn isExtended(self: Register) bool { + pub fn isExtended(self: AvxRegister) bool { return @enumToInt(self) & 0x08 != 0; } @@ -308,9 +308,9 @@ pub const Encoder = struct { register: u4 = 0b1111, length: u1 = 0b0, simd_prefix: u2 = 0b00, - wig: bool = false, - lig: bool = false, - lz: bool = false, + wig_desc: bool = false, + lig_desc: bool = false, + lz_desc: bool = false, pub fn rex(self: *Vex, r: Rex) void { self.rex_prefix = r; @@ -337,7 +337,7 @@ pub const Encoder = struct { } pub fn len_256(self: *Vex) void { - assert(!self.lz); + assert(!self.lz_desc); self.length = 1; } @@ -354,26 +354,26 @@ pub const Encoder = struct { } pub fn wig(self: *Vex) void { - self.wig = true; + self.wig_desc = true; } pub fn lig(self: *Vex) void { - self.lig = true; + self.lig_desc = true; } pub fn lz(self: *Vex) void { - self.lz = true; + self.lz_desc = true; } pub fn write(self: Vex, writer: anytype) usize { var buf: [3]u8 = .{0} ** 3; const form_3byte: bool = blk: { - if (self.rex_prefix.w and !self.wig) break :blk true; + if (self.rex_prefix.w and !self.wig_desc) break :blk true; if (self.rex_prefix.x or self.rex_prefix.b) break :blk true; break :blk self.lead_opc != 0b0_0001; }; - if (self.lz) { + if (self.lz_desc) { assert(self.length == 0); } From 357561840d705bafbeb22b4919d6a79b208cd8fe Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Fri, 13 May 2022 21:28:53 +0200 Subject: [PATCH 05/19] x64: load/store to/from AVX registers for f64 --- src/arch/x86_64/CodeGen.zig | 410 ++++++++++++++++++++++++------------ src/arch/x86_64/Emit.zig | 86 +++++--- src/arch/x86_64/Mir.zig | 55 +++-- src/arch/x86_64/abi.zig | 6 + src/codegen.zig | 24 +-- src/register_manager.zig | 18 +- src/test.zig | 18 +- 7 files changed, 410 insertions(+), 207 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index f1455b0591..d0bed75fbd 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -22,8 +22,6 @@ const Liveness = @import("../../Liveness.zig"); const Mir = @import("Mir.zig"); const Module = @import("../../Module.zig"); const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager; -const RegisterManager = RegisterManagerFn(Self, Register, &allocatable_registers); -const RegisterLock = RegisterManager.RegisterLock; const Target = std.Target; const Type = @import("../../type.zig").Type; const TypedValue = @import("../../TypedValue.zig"); @@ -31,12 +29,19 @@ const Value = @import("../../value.zig").Value; const bits = @import("bits.zig"); const abi = @import("abi.zig"); -const Register = bits.Register; + const callee_preserved_regs = abi.callee_preserved_regs; const caller_preserved_regs = abi.caller_preserved_regs; const allocatable_registers = abi.allocatable_registers; const c_abi_int_param_regs = abi.c_abi_int_param_regs; const c_abi_int_return_regs = abi.c_abi_int_return_regs; +const RegisterManager = RegisterManagerFn(Self, Register, &allocatable_registers, spillInstruction); +const RegisterLock = RegisterManager.RegisterLock; +const Register = bits.Register; + +const AvxRegisterManager = RegisterManagerFn(Self, AvxRegister, &abi.avx_regs, spillInstructionAvx); +const AvxRegisterLock = AvxRegisterManager.RegisterLock; +const AvxRegister = bits.AvxRegister; const InnerError = error{ OutOfMemory, @@ -87,7 +92,8 @@ branch_stack: *std.ArrayList(Branch), // Key is the block instruction blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .{}, -register_manager: RegisterManager = .{}, +register_manager: RegisterManager, +avx_register_manager: AvxRegisterManager, /// Maps offset to what is stored there. stack: std.AutoHashMapUnmanaged(u32, StackAllocation) = .{}, @@ -119,14 +125,16 @@ pub const MCValue = union(enum) { /// A pointer-sized integer that fits in a register. /// If the type is a pointer, this is the pointer address in virtual address space. immediate: u64, - /// The value is in a target-specific register. + /// The value is in a GP register. register: Register, - /// The value is a tuple { wrapped, overflow } where wrapped value is stored in the register, + /// The value is a tuple { wrapped, overflow } where wrapped value is stored in the GP register, /// and the operation is an unsigned operation. register_overflow_unsigned: Register, - /// The value is a tuple { wrapped, overflow } where wrapped value is stored in the register, + /// The value is a tuple { wrapped, overflow } where wrapped value is stored in the GP register, /// and the operation is a signed operation. register_overflow_signed: Register, + /// The value is in an AVX register. + avx_register: AvxRegister, /// The value is in memory at a hard-coded address. /// If the type is a pointer, it means the pointer address is at this memory location. memory: u64, @@ -295,7 +303,11 @@ pub fn generate( .mir_to_air_map = if (builtin.mode == .Debug) std.AutoHashMap(Mir.Inst.Index, Air.Inst.Index).init(bin_file.allocator) else {}, + .register_manager = undefined, + .avx_register_manager = undefined, }; + function.register_manager = .{ .function = &function }; + function.avx_register_manager = .{ .function = &function }; defer function.stack.deinit(bin_file.allocator); defer function.blocks.deinit(bin_file.allocator); defer function.exitlude_jump_relocs.deinit(bin_file.allocator); @@ -387,14 +399,14 @@ fn gen(self: *Self) InnerError!void { if (cc != .Naked) { _ = try self.addInst(.{ .tag = .push, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = .rbp, }).encode(), .data = undefined, // unused for push reg, }); _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = .rbp, .reg2 = .rsp, }).encode(), @@ -434,7 +446,7 @@ fn gen(self: *Self) InnerError!void { // push the callee_preserved_regs that were used const backpatch_push_callee_preserved_regs_i = try self.addInst(.{ .tag = .push_regs_from_callee_preserved_regs, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = .rbp, }).encode(), .data = .{ .payload = undefined }, // to be backpatched @@ -476,7 +488,7 @@ fn gen(self: *Self) InnerError!void { // pop the callee_preserved_regs _ = try self.addInst(.{ .tag = .pop_regs_from_callee_preserved_regs, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = .rbp, }).encode(), .data = .{ .payload = callee_preserved_regs_payload }, @@ -497,7 +509,7 @@ fn gen(self: *Self) InnerError!void { _ = try self.addInst(.{ .tag = .pop, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = .rbp, }).encode(), .data = undefined, @@ -505,7 +517,7 @@ fn gen(self: *Self) InnerError!void { _ = try self.addInst(.{ .tag = .ret, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .flags = 0b11, }).encode(), .data = undefined, @@ -521,14 +533,14 @@ fn gen(self: *Self) InnerError!void { if (aligned_stack_end > 0) { self.mir_instructions.set(backpatch_stack_sub, .{ .tag = .sub, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = .rsp, }).encode(), .data = .{ .imm = aligned_stack_end }, }); self.mir_instructions.set(backpatch_stack_add, .{ .tag = .add, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = .rsp, }).encode(), .data = .{ .imm = aligned_stack_end }, @@ -889,13 +901,27 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue { self.stack_align = abi_align; if (reg_ok) { - // Make sure the type can fit in a register before we try to allocate one. - const ptr_bits = self.target.cpu.arch.ptrBitWidth(); - const ptr_bytes: u64 = @divExact(ptr_bits, 8); - if (abi_size <= ptr_bytes) { - if (self.register_manager.tryAllocReg(inst)) |reg| { - return MCValue{ .register = registerAlias(reg, abi_size) }; - } + switch (elem_ty.zigTypeTag()) { + .Vector => return self.fail("TODO allocRegOrMem for Vector type", .{}), + .Float => { + // TODO check if AVX available + const ptr_bytes: u64 = 32; + if (abi_size <= ptr_bytes) { + if (self.avx_register_manager.tryAllocReg(inst)) |reg| { + return MCValue{ .avx_register = avxRegisterAlias(reg, abi_size) }; + } + } + }, + else => { + // Make sure the type can fit in a register before we try to allocate one. + const ptr_bits = self.target.cpu.arch.ptrBitWidth(); + const ptr_bytes: u64 = @divExact(ptr_bits, 8); + if (abi_size <= ptr_bytes) { + if (self.register_manager.tryAllocReg(inst)) |reg| { + return MCValue{ .register = registerAlias(reg, abi_size) }; + } + } + }, } } const stack_offset = try self.allocMem(inst, abi_size, abi_align); @@ -920,6 +946,21 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void try self.genSetStack(self.air.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv, .{}); } +pub fn spillInstructionAvx(self: *Self, reg: AvxRegister, inst: Air.Inst.Index) !void { + const stack_mcv = try self.allocRegOrMem(inst, false); + log.debug("spilling {d} to stack mcv {any}", .{ inst, stack_mcv }); + const reg_mcv = self.getResolvedInstValue(inst); + switch (reg_mcv) { + .avx_register => |other| { + assert(reg.to256() == other.to256()); + }, + else => {}, + } + const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; + try branch.inst_table.put(self.gpa, inst, stack_mcv); + try self.genSetStack(self.air.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv, .{}); +} + pub fn spillCompareFlagsIfOccupied(self: *Self) !void { if (self.compare_flags_inst) |inst_to_save| { const mcv = self.getResolvedInstValue(inst_to_save); @@ -1192,7 +1233,7 @@ fn airMin(self: *Self, inst: Air.Inst.Index) !void { const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ty, rhs_mcv); _ = try self.addInst(.{ .tag = if (signedness == .signed) .cond_mov_lt else .cond_mov_below, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = dst_mcv.register, .reg2 = lhs_reg, }).encode(), @@ -1396,7 +1437,7 @@ fn genSetStackTruncatedOverflowCompare( }; _ = try self.addInst(.{ .tag = .cond_set_byte_overflow, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = overflow_reg.to8(), .flags = flags, }).encode(), @@ -1416,7 +1457,7 @@ fn genSetStackTruncatedOverflowCompare( const eq_reg = temp_regs[2]; _ = try self.addInst(.{ .tag = .cond_set_byte_eq_ne, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = eq_reg.to8(), .flags = 0b00, }).encode(), @@ -1565,7 +1606,7 @@ fn genIntMulDivOpMir( .signed => { _ = try self.addInst(.{ .tag = .cwd, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .flags = 0b11, }).encode(), .data = undefined, @@ -1574,7 +1615,7 @@ fn genIntMulDivOpMir( .unsigned => { _ = try self.addInst(.{ .tag = .xor, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = .rdx, .reg2 = .rdx, }).encode(), @@ -1596,7 +1637,7 @@ fn genIntMulDivOpMir( .register => |reg| { _ = try self.addInst(.{ .tag = tag, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = reg, }).encode(), .data = undefined, @@ -1605,7 +1646,7 @@ fn genIntMulDivOpMir( .stack_offset => |off| { _ = try self.addInst(.{ .tag = tag, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg2 = .rbp, .flags = switch (abi_size) { 1 => 0b00, @@ -1647,7 +1688,7 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa _ = try self.addInst(.{ .tag = .xor, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = divisor.to64(), .reg2 = dividend.to64(), }).encode(), @@ -1655,7 +1696,7 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa }); _ = try self.addInst(.{ .tag = .sar, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = divisor.to64(), .flags = 0b10, }).encode(), @@ -1663,7 +1704,7 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa }); _ = try self.addInst(.{ .tag = .@"test", - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = .rdx, .reg2 = .rdx, }).encode(), @@ -1671,7 +1712,7 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa }); _ = try self.addInst(.{ .tag = .cond_mov_eq, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = divisor.to64(), .reg2 = .rdx, }).encode(), @@ -2058,7 +2099,7 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue { // mov reg, [rbp - 8] _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = addr_reg.to64(), .reg2 = .rbp, .flags = 0b01, @@ -2143,7 +2184,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void { // lea reg, [rbp] _ = try self.addInst(.{ .tag = .lea, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = addr_reg.to64(), .reg2 = .rbp, }).encode(), @@ -2154,7 +2195,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void { // lea reg, [rbp] _ = try self.addInst(.{ .tag = .lea, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = addr_reg.to64(), .reg2 = .rbp, }).encode(), @@ -2222,7 +2263,7 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void { // mov dst_mcv, [dst_mcv] _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .flags = 0b01, .reg1 = registerAlias(dst_mcv.register, @intCast(u32, elem_abi_size)), .reg2 = dst_mcv.register, @@ -2456,6 +2497,7 @@ fn reuseOperand( fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!void { const elem_ty = ptr_ty.elemType(); const abi_size = elem_ty.abiSize(self.target.*); + std.log.warn("{} => {}, {}", .{ ptr_ty.fmtDebug(), ptr, dst_mcv }); switch (ptr) { .none => unreachable, .undef => unreachable, @@ -2488,7 +2530,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo // mov dst_reg, [reg] _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = registerAlias(dst_reg, @intCast(u32, abi_size)), .reg2 = reg, .flags = 0b01, @@ -2508,6 +2550,9 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo else => return self.fail("TODO implement loading from register into {}", .{dst_mcv}), } }, + .avx_register => { + return self.fail("TODO load for AVX register", .{}); + }, .memory, .got_load, .direct_load, @@ -2559,7 +2604,7 @@ fn loadMemPtrIntoRegister(self: *Self, reg: Register, ptr_ty: Type, ptr: MCValue const fn_owner_decl = mod.declPtr(self.mod_fn.owner_decl); _ = try self.addInst(.{ .tag = .lea_pie, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = registerAlias(reg, abi_size), .flags = flags, }).encode(), @@ -2582,6 +2627,7 @@ fn loadMemPtrIntoRegister(self: *Self, reg: Register, ptr_ty: Type, ptr: MCValue fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type) InnerError!void { const abi_size = value_ty.abiSize(self.target.*); + std.log.warn("{} => {}, {} => {}", .{ ptr_ty.fmtDebug(), ptr, value_ty.fmtDebug(), value }); switch (ptr) { .none => unreachable, .undef => unreachable, @@ -2623,7 +2669,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type }); _ = try self.addInst(.{ .tag = .mov_mem_imm, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = reg.to64(), .flags = switch (abi_size) { 1 => 0b00, @@ -2645,7 +2691,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type const tmp_reg = try self.copyToTmpRegister(value_ty, value); _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = reg.to64(), .reg2 = tmp_reg.to64(), .flags = 0b10, @@ -2661,7 +2707,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type .register => |src_reg| { _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = reg.to64(), .reg2 = registerAlias(src_reg, @intCast(u32, abi_size)), .flags = 0b10, @@ -2689,6 +2735,9 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type }, } }, + .avx_register => { + return self.fail("TODO store for AVX register", .{}); + }, .got_load, .direct_load, .memory, @@ -2709,7 +2758,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type // mov reg, [reg] _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = addr_reg.to64(), .reg2 = addr_reg.to64(), .flags = 0b01, @@ -2748,7 +2797,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type } _ = try self.addInst(.{ .tag = .mov_mem_imm, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = addr_reg.to64(), .flags = flags, }).encode(), @@ -2758,7 +2807,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type .register => |reg| { _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = addr_reg.to64(), .reg2 = reg, .flags = 0b10, @@ -2779,7 +2828,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = tmp_reg, .reg2 = tmp_reg, .flags = 0b01, @@ -2788,7 +2837,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type }); _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = addr_reg.to64(), .reg2 = tmp_reg, .flags = 0b10, @@ -2806,7 +2855,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type const tmp_reg = try self.copyToTmpRegister(value_ty, value); _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = addr_reg.to64(), .reg2 = tmp_reg, .flags = 0b10, @@ -2967,7 +3016,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { if (signedness == .signed and field_size < 8) { _ = try self.addInst(.{ .tag = .mov_sign_extend, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = dst_mcv.register, .reg2 = registerAlias(dst_mcv.register, field_size), }).encode(), @@ -2998,7 +3047,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { }; _ = try self.addInst(.{ .tag = .cond_set_byte_overflow, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = dst_reg.to8(), .flags = flags, }).encode(), @@ -3042,7 +3091,7 @@ fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shi 1 => { _ = try self.addInst(.{ .tag = tag, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = registerAlias(reg, abi_size), .flags = 0b00, }).encode(), @@ -3053,7 +3102,7 @@ fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shi else => { _ = try self.addInst(.{ .tag = tag, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = registerAlias(reg, abi_size), .flags = 0b10, }).encode(), @@ -3074,7 +3123,7 @@ fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shi _ = try self.addInst(.{ .tag = tag, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = registerAlias(reg, abi_size), .flags = 0b01, }).encode(), @@ -3453,17 +3502,20 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu .register => |src_reg| { _ = try self.addInst(.{ .tag = mir_tag, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = registerAlias(dst_reg, abi_size), .reg2 = registerAlias(src_reg, abi_size), }).encode(), .data = undefined, }); }, + .avx_register => { + return self.fail("TODO genBinOp for AVX register", .{}); + }, .immediate => |imm| { _ = try self.addInst(.{ .tag = mir_tag, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = registerAlias(dst_reg, abi_size), }).encode(), .data = .{ .imm = @truncate(u32, imm) }, @@ -3488,7 +3540,7 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu } _ = try self.addInst(.{ .tag = mir_tag, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = registerAlias(dst_reg, abi_size), .reg2 = .rbp, .flags = 0b01, @@ -3498,6 +3550,9 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu }, } }, + .avx_register => { + return self.fail("TODO genBinOp for AVX register", .{}); + }, .ptr_stack_offset, .stack_offset => |off| { if (off > math.maxInt(i32)) { return self.fail("stack offset too large", .{}); @@ -3515,7 +3570,7 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu .register => |src_reg| { _ = try self.addInst(.{ .tag = mir_tag, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = .rbp, .reg2 = registerAlias(src_reg, abi_size), .flags = 0b10, @@ -3523,6 +3578,9 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu .data = .{ .imm = @bitCast(u32, -off) }, }); }, + .avx_register => { + return self.fail("TODO genBinOp for AVX register", .{}); + }, .immediate => |imm| { const tag: Mir.Inst.Tag = switch (mir_tag) { .add => .add_mem_imm, @@ -3546,7 +3604,7 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu }); _ = try self.addInst(.{ .tag = tag, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = .rbp, .flags = flags, }).encode(), @@ -3592,6 +3650,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M .ptr_stack_offset => unreachable, .register_overflow_unsigned => unreachable, .register_overflow_signed => unreachable, + .avx_register => unreachable, .register => |dst_reg| { switch (src_mcv) { .none => unreachable, @@ -3600,11 +3659,12 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M .ptr_stack_offset => unreachable, .register_overflow_unsigned => unreachable, .register_overflow_signed => unreachable, + .avx_register => unreachable, .register => |src_reg| { // register, register _ = try self.addInst(.{ .tag = .imul_complex, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = registerAlias(dst_reg, abi_size), .reg2 = registerAlias(src_reg, abi_size), }).encode(), @@ -3617,7 +3677,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M if (math.minInt(i32) <= imm and imm <= math.maxInt(i32)) { _ = try self.addInst(.{ .tag = .imul_complex, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = dst_reg.to32(), .reg2 = dst_reg.to32(), .flags = 0b10, @@ -3633,7 +3693,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M .stack_offset => |off| { _ = try self.addInst(.{ .tag = .imul_complex, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = registerAlias(dst_reg, abi_size), .reg2 = .rbp, .flags = 0b01, @@ -3663,6 +3723,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M .ptr_stack_offset => unreachable, .register_overflow_unsigned => unreachable, .register_overflow_signed => unreachable, + .avx_register => unreachable, .register => |src_reg| { // copy dst to a register const dst_reg = try self.copyToTmpRegister(dst_ty, dst_mcv); @@ -3670,7 +3731,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M // register, register _ = try self.addInst(.{ .tag = .imul_complex, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = registerAlias(dst_reg, abi_size), .reg2 = registerAlias(src_reg, abi_size), }).encode(), @@ -3865,6 +3926,9 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions. .ptr_stack_offset => { return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{}); }, + .avx_register => { + return self.fail("TODO implement calling with MCValue.avx_register arg", .{}); + }, .undef => unreachable, .immediate => unreachable, .unreach => unreachable, @@ -3883,7 +3947,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions. // Adjust the stack _ = try self.addInst(.{ .tag = .sub, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = .rsp, }).encode(), .data = .{ .imm = info.stack_byte_count }, @@ -3909,7 +3973,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions. unreachable; _ = try self.addInst(.{ .tag = .call, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .flags = 0b01, }).encode(), .data = .{ .imm = @truncate(u32, got_addr) }, @@ -3925,7 +3989,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions. try self.genSetReg(Type.initTag(.usize), .rax, mcv); _ = try self.addInst(.{ .tag = .call, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = .rax, .flags = 0b01, }).encode(), @@ -3943,7 +4007,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions. // callq *%rax _ = try self.addInst(.{ .tag = .call, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = .rax, .flags = 0b01, }).encode(), @@ -3978,7 +4042,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions. try self.genSetReg(Type.initTag(.usize), .rax, mcv); _ = try self.addInst(.{ .tag = .call, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = .rax, .flags = 0b01, }).encode(), @@ -3996,7 +4060,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions. const fn_got_addr = got_addr + got_index * ptr_bytes; _ = try self.addInst(.{ .tag = .call, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .flags = 0b01, }).encode(), .data = .{ .imm = @intCast(u32, fn_got_addr) }, @@ -4008,7 +4072,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions. try self.genSetReg(Type.initTag(.usize), .rax, mcv); _ = try self.addInst(.{ .tag = .call, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = .rax, .flags = 0b01, }).encode(), @@ -4021,7 +4085,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions. // Readjust the stack _ = try self.addInst(.{ .tag = .add, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = .rsp, }).encode(), .data = .{ .imm = info.stack_byte_count }, @@ -4081,7 +4145,7 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void { // which is available if the jump is 127 bytes or less forward. const jmp_reloc = try self.addInst(.{ .tag = .jmp, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .flags = 0b00, }).encode(), .data = .{ .inst = undefined }, @@ -4116,7 +4180,7 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void { // which is available if the jump is 127 bytes or less forward. const jmp_reloc = try self.addInst(.{ .tag = .jmp, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .flags = 0b00, }).encode(), .data = .{ .inst = undefined }, @@ -4362,7 +4426,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 { Mir.Inst.Tag.cond_jmp_greater_less; return self.addInst(.{ .tag = tag, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .flags = flags, }).encode(), .data = .{ .inst = undefined }, @@ -4372,7 +4436,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 { try self.spillCompareFlagsIfOccupied(); _ = try self.addInst(.{ .tag = .@"test", - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = reg, .flags = 0b00, }).encode(), @@ -4380,7 +4444,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 { }); return self.addInst(.{ .tag = .cond_jmp_eq_ne, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .flags = 0b01, }).encode(), .data = .{ .inst = undefined }, @@ -4776,7 +4840,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void { try self.genBody(body); _ = try self.addInst(.{ .tag = .jmp, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .flags = 0b00, }).encode(), .data = .{ .inst = jmp_target }, @@ -4829,7 +4893,7 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u .immediate => |imm| { _ = try self.addInst(.{ .tag = .xor, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = registerAlias(cond_reg, abi_size), }).encode(), .data = .{ .imm = @intCast(u32, imm) }, @@ -4838,7 +4902,7 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u .register => |reg| { _ = try self.addInst(.{ .tag = .xor, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = registerAlias(cond_reg, abi_size), .reg2 = registerAlias(reg, abi_size), }).encode(), @@ -4860,7 +4924,7 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u _ = try self.addInst(.{ .tag = .@"test", - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = registerAlias(cond_reg, abi_size), .reg2 = registerAlias(cond_reg, abi_size), }).encode(), @@ -4868,7 +4932,7 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u }); return self.addInst(.{ .tag = .cond_jmp_eq_ne, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .flags = 0b00, }).encode(), .data = .{ .inst = undefined }, @@ -5020,6 +5084,12 @@ fn br(self: *Self, block: Air.Inst.Index, operand: Air.Inst.Ref) !void { try self.setRegOrMem(self.air.typeOfIndex(block), new_mcv, operand_mcv); break :blk new_mcv; }, + .avx_register => blk: { + // TODO not needed; return operand_mcv ones we can transfer between XMM registers + const new_mcv = try self.allocRegOrMem(block, false); + try self.setRegOrMem(self.air.typeOfIndex(block), new_mcv, operand_mcv); + break :blk new_mcv; + }, else => return self.fail("TODO implement block_data.mcv = operand_mcv for {}", .{operand_mcv}), }; } else { @@ -5036,7 +5106,7 @@ fn brVoid(self: *Self, block: Air.Inst.Index) !void { // Leave the jump offset undefined const jmp_reloc = try self.addInst(.{ .tag = .jmp, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .flags = 0b00, }).encode(), .data = .{ .inst = undefined }, @@ -5126,7 +5196,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { }; _ = try self.addInst(.{ .tag = .push, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .flags = 0b10, }).encode(), .data = .{ .imm = n }, @@ -5137,7 +5207,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { return self.fail("unrecognized register: '{s}'", .{reg_name}); _ = try self.addInst(.{ .tag = .push, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = reg, }).encode(), .data = undefined, @@ -5151,7 +5221,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { return self.fail("unrecognized register: '{s}'", .{reg_name}); _ = try self.addInst(.{ .tag = .pop, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = reg, }).encode(), .data = undefined, @@ -5217,6 +5287,7 @@ fn setRegOrMem(self: *Self, ty: Type, loc: MCValue, val: MCValue) !void { .none => return, .immediate => unreachable, .register => |reg| return self.genSetReg(ty, reg, val), + .avx_register => |reg| return self.genSetAvxReg(ty, reg, val), .stack_offset => |off| return self.genSetStack(ty, off, val, .{}), .memory => { return self.fail("TODO implement setRegOrMem for memory", .{}); @@ -5247,6 +5318,9 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE .register_overflow_unsigned, .register_overflow_signed, => return self.fail("TODO genSetStackArg for register with overflow bit", .{}), + .avx_register => { + return self.fail("TODO genSetStackArg for AVX register", .{}); + }, .compare_flags_unsigned, .compare_flags_signed, => { @@ -5265,7 +5339,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE }); _ = try self.addInst(.{ .tag = .mov_mem_imm, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = .rsp, .flags = switch (abi_size) { 1 => 0b00, @@ -5301,7 +5375,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE .register => |reg| { _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = .rsp, .reg2 = registerAlias(reg, @intCast(u32, abi_size)), .flags = 0b10, @@ -5368,7 +5442,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl }; _ = try self.addInst(.{ .tag = .cond_set_byte_overflow, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = tmp_reg.to8(), .flags = flags, }).encode(), @@ -5398,7 +5472,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl }); _ = try self.addInst(.{ .tag = .mov_mem_imm, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = base_reg, .flags = switch (abi_size) { 1 => 0b00, @@ -5420,7 +5494,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl }); _ = try self.addInst(.{ .tag = .mov_mem_imm, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = base_reg, .flags = 0b10, }).encode(), @@ -5434,7 +5508,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl }); _ = try self.addInst(.{ .tag = .mov_mem_imm, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = base_reg, .flags = 0b10, }).encode(), @@ -5466,7 +5540,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = base_reg, .reg2 = registerAlias(tmp_reg, nearest_power_of_two), .flags = 0b10, @@ -5484,7 +5558,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl } else { _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = base_reg, .reg2 = registerAlias(reg, @intCast(u32, abi_size)), .flags = 0b10, @@ -5493,6 +5567,27 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl }); } }, + .avx_register => |reg| { + const base_reg = opts.dest_stack_base orelse .rbp; + switch (ty.zigTypeTag()) { + .Float => switch (ty.tag()) { + .f32 => return self.fail("TODO genSetStack for AVX register for f32", .{}), + .f64 => { + _ = try self.addInst(.{ + .tag = .mov_f64, + .ops = (Mir.Ops(Register, AvxRegister){ + .reg1 = base_reg, + .reg2 = reg.to128(), + .flags = 0b01, + }).encode(), + .data = .{ .imm = @bitCast(u32, -stack_offset) }, + }); + }, + else => return self.fail("TODO genSetStack for AVX register for type {}", .{ty.fmtDebug()}), + }, + else => return self.fail("TODO genSetStack for AVX register for type {}", .{ty.fmtDebug()}), + } + }, .memory, .got_load, .direct_load, @@ -5569,7 +5664,7 @@ fn genInlineMemcpy( .ptr_stack_offset, .stack_offset => |off| { _ = try self.addInst(.{ .tag = .lea, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = dst_addr_reg.to64(), .reg2 = opts.dest_stack_base orelse .rbp, }).encode(), @@ -5579,7 +5674,7 @@ fn genInlineMemcpy( .register => |reg| { _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = registerAlias(dst_addr_reg, @divExact(reg.size(), 8)), .reg2 = reg, }).encode(), @@ -5604,7 +5699,7 @@ fn genInlineMemcpy( .ptr_stack_offset, .stack_offset => |off| { _ = try self.addInst(.{ .tag = .lea, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = src_addr_reg.to64(), .reg2 = opts.source_stack_base orelse .rbp, }).encode(), @@ -5614,7 +5709,7 @@ fn genInlineMemcpy( .register => |reg| { _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = registerAlias(src_addr_reg, @divExact(reg.size(), 8)), .reg2 = reg, }).encode(), @@ -5637,7 +5732,7 @@ fn genInlineMemcpy( // mov rcx, 0 _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = .rcx, }).encode(), .data = .{ .imm = 0 }, @@ -5646,7 +5741,7 @@ fn genInlineMemcpy( // mov rax, 0 _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = .rax, }).encode(), .data = .{ .imm = 0 }, @@ -5656,7 +5751,7 @@ fn genInlineMemcpy( // cmp count, 0 const loop_start = try self.addInst(.{ .tag = .cmp, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = count_reg, }).encode(), .data = .{ .imm = 0 }, @@ -5665,14 +5760,14 @@ fn genInlineMemcpy( // je end const loop_reloc = try self.addInst(.{ .tag = .cond_jmp_eq_ne, - .ops = (Mir.Ops{ .flags = 0b01 }).encode(), + .ops = (Mir.Ops(Register, Register){ .flags = 0b01 }).encode(), .data = .{ .inst = undefined }, }); // mov tmp, [addr + rcx] _ = try self.addInst(.{ .tag = .mov_scale_src, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = tmp_reg.to8(), .reg2 = src_addr_reg, }).encode(), @@ -5682,7 +5777,7 @@ fn genInlineMemcpy( // mov [stack_offset + rax], tmp _ = try self.addInst(.{ .tag = .mov_scale_dst, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = dst_addr_reg, .reg2 = tmp_reg.to8(), }).encode(), @@ -5692,7 +5787,7 @@ fn genInlineMemcpy( // add rcx, 1 _ = try self.addInst(.{ .tag = .add, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = .rcx, }).encode(), .data = .{ .imm = 1 }, @@ -5701,7 +5796,7 @@ fn genInlineMemcpy( // add rax, 1 _ = try self.addInst(.{ .tag = .add, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = .rax, }).encode(), .data = .{ .imm = 1 }, @@ -5710,7 +5805,7 @@ fn genInlineMemcpy( // sub count, 1 _ = try self.addInst(.{ .tag = .sub, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = count_reg, }).encode(), .data = .{ .imm = 1 }, @@ -5719,7 +5814,7 @@ fn genInlineMemcpy( // jmp loop _ = try self.addInst(.{ .tag = .jmp, - .ops = (Mir.Ops{ .flags = 0b00 }).encode(), + .ops = (Mir.Ops(Register, Register){ .flags = 0b00 }).encode(), .data = .{ .inst = loop_start }, }); @@ -5751,7 +5846,7 @@ fn genInlineMemset( .ptr_stack_offset, .stack_offset => |off| { _ = try self.addInst(.{ .tag = .lea, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = addr_reg.to64(), .reg2 = opts.dest_stack_base orelse .rbp, }).encode(), @@ -5761,7 +5856,7 @@ fn genInlineMemset( .register => |reg| { _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = registerAlias(addr_reg, @divExact(reg.size(), 8)), .reg2 = reg, }).encode(), @@ -5782,7 +5877,7 @@ fn genInlineMemset( // cmp rax, -1 const loop_start = try self.addInst(.{ .tag = .cmp, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = .rax, }).encode(), .data = .{ .imm = @bitCast(u32, @as(i32, -1)) }, @@ -5791,7 +5886,7 @@ fn genInlineMemset( // je end const loop_reloc = try self.addInst(.{ .tag = .cond_jmp_eq_ne, - .ops = (Mir.Ops{ .flags = 0b01 }).encode(), + .ops = (Mir.Ops(Register, Register){ .flags = 0b01 }).encode(), .data = .{ .inst = undefined }, }); @@ -5807,7 +5902,7 @@ fn genInlineMemset( }); _ = try self.addInst(.{ .tag = .mov_mem_index_imm, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = addr_reg, }).encode(), .data = .{ .payload = payload }, @@ -5819,7 +5914,7 @@ fn genInlineMemset( // sub rax, 1 _ = try self.addInst(.{ .tag = .sub, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = .rax, }).encode(), .data = .{ .imm = 1 }, @@ -5828,7 +5923,7 @@ fn genInlineMemset( // jmp loop _ = try self.addInst(.{ .tag = .jmp, - .ops = (Mir.Ops{ .flags = 0b00 }).encode(), + .ops = (Mir.Ops(Register, Register){ .flags = 0b00 }).encode(), .data = .{ .inst = loop_start }, }); @@ -5836,6 +5931,43 @@ fn genInlineMemset( try self.performReloc(loop_reloc); } +fn genSetAvxReg(self: *Self, ty: Type, reg: AvxRegister, mcv: MCValue) InnerError!void { + switch (mcv) { + .dead => unreachable, + .register_overflow_unsigned, + .register_overflow_signed, + => unreachable, + .stack_offset => |off| { + if (off < std.math.minInt(i32) or off > std.math.maxInt(i32)) { + return self.fail("stack offset too large", .{}); + } + + switch (ty.zigTypeTag()) { + .Float => { + switch (ty.tag()) { + .f32 => return self.fail("TODO genSetAvxReg from stack offset for f32", .{}), + .f64 => { + _ = try self.addInst(.{ + .tag = .mov_f64, + .ops = (Mir.Ops(AvxRegister, Register){ + .reg1 = reg.to128(), + .reg2 = .rbp, + }).encode(), + .data = .{ .imm = @bitCast(u32, -off) }, + }); + }, + else => return self.fail("TODO genSetAvxReg from stack offset for {}", .{ty.fmtDebug()}), + } + }, + else => return self.fail("TODO genSetAvxReg from stack offset for type {}", .{ty.fmtDebug()}), + } + }, + else => |other| { + return self.fail("TODO genSetAvxReg from {}", .{other}); + }, + } +} + fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void { const abi_size = @intCast(u32, ty.abiSize(self.target.*)); switch (mcv) { @@ -5843,13 +5975,14 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void .register_overflow_unsigned, .register_overflow_signed, => unreachable, + .avx_register => unreachable, .ptr_stack_offset => |off| { if (off < std.math.minInt(i32) or off > std.math.maxInt(i32)) { return self.fail("stack offset too large", .{}); } _ = try self.addInst(.{ .tag = .lea, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = registerAlias(reg, abi_size), .reg2 = .rbp, }).encode(), @@ -5889,7 +6022,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void }; _ = try self.addInst(.{ .tag = tag, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = reg.to8(), .flags = flags, }).encode(), @@ -5902,7 +6035,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void if (x == 0) { _ = try self.addInst(.{ .tag = .xor, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = reg.to32(), .reg2 = reg.to32(), }).encode(), @@ -5914,7 +6047,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void // Next best case: if we set the lower four bytes, the upper four will be zeroed. _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = registerAlias(reg, abi_size), }).encode(), .data = .{ .imm = @truncate(u32, x) }, @@ -5931,7 +6064,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void const payload = try self.addExtra(Mir.Imm64.encode(x)); _ = try self.addInst(.{ .tag = .movabs, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = reg.to64(), }).encode(), .data = .{ .payload = payload }, @@ -5948,7 +6081,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void if (abi_size > 4) break :blk; _ = try self.addInst(.{ .tag = .mov_sign_extend, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = reg.to64(), .reg2 = registerAlias(src_reg, abi_size), }).encode(), @@ -5959,7 +6092,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void if (abi_size > 2) break :blk; _ = try self.addInst(.{ .tag = .mov_zero_extend, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = reg.to64(), .reg2 = registerAlias(src_reg, abi_size), }).encode(), @@ -5972,7 +6105,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = registerAlias(reg, abi_size), .reg2 = registerAlias(src_reg, abi_size), }).encode(), @@ -5985,7 +6118,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void try self.loadMemPtrIntoRegister(reg, Type.usize, mcv); _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = registerAlias(reg, abi_size), .reg2 = reg.to64(), .flags = 0b01, @@ -5998,7 +6131,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void // mov reg, [ds:imm32] _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = registerAlias(reg, abi_size), .flags = 0b01, }).encode(), @@ -6012,7 +6145,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void const payload = try self.addExtra(Mir.Imm64.encode(x)); _ = try self.addInst(.{ .tag = .movabs, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = .rax, .flags = 0b01, // imm64 will become moffs64 }).encode(), @@ -6025,7 +6158,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void // mov reg, [reg + 0x0] _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = registerAlias(reg, abi_size), .reg2 = reg.to64(), .flags = 0b01, @@ -6051,7 +6184,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void }; _ = try self.addInst(.{ .tag = .mov_sign_extend, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = reg.to64(), .reg2 = .rbp, .flags = flags, @@ -6067,7 +6200,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void }; _ = try self.addInst(.{ .tag = .mov_zero_extend, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = reg.to64(), .reg2 = .rbp, .flags = flags, @@ -6081,7 +6214,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = registerAlias(reg, abi_size), .reg2 = .rbp, .flags = 0b01, @@ -6152,7 +6285,7 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void { }; _ = try self.addInst(.{ .tag = .fld, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .flags = switch (src_ty.abiSize(self.target.*)) { 4 => 0b01, 8 => 0b10, @@ -6167,7 +6300,7 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void { const stack_dst = try self.allocRegOrMem(inst, false); _ = try self.addInst(.{ .tag = .fisttp, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .flags = switch (dst_ty.abiSize(self.target.*)) { 1...2 => 0b00, 3...4 => 0b01, @@ -6271,7 +6404,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void { try self.loadMemPtrIntoRegister(reg, src_ty, src_ptr); _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops{ + .ops = (Mir.Ops(Register, Register){ .reg1 = reg, .reg2 = reg, .flags = 0b01, @@ -6840,7 +6973,20 @@ fn registerAlias(reg: Register, size_bytes: u32) Register { } else if (size_bytes <= 8) { return reg.to64(); } else { - unreachable; // TODO handle floating-point registers + unreachable; + } +} + +/// Returns AVX register wide enough to hold at least `size_bytes`. +fn avxRegisterAlias(reg: AvxRegister, size_bytes: u32) AvxRegister { + if (size_bytes == 0) { + unreachable; // should be comptime known + } else if (size_bytes <= 16) { + return reg.to128(); + } else if (size_bytes <= 32) { + return reg.to256(); + } else { + unreachable; } } diff --git a/src/arch/x86_64/Emit.zig b/src/arch/x86_64/Emit.zig index 9015a805fb..80034ab536 100644 --- a/src/arch/x86_64/Emit.zig +++ b/src/arch/x86_64/Emit.zig @@ -68,6 +68,7 @@ pub fn lowerMir(emit: *Emit) InnerError!void { const inst = @intCast(u32, index); try emit.code_offset_mapping.putNoClobber(emit.bin_file.allocator, inst, emit.code.items.len); switch (tag) { + // GPR instructions .adc => try emit.mirArith(.adc, inst), .add => try emit.mirArith(.add, inst), .sub => try emit.mirArith(.sub, inst), @@ -182,6 +183,10 @@ pub fn lowerMir(emit: *Emit) InnerError!void { .interrupt => try emit.mirInterrupt(inst), .nop => try emit.mirNop(), + // AVX instructions + .mov_f64 => try emit.mirMovF64(inst), + + // Pseudo-instructions .call_extern => try emit.mirCallExtern(inst), .dbg_line => try emit.mirDbgLine(inst), @@ -245,7 +250,7 @@ fn mirSyscall(emit: *Emit) InnerError!void { } fn mirPushPop(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { - const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); + const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); switch (ops.flags) { 0b00 => { // PUSH/POP reg @@ -274,7 +279,7 @@ fn mirPushPop(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { } fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { - const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); + const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); const payload = emit.mir.instructions.items(.data)[inst].payload; const data = emit.mir.extraData(Mir.RegsToPushOrPop, payload).data; const regs = data.regs; @@ -297,7 +302,7 @@ fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Tag, inst: Mir.Inst.I } fn mirJmpCall(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { - const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); + const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); switch (ops.flags) { 0b00 => { const target = emit.mir.instructions.items(.data)[inst].inst; @@ -336,7 +341,7 @@ fn mirJmpCall(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { } fn mirCondJmp(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void { - const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); + const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); const target = emit.mir.instructions.items(.data)[inst].inst; const tag = switch (mir_tag) { .cond_jmp_greater_less => switch (ops.flags) { @@ -368,7 +373,7 @@ fn mirCondJmp(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerErr } fn mirCondSetByte(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void { - const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); + const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); const tag = switch (mir_tag) { .cond_set_byte_greater_less => switch (ops.flags) { 0b00 => Tag.setge, @@ -398,7 +403,7 @@ fn mirCondSetByte(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) Inne } fn mirCondMov(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { - const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); + const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); if (ops.flags == 0b00) { return lowerToRmEnc(tag, Register.reg(ops.reg1), RegisterOrMemory.reg(ops.reg2), emit.code); } @@ -418,7 +423,7 @@ fn mirCondMov(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const tag = emit.mir.instructions.items(.tag)[inst]; assert(tag == .@"test"); - const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); + const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); switch (ops.flags) { 0b00 => { if (ops.reg2 == .none) { @@ -447,7 +452,7 @@ fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { fn mirRet(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const tag = emit.mir.instructions.items(.tag)[inst]; assert(tag == .ret); - const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); + const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); switch (ops.flags) { 0b00 => { // RETF imm16 @@ -471,7 +476,7 @@ fn mirRet(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { } fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { - const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); + const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); switch (ops.flags) { 0b00 => { if (ops.reg2 == .none) { @@ -513,7 +518,7 @@ fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { } fn mirArithMemImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { - const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); + const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); assert(ops.reg2 == .none); const payload = emit.mir.instructions.items(.data)[inst].payload; const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data; @@ -554,7 +559,7 @@ inline fn immOpSize(u_imm: u32) u8 { } fn mirArithScaleSrc(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { - const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); + const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); const scale = ops.flags; const imm = emit.mir.instructions.items(.data)[inst].imm; // OP reg1, [reg2 + scale*rcx + imm32] @@ -570,7 +575,7 @@ fn mirArithScaleSrc(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void } fn mirArithScaleDst(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { - const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); + const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); const scale = ops.flags; const imm = emit.mir.instructions.items(.data)[inst].imm; const scale_index = ScaleIndex{ @@ -594,7 +599,7 @@ fn mirArithScaleDst(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void } fn mirArithScaleImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { - const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); + const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); const scale = ops.flags; const payload = emit.mir.instructions.items(.data)[inst].payload; const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data; @@ -611,7 +616,7 @@ fn mirArithScaleImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void } fn mirArithMemIndexImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { - const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); + const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); assert(ops.reg2 == .none); const payload = emit.mir.instructions.items(.data)[inst].payload; const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data; @@ -636,7 +641,7 @@ fn mirArithMemIndexImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!v fn mirMovSignExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const mir_tag = emit.mir.instructions.items(.tag)[inst]; assert(mir_tag == .mov_sign_extend); - const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); + const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); const imm = if (ops.flags != 0b00) emit.mir.instructions.items(.data)[inst].imm else undefined; switch (ops.flags) { 0b00 => { @@ -667,7 +672,7 @@ fn mirMovSignExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { fn mirMovZeroExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const mir_tag = emit.mir.instructions.items(.tag)[inst]; assert(mir_tag == .mov_zero_extend); - const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); + const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); const imm = if (ops.flags != 0b00) emit.mir.instructions.items(.data)[inst].imm else undefined; switch (ops.flags) { 0b00 => { @@ -694,7 +699,7 @@ fn mirMovZeroExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { fn mirMovabs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const tag = emit.mir.instructions.items(.tag)[inst]; assert(tag == .movabs); - const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); + const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); const imm: u64 = if (ops.reg1.size() == 64) blk: { const payload = emit.mir.instructions.items(.data)[inst].payload; const imm = emit.mir.extraData(Mir.Imm64, payload).data; @@ -718,7 +723,7 @@ fn mirMovabs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { fn mirFisttp(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const tag = emit.mir.instructions.items(.tag)[inst]; assert(tag == .fisttp); - const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); + const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); // the selecting between operand sizes for this particular `fisttp` instruction // is done via opcode instead of the usual prefixes. @@ -740,7 +745,7 @@ fn mirFisttp(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { fn mirFld(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const tag = emit.mir.instructions.items(.tag)[inst]; assert(tag == .fld); - const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); + const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); // the selecting between operand sizes for this particular `fisttp` instruction // is done via opcode instead of the usual prefixes. @@ -757,8 +762,9 @@ fn mirFld(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { }; return lowerToMEnc(opcode, .{ .memory = mem_or_reg }, emit.code); } + fn mirShift(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { - const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); + const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); switch (ops.flags) { 0b00 => { // sal reg1, 1 @@ -783,7 +789,7 @@ fn mirShift(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { } fn mirMulDiv(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { - const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); + const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); if (ops.reg1 != .none) { assert(ops.reg2 == .none); return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code); @@ -806,7 +812,7 @@ fn mirMulDiv(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { fn mirIMulComplex(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const tag = emit.mir.instructions.items(.tag)[inst]; assert(tag == .imul_complex); - const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); + const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); switch (ops.flags) { 0b00 => { return lowerToRmEnc(.imul, Register.reg(ops.reg1), RegisterOrMemory.reg(ops.reg2), emit.code); @@ -835,7 +841,7 @@ fn mirIMulComplex(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { } fn mirCwd(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { - const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); + const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); const tag: Tag = switch (ops.flags) { 0b00 => .cbw, 0b01 => .cwd, @@ -848,7 +854,7 @@ fn mirCwd(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const tag = emit.mir.instructions.items(.tag)[inst]; assert(tag == .lea); - const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); + const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); switch (ops.flags) { 0b00 => { // lea reg1, [reg2 + imm32] @@ -908,7 +914,7 @@ fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { fn mirLeaPie(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const tag = emit.mir.instructions.items(.tag)[inst]; assert(tag == .lea_pie); - const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); + const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); const load_reloc = emit.mir.instructions.items(.data)[inst].load_reloc; // lea reg1, [rip + reloc] @@ -947,6 +953,36 @@ fn mirLeaPie(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { } } +// AVX instructions + +fn mirMovF64(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { + const tag = emit.mir.instructions.items(.tag)[inst]; + assert(tag == .mov_f64); + const ops = emit.mir.instructions.items(.ops)[inst]; + const flags = @truncate(u2, ops); + const imm = emit.mir.instructions.items(.data)[inst].imm; + + switch (flags) { + 0b00 => { + const decoded = Mir.Ops(AvxRegister, GpRegister).decode(ops); + return lowerToRmEnc(.vmovsd, Register.avxReg(decoded.reg1), RegisterOrMemory.mem(.qword_ptr, .{ + .disp = imm, + .base = decoded.reg2, + }), emit.code); + }, + 0b01 => { + const decoded = Mir.Ops(GpRegister, AvxRegister).decode(ops); + return lowerToMrEnc(.vmovsd, RegisterOrMemory.mem(.qword_ptr, .{ + .disp = imm, + .base = decoded.reg1, + }), Register.avxReg(decoded.reg2), emit.code); + }, + else => return emit.fail("TODO unused variant 0b{b} for mov_f64", .{flags}), + } +} + +// Pseudo-instructions + fn mirCallExtern(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const tag = emit.mir.instructions.items(.tag)[inst]; assert(tag == .call_extern); diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig index 30f4351cb0..ef9679496d 100644 --- a/src/arch/x86_64/Mir.zig +++ b/src/arch/x86_64/Mir.zig @@ -14,7 +14,8 @@ const assert = std.debug.assert; const bits = @import("bits.zig"); const Air = @import("../../Air.zig"); const CodeGen = @import("CodeGen.zig"); -const Register = bits.Register; +const GpRegister = bits.Register; +const AvxRegister = bits.AvxRegister; instructions: std.MultiArrayList(Inst).Slice, /// The meaning of this data is determined by `Inst.Tag` value. @@ -349,6 +350,12 @@ pub const Inst = struct { /// Nop nop, + /// AVX instructions + /// ops flags: form: + /// 0b00 reg1, qword ptr [reg2 + imm32] + /// 0b10 qword ptr [reg1 + imm32], reg2 + mov_f64, + /// Pseudo-instructions /// call extern function /// Notes: @@ -450,30 +457,32 @@ pub const DbgLineColumn = struct { column: u32, }; -pub const Ops = struct { - reg1: Register = .none, - reg2: Register = .none, - flags: u2 = 0b00, +pub fn Ops(comptime Reg1: type, comptime Reg2: type) type { + return struct { + reg1: Reg1 = .none, + reg2: Reg2 = .none, + flags: u2 = 0b00, - pub fn encode(self: Ops) u16 { - var ops: u16 = 0; - ops |= @intCast(u16, @enumToInt(self.reg1)) << 9; - ops |= @intCast(u16, @enumToInt(self.reg2)) << 2; - ops |= self.flags; - return ops; - } + pub fn encode(self: @This()) u16 { + var ops: u16 = 0; + ops |= @intCast(u16, @enumToInt(self.reg1)) << 9; + ops |= @intCast(u16, @enumToInt(self.reg2)) << 2; + ops |= self.flags; + return ops; + } - pub fn decode(ops: u16) Ops { - const reg1 = @intToEnum(Register, @truncate(u7, ops >> 9)); - const reg2 = @intToEnum(Register, @truncate(u7, ops >> 2)); - const flags = @truncate(u2, ops); - return .{ - .reg1 = reg1, - .reg2 = reg2, - .flags = flags, - }; - } -}; + pub fn decode(ops: u16) @This() { + const reg1 = @intToEnum(Reg1, @truncate(u7, ops >> 9)); + const reg2 = @intToEnum(Reg2, @truncate(u7, ops >> 2)); + const flags = @truncate(u2, ops); + return .{ + .reg1 = reg1, + .reg2 = reg2, + .flags = flags, + }; + } + }; +} pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void { mir.instructions.deinit(gpa); diff --git a/src/arch/x86_64/abi.zig b/src/arch/x86_64/abi.zig index da2e3da394..a53c82530d 100644 --- a/src/arch/x86_64/abi.zig +++ b/src/arch/x86_64/abi.zig @@ -3,6 +3,7 @@ const Type = @import("../../type.zig").Type; const Target = std.Target; const assert = std.debug.assert; const Register = @import("bits.zig").Register; +const AvxRegister = @import("bits.zig").AvxRegister; pub const Class = enum { integer, sse, sseup, x87, x87up, complex_x87, memory, none }; @@ -381,3 +382,8 @@ pub const caller_preserved_regs = [_]Register{ .rax, .rcx, .rdx, .rsi, .rdi, .r8 pub const allocatable_registers = callee_preserved_regs ++ caller_preserved_regs; pub const c_abi_int_param_regs = [_]Register{ .rdi, .rsi, .rdx, .rcx, .r8, .r9 }; pub const c_abi_int_return_regs = [_]Register{ .rax, .rdx }; + +pub const avx_regs = [_]AvxRegister{ + .ymm0, .ymm1, .ymm2, .ymm3, .ymm4, .ymm5, .ymm6, .ymm7, + .ymm8, .ymm9, .ymm10, .ymm11, .ymm12, .ymm13, .ymm14, .ymm15, +}; diff --git a/src/codegen.zig b/src/codegen.zig index def69d952f..c14fb39629 100644 --- a/src/codegen.zig +++ b/src/codegen.zig @@ -78,13 +78,13 @@ pub fn generateFunction( debug_output: DebugInfoOutput, ) GenerateSymbolError!FnResult { switch (bin_file.options.target.cpu.arch) { - .arm, - .armeb, - => return @import("arch/arm/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output), - .aarch64, - .aarch64_be, - .aarch64_32, - => return @import("arch/aarch64/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output), + // .arm, + // .armeb, + // => return @import("arch/arm/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output), + // .aarch64, + // .aarch64_be, + // .aarch64_32, + // => return @import("arch/aarch64/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output), //.arc => return Function(.arc).generate(bin_file, src_loc, func, air, liveness, code, debug_output), //.avr => return Function(.avr).generate(bin_file, src_loc, func, air, liveness, code, debug_output), //.bpfel => return Function(.bpfel).generate(bin_file, src_loc, func, air, liveness, code, debug_output), @@ -101,9 +101,9 @@ pub fn generateFunction( //.r600 => return Function(.r600).generate(bin_file, src_loc, func, air, liveness, code, debug_output), //.amdgcn => return Function(.amdgcn).generate(bin_file, src_loc, func, air, liveness, code, debug_output), //.riscv32 => return Function(.riscv32).generate(bin_file, src_loc, func, air, liveness, code, debug_output), - .riscv64 => return @import("arch/riscv64/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output), + // .riscv64 => return @import("arch/riscv64/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output), //.sparc => return Function(.sparc).generate(bin_file, src_loc, func, air, liveness, code, debug_output), - .sparc64 => return @import("arch/sparc64/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output), + // .sparc64 => return @import("arch/sparc64/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output), //.sparcel => return Function(.sparcel).generate(bin_file, src_loc, func, air, liveness, code, debug_output), //.s390x => return Function(.s390x).generate(bin_file, src_loc, func, air, liveness, code, debug_output), //.tce => return Function(.tce).generate(bin_file, src_loc, func, air, liveness, code, debug_output), @@ -129,9 +129,9 @@ pub fn generateFunction( //.renderscript32 => return Function(.renderscript32).generate(bin_file, src_loc, func, air, liveness, code, debug_output), //.renderscript64 => return Function(.renderscript64).generate(bin_file, src_loc, func, air, liveness, code, debug_output), //.ve => return Function(.ve).generate(bin_file, src_loc, func, air, liveness, code, debug_output), - .wasm32, - .wasm64, - => return @import("arch/wasm/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output), + // .wasm32, + // .wasm64, + // => return @import("arch/wasm/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output), else => @panic("Backend architectures that don't have good support yet are commented out, to improve compilation performance. If you are interested in one of these other backends feel free to uncomment them. Eventually these will be completed, but stage1 is slow and a memory hog."), } } diff --git a/src/register_manager.zig b/src/register_manager.zig index 2c0502e867..bcec8e14e6 100644 --- a/src/register_manager.zig +++ b/src/register_manager.zig @@ -23,10 +23,15 @@ pub const AllocateRegistersError = error{ CodegenFail, }; +pub fn SpillFn(comptime Function: type, comptime Register: type) type { + return fn (*Function, Register, Air.Inst.Index) anyerror!void; +} + pub fn RegisterManager( comptime Function: type, comptime Register: type, comptime tracked_registers: []const Register, + comptime spill_fn: SpillFn(Function, Register), ) type { // architectures which do not have a concept of registers should // refrain from using RegisterManager @@ -47,6 +52,7 @@ pub fn RegisterManager( allocated_registers: FreeRegInt = 0, /// Tracks registers which are locked from being allocated locked_registers: FreeRegInt = 0, + function: *Function, const Self = @This(); @@ -55,8 +61,8 @@ pub fn RegisterManager( const FreeRegInt = std.meta.Int(.unsigned, tracked_registers.len); const ShiftInt = math.Log2Int(FreeRegInt); - fn getFunction(self: *Self) *Function { - return @fieldParentPtr(Function, "register_manager", self); + fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) AllocateRegistersError!void { + return try spill_fn(self.function, reg, inst); } fn getRegisterMask(reg: Register) ?FreeRegInt { @@ -251,14 +257,14 @@ pub fn RegisterManager( self.markRegUsed(reg); } else { const spilled_inst = self.registers[index]; - try self.getFunction().spillInstruction(reg, spilled_inst); + try self.spillInstruction(reg, spilled_inst); } self.registers[index] = inst; } else { // Don't track the register if (!self.isRegFree(reg)) { const spilled_inst = self.registers[index]; - try self.getFunction().spillInstruction(reg, spilled_inst); + try self.spillInstruction(reg, spilled_inst); self.freeReg(reg); } } @@ -293,7 +299,7 @@ pub fn RegisterManager( // stack allocation. const spilled_inst = self.registers[index]; self.registers[index] = tracked_inst; - try self.getFunction().spillInstruction(reg, spilled_inst); + try self.spillInstruction(reg, spilled_inst); } else { self.getRegAssumeFree(reg, tracked_inst); } @@ -302,7 +308,7 @@ pub fn RegisterManager( // Move the instruction that was previously there to a // stack allocation. const spilled_inst = self.registers[index]; - try self.getFunction().spillInstruction(reg, spilled_inst); + try self.spillInstruction(reg, spilled_inst); self.freeReg(reg); } } diff --git a/src/test.zig b/src/test.zig index a6537e77c7..b272d05718 100644 --- a/src/test.zig +++ b/src/test.zig @@ -34,18 +34,18 @@ test { var ctx = TestContext.init(std.testing.allocator, arena); defer ctx.deinit(); - { - const dir_path = try std.fs.path.join(arena, &.{ - std.fs.path.dirname(@src().file).?, "..", "test", "cases", - }); + // { + // const dir_path = try std.fs.path.join(arena, &.{ + // std.fs.path.dirname(@src().file).?, "..", "test", "cases", + // }); - var dir = try std.fs.cwd().openDir(dir_path, .{ .iterate = true }); - defer dir.close(); + // var dir = try std.fs.cwd().openDir(dir_path, .{ .iterate = true }); + // defer dir.close(); - ctx.addTestCasesFromDir(dir); - } + // ctx.addTestCasesFromDir(dir); + // } - try @import("test_cases").addCases(&ctx); + // try @import("test_cases").addCases(&ctx); try ctx.run(); } From 2aee2302515ba444999b82c2e40cbc35dee08baf Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Sat, 14 May 2022 01:50:07 +0200 Subject: [PATCH 06/19] x64: add unordered cmp with EFLAGS --- src/arch/x86_64/CodeGen.zig | 133 ++++++++++++- src/arch/x86_64/Emit.zig | 217 +++++++++++++++++++- src/arch/x86_64/Mir.zig | 11 +- src/register_manager.zig | 386 ++++++++++++++++++------------------ 4 files changed, 538 insertions(+), 209 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index d0bed75fbd..e81f2d5435 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -202,6 +202,7 @@ pub const MCValue = union(enum) { fn isRegister(mcv: MCValue) bool { return switch (mcv) { .register => true, + .avx_register => true, else => false, }; } @@ -971,6 +972,7 @@ pub fn spillCompareFlagsIfOccupied(self: *Self) !void { .compare_flags_signed, .compare_flags_unsigned, => try self.allocRegOrMem(inst_to_save, true), + .avx_register => try self.allocRegOrMem(inst_to_save, false), else => unreachable, }; @@ -988,6 +990,7 @@ pub fn spillCompareFlagsIfOccupied(self: *Self) !void { .register_overflow_signed, .register_overflow_unsigned, => |reg| self.register_manager.freeReg(reg), + .avx_register => |reg| self.avx_register_manager.freeReg(reg), else => {}, } } @@ -2497,7 +2500,6 @@ fn reuseOperand( fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!void { const elem_ty = ptr_ty.elemType(); const abi_size = elem_ty.abiSize(self.target.*); - std.log.warn("{} => {}, {}", .{ ptr_ty.fmtDebug(), ptr, dst_mcv }); switch (ptr) { .none => unreachable, .undef => unreachable, @@ -2627,7 +2629,6 @@ fn loadMemPtrIntoRegister(self: *Self, reg: Register, ptr_ty: Type, ptr: MCValue fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type) InnerError!void { const abi_size = value_ty.abiSize(self.target.*); - std.log.warn("{} => {}, {} => {}", .{ ptr_ty.fmtDebug(), ptr, value_ty.fmtDebug(), value }); switch (ptr) { .none => unreachable, .undef => unreachable, @@ -3375,13 +3376,39 @@ fn genBinOp( const rhs = try self.resolveInst(rhs_air); const lhs_ty = self.air.typeOf(lhs_air); const rhs_ty = self.air.typeOf(rhs_air); - if (lhs_ty.zigTypeTag() == .Vector or lhs_ty.zigTypeTag() == .Float) { + if (lhs_ty.zigTypeTag() == .Vector) { return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmtDebug()}); } if (lhs_ty.abiSize(self.target.*) > 8) { return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmtDebug()}); } + if (lhs_ty.zigTypeTag() == .Float) { + switch (tag) { + .add => { + const dst_reg: AvxRegister = blk: { + const reg = try self.avx_register_manager.allocReg(null); + try self.genSetAvxReg(lhs_ty, reg, lhs); + break :blk reg.to128(); + }; + const dst_lock = self.avx_register_manager.lockRegAssumeUnused(dst_reg); + defer self.avx_register_manager.unlockReg(dst_lock); + + const src_reg: AvxRegister = blk: { + const reg = try self.avx_register_manager.allocReg(null); + try self.genSetAvxReg(lhs_ty, reg, rhs); + break :blk reg.to128(); + }; + const src_lock = self.avx_register_manager.lockRegAssumeUnused(src_reg); + defer self.avx_register_manager.unlockReg(src_lock); + + try self.genBinOpMir(.add_f64, lhs_ty, .{ .avx_register = dst_reg }, .{ .avx_register = src_reg }); + return MCValue{ .avx_register = dst_reg }; + }, + else => unreachable, + } + } + const is_commutative: bool = switch (tag) { .add, .addwrap, @@ -3550,8 +3577,28 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu }, } }, - .avx_register => { - return self.fail("TODO genBinOp for AVX register", .{}); + .avx_register => |dst_reg| { + switch (src_mcv) { + .avx_register => |src_reg| { + switch (dst_ty.zigTypeTag()) { + .Float => switch (dst_ty.tag()) { + .f64 => { + _ = try self.addInst(.{ + .tag = mir_tag, + .ops = (Mir.Ops(AvxRegister, AvxRegister){ + .reg1 = dst_reg.to128(), + .reg2 = src_reg.to128(), + }).encode(), + .data = undefined, + }); + }, + else => return self.fail("TODO genBinOp for AVX register and type {}", .{dst_ty.fmtDebug()}), + }, + else => return self.fail("TODO genBinOp for AVX register and type {}", .{dst_ty.fmtDebug()}), + } + }, + else => return self.fail("TODO genBinOp for AVX register", .{}), + } }, .ptr_stack_offset, .stack_offset => |off| { if (off > math.maxInt(i32)) { @@ -4209,6 +4256,37 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void { self.compare_flags_inst = inst; const result: MCValue = result: { + if (ty.zigTypeTag() == .Float) { + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + + const dst_reg: AvxRegister = blk: { + const reg = try self.avx_register_manager.allocReg(null); + try self.genSetAvxReg(ty, reg, lhs); + break :blk reg.to128(); + }; + const dst_lock = self.avx_register_manager.lockRegAssumeUnused(dst_reg); + defer self.avx_register_manager.unlockReg(dst_lock); + + const src_reg: AvxRegister = blk: { + const reg = try self.avx_register_manager.allocReg(null); + try self.genSetAvxReg(ty, reg, rhs); + break :blk reg.to128(); + }; + const src_lock = self.avx_register_manager.lockRegAssumeUnused(src_reg); + defer self.avx_register_manager.unlockReg(src_lock); + + _ = try self.addInst(.{ + .tag = .cmp_f64, + .ops = (Mir.Ops(AvxRegister, AvxRegister){ + .reg1 = dst_reg, + .reg2 = src_reg, + }).encode(), + .data = undefined, + }); + + break :result MCValue{ .compare_flags_unsigned = op }; + } // There are 2 operands, destination and source. // Either one, but not both, can be a memory operand. // Source operand can be an immediate, 8 bits or 32 bits. @@ -5962,6 +6040,51 @@ fn genSetAvxReg(self: *Self, ty: Type, reg: AvxRegister, mcv: MCValue) InnerErro else => return self.fail("TODO genSetAvxReg from stack offset for type {}", .{ty.fmtDebug()}), } }, + .avx_register => |src_reg| { + switch (ty.zigTypeTag()) { + .Float => { + switch (ty.tag()) { + .f32 => return self.fail("TODO genSetAvxReg from register for f32", .{}), + .f64 => { + _ = try self.addInst(.{ + .tag = .mov_f64, + .ops = (Mir.Ops(AvxRegister, AvxRegister){ + .reg1 = reg.to128(), + .reg2 = src_reg.to128(), + .flags = 0b10, + }).encode(), + .data = undefined, + }); + }, + else => return self.fail("TODO genSetAvxReg from register for {}", .{ty.fmtDebug()}), + } + }, + else => return self.fail("TODO genSetAvxReg from register for type {}", .{ty.fmtDebug()}), + } + }, + .memory => { + switch (ty.zigTypeTag()) { + .Float => { + switch (ty.tag()) { + .f32 => return self.fail("TODO genSetAvxReg from memory for f32", .{}), + .f64 => { + const base_reg = try self.register_manager.allocReg(null); + try self.loadMemPtrIntoRegister(base_reg, Type.usize, mcv); + _ = try self.addInst(.{ + .tag = .mov_f64, + .ops = (Mir.Ops(AvxRegister, Register){ + .reg1 = reg.to128(), + .reg2 = base_reg.to64(), + }).encode(), + .data = .{ .imm = 0 }, + }); + }, + else => return self.fail("TODO genSetAvxReg from memory for {}", .{ty.fmtDebug()}), + } + }, + else => return self.fail("TODO genSetAvxReg from memory for type {}", .{ty.fmtDebug()}), + } + }, else => |other| { return self.fail("TODO genSetAvxReg from {}", .{other}); }, diff --git a/src/arch/x86_64/Emit.zig b/src/arch/x86_64/Emit.zig index 80034ab536..79341df8cd 100644 --- a/src/arch/x86_64/Emit.zig +++ b/src/arch/x86_64/Emit.zig @@ -186,6 +186,10 @@ pub fn lowerMir(emit: *Emit) InnerError!void { // AVX instructions .mov_f64 => try emit.mirMovF64(inst), + .add_f64 => try emit.mirAddF64(inst), + + .cmp_f64 => try emit.mirCmpF64(inst), + // Pseudo-instructions .call_extern => try emit.mirCallExtern(inst), @@ -960,11 +964,11 @@ fn mirMovF64(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { assert(tag == .mov_f64); const ops = emit.mir.instructions.items(.ops)[inst]; const flags = @truncate(u2, ops); - const imm = emit.mir.instructions.items(.data)[inst].imm; switch (flags) { 0b00 => { const decoded = Mir.Ops(AvxRegister, GpRegister).decode(ops); + const imm = emit.mir.instructions.items(.data)[inst].imm; return lowerToRmEnc(.vmovsd, Register.avxReg(decoded.reg1), RegisterOrMemory.mem(.qword_ptr, .{ .disp = imm, .base = decoded.reg2, @@ -972,11 +976,63 @@ fn mirMovF64(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { }, 0b01 => { const decoded = Mir.Ops(GpRegister, AvxRegister).decode(ops); + const imm = emit.mir.instructions.items(.data)[inst].imm; return lowerToMrEnc(.vmovsd, RegisterOrMemory.mem(.qword_ptr, .{ .disp = imm, .base = decoded.reg1, }), Register.avxReg(decoded.reg2), emit.code); }, + 0b10 => { + const decoded = Mir.Ops(AvxRegister, AvxRegister).decode(ops); + return lowerToRvmEnc( + .vmovsd, + Register.avxReg(decoded.reg1), + Register.avxReg(decoded.reg1), + RegisterOrMemory.avxReg(decoded.reg2), + emit.code, + ); + }, + else => return emit.fail("TODO unused variant 0b{b} for mov_f64", .{flags}), + } +} + +fn mirAddF64(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { + const tag = emit.mir.instructions.items(.tag)[inst]; + assert(tag == .add_f64); + const ops = emit.mir.instructions.items(.ops)[inst]; + const flags = @truncate(u2, ops); + + switch (flags) { + 0b00 => { + const decoded = Mir.Ops(AvxRegister, AvxRegister).decode(ops); + return lowerToRvmEnc( + .vaddsd, + Register.avxReg(decoded.reg1), + Register.avxReg(decoded.reg1), + RegisterOrMemory.avxReg(decoded.reg2), + emit.code, + ); + }, + else => return emit.fail("TODO unused variant 0b{b} for mov_f64", .{flags}), + } +} + +fn mirCmpF64(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { + const tag = emit.mir.instructions.items(.tag)[inst]; + assert(tag == .cmp_f64); + const ops = emit.mir.instructions.items(.ops)[inst]; + const flags = @truncate(u2, ops); + + switch (flags) { + 0b00 => { + const decoded = Mir.Ops(AvxRegister, AvxRegister).decode(ops); + return lowerToRmEnc( + .vucomisd, + Register.avxReg(decoded.reg1), + RegisterOrMemory.avxReg(decoded.reg2), + emit.code, + ); + }, else => return emit.fail("TODO unused variant 0b{b} for mov_f64", .{flags}), } } @@ -1217,6 +1273,9 @@ const Tag = enum { cmovb, cmovnae, vmovsd, + vaddsd, + vcmpsd, + vucomisd, fn isSetCC(tag: Tag) bool { return switch (tag) { @@ -1301,6 +1360,12 @@ const Encoding = enum { /// OP r64, r/m64, imm32 rmi, + + /// OP xmm1, xmm2, xmm3/m64 + rvm, + + /// OP xmm1, xmm2, xmm3/m64, imm8 + rvmi, }; const OpCode = union(enum) { @@ -1452,6 +1517,7 @@ inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode { .cmovb, .cmovnae => OpCode.twoByte(0x0f, 0x42), .cmovl, .cmovng => OpCode.twoByte(0x0f, 0x4c), .vmovsd => OpCode.oneByte(0x10), + .vucomisd => OpCode.oneByte(0x2e), else => null, }, .oi => return switch (tag) { @@ -1470,6 +1536,15 @@ inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode { .imul => OpCode.oneByte(if (is_one_byte) 0x6b else 0x69), else => null, }, + .rvm => return switch (tag) { + .vaddsd => OpCode.oneByte(0x58), + .vmovsd => OpCode.oneByte(0x10), + else => null, + }, + .rvmi => return switch (tag) { + .vcmpsd => OpCode.oneByte(0xc2), + else => null, + }, } } @@ -1578,6 +1653,16 @@ inline fn getVexPrefix(tag: Tag, enc: Encoding) ?VexPrefix { }, .rm => switch (tag) { .vmovsd => break :blk .{ .lig = true, .simd_prefix = .p_f2, .wig = true }, + .vucomisd => break :blk .{ .lig = true, .simd_prefix = .p_66, .wig = true }, + else => return null, + }, + .rvm => switch (tag) { + .vaddsd => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f2, .wig = true }, + .vmovsd => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f2, .wig = true }, + else => return null, + }, + .rvmi => switch (tag) { + .vcmpsd => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f2, .wig = true }, else => return null, }, else => unreachable, @@ -2013,15 +2098,33 @@ fn lowerToRmEnc( const opc = getOpCode(tag, .rm, reg.size() == 8 or reg_or_mem.size() == 8).?; switch (reg_or_mem) { .register => |src_reg| { - const encoder = try Encoder.init(code, 4); - if (reg.size() == 16) { - encoder.prefix16BitMode(); - } - encoder.rex(.{ - .w = setRexWRegister(reg) or setRexWRegister(src_reg), - .r = reg.isExtended(), - .b = src_reg.isExtended(), - }); + const encoder: Encoder = blk: { + switch (reg) { + .register => { + const encoder = try Encoder.init(code, 4); + if (reg.size() == 16) { + encoder.prefix16BitMode(); + } + encoder.rex(.{ + .w = setRexWRegister(reg) or setRexWRegister(src_reg), + .r = reg.isExtended(), + .b = src_reg.isExtended(), + }); + break :blk encoder; + }, + .avx_register => { + const encoder = try Encoder.init(code, 5); + var vex_prefix = getVexPrefix(tag, .rm).?; + const vex = &vex_prefix.prefix; + vex.rex(.{ + .r = reg.isExtended(), + .b = src_reg.isExtended(), + }); + encoder.vex(vex_prefix.prefix); + break :blk encoder; + }, + } + }; opc.encode(encoder); encoder.modRm_direct(reg.lowId(), src_reg.lowId()); }, @@ -2188,6 +2291,79 @@ fn lowerToRmiEnc( encodeImm(encoder, imm, reg.size()); } +fn lowerToRvmEnc( + tag: Tag, + reg1: Register, + reg2: Register, + reg_or_mem: RegisterOrMemory, + code: *std.ArrayList(u8), +) InnerError!void { + const opc = getOpCode(tag, .rvm, false).?; + var vex_prefix = getVexPrefix(tag, .rvm).?; + const vex = &vex_prefix.prefix; + switch (reg_or_mem) { + .register => |reg3| { + if (vex_prefix.reg) |vvvv| { + switch (vvvv) { + .nds => vex.reg(reg2.avx_register.id()), + else => unreachable, // TODO + } + } + const encoder = try Encoder.init(code, 5); + vex.rex(.{ + .r = reg1.isExtended(), + .b = reg3.isExtended(), + }); + encoder.vex(vex_prefix.prefix); + opc.encode(encoder); + encoder.modRm_direct(reg1.lowId(), reg3.lowId()); + }, + .memory => |dst_mem| { + _ = dst_mem; + unreachable; // TODO + }, + } +} + +fn lowerToRvmiEnc( + tag: Tag, + reg1: Register, + reg2: Register, + reg_or_mem: RegisterOrMemory, + imm: u32, + code: *std.ArrayList(u8), +) InnerError!void { + const opc = getOpCode(tag, .rvmi, false).?; + var vex_prefix = getVexPrefix(tag, .rvmi).?; + const vex = &vex_prefix.prefix; + const encoder: Encoder = blk: { + switch (reg_or_mem) { + .register => |reg3| { + if (vex_prefix.reg) |vvvv| { + switch (vvvv) { + .nds => vex.reg(reg2.avx_register.id()), + else => unreachable, // TODO + } + } + const encoder = try Encoder.init(code, 5); + vex.rex(.{ + .r = reg1.isExtended(), + .b = reg3.isExtended(), + }); + encoder.vex(vex_prefix.prefix); + opc.encode(encoder); + encoder.modRm_direct(reg1.lowId(), reg3.lowId()); + break :blk encoder; + }, + .memory => |dst_mem| { + _ = dst_mem; + unreachable; // TODO + }, + } + }; + encodeImm(encoder, imm, 8); // TODO +} + fn expectEqualHexStrings(expected: []const u8, given: []const u8, assembly: []const u8) !void { assert(expected.len > 0); if (mem.eql(u8, expected, given)) return; @@ -2598,3 +2774,24 @@ test "lower RMI encoding" { try lowerToRmiEnc(.imul, Register.reg(.r12w), RegisterOrMemory.reg(.r12w), 0x10, emit.code()); try expectEqualHexStrings("\x66\x45\x69\xE4\x10\x00", emit.lowered(), "imul r12w, r12w, 0x10"); } + +test "lower to RVM encoding" { + var emit = TestEmit.init(); + defer emit.deinit(); + try lowerToRvmEnc( + .vaddsd, + Register.avxReg(.xmm0), + Register.avxReg(.xmm1), + RegisterOrMemory.avxReg(.xmm2), + emit.code(), + ); + try expectEqualHexStrings("\xC5\xF3\x58\xC2", emit.lowered(), "vaddsd xmm0, xmm1, xmm2"); + try lowerToRvmEnc( + .vaddsd, + Register.avxReg(.xmm0), + Register.avxReg(.xmm0), + RegisterOrMemory.avxReg(.xmm1), + emit.code(), + ); + try expectEqualHexStrings("\xC5\xFB\x58\xC1", emit.lowered(), "vaddsd xmm0, xmm0, xmm1"); +} diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig index ef9679496d..ef50279d03 100644 --- a/src/arch/x86_64/Mir.zig +++ b/src/arch/x86_64/Mir.zig @@ -353,9 +353,18 @@ pub const Inst = struct { /// AVX instructions /// ops flags: form: /// 0b00 reg1, qword ptr [reg2 + imm32] - /// 0b10 qword ptr [reg1 + imm32], reg2 + /// 0b01 qword ptr [reg1 + imm32], reg2 + /// 0b10 reg1, reg2 mov_f64, + /// ops flags: form: + /// 0b00 reg1, reg1, reg2 + add_f64, + + /// ops flags: form: + /// + cmp_f64, + /// Pseudo-instructions /// call extern function /// Notes: diff --git a/src/register_manager.zig b/src/register_manager.zig index bcec8e14e6..6fc3641467 100644 --- a/src/register_manager.zig +++ b/src/register_manager.zig @@ -338,253 +338,253 @@ pub fn RegisterManager( }; } -const MockRegister1 = enum(u2) { - r0, - r1, - r2, - r3, +//const MockRegister1 = enum(u2) { +// r0, +// r1, +// r2, +// r3, - pub fn id(reg: MockRegister1) u2 { - return @enumToInt(reg); - } +// pub fn id(reg: MockRegister1) u2 { +// return @enumToInt(reg); +// } - const allocatable_registers = [_]MockRegister1{ .r2, .r3 }; -}; +// const allocatable_registers = [_]MockRegister1{ .r2, .r3 }; +//}; -const MockRegister2 = enum(u2) { - r0, - r1, - r2, - r3, +//const MockRegister2 = enum(u2) { +// r0, +// r1, +// r2, +// r3, - pub fn id(reg: MockRegister2) u2 { - return @enumToInt(reg); - } +// pub fn id(reg: MockRegister2) u2 { +// return @enumToInt(reg); +// } - const allocatable_registers = [_]MockRegister2{ .r0, .r1, .r2, .r3 }; -}; +// const allocatable_registers = [_]MockRegister2{ .r0, .r1, .r2, .r3 }; +//}; -fn MockFunction(comptime Register: type) type { - return struct { - allocator: Allocator, - register_manager: RegisterManager(Self, Register, &Register.allocatable_registers) = .{}, - spilled: std.ArrayListUnmanaged(Register) = .{}, +//fn MockFunction(comptime Register: type) type { +// return struct { +// allocator: Allocator, +// register_manager: RegisterManager(Self, Register, &Register.allocatable_registers) = .{}, +// spilled: std.ArrayListUnmanaged(Register) = .{}, - const Self = @This(); +// const Self = @This(); - pub fn deinit(self: *Self) void { - self.spilled.deinit(self.allocator); - } +// pub fn deinit(self: *Self) void { +// self.spilled.deinit(self.allocator); +// } - pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void { - _ = inst; - try self.spilled.append(self.allocator, reg); - } +// pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void { +// _ = inst; +// try self.spilled.append(self.allocator, reg); +// } - pub fn genAdd(self: *Self, res: Register, lhs: Register, rhs: Register) !void { - _ = self; - _ = res; - _ = lhs; - _ = rhs; - } - }; -} +// pub fn genAdd(self: *Self, res: Register, lhs: Register, rhs: Register) !void { +// _ = self; +// _ = res; +// _ = lhs; +// _ = rhs; +// } +// }; +//} -const MockFunction1 = MockFunction(MockRegister1); -const MockFunction2 = MockFunction(MockRegister2); +//const MockFunction1 = MockFunction(MockRegister1); +//const MockFunction2 = MockFunction(MockRegister2); -test "default state" { - const allocator = std.testing.allocator; +//test "default state" { +// const allocator = std.testing.allocator; - var function = MockFunction1{ - .allocator = allocator, - }; - defer function.deinit(); +// var function = MockFunction1{ +// .allocator = allocator, +// }; +// defer function.deinit(); - try expect(!function.register_manager.isRegAllocated(.r2)); - try expect(!function.register_manager.isRegAllocated(.r3)); - try expect(function.register_manager.isRegFree(.r2)); - try expect(function.register_manager.isRegFree(.r3)); -} +// try expect(!function.register_manager.isRegAllocated(.r2)); +// try expect(!function.register_manager.isRegAllocated(.r3)); +// try expect(function.register_manager.isRegFree(.r2)); +// try expect(function.register_manager.isRegFree(.r3)); +//} -test "tryAllocReg: no spilling" { - const allocator = std.testing.allocator; +//test "tryAllocReg: no spilling" { +// const allocator = std.testing.allocator; - var function = MockFunction1{ - .allocator = allocator, - }; - defer function.deinit(); +// var function = MockFunction1{ +// .allocator = allocator, +// }; +// defer function.deinit(); - const mock_instruction: Air.Inst.Index = 1; +// const mock_instruction: Air.Inst.Index = 1; - try expectEqual(@as(?MockRegister1, .r2), function.register_manager.tryAllocReg(mock_instruction)); - try expectEqual(@as(?MockRegister1, .r3), function.register_manager.tryAllocReg(mock_instruction)); - try expectEqual(@as(?MockRegister1, null), function.register_manager.tryAllocReg(mock_instruction)); +// try expectEqual(@as(?MockRegister1, .r2), function.register_manager.tryAllocReg(mock_instruction)); +// try expectEqual(@as(?MockRegister1, .r3), function.register_manager.tryAllocReg(mock_instruction)); +// try expectEqual(@as(?MockRegister1, null), function.register_manager.tryAllocReg(mock_instruction)); - try expect(function.register_manager.isRegAllocated(.r2)); - try expect(function.register_manager.isRegAllocated(.r3)); - try expect(!function.register_manager.isRegFree(.r2)); - try expect(!function.register_manager.isRegFree(.r3)); +// try expect(function.register_manager.isRegAllocated(.r2)); +// try expect(function.register_manager.isRegAllocated(.r3)); +// try expect(!function.register_manager.isRegFree(.r2)); +// try expect(!function.register_manager.isRegFree(.r3)); - function.register_manager.freeReg(.r2); - function.register_manager.freeReg(.r3); +// function.register_manager.freeReg(.r2); +// function.register_manager.freeReg(.r3); - try expect(function.register_manager.isRegAllocated(.r2)); - try expect(function.register_manager.isRegAllocated(.r3)); - try expect(function.register_manager.isRegFree(.r2)); - try expect(function.register_manager.isRegFree(.r3)); -} +// try expect(function.register_manager.isRegAllocated(.r2)); +// try expect(function.register_manager.isRegAllocated(.r3)); +// try expect(function.register_manager.isRegFree(.r2)); +// try expect(function.register_manager.isRegFree(.r3)); +//} -test "allocReg: spilling" { - const allocator = std.testing.allocator; +//test "allocReg: spilling" { +// const allocator = std.testing.allocator; - var function = MockFunction1{ - .allocator = allocator, - }; - defer function.deinit(); +// var function = MockFunction1{ +// .allocator = allocator, +// }; +// defer function.deinit(); - const mock_instruction: Air.Inst.Index = 1; +// const mock_instruction: Air.Inst.Index = 1; - try expectEqual(@as(?MockRegister1, .r2), try function.register_manager.allocReg(mock_instruction)); - try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction)); +// try expectEqual(@as(?MockRegister1, .r2), try function.register_manager.allocReg(mock_instruction)); +// try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction)); - // Spill a register - try expectEqual(@as(?MockRegister1, .r2), try function.register_manager.allocReg(mock_instruction)); - try expectEqualSlices(MockRegister1, &[_]MockRegister1{.r2}, function.spilled.items); +// // Spill a register +// try expectEqual(@as(?MockRegister1, .r2), try function.register_manager.allocReg(mock_instruction)); +// try expectEqualSlices(MockRegister1, &[_]MockRegister1{.r2}, function.spilled.items); - // No spilling necessary - function.register_manager.freeReg(.r3); - try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction)); - try expectEqualSlices(MockRegister1, &[_]MockRegister1{.r2}, function.spilled.items); +// // No spilling necessary +// function.register_manager.freeReg(.r3); +// try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction)); +// try expectEqualSlices(MockRegister1, &[_]MockRegister1{.r2}, function.spilled.items); - // Locked registers - function.register_manager.freeReg(.r3); - { - const lock = function.register_manager.lockReg(.r2); - defer if (lock) |reg| function.register_manager.unlockReg(reg); +// // Locked registers +// function.register_manager.freeReg(.r3); +// { +// const lock = function.register_manager.lockReg(.r2); +// defer if (lock) |reg| function.register_manager.unlockReg(reg); - try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction)); - } - try expect(!function.register_manager.lockedRegsExist()); -} +// try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction)); +// } +// try expect(!function.register_manager.lockedRegsExist()); +//} -test "tryAllocRegs" { - const allocator = std.testing.allocator; +//test "tryAllocRegs" { +// const allocator = std.testing.allocator; - var function = MockFunction2{ - .allocator = allocator, - }; - defer function.deinit(); +// var function = MockFunction2{ +// .allocator = allocator, +// }; +// defer function.deinit(); - try expectEqual([_]MockRegister2{ .r0, .r1, .r2 }, function.register_manager.tryAllocRegs(3, .{ null, null, null }).?); +// try expectEqual([_]MockRegister2{ .r0, .r1, .r2 }, function.register_manager.tryAllocRegs(3, .{ null, null, null }).?); - try expect(function.register_manager.isRegAllocated(.r0)); - try expect(function.register_manager.isRegAllocated(.r1)); - try expect(function.register_manager.isRegAllocated(.r2)); - try expect(!function.register_manager.isRegAllocated(.r3)); +// try expect(function.register_manager.isRegAllocated(.r0)); +// try expect(function.register_manager.isRegAllocated(.r1)); +// try expect(function.register_manager.isRegAllocated(.r2)); +// try expect(!function.register_manager.isRegAllocated(.r3)); - // Locked registers - function.register_manager.freeReg(.r0); - function.register_manager.freeReg(.r2); - function.register_manager.freeReg(.r3); - { - const lock = function.register_manager.lockReg(.r1); - defer if (lock) |reg| function.register_manager.unlockReg(reg); +// // Locked registers +// function.register_manager.freeReg(.r0); +// function.register_manager.freeReg(.r2); +// function.register_manager.freeReg(.r3); +// { +// const lock = function.register_manager.lockReg(.r1); +// defer if (lock) |reg| function.register_manager.unlockReg(reg); - try expectEqual([_]MockRegister2{ .r0, .r2, .r3 }, function.register_manager.tryAllocRegs(3, .{ null, null, null }).?); - } - try expect(!function.register_manager.lockedRegsExist()); +// try expectEqual([_]MockRegister2{ .r0, .r2, .r3 }, function.register_manager.tryAllocRegs(3, .{ null, null, null }).?); +// } +// try expect(!function.register_manager.lockedRegsExist()); - try expect(function.register_manager.isRegAllocated(.r0)); - try expect(function.register_manager.isRegAllocated(.r1)); - try expect(function.register_manager.isRegAllocated(.r2)); - try expect(function.register_manager.isRegAllocated(.r3)); -} +// try expect(function.register_manager.isRegAllocated(.r0)); +// try expect(function.register_manager.isRegAllocated(.r1)); +// try expect(function.register_manager.isRegAllocated(.r2)); +// try expect(function.register_manager.isRegAllocated(.r3)); +//} -test "allocRegs: normal usage" { - // TODO: convert this into a decltest once that is supported +//test "allocRegs: normal usage" { +// // TODO: convert this into a decltest once that is supported - const allocator = std.testing.allocator; +// const allocator = std.testing.allocator; - var function = MockFunction2{ - .allocator = allocator, - }; - defer function.deinit(); +// var function = MockFunction2{ +// .allocator = allocator, +// }; +// defer function.deinit(); - { - const result_reg: MockRegister2 = .r1; +// { +// const result_reg: MockRegister2 = .r1; - // The result register is known and fixed at this point, we - // don't want to accidentally allocate lhs or rhs to the - // result register, this is why we lock it. - // - // Using defer unlock right after lock is a good idea in - // most cases as you probably are using the locked registers - // in the remainder of this scope and don't need to use it - // after the end of this scope. However, in some situations, - // it may make sense to manually unlock registers before the - // end of the scope when you are certain that they don't - // contain any valuable data anymore and can be reused. For an - // example of that, see `selectively reducing register - // pressure`. - const lock = function.register_manager.lockReg(result_reg); - defer if (lock) |reg| function.register_manager.unlockReg(reg); +// // The result register is known and fixed at this point, we +// // don't want to accidentally allocate lhs or rhs to the +// // result register, this is why we lock it. +// // +// // Using defer unlock right after lock is a good idea in +// // most cases as you probably are using the locked registers +// // in the remainder of this scope and don't need to use it +// // after the end of this scope. However, in some situations, +// // it may make sense to manually unlock registers before the +// // end of the scope when you are certain that they don't +// // contain any valuable data anymore and can be reused. For an +// // example of that, see `selectively reducing register +// // pressure`. +// const lock = function.register_manager.lockReg(result_reg); +// defer if (lock) |reg| function.register_manager.unlockReg(reg); - const regs = try function.register_manager.allocRegs(2, .{ null, null }); - try function.genAdd(result_reg, regs[0], regs[1]); - } -} +// const regs = try function.register_manager.allocRegs(2, .{ null, null }); +// try function.genAdd(result_reg, regs[0], regs[1]); +// } +//} -test "allocRegs: selectively reducing register pressure" { - // TODO: convert this into a decltest once that is supported +//test "allocRegs: selectively reducing register pressure" { +// // TODO: convert this into a decltest once that is supported - const allocator = std.testing.allocator; +// const allocator = std.testing.allocator; - var function = MockFunction2{ - .allocator = allocator, - }; - defer function.deinit(); +// var function = MockFunction2{ +// .allocator = allocator, +// }; +// defer function.deinit(); - { - const result_reg: MockRegister2 = .r1; +// { +// const result_reg: MockRegister2 = .r1; - const lock = function.register_manager.lockReg(result_reg); +// const lock = function.register_manager.lockReg(result_reg); - // Here, we don't defer unlock because we manually unlock - // after genAdd - const regs = try function.register_manager.allocRegs(2, .{ null, null }); +// // Here, we don't defer unlock because we manually unlock +// // after genAdd +// const regs = try function.register_manager.allocRegs(2, .{ null, null }); - try function.genAdd(result_reg, regs[0], regs[1]); - function.register_manager.unlockReg(lock.?); +// try function.genAdd(result_reg, regs[0], regs[1]); +// function.register_manager.unlockReg(lock.?); - const extra_summand_reg = try function.register_manager.allocReg(null); - try function.genAdd(result_reg, result_reg, extra_summand_reg); - } -} +// const extra_summand_reg = try function.register_manager.allocReg(null); +// try function.genAdd(result_reg, result_reg, extra_summand_reg); +// } +//} -test "getReg" { - const allocator = std.testing.allocator; +//test "getReg" { +// const allocator = std.testing.allocator; - var function = MockFunction1{ - .allocator = allocator, - }; - defer function.deinit(); +// var function = MockFunction1{ +// .allocator = allocator, +// }; +// defer function.deinit(); - const mock_instruction: Air.Inst.Index = 1; +// const mock_instruction: Air.Inst.Index = 1; - try function.register_manager.getReg(.r3, mock_instruction); +// try function.register_manager.getReg(.r3, mock_instruction); - try expect(!function.register_manager.isRegAllocated(.r2)); - try expect(function.register_manager.isRegAllocated(.r3)); - try expect(function.register_manager.isRegFree(.r2)); - try expect(!function.register_manager.isRegFree(.r3)); +// try expect(!function.register_manager.isRegAllocated(.r2)); +// try expect(function.register_manager.isRegAllocated(.r3)); +// try expect(function.register_manager.isRegFree(.r2)); +// try expect(!function.register_manager.isRegFree(.r3)); - // Spill r3 - try function.register_manager.getReg(.r3, mock_instruction); +// // Spill r3 +// try function.register_manager.getReg(.r3, mock_instruction); - try expect(!function.register_manager.isRegAllocated(.r2)); - try expect(function.register_manager.isRegAllocated(.r3)); - try expect(function.register_manager.isRegFree(.r2)); - try expect(!function.register_manager.isRegFree(.r3)); - try expectEqualSlices(MockRegister1, &[_]MockRegister1{.r3}, function.spilled.items); -} +// try expect(!function.register_manager.isRegAllocated(.r2)); +// try expect(function.register_manager.isRegAllocated(.r3)); +// try expect(function.register_manager.isRegFree(.r2)); +// try expect(!function.register_manager.isRegFree(.r3)); +// try expectEqualSlices(MockRegister1, &[_]MockRegister1{.r3}, function.spilled.items); +//} From 9e5c8cb008f75c2e570a0e48d5d014e936c103ca Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Mon, 16 May 2022 19:45:30 +0200 Subject: [PATCH 07/19] x64: merge general purpose with simd register into one bitset This way, we do not have to tweak the `RegisterManager` to handle multiple register types - we have one linear space instead. Additionally we can use the bitset itself to separate the registers into overlapping (the ones that are aliases of differing bitwidths) and nonoverlapping classes (for example, AVX registers do not overlap general purpose registers, thus they can be allocated simultaneously). Another huge benefit of this simple approach is the fact that we can still refer to *all* registers regardless of their class via enum literals which makes the code so much more readable. Finally, `RegisterLock` is universal across different register classes. --- src/arch/x86_64/CodeGen.zig | 1060 ++++++++++++++--------------------- src/arch/x86_64/Emit.zig | 772 +++++++++++++------------ src/arch/x86_64/Mir.zig | 66 ++- src/arch/x86_64/abi.zig | 17 +- src/arch/x86_64/bits.zig | 100 ++-- src/codegen.zig | 24 +- src/register_manager.zig | 404 +++++++------ 7 files changed, 1102 insertions(+), 1341 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index e81f2d5435..aa6d04d5d9 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -35,14 +35,10 @@ const caller_preserved_regs = abi.caller_preserved_regs; const allocatable_registers = abi.allocatable_registers; const c_abi_int_param_regs = abi.c_abi_int_param_regs; const c_abi_int_return_regs = abi.c_abi_int_return_regs; -const RegisterManager = RegisterManagerFn(Self, Register, &allocatable_registers, spillInstruction); +const RegisterManager = RegisterManagerFn(Self, Register, &allocatable_registers); const RegisterLock = RegisterManager.RegisterLock; const Register = bits.Register; -const AvxRegisterManager = RegisterManagerFn(Self, AvxRegister, &abi.avx_regs, spillInstructionAvx); -const AvxRegisterLock = AvxRegisterManager.RegisterLock; -const AvxRegister = bits.AvxRegister; - const InnerError = error{ OutOfMemory, CodegenFail, @@ -92,8 +88,7 @@ branch_stack: *std.ArrayList(Branch), // Key is the block instruction blocks: std.AutoHashMapUnmanaged(Air.Inst.Index, BlockData) = .{}, -register_manager: RegisterManager, -avx_register_manager: AvxRegisterManager, +register_manager: RegisterManager = .{}, /// Maps offset to what is stored there. stack: std.AutoHashMapUnmanaged(u32, StackAllocation) = .{}, @@ -133,8 +128,6 @@ pub const MCValue = union(enum) { /// The value is a tuple { wrapped, overflow } where wrapped value is stored in the GP register, /// and the operation is a signed operation. register_overflow_signed: Register, - /// The value is in an AVX register. - avx_register: AvxRegister, /// The value is in memory at a hard-coded address. /// If the type is a pointer, it means the pointer address is at this memory location. memory: u64, @@ -202,7 +195,6 @@ pub const MCValue = union(enum) { fn isRegister(mcv: MCValue) bool { return switch (mcv) { .register => true, - .avx_register => true, else => false, }; } @@ -304,11 +296,7 @@ pub fn generate( .mir_to_air_map = if (builtin.mode == .Debug) std.AutoHashMap(Mir.Inst.Index, Air.Inst.Index).init(bin_file.allocator) else {}, - .register_manager = undefined, - .avx_register_manager = undefined, }; - function.register_manager = .{ .function = &function }; - function.avx_register_manager = .{ .function = &function }; defer function.stack.deinit(bin_file.allocator); defer function.blocks.deinit(bin_file.allocator); defer function.exitlude_jump_relocs.deinit(bin_file.allocator); @@ -400,17 +388,15 @@ fn gen(self: *Self) InnerError!void { if (cc != .Naked) { _ = try self.addInst(.{ .tag = .push, - .ops = (Mir.Ops(Register, Register){ - .reg1 = .rbp, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }), .data = undefined, // unused for push reg, }); _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp, .reg2 = .rsp, - }).encode(), + }), .data = undefined, }); // We want to subtract the aligned stack frame size from rsp here, but we don't @@ -447,9 +433,7 @@ fn gen(self: *Self) InnerError!void { // push the callee_preserved_regs that were used const backpatch_push_callee_preserved_regs_i = try self.addInst(.{ .tag = .push_regs_from_callee_preserved_regs, - .ops = (Mir.Ops(Register, Register){ - .reg1 = .rbp, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }), .data = .{ .payload = undefined }, // to be backpatched }); @@ -489,9 +473,7 @@ fn gen(self: *Self) InnerError!void { // pop the callee_preserved_regs _ = try self.addInst(.{ .tag = .pop_regs_from_callee_preserved_regs, - .ops = (Mir.Ops(Register, Register){ - .reg1 = .rbp, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }), .data = .{ .payload = callee_preserved_regs_payload }, }); @@ -510,17 +492,13 @@ fn gen(self: *Self) InnerError!void { _ = try self.addInst(.{ .tag = .pop, - .ops = (Mir.Ops(Register, Register){ - .reg1 = .rbp, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp }), .data = undefined, }); _ = try self.addInst(.{ .tag = .ret, - .ops = (Mir.Ops(Register, Register){ - .flags = 0b11, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .flags = 0b11 }), .data = undefined, }); @@ -534,16 +512,12 @@ fn gen(self: *Self) InnerError!void { if (aligned_stack_end > 0) { self.mir_instructions.set(backpatch_stack_sub, .{ .tag = .sub, - .ops = (Mir.Ops(Register, Register){ - .reg1 = .rsp, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }), .data = .{ .imm = aligned_stack_end }, }); self.mir_instructions.set(backpatch_stack_add, .{ .tag = .add, - .ops = (Mir.Ops(Register, Register){ - .reg1 = .rsp, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }), .data = .{ .imm = aligned_stack_end }, }); } @@ -908,8 +882,8 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue { // TODO check if AVX available const ptr_bytes: u64 = 32; if (abi_size <= ptr_bytes) { - if (self.avx_register_manager.tryAllocReg(inst)) |reg| { - return MCValue{ .avx_register = avxRegisterAlias(reg, abi_size) }; + if (self.register_manager.tryAllocReg(inst)) |reg| { + return MCValue{ .register = registerAlias(reg, abi_size) }; } } }, @@ -947,21 +921,6 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void try self.genSetStack(self.air.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv, .{}); } -pub fn spillInstructionAvx(self: *Self, reg: AvxRegister, inst: Air.Inst.Index) !void { - const stack_mcv = try self.allocRegOrMem(inst, false); - log.debug("spilling {d} to stack mcv {any}", .{ inst, stack_mcv }); - const reg_mcv = self.getResolvedInstValue(inst); - switch (reg_mcv) { - .avx_register => |other| { - assert(reg.to256() == other.to256()); - }, - else => {}, - } - const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; - try branch.inst_table.put(self.gpa, inst, stack_mcv); - try self.genSetStack(self.air.typeOfIndex(inst), stack_mcv.stack_offset, reg_mcv, .{}); -} - pub fn spillCompareFlagsIfOccupied(self: *Self) !void { if (self.compare_flags_inst) |inst_to_save| { const mcv = self.getResolvedInstValue(inst_to_save); @@ -972,7 +931,6 @@ pub fn spillCompareFlagsIfOccupied(self: *Self) !void { .compare_flags_signed, .compare_flags_unsigned, => try self.allocRegOrMem(inst_to_save, true), - .avx_register => try self.allocRegOrMem(inst_to_save, false), else => unreachable, }; @@ -990,7 +948,6 @@ pub fn spillCompareFlagsIfOccupied(self: *Self) !void { .register_overflow_signed, .register_overflow_unsigned, => |reg| self.register_manager.freeReg(reg), - .avx_register => |reg| self.avx_register_manager.freeReg(reg), else => {}, } } @@ -1236,10 +1193,10 @@ fn airMin(self: *Self, inst: Air.Inst.Index) !void { const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ty, rhs_mcv); _ = try self.addInst(.{ .tag = if (signedness == .signed) .cond_mov_lt else .cond_mov_below, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = dst_mcv.register, .reg2 = lhs_reg, - }).encode(), + }), .data = undefined, }); @@ -1440,10 +1397,10 @@ fn genSetStackTruncatedOverflowCompare( }; _ = try self.addInst(.{ .tag = .cond_set_byte_overflow, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = overflow_reg.to8(), .flags = flags, - }).encode(), + }), .data = undefined, }); @@ -1460,10 +1417,7 @@ fn genSetStackTruncatedOverflowCompare( const eq_reg = temp_regs[2]; _ = try self.addInst(.{ .tag = .cond_set_byte_eq_ne, - .ops = (Mir.Ops(Register, Register){ - .reg1 = eq_reg.to8(), - .flags = 0b00, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .reg1 = eq_reg.to8() }), .data = undefined, }); @@ -1609,19 +1563,17 @@ fn genIntMulDivOpMir( .signed => { _ = try self.addInst(.{ .tag = .cwd, - .ops = (Mir.Ops(Register, Register){ - .flags = 0b11, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .flags = 0b11 }), .data = undefined, }); }, .unsigned => { _ = try self.addInst(.{ .tag = .xor, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rdx, .reg2 = .rdx, - }).encode(), + }), .data = undefined, }); }, @@ -1640,16 +1592,14 @@ fn genIntMulDivOpMir( .register => |reg| { _ = try self.addInst(.{ .tag = tag, - .ops = (Mir.Ops(Register, Register){ - .reg1 = reg, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .reg1 = reg }), .data = undefined, }); }, .stack_offset => |off| { _ = try self.addInst(.{ .tag = tag, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg2 = .rbp, .flags = switch (abi_size) { 1 => 0b00, @@ -1658,7 +1608,7 @@ fn genIntMulDivOpMir( 8 => 0b11, else => unreachable, }, - }).encode(), + }), .data = .{ .imm = @bitCast(u32, -off) }, }); }, @@ -1691,34 +1641,34 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa _ = try self.addInst(.{ .tag = .xor, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = divisor.to64(), .reg2 = dividend.to64(), - }).encode(), + }), .data = undefined, }); _ = try self.addInst(.{ .tag = .sar, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = divisor.to64(), .flags = 0b10, - }).encode(), + }), .data = .{ .imm = 63 }, }); _ = try self.addInst(.{ .tag = .@"test", - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rdx, .reg2 = .rdx, - }).encode(), + }), .data = undefined, }); _ = try self.addInst(.{ .tag = .cond_mov_eq, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = divisor.to64(), .reg2 = .rdx, - }).encode(), + }), .data = undefined, }); try self.genBinOpMir(.add, Type.isize, .{ .register = divisor }, .{ .register = .rax }); @@ -2102,11 +2052,11 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue { // mov reg, [rbp - 8] _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = addr_reg.to64(), .reg2 = .rbp, .flags = 0b01, - }).encode(), + }), .data = .{ .imm = @bitCast(u32, -@intCast(i32, off)) }, }); }, @@ -2187,10 +2137,10 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void { // lea reg, [rbp] _ = try self.addInst(.{ .tag = .lea, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = addr_reg.to64(), .reg2 = .rbp, - }).encode(), + }), .data = .{ .imm = @bitCast(u32, -off) }, }); }, @@ -2198,10 +2148,10 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void { // lea reg, [rbp] _ = try self.addInst(.{ .tag = .lea, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = addr_reg.to64(), .reg2 = .rbp, - }).encode(), + }), .data = .{ .imm = @bitCast(u32, -off) }, }); }, @@ -2266,11 +2216,11 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void { // mov dst_mcv, [dst_mcv] _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops(Register, Register){ - .flags = 0b01, + .ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(dst_mcv.register, @intCast(u32, elem_abi_size)), .reg2 = dst_mcv.register, - }).encode(), + .flags = 0b01, + }), .data = .{ .imm = 0 }, }); break :result .{ .register = registerAlias(dst_mcv.register, @intCast(u32, elem_abi_size)) }; @@ -2532,11 +2482,11 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo // mov dst_reg, [reg] _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(dst_reg, @intCast(u32, abi_size)), .reg2 = reg, .flags = 0b01, - }).encode(), + }), .data = .{ .imm = 0 }, }); }, @@ -2552,9 +2502,6 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo else => return self.fail("TODO implement loading from register into {}", .{dst_mcv}), } }, - .avx_register => { - return self.fail("TODO load for AVX register", .{}); - }, .memory, .got_load, .direct_load, @@ -2606,10 +2553,10 @@ fn loadMemPtrIntoRegister(self: *Self, reg: Register, ptr_ty: Type, ptr: MCValue const fn_owner_decl = mod.declPtr(self.mod_fn.owner_decl); _ = try self.addInst(.{ .tag = .lea_pie, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(reg, abi_size), .flags = flags, - }).encode(), + }), .data = .{ .load_reloc = .{ .atom_index = fn_owner_decl.link.macho.local_sym_index, @@ -2670,7 +2617,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type }); _ = try self.addInst(.{ .tag = .mov_mem_imm, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = reg.to64(), .flags = switch (abi_size) { 1 => 0b00, @@ -2678,7 +2625,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type 4 => 0b10, else => unreachable, }, - }).encode(), + }), .data = .{ .payload = payload }, }); }, @@ -2692,11 +2639,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type const tmp_reg = try self.copyToTmpRegister(value_ty, value); _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = reg.to64(), .reg2 = tmp_reg.to64(), .flags = 0b10, - }).encode(), + }), .data = .{ .imm = 0 }, }); }, @@ -2708,11 +2655,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type .register => |src_reg| { _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = reg.to64(), .reg2 = registerAlias(src_reg, @intCast(u32, abi_size)), .flags = 0b10, - }).encode(), + }), .data = .{ .imm = 0 }, }); }, @@ -2736,9 +2683,6 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type }, } }, - .avx_register => { - return self.fail("TODO store for AVX register", .{}); - }, .got_load, .direct_load, .memory, @@ -2759,11 +2703,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type // mov reg, [reg] _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = addr_reg.to64(), .reg2 = addr_reg.to64(), .flags = 0b01, - }).encode(), + }), .data = .{ .imm = 0 }, }); @@ -2798,21 +2742,21 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type } _ = try self.addInst(.{ .tag = .mov_mem_imm, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = addr_reg.to64(), .flags = flags, - }).encode(), + }), .data = .{ .payload = payload }, }); }, .register => |reg| { _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = addr_reg.to64(), .reg2 = reg, .flags = 0b10, - }).encode(), + }), .data = .{ .imm = 0 }, }); }, @@ -2829,20 +2773,20 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = tmp_reg, .reg2 = tmp_reg, .flags = 0b01, - }).encode(), + }), .data = .{ .imm = 0 }, }); _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = addr_reg.to64(), .reg2 = tmp_reg, .flags = 0b10, - }).encode(), + }), .data = .{ .imm = 0 }, }); return; @@ -2856,11 +2800,11 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type const tmp_reg = try self.copyToTmpRegister(value_ty, value); _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = addr_reg.to64(), .reg2 = tmp_reg, .flags = 0b10, - }).encode(), + }), .data = .{ .imm = 0 }, }); return; @@ -3017,10 +2961,10 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { if (signedness == .signed and field_size < 8) { _ = try self.addInst(.{ .tag = .mov_sign_extend, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = dst_mcv.register, .reg2 = registerAlias(dst_mcv.register, field_size), - }).encode(), + }), .data = undefined, }); } @@ -3048,10 +2992,10 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { }; _ = try self.addInst(.{ .tag = .cond_set_byte_overflow, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = dst_reg.to8(), .flags = flags, - }).encode(), + }), .data = undefined, }); break :result MCValue{ .register = dst_reg.to8() }; @@ -3092,10 +3036,7 @@ fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shi 1 => { _ = try self.addInst(.{ .tag = tag, - .ops = (Mir.Ops(Register, Register){ - .reg1 = registerAlias(reg, abi_size), - .flags = 0b00, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(reg, abi_size) }), .data = undefined, }); return; @@ -3103,10 +3044,10 @@ fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shi else => { _ = try self.addInst(.{ .tag = tag, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(reg, abi_size), .flags = 0b10, - }).encode(), + }), .data = .{ .imm = @intCast(u8, imm) }, }); return; @@ -3124,10 +3065,10 @@ fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shi _ = try self.addInst(.{ .tag = tag, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(reg, abi_size), .flags = 0b01, - }).encode(), + }), .data = undefined, }); } @@ -3383,32 +3324,6 @@ fn genBinOp( return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmtDebug()}); } - if (lhs_ty.zigTypeTag() == .Float) { - switch (tag) { - .add => { - const dst_reg: AvxRegister = blk: { - const reg = try self.avx_register_manager.allocReg(null); - try self.genSetAvxReg(lhs_ty, reg, lhs); - break :blk reg.to128(); - }; - const dst_lock = self.avx_register_manager.lockRegAssumeUnused(dst_reg); - defer self.avx_register_manager.unlockReg(dst_lock); - - const src_reg: AvxRegister = blk: { - const reg = try self.avx_register_manager.allocReg(null); - try self.genSetAvxReg(lhs_ty, reg, rhs); - break :blk reg.to128(); - }; - const src_lock = self.avx_register_manager.lockRegAssumeUnused(src_reg); - defer self.avx_register_manager.unlockReg(src_lock); - - try self.genBinOpMir(.add_f64, lhs_ty, .{ .avx_register = dst_reg }, .{ .avx_register = src_reg }); - return MCValue{ .avx_register = dst_reg }; - }, - else => unreachable, - } - } - const is_commutative: bool = switch (tag) { .add, .addwrap, @@ -3526,25 +3441,39 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu const reg = try self.copyToTmpRegister(dst_ty, src_mcv); return self.genBinOpMir(mir_tag, dst_ty, dst_mcv, .{ .register = reg }); }, - .register => |src_reg| { - _ = try self.addInst(.{ - .tag = mir_tag, - .ops = (Mir.Ops(Register, Register){ - .reg1 = registerAlias(dst_reg, abi_size), - .reg2 = registerAlias(src_reg, abi_size), - }).encode(), - .data = undefined, - }); - }, - .avx_register => { - return self.fail("TODO genBinOp for AVX register", .{}); + .register => |src_reg| switch (dst_ty.zigTypeTag()) { + .Float => switch (dst_ty.tag()) { + .f64 => { + _ = try self.addInst(.{ + .tag = switch (mir_tag) { + .add => .add_f64, + .cmp => .cmp_f64, + else => return self.fail("TODO genBinOpMir for f64 register-register with MIR tag {}", .{mir_tag}), + }, + .ops = Mir.Inst.Ops.encode(.{ + .reg1 = dst_reg.to128(), + .reg2 = src_reg.to128(), + }), + .data = undefined, + }); + }, + else => return self.fail("TODO genBinOpMir for float register-register and type {}", .{dst_ty.fmtDebug()}), + }, + else => { + _ = try self.addInst(.{ + .tag = mir_tag, + .ops = Mir.Inst.Ops.encode(.{ + .reg1 = registerAlias(dst_reg, abi_size), + .reg2 = registerAlias(src_reg, abi_size), + }), + .data = undefined, + }); + }, }, .immediate => |imm| { _ = try self.addInst(.{ .tag = mir_tag, - .ops = (Mir.Ops(Register, Register){ - .reg1 = registerAlias(dst_reg, abi_size), - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(dst_reg, abi_size) }), .data = .{ .imm = @truncate(u32, imm) }, }); }, @@ -3567,39 +3496,16 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu } _ = try self.addInst(.{ .tag = mir_tag, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(dst_reg, abi_size), .reg2 = .rbp, .flags = 0b01, - }).encode(), + }), .data = .{ .imm = @bitCast(u32, -off) }, }); }, } }, - .avx_register => |dst_reg| { - switch (src_mcv) { - .avx_register => |src_reg| { - switch (dst_ty.zigTypeTag()) { - .Float => switch (dst_ty.tag()) { - .f64 => { - _ = try self.addInst(.{ - .tag = mir_tag, - .ops = (Mir.Ops(AvxRegister, AvxRegister){ - .reg1 = dst_reg.to128(), - .reg2 = src_reg.to128(), - }).encode(), - .data = undefined, - }); - }, - else => return self.fail("TODO genBinOp for AVX register and type {}", .{dst_ty.fmtDebug()}), - }, - else => return self.fail("TODO genBinOp for AVX register and type {}", .{dst_ty.fmtDebug()}), - } - }, - else => return self.fail("TODO genBinOp for AVX register", .{}), - } - }, .ptr_stack_offset, .stack_offset => |off| { if (off > math.maxInt(i32)) { return self.fail("stack offset too large", .{}); @@ -3617,17 +3523,14 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu .register => |src_reg| { _ = try self.addInst(.{ .tag = mir_tag, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp, .reg2 = registerAlias(src_reg, abi_size), .flags = 0b10, - }).encode(), + }), .data = .{ .imm = @bitCast(u32, -off) }, }); }, - .avx_register => { - return self.fail("TODO genBinOp for AVX register", .{}); - }, .immediate => |imm| { const tag: Mir.Inst.Tag = switch (mir_tag) { .add => .add_mem_imm, @@ -3651,10 +3554,10 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu }); _ = try self.addInst(.{ .tag = tag, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rbp, .flags = flags, - }).encode(), + }), .data = .{ .payload = payload }, }); }, @@ -3697,7 +3600,6 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M .ptr_stack_offset => unreachable, .register_overflow_unsigned => unreachable, .register_overflow_signed => unreachable, - .avx_register => unreachable, .register => |dst_reg| { switch (src_mcv) { .none => unreachable, @@ -3706,15 +3608,14 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M .ptr_stack_offset => unreachable, .register_overflow_unsigned => unreachable, .register_overflow_signed => unreachable, - .avx_register => unreachable, .register => |src_reg| { // register, register _ = try self.addInst(.{ .tag = .imul_complex, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(dst_reg, abi_size), .reg2 = registerAlias(src_reg, abi_size), - }).encode(), + }), .data = undefined, }); }, @@ -3724,11 +3625,11 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M if (math.minInt(i32) <= imm and imm <= math.maxInt(i32)) { _ = try self.addInst(.{ .tag = .imul_complex, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = dst_reg.to32(), .reg2 = dst_reg.to32(), .flags = 0b10, - }).encode(), + }), .data = .{ .imm = @truncate(u32, imm) }, }); } else { @@ -3740,11 +3641,11 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M .stack_offset => |off| { _ = try self.addInst(.{ .tag = .imul_complex, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(dst_reg, abi_size), .reg2 = .rbp, .flags = 0b01, - }).encode(), + }), .data = .{ .imm = @bitCast(u32, -off) }, }); }, @@ -3770,7 +3671,6 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M .ptr_stack_offset => unreachable, .register_overflow_unsigned => unreachable, .register_overflow_signed => unreachable, - .avx_register => unreachable, .register => |src_reg| { // copy dst to a register const dst_reg = try self.copyToTmpRegister(dst_ty, dst_mcv); @@ -3778,10 +3678,10 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M // register, register _ = try self.addInst(.{ .tag = .imul_complex, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(dst_reg, abi_size), .reg2 = registerAlias(src_reg, abi_size), - }).encode(), + }), .data = undefined, }); // copy dst_reg back out @@ -3888,9 +3788,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void { fn airBreakpoint(self: *Self) !void { _ = try self.addInst(.{ .tag = .interrupt, - .ops = (Mir.Ops{ - .flags = 0b00, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{}), .data = undefined, }); return self.finishAirBookkeeping(); @@ -3973,9 +3871,6 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions. .ptr_stack_offset => { return self.fail("TODO implement calling with MCValue.ptr_stack_offset arg", .{}); }, - .avx_register => { - return self.fail("TODO implement calling with MCValue.avx_register arg", .{}); - }, .undef => unreachable, .immediate => unreachable, .unreach => unreachable, @@ -3994,9 +3889,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions. // Adjust the stack _ = try self.addInst(.{ .tag = .sub, - .ops = (Mir.Ops(Register, Register){ - .reg1 = .rsp, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }), .data = .{ .imm = info.stack_byte_count }, }); } @@ -4020,9 +3913,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions. unreachable; _ = try self.addInst(.{ .tag = .call, - .ops = (Mir.Ops(Register, Register){ - .flags = 0b01, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .flags = 0b01 }), .data = .{ .imm = @truncate(u32, got_addr) }, }); } else if (func_value.castTag(.extern_fn)) |_| { @@ -4036,10 +3927,10 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions. try self.genSetReg(Type.initTag(.usize), .rax, mcv); _ = try self.addInst(.{ .tag = .call, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rax, .flags = 0b01, - }).encode(), + }), .data = undefined, }); } @@ -4054,10 +3945,10 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions. // callq *%rax _ = try self.addInst(.{ .tag = .call, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rax, .flags = 0b01, - }).encode(), + }), .data = undefined, }); } else if (func_value.castTag(.extern_fn)) |func_payload| { @@ -4089,10 +3980,10 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions. try self.genSetReg(Type.initTag(.usize), .rax, mcv); _ = try self.addInst(.{ .tag = .call, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rax, .flags = 0b01, - }).encode(), + }), .data = undefined, }); } @@ -4107,9 +3998,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions. const fn_got_addr = got_addr + got_index * ptr_bytes; _ = try self.addInst(.{ .tag = .call, - .ops = (Mir.Ops(Register, Register){ - .flags = 0b01, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .flags = 0b01 }), .data = .{ .imm = @intCast(u32, fn_got_addr) }, }); } else return self.fail("TODO implement calling extern fn on plan9", .{}); @@ -4119,10 +4008,10 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions. try self.genSetReg(Type.initTag(.usize), .rax, mcv); _ = try self.addInst(.{ .tag = .call, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rax, .flags = 0b01, - }).encode(), + }), .data = undefined, }); } @@ -4132,9 +4021,7 @@ fn airCall(self: *Self, inst: Air.Inst.Index, modifier: std.builtin.CallOptions. // Readjust the stack _ = try self.addInst(.{ .tag = .add, - .ops = (Mir.Ops(Register, Register){ - .reg1 = .rsp, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp }), .data = .{ .imm = info.stack_byte_count }, }); } @@ -4192,9 +4079,7 @@ fn airRet(self: *Self, inst: Air.Inst.Index) !void { // which is available if the jump is 127 bytes or less forward. const jmp_reloc = try self.addInst(.{ .tag = .jmp, - .ops = (Mir.Ops(Register, Register){ - .flags = 0b00, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{}), .data = .{ .inst = undefined }, }); try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc); @@ -4227,9 +4112,7 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void { // which is available if the jump is 127 bytes or less forward. const jmp_reloc = try self.addInst(.{ .tag = .jmp, - .ops = (Mir.Ops(Register, Register){ - .flags = 0b00, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{}), .data = .{ .inst = undefined }, }); try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc); @@ -4256,37 +4139,6 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void { self.compare_flags_inst = inst; const result: MCValue = result: { - if (ty.zigTypeTag() == .Float) { - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - - const dst_reg: AvxRegister = blk: { - const reg = try self.avx_register_manager.allocReg(null); - try self.genSetAvxReg(ty, reg, lhs); - break :blk reg.to128(); - }; - const dst_lock = self.avx_register_manager.lockRegAssumeUnused(dst_reg); - defer self.avx_register_manager.unlockReg(dst_lock); - - const src_reg: AvxRegister = blk: { - const reg = try self.avx_register_manager.allocReg(null); - try self.genSetAvxReg(ty, reg, rhs); - break :blk reg.to128(); - }; - const src_lock = self.avx_register_manager.lockRegAssumeUnused(src_reg); - defer self.avx_register_manager.unlockReg(src_lock); - - _ = try self.addInst(.{ - .tag = .cmp_f64, - .ops = (Mir.Ops(AvxRegister, AvxRegister){ - .reg1 = dst_reg, - .reg2 = src_reg, - }).encode(), - .data = undefined, - }); - - break :result MCValue{ .compare_flags_unsigned = op }; - } // There are 2 operands, destination and source. // Either one, but not both, can be a memory operand. // Source operand can be an immediate, 8 bits or 32 bits. @@ -4304,8 +4156,28 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void { const dst_mcv = MCValue{ .register = dst_reg }; + const rhs_ty = self.air.typeOf(bin_op.rhs); // This instruction supports only signed 32-bit immediates at most. - const src_mcv = try self.limitImmediateType(bin_op.rhs, i32); + const src_mcv: MCValue = blk: { + switch (rhs_ty.zigTypeTag()) { + .Float => { + const rhs = try self.resolveInst(bin_op.rhs); + const rhs_lock: ?RegisterLock = switch (rhs) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); + const src_reg = try self.copyToTmpRegister(rhs_ty, rhs); + break :blk MCValue{ .register = src_reg }; + }, + else => break :blk try self.limitImmediateType(bin_op.rhs, i32), + } + }; + const src_lock: ?RegisterLock = switch (src_mcv) { + .register => |reg| self.register_manager.lockReg(reg), + else => null, + }; + defer if (src_lock) |lock| self.register_manager.unlockReg(lock); try self.genBinOpMir(.cmp, ty, dst_mcv, src_mcv); break :result switch (signedness) { @@ -4504,9 +4376,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 { Mir.Inst.Tag.cond_jmp_greater_less; return self.addInst(.{ .tag = tag, - .ops = (Mir.Ops(Register, Register){ - .flags = flags, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .flags = flags }), .data = .{ .inst = undefined }, }); }, @@ -4514,17 +4384,12 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !u32 { try self.spillCompareFlagsIfOccupied(); _ = try self.addInst(.{ .tag = .@"test", - .ops = (Mir.Ops(Register, Register){ - .reg1 = reg, - .flags = 0b00, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .reg1 = reg }), .data = .{ .imm = 1 }, }); return self.addInst(.{ .tag = .cond_jmp_eq_ne, - .ops = (Mir.Ops(Register, Register){ - .flags = 0b01, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .flags = 0b01 }), .data = .{ .inst = undefined }, }); }, @@ -4918,9 +4783,7 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void { try self.genBody(body); _ = try self.addInst(.{ .tag = .jmp, - .ops = (Mir.Ops(Register, Register){ - .flags = 0b00, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{}), .data = .{ .inst = jmp_target }, }); return self.finishAirBookkeeping(); @@ -4971,19 +4834,17 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u .immediate => |imm| { _ = try self.addInst(.{ .tag = .xor, - .ops = (Mir.Ops(Register, Register){ - .reg1 = registerAlias(cond_reg, abi_size), - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(cond_reg, abi_size) }), .data = .{ .imm = @intCast(u32, imm) }, }); }, .register => |reg| { _ = try self.addInst(.{ .tag = .xor, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(cond_reg, abi_size), .reg2 = registerAlias(reg, abi_size), - }).encode(), + }), .data = undefined, }); }, @@ -5002,17 +4863,15 @@ fn genCondSwitchMir(self: *Self, ty: Type, condition: MCValue, case: MCValue) !u _ = try self.addInst(.{ .tag = .@"test", - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(cond_reg, abi_size), .reg2 = registerAlias(cond_reg, abi_size), - }).encode(), + }), .data = undefined, }); return self.addInst(.{ .tag = .cond_jmp_eq_ne, - .ops = (Mir.Ops(Register, Register){ - .flags = 0b00, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{}), .data = .{ .inst = undefined }, }); }, @@ -5156,17 +5015,20 @@ fn br(self: *Self, block: Air.Inst.Index, operand: Air.Inst.Ref) !void { if (block_mcv == .none) { block_data.mcv = switch (operand_mcv) { .none, .dead, .unreach => unreachable, - .register, .stack_offset, .memory => operand_mcv, + .stack_offset, .memory => operand_mcv, .compare_flags_signed, .compare_flags_unsigned, .immediate => blk: { const new_mcv = try self.allocRegOrMem(block, true); try self.setRegOrMem(self.air.typeOfIndex(block), new_mcv, operand_mcv); break :blk new_mcv; }, - .avx_register => blk: { - // TODO not needed; return operand_mcv ones we can transfer between XMM registers - const new_mcv = try self.allocRegOrMem(block, false); - try self.setRegOrMem(self.air.typeOfIndex(block), new_mcv, operand_mcv); - break :blk new_mcv; + .register => blk: { + if (self.air.typeOfIndex(block).zigTypeTag() == .Float) { + // TODO not needed; return operand_mcv ones we can transfer between XMM registers + const new_mcv = try self.allocRegOrMem(block, false); + try self.setRegOrMem(self.air.typeOfIndex(block), new_mcv, operand_mcv); + break :blk new_mcv; + } + break :blk operand_mcv; }, else => return self.fail("TODO implement block_data.mcv = operand_mcv for {}", .{operand_mcv}), }; @@ -5184,9 +5046,7 @@ fn brVoid(self: *Self, block: Air.Inst.Index) !void { // Leave the jump offset undefined const jmp_reloc = try self.addInst(.{ .tag = .jmp, - .ops = (Mir.Ops(Register, Register){ - .flags = 0b00, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{}), .data = .{ .inst = undefined }, }); block_data.relocs.appendAssumeCapacity(jmp_reloc); @@ -5274,9 +5134,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { }; _ = try self.addInst(.{ .tag = .push, - .ops = (Mir.Ops(Register, Register){ - .flags = 0b10, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .flags = 0b10 }), .data = .{ .imm = n }, }); } else if (mem.indexOf(u8, arg, "%%")) |l| { @@ -5285,9 +5143,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { return self.fail("unrecognized register: '{s}'", .{reg_name}); _ = try self.addInst(.{ .tag = .push, - .ops = (Mir.Ops(Register, Register){ - .reg1 = reg, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .reg1 = reg }), .data = undefined, }); } else return self.fail("TODO more push operands", .{}); @@ -5299,9 +5155,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { return self.fail("unrecognized register: '{s}'", .{reg_name}); _ = try self.addInst(.{ .tag = .pop, - .ops = (Mir.Ops(Register, Register){ - .reg1 = reg, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .reg1 = reg }), .data = undefined, }); } else return self.fail("TODO more pop operands", .{}); @@ -5365,7 +5219,6 @@ fn setRegOrMem(self: *Self, ty: Type, loc: MCValue, val: MCValue) !void { .none => return, .immediate => unreachable, .register => |reg| return self.genSetReg(ty, reg, val), - .avx_register => |reg| return self.genSetAvxReg(ty, reg, val), .stack_offset => |off| return self.genSetStack(ty, off, val, .{}), .memory => { return self.fail("TODO implement setRegOrMem for memory", .{}); @@ -5396,9 +5249,6 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE .register_overflow_unsigned, .register_overflow_signed, => return self.fail("TODO genSetStackArg for register with overflow bit", .{}), - .avx_register => { - return self.fail("TODO genSetStackArg for AVX register", .{}); - }, .compare_flags_unsigned, .compare_flags_signed, => { @@ -5417,7 +5267,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE }); _ = try self.addInst(.{ .tag = .mov_mem_imm, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp, .flags = switch (abi_size) { 1 => 0b00, @@ -5425,7 +5275,7 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE 4 => 0b10, else => unreachable, }, - }).encode(), + }), .data = .{ .payload = payload }, }); }, @@ -5453,11 +5303,11 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE .register => |reg| { _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rsp, .reg2 = registerAlias(reg, @intCast(u32, abi_size)), .flags = 0b10, - }).encode(), + }), .data = .{ .imm = @bitCast(u32, -stack_offset) }, }); }, @@ -5520,10 +5370,10 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl }; _ = try self.addInst(.{ .tag = .cond_set_byte_overflow, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = tmp_reg.to8(), .flags = flags, - }).encode(), + }), .data = undefined, }); @@ -5550,7 +5400,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl }); _ = try self.addInst(.{ .tag = .mov_mem_imm, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = base_reg, .flags = switch (abi_size) { 1 => 0b00, @@ -5558,7 +5408,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl 4 => 0b10, else => unreachable, }, - }).encode(), + }), .data = .{ .payload = payload }, }); }, @@ -5572,10 +5422,10 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl }); _ = try self.addInst(.{ .tag = .mov_mem_imm, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = base_reg, .flags = 0b10, - }).encode(), + }), .data = .{ .payload = payload }, }); } @@ -5586,10 +5436,10 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl }); _ = try self.addInst(.{ .tag = .mov_mem_imm, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = base_reg, .flags = 0b10, - }).encode(), + }), .data = .{ .payload = payload }, }); } @@ -5605,65 +5455,64 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl } const base_reg = opts.dest_stack_base orelse .rbp; - if (!math.isPowerOfTwo(abi_size)) { - const reg_lock = self.register_manager.lockReg(reg); - defer if (reg_lock) |lock| self.register_manager.unlockReg(lock); - const tmp_reg = try self.copyToTmpRegister(ty, mcv); - - var next_offset = stack_offset; - var remainder = abi_size; - while (remainder > 0) { - const nearest_power_of_two = @as(u6, 1) << math.log2_int(u3, @intCast(u3, remainder)); - - _ = try self.addInst(.{ - .tag = .mov, - .ops = (Mir.Ops(Register, Register){ - .reg1 = base_reg, - .reg2 = registerAlias(tmp_reg, nearest_power_of_two), - .flags = 0b10, - }).encode(), - .data = .{ .imm = @bitCast(u32, -next_offset) }, - }); - - if (nearest_power_of_two > 1) { - try self.genShiftBinOpMir(.shr, ty, tmp_reg, .{ .immediate = nearest_power_of_two * 8 }); - } - - remainder -= nearest_power_of_two; - next_offset -= nearest_power_of_two; - } - } else { - _ = try self.addInst(.{ - .tag = .mov, - .ops = (Mir.Ops(Register, Register){ - .reg1 = base_reg, - .reg2 = registerAlias(reg, @intCast(u32, abi_size)), - .flags = 0b10, - }).encode(), - .data = .{ .imm = @bitCast(u32, -stack_offset) }, - }); - } - }, - .avx_register => |reg| { - const base_reg = opts.dest_stack_base orelse .rbp; switch (ty.zigTypeTag()) { .Float => switch (ty.tag()) { - .f32 => return self.fail("TODO genSetStack for AVX register for f32", .{}), + .f32 => return self.fail("TODO genSetStack for register for f32", .{}), .f64 => { _ = try self.addInst(.{ .tag = .mov_f64, - .ops = (Mir.Ops(Register, AvxRegister){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = base_reg, .reg2 = reg.to128(), .flags = 0b01, - }).encode(), + }), .data = .{ .imm = @bitCast(u32, -stack_offset) }, }); }, - else => return self.fail("TODO genSetStack for AVX register for type {}", .{ty.fmtDebug()}), + else => return self.fail("TODO genSetStack for register for type {}", .{ty.fmtDebug()}), + }, + else => { + if (!math.isPowerOfTwo(abi_size)) { + const reg_lock = self.register_manager.lockReg(reg); + defer if (reg_lock) |lock| self.register_manager.unlockReg(lock); + + const tmp_reg = try self.copyToTmpRegister(ty, mcv); + + var next_offset = stack_offset; + var remainder = abi_size; + while (remainder > 0) { + const nearest_power_of_two = @as(u6, 1) << math.log2_int(u3, @intCast(u3, remainder)); + + _ = try self.addInst(.{ + .tag = .mov, + .ops = Mir.Inst.Ops.encode(.{ + .reg1 = base_reg, + .reg2 = registerAlias(tmp_reg, nearest_power_of_two), + .flags = 0b10, + }), + .data = .{ .imm = @bitCast(u32, -next_offset) }, + }); + + if (nearest_power_of_two > 1) { + try self.genShiftBinOpMir(.shr, ty, tmp_reg, .{ .immediate = nearest_power_of_two * 8 }); + } + + remainder -= nearest_power_of_two; + next_offset -= nearest_power_of_two; + } + } else { + _ = try self.addInst(.{ + .tag = .mov, + .ops = Mir.Inst.Ops.encode(.{ + .reg1 = base_reg, + .reg2 = registerAlias(reg, @intCast(u32, abi_size)), + .flags = 0b10, + }), + .data = .{ .imm = @bitCast(u32, -stack_offset) }, + }); + } }, - else => return self.fail("TODO genSetStack for AVX register for type {}", .{ty.fmtDebug()}), } }, .memory, @@ -5742,20 +5591,20 @@ fn genInlineMemcpy( .ptr_stack_offset, .stack_offset => |off| { _ = try self.addInst(.{ .tag = .lea, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = dst_addr_reg.to64(), .reg2 = opts.dest_stack_base orelse .rbp, - }).encode(), + }), .data = .{ .imm = @bitCast(u32, -off) }, }); }, .register => |reg| { _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(dst_addr_reg, @divExact(reg.size(), 8)), .reg2 = reg, - }).encode(), + }), .data = undefined, }); }, @@ -5777,20 +5626,20 @@ fn genInlineMemcpy( .ptr_stack_offset, .stack_offset => |off| { _ = try self.addInst(.{ .tag = .lea, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = src_addr_reg.to64(), .reg2 = opts.source_stack_base orelse .rbp, - }).encode(), + }), .data = .{ .imm = @bitCast(u32, -off) }, }); }, .register => |reg| { _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(src_addr_reg, @divExact(reg.size(), 8)), .reg2 = reg, - }).encode(), + }), .data = undefined, }); }, @@ -5810,18 +5659,14 @@ fn genInlineMemcpy( // mov rcx, 0 _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops(Register, Register){ - .reg1 = .rcx, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rcx }), .data = .{ .imm = 0 }, }); // mov rax, 0 _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops(Register, Register){ - .reg1 = .rax, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rax }), .data = .{ .imm = 0 }, }); @@ -5829,70 +5674,62 @@ fn genInlineMemcpy( // cmp count, 0 const loop_start = try self.addInst(.{ .tag = .cmp, - .ops = (Mir.Ops(Register, Register){ - .reg1 = count_reg, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .reg1 = count_reg }), .data = .{ .imm = 0 }, }); // je end const loop_reloc = try self.addInst(.{ .tag = .cond_jmp_eq_ne, - .ops = (Mir.Ops(Register, Register){ .flags = 0b01 }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .flags = 0b01 }), .data = .{ .inst = undefined }, }); // mov tmp, [addr + rcx] _ = try self.addInst(.{ .tag = .mov_scale_src, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = tmp_reg.to8(), .reg2 = src_addr_reg, - }).encode(), + }), .data = .{ .imm = 0 }, }); // mov [stack_offset + rax], tmp _ = try self.addInst(.{ .tag = .mov_scale_dst, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = dst_addr_reg, .reg2 = tmp_reg.to8(), - }).encode(), + }), .data = .{ .imm = 0 }, }); // add rcx, 1 _ = try self.addInst(.{ .tag = .add, - .ops = (Mir.Ops(Register, Register){ - .reg1 = .rcx, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rcx }), .data = .{ .imm = 1 }, }); // add rax, 1 _ = try self.addInst(.{ .tag = .add, - .ops = (Mir.Ops(Register, Register){ - .reg1 = .rax, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rax }), .data = .{ .imm = 1 }, }); // sub count, 1 _ = try self.addInst(.{ .tag = .sub, - .ops = (Mir.Ops(Register, Register){ - .reg1 = count_reg, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .reg1 = count_reg }), .data = .{ .imm = 1 }, }); // jmp loop _ = try self.addInst(.{ .tag = .jmp, - .ops = (Mir.Ops(Register, Register){ .flags = 0b00 }).encode(), + .ops = Mir.Inst.Ops.encode(.{}), .data = .{ .inst = loop_start }, }); @@ -5924,20 +5761,20 @@ fn genInlineMemset( .ptr_stack_offset, .stack_offset => |off| { _ = try self.addInst(.{ .tag = .lea, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = addr_reg.to64(), .reg2 = opts.dest_stack_base orelse .rbp, - }).encode(), + }), .data = .{ .imm = @bitCast(u32, -off) }, }); }, .register => |reg| { _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(addr_reg, @divExact(reg.size(), 8)), .reg2 = reg, - }).encode(), + }), .data = undefined, }); }, @@ -5955,16 +5792,14 @@ fn genInlineMemset( // cmp rax, -1 const loop_start = try self.addInst(.{ .tag = .cmp, - .ops = (Mir.Ops(Register, Register){ - .reg1 = .rax, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rax }), .data = .{ .imm = @bitCast(u32, @as(i32, -1)) }, }); // je end const loop_reloc = try self.addInst(.{ .tag = .cond_jmp_eq_ne, - .ops = (Mir.Ops(Register, Register){ .flags = 0b01 }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .flags = 0b01 }), .data = .{ .inst = undefined }, }); @@ -5980,9 +5815,7 @@ fn genInlineMemset( }); _ = try self.addInst(.{ .tag = .mov_mem_index_imm, - .ops = (Mir.Ops(Register, Register){ - .reg1 = addr_reg, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .reg1 = addr_reg }), .data = .{ .payload = payload }, }); }, @@ -5992,16 +5825,14 @@ fn genInlineMemset( // sub rax, 1 _ = try self.addInst(.{ .tag = .sub, - .ops = (Mir.Ops(Register, Register){ - .reg1 = .rax, - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .reg1 = .rax }), .data = .{ .imm = 1 }, }); // jmp loop _ = try self.addInst(.{ .tag = .jmp, - .ops = (Mir.Ops(Register, Register){ .flags = 0b00 }).encode(), + .ops = Mir.Inst.Ops.encode(.{}), .data = .{ .inst = loop_start }, }); @@ -6009,88 +5840,6 @@ fn genInlineMemset( try self.performReloc(loop_reloc); } -fn genSetAvxReg(self: *Self, ty: Type, reg: AvxRegister, mcv: MCValue) InnerError!void { - switch (mcv) { - .dead => unreachable, - .register_overflow_unsigned, - .register_overflow_signed, - => unreachable, - .stack_offset => |off| { - if (off < std.math.minInt(i32) or off > std.math.maxInt(i32)) { - return self.fail("stack offset too large", .{}); - } - - switch (ty.zigTypeTag()) { - .Float => { - switch (ty.tag()) { - .f32 => return self.fail("TODO genSetAvxReg from stack offset for f32", .{}), - .f64 => { - _ = try self.addInst(.{ - .tag = .mov_f64, - .ops = (Mir.Ops(AvxRegister, Register){ - .reg1 = reg.to128(), - .reg2 = .rbp, - }).encode(), - .data = .{ .imm = @bitCast(u32, -off) }, - }); - }, - else => return self.fail("TODO genSetAvxReg from stack offset for {}", .{ty.fmtDebug()}), - } - }, - else => return self.fail("TODO genSetAvxReg from stack offset for type {}", .{ty.fmtDebug()}), - } - }, - .avx_register => |src_reg| { - switch (ty.zigTypeTag()) { - .Float => { - switch (ty.tag()) { - .f32 => return self.fail("TODO genSetAvxReg from register for f32", .{}), - .f64 => { - _ = try self.addInst(.{ - .tag = .mov_f64, - .ops = (Mir.Ops(AvxRegister, AvxRegister){ - .reg1 = reg.to128(), - .reg2 = src_reg.to128(), - .flags = 0b10, - }).encode(), - .data = undefined, - }); - }, - else => return self.fail("TODO genSetAvxReg from register for {}", .{ty.fmtDebug()}), - } - }, - else => return self.fail("TODO genSetAvxReg from register for type {}", .{ty.fmtDebug()}), - } - }, - .memory => { - switch (ty.zigTypeTag()) { - .Float => { - switch (ty.tag()) { - .f32 => return self.fail("TODO genSetAvxReg from memory for f32", .{}), - .f64 => { - const base_reg = try self.register_manager.allocReg(null); - try self.loadMemPtrIntoRegister(base_reg, Type.usize, mcv); - _ = try self.addInst(.{ - .tag = .mov_f64, - .ops = (Mir.Ops(AvxRegister, Register){ - .reg1 = reg.to128(), - .reg2 = base_reg.to64(), - }).encode(), - .data = .{ .imm = 0 }, - }); - }, - else => return self.fail("TODO genSetAvxReg from memory for {}", .{ty.fmtDebug()}), - } - }, - else => return self.fail("TODO genSetAvxReg from memory for type {}", .{ty.fmtDebug()}), - } - }, - else => |other| { - return self.fail("TODO genSetAvxReg from {}", .{other}); - }, - } -} - fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void { const abi_size = @intCast(u32, ty.abiSize(self.target.*)); switch (mcv) { @@ -6098,17 +5847,16 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void .register_overflow_unsigned, .register_overflow_signed, => unreachable, - .avx_register => unreachable, .ptr_stack_offset => |off| { if (off < std.math.minInt(i32) or off > std.math.maxInt(i32)) { return self.fail("stack offset too large", .{}); } _ = try self.addInst(.{ .tag = .lea, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(reg, abi_size), .reg2 = .rbp, - }).encode(), + }), .data = .{ .imm = @bitCast(u32, -off) }, }); }, @@ -6145,10 +5893,10 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void }; _ = try self.addInst(.{ .tag = tag, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = reg.to8(), .flags = flags, - }).encode(), + }), .data = undefined, }); }, @@ -6158,10 +5906,10 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void if (x == 0) { _ = try self.addInst(.{ .tag = .xor, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = reg.to32(), .reg2 = reg.to32(), - }).encode(), + }), .data = undefined, }); return; @@ -6170,9 +5918,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void // Next best case: if we set the lower four bytes, the upper four will be zeroed. _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops(Register, Register){ - .reg1 = registerAlias(reg, abi_size), - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(reg, abi_size) }), .data = .{ .imm = @truncate(u32, x) }, }); return; @@ -6187,9 +5933,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void const payload = try self.addExtra(Mir.Imm64.encode(x)); _ = try self.addInst(.{ .tag = .movabs, - .ops = (Mir.Ops(Register, Register){ - .reg1 = reg.to64(), - }).encode(), + .ops = Mir.Inst.Ops.encode(.{ .reg1 = reg.to64() }), .data = .{ .payload = payload }, }); }, @@ -6198,40 +5942,60 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void if (src_reg.id() == reg.id()) return; - if (ty.zigTypeTag() == .Int) blk: { - switch (ty.intInfo(self.target.*).signedness) { + switch (ty.zigTypeTag()) { + .Int => switch (ty.intInfo(self.target.*).signedness) { .signed => { - if (abi_size > 4) break :blk; - _ = try self.addInst(.{ - .tag = .mov_sign_extend, - .ops = (Mir.Ops(Register, Register){ - .reg1 = reg.to64(), - .reg2 = registerAlias(src_reg, abi_size), - }).encode(), - .data = undefined, - }); + if (abi_size <= 4) { + _ = try self.addInst(.{ + .tag = .mov_sign_extend, + .ops = Mir.Inst.Ops.encode(.{ + .reg1 = reg.to64(), + .reg2 = registerAlias(src_reg, abi_size), + }), + .data = undefined, + }); + return; + } }, .unsigned => { - if (abi_size > 2) break :blk; + if (abi_size <= 2) { + _ = try self.addInst(.{ + .tag = .mov_zero_extend, + .ops = Mir.Inst.Ops.encode(.{ + .reg1 = reg.to64(), + .reg2 = registerAlias(src_reg, abi_size), + }), + .data = undefined, + }); + return; + } + }, + }, + .Float => switch (ty.tag()) { + .f32 => return self.fail("TODO genSetReg from register for f32", .{}), + .f64 => { _ = try self.addInst(.{ - .tag = .mov_zero_extend, - .ops = (Mir.Ops(Register, Register){ - .reg1 = reg.to64(), - .reg2 = registerAlias(src_reg, abi_size), - }).encode(), + .tag = .mov_f64, + .ops = Mir.Inst.Ops.encode(.{ + .reg1 = reg.to128(), + .reg2 = src_reg.to128(), + .flags = 0b10, + }), .data = undefined, }); + return; }, - } - return; + else => return self.fail("TODO genSetReg from register for {}", .{ty.fmtDebug()}), + }, + else => {}, } _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(reg, abi_size), .reg2 = registerAlias(src_reg, abi_size), - }).encode(), + }), .data = undefined, }); }, @@ -6241,107 +6005,148 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void try self.loadMemPtrIntoRegister(reg, Type.usize, mcv); _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(reg, abi_size), .reg2 = reg.to64(), .flags = 0b01, - }).encode(), + }), .data = .{ .imm = 0 }, }); }, - .memory => |x| { - if (x <= math.maxInt(i32)) { - // mov reg, [ds:imm32] - _ = try self.addInst(.{ - .tag = .mov, - .ops = (Mir.Ops(Register, Register){ - .reg1 = registerAlias(reg, abi_size), - .flags = 0b01, - }).encode(), - .data = .{ .imm = @truncate(u32, x) }, - }); - } else { - // If this is RAX, we can use a direct load. - // Otherwise, we need to load the address, then indirectly load the value. - if (reg.id() == 0) { - // movabs rax, ds:moffs64 - const payload = try self.addExtra(Mir.Imm64.encode(x)); - _ = try self.addInst(.{ - .tag = .movabs, - .ops = (Mir.Ops(Register, Register){ - .reg1 = .rax, - .flags = 0b01, // imm64 will become moffs64 - }).encode(), - .data = .{ .payload = payload }, - }); - } else { - // Rather than duplicate the logic used for the move, we just use a self-call with a new MCValue. - try self.genSetReg(ty, reg, MCValue{ .immediate = x }); - - // mov reg, [reg + 0x0] + .memory => |x| switch (ty.zigTypeTag()) { + .Float => { + switch (ty.tag()) { + .f32 => return self.fail("TODO genSetReg from memory for f32", .{}), + .f64 => { + const base_reg = try self.register_manager.allocReg(null); + try self.loadMemPtrIntoRegister(base_reg, Type.usize, mcv); + _ = try self.addInst(.{ + .tag = .mov_f64, + .ops = Mir.Inst.Ops.encode(.{ + .reg1 = reg.to128(), + .reg2 = base_reg.to64(), + }), + .data = .{ .imm = 0 }, + }); + }, + else => return self.fail("TODO genSetReg from memory for {}", .{ty.fmtDebug()}), + } + }, + else => { + if (x <= math.maxInt(i32)) { + // mov reg, [ds:imm32] _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(reg, abi_size), - .reg2 = reg.to64(), .flags = 0b01, - }).encode(), - .data = .{ .imm = 0 }, + }), + .data = .{ .imm = @truncate(u32, x) }, }); + } else { + // If this is RAX, we can use a direct load. + // Otherwise, we need to load the address, then indirectly load the value. + if (reg.id() == 0) { + // movabs rax, ds:moffs64 + const payload = try self.addExtra(Mir.Imm64.encode(x)); + _ = try self.addInst(.{ + .tag = .movabs, + .ops = Mir.Inst.Ops.encode(.{ + .reg1 = .rax, + .flags = 0b01, // imm64 will become moffs64 + }), + .data = .{ .payload = payload }, + }); + } else { + // Rather than duplicate the logic used for the move, we just use a self-call with a new MCValue. + try self.genSetReg(ty, reg, MCValue{ .immediate = x }); + + // mov reg, [reg + 0x0] + _ = try self.addInst(.{ + .tag = .mov, + .ops = Mir.Inst.Ops.encode(.{ + .reg1 = registerAlias(reg, abi_size), + .reg2 = reg.to64(), + .flags = 0b01, + }), + .data = .{ .imm = 0 }, + }); + } } - } + }, }, .stack_offset => |off| { if (off < std.math.minInt(i32) or off > std.math.maxInt(i32)) { return self.fail("stack offset too large", .{}); } - if (ty.zigTypeTag() == .Int) blk: { - switch (ty.intInfo(self.target.*).signedness) { + switch (ty.zigTypeTag()) { + .Int => switch (ty.intInfo(self.target.*).signedness) { .signed => { - const flags: u2 = switch (abi_size) { - 1 => 0b01, - 2 => 0b10, - 4 => 0b11, - else => break :blk, - }; - _ = try self.addInst(.{ - .tag = .mov_sign_extend, - .ops = (Mir.Ops(Register, Register){ - .reg1 = reg.to64(), - .reg2 = .rbp, - .flags = flags, - }).encode(), - .data = .{ .imm = @bitCast(u32, -off) }, - }); + if (abi_size <= 4) { + const flags: u2 = switch (abi_size) { + 1 => 0b01, + 2 => 0b10, + 4 => 0b11, + else => unreachable, + }; + _ = try self.addInst(.{ + .tag = .mov_sign_extend, + .ops = Mir.Inst.Ops.encode(.{ + .reg1 = reg.to64(), + .reg2 = .rbp, + .flags = flags, + }), + .data = .{ .imm = @bitCast(u32, -off) }, + }); + return; + } }, .unsigned => { - const flags: u2 = switch (abi_size) { - 1 => 0b01, - 2 => 0b10, - else => break :blk, - }; + if (abi_size <= 2) { + const flags: u2 = switch (abi_size) { + 1 => 0b01, + 2 => 0b10, + else => unreachable, + }; + _ = try self.addInst(.{ + .tag = .mov_zero_extend, + .ops = Mir.Inst.Ops.encode(.{ + .reg1 = reg.to64(), + .reg2 = .rbp, + .flags = flags, + }), + .data = .{ .imm = @bitCast(u32, -off) }, + }); + return; + } + }, + }, + .Float => switch (ty.tag()) { + .f32 => return self.fail("TODO genSetReg from stack offset for f32", .{}), + .f64 => { _ = try self.addInst(.{ - .tag = .mov_zero_extend, - .ops = (Mir.Ops(Register, Register){ - .reg1 = reg.to64(), + .tag = .mov_f64, + .ops = Mir.Inst.Ops.encode(.{ + .reg1 = reg.to128(), .reg2 = .rbp, - .flags = flags, - }).encode(), + }), .data = .{ .imm = @bitCast(u32, -off) }, }); + return; }, - } - return; + else => return self.fail("TODO genSetReg from stack offset for {}", .{ty.fmtDebug()}), + }, + else => {}, } _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = registerAlias(reg, abi_size), .reg2 = .rbp, .flags = 0b01, - }).encode(), + }), .data = .{ .imm = @bitCast(u32, -off) }, }); }, @@ -6408,14 +6213,14 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void { }; _ = try self.addInst(.{ .tag = .fld, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ + .reg1 = .rbp, .flags = switch (src_ty.abiSize(self.target.*)) { 4 => 0b01, 8 => 0b10, else => |size| return self.fail("TODO load ST(0) with abiSize={}", .{size}), }, - .reg1 = .rbp, - }).encode(), + }), .data = .{ .imm = @bitCast(u32, -stack_offset) }, }); @@ -6423,15 +6228,15 @@ fn airFloatToInt(self: *Self, inst: Air.Inst.Index) !void { const stack_dst = try self.allocRegOrMem(inst, false); _ = try self.addInst(.{ .tag = .fisttp, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ + .reg1 = .rbp, .flags = switch (dst_ty.abiSize(self.target.*)) { 1...2 => 0b00, 3...4 => 0b01, 5...8 => 0b10, else => |size| return self.fail("TODO convert float with abiSize={}", .{size}), }, - .reg1 = .rbp, - }).encode(), + }), .data = .{ .imm = @bitCast(u32, -stack_dst.stack_offset) }, }); @@ -6527,11 +6332,11 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void { try self.loadMemPtrIntoRegister(reg, src_ty, src_ptr); _ = try self.addInst(.{ .tag = .mov, - .ops = (Mir.Ops(Register, Register){ + .ops = Mir.Inst.Ops.encode(.{ .reg1 = reg, .reg2 = reg, .flags = 0b01, - }).encode(), + }), .data = .{ .imm = 0 }, }); break :blk MCValue{ .register = reg }; @@ -7095,22 +6900,11 @@ fn registerAlias(reg: Register, size_bytes: u32) Register { return reg.to32(); } else if (size_bytes <= 8) { return reg.to64(); - } else { - unreachable; - } -} - -/// Returns AVX register wide enough to hold at least `size_bytes`. -fn avxRegisterAlias(reg: AvxRegister, size_bytes: u32) AvxRegister { - if (size_bytes == 0) { - unreachable; // should be comptime known } else if (size_bytes <= 16) { return reg.to128(); } else if (size_bytes <= 32) { return reg.to256(); - } else { - unreachable; - } + } else unreachable; } /// Truncates the value in the register in place. diff --git a/src/arch/x86_64/Emit.zig b/src/arch/x86_64/Emit.zig index 79341df8cd..5ad8e86374 100644 --- a/src/arch/x86_64/Emit.zig +++ b/src/arch/x86_64/Emit.zig @@ -25,9 +25,8 @@ const MCValue = @import("CodeGen.zig").MCValue; const Mir = @import("Mir.zig"); const Module = @import("../../Module.zig"); const Instruction = bits.Instruction; -const GpRegister = bits.Register; -const AvxRegister = bits.AvxRegister; const Type = @import("../../type.zig").Type; +const Register = bits.Register; mir: Mir, bin_file: *link.File, @@ -238,7 +237,7 @@ fn fixupRelocs(emit: *Emit) InnerError!void { fn mirInterrupt(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const tag = emit.mir.instructions.items(.tag)[inst]; assert(tag == .interrupt); - const ops = Mir.Ops.decode(emit.mir.instructions.items(.ops)[inst]); + const ops = emit.mir.instructions.items(.ops)[inst].decode(); switch (ops.flags) { 0b00 => return lowerToZoEnc(.int3, emit.code), else => return emit.fail("TODO handle variant 0b{b} of interrupt instruction", .{ops.flags}), @@ -254,11 +253,11 @@ fn mirSyscall(emit: *Emit) InnerError!void { } fn mirPushPop(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { - const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); + const ops = emit.mir.instructions.items(.ops)[inst].decode(); switch (ops.flags) { 0b00 => { // PUSH/POP reg - return lowerToOEnc(tag, Register.reg(ops.reg1), emit.code); + return lowerToOEnc(tag, ops.reg1, emit.code); }, 0b01 => { // PUSH/POP r/m64 @@ -283,7 +282,7 @@ fn mirPushPop(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { } fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { - const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); + const ops = emit.mir.instructions.items(.ops)[inst].decode(); const payload = emit.mir.instructions.items(.data)[inst].payload; const data = emit.mir.extraData(Mir.RegsToPushOrPop, payload).data; const regs = data.regs; @@ -294,9 +293,9 @@ fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Tag, inst: Mir.Inst.I try lowerToMrEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{ .disp = @bitCast(u32, -@intCast(i32, disp)), .base = ops.reg1, - }), Register.reg(reg.to64()), emit.code); + }), reg.to64(), emit.code); } else { - try lowerToRmEnc(.mov, Register.reg(reg.to64()), RegisterOrMemory.mem(.qword_ptr, .{ + try lowerToRmEnc(.mov, reg.to64(), RegisterOrMemory.mem(.qword_ptr, .{ .disp = @bitCast(u32, -@intCast(i32, disp)), .base = ops.reg1, }), emit.code); @@ -306,7 +305,7 @@ fn mirPushPopRegsFromCalleePreservedRegs(emit: *Emit, tag: Tag, inst: Mir.Inst.I } fn mirJmpCall(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { - const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); + const ops = emit.mir.instructions.items(.ops)[inst].decode(); switch (ops.flags) { 0b00 => { const target = emit.mir.instructions.items(.data)[inst].inst; @@ -335,7 +334,7 @@ fn mirJmpCall(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { 0b10 => { // JMP/CALL r/m64 const imm = emit.mir.instructions.items(.data)[inst].imm; - return lowerToMEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{ + return lowerToMEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{ .disp = imm, .base = ops.reg1, }), emit.code); @@ -345,7 +344,7 @@ fn mirJmpCall(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { } fn mirCondJmp(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void { - const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); + const ops = emit.mir.instructions.items(.ops)[inst].decode(); const target = emit.mir.instructions.items(.data)[inst].inst; const tag = switch (mir_tag) { .cond_jmp_greater_less => switch (ops.flags) { @@ -377,7 +376,7 @@ fn mirCondJmp(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerErr } fn mirCondSetByte(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) InnerError!void { - const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); + const ops = emit.mir.instructions.items(.ops)[inst].decode(); const tag = switch (mir_tag) { .cond_set_byte_greater_less => switch (ops.flags) { 0b00 => Tag.setge, @@ -407,9 +406,9 @@ fn mirCondSetByte(emit: *Emit, mir_tag: Mir.Inst.Tag, inst: Mir.Inst.Index) Inne } fn mirCondMov(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { - const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); + const ops = emit.mir.instructions.items(.ops)[inst].decode(); if (ops.flags == 0b00) { - return lowerToRmEnc(tag, Register.reg(ops.reg1), RegisterOrMemory.reg(ops.reg2), emit.code); + return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code); } const imm = emit.mir.instructions.items(.data)[inst].imm; const ptr_size: Memory.PtrSize = switch (ops.flags) { @@ -418,7 +417,7 @@ fn mirCondMov(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { 0b10 => .dword_ptr, 0b11 => .qword_ptr, }; - return lowerToRmEnc(tag, Register.reg(ops.reg1), RegisterOrMemory.mem(ptr_size, .{ + return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(ptr_size, .{ .disp = imm, .base = ops.reg2, }), emit.code); @@ -427,7 +426,7 @@ fn mirCondMov(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const tag = emit.mir.instructions.items(.tag)[inst]; assert(tag == .@"test"); - const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); + const ops = emit.mir.instructions.items(.ops)[inst].decode(); switch (ops.flags) { 0b00 => { if (ops.reg2 == .none) { @@ -442,12 +441,7 @@ fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { return lowerToMiEnc(.@"test", RegisterOrMemory.reg(ops.reg1), imm, emit.code); } // TEST r/m64, r64 - return lowerToMrEnc( - .@"test", - RegisterOrMemory.reg(ops.reg1), - Register.reg(ops.reg2), - emit.code, - ); + return lowerToMrEnc(.@"test", RegisterOrMemory.reg(ops.reg1), ops.reg2, emit.code); }, else => return emit.fail("TODO more TEST alternatives", .{}), } @@ -456,7 +450,7 @@ fn mirTest(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { fn mirRet(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const tag = emit.mir.instructions.items(.tag)[inst]; assert(tag == .ret); - const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); + const ops = emit.mir.instructions.items(.ops)[inst].decode(); switch (ops.flags) { 0b00 => { // RETF imm16 @@ -480,7 +474,7 @@ fn mirRet(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { } fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { - const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); + const ops = emit.mir.instructions.items(.ops)[inst].decode(); switch (ops.flags) { 0b00 => { if (ops.reg2 == .none) { @@ -491,14 +485,14 @@ fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { } // mov reg1, reg2 // RM - return lowerToRmEnc(tag, Register.reg(ops.reg1), RegisterOrMemory.reg(ops.reg2), emit.code); + return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code); }, 0b01 => { // mov reg1, [reg2 + imm32] // RM const imm = emit.mir.instructions.items(.data)[inst].imm; - const src_reg: ?GpRegister = if (ops.reg2 == .none) null else ops.reg2; - return lowerToRmEnc(tag, Register.reg(ops.reg1), RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{ + const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null; + return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{ .disp = imm, .base = src_reg, }), emit.code); @@ -510,10 +504,10 @@ fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { // mov [reg1 + imm32], reg2 // MR const imm = emit.mir.instructions.items(.data)[inst].imm; - return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg2.size()), .{ + return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg2.size()), .{ .disp = imm, .base = ops.reg1, - }), Register.reg(ops.reg2), emit.code); + }), ops.reg2, emit.code); }, 0b11 => { return emit.fail("TODO unused variant: mov reg1, reg2, 0b11", .{}); @@ -522,7 +516,7 @@ fn mirArith(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { } fn mirArithMemImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { - const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); + const ops = emit.mir.instructions.items(.ops)[inst].decode(); assert(ops.reg2 == .none); const payload = emit.mir.instructions.items(.data)[inst].payload; const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data; @@ -539,19 +533,14 @@ fn mirArithMemImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { } inline fn setRexWRegister(reg: Register) bool { - switch (reg) { - .avx_register => return false, - .register => |r| { - if (r.size() == 64) return true; - return switch (r) { - .ah, .bh, .ch, .dh => true, - else => false, - }; - }, - } + if (reg.size() == 64) return true; + return switch (reg) { + .ah, .ch, .dh, .bh => true, + else => false, + }; } -inline fn immOpSize(u_imm: u32) u8 { +inline fn immOpSize(u_imm: u32) u6 { const imm = @bitCast(i32, u_imm); if (math.minInt(i8) <= imm and imm <= math.maxInt(i8)) { return 8; @@ -563,7 +552,7 @@ inline fn immOpSize(u_imm: u32) u8 { } fn mirArithScaleSrc(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { - const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); + const ops = emit.mir.instructions.items(.ops)[inst].decode(); const scale = ops.flags; const imm = emit.mir.instructions.items(.data)[inst].imm; // OP reg1, [reg2 + scale*rcx + imm32] @@ -571,7 +560,7 @@ fn mirArithScaleSrc(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void .scale = scale, .index = .rcx, }; - return lowerToRmEnc(tag, Register.reg(ops.reg1), RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{ + return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{ .disp = imm, .base = ops.reg2, .scale_index = scale_index, @@ -579,7 +568,7 @@ fn mirArithScaleSrc(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void } fn mirArithScaleDst(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { - const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); + const ops = emit.mir.instructions.items(.ops)[inst].decode(); const scale = ops.flags; const imm = emit.mir.instructions.items(.data)[inst].imm; const scale_index = ScaleIndex{ @@ -595,15 +584,15 @@ fn mirArithScaleDst(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void }), imm, emit.code); } // OP [reg1 + scale*rax + imm32], reg2 - return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg2.size()), .{ + return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg2.size()), .{ .disp = imm, .base = ops.reg1, .scale_index = scale_index, - }), Register.reg(ops.reg2), emit.code); + }), ops.reg2, emit.code); } fn mirArithScaleImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { - const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); + const ops = emit.mir.instructions.items(.ops)[inst].decode(); const scale = ops.flags; const payload = emit.mir.instructions.items(.data)[inst].payload; const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data; @@ -620,7 +609,7 @@ fn mirArithScaleImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void } fn mirArithMemIndexImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { - const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); + const ops = emit.mir.instructions.items(.ops)[inst].decode(); assert(ops.reg2 == .none); const payload = emit.mir.instructions.items(.data)[inst].payload; const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data; @@ -645,27 +634,27 @@ fn mirArithMemIndexImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!v fn mirMovSignExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const mir_tag = emit.mir.instructions.items(.tag)[inst]; assert(mir_tag == .mov_sign_extend); - const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); + const ops = emit.mir.instructions.items(.ops)[inst].decode(); const imm = if (ops.flags != 0b00) emit.mir.instructions.items(.data)[inst].imm else undefined; switch (ops.flags) { 0b00 => { const tag: Tag = if (ops.reg2.size() == 32) .movsxd else .movsx; - return lowerToRmEnc(tag, Register.reg(ops.reg1), RegisterOrMemory.reg(ops.reg2), emit.code); + return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code); }, 0b01 => { - return lowerToRmEnc(.movsx, Register.reg(ops.reg1), RegisterOrMemory.mem(.byte_ptr, .{ + return lowerToRmEnc(.movsx, ops.reg1, RegisterOrMemory.mem(.byte_ptr, .{ .disp = imm, .base = ops.reg2, }), emit.code); }, 0b10 => { - return lowerToRmEnc(.movsx, Register.reg(ops.reg1), RegisterOrMemory.mem(.word_ptr, .{ + return lowerToRmEnc(.movsx, ops.reg1, RegisterOrMemory.mem(.word_ptr, .{ .disp = imm, .base = ops.reg2, }), emit.code); }, 0b11 => { - return lowerToRmEnc(.movsxd, Register.reg(ops.reg1), RegisterOrMemory.mem(.dword_ptr, .{ + return lowerToRmEnc(.movsxd, ops.reg1, RegisterOrMemory.mem(.dword_ptr, .{ .disp = imm, .base = ops.reg2, }), emit.code); @@ -676,20 +665,20 @@ fn mirMovSignExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { fn mirMovZeroExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const mir_tag = emit.mir.instructions.items(.tag)[inst]; assert(mir_tag == .mov_zero_extend); - const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); + const ops = emit.mir.instructions.items(.ops)[inst].decode(); const imm = if (ops.flags != 0b00) emit.mir.instructions.items(.data)[inst].imm else undefined; switch (ops.flags) { 0b00 => { - return lowerToRmEnc(.movzx, Register.reg(ops.reg1), RegisterOrMemory.reg(ops.reg2), emit.code); + return lowerToRmEnc(.movzx, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code); }, 0b01 => { - return lowerToRmEnc(.movzx, Register.reg(ops.reg1), RegisterOrMemory.mem(.byte_ptr, .{ + return lowerToRmEnc(.movzx, ops.reg1, RegisterOrMemory.mem(.byte_ptr, .{ .disp = imm, .base = ops.reg2, }), emit.code); }, 0b10 => { - return lowerToRmEnc(.movzx, Register.reg(ops.reg1), RegisterOrMemory.mem(.word_ptr, .{ + return lowerToRmEnc(.movzx, ops.reg1, RegisterOrMemory.mem(.word_ptr, .{ .disp = imm, .base = ops.reg2, }), emit.code); @@ -703,31 +692,46 @@ fn mirMovZeroExtend(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { fn mirMovabs(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const tag = emit.mir.instructions.items(.tag)[inst]; assert(tag == .movabs); - const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); - const imm: u64 = if (ops.reg1.size() == 64) blk: { - const payload = emit.mir.instructions.items(.data)[inst].payload; - const imm = emit.mir.extraData(Mir.Imm64, payload).data; - break :blk imm.decode(); - } else emit.mir.instructions.items(.data)[inst].imm; - if (ops.flags == 0b00) { - // movabs reg, imm64 - // OI - return lowerToOiEnc(.mov, Register.reg(ops.reg1), imm, emit.code); + const ops = emit.mir.instructions.items(.ops)[inst].decode(); + switch (ops.flags) { + 0b00 => { + const imm: u64 = if (ops.reg1.size() == 64) blk: { + const payload = emit.mir.instructions.items(.data)[inst].payload; + const imm = emit.mir.extraData(Mir.Imm64, payload).data; + break :blk imm.decode(); + } else emit.mir.instructions.items(.data)[inst].imm; + // movabs reg, imm64 + // OI + return lowerToOiEnc(.mov, ops.reg1, imm, emit.code); + }, + 0b01 => { + if (ops.reg1 == .none) { + const imm: u64 = if (ops.reg2.size() == 64) blk: { + const payload = emit.mir.instructions.items(.data)[inst].payload; + const imm = emit.mir.extraData(Mir.Imm64, payload).data; + break :blk imm.decode(); + } else emit.mir.instructions.items(.data)[inst].imm; + // movabs moffs64, rax + // TD + return lowerToTdEnc(.mov, imm, ops.reg2, emit.code); + } + const imm: u64 = if (ops.reg1.size() == 64) blk: { + const payload = emit.mir.instructions.items(.data)[inst].payload; + const imm = emit.mir.extraData(Mir.Imm64, payload).data; + break :blk imm.decode(); + } else emit.mir.instructions.items(.data)[inst].imm; + // movabs rax, moffs64 + // FD + return lowerToFdEnc(.mov, ops.reg1, imm, emit.code); + }, + else => return emit.fail("TODO unused variant: movabs 0b{b}", .{ops.flags}), } - if (ops.reg1 == .none) { - // movabs moffs64, rax - // TD - return lowerToTdEnc(.mov, imm, Register.reg(ops.reg2), emit.code); - } - // movabs rax, moffs64 - // FD - return lowerToFdEnc(.mov, Register.reg(ops.reg1), imm, emit.code); } fn mirFisttp(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const tag = emit.mir.instructions.items(.tag)[inst]; assert(tag == .fisttp); - const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); + const ops = emit.mir.instructions.items(.ops)[inst].decode(); // the selecting between operand sizes for this particular `fisttp` instruction // is done via opcode instead of the usual prefixes. @@ -749,7 +753,7 @@ fn mirFisttp(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { fn mirFld(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const tag = emit.mir.instructions.items(.tag)[inst]; assert(tag == .fld); - const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); + const ops = emit.mir.instructions.items(.ops)[inst].decode(); // the selecting between operand sizes for this particular `fisttp` instruction // is done via opcode instead of the usual prefixes. @@ -768,7 +772,7 @@ fn mirFld(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { } fn mirShift(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { - const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); + const ops = emit.mir.instructions.items(.ops)[inst].decode(); switch (ops.flags) { 0b00 => { // sal reg1, 1 @@ -793,12 +797,11 @@ fn mirShift(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { } fn mirMulDiv(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { - const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); + const ops = emit.mir.instructions.items(.ops)[inst].decode(); if (ops.reg1 != .none) { assert(ops.reg2 == .none); return lowerToMEnc(tag, RegisterOrMemory.reg(ops.reg1), emit.code); } - assert(ops.reg1 == .none); assert(ops.reg2 != .none); const imm = emit.mir.instructions.items(.data)[inst].imm; const ptr_size: Memory.PtrSize = switch (ops.flags) { @@ -816,27 +819,27 @@ fn mirMulDiv(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { fn mirIMulComplex(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const tag = emit.mir.instructions.items(.tag)[inst]; assert(tag == .imul_complex); - const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); + const ops = emit.mir.instructions.items(.ops)[inst].decode(); switch (ops.flags) { 0b00 => { - return lowerToRmEnc(.imul, Register.reg(ops.reg1), RegisterOrMemory.reg(ops.reg2), emit.code); + return lowerToRmEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code); }, 0b01 => { const imm = emit.mir.instructions.items(.data)[inst].imm; - const src_reg: ?GpRegister = if (ops.reg2 == .none) null else ops.reg2; - return lowerToRmEnc(.imul, Register.reg(ops.reg1), RegisterOrMemory.mem(.qword_ptr, .{ + const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null; + return lowerToRmEnc(.imul, ops.reg1, RegisterOrMemory.mem(.qword_ptr, .{ .disp = imm, .base = src_reg, }), emit.code); }, 0b10 => { const imm = emit.mir.instructions.items(.data)[inst].imm; - return lowerToRmiEnc(.imul, Register.reg(ops.reg1), RegisterOrMemory.reg(ops.reg2), imm, emit.code); + return lowerToRmiEnc(.imul, ops.reg1, RegisterOrMemory.reg(ops.reg2), imm, emit.code); }, 0b11 => { const payload = emit.mir.instructions.items(.data)[inst].payload; const imm_pair = emit.mir.extraData(Mir.ImmPair, payload).data; - return lowerToRmiEnc(.imul, Register.reg(ops.reg1), RegisterOrMemory.mem(.qword_ptr, .{ + return lowerToRmiEnc(.imul, ops.reg1, RegisterOrMemory.mem(.qword_ptr, .{ .disp = imm_pair.dest_off, .base = ops.reg2, }), imm_pair.operand, emit.code); @@ -845,7 +848,7 @@ fn mirIMulComplex(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { } fn mirCwd(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { - const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); + const ops = emit.mir.instructions.items(.ops)[inst].decode(); const tag: Tag = switch (ops.flags) { 0b00 => .cbw, 0b01 => .cwd, @@ -858,17 +861,17 @@ fn mirCwd(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const tag = emit.mir.instructions.items(.tag)[inst]; assert(tag == .lea); - const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); + const ops = emit.mir.instructions.items(.ops)[inst].decode(); switch (ops.flags) { 0b00 => { // lea reg1, [reg2 + imm32] // RM const imm = emit.mir.instructions.items(.data)[inst].imm; - const src_reg: ?GpRegister = if (ops.reg2 == .none) null else ops.reg2; + const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null; return lowerToRmEnc( .lea, - Register.reg(ops.reg1), - RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{ + ops.reg1, + RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{ .disp = imm, .base = src_reg, }), @@ -881,8 +884,8 @@ fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const start_offset = emit.code.items.len; try lowerToRmEnc( .lea, - Register.reg(ops.reg1), - RegisterOrMemory.rip(Memory.PtrSize.fromBits(ops.reg1.size()), 0), + ops.reg1, + RegisterOrMemory.rip(Memory.PtrSize.new(ops.reg1.size()), 0), emit.code, ); const end_offset = emit.code.items.len; @@ -895,15 +898,15 @@ fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { 0b10 => { // lea reg, [rbp + rcx + imm32] const imm = emit.mir.instructions.items(.data)[inst].imm; - const src_reg: ?GpRegister = if (ops.reg2 == .none) null else ops.reg2; + const src_reg: ?Register = if (ops.reg2 != .none) ops.reg2 else null; const scale_index = ScaleIndex{ .scale = 0, .index = .rcx, }; return lowerToRmEnc( .lea, - Register.reg(ops.reg1), - RegisterOrMemory.mem(Memory.PtrSize.fromBits(ops.reg1.size()), .{ + ops.reg1, + RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{ .disp = imm, .base = src_reg, .scale_index = scale_index, @@ -918,15 +921,15 @@ fn mirLea(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { fn mirLeaPie(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const tag = emit.mir.instructions.items(.tag)[inst]; assert(tag == .lea_pie); - const ops = Mir.Ops(GpRegister, GpRegister).decode(emit.mir.instructions.items(.ops)[inst]); + const ops = emit.mir.instructions.items(.ops)[inst].decode(); const load_reloc = emit.mir.instructions.items(.data)[inst].load_reloc; // lea reg1, [rip + reloc] // RM try lowerToRmEnc( .lea, - Register.reg(ops.reg1), - RegisterOrMemory.rip(Memory.PtrSize.fromBits(ops.reg1.size()), 0), + ops.reg1, + RegisterOrMemory.rip(Memory.PtrSize.new(ops.reg1.size()), 0), emit.code, ); @@ -962,78 +965,70 @@ fn mirLeaPie(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { fn mirMovF64(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const tag = emit.mir.instructions.items(.tag)[inst]; assert(tag == .mov_f64); - const ops = emit.mir.instructions.items(.ops)[inst]; - const flags = @truncate(u2, ops); + const ops = emit.mir.instructions.items(.ops)[inst].decode(); - switch (flags) { + switch (ops.flags) { 0b00 => { - const decoded = Mir.Ops(AvxRegister, GpRegister).decode(ops); const imm = emit.mir.instructions.items(.data)[inst].imm; - return lowerToRmEnc(.vmovsd, Register.avxReg(decoded.reg1), RegisterOrMemory.mem(.qword_ptr, .{ + return lowerToVmEnc(.vmovsd, ops.reg1, RegisterOrMemory.mem(.qword_ptr, .{ .disp = imm, - .base = decoded.reg2, + .base = ops.reg2, }), emit.code); }, 0b01 => { - const decoded = Mir.Ops(GpRegister, AvxRegister).decode(ops); const imm = emit.mir.instructions.items(.data)[inst].imm; - return lowerToMrEnc(.vmovsd, RegisterOrMemory.mem(.qword_ptr, .{ + return lowerToMvEnc(.vmovsd, RegisterOrMemory.mem(.qword_ptr, .{ .disp = imm, - .base = decoded.reg1, - }), Register.avxReg(decoded.reg2), emit.code); + .base = ops.reg1, + }), ops.reg2, emit.code); }, 0b10 => { - const decoded = Mir.Ops(AvxRegister, AvxRegister).decode(ops); return lowerToRvmEnc( .vmovsd, - Register.avxReg(decoded.reg1), - Register.avxReg(decoded.reg1), - RegisterOrMemory.avxReg(decoded.reg2), + ops.reg1, + ops.reg1, + RegisterOrMemory.reg(ops.reg2), emit.code, ); }, - else => return emit.fail("TODO unused variant 0b{b} for mov_f64", .{flags}), + else => return emit.fail("TODO unused variant 0b{b} for mov_f64", .{ops.flags}), } } fn mirAddF64(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const tag = emit.mir.instructions.items(.tag)[inst]; assert(tag == .add_f64); - const ops = emit.mir.instructions.items(.ops)[inst]; - const flags = @truncate(u2, ops); + const ops = emit.mir.instructions.items(.ops)[inst].decode(); - switch (flags) { + switch (ops.flags) { 0b00 => { - const decoded = Mir.Ops(AvxRegister, AvxRegister).decode(ops); return lowerToRvmEnc( .vaddsd, - Register.avxReg(decoded.reg1), - Register.avxReg(decoded.reg1), - RegisterOrMemory.avxReg(decoded.reg2), + ops.reg1, + ops.reg1, + RegisterOrMemory.reg(ops.reg2), emit.code, ); }, - else => return emit.fail("TODO unused variant 0b{b} for mov_f64", .{flags}), + else => return emit.fail("TODO unused variant 0b{b} for mov_f64", .{ops.flags}), } } fn mirCmpF64(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { const tag = emit.mir.instructions.items(.tag)[inst]; assert(tag == .cmp_f64); - const ops = emit.mir.instructions.items(.ops)[inst]; - const flags = @truncate(u2, ops); + const ops = emit.mir.instructions.items(.ops)[inst].decode(); - switch (flags) { + switch (ops.flags) { 0b00 => { - const decoded = Mir.Ops(AvxRegister, AvxRegister).decode(ops); - return lowerToRmEnc( + return lowerToVmEnc( .vucomisd, - Register.avxReg(decoded.reg1), - RegisterOrMemory.avxReg(decoded.reg2), + ops.reg1, + RegisterOrMemory.reg(ops.reg2), emit.code, ); }, - else => return emit.fail("TODO unused variant 0b{b} for mov_f64", .{flags}), + else => return emit.fail("TODO unused variant 0b{b} for mov_f64", .{ops.flags}), } } @@ -1277,6 +1272,18 @@ const Tag = enum { vcmpsd, vucomisd, + fn isAvx(tag: Tag) bool { + return switch (tag) { + .vmovsd, + .vaddsd, + .vcmpsd, + .vucomisd, + => true, + + else => false, + }; + } + fn isSetCC(tag: Tag) bool { return switch (tag) { .seto, @@ -1361,6 +1368,12 @@ const Encoding = enum { /// OP r64, r/m64, imm32 rmi, + /// OP xmm1, xmm2/m64 + vm, + + /// OP m64, xmm1 + mv, + /// OP xmm1, xmm2, xmm3/m64 rvm, @@ -1389,7 +1402,7 @@ const OpCode = union(enum) { fn encodeWithReg(opc: OpCode, encoder: Encoder, reg: Register) void { assert(opc == .one_byte); - encoder.opcode_withReg(opc.one_byte, reg.lowId()); + encoder.opcode_withReg(opc.one_byte, reg.lowEnc()); } }; @@ -1536,6 +1549,15 @@ inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode { .imul => OpCode.oneByte(if (is_one_byte) 0x6b else 0x69), else => null, }, + .mv => return switch (tag) { + .vmovsd => OpCode.oneByte(0x11), + else => null, + }, + .vm => return switch (tag) { + .vmovsd => OpCode.oneByte(0x10), + .vucomisd => OpCode.oneByte(0x2e), + else => null, + }, .rvm => return switch (tag) { .vaddsd => OpCode.oneByte(0x58), .vmovsd => OpCode.oneByte(0x10), @@ -1647,11 +1669,11 @@ inline fn getVexPrefix(tag: Tag, enc: Encoding) ?VexPrefix { } = .none, } = blk: { switch (enc) { - .mr => switch (tag) { + .mv => switch (tag) { .vmovsd => break :blk .{ .lig = true, .simd_prefix = .p_f2, .wig = true }, else => return null, }, - .rm => switch (tag) { + .vm => switch (tag) { .vmovsd => break :blk .{ .lig = true, .simd_prefix = .p_f2, .wig = true }, .vucomisd => break :blk .{ .lig = true, .simd_prefix = .p_66, .wig = true }, else => return null, @@ -1697,61 +1719,50 @@ inline fn getVexPrefix(tag: Tag, enc: Encoding) ?VexPrefix { } }; } -const ScaleIndex = struct { +const ScaleIndex = packed struct { scale: u2, - index: GpRegister, + index: Register, }; const Memory = struct { - base: ?GpRegister, + base: ?Register, rip: bool = false, disp: u32, ptr_size: PtrSize, scale_index: ?ScaleIndex = null, - const PtrSize = enum { - byte_ptr, - word_ptr, - dword_ptr, - qword_ptr, + const PtrSize = enum(u2) { + byte_ptr = 0b00, + word_ptr = 0b01, + dword_ptr = 0b10, + qword_ptr = 0b11, - fn fromBits(in_bits: u64) PtrSize { - return switch (in_bits) { - 8 => .byte_ptr, - 16 => .word_ptr, - 32 => .dword_ptr, - 64 => .qword_ptr, - else => unreachable, - }; + fn new(bit_size: u64) PtrSize { + return @intToEnum(PtrSize, math.log2_int(u4, @intCast(u4, @divExact(bit_size, 8)))); } /// Returns size in bits. fn size(ptr_size: PtrSize) u64 { - return switch (ptr_size) { - .byte_ptr => 8, - .word_ptr => 16, - .dword_ptr => 32, - .qword_ptr => 64, - }; + return 8 * (math.powi(u8, 2, @enumToInt(ptr_size)) catch unreachable); } }; fn encode(mem_op: Memory, encoder: Encoder, operand: u3) void { if (mem_op.base) |base| { - const dst = base.lowId(); + const dst = base.lowEnc(); const src = operand; if (dst == 4 or mem_op.scale_index != null) { if (mem_op.disp == 0 and dst != 5) { encoder.modRm_SIBDisp0(src); if (mem_op.scale_index) |si| { - encoder.sib_scaleIndexBase(si.scale, si.index.lowId(), dst); + encoder.sib_scaleIndexBase(si.scale, si.index.lowEnc(), dst); } else { encoder.sib_base(dst); } } else if (immOpSize(mem_op.disp) == 8) { encoder.modRm_SIBDisp8(src); if (mem_op.scale_index) |si| { - encoder.sib_scaleIndexBaseDisp8(si.scale, si.index.lowId(), dst); + encoder.sib_scaleIndexBaseDisp8(si.scale, si.index.lowEnc(), dst); } else { encoder.sib_baseDisp8(dst); } @@ -1759,7 +1770,7 @@ const Memory = struct { } else { encoder.modRm_SIBDisp32(src); if (mem_op.scale_index) |si| { - encoder.sib_scaleIndexBaseDisp32(si.scale, si.index.lowId(), dst); + encoder.sib_scaleIndexBaseDisp32(si.scale, si.index.lowEnc(), dst); } else { encoder.sib_baseDisp32(dst); } @@ -1782,7 +1793,7 @@ const Memory = struct { } else { encoder.modRm_SIBDisp0(operand); if (mem_op.scale_index) |si| { - encoder.sib_scaleIndexDisp32(si.scale, si.index.lowId()); + encoder.sib_scaleIndexDisp32(si.scale, si.index.lowEnc()); } else { encoder.sib_disp32(); } @@ -1791,6 +1802,7 @@ const Memory = struct { } } + /// Returns size in bits. fn size(memory: Memory) u64 { return memory.ptr_size.size(); } @@ -1805,55 +1817,17 @@ fn encodeImm(encoder: Encoder, imm: u32, size: u64) void { } } -const Register = union(enum) { - register: GpRegister, - avx_register: AvxRegister, - - fn reg(register: GpRegister) Register { - return .{ .register = register }; - } - - fn avxReg(register: AvxRegister) Register { - return .{ .avx_register = register }; - } - - fn lowId(register: Register) u3 { - return switch (register) { - .register => |r| r.lowId(), - .avx_register => |r| r.lowId(), - }; - } - - fn size(register: Register) u64 { - return switch (register) { - .register => |r| r.size(), - .avx_register => |r| r.size(), - }; - } - - fn isExtended(register: Register) bool { - return switch (register) { - .register => |r| r.isExtended(), - .avx_register => |r| r.isExtended(), - }; - } -}; - const RegisterOrMemory = union(enum) { register: Register, memory: Memory, - fn reg(register: GpRegister) RegisterOrMemory { - return .{ .register = Register.reg(register) }; - } - - fn avxReg(register: AvxRegister) RegisterOrMemory { - return .{ .register = Register.avxReg(register) }; + fn reg(register: Register) RegisterOrMemory { + return .{ .register = register }; } fn mem(ptr_size: Memory.PtrSize, args: struct { disp: u32, - base: ?GpRegister = null, + base: ?Register = null, scale_index: ?ScaleIndex = null, }) RegisterOrMemory { return .{ @@ -1877,6 +1851,7 @@ const RegisterOrMemory = union(enum) { }; } + /// Returns size in bits. fn size(reg_or_mem: RegisterOrMemory) u64 { return switch (reg_or_mem) { .register => |reg| reg.size(), @@ -1886,6 +1861,7 @@ const RegisterOrMemory = union(enum) { }; fn lowerToZoEnc(tag: Tag, code: *std.ArrayList(u8)) InnerError!void { + assert(!tag.isAvx()); const opc = getOpCode(tag, .zo, false).?; const encoder = try Encoder.init(code, 2); switch (tag) { @@ -1900,6 +1876,7 @@ fn lowerToZoEnc(tag: Tag, code: *std.ArrayList(u8)) InnerError!void { } fn lowerToIEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) InnerError!void { + assert(!tag.isAvx()); if (tag == .ret_far or tag == .ret_near) { const encoder = try Encoder.init(code, 3); const opc = getOpCode(tag, .i, false).?; @@ -1917,6 +1894,7 @@ fn lowerToIEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) InnerError!void { } fn lowerToOEnc(tag: Tag, reg: Register, code: *std.ArrayList(u8)) InnerError!void { + assert(!tag.isAvx()); const opc = getOpCode(tag, .o, false).?; const encoder = try Encoder.init(code, 3); if (reg.size() == 16) { @@ -1930,6 +1908,7 @@ fn lowerToOEnc(tag: Tag, reg: Register, code: *std.ArrayList(u8)) InnerError!voi } fn lowerToDEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) InnerError!void { + assert(!tag.isAvx()); const opc = getOpCode(tag, .d, false).?; const encoder = try Encoder.init(code, 6); opc.encode(encoder); @@ -1937,6 +1916,7 @@ fn lowerToDEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) InnerError!void { } fn lowerToMxEnc(tag: Tag, reg_or_mem: RegisterOrMemory, enc: Encoding, code: *std.ArrayList(u8)) InnerError!void { + assert(!tag.isAvx()); const opc = getOpCode(tag, enc, reg_or_mem.size() == 8).?; const modrm_ext = getModRmExt(tag).?; switch (reg_or_mem) { @@ -1951,7 +1931,7 @@ fn lowerToMxEnc(tag: Tag, reg_or_mem: RegisterOrMemory, enc: Encoding, code: *st .b = reg.isExtended(), }); opc.encode(encoder); - encoder.modRm_direct(modrm_ext, reg.lowId()); + encoder.modRm_direct(modrm_ext, reg.lowEnc()); }, .memory => |mem_op| { const encoder = try Encoder.init(code, 8); @@ -1992,6 +1972,7 @@ fn lowerToFdEnc(tag: Tag, reg: Register, moffs: u64, code: *std.ArrayList(u8)) I } fn lowerToTdFdEnc(tag: Tag, reg: Register, moffs: u64, code: *std.ArrayList(u8), td: bool) InnerError!void { + assert(!tag.isAvx()); const opc = if (td) getOpCode(tag, .td, reg.size() == 8).? else @@ -2014,6 +1995,7 @@ fn lowerToTdFdEnc(tag: Tag, reg: Register, moffs: u64, code: *std.ArrayList(u8), } fn lowerToOiEnc(tag: Tag, reg: Register, imm: u64, code: *std.ArrayList(u8)) InnerError!void { + assert(!tag.isAvx()); const opc = getOpCode(tag, .oi, reg.size() == 8).?; const encoder = try Encoder.init(code, 10); if (reg.size() == 16) { @@ -2040,6 +2022,7 @@ fn lowerToMiXEnc( enc: Encoding, code: *std.ArrayList(u8), ) InnerError!void { + assert(!tag.isAvx()); const modrm_ext = getModRmExt(tag).?; const opc = getOpCode(tag, enc, reg_or_mem.size() == 8).?; switch (reg_or_mem) { @@ -2056,7 +2039,7 @@ fn lowerToMiXEnc( .b = dst_reg.isExtended(), }); opc.encode(encoder); - encoder.modRm_direct(modrm_ext, dst_reg.lowId()); + encoder.modRm_direct(modrm_ext, dst_reg.lowEnc()); encodeImm(encoder, imm, if (enc == .mi8) 8 else dst_reg.size()); }, .memory => |dst_mem| { @@ -2095,84 +2078,43 @@ fn lowerToRmEnc( reg_or_mem: RegisterOrMemory, code: *std.ArrayList(u8), ) InnerError!void { + assert(!tag.isAvx()); const opc = getOpCode(tag, .rm, reg.size() == 8 or reg_or_mem.size() == 8).?; switch (reg_or_mem) { .register => |src_reg| { - const encoder: Encoder = blk: { - switch (reg) { - .register => { - const encoder = try Encoder.init(code, 4); - if (reg.size() == 16) { - encoder.prefix16BitMode(); - } - encoder.rex(.{ - .w = setRexWRegister(reg) or setRexWRegister(src_reg), - .r = reg.isExtended(), - .b = src_reg.isExtended(), - }); - break :blk encoder; - }, - .avx_register => { - const encoder = try Encoder.init(code, 5); - var vex_prefix = getVexPrefix(tag, .rm).?; - const vex = &vex_prefix.prefix; - vex.rex(.{ - .r = reg.isExtended(), - .b = src_reg.isExtended(), - }); - encoder.vex(vex_prefix.prefix); - break :blk encoder; - }, - } - }; + const encoder = try Encoder.init(code, 5); + if (reg.size() == 16) { + encoder.prefix16BitMode(); + } + encoder.rex(.{ + .w = setRexWRegister(reg) or setRexWRegister(src_reg), + .r = reg.isExtended(), + .b = src_reg.isExtended(), + }); opc.encode(encoder); - encoder.modRm_direct(reg.lowId(), src_reg.lowId()); + encoder.modRm_direct(reg.lowEnc(), src_reg.lowEnc()); }, .memory => |src_mem| { - const encoder: Encoder = blk: { - switch (reg) { - .register => { - const encoder = try Encoder.init(code, 9); - if (reg.size() == 16) { - encoder.prefix16BitMode(); - } - if (src_mem.base) |base| { - // TODO handle 32-bit base register - requires prefix 0x67 - // Intel Manual, Vol 1, chapter 3.6 and 3.6.1 - encoder.rex(.{ - .w = setRexWRegister(reg), - .r = reg.isExtended(), - .b = base.isExtended(), - }); - } else { - encoder.rex(.{ - .w = setRexWRegister(reg), - .r = reg.isExtended(), - }); - } - break :blk encoder; - }, - .avx_register => { - const encoder = try Encoder.init(code, 10); - var vex_prefix = getVexPrefix(tag, .rm).?; - const vex = &vex_prefix.prefix; - if (src_mem.base) |base| { - vex.rex(.{ - .r = reg.isExtended(), - .b = base.isExtended(), - }); - } else { - vex.rex(.{ - .r = reg.isExtended(), - }); - } - encoder.vex(vex_prefix.prefix); - break :blk encoder; - }, - } - }; + const encoder = try Encoder.init(code, 9); + if (reg.size() == 16) { + encoder.prefix16BitMode(); + } + if (src_mem.base) |base| { + // TODO handle 32-bit base register - requires prefix 0x67 + // Intel Manual, Vol 1, chapter 3.6 and 3.6.1 + encoder.rex(.{ + .w = setRexWRegister(reg), + .r = reg.isExtended(), + .b = base.isExtended(), + }); + } else { + encoder.rex(.{ + .w = setRexWRegister(reg), + .r = reg.isExtended(), + }); + } opc.encode(encoder); - src_mem.encode(encoder, reg.lowId()); + src_mem.encode(encoder, reg.lowEnc()); }, } } @@ -2183,6 +2125,7 @@ fn lowerToMrEnc( reg: Register, code: *std.ArrayList(u8), ) InnerError!void { + assert(!tag.isAvx()); const opc = getOpCode(tag, .mr, reg.size() == 8 or reg_or_mem.size() == 8).?; switch (reg_or_mem) { .register => |dst_reg| { @@ -2196,53 +2139,27 @@ fn lowerToMrEnc( .b = dst_reg.isExtended(), }); opc.encode(encoder); - encoder.modRm_direct(reg.lowId(), dst_reg.lowId()); + encoder.modRm_direct(reg.lowEnc(), dst_reg.lowEnc()); }, .memory => |dst_mem| { - const encoder: Encoder = blk: { - switch (reg) { - .register => { - const encoder = try Encoder.init(code, 9); - if (reg.size() == 16) { - encoder.prefix16BitMode(); - } - if (dst_mem.base) |base| { - encoder.rex(.{ - .w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg), - .r = reg.isExtended(), - .b = base.isExtended(), - }); - } else { - encoder.rex(.{ - .w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg), - .r = reg.isExtended(), - }); - } - break :blk encoder; - }, - .avx_register => { - const encoder = try Encoder.init(code, 10); - var vex_prefix = getVexPrefix(tag, .mr).?; - const vex = &vex_prefix.prefix; - if (dst_mem.base) |base| { - vex.rex(.{ - .w = dst_mem.ptr_size == .qword_ptr, - .r = reg.isExtended(), - .b = base.isExtended(), - }); - } else { - vex.rex(.{ - .w = dst_mem.ptr_size == .qword_ptr, - .r = reg.isExtended(), - }); - } - encoder.vex(vex_prefix.prefix); - break :blk encoder; - }, - } - }; + const encoder = try Encoder.init(code, 9); + if (reg.size() == 16) { + encoder.prefix16BitMode(); + } + if (dst_mem.base) |base| { + encoder.rex(.{ + .w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg), + .r = reg.isExtended(), + .b = base.isExtended(), + }); + } else { + encoder.rex(.{ + .w = dst_mem.ptr_size == .qword_ptr or setRexWRegister(reg), + .r = reg.isExtended(), + }); + } opc.encode(encoder); - dst_mem.encode(encoder, reg.lowId()); + dst_mem.encode(encoder, reg.lowEnc()); }, } } @@ -2254,6 +2171,7 @@ fn lowerToRmiEnc( imm: u32, code: *std.ArrayList(u8), ) InnerError!void { + assert(!tag.isAvx()); const opc = getOpCode(tag, .rmi, false).?; const encoder = try Encoder.init(code, 13); if (reg.size() == 16) { @@ -2267,7 +2185,7 @@ fn lowerToRmiEnc( .b = src_reg.isExtended(), }); opc.encode(encoder); - encoder.modRm_direct(reg.lowId(), src_reg.lowId()); + encoder.modRm_direct(reg.lowEnc(), src_reg.lowEnc()); }, .memory => |src_mem| { if (src_mem.base) |base| { @@ -2285,12 +2203,94 @@ fn lowerToRmiEnc( }); } opc.encode(encoder); - src_mem.encode(encoder, reg.lowId()); + src_mem.encode(encoder, reg.lowEnc()); }, } encodeImm(encoder, imm, reg.size()); } +/// Also referred to as XM encoding in Intel manual. +fn lowerToVmEnc( + tag: Tag, + reg: Register, + reg_or_mem: RegisterOrMemory, + code: *std.ArrayList(u8), +) InnerError!void { + const opc = getOpCode(tag, .vm, false).?; + var vex_prefix = getVexPrefix(tag, .vm).?; + const vex = &vex_prefix.prefix; + switch (reg_or_mem) { + .register => |src_reg| { + const encoder = try Encoder.init(code, 5); + vex.rex(.{ + .r = reg.isExtended(), + .b = src_reg.isExtended(), + }); + encoder.vex(vex_prefix.prefix); + opc.encode(encoder); + encoder.modRm_direct(reg.lowEnc(), src_reg.lowEnc()); + }, + .memory => |src_mem| { + assert(src_mem.ptr_size == .qword_ptr); + const encoder = try Encoder.init(code, 10); + if (src_mem.base) |base| { + vex.rex(.{ + .r = reg.isExtended(), + .b = base.isExtended(), + }); + } else { + vex.rex(.{ + .r = reg.isExtended(), + }); + } + encoder.vex(vex_prefix.prefix); + opc.encode(encoder); + src_mem.encode(encoder, reg.lowEnc()); + }, + } +} + +/// Usually referred to as MR encoding with V/V in Intel manual. +fn lowerToMvEnc( + tag: Tag, + reg_or_mem: RegisterOrMemory, + reg: Register, + code: *std.ArrayList(u8), +) InnerError!void { + const opc = getOpCode(tag, .mv, false).?; + var vex_prefix = getVexPrefix(tag, .mv).?; + const vex = &vex_prefix.prefix; + switch (reg_or_mem) { + .register => |dst_reg| { + const encoder = try Encoder.init(code, 4); + vex.rex(.{ + .r = reg.isExtended(), + .b = dst_reg.isExtended(), + }); + encoder.vex(vex_prefix.prefix); + opc.encode(encoder); + encoder.modRm_direct(reg.lowEnc(), dst_reg.lowEnc()); + }, + .memory => |dst_mem| { + assert(dst_mem.ptr_size == .qword_ptr); + const encoder = try Encoder.init(code, 10); + if (dst_mem.base) |base| { + vex.rex(.{ + .r = reg.isExtended(), + .b = base.isExtended(), + }); + } else { + vex.rex(.{ + .r = reg.isExtended(), + }); + } + encoder.vex(vex_prefix.prefix); + opc.encode(encoder); + dst_mem.encode(encoder, reg.lowEnc()); + }, + } +} + fn lowerToRvmEnc( tag: Tag, reg1: Register, @@ -2305,7 +2305,7 @@ fn lowerToRvmEnc( .register => |reg3| { if (vex_prefix.reg) |vvvv| { switch (vvvv) { - .nds => vex.reg(reg2.avx_register.id()), + .nds => vex.reg(reg2.enc()), else => unreachable, // TODO } } @@ -2316,7 +2316,7 @@ fn lowerToRvmEnc( }); encoder.vex(vex_prefix.prefix); opc.encode(encoder); - encoder.modRm_direct(reg1.lowId(), reg3.lowId()); + encoder.modRm_direct(reg1.lowEnc(), reg3.lowEnc()); }, .memory => |dst_mem| { _ = dst_mem; @@ -2341,7 +2341,7 @@ fn lowerToRvmiEnc( .register => |reg3| { if (vex_prefix.reg) |vvvv| { switch (vvvv) { - .nds => vex.reg(reg2.avx_register.id()), + .nds => vex.reg(reg2.enc()), else => unreachable, // TODO } } @@ -2352,7 +2352,7 @@ fn lowerToRvmiEnc( }); encoder.vex(vex_prefix.prefix); opc.encode(encoder); - encoder.modRm_direct(reg1.lowId(), reg3.lowId()); + encoder.modRm_direct(reg1.lowEnc(), reg3.lowEnc()); break :blk encoder; }, .memory => |dst_mem| { @@ -2490,23 +2490,23 @@ test "lower MI encoding" { test "lower RM encoding" { var emit = TestEmit.init(); defer emit.deinit(); - try lowerToRmEnc(.mov, Register.reg(.rax), RegisterOrMemory.reg(.rbx), emit.code()); + try lowerToRmEnc(.mov, .rax, RegisterOrMemory.reg(.rbx), emit.code()); try expectEqualHexStrings("\x48\x8b\xc3", emit.lowered(), "mov rax, rbx"); - try lowerToRmEnc(.mov, Register.reg(.rax), RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0, .base = .r11 }), emit.code()); + try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0, .base = .r11 }), emit.code()); try expectEqualHexStrings("\x49\x8b\x03", emit.lowered(), "mov rax, qword ptr [r11 + 0]"); - try lowerToRmEnc(.add, Register.reg(.r11), RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10000000 }), emit.code()); + try lowerToRmEnc(.add, .r11, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10000000 }), emit.code()); try expectEqualHexStrings( "\x4C\x03\x1C\x25\x00\x00\x00\x10", emit.lowered(), "add r11, qword ptr [ds:0x10000000]", ); - try lowerToRmEnc(.add, Register.reg(.r12b), RegisterOrMemory.mem(.byte_ptr, .{ .disp = 0x10000000 }), emit.code()); + try lowerToRmEnc(.add, .r12b, RegisterOrMemory.mem(.byte_ptr, .{ .disp = 0x10000000 }), emit.code()); try expectEqualHexStrings( "\x44\x02\x24\x25\x00\x00\x00\x10", emit.lowered(), "add r11b, byte ptr [ds:0x10000000]", ); - try lowerToRmEnc(.sub, Register.reg(.r11), RegisterOrMemory.mem(.qword_ptr, .{ + try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10000000, .base = .r13, }), emit.code()); @@ -2515,7 +2515,7 @@ test "lower RM encoding" { emit.lowered(), "sub r11, qword ptr [r13 + 0x10000000]", ); - try lowerToRmEnc(.sub, Register.reg(.r11), RegisterOrMemory.mem(.qword_ptr, .{ + try lowerToRmEnc(.sub, .r11, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10000000, .base = .r12, }), emit.code()); @@ -2524,14 +2524,14 @@ test "lower RM encoding" { emit.lowered(), "sub r11, qword ptr [r12 + 0x10000000]", ); - try lowerToRmEnc(.mov, Register.reg(.rax), RegisterOrMemory.mem(.qword_ptr, .{ + try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{ .disp = @bitCast(u32, @as(i32, -4)), .base = .rbp, }), emit.code()); try expectEqualHexStrings("\x48\x8B\x45\xFC", emit.lowered(), "mov rax, qword ptr [rbp - 4]"); - try lowerToRmEnc(.lea, Register.reg(.rax), RegisterOrMemory.rip(.qword_ptr, 0x10), emit.code()); + try lowerToRmEnc(.lea, .rax, RegisterOrMemory.rip(.qword_ptr, 0x10), emit.code()); try expectEqualHexStrings("\x48\x8D\x05\x10\x00\x00\x00", emit.lowered(), "lea rax, [rip + 0x10]"); - try lowerToRmEnc(.mov, Register.reg(.rax), RegisterOrMemory.mem(.qword_ptr, .{ + try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{ .disp = @bitCast(u32, @as(i32, -8)), .base = .rbp, .scale_index = .{ @@ -2540,7 +2540,7 @@ test "lower RM encoding" { }, }), emit.code()); try expectEqualHexStrings("\x48\x8B\x44\x0D\xF8", emit.lowered(), "mov rax, qword ptr [rbp + rcx*1 - 8]"); - try lowerToRmEnc(.mov, Register.reg(.eax), RegisterOrMemory.mem(.dword_ptr, .{ + try lowerToRmEnc(.mov, .eax, RegisterOrMemory.mem(.dword_ptr, .{ .disp = @bitCast(u32, @as(i32, -4)), .base = .rbp, .scale_index = .{ @@ -2549,7 +2549,7 @@ test "lower RM encoding" { }, }), emit.code()); try expectEqualHexStrings("\x8B\x44\x95\xFC", emit.lowered(), "mov eax, dword ptr [rbp + rdx*4 - 4]"); - try lowerToRmEnc(.mov, Register.reg(.rax), RegisterOrMemory.mem(.qword_ptr, .{ + try lowerToRmEnc(.mov, .rax, RegisterOrMemory.mem(.qword_ptr, .{ .disp = @bitCast(u32, @as(i32, -8)), .base = .rbp, .scale_index = .{ @@ -2558,7 +2558,7 @@ test "lower RM encoding" { }, }), emit.code()); try expectEqualHexStrings("\x48\x8B\x44\xCD\xF8", emit.lowered(), "mov rax, qword ptr [rbp + rcx*8 - 8]"); - try lowerToRmEnc(.mov, Register.reg(.r8b), RegisterOrMemory.mem(.byte_ptr, .{ + try lowerToRmEnc(.mov, .r8b, RegisterOrMemory.mem(.byte_ptr, .{ .disp = @bitCast(u32, @as(i32, -24)), .base = .rsi, .scale_index = .{ @@ -2567,7 +2567,7 @@ test "lower RM encoding" { }, }), emit.code()); try expectEqualHexStrings("\x44\x8A\x44\x0E\xE8", emit.lowered(), "mov r8b, byte ptr [rsi + rcx*1 - 24]"); - try lowerToRmEnc(.lea, Register.reg(.rsi), RegisterOrMemory.mem(.qword_ptr, .{ + try lowerToRmEnc(.lea, .rsi, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0, .base = .rbp, .scale_index = .{ @@ -2576,33 +2576,25 @@ test "lower RM encoding" { }, }), emit.code()); try expectEqualHexStrings("\x48\x8D\x74\x0D\x00", emit.lowered(), "lea rsi, qword ptr [rbp + rcx*1 + 0]"); - - // AVX extension tests - try lowerToRmEnc(.vmovsd, Register.avxReg(.xmm1), RegisterOrMemory.rip(.qword_ptr, 0x10), emit.code()); - try expectEqualHexStrings( - "\xC5\xFB\x10\x0D\x10\x00\x00\x00", - emit.lowered(), - "vmovsd xmm1, qword ptr [rip + 0x10]", - ); } test "lower MR encoding" { var emit = TestEmit.init(); defer emit.deinit(); - try lowerToMrEnc(.mov, RegisterOrMemory.reg(.rax), Register.reg(.rbx), emit.code()); + try lowerToMrEnc(.mov, RegisterOrMemory.reg(.rax), .rbx, emit.code()); try expectEqualHexStrings("\x48\x89\xd8", emit.lowered(), "mov rax, rbx"); try lowerToMrEnc(.mov, RegisterOrMemory.mem(.qword_ptr, .{ .disp = @bitCast(u32, @as(i32, -4)), .base = .rbp, - }), Register.reg(.r11), emit.code()); + }), .r11, emit.code()); try expectEqualHexStrings("\x4c\x89\x5d\xfc", emit.lowered(), "mov qword ptr [rbp - 4], r11"); - try lowerToMrEnc(.add, RegisterOrMemory.mem(.byte_ptr, .{ .disp = 0x10000000 }), Register.reg(.r12b), emit.code()); + try lowerToMrEnc(.add, RegisterOrMemory.mem(.byte_ptr, .{ .disp = 0x10000000 }), .r12b, emit.code()); try expectEqualHexStrings( "\x44\x00\x24\x25\x00\x00\x00\x10", emit.lowered(), "add byte ptr [ds:0x10000000], r12b", ); - try lowerToMrEnc(.add, RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0x10000000 }), Register.reg(.r12d), emit.code()); + try lowerToMrEnc(.add, RegisterOrMemory.mem(.dword_ptr, .{ .disp = 0x10000000 }), .r12d, emit.code()); try expectEqualHexStrings( "\x44\x01\x24\x25\x00\x00\x00\x10", emit.lowered(), @@ -2611,61 +2603,53 @@ test "lower MR encoding" { try lowerToMrEnc(.sub, RegisterOrMemory.mem(.qword_ptr, .{ .disp = 0x10000000, .base = .r11, - }), Register.reg(.r12), emit.code()); + }), .r12, emit.code()); try expectEqualHexStrings( "\x4D\x29\xA3\x00\x00\x00\x10", emit.lowered(), "sub qword ptr [r11 + 0x10000000], r12", ); - try lowerToMrEnc(.mov, RegisterOrMemory.rip(.qword_ptr, 0x10), Register.reg(.r12), emit.code()); + try lowerToMrEnc(.mov, RegisterOrMemory.rip(.qword_ptr, 0x10), .r12, emit.code()); try expectEqualHexStrings("\x4C\x89\x25\x10\x00\x00\x00", emit.lowered(), "mov qword ptr [rip + 0x10], r12"); - - // AVX extension tests - try lowerToMrEnc(.vmovsd, RegisterOrMemory.rip(.qword_ptr, 0x10), Register.avxReg(.xmm1), emit.code()); - try expectEqualHexStrings( - "\xC5\xFB\x11\x0D\x10\x00\x00\x00", - emit.lowered(), - "vmovsd qword ptr [rip + 0x10], xmm1", - ); } test "lower OI encoding" { var emit = TestEmit.init(); defer emit.deinit(); - try lowerToOiEnc(.mov, Register.reg(.rax), 0x1000000000000000, emit.code()); + try lowerToOiEnc(.mov, .rax, 0x1000000000000000, emit.code()); try expectEqualHexStrings( "\x48\xB8\x00\x00\x00\x00\x00\x00\x00\x10", emit.lowered(), "movabs rax, 0x1000000000000000", ); - try lowerToOiEnc(.mov, Register.reg(.r11), 0x1000000000000000, emit.code()); + try lowerToOiEnc(.mov, .r11, 0x1000000000000000, emit.code()); try expectEqualHexStrings( "\x49\xBB\x00\x00\x00\x00\x00\x00\x00\x10", emit.lowered(), "movabs r11, 0x1000000000000000", ); - try lowerToOiEnc(.mov, Register.reg(.r11d), 0x10000000, emit.code()); + try lowerToOiEnc(.mov, .r11d, 0x10000000, emit.code()); try expectEqualHexStrings("\x41\xBB\x00\x00\x00\x10", emit.lowered(), "mov r11d, 0x10000000"); - try lowerToOiEnc(.mov, Register.reg(.r11w), 0x1000, emit.code()); + try lowerToOiEnc(.mov, .r11w, 0x1000, emit.code()); try expectEqualHexStrings("\x66\x41\xBB\x00\x10", emit.lowered(), "mov r11w, 0x1000"); - try lowerToOiEnc(.mov, Register.reg(.r11b), 0x10, emit.code()); + try lowerToOiEnc(.mov, .r11b, 0x10, emit.code()); try expectEqualHexStrings("\x41\xB3\x10", emit.lowered(), "mov r11b, 0x10"); } test "lower FD/TD encoding" { var emit = TestEmit.init(); defer emit.deinit(); - try lowerToFdEnc(.mov, Register.reg(.rax), 0x1000000000000000, emit.code()); + try lowerToFdEnc(.mov, .rax, 0x1000000000000000, emit.code()); try expectEqualHexStrings( "\x48\xa1\x00\x00\x00\x00\x00\x00\x00\x10", emit.lowered(), "mov rax, ds:0x1000000000000000", ); - try lowerToFdEnc(.mov, Register.reg(.eax), 0x10000000, emit.code()); + try lowerToFdEnc(.mov, .eax, 0x10000000, emit.code()); try expectEqualHexStrings("\xa1\x00\x00\x00\x10", emit.lowered(), "mov eax, ds:0x10000000"); - try lowerToFdEnc(.mov, Register.reg(.ax), 0x1000, emit.code()); + try lowerToFdEnc(.mov, .ax, 0x1000, emit.code()); try expectEqualHexStrings("\x66\xa1\x00\x10", emit.lowered(), "mov ax, ds:0x1000"); - try lowerToFdEnc(.mov, Register.reg(.al), 0x10, emit.code()); + try lowerToFdEnc(.mov, .al, 0x10, emit.code()); try expectEqualHexStrings("\xa0\x10", emit.lowered(), "mov al, ds:0x10"); } @@ -2741,16 +2725,16 @@ test "lower M1 and MC encodings" { test "lower O encoding" { var emit = TestEmit.init(); defer emit.deinit(); - try lowerToOEnc(.pop, Register.reg(.r12), emit.code()); + try lowerToOEnc(.pop, .r12, emit.code()); try expectEqualHexStrings("\x41\x5c", emit.lowered(), "pop r12"); - try lowerToOEnc(.push, Register.reg(.r12w), emit.code()); + try lowerToOEnc(.push, .r12w, emit.code()); try expectEqualHexStrings("\x66\x41\x54", emit.lowered(), "push r12w"); } test "lower RMI encoding" { var emit = TestEmit.init(); defer emit.deinit(); - try lowerToRmiEnc(.imul, Register.reg(.rax), RegisterOrMemory.mem(.qword_ptr, .{ + try lowerToRmiEnc(.imul, .rax, RegisterOrMemory.mem(.qword_ptr, .{ .disp = @bitCast(u32, @as(i32, -8)), .base = .rbp, }), 0x10, emit.code()); @@ -2759,39 +2743,49 @@ test "lower RMI encoding" { emit.lowered(), "imul rax, qword ptr [rbp - 8], 0x10", ); - try lowerToRmiEnc(.imul, Register.reg(.eax), RegisterOrMemory.mem(.dword_ptr, .{ + try lowerToRmiEnc(.imul, .eax, RegisterOrMemory.mem(.dword_ptr, .{ .disp = @bitCast(u32, @as(i32, -4)), .base = .rbp, }), 0x10, emit.code()); try expectEqualHexStrings("\x69\x45\xFC\x10\x00\x00\x00", emit.lowered(), "imul eax, dword ptr [rbp - 4], 0x10"); - try lowerToRmiEnc(.imul, Register.reg(.ax), RegisterOrMemory.mem(.word_ptr, .{ + try lowerToRmiEnc(.imul, .ax, RegisterOrMemory.mem(.word_ptr, .{ .disp = @bitCast(u32, @as(i32, -2)), .base = .rbp, }), 0x10, emit.code()); try expectEqualHexStrings("\x66\x69\x45\xFE\x10\x00", emit.lowered(), "imul ax, word ptr [rbp - 2], 0x10"); - try lowerToRmiEnc(.imul, Register.reg(.r12), RegisterOrMemory.reg(.r12), 0x10, emit.code()); + try lowerToRmiEnc(.imul, .r12, RegisterOrMemory.reg(.r12), 0x10, emit.code()); try expectEqualHexStrings("\x4D\x69\xE4\x10\x00\x00\x00", emit.lowered(), "imul r12, r12, 0x10"); - try lowerToRmiEnc(.imul, Register.reg(.r12w), RegisterOrMemory.reg(.r12w), 0x10, emit.code()); + try lowerToRmiEnc(.imul, .r12w, RegisterOrMemory.reg(.r12w), 0x10, emit.code()); try expectEqualHexStrings("\x66\x45\x69\xE4\x10\x00", emit.lowered(), "imul r12w, r12w, 0x10"); } +test "lower MV encoding" { + var emit = TestEmit.init(); + defer emit.deinit(); + try lowerToMvEnc(.vmovsd, RegisterOrMemory.rip(.qword_ptr, 0x10), .xmm1, emit.code()); + try expectEqualHexStrings( + "\xC5\xFB\x11\x0D\x10\x00\x00\x00", + emit.lowered(), + "vmovsd qword ptr [rip + 0x10], xmm1", + ); +} + +test "lower VM encoding" { + var emit = TestEmit.init(); + defer emit.deinit(); + try lowerToVmEnc(.vmovsd, .xmm1, RegisterOrMemory.rip(.qword_ptr, 0x10), emit.code()); + try expectEqualHexStrings( + "\xC5\xFB\x10\x0D\x10\x00\x00\x00", + emit.lowered(), + "vmovsd xmm1, qword ptr [rip + 0x10]", + ); +} + test "lower to RVM encoding" { var emit = TestEmit.init(); defer emit.deinit(); - try lowerToRvmEnc( - .vaddsd, - Register.avxReg(.xmm0), - Register.avxReg(.xmm1), - RegisterOrMemory.avxReg(.xmm2), - emit.code(), - ); + try lowerToRvmEnc(.vaddsd, .xmm0, .xmm1, RegisterOrMemory.reg(.xmm2), emit.code()); try expectEqualHexStrings("\xC5\xF3\x58\xC2", emit.lowered(), "vaddsd xmm0, xmm1, xmm2"); - try lowerToRvmEnc( - .vaddsd, - Register.avxReg(.xmm0), - Register.avxReg(.xmm0), - RegisterOrMemory.avxReg(.xmm1), - emit.code(), - ); + try lowerToRvmEnc(.vaddsd, .xmm0, .xmm0, RegisterOrMemory.reg(.xmm1), emit.code()); try expectEqualHexStrings("\xC5\xFB\x58\xC1", emit.lowered(), "vaddsd xmm0, xmm0, xmm1"); } diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig index ef50279d03..a1062ba6b4 100644 --- a/src/arch/x86_64/Mir.zig +++ b/src/arch/x86_64/Mir.zig @@ -14,8 +14,7 @@ const assert = std.debug.assert; const bits = @import("bits.zig"); const Air = @import("../../Air.zig"); const CodeGen = @import("CodeGen.zig"); -const GpRegister = bits.Register; -const AvxRegister = bits.AvxRegister; +const Register = bits.Register; instructions: std.MultiArrayList(Inst).Slice, /// The meaning of this data is determined by `Inst.Tag` value. @@ -23,11 +22,7 @@ extra: []const u32, pub const Inst = struct { tag: Tag, - /// This is 3 fields, and the meaning of each depends on `tag`. - /// reg1: Register - /// reg2: Register - /// flags: u2 - ops: u16, + ops: Ops, /// The meaning of this depends on `tag` and `ops`. data: Data, @@ -397,6 +392,36 @@ pub const Inst = struct { /// The position of an MIR instruction within the `Mir` instructions array. pub const Index = u32; + pub const Ops = packed struct { + reg1: u7, + reg2: u7, + flags: u2, + + pub fn encode(vals: struct { + reg1: Register = .none, + reg2: Register = .none, + flags: u2 = 0b00, + }) Ops { + return .{ + .reg1 = @enumToInt(vals.reg1), + .reg2 = @enumToInt(vals.reg2), + .flags = vals.flags, + }; + } + + pub fn decode(ops: Ops) struct { + reg1: Register, + reg2: Register, + flags: u2, + } { + return .{ + .reg1 = @intToEnum(Register, ops.reg1), + .reg2 = @intToEnum(Register, ops.reg2), + .flags = ops.flags, + }; + } + }; + /// All instructions have a 4-byte payload, which is contained within /// this union. `Tag` determines which union field is active, as well as /// how to interpret the data within. @@ -466,33 +491,6 @@ pub const DbgLineColumn = struct { column: u32, }; -pub fn Ops(comptime Reg1: type, comptime Reg2: type) type { - return struct { - reg1: Reg1 = .none, - reg2: Reg2 = .none, - flags: u2 = 0b00, - - pub fn encode(self: @This()) u16 { - var ops: u16 = 0; - ops |= @intCast(u16, @enumToInt(self.reg1)) << 9; - ops |= @intCast(u16, @enumToInt(self.reg2)) << 2; - ops |= self.flags; - return ops; - } - - pub fn decode(ops: u16) @This() { - const reg1 = @intToEnum(Reg1, @truncate(u7, ops >> 9)); - const reg2 = @intToEnum(Reg2, @truncate(u7, ops >> 2)); - const flags = @truncate(u2, ops); - return .{ - .reg1 = reg1, - .reg2 = reg2, - .flags = flags, - }; - } - }; -} - pub fn deinit(mir: *Mir, gpa: std.mem.Allocator) void { mir.instructions.deinit(gpa); gpa.free(mir.extra); diff --git a/src/arch/x86_64/abi.zig b/src/arch/x86_64/abi.zig index a53c82530d..85bf3a0790 100644 --- a/src/arch/x86_64/abi.zig +++ b/src/arch/x86_64/abi.zig @@ -3,7 +3,6 @@ const Type = @import("../../type.zig").Type; const Target = std.Target; const assert = std.debug.assert; const Register = @import("bits.zig").Register; -const AvxRegister = @import("bits.zig").AvxRegister; pub const Class = enum { integer, sse, sseup, x87, x87up, complex_x87, memory, none }; @@ -379,11 +378,17 @@ pub const callee_preserved_regs = [_]Register{ .rbx, .r12, .r13, .r14, .r15 }; /// the caller relinquishes control to a subroutine via call instruction (or similar). /// In other words, these registers are free to use by the callee. pub const caller_preserved_regs = [_]Register{ .rax, .rcx, .rdx, .rsi, .rdi, .r8, .r9, .r10, .r11 }; -pub const allocatable_registers = callee_preserved_regs ++ caller_preserved_regs; -pub const c_abi_int_param_regs = [_]Register{ .rdi, .rsi, .rdx, .rcx, .r8, .r9 }; -pub const c_abi_int_return_regs = [_]Register{ .rax, .rdx }; - -pub const avx_regs = [_]AvxRegister{ +pub const avx_regs = [_]Register{ .ymm0, .ymm1, .ymm2, .ymm3, .ymm4, .ymm5, .ymm6, .ymm7, .ymm8, .ymm9, .ymm10, .ymm11, .ymm12, .ymm13, .ymm14, .ymm15, }; +pub const allocatable_registers = callee_preserved_regs ++ caller_preserved_regs ++ avx_regs; + +// Masks for register manager +const FreeRegInt = std.meta.Int(.unsigned, allocatable_registers.len); +// TODO +pub const gp_mask: FreeRegInt = 0x3fff; +pub const avx_mask: FreeRegInt = 0x3fff_c000; + +pub const c_abi_int_param_regs = [_]Register{ .rdi, .rsi, .rdx, .rcx, .r8, .r9 }; +pub const c_abi_int_return_regs = [_]Register{ .rax, .rdx }; diff --git a/src/arch/x86_64/bits.zig b/src/arch/x86_64/bits.zig index 0bba295c20..85bd190e2b 100644 --- a/src/arch/x86_64/bits.zig +++ b/src/arch/x86_64/bits.zig @@ -43,17 +43,36 @@ pub const Register = enum(u7) { al, cl, dl, bl, ah, ch, dh, bh, r8b, r9b, r10b, r11b, r12b, r13b, r14b, r15b, - // Pseudo, used only for MIR to signify that the - // operand is not a register but an immediate, etc. + // 64-79, 256-bit registers. + // id is int value - 64. + ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7, + ymm8, ymm9, ymm10, ymm11, ymm12, ymm13, ymm14, ymm15, + + // 80-95, 128-bit registers. + // id is int value - 80. + xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, + xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, + + // Pseudo-value for MIR instructions. none, + pub fn id(self: Register) u5 { + return switch (@enumToInt(self)) { + 0...63 => @as(u5, @truncate(u4, @enumToInt(self))), + 64...79 => @truncate(u5, @enumToInt(self)), + else => unreachable, + }; + } + /// Returns the bit-width of the register. - pub fn size(self: Register) u7 { + pub fn size(self: Register) u9 { return switch (@enumToInt(self)) { 0...15 => 64, 16...31 => 32, 32...47 => 16, - 48...64 => 8, + 48...63 => 8, + 64...79 => 256, + 80...95 => 128, else => unreachable, }; } @@ -72,15 +91,23 @@ pub const Register = enum(u7) { /// an instruction (@see isExtended), and requires special handling. The /// lower three bits are often embedded directly in instructions (such as /// the B8 variant of moves), or used in R/M bytes. - pub fn id(self: Register) u4 { + pub fn enc(self: Register) u4 { return @truncate(u4, @enumToInt(self)); } - /// Like id, but only returns the lower 3 bits. - pub fn lowId(self: Register) u3 { + /// Like enc, but only returns the lower 3 bits. + pub fn lowEnc(self: Register) u3 { return @truncate(u3, @enumToInt(self)); } + pub fn to256(self: Register) Register { + return @intToEnum(Register, @as(u8, self.id()) + 64); + } + + pub fn to128(self: Register) Register { + return @intToEnum(Register, @as(u8, self.id()) + 80); + } + /// Convert from any register to its 64 bit alias. pub fn to64(self: Register) Register { return @intToEnum(Register, self.id()); @@ -126,57 +153,6 @@ pub const Register = enum(u7) { } }; -/// AVX registers. -/// TODO missing dwarfLocOp implementation. -/// TODO add support for AVX-512 -pub const AvxRegister = enum(u6) { - // 256-bit registers - ymm0, ymm1, ymm2, ymm3, ymm4, ymm5, ymm6, ymm7, - ymm8, ymm9, ymm10, ymm11, ymm12, ymm13, ymm14, ymm15, - - // 128-bit registers - xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7, - xmm8, xmm9, xmm10, xmm11, xmm12, xmm13, xmm14, xmm15, - - // Pseudo, used only for MIR to signify that the - // operand is not a register but an immediate, etc. - none, - - /// Returns the bit-width of the register. - pub fn size(self: AvxRegister) u9 { - return switch (@enumToInt(self)) { - 0...15 => 256, - 16...31 => 128, - else => unreachable, - }; - } - - /// Returns whether the register is *extended*. - pub fn isExtended(self: AvxRegister) bool { - return @enumToInt(self) & 0x08 != 0; - } - - /// This returns the 4-bit register ID. - pub fn id(self: AvxRegister) u4 { - return @truncate(u4, @enumToInt(self)); - } - - /// Like id, but only returns the lower 3 bits. - pub fn lowId(self: AvxRegister) u3 { - return @truncate(u3, @enumToInt(self)); - } - - /// Convert from any register to its 256 bit alias. - pub fn to256(self: AvxRegister) AvxRegister { - return @intToEnum(AvxRegister, self.id()); - } - - /// Convert from any register to its 128 bit alias. - pub fn to128(self: AvxRegister) AvxRegister { - return @intToEnum(AvxRegister, @as(u8, self.id()) + 16); - } -}; - // zig fmt: on /// Encoding helper functions for x86_64 instructions @@ -792,7 +768,7 @@ test "Encoder helpers - Vex prefix" { { stream.reset(); var vex_prefix = Encoder.Vex{}; - vex_prefix.reg(AvxRegister.xmm15.id()); + vex_prefix.reg(Register.xmm15.id()); const nwritten = vex_prefix.write(writer); try testing.expectEqualSlices(u8, &[_]u8{ 0xc5, 0x80 }, buf[0..nwritten]); } @@ -832,7 +808,7 @@ test "Encoder helpers - Vex prefix" { vex.simd_prefix_66(); encoder.vex(vex); // use 64 bit operation encoder.opcode_1byte(0x28); - encoder.modRm_direct(0, AvxRegister.xmm1.lowId()); + encoder.modRm_direct(0, Register.xmm1.lowId()); try testing.expectEqualSlices(u8, &[_]u8{ 0xC5, 0xF9, 0x28, 0xC1 }, code.items); } @@ -846,10 +822,10 @@ test "Encoder helpers - Vex prefix" { vex.simd_prefix_66(); vex.lead_opc_0f(); vex.rex(.{ .r = true }); - vex.reg(AvxRegister.xmm1.id()); + vex.reg(Register.xmm1.id()); encoder.vex(vex); encoder.opcode_1byte(0x16); - encoder.modRm_RIPDisp32(AvxRegister.xmm13.lowId()); + encoder.modRm_RIPDisp32(Register.xmm13.lowId()); encoder.disp32(0); try testing.expectEqualSlices(u8, &[_]u8{ 0xC5, 0x71, 0x16, 0x2D, 0x00, 0x00, 0x00, 0x00 }, code.items); } diff --git a/src/codegen.zig b/src/codegen.zig index c14fb39629..def69d952f 100644 --- a/src/codegen.zig +++ b/src/codegen.zig @@ -78,13 +78,13 @@ pub fn generateFunction( debug_output: DebugInfoOutput, ) GenerateSymbolError!FnResult { switch (bin_file.options.target.cpu.arch) { - // .arm, - // .armeb, - // => return @import("arch/arm/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output), - // .aarch64, - // .aarch64_be, - // .aarch64_32, - // => return @import("arch/aarch64/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output), + .arm, + .armeb, + => return @import("arch/arm/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output), + .aarch64, + .aarch64_be, + .aarch64_32, + => return @import("arch/aarch64/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output), //.arc => return Function(.arc).generate(bin_file, src_loc, func, air, liveness, code, debug_output), //.avr => return Function(.avr).generate(bin_file, src_loc, func, air, liveness, code, debug_output), //.bpfel => return Function(.bpfel).generate(bin_file, src_loc, func, air, liveness, code, debug_output), @@ -101,9 +101,9 @@ pub fn generateFunction( //.r600 => return Function(.r600).generate(bin_file, src_loc, func, air, liveness, code, debug_output), //.amdgcn => return Function(.amdgcn).generate(bin_file, src_loc, func, air, liveness, code, debug_output), //.riscv32 => return Function(.riscv32).generate(bin_file, src_loc, func, air, liveness, code, debug_output), - // .riscv64 => return @import("arch/riscv64/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output), + .riscv64 => return @import("arch/riscv64/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output), //.sparc => return Function(.sparc).generate(bin_file, src_loc, func, air, liveness, code, debug_output), - // .sparc64 => return @import("arch/sparc64/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output), + .sparc64 => return @import("arch/sparc64/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output), //.sparcel => return Function(.sparcel).generate(bin_file, src_loc, func, air, liveness, code, debug_output), //.s390x => return Function(.s390x).generate(bin_file, src_loc, func, air, liveness, code, debug_output), //.tce => return Function(.tce).generate(bin_file, src_loc, func, air, liveness, code, debug_output), @@ -129,9 +129,9 @@ pub fn generateFunction( //.renderscript32 => return Function(.renderscript32).generate(bin_file, src_loc, func, air, liveness, code, debug_output), //.renderscript64 => return Function(.renderscript64).generate(bin_file, src_loc, func, air, liveness, code, debug_output), //.ve => return Function(.ve).generate(bin_file, src_loc, func, air, liveness, code, debug_output), - // .wasm32, - // .wasm64, - // => return @import("arch/wasm/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output), + .wasm32, + .wasm64, + => return @import("arch/wasm/CodeGen.zig").generate(bin_file, src_loc, func, air, liveness, code, debug_output), else => @panic("Backend architectures that don't have good support yet are commented out, to improve compilation performance. If you are interested in one of these other backends feel free to uncomment them. Eventually these will be completed, but stage1 is slow and a memory hog."), } } diff --git a/src/register_manager.zig b/src/register_manager.zig index 6fc3641467..2c0502e867 100644 --- a/src/register_manager.zig +++ b/src/register_manager.zig @@ -23,15 +23,10 @@ pub const AllocateRegistersError = error{ CodegenFail, }; -pub fn SpillFn(comptime Function: type, comptime Register: type) type { - return fn (*Function, Register, Air.Inst.Index) anyerror!void; -} - pub fn RegisterManager( comptime Function: type, comptime Register: type, comptime tracked_registers: []const Register, - comptime spill_fn: SpillFn(Function, Register), ) type { // architectures which do not have a concept of registers should // refrain from using RegisterManager @@ -52,7 +47,6 @@ pub fn RegisterManager( allocated_registers: FreeRegInt = 0, /// Tracks registers which are locked from being allocated locked_registers: FreeRegInt = 0, - function: *Function, const Self = @This(); @@ -61,8 +55,8 @@ pub fn RegisterManager( const FreeRegInt = std.meta.Int(.unsigned, tracked_registers.len); const ShiftInt = math.Log2Int(FreeRegInt); - fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) AllocateRegistersError!void { - return try spill_fn(self.function, reg, inst); + fn getFunction(self: *Self) *Function { + return @fieldParentPtr(Function, "register_manager", self); } fn getRegisterMask(reg: Register) ?FreeRegInt { @@ -257,14 +251,14 @@ pub fn RegisterManager( self.markRegUsed(reg); } else { const spilled_inst = self.registers[index]; - try self.spillInstruction(reg, spilled_inst); + try self.getFunction().spillInstruction(reg, spilled_inst); } self.registers[index] = inst; } else { // Don't track the register if (!self.isRegFree(reg)) { const spilled_inst = self.registers[index]; - try self.spillInstruction(reg, spilled_inst); + try self.getFunction().spillInstruction(reg, spilled_inst); self.freeReg(reg); } } @@ -299,7 +293,7 @@ pub fn RegisterManager( // stack allocation. const spilled_inst = self.registers[index]; self.registers[index] = tracked_inst; - try self.spillInstruction(reg, spilled_inst); + try self.getFunction().spillInstruction(reg, spilled_inst); } else { self.getRegAssumeFree(reg, tracked_inst); } @@ -308,7 +302,7 @@ pub fn RegisterManager( // Move the instruction that was previously there to a // stack allocation. const spilled_inst = self.registers[index]; - try self.spillInstruction(reg, spilled_inst); + try self.getFunction().spillInstruction(reg, spilled_inst); self.freeReg(reg); } } @@ -338,253 +332,253 @@ pub fn RegisterManager( }; } -//const MockRegister1 = enum(u2) { -// r0, -// r1, -// r2, -// r3, +const MockRegister1 = enum(u2) { + r0, + r1, + r2, + r3, -// pub fn id(reg: MockRegister1) u2 { -// return @enumToInt(reg); -// } + pub fn id(reg: MockRegister1) u2 { + return @enumToInt(reg); + } -// const allocatable_registers = [_]MockRegister1{ .r2, .r3 }; -//}; + const allocatable_registers = [_]MockRegister1{ .r2, .r3 }; +}; -//const MockRegister2 = enum(u2) { -// r0, -// r1, -// r2, -// r3, +const MockRegister2 = enum(u2) { + r0, + r1, + r2, + r3, -// pub fn id(reg: MockRegister2) u2 { -// return @enumToInt(reg); -// } + pub fn id(reg: MockRegister2) u2 { + return @enumToInt(reg); + } -// const allocatable_registers = [_]MockRegister2{ .r0, .r1, .r2, .r3 }; -//}; + const allocatable_registers = [_]MockRegister2{ .r0, .r1, .r2, .r3 }; +}; -//fn MockFunction(comptime Register: type) type { -// return struct { -// allocator: Allocator, -// register_manager: RegisterManager(Self, Register, &Register.allocatable_registers) = .{}, -// spilled: std.ArrayListUnmanaged(Register) = .{}, +fn MockFunction(comptime Register: type) type { + return struct { + allocator: Allocator, + register_manager: RegisterManager(Self, Register, &Register.allocatable_registers) = .{}, + spilled: std.ArrayListUnmanaged(Register) = .{}, -// const Self = @This(); + const Self = @This(); -// pub fn deinit(self: *Self) void { -// self.spilled.deinit(self.allocator); -// } + pub fn deinit(self: *Self) void { + self.spilled.deinit(self.allocator); + } -// pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void { -// _ = inst; -// try self.spilled.append(self.allocator, reg); -// } + pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void { + _ = inst; + try self.spilled.append(self.allocator, reg); + } -// pub fn genAdd(self: *Self, res: Register, lhs: Register, rhs: Register) !void { -// _ = self; -// _ = res; -// _ = lhs; -// _ = rhs; -// } -// }; -//} + pub fn genAdd(self: *Self, res: Register, lhs: Register, rhs: Register) !void { + _ = self; + _ = res; + _ = lhs; + _ = rhs; + } + }; +} -//const MockFunction1 = MockFunction(MockRegister1); -//const MockFunction2 = MockFunction(MockRegister2); +const MockFunction1 = MockFunction(MockRegister1); +const MockFunction2 = MockFunction(MockRegister2); -//test "default state" { -// const allocator = std.testing.allocator; +test "default state" { + const allocator = std.testing.allocator; -// var function = MockFunction1{ -// .allocator = allocator, -// }; -// defer function.deinit(); + var function = MockFunction1{ + .allocator = allocator, + }; + defer function.deinit(); -// try expect(!function.register_manager.isRegAllocated(.r2)); -// try expect(!function.register_manager.isRegAllocated(.r3)); -// try expect(function.register_manager.isRegFree(.r2)); -// try expect(function.register_manager.isRegFree(.r3)); -//} + try expect(!function.register_manager.isRegAllocated(.r2)); + try expect(!function.register_manager.isRegAllocated(.r3)); + try expect(function.register_manager.isRegFree(.r2)); + try expect(function.register_manager.isRegFree(.r3)); +} -//test "tryAllocReg: no spilling" { -// const allocator = std.testing.allocator; +test "tryAllocReg: no spilling" { + const allocator = std.testing.allocator; -// var function = MockFunction1{ -// .allocator = allocator, -// }; -// defer function.deinit(); + var function = MockFunction1{ + .allocator = allocator, + }; + defer function.deinit(); -// const mock_instruction: Air.Inst.Index = 1; + const mock_instruction: Air.Inst.Index = 1; -// try expectEqual(@as(?MockRegister1, .r2), function.register_manager.tryAllocReg(mock_instruction)); -// try expectEqual(@as(?MockRegister1, .r3), function.register_manager.tryAllocReg(mock_instruction)); -// try expectEqual(@as(?MockRegister1, null), function.register_manager.tryAllocReg(mock_instruction)); + try expectEqual(@as(?MockRegister1, .r2), function.register_manager.tryAllocReg(mock_instruction)); + try expectEqual(@as(?MockRegister1, .r3), function.register_manager.tryAllocReg(mock_instruction)); + try expectEqual(@as(?MockRegister1, null), function.register_manager.tryAllocReg(mock_instruction)); -// try expect(function.register_manager.isRegAllocated(.r2)); -// try expect(function.register_manager.isRegAllocated(.r3)); -// try expect(!function.register_manager.isRegFree(.r2)); -// try expect(!function.register_manager.isRegFree(.r3)); + try expect(function.register_manager.isRegAllocated(.r2)); + try expect(function.register_manager.isRegAllocated(.r3)); + try expect(!function.register_manager.isRegFree(.r2)); + try expect(!function.register_manager.isRegFree(.r3)); -// function.register_manager.freeReg(.r2); -// function.register_manager.freeReg(.r3); + function.register_manager.freeReg(.r2); + function.register_manager.freeReg(.r3); -// try expect(function.register_manager.isRegAllocated(.r2)); -// try expect(function.register_manager.isRegAllocated(.r3)); -// try expect(function.register_manager.isRegFree(.r2)); -// try expect(function.register_manager.isRegFree(.r3)); -//} + try expect(function.register_manager.isRegAllocated(.r2)); + try expect(function.register_manager.isRegAllocated(.r3)); + try expect(function.register_manager.isRegFree(.r2)); + try expect(function.register_manager.isRegFree(.r3)); +} -//test "allocReg: spilling" { -// const allocator = std.testing.allocator; +test "allocReg: spilling" { + const allocator = std.testing.allocator; -// var function = MockFunction1{ -// .allocator = allocator, -// }; -// defer function.deinit(); + var function = MockFunction1{ + .allocator = allocator, + }; + defer function.deinit(); -// const mock_instruction: Air.Inst.Index = 1; + const mock_instruction: Air.Inst.Index = 1; -// try expectEqual(@as(?MockRegister1, .r2), try function.register_manager.allocReg(mock_instruction)); -// try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction)); + try expectEqual(@as(?MockRegister1, .r2), try function.register_manager.allocReg(mock_instruction)); + try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction)); -// // Spill a register -// try expectEqual(@as(?MockRegister1, .r2), try function.register_manager.allocReg(mock_instruction)); -// try expectEqualSlices(MockRegister1, &[_]MockRegister1{.r2}, function.spilled.items); + // Spill a register + try expectEqual(@as(?MockRegister1, .r2), try function.register_manager.allocReg(mock_instruction)); + try expectEqualSlices(MockRegister1, &[_]MockRegister1{.r2}, function.spilled.items); -// // No spilling necessary -// function.register_manager.freeReg(.r3); -// try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction)); -// try expectEqualSlices(MockRegister1, &[_]MockRegister1{.r2}, function.spilled.items); + // No spilling necessary + function.register_manager.freeReg(.r3); + try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction)); + try expectEqualSlices(MockRegister1, &[_]MockRegister1{.r2}, function.spilled.items); -// // Locked registers -// function.register_manager.freeReg(.r3); -// { -// const lock = function.register_manager.lockReg(.r2); -// defer if (lock) |reg| function.register_manager.unlockReg(reg); + // Locked registers + function.register_manager.freeReg(.r3); + { + const lock = function.register_manager.lockReg(.r2); + defer if (lock) |reg| function.register_manager.unlockReg(reg); -// try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction)); -// } -// try expect(!function.register_manager.lockedRegsExist()); -//} + try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction)); + } + try expect(!function.register_manager.lockedRegsExist()); +} -//test "tryAllocRegs" { -// const allocator = std.testing.allocator; +test "tryAllocRegs" { + const allocator = std.testing.allocator; -// var function = MockFunction2{ -// .allocator = allocator, -// }; -// defer function.deinit(); + var function = MockFunction2{ + .allocator = allocator, + }; + defer function.deinit(); -// try expectEqual([_]MockRegister2{ .r0, .r1, .r2 }, function.register_manager.tryAllocRegs(3, .{ null, null, null }).?); + try expectEqual([_]MockRegister2{ .r0, .r1, .r2 }, function.register_manager.tryAllocRegs(3, .{ null, null, null }).?); -// try expect(function.register_manager.isRegAllocated(.r0)); -// try expect(function.register_manager.isRegAllocated(.r1)); -// try expect(function.register_manager.isRegAllocated(.r2)); -// try expect(!function.register_manager.isRegAllocated(.r3)); + try expect(function.register_manager.isRegAllocated(.r0)); + try expect(function.register_manager.isRegAllocated(.r1)); + try expect(function.register_manager.isRegAllocated(.r2)); + try expect(!function.register_manager.isRegAllocated(.r3)); -// // Locked registers -// function.register_manager.freeReg(.r0); -// function.register_manager.freeReg(.r2); -// function.register_manager.freeReg(.r3); -// { -// const lock = function.register_manager.lockReg(.r1); -// defer if (lock) |reg| function.register_manager.unlockReg(reg); + // Locked registers + function.register_manager.freeReg(.r0); + function.register_manager.freeReg(.r2); + function.register_manager.freeReg(.r3); + { + const lock = function.register_manager.lockReg(.r1); + defer if (lock) |reg| function.register_manager.unlockReg(reg); -// try expectEqual([_]MockRegister2{ .r0, .r2, .r3 }, function.register_manager.tryAllocRegs(3, .{ null, null, null }).?); -// } -// try expect(!function.register_manager.lockedRegsExist()); + try expectEqual([_]MockRegister2{ .r0, .r2, .r3 }, function.register_manager.tryAllocRegs(3, .{ null, null, null }).?); + } + try expect(!function.register_manager.lockedRegsExist()); -// try expect(function.register_manager.isRegAllocated(.r0)); -// try expect(function.register_manager.isRegAllocated(.r1)); -// try expect(function.register_manager.isRegAllocated(.r2)); -// try expect(function.register_manager.isRegAllocated(.r3)); -//} + try expect(function.register_manager.isRegAllocated(.r0)); + try expect(function.register_manager.isRegAllocated(.r1)); + try expect(function.register_manager.isRegAllocated(.r2)); + try expect(function.register_manager.isRegAllocated(.r3)); +} -//test "allocRegs: normal usage" { -// // TODO: convert this into a decltest once that is supported +test "allocRegs: normal usage" { + // TODO: convert this into a decltest once that is supported -// const allocator = std.testing.allocator; + const allocator = std.testing.allocator; -// var function = MockFunction2{ -// .allocator = allocator, -// }; -// defer function.deinit(); + var function = MockFunction2{ + .allocator = allocator, + }; + defer function.deinit(); -// { -// const result_reg: MockRegister2 = .r1; + { + const result_reg: MockRegister2 = .r1; -// // The result register is known and fixed at this point, we -// // don't want to accidentally allocate lhs or rhs to the -// // result register, this is why we lock it. -// // -// // Using defer unlock right after lock is a good idea in -// // most cases as you probably are using the locked registers -// // in the remainder of this scope and don't need to use it -// // after the end of this scope. However, in some situations, -// // it may make sense to manually unlock registers before the -// // end of the scope when you are certain that they don't -// // contain any valuable data anymore and can be reused. For an -// // example of that, see `selectively reducing register -// // pressure`. -// const lock = function.register_manager.lockReg(result_reg); -// defer if (lock) |reg| function.register_manager.unlockReg(reg); + // The result register is known and fixed at this point, we + // don't want to accidentally allocate lhs or rhs to the + // result register, this is why we lock it. + // + // Using defer unlock right after lock is a good idea in + // most cases as you probably are using the locked registers + // in the remainder of this scope and don't need to use it + // after the end of this scope. However, in some situations, + // it may make sense to manually unlock registers before the + // end of the scope when you are certain that they don't + // contain any valuable data anymore and can be reused. For an + // example of that, see `selectively reducing register + // pressure`. + const lock = function.register_manager.lockReg(result_reg); + defer if (lock) |reg| function.register_manager.unlockReg(reg); -// const regs = try function.register_manager.allocRegs(2, .{ null, null }); -// try function.genAdd(result_reg, regs[0], regs[1]); -// } -//} + const regs = try function.register_manager.allocRegs(2, .{ null, null }); + try function.genAdd(result_reg, regs[0], regs[1]); + } +} -//test "allocRegs: selectively reducing register pressure" { -// // TODO: convert this into a decltest once that is supported +test "allocRegs: selectively reducing register pressure" { + // TODO: convert this into a decltest once that is supported -// const allocator = std.testing.allocator; + const allocator = std.testing.allocator; -// var function = MockFunction2{ -// .allocator = allocator, -// }; -// defer function.deinit(); + var function = MockFunction2{ + .allocator = allocator, + }; + defer function.deinit(); -// { -// const result_reg: MockRegister2 = .r1; + { + const result_reg: MockRegister2 = .r1; -// const lock = function.register_manager.lockReg(result_reg); + const lock = function.register_manager.lockReg(result_reg); -// // Here, we don't defer unlock because we manually unlock -// // after genAdd -// const regs = try function.register_manager.allocRegs(2, .{ null, null }); + // Here, we don't defer unlock because we manually unlock + // after genAdd + const regs = try function.register_manager.allocRegs(2, .{ null, null }); -// try function.genAdd(result_reg, regs[0], regs[1]); -// function.register_manager.unlockReg(lock.?); + try function.genAdd(result_reg, regs[0], regs[1]); + function.register_manager.unlockReg(lock.?); -// const extra_summand_reg = try function.register_manager.allocReg(null); -// try function.genAdd(result_reg, result_reg, extra_summand_reg); -// } -//} + const extra_summand_reg = try function.register_manager.allocReg(null); + try function.genAdd(result_reg, result_reg, extra_summand_reg); + } +} -//test "getReg" { -// const allocator = std.testing.allocator; +test "getReg" { + const allocator = std.testing.allocator; -// var function = MockFunction1{ -// .allocator = allocator, -// }; -// defer function.deinit(); + var function = MockFunction1{ + .allocator = allocator, + }; + defer function.deinit(); -// const mock_instruction: Air.Inst.Index = 1; + const mock_instruction: Air.Inst.Index = 1; -// try function.register_manager.getReg(.r3, mock_instruction); + try function.register_manager.getReg(.r3, mock_instruction); -// try expect(!function.register_manager.isRegAllocated(.r2)); -// try expect(function.register_manager.isRegAllocated(.r3)); -// try expect(function.register_manager.isRegFree(.r2)); -// try expect(!function.register_manager.isRegFree(.r3)); + try expect(!function.register_manager.isRegAllocated(.r2)); + try expect(function.register_manager.isRegAllocated(.r3)); + try expect(function.register_manager.isRegFree(.r2)); + try expect(!function.register_manager.isRegFree(.r3)); -// // Spill r3 -// try function.register_manager.getReg(.r3, mock_instruction); + // Spill r3 + try function.register_manager.getReg(.r3, mock_instruction); -// try expect(!function.register_manager.isRegAllocated(.r2)); -// try expect(function.register_manager.isRegAllocated(.r3)); -// try expect(function.register_manager.isRegFree(.r2)); -// try expect(!function.register_manager.isRegFree(.r3)); -// try expectEqualSlices(MockRegister1, &[_]MockRegister1{.r3}, function.spilled.items); -//} + try expect(!function.register_manager.isRegAllocated(.r2)); + try expect(function.register_manager.isRegAllocated(.r3)); + try expect(function.register_manager.isRegFree(.r2)); + try expect(!function.register_manager.isRegFree(.r3)); + try expectEqualSlices(MockRegister1, &[_]MockRegister1{.r3}, function.spilled.items); +} From 549174f743d3b56544e2d3d6993f4d43bba1e7fd Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 18 May 2022 11:15:04 +0200 Subject: [PATCH 08/19] regalloc: allow for optional selector mask when allocating --- src/arch/aarch64/CodeGen.zig | 56 +++++++++++++++---------------- src/arch/arm/CodeGen.zig | 64 ++++++++++++++++++------------------ src/arch/riscv64/CodeGen.zig | 16 ++++----- src/arch/sparc64/CodeGen.zig | 26 +++++++-------- src/arch/x86_64/CodeGen.zig | 40 +++++++++++----------- src/register_manager.zig | 17 +++++++--- 6 files changed, 113 insertions(+), 106 deletions(-) diff --git a/src/arch/aarch64/CodeGen.zig b/src/arch/aarch64/CodeGen.zig index 0993d352d7..a3116587d4 100644 --- a/src/arch/aarch64/CodeGen.zig +++ b/src/arch/aarch64/CodeGen.zig @@ -888,7 +888,7 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue { if (reg_ok) { // Make sure the type can fit in a register before we try to allocate one. if (abi_size <= 8) { - if (self.register_manager.tryAllocReg(inst)) |reg| { + if (self.register_manager.tryAllocReg(inst, .{})) |reg| { return MCValue{ .register = registerAlias(reg, abi_size) }; } } @@ -951,7 +951,7 @@ fn spillCompareFlagsIfOccupied(self: *Self) !void { /// allocated. A second call to `copyToTmpRegister` may return the same register. /// This can have a side effect of spilling instructions to the stack to free up a register. fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register { - const raw_reg = try self.register_manager.allocReg(null); + const raw_reg = try self.register_manager.allocReg(null, .{}); const reg = registerAlias(raw_reg, ty.abiSize(self.target.*)); try self.genSetReg(ty, reg, mcv); return reg; @@ -961,7 +961,7 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register { /// `reg_owner` is the instruction that gets associated with the register in the register table. /// This can have a side effect of spilling instructions to the stack to free up a register. fn copyToNewRegister(self: *Self, reg_owner: Air.Inst.Index, mcv: MCValue) !MCValue { - const raw_reg = try self.register_manager.allocReg(reg_owner); + const raw_reg = try self.register_manager.allocReg(reg_owner, .{}); const ty = self.air.typeOfIndex(reg_owner); const reg = registerAlias(raw_reg, ty.abiSize(self.target.*)); try self.genSetReg(self.air.typeOfIndex(reg_owner), reg, mcv); @@ -1074,11 +1074,11 @@ fn trunc( if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) { break :blk registerAlias(operand_reg, dest_ty.abiSize(self.target.*)); } else { - const raw_reg = try self.register_manager.allocReg(inst); + const raw_reg = try self.register_manager.allocReg(inst, .{}); break :blk registerAlias(raw_reg, dest_ty.abiSize(self.target.*)); } } else blk: { - const raw_reg = try self.register_manager.allocReg(null); + const raw_reg = try self.register_manager.allocReg(null, .{}); break :blk registerAlias(raw_reg, dest_ty.abiSize(self.target.*)); }; @@ -1160,7 +1160,7 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void { break :blk op_reg; } - const raw_reg = try self.register_manager.allocReg(null); + const raw_reg = try self.register_manager.allocReg(null, .{}); break :blk raw_reg.to32(); }; @@ -1193,7 +1193,7 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void { break :blk op_reg; } - const raw_reg = try self.register_manager.allocReg(null); + const raw_reg = try self.register_manager.allocReg(null, .{}); break :blk registerAlias(raw_reg, operand_ty.abiSize(self.target.*)); }; @@ -1293,7 +1293,7 @@ fn binOpRegister( break :inst Air.refToIndex(md.lhs).?; } else null; - const raw_reg = try self.register_manager.allocReg(track_inst); + const raw_reg = try self.register_manager.allocReg(track_inst, .{}); const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*)); if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg }); @@ -1308,7 +1308,7 @@ fn binOpRegister( break :inst Air.refToIndex(md.rhs).?; } else null; - const raw_reg = try self.register_manager.allocReg(track_inst); + const raw_reg = try self.register_manager.allocReg(track_inst, .{}); const reg = registerAlias(raw_reg, rhs_ty.abiAlignment(self.target.*)); if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg }); @@ -1326,11 +1326,11 @@ fn binOpRegister( } else if (rhs_is_register and self.reuseOperand(md.inst, md.rhs, 1, rhs)) { break :blk rhs_reg; } else { - const raw_reg = try self.register_manager.allocReg(md.inst); + const raw_reg = try self.register_manager.allocReg(md.inst, .{}); break :blk registerAlias(raw_reg, lhs_ty.abiSize(self.target.*)); } } else blk: { - const raw_reg = try self.register_manager.allocReg(null); + const raw_reg = try self.register_manager.allocReg(null, .{}); break :blk registerAlias(raw_reg, lhs_ty.abiSize(self.target.*)); }, }; @@ -1431,7 +1431,7 @@ fn binOpImmediate( ).?; } else null; - const raw_reg = try self.register_manager.allocReg(track_inst); + const raw_reg = try self.register_manager.allocReg(track_inst, .{}); const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*)); if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg }); @@ -1452,11 +1452,11 @@ fn binOpImmediate( )) { break :blk lhs_reg; } else { - const raw_reg = try self.register_manager.allocReg(md.inst); + const raw_reg = try self.register_manager.allocReg(md.inst, .{}); break :blk registerAlias(raw_reg, lhs_ty.abiSize(self.target.*)); } } else blk: { - const raw_reg = try self.register_manager.allocReg(null); + const raw_reg = try self.register_manager.allocReg(null, .{}); break :blk registerAlias(raw_reg, lhs_ty.abiSize(self.target.*)); }, }; @@ -1872,7 +1872,7 @@ fn airOverflow(self: *Self, inst: Air.Inst.Index) !void { const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg); defer self.register_manager.unlockReg(dest_reg_lock); - const raw_truncated_reg = try self.register_manager.allocReg(null); + const raw_truncated_reg = try self.register_manager.allocReg(null, .{}); const truncated_reg = registerAlias(raw_truncated_reg, lhs_ty.abiSize(self.target.*)); const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg); defer self.register_manager.unlockReg(truncated_reg_lock); @@ -1983,7 +1983,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg); defer self.register_manager.unlockReg(dest_reg_lock); - const truncated_reg = try self.register_manager.allocReg(null); + const truncated_reg = try self.register_manager.allocReg(null, .{}); const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg); defer self.register_manager.unlockReg(truncated_reg_lock); @@ -2048,7 +2048,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { defer if (rhs_lock) |reg| self.register_manager.unlockReg(reg); const lhs_reg = if (lhs_is_register) lhs.register else blk: { - const raw_reg = try self.register_manager.allocReg(null); + const raw_reg = try self.register_manager.allocReg(null, .{}); const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*)); break :blk reg; }; @@ -2056,7 +2056,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { defer if (new_lhs_lock) |reg| self.register_manager.unlockReg(reg); const rhs_reg = if (rhs_is_register) rhs.register else blk: { - const raw_reg = try self.register_manager.allocReg(null); + const raw_reg = try self.register_manager.allocReg(null, .{}); const reg = registerAlias(raw_reg, rhs_ty.abiAlignment(self.target.*)); break :blk reg; }; @@ -2067,7 +2067,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs); const dest_reg = blk: { - const raw_reg = try self.register_manager.allocReg(null); + const raw_reg = try self.register_manager.allocReg(null, .{}); const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*)); break :blk reg; }; @@ -2086,7 +2086,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { } }, }); - const dest_high_reg = try self.register_manager.allocReg(null); + const dest_high_reg = try self.register_manager.allocReg(null, .{}); const dest_high_reg_lock = self.register_manager.lockRegAssumeUnused(dest_high_reg); defer self.register_manager.unlockReg(dest_high_reg_lock); @@ -2136,7 +2136,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { } }, .unsigned => { - const dest_high_reg = try self.register_manager.allocReg(null); + const dest_high_reg = try self.register_manager.allocReg(null, .{}); const dest_high_reg_lock = self.register_manager.lockRegAssumeUnused(dest_high_reg); defer self.register_manager.unlockReg(dest_high_reg_lock); @@ -2192,7 +2192,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { }, } - const truncated_reg = try self.register_manager.allocReg(null); + const truncated_reg = try self.register_manager.allocReg(null, .{}); const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg); defer self.register_manager.unlockReg(truncated_reg_lock); @@ -2663,7 +2663,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo }, .stack_offset => |off| { if (elem_size <= 8) { - const raw_tmp_reg = try self.register_manager.allocReg(null); + const raw_tmp_reg = try self.register_manager.allocReg(null, .{}); const tmp_reg = registerAlias(raw_tmp_reg, elem_size); const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); defer self.register_manager.unlockReg(tmp_reg_lock); @@ -2672,7 +2672,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo try self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg }); } else { // TODO optimize the register allocation - const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null }); + const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null }, .{}); const regs_locks = self.register_manager.lockRegsAssumeUnused(4, regs); defer for (regs_locks) |reg| { self.register_manager.unlockReg(reg); @@ -2887,7 +2887,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type }, else => { if (abi_size <= 8) { - const raw_tmp_reg = try self.register_manager.allocReg(null); + const raw_tmp_reg = try self.register_manager.allocReg(null, .{}); const tmp_reg = registerAlias(raw_tmp_reg, abi_size); const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); defer self.register_manager.unlockReg(tmp_reg_lock); @@ -3002,7 +3002,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { // TODO return special MCValue condition flags // get overflow bit: set register to C flag // resp. V flag - const raw_dest_reg = try self.register_manager.allocReg(null); + const raw_dest_reg = try self.register_manager.allocReg(null, .{}); const dest_reg = raw_dest_reg.to32(); // C flag: cset reg, cs @@ -4065,7 +4065,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro const overflow_bit_ty = ty.structFieldType(1); const overflow_bit_offset = @intCast(u32, ty.structFieldOffset(1, self.target.*)); - const raw_cond_reg = try self.register_manager.allocReg(null); + const raw_cond_reg = try self.register_manager.allocReg(null, .{}); const cond_reg = registerAlias( raw_cond_reg, @intCast(u32, overflow_bit_ty.abiSize(self.target.*)), @@ -4113,7 +4113,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro const ptr_ty = Type.initPayload(&ptr_ty_payload.base); // TODO call extern memcpy - const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null }); + const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null }, .{}); const regs_locks = self.register_manager.lockRegsAssumeUnused(5, regs); defer for (regs_locks) |reg| { self.register_manager.unlockReg(reg); diff --git a/src/arch/arm/CodeGen.zig b/src/arch/arm/CodeGen.zig index bfc7f687fa..c792615c79 100644 --- a/src/arch/arm/CodeGen.zig +++ b/src/arch/arm/CodeGen.zig @@ -874,7 +874,7 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue { const ptr_bits = self.target.cpu.arch.ptrBitWidth(); const ptr_bytes: u64 = @divExact(ptr_bits, 8); if (abi_size <= ptr_bytes) { - if (self.register_manager.tryAllocReg(inst)) |reg| { + if (self.register_manager.tryAllocReg(inst, .{})) |reg| { return MCValue{ .register = reg }; } } @@ -939,7 +939,7 @@ fn spillCompareFlagsIfOccupied(self: *Self) !void { /// allocated. A second call to `copyToTmpRegister` may return the same register. /// This can have a side effect of spilling instructions to the stack to free up a register. fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register { - const reg = try self.register_manager.allocReg(null); + const reg = try self.register_manager.allocReg(null, .{}); try self.genSetReg(ty, reg, mcv); return reg; } @@ -948,7 +948,7 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register { /// `reg_owner` is the instruction that gets associated with the register in the register table. /// This can have a side effect of spilling instructions to the stack to free up a register. fn copyToNewRegister(self: *Self, reg_owner: Air.Inst.Index, mcv: MCValue) !MCValue { - const reg = try self.register_manager.allocReg(reg_owner); + const reg = try self.register_manager.allocReg(reg_owner, .{}); try self.genSetReg(self.air.typeOfIndex(reg_owner), reg, mcv); return MCValue{ .register = reg }; } @@ -1065,9 +1065,9 @@ fn trunc( if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) { break :blk operand_reg; } else { - break :blk try self.register_manager.allocReg(inst); + break :blk try self.register_manager.allocReg(inst, .{}); } - } else try self.register_manager.allocReg(null); + } else try self.register_manager.allocReg(null, .{}); switch (info_b.bits) { 32 => { @@ -1153,7 +1153,7 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void { break :blk op_reg; } - break :blk try self.register_manager.allocReg(null); + break :blk try self.register_manager.allocReg(null, .{}); }; _ = try self.addInst(.{ @@ -1183,7 +1183,7 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void { break :blk op_reg; } - break :blk try self.register_manager.allocReg(null); + break :blk try self.register_manager.allocReg(null, .{}); }; _ = try self.addInst(.{ @@ -1254,9 +1254,9 @@ fn minMax( } else if (rhs_is_register and self.reuseOperand(inst, bin_op.rhs, 1, rhs)) { break :blk rhs_reg; } else { - break :blk try self.register_manager.allocReg(inst); + break :blk try self.register_manager.allocReg(inst, .{}); } - } else try self.register_manager.allocReg(null); + } else try self.register_manager.allocReg(null, .{}); // lhs == reg should have been checked by airMinMax // @@ -1438,7 +1438,7 @@ fn airOverflow(self: *Self, inst: Air.Inst.Index) !void { const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg); defer self.register_manager.unlockReg(dest_reg_lock); - const truncated_reg = try self.register_manager.allocReg(null); + const truncated_reg = try self.register_manager.allocReg(null, .{}); const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg); defer self.register_manager.unlockReg(truncated_reg_lock); @@ -1543,7 +1543,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg); defer self.register_manager.unlockReg(dest_reg_lock); - const truncated_reg = try self.register_manager.allocReg(null); + const truncated_reg = try self.register_manager.allocReg(null, .{}); const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg); defer self.register_manager.unlockReg(truncated_reg_lock); @@ -1582,18 +1582,18 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { const lhs_reg = if (lhs_is_register) lhs.register else - try self.register_manager.allocReg(null); + try self.register_manager.allocReg(null, .{}); const new_lhs_lock = self.register_manager.lockReg(lhs_reg); defer if (new_lhs_lock) |reg| self.register_manager.unlockReg(reg); const rhs_reg = if (rhs_is_register) rhs.register else - try self.register_manager.allocReg(null); + try self.register_manager.allocReg(null, .{}); const new_rhs_lock = self.register_manager.lockReg(rhs_reg); defer if (new_rhs_lock) |reg| self.register_manager.unlockReg(reg); - const dest_regs = try self.register_manager.allocRegs(2, .{ null, null }); + const dest_regs = try self.register_manager.allocRegs(2, .{ null, null }, .{}); const dest_regs_locks = self.register_manager.lockRegsAssumeUnused(2, dest_regs); defer for (dest_regs_locks) |reg| { self.register_manager.unlockReg(reg); @@ -1604,7 +1604,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs); if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs); - const truncated_reg = try self.register_manager.allocReg(null); + const truncated_reg = try self.register_manager.allocReg(null, .{}); const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg); defer self.register_manager.unlockReg(truncated_reg_lock); @@ -2026,7 +2026,7 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void { const base_reg_lock = self.register_manager.lockRegAssumeUnused(base_reg); defer self.register_manager.unlockReg(base_reg_lock); - const dst_reg = try self.register_manager.allocReg(inst); + const dst_reg = try self.register_manager.allocReg(inst, .{}); const dst_mcv = MCValue{ .register = dst_reg }; const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg); defer self.register_manager.unlockReg(dst_reg_lock); @@ -2234,7 +2234,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo }, .stack_offset => |off| { if (elem_size <= 4) { - const tmp_reg = try self.register_manager.allocReg(null); + const tmp_reg = try self.register_manager.allocReg(null, .{}); const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); defer self.register_manager.unlockReg(tmp_reg_lock); @@ -2242,7 +2242,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo try self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg }); } else { // TODO optimize the register allocation - const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null }); + const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null }, .{}); const regs_locks = self.register_manager.lockRegsAssumeUnused(4, regs); defer for (regs_locks) |reg_locked| { self.register_manager.unlockReg(reg_locked); @@ -2271,7 +2271,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo .stack_offset, .stack_argument_offset, => { - const reg = try self.register_manager.allocReg(null); + const reg = try self.register_manager.allocReg(null, .{}); const reg_lock = self.register_manager.lockRegAssumeUnused(reg); defer self.register_manager.unlockReg(reg_lock); @@ -2338,14 +2338,14 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type }, else => { if (elem_size <= 4) { - const tmp_reg = try self.register_manager.allocReg(null); + const tmp_reg = try self.register_manager.allocReg(null, .{}); const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); defer self.register_manager.unlockReg(tmp_reg_lock); try self.genSetReg(value_ty, tmp_reg, value); try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty); } else { - const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null }); + const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null }, .{}); const regs_locks = self.register_manager.lockRegsAssumeUnused(4, regs); defer for (regs_locks) |reg| { self.register_manager.unlockReg(reg); @@ -2487,7 +2487,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { 1 => { // get overflow bit: set register to C flag // resp. V flag - const dest_reg = try self.register_manager.allocReg(null); + const dest_reg = try self.register_manager.allocReg(null, .{}); // mov reg, #0 _ = try self.addInst(.{ @@ -2567,7 +2567,7 @@ fn binOpRegister( break :inst Air.refToIndex(md.lhs).?; } else null; - const reg = try self.register_manager.allocReg(track_inst); + const reg = try self.register_manager.allocReg(track_inst, .{}); if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg }); @@ -2581,7 +2581,7 @@ fn binOpRegister( break :inst Air.refToIndex(md.rhs).?; } else null; - const reg = try self.register_manager.allocReg(track_inst); + const reg = try self.register_manager.allocReg(track_inst, .{}); if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg }); @@ -2598,9 +2598,9 @@ fn binOpRegister( } else if (rhs_is_register and self.reuseOperand(md.inst, md.rhs, 1, rhs)) { break :blk rhs_reg; } else { - break :blk try self.register_manager.allocReg(md.inst); + break :blk try self.register_manager.allocReg(md.inst, .{}); } - } else try self.register_manager.allocReg(null), + } else try self.register_manager.allocReg(null, .{}), }; if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs); @@ -2684,7 +2684,7 @@ fn binOpImmediate( ).?; } else null; - const reg = try self.register_manager.allocReg(track_inst); + const reg = try self.register_manager.allocReg(track_inst, .{}); if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg }); @@ -2704,9 +2704,9 @@ fn binOpImmediate( )) { break :blk lhs_reg; } else { - break :blk try self.register_manager.allocReg(md.inst); + break :blk try self.register_manager.allocReg(md.inst, .{}); } - } else try self.register_manager.allocReg(null), + } else try self.register_manager.allocReg(null, .{}), }; if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs); @@ -4363,7 +4363,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro const overflow_bit_ty = ty.structFieldType(1); const overflow_bit_offset = @intCast(u32, ty.structFieldOffset(1, self.target.*)); - const cond_reg = try self.register_manager.allocReg(null); + const cond_reg = try self.register_manager.allocReg(null, .{}); // C flag: movcs reg, #1 // V flag: movvs reg, #1 @@ -4408,7 +4408,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro const ptr_ty = Type.initPayload(&ptr_ty_payload.base); // TODO call extern memcpy - const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null }); + const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null }, .{}); const src_reg = regs[0]; const dst_reg = regs[1]; const len_reg = regs[2]; @@ -4782,7 +4782,7 @@ fn genSetStackArgument(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) I const ptr_ty = Type.initPayload(&ptr_ty_payload.base); // TODO call extern memcpy - const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null }); + const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null }, .{}); const src_reg = regs[0]; const dst_reg = regs[1]; const len_reg = regs[2]; diff --git a/src/arch/riscv64/CodeGen.zig b/src/arch/riscv64/CodeGen.zig index c2f9b2e36d..79bedb30d8 100644 --- a/src/arch/riscv64/CodeGen.zig +++ b/src/arch/riscv64/CodeGen.zig @@ -803,7 +803,7 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue { const ptr_bits = self.target.cpu.arch.ptrBitWidth(); const ptr_bytes: u64 = @divExact(ptr_bits, 8); if (abi_size <= ptr_bytes) { - if (self.register_manager.tryAllocReg(inst)) |reg| { + if (self.register_manager.tryAllocReg(inst, .{})) |reg| { return MCValue{ .register = reg }; } } @@ -826,7 +826,7 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void /// allocated. A second call to `copyToTmpRegister` may return the same register. /// This can have a side effect of spilling instructions to the stack to free up a register. fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register { - const reg = try self.register_manager.allocReg(null); + const reg = try self.register_manager.allocReg(null, .{}); try self.genSetReg(ty, reg, mcv); return reg; } @@ -835,7 +835,7 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register { /// `reg_owner` is the instruction that gets associated with the register in the register table. /// This can have a side effect of spilling instructions to the stack to free up a register. fn copyToNewRegister(self: *Self, reg_owner: Air.Inst.Index, mcv: MCValue) !MCValue { - const reg = try self.register_manager.allocReg(reg_owner); + const reg = try self.register_manager.allocReg(reg_owner, .{}); try self.genSetReg(self.air.typeOfIndex(reg_owner), reg, mcv); return MCValue{ .register = reg }; } @@ -958,7 +958,7 @@ fn binOpRegister( break :inst Air.refToIndex(bin_op.lhs).?; } else null; - const reg = try self.register_manager.allocReg(track_inst); + const reg = try self.register_manager.allocReg(track_inst, .{}); if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg }); @@ -973,7 +973,7 @@ fn binOpRegister( break :inst Air.refToIndex(bin_op.rhs).?; } else null; - const reg = try self.register_manager.allocReg(track_inst); + const reg = try self.register_manager.allocReg(track_inst, .{}); if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg }); @@ -990,9 +990,9 @@ fn binOpRegister( } else if (rhs_is_register and self.reuseOperand(inst, bin_op.rhs, 1, rhs)) { break :blk rhs_reg; } else { - break :blk try self.register_manager.allocReg(inst); + break :blk try self.register_manager.allocReg(inst, .{}); } - } else try self.register_manager.allocReg(null); + } else try self.register_manager.allocReg(null, .{}); if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs); if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs); @@ -1482,7 +1482,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo .memory, .stack_offset, => { - const reg = try self.register_manager.allocReg(null); + const reg = try self.register_manager.allocReg(null, .{}); const reg_lock = self.register_manager.lockRegAssumeUnused(reg); defer self.register_manager.unlockReg(reg_lock); diff --git a/src/arch/sparc64/CodeGen.zig b/src/arch/sparc64/CodeGen.zig index aa679cac6d..e8cb5d3100 100644 --- a/src/arch/sparc64/CodeGen.zig +++ b/src/arch/sparc64/CodeGen.zig @@ -1613,7 +1613,7 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue { if (reg_ok) { // Make sure the type can fit in a register before we try to allocate one. if (abi_size <= 8) { - if (self.register_manager.tryAllocReg(inst)) |reg| { + if (self.register_manager.tryAllocReg(inst, .{})) |reg| { return MCValue{ .register = reg }; } } @@ -1854,7 +1854,7 @@ fn binOpImmediate( ).?; } else null; - const reg = try self.register_manager.allocReg(track_inst); + const reg = try self.register_manager.allocReg(track_inst, .{}); if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg }); @@ -1873,10 +1873,10 @@ fn binOpImmediate( )) { break :blk lhs_reg; } else { - break :blk try self.register_manager.allocReg(md.inst); + break :blk try self.register_manager.allocReg(md.inst, .{}); } } else blk: { - break :blk try self.register_manager.allocReg(null); + break :blk try self.register_manager.allocReg(null, .{}); }, }; @@ -1953,7 +1953,7 @@ fn binOpRegister( break :inst Air.refToIndex(md.lhs).?; } else null; - const reg = try self.register_manager.allocReg(track_inst); + const reg = try self.register_manager.allocReg(track_inst, .{}); if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg }); break :blk reg; @@ -1966,7 +1966,7 @@ fn binOpRegister( break :inst Air.refToIndex(md.rhs).?; } else null; - const reg = try self.register_manager.allocReg(track_inst); + const reg = try self.register_manager.allocReg(track_inst, .{}); if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg }); break :blk reg; @@ -1981,10 +1981,10 @@ fn binOpRegister( } else if (rhs_is_register and self.reuseOperand(md.inst, md.rhs, 1, rhs)) { break :blk rhs_reg; } else { - break :blk try self.register_manager.allocReg(md.inst); + break :blk try self.register_manager.allocReg(md.inst, .{}); } } else blk: { - break :blk try self.register_manager.allocReg(null); + break :blk try self.register_manager.allocReg(null, .{}); }, }; @@ -2077,7 +2077,7 @@ fn brVoid(self: *Self, block: Air.Inst.Index) !void { /// allocated. A second call to `copyToTmpRegister` may return the same register. /// This can have a side effect of spilling instructions to the stack to free up a register. fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register { - const reg = try self.register_manager.allocReg(null); + const reg = try self.register_manager.allocReg(null, .{}); try self.genSetReg(ty, reg, mcv); return reg; } @@ -2364,7 +2364,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void }); } else { // Need to allocate a temporary register to load 64-bit immediates. - const tmp_reg = try self.register_manager.allocReg(null); + const tmp_reg = try self.register_manager.allocReg(null, .{}); try self.genSetReg(ty, tmp_reg, .{ .immediate = @truncate(u32, x) }); try self.genSetReg(ty, reg, .{ .immediate = @truncate(u32, x >> 32) }); @@ -2478,7 +2478,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro }; const ptr_ty = Type.initPayload(&ptr_ty_payload.base); - const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null }); + const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null }, .{}); const regs_locks = self.register_manager.lockRegsAssumeUnused(4, regs); defer for (regs_locks) |reg| { self.register_manager.unlockReg(reg); @@ -2717,14 +2717,14 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo }, .stack_offset => |off| { if (elem_size <= 8) { - const tmp_reg = try self.register_manager.allocReg(null); + const tmp_reg = try self.register_manager.allocReg(null, .{}); const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); defer self.register_manager.unlockReg(tmp_reg_lock); try self.load(.{ .register = tmp_reg }, ptr, ptr_ty); try self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg }); } else { - const regs = try self.register_manager.allocRegs(3, .{ null, null, null }); + const regs = try self.register_manager.allocRegs(3, .{ null, null, null }, .{}); const regs_locks = self.register_manager.lockRegsAssumeUnused(3, regs); defer for (regs_locks) |reg| { self.register_manager.unlockReg(reg); diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index aa6d04d5d9..aebbc03e46 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -882,7 +882,7 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue { // TODO check if AVX available const ptr_bytes: u64 = 32; if (abi_size <= ptr_bytes) { - if (self.register_manager.tryAllocReg(inst)) |reg| { + if (self.register_manager.tryAllocReg(inst, .{})) |reg| { return MCValue{ .register = registerAlias(reg, abi_size) }; } } @@ -892,7 +892,7 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue { const ptr_bits = self.target.cpu.arch.ptrBitWidth(); const ptr_bytes: u64 = @divExact(ptr_bits, 8); if (abi_size <= ptr_bytes) { - if (self.register_manager.tryAllocReg(inst)) |reg| { + if (self.register_manager.tryAllocReg(inst, .{})) |reg| { return MCValue{ .register = registerAlias(reg, abi_size) }; } } @@ -963,7 +963,7 @@ pub fn spillRegisters(self: *Self, comptime count: comptime_int, registers: [cou /// allocated. A second call to `copyToTmpRegister` may return the same register. /// This can have a side effect of spilling instructions to the stack to free up a register. fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register { - const reg = try self.register_manager.allocReg(null); + const reg = try self.register_manager.allocReg(null, .{}); try self.genSetReg(ty, reg, mcv); return reg; } @@ -973,7 +973,7 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register { /// This can have a side effect of spilling instructions to the stack to free up a register. /// WARNING make sure that the allocated register matches the returned MCValue from an instruction! fn copyToRegisterWithInstTracking(self: *Self, reg_owner: Air.Inst.Index, ty: Type, mcv: MCValue) !MCValue { - const reg = try self.register_manager.allocReg(reg_owner); + const reg = try self.register_manager.allocReg(reg_owner, .{}); try self.genSetReg(ty, reg, mcv); return MCValue{ .register = reg }; } @@ -1029,7 +1029,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void { }; defer if (operand_lock) |lock| self.register_manager.unlockReg(lock); - const reg = try self.register_manager.allocReg(inst); + const reg = try self.register_manager.allocReg(inst, .{}); try self.genSetReg(dest_ty, reg, .{ .immediate = 0 }); try self.genSetReg(operand_ty, reg, operand); break :blk MCValue{ .register = reg }; @@ -1384,7 +1384,7 @@ fn genSetStackTruncatedOverflowCompare( .unsigned => ty, }; - const temp_regs = try self.register_manager.allocRegs(3, .{ null, null, null }); + const temp_regs = try self.register_manager.allocRegs(3, .{ null, null, null }, .{}); const temp_regs_locks = self.register_manager.lockRegsAssumeUnused(3, temp_regs); defer for (temp_regs_locks) |rreg| { self.register_manager.unlockReg(rreg); @@ -2046,7 +2046,7 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue { const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg); defer self.register_manager.unlockReg(offset_reg_lock); - const addr_reg = try self.register_manager.allocReg(null); + const addr_reg = try self.register_manager.allocReg(null, .{}); switch (slice_mcv) { .stack_offset => |off| { // mov reg, [rbp - 8] @@ -2125,7 +2125,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void { const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg); defer self.register_manager.unlockReg(offset_reg_lock); - const addr_reg = try self.register_manager.allocReg(null); + const addr_reg = try self.register_manager.allocReg(null, .{}); switch (array) { .register => { const off = @intCast(i32, try self.allocMem( @@ -2492,7 +2492,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo }, .stack_offset => |off| { if (abi_size <= 8) { - const tmp_reg = try self.register_manager.allocReg(null); + const tmp_reg = try self.register_manager.allocReg(null, .{}); try self.load(.{ .register = tmp_reg }, ptr, ptr_ty); return self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg }, .{}); } @@ -2693,7 +2693,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type }; defer if (value_lock) |lock| self.register_manager.unlockReg(lock); - const addr_reg = try self.register_manager.allocReg(null); + const addr_reg = try self.register_manager.allocReg(null, .{}); const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg); defer self.register_manager.unlockReg(addr_reg_lock); @@ -2765,7 +2765,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type .memory, => { if (abi_size <= 8) { - const tmp_reg = try self.register_manager.allocReg(null); + const tmp_reg = try self.register_manager.allocReg(null, .{}); const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); defer self.register_manager.unlockReg(tmp_reg_lock); @@ -2883,7 +2883,7 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde if (can_reuse_operand) { break :blk reg; } else { - const result_reg = try self.register_manager.allocReg(inst); + const result_reg = try self.register_manager.allocReg(inst, .{}); try self.genSetReg(ptr_ty, result_reg, mcv); break :blk result_reg; } @@ -2984,7 +2984,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { const reg_lock = self.register_manager.lockRegAssumeUnused(reg); defer self.register_manager.unlockReg(reg_lock); - const dst_reg = try self.register_manager.allocReg(inst); + const dst_reg = try self.register_manager.allocReg(inst, .{}); const flags: u2 = switch (mcv) { .register_overflow_unsigned => 0b10, .register_overflow_signed => 0b00, @@ -5362,7 +5362,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl const overflow_bit_ty = ty.structFieldType(1); const overflow_bit_offset = ty.structFieldOffset(1, self.target.*); - const tmp_reg = try self.register_manager.allocReg(null); + const tmp_reg = try self.register_manager.allocReg(null, .{}); const flags: u2 = switch (mcv) { .register_overflow_unsigned => 0b10, .register_overflow_signed => 0b00, @@ -5580,7 +5580,7 @@ fn genInlineMemcpy( null; defer if (dsbase_lock) |lock| self.register_manager.unlockReg(lock); - const dst_addr_reg = try self.register_manager.allocReg(null); + const dst_addr_reg = try self.register_manager.allocReg(null, .{}); switch (dst_ptr) { .memory, .got_load, @@ -5615,7 +5615,7 @@ fn genInlineMemcpy( const dst_addr_reg_lock = self.register_manager.lockRegAssumeUnused(dst_addr_reg); defer self.register_manager.unlockReg(dst_addr_reg_lock); - const src_addr_reg = try self.register_manager.allocReg(null); + const src_addr_reg = try self.register_manager.allocReg(null, .{}); switch (src_ptr) { .memory, .got_load, @@ -5650,7 +5650,7 @@ fn genInlineMemcpy( const src_addr_reg_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg); defer self.register_manager.unlockReg(src_addr_reg_lock); - const regs = try self.register_manager.allocRegs(2, .{ null, null }); + const regs = try self.register_manager.allocRegs(2, .{ null, null }, .{}); const count_reg = regs[0].to64(); const tmp_reg = regs[1].to8(); @@ -5750,7 +5750,7 @@ fn genInlineMemset( const rax_lock = self.register_manager.lockRegAssumeUnused(.rax); defer self.register_manager.unlockReg(rax_lock); - const addr_reg = try self.register_manager.allocReg(null); + const addr_reg = try self.register_manager.allocReg(null, .{}); switch (dst_ptr) { .memory, .got_load, @@ -6018,7 +6018,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void switch (ty.tag()) { .f32 => return self.fail("TODO genSetReg from memory for f32", .{}), .f64 => { - const base_reg = try self.register_manager.allocReg(null); + const base_reg = try self.register_manager.allocReg(null, .{}); try self.loadMemPtrIntoRegister(base_reg, Type.usize, mcv); _ = try self.addInst(.{ .tag = .mov_f64, @@ -6328,7 +6328,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void { const src: MCValue = blk: { switch (src_ptr) { .got_load, .direct_load, .memory => { - const reg = try self.register_manager.allocReg(null); + const reg = try self.register_manager.allocReg(null, .{}); try self.loadMemPtrIntoRegister(reg, src_ty, src_ptr); _ = try self.addInst(.{ .tag = .mov, diff --git a/src/register_manager.zig b/src/register_manager.zig index 2c0502e867..25d8ef8675 100644 --- a/src/register_manager.zig +++ b/src/register_manager.zig @@ -173,6 +173,10 @@ pub fn RegisterManager( return self.locked_registers != 0; } + const AllocOpts = struct { + selector_mask: ?FreeRegInt = null, + }; + /// Allocates a specified number of registers, optionally /// tracking them. Returns `null` if not enough registers are /// free. @@ -180,7 +184,9 @@ pub fn RegisterManager( self: *Self, comptime count: comptime_int, insts: [count]?Air.Inst.Index, + opts: AllocOpts, ) ?[count]Register { + _ = opts; comptime assert(count > 0 and count <= tracked_registers.len); const free_and_not_locked_registers = self.free_registers & ~self.locked_registers; @@ -216,8 +222,8 @@ pub fn RegisterManager( /// Allocates a register and optionally tracks it with a /// corresponding instruction. Returns `null` if all registers /// are allocated. - pub fn tryAllocReg(self: *Self, inst: ?Air.Inst.Index) ?Register { - return if (tryAllocRegs(self, 1, .{inst})) |regs| regs[0] else null; + pub fn tryAllocReg(self: *Self, inst: ?Air.Inst.Index, opts: AllocOpts) ?Register { + return if (tryAllocRegs(self, 1, .{inst}, opts)) |regs| regs[0] else null; } /// Allocates a specified number of registers, optionally @@ -227,12 +233,13 @@ pub fn RegisterManager( self: *Self, comptime count: comptime_int, insts: [count]?Air.Inst.Index, + opts: AllocOpts, ) AllocateRegistersError![count]Register { comptime assert(count > 0 and count <= tracked_registers.len); const locked_registers_count = @popCount(FreeRegInt, self.locked_registers); if (count > tracked_registers.len - locked_registers_count) return error.OutOfRegisters; - const result = self.tryAllocRegs(count, insts) orelse blk: { + const result = self.tryAllocRegs(count, insts, opts) orelse blk: { // We'll take over the first count registers. Spill // the instructions that were previously there to a // stack allocations. @@ -275,8 +282,8 @@ pub fn RegisterManager( /// Allocates a register and optionally tracks it with a /// corresponding instruction. - pub fn allocReg(self: *Self, inst: ?Air.Inst.Index) AllocateRegistersError!Register { - return (try self.allocRegs(1, .{inst}))[0]; + pub fn allocReg(self: *Self, inst: ?Air.Inst.Index, opts: AllocOpts) AllocateRegistersError!Register { + return (try self.allocRegs(1, .{inst}, opts))[0]; } /// Spills the register if it is currently allocated. If a From f34615082058549d8945a39195cf0dc826b688e1 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 18 May 2022 12:23:07 +0200 Subject: [PATCH 09/19] x64: use register classes mask to select between gp and avx --- src/arch/x86_64/CodeGen.zig | 69 ++++++++++++++++++++++++++----------- src/arch/x86_64/abi.zig | 13 +++---- src/arch/x86_64/bits.zig | 18 +++++----- src/register_manager.zig | 20 ++++++++--- 4 files changed, 81 insertions(+), 39 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index aebbc03e46..7f41d62141 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -38,6 +38,9 @@ const c_abi_int_return_regs = abi.c_abi_int_return_regs; const RegisterManager = RegisterManagerFn(Self, Register, &allocatable_registers); const RegisterLock = RegisterManager.RegisterLock; const Register = bits.Register; +const RegisterClass = abi.RegisterClass; +const gp = RegisterClass.gp; +const avx = RegisterClass.avx; const InnerError = error{ OutOfMemory, @@ -882,7 +885,9 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue { // TODO check if AVX available const ptr_bytes: u64 = 32; if (abi_size <= ptr_bytes) { - if (self.register_manager.tryAllocReg(inst, .{})) |reg| { + if (self.register_manager.tryAllocReg(inst, .{ + .selector_mask = avx, + })) |reg| { return MCValue{ .register = registerAlias(reg, abi_size) }; } } @@ -892,7 +897,9 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue { const ptr_bits = self.target.cpu.arch.ptrBitWidth(); const ptr_bytes: u64 = @divExact(ptr_bits, 8); if (abi_size <= ptr_bytes) { - if (self.register_manager.tryAllocReg(inst, .{})) |reg| { + if (self.register_manager.tryAllocReg(inst, .{ + .selector_mask = gp, + })) |reg| { return MCValue{ .register = registerAlias(reg, abi_size) }; } } @@ -963,7 +970,13 @@ pub fn spillRegisters(self: *Self, comptime count: comptime_int, registers: [cou /// allocated. A second call to `copyToTmpRegister` may return the same register. /// This can have a side effect of spilling instructions to the stack to free up a register. fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register { - const reg = try self.register_manager.allocReg(null, .{}); + const mask = switch (ty.zigTypeTag()) { + .Float => avx, + else => gp, + }; + const reg: Register = try self.register_manager.allocReg(null, .{ + .selector_mask = mask, + }); try self.genSetReg(ty, reg, mcv); return reg; } @@ -973,7 +986,13 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register { /// This can have a side effect of spilling instructions to the stack to free up a register. /// WARNING make sure that the allocated register matches the returned MCValue from an instruction! fn copyToRegisterWithInstTracking(self: *Self, reg_owner: Air.Inst.Index, ty: Type, mcv: MCValue) !MCValue { - const reg = try self.register_manager.allocReg(reg_owner, .{}); + const mask = switch (ty.zigTypeTag()) { + .Float => avx, + else => gp, + }; + const reg: Register = try self.register_manager.allocReg(reg_owner, .{ + .selector_mask = mask, + }); try self.genSetReg(ty, reg, mcv); return MCValue{ .register = reg }; } @@ -1029,7 +1048,9 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void { }; defer if (operand_lock) |lock| self.register_manager.unlockReg(lock); - const reg = try self.register_manager.allocReg(inst, .{}); + const reg = try self.register_manager.allocReg(inst, .{ + .selector_mask = gp, + }); try self.genSetReg(dest_ty, reg, .{ .immediate = 0 }); try self.genSetReg(operand_ty, reg, operand); break :blk MCValue{ .register = reg }; @@ -1384,7 +1405,9 @@ fn genSetStackTruncatedOverflowCompare( .unsigned => ty, }; - const temp_regs = try self.register_manager.allocRegs(3, .{ null, null, null }, .{}); + const temp_regs = try self.register_manager.allocRegs(3, .{ null, null, null }, .{ + .selector_mask = gp, + }); const temp_regs_locks = self.register_manager.lockRegsAssumeUnused(3, temp_regs); defer for (temp_regs_locks) |rreg| { self.register_manager.unlockReg(rreg); @@ -2046,7 +2069,9 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue { const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg); defer self.register_manager.unlockReg(offset_reg_lock); - const addr_reg = try self.register_manager.allocReg(null, .{}); + const addr_reg = try self.register_manager.allocReg(null, .{ + .selector_mask = gp, + }); switch (slice_mcv) { .stack_offset => |off| { // mov reg, [rbp - 8] @@ -2125,7 +2150,9 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void { const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg); defer self.register_manager.unlockReg(offset_reg_lock); - const addr_reg = try self.register_manager.allocReg(null, .{}); + const addr_reg = try self.register_manager.allocReg(null, .{ + .selector_mask = gp, + }); switch (array) { .register => { const off = @intCast(i32, try self.allocMem( @@ -2492,7 +2519,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo }, .stack_offset => |off| { if (abi_size <= 8) { - const tmp_reg = try self.register_manager.allocReg(null, .{}); + const tmp_reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp }); try self.load(.{ .register = tmp_reg }, ptr, ptr_ty); return self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg }, .{}); } @@ -2693,7 +2720,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type }; defer if (value_lock) |lock| self.register_manager.unlockReg(lock); - const addr_reg = try self.register_manager.allocReg(null, .{}); + const addr_reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp }); const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg); defer self.register_manager.unlockReg(addr_reg_lock); @@ -2765,7 +2792,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type .memory, => { if (abi_size <= 8) { - const tmp_reg = try self.register_manager.allocReg(null, .{}); + const tmp_reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp }); const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); defer self.register_manager.unlockReg(tmp_reg_lock); @@ -2883,7 +2910,7 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde if (can_reuse_operand) { break :blk reg; } else { - const result_reg = try self.register_manager.allocReg(inst, .{}); + const result_reg = try self.register_manager.allocReg(inst, .{ .selector_mask = gp }); try self.genSetReg(ptr_ty, result_reg, mcv); break :blk result_reg; } @@ -2984,7 +3011,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { const reg_lock = self.register_manager.lockRegAssumeUnused(reg); defer self.register_manager.unlockReg(reg_lock); - const dst_reg = try self.register_manager.allocReg(inst, .{}); + const dst_reg = try self.register_manager.allocReg(inst, .{ .selector_mask = gp }); const flags: u2 = switch (mcv) { .register_overflow_unsigned => 0b10, .register_overflow_signed => 0b00, @@ -5362,7 +5389,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl const overflow_bit_ty = ty.structFieldType(1); const overflow_bit_offset = ty.structFieldOffset(1, self.target.*); - const tmp_reg = try self.register_manager.allocReg(null, .{}); + const tmp_reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp }); const flags: u2 = switch (mcv) { .register_overflow_unsigned => 0b10, .register_overflow_signed => 0b00, @@ -5580,7 +5607,7 @@ fn genInlineMemcpy( null; defer if (dsbase_lock) |lock| self.register_manager.unlockReg(lock); - const dst_addr_reg = try self.register_manager.allocReg(null, .{}); + const dst_addr_reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp }); switch (dst_ptr) { .memory, .got_load, @@ -5615,7 +5642,7 @@ fn genInlineMemcpy( const dst_addr_reg_lock = self.register_manager.lockRegAssumeUnused(dst_addr_reg); defer self.register_manager.unlockReg(dst_addr_reg_lock); - const src_addr_reg = try self.register_manager.allocReg(null, .{}); + const src_addr_reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp }); switch (src_ptr) { .memory, .got_load, @@ -5650,7 +5677,9 @@ fn genInlineMemcpy( const src_addr_reg_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg); defer self.register_manager.unlockReg(src_addr_reg_lock); - const regs = try self.register_manager.allocRegs(2, .{ null, null }, .{}); + const regs = try self.register_manager.allocRegs(2, .{ null, null }, .{ + .selector_mask = gp, + }); const count_reg = regs[0].to64(); const tmp_reg = regs[1].to8(); @@ -5750,7 +5779,7 @@ fn genInlineMemset( const rax_lock = self.register_manager.lockRegAssumeUnused(.rax); defer self.register_manager.unlockReg(rax_lock); - const addr_reg = try self.register_manager.allocReg(null, .{}); + const addr_reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp }); switch (dst_ptr) { .memory, .got_load, @@ -6018,7 +6047,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void switch (ty.tag()) { .f32 => return self.fail("TODO genSetReg from memory for f32", .{}), .f64 => { - const base_reg = try self.register_manager.allocReg(null, .{}); + const base_reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp }); try self.loadMemPtrIntoRegister(base_reg, Type.usize, mcv); _ = try self.addInst(.{ .tag = .mov_f64, @@ -6328,7 +6357,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void { const src: MCValue = blk: { switch (src_ptr) { .got_load, .direct_load, .memory => { - const reg = try self.register_manager.allocReg(null, .{}); + const reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp }); try self.loadMemPtrIntoRegister(reg, src_ty, src_ptr); _ = try self.addInst(.{ .tag = .mov, diff --git a/src/arch/x86_64/abi.zig b/src/arch/x86_64/abi.zig index 85bf3a0790..046948ff68 100644 --- a/src/arch/x86_64/abi.zig +++ b/src/arch/x86_64/abi.zig @@ -384,11 +384,12 @@ pub const avx_regs = [_]Register{ }; pub const allocatable_registers = callee_preserved_regs ++ caller_preserved_regs ++ avx_regs; -// Masks for register manager -const FreeRegInt = std.meta.Int(.unsigned, allocatable_registers.len); -// TODO -pub const gp_mask: FreeRegInt = 0x3fff; -pub const avx_mask: FreeRegInt = 0x3fff_c000; - pub const c_abi_int_param_regs = [_]Register{ .rdi, .rsi, .rdx, .rcx, .r8, .r9 }; pub const c_abi_int_return_regs = [_]Register{ .rax, .rdx }; + +// Masks for register manager +const FreeRegInt = std.meta.Int(.unsigned, allocatable_registers.len); +pub const RegisterClass = struct { + pub const gp: FreeRegInt = 0x3fff; + pub const avx: FreeRegInt = 0x3fff_c000; +}; diff --git a/src/arch/x86_64/bits.zig b/src/arch/x86_64/bits.zig index 85bd190e2b..a0ca774cae 100644 --- a/src/arch/x86_64/bits.zig +++ b/src/arch/x86_64/bits.zig @@ -56,10 +56,10 @@ pub const Register = enum(u7) { // Pseudo-value for MIR instructions. none, - pub fn id(self: Register) u5 { + pub fn id(self: Register) u7 { return switch (@enumToInt(self)) { - 0...63 => @as(u5, @truncate(u4, @enumToInt(self))), - 64...79 => @truncate(u5, @enumToInt(self)), + 0...63 => @as(u7, @truncate(u4, @enumToInt(self))), + 64...79 => @enumToInt(self), else => unreachable, }; } @@ -101,31 +101,31 @@ pub const Register = enum(u7) { } pub fn to256(self: Register) Register { - return @intToEnum(Register, @as(u8, self.id()) + 64); + return @intToEnum(Register, @as(u8, self.enc()) + 64); } pub fn to128(self: Register) Register { - return @intToEnum(Register, @as(u8, self.id()) + 80); + return @intToEnum(Register, @as(u8, self.enc()) + 80); } /// Convert from any register to its 64 bit alias. pub fn to64(self: Register) Register { - return @intToEnum(Register, self.id()); + return @intToEnum(Register, self.enc()); } /// Convert from any register to its 32 bit alias. pub fn to32(self: Register) Register { - return @intToEnum(Register, @as(u8, self.id()) + 16); + return @intToEnum(Register, @as(u8, self.enc()) + 16); } /// Convert from any register to its 16 bit alias. pub fn to16(self: Register) Register { - return @intToEnum(Register, @as(u8, self.id()) + 32); + return @intToEnum(Register, @as(u8, self.enc()) + 32); } /// Convert from any register to its 8 bit alias. pub fn to8(self: Register) Register { - return @intToEnum(Register, @as(u8, self.id()) + 48); + return @intToEnum(Register, @as(u8, self.enc()) + 48); } pub fn dwarfLocOp(self: Register) u8 { diff --git a/src/register_manager.zig b/src/register_manager.zig index 25d8ef8675..fbdf3ef2c3 100644 --- a/src/register_manager.zig +++ b/src/register_manager.zig @@ -66,6 +66,11 @@ pub fn RegisterManager( return mask; } + fn excludeRegister(reg: Register, mask: FreeRegInt) bool { + const reg_mask = getRegisterMask(reg) orelse return true; + return reg_mask & mask == 0; + } + fn markRegAllocated(self: *Self, reg: Register) void { const mask = getRegisterMask(reg) orelse return; self.allocated_registers |= mask; @@ -186,10 +191,11 @@ pub fn RegisterManager( insts: [count]?Air.Inst.Index, opts: AllocOpts, ) ?[count]Register { - _ = opts; comptime assert(count > 0 and count <= tracked_registers.len); - const free_and_not_locked_registers = self.free_registers & ~self.locked_registers; + const selector_mask = if (opts.selector_mask) |mask| mask else ~@as(FreeRegInt, 0); + const free_registers = self.free_registers & selector_mask; + const free_and_not_locked_registers = free_registers & ~self.locked_registers; const free_and_not_locked_registers_count = @popCount(FreeRegInt, free_and_not_locked_registers); if (free_and_not_locked_registers_count < count) return null; @@ -197,6 +203,7 @@ pub fn RegisterManager( var i: usize = 0; for (tracked_registers) |reg| { if (i >= count) break; + if (excludeRegister(reg, selector_mask)) continue; if (self.isRegLocked(reg)) continue; if (!self.isRegFree(reg)) continue; @@ -236,8 +243,12 @@ pub fn RegisterManager( opts: AllocOpts, ) AllocateRegistersError![count]Register { comptime assert(count > 0 and count <= tracked_registers.len); - const locked_registers_count = @popCount(FreeRegInt, self.locked_registers); - if (count > tracked_registers.len - locked_registers_count) return error.OutOfRegisters; + + const selector_mask = if (opts.selector_mask) |mask| mask else ~@as(FreeRegInt, 0); + const available_registers_count = @popCount(FreeRegInt, selector_mask); + const locked_registers = self.locked_registers & selector_mask; + const locked_registers_count = @popCount(FreeRegInt, locked_registers); + if (count > available_registers_count - locked_registers_count) return error.OutOfRegisters; const result = self.tryAllocRegs(count, insts, opts) orelse blk: { // We'll take over the first count registers. Spill @@ -247,6 +258,7 @@ pub fn RegisterManager( var i: usize = 0; for (tracked_registers) |reg| { if (i >= count) break; + if (excludeRegister(reg, selector_mask)) continue; if (self.isRegLocked(reg)) continue; regs[i] = reg; From 020f99d893be794376eea329af06838474744e80 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 18 May 2022 12:50:19 +0200 Subject: [PATCH 10/19] x64: remove special-casing of AVX for br() --- src/arch/x86_64/CodeGen.zig | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 7f41d62141..6d7684eef8 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -5042,21 +5042,12 @@ fn br(self: *Self, block: Air.Inst.Index, operand: Air.Inst.Ref) !void { if (block_mcv == .none) { block_data.mcv = switch (operand_mcv) { .none, .dead, .unreach => unreachable, - .stack_offset, .memory => operand_mcv, + .register, .stack_offset, .memory => operand_mcv, .compare_flags_signed, .compare_flags_unsigned, .immediate => blk: { const new_mcv = try self.allocRegOrMem(block, true); try self.setRegOrMem(self.air.typeOfIndex(block), new_mcv, operand_mcv); break :blk new_mcv; }, - .register => blk: { - if (self.air.typeOfIndex(block).zigTypeTag() == .Float) { - // TODO not needed; return operand_mcv ones we can transfer between XMM registers - const new_mcv = try self.allocRegOrMem(block, false); - try self.setRegOrMem(self.air.typeOfIndex(block), new_mcv, operand_mcv); - break :blk new_mcv; - } - break :blk operand_mcv; - }, else => return self.fail("TODO implement block_data.mcv = operand_mcv for {}", .{operand_mcv}), }; } else { From 36b939e8db902b01870d709faeb6d2725f4f73ae Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 18 May 2022 14:15:27 +0200 Subject: [PATCH 11/19] x64: handle basic f32 using AVX registers --- src/arch/x86_64/CodeGen.zig | 170 +++++++++++--------- src/arch/x86_64/Emit.zig | 208 +++++++++++------------- src/arch/x86_64/Mir.zig | 3 + test/behavior.zig | 312 ++++++++++++++++++------------------ 4 files changed, 352 insertions(+), 341 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 6d7684eef8..7896a2f3b7 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -3469,22 +3469,28 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu return self.genBinOpMir(mir_tag, dst_ty, dst_mcv, .{ .register = reg }); }, .register => |src_reg| switch (dst_ty.zigTypeTag()) { - .Float => switch (dst_ty.tag()) { - .f64 => { - _ = try self.addInst(.{ - .tag = switch (mir_tag) { - .add => .add_f64, - .cmp => .cmp_f64, - else => return self.fail("TODO genBinOpMir for f64 register-register with MIR tag {}", .{mir_tag}), - }, - .ops = Mir.Inst.Ops.encode(.{ - .reg1 = dst_reg.to128(), - .reg2 = src_reg.to128(), - }), - .data = undefined, - }); - }, - else => return self.fail("TODO genBinOpMir for float register-register and type {}", .{dst_ty.fmtDebug()}), + .Float => { + const actual_tag: Mir.Inst.Tag = switch (dst_ty.tag()) { + .f32 => switch (mir_tag) { + .add => Mir.Inst.Tag.add_f32, + .cmp => Mir.Inst.Tag.cmp_f32, + else => return self.fail("TODO genBinOpMir for f32 register-register with MIR tag {}", .{mir_tag}), + }, + .f64 => switch (mir_tag) { + .add => Mir.Inst.Tag.add_f64, + .cmp => Mir.Inst.Tag.cmp_f64, + else => return self.fail("TODO genBinOpMir for f64 register-register with MIR tag {}", .{mir_tag}), + }, + else => return self.fail("TODO genBinOpMir for float register-register and type {}", .{dst_ty.fmtDebug()}), + }; + _ = try self.addInst(.{ + .tag = actual_tag, + .ops = Mir.Inst.Ops.encode(.{ + .reg1 = dst_reg.to128(), + .reg2 = src_reg.to128(), + }), + .data = undefined, + }); }, else => { _ = try self.addInst(.{ @@ -5475,20 +5481,25 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl const base_reg = opts.dest_stack_base orelse .rbp; switch (ty.zigTypeTag()) { - .Float => switch (ty.tag()) { - .f32 => return self.fail("TODO genSetStack for register for f32", .{}), - .f64 => { - _ = try self.addInst(.{ - .tag = .mov_f64, - .ops = Mir.Inst.Ops.encode(.{ - .reg1 = base_reg, - .reg2 = reg.to128(), - .flags = 0b01, - }), - .data = .{ .imm = @bitCast(u32, -stack_offset) }, - }); - }, - else => return self.fail("TODO genSetStack for register for type {}", .{ty.fmtDebug()}), + .Float => { + const tag: Mir.Inst.Tag = switch (ty.tag()) { + .f32 => .mov_f32, + .f64 => .mov_f64, + else => return self.fail("TODO genSetStack for register for type {}", .{ty.fmtDebug()}), + }; + _ = try self.addInst(.{ + .tag = tag, + .ops = Mir.Inst.Ops.encode(.{ + .reg1 = switch (ty.tag()) { + .f32 => base_reg.to32(), + .f64 => base_reg.to64(), + else => unreachable, + }, + .reg2 = reg.to128(), + .flags = 0b01, + }), + .data = .{ .imm = @bitCast(u32, -stack_offset) }, + }); }, else => { if (!math.isPowerOfTwo(abi_size)) { @@ -5991,21 +6002,22 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void } }, }, - .Float => switch (ty.tag()) { - .f32 => return self.fail("TODO genSetReg from register for f32", .{}), - .f64 => { - _ = try self.addInst(.{ - .tag = .mov_f64, - .ops = Mir.Inst.Ops.encode(.{ - .reg1 = reg.to128(), - .reg2 = src_reg.to128(), - .flags = 0b10, - }), - .data = undefined, - }); - return; - }, - else => return self.fail("TODO genSetReg from register for {}", .{ty.fmtDebug()}), + .Float => { + const tag: Mir.Inst.Tag = switch (ty.tag()) { + .f32 => .mov_f32, + .f64 => .mov_f64, + else => return self.fail("TODO genSetReg from register for {}", .{ty.fmtDebug()}), + }; + _ = try self.addInst(.{ + .tag = tag, + .ops = Mir.Inst.Ops.encode(.{ + .reg1 = reg.to128(), + .reg2 = src_reg.to128(), + .flags = 0b10, + }), + .data = undefined, + }); + return; }, else => {}, } @@ -6035,22 +6047,27 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void }, .memory => |x| switch (ty.zigTypeTag()) { .Float => { - switch (ty.tag()) { - .f32 => return self.fail("TODO genSetReg from memory for f32", .{}), - .f64 => { - const base_reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp }); - try self.loadMemPtrIntoRegister(base_reg, Type.usize, mcv); - _ = try self.addInst(.{ - .tag = .mov_f64, - .ops = Mir.Inst.Ops.encode(.{ - .reg1 = reg.to128(), - .reg2 = base_reg.to64(), - }), - .data = .{ .imm = 0 }, - }); - }, + const base_reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp }); + try self.loadMemPtrIntoRegister(base_reg, Type.usize, mcv); + + const tag: Mir.Inst.Tag = switch (ty.tag()) { + .f32 => .mov_f32, + .f64 => .mov_f64, else => return self.fail("TODO genSetReg from memory for {}", .{ty.fmtDebug()}), - } + }; + + _ = try self.addInst(.{ + .tag = tag, + .ops = Mir.Inst.Ops.encode(.{ + .reg1 = reg.to128(), + .reg2 = switch (ty.tag()) { + .f32 => base_reg.to32(), + .f64 => base_reg.to64(), + else => unreachable, + }, + }), + .data = .{ .imm = 0 }, + }); }, else => { if (x <= math.maxInt(i32)) { @@ -6142,20 +6159,25 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void } }, }, - .Float => switch (ty.tag()) { - .f32 => return self.fail("TODO genSetReg from stack offset for f32", .{}), - .f64 => { - _ = try self.addInst(.{ - .tag = .mov_f64, - .ops = Mir.Inst.Ops.encode(.{ - .reg1 = reg.to128(), - .reg2 = .rbp, - }), - .data = .{ .imm = @bitCast(u32, -off) }, - }); - return; - }, - else => return self.fail("TODO genSetReg from stack offset for {}", .{ty.fmtDebug()}), + .Float => { + const tag: Mir.Inst.Tag = switch (ty.tag()) { + .f32 => .mov_f32, + .f64 => .mov_f64, + else => return self.fail("TODO genSetReg from stack offset for {}", .{ty.fmtDebug()}), + }; + _ = try self.addInst(.{ + .tag = tag, + .ops = Mir.Inst.Ops.encode(.{ + .reg1 = reg.to128(), + .reg2 = switch (ty.tag()) { + .f32 => .ebp, + .f64 => .rbp, + else => unreachable, + }, + }), + .data = .{ .imm = @bitCast(u32, -off) }, + }); + return; }, else => {}, } diff --git a/src/arch/x86_64/Emit.zig b/src/arch/x86_64/Emit.zig index 5ad8e86374..96f640b610 100644 --- a/src/arch/x86_64/Emit.zig +++ b/src/arch/x86_64/Emit.zig @@ -183,11 +183,14 @@ pub fn lowerMir(emit: *Emit) InnerError!void { .nop => try emit.mirNop(), // AVX instructions - .mov_f64 => try emit.mirMovF64(inst), + .mov_f64 => try emit.mirMovFloatAvx(.vmovsd, inst), + .mov_f32 => try emit.mirMovFloatAvx(.vmovss, inst), - .add_f64 => try emit.mirAddF64(inst), + .add_f64 => try emit.mirAddFloatAvx(.vaddsd, inst), + .add_f32 => try emit.mirAddFloatAvx(.vaddss, inst), - .cmp_f64 => try emit.mirCmpF64(inst), + .cmp_f64 => try emit.mirCmpFloatAvx(.vucomisd, inst), + .cmp_f32 => try emit.mirCmpFloatAvx(.vucomiss, inst), // Pseudo-instructions .call_extern => try emit.mirCallExtern(inst), @@ -962,71 +965,48 @@ fn mirLeaPie(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { // AVX instructions -fn mirMovF64(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { - const tag = emit.mir.instructions.items(.tag)[inst]; - assert(tag == .mov_f64); +fn mirMovFloatAvx(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { const ops = emit.mir.instructions.items(.ops)[inst].decode(); switch (ops.flags) { 0b00 => { const imm = emit.mir.instructions.items(.data)[inst].imm; - return lowerToVmEnc(.vmovsd, ops.reg1, RegisterOrMemory.mem(.qword_ptr, .{ + return lowerToVmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg2.size()), .{ .disp = imm, .base = ops.reg2, }), emit.code); }, 0b01 => { const imm = emit.mir.instructions.items(.data)[inst].imm; - return lowerToMvEnc(.vmovsd, RegisterOrMemory.mem(.qword_ptr, .{ + return lowerToMvEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{ .disp = imm, .base = ops.reg1, }), ops.reg2, emit.code); }, 0b10 => { - return lowerToRvmEnc( - .vmovsd, - ops.reg1, - ops.reg1, - RegisterOrMemory.reg(ops.reg2), - emit.code, - ); + return lowerToRvmEnc(tag, ops.reg1, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code); }, else => return emit.fail("TODO unused variant 0b{b} for mov_f64", .{ops.flags}), } } -fn mirAddF64(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { - const tag = emit.mir.instructions.items(.tag)[inst]; - assert(tag == .add_f64); +fn mirAddFloatAvx(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { const ops = emit.mir.instructions.items(.ops)[inst].decode(); switch (ops.flags) { 0b00 => { - return lowerToRvmEnc( - .vaddsd, - ops.reg1, - ops.reg1, - RegisterOrMemory.reg(ops.reg2), - emit.code, - ); + return lowerToRvmEnc(tag, ops.reg1, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code); }, else => return emit.fail("TODO unused variant 0b{b} for mov_f64", .{ops.flags}), } } -fn mirCmpF64(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { - const tag = emit.mir.instructions.items(.tag)[inst]; - assert(tag == .cmp_f64); +fn mirCmpFloatAvx(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { const ops = emit.mir.instructions.items(.ops)[inst].decode(); switch (ops.flags) { 0b00 => { - return lowerToVmEnc( - .vucomisd, - ops.reg1, - RegisterOrMemory.reg(ops.reg2), - emit.code, - ); + return lowerToVmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code); }, else => return emit.fail("TODO unused variant 0b{b} for mov_f64", .{ops.flags}), } @@ -1268,16 +1248,24 @@ const Tag = enum { cmovb, cmovnae, vmovsd, + vmovss, vaddsd, + vaddss, vcmpsd, + vcmpss, vucomisd, + vucomiss, fn isAvx(tag: Tag) bool { return switch (tag) { .vmovsd, + .vmovss, .vaddsd, + .vaddss, .vcmpsd, + .vcmpss, .vucomisd, + .vucomiss, => true, else => false, @@ -1406,7 +1394,7 @@ const OpCode = union(enum) { } }; -inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode { +inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) OpCode { switch (enc) { .zo => return switch (tag) { .ret_near => OpCode.oneByte(0xc3), @@ -1416,7 +1404,7 @@ inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode { .syscall => OpCode.twoByte(0x0f, 0x05), .cbw => OpCode.oneByte(0x98), .cwd, .cdq, .cqo => OpCode.oneByte(0x99), - else => null, + else => unreachable, }, .d => return switch (tag) { .jmp_near => OpCode.oneByte(0xe9), @@ -1437,7 +1425,7 @@ inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode { .jge, .jnl => if (is_one_byte) OpCode.oneByte(0x7d) else OpCode.twoByte(0x0f, 0x8d), .jle, .jng => if (is_one_byte) OpCode.oneByte(0x7e) else OpCode.twoByte(0x0f, 0x8e), .jg, .jnle => if (is_one_byte) OpCode.oneByte(0x7f) else OpCode.twoByte(0x0f, 0x8f), - else => null, + else => unreachable, }, .m => return switch (tag) { .jmp_near, .call_near, .push => OpCode.oneByte(0xff), @@ -1464,38 +1452,38 @@ inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode { .fisttp64 => OpCode.oneByte(0xdd), .fld32 => OpCode.oneByte(0xd9), .fld64 => OpCode.oneByte(0xdd), - else => null, + else => unreachable, }, .o => return switch (tag) { .push => OpCode.oneByte(0x50), .pop => OpCode.oneByte(0x58), - else => null, + else => unreachable, }, .i => return switch (tag) { .push => OpCode.oneByte(if (is_one_byte) 0x6a else 0x68), .@"test" => OpCode.oneByte(if (is_one_byte) 0xa8 else 0xa9), .ret_near => OpCode.oneByte(0xc2), .ret_far => OpCode.oneByte(0xca), - else => null, + else => unreachable, }, .m1 => return switch (tag) { .shl, .sal, .shr, .sar => OpCode.oneByte(if (is_one_byte) 0xd0 else 0xd1), - else => null, + else => unreachable, }, .mc => return switch (tag) { .shl, .sal, .shr, .sar => OpCode.oneByte(if (is_one_byte) 0xd2 else 0xd3), - else => null, + else => unreachable, }, .mi => return switch (tag) { .adc, .add, .sub, .xor, .@"and", .@"or", .sbb, .cmp => OpCode.oneByte(if (is_one_byte) 0x80 else 0x81), .mov => OpCode.oneByte(if (is_one_byte) 0xc6 else 0xc7), .@"test" => OpCode.oneByte(if (is_one_byte) 0xf6 else 0xf7), - else => null, + else => unreachable, }, .mi8 => return switch (tag) { .adc, .add, .sub, .xor, .@"and", .@"or", .sbb, .cmp => OpCode.oneByte(0x83), .shl, .sal, .shr, .sar => OpCode.oneByte(if (is_one_byte) 0xc0 else 0xc1), - else => null, + else => unreachable, }, .mr => return switch (tag) { .adc => OpCode.oneByte(if (is_one_byte) 0x10 else 0x11), @@ -1508,8 +1496,7 @@ inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode { .cmp => OpCode.oneByte(if (is_one_byte) 0x38 else 0x39), .mov => OpCode.oneByte(if (is_one_byte) 0x88 else 0x89), .@"test" => OpCode.oneByte(if (is_one_byte) 0x84 else 0x85), - .vmovsd => OpCode.oneByte(0x11), - else => null, + else => unreachable, }, .rm => return switch (tag) { .adc => OpCode.oneByte(if (is_one_byte) 0x12 else 0x13), @@ -1529,48 +1516,46 @@ inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) ?OpCode { .cmove, .cmovz => OpCode.twoByte(0x0f, 0x44), .cmovb, .cmovnae => OpCode.twoByte(0x0f, 0x42), .cmovl, .cmovng => OpCode.twoByte(0x0f, 0x4c), - .vmovsd => OpCode.oneByte(0x10), - .vucomisd => OpCode.oneByte(0x2e), - else => null, + else => unreachable, }, .oi => return switch (tag) { .mov => OpCode.oneByte(if (is_one_byte) 0xb0 else 0xb8), - else => null, + else => unreachable, }, .fd => return switch (tag) { .mov => OpCode.oneByte(if (is_one_byte) 0xa0 else 0xa1), - else => null, + else => unreachable, }, .td => return switch (tag) { .mov => OpCode.oneByte(if (is_one_byte) 0xa2 else 0xa3), - else => null, + else => unreachable, }, .rmi => return switch (tag) { .imul => OpCode.oneByte(if (is_one_byte) 0x6b else 0x69), - else => null, + else => unreachable, }, .mv => return switch (tag) { - .vmovsd => OpCode.oneByte(0x11), - else => null, + .vmovsd, .vmovss => OpCode.oneByte(0x11), + else => unreachable, }, .vm => return switch (tag) { - .vmovsd => OpCode.oneByte(0x10), - .vucomisd => OpCode.oneByte(0x2e), - else => null, + .vmovsd, .vmovss => OpCode.oneByte(0x10), + .vucomisd, .vucomiss => OpCode.oneByte(0x2e), + else => unreachable, }, .rvm => return switch (tag) { - .vaddsd => OpCode.oneByte(0x58), - .vmovsd => OpCode.oneByte(0x10), - else => null, + .vaddsd, .vaddss => OpCode.oneByte(0x58), + .vmovsd, .vmovss => OpCode.oneByte(0x10), + else => unreachable, }, .rvmi => return switch (tag) { - .vcmpsd => OpCode.oneByte(0xc2), - else => null, + .vcmpsd, .vcmpss => OpCode.oneByte(0xc2), + else => unreachable, }, } } -inline fn getModRmExt(tag: Tag) ?u3 { +inline fn getModRmExt(tag: Tag) u3 { return switch (tag) { .adc => 0x2, .add => 0x0, @@ -1631,11 +1616,11 @@ inline fn getModRmExt(tag: Tag) ?u3 { .fisttp64 => 0x1, .fld32 => 0x0, .fld64 => 0x0, - else => null, + else => unreachable, }; } -const VexPrefix = struct { +const VexEncoding = struct { prefix: Encoder.Vex, reg: ?enum { ndd, @@ -1644,7 +1629,7 @@ const VexPrefix = struct { }, }; -inline fn getVexPrefix(tag: Tag, enc: Encoding) ?VexPrefix { +inline fn getVexEncoding(tag: Tag, enc: Encoding) VexEncoding { const desc: struct { reg: enum { none, @@ -1671,21 +1656,27 @@ inline fn getVexPrefix(tag: Tag, enc: Encoding) ?VexPrefix { switch (enc) { .mv => switch (tag) { .vmovsd => break :blk .{ .lig = true, .simd_prefix = .p_f2, .wig = true }, - else => return null, + .vmovss => break :blk .{ .lig = true, .simd_prefix = .p_f3, .wig = true }, + else => unreachable, }, .vm => switch (tag) { .vmovsd => break :blk .{ .lig = true, .simd_prefix = .p_f2, .wig = true }, + .vmovss => break :blk .{ .lig = true, .simd_prefix = .p_f3, .wig = true }, .vucomisd => break :blk .{ .lig = true, .simd_prefix = .p_66, .wig = true }, - else => return null, + .vucomiss => break :blk .{ .lig = true, .wig = true }, + else => unreachable, }, .rvm => switch (tag) { .vaddsd => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f2, .wig = true }, + .vaddss => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f3, .wig = true }, .vmovsd => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f2, .wig = true }, - else => return null, + .vmovss => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f3, .wig = true }, + else => unreachable, }, .rvmi => switch (tag) { .vcmpsd => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f2, .wig = true }, - else => return null, + .vcmpss => break :blk .{ .reg = .nds, .lig = true, .simd_prefix = .p_f3, .wig = true }, + else => unreachable, }, else => unreachable, } @@ -1711,7 +1702,7 @@ inline fn getVexPrefix(tag: Tag, enc: Encoding) ?VexPrefix { .p_f3 => vex.simd_prefix_f3(), } - return VexPrefix{ .prefix = vex, .reg = switch (desc.reg) { + return VexEncoding{ .prefix = vex, .reg = switch (desc.reg) { .none => null, .nds => .nds, .dds => .dds, @@ -1862,7 +1853,7 @@ const RegisterOrMemory = union(enum) { fn lowerToZoEnc(tag: Tag, code: *std.ArrayList(u8)) InnerError!void { assert(!tag.isAvx()); - const opc = getOpCode(tag, .zo, false).?; + const opc = getOpCode(tag, .zo, false); const encoder = try Encoder.init(code, 2); switch (tag) { .cqo => { @@ -1879,12 +1870,12 @@ fn lowerToIEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) InnerError!void { assert(!tag.isAvx()); if (tag == .ret_far or tag == .ret_near) { const encoder = try Encoder.init(code, 3); - const opc = getOpCode(tag, .i, false).?; + const opc = getOpCode(tag, .i, false); opc.encode(encoder); encoder.imm16(@bitCast(i16, @truncate(u16, imm))); return; } - const opc = getOpCode(tag, .i, immOpSize(imm) == 8).?; + const opc = getOpCode(tag, .i, immOpSize(imm) == 8); const encoder = try Encoder.init(code, 5); if (immOpSize(imm) == 16) { encoder.prefix16BitMode(); @@ -1895,7 +1886,7 @@ fn lowerToIEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) InnerError!void { fn lowerToOEnc(tag: Tag, reg: Register, code: *std.ArrayList(u8)) InnerError!void { assert(!tag.isAvx()); - const opc = getOpCode(tag, .o, false).?; + const opc = getOpCode(tag, .o, false); const encoder = try Encoder.init(code, 3); if (reg.size() == 16) { encoder.prefix16BitMode(); @@ -1909,7 +1900,7 @@ fn lowerToOEnc(tag: Tag, reg: Register, code: *std.ArrayList(u8)) InnerError!voi fn lowerToDEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) InnerError!void { assert(!tag.isAvx()); - const opc = getOpCode(tag, .d, false).?; + const opc = getOpCode(tag, .d, false); const encoder = try Encoder.init(code, 6); opc.encode(encoder); encoder.imm32(@bitCast(i32, imm)); @@ -1917,8 +1908,8 @@ fn lowerToDEnc(tag: Tag, imm: u32, code: *std.ArrayList(u8)) InnerError!void { fn lowerToMxEnc(tag: Tag, reg_or_mem: RegisterOrMemory, enc: Encoding, code: *std.ArrayList(u8)) InnerError!void { assert(!tag.isAvx()); - const opc = getOpCode(tag, enc, reg_or_mem.size() == 8).?; - const modrm_ext = getModRmExt(tag).?; + const opc = getOpCode(tag, enc, reg_or_mem.size() == 8); + const modrm_ext = getModRmExt(tag); switch (reg_or_mem) { .register => |reg| { const encoder = try Encoder.init(code, 4); @@ -1973,10 +1964,7 @@ fn lowerToFdEnc(tag: Tag, reg: Register, moffs: u64, code: *std.ArrayList(u8)) I fn lowerToTdFdEnc(tag: Tag, reg: Register, moffs: u64, code: *std.ArrayList(u8), td: bool) InnerError!void { assert(!tag.isAvx()); - const opc = if (td) - getOpCode(tag, .td, reg.size() == 8).? - else - getOpCode(tag, .fd, reg.size() == 8).?; + const opc = if (td) getOpCode(tag, .td, reg.size() == 8) else getOpCode(tag, .fd, reg.size() == 8); const encoder = try Encoder.init(code, 10); if (reg.size() == 16) { encoder.prefix16BitMode(); @@ -1996,7 +1984,7 @@ fn lowerToTdFdEnc(tag: Tag, reg: Register, moffs: u64, code: *std.ArrayList(u8), fn lowerToOiEnc(tag: Tag, reg: Register, imm: u64, code: *std.ArrayList(u8)) InnerError!void { assert(!tag.isAvx()); - const opc = getOpCode(tag, .oi, reg.size() == 8).?; + const opc = getOpCode(tag, .oi, reg.size() == 8); const encoder = try Encoder.init(code, 10); if (reg.size() == 16) { encoder.prefix16BitMode(); @@ -2023,8 +2011,8 @@ fn lowerToMiXEnc( code: *std.ArrayList(u8), ) InnerError!void { assert(!tag.isAvx()); - const modrm_ext = getModRmExt(tag).?; - const opc = getOpCode(tag, enc, reg_or_mem.size() == 8).?; + const modrm_ext = getModRmExt(tag); + const opc = getOpCode(tag, enc, reg_or_mem.size() == 8); switch (reg_or_mem) { .register => |dst_reg| { const encoder = try Encoder.init(code, 7); @@ -2079,7 +2067,7 @@ fn lowerToRmEnc( code: *std.ArrayList(u8), ) InnerError!void { assert(!tag.isAvx()); - const opc = getOpCode(tag, .rm, reg.size() == 8 or reg_or_mem.size() == 8).?; + const opc = getOpCode(tag, .rm, reg.size() == 8 or reg_or_mem.size() == 8); switch (reg_or_mem) { .register => |src_reg| { const encoder = try Encoder.init(code, 5); @@ -2126,7 +2114,7 @@ fn lowerToMrEnc( code: *std.ArrayList(u8), ) InnerError!void { assert(!tag.isAvx()); - const opc = getOpCode(tag, .mr, reg.size() == 8 or reg_or_mem.size() == 8).?; + const opc = getOpCode(tag, .mr, reg.size() == 8 or reg_or_mem.size() == 8); switch (reg_or_mem) { .register => |dst_reg| { const encoder = try Encoder.init(code, 4); @@ -2172,7 +2160,7 @@ fn lowerToRmiEnc( code: *std.ArrayList(u8), ) InnerError!void { assert(!tag.isAvx()); - const opc = getOpCode(tag, .rmi, false).?; + const opc = getOpCode(tag, .rmi, false); const encoder = try Encoder.init(code, 13); if (reg.size() == 16) { encoder.prefix16BitMode(); @@ -2216,9 +2204,9 @@ fn lowerToVmEnc( reg_or_mem: RegisterOrMemory, code: *std.ArrayList(u8), ) InnerError!void { - const opc = getOpCode(tag, .vm, false).?; - var vex_prefix = getVexPrefix(tag, .vm).?; - const vex = &vex_prefix.prefix; + const opc = getOpCode(tag, .vm, false); + var enc = getVexEncoding(tag, .vm); + const vex = &enc.prefix; switch (reg_or_mem) { .register => |src_reg| { const encoder = try Encoder.init(code, 5); @@ -2226,12 +2214,11 @@ fn lowerToVmEnc( .r = reg.isExtended(), .b = src_reg.isExtended(), }); - encoder.vex(vex_prefix.prefix); + encoder.vex(enc.prefix); opc.encode(encoder); encoder.modRm_direct(reg.lowEnc(), src_reg.lowEnc()); }, .memory => |src_mem| { - assert(src_mem.ptr_size == .qword_ptr); const encoder = try Encoder.init(code, 10); if (src_mem.base) |base| { vex.rex(.{ @@ -2243,7 +2230,7 @@ fn lowerToVmEnc( .r = reg.isExtended(), }); } - encoder.vex(vex_prefix.prefix); + encoder.vex(enc.prefix); opc.encode(encoder); src_mem.encode(encoder, reg.lowEnc()); }, @@ -2257,9 +2244,9 @@ fn lowerToMvEnc( reg: Register, code: *std.ArrayList(u8), ) InnerError!void { - const opc = getOpCode(tag, .mv, false).?; - var vex_prefix = getVexPrefix(tag, .mv).?; - const vex = &vex_prefix.prefix; + const opc = getOpCode(tag, .mv, false); + var enc = getVexEncoding(tag, .mv); + const vex = &enc.prefix; switch (reg_or_mem) { .register => |dst_reg| { const encoder = try Encoder.init(code, 4); @@ -2267,12 +2254,11 @@ fn lowerToMvEnc( .r = reg.isExtended(), .b = dst_reg.isExtended(), }); - encoder.vex(vex_prefix.prefix); + encoder.vex(enc.prefix); opc.encode(encoder); encoder.modRm_direct(reg.lowEnc(), dst_reg.lowEnc()); }, .memory => |dst_mem| { - assert(dst_mem.ptr_size == .qword_ptr); const encoder = try Encoder.init(code, 10); if (dst_mem.base) |base| { vex.rex(.{ @@ -2284,7 +2270,7 @@ fn lowerToMvEnc( .r = reg.isExtended(), }); } - encoder.vex(vex_prefix.prefix); + encoder.vex(enc.prefix); opc.encode(encoder); dst_mem.encode(encoder, reg.lowEnc()); }, @@ -2298,12 +2284,12 @@ fn lowerToRvmEnc( reg_or_mem: RegisterOrMemory, code: *std.ArrayList(u8), ) InnerError!void { - const opc = getOpCode(tag, .rvm, false).?; - var vex_prefix = getVexPrefix(tag, .rvm).?; - const vex = &vex_prefix.prefix; + const opc = getOpCode(tag, .rvm, false); + var enc = getVexEncoding(tag, .rvm); + const vex = &enc.prefix; switch (reg_or_mem) { .register => |reg3| { - if (vex_prefix.reg) |vvvv| { + if (enc.reg) |vvvv| { switch (vvvv) { .nds => vex.reg(reg2.enc()), else => unreachable, // TODO @@ -2314,7 +2300,7 @@ fn lowerToRvmEnc( .r = reg1.isExtended(), .b = reg3.isExtended(), }); - encoder.vex(vex_prefix.prefix); + encoder.vex(enc.prefix); opc.encode(encoder); encoder.modRm_direct(reg1.lowEnc(), reg3.lowEnc()); }, @@ -2333,13 +2319,13 @@ fn lowerToRvmiEnc( imm: u32, code: *std.ArrayList(u8), ) InnerError!void { - const opc = getOpCode(tag, .rvmi, false).?; - var vex_prefix = getVexPrefix(tag, .rvmi).?; - const vex = &vex_prefix.prefix; + const opc = getOpCode(tag, .rvmi, false); + var enc = getVexEncoding(tag, .rvmi); + const vex = &enc.prefix; const encoder: Encoder = blk: { switch (reg_or_mem) { .register => |reg3| { - if (vex_prefix.reg) |vvvv| { + if (enc.reg) |vvvv| { switch (vvvv) { .nds => vex.reg(reg2.enc()), else => unreachable, // TODO @@ -2350,7 +2336,7 @@ fn lowerToRvmiEnc( .r = reg1.isExtended(), .b = reg3.isExtended(), }); - encoder.vex(vex_prefix.prefix); + encoder.vex(enc.prefix); opc.encode(encoder); encoder.modRm_direct(reg1.lowEnc(), reg3.lowEnc()); break :blk encoder; diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig index a1062ba6b4..dc8c1fa0b2 100644 --- a/src/arch/x86_64/Mir.zig +++ b/src/arch/x86_64/Mir.zig @@ -351,14 +351,17 @@ pub const Inst = struct { /// 0b01 qword ptr [reg1 + imm32], reg2 /// 0b10 reg1, reg2 mov_f64, + mov_f32, /// ops flags: form: /// 0b00 reg1, reg1, reg2 add_f64, + add_f32, /// ops flags: form: /// cmp_f64, + cmp_f32, /// Pseudo-instructions /// call extern function diff --git a/test/behavior.zig b/test/behavior.zig index 4d8b3587bc..cea8ded1ba 100644 --- a/test/behavior.zig +++ b/test/behavior.zig @@ -1,163 +1,163 @@ const builtin = @import("builtin"); test { - _ = @import("behavior/align.zig"); - _ = @import("behavior/alignof.zig"); - _ = @import("behavior/array.zig"); - _ = @import("behavior/async_fn.zig"); - _ = @import("behavior/atomics.zig"); - _ = @import("behavior/await_struct.zig"); + // _ = @import("behavior/align.zig"); + // _ = @import("behavior/alignof.zig"); + // _ = @import("behavior/array.zig"); + // _ = @import("behavior/async_fn.zig"); + // _ = @import("behavior/atomics.zig"); + // _ = @import("behavior/await_struct.zig"); _ = @import("behavior/basic.zig"); - _ = @import("behavior/bit_shifting.zig"); - _ = @import("behavior/bitcast.zig"); - _ = @import("behavior/bitreverse.zig"); - _ = @import("behavior/bool.zig"); - _ = @import("behavior/bugs/394.zig"); - _ = @import("behavior/bugs/421.zig"); - _ = @import("behavior/bugs/529.zig"); - _ = @import("behavior/bugs/624.zig"); - _ = @import("behavior/bugs/655.zig"); - _ = @import("behavior/bugs/656.zig"); - _ = @import("behavior/bugs/679.zig"); - _ = @import("behavior/bugs/704.zig"); - _ = @import("behavior/bugs/718.zig"); - _ = @import("behavior/bugs/726.zig"); - _ = @import("behavior/bugs/828.zig"); - _ = @import("behavior/bugs/920.zig"); - _ = @import("behavior/bugs/1025.zig"); - _ = @import("behavior/bugs/1076.zig"); - _ = @import("behavior/bugs/1111.zig"); - _ = @import("behavior/bugs/1120.zig"); - _ = @import("behavior/bugs/1277.zig"); - _ = @import("behavior/bugs/1310.zig"); - _ = @import("behavior/bugs/1381.zig"); - _ = @import("behavior/bugs/1421.zig"); - _ = @import("behavior/bugs/1442.zig"); - _ = @import("behavior/bugs/1486.zig"); - _ = @import("behavior/bugs/1500.zig"); - _ = @import("behavior/bugs/1607.zig"); - _ = @import("behavior/bugs/1735.zig"); - _ = @import("behavior/bugs/1741.zig"); - _ = @import("behavior/bugs/1851.zig"); - _ = @import("behavior/bugs/1914.zig"); - _ = @import("behavior/bugs/2006.zig"); - _ = @import("behavior/bugs/2114.zig"); - _ = @import("behavior/bugs/2346.zig"); - _ = @import("behavior/bugs/2578.zig"); - _ = @import("behavior/bugs/2692.zig"); - _ = @import("behavior/bugs/2889.zig"); - _ = @import("behavior/bugs/3007.zig"); - _ = @import("behavior/bugs/3046.zig"); - _ = @import("behavior/bugs/3112.zig"); - _ = @import("behavior/bugs/3367.zig"); - _ = @import("behavior/bugs/3384.zig"); - _ = @import("behavior/bugs/3586.zig"); - _ = @import("behavior/bugs/3742.zig"); - _ = @import("behavior/bugs/3779.zig"); - _ = @import("behavior/bugs/4328.zig"); - _ = @import("behavior/bugs/4560.zig"); - _ = @import("behavior/bugs/4769_a.zig"); - _ = @import("behavior/bugs/4769_b.zig"); - _ = @import("behavior/bugs/4954.zig"); - _ = @import("behavior/bugs/5398.zig"); - _ = @import("behavior/bugs/5413.zig"); - _ = @import("behavior/bugs/5474.zig"); - _ = @import("behavior/bugs/5487.zig"); - _ = @import("behavior/bugs/6456.zig"); - _ = @import("behavior/bugs/6781.zig"); - _ = @import("behavior/bugs/6850.zig"); - _ = @import("behavior/bugs/7003.zig"); - _ = @import("behavior/bugs/7027.zig"); - _ = @import("behavior/bugs/7047.zig"); - _ = @import("behavior/bugs/7187.zig"); - _ = @import("behavior/bugs/7250.zig"); - _ = @import("behavior/bugs/9584.zig"); - _ = @import("behavior/bugs/10138.zig"); - _ = @import("behavior/bugs/10147.zig"); - _ = @import("behavior/bugs/10970.zig"); - _ = @import("behavior/bugs/11046.zig"); - _ = @import("behavior/bugs/11100.zig"); - _ = @import("behavior/bugs/11139.zig"); - _ = @import("behavior/bugs/11159.zig"); - _ = @import("behavior/bugs/11162.zig"); - _ = @import("behavior/bugs/11165.zig"); - _ = @import("behavior/bugs/11181.zig"); - _ = @import("behavior/bugs/11182.zig"); - _ = @import("behavior/bugs/11213.zig"); - _ = @import("behavior/byteswap.zig"); - _ = @import("behavior/byval_arg_var.zig"); - _ = @import("behavior/call.zig"); - _ = @import("behavior/cast.zig"); - _ = @import("behavior/cast_int.zig"); - _ = @import("behavior/comptime_memory.zig"); - _ = @import("behavior/const_slice_child.zig"); - _ = @import("behavior/defer.zig"); - _ = @import("behavior/enum.zig"); - _ = @import("behavior/error.zig"); - _ = @import("behavior/eval.zig"); - _ = @import("behavior/field_parent_ptr.zig"); - _ = @import("behavior/floatop.zig"); - _ = @import("behavior/fn.zig"); - _ = @import("behavior/fn_delegation.zig"); - _ = @import("behavior/fn_in_struct_in_comptime.zig"); - _ = @import("behavior/for.zig"); - _ = @import("behavior/generics.zig"); - _ = @import("behavior/hasdecl.zig"); - _ = @import("behavior/hasfield.zig"); - _ = @import("behavior/if.zig"); - _ = @import("behavior/import.zig"); - _ = @import("behavior/incomplete_struct_param_tld.zig"); - _ = @import("behavior/int128.zig"); - _ = @import("behavior/int_div.zig"); - _ = @import("behavior/inttoptr.zig"); - _ = @import("behavior/ir_block_deps.zig"); - _ = @import("behavior/math.zig"); - _ = @import("behavior/maximum_minimum.zig"); - _ = @import("behavior/member_func.zig"); - _ = @import("behavior/merge_error_sets.zig"); - _ = @import("behavior/muladd.zig"); - _ = @import("behavior/namespace_depends_on_compile_var.zig"); - _ = @import("behavior/null.zig"); - _ = @import("behavior/optional.zig"); - _ = @import("behavior/pointers.zig"); - _ = @import("behavior/popcount.zig"); - _ = @import("behavior/prefetch.zig"); - _ = @import("behavior/ptrcast.zig"); - _ = @import("behavior/pub_enum.zig"); - _ = @import("behavior/ref_var_in_if_after_if_2nd_switch_prong.zig"); - _ = @import("behavior/reflection.zig"); - _ = @import("behavior/saturating_arithmetic.zig"); - _ = @import("behavior/select.zig"); - _ = @import("behavior/shuffle.zig"); - _ = @import("behavior/sizeof_and_typeof.zig"); - _ = @import("behavior/slice.zig"); - _ = @import("behavior/slice_sentinel_comptime.zig"); - _ = @import("behavior/src.zig"); - _ = @import("behavior/struct.zig"); - _ = @import("behavior/packed-struct.zig"); - _ = @import("behavior/struct_contains_null_ptr_itself.zig"); - _ = @import("behavior/struct_contains_slice_of_itself.zig"); - _ = @import("behavior/switch.zig"); - _ = @import("behavior/switch_prong_err_enum.zig"); - _ = @import("behavior/switch_prong_implicit_cast.zig"); - _ = @import("behavior/this.zig"); - _ = @import("behavior/translate_c_macros.zig"); - _ = @import("behavior/truncate.zig"); - _ = @import("behavior/try.zig"); - _ = @import("behavior/tuple.zig"); - _ = @import("behavior/type.zig"); - _ = @import("behavior/type_info.zig"); - _ = @import("behavior/typename.zig"); - _ = @import("behavior/undefined.zig"); - _ = @import("behavior/underscore.zig"); - _ = @import("behavior/union.zig"); - _ = @import("behavior/union_with_members.zig"); - _ = @import("behavior/usingnamespace.zig"); - _ = @import("behavior/var_args.zig"); - _ = @import("behavior/vector.zig"); - _ = @import("behavior/void.zig"); - _ = @import("behavior/while.zig"); - _ = @import("behavior/widening.zig"); + // _ = @import("behavior/bit_shifting.zig"); + // _ = @import("behavior/bitcast.zig"); + // _ = @import("behavior/bitreverse.zig"); + // _ = @import("behavior/bool.zig"); + // _ = @import("behavior/bugs/394.zig"); + // _ = @import("behavior/bugs/421.zig"); + // _ = @import("behavior/bugs/529.zig"); + // _ = @import("behavior/bugs/624.zig"); + // _ = @import("behavior/bugs/655.zig"); + // _ = @import("behavior/bugs/656.zig"); + // _ = @import("behavior/bugs/679.zig"); + // _ = @import("behavior/bugs/704.zig"); + // _ = @import("behavior/bugs/718.zig"); + // _ = @import("behavior/bugs/726.zig"); + // _ = @import("behavior/bugs/828.zig"); + // _ = @import("behavior/bugs/920.zig"); + // _ = @import("behavior/bugs/1025.zig"); + // _ = @import("behavior/bugs/1076.zig"); + // _ = @import("behavior/bugs/1111.zig"); + // _ = @import("behavior/bugs/1120.zig"); + // _ = @import("behavior/bugs/1277.zig"); + // _ = @import("behavior/bugs/1310.zig"); + // _ = @import("behavior/bugs/1381.zig"); + // _ = @import("behavior/bugs/1421.zig"); + // _ = @import("behavior/bugs/1442.zig"); + // _ = @import("behavior/bugs/1486.zig"); + // _ = @import("behavior/bugs/1500.zig"); + // _ = @import("behavior/bugs/1607.zig"); + // _ = @import("behavior/bugs/1735.zig"); + // _ = @import("behavior/bugs/1741.zig"); + // _ = @import("behavior/bugs/1851.zig"); + // _ = @import("behavior/bugs/1914.zig"); + // _ = @import("behavior/bugs/2006.zig"); + // _ = @import("behavior/bugs/2114.zig"); + // _ = @import("behavior/bugs/2346.zig"); + // _ = @import("behavior/bugs/2578.zig"); + // _ = @import("behavior/bugs/2692.zig"); + // _ = @import("behavior/bugs/2889.zig"); + // _ = @import("behavior/bugs/3007.zig"); + // _ = @import("behavior/bugs/3046.zig"); + // _ = @import("behavior/bugs/3112.zig"); + // _ = @import("behavior/bugs/3367.zig"); + // _ = @import("behavior/bugs/3384.zig"); + // _ = @import("behavior/bugs/3586.zig"); + // _ = @import("behavior/bugs/3742.zig"); + // _ = @import("behavior/bugs/3779.zig"); + // _ = @import("behavior/bugs/4328.zig"); + // _ = @import("behavior/bugs/4560.zig"); + // _ = @import("behavior/bugs/4769_a.zig"); + // _ = @import("behavior/bugs/4769_b.zig"); + // _ = @import("behavior/bugs/4954.zig"); + // _ = @import("behavior/bugs/5398.zig"); + // _ = @import("behavior/bugs/5413.zig"); + // _ = @import("behavior/bugs/5474.zig"); + // _ = @import("behavior/bugs/5487.zig"); + // _ = @import("behavior/bugs/6456.zig"); + // _ = @import("behavior/bugs/6781.zig"); + // _ = @import("behavior/bugs/6850.zig"); + // _ = @import("behavior/bugs/7003.zig"); + // _ = @import("behavior/bugs/7027.zig"); + // _ = @import("behavior/bugs/7047.zig"); + // _ = @import("behavior/bugs/7187.zig"); + // _ = @import("behavior/bugs/7250.zig"); + // _ = @import("behavior/bugs/9584.zig"); + // _ = @import("behavior/bugs/10138.zig"); + // _ = @import("behavior/bugs/10147.zig"); + // _ = @import("behavior/bugs/10970.zig"); + // _ = @import("behavior/bugs/11046.zig"); + // _ = @import("behavior/bugs/11100.zig"); + // _ = @import("behavior/bugs/11139.zig"); + // _ = @import("behavior/bugs/11159.zig"); + // _ = @import("behavior/bugs/11162.zig"); + // _ = @import("behavior/bugs/11165.zig"); + // _ = @import("behavior/bugs/11181.zig"); + // _ = @import("behavior/bugs/11182.zig"); + // _ = @import("behavior/bugs/11213.zig"); + // _ = @import("behavior/byteswap.zig"); + // _ = @import("behavior/byval_arg_var.zig"); + // _ = @import("behavior/call.zig"); + // _ = @import("behavior/cast.zig"); + // _ = @import("behavior/cast_int.zig"); + // _ = @import("behavior/comptime_memory.zig"); + // _ = @import("behavior/const_slice_child.zig"); + // _ = @import("behavior/defer.zig"); + // _ = @import("behavior/enum.zig"); + // _ = @import("behavior/error.zig"); + // _ = @import("behavior/eval.zig"); + // _ = @import("behavior/field_parent_ptr.zig"); + // _ = @import("behavior/floatop.zig"); + // _ = @import("behavior/fn.zig"); + // _ = @import("behavior/fn_delegation.zig"); + // _ = @import("behavior/fn_in_struct_in_comptime.zig"); + // _ = @import("behavior/for.zig"); + // _ = @import("behavior/generics.zig"); + // _ = @import("behavior/hasdecl.zig"); + // _ = @import("behavior/hasfield.zig"); + // _ = @import("behavior/if.zig"); + // _ = @import("behavior/import.zig"); + // _ = @import("behavior/incomplete_struct_param_tld.zig"); + // _ = @import("behavior/int128.zig"); + // _ = @import("behavior/int_div.zig"); + // _ = @import("behavior/inttoptr.zig"); + // _ = @import("behavior/ir_block_deps.zig"); + // _ = @import("behavior/math.zig"); + // _ = @import("behavior/maximum_minimum.zig"); + // _ = @import("behavior/member_func.zig"); + // _ = @import("behavior/merge_error_sets.zig"); + // _ = @import("behavior/muladd.zig"); + // _ = @import("behavior/namespace_depends_on_compile_var.zig"); + // _ = @import("behavior/null.zig"); + // _ = @import("behavior/optional.zig"); + // _ = @import("behavior/pointers.zig"); + // _ = @import("behavior/popcount.zig"); + // _ = @import("behavior/prefetch.zig"); + // _ = @import("behavior/ptrcast.zig"); + // _ = @import("behavior/pub_enum.zig"); + // _ = @import("behavior/ref_var_in_if_after_if_2nd_switch_prong.zig"); + // _ = @import("behavior/reflection.zig"); + // _ = @import("behavior/saturating_arithmetic.zig"); + // _ = @import("behavior/select.zig"); + // _ = @import("behavior/shuffle.zig"); + // _ = @import("behavior/sizeof_and_typeof.zig"); + // _ = @import("behavior/slice.zig"); + // _ = @import("behavior/slice_sentinel_comptime.zig"); + // _ = @import("behavior/src.zig"); + // _ = @import("behavior/struct.zig"); + // _ = @import("behavior/packed-struct.zig"); + // _ = @import("behavior/struct_contains_null_ptr_itself.zig"); + // _ = @import("behavior/struct_contains_slice_of_itself.zig"); + // _ = @import("behavior/switch.zig"); + // _ = @import("behavior/switch_prong_err_enum.zig"); + // _ = @import("behavior/switch_prong_implicit_cast.zig"); + // _ = @import("behavior/this.zig"); + // _ = @import("behavior/translate_c_macros.zig"); + // _ = @import("behavior/truncate.zig"); + // _ = @import("behavior/try.zig"); + // _ = @import("behavior/tuple.zig"); + // _ = @import("behavior/type.zig"); + // _ = @import("behavior/type_info.zig"); + // _ = @import("behavior/typename.zig"); + // _ = @import("behavior/undefined.zig"); + // _ = @import("behavior/underscore.zig"); + // _ = @import("behavior/union.zig"); + // _ = @import("behavior/union_with_members.zig"); + // _ = @import("behavior/usingnamespace.zig"); + // _ = @import("behavior/var_args.zig"); + // _ = @import("behavior/vector.zig"); + // _ = @import("behavior/void.zig"); + // _ = @import("behavior/while.zig"); + // _ = @import("behavior/widening.zig"); if (builtin.stage2_arch == .wasm32) { _ = @import("behavior/wasm.zig"); From 6d32498c5522b3385d8adeb55f94f8f56ac108b3 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 18 May 2022 17:37:17 +0200 Subject: [PATCH 12/19] x64: re-enable behavior tests --- src/arch/x86_64/CodeGen.zig | 42 +++-- test/behavior.zig | 312 ++++++++++++++++++------------------ test/behavior/basic.zig | 3 + test/behavior/math.zig | 1 - test/behavior/union.zig | 1 + 5 files changed, 193 insertions(+), 166 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 7896a2f3b7..0cae799e02 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -5325,15 +5325,39 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE }); }, .register => |reg| { - _ = try self.addInst(.{ - .tag = .mov, - .ops = Mir.Inst.Ops.encode(.{ - .reg1 = .rsp, - .reg2 = registerAlias(reg, @intCast(u32, abi_size)), - .flags = 0b10, - }), - .data = .{ .imm = @bitCast(u32, -stack_offset) }, - }); + switch (ty.zigTypeTag()) { + .Float => { + const tag: Mir.Inst.Tag = switch (ty.tag()) { + .f32 => .mov_f32, + .f64 => .mov_f64, + else => return self.fail("TODO genSetStackArg for register for type {}", .{ty.fmtDebug()}), + }; + _ = try self.addInst(.{ + .tag = tag, + .ops = Mir.Inst.Ops.encode(.{ + .reg1 = switch (ty.tag()) { + .f32 => .esp, + .f64 => .rsp, + else => unreachable, + }, + .reg2 = reg.to128(), + .flags = 0b01, + }), + .data = .{ .imm = @bitCast(u32, -stack_offset) }, + }); + }, + else => { + _ = try self.addInst(.{ + .tag = .mov, + .ops = Mir.Inst.Ops.encode(.{ + .reg1 = .rsp, + .reg2 = registerAlias(reg, @intCast(u32, abi_size)), + .flags = 0b10, + }), + .data = .{ .imm = @bitCast(u32, -stack_offset) }, + }); + }, + } }, .ptr_stack_offset => { const reg = try self.copyToTmpRegister(ty, mcv); diff --git a/test/behavior.zig b/test/behavior.zig index cea8ded1ba..4d8b3587bc 100644 --- a/test/behavior.zig +++ b/test/behavior.zig @@ -1,163 +1,163 @@ const builtin = @import("builtin"); test { - // _ = @import("behavior/align.zig"); - // _ = @import("behavior/alignof.zig"); - // _ = @import("behavior/array.zig"); - // _ = @import("behavior/async_fn.zig"); - // _ = @import("behavior/atomics.zig"); - // _ = @import("behavior/await_struct.zig"); + _ = @import("behavior/align.zig"); + _ = @import("behavior/alignof.zig"); + _ = @import("behavior/array.zig"); + _ = @import("behavior/async_fn.zig"); + _ = @import("behavior/atomics.zig"); + _ = @import("behavior/await_struct.zig"); _ = @import("behavior/basic.zig"); - // _ = @import("behavior/bit_shifting.zig"); - // _ = @import("behavior/bitcast.zig"); - // _ = @import("behavior/bitreverse.zig"); - // _ = @import("behavior/bool.zig"); - // _ = @import("behavior/bugs/394.zig"); - // _ = @import("behavior/bugs/421.zig"); - // _ = @import("behavior/bugs/529.zig"); - // _ = @import("behavior/bugs/624.zig"); - // _ = @import("behavior/bugs/655.zig"); - // _ = @import("behavior/bugs/656.zig"); - // _ = @import("behavior/bugs/679.zig"); - // _ = @import("behavior/bugs/704.zig"); - // _ = @import("behavior/bugs/718.zig"); - // _ = @import("behavior/bugs/726.zig"); - // _ = @import("behavior/bugs/828.zig"); - // _ = @import("behavior/bugs/920.zig"); - // _ = @import("behavior/bugs/1025.zig"); - // _ = @import("behavior/bugs/1076.zig"); - // _ = @import("behavior/bugs/1111.zig"); - // _ = @import("behavior/bugs/1120.zig"); - // _ = @import("behavior/bugs/1277.zig"); - // _ = @import("behavior/bugs/1310.zig"); - // _ = @import("behavior/bugs/1381.zig"); - // _ = @import("behavior/bugs/1421.zig"); - // _ = @import("behavior/bugs/1442.zig"); - // _ = @import("behavior/bugs/1486.zig"); - // _ = @import("behavior/bugs/1500.zig"); - // _ = @import("behavior/bugs/1607.zig"); - // _ = @import("behavior/bugs/1735.zig"); - // _ = @import("behavior/bugs/1741.zig"); - // _ = @import("behavior/bugs/1851.zig"); - // _ = @import("behavior/bugs/1914.zig"); - // _ = @import("behavior/bugs/2006.zig"); - // _ = @import("behavior/bugs/2114.zig"); - // _ = @import("behavior/bugs/2346.zig"); - // _ = @import("behavior/bugs/2578.zig"); - // _ = @import("behavior/bugs/2692.zig"); - // _ = @import("behavior/bugs/2889.zig"); - // _ = @import("behavior/bugs/3007.zig"); - // _ = @import("behavior/bugs/3046.zig"); - // _ = @import("behavior/bugs/3112.zig"); - // _ = @import("behavior/bugs/3367.zig"); - // _ = @import("behavior/bugs/3384.zig"); - // _ = @import("behavior/bugs/3586.zig"); - // _ = @import("behavior/bugs/3742.zig"); - // _ = @import("behavior/bugs/3779.zig"); - // _ = @import("behavior/bugs/4328.zig"); - // _ = @import("behavior/bugs/4560.zig"); - // _ = @import("behavior/bugs/4769_a.zig"); - // _ = @import("behavior/bugs/4769_b.zig"); - // _ = @import("behavior/bugs/4954.zig"); - // _ = @import("behavior/bugs/5398.zig"); - // _ = @import("behavior/bugs/5413.zig"); - // _ = @import("behavior/bugs/5474.zig"); - // _ = @import("behavior/bugs/5487.zig"); - // _ = @import("behavior/bugs/6456.zig"); - // _ = @import("behavior/bugs/6781.zig"); - // _ = @import("behavior/bugs/6850.zig"); - // _ = @import("behavior/bugs/7003.zig"); - // _ = @import("behavior/bugs/7027.zig"); - // _ = @import("behavior/bugs/7047.zig"); - // _ = @import("behavior/bugs/7187.zig"); - // _ = @import("behavior/bugs/7250.zig"); - // _ = @import("behavior/bugs/9584.zig"); - // _ = @import("behavior/bugs/10138.zig"); - // _ = @import("behavior/bugs/10147.zig"); - // _ = @import("behavior/bugs/10970.zig"); - // _ = @import("behavior/bugs/11046.zig"); - // _ = @import("behavior/bugs/11100.zig"); - // _ = @import("behavior/bugs/11139.zig"); - // _ = @import("behavior/bugs/11159.zig"); - // _ = @import("behavior/bugs/11162.zig"); - // _ = @import("behavior/bugs/11165.zig"); - // _ = @import("behavior/bugs/11181.zig"); - // _ = @import("behavior/bugs/11182.zig"); - // _ = @import("behavior/bugs/11213.zig"); - // _ = @import("behavior/byteswap.zig"); - // _ = @import("behavior/byval_arg_var.zig"); - // _ = @import("behavior/call.zig"); - // _ = @import("behavior/cast.zig"); - // _ = @import("behavior/cast_int.zig"); - // _ = @import("behavior/comptime_memory.zig"); - // _ = @import("behavior/const_slice_child.zig"); - // _ = @import("behavior/defer.zig"); - // _ = @import("behavior/enum.zig"); - // _ = @import("behavior/error.zig"); - // _ = @import("behavior/eval.zig"); - // _ = @import("behavior/field_parent_ptr.zig"); - // _ = @import("behavior/floatop.zig"); - // _ = @import("behavior/fn.zig"); - // _ = @import("behavior/fn_delegation.zig"); - // _ = @import("behavior/fn_in_struct_in_comptime.zig"); - // _ = @import("behavior/for.zig"); - // _ = @import("behavior/generics.zig"); - // _ = @import("behavior/hasdecl.zig"); - // _ = @import("behavior/hasfield.zig"); - // _ = @import("behavior/if.zig"); - // _ = @import("behavior/import.zig"); - // _ = @import("behavior/incomplete_struct_param_tld.zig"); - // _ = @import("behavior/int128.zig"); - // _ = @import("behavior/int_div.zig"); - // _ = @import("behavior/inttoptr.zig"); - // _ = @import("behavior/ir_block_deps.zig"); - // _ = @import("behavior/math.zig"); - // _ = @import("behavior/maximum_minimum.zig"); - // _ = @import("behavior/member_func.zig"); - // _ = @import("behavior/merge_error_sets.zig"); - // _ = @import("behavior/muladd.zig"); - // _ = @import("behavior/namespace_depends_on_compile_var.zig"); - // _ = @import("behavior/null.zig"); - // _ = @import("behavior/optional.zig"); - // _ = @import("behavior/pointers.zig"); - // _ = @import("behavior/popcount.zig"); - // _ = @import("behavior/prefetch.zig"); - // _ = @import("behavior/ptrcast.zig"); - // _ = @import("behavior/pub_enum.zig"); - // _ = @import("behavior/ref_var_in_if_after_if_2nd_switch_prong.zig"); - // _ = @import("behavior/reflection.zig"); - // _ = @import("behavior/saturating_arithmetic.zig"); - // _ = @import("behavior/select.zig"); - // _ = @import("behavior/shuffle.zig"); - // _ = @import("behavior/sizeof_and_typeof.zig"); - // _ = @import("behavior/slice.zig"); - // _ = @import("behavior/slice_sentinel_comptime.zig"); - // _ = @import("behavior/src.zig"); - // _ = @import("behavior/struct.zig"); - // _ = @import("behavior/packed-struct.zig"); - // _ = @import("behavior/struct_contains_null_ptr_itself.zig"); - // _ = @import("behavior/struct_contains_slice_of_itself.zig"); - // _ = @import("behavior/switch.zig"); - // _ = @import("behavior/switch_prong_err_enum.zig"); - // _ = @import("behavior/switch_prong_implicit_cast.zig"); - // _ = @import("behavior/this.zig"); - // _ = @import("behavior/translate_c_macros.zig"); - // _ = @import("behavior/truncate.zig"); - // _ = @import("behavior/try.zig"); - // _ = @import("behavior/tuple.zig"); - // _ = @import("behavior/type.zig"); - // _ = @import("behavior/type_info.zig"); - // _ = @import("behavior/typename.zig"); - // _ = @import("behavior/undefined.zig"); - // _ = @import("behavior/underscore.zig"); - // _ = @import("behavior/union.zig"); - // _ = @import("behavior/union_with_members.zig"); - // _ = @import("behavior/usingnamespace.zig"); - // _ = @import("behavior/var_args.zig"); - // _ = @import("behavior/vector.zig"); - // _ = @import("behavior/void.zig"); - // _ = @import("behavior/while.zig"); - // _ = @import("behavior/widening.zig"); + _ = @import("behavior/bit_shifting.zig"); + _ = @import("behavior/bitcast.zig"); + _ = @import("behavior/bitreverse.zig"); + _ = @import("behavior/bool.zig"); + _ = @import("behavior/bugs/394.zig"); + _ = @import("behavior/bugs/421.zig"); + _ = @import("behavior/bugs/529.zig"); + _ = @import("behavior/bugs/624.zig"); + _ = @import("behavior/bugs/655.zig"); + _ = @import("behavior/bugs/656.zig"); + _ = @import("behavior/bugs/679.zig"); + _ = @import("behavior/bugs/704.zig"); + _ = @import("behavior/bugs/718.zig"); + _ = @import("behavior/bugs/726.zig"); + _ = @import("behavior/bugs/828.zig"); + _ = @import("behavior/bugs/920.zig"); + _ = @import("behavior/bugs/1025.zig"); + _ = @import("behavior/bugs/1076.zig"); + _ = @import("behavior/bugs/1111.zig"); + _ = @import("behavior/bugs/1120.zig"); + _ = @import("behavior/bugs/1277.zig"); + _ = @import("behavior/bugs/1310.zig"); + _ = @import("behavior/bugs/1381.zig"); + _ = @import("behavior/bugs/1421.zig"); + _ = @import("behavior/bugs/1442.zig"); + _ = @import("behavior/bugs/1486.zig"); + _ = @import("behavior/bugs/1500.zig"); + _ = @import("behavior/bugs/1607.zig"); + _ = @import("behavior/bugs/1735.zig"); + _ = @import("behavior/bugs/1741.zig"); + _ = @import("behavior/bugs/1851.zig"); + _ = @import("behavior/bugs/1914.zig"); + _ = @import("behavior/bugs/2006.zig"); + _ = @import("behavior/bugs/2114.zig"); + _ = @import("behavior/bugs/2346.zig"); + _ = @import("behavior/bugs/2578.zig"); + _ = @import("behavior/bugs/2692.zig"); + _ = @import("behavior/bugs/2889.zig"); + _ = @import("behavior/bugs/3007.zig"); + _ = @import("behavior/bugs/3046.zig"); + _ = @import("behavior/bugs/3112.zig"); + _ = @import("behavior/bugs/3367.zig"); + _ = @import("behavior/bugs/3384.zig"); + _ = @import("behavior/bugs/3586.zig"); + _ = @import("behavior/bugs/3742.zig"); + _ = @import("behavior/bugs/3779.zig"); + _ = @import("behavior/bugs/4328.zig"); + _ = @import("behavior/bugs/4560.zig"); + _ = @import("behavior/bugs/4769_a.zig"); + _ = @import("behavior/bugs/4769_b.zig"); + _ = @import("behavior/bugs/4954.zig"); + _ = @import("behavior/bugs/5398.zig"); + _ = @import("behavior/bugs/5413.zig"); + _ = @import("behavior/bugs/5474.zig"); + _ = @import("behavior/bugs/5487.zig"); + _ = @import("behavior/bugs/6456.zig"); + _ = @import("behavior/bugs/6781.zig"); + _ = @import("behavior/bugs/6850.zig"); + _ = @import("behavior/bugs/7003.zig"); + _ = @import("behavior/bugs/7027.zig"); + _ = @import("behavior/bugs/7047.zig"); + _ = @import("behavior/bugs/7187.zig"); + _ = @import("behavior/bugs/7250.zig"); + _ = @import("behavior/bugs/9584.zig"); + _ = @import("behavior/bugs/10138.zig"); + _ = @import("behavior/bugs/10147.zig"); + _ = @import("behavior/bugs/10970.zig"); + _ = @import("behavior/bugs/11046.zig"); + _ = @import("behavior/bugs/11100.zig"); + _ = @import("behavior/bugs/11139.zig"); + _ = @import("behavior/bugs/11159.zig"); + _ = @import("behavior/bugs/11162.zig"); + _ = @import("behavior/bugs/11165.zig"); + _ = @import("behavior/bugs/11181.zig"); + _ = @import("behavior/bugs/11182.zig"); + _ = @import("behavior/bugs/11213.zig"); + _ = @import("behavior/byteswap.zig"); + _ = @import("behavior/byval_arg_var.zig"); + _ = @import("behavior/call.zig"); + _ = @import("behavior/cast.zig"); + _ = @import("behavior/cast_int.zig"); + _ = @import("behavior/comptime_memory.zig"); + _ = @import("behavior/const_slice_child.zig"); + _ = @import("behavior/defer.zig"); + _ = @import("behavior/enum.zig"); + _ = @import("behavior/error.zig"); + _ = @import("behavior/eval.zig"); + _ = @import("behavior/field_parent_ptr.zig"); + _ = @import("behavior/floatop.zig"); + _ = @import("behavior/fn.zig"); + _ = @import("behavior/fn_delegation.zig"); + _ = @import("behavior/fn_in_struct_in_comptime.zig"); + _ = @import("behavior/for.zig"); + _ = @import("behavior/generics.zig"); + _ = @import("behavior/hasdecl.zig"); + _ = @import("behavior/hasfield.zig"); + _ = @import("behavior/if.zig"); + _ = @import("behavior/import.zig"); + _ = @import("behavior/incomplete_struct_param_tld.zig"); + _ = @import("behavior/int128.zig"); + _ = @import("behavior/int_div.zig"); + _ = @import("behavior/inttoptr.zig"); + _ = @import("behavior/ir_block_deps.zig"); + _ = @import("behavior/math.zig"); + _ = @import("behavior/maximum_minimum.zig"); + _ = @import("behavior/member_func.zig"); + _ = @import("behavior/merge_error_sets.zig"); + _ = @import("behavior/muladd.zig"); + _ = @import("behavior/namespace_depends_on_compile_var.zig"); + _ = @import("behavior/null.zig"); + _ = @import("behavior/optional.zig"); + _ = @import("behavior/pointers.zig"); + _ = @import("behavior/popcount.zig"); + _ = @import("behavior/prefetch.zig"); + _ = @import("behavior/ptrcast.zig"); + _ = @import("behavior/pub_enum.zig"); + _ = @import("behavior/ref_var_in_if_after_if_2nd_switch_prong.zig"); + _ = @import("behavior/reflection.zig"); + _ = @import("behavior/saturating_arithmetic.zig"); + _ = @import("behavior/select.zig"); + _ = @import("behavior/shuffle.zig"); + _ = @import("behavior/sizeof_and_typeof.zig"); + _ = @import("behavior/slice.zig"); + _ = @import("behavior/slice_sentinel_comptime.zig"); + _ = @import("behavior/src.zig"); + _ = @import("behavior/struct.zig"); + _ = @import("behavior/packed-struct.zig"); + _ = @import("behavior/struct_contains_null_ptr_itself.zig"); + _ = @import("behavior/struct_contains_slice_of_itself.zig"); + _ = @import("behavior/switch.zig"); + _ = @import("behavior/switch_prong_err_enum.zig"); + _ = @import("behavior/switch_prong_implicit_cast.zig"); + _ = @import("behavior/this.zig"); + _ = @import("behavior/translate_c_macros.zig"); + _ = @import("behavior/truncate.zig"); + _ = @import("behavior/try.zig"); + _ = @import("behavior/tuple.zig"); + _ = @import("behavior/type.zig"); + _ = @import("behavior/type_info.zig"); + _ = @import("behavior/typename.zig"); + _ = @import("behavior/undefined.zig"); + _ = @import("behavior/underscore.zig"); + _ = @import("behavior/union.zig"); + _ = @import("behavior/union_with_members.zig"); + _ = @import("behavior/usingnamespace.zig"); + _ = @import("behavior/var_args.zig"); + _ = @import("behavior/vector.zig"); + _ = @import("behavior/void.zig"); + _ = @import("behavior/while.zig"); + _ = @import("behavior/widening.zig"); if (builtin.stage2_arch == .wasm32) { _ = @import("behavior/wasm.zig"); diff --git a/test/behavior/basic.zig b/test/behavior/basic.zig index 10e48c6c7b..d62ba75dee 100644 --- a/test/behavior/basic.zig +++ b/test/behavior/basic.zig @@ -402,6 +402,7 @@ fn testPointerToVoidReturnType2() *const void { test "array 2D const double ptr" { if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; + if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO const rect_2d_vertexes = [_][1]f32{ @@ -414,6 +415,7 @@ test "array 2D const double ptr" { test "array 2D const double ptr with offset" { if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; + if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; const rect_2d_vertexes = [_][2]f32{ @@ -426,6 +428,7 @@ test "array 2D const double ptr with offset" { test "array 3D const double ptr with offset" { if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; + if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO const rect_3d_vertexes = [_][2][2]f32{ diff --git a/test/behavior/math.zig b/test/behavior/math.zig index 60c71010d4..3b73d93c01 100644 --- a/test/behavior/math.zig +++ b/test/behavior/math.zig @@ -198,7 +198,6 @@ test "const number literal" { const ten = 10; test "float equality" { - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO diff --git a/test/behavior/union.zig b/test/behavior/union.zig index 8315ea8a22..7f17ff50c6 100644 --- a/test/behavior/union.zig +++ b/test/behavior/union.zig @@ -882,6 +882,7 @@ test "extern union doesn't trigger field check at comptime" { test "anonymous union literal syntax" { if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO + if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO const S = struct { const Number = union { From 080d138b9d06402a8fbb70b4addc90751c6b798e Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Wed, 18 May 2022 17:58:05 +0200 Subject: [PATCH 13/19] x64: re-enable incremental tests --- src/arch/x86_64/bits.zig | 18 +++++++++--------- src/register_manager.zig | 26 +++++++++++++------------- src/test.zig | 18 +++++++++--------- 3 files changed, 31 insertions(+), 31 deletions(-) diff --git a/src/arch/x86_64/bits.zig b/src/arch/x86_64/bits.zig index a0ca774cae..64a067d050 100644 --- a/src/arch/x86_64/bits.zig +++ b/src/arch/x86_64/bits.zig @@ -696,8 +696,8 @@ test "Encoder helpers - general purpose registers" { }); encoder.opcode_2byte(0x0f, 0xaf); encoder.modRm_direct( - Register.eax.lowId(), - Register.edi.lowId(), + Register.eax.lowEnc(), + Register.edi.lowEnc(), ); try testing.expectEqualSlices(u8, &[_]u8{ 0x0f, 0xaf, 0xc7 }, code.items); @@ -716,8 +716,8 @@ test "Encoder helpers - general purpose registers" { }); encoder.opcode_1byte(0x89); encoder.modRm_direct( - Register.edi.lowId(), - Register.eax.lowId(), + Register.edi.lowEnc(), + Register.eax.lowEnc(), ); try testing.expectEqualSlices(u8, &[_]u8{ 0x89, 0xf8 }, code.items); @@ -743,7 +743,7 @@ test "Encoder helpers - general purpose registers" { encoder.opcode_1byte(0x81); encoder.modRm_direct( 0, - Register.rcx.lowId(), + Register.rcx.lowEnc(), ); encoder.imm32(2147483647); @@ -768,7 +768,7 @@ test "Encoder helpers - Vex prefix" { { stream.reset(); var vex_prefix = Encoder.Vex{}; - vex_prefix.reg(Register.xmm15.id()); + vex_prefix.reg(Register.xmm15.enc()); const nwritten = vex_prefix.write(writer); try testing.expectEqualSlices(u8, &[_]u8{ 0xc5, 0x80 }, buf[0..nwritten]); } @@ -808,7 +808,7 @@ test "Encoder helpers - Vex prefix" { vex.simd_prefix_66(); encoder.vex(vex); // use 64 bit operation encoder.opcode_1byte(0x28); - encoder.modRm_direct(0, Register.xmm1.lowId()); + encoder.modRm_direct(0, Register.xmm1.lowEnc()); try testing.expectEqualSlices(u8, &[_]u8{ 0xC5, 0xF9, 0x28, 0xC1 }, code.items); } @@ -822,10 +822,10 @@ test "Encoder helpers - Vex prefix" { vex.simd_prefix_66(); vex.lead_opc_0f(); vex.rex(.{ .r = true }); - vex.reg(Register.xmm1.id()); + vex.reg(Register.xmm1.enc()); encoder.vex(vex); encoder.opcode_1byte(0x16); - encoder.modRm_RIPDisp32(Register.xmm13.lowId()); + encoder.modRm_RIPDisp32(Register.xmm13.lowEnc()); encoder.disp32(0); try testing.expectEqualSlices(u8, &[_]u8{ 0xC5, 0x71, 0x16, 0x2D, 0x00, 0x00, 0x00, 0x00 }, code.items); } diff --git a/src/register_manager.zig b/src/register_manager.zig index fbdf3ef2c3..a5ab4b904b 100644 --- a/src/register_manager.zig +++ b/src/register_manager.zig @@ -430,9 +430,9 @@ test "tryAllocReg: no spilling" { const mock_instruction: Air.Inst.Index = 1; - try expectEqual(@as(?MockRegister1, .r2), function.register_manager.tryAllocReg(mock_instruction)); - try expectEqual(@as(?MockRegister1, .r3), function.register_manager.tryAllocReg(mock_instruction)); - try expectEqual(@as(?MockRegister1, null), function.register_manager.tryAllocReg(mock_instruction)); + try expectEqual(@as(?MockRegister1, .r2), function.register_manager.tryAllocReg(mock_instruction, .{})); + try expectEqual(@as(?MockRegister1, .r3), function.register_manager.tryAllocReg(mock_instruction, .{})); + try expectEqual(@as(?MockRegister1, null), function.register_manager.tryAllocReg(mock_instruction, .{})); try expect(function.register_manager.isRegAllocated(.r2)); try expect(function.register_manager.isRegAllocated(.r3)); @@ -458,16 +458,16 @@ test "allocReg: spilling" { const mock_instruction: Air.Inst.Index = 1; - try expectEqual(@as(?MockRegister1, .r2), try function.register_manager.allocReg(mock_instruction)); - try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction)); + try expectEqual(@as(?MockRegister1, .r2), try function.register_manager.allocReg(mock_instruction, .{})); + try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction, .{})); // Spill a register - try expectEqual(@as(?MockRegister1, .r2), try function.register_manager.allocReg(mock_instruction)); + try expectEqual(@as(?MockRegister1, .r2), try function.register_manager.allocReg(mock_instruction, .{})); try expectEqualSlices(MockRegister1, &[_]MockRegister1{.r2}, function.spilled.items); // No spilling necessary function.register_manager.freeReg(.r3); - try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction)); + try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction, .{})); try expectEqualSlices(MockRegister1, &[_]MockRegister1{.r2}, function.spilled.items); // Locked registers @@ -476,7 +476,7 @@ test "allocReg: spilling" { const lock = function.register_manager.lockReg(.r2); defer if (lock) |reg| function.register_manager.unlockReg(reg); - try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction)); + try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction, .{})); } try expect(!function.register_manager.lockedRegsExist()); } @@ -489,7 +489,7 @@ test "tryAllocRegs" { }; defer function.deinit(); - try expectEqual([_]MockRegister2{ .r0, .r1, .r2 }, function.register_manager.tryAllocRegs(3, .{ null, null, null }).?); + try expectEqual([_]MockRegister2{ .r0, .r1, .r2 }, function.register_manager.tryAllocRegs(3, .{ null, null, null }, .{}).?); try expect(function.register_manager.isRegAllocated(.r0)); try expect(function.register_manager.isRegAllocated(.r1)); @@ -504,7 +504,7 @@ test "tryAllocRegs" { const lock = function.register_manager.lockReg(.r1); defer if (lock) |reg| function.register_manager.unlockReg(reg); - try expectEqual([_]MockRegister2{ .r0, .r2, .r3 }, function.register_manager.tryAllocRegs(3, .{ null, null, null }).?); + try expectEqual([_]MockRegister2{ .r0, .r2, .r3 }, function.register_manager.tryAllocRegs(3, .{ null, null, null }, .{}).?); } try expect(!function.register_manager.lockedRegsExist()); @@ -543,7 +543,7 @@ test "allocRegs: normal usage" { const lock = function.register_manager.lockReg(result_reg); defer if (lock) |reg| function.register_manager.unlockReg(reg); - const regs = try function.register_manager.allocRegs(2, .{ null, null }); + const regs = try function.register_manager.allocRegs(2, .{ null, null }, .{}); try function.genAdd(result_reg, regs[0], regs[1]); } } @@ -565,12 +565,12 @@ test "allocRegs: selectively reducing register pressure" { // Here, we don't defer unlock because we manually unlock // after genAdd - const regs = try function.register_manager.allocRegs(2, .{ null, null }); + const regs = try function.register_manager.allocRegs(2, .{ null, null }, .{}); try function.genAdd(result_reg, regs[0], regs[1]); function.register_manager.unlockReg(lock.?); - const extra_summand_reg = try function.register_manager.allocReg(null); + const extra_summand_reg = try function.register_manager.allocReg(null, .{}); try function.genAdd(result_reg, result_reg, extra_summand_reg); } } diff --git a/src/test.zig b/src/test.zig index b272d05718..a6537e77c7 100644 --- a/src/test.zig +++ b/src/test.zig @@ -34,18 +34,18 @@ test { var ctx = TestContext.init(std.testing.allocator, arena); defer ctx.deinit(); - // { - // const dir_path = try std.fs.path.join(arena, &.{ - // std.fs.path.dirname(@src().file).?, "..", "test", "cases", - // }); + { + const dir_path = try std.fs.path.join(arena, &.{ + std.fs.path.dirname(@src().file).?, "..", "test", "cases", + }); - // var dir = try std.fs.cwd().openDir(dir_path, .{ .iterate = true }); - // defer dir.close(); + var dir = try std.fs.cwd().openDir(dir_path, .{ .iterate = true }); + defer dir.close(); - // ctx.addTestCasesFromDir(dir); - // } + ctx.addTestCasesFromDir(dir); + } - // try @import("test_cases").addCases(&ctx); + try @import("test_cases").addCases(&ctx); try ctx.run(); } From 283f40e4e9c44986353ba8abcc760684e9adf6cc Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 19 May 2022 15:34:13 +0200 Subject: [PATCH 14/19] x64: use StaticBitSet instead of an integer internally in RegisterManager --- src/arch/x86_64/CodeGen.zig | 11 ++-- src/arch/x86_64/abi.zig | 37 +++++++++---- src/register_manager.zig | 100 +++++++++++++++++------------------- 3 files changed, 79 insertions(+), 69 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 0cae799e02..3109470620 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -21,7 +21,6 @@ const Emit = @import("Emit.zig"); const Liveness = @import("../../Liveness.zig"); const Mir = @import("Mir.zig"); const Module = @import("../../Module.zig"); -const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager; const Target = std.Target; const Type = @import("../../type.zig").Type; const TypedValue = @import("../../TypedValue.zig"); @@ -32,15 +31,15 @@ const abi = @import("abi.zig"); const callee_preserved_regs = abi.callee_preserved_regs; const caller_preserved_regs = abi.caller_preserved_regs; -const allocatable_registers = abi.allocatable_registers; const c_abi_int_param_regs = abi.c_abi_int_param_regs; const c_abi_int_return_regs = abi.c_abi_int_return_regs; -const RegisterManager = RegisterManagerFn(Self, Register, &allocatable_registers); + +const RegisterManager = abi.RegisterManager; const RegisterLock = RegisterManager.RegisterLock; const Register = bits.Register; -const RegisterClass = abi.RegisterClass; -const gp = RegisterClass.gp; -const avx = RegisterClass.avx; + +const gp = abi.RegisterClass.gp; +const avx = abi.RegisterClass.avx; const InnerError = error{ OutOfMemory, diff --git a/src/arch/x86_64/abi.zig b/src/arch/x86_64/abi.zig index 046948ff68..bf85f002d1 100644 --- a/src/arch/x86_64/abi.zig +++ b/src/arch/x86_64/abi.zig @@ -3,6 +3,7 @@ const Type = @import("../../type.zig").Type; const Target = std.Target; const assert = std.debug.assert; const Register = @import("bits.zig").Register; +const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager; pub const Class = enum { integer, sse, sseup, x87, x87up, complex_x87, memory, none }; @@ -378,18 +379,34 @@ pub const callee_preserved_regs = [_]Register{ .rbx, .r12, .r13, .r14, .r15 }; /// the caller relinquishes control to a subroutine via call instruction (or similar). /// In other words, these registers are free to use by the callee. pub const caller_preserved_regs = [_]Register{ .rax, .rcx, .rdx, .rsi, .rdi, .r8, .r9, .r10, .r11 }; -pub const avx_regs = [_]Register{ - .ymm0, .ymm1, .ymm2, .ymm3, .ymm4, .ymm5, .ymm6, .ymm7, - .ymm8, .ymm9, .ymm10, .ymm11, .ymm12, .ymm13, .ymm14, .ymm15, -}; -pub const allocatable_registers = callee_preserved_regs ++ caller_preserved_regs ++ avx_regs; pub const c_abi_int_param_regs = [_]Register{ .rdi, .rsi, .rdx, .rcx, .r8, .r9 }; pub const c_abi_int_return_regs = [_]Register{ .rax, .rdx }; -// Masks for register manager -const FreeRegInt = std.meta.Int(.unsigned, allocatable_registers.len); -pub const RegisterClass = struct { - pub const gp: FreeRegInt = 0x3fff; - pub const avx: FreeRegInt = 0x3fff_c000; +const avx_regs = [_]Register{ + .ymm0, .ymm1, .ymm2, .ymm3, .ymm4, .ymm5, .ymm6, .ymm7, + .ymm8, .ymm9, .ymm10, .ymm11, .ymm12, .ymm13, .ymm14, .ymm15, +}; +const allocatable_registers = callee_preserved_regs ++ caller_preserved_regs ++ avx_regs; +pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, &allocatable_registers); + +// Register classes +const RegisterBitSet = RegisterManager.RegisterBitSet; +pub const RegisterClass = struct { + pub const gp: RegisterBitSet = blk: { + var set = RegisterBitSet.initEmpty(); + set.setRangeValue(.{ + .start = 0, + .end = caller_preserved_regs.len + callee_preserved_regs.len, + }, true); + break :blk set; + }; + pub const avx: RegisterBitSet = blk: { + var set = RegisterBitSet.initEmpty(); + set.setRangeValue(.{ + .start = caller_preserved_regs.len + callee_preserved_regs.len, + .end = allocatable_registers.len, + }, true); + break :blk set; + }; }; diff --git a/src/register_manager.zig b/src/register_manager.zig index a5ab4b904b..7b82253e22 100644 --- a/src/register_manager.zig +++ b/src/register_manager.zig @@ -4,6 +4,7 @@ const mem = std.mem; const assert = std.debug.assert; const Allocator = std.mem.Allocator; const Air = @import("Air.zig"); +const StaticBitSet = std.bit_set.StaticBitSet; const Type = @import("type.zig").Type; const Module = @import("Module.zig"); const expect = std.testing.expect; @@ -41,66 +42,54 @@ pub fn RegisterManager( registers: [tracked_registers.len]Air.Inst.Index = undefined, /// Tracks which registers are free (in which case the /// corresponding bit is set to 1) - free_registers: FreeRegInt = math.maxInt(FreeRegInt), + free_registers: RegisterBitSet = RegisterBitSet.initFull(), /// Tracks all registers allocated in the course of this /// function - allocated_registers: FreeRegInt = 0, + allocated_registers: RegisterBitSet = RegisterBitSet.initEmpty(), /// Tracks registers which are locked from being allocated - locked_registers: FreeRegInt = 0, + locked_registers: RegisterBitSet = RegisterBitSet.initEmpty(), const Self = @This(); - /// An integer whose bits represent all the registers and - /// whether they are free. - const FreeRegInt = std.meta.Int(.unsigned, tracked_registers.len); - const ShiftInt = math.Log2Int(FreeRegInt); + pub const RegisterBitSet = StaticBitSet(tracked_registers.len); fn getFunction(self: *Self) *Function { return @fieldParentPtr(Function, "register_manager", self); } - fn getRegisterMask(reg: Register) ?FreeRegInt { - const index = indexOfRegIntoTracked(reg) orelse return null; - const shift = @intCast(ShiftInt, index); - const mask = @as(FreeRegInt, 1) << shift; - return mask; - } - - fn excludeRegister(reg: Register, mask: FreeRegInt) bool { - const reg_mask = getRegisterMask(reg) orelse return true; - return reg_mask & mask == 0; - } - fn markRegAllocated(self: *Self, reg: Register) void { - const mask = getRegisterMask(reg) orelse return; - self.allocated_registers |= mask; + const index = indexOfRegIntoTracked(reg) orelse return; + self.allocated_registers.set(index); } fn markRegUsed(self: *Self, reg: Register) void { - const mask = getRegisterMask(reg) orelse return; - self.free_registers &= ~mask; + const index = indexOfRegIntoTracked(reg) orelse return; + self.free_registers.unset(index); } fn markRegFree(self: *Self, reg: Register) void { - const mask = getRegisterMask(reg) orelse return; - self.free_registers |= mask; + const index = indexOfRegIntoTracked(reg) orelse return; + self.free_registers.set(index); } - pub fn indexOfReg(comptime registers: []const Register, reg: Register) ?std.math.IntFittingRange(0, registers.len - 1) { + pub fn indexOfReg( + comptime registers: []const Register, + reg: Register, + ) ?std.math.IntFittingRange(0, registers.len - 1) { inline for (tracked_registers) |cpreg, i| { if (reg.id() == cpreg.id()) return i; } return null; } - pub fn indexOfRegIntoTracked(reg: Register) ?ShiftInt { + pub fn indexOfRegIntoTracked(reg: Register) ?RegisterBitSet.ShiftInt { return indexOfReg(tracked_registers, reg); } /// Returns true when this register is not tracked pub fn isRegFree(self: Self, reg: Register) bool { - const mask = getRegisterMask(reg) orelse return true; - return self.free_registers & mask != 0; + const index = indexOfRegIntoTracked(reg) orelse return true; + return self.free_registers.isSet(index); } /// Returns whether this register was allocated in the course @@ -108,16 +97,16 @@ pub fn RegisterManager( /// /// Returns false when this register is not tracked pub fn isRegAllocated(self: Self, reg: Register) bool { - const mask = getRegisterMask(reg) orelse return false; - return self.allocated_registers & mask != 0; + const index = indexOfRegIntoTracked(reg) orelse return false; + return self.allocated_registers.isSet(index); } /// Returns whether this register is locked /// /// Returns false when this register is not tracked pub fn isRegLocked(self: Self, reg: Register) bool { - const mask = getRegisterMask(reg) orelse return false; - return self.locked_registers & mask != 0; + const index = indexOfRegIntoTracked(reg) orelse return false; + return self.locked_registers.isSet(index); } pub const RegisterLock = struct { @@ -136,8 +125,8 @@ pub fn RegisterManager( log.debug(" register already locked", .{}); return null; } - const mask = getRegisterMask(reg) orelse return null; - self.locked_registers |= mask; + const index = indexOfRegIntoTracked(reg) orelse return null; + self.locked_registers.set(index); return RegisterLock{ .register = reg }; } @@ -146,8 +135,8 @@ pub fn RegisterManager( pub fn lockRegAssumeUnused(self: *Self, reg: Register) RegisterLock { log.debug("locking asserting free {}", .{reg}); assert(!self.isRegLocked(reg)); - const mask = getRegisterMask(reg) orelse unreachable; - self.locked_registers |= mask; + const index = indexOfRegIntoTracked(reg) orelse unreachable; + self.locked_registers.set(index); return RegisterLock{ .register = reg }; } @@ -169,17 +158,17 @@ pub fn RegisterManager( /// Call `lockReg` to obtain the lock first. pub fn unlockReg(self: *Self, lock: RegisterLock) void { log.debug("unlocking {}", .{lock.register}); - const mask = getRegisterMask(lock.register) orelse return; - self.locked_registers &= ~mask; + const index = indexOfRegIntoTracked(lock.register) orelse return; + self.locked_registers.unset(index); } /// Returns true when at least one register is locked pub fn lockedRegsExist(self: Self) bool { - return self.locked_registers != 0; + return self.locked_registers.count() > 0; } const AllocOpts = struct { - selector_mask: ?FreeRegInt = null, + selector_mask: ?RegisterBitSet = null, }; /// Allocates a specified number of registers, optionally @@ -193,17 +182,22 @@ pub fn RegisterManager( ) ?[count]Register { comptime assert(count > 0 and count <= tracked_registers.len); - const selector_mask = if (opts.selector_mask) |mask| mask else ~@as(FreeRegInt, 0); - const free_registers = self.free_registers & selector_mask; - const free_and_not_locked_registers = free_registers & ~self.locked_registers; - const free_and_not_locked_registers_count = @popCount(FreeRegInt, free_and_not_locked_registers); - if (free_and_not_locked_registers_count < count) return null; + const available_registers = opts.selector_mask orelse RegisterBitSet.initFull(); + + var free_and_not_locked_registers = self.free_registers; + free_and_not_locked_registers.setIntersection(available_registers); + + var unlocked_registers = self.locked_registers; + unlocked_registers.toggleAll(); + + free_and_not_locked_registers.setIntersection(unlocked_registers); + + if (free_and_not_locked_registers.count() < count) return null; var regs: [count]Register = undefined; var i: usize = 0; for (tracked_registers) |reg| { if (i >= count) break; - if (excludeRegister(reg, selector_mask)) continue; if (self.isRegLocked(reg)) continue; if (!self.isRegFree(reg)) continue; @@ -244,11 +238,12 @@ pub fn RegisterManager( ) AllocateRegistersError![count]Register { comptime assert(count > 0 and count <= tracked_registers.len); - const selector_mask = if (opts.selector_mask) |mask| mask else ~@as(FreeRegInt, 0); - const available_registers_count = @popCount(FreeRegInt, selector_mask); - const locked_registers = self.locked_registers & selector_mask; - const locked_registers_count = @popCount(FreeRegInt, locked_registers); - if (count > available_registers_count - locked_registers_count) return error.OutOfRegisters; + const available_registers = opts.selector_mask orelse RegisterBitSet.initFull(); + + var locked_registers = self.locked_registers; + locked_registers.setIntersection(available_registers); + + if (count > available_registers.count() - locked_registers.count()) return error.OutOfRegisters; const result = self.tryAllocRegs(count, insts, opts) orelse blk: { // We'll take over the first count registers. Spill @@ -258,7 +253,6 @@ pub fn RegisterManager( var i: usize = 0; for (tracked_registers) |reg| { if (i >= count) break; - if (excludeRegister(reg, selector_mask)) continue; if (self.isRegLocked(reg)) continue; regs[i] = reg; From 5cbfd5819e423cdc6b092d1eb687189fb204b075 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 19 May 2022 17:36:04 +0200 Subject: [PATCH 15/19] x64: check for floating-point intrinsics in codegen --- src/arch/x86_64/CodeGen.zig | 283 +++++++++++++++++++++--------------- src/arch/x86_64/Emit.zig | 12 +- src/arch/x86_64/Mir.zig | 12 +- src/arch/x86_64/abi.zig | 6 +- 4 files changed, 179 insertions(+), 134 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 3109470620..2e4a396c9f 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -39,7 +39,7 @@ const RegisterLock = RegisterManager.RegisterLock; const Register = bits.Register; const gp = abi.RegisterClass.gp; -const avx = abi.RegisterClass.avx; +const sse = abi.RegisterClass.sse; const InnerError = error{ OutOfMemory, @@ -881,15 +881,18 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue { switch (elem_ty.zigTypeTag()) { .Vector => return self.fail("TODO allocRegOrMem for Vector type", .{}), .Float => { - // TODO check if AVX available - const ptr_bytes: u64 = 32; - if (abi_size <= ptr_bytes) { - if (self.register_manager.tryAllocReg(inst, .{ - .selector_mask = avx, - })) |reg| { - return MCValue{ .register = registerAlias(reg, abi_size) }; + if (self.intrinsicsAllowed(elem_ty)) { + const ptr_bytes: u64 = 32; + if (abi_size <= ptr_bytes) { + if (self.register_manager.tryAllocReg(inst, .{ + .selector_mask = sse, + })) |reg| { + return MCValue{ .register = registerAlias(reg, abi_size) }; + } } } + + return self.fail("TODO allocRegOrMem for Float type without SSE/AVX support", .{}); }, else => { // Make sure the type can fit in a register before we try to allocate one. @@ -969,8 +972,11 @@ pub fn spillRegisters(self: *Self, comptime count: comptime_int, registers: [cou /// allocated. A second call to `copyToTmpRegister` may return the same register. /// This can have a side effect of spilling instructions to the stack to free up a register. fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register { - const mask = switch (ty.zigTypeTag()) { - .Float => avx, + const mask: RegisterManager.RegisterBitSet = switch (ty.zigTypeTag()) { + .Float => blk: { + if (self.intrinsicsAllowed(ty)) break :blk sse; + return self.fail("TODO copy {} to register", .{ty.fmtDebug()}); + }, else => gp, }; const reg: Register = try self.register_manager.allocReg(null, .{ @@ -985,8 +991,11 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register { /// This can have a side effect of spilling instructions to the stack to free up a register. /// WARNING make sure that the allocated register matches the returned MCValue from an instruction! fn copyToRegisterWithInstTracking(self: *Self, reg_owner: Air.Inst.Index, ty: Type, mcv: MCValue) !MCValue { - const mask = switch (ty.zigTypeTag()) { - .Float => avx, + const mask: RegisterManager.RegisterBitSet = switch (ty.zigTypeTag()) { + .Float => blk: { + if (self.intrinsicsAllowed(ty)) break :blk sse; + return self.fail("TODO copy {} to register", .{ty.fmtDebug()}); + }, else => gp, }; const reg: Register = try self.register_manager.allocReg(reg_owner, .{ @@ -3469,27 +3478,32 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu }, .register => |src_reg| switch (dst_ty.zigTypeTag()) { .Float => { - const actual_tag: Mir.Inst.Tag = switch (dst_ty.tag()) { - .f32 => switch (mir_tag) { - .add => Mir.Inst.Tag.add_f32, - .cmp => Mir.Inst.Tag.cmp_f32, - else => return self.fail("TODO genBinOpMir for f32 register-register with MIR tag {}", .{mir_tag}), - }, - .f64 => switch (mir_tag) { - .add => Mir.Inst.Tag.add_f64, - .cmp => Mir.Inst.Tag.cmp_f64, - else => return self.fail("TODO genBinOpMir for f64 register-register with MIR tag {}", .{mir_tag}), - }, - else => return self.fail("TODO genBinOpMir for float register-register and type {}", .{dst_ty.fmtDebug()}), - }; - _ = try self.addInst(.{ - .tag = actual_tag, - .ops = Mir.Inst.Ops.encode(.{ - .reg1 = dst_reg.to128(), - .reg2 = src_reg.to128(), - }), - .data = undefined, - }); + if (self.intrinsicsAllowed(dst_ty)) { + const actual_tag: Mir.Inst.Tag = switch (dst_ty.tag()) { + .f32 => switch (mir_tag) { + .add => Mir.Inst.Tag.add_f32_avx, + .cmp => Mir.Inst.Tag.cmp_f32_avx, + else => return self.fail("TODO genBinOpMir for f32 register-register with MIR tag {}", .{mir_tag}), + }, + .f64 => switch (mir_tag) { + .add => Mir.Inst.Tag.add_f64_avx, + .cmp => Mir.Inst.Tag.cmp_f64_avx, + else => return self.fail("TODO genBinOpMir for f64 register-register with MIR tag {}", .{mir_tag}), + }, + else => return self.fail("TODO genBinOpMir for float register-register and type {}", .{dst_ty.fmtDebug()}), + }; + _ = try self.addInst(.{ + .tag = actual_tag, + .ops = Mir.Inst.Ops.encode(.{ + .reg1 = dst_reg.to128(), + .reg2 = src_reg.to128(), + }), + .data = undefined, + }); + return; + } + + return self.fail("TODO genBinOpMir for float register-register and no intrinsics", .{}); }, else => { _ = try self.addInst(.{ @@ -5326,24 +5340,29 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE .register => |reg| { switch (ty.zigTypeTag()) { .Float => { - const tag: Mir.Inst.Tag = switch (ty.tag()) { - .f32 => .mov_f32, - .f64 => .mov_f64, - else => return self.fail("TODO genSetStackArg for register for type {}", .{ty.fmtDebug()}), - }; - _ = try self.addInst(.{ - .tag = tag, - .ops = Mir.Inst.Ops.encode(.{ - .reg1 = switch (ty.tag()) { - .f32 => .esp, - .f64 => .rsp, - else => unreachable, - }, - .reg2 = reg.to128(), - .flags = 0b01, - }), - .data = .{ .imm = @bitCast(u32, -stack_offset) }, - }); + if (self.intrinsicsAllowed(ty)) { + const tag: Mir.Inst.Tag = switch (ty.tag()) { + .f32 => .mov_f32_avx, + .f64 => .mov_f64_avx, + else => return self.fail("TODO genSetStackArg for register for type {}", .{ty.fmtDebug()}), + }; + _ = try self.addInst(.{ + .tag = tag, + .ops = Mir.Inst.Ops.encode(.{ + .reg1 = switch (ty.tag()) { + .f32 => .esp, + .f64 => .rsp, + else => unreachable, + }, + .reg2 = reg.to128(), + .flags = 0b01, + }), + .data = .{ .imm = @bitCast(u32, -stack_offset) }, + }); + return; + } + + return self.fail("TODO genSetStackArg for register with no intrinsics", .{}); }, else => { _ = try self.addInst(.{ @@ -5505,24 +5524,29 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl switch (ty.zigTypeTag()) { .Float => { - const tag: Mir.Inst.Tag = switch (ty.tag()) { - .f32 => .mov_f32, - .f64 => .mov_f64, - else => return self.fail("TODO genSetStack for register for type {}", .{ty.fmtDebug()}), - }; - _ = try self.addInst(.{ - .tag = tag, - .ops = Mir.Inst.Ops.encode(.{ - .reg1 = switch (ty.tag()) { - .f32 => base_reg.to32(), - .f64 => base_reg.to64(), - else => unreachable, - }, - .reg2 = reg.to128(), - .flags = 0b01, - }), - .data = .{ .imm = @bitCast(u32, -stack_offset) }, - }); + if (self.intrinsicsAllowed(ty)) { + const tag: Mir.Inst.Tag = switch (ty.tag()) { + .f32 => .mov_f32_avx, + .f64 => .mov_f64_avx, + else => return self.fail("TODO genSetStack for register for type {}", .{ty.fmtDebug()}), + }; + _ = try self.addInst(.{ + .tag = tag, + .ops = Mir.Inst.Ops.encode(.{ + .reg1 = switch (ty.tag()) { + .f32 => base_reg.to32(), + .f64 => base_reg.to64(), + else => unreachable, + }, + .reg2 = reg.to128(), + .flags = 0b01, + }), + .data = .{ .imm = @bitCast(u32, -stack_offset) }, + }); + return; + } + + return self.fail("TODO genSetStack for register for type float with no intrinsics", .{}); }, else => { if (!math.isPowerOfTwo(abi_size)) { @@ -6026,21 +6050,25 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void }, }, .Float => { - const tag: Mir.Inst.Tag = switch (ty.tag()) { - .f32 => .mov_f32, - .f64 => .mov_f64, - else => return self.fail("TODO genSetReg from register for {}", .{ty.fmtDebug()}), - }; - _ = try self.addInst(.{ - .tag = tag, - .ops = Mir.Inst.Ops.encode(.{ - .reg1 = reg.to128(), - .reg2 = src_reg.to128(), - .flags = 0b10, - }), - .data = undefined, - }); - return; + if (self.intrinsicsAllowed(ty)) { + const tag: Mir.Inst.Tag = switch (ty.tag()) { + .f32 => .mov_f32_avx, + .f64 => .mov_f64_avx, + else => return self.fail("TODO genSetReg from register for {}", .{ty.fmtDebug()}), + }; + _ = try self.addInst(.{ + .tag = tag, + .ops = Mir.Inst.Ops.encode(.{ + .reg1 = reg.to128(), + .reg2 = src_reg.to128(), + .flags = 0b10, + }), + .data = undefined, + }); + return; + } + + return self.fail("TODO genSetReg from register for float with no intrinsics", .{}); }, else => {}, } @@ -6073,24 +6101,29 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void const base_reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp }); try self.loadMemPtrIntoRegister(base_reg, Type.usize, mcv); - const tag: Mir.Inst.Tag = switch (ty.tag()) { - .f32 => .mov_f32, - .f64 => .mov_f64, - else => return self.fail("TODO genSetReg from memory for {}", .{ty.fmtDebug()}), - }; + if (self.intrinsicsAllowed(ty)) { + const tag: Mir.Inst.Tag = switch (ty.tag()) { + .f32 => .mov_f32_avx, + .f64 => .mov_f64_avx, + else => return self.fail("TODO genSetReg from memory for {}", .{ty.fmtDebug()}), + }; - _ = try self.addInst(.{ - .tag = tag, - .ops = Mir.Inst.Ops.encode(.{ - .reg1 = reg.to128(), - .reg2 = switch (ty.tag()) { - .f32 => base_reg.to32(), - .f64 => base_reg.to64(), - else => unreachable, - }, - }), - .data = .{ .imm = 0 }, - }); + _ = try self.addInst(.{ + .tag = tag, + .ops = Mir.Inst.Ops.encode(.{ + .reg1 = reg.to128(), + .reg2 = switch (ty.tag()) { + .f32 => base_reg.to32(), + .f64 => base_reg.to64(), + else => unreachable, + }, + }), + .data = .{ .imm = 0 }, + }); + return; + } + + return self.fail("TODO genSetReg from memory for float with no intrinsics", .{}); }, else => { if (x <= math.maxInt(i32)) { @@ -6183,24 +6216,27 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void }, }, .Float => { - const tag: Mir.Inst.Tag = switch (ty.tag()) { - .f32 => .mov_f32, - .f64 => .mov_f64, - else => return self.fail("TODO genSetReg from stack offset for {}", .{ty.fmtDebug()}), - }; - _ = try self.addInst(.{ - .tag = tag, - .ops = Mir.Inst.Ops.encode(.{ - .reg1 = reg.to128(), - .reg2 = switch (ty.tag()) { - .f32 => .ebp, - .f64 => .rbp, - else => unreachable, - }, - }), - .data = .{ .imm = @bitCast(u32, -off) }, - }); - return; + if (self.intrinsicsAllowed(ty)) { + const tag: Mir.Inst.Tag = switch (ty.tag()) { + .f32 => .mov_f32_avx, + .f64 => .mov_f64_avx, + else => return self.fail("TODO genSetReg from stack offset for {}", .{ty.fmtDebug()}), + }; + _ = try self.addInst(.{ + .tag = tag, + .ops = Mir.Inst.Ops.encode(.{ + .reg1 = reg.to128(), + .reg2 = switch (ty.tag()) { + .f32 => .ebp, + .f64 => .rbp, + else => unreachable, + }, + }), + .data = .{ .imm = @bitCast(u32, -off) }, + }); + return; + } + return self.fail("TODO genSetReg from stack offset for float with no intrinsics", .{}); }, else => {}, } @@ -6995,3 +7031,12 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void { }, } } + +fn intrinsicsAllowed(self: *Self, ty: Type) bool { + return switch (ty.tag()) { + .f32, + .f64, + => Target.x86.featureSetHasAny(self.target.cpu.features, .{ .avx, .avx2 }), + else => unreachable, // TODO finish this off + }; +} diff --git a/src/arch/x86_64/Emit.zig b/src/arch/x86_64/Emit.zig index 96f640b610..fbcd8359f7 100644 --- a/src/arch/x86_64/Emit.zig +++ b/src/arch/x86_64/Emit.zig @@ -183,14 +183,14 @@ pub fn lowerMir(emit: *Emit) InnerError!void { .nop => try emit.mirNop(), // AVX instructions - .mov_f64 => try emit.mirMovFloatAvx(.vmovsd, inst), - .mov_f32 => try emit.mirMovFloatAvx(.vmovss, inst), + .mov_f64_avx => try emit.mirMovFloatAvx(.vmovsd, inst), + .mov_f32_avx => try emit.mirMovFloatAvx(.vmovss, inst), - .add_f64 => try emit.mirAddFloatAvx(.vaddsd, inst), - .add_f32 => try emit.mirAddFloatAvx(.vaddss, inst), + .add_f64_avx => try emit.mirAddFloatAvx(.vaddsd, inst), + .add_f32_avx => try emit.mirAddFloatAvx(.vaddss, inst), - .cmp_f64 => try emit.mirCmpFloatAvx(.vucomisd, inst), - .cmp_f32 => try emit.mirCmpFloatAvx(.vucomiss, inst), + .cmp_f64_avx => try emit.mirCmpFloatAvx(.vucomisd, inst), + .cmp_f32_avx => try emit.mirCmpFloatAvx(.vucomiss, inst), // Pseudo-instructions .call_extern => try emit.mirCallExtern(inst), diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig index dc8c1fa0b2..0f200d43e6 100644 --- a/src/arch/x86_64/Mir.zig +++ b/src/arch/x86_64/Mir.zig @@ -350,18 +350,18 @@ pub const Inst = struct { /// 0b00 reg1, qword ptr [reg2 + imm32] /// 0b01 qword ptr [reg1 + imm32], reg2 /// 0b10 reg1, reg2 - mov_f64, - mov_f32, + mov_f64_avx, + mov_f32_avx, /// ops flags: form: /// 0b00 reg1, reg1, reg2 - add_f64, - add_f32, + add_f64_avx, + add_f32_avx, /// ops flags: form: /// - cmp_f64, - cmp_f32, + cmp_f64_avx, + cmp_f32_avx, /// Pseudo-instructions /// call extern function diff --git a/src/arch/x86_64/abi.zig b/src/arch/x86_64/abi.zig index bf85f002d1..7e2025a23d 100644 --- a/src/arch/x86_64/abi.zig +++ b/src/arch/x86_64/abi.zig @@ -383,11 +383,11 @@ pub const caller_preserved_regs = [_]Register{ .rax, .rcx, .rdx, .rsi, .rdi, .r8 pub const c_abi_int_param_regs = [_]Register{ .rdi, .rsi, .rdx, .rcx, .r8, .r9 }; pub const c_abi_int_return_regs = [_]Register{ .rax, .rdx }; -const avx_regs = [_]Register{ +const sse_avx_regs = [_]Register{ .ymm0, .ymm1, .ymm2, .ymm3, .ymm4, .ymm5, .ymm6, .ymm7, .ymm8, .ymm9, .ymm10, .ymm11, .ymm12, .ymm13, .ymm14, .ymm15, }; -const allocatable_registers = callee_preserved_regs ++ caller_preserved_regs ++ avx_regs; +const allocatable_registers = callee_preserved_regs ++ caller_preserved_regs ++ sse_avx_regs; pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, &allocatable_registers); // Register classes @@ -401,7 +401,7 @@ pub const RegisterClass = struct { }, true); break :blk set; }; - pub const avx: RegisterBitSet = blk: { + pub const sse: RegisterBitSet = blk: { var set = RegisterBitSet.initEmpty(); set.setRangeValue(.{ .start = caller_preserved_regs.len + callee_preserved_regs.len, From f766b25f82a6c94d6b77f6a4172bfc4feb1d9271 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 19 May 2022 20:24:06 +0200 Subject: [PATCH 16/19] x64: load float from memory to register on PIE targets --- src/arch/x86_64/CodeGen.zig | 52 ++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 2e4a396c9f..bd17b2a9ae 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -6085,16 +6085,48 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void .direct_load, .got_load, => { - try self.loadMemPtrIntoRegister(reg, Type.usize, mcv); - _ = try self.addInst(.{ - .tag = .mov, - .ops = Mir.Inst.Ops.encode(.{ - .reg1 = registerAlias(reg, abi_size), - .reg2 = reg.to64(), - .flags = 0b01, - }), - .data = .{ .imm = 0 }, - }); + switch (ty.zigTypeTag()) { + .Float => { + const base_reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp }); + try self.loadMemPtrIntoRegister(base_reg, Type.usize, mcv); + + if (self.intrinsicsAllowed(ty)) { + const tag: Mir.Inst.Tag = switch (ty.tag()) { + .f32 => .mov_f32_avx, + .f64 => .mov_f64_avx, + else => return self.fail("TODO genSetReg from memory for {}", .{ty.fmtDebug()}), + }; + + _ = try self.addInst(.{ + .tag = tag, + .ops = Mir.Inst.Ops.encode(.{ + .reg1 = reg.to128(), + .reg2 = switch (ty.tag()) { + .f32 => base_reg.to32(), + .f64 => base_reg.to64(), + else => unreachable, + }, + }), + .data = .{ .imm = 0 }, + }); + return; + } + + return self.fail("TODO genSetReg from memory for float with no intrinsics", .{}); + }, + else => { + try self.loadMemPtrIntoRegister(reg, Type.usize, mcv); + _ = try self.addInst(.{ + .tag = .mov, + .ops = Mir.Inst.Ops.encode(.{ + .reg1 = registerAlias(reg, abi_size), + .reg2 = reg.to64(), + .flags = 0b01, + }), + .data = .{ .imm = 0 }, + }); + }, + } }, .memory => |x| switch (ty.zigTypeTag()) { .Float => { From e95dfac03eea942b53bbdf6117a716aaeba5eb4b Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Thu, 19 May 2022 23:17:43 +0200 Subject: [PATCH 17/19] regalloc: make register class bitmask non-optional --- src/arch/aarch64/CodeGen.zig | 62 +++++++++---------- src/arch/aarch64/abi.zig | 17 ++++++ src/arch/arm/CodeGen.zig | 71 +++++++++++----------- src/arch/arm/abi.zig | 18 +++++- src/arch/riscv64/CodeGen.zig | 22 +++---- src/arch/riscv64/abi.zig | 17 ++++++ src/arch/sparc64/CodeGen.zig | 32 +++++----- src/arch/sparc64/abi.zig | 18 +++++- src/arch/x86_64/CodeGen.zig | 64 ++++++++------------ src/register_manager.zig | 111 +++++++++++++++++++++++++---------- 10 files changed, 263 insertions(+), 169 deletions(-) diff --git a/src/arch/aarch64/CodeGen.zig b/src/arch/aarch64/CodeGen.zig index a3116587d4..f4f2b1e5e5 100644 --- a/src/arch/aarch64/CodeGen.zig +++ b/src/arch/aarch64/CodeGen.zig @@ -21,9 +21,6 @@ const DW = std.dwarf; const leb128 = std.leb; const log = std.log.scoped(.codegen); const build_options = @import("build_options"); -const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager; -const RegisterManager = RegisterManagerFn(Self, Register, &callee_preserved_regs); -const RegisterLock = RegisterManager.RegisterLock; const GenerateSymbolError = @import("../../codegen.zig").GenerateSymbolError; const FnResult = @import("../../codegen.zig").FnResult; @@ -31,11 +28,14 @@ const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput; const bits = @import("bits.zig"); const abi = @import("abi.zig"); +const RegisterManager = abi.RegisterManager; +const RegisterLock = RegisterManager.RegisterLock; const Register = bits.Register; const Instruction = bits.Instruction; const callee_preserved_regs = abi.callee_preserved_regs; const c_abi_int_param_regs = abi.c_abi_int_param_regs; const c_abi_int_return_regs = abi.c_abi_int_return_regs; +const gp = abi.RegisterClass.gp; const InnerError = error{ OutOfMemory, @@ -888,7 +888,7 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue { if (reg_ok) { // Make sure the type can fit in a register before we try to allocate one. if (abi_size <= 8) { - if (self.register_manager.tryAllocReg(inst, .{})) |reg| { + if (self.register_manager.tryAllocReg(inst, gp)) |reg| { return MCValue{ .register = registerAlias(reg, abi_size) }; } } @@ -951,7 +951,7 @@ fn spillCompareFlagsIfOccupied(self: *Self) !void { /// allocated. A second call to `copyToTmpRegister` may return the same register. /// This can have a side effect of spilling instructions to the stack to free up a register. fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register { - const raw_reg = try self.register_manager.allocReg(null, .{}); + const raw_reg = try self.register_manager.allocReg(null, gp); const reg = registerAlias(raw_reg, ty.abiSize(self.target.*)); try self.genSetReg(ty, reg, mcv); return reg; @@ -961,7 +961,7 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register { /// `reg_owner` is the instruction that gets associated with the register in the register table. /// This can have a side effect of spilling instructions to the stack to free up a register. fn copyToNewRegister(self: *Self, reg_owner: Air.Inst.Index, mcv: MCValue) !MCValue { - const raw_reg = try self.register_manager.allocReg(reg_owner, .{}); + const raw_reg = try self.register_manager.allocReg(reg_owner, gp); const ty = self.air.typeOfIndex(reg_owner); const reg = registerAlias(raw_reg, ty.abiSize(self.target.*)); try self.genSetReg(self.air.typeOfIndex(reg_owner), reg, mcv); @@ -1074,11 +1074,11 @@ fn trunc( if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) { break :blk registerAlias(operand_reg, dest_ty.abiSize(self.target.*)); } else { - const raw_reg = try self.register_manager.allocReg(inst, .{}); + const raw_reg = try self.register_manager.allocReg(inst, gp); break :blk registerAlias(raw_reg, dest_ty.abiSize(self.target.*)); } } else blk: { - const raw_reg = try self.register_manager.allocReg(null, .{}); + const raw_reg = try self.register_manager.allocReg(null, gp); break :blk registerAlias(raw_reg, dest_ty.abiSize(self.target.*)); }; @@ -1160,7 +1160,7 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void { break :blk op_reg; } - const raw_reg = try self.register_manager.allocReg(null, .{}); + const raw_reg = try self.register_manager.allocReg(null, gp); break :blk raw_reg.to32(); }; @@ -1193,7 +1193,7 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void { break :blk op_reg; } - const raw_reg = try self.register_manager.allocReg(null, .{}); + const raw_reg = try self.register_manager.allocReg(null, gp); break :blk registerAlias(raw_reg, operand_ty.abiSize(self.target.*)); }; @@ -1293,7 +1293,7 @@ fn binOpRegister( break :inst Air.refToIndex(md.lhs).?; } else null; - const raw_reg = try self.register_manager.allocReg(track_inst, .{}); + const raw_reg = try self.register_manager.allocReg(track_inst, gp); const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*)); if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg }); @@ -1308,7 +1308,7 @@ fn binOpRegister( break :inst Air.refToIndex(md.rhs).?; } else null; - const raw_reg = try self.register_manager.allocReg(track_inst, .{}); + const raw_reg = try self.register_manager.allocReg(track_inst, gp); const reg = registerAlias(raw_reg, rhs_ty.abiAlignment(self.target.*)); if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg }); @@ -1326,11 +1326,11 @@ fn binOpRegister( } else if (rhs_is_register and self.reuseOperand(md.inst, md.rhs, 1, rhs)) { break :blk rhs_reg; } else { - const raw_reg = try self.register_manager.allocReg(md.inst, .{}); + const raw_reg = try self.register_manager.allocReg(md.inst, gp); break :blk registerAlias(raw_reg, lhs_ty.abiSize(self.target.*)); } } else blk: { - const raw_reg = try self.register_manager.allocReg(null, .{}); + const raw_reg = try self.register_manager.allocReg(null, gp); break :blk registerAlias(raw_reg, lhs_ty.abiSize(self.target.*)); }, }; @@ -1431,7 +1431,7 @@ fn binOpImmediate( ).?; } else null; - const raw_reg = try self.register_manager.allocReg(track_inst, .{}); + const raw_reg = try self.register_manager.allocReg(track_inst, gp); const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*)); if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg }); @@ -1452,11 +1452,11 @@ fn binOpImmediate( )) { break :blk lhs_reg; } else { - const raw_reg = try self.register_manager.allocReg(md.inst, .{}); + const raw_reg = try self.register_manager.allocReg(md.inst, gp); break :blk registerAlias(raw_reg, lhs_ty.abiSize(self.target.*)); } } else blk: { - const raw_reg = try self.register_manager.allocReg(null, .{}); + const raw_reg = try self.register_manager.allocReg(null, gp); break :blk registerAlias(raw_reg, lhs_ty.abiSize(self.target.*)); }, }; @@ -1872,7 +1872,7 @@ fn airOverflow(self: *Self, inst: Air.Inst.Index) !void { const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg); defer self.register_manager.unlockReg(dest_reg_lock); - const raw_truncated_reg = try self.register_manager.allocReg(null, .{}); + const raw_truncated_reg = try self.register_manager.allocReg(null, gp); const truncated_reg = registerAlias(raw_truncated_reg, lhs_ty.abiSize(self.target.*)); const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg); defer self.register_manager.unlockReg(truncated_reg_lock); @@ -1983,7 +1983,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg); defer self.register_manager.unlockReg(dest_reg_lock); - const truncated_reg = try self.register_manager.allocReg(null, .{}); + const truncated_reg = try self.register_manager.allocReg(null, gp); const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg); defer self.register_manager.unlockReg(truncated_reg_lock); @@ -2048,7 +2048,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { defer if (rhs_lock) |reg| self.register_manager.unlockReg(reg); const lhs_reg = if (lhs_is_register) lhs.register else blk: { - const raw_reg = try self.register_manager.allocReg(null, .{}); + const raw_reg = try self.register_manager.allocReg(null, gp); const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*)); break :blk reg; }; @@ -2056,7 +2056,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { defer if (new_lhs_lock) |reg| self.register_manager.unlockReg(reg); const rhs_reg = if (rhs_is_register) rhs.register else blk: { - const raw_reg = try self.register_manager.allocReg(null, .{}); + const raw_reg = try self.register_manager.allocReg(null, gp); const reg = registerAlias(raw_reg, rhs_ty.abiAlignment(self.target.*)); break :blk reg; }; @@ -2067,7 +2067,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs); const dest_reg = blk: { - const raw_reg = try self.register_manager.allocReg(null, .{}); + const raw_reg = try self.register_manager.allocReg(null, gp); const reg = registerAlias(raw_reg, lhs_ty.abiSize(self.target.*)); break :blk reg; }; @@ -2086,7 +2086,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { } }, }); - const dest_high_reg = try self.register_manager.allocReg(null, .{}); + const dest_high_reg = try self.register_manager.allocReg(null, gp); const dest_high_reg_lock = self.register_manager.lockRegAssumeUnused(dest_high_reg); defer self.register_manager.unlockReg(dest_high_reg_lock); @@ -2136,7 +2136,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { } }, .unsigned => { - const dest_high_reg = try self.register_manager.allocReg(null, .{}); + const dest_high_reg = try self.register_manager.allocReg(null, gp); const dest_high_reg_lock = self.register_manager.lockRegAssumeUnused(dest_high_reg); defer self.register_manager.unlockReg(dest_high_reg_lock); @@ -2192,7 +2192,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { }, } - const truncated_reg = try self.register_manager.allocReg(null, .{}); + const truncated_reg = try self.register_manager.allocReg(null, gp); const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg); defer self.register_manager.unlockReg(truncated_reg_lock); @@ -2663,7 +2663,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo }, .stack_offset => |off| { if (elem_size <= 8) { - const raw_tmp_reg = try self.register_manager.allocReg(null, .{}); + const raw_tmp_reg = try self.register_manager.allocReg(null, gp); const tmp_reg = registerAlias(raw_tmp_reg, elem_size); const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); defer self.register_manager.unlockReg(tmp_reg_lock); @@ -2672,7 +2672,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo try self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg }); } else { // TODO optimize the register allocation - const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null }, .{}); + const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null }, gp); const regs_locks = self.register_manager.lockRegsAssumeUnused(4, regs); defer for (regs_locks) |reg| { self.register_manager.unlockReg(reg); @@ -2887,7 +2887,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type }, else => { if (abi_size <= 8) { - const raw_tmp_reg = try self.register_manager.allocReg(null, .{}); + const raw_tmp_reg = try self.register_manager.allocReg(null, gp); const tmp_reg = registerAlias(raw_tmp_reg, abi_size); const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); defer self.register_manager.unlockReg(tmp_reg_lock); @@ -3002,7 +3002,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { // TODO return special MCValue condition flags // get overflow bit: set register to C flag // resp. V flag - const raw_dest_reg = try self.register_manager.allocReg(null, .{}); + const raw_dest_reg = try self.register_manager.allocReg(null, gp); const dest_reg = raw_dest_reg.to32(); // C flag: cset reg, cs @@ -4065,7 +4065,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro const overflow_bit_ty = ty.structFieldType(1); const overflow_bit_offset = @intCast(u32, ty.structFieldOffset(1, self.target.*)); - const raw_cond_reg = try self.register_manager.allocReg(null, .{}); + const raw_cond_reg = try self.register_manager.allocReg(null, gp); const cond_reg = registerAlias( raw_cond_reg, @intCast(u32, overflow_bit_ty.abiSize(self.target.*)), @@ -4113,7 +4113,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro const ptr_ty = Type.initPayload(&ptr_ty_payload.base); // TODO call extern memcpy - const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null }, .{}); + const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null }, gp); const regs_locks = self.register_manager.lockRegsAssumeUnused(5, regs); defer for (regs_locks) |reg| { self.register_manager.unlockReg(reg); diff --git a/src/arch/aarch64/abi.zig b/src/arch/aarch64/abi.zig index 1c5225104a..839b2789e4 100644 --- a/src/arch/aarch64/abi.zig +++ b/src/arch/aarch64/abi.zig @@ -1,6 +1,7 @@ const builtin = @import("builtin"); const bits = @import("bits.zig"); const Register = bits.Register; +const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager; const callee_preserved_regs_impl = if (builtin.os.tag.isDarwin()) struct { pub const callee_preserved_regs = [_]Register{ @@ -18,3 +19,19 @@ pub const callee_preserved_regs = callee_preserved_regs_impl.callee_preserved_re pub const c_abi_int_param_regs = [_]Register{ .x0, .x1, .x2, .x3, .x4, .x5, .x6, .x7 }; pub const c_abi_int_return_regs = [_]Register{ .x0, .x1, .x2, .x3, .x4, .x5, .x6, .x7 }; + +const allocatable_registers = callee_preserved_regs; +pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, &allocatable_registers); + +// Register classes +const RegisterBitSet = RegisterManager.RegisterBitSet; +pub const RegisterClass = struct { + pub const gp: RegisterBitSet = blk: { + var set = RegisterBitSet.initEmpty(); + set.setRangeValue(.{ + .start = 0, + .end = callee_preserved_regs.len, + }, true); + break :blk set; + }; +}; diff --git a/src/arch/arm/CodeGen.zig b/src/arch/arm/CodeGen.zig index c792615c79..4f121dd56e 100644 --- a/src/arch/arm/CodeGen.zig +++ b/src/arch/arm/CodeGen.zig @@ -21,9 +21,6 @@ const DW = std.dwarf; const leb128 = std.leb; const log = std.log.scoped(.codegen); const build_options = @import("build_options"); -const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager; -const RegisterManager = RegisterManagerFn(Self, Register, &allocatable_registers); -const RegisterLock = RegisterManager.RegisterLock; const FnResult = @import("../../codegen.zig").FnResult; const GenerateSymbolError = @import("../../codegen.zig").GenerateSymbolError; @@ -31,14 +28,16 @@ const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput; const bits = @import("bits.zig"); const abi = @import("abi.zig"); +const RegisterManager = abi.RegisterManager; +const RegisterLock = RegisterManager.RegisterLock; const Register = bits.Register; const Instruction = bits.Instruction; const Condition = bits.Condition; const callee_preserved_regs = abi.callee_preserved_regs; const caller_preserved_regs = abi.caller_preserved_regs; -const allocatable_registers = abi.allocatable_registers; const c_abi_int_param_regs = abi.c_abi_int_param_regs; const c_abi_int_return_regs = abi.c_abi_int_return_regs; +const gp = abi.RegisterClass.gp; const InnerError = error{ OutOfMemory, @@ -874,7 +873,7 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue { const ptr_bits = self.target.cpu.arch.ptrBitWidth(); const ptr_bytes: u64 = @divExact(ptr_bits, 8); if (abi_size <= ptr_bytes) { - if (self.register_manager.tryAllocReg(inst, .{})) |reg| { + if (self.register_manager.tryAllocReg(inst, gp)) |reg| { return MCValue{ .register = reg }; } } @@ -939,7 +938,7 @@ fn spillCompareFlagsIfOccupied(self: *Self) !void { /// allocated. A second call to `copyToTmpRegister` may return the same register. /// This can have a side effect of spilling instructions to the stack to free up a register. fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register { - const reg = try self.register_manager.allocReg(null, .{}); + const reg = try self.register_manager.allocReg(null, gp); try self.genSetReg(ty, reg, mcv); return reg; } @@ -948,7 +947,7 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register { /// `reg_owner` is the instruction that gets associated with the register in the register table. /// This can have a side effect of spilling instructions to the stack to free up a register. fn copyToNewRegister(self: *Self, reg_owner: Air.Inst.Index, mcv: MCValue) !MCValue { - const reg = try self.register_manager.allocReg(reg_owner, .{}); + const reg = try self.register_manager.allocReg(reg_owner, gp); try self.genSetReg(self.air.typeOfIndex(reg_owner), reg, mcv); return MCValue{ .register = reg }; } @@ -1065,9 +1064,9 @@ fn trunc( if (operand == .register and self.reuseOperand(inst, ty_op.operand, 0, operand)) { break :blk operand_reg; } else { - break :blk try self.register_manager.allocReg(inst, .{}); + break :blk try self.register_manager.allocReg(inst, gp); } - } else try self.register_manager.allocReg(null, .{}); + } else try self.register_manager.allocReg(null, gp); switch (info_b.bits) { 32 => { @@ -1153,7 +1152,7 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void { break :blk op_reg; } - break :blk try self.register_manager.allocReg(null, .{}); + break :blk try self.register_manager.allocReg(null, gp); }; _ = try self.addInst(.{ @@ -1183,7 +1182,7 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void { break :blk op_reg; } - break :blk try self.register_manager.allocReg(null, .{}); + break :blk try self.register_manager.allocReg(null, gp); }; _ = try self.addInst(.{ @@ -1254,9 +1253,9 @@ fn minMax( } else if (rhs_is_register and self.reuseOperand(inst, bin_op.rhs, 1, rhs)) { break :blk rhs_reg; } else { - break :blk try self.register_manager.allocReg(inst, .{}); + break :blk try self.register_manager.allocReg(inst, gp); } - } else try self.register_manager.allocReg(null, .{}); + } else try self.register_manager.allocReg(null, gp); // lhs == reg should have been checked by airMinMax // @@ -1438,7 +1437,7 @@ fn airOverflow(self: *Self, inst: Air.Inst.Index) !void { const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg); defer self.register_manager.unlockReg(dest_reg_lock); - const truncated_reg = try self.register_manager.allocReg(null, .{}); + const truncated_reg = try self.register_manager.allocReg(null, gp); const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg); defer self.register_manager.unlockReg(truncated_reg_lock); @@ -1543,7 +1542,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { const dest_reg_lock = self.register_manager.lockRegAssumeUnused(dest_reg); defer self.register_manager.unlockReg(dest_reg_lock); - const truncated_reg = try self.register_manager.allocReg(null, .{}); + const truncated_reg = try self.register_manager.allocReg(null, gp); const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg); defer self.register_manager.unlockReg(truncated_reg_lock); @@ -1582,18 +1581,18 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { const lhs_reg = if (lhs_is_register) lhs.register else - try self.register_manager.allocReg(null, .{}); + try self.register_manager.allocReg(null, gp); const new_lhs_lock = self.register_manager.lockReg(lhs_reg); defer if (new_lhs_lock) |reg| self.register_manager.unlockReg(reg); const rhs_reg = if (rhs_is_register) rhs.register else - try self.register_manager.allocReg(null, .{}); + try self.register_manager.allocReg(null, gp); const new_rhs_lock = self.register_manager.lockReg(rhs_reg); defer if (new_rhs_lock) |reg| self.register_manager.unlockReg(reg); - const dest_regs = try self.register_manager.allocRegs(2, .{ null, null }, .{}); + const dest_regs = try self.register_manager.allocRegs(2, .{ null, null }, gp); const dest_regs_locks = self.register_manager.lockRegsAssumeUnused(2, dest_regs); defer for (dest_regs_locks) |reg| { self.register_manager.unlockReg(reg); @@ -1604,7 +1603,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs); if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs); - const truncated_reg = try self.register_manager.allocReg(null, .{}); + const truncated_reg = try self.register_manager.allocReg(null, gp); const truncated_reg_lock = self.register_manager.lockRegAssumeUnused(truncated_reg); defer self.register_manager.unlockReg(truncated_reg_lock); @@ -2026,7 +2025,7 @@ fn airSliceElemVal(self: *Self, inst: Air.Inst.Index) !void { const base_reg_lock = self.register_manager.lockRegAssumeUnused(base_reg); defer self.register_manager.unlockReg(base_reg_lock); - const dst_reg = try self.register_manager.allocReg(inst, .{}); + const dst_reg = try self.register_manager.allocReg(inst, gp); const dst_mcv = MCValue{ .register = dst_reg }; const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg); defer self.register_manager.unlockReg(dst_reg_lock); @@ -2234,7 +2233,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo }, .stack_offset => |off| { if (elem_size <= 4) { - const tmp_reg = try self.register_manager.allocReg(null, .{}); + const tmp_reg = try self.register_manager.allocReg(null, gp); const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); defer self.register_manager.unlockReg(tmp_reg_lock); @@ -2242,7 +2241,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo try self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg }); } else { // TODO optimize the register allocation - const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null }, .{}); + const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null }, gp); const regs_locks = self.register_manager.lockRegsAssumeUnused(4, regs); defer for (regs_locks) |reg_locked| { self.register_manager.unlockReg(reg_locked); @@ -2271,7 +2270,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo .stack_offset, .stack_argument_offset, => { - const reg = try self.register_manager.allocReg(null, .{}); + const reg = try self.register_manager.allocReg(null, gp); const reg_lock = self.register_manager.lockRegAssumeUnused(reg); defer self.register_manager.unlockReg(reg_lock); @@ -2338,14 +2337,14 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type }, else => { if (elem_size <= 4) { - const tmp_reg = try self.register_manager.allocReg(null, .{}); + const tmp_reg = try self.register_manager.allocReg(null, gp); const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); defer self.register_manager.unlockReg(tmp_reg_lock); try self.genSetReg(value_ty, tmp_reg, value); try self.store(ptr, .{ .register = tmp_reg }, ptr_ty, value_ty); } else { - const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null }, .{}); + const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null }, gp); const regs_locks = self.register_manager.lockRegsAssumeUnused(4, regs); defer for (regs_locks) |reg| { self.register_manager.unlockReg(reg); @@ -2487,7 +2486,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { 1 => { // get overflow bit: set register to C flag // resp. V flag - const dest_reg = try self.register_manager.allocReg(null, .{}); + const dest_reg = try self.register_manager.allocReg(null, gp); // mov reg, #0 _ = try self.addInst(.{ @@ -2567,7 +2566,7 @@ fn binOpRegister( break :inst Air.refToIndex(md.lhs).?; } else null; - const reg = try self.register_manager.allocReg(track_inst, .{}); + const reg = try self.register_manager.allocReg(track_inst, gp); if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg }); @@ -2581,7 +2580,7 @@ fn binOpRegister( break :inst Air.refToIndex(md.rhs).?; } else null; - const reg = try self.register_manager.allocReg(track_inst, .{}); + const reg = try self.register_manager.allocReg(track_inst, gp); if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg }); @@ -2598,9 +2597,9 @@ fn binOpRegister( } else if (rhs_is_register and self.reuseOperand(md.inst, md.rhs, 1, rhs)) { break :blk rhs_reg; } else { - break :blk try self.register_manager.allocReg(md.inst, .{}); + break :blk try self.register_manager.allocReg(md.inst, gp); } - } else try self.register_manager.allocReg(null, .{}), + } else try self.register_manager.allocReg(null, gp), }; if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs); @@ -2684,7 +2683,7 @@ fn binOpImmediate( ).?; } else null; - const reg = try self.register_manager.allocReg(track_inst, .{}); + const reg = try self.register_manager.allocReg(track_inst, gp); if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg }); @@ -2704,9 +2703,9 @@ fn binOpImmediate( )) { break :blk lhs_reg; } else { - break :blk try self.register_manager.allocReg(md.inst, .{}); + break :blk try self.register_manager.allocReg(md.inst, gp); } - } else try self.register_manager.allocReg(null, .{}), + } else try self.register_manager.allocReg(null, gp), }; if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs); @@ -4363,7 +4362,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro const overflow_bit_ty = ty.structFieldType(1); const overflow_bit_offset = @intCast(u32, ty.structFieldOffset(1, self.target.*)); - const cond_reg = try self.register_manager.allocReg(null, .{}); + const cond_reg = try self.register_manager.allocReg(null, gp); // C flag: movcs reg, #1 // V flag: movvs reg, #1 @@ -4408,7 +4407,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro const ptr_ty = Type.initPayload(&ptr_ty_payload.base); // TODO call extern memcpy - const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null }, .{}); + const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null }, gp); const src_reg = regs[0]; const dst_reg = regs[1]; const len_reg = regs[2]; @@ -4782,7 +4781,7 @@ fn genSetStackArgument(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) I const ptr_ty = Type.initPayload(&ptr_ty_payload.base); // TODO call extern memcpy - const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null }, .{}); + const regs = try self.register_manager.allocRegs(5, .{ null, null, null, null, null }, gp); const src_reg = regs[0]; const dst_reg = regs[1]; const len_reg = regs[2]; diff --git a/src/arch/arm/abi.zig b/src/arch/arm/abi.zig index 4073b92222..dbee1f9b90 100644 --- a/src/arch/arm/abi.zig +++ b/src/arch/arm/abi.zig @@ -1,9 +1,25 @@ const bits = @import("bits.zig"); const Register = bits.Register; +const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager; pub const callee_preserved_regs = [_]Register{ .r4, .r5, .r6, .r7, .r8, .r10 }; pub const caller_preserved_regs = [_]Register{ .r0, .r1, .r2, .r3 }; -pub const allocatable_registers = callee_preserved_regs ++ caller_preserved_regs; pub const c_abi_int_param_regs = [_]Register{ .r0, .r1, .r2, .r3 }; pub const c_abi_int_return_regs = [_]Register{ .r0, .r1 }; + +const allocatable_registers = callee_preserved_regs ++ caller_preserved_regs; +pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, &allocatable_registers); + +// Register classes +const RegisterBitSet = RegisterManager.RegisterBitSet; +pub const RegisterClass = struct { + pub const gp: RegisterBitSet = blk: { + var set = RegisterBitSet.initEmpty(); + set.setRangeValue(.{ + .start = 0, + .end = caller_preserved_regs.len + callee_preserved_regs.len, + }, true); + break :blk set; + }; +}; diff --git a/src/arch/riscv64/CodeGen.zig b/src/arch/riscv64/CodeGen.zig index 79bedb30d8..b713161053 100644 --- a/src/arch/riscv64/CodeGen.zig +++ b/src/arch/riscv64/CodeGen.zig @@ -21,9 +21,6 @@ const DW = std.dwarf; const leb128 = std.leb; const log = std.log.scoped(.codegen); const build_options = @import("build_options"); -const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager; -const RegisterManager = RegisterManagerFn(Self, Register, &callee_preserved_regs); -const RegisterLock = RegisterManager.RegisterLock; const FnResult = @import("../../codegen.zig").FnResult; const GenerateSymbolError = @import("../../codegen.zig").GenerateSymbolError; @@ -32,8 +29,11 @@ const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput; const bits = @import("bits.zig"); const abi = @import("abi.zig"); const Register = bits.Register; +const RegisterManager = abi.RegisterManager; +const RegisterLock = RegisterManager.RegisterLock; const Instruction = abi.Instruction; const callee_preserved_regs = abi.callee_preserved_regs; +const gp = abi.RegisterClass.gp; const InnerError = error{ OutOfMemory, @@ -803,7 +803,7 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue { const ptr_bits = self.target.cpu.arch.ptrBitWidth(); const ptr_bytes: u64 = @divExact(ptr_bits, 8); if (abi_size <= ptr_bytes) { - if (self.register_manager.tryAllocReg(inst, .{})) |reg| { + if (self.register_manager.tryAllocReg(inst, gp)) |reg| { return MCValue{ .register = reg }; } } @@ -826,7 +826,7 @@ pub fn spillInstruction(self: *Self, reg: Register, inst: Air.Inst.Index) !void /// allocated. A second call to `copyToTmpRegister` may return the same register. /// This can have a side effect of spilling instructions to the stack to free up a register. fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register { - const reg = try self.register_manager.allocReg(null, .{}); + const reg = try self.register_manager.allocReg(null, gp); try self.genSetReg(ty, reg, mcv); return reg; } @@ -835,7 +835,7 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register { /// `reg_owner` is the instruction that gets associated with the register in the register table. /// This can have a side effect of spilling instructions to the stack to free up a register. fn copyToNewRegister(self: *Self, reg_owner: Air.Inst.Index, mcv: MCValue) !MCValue { - const reg = try self.register_manager.allocReg(reg_owner, .{}); + const reg = try self.register_manager.allocReg(reg_owner, gp); try self.genSetReg(self.air.typeOfIndex(reg_owner), reg, mcv); return MCValue{ .register = reg }; } @@ -958,7 +958,7 @@ fn binOpRegister( break :inst Air.refToIndex(bin_op.lhs).?; } else null; - const reg = try self.register_manager.allocReg(track_inst, .{}); + const reg = try self.register_manager.allocReg(track_inst, gp); if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg }); @@ -973,7 +973,7 @@ fn binOpRegister( break :inst Air.refToIndex(bin_op.rhs).?; } else null; - const reg = try self.register_manager.allocReg(track_inst, .{}); + const reg = try self.register_manager.allocReg(track_inst, gp); if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg }); @@ -990,9 +990,9 @@ fn binOpRegister( } else if (rhs_is_register and self.reuseOperand(inst, bin_op.rhs, 1, rhs)) { break :blk rhs_reg; } else { - break :blk try self.register_manager.allocReg(inst, .{}); + break :blk try self.register_manager.allocReg(inst, gp); } - } else try self.register_manager.allocReg(null, .{}); + } else try self.register_manager.allocReg(null, gp); if (!lhs_is_register) try self.genSetReg(lhs_ty, lhs_reg, lhs); if (!rhs_is_register) try self.genSetReg(rhs_ty, rhs_reg, rhs); @@ -1482,7 +1482,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo .memory, .stack_offset, => { - const reg = try self.register_manager.allocReg(null, .{}); + const reg = try self.register_manager.allocReg(null, gp); const reg_lock = self.register_manager.lockRegAssumeUnused(reg); defer self.register_manager.unlockReg(reg_lock); diff --git a/src/arch/riscv64/abi.zig b/src/arch/riscv64/abi.zig index dd0feeea49..459fd40c9c 100644 --- a/src/arch/riscv64/abi.zig +++ b/src/arch/riscv64/abi.zig @@ -1,6 +1,23 @@ const bits = @import("bits.zig"); const Register = bits.Register; +const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager; pub const callee_preserved_regs = [_]Register{ .s0, .s1, .s2, .s3, .s4, .s5, .s6, .s7, .s8, .s9, .s10, .s11, }; + +const allocatable_registers = callee_preserved_regs; +pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, &allocatable_registers); + +// Register classes +const RegisterBitSet = RegisterManager.RegisterBitSet; +pub const RegisterClass = struct { + pub const gp: RegisterBitSet = blk: { + var set = RegisterBitSet.initEmpty(); + set.setRangeValue(.{ + .start = 0, + .end = callee_preserved_regs.len, + }, true); + break :blk set; + }; +}; diff --git a/src/arch/sparc64/CodeGen.zig b/src/arch/sparc64/CodeGen.zig index e8cb5d3100..ff066f78f2 100644 --- a/src/arch/sparc64/CodeGen.zig +++ b/src/arch/sparc64/CodeGen.zig @@ -21,9 +21,6 @@ const Type = @import("../../type.zig").Type; const GenerateSymbolError = @import("../../codegen.zig").GenerateSymbolError; const FnResult = @import("../../codegen.zig").FnResult; const DebugInfoOutput = @import("../../codegen.zig").DebugInfoOutput; -const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager; -const RegisterManager = RegisterManagerFn(Self, Register, &abi.allocatable_regs); -const RegisterLock = RegisterManager.RegisterLock; const build_options = @import("build_options"); @@ -31,7 +28,10 @@ const bits = @import("bits.zig"); const abi = @import("abi.zig"); const Instruction = bits.Instruction; const ShiftWidth = Instruction.ShiftWidth; +const RegisterManager = abi.RegisterManager; +const RegisterLock = RegisterManager.RegisterLock; const Register = bits.Register; +const gp = abi.RegisterClass.gp; const Self = @This(); @@ -1613,7 +1613,7 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue { if (reg_ok) { // Make sure the type can fit in a register before we try to allocate one. if (abi_size <= 8) { - if (self.register_manager.tryAllocReg(inst, .{})) |reg| { + if (self.register_manager.tryAllocReg(inst, gp)) |reg| { return MCValue{ .register = reg }; } } @@ -1854,7 +1854,7 @@ fn binOpImmediate( ).?; } else null; - const reg = try self.register_manager.allocReg(track_inst, .{}); + const reg = try self.register_manager.allocReg(track_inst, gp); if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg }); @@ -1873,10 +1873,10 @@ fn binOpImmediate( )) { break :blk lhs_reg; } else { - break :blk try self.register_manager.allocReg(md.inst, .{}); + break :blk try self.register_manager.allocReg(md.inst, gp); } } else blk: { - break :blk try self.register_manager.allocReg(null, .{}); + break :blk try self.register_manager.allocReg(null, gp); }, }; @@ -1953,7 +1953,7 @@ fn binOpRegister( break :inst Air.refToIndex(md.lhs).?; } else null; - const reg = try self.register_manager.allocReg(track_inst, .{}); + const reg = try self.register_manager.allocReg(track_inst, gp); if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg }); break :blk reg; @@ -1966,7 +1966,7 @@ fn binOpRegister( break :inst Air.refToIndex(md.rhs).?; } else null; - const reg = try self.register_manager.allocReg(track_inst, .{}); + const reg = try self.register_manager.allocReg(track_inst, gp); if (track_inst) |inst| branch.inst_table.putAssumeCapacity(inst, .{ .register = reg }); break :blk reg; @@ -1981,10 +1981,10 @@ fn binOpRegister( } else if (rhs_is_register and self.reuseOperand(md.inst, md.rhs, 1, rhs)) { break :blk rhs_reg; } else { - break :blk try self.register_manager.allocReg(md.inst, .{}); + break :blk try self.register_manager.allocReg(md.inst, gp); } } else blk: { - break :blk try self.register_manager.allocReg(null, .{}); + break :blk try self.register_manager.allocReg(null, gp); }, }; @@ -2077,7 +2077,7 @@ fn brVoid(self: *Self, block: Air.Inst.Index) !void { /// allocated. A second call to `copyToTmpRegister` may return the same register. /// This can have a side effect of spilling instructions to the stack to free up a register. fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register { - const reg = try self.register_manager.allocReg(null, .{}); + const reg = try self.register_manager.allocReg(null, gp); try self.genSetReg(ty, reg, mcv); return reg; } @@ -2364,7 +2364,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void }); } else { // Need to allocate a temporary register to load 64-bit immediates. - const tmp_reg = try self.register_manager.allocReg(null, .{}); + const tmp_reg = try self.register_manager.allocReg(null, gp); try self.genSetReg(ty, tmp_reg, .{ .immediate = @truncate(u32, x) }); try self.genSetReg(ty, reg, .{ .immediate = @truncate(u32, x >> 32) }); @@ -2478,7 +2478,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: u32, mcv: MCValue) InnerErro }; const ptr_ty = Type.initPayload(&ptr_ty_payload.base); - const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null }, .{}); + const regs = try self.register_manager.allocRegs(4, .{ null, null, null, null }, gp); const regs_locks = self.register_manager.lockRegsAssumeUnused(4, regs); defer for (regs_locks) |reg| { self.register_manager.unlockReg(reg); @@ -2717,14 +2717,14 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo }, .stack_offset => |off| { if (elem_size <= 8) { - const tmp_reg = try self.register_manager.allocReg(null, .{}); + const tmp_reg = try self.register_manager.allocReg(null, gp); const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); defer self.register_manager.unlockReg(tmp_reg_lock); try self.load(.{ .register = tmp_reg }, ptr, ptr_ty); try self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg }); } else { - const regs = try self.register_manager.allocRegs(3, .{ null, null, null }, .{}); + const regs = try self.register_manager.allocRegs(3, .{ null, null, null }, gp); const regs_locks = self.register_manager.lockRegsAssumeUnused(3, regs); defer for (regs_locks) |reg| { self.register_manager.unlockReg(reg); diff --git a/src/arch/sparc64/abi.zig b/src/arch/sparc64/abi.zig index a9001c7dc7..b1985a7b0b 100644 --- a/src/arch/sparc64/abi.zig +++ b/src/arch/sparc64/abi.zig @@ -1,5 +1,6 @@ const bits = @import("bits.zig"); const Register = bits.Register; +const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager; // SPARCv9 stack constants. // See: Registers and the Stack Frame, page 3P-8, SCD 2.4.1. @@ -21,7 +22,7 @@ pub const stack_save_area = 176; pub const caller_preserved_regs = [_]Register{ .o0, .o1, .o2, .o3, .o4, .o5, .g1, .g4, .g5 }; // Try to allocate i, l, o, then g sets of registers, in order of priority. -pub const allocatable_regs = [_]Register{ +const allocatable_regs = [_]Register{ // zig fmt: off .@"i0", .@"i1", .@"i2", .@"i3", .@"i4", .@"i5", .l0, .l1, .l2, .l3, .l4, .l5, .l6, .l7, @@ -35,3 +36,18 @@ pub const c_abi_int_param_regs_callee_view = [_]Register{ .@"i0", .@"i1", .@"i2" pub const c_abi_int_return_regs_caller_view = [_]Register{ .o0, .o1, .o2, .o3 }; pub const c_abi_int_return_regs_callee_view = [_]Register{ .@"i0", .@"i1", .@"i2", .@"i3" }; + +pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, &allocatable_regs); + +// Register classes +const RegisterBitSet = RegisterManager.RegisterBitSet; +pub const RegisterClass = struct { + pub const gp: RegisterBitSet = blk: { + var set = RegisterBitSet.initEmpty(); + set.setRangeValue(.{ + .start = 0, + .end = allocatable_regs.len, + }, true); + break :blk set; + }; +}; diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index bd17b2a9ae..8ce0d5884b 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -884,9 +884,7 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue { if (self.intrinsicsAllowed(elem_ty)) { const ptr_bytes: u64 = 32; if (abi_size <= ptr_bytes) { - if (self.register_manager.tryAllocReg(inst, .{ - .selector_mask = sse, - })) |reg| { + if (self.register_manager.tryAllocReg(inst, sse)) |reg| { return MCValue{ .register = registerAlias(reg, abi_size) }; } } @@ -899,9 +897,7 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue { const ptr_bits = self.target.cpu.arch.ptrBitWidth(); const ptr_bytes: u64 = @divExact(ptr_bits, 8); if (abi_size <= ptr_bytes) { - if (self.register_manager.tryAllocReg(inst, .{ - .selector_mask = gp, - })) |reg| { + if (self.register_manager.tryAllocReg(inst, gp)) |reg| { return MCValue{ .register = registerAlias(reg, abi_size) }; } } @@ -972,16 +968,14 @@ pub fn spillRegisters(self: *Self, comptime count: comptime_int, registers: [cou /// allocated. A second call to `copyToTmpRegister` may return the same register. /// This can have a side effect of spilling instructions to the stack to free up a register. fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register { - const mask: RegisterManager.RegisterBitSet = switch (ty.zigTypeTag()) { + const reg_class: RegisterManager.RegisterBitSet = switch (ty.zigTypeTag()) { .Float => blk: { if (self.intrinsicsAllowed(ty)) break :blk sse; return self.fail("TODO copy {} to register", .{ty.fmtDebug()}); }, else => gp, }; - const reg: Register = try self.register_manager.allocReg(null, .{ - .selector_mask = mask, - }); + const reg: Register = try self.register_manager.allocReg(null, reg_class); try self.genSetReg(ty, reg, mcv); return reg; } @@ -991,16 +985,14 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register { /// This can have a side effect of spilling instructions to the stack to free up a register. /// WARNING make sure that the allocated register matches the returned MCValue from an instruction! fn copyToRegisterWithInstTracking(self: *Self, reg_owner: Air.Inst.Index, ty: Type, mcv: MCValue) !MCValue { - const mask: RegisterManager.RegisterBitSet = switch (ty.zigTypeTag()) { + const reg_class: RegisterManager.RegisterBitSet = switch (ty.zigTypeTag()) { .Float => blk: { if (self.intrinsicsAllowed(ty)) break :blk sse; return self.fail("TODO copy {} to register", .{ty.fmtDebug()}); }, else => gp, }; - const reg: Register = try self.register_manager.allocReg(reg_owner, .{ - .selector_mask = mask, - }); + const reg: Register = try self.register_manager.allocReg(reg_owner, reg_class); try self.genSetReg(ty, reg, mcv); return MCValue{ .register = reg }; } @@ -1056,9 +1048,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void { }; defer if (operand_lock) |lock| self.register_manager.unlockReg(lock); - const reg = try self.register_manager.allocReg(inst, .{ - .selector_mask = gp, - }); + const reg = try self.register_manager.allocReg(inst, gp); try self.genSetReg(dest_ty, reg, .{ .immediate = 0 }); try self.genSetReg(operand_ty, reg, operand); break :blk MCValue{ .register = reg }; @@ -1413,9 +1403,7 @@ fn genSetStackTruncatedOverflowCompare( .unsigned => ty, }; - const temp_regs = try self.register_manager.allocRegs(3, .{ null, null, null }, .{ - .selector_mask = gp, - }); + const temp_regs = try self.register_manager.allocRegs(3, .{ null, null, null }, gp); const temp_regs_locks = self.register_manager.lockRegsAssumeUnused(3, temp_regs); defer for (temp_regs_locks) |rreg| { self.register_manager.unlockReg(rreg); @@ -2077,9 +2065,7 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue { const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg); defer self.register_manager.unlockReg(offset_reg_lock); - const addr_reg = try self.register_manager.allocReg(null, .{ - .selector_mask = gp, - }); + const addr_reg = try self.register_manager.allocReg(null, gp); switch (slice_mcv) { .stack_offset => |off| { // mov reg, [rbp - 8] @@ -2158,9 +2144,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void { const offset_reg_lock = self.register_manager.lockRegAssumeUnused(offset_reg); defer self.register_manager.unlockReg(offset_reg_lock); - const addr_reg = try self.register_manager.allocReg(null, .{ - .selector_mask = gp, - }); + const addr_reg = try self.register_manager.allocReg(null, gp); switch (array) { .register => { const off = @intCast(i32, try self.allocMem( @@ -2527,7 +2511,7 @@ fn load(self: *Self, dst_mcv: MCValue, ptr: MCValue, ptr_ty: Type) InnerError!vo }, .stack_offset => |off| { if (abi_size <= 8) { - const tmp_reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp }); + const tmp_reg = try self.register_manager.allocReg(null, gp); try self.load(.{ .register = tmp_reg }, ptr, ptr_ty); return self.genSetStack(elem_ty, off, MCValue{ .register = tmp_reg }, .{}); } @@ -2728,7 +2712,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type }; defer if (value_lock) |lock| self.register_manager.unlockReg(lock); - const addr_reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp }); + const addr_reg = try self.register_manager.allocReg(null, gp); const addr_reg_lock = self.register_manager.lockRegAssumeUnused(addr_reg); defer self.register_manager.unlockReg(addr_reg_lock); @@ -2800,7 +2784,7 @@ fn store(self: *Self, ptr: MCValue, value: MCValue, ptr_ty: Type, value_ty: Type .memory, => { if (abi_size <= 8) { - const tmp_reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp }); + const tmp_reg = try self.register_manager.allocReg(null, gp); const tmp_reg_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); defer self.register_manager.unlockReg(tmp_reg_lock); @@ -2918,7 +2902,7 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde if (can_reuse_operand) { break :blk reg; } else { - const result_reg = try self.register_manager.allocReg(inst, .{ .selector_mask = gp }); + const result_reg = try self.register_manager.allocReg(inst, gp); try self.genSetReg(ptr_ty, result_reg, mcv); break :blk result_reg; } @@ -3019,7 +3003,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { const reg_lock = self.register_manager.lockRegAssumeUnused(reg); defer self.register_manager.unlockReg(reg_lock); - const dst_reg = try self.register_manager.allocReg(inst, .{ .selector_mask = gp }); + const dst_reg = try self.register_manager.allocReg(inst, gp); const flags: u2 = switch (mcv) { .register_overflow_unsigned => 0b10, .register_overflow_signed => 0b00, @@ -5428,7 +5412,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl const overflow_bit_ty = ty.structFieldType(1); const overflow_bit_offset = ty.structFieldOffset(1, self.target.*); - const tmp_reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp }); + const tmp_reg = try self.register_manager.allocReg(null, gp); const flags: u2 = switch (mcv) { .register_overflow_unsigned => 0b10, .register_overflow_signed => 0b00, @@ -5656,7 +5640,7 @@ fn genInlineMemcpy( null; defer if (dsbase_lock) |lock| self.register_manager.unlockReg(lock); - const dst_addr_reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp }); + const dst_addr_reg = try self.register_manager.allocReg(null, gp); switch (dst_ptr) { .memory, .got_load, @@ -5691,7 +5675,7 @@ fn genInlineMemcpy( const dst_addr_reg_lock = self.register_manager.lockRegAssumeUnused(dst_addr_reg); defer self.register_manager.unlockReg(dst_addr_reg_lock); - const src_addr_reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp }); + const src_addr_reg = try self.register_manager.allocReg(null, gp); switch (src_ptr) { .memory, .got_load, @@ -5726,9 +5710,7 @@ fn genInlineMemcpy( const src_addr_reg_lock = self.register_manager.lockRegAssumeUnused(src_addr_reg); defer self.register_manager.unlockReg(src_addr_reg_lock); - const regs = try self.register_manager.allocRegs(2, .{ null, null }, .{ - .selector_mask = gp, - }); + const regs = try self.register_manager.allocRegs(2, .{ null, null }, gp); const count_reg = regs[0].to64(); const tmp_reg = regs[1].to8(); @@ -5828,7 +5810,7 @@ fn genInlineMemset( const rax_lock = self.register_manager.lockRegAssumeUnused(.rax); defer self.register_manager.unlockReg(rax_lock); - const addr_reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp }); + const addr_reg = try self.register_manager.allocReg(null, gp); switch (dst_ptr) { .memory, .got_load, @@ -6087,7 +6069,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void => { switch (ty.zigTypeTag()) { .Float => { - const base_reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp }); + const base_reg = try self.register_manager.allocReg(null, gp); try self.loadMemPtrIntoRegister(base_reg, Type.usize, mcv); if (self.intrinsicsAllowed(ty)) { @@ -6130,7 +6112,7 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void }, .memory => |x| switch (ty.zigTypeTag()) { .Float => { - const base_reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp }); + const base_reg = try self.register_manager.allocReg(null, gp); try self.loadMemPtrIntoRegister(base_reg, Type.usize, mcv); if (self.intrinsicsAllowed(ty)) { @@ -6461,7 +6443,7 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void { const src: MCValue = blk: { switch (src_ptr) { .got_load, .direct_load, .memory => { - const reg = try self.register_manager.allocReg(null, .{ .selector_mask = gp }); + const reg = try self.register_manager.allocReg(null, gp); try self.loadMemPtrIntoRegister(reg, src_ty, src_ptr); _ = try self.addInst(.{ .tag = .mov, diff --git a/src/register_manager.zig b/src/register_manager.zig index 7b82253e22..a73f345e0c 100644 --- a/src/register_manager.zig +++ b/src/register_manager.zig @@ -57,6 +57,11 @@ pub fn RegisterManager( return @fieldParentPtr(Function, "register_manager", self); } + fn excludeRegister(reg: Register, register_class: RegisterBitSet) bool { + const index = indexOfRegIntoTracked(reg) orelse return true; + return !register_class.isSet(index); + } + fn markRegAllocated(self: *Self, reg: Register) void { const index = indexOfRegIntoTracked(reg) orelse return; self.allocated_registers.set(index); @@ -167,10 +172,6 @@ pub fn RegisterManager( return self.locked_registers.count() > 0; } - const AllocOpts = struct { - selector_mask: ?RegisterBitSet = null, - }; - /// Allocates a specified number of registers, optionally /// tracking them. Returns `null` if not enough registers are /// free. @@ -178,14 +179,12 @@ pub fn RegisterManager( self: *Self, comptime count: comptime_int, insts: [count]?Air.Inst.Index, - opts: AllocOpts, + register_class: RegisterBitSet, ) ?[count]Register { comptime assert(count > 0 and count <= tracked_registers.len); - const available_registers = opts.selector_mask orelse RegisterBitSet.initFull(); - var free_and_not_locked_registers = self.free_registers; - free_and_not_locked_registers.setIntersection(available_registers); + free_and_not_locked_registers.setIntersection(register_class); var unlocked_registers = self.locked_registers; unlocked_registers.toggleAll(); @@ -198,6 +197,7 @@ pub fn RegisterManager( var i: usize = 0; for (tracked_registers) |reg| { if (i >= count) break; + if (excludeRegister(reg, register_class)) continue; if (self.isRegLocked(reg)) continue; if (!self.isRegFree(reg)) continue; @@ -223,8 +223,8 @@ pub fn RegisterManager( /// Allocates a register and optionally tracks it with a /// corresponding instruction. Returns `null` if all registers /// are allocated. - pub fn tryAllocReg(self: *Self, inst: ?Air.Inst.Index, opts: AllocOpts) ?Register { - return if (tryAllocRegs(self, 1, .{inst}, opts)) |regs| regs[0] else null; + pub fn tryAllocReg(self: *Self, inst: ?Air.Inst.Index, register_class: RegisterBitSet) ?Register { + return if (tryAllocRegs(self, 1, .{inst}, register_class)) |regs| regs[0] else null; } /// Allocates a specified number of registers, optionally @@ -234,18 +234,16 @@ pub fn RegisterManager( self: *Self, comptime count: comptime_int, insts: [count]?Air.Inst.Index, - opts: AllocOpts, + register_class: RegisterBitSet, ) AllocateRegistersError![count]Register { comptime assert(count > 0 and count <= tracked_registers.len); - const available_registers = opts.selector_mask orelse RegisterBitSet.initFull(); - var locked_registers = self.locked_registers; - locked_registers.setIntersection(available_registers); + locked_registers.setIntersection(register_class); - if (count > available_registers.count() - locked_registers.count()) return error.OutOfRegisters; + if (count > register_class.count() - locked_registers.count()) return error.OutOfRegisters; - const result = self.tryAllocRegs(count, insts, opts) orelse blk: { + const result = self.tryAllocRegs(count, insts, register_class) orelse blk: { // We'll take over the first count registers. Spill // the instructions that were previously there to a // stack allocations. @@ -253,6 +251,7 @@ pub fn RegisterManager( var i: usize = 0; for (tracked_registers) |reg| { if (i >= count) break; + if (excludeRegister(reg, register_class)) continue; if (self.isRegLocked(reg)) continue; regs[i] = reg; @@ -288,8 +287,12 @@ pub fn RegisterManager( /// Allocates a register and optionally tracks it with a /// corresponding instruction. - pub fn allocReg(self: *Self, inst: ?Air.Inst.Index, opts: AllocOpts) AllocateRegistersError!Register { - return (try self.allocRegs(1, .{inst}, opts))[0]; + pub fn allocReg( + self: *Self, + inst: ?Air.Inst.Index, + register_class: RegisterBitSet, + ) AllocateRegistersError!Register { + return (try self.allocRegs(1, .{inst}, register_class))[0]; } /// Spills the register if it is currently allocated. If a @@ -374,11 +377,15 @@ const MockRegister2 = enum(u2) { fn MockFunction(comptime Register: type) type { return struct { allocator: Allocator, - register_manager: RegisterManager(Self, Register, &Register.allocatable_registers) = .{}, + register_manager: RegisterManagerT = .{}, spilled: std.ArrayListUnmanaged(Register) = .{}, const Self = @This(); + const RegisterManagerT = RegisterManager(Self, Register, &Register.allocatable_registers); + + pub const reg_class: RegisterManagerT.RegisterBitSet = RegisterManagerT.RegisterBitSet.initFull(); + pub fn deinit(self: *Self) void { self.spilled.deinit(self.allocator); } @@ -423,10 +430,20 @@ test "tryAllocReg: no spilling" { defer function.deinit(); const mock_instruction: Air.Inst.Index = 1; + const reg_class = MockFunction1.reg_class; - try expectEqual(@as(?MockRegister1, .r2), function.register_manager.tryAllocReg(mock_instruction, .{})); - try expectEqual(@as(?MockRegister1, .r3), function.register_manager.tryAllocReg(mock_instruction, .{})); - try expectEqual(@as(?MockRegister1, null), function.register_manager.tryAllocReg(mock_instruction, .{})); + try expectEqual(@as(?MockRegister1, .r2), function.register_manager.tryAllocReg( + mock_instruction, + reg_class, + )); + try expectEqual(@as(?MockRegister1, .r3), function.register_manager.tryAllocReg( + mock_instruction, + reg_class, + )); + try expectEqual(@as(?MockRegister1, null), function.register_manager.tryAllocReg( + mock_instruction, + reg_class, + )); try expect(function.register_manager.isRegAllocated(.r2)); try expect(function.register_manager.isRegAllocated(.r3)); @@ -451,17 +468,30 @@ test "allocReg: spilling" { defer function.deinit(); const mock_instruction: Air.Inst.Index = 1; + const reg_class = MockFunction1.reg_class; - try expectEqual(@as(?MockRegister1, .r2), try function.register_manager.allocReg(mock_instruction, .{})); - try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction, .{})); + try expectEqual(@as(?MockRegister1, .r2), try function.register_manager.allocReg( + mock_instruction, + reg_class, + )); + try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg( + mock_instruction, + reg_class, + )); // Spill a register - try expectEqual(@as(?MockRegister1, .r2), try function.register_manager.allocReg(mock_instruction, .{})); + try expectEqual(@as(?MockRegister1, .r2), try function.register_manager.allocReg( + mock_instruction, + reg_class, + )); try expectEqualSlices(MockRegister1, &[_]MockRegister1{.r2}, function.spilled.items); // No spilling necessary function.register_manager.freeReg(.r3); - try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction, .{})); + try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg( + mock_instruction, + reg_class, + )); try expectEqualSlices(MockRegister1, &[_]MockRegister1{.r2}, function.spilled.items); // Locked registers @@ -470,7 +500,10 @@ test "allocReg: spilling" { const lock = function.register_manager.lockReg(.r2); defer if (lock) |reg| function.register_manager.unlockReg(reg); - try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg(mock_instruction, .{})); + try expectEqual(@as(?MockRegister1, .r3), try function.register_manager.allocReg( + mock_instruction, + reg_class, + )); } try expect(!function.register_manager.lockedRegsExist()); } @@ -483,7 +516,13 @@ test "tryAllocRegs" { }; defer function.deinit(); - try expectEqual([_]MockRegister2{ .r0, .r1, .r2 }, function.register_manager.tryAllocRegs(3, .{ null, null, null }, .{}).?); + const reg_class = MockFunction2.reg_class; + + try expectEqual([_]MockRegister2{ .r0, .r1, .r2 }, function.register_manager.tryAllocRegs( + 3, + .{ null, null, null }, + reg_class, + ).?); try expect(function.register_manager.isRegAllocated(.r0)); try expect(function.register_manager.isRegAllocated(.r1)); @@ -498,7 +537,11 @@ test "tryAllocRegs" { const lock = function.register_manager.lockReg(.r1); defer if (lock) |reg| function.register_manager.unlockReg(reg); - try expectEqual([_]MockRegister2{ .r0, .r2, .r3 }, function.register_manager.tryAllocRegs(3, .{ null, null, null }, .{}).?); + try expectEqual([_]MockRegister2{ .r0, .r2, .r3 }, function.register_manager.tryAllocRegs( + 3, + .{ null, null, null }, + reg_class, + ).?); } try expect(!function.register_manager.lockedRegsExist()); @@ -518,6 +561,8 @@ test "allocRegs: normal usage" { }; defer function.deinit(); + const reg_class = MockFunction2.reg_class; + { const result_reg: MockRegister2 = .r1; @@ -537,7 +582,7 @@ test "allocRegs: normal usage" { const lock = function.register_manager.lockReg(result_reg); defer if (lock) |reg| function.register_manager.unlockReg(reg); - const regs = try function.register_manager.allocRegs(2, .{ null, null }, .{}); + const regs = try function.register_manager.allocRegs(2, .{ null, null }, reg_class); try function.genAdd(result_reg, regs[0], regs[1]); } } @@ -552,6 +597,8 @@ test "allocRegs: selectively reducing register pressure" { }; defer function.deinit(); + const reg_class = MockFunction2.reg_class; + { const result_reg: MockRegister2 = .r1; @@ -559,12 +606,12 @@ test "allocRegs: selectively reducing register pressure" { // Here, we don't defer unlock because we manually unlock // after genAdd - const regs = try function.register_manager.allocRegs(2, .{ null, null }, .{}); + const regs = try function.register_manager.allocRegs(2, .{ null, null }, reg_class); try function.genAdd(result_reg, regs[0], regs[1]); function.register_manager.unlockReg(lock.?); - const extra_summand_reg = try function.register_manager.allocReg(null, .{}); + const extra_summand_reg = try function.register_manager.allocReg(null, reg_class); try function.genAdd(result_reg, result_reg, extra_summand_reg); } } From 0e43d007c098837d1863f2dab80b99e95910e0aa Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Fri, 20 May 2022 00:04:56 +0200 Subject: [PATCH 18/19] regalloc: temporarily nerf back to raw ints until stage2 catches up --- src/arch/aarch64/abi.zig | 19 +- src/arch/arm/abi.zig | 19 +- src/arch/riscv64/abi.zig | 19 +- src/arch/sparc64/abi.zig | 19 +- src/arch/x86_64/abi.zig | 38 ++-- src/register_manager.zig | 414 +++++++++++++++++++++++++++++++++++---- 6 files changed, 439 insertions(+), 89 deletions(-) diff --git a/src/arch/aarch64/abi.zig b/src/arch/aarch64/abi.zig index 839b2789e4..89a3a6c21d 100644 --- a/src/arch/aarch64/abi.zig +++ b/src/arch/aarch64/abi.zig @@ -1,3 +1,4 @@ +const std = @import("std"); const builtin = @import("builtin"); const bits = @import("bits.zig"); const Register = bits.Register; @@ -26,12 +27,14 @@ pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, // Register classes const RegisterBitSet = RegisterManager.RegisterBitSet; pub const RegisterClass = struct { - pub const gp: RegisterBitSet = blk: { - var set = RegisterBitSet.initEmpty(); - set.setRangeValue(.{ - .start = 0, - .end = callee_preserved_regs.len, - }, true); - break :blk set; - }; + pub const gp: RegisterBitSet = std.math.maxInt(RegisterBitSet); + // TODO uncomment once #11680 is fixed. + // pub const gp: RegisterBitSet = blk: { + // var set = RegisterBitSet.initEmpty(); + // set.setRangeValue(.{ + // .start = 0, + // .end = callee_preserved_regs.len, + // }, true); + // break :blk set; + // }; }; diff --git a/src/arch/arm/abi.zig b/src/arch/arm/abi.zig index dbee1f9b90..c76c3b0ea0 100644 --- a/src/arch/arm/abi.zig +++ b/src/arch/arm/abi.zig @@ -1,3 +1,4 @@ +const std = @import("std"); const bits = @import("bits.zig"); const Register = bits.Register; const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager; @@ -14,12 +15,14 @@ pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, // Register classes const RegisterBitSet = RegisterManager.RegisterBitSet; pub const RegisterClass = struct { - pub const gp: RegisterBitSet = blk: { - var set = RegisterBitSet.initEmpty(); - set.setRangeValue(.{ - .start = 0, - .end = caller_preserved_regs.len + callee_preserved_regs.len, - }, true); - break :blk set; - }; + pub const gp: RegisterBitSet = std.math.maxInt(RegisterBitSet); + // TODO uncomment once #11680 is fixed. + // pub const gp: RegisterBitSet = blk: { + // var set = RegisterBitSet.initEmpty(); + // set.setRangeValue(.{ + // .start = 0, + // .end = caller_preserved_regs.len + callee_preserved_regs.len, + // }, true); + // break :blk set; + // }; }; diff --git a/src/arch/riscv64/abi.zig b/src/arch/riscv64/abi.zig index 459fd40c9c..30d3719a46 100644 --- a/src/arch/riscv64/abi.zig +++ b/src/arch/riscv64/abi.zig @@ -1,3 +1,4 @@ +const std = @import("std"); const bits = @import("bits.zig"); const Register = bits.Register; const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager; @@ -12,12 +13,14 @@ pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, // Register classes const RegisterBitSet = RegisterManager.RegisterBitSet; pub const RegisterClass = struct { - pub const gp: RegisterBitSet = blk: { - var set = RegisterBitSet.initEmpty(); - set.setRangeValue(.{ - .start = 0, - .end = callee_preserved_regs.len, - }, true); - break :blk set; - }; + pub const gp: RegisterBitSet = std.math.maxInt(RegisterBitSet); + // TODO uncomment once #11680 is fixed. + // pub const gp: RegisterBitSet = blk: { + // var set = RegisterBitSet.initEmpty(); + // set.setRangeValue(.{ + // .start = 0, + // .end = callee_preserved_regs.len, + // }, true); + // break :blk set; + // }; }; diff --git a/src/arch/sparc64/abi.zig b/src/arch/sparc64/abi.zig index b1985a7b0b..1c6d40941f 100644 --- a/src/arch/sparc64/abi.zig +++ b/src/arch/sparc64/abi.zig @@ -1,3 +1,4 @@ +const std = @import("std"); const bits = @import("bits.zig"); const Register = bits.Register; const RegisterManagerFn = @import("../../register_manager.zig").RegisterManager; @@ -42,12 +43,14 @@ pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, // Register classes const RegisterBitSet = RegisterManager.RegisterBitSet; pub const RegisterClass = struct { - pub const gp: RegisterBitSet = blk: { - var set = RegisterBitSet.initEmpty(); - set.setRangeValue(.{ - .start = 0, - .end = allocatable_regs.len, - }, true); - break :blk set; - }; + pub const gp: RegisterBitSet = std.math.maxInt(RegisterBitSet); + // TODO uncomment once #11680 is fixed. + // pub const gp: RegisterBitSet = blk: { + // var set = RegisterBitSet.initEmpty(); + // set.setRangeValue(.{ + // .start = 0, + // .end = allocatable_regs.len, + // }, true); + // break :blk set; + // }; }; diff --git a/src/arch/x86_64/abi.zig b/src/arch/x86_64/abi.zig index 7e2025a23d..77f28c11f4 100644 --- a/src/arch/x86_64/abi.zig +++ b/src/arch/x86_64/abi.zig @@ -393,20 +393,26 @@ pub const RegisterManager = RegisterManagerFn(@import("CodeGen.zig"), Register, // Register classes const RegisterBitSet = RegisterManager.RegisterBitSet; pub const RegisterClass = struct { - pub const gp: RegisterBitSet = blk: { - var set = RegisterBitSet.initEmpty(); - set.setRangeValue(.{ - .start = 0, - .end = caller_preserved_regs.len + callee_preserved_regs.len, - }, true); - break :blk set; - }; - pub const sse: RegisterBitSet = blk: { - var set = RegisterBitSet.initEmpty(); - set.setRangeValue(.{ - .start = caller_preserved_regs.len + callee_preserved_regs.len, - .end = allocatable_registers.len, - }, true); - break :blk set; - }; + pub const gp: RegisterBitSet = @as(RegisterBitSet, std.math.maxInt(std.meta.Int( + .unsigned, + caller_preserved_regs.len + callee_preserved_regs.len, + ))); + pub const sse: RegisterBitSet = std.math.maxInt(RegisterBitSet) - gp; + // TODO uncomment once #11680 is fixed. + // pub const gp: RegisterBitSet = blk: { + // var set = RegisterBitSet.initEmpty(); + // set.setRangeValue(.{ + // .start = 0, + // .end = caller_preserved_regs.len + callee_preserved_regs.len, + // }, true); + // break :blk set; + // }; + // pub const sse: RegisterBitSet = blk: { + // var set = RegisterBitSet.initEmpty(); + // set.setRangeValue(.{ + // .start = caller_preserved_regs.len + callee_preserved_regs.len, + // .end = allocatable_registers.len, + // }, true); + // break :blk set; + // }; }; diff --git a/src/register_manager.zig b/src/register_manager.zig index a73f345e0c..347c916769 100644 --- a/src/register_manager.zig +++ b/src/register_manager.zig @@ -4,7 +4,6 @@ const mem = std.mem; const assert = std.debug.assert; const Allocator = std.mem.Allocator; const Air = @import("Air.zig"); -const StaticBitSet = std.bit_set.StaticBitSet; const Type = @import("type.zig").Type; const Module = @import("Module.zig"); const expect = std.testing.expect; @@ -42,39 +41,49 @@ pub fn RegisterManager( registers: [tracked_registers.len]Air.Inst.Index = undefined, /// Tracks which registers are free (in which case the /// corresponding bit is set to 1) - free_registers: RegisterBitSet = RegisterBitSet.initFull(), + free_registers: RegisterBitSet = math.maxInt(RegisterBitSet), /// Tracks all registers allocated in the course of this /// function - allocated_registers: RegisterBitSet = RegisterBitSet.initEmpty(), + allocated_registers: RegisterBitSet = 0, /// Tracks registers which are locked from being allocated - locked_registers: RegisterBitSet = RegisterBitSet.initEmpty(), + locked_registers: RegisterBitSet = 0, const Self = @This(); - pub const RegisterBitSet = StaticBitSet(tracked_registers.len); + /// An integer whose bits represent all the registers and + /// whether they are free. + pub const RegisterBitSet = std.meta.Int(.unsigned, tracked_registers.len); + const ShiftInt = math.Log2Int(RegisterBitSet); fn getFunction(self: *Self) *Function { return @fieldParentPtr(Function, "register_manager", self); } fn excludeRegister(reg: Register, register_class: RegisterBitSet) bool { - const index = indexOfRegIntoTracked(reg) orelse return true; - return !register_class.isSet(index); + const mask = getRegisterMask(reg) orelse return true; + return mask & register_class == 0; + } + + fn getRegisterMask(reg: Register) ?RegisterBitSet { + const index = indexOfRegIntoTracked(reg) orelse return null; + const shift = @intCast(ShiftInt, index); + const mask = @as(RegisterBitSet, 1) << shift; + return mask; } fn markRegAllocated(self: *Self, reg: Register) void { - const index = indexOfRegIntoTracked(reg) orelse return; - self.allocated_registers.set(index); + const mask = getRegisterMask(reg) orelse return; + self.allocated_registers |= mask; } fn markRegUsed(self: *Self, reg: Register) void { - const index = indexOfRegIntoTracked(reg) orelse return; - self.free_registers.unset(index); + const mask = getRegisterMask(reg) orelse return; + self.free_registers &= ~mask; } fn markRegFree(self: *Self, reg: Register) void { - const index = indexOfRegIntoTracked(reg) orelse return; - self.free_registers.set(index); + const mask = getRegisterMask(reg) orelse return; + self.free_registers |= mask; } pub fn indexOfReg( @@ -87,14 +96,14 @@ pub fn RegisterManager( return null; } - pub fn indexOfRegIntoTracked(reg: Register) ?RegisterBitSet.ShiftInt { + pub fn indexOfRegIntoTracked(reg: Register) ?ShiftInt { return indexOfReg(tracked_registers, reg); } /// Returns true when this register is not tracked pub fn isRegFree(self: Self, reg: Register) bool { - const index = indexOfRegIntoTracked(reg) orelse return true; - return self.free_registers.isSet(index); + const mask = getRegisterMask(reg) orelse return true; + return self.free_registers & mask != 0; } /// Returns whether this register was allocated in the course @@ -102,16 +111,16 @@ pub fn RegisterManager( /// /// Returns false when this register is not tracked pub fn isRegAllocated(self: Self, reg: Register) bool { - const index = indexOfRegIntoTracked(reg) orelse return false; - return self.allocated_registers.isSet(index); + const mask = getRegisterMask(reg) orelse return false; + return self.allocated_registers & mask != 0; } /// Returns whether this register is locked /// /// Returns false when this register is not tracked pub fn isRegLocked(self: Self, reg: Register) bool { - const index = indexOfRegIntoTracked(reg) orelse return false; - return self.locked_registers.isSet(index); + const mask = getRegisterMask(reg) orelse return false; + return self.locked_registers & mask != 0; } pub const RegisterLock = struct { @@ -130,8 +139,8 @@ pub fn RegisterManager( log.debug(" register already locked", .{}); return null; } - const index = indexOfRegIntoTracked(reg) orelse return null; - self.locked_registers.set(index); + const mask = getRegisterMask(reg) orelse return null; + self.locked_registers |= mask; return RegisterLock{ .register = reg }; } @@ -140,8 +149,8 @@ pub fn RegisterManager( pub fn lockRegAssumeUnused(self: *Self, reg: Register) RegisterLock { log.debug("locking asserting free {}", .{reg}); assert(!self.isRegLocked(reg)); - const index = indexOfRegIntoTracked(reg) orelse unreachable; - self.locked_registers.set(index); + const mask = getRegisterMask(reg) orelse unreachable; + self.locked_registers |= mask; return RegisterLock{ .register = reg }; } @@ -163,13 +172,13 @@ pub fn RegisterManager( /// Call `lockReg` to obtain the lock first. pub fn unlockReg(self: *Self, lock: RegisterLock) void { log.debug("unlocking {}", .{lock.register}); - const index = indexOfRegIntoTracked(lock.register) orelse return; - self.locked_registers.unset(index); + const mask = getRegisterMask(lock.register) orelse return; + self.locked_registers &= ~mask; } /// Returns true when at least one register is locked pub fn lockedRegsExist(self: Self) bool { - return self.locked_registers.count() > 0; + return self.locked_registers != 0; } /// Allocates a specified number of registers, optionally @@ -183,15 +192,10 @@ pub fn RegisterManager( ) ?[count]Register { comptime assert(count > 0 and count <= tracked_registers.len); - var free_and_not_locked_registers = self.free_registers; - free_and_not_locked_registers.setIntersection(register_class); - - var unlocked_registers = self.locked_registers; - unlocked_registers.toggleAll(); - - free_and_not_locked_registers.setIntersection(unlocked_registers); - - if (free_and_not_locked_registers.count() < count) return null; + const free_registers = self.free_registers & register_class; + const free_and_not_locked_registers = free_registers & ~self.locked_registers; + const free_and_not_locked_registers_count = @popCount(RegisterBitSet, free_and_not_locked_registers); + if (free_and_not_locked_registers_count < count) return null; var regs: [count]Register = undefined; var i: usize = 0; @@ -238,10 +242,10 @@ pub fn RegisterManager( ) AllocateRegistersError![count]Register { comptime assert(count > 0 and count <= tracked_registers.len); - var locked_registers = self.locked_registers; - locked_registers.setIntersection(register_class); - - if (count > register_class.count() - locked_registers.count()) return error.OutOfRegisters; + const available_registers_count = @popCount(RegisterBitSet, register_class); + const locked_registers = self.locked_registers & register_class; + const locked_registers_count = @popCount(RegisterBitSet, locked_registers); + if (count > available_registers_count - locked_registers_count) return error.OutOfRegisters; const result = self.tryAllocRegs(count, insts, register_class) orelse blk: { // We'll take over the first count registers. Spill @@ -348,6 +352,334 @@ pub fn RegisterManager( }; } +// TODO delete current implementation of RegisterManager above, and uncomment the one +// below once #11680 is fixed: +// https://github.com/ziglang/zig/issues/11680 + +//pub fn RegisterManager( +// comptime Function: type, +// comptime Register: type, +// comptime tracked_registers: []const Register, +//) type { +// // architectures which do not have a concept of registers should +// // refrain from using RegisterManager +// assert(tracked_registers.len > 0); // see note above + +// return struct { +// /// Tracks the AIR instruction allocated to every register. If +// /// no instruction is allocated to a register (i.e. the +// /// register is free), the value in that slot is undefined. +// /// +// /// The key must be canonical register. +// registers: [tracked_registers.len]Air.Inst.Index = undefined, +// /// Tracks which registers are free (in which case the +// /// corresponding bit is set to 1) +// free_registers: RegisterBitSet = RegisterBitSet.initFull(), +// /// Tracks all registers allocated in the course of this +// /// function +// allocated_registers: RegisterBitSet = RegisterBitSet.initEmpty(), +// /// Tracks registers which are locked from being allocated +// locked_registers: RegisterBitSet = RegisterBitSet.initEmpty(), + +// const Self = @This(); + +// pub const RegisterBitSet = StaticBitSet(tracked_registers.len); + +// fn getFunction(self: *Self) *Function { +// return @fieldParentPtr(Function, "register_manager", self); +// } + +// fn excludeRegister(reg: Register, register_class: RegisterBitSet) bool { +// const index = indexOfRegIntoTracked(reg) orelse return true; +// return !register_class.isSet(index); +// } + +// fn markRegAllocated(self: *Self, reg: Register) void { +// const index = indexOfRegIntoTracked(reg) orelse return; +// self.allocated_registers.set(index); +// } + +// fn markRegUsed(self: *Self, reg: Register) void { +// const index = indexOfRegIntoTracked(reg) orelse return; +// self.free_registers.unset(index); +// } + +// fn markRegFree(self: *Self, reg: Register) void { +// const index = indexOfRegIntoTracked(reg) orelse return; +// self.free_registers.set(index); +// } + +// pub fn indexOfReg( +// comptime registers: []const Register, +// reg: Register, +// ) ?std.math.IntFittingRange(0, registers.len - 1) { +// inline for (tracked_registers) |cpreg, i| { +// if (reg.id() == cpreg.id()) return i; +// } +// return null; +// } + +// pub fn indexOfRegIntoTracked(reg: Register) ?RegisterBitSet.ShiftInt { +// return indexOfReg(tracked_registers, reg); +// } + +// /// Returns true when this register is not tracked +// pub fn isRegFree(self: Self, reg: Register) bool { +// const index = indexOfRegIntoTracked(reg) orelse return true; +// return self.free_registers.isSet(index); +// } + +// /// Returns whether this register was allocated in the course +// /// of this function. +// /// +// /// Returns false when this register is not tracked +// pub fn isRegAllocated(self: Self, reg: Register) bool { +// const index = indexOfRegIntoTracked(reg) orelse return false; +// return self.allocated_registers.isSet(index); +// } + +// /// Returns whether this register is locked +// /// +// /// Returns false when this register is not tracked +// pub fn isRegLocked(self: Self, reg: Register) bool { +// const index = indexOfRegIntoTracked(reg) orelse return false; +// return self.locked_registers.isSet(index); +// } + +// pub const RegisterLock = struct { +// register: Register, +// }; + +// /// Prevents the register from being allocated until they are +// /// unlocked again. +// /// Returns `RegisterLock` if the register was not already +// /// locked, or `null` otherwise. +// /// Only the owner of the `RegisterLock` can unlock the +// /// register later. +// pub fn lockReg(self: *Self, reg: Register) ?RegisterLock { +// log.debug("locking {}", .{reg}); +// if (self.isRegLocked(reg)) { +// log.debug(" register already locked", .{}); +// return null; +// } +// const index = indexOfRegIntoTracked(reg) orelse return null; +// self.locked_registers.set(index); +// return RegisterLock{ .register = reg }; +// } + +// /// Like `lockReg` but asserts the register was unused always +// /// returning a valid lock. +// pub fn lockRegAssumeUnused(self: *Self, reg: Register) RegisterLock { +// log.debug("locking asserting free {}", .{reg}); +// assert(!self.isRegLocked(reg)); +// const index = indexOfRegIntoTracked(reg) orelse unreachable; +// self.locked_registers.set(index); +// return RegisterLock{ .register = reg }; +// } + +// /// Like `lockRegAssumeUnused` but locks multiple registers. +// pub fn lockRegsAssumeUnused( +// self: *Self, +// comptime count: comptime_int, +// regs: [count]Register, +// ) [count]RegisterLock { +// var buf: [count]RegisterLock = undefined; +// for (regs) |reg, i| { +// buf[i] = self.lockRegAssumeUnused(reg); +// } +// return buf; +// } + +// /// Unlocks the register allowing its re-allocation and re-use. +// /// Requires `RegisterLock` to unlock a register. +// /// Call `lockReg` to obtain the lock first. +// pub fn unlockReg(self: *Self, lock: RegisterLock) void { +// log.debug("unlocking {}", .{lock.register}); +// const index = indexOfRegIntoTracked(lock.register) orelse return; +// self.locked_registers.unset(index); +// } + +// /// Returns true when at least one register is locked +// pub fn lockedRegsExist(self: Self) bool { +// return self.locked_registers.count() > 0; +// } + +// /// Allocates a specified number of registers, optionally +// /// tracking them. Returns `null` if not enough registers are +// /// free. +// pub fn tryAllocRegs( +// self: *Self, +// comptime count: comptime_int, +// insts: [count]?Air.Inst.Index, +// register_class: RegisterBitSet, +// ) ?[count]Register { +// comptime assert(count > 0 and count <= tracked_registers.len); + +// var free_and_not_locked_registers = self.free_registers; +// free_and_not_locked_registers.setIntersection(register_class); + +// var unlocked_registers = self.locked_registers; +// unlocked_registers.toggleAll(); + +// free_and_not_locked_registers.setIntersection(unlocked_registers); + +// if (free_and_not_locked_registers.count() < count) return null; + +// var regs: [count]Register = undefined; +// var i: usize = 0; +// for (tracked_registers) |reg| { +// if (i >= count) break; +// if (excludeRegister(reg, register_class)) continue; +// if (self.isRegLocked(reg)) continue; +// if (!self.isRegFree(reg)) continue; + +// regs[i] = reg; +// i += 1; +// } +// assert(i == count); + +// for (regs) |reg, j| { +// self.markRegAllocated(reg); + +// if (insts[j]) |inst| { +// // Track the register +// const index = indexOfRegIntoTracked(reg).?; // indexOfReg() on a callee-preserved reg should never return null +// self.registers[index] = inst; +// self.markRegUsed(reg); +// } +// } + +// return regs; +// } + +// /// Allocates a register and optionally tracks it with a +// /// corresponding instruction. Returns `null` if all registers +// /// are allocated. +// pub fn tryAllocReg(self: *Self, inst: ?Air.Inst.Index, register_class: RegisterBitSet) ?Register { +// return if (tryAllocRegs(self, 1, .{inst}, register_class)) |regs| regs[0] else null; +// } + +// /// Allocates a specified number of registers, optionally +// /// tracking them. Asserts that count is not +// /// larger than the total number of registers available. +// pub fn allocRegs( +// self: *Self, +// comptime count: comptime_int, +// insts: [count]?Air.Inst.Index, +// register_class: RegisterBitSet, +// ) AllocateRegistersError![count]Register { +// comptime assert(count > 0 and count <= tracked_registers.len); + +// var locked_registers = self.locked_registers; +// locked_registers.setIntersection(register_class); + +// if (count > register_class.count() - locked_registers.count()) return error.OutOfRegisters; + +// const result = self.tryAllocRegs(count, insts, register_class) orelse blk: { +// // We'll take over the first count registers. Spill +// // the instructions that were previously there to a +// // stack allocations. +// var regs: [count]Register = undefined; +// var i: usize = 0; +// for (tracked_registers) |reg| { +// if (i >= count) break; +// if (excludeRegister(reg, register_class)) break; +// if (self.isRegLocked(reg)) continue; + +// regs[i] = reg; +// self.markRegAllocated(reg); +// const index = indexOfRegIntoTracked(reg).?; // indexOfReg() on a callee-preserved reg should never return null +// if (insts[i]) |inst| { +// // Track the register +// if (self.isRegFree(reg)) { +// self.markRegUsed(reg); +// } else { +// const spilled_inst = self.registers[index]; +// try self.getFunction().spillInstruction(reg, spilled_inst); +// } +// self.registers[index] = inst; +// } else { +// // Don't track the register +// if (!self.isRegFree(reg)) { +// const spilled_inst = self.registers[index]; +// try self.getFunction().spillInstruction(reg, spilled_inst); +// self.freeReg(reg); +// } +// } + +// i += 1; +// } + +// break :blk regs; +// }; + +// log.debug("allocated registers {any} for insts {any}", .{ result, insts }); +// return result; +// } + +// /// Allocates a register and optionally tracks it with a +// /// corresponding instruction. +// pub fn allocReg( +// self: *Self, +// inst: ?Air.Inst.Index, +// register_class: RegisterBitSet, +// ) AllocateRegistersError!Register { +// return (try self.allocRegs(1, .{inst}, register_class))[0]; +// } + +// /// Spills the register if it is currently allocated. If a +// /// corresponding instruction is passed, will also track this +// /// register. +// pub fn getReg(self: *Self, reg: Register, inst: ?Air.Inst.Index) AllocateRegistersError!void { +// const index = indexOfRegIntoTracked(reg) orelse return; +// log.debug("getReg {} for inst {}", .{ reg, inst }); +// self.markRegAllocated(reg); + +// if (inst) |tracked_inst| +// if (!self.isRegFree(reg)) { +// // Move the instruction that was previously there to a +// // stack allocation. +// const spilled_inst = self.registers[index]; +// self.registers[index] = tracked_inst; +// try self.getFunction().spillInstruction(reg, spilled_inst); +// } else { +// self.getRegAssumeFree(reg, tracked_inst); +// } +// else { +// if (!self.isRegFree(reg)) { +// // Move the instruction that was previously there to a +// // stack allocation. +// const spilled_inst = self.registers[index]; +// try self.getFunction().spillInstruction(reg, spilled_inst); +// self.freeReg(reg); +// } +// } +// } + +// /// Allocates the specified register with the specified +// /// instruction. Asserts that the register is free and no +// /// spilling is necessary. +// pub fn getRegAssumeFree(self: *Self, reg: Register, inst: Air.Inst.Index) void { +// const index = indexOfRegIntoTracked(reg) orelse return; +// log.debug("getRegAssumeFree {} for inst {}", .{ reg, inst }); +// self.markRegAllocated(reg); + +// assert(self.isRegFree(reg)); +// self.registers[index] = inst; +// self.markRegUsed(reg); +// } + +// /// Marks the specified register as free +// pub fn freeReg(self: *Self, reg: Register) void { +// const index = indexOfRegIntoTracked(reg) orelse return; +// log.debug("freeing register {}", .{reg}); + +// self.registers[index] = undefined; +// self.markRegFree(reg); +// } +// }; +//} + const MockRegister1 = enum(u2) { r0, r1, @@ -384,7 +716,7 @@ fn MockFunction(comptime Register: type) type { const RegisterManagerT = RegisterManager(Self, Register, &Register.allocatable_registers); - pub const reg_class: RegisterManagerT.RegisterBitSet = RegisterManagerT.RegisterBitSet.initFull(); + pub const reg_class: RegisterManagerT.RegisterBitSet = math.maxInt(RegisterManagerT.RegisterBitSet); pub fn deinit(self: *Self) void { self.spilled.deinit(self.allocator); From 274654d73e36df29fdc017a009c8607fb46a15e4 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Fri, 20 May 2022 13:00:59 +0200 Subject: [PATCH 19/19] x64: implement matching SSE instructions for generic cross-comp target --- src/arch/x86_64/CodeGen.zig | 108 +++++++--- src/arch/x86_64/Emit.zig | 403 ++++++++++++++++++++++++------------ src/arch/x86_64/Mir.zig | 22 +- src/arch/x86_64/bits.zig | 11 + 4 files changed, 384 insertions(+), 160 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 8ce0d5884b..eeb4cab04f 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -881,7 +881,7 @@ fn allocRegOrMem(self: *Self, inst: Air.Inst.Index, reg_ok: bool) !MCValue { switch (elem_ty.zigTypeTag()) { .Vector => return self.fail("TODO allocRegOrMem for Vector type", .{}), .Float => { - if (self.intrinsicsAllowed(elem_ty)) { + if (intrinsicsAllowed(self.target.*, elem_ty)) { const ptr_bytes: u64 = 32; if (abi_size <= ptr_bytes) { if (self.register_manager.tryAllocReg(inst, sse)) |reg| { @@ -970,7 +970,7 @@ pub fn spillRegisters(self: *Self, comptime count: comptime_int, registers: [cou fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register { const reg_class: RegisterManager.RegisterBitSet = switch (ty.zigTypeTag()) { .Float => blk: { - if (self.intrinsicsAllowed(ty)) break :blk sse; + if (intrinsicsAllowed(self.target.*, ty)) break :blk sse; return self.fail("TODO copy {} to register", .{ty.fmtDebug()}); }, else => gp, @@ -987,7 +987,7 @@ fn copyToTmpRegister(self: *Self, ty: Type, mcv: MCValue) !Register { fn copyToRegisterWithInstTracking(self: *Self, reg_owner: Air.Inst.Index, ty: Type, mcv: MCValue) !MCValue { const reg_class: RegisterManager.RegisterBitSet = switch (ty.zigTypeTag()) { .Float => blk: { - if (self.intrinsicsAllowed(ty)) break :blk sse; + if (intrinsicsAllowed(self.target.*, ty)) break :blk sse; return self.fail("TODO copy {} to register", .{ty.fmtDebug()}); }, else => gp, @@ -3462,16 +3462,28 @@ fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValu }, .register => |src_reg| switch (dst_ty.zigTypeTag()) { .Float => { - if (self.intrinsicsAllowed(dst_ty)) { + if (intrinsicsAllowed(self.target.*, dst_ty)) { const actual_tag: Mir.Inst.Tag = switch (dst_ty.tag()) { .f32 => switch (mir_tag) { - .add => Mir.Inst.Tag.add_f32_avx, - .cmp => Mir.Inst.Tag.cmp_f32_avx, + .add => if (hasAvxSupport(self.target.*)) + Mir.Inst.Tag.add_f32_avx + else + Mir.Inst.Tag.add_f32_sse, + .cmp => if (hasAvxSupport(self.target.*)) + Mir.Inst.Tag.cmp_f32_avx + else + Mir.Inst.Tag.cmp_f32_sse, else => return self.fail("TODO genBinOpMir for f32 register-register with MIR tag {}", .{mir_tag}), }, .f64 => switch (mir_tag) { - .add => Mir.Inst.Tag.add_f64_avx, - .cmp => Mir.Inst.Tag.cmp_f64_avx, + .add => if (hasAvxSupport(self.target.*)) + Mir.Inst.Tag.add_f64_avx + else + Mir.Inst.Tag.add_f64_sse, + .cmp => if (hasAvxSupport(self.target.*)) + Mir.Inst.Tag.cmp_f64_avx + else + Mir.Inst.Tag.cmp_f64_sse, else => return self.fail("TODO genBinOpMir for f64 register-register with MIR tag {}", .{mir_tag}), }, else => return self.fail("TODO genBinOpMir for float register-register and type {}", .{dst_ty.fmtDebug()}), @@ -5324,10 +5336,16 @@ fn genSetStackArg(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue) InnerE .register => |reg| { switch (ty.zigTypeTag()) { .Float => { - if (self.intrinsicsAllowed(ty)) { + if (intrinsicsAllowed(self.target.*, ty)) { const tag: Mir.Inst.Tag = switch (ty.tag()) { - .f32 => .mov_f32_avx, - .f64 => .mov_f64_avx, + .f32 => if (hasAvxSupport(self.target.*)) + Mir.Inst.Tag.mov_f32_avx + else + Mir.Inst.Tag.mov_f32_sse, + .f64 => if (hasAvxSupport(self.target.*)) + Mir.Inst.Tag.mov_f64_avx + else + Mir.Inst.Tag.mov_f64_sse, else => return self.fail("TODO genSetStackArg for register for type {}", .{ty.fmtDebug()}), }; _ = try self.addInst(.{ @@ -5508,10 +5526,16 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl switch (ty.zigTypeTag()) { .Float => { - if (self.intrinsicsAllowed(ty)) { + if (intrinsicsAllowed(self.target.*, ty)) { const tag: Mir.Inst.Tag = switch (ty.tag()) { - .f32 => .mov_f32_avx, - .f64 => .mov_f64_avx, + .f32 => if (hasAvxSupport(self.target.*)) + Mir.Inst.Tag.mov_f32_avx + else + Mir.Inst.Tag.mov_f32_sse, + .f64 => if (hasAvxSupport(self.target.*)) + Mir.Inst.Tag.mov_f64_avx + else + Mir.Inst.Tag.mov_f64_sse, else => return self.fail("TODO genSetStack for register for type {}", .{ty.fmtDebug()}), }; _ = try self.addInst(.{ @@ -6032,10 +6056,16 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void }, }, .Float => { - if (self.intrinsicsAllowed(ty)) { + if (intrinsicsAllowed(self.target.*, ty)) { const tag: Mir.Inst.Tag = switch (ty.tag()) { - .f32 => .mov_f32_avx, - .f64 => .mov_f64_avx, + .f32 => if (hasAvxSupport(self.target.*)) + Mir.Inst.Tag.mov_f32_avx + else + Mir.Inst.Tag.mov_f32_sse, + .f64 => if (hasAvxSupport(self.target.*)) + Mir.Inst.Tag.mov_f64_avx + else + Mir.Inst.Tag.mov_f64_sse, else => return self.fail("TODO genSetReg from register for {}", .{ty.fmtDebug()}), }; _ = try self.addInst(.{ @@ -6072,10 +6102,16 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void const base_reg = try self.register_manager.allocReg(null, gp); try self.loadMemPtrIntoRegister(base_reg, Type.usize, mcv); - if (self.intrinsicsAllowed(ty)) { + if (intrinsicsAllowed(self.target.*, ty)) { const tag: Mir.Inst.Tag = switch (ty.tag()) { - .f32 => .mov_f32_avx, - .f64 => .mov_f64_avx, + .f32 => if (hasAvxSupport(self.target.*)) + Mir.Inst.Tag.mov_f32_avx + else + Mir.Inst.Tag.mov_f32_sse, + .f64 => if (hasAvxSupport(self.target.*)) + Mir.Inst.Tag.mov_f64_avx + else + Mir.Inst.Tag.mov_f64_sse, else => return self.fail("TODO genSetReg from memory for {}", .{ty.fmtDebug()}), }; @@ -6115,10 +6151,16 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void const base_reg = try self.register_manager.allocReg(null, gp); try self.loadMemPtrIntoRegister(base_reg, Type.usize, mcv); - if (self.intrinsicsAllowed(ty)) { + if (intrinsicsAllowed(self.target.*, ty)) { const tag: Mir.Inst.Tag = switch (ty.tag()) { - .f32 => .mov_f32_avx, - .f64 => .mov_f64_avx, + .f32 => if (hasAvxSupport(self.target.*)) + Mir.Inst.Tag.mov_f32_avx + else + Mir.Inst.Tag.mov_f32_sse, + .f64 => if (hasAvxSupport(self.target.*)) + Mir.Inst.Tag.mov_f64_avx + else + Mir.Inst.Tag.mov_f64_sse, else => return self.fail("TODO genSetReg from memory for {}", .{ty.fmtDebug()}), }; @@ -6230,10 +6272,16 @@ fn genSetReg(self: *Self, ty: Type, reg: Register, mcv: MCValue) InnerError!void }, }, .Float => { - if (self.intrinsicsAllowed(ty)) { + if (intrinsicsAllowed(self.target.*, ty)) { const tag: Mir.Inst.Tag = switch (ty.tag()) { - .f32 => .mov_f32_avx, - .f64 => .mov_f64_avx, + .f32 => if (hasAvxSupport(self.target.*)) + Mir.Inst.Tag.mov_f32_avx + else + Mir.Inst.Tag.mov_f32_sse, + .f64 => if (hasAvxSupport(self.target.*)) + Mir.Inst.Tag.mov_f64_avx + else + Mir.Inst.Tag.mov_f64_sse, else => return self.fail("TODO genSetReg from stack offset for {}", .{ty.fmtDebug()}), }; _ = try self.addInst(.{ @@ -7046,11 +7094,15 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void { } } -fn intrinsicsAllowed(self: *Self, ty: Type) bool { +fn intrinsicsAllowed(target: Target, ty: Type) bool { return switch (ty.tag()) { .f32, .f64, - => Target.x86.featureSetHasAny(self.target.cpu.features, .{ .avx, .avx2 }), + => Target.x86.featureSetHasAny(target.cpu.features, .{ .sse2, .avx, .avx2 }), else => unreachable, // TODO finish this off }; } + +fn hasAvxSupport(target: Target) bool { + return Target.x86.featureSetHasAny(target.cpu.features, .{ .avx, .avx2 }); +} diff --git a/src/arch/x86_64/Emit.zig b/src/arch/x86_64/Emit.zig index fbcd8359f7..84955a8aac 100644 --- a/src/arch/x86_64/Emit.zig +++ b/src/arch/x86_64/Emit.zig @@ -182,6 +182,16 @@ pub fn lowerMir(emit: *Emit) InnerError!void { .interrupt => try emit.mirInterrupt(inst), .nop => try emit.mirNop(), + // SSE instructions + .mov_f64_sse => try emit.mirMovFloatSse(.movsd, inst), + .mov_f32_sse => try emit.mirMovFloatSse(.movss, inst), + + .add_f64_sse => try emit.mirAddFloatSse(.addsd, inst), + .add_f32_sse => try emit.mirAddFloatSse(.addss, inst), + + .cmp_f64_sse => try emit.mirCmpFloatSse(.ucomisd, inst), + .cmp_f32_sse => try emit.mirCmpFloatSse(.ucomiss, inst), + // AVX instructions .mov_f64_avx => try emit.mirMovFloatAvx(.vmovsd, inst), .mov_f32_avx => try emit.mirMovFloatAvx(.vmovss, inst), @@ -536,6 +546,7 @@ fn mirArithMemImm(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { } inline fn setRexWRegister(reg: Register) bool { + if (reg.size() > 64) return false; if (reg.size() == 64) return true; return switch (reg) { .ah, .ch, .dh, .bh => true, @@ -963,11 +974,55 @@ fn mirLeaPie(emit: *Emit, inst: Mir.Inst.Index) InnerError!void { } } +// SSE instructions + +fn mirMovFloatSse(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { + const ops = emit.mir.instructions.items(.ops)[inst].decode(); + switch (ops.flags) { + 0b00 => { + const imm = emit.mir.instructions.items(.data)[inst].imm; + return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg2.size()), .{ + .disp = imm, + .base = ops.reg2, + }), emit.code); + }, + 0b01 => { + const imm = emit.mir.instructions.items(.data)[inst].imm; + return lowerToMrEnc(tag, RegisterOrMemory.mem(Memory.PtrSize.new(ops.reg1.size()), .{ + .disp = imm, + .base = ops.reg1, + }), ops.reg2, emit.code); + }, + 0b10 => { + return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code); + }, + else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, tag }), + } +} + +fn mirAddFloatSse(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { + const ops = emit.mir.instructions.items(.ops)[inst].decode(); + switch (ops.flags) { + 0b00 => { + return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code); + }, + else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, tag }), + } +} + +fn mirCmpFloatSse(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { + const ops = emit.mir.instructions.items(.ops)[inst].decode(); + switch (ops.flags) { + 0b00 => { + return lowerToRmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code); + }, + else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, tag }), + } +} // AVX instructions fn mirMovFloatAvx(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { const ops = emit.mir.instructions.items(.ops)[inst].decode(); - switch (ops.flags) { 0b00 => { const imm = emit.mir.instructions.items(.data)[inst].imm; @@ -986,24 +1041,22 @@ fn mirMovFloatAvx(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { 0b10 => { return lowerToRvmEnc(tag, ops.reg1, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code); }, - else => return emit.fail("TODO unused variant 0b{b} for mov_f64", .{ops.flags}), + else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, tag }), } } fn mirAddFloatAvx(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { const ops = emit.mir.instructions.items(.ops)[inst].decode(); - switch (ops.flags) { 0b00 => { return lowerToRvmEnc(tag, ops.reg1, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code); }, - else => return emit.fail("TODO unused variant 0b{b} for mov_f64", .{ops.flags}), + else => return emit.fail("TODO unused variant 0b{b} for {}", .{ ops.flags, tag }), } } fn mirCmpFloatAvx(emit: *Emit, tag: Tag, inst: Mir.Inst.Index) InnerError!void { const ops = emit.mir.instructions.items(.ops)[inst].decode(); - switch (ops.flags) { 0b00 => { return lowerToVmEnc(tag, ops.reg1, RegisterOrMemory.reg(ops.reg2), emit.code); @@ -1247,6 +1300,14 @@ const Tag = enum { cmovng, cmovb, cmovnae, + movsd, + movss, + addsd, + addss, + cmpsd, + cmpss, + ucomisd, + ucomiss, vmovsd, vmovss, vaddsd, @@ -1256,6 +1317,22 @@ const Tag = enum { vucomisd, vucomiss, + fn isSse(tag: Tag) bool { + return switch (tag) { + .movsd, + .movss, + .addsd, + .addss, + .cmpsd, + .cmpss, + .ucomisd, + .ucomiss, + => true, + + else => false, + }; + } + fn isAvx(tag: Tag) bool { return switch (tag) { .vmovsd, @@ -1369,190 +1446,256 @@ const Encoding = enum { rvmi, }; -const OpCode = union(enum) { - one_byte: u8, - two_byte: struct { _1: u8, _2: u8 }, +const OpCode = struct { + bytes: [3]u8, + count: usize, - fn oneByte(opc: u8) OpCode { - return .{ .one_byte = opc }; - } - - fn twoByte(opc1: u8, opc2: u8) OpCode { - return .{ .two_byte = .{ ._1 = opc1, ._2 = opc2 } }; + fn init(comptime in_bytes: []const u8) OpCode { + comptime assert(in_bytes.len <= 3); + comptime var bytes: [3]u8 = undefined; + inline for (in_bytes) |x, i| { + bytes[i] = x; + } + return .{ .bytes = bytes, .count = in_bytes.len }; } fn encode(opc: OpCode, encoder: Encoder) void { - switch (opc) { - .one_byte => |v| encoder.opcode_1byte(v), - .two_byte => |v| encoder.opcode_2byte(v._1, v._2), + switch (opc.count) { + 1 => encoder.opcode_1byte(opc.bytes[0]), + 2 => encoder.opcode_2byte(opc.bytes[0], opc.bytes[1]), + 3 => encoder.opcode_3byte(opc.bytes[0], opc.bytes[1], opc.bytes[2]), + else => unreachable, } } fn encodeWithReg(opc: OpCode, encoder: Encoder, reg: Register) void { - assert(opc == .one_byte); - encoder.opcode_withReg(opc.one_byte, reg.lowEnc()); + assert(opc.count == 1); + encoder.opcode_withReg(opc.bytes[0], reg.lowEnc()); } }; inline fn getOpCode(tag: Tag, enc: Encoding, is_one_byte: bool) OpCode { + // zig fmt: off switch (enc) { .zo => return switch (tag) { - .ret_near => OpCode.oneByte(0xc3), - .ret_far => OpCode.oneByte(0xcb), - .int3 => OpCode.oneByte(0xcc), - .nop => OpCode.oneByte(0x90), - .syscall => OpCode.twoByte(0x0f, 0x05), - .cbw => OpCode.oneByte(0x98), - .cwd, .cdq, .cqo => OpCode.oneByte(0x99), - else => unreachable, + .ret_near => OpCode.init(&.{0xc3}), + .ret_far => OpCode.init(&.{0xcb}), + .int3 => OpCode.init(&.{0xcc}), + .nop => OpCode.init(&.{0x90}), + .syscall => OpCode.init(&.{ 0x0f, 0x05 }), + .cbw => OpCode.init(&.{0x98}), + .cwd, + .cdq, + .cqo => OpCode.init(&.{0x99}), + else => unreachable, }, .d => return switch (tag) { - .jmp_near => OpCode.oneByte(0xe9), - .call_near => OpCode.oneByte(0xe8), - .jo => if (is_one_byte) OpCode.oneByte(0x70) else OpCode.twoByte(0x0f, 0x80), - .jno => if (is_one_byte) OpCode.oneByte(0x71) else OpCode.twoByte(0x0f, 0x81), - .jb, .jc, .jnae => if (is_one_byte) OpCode.oneByte(0x72) else OpCode.twoByte(0x0f, 0x82), - .jnb, .jnc, .jae => if (is_one_byte) OpCode.oneByte(0x73) else OpCode.twoByte(0x0f, 0x83), - .je, .jz => if (is_one_byte) OpCode.oneByte(0x74) else OpCode.twoByte(0x0f, 0x84), - .jne, .jnz => if (is_one_byte) OpCode.oneByte(0x75) else OpCode.twoByte(0x0f, 0x85), - .jna, .jbe => if (is_one_byte) OpCode.oneByte(0x76) else OpCode.twoByte(0x0f, 0x86), - .jnbe, .ja => if (is_one_byte) OpCode.oneByte(0x77) else OpCode.twoByte(0x0f, 0x87), - .js => if (is_one_byte) OpCode.oneByte(0x78) else OpCode.twoByte(0x0f, 0x88), - .jns => if (is_one_byte) OpCode.oneByte(0x79) else OpCode.twoByte(0x0f, 0x89), - .jpe, .jp => if (is_one_byte) OpCode.oneByte(0x7a) else OpCode.twoByte(0x0f, 0x8a), - .jpo, .jnp => if (is_one_byte) OpCode.oneByte(0x7b) else OpCode.twoByte(0x0f, 0x8b), - .jnge, .jl => if (is_one_byte) OpCode.oneByte(0x7c) else OpCode.twoByte(0x0f, 0x8c), - .jge, .jnl => if (is_one_byte) OpCode.oneByte(0x7d) else OpCode.twoByte(0x0f, 0x8d), - .jle, .jng => if (is_one_byte) OpCode.oneByte(0x7e) else OpCode.twoByte(0x0f, 0x8e), - .jg, .jnle => if (is_one_byte) OpCode.oneByte(0x7f) else OpCode.twoByte(0x0f, 0x8f), - else => unreachable, + .jmp_near => OpCode.init(&.{0xe9}), + .call_near => OpCode.init(&.{0xe8}), + .jo => if (is_one_byte) OpCode.init(&.{0x70}) else OpCode.init(&.{0x0f,0x80}), + .jno => if (is_one_byte) OpCode.init(&.{0x71}) else OpCode.init(&.{0x0f,0x81}), + .jb, + .jc, + .jnae => if (is_one_byte) OpCode.init(&.{0x72}) else OpCode.init(&.{0x0f,0x82}), + .jnb, + .jnc, + .jae => if (is_one_byte) OpCode.init(&.{0x73}) else OpCode.init(&.{0x0f,0x83}), + .je, + .jz => if (is_one_byte) OpCode.init(&.{0x74}) else OpCode.init(&.{0x0f,0x84}), + .jne, + .jnz => if (is_one_byte) OpCode.init(&.{0x75}) else OpCode.init(&.{0x0f,0x85}), + .jna, + .jbe => if (is_one_byte) OpCode.init(&.{0x76}) else OpCode.init(&.{0x0f,0x86}), + .jnbe, + .ja => if (is_one_byte) OpCode.init(&.{0x77}) else OpCode.init(&.{0x0f,0x87}), + .js => if (is_one_byte) OpCode.init(&.{0x78}) else OpCode.init(&.{0x0f,0x88}), + .jns => if (is_one_byte) OpCode.init(&.{0x79}) else OpCode.init(&.{0x0f,0x89}), + .jpe, + .jp => if (is_one_byte) OpCode.init(&.{0x7a}) else OpCode.init(&.{0x0f,0x8a}), + .jpo, + .jnp => if (is_one_byte) OpCode.init(&.{0x7b}) else OpCode.init(&.{0x0f,0x8b}), + .jnge, + .jl => if (is_one_byte) OpCode.init(&.{0x7c}) else OpCode.init(&.{0x0f,0x8c}), + .jge, + .jnl => if (is_one_byte) OpCode.init(&.{0x7d}) else OpCode.init(&.{0x0f,0x8d}), + .jle, + .jng => if (is_one_byte) OpCode.init(&.{0x7e}) else OpCode.init(&.{0x0f,0x8e}), + .jg, + .jnle => if (is_one_byte) OpCode.init(&.{0x7f}) else OpCode.init(&.{0x0f,0x8f}), + else => unreachable, }, .m => return switch (tag) { - .jmp_near, .call_near, .push => OpCode.oneByte(0xff), - .pop => OpCode.oneByte(0x8f), - .seto => OpCode.twoByte(0x0f, 0x90), - .setno => OpCode.twoByte(0x0f, 0x91), - .setb, .setc, .setnae => OpCode.twoByte(0x0f, 0x92), - .setnb, .setnc, .setae => OpCode.twoByte(0x0f, 0x93), - .sete, .setz => OpCode.twoByte(0x0f, 0x94), - .setne, .setnz => OpCode.twoByte(0x0f, 0x95), - .setbe, .setna => OpCode.twoByte(0x0f, 0x96), - .seta, .setnbe => OpCode.twoByte(0x0f, 0x97), - .sets => OpCode.twoByte(0x0f, 0x98), - .setns => OpCode.twoByte(0x0f, 0x99), - .setp, .setpe => OpCode.twoByte(0x0f, 0x9a), - .setnp, .setop => OpCode.twoByte(0x0f, 0x9b), - .setl, .setnge => OpCode.twoByte(0x0f, 0x9c), - .setnl, .setge => OpCode.twoByte(0x0f, 0x9d), - .setle, .setng => OpCode.twoByte(0x0f, 0x9e), - .setnle, .setg => OpCode.twoByte(0x0f, 0x9f), - .idiv, .div, .imul, .mul => OpCode.oneByte(if (is_one_byte) 0xf6 else 0xf7), - .fisttp16 => OpCode.oneByte(0xdf), - .fisttp32 => OpCode.oneByte(0xdb), - .fisttp64 => OpCode.oneByte(0xdd), - .fld32 => OpCode.oneByte(0xd9), - .fld64 => OpCode.oneByte(0xdd), - else => unreachable, + .jmp_near, + .call_near, + .push => OpCode.init(&.{0xff}), + .pop => OpCode.init(&.{0x8f}), + .seto => OpCode.init(&.{0x0f,0x90}), + .setno => OpCode.init(&.{0x0f,0x91}), + .setb, + .setc, + .setnae => OpCode.init(&.{0x0f,0x92}), + .setnb, + .setnc, + .setae => OpCode.init(&.{0x0f,0x93}), + .sete, + .setz => OpCode.init(&.{0x0f,0x94}), + .setne, + .setnz => OpCode.init(&.{0x0f,0x95}), + .setbe, + .setna => OpCode.init(&.{0x0f,0x96}), + .seta, + .setnbe => OpCode.init(&.{0x0f,0x97}), + .sets => OpCode.init(&.{0x0f,0x98}), + .setns => OpCode.init(&.{0x0f,0x99}), + .setp, + .setpe => OpCode.init(&.{0x0f,0x9a}), + .setnp, + .setop => OpCode.init(&.{0x0f,0x9b}), + .setl, + .setnge => OpCode.init(&.{0x0f,0x9c}), + .setnl, + .setge => OpCode.init(&.{0x0f,0x9d}), + .setle, + .setng => OpCode.init(&.{0x0f,0x9e}), + .setnle, + .setg => OpCode.init(&.{0x0f,0x9f}), + .idiv, + .div, + .imul, + .mul => if (is_one_byte) OpCode.init(&.{0xf6}) else OpCode.init(&.{0xf7}), + .fisttp16 => OpCode.init(&.{0xdf}), + .fisttp32 => OpCode.init(&.{0xdb}), + .fisttp64 => OpCode.init(&.{0xdd}), + .fld32 => OpCode.init(&.{0xd9}), + .fld64 => OpCode.init(&.{0xdd}), + else => unreachable, }, .o => return switch (tag) { - .push => OpCode.oneByte(0x50), - .pop => OpCode.oneByte(0x58), - else => unreachable, + .push => OpCode.init(&.{0x50}), + .pop => OpCode.init(&.{0x58}), + else => unreachable, }, .i => return switch (tag) { - .push => OpCode.oneByte(if (is_one_byte) 0x6a else 0x68), - .@"test" => OpCode.oneByte(if (is_one_byte) 0xa8 else 0xa9), - .ret_near => OpCode.oneByte(0xc2), - .ret_far => OpCode.oneByte(0xca), - else => unreachable, + .push => if (is_one_byte) OpCode.init(&.{0x6a}) else OpCode.init(&.{0x68}), + .@"test" => if (is_one_byte) OpCode.init(&.{0xa8}) else OpCode.init(&.{0xa9}), + .ret_near => OpCode.init(&.{0xc2}), + .ret_far => OpCode.init(&.{0xca}), + else => unreachable, }, .m1 => return switch (tag) { - .shl, .sal, .shr, .sar => OpCode.oneByte(if (is_one_byte) 0xd0 else 0xd1), - else => unreachable, + .shl, .sal, + .shr, .sar => if (is_one_byte) OpCode.init(&.{0xd0}) else OpCode.init(&.{0xd1}), + else => unreachable, }, .mc => return switch (tag) { - .shl, .sal, .shr, .sar => OpCode.oneByte(if (is_one_byte) 0xd2 else 0xd3), - else => unreachable, + .shl, .sal, + .shr, .sar => if (is_one_byte) OpCode.init(&.{0xd2}) else OpCode.init(&.{0xd3}), + else => unreachable, }, .mi => return switch (tag) { - .adc, .add, .sub, .xor, .@"and", .@"or", .sbb, .cmp => OpCode.oneByte(if (is_one_byte) 0x80 else 0x81), - .mov => OpCode.oneByte(if (is_one_byte) 0xc6 else 0xc7), - .@"test" => OpCode.oneByte(if (is_one_byte) 0xf6 else 0xf7), - else => unreachable, + .adc, .add, + .sub, .xor, + .@"and", .@"or", + .sbb, .cmp => if (is_one_byte) OpCode.init(&.{0x80}) else OpCode.init(&.{0x81}), + .mov => if (is_one_byte) OpCode.init(&.{0xc6}) else OpCode.init(&.{0xc7}), + .@"test" => if (is_one_byte) OpCode.init(&.{0xf6}) else OpCode.init(&.{0xf7}), + else => unreachable, }, .mi8 => return switch (tag) { - .adc, .add, .sub, .xor, .@"and", .@"or", .sbb, .cmp => OpCode.oneByte(0x83), - .shl, .sal, .shr, .sar => OpCode.oneByte(if (is_one_byte) 0xc0 else 0xc1), - else => unreachable, + .adc, .add, + .sub, .xor, + .@"and", .@"or", + .sbb, .cmp => OpCode.init(&.{0x83}), + .shl, .sal, + .shr, .sar => if (is_one_byte) OpCode.init(&.{0xc0}) else OpCode.init(&.{0xc1}), + else => unreachable, }, .mr => return switch (tag) { - .adc => OpCode.oneByte(if (is_one_byte) 0x10 else 0x11), - .add => OpCode.oneByte(if (is_one_byte) 0x00 else 0x01), - .sub => OpCode.oneByte(if (is_one_byte) 0x28 else 0x29), - .xor => OpCode.oneByte(if (is_one_byte) 0x30 else 0x31), - .@"and" => OpCode.oneByte(if (is_one_byte) 0x20 else 0x21), - .@"or" => OpCode.oneByte(if (is_one_byte) 0x08 else 0x09), - .sbb => OpCode.oneByte(if (is_one_byte) 0x18 else 0x19), - .cmp => OpCode.oneByte(if (is_one_byte) 0x38 else 0x39), - .mov => OpCode.oneByte(if (is_one_byte) 0x88 else 0x89), - .@"test" => OpCode.oneByte(if (is_one_byte) 0x84 else 0x85), - else => unreachable, + .adc => if (is_one_byte) OpCode.init(&.{0x10}) else OpCode.init(&.{0x11}), + .add => if (is_one_byte) OpCode.init(&.{0x00}) else OpCode.init(&.{0x01}), + .sub => if (is_one_byte) OpCode.init(&.{0x28}) else OpCode.init(&.{0x29}), + .xor => if (is_one_byte) OpCode.init(&.{0x30}) else OpCode.init(&.{0x31}), + .@"and" => if (is_one_byte) OpCode.init(&.{0x20}) else OpCode.init(&.{0x21}), + .@"or" => if (is_one_byte) OpCode.init(&.{0x08}) else OpCode.init(&.{0x09}), + .sbb => if (is_one_byte) OpCode.init(&.{0x18}) else OpCode.init(&.{0x19}), + .cmp => if (is_one_byte) OpCode.init(&.{0x38}) else OpCode.init(&.{0x39}), + .mov => if (is_one_byte) OpCode.init(&.{0x88}) else OpCode.init(&.{0x89}), + .@"test" => if (is_one_byte) OpCode.init(&.{0x84}) else OpCode.init(&.{0x85}), + .movsd => OpCode.init(&.{0xf2,0x0f,0x11}), + .movss => OpCode.init(&.{0xf3,0x0f,0x11}), + else => unreachable, }, .rm => return switch (tag) { - .adc => OpCode.oneByte(if (is_one_byte) 0x12 else 0x13), - .add => OpCode.oneByte(if (is_one_byte) 0x02 else 0x03), - .sub => OpCode.oneByte(if (is_one_byte) 0x2a else 0x2b), - .xor => OpCode.oneByte(if (is_one_byte) 0x32 else 0x33), - .@"and" => OpCode.oneByte(if (is_one_byte) 0x22 else 0x23), - .@"or" => OpCode.oneByte(if (is_one_byte) 0x0a else 0x0b), - .sbb => OpCode.oneByte(if (is_one_byte) 0x1a else 0x1b), - .cmp => OpCode.oneByte(if (is_one_byte) 0x3a else 0x3b), - .mov => OpCode.oneByte(if (is_one_byte) 0x8a else 0x8b), - .movsx => OpCode.twoByte(0x0f, if (is_one_byte) 0xbe else 0xbf), - .movsxd => OpCode.oneByte(0x63), - .movzx => OpCode.twoByte(0x0f, if (is_one_byte) 0xb6 else 0xb7), - .lea => OpCode.oneByte(if (is_one_byte) 0x8c else 0x8d), - .imul => OpCode.twoByte(0x0f, 0xaf), - .cmove, .cmovz => OpCode.twoByte(0x0f, 0x44), - .cmovb, .cmovnae => OpCode.twoByte(0x0f, 0x42), - .cmovl, .cmovng => OpCode.twoByte(0x0f, 0x4c), + .adc => if (is_one_byte) OpCode.init(&.{0x12}) else OpCode.init(&.{0x13}), + .add => if (is_one_byte) OpCode.init(&.{0x02}) else OpCode.init(&.{0x03}), + .sub => if (is_one_byte) OpCode.init(&.{0x2a}) else OpCode.init(&.{0x2b}), + .xor => if (is_one_byte) OpCode.init(&.{0x32}) else OpCode.init(&.{0x33}), + .@"and" => if (is_one_byte) OpCode.init(&.{0x22}) else OpCode.init(&.{0x23}), + .@"or" => if (is_one_byte) OpCode.init(&.{0x0a}) else OpCode.init(&.{0x0b}), + .sbb => if (is_one_byte) OpCode.init(&.{0x1a}) else OpCode.init(&.{0x1b}), + .cmp => if (is_one_byte) OpCode.init(&.{0x3a}) else OpCode.init(&.{0x3b}), + .mov => if (is_one_byte) OpCode.init(&.{0x8a}) else OpCode.init(&.{0x8b}), + .movsx => if (is_one_byte) OpCode.init(&.{0x0f,0xbe}) else OpCode.init(&.{0x0f,0xbf}), + .movsxd => OpCode.init(&.{0x63}), + .movzx => if (is_one_byte) OpCode.init(&.{0x0f,0xb6}) else OpCode.init(&.{0x0f,0xb7}), + .lea => if (is_one_byte) OpCode.init(&.{0x8c}) else OpCode.init(&.{0x8d}), + .imul => OpCode.init(&.{0x0f,0xaf}), + .cmove, + .cmovz => OpCode.init(&.{0x0f,0x44}), + .cmovb, + .cmovnae => OpCode.init(&.{0x0f,0x42}), + .cmovl, + .cmovng => OpCode.init(&.{0x0f,0x4c}), + .movsd => OpCode.init(&.{0xf2,0x0f,0x10}), + .movss => OpCode.init(&.{0xf3,0x0f,0x10}), + .addsd => OpCode.init(&.{0xf2,0x0f,0x58}), + .addss => OpCode.init(&.{0xf3,0x0f,0x58}), + .ucomisd => OpCode.init(&.{0x66,0x0f,0x2e}), + .ucomiss => OpCode.init(&.{0x0f,0x2e}), else => unreachable, }, .oi => return switch (tag) { - .mov => OpCode.oneByte(if (is_one_byte) 0xb0 else 0xb8), + .mov => if (is_one_byte) OpCode.init(&.{0xb0}) else OpCode.init(&.{0xb8}), else => unreachable, }, .fd => return switch (tag) { - .mov => OpCode.oneByte(if (is_one_byte) 0xa0 else 0xa1), + .mov => if (is_one_byte) OpCode.init(&.{0xa0}) else OpCode.init(&.{0xa1}), else => unreachable, }, .td => return switch (tag) { - .mov => OpCode.oneByte(if (is_one_byte) 0xa2 else 0xa3), + .mov => if (is_one_byte) OpCode.init(&.{0xa2}) else OpCode.init(&.{0xa3}), else => unreachable, }, .rmi => return switch (tag) { - .imul => OpCode.oneByte(if (is_one_byte) 0x6b else 0x69), - else => unreachable, + .imul => if (is_one_byte) OpCode.init(&.{0x6b}) else OpCode.init(&.{0x69}), + else => unreachable, }, .mv => return switch (tag) { - .vmovsd, .vmovss => OpCode.oneByte(0x11), + .vmovsd, + .vmovss => OpCode.init(&.{0x11}), else => unreachable, }, .vm => return switch (tag) { - .vmovsd, .vmovss => OpCode.oneByte(0x10), - .vucomisd, .vucomiss => OpCode.oneByte(0x2e), + .vmovsd, + .vmovss => OpCode.init(&.{0x10}), + .vucomisd, + .vucomiss => OpCode.init(&.{0x2e}), else => unreachable, }, .rvm => return switch (tag) { - .vaddsd, .vaddss => OpCode.oneByte(0x58), - .vmovsd, .vmovss => OpCode.oneByte(0x10), + .vaddsd, + .vaddss => OpCode.init(&.{0x58}), + .vmovsd, + .vmovss => OpCode.init(&.{0x10}), else => unreachable, }, .rvmi => return switch (tag) { - .vcmpsd, .vcmpss => OpCode.oneByte(0xc2), - else => unreachable, + .vcmpsd, + .vcmpss => OpCode.init(&.{0xc2}), + else => unreachable, }, } + // zig fmt: on } inline fn getModRmExt(tag: Tag) u3 { diff --git a/src/arch/x86_64/Mir.zig b/src/arch/x86_64/Mir.zig index 0f200d43e6..a35231a9b8 100644 --- a/src/arch/x86_64/Mir.zig +++ b/src/arch/x86_64/Mir.zig @@ -345,11 +345,29 @@ pub const Inst = struct { /// Nop nop, - /// AVX instructions + /// SSE instructions /// ops flags: form: /// 0b00 reg1, qword ptr [reg2 + imm32] /// 0b01 qword ptr [reg1 + imm32], reg2 /// 0b10 reg1, reg2 + mov_f64_sse, + mov_f32_sse, + + /// ops flags: form: + /// 0b00 reg1, reg2 + add_f64_sse, + add_f32_sse, + + /// ops flags: form: + /// 0b00 reg1, reg2 + cmp_f64_sse, + cmp_f32_sse, + + /// AVX instructions + /// ops flags: form: + /// 0b00 reg1, qword ptr [reg2 + imm32] + /// 0b01 qword ptr [reg1 + imm32], reg2 + /// 0b10 reg1, reg1, reg2 mov_f64_avx, mov_f32_avx, @@ -359,7 +377,7 @@ pub const Inst = struct { add_f32_avx, /// ops flags: form: - /// + /// 0b00 reg1, reg1, reg2 cmp_f64_avx, cmp_f32_avx, diff --git a/src/arch/x86_64/bits.zig b/src/arch/x86_64/bits.zig index 64a067d050..6429781516 100644 --- a/src/arch/x86_64/bits.zig +++ b/src/arch/x86_64/bits.zig @@ -441,6 +441,17 @@ pub const Encoder = struct { self.code.appendAssumeCapacity(opcode); } + /// Encodes a 3 byte opcode + /// + /// e.g. MOVSD has the opcode 0xf2 0x0f 0x10 + /// + /// encoder.opcode_3byte(0xf2, 0x0f, 0x10); + pub fn opcode_3byte(self: Self, prefix_1: u8, prefix_2: u8, opcode: u8) void { + self.code.appendAssumeCapacity(prefix_1); + self.code.appendAssumeCapacity(prefix_2); + self.code.appendAssumeCapacity(opcode); + } + /// Encodes a 1 byte opcode with a reg field /// /// Remember to add a REX prefix byte if reg is extended!