From 9c3d24ea0bffa53fb08a73493d72a6a0866f6432 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Mon, 9 May 2022 17:39:19 +0200 Subject: [PATCH 01/18] x64: add naive impl of shr --- src/arch/x86_64/CodeGen.zig | 75 ++++++++++++++++++++++++++++++++++--- test/behavior/math.zig | 2 - 2 files changed, 70 insertions(+), 7 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index ee472eeac8..28de88ff86 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -2052,11 +2052,76 @@ fn airShlSat(self: *Self, inst: Air.Inst.Index) !void { fn airShr(self: *Self, inst: Air.Inst.Index) !void { const bin_op = self.air.instructions.items(.data)[inst].bin_op; - const result: MCValue = if (self.liveness.isUnused(inst)) - .dead - else - return self.fail("TODO implement shr for {}", .{self.target.cpu.arch}); - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); + + if (self.liveness.isUnused(inst)) { + return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none }); + } + + const ty = self.air.typeOfIndex(inst); + const tag = self.air.instructions.items(.tag)[inst]; + switch (tag) { + .shr_exact => return self.fail("TODO implement shr_exact for type {}", .{ty.fmtDebug()}), + .shr => {}, + else => unreachable, + } + + if (ty.zigTypeTag() != .Int) { + return self.fail("TODO implement shr for type {}", .{ty.fmtDebug()}); + } + if (ty.abiSize(self.target.*) > 8) { + return self.fail("TODO implement shr for integers larger than 8 bytes", .{}); + } + + // TODO look into reusing the operands + // TODO audit register allocation mechanics + const shift = try self.resolveInst(bin_op.rhs); + const shift_ty = self.air.typeOf(bin_op.rhs); + + blk: { + switch (shift) { + .register => |reg| { + if (reg.to64() == .rcx) break :blk; + }, + else => {}, + } + try self.register_manager.getReg(.rcx, null); + try self.genSetReg(shift_ty, .rcx, shift); + } + const rcx_lock = self.register_manager.lockRegAssumeUnused(.rcx); + defer self.register_manager.unlockReg(rcx_lock); + + const value = try self.resolveInst(bin_op.lhs); + const value_lock: ?RegisterLock = switch (value) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (value_lock) |lock| self.register_manager.unlockReg(lock); + + const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ty, value); + switch (ty.intInfo(self.target.*).signedness) { + .signed => { + _ = try self.addInst(.{ + .tag = .sar, + .ops = (Mir.Ops{ + .reg1 = dst_mcv.register, + .flags = 0b01, + }).encode(), + .data = undefined, + }); + }, + .unsigned => { + _ = try self.addInst(.{ + .tag = .shr, + .ops = (Mir.Ops{ + .reg1 = dst_mcv.register, + .flags = 0b01, + }).encode(), + .data = undefined, + }); + }, + } + + return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none }); } fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void { diff --git a/test/behavior/math.zig b/test/behavior/math.zig index 42f2635afd..dd609c9b08 100644 --- a/test/behavior/math.zig +++ b/test/behavior/math.zig @@ -570,8 +570,6 @@ test "bit shift a u1" { } test "truncating shift right" { - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO - try testShrTrunc(maxInt(u16)); comptime try testShrTrunc(maxInt(u16)); } From 20e7f1218b997e3da5d10cb5d038ed782d772716 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Mon, 9 May 2022 22:31:36 +0200 Subject: [PATCH 02/18] x64: make one entry point for binary ops * rename `genBinMathOp` into `genBinOp` and handle commutativity * rename `genBinMathOpMir` into `genBinOpMir` --- src/arch/x86_64/CodeGen.zig | 323 +++++++++++++----------------------- 1 file changed, 113 insertions(+), 210 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 28de88ff86..8dbb6aef7a 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -586,11 +586,11 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { switch (air_tags[inst]) { // zig fmt: off - .add => try self.airAdd(inst), - .addwrap => try self.airAdd(inst), + .add => try self.airBinOp(inst), + .addwrap => try self.airBinOp(inst), .add_sat => try self.airAddSat(inst), - .sub => try self.airSub(inst), - .subwrap => try self.airSub(inst), + .sub => try self.airBinOp(inst), + .subwrap => try self.airBinOp(inst), .sub_sat => try self.airSubSat(inst), .mul => try self.airMul(inst), .mulwrap => try self.airMul(inst), @@ -601,8 +601,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { .shl_sat => try self.airShlSat(inst), .min => try self.airMin(inst), .max => try self.airMax(inst), - .ptr_add => try self.airPtrAdd(inst), - .ptr_sub => try self.airPtrSub(inst), + .ptr_add => try self.airBinOp(inst), + .ptr_sub => try self.airBinOp(inst), .slice => try self.airSlice(inst), .sqrt, @@ -638,11 +638,11 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { .cmp_vector => try self.airCmpVector(inst), .cmp_lt_errors_len => try self.airCmpLtErrorsLen(inst), - .bool_and => try self.airBoolOp(inst), - .bool_or => try self.airBoolOp(inst), - .bit_and => try self.airBitAnd(inst), - .bit_or => try self.airBitOr(inst), - .xor => try self.airXor(inst), + .bool_and => try self.airBinOp(inst), + .bool_or => try self.airBinOp(inst), + .bit_and => try self.airBinOp(inst), + .bit_or => try self.airBinOp(inst), + .xor => try self.airBinOp(inst), .shr, .shr_exact => try self.airShr(inst), .alloc => try self.airAlloc(inst), @@ -1122,7 +1122,7 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void { }, else => {}, } - break :result try self.genBinMathOp(inst, ty_op.operand, .bool_true); + break :result try self.genBinOp(inst, ty_op.operand, .bool_true); }; return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); } @@ -1159,7 +1159,7 @@ fn airMin(self: *Self, inst: Air.Inst.Index) !void { }; defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); - try self.genBinMathOpMir(.cmp, ty, .{ .register = lhs_reg }, rhs_mcv); + try self.genBinOpMir(.cmp, ty, .{ .register = lhs_reg }, rhs_mcv); const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ty, rhs_mcv); _ = try self.addInst(.{ @@ -1185,63 +1185,12 @@ fn airMax(self: *Self, inst: Air.Inst.Index) !void { return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); } -fn genPtrBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air.Inst.Ref) !MCValue { - const dst_ty = self.air.typeOfIndex(inst); - const elem_size = dst_ty.elemType2().abiSize(self.target.*); - const ptr = try self.resolveInst(op_lhs); - const offset = try self.resolveInst(op_rhs); - const offset_ty = self.air.typeOf(op_rhs); - - const offset_lock: ?RegisterLock = switch (offset) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (offset_lock) |lock| self.register_manager.unlockReg(lock); - - const dst_mcv: MCValue = blk: { - if (self.reuseOperand(inst, op_lhs, 0, ptr)) { - if (ptr.isMemory() or ptr.isRegister()) break :blk ptr; - } - break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, ptr) }; - }; - - const dst_mcv_lock: ?RegisterLock = switch (dst_mcv) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (dst_mcv_lock) |lock| self.register_manager.unlockReg(lock); - - const offset_mcv: MCValue = blk: { - if (self.reuseOperand(inst, op_rhs, 1, offset)) { - if (offset.isRegister()) break :blk offset; - } - break :blk MCValue{ .register = try self.copyToTmpRegister(offset_ty, offset) }; - }; - - const offset_mcv_lock: ?RegisterLock = switch (offset_mcv) { - .register => |reg| self.register_manager.lockReg(reg), - else => null, - }; - defer if (offset_mcv_lock) |lock| self.register_manager.unlockReg(lock); - - try self.genIntMulComplexOpMir(offset_ty, offset_mcv, .{ .immediate = elem_size }); - - const tag = self.air.instructions.items(.tag)[inst]; - switch (tag) { - .ptr_add => try self.genBinMathOpMir(.add, dst_ty, dst_mcv, offset_mcv), - .ptr_sub => try self.genBinMathOpMir(.sub, dst_ty, dst_mcv, offset_mcv), - else => unreachable, - } - - return dst_mcv; -} - fn airPtrAdd(self: *Self, inst: Air.Inst.Index) !void { const bin_op = self.air.instructions.items(.data)[inst].bin_op; const result = if (self.liveness.isUnused(inst)) .dead else - try self.genPtrBinMathOp(inst, bin_op.lhs, bin_op.rhs); + try self.genBinOp(inst, bin_op.lhs, bin_op.rhs); return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); } @@ -1250,7 +1199,7 @@ fn airPtrSub(self: *Self, inst: Air.Inst.Index) !void { const result = if (self.liveness.isUnused(inst)) .dead else - try self.genPtrBinMathOp(inst, bin_op.lhs, bin_op.rhs); + try self.genBinOp(inst, bin_op.lhs, bin_op.rhs); return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); } @@ -1275,21 +1224,12 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void { return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); } -fn airAdd(self: *Self, inst: Air.Inst.Index) !void { +fn airBinOp(self: *Self, inst: Air.Inst.Index) !void { const bin_op = self.air.instructions.items(.data)[inst].bin_op; const result: MCValue = if (self.liveness.isUnused(inst)) .dead else - try self.genBinMathOp(inst, bin_op.lhs, bin_op.rhs); - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airAddWrap(self: *Self, inst: Air.Inst.Index) !void { - const bin_op = self.air.instructions.items(.data)[inst].bin_op; - const result: MCValue = if (self.liveness.isUnused(inst)) - .dead - else - try self.genBinMathOp(inst, bin_op.lhs, bin_op.rhs); + try self.genBinOp(inst, bin_op.lhs, bin_op.rhs); return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); } @@ -1302,60 +1242,6 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void { return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); } -/// Result is always a register. -fn genSubOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air.Inst.Ref) !MCValue { - const dst_ty = self.air.typeOf(op_lhs); - - const lhs = try self.resolveInst(op_lhs); - const lhs_lock: ?RegisterLock = switch (lhs) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock); - - const rhs = try self.resolveInst(op_rhs); - const rhs_lock: ?RegisterLock = switch (rhs) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); - - const dst_mcv: MCValue = blk: { - if (self.reuseOperand(inst, op_lhs, 0, lhs) and lhs.isRegister()) { - break :blk lhs; - } - break :blk try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs); - }; - const dst_mcv_lock: ?RegisterLock = switch (dst_mcv) { - .register => |reg| self.register_manager.lockReg(reg), - else => null, - }; - defer if (dst_mcv_lock) |lock| self.register_manager.unlockReg(lock); - - const rhs_mcv: MCValue = blk: { - if (rhs.isMemory() or rhs.isRegister()) break :blk rhs; - break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, rhs) }; - }; - const rhs_mcv_lock: ?RegisterLock = switch (rhs_mcv) { - .register => |reg| self.register_manager.lockReg(reg), - else => null, - }; - defer if (rhs_mcv_lock) |lock| self.register_manager.unlockReg(lock); - - try self.genBinMathOpMir(.sub, dst_ty, dst_mcv, rhs_mcv); - - return dst_mcv; -} - -fn airSub(self: *Self, inst: Air.Inst.Index) !void { - const bin_op = self.air.instructions.items(.data)[inst].bin_op; - const result: MCValue = if (self.liveness.isUnused(inst)) - .dead - else - try self.genSubOp(inst, bin_op.lhs, bin_op.rhs); - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - fn airSubSat(self: *Self, inst: Air.Inst.Index) !void { const bin_op = self.air.instructions.items(.data)[inst].bin_op; const result: MCValue = if (self.liveness.isUnused(inst)) @@ -1422,7 +1308,7 @@ fn airAddWithOverflow(self: *Self, inst: Air.Inst.Index) !void { try self.spillCompareFlagsIfOccupied(); self.compare_flags_inst = inst; - const partial = try self.genBinMathOp(inst, bin_op.lhs, bin_op.rhs); + const partial = try self.genBinOp(inst, bin_op.lhs, bin_op.rhs); const result: MCValue = switch (int_info.signedness) { .signed => .{ .register_overflow_signed = partial.register }, .unsigned => .{ .register_overflow_unsigned = partial.register }, @@ -1454,7 +1340,7 @@ fn airSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void { try self.spillCompareFlagsIfOccupied(); self.compare_flags_inst = inst; - const partial = try self.genSubOp(inst, bin_op.lhs, bin_op.rhs); + const partial = try self.genBinOp(inst, bin_op.lhs, bin_op.rhs); const result: MCValue = switch (int_info.signedness) { .signed => .{ .register_overflow_signed = partial.register }, .unsigned => .{ .register_overflow_unsigned = partial.register }, @@ -1605,7 +1491,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { const scratch_reg = temp_regs[1]; try self.genSetReg(extended_ty, scratch_reg, .{ .register = dst_reg }); try self.truncateRegister(ty, scratch_reg); - try self.genBinMathOpMir( + try self.genBinOpMir( .cmp, extended_ty, .{ .register = dst_reg }, @@ -1622,7 +1508,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { .data = undefined, }); - try self.genBinMathOpMir( + try self.genBinOpMir( .@"or", Type.u8, .{ .register = overflow_reg }, @@ -1781,7 +1667,7 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa }).encode(), .data = undefined, }); - try self.genBinMathOpMir(.add, Type.isize, .{ .register = divisor }, .{ .register = .rax }); + try self.genBinOpMir(.add, Type.isize, .{ .register = divisor }, .{ .register = .rax }); return MCValue{ .register = divisor }; } @@ -1945,7 +1831,7 @@ fn airMod(self: *Self, inst: Air.Inst.Index) !void { try self.genIntMulComplexOpMir(ty, div_floor, rhs); const result = try self.copyToRegisterWithInstTracking(inst, ty, lhs); - try self.genBinMathOpMir(.sub, ty, result, div_floor); + try self.genBinOpMir(.sub, ty, result, div_floor); break :result result; }, @@ -1955,33 +1841,6 @@ fn airMod(self: *Self, inst: Air.Inst.Index) !void { return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); } -fn airBitAnd(self: *Self, inst: Air.Inst.Index) !void { - const bin_op = self.air.instructions.items(.data)[inst].bin_op; - const result: MCValue = if (self.liveness.isUnused(inst)) - .dead - else - try self.genBinMathOp(inst, bin_op.lhs, bin_op.rhs); - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airBitOr(self: *Self, inst: Air.Inst.Index) !void { - const bin_op = self.air.instructions.items(.data)[inst].bin_op; - const result: MCValue = if (self.liveness.isUnused(inst)) - .dead - else - try self.genBinMathOp(inst, bin_op.lhs, bin_op.rhs); - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airXor(self: *Self, inst: Air.Inst.Index) !void { - const bin_op = self.air.instructions.items(.data)[inst].bin_op; - const result: MCValue = if (self.liveness.isUnused(inst)) - .dead - else - try self.genBinMathOp(inst, bin_op.lhs, bin_op.rhs); - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - fn airShl(self: *Self, inst: Air.Inst.Index) !void { const bin_op = self.air.instructions.items(.data)[inst].bin_op; if (self.liveness.isUnused(inst)) { @@ -1989,6 +1848,7 @@ fn airShl(self: *Self, inst: Air.Inst.Index) !void { } const ty = self.air.typeOfIndex(inst); + const tag = self.air.instructions.items(.tag)[inst]; switch (tag) { .shl_exact => return self.fail("TODO implement {} for type {}", .{ tag, ty.fmtDebug() }), @@ -2470,7 +2330,7 @@ fn genSliceElemPtr(self: *Self, lhs: Air.Inst.Ref, rhs: Air.Inst.Ref) !MCValue { } // TODO we could allocate register here, but need to expect addr register and potentially // offset register. - try self.genBinMathOpMir(.add, slice_ptr_field_type, .{ .register = addr_reg }, .{ + try self.genBinOpMir(.add, slice_ptr_field_type, .{ .register = addr_reg }, .{ .register = offset_reg, }); return MCValue{ .register = addr_reg.to64() }; @@ -2573,7 +2433,7 @@ fn airArrayElemVal(self: *Self, inst: Air.Inst.Index) !void { // TODO we could allocate register here, but need to expect addr register and potentially // offset register. const dst_mcv = try self.allocRegOrMem(inst, false); - try self.genBinMathOpMir(.add, Type.usize, .{ .register = addr_reg }, .{ .register = offset_reg }); + try self.genBinOpMir(.add, Type.usize, .{ .register = addr_reg }, .{ .register = offset_reg }); try self.load(dst_mcv, .{ .register = addr_reg.to64() }, array_ty); return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none }); @@ -2613,7 +2473,7 @@ fn airPtrElemVal(self: *Self, inst: Air.Inst.Index) !void { defer self.register_manager.unlockReg(offset_reg_lock); const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr); - try self.genBinMathOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg }); + try self.genBinOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg }); const result: MCValue = result: { if (elem_abi_size > 8) { @@ -2667,7 +2527,7 @@ fn airPtrElemPtr(self: *Self, inst: Air.Inst.Index) !void { defer self.register_manager.unlockReg(offset_reg_lock); const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr); - try self.genBinMathOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg }); + try self.genBinOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg }); return self.finishAir(inst, dst_mcv, .{ extra.lhs, extra.rhs, .none }); } @@ -2700,7 +2560,7 @@ fn airSetUnionTag(self: *Self, inst: Air.Inst.Index) !void { const adjusted_ptr: MCValue = if (layout.payload_size > 0 and layout.tag_align < layout.payload_align) blk: { // TODO reusing the operand const reg = try self.copyToTmpRegister(ptr_ty, ptr); - try self.genBinMathOpMir(.add, ptr_ty, .{ .register = reg }, .{ .immediate = layout.payload_size }); + try self.genBinOpMir(.add, ptr_ty, .{ .register = reg }, .{ .immediate = layout.payload_size }); break :blk MCValue{ .register = reg }; } else ptr; @@ -3267,7 +3127,7 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde defer self.register_manager.unlockReg(offset_reg_lock); const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ptr_ty, mcv); - try self.genBinMathOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg }); + try self.genBinOpMir(.add, ptr_ty, dst_mcv, .{ .register = offset_reg }); break :result dst_mcv; }, .ptr_stack_offset => |off| { @@ -3297,7 +3157,7 @@ fn structFieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, inde const result_reg_lock = self.register_manager.lockReg(result_reg); defer if (result_reg_lock) |lock| self.register_manager.unlockReg(lock); - try self.genBinMathOpMir(.add, ptr_ty, .{ .register = result_reg }, .{ .register = offset_reg }); + try self.genBinOpMir(.add, ptr_ty, .{ .register = result_reg }, .{ .register = offset_reg }); break :result MCValue{ .register = result_reg }; }, else => return self.fail("TODO implement codegen struct_field_ptr for {}", .{mcv}), @@ -3357,7 +3217,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { const mask = (~@as(u64, 0)) >> mask_shift; const tmp_reg = try self.copyToTmpRegister(Type.usize, .{ .immediate = mask }); - try self.genBinMathOpMir(.@"and", Type.usize, dst_mcv, .{ .register = tmp_reg }); + try self.genBinOpMir(.@"and", Type.usize, dst_mcv, .{ .register = tmp_reg }); const signedness: std.builtin.Signedness = blk: { if (struct_field_ty.zigTypeTag() != .Int) break :blk .unsigned; @@ -3426,8 +3286,39 @@ fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void { } /// Result is always a register. -fn genBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air.Inst.Ref) !MCValue { +fn genBinOp( + self: *Self, + inst: Air.Inst.Index, + op_lhs: Air.Inst.Ref, + op_rhs: Air.Inst.Ref, +) !MCValue { + const tag = self.air.instructions.items(.tag)[inst]; + const is_commutative: bool = switch (tag) { + .add, + .addwrap, + .add_with_overflow, + .bool_or, + .bit_or, + .bool_and, + .bit_and, + .xor, + .not, + => true, + + .sub, + .subwrap, + .sub_with_overflow, + .mul, + .shl, + .shr, + .ptr_add, + .ptr_sub, + => false, + + else => unreachable, + }; const dst_ty = self.air.typeOf(op_lhs); + const src_ty = self.air.typeOf(op_rhs); const lhs = try self.resolveInst(op_lhs); const lhs_lock: ?RegisterLock = switch (lhs) { @@ -3448,7 +3339,7 @@ fn genBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: if (self.reuseOperand(inst, op_lhs, 0, lhs) and lhs.isRegister()) { break :blk lhs; } - if (self.reuseOperand(inst, op_rhs, 1, rhs) and rhs.isRegister()) { + if (is_commutative and self.reuseOperand(inst, op_rhs, 1, rhs) and rhs.isRegister()) { flipped = true; break :blk rhs; } @@ -3463,7 +3354,7 @@ fn genBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: const src_mcv: MCValue = blk: { const mcv = if (flipped) lhs else rhs; if (mcv.isRegister() or mcv.isMemory()) break :blk mcv; - break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, mcv) }; + break :blk MCValue{ .register = try self.copyToTmpRegister(src_ty, mcv) }; }; const src_mcv_lock: ?RegisterLock = switch (src_mcv) { .register => |reg| self.register_manager.lockReg(reg), @@ -3471,18 +3362,39 @@ fn genBinMathOp(self: *Self, inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: }; defer if (src_mcv_lock) |lock| self.register_manager.unlockReg(lock); - const tag = self.air.instructions.items(.tag)[inst]; switch (tag) { - .add, .addwrap, .add_with_overflow => try self.genBinMathOpMir(.add, dst_ty, dst_mcv, src_mcv), - .bool_or, .bit_or => try self.genBinMathOpMir(.@"or", dst_ty, dst_mcv, src_mcv), - .bool_and, .bit_and => try self.genBinMathOpMir(.@"and", dst_ty, dst_mcv, src_mcv), - .xor, .not => try self.genBinMathOpMir(.xor, dst_ty, dst_mcv, src_mcv), + .add, + .addwrap, + .add_with_overflow, + => try self.genBinOpMir(.add, dst_ty, dst_mcv, src_mcv), + + .sub, + .subwrap, + .sub_with_overflow, + => try self.genBinOpMir(.sub, dst_ty, dst_mcv, src_mcv), + + .ptr_add, + .ptr_sub, + => { + const mir_tag: Mir.Inst.Tag = switch (tag) { + .ptr_add => .add, + .ptr_sub => .sub, + else => unreachable, + }; + const elem_size = dst_ty.elemType2().abiSize(self.target.*); + try self.genIntMulComplexOpMir(src_ty, src_mcv, .{ .immediate = elem_size }); + try self.genBinOpMir(mir_tag, dst_ty, dst_mcv, src_mcv); + }, + + .bool_or, .bit_or => try self.genBinOpMir(.@"or", dst_ty, dst_mcv, src_mcv), + .bool_and, .bit_and => try self.genBinOpMir(.@"and", dst_ty, dst_mcv, src_mcv), + .xor, .not => try self.genBinOpMir(.xor, dst_ty, dst_mcv, src_mcv), else => unreachable, } return dst_mcv; } -fn genBinMathOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void { +fn genBinOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void { const abi_size = @intCast(u32, dst_ty.abiSize(self.target.*)); switch (dst_mcv) { .none => unreachable, @@ -3504,7 +3416,7 @@ fn genBinMathOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MC defer if (dst_reg_lock) |lock| self.register_manager.unlockReg(lock); const reg = try self.copyToTmpRegister(dst_ty, src_mcv); - return self.genBinMathOpMir(mir_tag, dst_ty, dst_mcv, .{ .register = reg }); + return self.genBinOpMir(mir_tag, dst_ty, dst_mcv, .{ .register = reg }); }, .register => |src_reg| { _ = try self.addInst(.{ @@ -3536,7 +3448,7 @@ fn genBinMathOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MC defer if (dst_reg_lock) |lock| self.register_manager.unlockReg(lock); const reg = try self.copyToTmpRegister(dst_ty, src_mcv); - return self.genBinMathOpMir(mir_tag, dst_ty, dst_mcv, .{ .register = reg }); + return self.genBinOpMir(mir_tag, dst_ty, dst_mcv, .{ .register = reg }); }, .stack_offset => |off| { if (off > math.maxInt(i32)) { @@ -3637,7 +3549,7 @@ fn genBinMathOpMir(self: *Self, mir_tag: Mir.Inst.Tag, dst_ty: Type, dst_mcv: MC /// Performs multi-operand integer multiplication between dst_mcv and src_mcv, storing the result in dst_mcv. /// Does not support byte-size operands. -fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) !void { +fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: MCValue) InnerError!void { const abi_size = @intCast(u32, dst_ty.abiSize(self.target.*)); switch (dst_mcv) { .none => unreachable, @@ -3733,11 +3645,17 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M .data = undefined, }); // copy dst_reg back out - return self.genSetStack(dst_ty, off, MCValue{ .register = dst_reg }, .{}); + return self.genSetStack(dst_ty, off, .{ .register = dst_reg }, .{}); }, - .immediate => |imm| { - _ = imm; - return self.fail("TODO implement x86 multiply source immediate", .{}); + .immediate => { + // copy dst to a register + const dst_reg = try self.copyToTmpRegister(dst_ty, dst_mcv); + const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg); + defer self.register_manager.unlockReg(dst_reg_lock); + + try self.genIntMulComplexOpMir(dst_ty, .{ .register = dst_reg }, src_mcv); + + return self.genSetStack(dst_ty, off, .{ .register = dst_reg }, .{}); }, .memory, .stack_offset => { return self.fail("TODO implement x86 multiply source memory", .{}); @@ -4213,7 +4131,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void { // This instruction supports only signed 32-bit immediates at most. const src_mcv = try self.limitImmediateType(bin_op.rhs, i32); - try self.genBinMathOpMir(.cmp, ty, dst_mcv, src_mcv); + try self.genBinOpMir(.cmp, ty, dst_mcv, src_mcv); break :result switch (signedness) { .signed => MCValue{ .compare_flags_signed = op }, .unsigned => MCValue{ .compare_flags_unsigned = op }, @@ -4606,7 +4524,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, ty: Type, operand: MCValue) !MCValu break :blk if (payload_ty.hasRuntimeBits()) Type.bool else ty; } else ty; - try self.genBinMathOpMir(.cmp, cmp_ty, operand, MCValue{ .immediate = 0 }); + try self.genBinOpMir(.cmp, cmp_ty, operand, MCValue{ .immediate = 0 }); return MCValue{ .compare_flags_unsigned = .eq }; } @@ -4629,13 +4547,13 @@ fn isErr(self: *Self, inst: Air.Inst.Index, ty: Type, operand: MCValue) !MCValue if (!payload_type.hasRuntimeBits()) { if (err_type.abiSize(self.target.*) <= 8) { - try self.genBinMathOpMir(.cmp, err_type, operand, MCValue{ .immediate = 0 }); + try self.genBinOpMir(.cmp, err_type, operand, MCValue{ .immediate = 0 }); return MCValue{ .compare_flags_unsigned = .gt }; } else { return self.fail("TODO isErr for errors with size larger than register size", .{}); } } else { - try self.genBinMathOpMir(.cmp, err_type, operand, MCValue{ .immediate = 0 }); + try self.genBinOpMir(.cmp, err_type, operand, MCValue{ .immediate = 0 }); return MCValue{ .compare_flags_unsigned = .gt }; } } @@ -5053,21 +4971,6 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void { return self.finishAir(inst, .dead, .{ branch.operand, .none, .none }); } -fn airBoolOp(self: *Self, inst: Air.Inst.Index) !void { - const bin_op = self.air.instructions.items(.data)[inst].bin_op; - const air_tags = self.air.instructions.items(.tag); - const result: MCValue = if (self.liveness.isUnused(inst)) - .dead - else switch (air_tags[inst]) { - // lhs AND rhs - .bool_and => try self.genBinMathOp(inst, bin_op.lhs, bin_op.rhs), - // lhs OR rhs - .bool_or => try self.genBinMathOp(inst, bin_op.lhs, bin_op.rhs), - else => unreachable, // Not a boolean operation - }; - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - fn br(self: *Self, block: Air.Inst.Index, operand: Air.Inst.Ref) !void { const block_data = self.blocks.getPtr(block).?; @@ -5844,7 +5747,7 @@ fn genInlineMemset( defer self.register_manager.unlockReg(addr_reg_lock); try self.genSetReg(Type.usize, .rax, len); - try self.genBinMathOpMir(.sub, Type.usize, .{ .register = .rax }, .{ .immediate = 1 }); + try self.genBinOpMir(.sub, Type.usize, .{ .register = .rax }, .{ .immediate = 1 }); // loop: // cmp rax, -1 @@ -6963,10 +6866,10 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void { const shift = @intCast(u6, max_reg_bit_width - int_info.bits); const mask = (~@as(u64, 0)) >> shift; if (int_info.bits <= 32) { - try self.genBinMathOpMir(.@"and", Type.usize, .{ .register = reg }, .{ .immediate = mask }); + try self.genBinOpMir(.@"and", Type.usize, .{ .register = reg }, .{ .immediate = mask }); } else { const tmp_reg = try self.copyToTmpRegister(Type.usize, .{ .immediate = mask }); - try self.genBinMathOpMir(.@"and", Type.usize, .{ .register = reg }, .{ .register = tmp_reg }); + try self.genBinOpMir(.@"and", Type.usize, .{ .register = reg }, .{ .register = tmp_reg }); } }, } From 7b9f8bfbd80aa37afc160c39b341c59fdd3cbad8 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Mon, 9 May 2022 23:50:01 +0200 Subject: [PATCH 03/18] x64: migrate mul to new genBinOp helper --- src/arch/x86_64/CodeGen.zig | 151 ++++++++++++++---------------------- 1 file changed, 59 insertions(+), 92 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 8dbb6aef7a..726476f6c7 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -592,8 +592,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { .sub => try self.airBinOp(inst), .subwrap => try self.airBinOp(inst), .sub_sat => try self.airSubSat(inst), - .mul => try self.airMul(inst), - .mulwrap => try self.airMul(inst), + .mul => try self.airBinOp(inst), + .mulwrap => try self.airBinOp(inst), .mul_sat => try self.airMulSat(inst), .rem => try self.airRem(inst), .mod => try self.airMod(inst), @@ -1122,7 +1122,7 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void { }, else => {}, } - break :result try self.genBinOp(inst, ty_op.operand, .bool_true); + break :result try self.genBinOp(inst, ty_op.operand, .bool_true, true); }; return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); } @@ -1185,24 +1185,6 @@ fn airMax(self: *Self, inst: Air.Inst.Index) !void { return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); } -fn airPtrAdd(self: *Self, inst: Air.Inst.Index) !void { - const bin_op = self.air.instructions.items(.data)[inst].bin_op; - const result = if (self.liveness.isUnused(inst)) - .dead - else - try self.genBinOp(inst, bin_op.lhs, bin_op.rhs); - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airPtrSub(self: *Self, inst: Air.Inst.Index) !void { - const bin_op = self.air.instructions.items(.data)[inst].bin_op; - const result = if (self.liveness.isUnused(inst)) - .dead - else - try self.genBinOp(inst, bin_op.lhs, bin_op.rhs); - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - fn airSlice(self: *Self, inst: Air.Inst.Index) !void { const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; @@ -1229,7 +1211,7 @@ fn airBinOp(self: *Self, inst: Air.Inst.Index) !void { const result: MCValue = if (self.liveness.isUnused(inst)) .dead else - try self.genBinOp(inst, bin_op.lhs, bin_op.rhs); + try self.genBinOp(inst, bin_op.lhs, bin_op.rhs, true); return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); } @@ -1251,36 +1233,6 @@ fn airSubSat(self: *Self, inst: Air.Inst.Index) !void { return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); } -fn airMul(self: *Self, inst: Air.Inst.Index) !void { - const bin_op = self.air.instructions.items(.data)[inst].bin_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { - const ty = self.air.typeOfIndex(inst); - - if (ty.zigTypeTag() != .Int) { - return self.fail("TODO implement 'mul' for operands of dst type {}", .{ty.zigTypeTag()}); - } - - // Spill .rax and .rdx upfront to ensure we don't spill the operands too late. - try self.register_manager.getReg(.rax, inst); - try self.register_manager.getReg(.rdx, null); - const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx }); - defer for (reg_locks) |reg| { - self.register_manager.unlockReg(reg); - }; - - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - - const signedness = ty.intInfo(self.target.*).signedness; - try self.genIntMulDivOpMir(switch (signedness) { - .signed => .imul, - .unsigned => .mul, - }, ty, signedness, lhs, rhs); - break :result MCValue{ .register = .rax }; - }; - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - fn airMulSat(self: *Self, inst: Air.Inst.Index) !void { const bin_op = self.air.instructions.items(.data)[inst].bin_op; const result: MCValue = if (self.liveness.isUnused(inst)) @@ -1308,7 +1260,7 @@ fn airAddWithOverflow(self: *Self, inst: Air.Inst.Index) !void { try self.spillCompareFlagsIfOccupied(); self.compare_flags_inst = inst; - const partial = try self.genBinOp(inst, bin_op.lhs, bin_op.rhs); + const partial = try self.genBinOp(inst, bin_op.lhs, bin_op.rhs, true); const result: MCValue = switch (int_info.signedness) { .signed => .{ .register_overflow_signed = partial.register }, .unsigned => .{ .register_overflow_unsigned = partial.register }, @@ -1340,7 +1292,7 @@ fn airSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void { try self.spillCompareFlagsIfOccupied(); self.compare_flags_inst = inst; - const partial = try self.genBinOp(inst, bin_op.lhs, bin_op.rhs); + const partial = try self.genBinOp(inst, bin_op.lhs, bin_op.rhs, true); const result: MCValue = switch (int_info.signedness) { .signed => .{ .register_overflow_signed = partial.register }, .unsigned => .{ .register_overflow_unsigned = partial.register }, @@ -1373,34 +1325,17 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { return self.fail("TODO implement mul_with_overflow for Ints larger than 64bits", .{}); } + try self.spillCompareFlagsIfOccupied(); + if (math.isPowerOfTwo(int_info.bits)) { - try self.spillCompareFlagsIfOccupied(); self.compare_flags_inst = inst; - - // Spill .rax and .rdx upfront to ensure we don't spill the operands too late. - try self.register_manager.getReg(.rax, inst); - try self.register_manager.getReg(.rdx, null); - const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx }); - defer for (reg_locks) |reg| { - self.register_manager.unlockReg(reg); + const partial = try self.genBinOp(inst, bin_op.lhs, bin_op.rhs, true); + break :result switch (int_info.signedness) { + .signed => MCValue{ .register_overflow_signed = partial.register }, + .unsigned => MCValue{ .register_overflow_unsigned = partial.register }, }; - - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - - try self.genIntMulDivOpMir(switch (int_info.signedness) { - .signed => .imul, - .unsigned => .mul, - }, ty, int_info.signedness, lhs, rhs); - - const result: MCValue = switch (int_info.signedness) { - .signed => .{ .register_overflow_signed = .rax }, - .unsigned => .{ .register_overflow_unsigned = .rax }, - }; - break :result result; } - try self.spillCompareFlagsIfOccupied(); self.compare_flags_inst = null; const dst_reg: Register = dst_reg: { @@ -1437,20 +1372,8 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { break :dst_reg dst_reg; }, .unsigned => { - // Spill .rax and .rdx upfront to ensure we don't spill the operands too late. - try self.register_manager.getReg(.rax, null); - try self.register_manager.getReg(.rdx, null); - const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx }); - defer for (reg_locks) |reg| { - self.register_manager.unlockReg(reg); - }; - - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - - try self.genIntMulDivOpMir(.mul, ty, .unsigned, lhs, rhs); - - break :dst_reg registerAlias(.rax, @intCast(u32, ty.abiSize(self.target.*))); + const dst_mcv = try self.genBinOp(inst, bin_op.lhs, bin_op.rhs, false); + break :dst_reg dst_mcv.register; }, } }; @@ -3291,6 +3214,7 @@ fn genBinOp( inst: Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air.Inst.Ref, + track: bool, ) !MCValue { const tag = self.air.instructions.items(.tag)[inst]; const is_commutative: bool = switch (tag) { @@ -3309,6 +3233,8 @@ fn genBinOp( .subwrap, .sub_with_overflow, .mul, + .mulwrap, + .mul_with_overflow, .shl, .shr, .ptr_add, @@ -3320,6 +3246,43 @@ fn genBinOp( const dst_ty = self.air.typeOf(op_lhs); const src_ty = self.air.typeOf(op_rhs); + if (dst_ty.zigTypeTag() == .Vector or dst_ty.zigTypeTag() == .Float) { + return self.fail("TODO implement genBinOp for {}", .{dst_ty.fmtDebug()}); + } + if (dst_ty.abiSize(self.target.*) > 8) { + return self.fail("TODO implement genBinOp for {}", .{dst_ty.fmtDebug()}); + } + + switch (tag) { + .mul, + .mulwrap, + .mul_with_overflow, + => { + // Spill .rax and .rdx upfront to ensure we don't spill the operands too late. + try self.register_manager.getReg(.rax, if (track) inst else null); + try self.register_manager.getReg(.rdx, null); + const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx }); + defer for (reg_locks) |reg| { + self.register_manager.unlockReg(reg); + }; + + const lhs = try self.resolveInst(op_lhs); + const rhs = try self.resolveInst(op_rhs); + + const int_info = dst_ty.intInfo(self.target.*); + try self.genIntMulDivOpMir(switch (int_info.signedness) { + .signed => .imul, + .unsigned => .mul, + }, dst_ty, int_info.signedness, lhs, rhs); + + return switch (int_info.signedness) { + .signed => MCValue{ .register = .rax }, + .unsigned => MCValue{ .register = registerAlias(.rax, @intCast(u32, dst_ty.abiSize(self.target.*))) }, + }; + }, + else => {}, + } + const lhs = try self.resolveInst(op_lhs); const lhs_lock: ?RegisterLock = switch (lhs) { .register => |reg| self.register_manager.lockRegAssumeUnused(reg), @@ -3343,7 +3306,11 @@ fn genBinOp( flipped = true; break :blk rhs; } - break :blk try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs); + if (track) { + break :blk try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs); + } else { + break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, lhs) }; + } }; const dst_mcv_lock: ?RegisterLock = switch (dst_mcv) { .register => |reg| self.register_manager.lockReg(reg), From c3b7a5cc26d11a0349ff1d7812b00687cfa41c2e Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Mon, 9 May 2022 23:58:46 +0200 Subject: [PATCH 04/18] x64: pass tag and maybe_inst explictly to genBinOp --- src/arch/x86_64/CodeGen.zig | 68 ++++++++++++++++++++----------------- 1 file changed, 36 insertions(+), 32 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 726476f6c7..94f1195a3c 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -1122,7 +1122,7 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void { }, else => {}, } - break :result try self.genBinOp(inst, ty_op.operand, .bool_true, true); + break :result try self.genBinOp(.not, inst, ty_op.operand, .bool_true); }; return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); } @@ -1208,10 +1208,13 @@ fn airSlice(self: *Self, inst: Air.Inst.Index) !void { fn airBinOp(self: *Self, inst: Air.Inst.Index) !void { const bin_op = self.air.instructions.items(.data)[inst].bin_op; - const result: MCValue = if (self.liveness.isUnused(inst)) - .dead - else - try self.genBinOp(inst, bin_op.lhs, bin_op.rhs, true); + + if (self.liveness.isUnused(inst)) { + return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none }); + } + + const tag = self.air.instructions.items(.tag)[inst]; + const result = try self.genBinOp(tag, inst, bin_op.lhs, bin_op.rhs); return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); } @@ -1260,7 +1263,7 @@ fn airAddWithOverflow(self: *Self, inst: Air.Inst.Index) !void { try self.spillCompareFlagsIfOccupied(); self.compare_flags_inst = inst; - const partial = try self.genBinOp(inst, bin_op.lhs, bin_op.rhs, true); + const partial = try self.genBinOp(.add, inst, bin_op.lhs, bin_op.rhs); const result: MCValue = switch (int_info.signedness) { .signed => .{ .register_overflow_signed = partial.register }, .unsigned => .{ .register_overflow_unsigned = partial.register }, @@ -1292,7 +1295,7 @@ fn airSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void { try self.spillCompareFlagsIfOccupied(); self.compare_flags_inst = inst; - const partial = try self.genBinOp(inst, bin_op.lhs, bin_op.rhs, true); + const partial = try self.genBinOp(.sub, inst, bin_op.lhs, bin_op.rhs); const result: MCValue = switch (int_info.signedness) { .signed => .{ .register_overflow_signed = partial.register }, .unsigned => .{ .register_overflow_unsigned = partial.register }, @@ -1329,7 +1332,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { if (math.isPowerOfTwo(int_info.bits)) { self.compare_flags_inst = inst; - const partial = try self.genBinOp(inst, bin_op.lhs, bin_op.rhs, true); + const partial = try self.genBinOp(.mul, inst, bin_op.lhs, bin_op.rhs); break :result switch (int_info.signedness) { .signed => MCValue{ .register_overflow_signed = partial.register }, .unsigned => MCValue{ .register_overflow_unsigned = partial.register }, @@ -1372,7 +1375,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { break :dst_reg dst_reg; }, .unsigned => { - const dst_mcv = try self.genBinOp(inst, bin_op.lhs, bin_op.rhs, false); + const dst_mcv = try self.genBinOp(.mul, null, bin_op.lhs, bin_op.rhs); break :dst_reg dst_mcv.register; }, } @@ -3211,16 +3214,14 @@ fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void { /// Result is always a register. fn genBinOp( self: *Self, - inst: Air.Inst.Index, + tag: Air.Inst.Tag, + maybe_inst: ?Air.Inst.Index, op_lhs: Air.Inst.Ref, op_rhs: Air.Inst.Ref, - track: bool, ) !MCValue { - const tag = self.air.instructions.items(.tag)[inst]; const is_commutative: bool = switch (tag) { .add, .addwrap, - .add_with_overflow, .bool_or, .bit_or, .bool_and, @@ -3231,10 +3232,8 @@ fn genBinOp( .sub, .subwrap, - .sub_with_overflow, .mul, .mulwrap, - .mul_with_overflow, .shl, .shr, .ptr_add, @@ -3256,10 +3255,9 @@ fn genBinOp( switch (tag) { .mul, .mulwrap, - .mul_with_overflow, => { // Spill .rax and .rdx upfront to ensure we don't spill the operands too late. - try self.register_manager.getReg(.rax, if (track) inst else null); + try self.register_manager.getReg(.rax, maybe_inst); try self.register_manager.getReg(.rdx, null); const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx }); defer for (reg_locks) |reg| { @@ -3299,18 +3297,17 @@ fn genBinOp( var flipped: bool = false; const dst_mcv: MCValue = blk: { - if (self.reuseOperand(inst, op_lhs, 0, lhs) and lhs.isRegister()) { - break :blk lhs; - } - if (is_commutative and self.reuseOperand(inst, op_rhs, 1, rhs) and rhs.isRegister()) { - flipped = true; - break :blk rhs; - } - if (track) { + if (maybe_inst) |inst| { + if (self.reuseOperand(inst, op_lhs, 0, lhs) and lhs.isRegister()) { + break :blk lhs; + } + if (is_commutative and self.reuseOperand(inst, op_rhs, 1, rhs) and rhs.isRegister()) { + flipped = true; + break :blk rhs; + } break :blk try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs); - } else { - break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, lhs) }; } + break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, lhs) }; }; const dst_mcv_lock: ?RegisterLock = switch (dst_mcv) { .register => |reg| self.register_manager.lockReg(reg), @@ -3332,12 +3329,10 @@ fn genBinOp( switch (tag) { .add, .addwrap, - .add_with_overflow, => try self.genBinOpMir(.add, dst_ty, dst_mcv, src_mcv), .sub, .subwrap, - .sub_with_overflow, => try self.genBinOpMir(.sub, dst_ty, dst_mcv, src_mcv), .ptr_add, @@ -3353,9 +3348,18 @@ fn genBinOp( try self.genBinOpMir(mir_tag, dst_ty, dst_mcv, src_mcv); }, - .bool_or, .bit_or => try self.genBinOpMir(.@"or", dst_ty, dst_mcv, src_mcv), - .bool_and, .bit_and => try self.genBinOpMir(.@"and", dst_ty, dst_mcv, src_mcv), - .xor, .not => try self.genBinOpMir(.xor, dst_ty, dst_mcv, src_mcv), + .bool_or, + .bit_or, + => try self.genBinOpMir(.@"or", dst_ty, dst_mcv, src_mcv), + + .bool_and, + .bit_and, + => try self.genBinOpMir(.@"and", dst_ty, dst_mcv, src_mcv), + + .xor, + .not, + => try self.genBinOpMir(.xor, dst_ty, dst_mcv, src_mcv), + else => unreachable, } return dst_mcv; From 252c5a2339820d15bc7a063cdd68cc65b49ed413 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 10 May 2022 01:13:13 +0200 Subject: [PATCH 05/18] x64: migrate mod and rem into genBinOp --- src/arch/x86_64/CodeGen.zig | 142 +++++++++++++++--------------------- 1 file changed, 60 insertions(+), 82 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 94f1195a3c..65293e0f46 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -595,8 +595,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { .mul => try self.airBinOp(inst), .mulwrap => try self.airBinOp(inst), .mul_sat => try self.airMulSat(inst), - .rem => try self.airRem(inst), - .mod => try self.airMod(inst), + .rem => try self.airBinOp(inst), + .mod => try self.airBinOp(inst), .shl, .shl_exact => try self.airShl(inst), .shl_sat => try self.airShlSat(inst), .min => try self.airMin(inst), @@ -1539,6 +1539,7 @@ fn genIntMulDivOpMir( } } +/// Always returns a register. /// Clobbers .rax and .rdx registers. fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCValue { const signedness = ty.intInfo(self.target.*).signedness; @@ -1687,86 +1688,6 @@ fn airDiv(self: *Self, inst: Air.Inst.Index) !void { return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); } -fn airRem(self: *Self, inst: Air.Inst.Index) !void { - const bin_op = self.air.instructions.items(.data)[inst].bin_op; - - if (self.liveness.isUnused(inst)) { - return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none }); - } - const ty = self.air.typeOfIndex(inst); - if (ty.zigTypeTag() != .Int) { - return self.fail("TODO implement .rem for operands of dst type {}", .{ty.zigTypeTag()}); - } - // Spill .rax and .rdx upfront to ensure we don't spill the operands too late. - try self.register_manager.getReg(.rax, null); - try self.register_manager.getReg(.rdx, inst); - const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx }); - defer for (reg_locks) |reg| { - self.register_manager.unlockReg(reg); - }; - - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - - const signedness = ty.intInfo(self.target.*).signedness; - try self.genIntMulDivOpMir(switch (signedness) { - .signed => .idiv, - .unsigned => .div, - }, ty, signedness, lhs, rhs); - - const result: MCValue = .{ .register = .rdx }; - - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airMod(self: *Self, inst: Air.Inst.Index) !void { - const bin_op = self.air.instructions.items(.data)[inst].bin_op; - - if (self.liveness.isUnused(inst)) { - return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none }); - } - - const ty = self.air.typeOfIndex(inst); - if (ty.zigTypeTag() != .Int) { - return self.fail("TODO implement .mod for operands of dst type {}", .{ty.zigTypeTag()}); - } - const signedness = ty.intInfo(self.target.*).signedness; - - // Spill .rax and .rdx upfront to ensure we don't spill the operands too late. - try self.register_manager.getReg(.rax, null); - try self.register_manager.getReg(.rdx, if (signedness == .unsigned) inst else null); - const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx }); - defer for (reg_locks) |reg| { - self.register_manager.unlockReg(reg); - }; - - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - - const result: MCValue = result: { - switch (signedness) { - .unsigned => { - try self.genIntMulDivOpMir(switch (signedness) { - .signed => .idiv, - .unsigned => .div, - }, ty, signedness, lhs, rhs); - break :result MCValue{ .register = .rdx }; - }, - .signed => { - const div_floor = try self.genInlineIntDivFloor(ty, lhs, rhs); - try self.genIntMulComplexOpMir(ty, div_floor, rhs); - - const result = try self.copyToRegisterWithInstTracking(inst, ty, lhs); - try self.genBinOpMir(.sub, ty, result, div_floor); - - break :result result; - }, - } - }; - - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - fn airShl(self: *Self, inst: Air.Inst.Index) !void { const bin_op = self.air.instructions.items(.data)[inst].bin_op; if (self.liveness.isUnused(inst)) { @@ -3234,12 +3155,18 @@ fn genBinOp( .subwrap, .mul, .mulwrap, + .div_exact, + .div_trunc, + .rem, + .mod, .shl, .shr, .ptr_add, .ptr_sub, => false, + .div_float => return self.fail("TODO implement genBinOp for {}", .{tag}), + else => unreachable, }; const dst_ty = self.air.typeOf(op_lhs); @@ -3278,6 +3205,57 @@ fn genBinOp( .unsigned => MCValue{ .register = registerAlias(.rax, @intCast(u32, dst_ty.abiSize(self.target.*))) }, }; }, + .mod, + .rem, + => { + const int_info = dst_ty.intInfo(self.target.*); + const track_inst_rdx: ?Air.Inst.Index = switch (tag) { + .mod => if (int_info.signedness == .unsigned) maybe_inst else null, + .rem => maybe_inst, + else => unreachable, + }; + + // Spill .rax and .rdx upfront to ensure we don't spill the operands too late. + try self.register_manager.getReg(.rax, null); + try self.register_manager.getReg(.rdx, track_inst_rdx); + const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx }); + defer for (reg_locks) |reg| { + self.register_manager.unlockReg(reg); + }; + + const lhs = try self.resolveInst(op_lhs); + const rhs = try self.resolveInst(op_rhs); + + switch (int_info.signedness) { + .signed => { + switch (tag) { + .rem => { + try self.genIntMulDivOpMir(.idiv, dst_ty, .signed, lhs, rhs); + return MCValue{ .register = .rdx }; + }, + .mod => { + const div_floor = try self.genInlineIntDivFloor(dst_ty, lhs, rhs); + try self.genIntMulComplexOpMir(dst_ty, div_floor, rhs); + const div_floor_lock = self.register_manager.lockReg(div_floor.register); + defer if (div_floor_lock) |lock| self.register_manager.unlockReg(lock); + + const result: MCValue = if (maybe_inst) |inst| + try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs) + else + MCValue{ .register = try self.copyToTmpRegister(dst_ty, lhs) }; + try self.genBinOpMir(.sub, dst_ty, result, div_floor); + + return result; + }, + else => unreachable, + } + }, + .unsigned => { + try self.genIntMulDivOpMir(.div, dst_ty, .unsigned, lhs, rhs); + return MCValue{ .register = .rdx }; + }, + } + }, else => {}, } From af3ecd04b4377db7104a2d37d462f0f4b3e3db84 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 10 May 2022 09:04:39 +0200 Subject: [PATCH 06/18] x64: make genBinOp operate on MCValues directly --- src/arch/x86_64/CodeGen.zig | 138 ++++++++++++++++++++++-------------- 1 file changed, 84 insertions(+), 54 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 65293e0f46..5e27026b7c 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -1089,8 +1089,15 @@ fn airBoolToInt(self: *Self, inst: Air.Inst.Index) !void { fn airNot(self: *Self, inst: Air.Inst.Index) !void { const ty_op = self.air.instructions.items(.data)[inst].ty_op; - const result: MCValue = if (self.liveness.isUnused(inst)) .dead else result: { - const operand = try self.resolveInst(ty_op.operand); + + if (self.liveness.isUnused(inst)) { + return self.finishAir(inst, .dead, .{ ty_op.operand, .none, .none }); + } + + const operand_ty = self.air.typeOf(ty_op.operand); + const operand = try self.resolveInst(ty_op.operand); + + const result: MCValue = result: { switch (operand) { .dead => unreachable, .unreach => unreachable, @@ -1122,7 +1129,28 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void { }, else => {}, } - break :result try self.genBinOp(.not, inst, ty_op.operand, .bool_true); + + const operand_lock: ?RegisterLock = switch (operand) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (operand_lock) |lock| self.register_manager.unlockReg(lock); + + const dst_mcv: MCValue = blk: { + if (self.reuseOperand(inst, ty_op.operand, 0, operand) and operand.isRegister()) { + break :blk operand; + } + break :blk try self.copyToRegisterWithInstTracking(inst, operand_ty, operand); + }; + const dst_mcv_lock: ?RegisterLock = switch (dst_mcv) { + .register => |reg| self.register_manager.lockReg(reg), + else => null, + }; + defer if (dst_mcv_lock) |lock| self.register_manager.unlockReg(lock); + + try self.genBinOpMir(.xor, operand_ty, dst_mcv, .{ .immediate = 1 }); + + break :result dst_mcv; }; return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); } @@ -1214,7 +1242,13 @@ fn airBinOp(self: *Self, inst: Air.Inst.Index) !void { } const tag = self.air.instructions.items(.tag)[inst]; - const result = try self.genBinOp(tag, inst, bin_op.lhs, bin_op.rhs); + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + const lhs_ty = self.air.typeOf(bin_op.lhs); + const rhs_ty = self.air.typeOf(bin_op.rhs); + + const result = try self.genBinOp(tag, inst, lhs, rhs, lhs_ty, rhs_ty); + return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); } @@ -1263,7 +1297,10 @@ fn airAddWithOverflow(self: *Self, inst: Air.Inst.Index) !void { try self.spillCompareFlagsIfOccupied(); self.compare_flags_inst = inst; - const partial = try self.genBinOp(.add, inst, bin_op.lhs, bin_op.rhs); + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + + const partial = try self.genBinOp(.add, null, lhs, rhs, ty, ty); const result: MCValue = switch (int_info.signedness) { .signed => .{ .register_overflow_signed = partial.register }, .unsigned => .{ .register_overflow_unsigned = partial.register }, @@ -1295,7 +1332,10 @@ fn airSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void { try self.spillCompareFlagsIfOccupied(); self.compare_flags_inst = inst; - const partial = try self.genBinOp(.sub, inst, bin_op.lhs, bin_op.rhs); + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + + const partial = try self.genBinOp(.sub, null, lhs, rhs, ty, ty); const result: MCValue = switch (int_info.signedness) { .signed => .{ .register_overflow_signed = partial.register }, .unsigned => .{ .register_overflow_unsigned = partial.register }, @@ -1330,9 +1370,12 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { try self.spillCompareFlagsIfOccupied(); + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + if (math.isPowerOfTwo(int_info.bits)) { self.compare_flags_inst = inst; - const partial = try self.genBinOp(.mul, inst, bin_op.lhs, bin_op.rhs); + const partial = try self.genBinOp(.mul, null, lhs, rhs, ty, ty); break :result switch (int_info.signedness) { .signed => MCValue{ .register_overflow_signed = partial.register }, .unsigned => MCValue{ .register_overflow_unsigned = partial.register }, @@ -1344,9 +1387,6 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { const dst_reg: Register = dst_reg: { switch (int_info.signedness) { .signed => { - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - const rhs_lock: ?RegisterLock = switch (rhs) { .register => |reg| self.register_manager.lockRegAssumeUnused(reg), else => null, @@ -1375,7 +1415,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { break :dst_reg dst_reg; }, .unsigned => { - const dst_mcv = try self.genBinOp(.mul, null, bin_op.lhs, bin_op.rhs); + const dst_mcv = try self.genBinOp(.mul, null, lhs, rhs, ty, ty); break :dst_reg dst_mcv.register; }, } @@ -3137,8 +3177,10 @@ fn genBinOp( self: *Self, tag: Air.Inst.Tag, maybe_inst: ?Air.Inst.Index, - op_lhs: Air.Inst.Ref, - op_rhs: Air.Inst.Ref, + lhs: MCValue, + rhs: MCValue, + lhs_ty: Type, + rhs_ty: Type, ) !MCValue { const is_commutative: bool = switch (tag) { .add, @@ -3148,7 +3190,6 @@ fn genBinOp( .bool_and, .bit_and, .xor, - .not, => true, .sub, @@ -3169,14 +3210,12 @@ fn genBinOp( else => unreachable, }; - const dst_ty = self.air.typeOf(op_lhs); - const src_ty = self.air.typeOf(op_rhs); - if (dst_ty.zigTypeTag() == .Vector or dst_ty.zigTypeTag() == .Float) { - return self.fail("TODO implement genBinOp for {}", .{dst_ty.fmtDebug()}); + if (lhs_ty.zigTypeTag() == .Vector or lhs_ty.zigTypeTag() == .Float) { + return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmtDebug()}); } - if (dst_ty.abiSize(self.target.*) > 8) { - return self.fail("TODO implement genBinOp for {}", .{dst_ty.fmtDebug()}); + if (lhs_ty.abiSize(self.target.*) > 8) { + return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmtDebug()}); } switch (tag) { @@ -3191,24 +3230,21 @@ fn genBinOp( self.register_manager.unlockReg(reg); }; - const lhs = try self.resolveInst(op_lhs); - const rhs = try self.resolveInst(op_rhs); - - const int_info = dst_ty.intInfo(self.target.*); + const int_info = lhs_ty.intInfo(self.target.*); try self.genIntMulDivOpMir(switch (int_info.signedness) { .signed => .imul, .unsigned => .mul, - }, dst_ty, int_info.signedness, lhs, rhs); + }, lhs_ty, int_info.signedness, lhs, rhs); return switch (int_info.signedness) { .signed => MCValue{ .register = .rax }, - .unsigned => MCValue{ .register = registerAlias(.rax, @intCast(u32, dst_ty.abiSize(self.target.*))) }, + .unsigned => MCValue{ .register = registerAlias(.rax, @intCast(u32, lhs_ty.abiSize(self.target.*))) }, }; }, .mod, .rem, => { - const int_info = dst_ty.intInfo(self.target.*); + const int_info = lhs_ty.intInfo(self.target.*); const track_inst_rdx: ?Air.Inst.Index = switch (tag) { .mod => if (int_info.signedness == .unsigned) maybe_inst else null, .rem => maybe_inst, @@ -3223,27 +3259,24 @@ fn genBinOp( self.register_manager.unlockReg(reg); }; - const lhs = try self.resolveInst(op_lhs); - const rhs = try self.resolveInst(op_rhs); - switch (int_info.signedness) { .signed => { switch (tag) { .rem => { - try self.genIntMulDivOpMir(.idiv, dst_ty, .signed, lhs, rhs); + try self.genIntMulDivOpMir(.idiv, lhs_ty, .signed, lhs, rhs); return MCValue{ .register = .rdx }; }, .mod => { - const div_floor = try self.genInlineIntDivFloor(dst_ty, lhs, rhs); - try self.genIntMulComplexOpMir(dst_ty, div_floor, rhs); + const div_floor = try self.genInlineIntDivFloor(lhs_ty, lhs, rhs); + try self.genIntMulComplexOpMir(lhs_ty, div_floor, rhs); const div_floor_lock = self.register_manager.lockReg(div_floor.register); defer if (div_floor_lock) |lock| self.register_manager.unlockReg(lock); const result: MCValue = if (maybe_inst) |inst| - try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs) + try self.copyToRegisterWithInstTracking(inst, lhs_ty, lhs) else - MCValue{ .register = try self.copyToTmpRegister(dst_ty, lhs) }; - try self.genBinOpMir(.sub, dst_ty, result, div_floor); + MCValue{ .register = try self.copyToTmpRegister(lhs_ty, lhs) }; + try self.genBinOpMir(.sub, lhs_ty, result, div_floor); return result; }, @@ -3251,7 +3284,7 @@ fn genBinOp( } }, .unsigned => { - try self.genIntMulDivOpMir(.div, dst_ty, .unsigned, lhs, rhs); + try self.genIntMulDivOpMir(.div, lhs_ty, .unsigned, lhs, rhs); return MCValue{ .register = .rdx }; }, } @@ -3259,14 +3292,12 @@ fn genBinOp( else => {}, } - const lhs = try self.resolveInst(op_lhs); const lhs_lock: ?RegisterLock = switch (lhs) { .register => |reg| self.register_manager.lockRegAssumeUnused(reg), else => null, }; defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock); - const rhs = try self.resolveInst(op_rhs); const rhs_lock: ?RegisterLock = switch (rhs) { .register => |reg| self.register_manager.lockRegAssumeUnused(reg), else => null, @@ -3276,16 +3307,17 @@ fn genBinOp( var flipped: bool = false; const dst_mcv: MCValue = blk: { if (maybe_inst) |inst| { - if (self.reuseOperand(inst, op_lhs, 0, lhs) and lhs.isRegister()) { + const bin_op = self.air.instructions.items(.data)[inst].bin_op; + if (self.reuseOperand(inst, bin_op.lhs, 0, lhs) and lhs.isRegister()) { break :blk lhs; } - if (is_commutative and self.reuseOperand(inst, op_rhs, 1, rhs) and rhs.isRegister()) { + if (is_commutative and self.reuseOperand(inst, bin_op.rhs, 1, rhs) and rhs.isRegister()) { flipped = true; break :blk rhs; } - break :blk try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs); + break :blk try self.copyToRegisterWithInstTracking(inst, lhs_ty, lhs); } - break :blk MCValue{ .register = try self.copyToTmpRegister(dst_ty, lhs) }; + break :blk MCValue{ .register = try self.copyToTmpRegister(lhs_ty, lhs) }; }; const dst_mcv_lock: ?RegisterLock = switch (dst_mcv) { .register => |reg| self.register_manager.lockReg(reg), @@ -3296,7 +3328,7 @@ fn genBinOp( const src_mcv: MCValue = blk: { const mcv = if (flipped) lhs else rhs; if (mcv.isRegister() or mcv.isMemory()) break :blk mcv; - break :blk MCValue{ .register = try self.copyToTmpRegister(src_ty, mcv) }; + break :blk MCValue{ .register = try self.copyToTmpRegister(rhs_ty, mcv) }; }; const src_mcv_lock: ?RegisterLock = switch (src_mcv) { .register => |reg| self.register_manager.lockReg(reg), @@ -3307,11 +3339,11 @@ fn genBinOp( switch (tag) { .add, .addwrap, - => try self.genBinOpMir(.add, dst_ty, dst_mcv, src_mcv), + => try self.genBinOpMir(.add, lhs_ty, dst_mcv, src_mcv), .sub, .subwrap, - => try self.genBinOpMir(.sub, dst_ty, dst_mcv, src_mcv), + => try self.genBinOpMir(.sub, lhs_ty, dst_mcv, src_mcv), .ptr_add, .ptr_sub, @@ -3321,22 +3353,20 @@ fn genBinOp( .ptr_sub => .sub, else => unreachable, }; - const elem_size = dst_ty.elemType2().abiSize(self.target.*); - try self.genIntMulComplexOpMir(src_ty, src_mcv, .{ .immediate = elem_size }); - try self.genBinOpMir(mir_tag, dst_ty, dst_mcv, src_mcv); + const elem_size = lhs_ty.elemType2().abiSize(self.target.*); + try self.genIntMulComplexOpMir(rhs_ty, src_mcv, .{ .immediate = elem_size }); + try self.genBinOpMir(mir_tag, lhs_ty, dst_mcv, src_mcv); }, .bool_or, .bit_or, - => try self.genBinOpMir(.@"or", dst_ty, dst_mcv, src_mcv), + => try self.genBinOpMir(.@"or", lhs_ty, dst_mcv, src_mcv), .bool_and, .bit_and, - => try self.genBinOpMir(.@"and", dst_ty, dst_mcv, src_mcv), + => try self.genBinOpMir(.@"and", lhs_ty, dst_mcv, src_mcv), - .xor, - .not, - => try self.genBinOpMir(.xor, dst_ty, dst_mcv, src_mcv), + .xor => try self.genBinOpMir(.xor, lhs_ty, dst_mcv, src_mcv), else => unreachable, } From 85ca14e35a6be6591faaac42a4c95cf183ef4c3b Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 10 May 2022 09:08:33 +0200 Subject: [PATCH 07/18] x64: converge add_with_overflow and sub_with_overflow --- src/arch/x86_64/CodeGen.zig | 47 +++++++------------------------------ 1 file changed, 9 insertions(+), 38 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 5e27026b7c..0586514a62 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -621,8 +621,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { .trunc_float, => try self.airUnaryMath(inst), - .add_with_overflow => try self.airAddWithOverflow(inst), - .sub_with_overflow => try self.airSubWithOverflow(inst), + .add_with_overflow => try self.airAddSubWithOverflow(inst), + .sub_with_overflow => try self.airAddSubWithOverflow(inst), .mul_with_overflow => try self.airMulWithOverflow(inst), .shl_with_overflow => try self.airShlWithOverflow(inst), @@ -1279,8 +1279,9 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void { return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); } -fn airAddWithOverflow(self: *Self, inst: Air.Inst.Index) !void { +fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void { const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; + const tag = self.air.instructions.items(.tag)[inst]; const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; const result = if (self.liveness.isUnused(inst)) .dead else result: { const ty = self.air.typeOf(bin_op.lhs); @@ -1300,42 +1301,12 @@ fn airAddWithOverflow(self: *Self, inst: Air.Inst.Index) !void { const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); - const partial = try self.genBinOp(.add, null, lhs, rhs, ty, ty); - const result: MCValue = switch (int_info.signedness) { - .signed => .{ .register_overflow_signed = partial.register }, - .unsigned => .{ .register_overflow_unsigned = partial.register }, + const base_tag: Air.Inst.Tag = switch (tag) { + .add_with_overflow => .add, + .sub_with_overflow => .sub, + else => unreachable, }; - break :result result; - }, - else => unreachable, - } - }; - - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - -fn airSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void { - const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; - const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; - const result = if (self.liveness.isUnused(inst)) .dead else result: { - const ty = self.air.typeOf(bin_op.lhs); - - switch (ty.zigTypeTag()) { - .Vector => return self.fail("TODO implement sub_with_overflow for Vector type", .{}), - .Int => { - const int_info = ty.intInfo(self.target.*); - - if (int_info.bits > 64) { - return self.fail("TODO implement sub_with_overflow for Ints larger than 64bits", .{}); - } - - try self.spillCompareFlagsIfOccupied(); - self.compare_flags_inst = inst; - - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - - const partial = try self.genBinOp(.sub, null, lhs, rhs, ty, ty); + const partial = try self.genBinOp(base_tag, null, lhs, rhs, ty, ty); const result: MCValue = switch (int_info.signedness) { .signed => .{ .register_overflow_signed = partial.register }, .unsigned => .{ .register_overflow_unsigned = partial.register }, From ef9e3fb2b6cabb711b3421250b033f1696f961ae Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 10 May 2022 09:27:14 +0200 Subject: [PATCH 08/18] x64: migrate div to genMulDivBinOp --- src/arch/x86_64/CodeGen.zig | 431 +++++++++++++++++++----------------- 1 file changed, 230 insertions(+), 201 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 0586514a62..79a3605e23 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -592,11 +592,11 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { .sub => try self.airBinOp(inst), .subwrap => try self.airBinOp(inst), .sub_sat => try self.airSubSat(inst), - .mul => try self.airBinOp(inst), - .mulwrap => try self.airBinOp(inst), + .mul => try self.airMulDivBinOp(inst), + .mulwrap => try self.airMulDivBinOp(inst), .mul_sat => try self.airMulSat(inst), - .rem => try self.airBinOp(inst), - .mod => try self.airBinOp(inst), + .rem => try self.airMulDivBinOp(inst), + .mod => try self.airMulDivBinOp(inst), .shl, .shl_exact => try self.airShl(inst), .shl_sat => try self.airShlSat(inst), .min => try self.airMin(inst), @@ -626,7 +626,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { .mul_with_overflow => try self.airMulWithOverflow(inst), .shl_with_overflow => try self.airShlWithOverflow(inst), - .div_float, .div_trunc, .div_floor, .div_exact => try self.airDiv(inst), + .div_float, .div_trunc, .div_floor, .div_exact => try self.airMulDivBinOp(inst), .cmp_lt => try self.airCmp(inst, .lt), .cmp_lte => try self.airCmp(inst, .lte), @@ -959,6 +959,12 @@ pub fn spillCompareFlagsIfOccupied(self: *Self) !void { } } +pub fn spillRegisters(self: *Self, comptime count: comptime_int, registers: [count]Register) !void { + for (registers) |reg| { + try self.register_manager.getReg(reg, null); + } +} + /// Copies a value to a register without tracking the register. The register is not considered /// allocated. A second call to `copyToTmpRegister` may return the same register. /// This can have a side effect of spilling instructions to the stack to free up a register. @@ -1252,6 +1258,26 @@ fn airBinOp(self: *Self, inst: Air.Inst.Index) !void { return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); } +fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void { + const bin_op = self.air.instructions.items(.data)[inst].bin_op; + + if (self.liveness.isUnused(inst)) { + return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none }); + } + + const tag = self.air.instructions.items(.tag)[inst]; + const ty = self.air.typeOfIndex(inst); + + try self.spillRegisters(2, .{ .rax, .rdx }); + + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + + const result = try self.genMulDivBinOp(tag, inst, ty, lhs, rhs); + + return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); +} + fn airAddSat(self: *Self, inst: Air.Inst.Index) !void { const bin_op = self.air.instructions.items(.data)[inst].bin_op; const result: MCValue = if (self.liveness.isUnused(inst)) @@ -1339,25 +1365,31 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { return self.fail("TODO implement mul_with_overflow for Ints larger than 64bits", .{}); } - try self.spillCompareFlagsIfOccupied(); - - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - if (math.isPowerOfTwo(int_info.bits)) { + try self.spillCompareFlagsIfOccupied(); self.compare_flags_inst = inst; - const partial = try self.genBinOp(.mul, null, lhs, rhs, ty, ty); + + try self.spillRegisters(2, .{ .rax, .rdx }); + + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + + const partial = try self.genMulDivBinOp(.mul, null, ty, lhs, rhs); break :result switch (int_info.signedness) { .signed => MCValue{ .register_overflow_signed = partial.register }, .unsigned => MCValue{ .register_overflow_unsigned = partial.register }, }; } + try self.spillCompareFlagsIfOccupied(); self.compare_flags_inst = null; const dst_reg: Register = dst_reg: { switch (int_info.signedness) { .signed => { + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + const rhs_lock: ?RegisterLock = switch (rhs) { .register => |reg| self.register_manager.lockRegAssumeUnused(reg), else => null, @@ -1386,7 +1418,12 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { break :dst_reg dst_reg; }, .unsigned => { - const dst_mcv = try self.genBinOp(.mul, null, lhs, rhs, ty, ty); + try self.spillRegisters(2, .{ .rax, .rdx }); + + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + + const dst_mcv = try self.genMulDivBinOp(.mul, null, ty, lhs, rhs); break :dst_reg dst_mcv.register; }, } @@ -1487,7 +1524,13 @@ fn genIntMulDivOpMir( return self.fail("TODO implement genIntMulDivOpMir for ABI size larger than 8", .{}); } - try self.genSetReg(ty, .rax, lhs); + lhs: { + switch (lhs) { + .register => |reg| if (reg.to64() == .rax) break :lhs, + else => {}, + } + try self.genSetReg(ty, .rax, lhs); + } switch (signedness) { .signed => { @@ -1609,96 +1652,6 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa return MCValue{ .register = divisor }; } -fn airDiv(self: *Self, inst: Air.Inst.Index) !void { - const bin_op = self.air.instructions.items(.data)[inst].bin_op; - - if (self.liveness.isUnused(inst)) { - return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none }); - } - - const tag = self.air.instructions.items(.tag)[inst]; - const ty = self.air.typeOfIndex(inst); - - if (ty.zigTypeTag() != .Int) { - return self.fail("TODO implement {} for operands of dst type {}", .{ tag, ty.zigTypeTag() }); - } - - if (tag == .div_float) { - return self.fail("TODO implement {}", .{tag}); - } - - const signedness = ty.intInfo(self.target.*).signedness; - - // Spill .rax and .rdx upfront to ensure we don't spill the operands too late. - const track_rax: ?Air.Inst.Index = blk: { - if (signedness == .unsigned) break :blk inst; - switch (tag) { - .div_exact, .div_trunc => break :blk inst, - else => break :blk null, - } - }; - try self.register_manager.getReg(.rax, track_rax); - try self.register_manager.getReg(.rdx, null); - const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx }); - defer for (reg_locks) |reg| { - self.register_manager.unlockReg(reg); - }; - - const lhs = try self.resolveInst(bin_op.lhs); - const lhs_lock: ?RegisterLock = switch (lhs) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock); - - const rhs: MCValue = blk: { - const rhs = try self.resolveInst(bin_op.rhs); - if (signedness == .signed) { - switch (tag) { - .div_floor => { - const rhs_lock: ?RegisterLock = switch (rhs) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); - - break :blk try self.copyToRegisterWithInstTracking(inst, ty, rhs); - }, - else => {}, - } - } - break :blk rhs; - }; - const rhs_lock: ?RegisterLock = switch (rhs) { - .register => |reg| self.register_manager.lockReg(reg), - else => null, - }; - defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); - - const result: MCValue = result: { - if (signedness == .unsigned) { - try self.genIntMulDivOpMir(.div, ty, signedness, lhs, rhs); - break :result MCValue{ .register = .rax }; - } - - switch (tag) { - .div_exact, .div_trunc => { - try self.genIntMulDivOpMir(switch (signedness) { - .signed => .idiv, - .unsigned => .div, - }, ty, signedness, lhs, rhs); - break :result MCValue{ .register = .rax }; - }, - .div_floor => { - break :result try self.genInlineIntDivFloor(ty, lhs, rhs); - }, - else => unreachable, - } - }; - - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - fn airShl(self: *Self, inst: Air.Inst.Index) !void { const bin_op = self.air.instructions.items(.data)[inst].bin_op; if (self.liveness.isUnused(inst)) { @@ -3143,6 +3096,170 @@ fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void { return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); } +/// Result is always a register. +/// Clobbers .rax and .rdx therefore care needs to be taken to spill operands upfront. +/// Asserts .rax and .rdx are free. +fn genMulDivBinOp( + self: *Self, + tag: Air.Inst.Tag, + maybe_inst: ?Air.Inst.Index, + ty: Type, + lhs: MCValue, + rhs: MCValue, +) !MCValue { + if (ty.zigTypeTag() == .Vector or ty.zigTypeTag() == .Float) { + return self.fail("TODO implement genBinOp for {}", .{ty.fmtDebug()}); + } + if (ty.abiSize(self.target.*) > 8) { + return self.fail("TODO implement genBinOp for {}", .{ty.fmtDebug()}); + } + if (tag == .div_float) { + return self.fail("TODO implement genMulDivBinOp for div_float", .{}); + } + + assert(self.register_manager.isRegFree(.rax)); + assert(self.register_manager.isRegFree(.rdx)); + + const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx }); + defer for (reg_locks) |reg| { + self.register_manager.unlockReg(reg); + }; + + const int_info = ty.intInfo(self.target.*); + const signedness = int_info.signedness; + + switch (tag) { + .mul, + .mulwrap, + => { + try self.register_manager.getReg(.rax, maybe_inst); + try self.register_manager.getReg(.rdx, null); + + try self.genIntMulDivOpMir(switch (signedness) { + .signed => .imul, + .unsigned => .mul, + }, ty, signedness, lhs, rhs); + + return switch (signedness) { + .signed => MCValue{ .register = .rax }, + .unsigned => MCValue{ .register = registerAlias(.rax, @intCast(u32, ty.abiSize(self.target.*))) }, + }; + }, + .mod, + .rem, + => { + const track_inst_rdx: ?Air.Inst.Index = switch (tag) { + .mod => if (signedness == .unsigned) maybe_inst else null, + .rem => maybe_inst, + else => unreachable, + }; + + try self.register_manager.getReg(.rax, null); + try self.register_manager.getReg(.rdx, track_inst_rdx); + + switch (signedness) { + .signed => { + switch (tag) { + .rem => { + try self.genIntMulDivOpMir(.idiv, ty, .signed, lhs, rhs); + return MCValue{ .register = .rdx }; + }, + .mod => { + const div_floor = try self.genInlineIntDivFloor(ty, lhs, rhs); + try self.genIntMulComplexOpMir(ty, div_floor, rhs); + const div_floor_lock = self.register_manager.lockReg(div_floor.register); + defer if (div_floor_lock) |lock| self.register_manager.unlockReg(lock); + + const result: MCValue = if (maybe_inst) |inst| + try self.copyToRegisterWithInstTracking(inst, ty, lhs) + else + MCValue{ .register = try self.copyToTmpRegister(ty, lhs) }; + try self.genBinOpMir(.sub, ty, result, div_floor); + + return result; + }, + else => unreachable, + } + }, + .unsigned => { + try self.genIntMulDivOpMir(.div, ty, .unsigned, lhs, rhs); + return MCValue{ .register = .rdx }; + }, + } + }, + .div_trunc, + .div_floor, + .div_exact, + => { + const track_inst_rax: ?Air.Inst.Index = blk: { + if (signedness == .unsigned) break :blk maybe_inst; + switch (tag) { + .div_exact, .div_trunc => break :blk maybe_inst, + else => break :blk null, + } + }; + + try self.register_manager.getReg(.rax, track_inst_rax); + try self.register_manager.getReg(.rdx, null); + + const lhs_lock: ?RegisterLock = switch (lhs) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock); + + const actual_rhs: MCValue = blk: { + if (signedness == .signed) { + switch (tag) { + .div_floor => { + const rhs_lock: ?RegisterLock = switch (rhs) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); + + if (maybe_inst) |inst| { + break :blk try self.copyToRegisterWithInstTracking(inst, ty, rhs); + } + break :blk MCValue{ .register = try self.copyToTmpRegister(ty, rhs) }; + }, + else => {}, + } + } + break :blk rhs; + }; + const rhs_lock: ?RegisterLock = switch (actual_rhs) { + .register => |reg| self.register_manager.lockReg(reg), + else => null, + }; + defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); + + const result: MCValue = result: { + if (signedness == .unsigned) { + try self.genIntMulDivOpMir(.div, ty, signedness, lhs, actual_rhs); + break :result MCValue{ .register = .rax }; + } + + switch (tag) { + .div_exact, .div_trunc => { + try self.genIntMulDivOpMir(switch (signedness) { + .signed => .idiv, + .unsigned => .div, + }, ty, signedness, lhs, actual_rhs); + break :result MCValue{ .register = .rax }; + }, + .div_floor => { + break :result try self.genInlineIntDivFloor(ty, lhs, actual_rhs); + }, + else => unreachable, + } + }; + return result; + }, + else => unreachable, + } +} + /// Result is always a register. fn genBinOp( self: *Self, @@ -3153,6 +3270,13 @@ fn genBinOp( lhs_ty: Type, rhs_ty: Type, ) !MCValue { + if (lhs_ty.zigTypeTag() == .Vector or lhs_ty.zigTypeTag() == .Float) { + return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmtDebug()}); + } + if (lhs_ty.abiSize(self.target.*) > 8) { + return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmtDebug()}); + } + const is_commutative: bool = switch (tag) { .add, .addwrap, @@ -3163,106 +3287,9 @@ fn genBinOp( .xor, => true, - .sub, - .subwrap, - .mul, - .mulwrap, - .div_exact, - .div_trunc, - .rem, - .mod, - .shl, - .shr, - .ptr_add, - .ptr_sub, - => false, - - .div_float => return self.fail("TODO implement genBinOp for {}", .{tag}), - - else => unreachable, + else => false, }; - if (lhs_ty.zigTypeTag() == .Vector or lhs_ty.zigTypeTag() == .Float) { - return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmtDebug()}); - } - if (lhs_ty.abiSize(self.target.*) > 8) { - return self.fail("TODO implement genBinOp for {}", .{lhs_ty.fmtDebug()}); - } - - switch (tag) { - .mul, - .mulwrap, - => { - // Spill .rax and .rdx upfront to ensure we don't spill the operands too late. - try self.register_manager.getReg(.rax, maybe_inst); - try self.register_manager.getReg(.rdx, null); - const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx }); - defer for (reg_locks) |reg| { - self.register_manager.unlockReg(reg); - }; - - const int_info = lhs_ty.intInfo(self.target.*); - try self.genIntMulDivOpMir(switch (int_info.signedness) { - .signed => .imul, - .unsigned => .mul, - }, lhs_ty, int_info.signedness, lhs, rhs); - - return switch (int_info.signedness) { - .signed => MCValue{ .register = .rax }, - .unsigned => MCValue{ .register = registerAlias(.rax, @intCast(u32, lhs_ty.abiSize(self.target.*))) }, - }; - }, - .mod, - .rem, - => { - const int_info = lhs_ty.intInfo(self.target.*); - const track_inst_rdx: ?Air.Inst.Index = switch (tag) { - .mod => if (int_info.signedness == .unsigned) maybe_inst else null, - .rem => maybe_inst, - else => unreachable, - }; - - // Spill .rax and .rdx upfront to ensure we don't spill the operands too late. - try self.register_manager.getReg(.rax, null); - try self.register_manager.getReg(.rdx, track_inst_rdx); - const reg_locks = self.register_manager.lockRegsAssumeUnused(2, .{ .rax, .rdx }); - defer for (reg_locks) |reg| { - self.register_manager.unlockReg(reg); - }; - - switch (int_info.signedness) { - .signed => { - switch (tag) { - .rem => { - try self.genIntMulDivOpMir(.idiv, lhs_ty, .signed, lhs, rhs); - return MCValue{ .register = .rdx }; - }, - .mod => { - const div_floor = try self.genInlineIntDivFloor(lhs_ty, lhs, rhs); - try self.genIntMulComplexOpMir(lhs_ty, div_floor, rhs); - const div_floor_lock = self.register_manager.lockReg(div_floor.register); - defer if (div_floor_lock) |lock| self.register_manager.unlockReg(lock); - - const result: MCValue = if (maybe_inst) |inst| - try self.copyToRegisterWithInstTracking(inst, lhs_ty, lhs) - else - MCValue{ .register = try self.copyToTmpRegister(lhs_ty, lhs) }; - try self.genBinOpMir(.sub, lhs_ty, result, div_floor); - - return result; - }, - else => unreachable, - } - }, - .unsigned => { - try self.genIntMulDivOpMir(.div, lhs_ty, .unsigned, lhs, rhs); - return MCValue{ .register = .rdx }; - }, - } - }, - else => {}, - } - const lhs_lock: ?RegisterLock = switch (lhs) { .register => |reg| self.register_manager.lockRegAssumeUnused(reg), else => null, @@ -5460,6 +5487,7 @@ fn genInlineMemcpy( len: MCValue, opts: InlineMemcpyOpts, ) InnerError!void { + // TODO preserve contents of .rax and .rcx if not free, and then restore try self.register_manager.getReg(.rax, null); try self.register_manager.getReg(.rcx, null); @@ -5657,6 +5685,7 @@ fn genInlineMemset( len: MCValue, opts: InlineMemcpyOpts, ) InnerError!void { + // TODO preserve contents of .rax and then restore try self.register_manager.getReg(.rax, null); const rax_lock = self.register_manager.lockRegAssumeUnused(.rax); defer self.register_manager.unlockReg(rax_lock); From aef3c149e64746dac01f4d48a2835abd4204e625 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 10 May 2022 17:25:49 +0200 Subject: [PATCH 09/18] x64: refactor genMulDivBinOp helper --- src/arch/x86_64/CodeGen.zig | 178 ++++++++++++++++++------------------ 1 file changed, 87 insertions(+), 91 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 79a3605e23..82c7593a85 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -3131,75 +3131,82 @@ fn genMulDivBinOp( switch (tag) { .mul, .mulwrap, - => { - try self.register_manager.getReg(.rax, maybe_inst); - try self.register_manager.getReg(.rdx, null); - - try self.genIntMulDivOpMir(switch (signedness) { - .signed => .imul, - .unsigned => .mul, - }, ty, signedness, lhs, rhs); - - return switch (signedness) { - .signed => MCValue{ .register = .rax }, - .unsigned => MCValue{ .register = registerAlias(.rax, @intCast(u32, ty.abiSize(self.target.*))) }, - }; - }, - .mod, .rem, + .div_trunc, + .div_exact, => { - const track_inst_rdx: ?Air.Inst.Index = switch (tag) { - .mod => if (signedness == .unsigned) maybe_inst else null, - .rem => maybe_inst, - else => unreachable, + const track_inst_rax: ?Air.Inst.Index = switch (tag) { + .mul, .mulwrap, .div_exact, .div_trunc => maybe_inst, + else => null, }; - - try self.register_manager.getReg(.rax, null); + const track_inst_rdx: ?Air.Inst.Index = switch (tag) { + .rem => maybe_inst, + else => null, + }; + try self.register_manager.getReg(.rax, track_inst_rax); try self.register_manager.getReg(.rdx, track_inst_rdx); - switch (signedness) { - .signed => { - switch (tag) { - .rem => { - try self.genIntMulDivOpMir(.idiv, ty, .signed, lhs, rhs); - return MCValue{ .register = .rdx }; - }, - .mod => { - const div_floor = try self.genInlineIntDivFloor(ty, lhs, rhs); - try self.genIntMulComplexOpMir(ty, div_floor, rhs); - const div_floor_lock = self.register_manager.lockReg(div_floor.register); - defer if (div_floor_lock) |lock| self.register_manager.unlockReg(lock); - - const result: MCValue = if (maybe_inst) |inst| - try self.copyToRegisterWithInstTracking(inst, ty, lhs) - else - MCValue{ .register = try self.copyToTmpRegister(ty, lhs) }; - try self.genBinOpMir(.sub, ty, result, div_floor); - - return result; - }, - else => unreachable, - } + const mir_tag: Mir.Inst.Tag = switch (signedness) { + .signed => switch (tag) { + .mul, .mulwrap => Mir.Inst.Tag.imul, + .div_trunc, .div_exact, .rem => Mir.Inst.Tag.idiv, + else => unreachable, }, - .unsigned => { - try self.genIntMulDivOpMir(.div, ty, .unsigned, lhs, rhs); - return MCValue{ .register = .rdx }; + .unsigned => switch (tag) { + .mul, .mulwrap => Mir.Inst.Tag.mul, + .div_trunc, .div_exact, .rem => Mir.Inst.Tag.div, + else => unreachable, + }, + }; + + try self.genIntMulDivOpMir(mir_tag, ty, .signed, lhs, rhs); + + switch (signedness) { + .signed => switch (tag) { + .mul, .mulwrap, .div_trunc, .div_exact => return MCValue{ .register = .rax }, + .rem => return MCValue{ .register = .rdx }, + else => unreachable, + }, + .unsigned => switch (tag) { + .mul, .mulwrap, .div_trunc, .div_exact => return MCValue{ + .register = registerAlias(.rax, @intCast(u32, ty.abiSize(self.target.*))), + }, + .rem => return MCValue{ + .register = registerAlias(.rdx, @intCast(u32, ty.abiSize(self.target.*))), + }, + else => unreachable, }, } }, - .div_trunc, - .div_floor, - .div_exact, - => { - const track_inst_rax: ?Air.Inst.Index = blk: { - if (signedness == .unsigned) break :blk maybe_inst; - switch (tag) { - .div_exact, .div_trunc => break :blk maybe_inst, - else => break :blk null, - } - }; - try self.register_manager.getReg(.rax, track_inst_rax); + .mod => { + try self.register_manager.getReg(.rax, null); + try self.register_manager.getReg(.rdx, if (signedness == .unsigned) maybe_inst else null); + + switch (signedness) { + .signed => { + const div_floor = try self.genInlineIntDivFloor(ty, lhs, rhs); + try self.genIntMulComplexOpMir(ty, div_floor, rhs); + const div_floor_lock = self.register_manager.lockReg(div_floor.register); + defer if (div_floor_lock) |lock| self.register_manager.unlockReg(lock); + + const result: MCValue = if (maybe_inst) |inst| + try self.copyToRegisterWithInstTracking(inst, ty, lhs) + else + MCValue{ .register = try self.copyToTmpRegister(ty, lhs) }; + try self.genBinOpMir(.sub, ty, result, div_floor); + + return result; + }, + .unsigned => { + try self.genIntMulDivOpMir(.div, ty, .unsigned, lhs, rhs); + return MCValue{ .register = registerAlias(.rdx, @intCast(u32, ty.abiSize(self.target.*))) }; + }, + } + }, + + .div_floor => { + try self.register_manager.getReg(.rax, if (signedness == .unsigned) maybe_inst else null); try self.register_manager.getReg(.rdx, null); const lhs_lock: ?RegisterLock = switch (lhs) { @@ -3209,24 +3216,21 @@ fn genMulDivBinOp( defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock); const actual_rhs: MCValue = blk: { - if (signedness == .signed) { - switch (tag) { - .div_floor => { - const rhs_lock: ?RegisterLock = switch (rhs) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); + switch (signedness) { + .signed => { + const rhs_lock: ?RegisterLock = switch (rhs) { + .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + else => null, + }; + defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); - if (maybe_inst) |inst| { - break :blk try self.copyToRegisterWithInstTracking(inst, ty, rhs); - } - break :blk MCValue{ .register = try self.copyToTmpRegister(ty, rhs) }; - }, - else => {}, - } + if (maybe_inst) |inst| { + break :blk try self.copyToRegisterWithInstTracking(inst, ty, rhs); + } + break :blk MCValue{ .register = try self.copyToTmpRegister(ty, rhs) }; + }, + .unsigned => break :blk rhs, } - break :blk rhs; }; const rhs_lock: ?RegisterLock = switch (actual_rhs) { .register => |reg| self.register_manager.lockReg(reg), @@ -3235,27 +3239,19 @@ fn genMulDivBinOp( defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); const result: MCValue = result: { - if (signedness == .unsigned) { - try self.genIntMulDivOpMir(.div, ty, signedness, lhs, actual_rhs); - break :result MCValue{ .register = .rax }; - } - - switch (tag) { - .div_exact, .div_trunc => { - try self.genIntMulDivOpMir(switch (signedness) { - .signed => .idiv, - .unsigned => .div, - }, ty, signedness, lhs, actual_rhs); - break :result MCValue{ .register = .rax }; + switch (signedness) { + .signed => break :result try self.genInlineIntDivFloor(ty, lhs, actual_rhs), + .unsigned => { + try self.genIntMulDivOpMir(.div, ty, .unsigned, lhs, actual_rhs); + break :result MCValue{ + .register = registerAlias(.rax, @intCast(u32, ty.abiSize(self.target.*))), + }; }, - .div_floor => { - break :result try self.genInlineIntDivFloor(ty, lhs, actual_rhs); - }, - else => unreachable, } }; return result; }, + else => unreachable, } } From 6a4e445f5aaadf8e87ec08499c84f1d5f279e8b3 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 10 May 2022 17:53:53 +0200 Subject: [PATCH 10/18] x64: pull shl and shr into one helper fn --- src/arch/x86_64/CodeGen.zig | 253 +++++++++++++++++------------------- 1 file changed, 121 insertions(+), 132 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 82c7593a85..d26ec613fd 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -597,7 +597,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { .mul_sat => try self.airMulSat(inst), .rem => try self.airMulDivBinOp(inst), .mod => try self.airMulDivBinOp(inst), - .shl, .shl_exact => try self.airShl(inst), + .shl, .shl_exact => try self.airShlShrBinOp(inst), .shl_sat => try self.airShlSat(inst), .min => try self.airMin(inst), .max => try self.airMax(inst), @@ -643,7 +643,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { .bit_and => try self.airBinOp(inst), .bit_or => try self.airBinOp(inst), .xor => try self.airBinOp(inst), - .shr, .shr_exact => try self.airShr(inst), + .shr, .shr_exact => try self.airShlShrBinOp(inst), .alloc => try self.airAlloc(inst), .ret_ptr => try self.airRetPtr(inst), @@ -1652,64 +1652,22 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa return MCValue{ .register = divisor }; } -fn airShl(self: *Self, inst: Air.Inst.Index) !void { +fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void { const bin_op = self.air.instructions.items(.data)[inst].bin_op; + if (self.liveness.isUnused(inst)) { return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none }); } - const ty = self.air.typeOfIndex(inst); - const tag = self.air.instructions.items(.tag)[inst]; - switch (tag) { - .shl_exact => return self.fail("TODO implement {} for type {}", .{ tag, ty.fmtDebug() }), - .shl => {}, - else => unreachable, - } + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + const lhs_ty = self.air.typeOf(bin_op.lhs); + const rhs_ty = self.air.typeOf(bin_op.rhs); - if (ty.zigTypeTag() != .Int) { - return self.fail("TODO implement .shl for type {}", .{ty.fmtDebug()}); - } - if (ty.abiSize(self.target.*) > 8) { - return self.fail("TODO implement .shl for integers larger than 8 bytes", .{}); - } + const result = try self.genShiftBinOp(tag, inst, lhs, rhs, lhs_ty, rhs_ty); - // TODO look into reusing the operands - // TODO audit register allocation mechanics - const shift = try self.resolveInst(bin_op.rhs); - const shift_ty = self.air.typeOf(bin_op.rhs); - - blk: { - switch (shift) { - .register => |reg| { - if (reg.to64() == .rcx) break :blk; - }, - else => {}, - } - try self.register_manager.getReg(.rcx, null); - try self.genSetReg(shift_ty, .rcx, shift); - } - const rcx_lock = self.register_manager.lockRegAssumeUnused(.rcx); - defer self.register_manager.unlockReg(rcx_lock); - - const value = try self.resolveInst(bin_op.lhs); - const value_lock: ?RegisterLock = switch (value) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (value_lock) |lock| self.register_manager.unlockReg(lock); - - const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ty, value); - _ = try self.addInst(.{ - .tag = .sal, - .ops = (Mir.Ops{ - .reg1 = dst_mcv.register, - .flags = 0b01, - }).encode(), - .data = undefined, - }); - - return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none }); + return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); } fn airShlSat(self: *Self, inst: Air.Inst.Index) !void { @@ -1721,80 +1679,6 @@ fn airShlSat(self: *Self, inst: Air.Inst.Index) !void { return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); } -fn airShr(self: *Self, inst: Air.Inst.Index) !void { - const bin_op = self.air.instructions.items(.data)[inst].bin_op; - - if (self.liveness.isUnused(inst)) { - return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none }); - } - - const ty = self.air.typeOfIndex(inst); - const tag = self.air.instructions.items(.tag)[inst]; - switch (tag) { - .shr_exact => return self.fail("TODO implement shr_exact for type {}", .{ty.fmtDebug()}), - .shr => {}, - else => unreachable, - } - - if (ty.zigTypeTag() != .Int) { - return self.fail("TODO implement shr for type {}", .{ty.fmtDebug()}); - } - if (ty.abiSize(self.target.*) > 8) { - return self.fail("TODO implement shr for integers larger than 8 bytes", .{}); - } - - // TODO look into reusing the operands - // TODO audit register allocation mechanics - const shift = try self.resolveInst(bin_op.rhs); - const shift_ty = self.air.typeOf(bin_op.rhs); - - blk: { - switch (shift) { - .register => |reg| { - if (reg.to64() == .rcx) break :blk; - }, - else => {}, - } - try self.register_manager.getReg(.rcx, null); - try self.genSetReg(shift_ty, .rcx, shift); - } - const rcx_lock = self.register_manager.lockRegAssumeUnused(.rcx); - defer self.register_manager.unlockReg(rcx_lock); - - const value = try self.resolveInst(bin_op.lhs); - const value_lock: ?RegisterLock = switch (value) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, - }; - defer if (value_lock) |lock| self.register_manager.unlockReg(lock); - - const dst_mcv = try self.copyToRegisterWithInstTracking(inst, ty, value); - switch (ty.intInfo(self.target.*).signedness) { - .signed => { - _ = try self.addInst(.{ - .tag = .sar, - .ops = (Mir.Ops{ - .reg1 = dst_mcv.register, - .flags = 0b01, - }).encode(), - .data = undefined, - }); - }, - .unsigned => { - _ = try self.addInst(.{ - .tag = .shr, - .ops = (Mir.Ops{ - .reg1 = dst_mcv.register, - .flags = 0b01, - }).encode(), - .data = undefined, - }); - }, - } - - return self.finishAir(inst, dst_mcv, .{ bin_op.lhs, bin_op.rhs, .none }); -} - fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void { const ty_op = self.air.instructions.items(.data)[inst].ty_op; if (self.liveness.isUnused(inst)) { @@ -1822,7 +1706,7 @@ fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void { // TODO reuse the operand const result = try self.copyToRegisterWithInstTracking(inst, optional_ty, operand); const shift = @intCast(u8, offset * @sizeOf(usize)); - try self.shiftRegister(result.register, @intCast(u8, shift)); + try self.shiftRegisterRightUnsigned(result.register, @intCast(u8, shift)); break :result result; }, else => return self.fail("TODO implement optional_payload when operand is {}", .{operand}), @@ -1909,7 +1793,7 @@ fn airUnwrapErrPayload(self: *Self, inst: Air.Inst.Index) !void { // TODO reuse operand const shift = @intCast(u6, err_abi_size * @sizeOf(usize)); const result = try self.copyToRegisterWithInstTracking(inst, err_union_ty, operand); - try self.shiftRegister(result.register.to64(), shift); + try self.shiftRegisterRightUnsigned(result.register.to64(), shift); break :result MCValue{ .register = registerAlias(result.register, @intCast(u32, payload_ty.abiSize(self.target.*))), }; @@ -2421,7 +2305,7 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void { else 0; const result = try self.copyToRegisterWithInstTracking(inst, union_ty, operand); - try self.shiftRegister(result.register.to64(), shift); + try self.shiftRegisterRightUnsigned(result.register.to64(), shift); break :blk MCValue{ .register = registerAlias(result.register, @intCast(u32, layout.tag_size)), }; @@ -3020,7 +2904,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { // Shift by struct_field_offset. const shift = @intCast(u8, struct_field_offset * @sizeOf(usize)); - try self.shiftRegister(dst_mcv.register, shift); + try self.shiftRegisterRightUnsigned(dst_mcv.register, shift); // Mask with reg.size() - struct_field_size const max_reg_bit_width = Register.rax.size(); @@ -3097,7 +2981,111 @@ fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void { } /// Result is always a register. -/// Clobbers .rax and .rdx therefore care needs to be taken to spill operands upfront. +/// Clobbers .rcx therefore care is needed to spill .rcx upfront. +/// Asserts .rcx is free. +fn genShiftBinOp( + self: *Self, + tag: Air.Inst.Tag, + maybe_inst: ?Air.Inst.Index, + lhs: MCValue, + rhs: MCValue, + lhs_ty: Type, + rhs_ty: Type, +) !MCValue { + if (lhs_ty.zigTypeTag() == .Vector or lhs_ty.zigTypeTag() == .Float) { + return self.fail("TODO implement genShiftBinOp for {}", .{lhs_ty.fmtDebug()}); + } + if (lhs_ty.abiSize(self.target.*) > 8) { + return self.fail("TODO implement genShiftBinOp for {}", .{lhs_ty.fmtDebug()}); + } + + assert(self.register_manager.isRegFree(.rcx)); + + try self.register_manager.getReg(.rcx, null); + try self.genSetReg(rhs_ty, .rcx, rhs); + const rcx_lock = self.register_manager.lockRegAssumeUnused(.rcx); + defer self.register_manager.unlockReg(rcx_lock); + + const int_info = lhs_ty.intInfo(self.target.*); + const signedness = int_info.signedness; + + const lhs_lock: ?RegisterLock = switch (lhs) { + .register => |reg| self.register_manager.lockReg(reg), + else => null, + }; + defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock); + + const rhs_lock: ?RegisterLock = switch (rhs) { + .register => |reg| self.register_manager.lockReg(reg), + else => null, + }; + defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); + + const dst: MCValue = blk: { + if (maybe_inst) |inst| { + const bin_op = self.air.instructions.items(.data)[inst].bin_op; + // TODO dst can also be a memory location + if (self.reuseOperand(inst, bin_op.lhs, 0, lhs) and lhs.isRegister()) { + break :blk lhs; + } + break :blk try self.copyToRegisterWithInstTracking(inst, lhs_ty, lhs); + } + break :blk MCValue{ .register = try self.copyToTmpRegister(lhs_ty, lhs) }; + }; + + switch (tag) { + .shl => switch (signedness) { + .signed => { + _ = try self.addInst(.{ + .tag = .sal, + .ops = (Mir.Ops{ + .reg1 = dst.register, + .flags = 0b01, + }).encode(), + .data = undefined, + }); + }, + .unsigned => { + _ = try self.addInst(.{ + .tag = .shl, + .ops = (Mir.Ops{ + .reg1 = dst.register, + .flags = 0b01, + }).encode(), + .data = undefined, + }); + }, + }, + .shr => switch (signedness) { + .signed => { + _ = try self.addInst(.{ + .tag = .sar, + .ops = (Mir.Ops{ + .reg1 = dst.register, + .flags = 0b01, + }).encode(), + .data = undefined, + }); + }, + .unsigned => { + _ = try self.addInst(.{ + .tag = .shr, + .ops = (Mir.Ops{ + .reg1 = dst.register, + .flags = 0b01, + }).encode(), + .data = undefined, + }); + }, + }, + else => unreachable, + } + + return dst; +} + +/// Result is always a register. +/// Clobbers .rax and .rdx therefore care is needed to spill .rax and .rdx upfront. /// Asserts .rax and .rdx are free. fn genMulDivBinOp( self: *Self, @@ -6790,7 +6778,8 @@ fn registerAlias(reg: Register, size_bytes: u32) Register { } } -fn shiftRegister(self: *Self, reg: Register, shift: u8) !void { +/// Shifts register right without sign-extension. +fn shiftRegisterRightUnsigned(self: *Self, reg: Register, shift: u8) !void { if (shift == 0) return; if (shift == 1) { _ = try self.addInst(.{ From a9514ae1732df1fdc142b602e62ab0d12f693afa Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 10 May 2022 18:16:14 +0200 Subject: [PATCH 11/18] x64: handle immediate as RHS of shift bin ops --- src/arch/x86_64/CodeGen.zig | 50 ++++++++++++++++++++++++++----------- 1 file changed, 35 insertions(+), 15 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index d26ec613fd..d129fc493b 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -1659,6 +1659,8 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void { return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none }); } + try self.spillRegisters(1, .{.rcx}); + const tag = self.air.instructions.items(.tag)[inst]; const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); @@ -2981,7 +2983,7 @@ fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void { } /// Result is always a register. -/// Clobbers .rcx therefore care is needed to spill .rcx upfront. +/// Clobbers .rcx for non-immediate rhs, therefore care is needed to spill .rcx upfront. /// Asserts .rcx is free. fn genShiftBinOp( self: *Self, @@ -2999,12 +3001,7 @@ fn genShiftBinOp( return self.fail("TODO implement genShiftBinOp for {}", .{lhs_ty.fmtDebug()}); } - assert(self.register_manager.isRegFree(.rcx)); - - try self.register_manager.getReg(.rcx, null); - try self.genSetReg(rhs_ty, .rcx, rhs); - const rcx_lock = self.register_manager.lockRegAssumeUnused(.rcx); - defer self.register_manager.unlockReg(rcx_lock); + assert(rhs_ty.abiSize(self.target.*) == 1); const int_info = lhs_ty.intInfo(self.target.*); const signedness = int_info.signedness; @@ -3021,6 +3018,29 @@ fn genShiftBinOp( }; defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); + const flags: u2 = blk: { + if (rhs.isImmediate()) { + const flags: u2 = switch (rhs.immediate) { + 0 => unreachable, // TODO is this valid? + 1 => 0b00, + else => 0b10, + }; + break :blk flags; + } + + assert(self.register_manager.isRegFree(.rcx)); + + try self.register_manager.getReg(.rcx, null); + try self.genSetReg(rhs_ty, .rcx, rhs); + const rcx_lock = self.register_manager.lockRegAssumeUnused(.rcx); + defer self.register_manager.unlockReg(rcx_lock); + + break :blk 0b01; + }; + const data: Mir.Inst.Data = if (rhs.isImmediate()) .{ + .imm = @intCast(u8, rhs.immediate), + } else undefined; + const dst: MCValue = blk: { if (maybe_inst) |inst| { const bin_op = self.air.instructions.items(.data)[inst].bin_op; @@ -3040,9 +3060,9 @@ fn genShiftBinOp( .tag = .sal, .ops = (Mir.Ops{ .reg1 = dst.register, - .flags = 0b01, + .flags = flags, }).encode(), - .data = undefined, + .data = data, }); }, .unsigned => { @@ -3050,9 +3070,9 @@ fn genShiftBinOp( .tag = .shl, .ops = (Mir.Ops{ .reg1 = dst.register, - .flags = 0b01, + .flags = flags, }).encode(), - .data = undefined, + .data = data, }); }, }, @@ -3062,9 +3082,9 @@ fn genShiftBinOp( .tag = .sar, .ops = (Mir.Ops{ .reg1 = dst.register, - .flags = 0b01, + .flags = flags, }).encode(), - .data = undefined, + .data = data, }); }, .unsigned => { @@ -3072,9 +3092,9 @@ fn genShiftBinOp( .tag = .shr, .ops = (Mir.Ops{ .reg1 = dst.register, - .flags = 0b01, + .flags = flags, }).encode(), - .data = undefined, + .data = data, }); }, }, From 9725205859517ce81e201b79b55f7a7c2cb87f20 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 10 May 2022 18:52:49 +0200 Subject: [PATCH 12/18] x64: consolidate shifts into single MIR helper fn --- src/arch/x86_64/CodeGen.zig | 201 +++++++++++++++--------------------- 1 file changed, 81 insertions(+), 120 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index d129fc493b..72c65e84de 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -1708,7 +1708,7 @@ fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void { // TODO reuse the operand const result = try self.copyToRegisterWithInstTracking(inst, optional_ty, operand); const shift = @intCast(u8, offset * @sizeOf(usize)); - try self.shiftRegisterRightUnsigned(result.register, @intCast(u8, shift)); + try self.genShiftBinOpMir(optional_ty, result.register, .{ .immediate = @intCast(u8, shift) }, .right); break :result result; }, else => return self.fail("TODO implement optional_payload when operand is {}", .{operand}), @@ -1795,7 +1795,7 @@ fn airUnwrapErrPayload(self: *Self, inst: Air.Inst.Index) !void { // TODO reuse operand const shift = @intCast(u6, err_abi_size * @sizeOf(usize)); const result = try self.copyToRegisterWithInstTracking(inst, err_union_ty, operand); - try self.shiftRegisterRightUnsigned(result.register.to64(), shift); + try self.genShiftBinOpMir(Type.usize, result.register, .{ .immediate = shift }, .right); break :result MCValue{ .register = registerAlias(result.register, @intCast(u32, payload_ty.abiSize(self.target.*))), }; @@ -2307,7 +2307,7 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void { else 0; const result = try self.copyToRegisterWithInstTracking(inst, union_ty, operand); - try self.shiftRegisterRightUnsigned(result.register.to64(), shift); + try self.genShiftBinOpMir(Type.usize, result.register, .{ .immediate = shift }, .right); break :blk MCValue{ .register = registerAlias(result.register, @intCast(u32, layout.tag_size)), }; @@ -2906,7 +2906,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { // Shift by struct_field_offset. const shift = @intCast(u8, struct_field_offset * @sizeOf(usize)); - try self.shiftRegisterRightUnsigned(dst_mcv.register, shift); + try self.genShiftBinOpMir(Type.usize, dst_mcv.register, .{ .immediate = shift }, .right); // Mask with reg.size() - struct_field_size const max_reg_bit_width = Register.rax.size(); @@ -2982,6 +2982,74 @@ fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void { return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); } +/// Clobbers .rcx for non-immediate shift value. +fn genShiftBinOpMir(self: *Self, ty: Type, reg: Register, shift: MCValue, direction: enum { left, right }) !void { + assert(reg.to64() != .rcx); + + const abi_size = @intCast(u32, ty.abiSize(self.target.*)); + const signedness: std.builtin.Signedness = blk: { + if (ty.zigTypeTag() != .Int) break :blk .unsigned; + break :blk ty.intInfo(self.target.*).signedness; + }; + + const tag: Mir.Inst.Tag = switch (signedness) { + .signed => switch (direction) { + .left => Mir.Inst.Tag.sal, + .right => Mir.Inst.Tag.sar, + }, + .unsigned => switch (direction) { + .left => Mir.Inst.Tag.shl, + .right => Mir.Inst.Tag.shr, + }, + }; + + blk: { + switch (shift) { + .immediate => |imm| switch (imm) { + 0 => return, + 1 => { + _ = try self.addInst(.{ + .tag = tag, + .ops = (Mir.Ops{ + .reg1 = registerAlias(reg, abi_size), + .flags = 0b00, + }).encode(), + .data = undefined, + }); + return; + }, + else => { + _ = try self.addInst(.{ + .tag = tag, + .ops = (Mir.Ops{ + .reg1 = registerAlias(reg, abi_size), + .flags = 0b10, + }).encode(), + .data = .{ .imm = @intCast(u8, imm) }, + }); + return; + }, + }, + .register => |shift_reg| { + if (shift_reg == .rcx) break :blk; + }, + else => {}, + } + assert(self.register_manager.isRegFree(.rcx)); + try self.register_manager.getReg(.rcx, null); + try self.genSetReg(Type.u8, .rcx, shift); + } + + _ = try self.addInst(.{ + .tag = tag, + .ops = (Mir.Ops{ + .reg1 = registerAlias(reg, abi_size), + .flags = 0b01, + }).encode(), + .data = undefined, + }); +} + /// Result is always a register. /// Clobbers .rcx for non-immediate rhs, therefore care is needed to spill .rcx upfront. /// Asserts .rcx is free. @@ -3003,9 +3071,6 @@ fn genShiftBinOp( assert(rhs_ty.abiSize(self.target.*) == 1); - const int_info = lhs_ty.intInfo(self.target.*); - const signedness = int_info.signedness; - const lhs_lock: ?RegisterLock = switch (lhs) { .register => |reg| self.register_manager.lockReg(reg), else => null, @@ -3018,28 +3083,10 @@ fn genShiftBinOp( }; defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); - const flags: u2 = blk: { - if (rhs.isImmediate()) { - const flags: u2 = switch (rhs.immediate) { - 0 => unreachable, // TODO is this valid? - 1 => 0b00, - else => 0b10, - }; - break :blk flags; - } - - assert(self.register_manager.isRegFree(.rcx)); - - try self.register_manager.getReg(.rcx, null); - try self.genSetReg(rhs_ty, .rcx, rhs); - const rcx_lock = self.register_manager.lockRegAssumeUnused(.rcx); - defer self.register_manager.unlockReg(rcx_lock); - - break :blk 0b01; - }; - const data: Mir.Inst.Data = if (rhs.isImmediate()) .{ - .imm = @intCast(u8, rhs.immediate), - } else undefined; + assert(self.register_manager.isRegFree(.rcx)); + try self.register_manager.getReg(.rcx, null); + const rcx_lock = self.register_manager.lockRegAssumeUnused(.rcx); + defer self.register_manager.unlockReg(rcx_lock); const dst: MCValue = blk: { if (maybe_inst) |inst| { @@ -3054,50 +3101,8 @@ fn genShiftBinOp( }; switch (tag) { - .shl => switch (signedness) { - .signed => { - _ = try self.addInst(.{ - .tag = .sal, - .ops = (Mir.Ops{ - .reg1 = dst.register, - .flags = flags, - }).encode(), - .data = data, - }); - }, - .unsigned => { - _ = try self.addInst(.{ - .tag = .shl, - .ops = (Mir.Ops{ - .reg1 = dst.register, - .flags = flags, - }).encode(), - .data = data, - }); - }, - }, - .shr => switch (signedness) { - .signed => { - _ = try self.addInst(.{ - .tag = .sar, - .ops = (Mir.Ops{ - .reg1 = dst.register, - .flags = flags, - }).encode(), - .data = data, - }); - }, - .unsigned => { - _ = try self.addInst(.{ - .tag = .shr, - .ops = (Mir.Ops{ - .reg1 = dst.register, - .flags = flags, - }).encode(), - .data = data, - }); - }, - }, + .shl => try self.genShiftBinOpMir(lhs_ty, dst.register, rhs, .left), + .shr => try self.genShiftBinOpMir(lhs_ty, dst.register, rhs, .right), else => unreachable, } @@ -5422,14 +5427,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl }); if (nearest_power_of_two > 1) { - _ = try self.addInst(.{ - .tag = .shr, - .ops = (Mir.Ops{ - .reg1 = tmp_reg, - .flags = 0b10, - }).encode(), - .data = .{ .imm = nearest_power_of_two * 8 }, - }); + try self.genShiftBinOpMir(ty, tmp_reg, .{ .immediate = nearest_power_of_two * 8 }, .right); } remainder -= nearest_power_of_two; @@ -6798,29 +6796,6 @@ fn registerAlias(reg: Register, size_bytes: u32) Register { } } -/// Shifts register right without sign-extension. -fn shiftRegisterRightUnsigned(self: *Self, reg: Register, shift: u8) !void { - if (shift == 0) return; - if (shift == 1) { - _ = try self.addInst(.{ - .tag = .shr, - .ops = (Mir.Ops{ - .reg1 = reg, - }).encode(), - .data = undefined, - }); - } else { - _ = try self.addInst(.{ - .tag = .shr, - .ops = (Mir.Ops{ - .reg1 = reg, - .flags = 0b10, - }).encode(), - .data = .{ .imm = shift }, - }); - } -} - /// Truncates the value in the register in place. /// Clobbers any remaining bits. fn truncateRegister(self: *Self, ty: Type, reg: Register) !void { @@ -6829,22 +6804,8 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void { switch (int_info.signedness) { .signed => { const shift = @intCast(u6, max_reg_bit_width - int_info.bits); - _ = try self.addInst(.{ - .tag = .sal, - .ops = (Mir.Ops{ - .reg1 = reg.to64(), - .flags = 0b10, - }).encode(), - .data = .{ .imm = shift }, - }); - _ = try self.addInst(.{ - .tag = .sar, - .ops = (Mir.Ops{ - .reg1 = reg.to64(), - .flags = 0b10, - }).encode(), - .data = .{ .imm = shift }, - }); + try self.genShiftBinOpMir(Type.isize, reg, .{ .immediate = shift }, .left); + try self.genShiftBinOpMir(Type.isize, reg, .{ .immediate = shift }, .right); }, .unsigned => { const shift = @intCast(u6, max_reg_bit_width - int_info.bits); From f131e41db9f8b1b1bbd678b52ef377821d83bddc Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 10 May 2022 19:34:20 +0200 Subject: [PATCH 13/18] x64: implement shl_exact and shr_exact --- src/arch/x86_64/CodeGen.zig | 54 +++++++++++++++++++------------------ test/behavior/math.zig | 4 --- 2 files changed, 28 insertions(+), 30 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 72c65e84de..2ed61006f2 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -1708,7 +1708,7 @@ fn airOptionalPayload(self: *Self, inst: Air.Inst.Index) !void { // TODO reuse the operand const result = try self.copyToRegisterWithInstTracking(inst, optional_ty, operand); const shift = @intCast(u8, offset * @sizeOf(usize)); - try self.genShiftBinOpMir(optional_ty, result.register, .{ .immediate = @intCast(u8, shift) }, .right); + try self.genShiftBinOpMir(.shr, optional_ty, result.register, .{ .immediate = @intCast(u8, shift) }); break :result result; }, else => return self.fail("TODO implement optional_payload when operand is {}", .{operand}), @@ -1795,7 +1795,7 @@ fn airUnwrapErrPayload(self: *Self, inst: Air.Inst.Index) !void { // TODO reuse operand const shift = @intCast(u6, err_abi_size * @sizeOf(usize)); const result = try self.copyToRegisterWithInstTracking(inst, err_union_ty, operand); - try self.genShiftBinOpMir(Type.usize, result.register, .{ .immediate = shift }, .right); + try self.genShiftBinOpMir(.shr, Type.usize, result.register, .{ .immediate = shift }); break :result MCValue{ .register = registerAlias(result.register, @intCast(u32, payload_ty.abiSize(self.target.*))), }; @@ -2307,7 +2307,7 @@ fn airGetUnionTag(self: *Self, inst: Air.Inst.Index) !void { else 0; const result = try self.copyToRegisterWithInstTracking(inst, union_ty, operand); - try self.genShiftBinOpMir(Type.usize, result.register, .{ .immediate = shift }, .right); + try self.genShiftBinOpMir(.shr, Type.usize, result.register, .{ .immediate = shift }); break :blk MCValue{ .register = registerAlias(result.register, @intCast(u32, layout.tag_size)), }; @@ -2906,7 +2906,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { // Shift by struct_field_offset. const shift = @intCast(u8, struct_field_offset * @sizeOf(usize)); - try self.genShiftBinOpMir(Type.usize, dst_mcv.register, .{ .immediate = shift }, .right); + try self.genShiftBinOpMir(.shr, Type.usize, dst_mcv.register, .{ .immediate = shift }); // Mask with reg.size() - struct_field_size const max_reg_bit_width = Register.rax.size(); @@ -2983,26 +2983,15 @@ fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void { } /// Clobbers .rcx for non-immediate shift value. -fn genShiftBinOpMir(self: *Self, ty: Type, reg: Register, shift: MCValue, direction: enum { left, right }) !void { +fn genShiftBinOpMir(self: *Self, tag: Mir.Inst.Tag, ty: Type, reg: Register, shift: MCValue) !void { assert(reg.to64() != .rcx); + switch (tag) { + .sal, .sar, .shl, .shr => {}, + else => unreachable, + } + const abi_size = @intCast(u32, ty.abiSize(self.target.*)); - const signedness: std.builtin.Signedness = blk: { - if (ty.zigTypeTag() != .Int) break :blk .unsigned; - break :blk ty.intInfo(self.target.*).signedness; - }; - - const tag: Mir.Inst.Tag = switch (signedness) { - .signed => switch (direction) { - .left => Mir.Inst.Tag.sal, - .right => Mir.Inst.Tag.sar, - }, - .unsigned => switch (direction) { - .left => Mir.Inst.Tag.shl, - .right => Mir.Inst.Tag.shr, - }, - }; - blk: { switch (shift) { .immediate => |imm| switch (imm) { @@ -3100,9 +3089,22 @@ fn genShiftBinOp( break :blk MCValue{ .register = try self.copyToTmpRegister(lhs_ty, lhs) }; }; + const signedness = lhs_ty.intInfo(self.target.*).signedness; switch (tag) { - .shl => try self.genShiftBinOpMir(lhs_ty, dst.register, rhs, .left), - .shr => try self.genShiftBinOpMir(lhs_ty, dst.register, rhs, .right), + .shl => try self.genShiftBinOpMir(switch (signedness) { + .signed => .sal, + .unsigned => .shl, + }, lhs_ty, dst.register, rhs), + + .shl_exact => try self.genShiftBinOpMir(.shl, lhs_ty, dst.register, rhs), + + .shr, + .shr_exact, + => try self.genShiftBinOpMir(switch (signedness) { + .signed => .sar, + .unsigned => .shr, + }, lhs_ty, dst.register, rhs), + else => unreachable, } @@ -5427,7 +5429,7 @@ fn genSetStack(self: *Self, ty: Type, stack_offset: i32, mcv: MCValue, opts: Inl }); if (nearest_power_of_two > 1) { - try self.genShiftBinOpMir(ty, tmp_reg, .{ .immediate = nearest_power_of_two * 8 }, .right); + try self.genShiftBinOpMir(.shr, ty, tmp_reg, .{ .immediate = nearest_power_of_two * 8 }); } remainder -= nearest_power_of_two; @@ -6804,8 +6806,8 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void { switch (int_info.signedness) { .signed => { const shift = @intCast(u6, max_reg_bit_width - int_info.bits); - try self.genShiftBinOpMir(Type.isize, reg, .{ .immediate = shift }, .left); - try self.genShiftBinOpMir(Type.isize, reg, .{ .immediate = shift }, .right); + try self.genShiftBinOpMir(.sal, Type.isize, reg, .{ .immediate = shift }); + try self.genShiftBinOpMir(.sar, Type.isize, reg, .{ .immediate = shift }); }, .unsigned => { const shift = @intCast(u6, max_reg_bit_width - int_info.bits); diff --git a/test/behavior/math.zig b/test/behavior/math.zig index dd609c9b08..53b11d1d9f 100644 --- a/test/behavior/math.zig +++ b/test/behavior/math.zig @@ -1065,8 +1065,6 @@ fn testShlTrunc(x: u16) !void { } test "exact shift left" { - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO - try testShlExact(0b00110101); comptime try testShlExact(0b00110101); } @@ -1076,8 +1074,6 @@ fn testShlExact(x: u8) !void { } test "exact shift right" { - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO - try testShrExact(0b10110100); comptime try testShrExact(0b10110100); } From 2a738599a0180b6f12439d33722a52182b11c21e Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 10 May 2022 20:45:57 +0200 Subject: [PATCH 14/18] x64: implement missing bits in add_with_overflow and sub_with_overflow --- src/arch/x86_64/CodeGen.zig | 173 +++++++++++++++++++++--------------- test/behavior/math.zig | 1 - 2 files changed, 101 insertions(+), 73 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 2ed61006f2..15610f1bd2 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -1311,18 +1311,15 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void { const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; const result = if (self.liveness.isUnused(inst)) .dead else result: { const ty = self.air.typeOf(bin_op.lhs); - + const abi_size = ty.abiSize(self.target.*); switch (ty.zigTypeTag()) { .Vector => return self.fail("TODO implement add_with_overflow for Vector type", .{}), .Int => { - const int_info = ty.intInfo(self.target.*); - - if (int_info.bits > 64) { + if (abi_size > 8) { return self.fail("TODO implement add_with_overflow for Ints larger than 64bits", .{}); } try self.spillCompareFlagsIfOccupied(); - self.compare_flags_inst = inst; const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); @@ -1333,11 +1330,30 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void { else => unreachable, }; const partial = try self.genBinOp(base_tag, null, lhs, rhs, ty, ty); - const result: MCValue = switch (int_info.signedness) { - .signed => .{ .register_overflow_signed = partial.register }, - .unsigned => .{ .register_overflow_unsigned = partial.register }, - }; - break :result result; + + const int_info = ty.intInfo(self.target.*); + + if (math.isPowerOfTwo(int_info.bits) and int_info.bits >= 8) { + self.compare_flags_inst = inst; + + const result: MCValue = switch (int_info.signedness) { + .signed => .{ .register_overflow_signed = partial.register }, + .unsigned => .{ .register_overflow_unsigned = partial.register }, + }; + break :result result; + } + + self.compare_flags_inst = null; + + const tuple_ty = self.air.typeOfIndex(inst); + const tuple_size = @intCast(u32, tuple_ty.abiSize(self.target.*)); + const tuple_align = tuple_ty.abiAlignment(self.target.*); + const overflow_bit_offset = @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)); + const stack_offset = @intCast(i32, try self.allocMem(inst, tuple_size, tuple_align)); + + try self.genSetStackTruncatedOverflowCompare(ty, stack_offset, overflow_bit_offset, partial.register); + + break :result MCValue{ .stack_offset = stack_offset }; }, else => unreachable, } @@ -1346,6 +1362,75 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void { return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); } +fn genSetStackTruncatedOverflowCompare( + self: *Self, + ty: Type, + stack_offset: i32, + overflow_bit_offset: i32, + reg: Register, +) !void { + const reg_lock = self.register_manager.lockReg(reg); + defer if (reg_lock) |lock| self.register_manager.unlockReg(lock); + + const int_info = ty.intInfo(self.target.*); + const extended_ty = switch (int_info.signedness) { + .signed => Type.isize, + .unsigned => ty, + }; + + const temp_regs = try self.register_manager.allocRegs(3, .{ null, null, null }); + const temp_regs_locks = self.register_manager.lockRegsAssumeUnused(3, temp_regs); + defer for (temp_regs_locks) |rreg| { + self.register_manager.unlockReg(rreg); + }; + + const overflow_reg = temp_regs[0]; + const flags: u2 = switch (int_info.signedness) { + .signed => 0b00, + .unsigned => 0b10, + }; + _ = try self.addInst(.{ + .tag = .cond_set_byte_overflow, + .ops = (Mir.Ops{ + .reg1 = overflow_reg.to8(), + .flags = flags, + }).encode(), + .data = undefined, + }); + + const scratch_reg = temp_regs[1]; + try self.genSetReg(extended_ty, scratch_reg, .{ .register = reg }); + try self.truncateRegister(ty, scratch_reg); + try self.genBinOpMir( + .cmp, + extended_ty, + .{ .register = reg }, + .{ .register = scratch_reg }, + ); + + const eq_reg = temp_regs[2]; + _ = try self.addInst(.{ + .tag = .cond_set_byte_eq_ne, + .ops = (Mir.Ops{ + .reg1 = eq_reg.to8(), + .flags = 0b00, + }).encode(), + .data = undefined, + }); + + try self.genBinOpMir( + .@"or", + Type.u8, + .{ .register = overflow_reg }, + .{ .register = eq_reg }, + ); + + try self.genSetStack(ty, stack_offset, .{ .register = scratch_reg }, .{}); + try self.genSetStack(Type.initTag(.u1), stack_offset - overflow_bit_offset, .{ + .register = overflow_reg.to8(), + }, .{}); +} + fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; @@ -1355,17 +1440,18 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { } const ty = self.air.typeOf(bin_op.lhs); + const abi_size = ty.abiSize(self.target.*); const result: MCValue = result: { switch (ty.zigTypeTag()) { .Vector => return self.fail("TODO implement mul_with_overflow for Vector type", .{}), .Int => { - const int_info = ty.intInfo(self.target.*); - - if (int_info.bits > 64) { + if (abi_size > 8) { return self.fail("TODO implement mul_with_overflow for Ints larger than 64bits", .{}); } - if (math.isPowerOfTwo(int_info.bits)) { + const int_info = ty.intInfo(self.target.*); + + if (math.isPowerOfTwo(int_info.bits) and int_info.bits >= 8) { try self.spillCompareFlagsIfOccupied(); self.compare_flags_inst = inst; @@ -1428,71 +1514,14 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { }, } }; - const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg); - defer self.register_manager.unlockReg(dst_reg_lock); const tuple_ty = self.air.typeOfIndex(inst); const tuple_size = @intCast(u32, tuple_ty.abiSize(self.target.*)); const tuple_align = tuple_ty.abiAlignment(self.target.*); const overflow_bit_offset = @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)); - const stack_offset = @intCast(i32, try self.allocMem(inst, tuple_size, tuple_align)); - const extended_ty = switch (int_info.signedness) { - .signed => Type.isize, - .unsigned => ty, - }; - const temp_regs = try self.register_manager.allocRegs(3, .{ null, null, null }); - const temp_regs_locks = self.register_manager.lockRegsAssumeUnused(3, temp_regs); - defer for (temp_regs_locks) |reg| { - self.register_manager.unlockReg(reg); - }; - - const overflow_reg = temp_regs[0]; - const flags: u2 = switch (int_info.signedness) { - .signed => 0b00, - .unsigned => 0b10, - }; - _ = try self.addInst(.{ - .tag = .cond_set_byte_overflow, - .ops = (Mir.Ops{ - .reg1 = overflow_reg.to8(), - .flags = flags, - }).encode(), - .data = undefined, - }); - - const scratch_reg = temp_regs[1]; - try self.genSetReg(extended_ty, scratch_reg, .{ .register = dst_reg }); - try self.truncateRegister(ty, scratch_reg); - try self.genBinOpMir( - .cmp, - extended_ty, - .{ .register = dst_reg }, - .{ .register = scratch_reg }, - ); - - const eq_reg = temp_regs[2]; - _ = try self.addInst(.{ - .tag = .cond_set_byte_eq_ne, - .ops = (Mir.Ops{ - .reg1 = eq_reg.to8(), - .flags = 0b00, - }).encode(), - .data = undefined, - }); - - try self.genBinOpMir( - .@"or", - Type.u8, - .{ .register = overflow_reg }, - .{ .register = eq_reg }, - ); - - try self.genSetStack(ty, stack_offset, .{ .register = scratch_reg }, .{}); - try self.genSetStack(Type.initTag(.u1), stack_offset - overflow_bit_offset, .{ - .register = overflow_reg.to8(), - }, .{}); + try self.genSetStackTruncatedOverflowCompare(ty, stack_offset, overflow_bit_offset, dst_reg); break :result MCValue{ .stack_offset = stack_offset }; }, diff --git a/test/behavior/math.zig b/test/behavior/math.zig index 53b11d1d9f..201fccb8ae 100644 --- a/test/behavior/math.zig +++ b/test/behavior/math.zig @@ -640,7 +640,6 @@ test "@addWithOverflow" { test "small int addition" { if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO From d31875f7abcad00dc95211e7395d80baa879cb93 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 10 May 2022 20:53:44 +0200 Subject: [PATCH 15/18] x64: implement shl_with_overflow for powers of two --- src/arch/x86_64/CodeGen.zig | 104 +++++++++++++++++++++++++++++++++++- test/behavior/math.zig | 1 - 2 files changed, 102 insertions(+), 3 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 15610f1bd2..8353f9b85e 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -1533,8 +1533,108 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { } fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void { - _ = inst; - return self.fail("TODO implement airShlWithOverflow for {}", .{self.target.cpu.arch}); + const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; + const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; + + if (self.liveness.isUnused(inst)) { + return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none }); + } + + const lhs_ty = self.air.typeOf(bin_op.lhs); + const abi_size = lhs_ty.abiSize(self.target.*); + const rhs_ty = self.air.typeOf(bin_op.rhs); + + const result: MCValue = result: { + switch (lhs_ty.zigTypeTag()) { + .Vector => return self.fail("TODO implement shl_with_overflow for Vector type", .{}), + .Int => { + if (abi_size > 8) { + return self.fail("TODO implement shl_with_overflow for Ints larger than 64bits", .{}); + } + + const int_info = lhs_ty.intInfo(self.target.*); + + if (math.isPowerOfTwo(int_info.bits) and int_info.bits >= 8) { + try self.spillCompareFlagsIfOccupied(); + self.compare_flags_inst = inst; + + try self.spillRegisters(1, .{.rcx}); + + const lhs = try self.resolveInst(bin_op.lhs); + const rhs = try self.resolveInst(bin_op.rhs); + + const partial = try self.genShiftBinOp(.shl, null, lhs, rhs, lhs_ty, rhs_ty); + break :result switch (int_info.signedness) { + .signed => MCValue{ .register_overflow_signed = partial.register }, + .unsigned => MCValue{ .register_overflow_unsigned = partial.register }, + }; + } + + return self.fail("TODO shl_with_overflow non-power-of-two", .{}); + + // try self.spillCompareFlagsIfOccupied(); + // self.compare_flags_inst = null; + + // const dst_reg: Register = dst_reg: { + // switch (int_info.signedness) { + // .signed => { + // const lhs = try self.resolveInst(bin_op.lhs); + // const rhs = try self.resolveInst(bin_op.rhs); + + // const rhs_lock: ?RegisterLock = switch (rhs) { + // .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + // else => null, + // }; + // defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); + + // const dst_reg: Register = blk: { + // if (lhs.isRegister()) break :blk lhs.register; + // break :blk try self.copyToTmpRegister(ty, lhs); + // }; + // const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg); + // defer self.register_manager.unlockReg(dst_reg_lock); + + // const rhs_mcv: MCValue = blk: { + // if (rhs.isRegister() or rhs.isMemory()) break :blk rhs; + // break :blk MCValue{ .register = try self.copyToTmpRegister(ty, rhs) }; + // }; + // const rhs_mcv_lock: ?RegisterLock = switch (rhs_mcv) { + // .register => |reg| self.register_manager.lockReg(reg), + // else => null, + // }; + // defer if (rhs_mcv_lock) |lock| self.register_manager.unlockReg(lock); + + // try self.genIntMulComplexOpMir(Type.isize, .{ .register = dst_reg }, rhs_mcv); + + // break :dst_reg dst_reg; + // }, + // .unsigned => { + // try self.spillRegisters(2, .{ .rax, .rdx }); + + // const lhs = try self.resolveInst(bin_op.lhs); + // const rhs = try self.resolveInst(bin_op.rhs); + + // const dst_mcv = try self.genMulDivBinOp(.mul, null, ty, lhs, rhs); + // break :dst_reg dst_mcv.register; + // }, + // } + // }; + + // const tuple_ty = self.air.typeOfIndex(inst); + // const tuple_size = @intCast(u32, tuple_ty.abiSize(self.target.*)); + // const tuple_align = tuple_ty.abiAlignment(self.target.*); + // const overflow_bit_offset = @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)); + // const stack_offset = @intCast(i32, try self.allocMem(inst, tuple_size, tuple_align)); + + // try self.genSetStackTruncatedOverflowCompare(ty, stack_offset, overflow_bit_offset, dst_reg); + + // break :result MCValue{ .stack_offset = stack_offset }; + }, + else => unreachable, + } + }; + + return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); } /// Generates signed or unsigned integer multiplication/division. diff --git a/test/behavior/math.zig b/test/behavior/math.zig index 201fccb8ae..91ce4b34cb 100644 --- a/test/behavior/math.zig +++ b/test/behavior/math.zig @@ -905,7 +905,6 @@ test "@subWithOverflow" { test "@shlWithOverflow" { if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO var result: u16 = undefined; try expect(@shlWithOverflow(u16, 0b0010111111111111, 3, &result)); From 1d3b714125e98e4c135dbb9d2718e58984ab92eb Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 10 May 2022 21:19:05 +0200 Subject: [PATCH 16/18] x64: implement shl with overflow for non-pow-2 --- src/arch/x86_64/CodeGen.zig | 132 +++++------------------------------- test/behavior/math.zig | 51 ++++++++++---- 2 files changed, 56 insertions(+), 127 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 8353f9b85e..087f7e3d62 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -621,10 +621,10 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { .trunc_float, => try self.airUnaryMath(inst), - .add_with_overflow => try self.airAddSubWithOverflow(inst), - .sub_with_overflow => try self.airAddSubWithOverflow(inst), + .add_with_overflow => try self.airAddSubShlWithOverflow(inst), + .sub_with_overflow => try self.airAddSubShlWithOverflow(inst), .mul_with_overflow => try self.airMulWithOverflow(inst), - .shl_with_overflow => try self.airShlWithOverflow(inst), + .shl_with_overflow => try self.airAddSubShlWithOverflow(inst), .div_float, .div_trunc, .div_floor, .div_exact => try self.airMulDivBinOp(inst), @@ -1305,7 +1305,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void { return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); } -fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void { +fn airAddSubShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void { const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; const tag = self.air.instructions.items(.tag)[inst]; const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; @@ -1313,23 +1313,30 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void { const ty = self.air.typeOf(bin_op.lhs); const abi_size = ty.abiSize(self.target.*); switch (ty.zigTypeTag()) { - .Vector => return self.fail("TODO implement add_with_overflow for Vector type", .{}), + .Vector => return self.fail("TODO implement add/sub/shl with overflow for Vector type", .{}), .Int => { if (abi_size > 8) { - return self.fail("TODO implement add_with_overflow for Ints larger than 64bits", .{}); + return self.fail("TODO implement add/sub/shl with overflow for Ints larger than 64bits", .{}); } try self.spillCompareFlagsIfOccupied(); + if (tag == .shl_with_overflow) { + try self.spillRegisters(1, .{.rcx}); + } + const lhs = try self.resolveInst(bin_op.lhs); const rhs = try self.resolveInst(bin_op.rhs); - const base_tag: Air.Inst.Tag = switch (tag) { - .add_with_overflow => .add, - .sub_with_overflow => .sub, + const partial: MCValue = switch (tag) { + .add_with_overflow => try self.genBinOp(.add, null, lhs, rhs, ty, ty), + .sub_with_overflow => try self.genBinOp(.sub, null, lhs, rhs, ty, ty), + .shl_with_overflow => blk: { + const shift_ty = self.air.typeOf(bin_op.rhs); + break :blk try self.genShiftBinOp(.shl, null, lhs, rhs, ty, shift_ty); + }, else => unreachable, }; - const partial = try self.genBinOp(base_tag, null, lhs, rhs, ty, ty); const int_info = ty.intInfo(self.target.*); @@ -1532,111 +1539,6 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); } -fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void { - const ty_pl = self.air.instructions.items(.data)[inst].ty_pl; - const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; - - if (self.liveness.isUnused(inst)) { - return self.finishAir(inst, .dead, .{ bin_op.lhs, bin_op.rhs, .none }); - } - - const lhs_ty = self.air.typeOf(bin_op.lhs); - const abi_size = lhs_ty.abiSize(self.target.*); - const rhs_ty = self.air.typeOf(bin_op.rhs); - - const result: MCValue = result: { - switch (lhs_ty.zigTypeTag()) { - .Vector => return self.fail("TODO implement shl_with_overflow for Vector type", .{}), - .Int => { - if (abi_size > 8) { - return self.fail("TODO implement shl_with_overflow for Ints larger than 64bits", .{}); - } - - const int_info = lhs_ty.intInfo(self.target.*); - - if (math.isPowerOfTwo(int_info.bits) and int_info.bits >= 8) { - try self.spillCompareFlagsIfOccupied(); - self.compare_flags_inst = inst; - - try self.spillRegisters(1, .{.rcx}); - - const lhs = try self.resolveInst(bin_op.lhs); - const rhs = try self.resolveInst(bin_op.rhs); - - const partial = try self.genShiftBinOp(.shl, null, lhs, rhs, lhs_ty, rhs_ty); - break :result switch (int_info.signedness) { - .signed => MCValue{ .register_overflow_signed = partial.register }, - .unsigned => MCValue{ .register_overflow_unsigned = partial.register }, - }; - } - - return self.fail("TODO shl_with_overflow non-power-of-two", .{}); - - // try self.spillCompareFlagsIfOccupied(); - // self.compare_flags_inst = null; - - // const dst_reg: Register = dst_reg: { - // switch (int_info.signedness) { - // .signed => { - // const lhs = try self.resolveInst(bin_op.lhs); - // const rhs = try self.resolveInst(bin_op.rhs); - - // const rhs_lock: ?RegisterLock = switch (rhs) { - // .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - // else => null, - // }; - // defer if (rhs_lock) |lock| self.register_manager.unlockReg(lock); - - // const dst_reg: Register = blk: { - // if (lhs.isRegister()) break :blk lhs.register; - // break :blk try self.copyToTmpRegister(ty, lhs); - // }; - // const dst_reg_lock = self.register_manager.lockRegAssumeUnused(dst_reg); - // defer self.register_manager.unlockReg(dst_reg_lock); - - // const rhs_mcv: MCValue = blk: { - // if (rhs.isRegister() or rhs.isMemory()) break :blk rhs; - // break :blk MCValue{ .register = try self.copyToTmpRegister(ty, rhs) }; - // }; - // const rhs_mcv_lock: ?RegisterLock = switch (rhs_mcv) { - // .register => |reg| self.register_manager.lockReg(reg), - // else => null, - // }; - // defer if (rhs_mcv_lock) |lock| self.register_manager.unlockReg(lock); - - // try self.genIntMulComplexOpMir(Type.isize, .{ .register = dst_reg }, rhs_mcv); - - // break :dst_reg dst_reg; - // }, - // .unsigned => { - // try self.spillRegisters(2, .{ .rax, .rdx }); - - // const lhs = try self.resolveInst(bin_op.lhs); - // const rhs = try self.resolveInst(bin_op.rhs); - - // const dst_mcv = try self.genMulDivBinOp(.mul, null, ty, lhs, rhs); - // break :dst_reg dst_mcv.register; - // }, - // } - // }; - - // const tuple_ty = self.air.typeOfIndex(inst); - // const tuple_size = @intCast(u32, tuple_ty.abiSize(self.target.*)); - // const tuple_align = tuple_ty.abiAlignment(self.target.*); - // const overflow_bit_offset = @intCast(i32, tuple_ty.structFieldOffset(1, self.target.*)); - // const stack_offset = @intCast(i32, try self.allocMem(inst, tuple_size, tuple_align)); - - // try self.genSetStackTruncatedOverflowCompare(ty, stack_offset, overflow_bit_offset, dst_reg); - - // break :result MCValue{ .stack_offset = stack_offset }; - }, - else => unreachable, - } - }; - - return self.finishAir(inst, result, .{ bin_op.lhs, bin_op.rhs, .none }); -} - /// Generates signed or unsigned integer multiplication/division. /// Clobbers .rax and .rdx registers. /// Quotient is saved in .rax and remainder in .rdx. diff --git a/test/behavior/math.zig b/test/behavior/math.zig index 91ce4b34cb..8dc21c0598 100644 --- a/test/behavior/math.zig +++ b/test/behavior/math.zig @@ -905,20 +905,47 @@ test "@subWithOverflow" { test "@shlWithOverflow" { if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO + if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO - var result: u16 = undefined; - try expect(@shlWithOverflow(u16, 0b0010111111111111, 3, &result)); - try expect(result == 0b0111111111111000); - try expect(!@shlWithOverflow(u16, 0b0010111111111111, 2, &result)); - try expect(result == 0b1011111111111100); + { + var result: u4 = undefined; + var a: u4 = 2; + var b: u2 = 1; + try expect(!@shlWithOverflow(u4, a, b, &result)); + try expect(result == 4); - var a: u16 = 0b0000_0000_0000_0011; - var b: u4 = 15; - try expect(@shlWithOverflow(u16, a, b, &result)); - try expect(result == 0b1000_0000_0000_0000); - b = 14; - try expect(!@shlWithOverflow(u16, a, b, &result)); - try expect(result == 0b1100_0000_0000_0000); + b = 3; + try expect(@shlWithOverflow(u4, a, b, &result)); + try expect(result == 0); + } + + { + var result: i9 = undefined; + var a: i9 = 127; + var b: u4 = 1; + try expect(!@shlWithOverflow(i9, a, b, &result)); + try expect(result == 254); + + b = 2; + try expect(@shlWithOverflow(i9, a, b, &result)); + try expect(result == -4); + } + + { + var result: u16 = undefined; + try expect(@shlWithOverflow(u16, 0b0010111111111111, 3, &result)); + try expect(result == 0b0111111111111000); + try expect(!@shlWithOverflow(u16, 0b0010111111111111, 2, &result)); + try expect(result == 0b1011111111111100); + + var a: u16 = 0b0000_0000_0000_0011; + var b: u4 = 15; + try expect(@shlWithOverflow(u16, a, b, &result)); + try expect(result == 0b1000_0000_0000_0000); + b = 14; + try expect(!@shlWithOverflow(u16, a, b, &result)); + try expect(result == 0b1100_0000_0000_0000); + } } test "overflow arithmetic with u0 values" { From f6f98a621f70a10078fe3de688fe4f8c2e027a30 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 10 May 2022 21:21:09 +0200 Subject: [PATCH 17/18] x64: enable additional math test --- test/behavior/math.zig | 1 - 1 file changed, 1 deletion(-) diff --git a/test/behavior/math.zig b/test/behavior/math.zig index 8dc21c0598..85b62e0275 100644 --- a/test/behavior/math.zig +++ b/test/behavior/math.zig @@ -9,7 +9,6 @@ const mem = std.mem; const math = std.math; test "assignment operators" { - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO From 3c69810fe6660fa3115d345bb60b3f811ac7c8c0 Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 10 May 2022 21:30:39 +0200 Subject: [PATCH 18/18] x64: fix binary not implementation --- src/arch/x86_64/CodeGen.zig | 3 ++- test/behavior/math.zig | 1 - 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 087f7e3d62..be053f310c 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -1154,7 +1154,8 @@ fn airNot(self: *Self, inst: Air.Inst.Index) !void { }; defer if (dst_mcv_lock) |lock| self.register_manager.unlockReg(lock); - try self.genBinOpMir(.xor, operand_ty, dst_mcv, .{ .immediate = 1 }); + const mask = ~@as(u64, 0); + try self.genBinOpMir(.xor, operand_ty, dst_mcv, .{ .immediate = mask }); break :result dst_mcv; }; diff --git a/test/behavior/math.zig b/test/behavior/math.zig index 85b62e0275..53a3f61d14 100644 --- a/test/behavior/math.zig +++ b/test/behavior/math.zig @@ -362,7 +362,6 @@ fn comptimeAdd(comptime a: comptime_int, comptime b: comptime_int) comptime_int test "binary not" { if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO try expect(comptime x: { break :x ~@as(u16, 0b1010101010101010) == 0b0101010101010101;