diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index 88d186acef..5bba03b410 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -2418,7 +2418,7 @@ fn genBodyBlock(self: *CodeGen, body: []const Air.Inst.Index) InnerError!void { } fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { - @setEvalBranchQuota(13_900); + @setEvalBranchQuota(14_100); const pt = cg.pt; const zcu = pt.zcu; const ip = &zcu.intern_pool; @@ -2459,7 +2459,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .mul_sat => try cg.airMulSat(inst), .shl_sat => try cg.airShlSat(inst), - .add_with_overflow => try cg.airAddSubWithOverflow(inst), .sub_with_overflow => try cg.airAddSubWithOverflow(inst), .mul_with_overflow => try cg.airMulWithOverflow(inst), .shl_with_overflow => try cg.airShlWithOverflow(inst), @@ -28019,6 +28018,857 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { }; try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); }, + .add_with_overflow => |air_tag| if (use_old) try cg.airAddSubWithOverflow(inst) else fallback: { + const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; + const bin_op = cg.air.extraData(Air.Bin, ty_pl.payload).data; + if (cg.typeOf(bin_op.lhs).isVector(zcu)) break :fallback try cg.airAddSubWithOverflow(inst); + var ops = try cg.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + var res: [2]Temp = undefined; + cg.select(&res, &.{ ty_pl.ty.toType(), .u1 }, &ops, comptime &.{ .{ + .src_constraints = .{ .{ .exact_signed_int = 8 }, .{ .exact_signed_int = 8 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0b, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 8 }, .{ .exact_unsigned_int = 8 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0b, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .byte }, .{ .signed_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i8, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .po } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0b, .src1b, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._l, .sa, .dst0b, .uia(8, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .sa, .dst0b, .uia(8, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .@"test", .tmp0b, .sia(-1 << 7, .src0, .sub_smin), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .byte }, .{ .unsigned_int = .byte }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm8, .none } }, + .{ .src = .{ .imm8, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .b } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0b, .src1b, ._, ._ }, + .{ ._, ._, .@"and", .dst0b, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .cmp, .dst0b, .src1b, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 16 }, .{ .exact_signed_int = 16 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 16 }, .{ .exact_unsigned_int = 16 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, + .{ ._, ._l, .ro, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sa, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .ro, .tmp0d, .ui(1), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .word }, .{ .signed_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, + .{ ._, ._l, .ro, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sa, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .ro, .tmp0d, .ui(1), ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .b } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0w, .src1w, ._, ._ }, + .{ ._, ._, .@"and", .dst0w, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .cmp, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .required_features = .{ .fast_imm16, null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .b } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .@"and", .dst0w, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .cmp, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .word }, .{ .unsigned_int = .word }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm16, .none } }, + .{ .src = .{ .imm16, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .b } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .cmp, .dst0w, .src1w, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_signed_int = 32 }, .{ .exact_signed_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .exact_unsigned_int = 32 }, .{ .exact_unsigned_int = 32 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .bmi2, null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 31 }, .{ .exact_signed_int = 31 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ui(1), ._, ._ }, + .{ ._, ._, .add, .dst0d, .dst0d, ._, ._ }, + .{ ._, ._rx, .sa, .dst0d, .dst0d, .tmp0d, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .signed_int = .dword }, .{ .signed_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i32, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._l, .ro, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .dst0d, ._, ._ }, + .{ ._, ._r, .sa, .dst0d, .uia(32, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .ro, .tmp0d, .ui(1), ._, ._ }, + } }, + }, .{ + .src_constraints = .{ .{ .unsigned_int = .dword }, .{ .unsigned_int = .dword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .imm32, .none } }, + .{ .src = .{ .imm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .b } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0d, .src1d, ._, ._ }, + .{ ._, ._, .@"and", .dst0d, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .cmp, .dst0d, .src1d, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 64 }, .{ .exact_signed_int = 64 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_unsigned_int = 64 }, .{ .exact_unsigned_int = 64 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ .{ .exact_signed_int = 63 }, .{ .exact_signed_int = 63 }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ui(1), ._, ._ }, + .{ ._, ._, .add, .dst0q, .dst0q, ._, ._ }, + .{ ._, ._rx, .sa, .dst0q, .dst0q, .tmp0q, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .signed_int = .qword }, .{ .signed_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .mem, .none } }, + .{ .src = .{ .mem, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .i64, .kind = .{ .mut_rc = .{ .ref = .src1, .rc = .general_purpose } } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, + .{ ._, ._l, .ro, .dst0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .dst0q, ._, ._ }, + .{ ._, ._r, .sa, .dst0q, .uia(64, .src0, .sub_bit_size), ._, ._ }, + .{ ._, ._r, .ro, .tmp0q, .ui(1), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .extra_temps = .{ + .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .{ .cc = .b } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size), ._, ._ }, + .{ ._, ._, .add, .dst0q, .src1q, ._, ._ }, + .{ ._, ._, .bzhi, .dst0q, .dst0q, .tmp0q, ._ }, + .{ ._, ._, .cmp, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .unsigned_int = .qword }, .{ .unsigned_int = .qword }, .any }, + .patterns = &.{ + .{ .src = .{ .to_mut_gpr, .simm32, .none } }, + .{ .src = .{ .simm32, .to_mut_gpr, .none }, .commute = .{ 0, 1 } }, + .{ .src = .{ .to_mut_gpr, .to_gpr, .none } }, + }, + .dst_temps = .{ .{ .rc = .general_purpose }, .{ .cc = .b } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .dst0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .add, .src0q, .src1q, ._, ._ }, + .{ ._, ._, .@"and", .dst0q, .src0q, ._, ._ }, + .{ ._, ._, .cmp, .dst0q, .src1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .none }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._, .adc, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -16), ._, ._ }, + .{ ._, ._, .adc, .tmp1q, .memad(.src1q, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._o, .set, .mema(.dst0b, .add_src0_size), ._, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .{ .exact_remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .c } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sa(.src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsia(.src0q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .adc, .tmp1q, .memsia(.src1q, .@"8", .tmp0, .add_size), ._, ._ }, + .{ ._, ._, .mov, .memsia(.dst0q, .@"8", .tmp0, .add_src0_size), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._, .adc, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -16), ._, ._ }, + .{ ._, ._, .adc, .tmp1q, .memad(.src1q, .add_size, -16), ._, ._ }, + .{ ._, ._l, .ro, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp1q, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .ui(63), ._, ._ }, + .{ ._, ._r, .ro, .tmp0q, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .rc = .general_purpose } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .dst1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._, .adc, .dst1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -16), .dst1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .dst1q, .memad(.src0q, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .adc, .dst1q, .memad(.src1q, .add_size, -16), ._, ._ }, + .{ ._, ._, .bzhi, .tmp0q, .dst1q, .tmp0q, ._ }, + .{ ._, ._r, .sh, .dst1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .rc = .general_purpose } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(2, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .dst1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._, .adc, .dst1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -16), .dst1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .dst1q, .memad(.src0q, .add_size, -16), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .adc, .dst1q, .memad(.src1q, .add_size, -16), ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .dst1q, ._, ._ }, + .{ ._, ._r, .sh, .dst1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -16), .tmp0q, ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .ui(0), ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_signed_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .cc = .o } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .tmp1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .tmp1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .tmp1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .tmp1q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._l, .ro, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .tmp1q, ._, ._ }, + .{ ._, ._r, .sa, .tmp1q, .uia(64, .src0, .sub_bit_size_rem_64), ._, ._ }, + .{ ._, ._r, .ro, .tmp0q, .ui(1), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp1q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", .bmi2, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .rc = .general_purpose } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .dst1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .dst1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .dst1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .dst1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp0d, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .adc, .dst1q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .bzhi, .tmp0q, .dst1q, .tmp0q, ._ }, + .{ ._, ._r, .sh, .dst1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .{ .remainder_unsigned_int = .{ .of = .xword, .is = .xword } }, + .any, + }, + .patterns = &.{ + .{ .src = .{ .to_mem, .to_mem, .none } }, + }, + .extra_temps = .{ + .{ .type = .isize, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .mem, .{ .rc = .general_purpose } }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0p, .sia(1, .src0, .sub_size_div_8), ._, ._ }, + .{ ._, ._c, .cl, ._, ._, ._, ._ }, + .{ .@"0:", ._, .mov, .dst1q, .memsiad(.src0q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .adc, .dst1q, .memsiad(.src1q, .@"8", .tmp0, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .memsiad(.dst0q, .@"8", .tmp0, .add_src0_size, -8), .dst1q, ._, ._ }, + .{ ._, ._c, .in, .tmp0p, ._, ._, ._ }, + .{ ._, ._nz, .j, .@"0b", ._, ._, ._ }, + .{ ._, ._, .mov, .dst1q, .memad(.src0q, .add_size, -8), ._, ._ }, + .{ ._, ._, .mov, .tmp0q, .ua(.src0, .add_umax), ._, ._ }, + .{ ._, ._, .adc, .dst1q, .memad(.src1q, .add_size, -8), ._, ._ }, + .{ ._, ._, .@"and", .tmp0q, .dst1q, ._, ._ }, + .{ ._, ._r, .sh, .dst1q, .ua(.src0, .add_bit_size_rem_64), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_src0_size, -8), .tmp0q, ._, ._ }, + } }, + } }) catch |err| switch (err) { + error.SelectFailed => return cg.fail("failed to select {s} {} {} {}", .{ + @tagName(air_tag), + ty_pl.ty.toType().fmt(pt), + ops[0].tracking(cg), + ops[1].tracking(cg), + }), + else => |e| return e, + }; + try res[0].withOverflow(&res[1], cg); + try res[0].finish(inst, &.{ bin_op.lhs, bin_op.rhs }, &ops, cg); + }, .alloc => if (use_old) try cg.airAlloc(inst) else { const ty = air_datas[@intFromEnum(inst)].ty; const slot = try cg.tempInit(ty, .{ .lea_frame = .{ @@ -104205,13 +105055,21 @@ fn memSize(self: *CodeGen, ty: Type) Memory.Size { fn splitType(self: *CodeGen, comptime parts_len: usize, ty: Type) ![parts_len]Type { const pt = self.pt; const zcu = pt.zcu; + const ip = &zcu.intern_pool; var parts: [parts_len]Type = undefined; - if (ty.isVector(zcu)) if (std.math.divExact(u32, ty.vectorLen(zcu), parts_len)) |vec_len| return .{ - try pt.vectorType(.{ .len = vec_len, .child = ty.scalarType(zcu).toIntern() }), - } ** parts_len else |err| switch (err) { - error.DivisionByZero => unreachable, - error.UnexpectedRemainder => {}, - }; + switch (ip.indexToKey(ty.toIntern())) { + .vector_type => |vector_type| if (std.math.divExact(u32, vector_type.len, parts_len)) |vec_len| return .{ + try pt.vectorType(.{ .len = vec_len, .child = vector_type.child }), + } ** parts_len else |err| switch (err) { + error.DivisionByZero => unreachable, + error.UnexpectedRemainder => {}, + }, + .tuple_type => |tuple_type| if (tuple_type.types.len == parts_len) { + for (&parts, tuple_type.types.get(ip)) |*part, part_ty| part.* = .fromInterned(part_ty); + return parts; + }, + else => {}, + } const classes = std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target, .other), .none); if (classes.len == parts_len) for (&parts, classes, 0..) |*part, class, part_i| { part.* = switch (class) { @@ -104896,6 +105754,52 @@ const Temp = struct { second_temp.* = result_temp; } + fn withOverflow(temp: *Temp, overflow_temp: *Temp, cg: *CodeGen) InnerError!void { + const temp_index = temp.unwrap(cg).temp; + const temp_tracking = temp_index.tracking(cg); + const overflow_temp_tracking = overflow_temp.unwrap(cg).temp.tracking(cg); + switch (temp_tracking.short) { + .register => |reg| switch (overflow_temp_tracking.short) { + .eflags => |overflow_cc| { + const result_temp_index = cg.next_temp_index; + const result_temp: Temp = .{ .index = result_temp_index.toIndex() }; + assert(cg.reuseTemp(result_temp.index, temp.index, temp_tracking)); + assert(cg.reuseTemp(result_temp.index, overflow_temp.index, overflow_temp_tracking)); + result_temp_index.tracking(cg).* = .init(.{ .register_overflow = .{ .reg = reg, .eflags = overflow_cc } }); + cg.temp_type[@intFromEnum(result_temp_index)] = .slice_const_u8; + cg.next_temp_index = @enumFromInt(@intFromEnum(result_temp_index) + 1); + temp.* = result_temp; + overflow_temp.* = result_temp; + return; + }, + .register => |overflow_reg| { + const result_temp_index = cg.next_temp_index; + const result_temp: Temp = .{ .index = result_temp_index.toIndex() }; + assert(cg.reuseTemp(result_temp.index, temp.index, temp_tracking)); + assert(cg.reuseTemp(result_temp.index, overflow_temp.index, overflow_temp_tracking)); + result_temp_index.tracking(cg).* = .init(.{ .register_pair = .{ reg, overflow_reg } }); + cg.temp_type[@intFromEnum(result_temp_index)] = .slice_const_u8; + cg.next_temp_index = @enumFromInt(@intFromEnum(result_temp_index) + 1); + temp.* = result_temp; + overflow_temp.* = result_temp; + return; + }, + else => {}, + }, + .load_frame => { + const zcu = cg.pt.zcu; + const ip = &zcu.intern_pool; + const field_tys = ip.indexToKey(temp_index.typeOf(cg).toIntern()).tuple_type.types.get(ip); + try temp.write(overflow_temp, .{ .disp = @intCast(Type.fromInterned(field_tys[0]).abiSize(zcu)) }, cg); + try overflow_temp.die(cg); + overflow_temp.* = temp.*; + return; + }, + else => {}, + } + std.debug.panic("{s}: {} {}\n", .{ @src().fn_name, temp_tracking, overflow_temp_tracking }); + } + fn asMask(temp: Temp, info: MaskInfo, cg: *CodeGen) void { assert(info.scalar != .none); const mcv = &temp.unwrap(cg).temp.tracking(cg).short; @@ -105581,6 +106485,42 @@ const Temp = struct { }, .dst_temps = .{ .{ .ref = .src0 }, .unused }, .each = .{ .once = &.{} }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + }, + .extra_temps = .{ + .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + .unused, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .clobbers = .{ .eflags = true }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .tmp0q, .memad(.src0q, .add_size, -16), ._, ._ }, + .{ ._, ._r, .sa, .tmp0q, .ui(63), ._, ._ }, + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .tmp0q, ._, ._ }, + } }, + }, .{ + .required_features = .{ .@"64bit", null, null, null }, + .src_constraints = .{ .{ .exact_remainder_unsigned_int = .{ .of = .xword, .is = .qword } }, .any, .any }, + .patterns = &.{ + .{ .src = .{ .mut_mem, .none, .none } }, + }, + .dst_temps = .{ .{ .ref = .src0 }, .unused }, + .each = .{ .once = &.{ + .{ ._, ._, .mov, .memad(.dst0q, .add_size, -8), .si(0), ._, ._ }, + } }, }, .{ .required_features = .{ .@"64bit", null, null, null }, .src_constraints = .{ .{ .remainder_signed_int = .{ .of = .xword, .is = .qword } }, .any, .any }, @@ -110362,15 +111302,18 @@ const Select = struct { return switch (src) { .none => temp.tracking(cg).short == .none, .imm8 => switch (temp.tracking(cg).short) { - .immediate => |imm| std.math.cast(u8, imm) != null, + .immediate => |imm| std.math.cast(u8, imm) != null or + std.math.cast(i8, @as(i64, @bitCast(imm))) != null, else => false, }, .imm16 => switch (temp.tracking(cg).short) { - .immediate => |imm| std.math.cast(u16, imm) != null, + .immediate => |imm| std.math.cast(u16, imm) != null or + std.math.cast(i16, @as(i64, @bitCast(imm))) != null, else => false, }, .imm32 => switch (temp.tracking(cg).short) { - .immediate => |imm| std.math.cast(u32, imm) != null, + .immediate => |imm| std.math.cast(u32, imm) != null or + std.math.cast(i32, @as(i64, @bitCast(imm))) != null, else => false, }, .simm32 => switch (temp.tracking(cg).short) { @@ -110494,6 +111437,8 @@ const Select = struct { const Kind = union(enum) { unused, any, + none, + undef, cc: Condition, ref: Select.Operand.Ref, reg: Register, @@ -110580,6 +111525,8 @@ const Select = struct { return switch (spec.kind) { .unused => .{ undefined, false }, .any => .{ try cg.tempAlloc(spec.type), true }, + .none => .{ try cg.tempInit(spec.type, .none), true }, + .undef => .{ try cg.tempInit(spec.type, .undef), true }, .cc => |cc| .{ try cg.tempInit(spec.type, .{ .eflags = cc }), true }, .ref => |ref| .{ ref.tempOf(s), false }, .reg => |reg| .{ try cg.tempInit(spec.type, .{ .register = reg }), true }, @@ -110922,6 +111869,7 @@ const Select = struct { const sub_size: Adjust = .{ .sign = .neg, .lhs = .size, .op = .mul, .rhs = .@"1" }; const sub_src0_size_div_8: Adjust = .{ .sign = .neg, .lhs = .src0_size, .op = .div, .rhs = .@"8" }; const sub_src0_size: Adjust = .{ .sign = .neg, .lhs = .src0_size, .op = .mul, .rhs = .@"1" }; + const add_src0_size: Adjust = .{ .sign = .pos, .lhs = .src0_size, .op = .mul, .rhs = .@"1" }; const add_8_src0_size: Adjust = .{ .sign = .pos, .lhs = .src0_size, .op = .mul, .rhs = .@"8" }; const add_delta_size_div_8: Adjust = .{ .sign = .pos, .lhs = .delta_size, .op = .div, .rhs = .@"8" }; const add_delta_elem_size: Adjust = .{ .sign = .pos, .lhs = .delta_elem_size, .op = .mul, .rhs = .@"1" }; @@ -110965,6 +111913,7 @@ const Select = struct { const add_dst0_elem_size: Adjust = .{ .sign = .pos, .lhs = .dst0_elem_size, .op = .mul, .rhs = .@"1" }; const add_elem_limbs: Adjust = .{ .sign = .pos, .lhs = .elem_limbs, .op = .mul, .rhs = .@"1" }; const add_smin: Adjust = .{ .sign = .pos, .lhs = .smin, .op = .mul, .rhs = .@"1" }; + const sub_smin: Adjust = .{ .sign = .neg, .lhs = .smin, .op = .mul, .rhs = .@"1" }; const add_umax: Adjust = .{ .sign = .pos, .lhs = .umax, .op = .mul, .rhs = .@"1" }; const repeat: Adjust = .{ .sign = .pos, .lhs = .repeat, .op = .mul, .rhs = .@"1" }; }; @@ -111783,6 +112732,7 @@ fn select( if (std.debug.runtime_safety) { for (case.src_constraints[src_temps.len..]) |src_constraint| assert(src_constraint == .any); for (case.dst_constraints[dst_temps.len..]) |dst_constraint| assert(dst_constraint == .any); + for (case.dst_temps[dst_temps.len..]) |dst_temp| assert(dst_temp == .unused); } patterns: for (case.patterns) |pattern| { for (pattern.src[0..src_temps.len], src_temps) |src_pattern, src_temp| if (!src_pattern.matches(src_temp, cg)) continue :patterns; diff --git a/src/arch/x86_64/abi.zig b/src/arch/x86_64/abi.zig index 6d11a44c2a..f219cc118d 100644 --- a/src/arch/x86_64/abi.zig +++ b/src/arch/x86_64/abi.zig @@ -266,7 +266,8 @@ pub fn classifySystemV(ty: Type, zcu: *Zcu, target: *const std.Target, ctx: Cont // separately.". const ty_size = ty.abiSize(zcu); switch (ty.containerLayout(zcu)) { - .auto, .@"extern" => {}, + .auto => unreachable, + .@"extern" => {}, .@"packed" => { assert(ty_size <= 16); result[0] = .integer; @@ -345,7 +346,8 @@ fn classifySystemVStruct( ); if (zcu.typeToStruct(field_ty)) |field_loaded_struct| { switch (field_loaded_struct.layout) { - .auto, .@"extern" => { + .auto => unreachable, + .@"extern" => { byte_offset = classifySystemVStruct(result, byte_offset, field_loaded_struct, zcu, target); continue; }, @@ -353,7 +355,8 @@ fn classifySystemVStruct( } } else if (zcu.typeToUnion(field_ty)) |field_loaded_union| { switch (field_loaded_union.flagsUnordered(ip).layout) { - .auto, .@"extern" => { + .auto => unreachable, + .@"extern" => { byte_offset = classifySystemVUnion(result, byte_offset, field_loaded_union, zcu, target); continue; }, @@ -386,7 +389,8 @@ fn classifySystemVUnion( const field_ty = Type.fromInterned(loaded_union.field_types.get(ip)[field_index]); if (zcu.typeToStruct(field_ty)) |field_loaded_struct| { switch (field_loaded_struct.layout) { - .auto, .@"extern" => { + .auto => unreachable, + .@"extern" => { _ = classifySystemVStruct(result, starting_byte_offset, field_loaded_struct, zcu, target); continue; }, @@ -394,7 +398,8 @@ fn classifySystemVUnion( } } else if (zcu.typeToUnion(field_ty)) |field_loaded_union| { switch (field_loaded_union.flagsUnordered(ip).layout) { - .auto, .@"extern" => { + .auto => unreachable, + .@"extern" => { _ = classifySystemVUnion(result, starting_byte_offset, field_loaded_union, zcu, target); continue; }, diff --git a/test/behavior/x86_64/binary.zig b/test/behavior/x86_64/binary.zig index a7aeda1665..b5a7762fcc 100644 --- a/test/behavior/x86_64/binary.zig +++ b/test/behavior/x86_64/binary.zig @@ -5262,6 +5262,17 @@ test addUnsafe { try test_add_unsafe.testFloatVectors(); } +inline fn addSafe(comptime Type: type, lhs: Type, rhs: Type) AddOneBit(Type) { + @setRuntimeSafety(true); + return @as(AddOneBit(Type), lhs) + rhs; +} +test addSafe { + const test_add_safe = binary(addSafe, .{}); + try test_add_safe.testInts(); + try test_add_safe.testFloats(); + try test_add_safe.testFloatVectors(); +} + inline fn addWrap(comptime Type: type, lhs: Type, rhs: Type) Type { return lhs +% rhs; } @@ -5416,6 +5427,14 @@ test min { try test_min.testFloatVectors(); } +inline fn addWithOverflow(comptime Type: type, lhs: Type, rhs: Type) struct { Type, u1 } { + return @addWithOverflow(lhs, rhs); +} +test addWithOverflow { + const test_add_with_overflow = binary(addWithOverflow, .{}); + try test_add_with_overflow.testInts(); +} + inline fn equal(comptime Type: type, lhs: Type, rhs: Type) @TypeOf(lhs == rhs) { return lhs == rhs; } @@ -5519,15 +5538,16 @@ test shl { try test_shl.testInts(); } -inline fn shlExact(comptime Type: type, lhs: Type, rhs: Type) Type { +inline fn shlExactUnsafe(comptime Type: type, lhs: Type, rhs: Type) Type { + @setRuntimeSafety(false); const bit_cast_rhs: @Type(.{ .int = .{ .signedness = .unsigned, .bits = @bitSizeOf(Type) } }) = @bitCast(rhs); const truncate_rhs: Log2Int(Type) = @truncate(bit_cast_rhs); const final_rhs = if (comptime cast(Log2Int(Type), @bitSizeOf(Type))) |bits| truncate_rhs % bits else truncate_rhs; return @shlExact(lhs << final_rhs >> final_rhs, final_rhs); } -test shlExact { - const test_shl_exact = binary(shlExact, .{}); - try test_shl_exact.testInts(); +test shlExactUnsafe { + const test_shl_exact_unsafe = binary(shlExactUnsafe, .{}); + try test_shl_exact_unsafe.testInts(); } inline fn bitXor(comptime Type: type, lhs: Type, rhs: Type) @TypeOf(lhs ^ rhs) { diff --git a/test/behavior/x86_64/math.zig b/test/behavior/x86_64/math.zig index f168737830..5b31876edc 100644 --- a/test/behavior/x86_64/math.zig +++ b/test/behavior/x86_64/math.zig @@ -153,6 +153,9 @@ pub noinline fn checkExpected(expected: anytype, actual: @TypeOf(expected), comp ); }, }, + .@"struct" => |@"struct"| inline for (@"struct".fields) |field| { + try checkExpected(@field(expected, field.name), @field(actual, field.name), compare); + } else return, }; if (switch (@typeInfo(Expected)) { else => unexpected,