diff --git a/src/Air.zig b/src/Air.zig index 4589bb1557..e76ffb3c39 100644 --- a/src/Air.zig +++ b/src/Air.zig @@ -893,14 +893,38 @@ pub const Inst = struct { pub const Index = enum(u32) { _, - pub fn toRef(i: Index) Inst.Ref { - assert(@intFromEnum(i) >> 31 == 0); - return @enumFromInt((1 << 31) | @intFromEnum(i)); + pub fn unwrap(index: Index) union(enum) { ref: Inst.Ref, target: u31 } { + const low_index: u31 = @truncate(@intFromEnum(index)); + return switch (@as(u1, @intCast(@intFromEnum(index) >> 31))) { + 0 => .{ .ref = @enumFromInt(@as(u32, 1 << 31) | low_index) }, + 1 => .{ .target = low_index }, + }; } - pub fn toTargetIndex(i: Index) u31 { - assert(@intFromEnum(i) >> 31 == 1); - return @truncate(@intFromEnum(i)); + pub fn toRef(index: Index) Inst.Ref { + return index.unwrap().ref; + } + + pub fn fromTargetIndex(index: u31) Index { + return @enumFromInt((1 << 31) | @as(u32, index)); + } + + pub fn toTargetIndex(index: Index) u31 { + return index.unwrap().target; + } + + pub fn format( + index: Index, + comptime _: []const u8, + _: std.fmt.FormatOptions, + writer: anytype, + ) @TypeOf(writer).Error!void { + try writer.writeByte('%'); + switch (index.unwrap()) { + .ref => {}, + .target => try writer.writeByte('t'), + } + try writer.print("{d}", .{@as(u31, @truncate(@intFromEnum(index)))}); } }; diff --git a/src/Compilation.zig b/src/Compilation.zig index 77596e34ec..1260bc028f 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -3067,6 +3067,7 @@ pub fn saveState(comp: *Compilation) !void { // linker state switch (lf.tag) { .wasm => { + dev.check(link.File.Tag.wasm.devFeature()); const wasm = lf.cast(.wasm).?; const is_obj = comp.config.output_mode == .Obj; try bufs.ensureUnusedCapacity(85); diff --git a/src/Liveness.zig b/src/Liveness.zig index 709844c0ac..e6ed782a21 100644 --- a/src/Liveness.zig +++ b/src/Liveness.zig @@ -202,14 +202,6 @@ pub fn operandDies(l: Liveness, inst: Air.Inst.Index, operand: OperandInt) bool return (l.tomb_bits[usize_index] & mask) != 0; } -pub fn clearOperandDeath(l: Liveness, inst: Air.Inst.Index, operand: OperandInt) void { - assert(operand < bpi - 1); - const usize_index = (@intFromEnum(inst) * bpi) / @bitSizeOf(usize); - const mask = @as(usize, 1) << - @as(Log2Int(usize), @intCast((@intFromEnum(inst) % (@bitSizeOf(usize) / bpi)) * bpi + operand)); - l.tomb_bits[usize_index] &= ~mask; -} - const OperandCategory = enum { /// The operand lives on, but this instruction cannot possibly mutate memory. none, @@ -844,12 +836,6 @@ const Analysis = struct { special: std.AutoHashMapUnmanaged(Air.Inst.Index, u32), extra: std.ArrayListUnmanaged(u32), - fn storeTombBits(a: *Analysis, inst: Air.Inst.Index, tomb_bits: Bpi) void { - const usize_index = (inst * bpi) / @bitSizeOf(usize); - a.tomb_bits[usize_index] |= @as(usize, tomb_bits) << - @as(Log2Int(usize), @intCast((inst % (@bitSizeOf(usize) / bpi)) * bpi)); - } - fn addExtra(a: *Analysis, extra: anytype) Allocator.Error!u32 { const fields = std.meta.fields(@TypeOf(extra)); try a.extra.ensureUnusedCapacity(a.gpa, fields.len); diff --git a/src/arch/aarch64/CodeGen.zig b/src/arch/aarch64/CodeGen.zig index c1657396ad..64230dfc9e 100644 --- a/src/arch/aarch64/CodeGen.zig +++ b/src/arch/aarch64/CodeGen.zig @@ -71,6 +71,8 @@ end_di_column: u32, /// which is a relative jump, based on the address following the reloc. exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty, +reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined, + /// We postpone the creation of debug info for function args and locals /// until after all Mir instructions have been generated. Only then we /// will know saved_regs_stack_space which is necessary in order to @@ -646,6 +648,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { const old_air_bookkeeping = self.air_bookkeeping; try self.ensureProcessDeathCapacity(Liveness.bpi); + self.reused_operands = @TypeOf(self.reused_operands).initEmpty(); switch (air_tags[@intFromEnum(inst)]) { // zig fmt: off .add => try self.airBinOp(inst, .add), @@ -927,16 +930,13 @@ fn finishAirBookkeeping(self: *Self) void { } fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Liveness.bpi - 1]Air.Inst.Ref) void { - var tomb_bits = self.liveness.getTombBits(inst); - for (operands) |op| { - const dies = @as(u1, @truncate(tomb_bits)) != 0; - tomb_bits >>= 1; - if (!dies) continue; - const op_index = op.toIndex() orelse continue; - self.processDeath(op_index); + const tomb_bits = self.liveness.getTombBits(inst); + for (0.., operands) |op_index, op| { + if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; + if (self.reused_operands.isSet(op_index)) continue; + self.processDeath(op.toIndexAllowNone() orelse continue); } - const is_used = @as(u1, @truncate(tomb_bits)) == 0; - if (is_used) { + if (tomb_bits & 1 << (Liveness.bpi - 1) == 0) { log.debug("%{d} => {}", .{ inst, result }); const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; branch.inst_table.putAssumeCapacityNoClobber(inst, result); @@ -3614,7 +3614,7 @@ fn reuseOperand( } // Prevent the operand deaths processing code from deallocating it. - self.liveness.clearOperandDeath(inst, op_index); + self.reused_operands.set(op_index); // That makes us responsible for doing the rest of the stuff that processDeath would have done. const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; diff --git a/src/arch/arm/CodeGen.zig b/src/arch/arm/CodeGen.zig index 10b0473ecf..fa892117f3 100644 --- a/src/arch/arm/CodeGen.zig +++ b/src/arch/arm/CodeGen.zig @@ -72,6 +72,8 @@ end_di_column: u32, /// which is a relative jump, based on the address following the reloc. exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty, +reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined, + /// We postpone the creation of debug info for function args and locals /// until after all Mir instructions have been generated. Only then we /// will know saved_regs_stack_space which is necessary in order to @@ -635,6 +637,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { const old_air_bookkeeping = self.air_bookkeeping; try self.ensureProcessDeathCapacity(Liveness.bpi); + self.reused_operands = @TypeOf(self.reused_operands).initEmpty(); switch (air_tags[@intFromEnum(inst)]) { // zig fmt: off .add, => try self.airBinOp(inst, .add), @@ -918,16 +921,13 @@ fn finishAirBookkeeping(self: *Self) void { } fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Liveness.bpi - 1]Air.Inst.Ref) void { - var tomb_bits = self.liveness.getTombBits(inst); - for (operands) |op| { - const dies = @as(u1, @truncate(tomb_bits)) != 0; - tomb_bits >>= 1; - if (!dies) continue; - const op_index = op.toIndex() orelse continue; - self.processDeath(op_index); + const tomb_bits = self.liveness.getTombBits(inst); + for (0.., operands) |op_index, op| { + if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; + if (self.reused_operands.isSet(op_index)) continue; + self.processDeath(op.toIndexAllowNone() orelse continue); } - const is_used = @as(u1, @truncate(tomb_bits)) == 0; - if (is_used) { + if (tomb_bits & 1 << (Liveness.bpi - 1) == 0) { log.debug("%{d} => {}", .{ inst, result }); const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; branch.inst_table.putAssumeCapacityNoClobber(inst, result); @@ -2650,7 +2650,7 @@ fn reuseOperand( } // Prevent the operand deaths processing code from deallocating it. - self.liveness.clearOperandDeath(inst, op_index); + self.reused_operands.set(op_index); // That makes us responsible for doing the rest of the stuff that processDeath would have done. const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; diff --git a/src/arch/riscv64/CodeGen.zig b/src/arch/riscv64/CodeGen.zig index c180b3aac9..66f4ce6f0d 100644 --- a/src/arch/riscv64/CodeGen.zig +++ b/src/arch/riscv64/CodeGen.zig @@ -82,6 +82,8 @@ scope_generation: u32, /// which is a relative jump, based on the address following the reloc. exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty, +reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined, + /// Whenever there is a runtime branch, we push a Branch onto this stack, /// and pop it off when the runtime branch joins. This provides an "overlay" /// of the table of mappings from instructions to `MCValue` from within the branch. @@ -1443,8 +1445,11 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void { verbose_tracking_log.debug("{}", .{func.fmtTracking()}); const old_air_bookkeeping = func.air_bookkeeping; + try func.ensureProcessDeathCapacity(Liveness.bpi); + + func.reused_operands = @TypeOf(func.reused_operands).initEmpty(); try func.inst_tracking.ensureUnusedCapacity(func.gpa, 1); - const tag: Air.Inst.Tag = air_tags[@intFromEnum(inst)]; + const tag = air_tags[@intFromEnum(inst)]; switch (tag) { // zig fmt: off .add, @@ -1783,11 +1788,10 @@ fn finishAir( result: MCValue, operands: [Liveness.bpi - 1]Air.Inst.Ref, ) !void { - var tomb_bits = func.liveness.getTombBits(inst); - for (operands) |op| { - const dies = @as(u1, @truncate(tomb_bits)) != 0; - tomb_bits >>= 1; - if (!dies) continue; + const tomb_bits = func.liveness.getTombBits(inst); + for (0.., operands) |op_index, op| { + if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; + if (func.reused_operands.isSet(op_index)) continue; try func.processDeath(op.toIndexAllowNone() orelse continue); } func.finishAirResult(inst, result); @@ -4424,7 +4428,7 @@ fn reuseOperandAdvanced( } // Prevent the operand deaths processing code from deallocating it. - func.liveness.clearOperandDeath(inst, op_index); + func.reused_operands.set(op_index); const op_inst = operand.toIndex().?; func.getResolvedInstValue(op_inst).reuse(func, maybe_tracked_inst, op_inst); diff --git a/src/arch/sparc64/CodeGen.zig b/src/arch/sparc64/CodeGen.zig index 0957789325..240123ee51 100644 --- a/src/arch/sparc64/CodeGen.zig +++ b/src/arch/sparc64/CodeGen.zig @@ -78,6 +78,8 @@ end_di_column: u32, /// which is a relative jump, based on the address following the reloc. exitlude_jump_relocs: std.ArrayListUnmanaged(usize) = .empty, +reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined, + /// Whenever there is a runtime branch, we push a Branch onto this stack, /// and pop it off when the runtime branch joins. This provides an "overlay" /// of the table of mappings from instructions to `MCValue` from within the branch. @@ -493,6 +495,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { const old_air_bookkeeping = self.air_bookkeeping; try self.ensureProcessDeathCapacity(Liveness.bpi); + self.reused_operands = @TypeOf(self.reused_operands).initEmpty(); switch (air_tags[@intFromEnum(inst)]) { // zig fmt: off .ptr_add => try self.airPtrArithmetic(inst, .ptr_add), @@ -3523,16 +3526,13 @@ fn finishAirBookkeeping(self: *Self) void { } fn finishAir(self: *Self, inst: Air.Inst.Index, result: MCValue, operands: [Liveness.bpi - 1]Air.Inst.Ref) void { - var tomb_bits = self.liveness.getTombBits(inst); - for (operands) |op| { - const dies = @as(u1, @truncate(tomb_bits)) != 0; - tomb_bits >>= 1; - if (!dies) continue; - const op_index = op.toIndex() orelse continue; - self.processDeath(op_index); + const tomb_bits = self.liveness.getTombBits(inst); + for (0.., operands) |op_index, op| { + if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; + if (self.reused_operands.isSet(op_index)) continue; + self.processDeath(op.toIndexAllowNone() orelse continue); } - const is_used = @as(u1, @truncate(tomb_bits)) == 0; - if (is_used) { + if (tomb_bits & 1 << (Liveness.bpi - 1) == 0) { log.debug("%{d} => {}", .{ inst, result }); const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; branch.inst_table.putAssumeCapacityNoClobber(inst, result); @@ -4568,7 +4568,7 @@ fn reuseOperand(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, op_ind } // Prevent the operand deaths processing code from deallocating it. - self.liveness.clearOperandDeath(inst, op_index); + self.reused_operands.set(op_index); // That makes us responsible for doing the rest of the stuff that processDeath would have done. const branch = &self.branch_stack.items[self.branch_stack.items.len - 1]; diff --git a/src/arch/x86_64/CodeGen.zig b/src/arch/x86_64/CodeGen.zig index dbb0179ebb..3bbc80999a 100644 --- a/src/arch/x86_64/CodeGen.zig +++ b/src/arch/x86_64/CodeGen.zig @@ -1,41 +1,26 @@ const std = @import("std"); -const build_options = @import("build_options"); -const builtin = @import("builtin"); const assert = std.debug.assert; const codegen = @import("../../codegen.zig"); -const leb128 = std.leb; const link = @import("../../link.zig"); const log = std.log.scoped(.codegen); const tracking_log = std.log.scoped(.tracking); const verbose_tracking_log = std.log.scoped(.verbose_tracking); const wip_mir_log = std.log.scoped(.wip_mir); -const math = std.math; -const mem = std.mem; -const target_util = @import("../../target.zig"); -const trace = @import("../../tracy.zig").trace; const Air = @import("../../Air.zig"); -const Allocator = mem.Allocator; -const CodeGenError = codegen.CodeGenError; -const Compilation = @import("../../Compilation.zig"); -const ErrorMsg = Zcu.ErrorMsg; +const Allocator = std.mem.Allocator; const Emit = @import("Emit.zig"); const Liveness = @import("../../Liveness.zig"); const Lower = @import("Lower.zig"); const Mir = @import("Mir.zig"); -const Package = @import("../../Package.zig"); const Zcu = @import("../../Zcu.zig"); +const Module = @import("../../Package/Module.zig"); const InternPool = @import("../../InternPool.zig"); -const Alignment = InternPool.Alignment; -const Target = std.Target; const Type = @import("../../Type.zig"); const Value = @import("../../Value.zig"); -const Instruction = @import("encoder.zig").Instruction; const abi = @import("abi.zig"); const bits = @import("bits.zig"); -const errUnionErrorOffset = codegen.errUnionErrorOffset; -const errUnionPayloadOffset = codegen.errUnionPayloadOffset; const encoder = @import("encoder.zig"); const Condition = bits.Condition; @@ -46,7 +31,7 @@ const RegisterManager = abi.RegisterManager; const RegisterLock = RegisterManager.RegisterLock; const FrameIndex = bits.FrameIndex; -const InnerError = CodeGenError || error{OutOfRegisters}; +const InnerError = codegen.CodeGenError || error{OutOfRegisters}; gpa: Allocator, pt: Zcu.PerThread, @@ -57,7 +42,7 @@ debug_output: link.File.DebugInfoOutput, target: *const std.Target, owner: Owner, inline_func: InternPool.Index, -mod: *Package.Module, +mod: *Module, arg_index: u32, args: []MCValue, va_info: union { @@ -89,6 +74,7 @@ end_di_column: u32, /// which is a relative jump, based on the address following the reloc. exitlude_jump_relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty, +reused_operands: std.StaticBitSet(Liveness.bpi - 1) = undefined, const_tracking: ConstTrackingMap = .{}, inst_tracking: InstTrackingMap = .{}, @@ -108,13 +94,11 @@ loops: std.AutoHashMapUnmanaged(Air.Inst.Index, struct { /// The state to restore before branching. state: State, /// The branch target. - jmp_target: Mir.Inst.Index, + target: Mir.Inst.Index, }) = .{}, -/// Debug field, used to find bugs in the compiler. -air_bookkeeping: @TypeOf(air_bookkeeping_init) = air_bookkeeping_init, - -const air_bookkeeping_init = if (std.debug.runtime_safety) @as(usize, 0) else {}; +next_temp_index: Temp.Index = @enumFromInt(0), +temp_type: [Temp.Index.max]Type = undefined, const Owner = union(enum) { nav_index: InternPool.Nav.Index, @@ -433,7 +417,7 @@ pub const MCValue = union(enum) { .reserved_frame, .lea_symbol, => unreachable, - .memory => |addr| if (math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr| .{ + .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr| .{ .base = .{ .reg = .ds }, .mod = .{ .rm = .{ .size = size, @@ -484,8 +468,8 @@ pub const MCValue = union(enum) { .register_overflow => |pl| try writer.print("{s}:{s}", .{ @tagName(pl.eflags), @tagName(pl.reg), }), - .load_symbol => |pl| try writer.print("[{} + 0x{x}]", .{ pl.sym_index, pl.off }), - .lea_symbol => |pl| try writer.print("{} + 0x{x}", .{ pl.sym_index, pl.off }), + .load_symbol => |pl| try writer.print("[sym:{} + 0x{x}]", .{ pl.sym_index, pl.off }), + .lea_symbol => |pl| try writer.print("sym:{} + 0x{x}", .{ pl.sym_index, pl.off }), .indirect => |pl| try writer.print("[{s} + 0x{x}]", .{ @tagName(pl.reg), pl.off }), .load_direct => |pl| try writer.print("[direct:{d}]", .{pl}), .lea_direct => |pl| try writer.print("direct:{d}", .{pl}), @@ -562,7 +546,7 @@ const InstTracking = struct { .reserved_frame => |index| self.long = .{ .load_frame = .{ .index = index } }, else => unreachable, } - tracking_log.debug("spill %{d} from {} to {}", .{ inst, self.short, self.long }); + tracking_log.debug("spill {} from {} to {}", .{ inst, self.short, self.long }); try function.genCopy(function.typeOfIndex(inst), self.long, self.short, .{}); } @@ -605,7 +589,7 @@ const InstTracking = struct { fn trackSpill(self: *InstTracking, function: *Self, inst: Air.Inst.Index) !void { try function.freeValue(self.short); self.reuseFrame(); - tracking_log.debug("%{d} => {} (spilled)", .{ inst, self.* }); + tracking_log.debug("{} => {} (spilled)", .{ inst, self.* }); } fn verifyMaterialize(self: InstTracking, target: InstTracking) void { @@ -678,14 +662,14 @@ const InstTracking = struct { else => target.long, } else target.long; self.short = target.short; - tracking_log.debug("%{d} => {} (materialize)", .{ inst, self.* }); + tracking_log.debug("{} => {} (materialize)", .{ inst, self.* }); } fn resurrect(self: *InstTracking, inst: Air.Inst.Index, scope_generation: u32) void { switch (self.short) { .dead => |die_generation| if (die_generation >= scope_generation) { self.reuseFrame(); - tracking_log.debug("%{d} => {} (resurrect)", .{ inst, self.* }); + tracking_log.debug("{} => {} (resurrect)", .{ inst, self.* }); }, else => {}, } @@ -695,7 +679,7 @@ const InstTracking = struct { if (self.short == .dead) return; try function.freeValue(self.short); self.short = .{ .dead = function.scope_generation }; - tracking_log.debug("%{d} => {} (death)", .{ inst, self.* }); + tracking_log.debug("{} => {} (death)", .{ inst, self.* }); } fn reuse( @@ -705,16 +689,13 @@ const InstTracking = struct { old_inst: Air.Inst.Index, ) void { self.short = .{ .dead = function.scope_generation }; - if (new_inst) |inst| - tracking_log.debug("%{d} => {} (reuse %{d})", .{ inst, self.*, old_inst }) - else - tracking_log.debug("tmp => {} (reuse %{d})", .{ self.*, old_inst }); + tracking_log.debug("{?} => {} (reuse {})", .{ new_inst, self.*, old_inst }); } fn liveOut(self: *InstTracking, function: *Self, inst: Air.Inst.Index) void { for (self.getRegs()) |reg| { if (function.register_manager.isRegFree(reg)) { - tracking_log.debug("%{d} => {} (live-out)", .{ inst, self.* }); + tracking_log.debug("{} => {} (live-out)", .{ inst, self.* }); continue; } @@ -741,7 +722,7 @@ const InstTracking = struct { // Perform side-effects of freeValue manually. function.register_manager.freeReg(reg); - tracking_log.debug("%{d} => {} (live-out %{d})", .{ inst, self.*, tracked_inst }); + tracking_log.debug("{} => {} (live-out {})", .{ inst, self.*, tracked_inst }); } } @@ -759,10 +740,10 @@ const InstTracking = struct { const FrameAlloc = struct { abi_size: u31, spill_pad: u3, - abi_align: Alignment, + abi_align: InternPool.Alignment, ref_count: u16, - fn init(alloc_abi: struct { size: u64, pad: u3 = 0, alignment: Alignment }) FrameAlloc { + fn init(alloc_abi: struct { size: u64, pad: u3 = 0, alignment: InternPool.Alignment }) FrameAlloc { return .{ .abi_size = @intCast(alloc_abi.size), .spill_pad = alloc_abi.pad, @@ -779,14 +760,14 @@ const FrameAlloc = struct { fn initSpill(ty: Type, zcu: *Zcu) FrameAlloc { const abi_size = ty.abiSize(zcu); const spill_size = if (abi_size < 8) - math.ceilPowerOfTwoAssert(u64, abi_size) + std.math.ceilPowerOfTwoAssert(u64, abi_size) else std.mem.alignForward(u64, abi_size, 8); return init(.{ .size = spill_size, .pad = @intCast(spill_size - abi_size), .alignment = ty.abiAlignment(zcu).maxStrict( - Alignment.fromNonzeroByteUnits(@min(spill_size, 8)), + InternPool.Alignment.fromNonzeroByteUnits(@min(spill_size, 8)), ), }); } @@ -819,7 +800,7 @@ pub fn generate( liveness: Liveness, code: *std.ArrayListUnmanaged(u8), debug_output: link.File.DebugInfoOutput, -) CodeGenError!void { +) codegen.CodeGenError!void { const zcu = pt.zcu; const comp = zcu.comp; const gpa = zcu.gpa; @@ -862,6 +843,11 @@ pub fn generate( function.mir_instructions.deinit(gpa); function.mir_extra.deinit(gpa); } + try function.inst_tracking.ensureTotalCapacity(gpa, Temp.Index.max); + for (0..Temp.Index.max) |temp_index| { + const temp: Temp.Index = @enumFromInt(temp_index); + function.inst_tracking.putAssumeCapacityNoClobber(temp.toIndex(), InstTracking.init(.none)); + } wip_mir_log.debug("{}:", .{fmtNav(func.owner_nav, ip)}); @@ -891,9 +877,9 @@ pub fn generate( })); function.frame_allocs.set(@intFromEnum(FrameIndex.base_ptr), FrameAlloc.init(.{ .size = Type.usize.abiSize(zcu), - .alignment = Alignment.min( + .alignment = InternPool.Alignment.min( call_info.stack_align, - Alignment.fromNonzeroByteUnits(function.target.stackAlignment()), + InternPool.Alignment.fromNonzeroByteUnits(function.target.stackAlignment()), ), })); function.frame_allocs.set( @@ -972,7 +958,7 @@ pub fn generateLazy( lazy_sym: link.File.LazySymbol, code: *std.ArrayListUnmanaged(u8), debug_output: link.File.DebugInfoOutput, -) CodeGenError!void { +) codegen.CodeGenError!void { const comp = bin_file.comp; const gpa = comp.gpa; // This function is for generating global code, so we use the root module. @@ -1169,14 +1155,14 @@ fn formatWipMir( lower.mir.extraData(Mir.Imm64, mir_inst.data.ai.i).data.decode(), }), .pseudo_dbg_local_as => { - const mem_op: Instruction.Operand = .{ .mem = .initSib(.qword, .{ + const mem_op: encoder.Instruction.Operand = .{ .mem = .initSib(.qword, .{ .base = .{ .reloc = mir_inst.data.as.sym_index }, }) }; try writer.print(" {}, {}", .{ mir_inst.data.as.air_inst, mem_op.fmt(.m) }); }, .pseudo_dbg_local_aso => { const sym_off = lower.mir.extraData(bits.SymbolOffset, mir_inst.data.ax.payload).data; - const mem_op: Instruction.Operand = .{ .mem = .initSib(.qword, .{ + const mem_op: encoder.Instruction.Operand = .{ .mem = .initSib(.qword, .{ .base = .{ .reloc = sym_off.sym_index }, .disp = sym_off.off, }) }; @@ -1184,7 +1170,7 @@ fn formatWipMir( }, .pseudo_dbg_local_aro => { const air_off = lower.mir.extraData(Mir.AirOffset, mir_inst.data.rx.payload).data; - const mem_op: Instruction.Operand = .{ .mem = .initSib(.qword, .{ + const mem_op: encoder.Instruction.Operand = .{ .mem = .initSib(.qword, .{ .base = .{ .reg = mir_inst.data.rx.r1 }, .disp = air_off.off, }) }; @@ -1192,14 +1178,14 @@ fn formatWipMir( }, .pseudo_dbg_local_af => { const frame_addr = lower.mir.extraData(bits.FrameAddr, mir_inst.data.ax.payload).data; - const mem_op: Instruction.Operand = .{ .mem = .initSib(.qword, .{ + const mem_op: encoder.Instruction.Operand = .{ .mem = .initSib(.qword, .{ .base = .{ .frame = frame_addr.index }, .disp = frame_addr.off, }) }; - try writer.print(" {}, {d}", .{ mir_inst.data.ax.air_inst, mem_op.fmt(.m) }); + try writer.print(" {}, {}", .{ mir_inst.data.ax.air_inst, mem_op.fmt(.m) }); }, .pseudo_dbg_local_am => { - const mem_op: Instruction.Operand = .{ + const mem_op: encoder.Instruction.Operand = .{ .mem = lower.mir.extraData(Mir.Memory, mir_inst.data.ax.payload).data.decode(), }; try writer.print(" {}, {}", .{ mir_inst.data.ax.air_inst, mem_op.fmt(.m) }); @@ -1221,7 +1207,7 @@ fn formatTracking( writer: anytype, ) @TypeOf(writer).Error!void { var it = data.self.inst_tracking.iterator(); - while (it.next()) |entry| try writer.print("\n%{d} = {}", .{ entry.key_ptr.*, entry.value_ptr.* }); + while (it.next()) |entry| try writer.print("\n{} = {}", .{ entry.key_ptr.*, entry.value_ptr.* }); } fn fmtTracking(self: *Self) std.fmt.Formatter(formatTracking) { return .{ .data = .{ .self = self } }; @@ -1427,7 +1413,7 @@ fn asmAirImmediate(self: *Self, tag: MirTagAir, inst: Air.Inst.Index, imm: Immed .i = @bitCast(s), } }, }), - .unsigned => |u| _ = if (math.cast(u32, u)) |small| try self.addInst(.{ + .unsigned => |u| _ = if (std.math.cast(u32, u)) |small| try self.addInst(.{ .tag = .pseudo, .ops = switch (tag) { .dbg_local => .pseudo_dbg_local_ai_u, @@ -1632,7 +1618,7 @@ fn asmRegisterRegister(self: *Self, tag: Mir.Inst.FixedTag, reg1: Register, reg2 fn asmRegisterImmediate(self: *Self, tag: Mir.Inst.FixedTag, reg: Register, imm: Immediate) !void { const ops: Mir.Inst.Ops, const i: u32 = switch (imm) { .signed => |s| .{ .ri_s, @bitCast(s) }, - .unsigned => |u| if (math.cast(u32, u)) |small| + .unsigned => |u| if (std.math.cast(u32, u)) |small| .{ .ri_u, small } else .{ .ri_64, try self.addExtra(Mir.Imm64.encode(imm.unsigned)) }, @@ -1831,8 +1817,8 @@ fn asmRegisterMemoryImmediate( imm: Immediate, ) !void { if (switch (imm) { - .signed => |s| if (math.cast(i16, s)) |x| @as(u16, @bitCast(x)) else null, - .unsigned => |u| math.cast(u16, u), + .signed => |s| if (std.math.cast(i16, s)) |x| @as(u16, @bitCast(x)) else null, + .unsigned => |u| std.math.cast(u16, u), .reloc => unreachable, }) |small_imm| { _ = try self.addInst(.{ @@ -1967,8 +1953,8 @@ fn gen(self: *Self) InnerError!void { const cc = abi.resolveCallingConvention(fn_info.cc, self.target.*); if (cc != .naked) { try self.asmRegister(.{ ._, .push }, .rbp); - try self.asmPseudoImmediate(.pseudo_cfi_adjust_cfa_offset_i_s, Immediate.s(8)); - try self.asmPseudoRegisterImmediate(.pseudo_cfi_rel_offset_ri_s, .rbp, Immediate.s(0)); + try self.asmPseudoImmediate(.pseudo_cfi_adjust_cfa_offset_i_s, .s(8)); + try self.asmPseudoRegisterImmediate(.pseudo_cfi_rel_offset_ri_s, .rbp, .s(0)); try self.asmRegisterRegister(.{ ._, .mov }, .rbp, .rsp); try self.asmPseudoRegister(.pseudo_cfi_def_cfa_register_r, .rbp); const backpatch_push_callee_preserved_regs = try self.asmPlaceholder(); @@ -2016,7 +2002,7 @@ fn gen(self: *Self) InnerError!void { .{}, ); - try self.asmRegisterImmediate(.{ ._, .cmp }, .al, Immediate.u(info.fp_count)); + try self.asmRegisterImmediate(.{ ._, .cmp }, .al, .u(info.fp_count)); const skip_sse_reloc = try self.asmJccReloc(.na, undefined); const vec_2_f64 = try pt.vectorType(.{ .len = 2, .child = .f64_type }); @@ -2055,15 +2041,15 @@ fn gen(self: *Self) InnerError!void { const backpatch_stack_dealloc = try self.asmPlaceholder(); const backpatch_pop_callee_preserved_regs = try self.asmPlaceholder(); try self.asmRegister(.{ ._, .pop }, .rbp); - try self.asmPseudoRegisterImmediate(.pseudo_cfi_def_cfa_ri_s, .rsp, Immediate.s(8)); + try self.asmPseudoRegisterImmediate(.pseudo_cfi_def_cfa_ri_s, .rsp, .s(8)); try self.asmOpOnly(.{ ._, .ret }); const frame_layout = try self.computeFrameLayout(cc); - const need_frame_align = frame_layout.stack_mask != math.maxInt(u32); + const need_frame_align = frame_layout.stack_mask != std.math.maxInt(u32); const need_stack_adjust = frame_layout.stack_adjust > 0; const need_save_reg = frame_layout.save_reg_list.count() > 0; if (need_frame_align) { - const page_align = @as(u32, math.maxInt(u32)) << 12; + const page_align = @as(u32, std.math.maxInt(u32)) << 12; self.mir_instructions.set(backpatch_frame_align, .{ .tag = .@"and", .ops = .ri_s, @@ -2170,23 +2156,18 @@ fn gen(self: *Self) InnerError!void { }); } -fn checkInvariantsAfterAirInst(self: *Self, inst: Air.Inst.Index, old_air_bookkeeping: @TypeOf(air_bookkeeping_init)) void { +fn checkInvariantsAfterAirInst(self: *Self) void { assert(!self.register_manager.lockedRegsExist()); if (std.debug.runtime_safety) { - if (self.air_bookkeeping < old_air_bookkeeping + 1) { - std.debug.panic("in codegen.zig, handling of AIR instruction %{d} ('{}') did not do proper bookkeeping. Look for a missing call to finishAir.", .{ inst, self.air.instructions.items(.tag)[@intFromEnum(inst)] }); - } - - { // check consistency of tracked registers - var it = self.register_manager.free_registers.iterator(.{ .kind = .unset }); - while (it.next()) |index| { - const tracked_inst = self.register_manager.registers[index]; - const tracking = self.getResolvedInstValue(tracked_inst); - for (tracking.getRegs()) |reg| { - if (RegisterManager.indexOfRegIntoTracked(reg).? == index) break; - } else unreachable; // tracked register not in use - } + // check consistency of tracked registers + var it = self.register_manager.free_registers.iterator(.{ .kind = .unset }); + while (it.next()) |index| { + const tracked_inst = self.register_manager.registers[index]; + const tracking = self.getResolvedInstValue(tracked_inst); + for (tracking.getRegs()) |reg| { + if (RegisterManager.indexOfRegIntoTracked(reg).? == index) break; + } else unreachable; // tracked register not in use } } } @@ -2202,6 +2183,8 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { const zcu = pt.zcu; const ip = &zcu.intern_pool; const air_tags = self.air.instructions.items(.tag); + const air_datas = self.air.instructions.items(.data); + const use_old = self.target.ofmt == .coff; self.arg_index = 0; for (body) |inst| switch (air_tags[@intFromEnum(inst)]) { @@ -2209,12 +2192,13 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { wip_mir_log.debug("{}", .{self.fmtAir(inst)}); verbose_tracking_log.debug("{}", .{self.fmtTracking()}); - const old_air_bookkeeping = self.air_bookkeeping; + self.reused_operands = @TypeOf(self.reused_operands).initEmpty(); try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1); try self.airArg(inst); - self.checkInvariantsAfterAirInst(inst, old_air_bookkeeping); + self.resetTemps(); + self.checkInvariantsAfterAirInst(); }, else => break, }; @@ -2226,7 +2210,7 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { wip_mir_log.debug("{}", .{self.fmtAir(inst)}); verbose_tracking_log.debug("{}", .{self.fmtTracking()}); - const old_air_bookkeeping = self.air_bookkeeping; + self.reused_operands = @TypeOf(self.reused_operands).initEmpty(); try self.inst_tracking.ensureUnusedCapacity(self.gpa, 1); switch (air_tags[@intFromEnum(inst)]) { // zig fmt: off @@ -2260,7 +2244,6 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { .sub_sat => try self.airSubSat(inst), .mul_sat => try self.airMulSat(inst), .shl_sat => try self.airShlSat(inst), - .slice => try self.airSlice(inst), .sin, .cos, @@ -2298,127 +2281,58 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { .cmp_vector => try self.airCmpVector(inst), .cmp_lt_errors_len => try self.airCmpLtErrorsLen(inst), - .alloc => try self.airAlloc(inst), - .ret_ptr => try self.airRetPtr(inst), - .arg => try self.airDbgArg(inst), - .assembly => try self.airAsm(inst), - .bitcast => try self.airBitCast(inst), - .block => try self.airBlock(inst), - .br => try self.airBr(inst), - .repeat => try self.airRepeat(inst), - .switch_dispatch => try self.airSwitchDispatch(inst), - .trap => try self.airTrap(), - .breakpoint => try self.airBreakpoint(), - .ret_addr => try self.airRetAddr(inst), - .frame_addr => try self.airFrameAddress(inst), - .cond_br => try self.airCondBr(inst), - .fptrunc => try self.airFptrunc(inst), - .fpext => try self.airFpext(inst), - .intcast => try self.airIntCast(inst), - .trunc => try self.airTrunc(inst), - .int_from_bool => try self.airIntFromBool(inst), - .is_non_null => try self.airIsNonNull(inst), - .is_non_null_ptr => try self.airIsNonNullPtr(inst), - .is_null => try self.airIsNull(inst), - .is_null_ptr => try self.airIsNullPtr(inst), - .is_non_err => try self.airIsNonErr(inst), - .is_non_err_ptr => try self.airIsNonErrPtr(inst), - .is_err => try self.airIsErr(inst), - .is_err_ptr => try self.airIsErrPtr(inst), - .load => try self.airLoad(inst), - .loop => try self.airLoop(inst), - .int_from_ptr => try self.airIntFromPtr(inst), - .ret => try self.airRet(inst, false), - .ret_safe => try self.airRet(inst, true), - .ret_load => try self.airRetLoad(inst), - .store => try self.airStore(inst, false), - .store_safe => try self.airStore(inst, true), - .struct_field_ptr=> try self.airStructFieldPtr(inst), - .struct_field_val=> try self.airStructFieldVal(inst), - .array_to_slice => try self.airArrayToSlice(inst), - .float_from_int => try self.airFloatFromInt(inst), - .int_from_float => try self.airIntFromFloat(inst), - .cmpxchg_strong => try self.airCmpxchg(inst), - .cmpxchg_weak => try self.airCmpxchg(inst), - .atomic_rmw => try self.airAtomicRmw(inst), - .atomic_load => try self.airAtomicLoad(inst), - .memcpy => try self.airMemcpy(inst), - .memset => try self.airMemset(inst, false), - .memset_safe => try self.airMemset(inst, true), - .set_union_tag => try self.airSetUnionTag(inst), - .get_union_tag => try self.airGetUnionTag(inst), - .clz => try self.airClz(inst), - .ctz => try self.airCtz(inst), - .popcount => try self.airPopCount(inst), - .byte_swap => try self.airByteSwap(inst), - .bit_reverse => try self.airBitReverse(inst), - .tag_name => try self.airTagName(inst), - .error_name => try self.airErrorName(inst), - .splat => try self.airSplat(inst), - .select => try self.airSelect(inst), - .shuffle => try self.airShuffle(inst), - .reduce => try self.airReduce(inst), - .aggregate_init => try self.airAggregateInit(inst), - .union_init => try self.airUnionInit(inst), - .prefetch => try self.airPrefetch(inst), - .mul_add => try self.airMulAdd(inst), - .addrspace_cast => return self.fail("TODO implement addrspace_cast", .{}), - - .@"try" => try self.airTry(inst), - .try_cold => try self.airTry(inst), // TODO - .try_ptr => try self.airTryPtr(inst), - .try_ptr_cold => try self.airTryPtr(inst), // TODO - - .dbg_stmt => try self.airDbgStmt(inst), - .dbg_empty_stmt => try self.airDbgEmptyStmt(), - .dbg_inline_block => try self.airDbgInlineBlock(inst), - .dbg_var_ptr, - .dbg_var_val, - .dbg_arg_inline, - => try self.airDbgVar(inst), - - .call => try self.airCall(inst, .auto), - .call_always_tail => try self.airCall(inst, .always_tail), - .call_never_tail => try self.airCall(inst, .never_tail), - .call_never_inline => try self.airCall(inst, .never_inline), + .bitcast => try self.airBitCast(inst), + .fptrunc => try self.airFptrunc(inst), + .fpext => try self.airFpext(inst), + .intcast => try self.airIntCast(inst), + .trunc => try self.airTrunc(inst), + .is_non_null => try self.airIsNonNull(inst), + .is_null => try self.airIsNull(inst), + .is_non_err => try self.airIsNonErr(inst), + .is_err => try self.airIsErr(inst), + .load => try self.airLoad(inst), + .store => try self.airStore(inst, false), + .store_safe => try self.airStore(inst, true), + .struct_field_val => try self.airStructFieldVal(inst), + .float_from_int => try self.airFloatFromInt(inst), + .int_from_float => try self.airIntFromFloat(inst), + .cmpxchg_strong => try self.airCmpxchg(inst), + .cmpxchg_weak => try self.airCmpxchg(inst), + .atomic_rmw => try self.airAtomicRmw(inst), + .atomic_load => try self.airAtomicLoad(inst), + .memcpy => try self.airMemcpy(inst), + .memset => try self.airMemset(inst, false), + .memset_safe => try self.airMemset(inst, true), + .set_union_tag => try self.airSetUnionTag(inst), + .get_union_tag => try self.airGetUnionTag(inst), + .clz => try self.airClz(inst), + .ctz => try self.airCtz(inst), + .popcount => try self.airPopCount(inst), + .byte_swap => try self.airByteSwap(inst), + .bit_reverse => try self.airBitReverse(inst), + .tag_name => try self.airTagName(inst), + .error_name => try self.airErrorName(inst), + .splat => try self.airSplat(inst), + .select => try self.airSelect(inst), + .shuffle => try self.airShuffle(inst), + .reduce => try self.airReduce(inst), + .aggregate_init => try self.airAggregateInit(inst), + .union_init => try self.airUnionInit(inst), + .prefetch => try self.airPrefetch(inst), + .mul_add => try self.airMulAdd(inst), .atomic_store_unordered => try self.airAtomicStore(inst, .unordered), .atomic_store_monotonic => try self.airAtomicStore(inst, .monotonic), .atomic_store_release => try self.airAtomicStore(inst, .release), .atomic_store_seq_cst => try self.airAtomicStore(inst, .seq_cst), - .struct_field_ptr_index_0 => try self.airStructFieldPtrIndex(inst, 0), - .struct_field_ptr_index_1 => try self.airStructFieldPtrIndex(inst, 1), - .struct_field_ptr_index_2 => try self.airStructFieldPtrIndex(inst, 2), - .struct_field_ptr_index_3 => try self.airStructFieldPtrIndex(inst, 3), - - .field_parent_ptr => try self.airFieldParentPtr(inst), - - .switch_br => try self.airSwitchBr(inst), - .loop_switch_br => try self.airLoopSwitchBr(inst), - .slice_ptr => try self.airSlicePtr(inst), - .slice_len => try self.airSliceLen(inst), - - .ptr_slice_len_ptr => try self.airPtrSliceLenPtr(inst), - .ptr_slice_ptr_ptr => try self.airPtrSlicePtrPtr(inst), - .array_elem_val => try self.airArrayElemVal(inst), .slice_elem_val => try self.airSliceElemVal(inst), - .slice_elem_ptr => try self.airSliceElemPtr(inst), .ptr_elem_val => try self.airPtrElemVal(inst), - .ptr_elem_ptr => try self.airPtrElemPtr(inst), - - .inferred_alloc, .inferred_alloc_comptime => unreachable, - .unreach => self.finishAirBookkeeping(), .optional_payload => try self.airOptionalPayload(inst), - .optional_payload_ptr => try self.airOptionalPayloadPtr(inst), - .optional_payload_ptr_set => try self.airOptionalPayloadPtrSet(inst), .unwrap_errunion_err => try self.airUnwrapErrUnionErr(inst), .unwrap_errunion_payload => try self.airUnwrapErrUnionPayload(inst), - .unwrap_errunion_err_ptr => try self.airUnwrapErrUnionErrPtr(inst), - .unwrap_errunion_payload_ptr=> try self.airUnwrapErrUnionPayloadPtr(inst), - .errunion_payload_ptr_set => try self.airErrUnionPayloadPtrSet(inst), .err_return_trace => try self.airErrReturnTrace(inst), .set_err_return_trace => try self.airSetErrReturnTrace(inst), .save_err_return_trace_index=> try self.airSaveErrReturnTraceIndex(inst), @@ -2426,7 +2340,12 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { .wrap_optional => try self.airWrapOptional(inst), .wrap_errunion_payload => try self.airWrapErrUnionPayload(inst), .wrap_errunion_err => try self.airWrapErrUnionErr(inst), + // zig fmt: on + .add_safe, + .sub_safe, + .mul_safe, + => return self.fail("TODO implement safety_checked_instructions", .{}), .add_optimized, .sub_optimized, .mul_optimized, @@ -2448,13 +2367,429 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { .int_from_float_optimized, => return self.fail("TODO implement optimized float mode", .{}), - .add_safe, - .sub_safe, - .mul_safe, - => return self.fail("TODO implement safety_checked_instructions", .{}), + .arg => try self.airDbgArg(inst), + .alloc => if (use_old) try self.airAlloc(inst) else { + var slot = try self.tempFromValue(self.typeOfIndex(inst), .{ .lea_frame = .{ + .index = try self.allocMemPtr(inst), + } }); + try slot.moveTo(inst, self); + }, + .inferred_alloc => unreachable, + .inferred_alloc_comptime => unreachable, + .ret_ptr => if (use_old) try self.airRetPtr(inst) else { + var slot = switch (self.ret_mcv.long) { + else => unreachable, + .none => try self.tempFromValue(self.typeOfIndex(inst), .{ .lea_frame = .{ + .index = try self.allocMemPtr(inst), + } }), + .load_frame => slot: { + var slot = try self.tempFromValue(self.typeOfIndex(inst), self.ret_mcv.long); + try slot.toOffset(self.ret_mcv.short.indirect.off, self); + break :slot slot; + }, + }; + try slot.moveTo(inst, self); + }, + .assembly => try self.airAsm(inst), + .block => if (use_old) try self.airBlock(inst) else { + const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.Block, ty_pl.payload); + try self.asmPseudo(.pseudo_dbg_enter_block_none); + try self.lowerBlock(inst, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len])); + try self.asmPseudo(.pseudo_dbg_leave_block_none); + }, + .loop => if (use_old) try self.airLoop(inst) else { + const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.Block, ty_pl.payload); + self.scope_generation += 1; + try self.loops.putNoClobber(self.gpa, inst, .{ + .state = try self.saveState(), + .target = @intCast(self.mir_instructions.len), + }); + defer assert(self.loops.remove(inst)); + try self.genBodyBlock(@ptrCast(self.air.extra[extra.end..][0..extra.data.body_len])); + }, + .repeat => if (use_old) try self.airRepeat(inst) else { + const repeat = air_datas[@intFromEnum(inst)].repeat; + const loop = self.loops.get(repeat.loop_inst).?; + try self.restoreState(loop.state, &.{}, .{ + .emit_instructions = true, + .update_tracking = false, + .resurrect = false, + .close_scope = true, + }); + _ = try self.asmJmpReloc(loop.target); + }, + .br => try self.airBr(inst), + .trap => try self.asmOpOnly(.{ ._, .ud2 }), + .breakpoint => try self.asmOpOnly(.{ ._, .int3 }), + .ret_addr => if (use_old) try self.airRetAddr(inst) else { + var slot = try self.tempFromValue(self.typeOfIndex(inst), .{ .load_frame = .{ + .index = .ret_addr, + } }); + while (try slot.toAnyReg(self)) {} + try slot.moveTo(inst, self); + }, + .frame_addr => if (use_old) try self.airFrameAddress(inst) else { + var slot = try self.tempFromValue(self.typeOfIndex(inst), .{ .lea_frame = .{ + .index = .base_ptr, + } }); + try slot.moveTo(inst, self); + }, + .call => try self.airCall(inst, .auto), + .call_always_tail => try self.airCall(inst, .always_tail), + .call_never_tail => try self.airCall(inst, .never_tail), + .call_never_inline => try self.airCall(inst, .never_inline), + + .cond_br => try self.airCondBr(inst), + .switch_br => try self.airSwitchBr(inst), + .loop_switch_br => try self.airLoopSwitchBr(inst), + .switch_dispatch => try self.airSwitchDispatch(inst), + .@"try", .try_cold => try self.airTry(inst), + .try_ptr, .try_ptr_cold => try self.airTryPtr(inst), + .dbg_stmt => if (use_old) try self.airDbgStmt(inst) else { + const dbg_stmt = air_datas[@intFromEnum(inst)].dbg_stmt; + _ = try self.addInst(.{ + .tag = .pseudo, + .ops = .pseudo_dbg_line_line_column, + .data = .{ .line_column = .{ + .line = dbg_stmt.line, + .column = dbg_stmt.column, + } }, + }); + }, + .dbg_empty_stmt => if (use_old) try self.airDbgEmptyStmt() else { + if (self.mir_instructions.len > 0) { + const prev_mir_op = &self.mir_instructions.items(.ops)[self.mir_instructions.len - 1]; + if (prev_mir_op.* == .pseudo_dbg_line_stmt_line_column) + prev_mir_op.* = .pseudo_dbg_line_line_column; + } + try self.asmOpOnly(.{ ._, .nop }); + }, + .dbg_inline_block => if (use_old) try self.airDbgInlineBlock(inst) else { + const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.DbgInlineBlock, ty_pl.payload); + const old_inline_func = self.inline_func; + defer self.inline_func = old_inline_func; + self.inline_func = extra.data.func; + _ = try self.addInst(.{ + .tag = .pseudo, + .ops = .pseudo_dbg_enter_inline_func, + .data = .{ .func = extra.data.func }, + }); + try self.lowerBlock(inst, @ptrCast(self.air.extra[extra.end..][0..extra.data.body_len])); + _ = try self.addInst(.{ + .tag = .pseudo, + .ops = .pseudo_dbg_leave_inline_func, + .data = .{ .func = old_inline_func }, + }); + }, + .dbg_var_ptr, .dbg_var_val, .dbg_arg_inline => if (use_old) try self.airDbgVar(inst) else { + const pl_op = air_datas[@intFromEnum(inst)].pl_op; + var ops = try self.tempsFromOperands(inst, .{pl_op.operand}); + try self.genLocalDebugInfo(inst, ops[0].tracking(self).short); + try ops[0].die(self); + }, + .is_null_ptr => if (use_old) try self.airIsNullPtr(inst) else { + const un_op = air_datas[@intFromEnum(inst)].un_op; + const opt_ty = self.typeOf(un_op).childType(zcu); + const opt_repr_is_pl = opt_ty.optionalReprIsPayload(zcu); + const opt_child_ty = opt_ty.optionalChild(zcu); + const opt_child_abi_size: u31 = @intCast(opt_child_ty.abiSize(zcu)); + var ops = try self.tempsFromOperands(inst, .{un_op}); + if (!opt_repr_is_pl) try ops[0].toOffset(opt_child_abi_size, self); + while (try ops[0].toLea(self)) {} + try self.asmMemoryImmediate( + .{ ._, .cmp }, + try ops[0].tracking(self).short.deref().mem(self, if (!opt_repr_is_pl) + .byte + else if (opt_child_ty.isSlice(zcu)) + .qword + else + Memory.Size.fromSize(opt_child_abi_size)), + .u(0), + ); + var is_null = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .e }); + try ops[0].die(self); + try is_null.moveTo(inst, self); + }, + .is_non_null_ptr => if (use_old) try self.airIsNonNullPtr(inst) else { + const un_op = air_datas[@intFromEnum(inst)].un_op; + const opt_ty = self.typeOf(un_op).childType(zcu); + const opt_repr_is_pl = opt_ty.optionalReprIsPayload(zcu); + const opt_child_ty = opt_ty.optionalChild(zcu); + const opt_child_abi_size: u31 = @intCast(opt_child_ty.abiSize(zcu)); + var ops = try self.tempsFromOperands(inst, .{un_op}); + if (!opt_repr_is_pl) try ops[0].toOffset(opt_child_abi_size, self); + while (try ops[0].toLea(self)) {} + try self.asmMemoryImmediate( + .{ ._, .cmp }, + try ops[0].tracking(self).short.deref().mem(self, if (!opt_repr_is_pl) + .byte + else if (opt_child_ty.isSlice(zcu)) + .qword + else + Memory.Size.fromSize(opt_child_abi_size)), + .u(0), + ); + var is_non_null = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .ne }); + try ops[0].die(self); + try is_non_null.moveTo(inst, self); + }, + .is_err_ptr => if (use_old) try self.airIsErrPtr(inst) else { + const un_op = air_datas[@intFromEnum(inst)].un_op; + const eu_ty = self.typeOf(un_op).childType(zcu); + const eu_err_ty = eu_ty.errorUnionSet(zcu); + const eu_pl_ty = eu_ty.errorUnionPayload(zcu); + const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); + var ops = try self.tempsFromOperands(inst, .{un_op}); + try ops[0].toOffset(eu_err_off, self); + while (try ops[0].toLea(self)) {} + try self.asmMemoryImmediate( + .{ ._, .cmp }, + try ops[0].tracking(self).short.deref().mem(self, self.memSize(eu_err_ty)), + .u(0), + ); + var is_err = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .ne }); + try ops[0].die(self); + try is_err.moveTo(inst, self); + }, + .is_non_err_ptr => if (use_old) try self.airIsNonErrPtr(inst) else { + const un_op = air_datas[@intFromEnum(inst)].un_op; + const eu_ty = self.typeOf(un_op).childType(zcu); + const eu_err_ty = eu_ty.errorUnionSet(zcu); + const eu_pl_ty = eu_ty.errorUnionPayload(zcu); + const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); + var ops = try self.tempsFromOperands(inst, .{un_op}); + try ops[0].toOffset(eu_err_off, self); + while (try ops[0].toLea(self)) {} + try self.asmMemoryImmediate( + .{ ._, .cmp }, + try ops[0].tracking(self).short.deref().mem(self, self.memSize(eu_err_ty)), + .u(0), + ); + var is_non_err = try self.tempFromValue(self.typeOfIndex(inst), .{ .eflags = .e }); + try ops[0].die(self); + try is_non_err.moveTo(inst, self); + }, + .int_from_ptr => if (use_old) try self.airIntFromPtr(inst) else { + const un_op = air_datas[@intFromEnum(inst)].un_op; + var ops = try self.tempsFromOperands(inst, .{un_op}); + try ops[0].toLimb(0, self); + try ops[0].moveTo(inst, self); + }, + .int_from_bool => if (use_old) try self.airIntFromBool(inst) else { + const un_op = air_datas[@intFromEnum(inst)].un_op; + var ops = try self.tempsFromOperands(inst, .{un_op}); + try ops[0].moveTo(inst, self); + }, + .ret => try self.airRet(inst, false), + .ret_safe => try self.airRet(inst, true), + .ret_load => try self.airRetLoad(inst), + .unreach => {}, + .optional_payload_ptr => if (use_old) try self.airOptionalPayloadPtr(inst) else { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); + try ops[0].moveTo(inst, self); + }, + .optional_payload_ptr_set => if (use_old) try self.airOptionalPayloadPtrSet(inst) else { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + const opt_ty = self.typeOf(ty_op.operand).childType(zcu); + var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); + if (!opt_ty.optionalReprIsPayload(zcu)) { + const opt_child_ty = opt_ty.optionalChild(zcu); + const opt_child_abi_size: i32 = @intCast(opt_child_ty.abiSize(zcu)); + try ops[0].toOffset(opt_child_abi_size, self); + var has_value = try self.tempFromValue(Type.bool, .{ .immediate = 1 }); + try ops[0].store(&has_value, self); + try has_value.die(self); + try ops[0].toOffset(-opt_child_abi_size, self); + } + try ops[0].moveTo(inst, self); + }, + .unwrap_errunion_payload_ptr => if (use_old) try self.airUnwrapErrUnionPayloadPtr(inst) else { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + const eu_ty = self.typeOf(ty_op.operand).childType(zcu); + const eu_pl_ty = eu_ty.errorUnionPayload(zcu); + const eu_pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(eu_pl_ty, zcu)); + var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); + try ops[0].toOffset(eu_pl_off, self); + try ops[0].moveTo(inst, self); + }, + .unwrap_errunion_err_ptr => if (use_old) try self.airUnwrapErrUnionErrPtr(inst) else { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + const eu_ty = self.typeOf(ty_op.operand).childType(zcu); + const eu_pl_ty = eu_ty.errorUnionPayload(zcu); + const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); + var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); + try ops[0].toOffset(eu_err_off, self); + var err = try ops[0].load(eu_ty.errorUnionSet(zcu), self); + try ops[0].die(self); + try err.moveTo(inst, self); + }, + .errunion_payload_ptr_set => if (use_old) try self.airErrUnionPayloadPtrSet(inst) else { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + const eu_ty = self.typeOf(ty_op.operand).childType(zcu); + const eu_err_ty = eu_ty.errorUnionSet(zcu); + const eu_pl_ty = eu_ty.errorUnionPayload(zcu); + const eu_err_off: i32 = @intCast(codegen.errUnionErrorOffset(eu_pl_ty, zcu)); + const eu_pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(eu_pl_ty, zcu)); + var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); + try ops[0].toOffset(eu_err_off, self); + var no_err = try self.tempFromValue(eu_err_ty, .{ .immediate = 0 }); + try ops[0].store(&no_err, self); + try no_err.die(self); + try ops[0].toOffset(eu_pl_off - eu_err_off, self); + try ops[0].moveTo(inst, self); + }, + .struct_field_ptr => if (use_old) try self.airStructFieldPtr(inst) else { + const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.StructField, ty_pl.payload).data; + var ops = try self.tempsFromOperands(inst, .{extra.struct_operand}); + try ops[0].toOffset(self.fieldOffset(self.typeOf(extra.struct_operand), self.typeOfIndex(inst), extra.field_index), self); + try ops[0].moveTo(inst, self); + }, + .struct_field_ptr_index_0 => if (use_old) try self.airStructFieldPtrIndex(inst, 0) else { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); + try ops[0].toOffset(self.fieldOffset(self.typeOf(ty_op.operand), self.typeOfIndex(inst), 0), self); + try ops[0].moveTo(inst, self); + }, + .struct_field_ptr_index_1 => if (use_old) try self.airStructFieldPtrIndex(inst, 1) else { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); + try ops[0].toOffset(self.fieldOffset(self.typeOf(ty_op.operand), self.typeOfIndex(inst), 1), self); + try ops[0].moveTo(inst, self); + }, + .struct_field_ptr_index_2 => if (use_old) try self.airStructFieldPtrIndex(inst, 2) else { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); + try ops[0].toOffset(self.fieldOffset(self.typeOf(ty_op.operand), self.typeOfIndex(inst), 2), self); + try ops[0].moveTo(inst, self); + }, + .struct_field_ptr_index_3 => if (use_old) try self.airStructFieldPtrIndex(inst, 3) else { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); + try ops[0].toOffset(self.fieldOffset(self.typeOf(ty_op.operand), self.typeOfIndex(inst), 3), self); + try ops[0].moveTo(inst, self); + }, + .slice => if (use_old) try self.airSlice(inst) else { + const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; + const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; + var ops = try self.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + try ops[0].toPair(&ops[1], self); + try ops[0].moveTo(inst, self); + }, + .slice_len => if (use_old) try self.airSliceLen(inst) else { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); + try ops[0].toLimb(1, self); + try ops[0].moveTo(inst, self); + }, + .slice_ptr => if (use_old) try self.airSlicePtr(inst) else { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); + try ops[0].toLimb(0, self); + try ops[0].moveTo(inst, self); + }, + .ptr_slice_len_ptr => if (use_old) try self.airPtrSliceLenPtr(inst) else { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); + try ops[0].toOffset(8, self); + try ops[0].moveTo(inst, self); + }, + .ptr_slice_ptr_ptr => if (use_old) try self.airPtrSlicePtrPtr(inst) else { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); + try ops[0].toOffset(0, self); + try ops[0].moveTo(inst, self); + }, + .slice_elem_ptr, .ptr_elem_ptr => |tag| if (use_old) switch (tag) { + else => unreachable, + .slice_elem_ptr => try self.airSliceElemPtr(inst), + .ptr_elem_ptr => try self.airPtrElemPtr(inst), + } else { + const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; + const bin_op = self.air.extraData(Air.Bin, ty_pl.payload).data; + var ops = try self.tempsFromOperands(inst, .{ bin_op.lhs, bin_op.rhs }); + switch (tag) { + else => unreachable, + .slice_elem_ptr => try ops[0].toLimb(0, self), + .ptr_elem_ptr => {}, + } + const dst_ty = self.typeOfIndex(inst); + if (dst_ty.ptrInfo(zcu).flags.vector_index == .none) zero_offset: { + const elem_size = dst_ty.childType(zcu).abiSize(zcu); + if (elem_size == 0) break :zero_offset; + while (true) for (&ops) |*op| { + if (try op.toAnyReg(self)) break; + } else break; + const lhs_reg = ops[0].unwrap(self).temp.tracking(self).short.register.to64(); + const rhs_reg = ops[1].unwrap(self).temp.tracking(self).short.register.to64(); + if (!std.math.isPowerOfTwo(elem_size)) { + try self.spillEflagsIfOccupied(); + try self.asmRegisterRegisterImmediate( + .{ .i_, .mul }, + rhs_reg, + rhs_reg, + .u(elem_size), + ); + try self.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ + .base = .{ .reg = lhs_reg }, + .mod = .{ .rm = .{ .size = .qword, .index = rhs_reg } }, + }); + } else if (elem_size > 8) { + try self.spillEflagsIfOccupied(); + try self.asmRegisterImmediate( + .{ ._l, .sh }, + rhs_reg, + .u(std.math.log2_int(u64, elem_size)), + ); + try self.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ + .base = .{ .reg = lhs_reg }, + .mod = .{ .rm = .{ .size = .qword, .index = rhs_reg } }, + }); + } else try self.asmRegisterMemory(.{ ._, .lea }, lhs_reg, .{ + .base = .{ .reg = lhs_reg }, + .mod = .{ .rm = .{ + .size = .qword, + .index = rhs_reg, + .scale = .fromFactor(@intCast(elem_size)), + } }, + }); + } + try ops[1].die(self); + try ops[0].moveTo(inst, self); + }, + .array_to_slice => if (use_old) try self.airArrayToSlice(inst) else { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); + var len = try self.tempFromValue(Type.usize, .{ + .immediate = self.typeOf(ty_op.operand).childType(zcu).arrayLen(zcu), + }); + try ops[0].toPair(&len, self); + try ops[0].moveTo(inst, self); + }, + .error_set_has_value => return self.fail("TODO implement error_set_has_value", .{}), + .field_parent_ptr => if (use_old) try self.airFieldParentPtr(inst) else { + const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; + const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data; + var ops = try self.tempsFromOperands(inst, .{extra.field_ptr}); + try ops[0].toOffset(-self.fieldOffset(self.typeOfIndex(inst), self.typeOf(extra.field_ptr), extra.field_index), self); + try ops[0].moveTo(inst, self); + }, .is_named_enum_value => return self.fail("TODO implement is_named_enum_value", .{}), - .error_set_has_value => return self.fail("TODO implement error_set_has_value", .{}), + + .wasm_memory_size => unreachable, + .wasm_memory_grow => unreachable, + + .addrspace_cast => { + const ty_op = air_datas[@intFromEnum(inst)].ty_op; + var ops = try self.tempsFromOperands(inst, .{ty_op.operand}); + try ops[0].moveTo(inst, self); + }, + .vector_store_elem => return self.fail("TODO implement vector_store_elem", .{}), .c_va_arg => try self.airVaArg(inst), @@ -2462,15 +2797,12 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { .c_va_end => try self.airVaEnd(inst), .c_va_start => try self.airVaStart(inst), - .wasm_memory_size => unreachable, - .wasm_memory_grow => unreachable, - .work_item_id => unreachable, .work_group_size => unreachable, .work_group_id => unreachable, - // zig fmt: on } - self.checkInvariantsAfterAirInst(inst, old_air_bookkeeping); + self.resetTemps(); + self.checkInvariantsAfterAirInst(); } verbose_tracking_log.debug("{}", .{self.fmtTracking()}); } @@ -2530,7 +2862,7 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void { data_off += @intCast(tag_name_len + 1); } - try self.airTrap(); + try self.asmOpOnly(.{ ._, .ud2 }); for (exitlude_jump_relocs) |reloc| self.performReloc(reloc); try self.asmOpOnly(.{ ._, .ret }); @@ -2544,6 +2876,10 @@ fn genLazy(self: *Self, lazy_sym: link.File.LazySymbol) InnerError!void { fn getValue(self: *Self, value: MCValue, inst: ?Air.Inst.Index) !void { for (value.getRegs()) |reg| try self.register_manager.getReg(reg, inst); + switch (value) { + else => {}, + .eflags, .register_overflow => self.eflags_inst = inst, + } } fn getValueIfFree(self: *Self, value: MCValue, inst: ?Air.Inst.Index) void { @@ -2577,26 +2913,18 @@ fn processDeath(self: *Self, inst: Air.Inst.Index) !void { try self.inst_tracking.getPtr(inst).?.die(self, inst); } -/// Called when there are no operands, and the instruction is always unreferenced. -fn finishAirBookkeeping(self: *Self) void { - if (std.debug.runtime_safety) { - self.air_bookkeeping += 1; - } -} - fn finishAirResult(self: *Self, inst: Air.Inst.Index, result: MCValue) void { if (self.liveness.isUnused(inst) and self.air.instructions.items(.tag)[@intFromEnum(inst)] != .arg) switch (result) { .none, .dead, .unreach => {}, else => unreachable, // Why didn't the result die? } else { - tracking_log.debug("%{d} => {} (birth)", .{ inst, result }); + tracking_log.debug("{} => {} (birth)", .{ inst, result }); self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(result)); // In some cases, an operand may be reused as the result. // If that operand died and was a register, it was freed by // processDeath, so we have to "re-allocate" the register. self.getValueIfFree(result, inst); } - self.finishAirBookkeeping(); } fn finishAir( @@ -2605,11 +2933,10 @@ fn finishAir( result: MCValue, operands: [Liveness.bpi - 1]Air.Inst.Ref, ) !void { - var tomb_bits = self.liveness.getTombBits(inst); - for (operands) |op| { - const dies = @as(u1, @truncate(tomb_bits)) != 0; - tomb_bits >>= 1; - if (!dies) continue; + const tomb_bits = self.liveness.getTombBits(inst); + for (0.., operands) |op_index, op| { + if (tomb_bits & @as(Liveness.Bpi, 1) << @intCast(op_index) == 0) continue; + if (self.reused_operands.isSet(op_index)) continue; try self.processDeath(op.toIndexAllowNone() orelse continue); } self.finishAirResult(inst, result); @@ -2657,7 +2984,7 @@ fn computeFrameLayout(self: *Self, cc: std.builtin.CallingConvention) !FrameLayo } }; const sort_context = SortContext{ .frame_align = frame_align }; - mem.sort(FrameIndex, stack_frame_order, sort_context, SortContext.lessThan); + std.mem.sort(FrameIndex, stack_frame_order, sort_context, SortContext.lessThan); } const call_frame_align = frame_align[@intFromEnum(FrameIndex.call_frame)]; @@ -2697,13 +3024,13 @@ fn computeFrameLayout(self: *Self, cc: std.builtin.CallingConvention) !FrameLayo @intCast(rsp_offset - frame_offset[@intFromEnum(FrameIndex.stack_frame)]); return .{ - .stack_mask = @as(u32, math.maxInt(u32)) << @intCast(if (need_align_stack) @intFromEnum(needed_align) else 0), + .stack_mask = @as(u32, std.math.maxInt(u32)) << @intCast(if (need_align_stack) @intFromEnum(needed_align) else 0), .stack_adjust = @intCast(rsp_offset - frame_offset[@intFromEnum(FrameIndex.call_frame)]), .save_reg_list = save_reg_list, }; } -fn getFrameAddrAlignment(self: *Self, frame_addr: bits.FrameAddr) Alignment { +fn getFrameAddrAlignment(self: *Self, frame_addr: bits.FrameAddr) InternPool.Alignment { const alloc_align = self.frame_allocs.get(@intFromEnum(frame_addr.index)).abi_align; return @enumFromInt(@min(@intFromEnum(alloc_align), @ctz(frame_addr.off))); } @@ -2741,7 +3068,7 @@ fn allocMemPtr(self: *Self, inst: Air.Inst.Index) !FrameIndex { const ptr_ty = self.typeOfIndex(inst); const val_ty = ptr_ty.childType(zcu); return self.allocFrameIndex(FrameAlloc.init(.{ - .size = math.cast(u32, val_ty.abiSize(zcu)) orelse { + .size = std.math.cast(u32, val_ty.abiSize(zcu)) orelse { return self.fail("type '{}' too big to fit into stack frame", .{val_ty.fmt(pt)}); }, .alignment = ptr_ty.ptrAlignment(zcu).max(.@"1"), @@ -2759,7 +3086,7 @@ fn allocTempRegOrMem(self: *Self, elem_ty: Type, reg_ok: bool) !MCValue { fn allocRegOrMemAdvanced(self: *Self, ty: Type, inst: ?Air.Inst.Index, reg_ok: bool) !MCValue { const pt = self.pt; const zcu = pt.zcu; - const abi_size = math.cast(u32, ty.abiSize(zcu)) orelse { + const abi_size = std.math.cast(u32, ty.abiSize(zcu)) orelse { return self.fail("type '{}' too big to fit into stack frame", .{ty.fmt(pt)}); }; @@ -2857,8 +3184,8 @@ fn restoreState(self: *Self, state: State, deaths: []const Air.Inst.Index, compt } if (opts.resurrect) for ( - self.inst_tracking.keys()[0..state.inst_tracking_len], - self.inst_tracking.values()[0..state.inst_tracking_len], + self.inst_tracking.keys()[Temp.Index.max..state.inst_tracking_len], + self.inst_tracking.values()[Temp.Index.max..state.inst_tracking_len], ) |inst, *tracking| tracking.resurrect(inst, state.scope_generation); for (deaths) |death| try self.processDeath(death); @@ -3067,7 +3394,7 @@ fn airFptrunc(self: *Self, inst: Air.Inst.Index) !void { .{ .v_, .cvtps2ph }, dst_reg, mat_src_reg.to128(), - Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))), + .u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))), ); }, else => unreachable, @@ -3267,7 +3594,7 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void { const dst_elem_abi_size = dst_ty.childType(zcu).abiSize(zcu); const src_elem_abi_size = src_ty.childType(zcu).abiSize(zcu); - switch (math.order(dst_elem_abi_size, src_elem_abi_size)) { + switch (std.math.order(dst_elem_abi_size, src_elem_abi_size)) { .lt => { const mir_tag: Mir.Inst.FixedTag = switch (dst_elem_abi_size) { else => break :result null, @@ -3431,8 +3758,8 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void { }; const dst_mcv = if (dst_int_info.bits <= src_storage_bits and - math.divCeil(u16, dst_int_info.bits, 64) catch unreachable == - math.divCeil(u32, src_storage_bits, 64) catch unreachable and + std.math.divCeil(u16, dst_int_info.bits, 64) catch unreachable == + std.math.divCeil(u32, src_storage_bits, 64) catch unreachable and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: { const dst_mcv = try self.allocRegOrMem(inst, true); try self.genCopy(min_ty, dst_mcv, src_mcv, .{}); @@ -3449,8 +3776,8 @@ fn airIntCast(self: *Self, inst: Air.Inst.Index) !void { break :result .{ .register = registerAlias(dst_mcv.getReg().?, dst_abi_size) }; } - const src_limbs_len = math.divCeil(u16, src_int_info.bits, 64) catch unreachable; - const dst_limbs_len = math.divCeil(u16, dst_int_info.bits, 64) catch unreachable; + const src_limbs_len = std.math.divCeil(u16, src_int_info.bits, 64) catch unreachable; + const dst_limbs_len = std.math.divCeil(u16, dst_int_info.bits, 64) catch unreachable; const high_mcv: MCValue = if (dst_mcv.isMemory()) dst_mcv.address().offset((src_limbs_len - 1) * 8).deref() @@ -3570,7 +3897,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void { const dst_info = dst_elem_ty.intInfo(zcu); const src_info = src_elem_ty.intInfo(zcu); - const mask_val = try pt.intValue(src_elem_ty, @as(u64, math.maxInt(u64)) >> @intCast(64 - dst_info.bits)); + const mask_val = try pt.intValue(src_elem_ty, @as(u64, std.math.maxInt(u64)) >> @intCast(64 - dst_info.bits)); const splat_ty = try pt.vectorType(.{ .len = @intCast(@divExact(@as(u64, if (src_abi_size > 16) 256 else 128), src_info.bits)), @@ -3607,7 +3934,7 @@ fn airTrunc(self: *Self, inst: Air.Inst.Index) !void { .{ if (self.hasFeature(.avx2)) .v_i128 else .v_f128, .extract }, registerAlias(temp_reg, dst_abi_size), dst_alias, - Immediate.u(1), + .u(1), ); try self.asmRegisterRegisterRegister( mir_tag, @@ -3806,7 +4133,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void { try self.asmMemoryImmediate( .{ ._, .mov }, .{ .base = .{ .frame = frame_index }, .mod = .{ .rm = .{ .size = .qword } } }, - Immediate.u(0), + .u(0), ); const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); @@ -3920,11 +4247,7 @@ fn airMulDivBinOp(self: *Self, inst: Air.Inst.Index) !void { .mod = .{ .rm = .{ .size = .qword } }, }, ); - try self.asmRegisterImmediate( - .{ ._, .sbb }, - dst_mcv.register_pair[1], - Immediate.u(0), - ); + try self.asmRegisterImmediate(.{ ._, .sbb }, dst_mcv.register_pair[1], .u(0)); try self.freeValue( .{ .load_frame = .{ .index = signed_div_floor_state.frame_index } }, ); @@ -4068,7 +4391,7 @@ fn airAddSat(self: *Self, inst: Air.Inst.Index) !void { break :cc .o; } else cc: { try self.genSetReg(limit_reg, ty, .{ - .immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - ty.bitSize(zcu)), + .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - ty.bitSize(zcu)), }, .{}); try self.genBinOpMir(.{ ._, .add }, ty, dst_mcv, rhs_mcv); @@ -4266,12 +4589,12 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void { mat_rhs_mcv.register_pair[1], ); - try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, Immediate.u(63)); + try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, .u(63)); try self.asmRegister(.{ ._, .not }, tmp_reg); - try self.asmMemoryImmediate(.{ ._, .cmp }, try overflow.mem(self, .dword), Immediate.s(0)); + try self.asmMemoryImmediate(.{ ._, .cmp }, try overflow.mem(self, .dword), .s(0)); try self.freeValue(overflow); try self.asmCmovccRegisterRegister(.ne, dst_mcv.register_pair[0], tmp_reg); - try self.asmRegisterImmediate(.{ ._c, .bt }, tmp_reg, Immediate.u(63)); + try self.asmRegisterImmediate(.{ ._c, .bt }, tmp_reg, .u(63)); try self.asmCmovccRegisterRegister(.ne, dst_mcv.register_pair[1], tmp_reg); break :result dst_mcv; } @@ -4321,7 +4644,7 @@ fn airMulSat(self: *Self, inst: Air.Inst.Index) !void { break :cc .o; } else cc: { try self.genSetReg(limit_reg, ty, .{ - .immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - reg_bits), + .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - reg_bits), }, .{}); break :cc .c; }; @@ -4366,7 +4689,7 @@ fn airAddSubWithOverflow(self: *Self, inst: Air.Inst.Index) !void { }; const tuple_ty = self.typeOfIndex(inst); - if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) { + if (int_info.bits >= 8 and std.math.isPowerOfTwo(int_info.bits)) { switch (partial_mcv) { .register => |reg| { self.eflags_inst = inst; @@ -4444,7 +4767,7 @@ fn airShlWithOverflow(self: *Self, inst: Air.Inst.Index) !void { const cc = Condition.ne; const tuple_ty = self.typeOfIndex(inst); - if (int_info.bits >= 8 and math.isPowerOfTwo(int_info.bits)) { + if (int_info.bits >= 8 and std.math.isPowerOfTwo(int_info.bits)) { switch (partial_mcv) { .register => |reg| { self.eflags_inst = inst; @@ -4576,7 +4899,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { if (dst_info.bits > 128 and dst_info.signedness == .unsigned) { const slow_inc = self.hasFeature(.slow_incdec); const abi_size: u32 = @intCast(dst_ty.abiSize(zcu)); - const limb_len = math.divCeil(u32, abi_size, 8) catch unreachable; + const limb_len = std.math.divCeil(u32, abi_size, 8) catch unreachable; try self.spillRegisters(&.{ .rax, .rcx, .rdx }); const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rax, .rcx, .rdx }); @@ -4618,7 +4941,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { try self.asmRegisterRegister(.{ ._, .xor }, .edx, .edx); const inner_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); - try self.asmRegisterImmediate(.{ ._r, .sh }, .cl, Immediate.u(1)); + try self.asmRegisterImmediate(.{ ._r, .sh }, .cl, .u(1)); try self.asmMemoryRegister(.{ ._, .adc }, .{ .base = .{ .frame = dst_mcv.load_frame.index }, .mod = .{ .rm = .{ @@ -4642,7 +4965,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { }); try self.asmRegister(.{ ._, .mul }, temp_regs[1].to64()); - try self.asmRegisterImmediate(.{ ._r, .sh }, .ch, Immediate.u(1)); + try self.asmRegisterImmediate(.{ ._r, .sh }, .ch, .u(1)); try self.asmMemoryRegister(.{ ._, .adc }, .{ .base = .{ .frame = dst_mcv.load_frame.index }, .mod = .{ .rm = .{ @@ -4656,30 +4979,22 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { try self.asmSetccRegister(.c, .ch); if (slow_inc) { - try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), Immediate.u(1)); - try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[3].to32(), Immediate.u(1)); + try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), .u(1)); + try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[3].to32(), .u(1)); } else { try self.asmRegister(.{ ._, .inc }, temp_regs[2].to32()); try self.asmRegister(.{ ._, .inc }, temp_regs[3].to32()); } - try self.asmRegisterImmediate( - .{ ._, .cmp }, - temp_regs[3].to32(), - Immediate.u(limb_len), - ); + try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[3].to32(), .u(limb_len)); _ = try self.asmJccReloc(.b, inner_loop); try self.asmRegisterRegister(.{ ._, .@"or" }, .rdx, .rcx); const overflow = try self.asmJccReloc(.nz, undefined); const overflow_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); - try self.asmRegisterImmediate( - .{ ._, .cmp }, - temp_regs[2].to32(), - Immediate.u(limb_len), - ); + try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[2].to32(), .u(limb_len)); const no_overflow = try self.asmJccReloc(.nb, undefined); if (slow_inc) { - try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), Immediate.u(1)); + try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), .u(1)); } else { try self.asmRegister(.{ ._, .inc }, temp_regs[2].to32()); } @@ -4691,7 +5006,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { .scale = .@"8", .disp = lhs_mcv.load_frame.off - 8, } }, - }, Immediate.u(0)); + }, .u(0)); _ = try self.asmJccReloc(.z, overflow_loop); self.performReloc(overflow); try self.asmMemoryImmediate(.{ ._, .mov }, .{ @@ -4701,20 +5016,16 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { .disp = dst_mcv.load_frame.off + @as(i32, @intCast(tuple_ty.structFieldOffset(1, zcu))), } }, - }, Immediate.u(1)); + }, .u(1)); self.performReloc(no_overflow); self.performReloc(skip_inner); if (slow_inc) { - try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), Immediate.u(1)); + try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), .u(1)); } else { try self.asmRegister(.{ ._, .inc }, temp_regs[0].to32()); } - try self.asmRegisterImmediate( - .{ ._, .cmp }, - temp_regs[0].to32(), - Immediate.u(limb_len), - ); + try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[0].to32(), .u(limb_len)); _ = try self.asmJccReloc(.b, outer_loop); break :result dst_mcv; @@ -4750,7 +5061,7 @@ fn airMulWithOverflow(self: *Self, inst: Air.Inst.Index) !void { try self.asmMemoryImmediate( .{ ._, .cmp }, try overflow.mem(self, self.memSize(Type.c_int)), - Immediate.s(0), + .s(0), ); try self.genSetMem( .{ .frame = dst_mcv.load_frame.index }, @@ -5038,7 +5349,7 @@ fn genInlineIntDivFloor(self: *Self, ty: Type, lhs: MCValue, rhs: MCValue) !MCVa try self.asmRegisterImmediate( .{ ._r, .sa }, registerAlias(divisor, abi_size), - Immediate.u(int_info.bits - 1), + .u(int_info.bits - 1), ); try self.asmRegisterRegister( .{ ._, .@"test" }, @@ -5217,8 +5528,8 @@ fn airShlShrBinOp(self: *Self, inst: Air.Inst.Index) !void { defer for (reg_locks) |reg_lock| if (reg_lock) |lock| self.register_manager.unlockReg(lock); - const shift_imm = - Immediate.u(@intCast(Value.fromInterned(rhs_elem).toUnsignedInt(zcu))); + const shift_imm: Immediate = + .u(@intCast(Value.fromInterned(rhs_elem).toUnsignedInt(zcu))); if (self.hasFeature(.avx)) try self.asmRegisterRegisterImmediate( mir_tag, registerAlias(dst_reg, abi_size), @@ -5434,7 +5745,7 @@ fn airUnwrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void { break :result operand; } - const err_off = errUnionErrorOffset(payload_ty, zcu); + const err_off = codegen.errUnionErrorOffset(payload_ty, zcu); switch (operand) { .register => |reg| { // TODO reuse operand @@ -5492,7 +5803,7 @@ fn airUnwrapErrUnionErrPtr(self: *Self, inst: Air.Inst.Index) !void { const eu_ty = src_ty.childType(zcu); const pl_ty = eu_ty.errorUnionPayload(zcu); const err_ty = eu_ty.errorUnionSet(zcu); - const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu)); + const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu)); const err_abi_size: u32 = @intCast(err_ty.abiSize(zcu)); try self.asmRegisterMemory( .{ ._, .mov }, @@ -5535,7 +5846,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void { const eu_ty = src_ty.childType(zcu); const pl_ty = eu_ty.errorUnionPayload(zcu); const err_ty = eu_ty.errorUnionSet(zcu); - const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu)); + const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu)); const err_abi_size: u32 = @intCast(err_ty.abiSize(zcu)); try self.asmMemoryImmediate( .{ ._, .mov }, @@ -5546,7 +5857,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void { .disp = err_off, } }, }, - Immediate.u(0), + .u(0), ); if (self.liveness.isUnused(inst)) break :result .unreach; @@ -5559,7 +5870,7 @@ fn airErrUnionPayloadPtrSet(self: *Self, inst: Air.Inst.Index) !void { const dst_lock = self.register_manager.lockReg(dst_reg); defer if (dst_lock) |lock| self.register_manager.unlockReg(lock); - const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, zcu)); + const pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(pl_ty, zcu)); const dst_abi_size: u32 = @intCast(dst_ty.abiSize(zcu)); try self.asmRegisterMemory( .{ ._, .lea }, @@ -5587,7 +5898,7 @@ fn genUnwrapErrUnionPayloadMir( const result: MCValue = result: { if (!payload_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .none; - const payload_off: u31 = @intCast(errUnionPayloadOffset(payload_ty, zcu)); + const payload_off: u31 = @intCast(codegen.errUnionPayloadOffset(payload_ty, zcu)); switch (err_union) { .load_frame => |frame_addr| break :result .{ .load_frame = .{ .index = frame_addr.index, @@ -5636,7 +5947,7 @@ fn genUnwrapErrUnionPayloadPtrMir( const payload_ty = err_union_ty.errorUnionPayload(zcu); const result: MCValue = result: { - const payload_off = errUnionPayloadOffset(payload_ty, zcu); + const payload_off = codegen.errUnionPayloadOffset(payload_ty, zcu); const result_mcv: MCValue = if (maybe_inst) |inst| try self.copyToRegisterWithInstTracking(inst, ptr_ty, ptr_mcv) else @@ -5696,7 +6007,7 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void { try self.asmRegisterImmediate( .{ ._s, .bt }, opt_reg, - Immediate.u(@as(u6, @intCast(pl_abi_size * 8))), + .u(@as(u6, @intCast(pl_abi_size * 8))), ); }, @@ -5709,7 +6020,7 @@ fn airWrapOptional(self: *Self, inst: Air.Inst.Index) !void { .disp = frame_addr.off + pl_abi_size, } }, }, - Immediate.u(1), + .u(1), ), } } @@ -5733,8 +6044,8 @@ fn airWrapErrUnionPayload(self: *Self, inst: Air.Inst.Index) !void { if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result .{ .immediate = 0 }; const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(eu_ty, zcu)); - const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, zcu)); - const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu)); + const pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(pl_ty, zcu)); + const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu)); try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, operand, .{}); try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, .{ .immediate = 0 }, .{}); break :result .{ .load_frame = .{ .index = frame_index } }; @@ -5756,8 +6067,8 @@ fn airWrapErrUnionErr(self: *Self, inst: Air.Inst.Index) !void { if (!pl_ty.hasRuntimeBitsIgnoreComptime(zcu)) break :result try self.resolveInst(ty_op.operand); const frame_index = try self.allocFrameIndex(FrameAlloc.initSpill(eu_ty, zcu)); - const pl_off: i32 = @intCast(errUnionPayloadOffset(pl_ty, zcu)); - const err_off: i32 = @intCast(errUnionErrorOffset(pl_ty, zcu)); + const pl_off: i32 = @intCast(codegen.errUnionPayloadOffset(pl_ty, zcu)); + const err_off: i32 = @intCast(codegen.errUnionErrorOffset(pl_ty, zcu)); try self.genSetMem(.{ .frame = frame_index }, pl_off, pl_ty, .undef, .{}); const operand = try self.resolveInst(ty_op.operand); try self.genSetMem(.{ .frame = frame_index }, err_off, err_ty, operand, .{}); @@ -5770,11 +6081,20 @@ fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void { const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; const result = result: { const src_mcv = try self.resolveInst(ty_op.operand); - if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result src_mcv; + const ptr_mcv: MCValue = switch (src_mcv) { + .register_pair => |regs| .{ .register = regs[0] }, + else => src_mcv, + }; + if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) { + switch (src_mcv) { + .register_pair => |regs| try self.freeValue(.{ .register = regs[1] }), + else => {}, + } + break :result ptr_mcv; + } const dst_mcv = try self.allocRegOrMem(inst, true); - const dst_ty = self.typeOfIndex(inst); - try self.genCopy(dst_ty, dst_mcv, src_mcv, .{}); + try self.genCopy(self.typeOfIndex(inst), dst_mcv, ptr_mcv, .{}); break :result dst_mcv; }; return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); @@ -5782,23 +6102,28 @@ fn airSlicePtr(self: *Self, inst: Air.Inst.Index) !void { fn airSliceLen(self: *Self, inst: Air.Inst.Index) !void { const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - - const result: MCValue = result: { + const result = result: { const src_mcv = try self.resolveInst(ty_op.operand); - switch (src_mcv) { - .load_frame => |frame_addr| { - const len_mcv: MCValue = .{ .load_frame = .{ - .index = frame_addr.index, - .off = frame_addr.off + 8, - } }; - if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) break :result len_mcv; - - const dst_mcv = try self.allocRegOrMem(inst, true); - try self.genCopy(Type.usize, dst_mcv, len_mcv, .{}); - break :result dst_mcv; - }, + const len_mcv: MCValue = switch (src_mcv) { + .register_pair => |regs| .{ .register = regs[1] }, + .load_frame => |frame_addr| .{ .load_frame = .{ + .index = frame_addr.index, + .off = frame_addr.off + 8, + } }, else => return self.fail("TODO implement slice_len for {}", .{src_mcv}), + }; + if (self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) { + switch (src_mcv) { + .register_pair => |regs| try self.freeValue(.{ .register = regs[0] }), + .load_frame => {}, + else => unreachable, + } + break :result len_mcv; } + + const dst_mcv = try self.allocRegOrMem(inst, true); + try self.genCopy(self.typeOfIndex(inst), dst_mcv, len_mcv, .{}); + break :result dst_mcv; }; return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); } @@ -6296,27 +6621,27 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void { const src_bits: u31 = @intCast(src_ty.bitSize(zcu)); const has_lzcnt = self.hasFeature(.lzcnt); if (src_bits > @as(u32, if (has_lzcnt) 128 else 64)) { - const limbs_len = math.divCeil(u32, abi_size, 8) catch unreachable; + const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable; const extra_bits = abi_size * 8 - src_bits; const index_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); const index_lock = self.register_manager.lockRegAssumeUnused(index_reg); defer self.register_manager.unlockReg(index_lock); - try self.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), Immediate.u(limbs_len)); + try self.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), .u(limbs_len)); switch (extra_bits) { 1 => try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()), else => try self.asmRegisterImmediate( .{ ._, .mov }, dst_reg.to32(), - Immediate.s(@as(i32, extra_bits) - 1), + .s(@as(i32, extra_bits) - 1), ), } const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); try self.asmRegisterRegister(.{ ._, .@"test" }, index_reg.to32(), index_reg.to32()); const zero = try self.asmJccReloc(.z, undefined); if (self.hasFeature(.slow_incdec)) { - try self.asmRegisterImmediate(.{ ._, .sub }, index_reg.to32(), Immediate.u(1)); + try self.asmRegisterImmediate(.{ ._, .sub }, index_reg.to32(), .u(1)); } else { try self.asmRegister(.{ ._, .dec }, index_reg.to32()); } @@ -6328,7 +6653,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void { .scale = .@"8", .disp = src_mcv.load_frame.off, } }, - }, Immediate.u(0)); + }, .u(0)); _ = try self.asmJccReloc(.e, loop); try self.asmRegisterMemory(.{ ._, .bsr }, dst_reg.to64(), .{ .base = .{ .frame = src_mcv.load_frame.index }, @@ -6340,9 +6665,9 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void { } }, }); self.performReloc(zero); - try self.asmRegisterImmediate(.{ ._l, .sh }, index_reg.to32(), Immediate.u(6)); + try self.asmRegisterImmediate(.{ ._l, .sh }, index_reg.to32(), .u(6)); try self.asmRegisterRegister(.{ ._, .add }, index_reg.to32(), dst_reg.to32()); - try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), Immediate.u(src_bits - 1)); + try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), .u(src_bits - 1)); try self.asmRegisterRegister(.{ ._, .sub }, dst_reg.to32(), index_reg.to32()); break :result dst_mcv; } @@ -6404,7 +6729,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void { assert(src_bits <= 64); const cmov_abi_size = @max(@as(u32, @intCast(dst_ty.abiSize(zcu))), 2); - if (math.isPowerOfTwo(src_bits)) { + if (std.math.isPowerOfTwo(src_bits)) { const imm_reg = try self.copyToTmpRegister(dst_ty, .{ .immediate = src_bits ^ (src_bits - 1), }); @@ -6429,7 +6754,7 @@ fn airClz(self: *Self, inst: Air.Inst.Index) !void { try self.genBinOpMir(.{ ._, .xor }, dst_ty, dst_mcv, .{ .immediate = src_bits - 1 }); } else { const imm_reg = try self.copyToTmpRegister(dst_ty, .{ - .immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - self.regBitSize(dst_ty)), + .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - self.regBitSize(dst_ty)), }); const imm_lock = self.register_manager.lockRegAssumeUnused(imm_reg); defer self.register_manager.unlockReg(imm_lock); @@ -6493,30 +6818,30 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void { const src_bits: u31 = @intCast(src_ty.bitSize(zcu)); const has_bmi = self.hasFeature(.bmi); if (src_bits > @as(u32, if (has_bmi) 128 else 64)) { - const limbs_len = math.divCeil(u32, abi_size, 8) catch unreachable; + const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable; const extra_bits = abi_size * 8 - src_bits; const index_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); const index_lock = self.register_manager.lockRegAssumeUnused(index_reg); defer self.register_manager.unlockReg(index_lock); - try self.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), Immediate.s(-1)); + try self.asmRegisterImmediate(.{ ._, .mov }, index_reg.to32(), .s(-1)); switch (extra_bits) { 0 => try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()), 1 => try self.asmRegisterRegister(.{ ._, .mov }, dst_reg.to32(), dst_reg.to32()), else => try self.asmRegisterImmediate( .{ ._, .mov }, dst_reg.to32(), - Immediate.s(-@as(i32, extra_bits)), + .s(-@as(i32, extra_bits)), ), } const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); if (self.hasFeature(.slow_incdec)) { - try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), Immediate.u(1)); + try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), .u(1)); } else { try self.asmRegister(.{ ._, .inc }, index_reg.to32()); } - try self.asmRegisterImmediate(.{ ._, .cmp }, index_reg.to32(), Immediate.u(limbs_len)); + try self.asmRegisterImmediate(.{ ._, .cmp }, index_reg.to32(), .u(limbs_len)); const zero = try self.asmJccReloc(.nb, undefined); try self.asmMemoryImmediate(.{ ._, .cmp }, .{ .base = .{ .frame = src_mcv.load_frame.index }, @@ -6526,7 +6851,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void { .scale = .@"8", .disp = src_mcv.load_frame.off, } }, - }, Immediate.u(0)); + }, .u(0)); _ = try self.asmJccReloc(.e, loop); try self.asmRegisterMemory(.{ ._, .bsf }, dst_reg.to64(), .{ .base = .{ .frame = src_mcv.load_frame.index }, @@ -6538,7 +6863,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void { } }, }); self.performReloc(zero); - try self.asmRegisterImmediate(.{ ._l, .sh }, index_reg.to32(), Immediate.u(6)); + try self.asmRegisterImmediate(.{ ._l, .sh }, index_reg.to32(), .u(6)); try self.asmRegisterRegister(.{ ._, .add }, dst_reg.to32(), index_reg.to32()); break :result dst_mcv; } @@ -6558,7 +6883,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void { .{ ._, .@"or" }, wide_ty, tmp_mcv, - .{ .immediate = (@as(u64, math.maxInt(u64)) >> @intCast(64 - extra_bits)) << + .{ .immediate = (@as(u64, std.math.maxInt(u64)) >> @intCast(64 - extra_bits)) << @intCast(src_bits) }, ); break :masked tmp_mcv; @@ -6585,7 +6910,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void { .{ ._, .@"or" }, Type.u64, dst_mcv, - .{ .immediate = @as(u64, math.maxInt(u64)) << @intCast(src_bits - 64) }, + .{ .immediate = @as(u64, std.math.maxInt(u64)) << @intCast(src_bits - 64) }, ); break :masked dst_mcv; } else hi_mat_src_mcv; @@ -6602,7 +6927,7 @@ fn airCtz(self: *Self, inst: Air.Inst.Index) !void { const width_lock = self.register_manager.lockRegAssumeUnused(width_reg); defer self.register_manager.unlockReg(width_lock); - if (src_bits <= 8 or !math.isPowerOfTwo(src_bits)) { + if (src_bits <= 8 or !std.math.isPowerOfTwo(src_bits)) { const wide_reg = try self.copyToTmpRegister(src_ty, mat_src_mcv); const wide_lock = self.register_manager.lockRegAssumeUnused(wide_reg); defer self.register_manager.unlockReg(wide_lock); @@ -6701,11 +7026,11 @@ fn genPopCount( }, ); - const mask = @as(u64, math.maxInt(u64)) >> @intCast(64 - src_abi_size * 8); - const imm_0_1 = Immediate.u(mask / 0b1_1); - const imm_00_11 = Immediate.u(mask / 0b01_01); - const imm_0000_1111 = Immediate.u(mask / 0b0001_0001); - const imm_0000_0001 = Immediate.u(mask / 0b1111_1111); + const mask = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - src_abi_size * 8); + const imm_0_1: Immediate = .u(mask / 0b1_1); + const imm_00_11: Immediate = .u(mask / 0b01_01); + const imm_0000_1111: Immediate = .u(mask / 0b0001_0001); + const imm_0000_0001: Immediate = .u(mask / 0b1111_1111); const tmp_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); const tmp_lock = self.register_manager.lockRegAssumeUnused(tmp_reg); @@ -6722,7 +7047,7 @@ fn genPopCount( // dst = operand try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst); // tmp = operand - try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, Immediate.u(1)); + try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, .u(1)); // tmp = operand >> 1 if (src_abi_size > 4) { try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0_1); @@ -6733,7 +7058,7 @@ fn genPopCount( // dst = temp1 = operand - ((operand >> 1) & 0x55...55) try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst); // tmp = temp1 - try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u(2)); + try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u(2)); // dst = temp1 >> 2 if (src_abi_size > 4) { try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_00_11); @@ -6749,7 +7074,7 @@ fn genPopCount( // tmp = temp2 = (temp1 & 0x33...33) + ((temp1 >> 2) & 0x33...33) try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp); // dst = temp2 - try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, Immediate.u(4)); + try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, .u(4)); // tmp = temp2 >> 4 try self.asmRegisterRegister(.{ ._, .add }, dst, tmp); // dst = temp2 + (temp2 >> 4) @@ -6767,7 +7092,7 @@ fn genPopCount( // dst = temp3 = (temp2 + (temp2 >> 4)) & 0x0f...0f // dst = temp3 * 0x01...01 if (src_abi_size > 1) { - try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u((src_abi_size - 1) * 8)); + try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u((src_abi_size - 1) * 8)); } // dst = (temp3 * 0x01...01) >> (bits - 8) } @@ -6847,7 +7172,7 @@ fn genByteSwap( return .{ .register_pair = .{ dst_regs[1], dst_regs[0] } }; }, else => { - const limbs_len = math.divCeil(u32, abi_size, 8) catch unreachable; + const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable; const temp_regs = try self.register_manager.allocRegs(4, .{null} ** 4, abi.RegisterClass.gp); @@ -6856,11 +7181,7 @@ fn genByteSwap( const dst_mcv = try self.allocRegOrMem(inst, false); try self.asmRegisterRegister(.{ ._, .xor }, temp_regs[0].to32(), temp_regs[0].to32()); - try self.asmRegisterImmediate( - .{ ._, .mov }, - temp_regs[1].to32(), - Immediate.u(limbs_len - 1), - ); + try self.asmRegisterImmediate(.{ ._, .mov }, temp_regs[1].to32(), .u(limbs_len - 1)); const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); try self.asmRegisterMemory( @@ -6912,8 +7233,8 @@ fn genByteSwap( } }, }, temp_regs[2].to64()); if (self.hasFeature(.slow_incdec)) { - try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), Immediate.u(1)); - try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), Immediate.u(1)); + try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), .u(1)); + try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), .u(1)); } else { try self.asmRegister(.{ ._, .inc }, temp_regs[0].to32()); try self.asmRegister(.{ ._, .dec }, temp_regs[1].to32()); @@ -6994,10 +7315,10 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void { else undefined; - const mask = @as(u64, math.maxInt(u64)) >> @intCast(64 - limb_abi_size * 8); - const imm_0000_1111 = Immediate.u(mask / 0b0001_0001); - const imm_00_11 = Immediate.u(mask / 0b01_01); - const imm_0_1 = Immediate.u(mask / 0b1_1); + const mask = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - limb_abi_size * 8); + const imm_0000_1111: Immediate = .u(mask / 0b0001_0001); + const imm_00_11: Immediate = .u(mask / 0b01_01); + const imm_0_1: Immediate = .u(mask / 0b1_1); for (dst_mcv.getRegs()) |dst_reg| { const dst = registerAlias(dst_reg, limb_abi_size); @@ -7005,7 +7326,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void { // dst = temp1 = bswap(operand) try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst); // tmp = temp1 - try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u(4)); + try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u(4)); // dst = temp1 >> 4 if (limb_abi_size > 4) { try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0000_1111); @@ -7017,13 +7338,13 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void { } // tmp = temp1 & 0x0F...0F // dst = (temp1 >> 4) & 0x0F...0F - try self.asmRegisterImmediate(.{ ._l, .sh }, tmp, Immediate.u(4)); + try self.asmRegisterImmediate(.{ ._l, .sh }, tmp, .u(4)); // tmp = (temp1 & 0x0F...0F) << 4 try self.asmRegisterRegister(.{ ._, .@"or" }, dst, tmp); // dst = temp2 = ((temp1 >> 4) & 0x0F...0F) | ((temp1 & 0x0F...0F) << 4) try self.asmRegisterRegister(.{ ._, .mov }, tmp, dst); // tmp = temp2 - try self.asmRegisterImmediate(.{ ._r, .sh }, dst, Immediate.u(2)); + try self.asmRegisterImmediate(.{ ._r, .sh }, dst, .u(2)); // dst = temp2 >> 2 if (limb_abi_size > 4) { try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_00_11); @@ -7050,7 +7371,7 @@ fn airBitReverse(self: *Self, inst: Air.Inst.Index) !void { // tmp = temp3 = ((temp2 >> 2) & 0x33...33) + ((temp2 & 0x33...33) << 2) try self.asmRegisterRegister(.{ ._, .mov }, dst, tmp); // dst = temp3 - try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, Immediate.u(1)); + try self.asmRegisterImmediate(.{ ._r, .sh }, tmp, .u(1)); // tmp = temp3 >> 1 if (limb_abi_size > 4) { try self.asmRegisterImmediate(.{ ._, .mov }, imm, imm_0_1); @@ -7337,7 +7658,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro dst_alias, dst_alias, try src_mcv.mem(self, Memory.Size.fromSize(abi_size)), - Immediate.u(@as(u5, @bitCast(mode))), + .u(@as(u5, @bitCast(mode))), ) else try self.asmRegisterRegisterRegisterImmediate( mir_tag, dst_alias, @@ -7346,13 +7667,13 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro src_mcv.getReg().? else try self.copyToTmpRegister(ty, src_mcv), abi_size), - Immediate.u(@as(u5, @bitCast(mode))), + .u(@as(u5, @bitCast(mode))), ), else => if (src_mcv.isMemory()) try self.asmRegisterMemoryImmediate( mir_tag, dst_alias, try src_mcv.mem(self, Memory.Size.fromSize(abi_size)), - Immediate.u(@as(u5, @bitCast(mode))), + .u(@as(u5, @bitCast(mode))), ) else try self.asmRegisterRegisterImmediate( mir_tag, dst_alias, @@ -7360,7 +7681,7 @@ fn genRound(self: *Self, ty: Type, dst_reg: Register, src_mcv: MCValue, mode: Ro src_mcv.getReg().? else try self.copyToTmpRegister(ty, src_mcv), abi_size), - Immediate.u(@as(u5, @bitCast(mode))), + .u(@as(u5, @bitCast(mode))), ), } } @@ -7433,7 +7754,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void { defer self.register_manager.unlockReg(tmp_lock); try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, dst_regs[1]); - try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, Immediate.u(63)); + try self.asmRegisterImmediate(.{ ._r, .sa }, tmp_reg, .u(63)); try self.asmRegisterRegister(.{ ._, .xor }, dst_regs[0], tmp_reg); try self.asmRegisterRegister(.{ ._, .xor }, dst_regs[1], tmp_reg); try self.asmRegisterRegister(.{ ._, .sub }, dst_regs[0], tmp_reg); @@ -7443,7 +7764,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void { }, else => { const abi_size: u31 = @intCast(ty.abiSize(zcu)); - const limb_len = math.divCeil(u31, abi_size, 8) catch unreachable; + const limb_len = std.math.divCeil(u31, abi_size, 8) catch unreachable; const tmp_regs = try self.register_manager.allocRegs(3, .{null} ** 3, abi.RegisterClass.gp); @@ -7460,7 +7781,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void { try self.asmMemoryImmediate( .{ ._, .cmp }, try dst_mcv.address().offset((limb_len - 1) * 8).deref().mem(self, .qword), - Immediate.u(0), + .u(0), ); const positive = try self.asmJccReloc(.ns, undefined); @@ -7469,7 +7790,7 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void { const neg_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); try self.asmRegisterRegister(.{ ._, .xor }, tmp_regs[2].to32(), tmp_regs[2].to32()); - try self.asmRegisterImmediate(.{ ._r, .sh }, tmp_regs[1].to8(), Immediate.u(1)); + try self.asmRegisterImmediate(.{ ._r, .sh }, tmp_regs[1].to8(), .u(1)); try self.asmRegisterMemory(.{ ._, .sbb }, tmp_regs[2].to64(), .{ .base = .{ .frame = dst_mcv.load_frame.index }, .mod = .{ .rm = .{ @@ -7491,11 +7812,11 @@ fn airAbs(self: *Self, inst: Air.Inst.Index) !void { }, tmp_regs[2].to64()); if (self.hasFeature(.slow_incdec)) { - try self.asmRegisterImmediate(.{ ._, .add }, tmp_regs[0].to32(), Immediate.u(1)); + try self.asmRegisterImmediate(.{ ._, .add }, tmp_regs[0].to32(), .u(1)); } else { try self.asmRegister(.{ ._, .inc }, tmp_regs[0].to32()); } - try self.asmRegisterImmediate(.{ ._, .cmp }, tmp_regs[0].to32(), Immediate.u(limb_len)); + try self.asmRegisterImmediate(.{ ._, .cmp }, tmp_regs[0].to32(), .u(limb_len)); _ = try self.asmJccReloc(.b, neg_loop); self.performReloc(positive); @@ -7620,7 +7941,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void { .{ .v_, .cvtps2ph }, dst_reg, dst_reg, - Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))), + .u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))), ); break :result dst_mcv; }, @@ -7650,7 +7971,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void { .{ .v_, .cvtps2ph }, dst_reg, dst_reg, - Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))), + .u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))), ); break :result dst_mcv; }, @@ -7675,7 +7996,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void { .{ .v_, .cvtps2ph }, dst_reg, wide_reg, - Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))), + .u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))), ); break :result dst_mcv; }, @@ -7699,9 +8020,7 @@ fn airSqrt(self: *Self, inst: Air.Inst.Index) !void { else => unreachable, }, else => unreachable, - }) orelse return self.fail("TODO implement airSqrt for {}", .{ - ty.fmt(pt), - }); + }) orelse return self.fail("TODO implement airSqrt for {}", .{ty.fmt(pt)}); switch (mir_tag[0]) { .v_ss, .v_sd => if (src_mcv.isMemory()) try self.asmRegisterRegisterMemory( mir_tag, @@ -7805,7 +8124,7 @@ fn reuseOperandAdvanced( } // Prevent the operand deaths processing code from deallocating it. - self.liveness.clearOperandDeath(inst, op_index); + self.reused_operands.set(op_index); const op_inst = operand.toIndex().?; self.getResolvedInstValue(op_inst).reuse(self, maybe_tracked_inst, op_inst); @@ -7890,7 +8209,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn } }, }); try self.spillEflagsIfOccupied(); - try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(val_bit_off)); + try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, .u(val_bit_off)); } else { const tmp_reg = registerAlias(try self.register_manager.allocReg(null, abi.RegisterClass.gp), val_abi_size); @@ -7913,12 +8232,7 @@ fn packedLoad(self: *Self, dst_mcv: MCValue, ptr_ty: Type, ptr_mcv: MCValue) Inn } }, }); try self.spillEflagsIfOccupied(); - try self.asmRegisterRegisterImmediate( - .{ ._rd, .sh }, - dst_alias, - tmp_reg, - Immediate.u(val_bit_off), - ); + try self.asmRegisterRegisterImmediate(.{ ._rd, .sh }, dst_alias, tmp_reg, .u(val_bit_off)); } if (val_extra_bits > 0) try self.truncateRegister(val_ty, dst_reg); @@ -8064,16 +8378,16 @@ fn packedStore(self: *Self, ptr_ty: Type, ptr_mcv: MCValue, src_mcv: MCValue) In } }, }; - const part_mask = (@as(u64, math.maxInt(u64)) >> @intCast(64 - part_bit_size)) << + const part_mask = (@as(u64, std.math.maxInt(u64)) >> @intCast(64 - part_bit_size)) << @intCast(part_bit_off); - const part_mask_not = part_mask ^ (@as(u64, math.maxInt(u64)) >> @intCast(64 - limb_abi_bits)); + const part_mask_not = part_mask ^ (@as(u64, std.math.maxInt(u64)) >> @intCast(64 - limb_abi_bits)); if (limb_abi_size <= 4) { - try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, Immediate.u(part_mask_not)); - } else if (math.cast(i32, @as(i64, @bitCast(part_mask_not)))) |small| { - try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, Immediate.s(small)); + try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, .u(part_mask_not)); + } else if (std.math.cast(i32, @as(i64, @bitCast(part_mask_not)))) |small| { + try self.asmMemoryImmediate(.{ ._, .@"and" }, limb_mem, .s(small)); } else { const part_mask_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); - try self.asmRegisterImmediate(.{ ._, .mov }, part_mask_reg, Immediate.u(part_mask_not)); + try self.asmRegisterImmediate(.{ ._, .mov }, part_mask_reg, .u(part_mask_not)); try self.asmMemoryRegister(.{ ._, .@"and" }, limb_mem, part_mask_reg); } @@ -8209,25 +8523,14 @@ fn airStructFieldPtr(self: *Self, inst: Air.Inst.Index) !void { return self.finishAir(inst, result, .{ extra.struct_operand, .none, .none }); } -fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, index: u8) !void { +fn airStructFieldPtrIndex(self: *Self, inst: Air.Inst.Index, field_index: u8) !void { const ty_op = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_op; - const result = try self.fieldPtr(inst, ty_op.operand, index); + const result = try self.fieldPtr(inst, ty_op.operand, field_index); return self.finishAir(inst, result, .{ ty_op.operand, .none, .none }); } -fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32) !MCValue { - const pt = self.pt; - const zcu = pt.zcu; +fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, field_index: u32) !MCValue { const ptr_field_ty = self.typeOfIndex(inst); - const ptr_container_ty = self.typeOf(operand); - const container_ty = ptr_container_ty.childType(zcu); - - const field_off: i32 = switch (container_ty.containerLayout(zcu)) { - .auto, .@"extern" => @intCast(container_ty.structFieldOffset(index, zcu)), - .@"packed" => @divExact(@as(i32, ptr_container_ty.ptrInfo(zcu).packed_offset.bit_offset) + - (if (zcu.typeToStruct(container_ty)) |struct_obj| pt.structPackedFieldBitOffset(struct_obj, index) else 0) - - ptr_field_ty.ptrInfo(zcu).packed_offset.bit_offset, 8), - }; const src_mcv = try self.resolveInst(operand); const dst_mcv = if (switch (src_mcv) { @@ -8235,7 +8538,19 @@ fn fieldPtr(self: *Self, inst: Air.Inst.Index, operand: Air.Inst.Ref, index: u32 .register, .register_offset => self.reuseOperand(inst, operand, 0, src_mcv), else => false, }) src_mcv else try self.copyToRegisterWithInstTracking(inst, ptr_field_ty, src_mcv); - return dst_mcv.offset(field_off); + return dst_mcv.offset(self.fieldOffset(self.typeOf(operand), ptr_field_ty, field_index)); +} + +fn fieldOffset(self: *Self, ptr_agg_ty: Type, ptr_field_ty: Type, field_index: u32) i32 { + const pt = self.pt; + const zcu = pt.zcu; + const agg_ty = ptr_agg_ty.childType(zcu); + return switch (agg_ty.containerLayout(zcu)) { + .auto, .@"extern" => @intCast(agg_ty.structFieldOffset(field_index, zcu)), + .@"packed" => @divExact(@as(i32, ptr_agg_ty.ptrInfo(zcu).packed_offset.bit_offset) + + (if (zcu.typeToStruct(agg_ty)) |struct_obj| pt.structPackedFieldBitOffset(struct_obj, field_index) else 0) - + ptr_field_ty.ptrInfo(zcu).packed_offset.bit_offset, 8), + }; } fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { @@ -8476,7 +8791,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { } }, }); try self.spillEflagsIfOccupied(); - try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, Immediate.u(field_bit_off)); + try self.asmRegisterImmediate(.{ ._r, .sh }, load_reg, .u(field_bit_off)); } else { const tmp_reg = registerAlias( try self.register_manager.allocReg(null, abi.RegisterClass.gp), @@ -8509,7 +8824,7 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { .{ ._rd, .sh }, dst_alias, tmp_reg, - Immediate.u(field_bit_off), + .u(field_bit_off), ); } @@ -8528,27 +8843,17 @@ fn airStructFieldVal(self: *Self, inst: Air.Inst.Index) !void { } fn airFieldParentPtr(self: *Self, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const extra = self.air.extraData(Air.FieldParentPtr, ty_pl.payload).data; - const inst_ty = self.typeOfIndex(inst); - const parent_ty = inst_ty.childType(zcu); - const field_off: i32 = switch (parent_ty.containerLayout(zcu)) { - .auto, .@"extern" => @intCast(parent_ty.structFieldOffset(extra.field_index, zcu)), - .@"packed" => @divExact(@as(i32, inst_ty.ptrInfo(zcu).packed_offset.bit_offset) + - (if (zcu.typeToStruct(parent_ty)) |struct_obj| pt.structPackedFieldBitOffset(struct_obj, extra.field_index) else 0) - - self.typeOf(extra.field_ptr).ptrInfo(zcu).packed_offset.bit_offset, 8), - }; - + const ptr_agg_ty = self.typeOfIndex(inst); const src_mcv = try self.resolveInst(extra.field_ptr); const dst_mcv = if (src_mcv.isRegisterOffset() and self.reuseOperand(inst, extra.field_ptr, 0, src_mcv)) src_mcv else - try self.copyToRegisterWithInstTracking(inst, inst_ty, src_mcv); - const result = dst_mcv.offset(-field_off); + try self.copyToRegisterWithInstTracking(inst, ptr_agg_ty, src_mcv); + const result = dst_mcv.offset(-self.fieldOffset(ptr_agg_ty, self.typeOf(extra.field_ptr), extra.field_index)); return self.finishAir(inst, result, .{ extra.field_ptr, .none, .none }); } @@ -8613,7 +8918,7 @@ fn genUnOp(self: *Self, maybe_inst: ?Air.Inst.Index, tag: Air.Inst.Tag, src_air: }; if (int_info.signedness == .unsigned and self.regExtraBits(limb_ty) > 0) { - const mask = @as(u64, math.maxInt(u64)) >> @intCast(64 - limb_bits); + const mask = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - limb_bits); try self.genBinOpMir(.{ ._, .xor }, limb_ty, limb_mcv, .{ .immediate = mask }); } else try self.genUnOpMir(.{ ._, .not }, limb_ty, limb_mcv); } @@ -8698,7 +9003,7 @@ fn genShiftBinOpMir( try self.spillEflagsIfOccupied(); if (abi_size > 16) { - const limbs_len = math.divCeil(u32, abi_size, 8) catch unreachable; + const limbs_len = std.math.divCeil(u32, abi_size, 8) catch unreachable; assert(shift_abi_size >= 1 and shift_abi_size <= 2); const rcx_lock: ?RegisterLock = switch (rhs_mcv) { @@ -8725,12 +9030,12 @@ fn genShiftBinOpMir( switch (tag[0]) { ._l => { - try self.asmRegisterImmediate(.{ ._, .mov }, temp_regs[1].to32(), Immediate.u(limbs_len - 1)); + try self.asmRegisterImmediate(.{ ._, .mov }, temp_regs[1].to32(), .u(limbs_len - 1)); switch (rhs_mcv) { .immediate => |shift_imm| try self.asmRegisterImmediate( .{ ._, .mov }, temp_regs[0].to32(), - Immediate.u(limbs_len - (shift_imm >> 6) - 1), + .u(limbs_len - (shift_imm >> 6) - 1), ), else => { try self.asmRegisterRegister( @@ -8738,16 +9043,8 @@ fn genShiftBinOpMir( temp_regs[2].to32(), registerAlias(.rcx, shift_abi_size), ); - try self.asmRegisterImmediate( - .{ ._, .@"and" }, - .cl, - Immediate.u(math.maxInt(u6)), - ); - try self.asmRegisterImmediate( - .{ ._r, .sh }, - temp_regs[2].to32(), - Immediate.u(6), - ); + try self.asmRegisterImmediate(.{ ._, .@"and" }, .cl, .u(std.math.maxInt(u6))); + try self.asmRegisterImmediate(.{ ._r, .sh }, temp_regs[2].to32(), .u(6)); try self.asmRegisterRegister( .{ ._, .mov }, temp_regs[0].to32(), @@ -8767,7 +9064,7 @@ fn genShiftBinOpMir( .immediate => |shift_imm| try self.asmRegisterImmediate( .{ ._, .mov }, temp_regs[0].to32(), - Immediate.u(shift_imm >> 6), + .u(shift_imm >> 6), ), else => { try self.asmRegisterRegister( @@ -8775,16 +9072,8 @@ fn genShiftBinOpMir( temp_regs[0].to32(), registerAlias(.rcx, shift_abi_size), ); - try self.asmRegisterImmediate( - .{ ._, .@"and" }, - .cl, - Immediate.u(math.maxInt(u6)), - ); - try self.asmRegisterImmediate( - .{ ._r, .sh }, - temp_regs[0].to32(), - Immediate.u(6), - ); + try self.asmRegisterImmediate(.{ ._, .@"and" }, .cl, .u(std.math.maxInt(u6))); + try self.asmRegisterImmediate(.{ ._r, .sh }, temp_regs[0].to32(), .u(6)); }, } }, @@ -8813,7 +9102,7 @@ fn genShiftBinOpMir( try self.asmRegisterImmediate( .{ ._, .cmp }, temp_regs[0].to32(), - Immediate.u(limbs_len - 1), + .u(limbs_len - 1), ); break :skip try self.asmJccReloc(.nb, undefined); }, @@ -8843,7 +9132,7 @@ fn genShiftBinOpMir( }, .sh }, temp_regs[2].to64(), temp_regs[3].to64(), - Immediate.u(shift_imm & math.maxInt(u6)), + .u(shift_imm & std.math.maxInt(u6)), ), else => try self.asmRegisterRegisterRegister(.{ switch (tag[0]) { ._l => ._ld, @@ -8864,8 +9153,8 @@ fn genShiftBinOpMir( switch (tag[0]) { ._l => { if (slow_inc_dec) { - try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), Immediate.u(1)); - try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[0].to32(), Immediate.u(1)); + try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), .u(1)); + try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[0].to32(), .u(1)); } else { try self.asmRegister(.{ ._, .dec }, temp_regs[1].to32()); try self.asmRegister(.{ ._, .dec }, temp_regs[0].to32()); @@ -8874,8 +9163,8 @@ fn genShiftBinOpMir( }, ._r => { if (slow_inc_dec) { - try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[1].to32(), Immediate.u(1)); - try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), Immediate.u(1)); + try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[1].to32(), .u(1)); + try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), .u(1)); } else { try self.asmRegister(.{ ._, .inc }, temp_regs[1].to32()); try self.asmRegister(.{ ._, .inc }, temp_regs[0].to32()); @@ -8883,7 +9172,7 @@ fn genShiftBinOpMir( try self.asmRegisterImmediate( .{ ._, .cmp }, temp_regs[0].to32(), - Immediate.u(limbs_len - 1), + .u(limbs_len - 1), ); _ = try self.asmJccReloc(.b, loop); }, @@ -8898,7 +9187,7 @@ fn genShiftBinOpMir( .immediate => |shift_imm| try self.asmRegisterImmediate( tag, temp_regs[2].to64(), - Immediate.u(shift_imm & math.maxInt(u6)), + .u(shift_imm & std.math.maxInt(u6)), ), else => try self.asmRegisterRegister(tag, temp_regs[2].to64(), .cl), } @@ -8914,7 +9203,7 @@ fn genShiftBinOpMir( if (tag[0] == ._r and tag[1] == .sa) try self.asmRegisterImmediate( tag, temp_regs[2].to64(), - Immediate.u(63), + .u(63), ); if (switch (rhs_mcv) { .immediate => |shift_imm| shift_imm >> 6 > 0, @@ -8935,7 +9224,7 @@ fn genShiftBinOpMir( try self.asmRegisterImmediate( .{ ._, .cmp }, temp_regs[1].to32(), - Immediate.u(limbs_len - 1), + .u(limbs_len - 1), ); break :skip try self.asmJccReloc(.nb, undefined); }, @@ -8945,12 +9234,12 @@ fn genShiftBinOpMir( const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); switch (tag[0]) { ._l => if (slow_inc_dec) { - try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), Immediate.u(1)); + try self.asmRegisterImmediate(.{ ._, .sub }, temp_regs[1].to32(), .u(1)); } else { try self.asmRegister(.{ ._, .dec }, temp_regs[1].to32()); }, ._r => if (slow_inc_dec) { - try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[1].to32(), Immediate.u(1)); + try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[1].to32(), .u(1)); } else { try self.asmRegister(.{ ._, .inc }, temp_regs[1].to32()); }, @@ -8972,14 +9261,14 @@ fn genShiftBinOpMir( .scale = .@"8", .disp = lhs_mcv.load_frame.off, } }, - }, Immediate.u(0)); + }, .u(0)); switch (tag[0]) { ._l => _ = try self.asmJccReloc(.nz, loop), ._r => { try self.asmRegisterImmediate( .{ ._, .cmp }, temp_regs[1].to32(), - Immediate.u(limbs_len - 1), + .u(limbs_len - 1), ); _ = try self.asmJccReloc(.b, loop); }, @@ -9021,12 +9310,12 @@ fn genShiftBinOpMir( info.double_tag, lhs_regs[info.indices[1]], lhs_regs[info.indices[0]], - Immediate.u(shift_imm), + .u(shift_imm), ); try self.asmRegisterImmediate( tag, lhs_regs[info.indices[0]], - Immediate.u(shift_imm), + .u(shift_imm), ); return; } else { @@ -9039,7 +9328,7 @@ fn genShiftBinOpMir( if (tag[0] == ._r and tag[1] == .sa) try self.asmRegisterImmediate( tag, lhs_regs[info.indices[0]], - Immediate.u(63), + .u(63), ) else try self.asmRegisterRegister( .{ ._, .xor }, lhs_regs[info.indices[0]], @@ -9048,7 +9337,7 @@ fn genShiftBinOpMir( if (shift_imm > 64) try self.asmRegisterImmediate( tag, lhs_regs[info.indices[1]], - Immediate.u(shift_imm - 64), + .u(shift_imm - 64), ); return; }, @@ -9059,7 +9348,7 @@ fn genShiftBinOpMir( if (tag[0] == ._r and tag[1] == .sa) { try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, lhs_regs[info.indices[0]]); - try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(63)); + try self.asmRegisterImmediate(tag, tmp_reg, .u(63)); } else try self.asmRegisterRegister( .{ ._, .xor }, tmp_reg.to32(), @@ -9076,11 +9365,7 @@ fn genShiftBinOpMir( lhs_regs[info.indices[0]], registerAlias(shift_reg, 1), ); - try self.asmRegisterImmediate( - .{ ._, .cmp }, - registerAlias(shift_reg, 1), - Immediate.u(64), - ); + try self.asmRegisterImmediate(.{ ._, .cmp }, registerAlias(shift_reg, 1), .u(64)); try self.asmCmovccRegisterRegister( .ae, lhs_regs[info.indices[1]], @@ -9119,7 +9404,7 @@ fn genShiftBinOpMir( } }, }, tmp_reg, - Immediate.u(shift_imm), + .u(shift_imm), ); try self.asmMemoryImmediate( tag, @@ -9130,7 +9415,7 @@ fn genShiftBinOpMir( .disp = dst_frame_addr.off + info.indices[0] * 8, } }, }, - Immediate.u(shift_imm), + .u(shift_imm), ); return; } else { @@ -9149,7 +9434,7 @@ fn genShiftBinOpMir( if (shift_imm > 64) try self.asmRegisterImmediate( tag, tmp_reg, - Immediate.u(shift_imm - 64), + .u(shift_imm - 64), ); try self.asmMemoryRegister( .{ ._, .mov }, @@ -9171,7 +9456,7 @@ fn genShiftBinOpMir( .disp = dst_frame_addr.off + info.indices[0] * 8, } }, }, - Immediate.u(63), + .u(63), ) else { try self.asmRegisterRegister(.{ ._, .xor }, tmp_reg.to32(), tmp_reg.to32()); try self.asmMemoryRegister( @@ -9223,7 +9508,7 @@ fn genShiftBinOpMir( ); if (tag[0] == ._r and tag[1] == .sa) { try self.asmRegisterRegister(.{ ._, .mov }, tmp_reg, first_reg); - try self.asmRegisterImmediate(tag, tmp_reg, Immediate.u(63)); + try self.asmRegisterImmediate(tag, tmp_reg, .u(63)); } else try self.asmRegisterRegister( .{ ._, .xor }, tmp_reg.to32(), @@ -9239,7 +9524,7 @@ fn genShiftBinOpMir( try self.asmRegisterImmediate( .{ ._, .cmp }, registerAlias(shift_reg, 1), - Immediate.u(64), + .u(64), ); try self.asmCmovccRegisterRegister(.ae, second_reg, first_reg); try self.asmCmovccRegisterRegister(.ae, first_reg, tmp_reg); @@ -9277,7 +9562,7 @@ fn genShiftBinOpMir( .immediate => |shift_imm| return self.asmRegisterImmediate( tag, registerAlias(lhs_reg, abi_size), - Immediate.u(shift_imm), + .u(shift_imm), ), .register => |shift_reg| return self.asmRegisterRegister( tag, @@ -9292,7 +9577,7 @@ fn genShiftBinOpMir( .base = .{ .reg = .ds }, .mod = .{ .rm = .{ .size = Memory.Size.fromSize(abi_size), - .disp = math.cast(i32, @as(i64, @bitCast(addr))) orelse + .disp = std.math.cast(i32, @as(i64, @bitCast(addr))) orelse return self.fail("TODO genShiftBinOpMir between {s} and {s}", .{ @tagName(lhs_mcv), @tagName(shift_mcv), @@ -9316,11 +9601,7 @@ fn genShiftBinOpMir( else => unreachable, }; switch (shift_mcv) { - .immediate => |shift_imm| return self.asmMemoryImmediate( - tag, - lhs_mem, - Immediate.u(shift_imm), - ), + .immediate => |shift_imm| return self.asmMemoryImmediate(tag, lhs_mem, .u(shift_imm)), .register => |shift_reg| return self.asmMemoryRegister( tag, lhs_mem, @@ -9495,7 +9776,7 @@ fn genMulDivBinOp( switch (tag) { .mul, .mul_wrap => { const slow_inc = self.hasFeature(.slow_incdec); - const limb_len = math.divCeil(u32, src_abi_size, 8) catch unreachable; + const limb_len = std.math.divCeil(u32, src_abi_size, 8) catch unreachable; try self.spillRegisters(&.{ .rax, .rcx, .rdx }); const reg_locks = self.register_manager.lockRegs(3, .{ .rax, .rcx, .rdx }); @@ -9536,7 +9817,7 @@ fn genMulDivBinOp( try self.asmRegisterRegister(.{ ._, .xor }, .edx, .edx); const inner_loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); - try self.asmRegisterImmediate(.{ ._r, .sh }, .cl, Immediate.u(1)); + try self.asmRegisterImmediate(.{ ._r, .sh }, .cl, .u(1)); try self.asmMemoryRegister(.{ ._, .adc }, .{ .base = .{ .frame = dst_mcv.load_frame.index }, .mod = .{ .rm = .{ @@ -9559,7 +9840,7 @@ fn genMulDivBinOp( }); try self.asmRegister(.{ ._, .mul }, temp_regs[1].to64()); - try self.asmRegisterImmediate(.{ ._r, .sh }, .ch, Immediate.u(1)); + try self.asmRegisterImmediate(.{ ._r, .sh }, .ch, .u(1)); try self.asmMemoryRegister(.{ ._, .adc }, .{ .base = .{ .frame = dst_mcv.load_frame.index }, .mod = .{ .rm = .{ @@ -9572,30 +9853,22 @@ fn genMulDivBinOp( try self.asmSetccRegister(.c, .ch); if (slow_inc) { - try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), Immediate.u(1)); - try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[3].to32(), Immediate.u(1)); + try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[2].to32(), .u(1)); + try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[3].to32(), .u(1)); } else { try self.asmRegister(.{ ._, .inc }, temp_regs[2].to32()); try self.asmRegister(.{ ._, .inc }, temp_regs[3].to32()); } - try self.asmRegisterImmediate( - .{ ._, .cmp }, - temp_regs[3].to32(), - Immediate.u(limb_len), - ); + try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[3].to32(), .u(limb_len)); _ = try self.asmJccReloc(.b, inner_loop); self.performReloc(skip_inner); if (slow_inc) { - try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), Immediate.u(1)); + try self.asmRegisterImmediate(.{ ._, .add }, temp_regs[0].to32(), .u(1)); } else { try self.asmRegister(.{ ._, .inc }, temp_regs[0].to32()); } - try self.asmRegisterImmediate( - .{ ._, .cmp }, - temp_regs[0].to32(), - Immediate.u(limb_len), - ); + try self.asmRegisterImmediate(.{ ._, .cmp }, temp_regs[0].to32(), .u(limb_len)); _ = try self.asmJccReloc(.b, outer_loop); return dst_mcv; @@ -9911,7 +10184,7 @@ fn genBinOp( dst_reg, dst_reg, try rhs_mcv.mem(self, .word), - Immediate.u(1), + .u(1), ) else try self.asmRegisterRegisterRegister( .{ .vp_, .unpcklwd }, dst_reg, @@ -9969,7 +10242,7 @@ fn genBinOp( .{ .v_, .cvtps2ph }, dst_reg, dst_reg, - Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))), + .u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))), ); break :adjusted .{ .register = dst_reg }; }, @@ -10278,7 +10551,7 @@ fn genBinOp( .lea_tlv, .lea_frame, => true, - .memory => |addr| math.cast(i32, @as(i64, @bitCast(addr))) == null, + .memory => |addr| std.math.cast(i32, @as(i64, @bitCast(addr))) == null, else => false, .register_pair, .register_overflow, @@ -10410,7 +10683,7 @@ fn genBinOp( dst_reg, dst_reg, try src_mcv.mem(self, .word), - Immediate.u(1), + .u(1), ) else try self.asmRegisterRegisterRegister( .{ .vp_, .unpcklwd }, dst_reg, @@ -10442,7 +10715,7 @@ fn genBinOp( dst_reg, dst_reg, dst_reg, - Immediate.u(@as(u5, @bitCast(RoundMode{ + .u(@as(u5, @bitCast(RoundMode{ .mode = switch (air_tag) { .div_trunc => .zero, .div_floor => .down, @@ -10457,7 +10730,7 @@ fn genBinOp( .{ .v_, .cvtps2ph }, dst_reg, dst_reg, - Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))), + .u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))), ); return dst_mcv; }, @@ -10856,7 +11129,7 @@ fn genBinOp( dst_reg, dst_reg, try src_mcv.mem(self, .word), - Immediate.u(1), + .u(1), ) else try self.asmRegisterRegisterRegister( .{ .vp_, .unpcklwd }, dst_reg, @@ -10886,7 +11159,7 @@ fn genBinOp( .{ .v_, .cvtps2ph }, dst_reg, dst_reg, - Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))), + .u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))), ); return dst_mcv; }, @@ -10902,7 +11175,7 @@ fn genBinOp( .{ .vp_d, .insr }, dst_reg, try src_mcv.mem(self, .dword), - Immediate.u(1), + .u(1), ) else try self.asmRegisterRegisterRegister( .{ .v_ps, .unpckl }, dst_reg, @@ -10937,7 +11210,7 @@ fn genBinOp( .{ .v_, .cvtps2ph }, dst_reg, dst_reg, - Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))), + .u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))), ); return dst_mcv; }, @@ -10980,7 +11253,7 @@ fn genBinOp( .{ .v_, .cvtps2ph }, dst_reg, dst_reg, - Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))), + .u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))), ); return dst_mcv; }, @@ -11023,7 +11296,7 @@ fn genBinOp( .{ .v_, .cvtps2ph }, dst_reg, dst_reg.to256(), - Immediate.u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))), + .u(@as(u5, @bitCast(RoundMode{ .mode = .mxcsr }))), ); return dst_mcv; }, @@ -11191,7 +11464,7 @@ fn genBinOp( ); }, .cmp => { - const imm = Immediate.u(switch (air_tag) { + const imm: Immediate = .u(switch (air_tag) { .cmp_eq => 0, .cmp_lt, .cmp_gt => 1, .cmp_lte, .cmp_gte => 2, @@ -11289,7 +11562,7 @@ fn genBinOp( mask_reg, rhs_copy_reg, rhs_copy_reg, - Immediate.u(3), // unord + .u(3), // unord ); try self.asmRegisterRegisterRegisterRegister( @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(zcu)) { @@ -11356,7 +11629,7 @@ fn genBinOp( }), mask_reg, mask_reg, - Immediate.u(if (has_blend) 3 else 7), // unord, ord + .u(if (has_blend) 3 else 7), // unord, ord ); if (has_blend) try self.asmRegisterRegisterRegister( @as(?Mir.Inst.FixedTag, switch (lhs_ty.zigTypeTag(zcu)) { @@ -11567,29 +11840,29 @@ fn genBinOpMir( 8 => try self.asmRegisterImmediate( mir_limb_tag, dst_alias, - if (math.cast(i8, @as(i64, @bitCast(imm)))) |small| - Immediate.s(small) + if (std.math.cast(i8, @as(i64, @bitCast(imm)))) |small| + .s(small) else - Immediate.u(@as(u8, @intCast(imm))), + .u(@as(u8, @intCast(imm))), ), 16 => try self.asmRegisterImmediate( mir_limb_tag, dst_alias, - if (math.cast(i16, @as(i64, @bitCast(imm)))) |small| - Immediate.s(small) + if (std.math.cast(i16, @as(i64, @bitCast(imm)))) |small| + .s(small) else - Immediate.u(@as(u16, @intCast(imm))), + .u(@as(u16, @intCast(imm))), ), 32 => try self.asmRegisterImmediate( mir_limb_tag, dst_alias, - if (math.cast(i32, @as(i64, @bitCast(imm)))) |small| - Immediate.s(small) + if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |small| + .s(small) else - Immediate.u(@as(u32, @intCast(imm))), + .u(@as(u32, @intCast(imm))), ), - 64 => if (math.cast(i32, @as(i64, @bitCast(imm)))) |small| - try self.asmRegisterImmediate(mir_limb_tag, dst_alias, Immediate.s(small)) + 64 => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |small| + try self.asmRegisterImmediate(mir_limb_tag, dst_alias, .s(small)) else try self.asmRegisterRegister(mir_limb_tag, dst_alias, registerAlias( try self.copyToTmpRegister(ty, src_mcv), @@ -11619,7 +11892,7 @@ fn genBinOpMir( .base = .{ .reg = .ds }, .mod = .{ .rm = .{ .size = Memory.Size.fromSize(limb_abi_size), - .disp = math.cast(i32, addr + off) orelse break :direct, + .disp = std.math.cast(i32, addr + off) orelse break :direct, } }, }, .indirect => |reg_off| .{ @@ -11731,8 +12004,8 @@ fn genBinOpMir( => null, .memory, .load_symbol, .load_got, .load_direct, .load_tlv => src: { switch (resolved_src_mcv) { - .memory => |addr| if (math.cast(i32, @as(i64, @bitCast(addr))) != null and - math.cast(i32, @as(i64, @bitCast(addr)) + abi_size - limb_abi_size) != null) + .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr))) != null and + std.math.cast(i32, @as(i64, @bitCast(addr)) + abi_size - limb_abi_size) != null) break :src null, .load_symbol, .load_got, .load_direct, .load_tlv => {}, else => unreachable, @@ -11823,33 +12096,29 @@ fn genBinOpMir( 8 => try self.asmMemoryImmediate( mir_limb_tag, dst_limb_mem, - if (math.cast(i8, @as(i64, @bitCast(imm)))) |small| - Immediate.s(small) + if (std.math.cast(i8, @as(i64, @bitCast(imm)))) |small| + .s(small) else - Immediate.u(@as(u8, @intCast(imm))), + .u(@as(u8, @intCast(imm))), ), 16 => try self.asmMemoryImmediate( mir_limb_tag, dst_limb_mem, - if (math.cast(i16, @as(i64, @bitCast(imm)))) |small| - Immediate.s(small) + if (std.math.cast(i16, @as(i64, @bitCast(imm)))) |small| + .s(small) else - Immediate.u(@as(u16, @intCast(imm))), + .u(@as(u16, @intCast(imm))), ), 32 => try self.asmMemoryImmediate( mir_limb_tag, dst_limb_mem, - if (math.cast(i32, @as(i64, @bitCast(imm)))) |small| - Immediate.s(small) + if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |small| + .s(small) else - Immediate.u(@as(u32, @intCast(imm))), + .u(@as(u32, @intCast(imm))), ), - 64 => if (math.cast(i32, @as(i64, @bitCast(imm)))) |small| - try self.asmMemoryImmediate( - mir_limb_tag, - dst_limb_mem, - Immediate.s(small), - ) + 64 => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |small| + try self.asmMemoryImmediate(mir_limb_tag, dst_limb_mem, .s(small)) else try self.asmMemoryRegister( mir_limb_tag, @@ -11973,12 +12242,12 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M registerAlias(src_reg, abi_size), ), .immediate => |imm| { - if (math.cast(i32, imm)) |small| { + if (std.math.cast(i32, imm)) |small| { try self.asmRegisterRegisterImmediate( .{ .i_, .mul }, dst_alias, dst_alias, - Immediate.s(small), + .s(small), ); } else { const src_reg = try self.copyToTmpRegister(dst_ty, resolved_src_mcv); @@ -12009,7 +12278,7 @@ fn genIntMulComplexOpMir(self: *Self, dst_ty: Type, dst_mcv: MCValue, src_mcv: M .base = .{ .reg = .ds }, .mod = .{ .rm = .{ .size = Memory.Size.fromSize(abi_size), - .disp = math.cast(i32, @as(i64, @bitCast(addr))) orelse + .disp = std.math.cast(i32, @as(i64, @bitCast(addr))) orelse return self.asmRegisterRegister( .{ .i_, .mul }, dst_alias, @@ -12087,11 +12356,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void { ) |dst_reg, elem_index| { assert(self.register_manager.isRegFree(dst_reg)); if (elem_index > 0) { - try self.asmRegisterImmediate( - .{ ._l, .sh }, - dst_reg.to8(), - Immediate.u(elem_index), - ); + try self.asmRegisterImmediate(.{ ._l, .sh }, dst_reg.to8(), .u(elem_index)); try self.asmRegisterRegister( .{ ._, .@"or" }, dst_reg.to8(), @@ -12127,7 +12392,7 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void { try self.asmRegisterImmediate( .{ ._, .mov }, index_reg.to32(), - Immediate.u(regs_frame_addr.regs), + .u(regs_frame_addr.regs), ); const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); try self.asmMemoryImmediate(.{ ._, .cmp }, .{ @@ -12147,14 +12412,14 @@ fn airArg(self: *Self, inst: Air.Inst.Index) !void { ); self.performReloc(unset); if (self.hasFeature(.slow_incdec)) { - try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), Immediate.u(1)); + try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), .u(1)); } else { try self.asmRegister(.{ ._, .inc }, index_reg.to32()); } try self.asmRegisterImmediate( .{ ._, .cmp }, index_reg.to32(), - Immediate.u(arg_ty.vectorLen(zcu)), + .u(arg_ty.vectorLen(zcu)), ); _ = try self.asmJccReloc(.b, loop); @@ -12180,7 +12445,6 @@ fn airDbgArg(self: *Self, inst: Air.Inst.Index) !void { for (self.args[self.arg_index..]) |arg| { if (arg != .none) break; } else try self.airDbgVarArgs(); - self.finishAirBookkeeping(); } fn airDbgVarArgs(self: *Self) !void { @@ -12200,9 +12464,9 @@ fn genLocalDebugInfo( switch (mcv) { .none => try self.asmAir(.dbg_local, inst), .unreach, .dead, .elementwise_regs_then_frame, .reserved_frame, .air_ref => unreachable, - .immediate => |imm| try self.asmAirImmediate(.dbg_local, inst, Immediate.u(imm)), + .immediate => |imm| try self.asmAirImmediate(.dbg_local, inst, .u(imm)), .lea_frame => |frame_addr| try self.asmAirFrameAddress(.dbg_local, inst, frame_addr), - .lea_symbol => |sym_off| try self.asmAirImmediate(.dbg_local, inst, Immediate.rel(sym_off)), + .lea_symbol => |sym_off| try self.asmAirImmediate(.dbg_local, inst, .rel(sym_off)), else => { const ty = switch (tag) { else => unreachable, @@ -12245,16 +12509,6 @@ fn genLocalDebugInfo( } } -fn airTrap(self: *Self) !void { - try self.asmOpOnly(.{ ._, .ud2 }); - self.finishAirBookkeeping(); -} - -fn airBreakpoint(self: *Self) !void { - try self.asmOpOnly(.{ ._, .int3 }); - self.finishAirBookkeeping(); -} - fn airRetAddr(self: *Self, inst: Air.Inst.Index) !void { const dst_mcv = try self.allocRegOrMem(inst, true); try self.genCopy(Type.usize, dst_mcv, .{ .load_frame = .{ .index = .ret_addr } }, .{}); @@ -12426,7 +12680,7 @@ fn genCall(self: *Self, info: union(enum) { try self.asmRegisterImmediate( .{ ._, .mov }, index_reg.to32(), - Immediate.u(regs_frame_addr.regs), + .u(regs_frame_addr.regs), ); const loop: Mir.Inst.Index = @intCast(self.mir_instructions.len); try self.asmMemoryRegister(.{ ._, .bt }, src_mem, index_reg.to32()); @@ -12440,14 +12694,14 @@ fn genCall(self: *Self, info: union(enum) { } }, }); if (self.hasFeature(.slow_incdec)) { - try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), Immediate.u(1)); + try self.asmRegisterImmediate(.{ ._, .add }, index_reg.to32(), .u(1)); } else { try self.asmRegister(.{ ._, .inc }, index_reg.to32()); } try self.asmRegisterImmediate( .{ ._, .cmp }, index_reg.to32(), - Immediate.u(arg_ty.vectorLen(zcu)), + .u(arg_ty.vectorLen(zcu)), ); _ = try self.asmJccReloc(.b, loop); @@ -12521,11 +12775,7 @@ fn genCall(self: *Self, info: union(enum) { 0.., ) |dst_reg, elem_index| { try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()); - try self.asmMemoryImmediate( - .{ ._, .bt }, - src_mem, - Immediate.u(elem_index), - ); + try self.asmMemoryImmediate(.{ ._, .bt }, src_mem, .u(elem_index)); try self.asmSetccRegister(.c, dst_reg.to8()); } }, @@ -12533,7 +12783,7 @@ fn genCall(self: *Self, info: union(enum) { }; if (fn_info.is_var_args) - try self.asmRegisterImmediate(.{ ._, .mov }, .al, Immediate.u(call_info.fp_count)); + try self.asmRegisterImmediate(.{ ._, .mov }, .al, .u(call_info.fp_count)); // Due to incremental compilation, how function calls are generated depends // on linking. @@ -12551,7 +12801,7 @@ fn genCall(self: *Self, info: union(enum) { if (self.bin_file.cast(.elf)) |elf_file| { const zo = elf_file.zigObjectPtr().?; const sym_index = try zo.getOrCreateMetadataForNav(zcu, func.owner_nav); - try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = sym_index })); + try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = sym_index })); } else if (self.bin_file.cast(.coff)) |coff_file| { const atom = try coff_file.getOrCreateAtomForNav(func.owner_nav); const sym_index = coff_file.getAtom(atom).getSymbolIndex().?; @@ -12561,7 +12811,7 @@ fn genCall(self: *Self, info: union(enum) { const zo = macho_file.getZigObject().?; const sym_index = try zo.getOrCreateMetadataForNav(macho_file, func.owner_nav); const sym = zo.symbols.items[sym_index]; - try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = sym.nlist_idx })); + try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = sym.nlist_idx })); } else if (self.bin_file.cast(.plan9)) |p9| { const atom_index = try p9.seeNav(pt, func.owner_nav); const atom = p9.getAtom(atom_index); @@ -12579,13 +12829,13 @@ fn genCall(self: *Self, info: union(enum) { @"extern".name.toSlice(ip), @"extern".lib_name.toSlice(ip), ); - try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = target_sym_index })); + try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = target_sym_index })); } else if (self.bin_file.cast(.macho)) |macho_file| { const target_sym_index = try macho_file.getGlobalSymbol( @"extern".name.toSlice(ip), @"extern".lib_name.toSlice(ip), ); - try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = target_sym_index })); + try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = target_sym_index })); } else try self.genExternSymbolRef( .call, @"extern".lib_name.toSlice(ip), @@ -12600,10 +12850,10 @@ fn genCall(self: *Self, info: union(enum) { }, .lib => |lib| if (self.bin_file.cast(.elf)) |elf_file| { const target_sym_index = try elf_file.getGlobalSymbol(lib.callee, lib.lib); - try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = target_sym_index })); + try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = target_sym_index })); } else if (self.bin_file.cast(.macho)) |macho_file| { const target_sym_index = try macho_file.getGlobalSymbol(lib.callee, lib.lib); - try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = target_sym_index })); + try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = target_sym_index })); } else try self.genExternSymbolRef(.call, lib.lib, lib.callee), } return call_info.return_value.short; @@ -12665,7 +12915,7 @@ fn airRetLoad(self: *Self, inst: Air.Inst.Index) !void { try self.exitlude_jump_relocs.append(self.gpa, jmp_reloc); } -fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void { +fn airCmp(self: *Self, inst: Air.Inst.Index, op: std.math.CompareOperator) !void { const pt = self.pt; const zcu = pt.zcu; const bin_op = self.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; @@ -12754,7 +13004,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void { try self.asmRegisterImmediate( .{ ._r, .sh }, registerAlias(temp_lhs_reg, opt_abi_size), - Immediate.u(payload_abi_size * 8), + .u(payload_abi_size * 8), ); } @@ -12775,7 +13025,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void { try self.asmRegisterImmediate( .{ ._r, .sh }, registerAlias(temp_rhs_reg, opt_abi_size), - Immediate.u(payload_abi_size * 8), + .u(payload_abi_size * 8), ); try self.asmRegisterRegister( .{ ._, .@"test" }, @@ -12867,10 +13117,10 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void { .register_pair, .load_frame => null, .memory, .load_symbol, .load_got, .load_direct, .load_tlv => dst: { switch (resolved_dst_mcv) { - .memory => |addr| if (math.cast( + .memory => |addr| if (std.math.cast( i32, @as(i64, @bitCast(addr)), - ) != null and math.cast( + ) != null and std.math.cast( i32, @as(i64, @bitCast(addr)) + abi_size - 8, ) != null) break :dst null, @@ -12928,10 +13178,10 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void { .register_pair, .load_frame => null, .memory, .load_symbol, .load_got, .load_direct, .load_tlv => src: { switch (resolved_src_mcv) { - .memory => |addr| if (math.cast( + .memory => |addr| if (std.math.cast( i32, @as(i64, @bitCast(addr)), - ) != null and math.cast( + ) != null and std.math.cast( i32, @as(i64, @bitCast(addr)) + abi_size - 8, ) != null) break :src null, @@ -12971,7 +13221,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void { const locks = self.register_manager.lockRegsAssumeUnused(2, regs); defer for (locks) |lock| self.register_manager.unlockReg(lock); - const limbs_len = math.divCeil(u16, abi_size, 8) catch unreachable; + const limbs_len = std.math.divCeil(u16, abi_size, 8) catch unreachable; var limb_i: u16 = 0; while (limb_i < limbs_len) : (limb_i += 1) { const off = limb_i * 8; @@ -13067,7 +13317,7 @@ fn airCmp(self: *Self, inst: Air.Inst.Index, op: math.CompareOperator) !void { tmp1_reg, dst_reg.to128(), try src_mcv.mem(self, .word), - Immediate.u(1), + .u(1), ) else try self.asmRegisterRegisterRegister( .{ .vp_, .unpcklwd }, tmp1_reg, @@ -13232,7 +13482,6 @@ fn airDbgStmt(self: *Self, inst: Air.Inst.Index) !void { .column = dbg_stmt.column, } }, }); - self.finishAirBookkeeping(); } fn airDbgEmptyStmt(self: *Self) !void { @@ -13240,7 +13489,6 @@ fn airDbgEmptyStmt(self: *Self) !void { self.mir_instructions.items(.ops)[self.mir_instructions.len - 1] == .pseudo_dbg_line_stmt_line_column) self.mir_instructions.items(.ops)[self.mir_instructions.len - 1] = .pseudo_dbg_line_line_column; try self.asmOpOnly(.{ ._, .nop }); - self.finishAirBookkeeping(); } fn airDbgInlineBlock(self: *Self, inst: Air.Inst.Index) !void { @@ -13278,7 +13526,7 @@ fn genCondBrMir(self: *Self, ty: Type, mcv: MCValue) !Mir.Inst.Index { }, .register => |reg| { try self.spillEflagsIfOccupied(); - try self.asmRegisterImmediate(.{ ._, .@"test" }, reg.to8(), Immediate.u(1)); + try self.asmRegisterImmediate(.{ ._, .@"test" }, reg.to8(), .u(1)); return self.asmJccReloc(.z, undefined); }, .immediate, @@ -13338,7 +13586,6 @@ fn airCondBr(self: *Self, inst: Air.Inst.Index) !void { }); // We already took care of pl_op.operand earlier, so there's nothing left to do. - self.finishAirBookkeeping(); } fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MCValue { @@ -13353,7 +13600,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC const pl_ty = opt_ty.optionalChild(zcu); - const some_info: struct { off: i32, ty: Type } = if (opt_ty.optionalReprIsPayload(zcu)) + const some_info: struct { off: u31, ty: Type } = if (opt_ty.optionalReprIsPayload(zcu)) .{ .off = 0, .ty = if (pl_ty.isSlice(zcu)) pl_ty.slicePtrFieldType(zcu) else pl_ty } else .{ .off = @intCast(pl_ty.abiSize(zcu)), .ty = Type.bool }; @@ -13366,7 +13613,6 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC .undef, .immediate, .eflags, - .register_pair, .register_offset, .register_overflow, .lea_direct, @@ -13396,7 +13642,25 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC try self.asmRegisterImmediate( .{ ._, .bt }, registerAlias(opt_reg, opt_abi_size), - Immediate.u(@as(u6, @intCast(some_info.off * 8))), + .u(@as(u6, @intCast(some_info.off * 8))), + ); + return .{ .eflags = .nc }; + }, + + .register_pair => |opt_regs| { + if (some_info.off == 0) { + const some_abi_size: u32 = @intCast(some_info.ty.abiSize(zcu)); + const alias_reg = registerAlias(opt_regs[0], some_abi_size); + assert(some_abi_size * 8 == alias_reg.bitSize()); + try self.asmRegisterRegister(.{ ._, .@"test" }, alias_reg, alias_reg); + return .{ .eflags = .z }; + } + assert(some_info.ty.ip_index == .bool_type); + const opt_abi_size: u32 = @intCast(opt_ty.abiSize(zcu)); + try self.asmRegisterImmediate( + .{ ._, .bt }, + registerAlias(opt_regs[some_info.off / 8], opt_abi_size), + .u(@as(u6, @truncate(some_info.off * 8))), ); return .{ .eflags = .nc }; }, @@ -13422,7 +13686,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC .disp = some_info.off, } }, }, - Immediate.u(0), + .u(0), ); return .{ .eflags = .e }; }, @@ -13448,7 +13712,7 @@ fn isNull(self: *Self, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC }, else => unreachable, }, - Immediate.u(0), + .u(0), ); return .{ .eflags = .e }; }, @@ -13485,7 +13749,7 @@ fn isNullPtr(self: *Self, inst: Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCValue) .disp = some_info.off, } }, }, - Immediate.u(0), + .u(0), ); self.eflags_inst = inst; @@ -13500,7 +13764,7 @@ fn isErr(self: *Self, maybe_inst: ?Air.Inst.Index, eu_ty: Type, eu_mcv: MCValue) try self.spillEflagsIfOccupied(); - const err_off: u31 = @intCast(errUnionErrorOffset(eu_ty.errorUnionPayload(zcu), zcu)); + const err_off: u31 = @intCast(codegen.errUnionErrorOffset(eu_ty.errorUnionPayload(zcu), zcu)); switch (eu_mcv) { .register => |reg| { const eu_lock = self.register_manager.lockReg(reg); @@ -13557,7 +13821,7 @@ fn isErrPtr(self: *Self, maybe_inst: ?Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCV const ptr_lock = self.register_manager.lockReg(ptr_reg); defer if (ptr_lock) |lock| self.register_manager.unlockReg(lock); - const err_off: u31 = @intCast(errUnionErrorOffset(eu_ty.errorUnionPayload(zcu), zcu)); + const err_off: u31 = @intCast(codegen.errUnionErrorOffset(eu_ty.errorUnionPayload(zcu), zcu)); try self.asmMemoryImmediate( .{ ._, .cmp }, .{ @@ -13567,7 +13831,7 @@ fn isErrPtr(self: *Self, maybe_inst: ?Air.Inst.Index, ptr_ty: Type, ptr_mcv: MCV .disp = err_off, } }, }, - Immediate.u(0), + .u(0), ); if (maybe_inst) |inst| self.eflags_inst = inst; @@ -13686,12 +13950,11 @@ fn airLoop(self: *Self, inst: Air.Inst.Index) !void { try self.loops.putNoClobber(self.gpa, inst, .{ .state = state, - .jmp_target = @intCast(self.mir_instructions.len), + .target = @intCast(self.mir_instructions.len), }); defer assert(self.loops.remove(inst)); try self.genBodyBlock(body); - self.finishAirBookkeeping(); } fn airBlock(self: *Self, inst: Air.Inst.Index) !void { @@ -13729,7 +13992,6 @@ fn lowerBlock(self: *Self, inst: Air.Inst.Index, body: []const Air.Inst.Index) ! const tracking = &self.inst_tracking.values()[inst_tracking_i]; if (self.liveness.isUnused(inst)) try tracking.die(self, inst); self.getValueIfFree(tracking.short, inst); - self.finishAirBookkeeping(); } fn lowerSwitchBr(self: *Self, inst: Air.Inst.Index, switch_br: Air.UnwrappedSwitch, condition: MCValue) !void { @@ -13864,7 +14126,6 @@ fn airSwitchBr(self: *Self, inst: Air.Inst.Index) !void { try self.lowerSwitchBr(inst, switch_br, condition); // We already took care of pl_op.operand earlier, so there's nothing left to do - self.finishAirBookkeeping(); } fn airLoopSwitchBr(self: *Self, inst: Air.Inst.Index) !void { @@ -13893,7 +14154,7 @@ fn airLoopSwitchBr(self: *Self, inst: Air.Inst.Index) !void { try self.loops.putNoClobber(self.gpa, inst, .{ .state = state, - .jmp_target = @intCast(self.mir_instructions.len), + .target = @intCast(self.mir_instructions.len), }); defer assert(self.loops.remove(inst)); @@ -13903,7 +14164,6 @@ fn airLoopSwitchBr(self: *Self, inst: Air.Inst.Index) !void { try self.lowerSwitchBr(inst, switch_br, mat_cond); try self.processDeath(inst); - self.finishAirBookkeeping(); } fn airSwitchDispatch(self: *Self, inst: Air.Inst.Index) !void { @@ -13945,12 +14205,10 @@ fn airSwitchDispatch(self: *Self, inst: Air.Inst.Index) !void { // Emit a jump with a relocation. It will be patched up after the block ends. // Leave the jump offset undefined - _ = try self.asmJmpReloc(loop_data.jmp_target); + _ = try self.asmJmpReloc(loop_data.target); // Stop tracking block result without forgetting tracking info try self.freeValue(block_tracking.short); - - self.finishAirBookkeeping(); } fn performReloc(self: *Self, reloc: Mir.Inst.Index) void { @@ -14023,8 +14281,6 @@ fn airBr(self: *Self, inst: Air.Inst.Index) !void { // Stop tracking block result without forgetting tracking info try self.freeValue(block_tracking.short); - - self.finishAirBookkeeping(); } fn airRepeat(self: *Self, inst: Air.Inst.Index) !void { @@ -14036,8 +14292,7 @@ fn airRepeat(self: *Self, inst: Air.Inst.Index) !void { .resurrect = false, .close_scope = true, }); - _ = try self.asmJmpReloc(repeat_info.jmp_target); - self.finishAirBookkeeping(); + _ = try self.asmJmpReloc(repeat_info.target); } fn airAsm(self: *Self, inst: Air.Inst.Index) !void { @@ -14068,9 +14323,9 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { var outputs_extra_i = extra_i; for (outputs) |output| { - const extra_bytes = mem.sliceAsBytes(self.air.extra[extra_i..]); - const constraint = mem.sliceTo(mem.sliceAsBytes(self.air.extra[extra_i..]), 0); - const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); + const extra_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]); + const constraint = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0); + const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); // This equation accounts for the fact that even if we have exactly 4 bytes // for the string, we still use the next u32 for the null terminator. extra_i += (constraint.len + name.len + (2 + 3)) / 4; @@ -14097,8 +14352,8 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { const is_early_clobber = constraint[1] == '&'; const rest = constraint[@as(usize, 1) + @intFromBool(is_early_clobber) ..]; const arg_mcv: MCValue = arg_mcv: { - const arg_maybe_reg: ?Register = if (mem.eql(u8, rest, "r") or - mem.eql(u8, rest, "f") or mem.eql(u8, rest, "x")) + const arg_maybe_reg: ?Register = if (std.mem.eql(u8, rest, "r") or + std.mem.eql(u8, rest, "f") or std.mem.eql(u8, rest, "x")) registerAlias( self.register_manager.tryAllocReg(maybe_inst, switch (rest[0]) { 'r' => abi.RegisterClass.gp, @@ -14108,20 +14363,20 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { }) orelse return self.fail("ran out of registers lowering inline asm", .{}), @intCast(ty.abiSize(zcu)), ) - else if (mem.eql(u8, rest, "m")) + else if (std.mem.eql(u8, rest, "m")) if (output != .none) null else return self.fail( "memory constraint unsupported for asm result: '{s}'", .{constraint}, ) - else if (mem.eql(u8, rest, "g") or - mem.eql(u8, rest, "rm") or mem.eql(u8, rest, "mr") or - mem.eql(u8, rest, "r,m") or mem.eql(u8, rest, "m,r")) + else if (std.mem.eql(u8, rest, "g") or + std.mem.eql(u8, rest, "rm") or std.mem.eql(u8, rest, "mr") or + std.mem.eql(u8, rest, "r,m") or std.mem.eql(u8, rest, "m,r")) self.register_manager.tryAllocReg(maybe_inst, abi.RegisterClass.gp) orelse if (output != .none) null else return self.fail("ran out of registers lowering inline asm", .{}) - else if (mem.startsWith(u8, rest, "{") and mem.endsWith(u8, rest, "}")) + else if (std.mem.startsWith(u8, rest, "{") and std.mem.endsWith(u8, rest, "}")) parseRegName(rest["{".len .. rest.len - "}".len]) orelse return self.fail("invalid register constraint: '{s}'", .{constraint}) else if (rest.len == 1 and std.ascii.isDigit(rest[0])) { @@ -14134,7 +14389,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { break :arg_mcv if (arg_maybe_reg) |reg| .{ .register = reg } else arg: { const ptr_mcv = try self.resolveInst(output); switch (ptr_mcv) { - .immediate => |addr| if (math.cast(i32, @as(i64, @bitCast(addr)))) |_| + .immediate => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |_| break :arg ptr_mcv.deref(), .register, .register_offset, .lea_frame => break :arg ptr_mcv.deref(), else => {}, @@ -14145,7 +14400,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { if (arg_mcv.getReg()) |reg| if (RegisterManager.indexOfRegIntoTracked(reg)) |_| { _ = self.register_manager.lockReg(reg); }; - if (!mem.eql(u8, name, "_")) + if (!std.mem.eql(u8, name, "_")) arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len)); args.appendAssumeCapacity(arg_mcv); if (output == .none) result = arg_mcv; @@ -14153,17 +14408,17 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { } for (inputs) |input| { - const input_bytes = mem.sliceAsBytes(self.air.extra[extra_i..]); - const constraint = mem.sliceTo(input_bytes, 0); - const name = mem.sliceTo(input_bytes[constraint.len + 1 ..], 0); + const input_bytes = std.mem.sliceAsBytes(self.air.extra[extra_i..]); + const constraint = std.mem.sliceTo(input_bytes, 0); + const name = std.mem.sliceTo(input_bytes[constraint.len + 1 ..], 0); // This equation accounts for the fact that even if we have exactly 4 bytes // for the string, we still use the next u32 for the null terminator. extra_i += (constraint.len + name.len + (2 + 3)) / 4; const ty = self.typeOf(input); const input_mcv = try self.resolveInst(input); - const arg_mcv: MCValue = if (mem.eql(u8, constraint, "r") or - mem.eql(u8, constraint, "f") or mem.eql(u8, constraint, "x")) + const arg_mcv: MCValue = if (std.mem.eql(u8, constraint, "r") or + std.mem.eql(u8, constraint, "f") or std.mem.eql(u8, constraint, "x")) arg: { const rc = switch (constraint[0]) { 'r' => abi.RegisterClass.gp, @@ -14177,16 +14432,16 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { const reg = try self.register_manager.allocReg(null, rc); try self.genSetReg(reg, ty, input_mcv, .{}); break :arg .{ .register = registerAlias(reg, @intCast(ty.abiSize(zcu))) }; - } else if (mem.eql(u8, constraint, "i") or mem.eql(u8, constraint, "n")) + } else if (std.mem.eql(u8, constraint, "i") or std.mem.eql(u8, constraint, "n")) switch (input_mcv) { .immediate => |imm| .{ .immediate = imm }, else => return self.fail("immediate operand requires comptime value: '{s}'", .{ constraint, }), } - else if (mem.eql(u8, constraint, "m")) arg: { + else if (std.mem.eql(u8, constraint, "m")) arg: { switch (input_mcv) { - .memory => |addr| if (math.cast(i32, @as(i64, @bitCast(addr)))) |_| + .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |_| break :arg input_mcv, .indirect, .load_frame => break :arg input_mcv, .load_symbol, .load_direct, .load_got, .load_tlv => {}, @@ -14203,22 +14458,22 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { }; try self.genSetReg(addr_reg, Type.usize, input_mcv.address(), .{}); break :arg .{ .indirect = .{ .reg = addr_reg } }; - } else if (mem.eql(u8, constraint, "g") or - mem.eql(u8, constraint, "rm") or mem.eql(u8, constraint, "mr") or - mem.eql(u8, constraint, "r,m") or mem.eql(u8, constraint, "m,r")) + } else if (std.mem.eql(u8, constraint, "g") or + std.mem.eql(u8, constraint, "rm") or std.mem.eql(u8, constraint, "mr") or + std.mem.eql(u8, constraint, "r,m") or std.mem.eql(u8, constraint, "m,r")) arg: { switch (input_mcv) { .register, .indirect, .load_frame => break :arg input_mcv, - .memory => |addr| if (math.cast(i32, @as(i64, @bitCast(addr)))) |_| + .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |_| break :arg input_mcv, else => {}, } const temp_mcv = try self.allocTempRegOrMem(ty, true); try self.genCopy(ty, temp_mcv, input_mcv, .{}); break :arg temp_mcv; - } else if (mem.eql(u8, constraint, "X")) + } else if (std.mem.eql(u8, constraint, "X")) input_mcv - else if (mem.startsWith(u8, constraint, "{") and mem.endsWith(u8, constraint, "}")) arg: { + else if (std.mem.startsWith(u8, constraint, "{") and std.mem.endsWith(u8, constraint, "}")) arg: { const reg = parseRegName(constraint["{".len .. constraint.len - "}".len]) orelse return self.fail("invalid register constraint: '{s}'", .{constraint}); try self.register_manager.getReg(reg, null); @@ -14233,7 +14488,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { if (arg_mcv.getReg()) |reg| if (RegisterManager.indexOfRegIntoTracked(reg)) |_| { _ = self.register_manager.lockReg(reg); }; - if (!mem.eql(u8, name, "_")) + if (!std.mem.eql(u8, name, "_")) arg_map.putAssumeCapacityNoClobber(name, @intCast(args.items.len)); args.appendAssumeCapacity(arg_mcv); } @@ -14241,7 +14496,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { { var clobber_i: u32 = 0; while (clobber_i < clobbers_len) : (clobber_i += 1) { - const clobber = mem.sliceTo(mem.sliceAsBytes(self.air.extra[extra_i..]), 0); + const clobber = std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[extra_i..]), 0); // This equation accounts for the fact that even if we have exactly 4 bytes // for the string, we still use the next u32 for the null terminator. extra_i += clobber.len / 4 + 1; @@ -14294,20 +14549,20 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { labels.deinit(self.gpa); } - const asm_source = mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len]; - var line_it = mem.tokenizeAny(u8, asm_source, "\n\r;"); + const asm_source = std.mem.sliceAsBytes(self.air.extra[extra_i..])[0..extra.data.source_len]; + var line_it = std.mem.tokenizeAny(u8, asm_source, "\n\r;"); next_line: while (line_it.next()) |line| { - var mnem_it = mem.tokenizeAny(u8, line, " \t"); - var prefix: Instruction.Prefix = .none; + var mnem_it = std.mem.tokenizeAny(u8, line, " \t"); + var prefix: encoder.Instruction.Prefix = .none; const mnem_str = while (mnem_it.next()) |mnem_str| { if (mnem_str[0] == '#') continue :next_line; - if (mem.startsWith(u8, mnem_str, "//")) continue :next_line; - if (std.meta.stringToEnum(Instruction.Prefix, mnem_str)) |pre| { + if (std.mem.startsWith(u8, mnem_str, "//")) continue :next_line; + if (std.meta.stringToEnum(encoder.Instruction.Prefix, mnem_str)) |pre| { if (prefix != .none) return self.fail("extra prefix: '{s}'", .{mnem_str}); prefix = pre; continue; } - if (!mem.endsWith(u8, mnem_str, ":")) break mnem_str; + if (!std.mem.endsWith(u8, mnem_str, ":")) break mnem_str; const label_name = mnem_str[0 .. mnem_str.len - ":".len]; if (!Label.isValid(.definition, label_name)) return self.fail("invalid label: '{s}'", .{label_name}); @@ -14332,21 +14587,21 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { var mnem_size: ?Memory.Size = if (prefix == .directive) null - else if (mem.endsWith(u8, mnem_str, "b")) + else if (std.mem.endsWith(u8, mnem_str, "b")) .byte - else if (mem.endsWith(u8, mnem_str, "w")) + else if (std.mem.endsWith(u8, mnem_str, "w")) .word - else if (mem.endsWith(u8, mnem_str, "l")) + else if (std.mem.endsWith(u8, mnem_str, "l")) .dword - else if (mem.endsWith(u8, mnem_str, "q") and - (std.mem.indexOfScalar(u8, "vp", mnem_str[0]) == null or !mem.endsWith(u8, mnem_str, "dq"))) + else if (std.mem.endsWith(u8, mnem_str, "q") and + (std.mem.indexOfScalar(u8, "vp", mnem_str[0]) == null or !std.mem.endsWith(u8, mnem_str, "dq"))) .qword - else if (mem.endsWith(u8, mnem_str, "t")) + else if (std.mem.endsWith(u8, mnem_str, "t")) .tbyte else null; const mnem_tag = while (true) break std.meta.stringToEnum( - Instruction.Mnemonic, + encoder.Instruction.Mnemonic, mnem_str[0 .. mnem_str.len - @intFromBool(mnem_size != null)], ) orelse if (mnem_size) |_| { mnem_size = null; @@ -14367,18 +14622,18 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { .{ ._, .pseudo } else for (std.enums.values(Mir.Inst.Fixes)) |fixes| { const fixes_name = @tagName(fixes); - const space_i = mem.indexOfScalar(u8, fixes_name, ' '); + const space_i = std.mem.indexOfScalar(u8, fixes_name, ' '); const fixes_prefix = if (space_i) |i| - std.meta.stringToEnum(Instruction.Prefix, fixes_name[0..i]).? + std.meta.stringToEnum(encoder.Instruction.Prefix, fixes_name[0..i]).? else .none; if (fixes_prefix != prefix) continue; const pattern = fixes_name[if (space_i) |i| i + " ".len else 0..]; - const wildcard_i = mem.indexOfScalar(u8, pattern, '_').?; + const wildcard_i = std.mem.indexOfScalar(u8, pattern, '_').?; const mnem_prefix = pattern[0..wildcard_i]; const mnem_suffix = pattern[wildcard_i + "_".len ..]; - if (!mem.startsWith(u8, mnem_name, mnem_prefix)) continue; - if (!mem.endsWith(u8, mnem_name, mnem_suffix)) continue; + if (!std.mem.startsWith(u8, mnem_name, mnem_prefix)) continue; + if (!std.mem.endsWith(u8, mnem_name, mnem_suffix)) continue; break .{ fixes, std.meta.stringToEnum( Mir.Inst.Tag, mnem_name[mnem_prefix.len .. mnem_name.len - mnem_suffix.len], @@ -14400,21 +14655,21 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { var ops: [4]Operand = .{.none} ** 4; var last_op = false; - var op_it = mem.splitScalar(u8, mnem_it.rest(), ','); + var op_it = std.mem.splitScalar(u8, mnem_it.rest(), ','); next_op: for (&ops) |*op| { const op_str = while (!last_op) { const full_str = op_it.next() orelse break :next_op; - const code_str = if (mem.indexOfScalar(u8, full_str, '#') orelse - mem.indexOf(u8, full_str, "//")) |comment| + const code_str = if (std.mem.indexOfScalar(u8, full_str, '#') orelse + std.mem.indexOf(u8, full_str, "//")) |comment| code: { last_op = true; break :code full_str[0..comment]; } else full_str; - const trim_str = mem.trim(u8, code_str, " \t*"); + const trim_str = std.mem.trim(u8, code_str, " \t*"); if (trim_str.len > 0) break trim_str; } else break; - if (mem.startsWith(u8, op_str, "%%")) { - const colon = mem.indexOfScalarPos(u8, op_str, "%%".len + 2, ':'); + if (std.mem.startsWith(u8, op_str, "%%")) { + const colon = std.mem.indexOfScalarPos(u8, op_str, "%%".len + 2, ':'); const reg = parseRegName(op_str["%%".len .. colon orelse op_str.len]) orelse return self.fail("invalid register: '{s}'", .{op_str}); if (colon) |colon_pos| { @@ -14432,8 +14687,8 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { return self.fail("invalid register size: '{s}'", .{op_str}); op.* = .{ .reg = reg }; } - } else if (mem.startsWith(u8, op_str, "%[") and mem.endsWith(u8, op_str, "]")) { - const colon = mem.indexOfScalarPos(u8, op_str, "%[".len, ':'); + } else if (std.mem.startsWith(u8, op_str, "%[") and std.mem.endsWith(u8, op_str, "]")) { + const colon = std.mem.indexOfScalarPos(u8, op_str, "%[".len, ':'); const modifier = if (colon) |colon_pos| op_str[colon_pos + ":".len .. op_str.len - "]".len] else @@ -14442,15 +14697,15 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { arg_map.get(op_str["%[".len .. colon orelse op_str.len - "]".len]) orelse return self.fail("no matching constraint: '{s}'", .{op_str}) ]) { - .immediate => |imm| if (mem.eql(u8, modifier, "") or mem.eql(u8, modifier, "c")) - .{ .imm = Immediate.u(imm) } + .immediate => |imm| if (std.mem.eql(u8, modifier, "") or std.mem.eql(u8, modifier, "c")) + .{ .imm = .u(imm) } else return self.fail("invalid modifier: '{s}'", .{modifier}), - .register => |reg| if (mem.eql(u8, modifier, "")) + .register => |reg| if (std.mem.eql(u8, modifier, "")) .{ .reg = reg } else return self.fail("invalid modifier: '{s}'", .{modifier}), - .memory => |addr| if (mem.eql(u8, modifier, "") or mem.eql(u8, modifier, "P")) + .memory => |addr| if (std.mem.eql(u8, modifier, "") or std.mem.eql(u8, modifier, "P")) .{ .mem = .{ .base = .{ .reg = .ds }, .mod = .{ .rm = .{ @@ -14461,7 +14716,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { } } else return self.fail("invalid modifier: '{s}'", .{modifier}), - .indirect => |reg_off| if (mem.eql(u8, modifier, "")) + .indirect => |reg_off| if (std.mem.eql(u8, modifier, "")) .{ .mem = .{ .base = .{ .reg = reg_off.reg }, .mod = .{ .rm = .{ @@ -14472,7 +14727,7 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { } } else return self.fail("invalid modifier: '{s}'", .{modifier}), - .load_frame => |frame_addr| if (mem.eql(u8, modifier, "")) + .load_frame => |frame_addr| if (std.mem.eql(u8, modifier, "")) .{ .mem = .{ .base = .{ .frame = frame_addr.index }, .mod = .{ .rm = .{ @@ -14483,42 +14738,42 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { } } else return self.fail("invalid modifier: '{s}'", .{modifier}), - .lea_got => |sym_index| if (mem.eql(u8, modifier, "P")) + .lea_got => |sym_index| if (std.mem.eql(u8, modifier, "P")) .{ .reg = try self.copyToTmpRegister(Type.usize, .{ .lea_got = sym_index }) } else return self.fail("invalid modifier: '{s}'", .{modifier}), - .lea_symbol => |sym_off| if (mem.eql(u8, modifier, "P")) + .lea_symbol => |sym_off| if (std.mem.eql(u8, modifier, "P")) .{ .reg = try self.copyToTmpRegister(Type.usize, .{ .lea_symbol = sym_off }) } else return self.fail("invalid modifier: '{s}'", .{modifier}), else => return self.fail("invalid constraint: '{s}'", .{op_str}), }; - } else if (mem.startsWith(u8, op_str, "$")) { + } else if (std.mem.startsWith(u8, op_str, "$")) { if (std.fmt.parseInt(i32, op_str["$".len..], 0)) |s| { if (mnem_size) |size| { - const max = @as(u64, math.maxInt(u64)) >> @intCast(64 - (size.bitSize() - 1)); + const max = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - (size.bitSize() - 1)); if ((if (s < 0) ~s else s) > max) return self.fail("invalid immediate size: '{s}'", .{op_str}); } - op.* = .{ .imm = Immediate.s(s) }; + op.* = .{ .imm = .s(s) }; } else |_| if (std.fmt.parseInt(u64, op_str["$".len..], 0)) |u| { if (mnem_size) |size| { - const max = @as(u64, math.maxInt(u64)) >> @intCast(64 - size.bitSize()); + const max = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - size.bitSize()); if (u > max) return self.fail("invalid immediate size: '{s}'", .{op_str}); } - op.* = .{ .imm = Immediate.u(u) }; + op.* = .{ .imm = .u(u) }; } else |_| return self.fail("invalid immediate: '{s}'", .{op_str}); - } else if (mem.endsWith(u8, op_str, ")")) { - const open = mem.indexOfScalar(u8, op_str, '(') orelse + } else if (std.mem.endsWith(u8, op_str, ")")) { + const open = std.mem.indexOfScalar(u8, op_str, '(') orelse return self.fail("invalid operand: '{s}'", .{op_str}); - var sib_it = mem.splitScalar(u8, op_str[open + "(".len .. op_str.len - ")".len], ','); + var sib_it = std.mem.splitScalar(u8, op_str[open + "(".len .. op_str.len - ")".len], ','); const base_str = sib_it.next() orelse return self.fail("invalid memory operand: '{s}'", .{op_str}); - if (base_str.len > 0 and !mem.startsWith(u8, base_str, "%%")) + if (base_str.len > 0 and !std.mem.startsWith(u8, base_str, "%%")) return self.fail("invalid memory operand: '{s}'", .{op_str}); const index_str = sib_it.next() orelse ""; - if (index_str.len > 0 and !mem.startsWith(u8, base_str, "%%")) + if (index_str.len > 0 and !std.mem.startsWith(u8, base_str, "%%")) return self.fail("invalid memory operand: '{s}'", .{op_str}); const scale_str = sib_it.next() orelse ""; if (index_str.len == 0 and scale_str.len > 0) @@ -14550,10 +14805,10 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { else .none, .scale = scale, - .disp = if (mem.startsWith(u8, op_str[0..open], "%[") and - mem.endsWith(u8, op_str[0..open], "]")) + .disp = if (std.mem.startsWith(u8, op_str[0..open], "%[") and + std.mem.endsWith(u8, op_str[0..open], "]")) disp: { - const colon = mem.indexOfScalarPos(u8, op_str[0..open], "%[".len, ':'); + const colon = std.mem.indexOfScalarPos(u8, op_str[0..open], "%[".len, ':'); const modifier = if (colon) |colon_pos| op_str[colon_pos + ":".len .. open - "]".len] else @@ -14562,9 +14817,9 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { arg_map.get(op_str["%[".len .. colon orelse open - "]".len]) orelse return self.fail("no matching constraint: '{s}'", .{op_str}) ]) { - .immediate => |imm| if (mem.eql(u8, modifier, "") or - mem.eql(u8, modifier, "c")) - math.cast(i32, @as(i64, @bitCast(imm))) orelse + .immediate => |imm| if (std.mem.eql(u8, modifier, "") or + std.mem.eql(u8, modifier, "c")) + std.math.cast(i32, @as(i64, @bitCast(imm))) orelse return self.fail("invalid displacement: '{s}'", .{op_str}) else return self.fail("invalid modifier: '{s}'", .{modifier}), @@ -14730,10 +14985,10 @@ fn airAsm(self: *Self, inst: Air.Inst.Index) !void { return self.fail("undefined label: '{s}'", .{label.key_ptr.*}); for (outputs, args.items[0..outputs.len]) |output, arg_mcv| { - const extra_bytes = mem.sliceAsBytes(self.air.extra[outputs_extra_i..]); + const extra_bytes = std.mem.sliceAsBytes(self.air.extra[outputs_extra_i..]); const constraint = - mem.sliceTo(mem.sliceAsBytes(self.air.extra[outputs_extra_i..]), 0); - const name = mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); + std.mem.sliceTo(std.mem.sliceAsBytes(self.air.extra[outputs_extra_i..]), 0); + const name = std.mem.sliceTo(extra_bytes[constraint.len + 1 ..], 0); // This equation accounts for the fact that even if we have exactly 4 bytes // for the string, we still use the next u32 for the null terminator. outputs_extra_i += (constraint.len + name.len + (2 + 3)) / 4; @@ -14777,7 +15032,10 @@ const MoveStrategy = union(enum) { pub fn read(strat: MoveStrategy, self: *Self, dst_reg: Register, src_mem: Memory) !void { switch (strat) { - .move => |tag| try self.asmRegisterMemory(tag, dst_reg, src_mem), + .move => |tag| try self.asmRegisterMemory(tag, switch (tag[1]) { + else => dst_reg, + .lea => if (dst_reg.bitSize() >= 32) dst_reg else dst_reg.to32(), + }, src_mem), .x87_load_store => { try self.asmMemory(.{ .f_, .ld }, src_mem); assert(dst_reg != .st7); @@ -14787,14 +15045,14 @@ const MoveStrategy = union(enum) { ie.insert, dst_reg, src_mem, - Immediate.u(0), + .u(0), ), .vex_insert_extract => |ie| try self.asmRegisterRegisterMemoryImmediate( ie.insert, dst_reg, dst_reg, src_mem, - Immediate.u(0), + .u(0), ), } } @@ -14809,7 +15067,7 @@ const MoveStrategy = union(enum) { ie.extract, dst_mem, src_reg, - Immediate.u(0), + .u(0), ), } } @@ -14823,7 +15081,7 @@ fn moveStrategy(self: *Self, ty: Type, class: Register.Class, aligned: bool) !Mo .mmx => {}, .sse => switch (ty.zigTypeTag(zcu)) { else => { - const classes = mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .other), .none); + const classes = std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .other), .none); assert(std.mem.indexOfNone(abi.Class, classes, &.{ .integer, .sse, .sseup, .memory, .float, .float_combine, }) == null); @@ -15135,7 +15393,7 @@ fn genCopy(self: *Self, ty: Type, dst_mcv: MCValue, src_mcv: MCValue, opts: Copy ), .memory, .load_symbol, .load_direct, .load_got, .load_tlv => { switch (dst_mcv) { - .memory => |addr| if (math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr| + .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr| return self.genSetMem(.{ .reg = .ds }, small_addr, ty, src_mcv, opts), .load_symbol, .load_direct, .load_got, .load_tlv => {}, else => unreachable, @@ -15179,17 +15437,17 @@ fn genSetReg( => unreachable, .undef => if (opts.safety) switch (dst_reg.class()) { .general_purpose => switch (abi_size) { - 1 => try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to8(), Immediate.u(0xAA)), - 2 => try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to16(), Immediate.u(0xAAAA)), + 1 => try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to8(), .u(0xAA)), + 2 => try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to16(), .u(0xAAAA)), 3...4 => try self.asmRegisterImmediate( .{ ._, .mov }, dst_reg.to32(), - Immediate.s(@as(i32, @bitCast(@as(u32, 0xAAAAAAAA)))), + .s(@as(i32, @bitCast(@as(u32, 0xAAAAAAAA)))), ), 5...8 => try self.asmRegisterImmediate( .{ ._, .mov }, dst_reg.to64(), - Immediate.u(0xAAAAAAAAAAAAAAAA), + .u(0xAAAAAAAAAAAAAAAA), ), else => unreachable, }, @@ -15203,20 +15461,20 @@ fn genSetReg( // register is the fastest way to zero a register. try self.spillEflagsIfOccupied(); try self.asmRegisterRegister(.{ ._, .xor }, dst_reg.to32(), dst_reg.to32()); - } else if (abi_size > 4 and math.cast(u32, imm) != null) { + } else if (abi_size > 4 and std.math.cast(u32, imm) != null) { // 32-bit moves zero-extend to 64-bit. - try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), Immediate.u(imm)); + try self.asmRegisterImmediate(.{ ._, .mov }, dst_reg.to32(), .u(imm)); } else if (abi_size <= 4 and @as(i64, @bitCast(imm)) < 0) { try self.asmRegisterImmediate( .{ ._, .mov }, registerAlias(dst_reg, abi_size), - Immediate.s(@intCast(@as(i64, @bitCast(imm)))), + .s(@intCast(@as(i64, @bitCast(imm)))), ); } else { try self.asmRegisterImmediate( .{ ._, .mov }, registerAlias(dst_reg, abi_size), - Immediate.u(imm), + .u(imm), ); } }, @@ -15325,15 +15583,15 @@ fn genSetReg( .load_frame => |frame_addr| try self.moveStrategy( ty, dst_reg.class(), - self.getFrameAddrAlignment(frame_addr).compare(.gte, Alignment.fromLog2Units( - math.log2_int_ceil(u10, @divExact(dst_reg.bitSize(), 8)), + self.getFrameAddrAlignment(frame_addr).compare(.gte, InternPool.Alignment.fromLog2Units( + std.math.log2_int_ceil(u10, @divExact(dst_reg.bitSize(), 8)), )), ), .lea_frame => .{ .move = .{ ._, .lea } }, else => unreachable, }).read(self, registerAlias(dst_reg, abi_size), switch (src_mcv) { .register_offset, .indirect => |reg_off| .{ - .base = .{ .reg = reg_off.reg }, + .base = .{ .reg = reg_off.reg.to64() }, .mod = .{ .rm = .{ .size = self.memSize(ty), .disp = reg_off.off, @@ -15350,7 +15608,7 @@ fn genSetReg( }), .memory, .load_symbol, .load_direct, .load_got, .load_tlv => { switch (src_mcv) { - .memory => |addr| if (math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr| + .memory => |addr| if (std.math.cast(i32, @as(i64, @bitCast(addr)))) |small_addr| return (try self.moveStrategy( ty, dst_reg.class(), @@ -15400,14 +15658,10 @@ fn genSetReg( const addr_lock = self.register_manager.lockRegAssumeUnused(addr_reg); defer self.register_manager.unlockReg(addr_lock); - try (try self.moveStrategy(ty, dst_reg.class(), false)).read( - self, - registerAlias(dst_reg, abi_size), - .{ - .base = .{ .reg = addr_reg }, - .mod = .{ .rm = .{ .size = self.memSize(ty) } }, - }, - ); + try (try self.moveStrategy(ty, dst_reg.class(), false)).read(self, registerAlias(dst_reg, abi_size), .{ + .base = .{ .reg = addr_reg.to64() }, + .mod = .{ .rm = .{ .size = self.memSize(ty) } }, + }); }, .lea_symbol => |sym_off| switch (self.bin_file.tag) { .elf, .macho => try self.asmRegisterMemory( @@ -15478,12 +15732,12 @@ fn genSetMem( ), .immediate => |imm| switch (abi_size) { 1, 2, 4 => { - const immediate = switch (if (ty.isAbiInt(zcu)) + const immediate: Immediate = switch (if (ty.isAbiInt(zcu)) ty.intInfo(zcu).signedness else .unsigned) { - .signed => Immediate.s(@truncate(@as(i64, @bitCast(imm)))), - .unsigned => Immediate.u(@as(u32, @intCast(imm))), + .signed => .s(@truncate(@as(i64, @bitCast(imm)))), + .unsigned => .u(@as(u32, @intCast(imm))), }; try self.asmMemoryImmediate( .{ ._, .mov }, @@ -15495,14 +15749,14 @@ fn genSetMem( ); }, 3, 5...7 => unreachable, - else => if (math.cast(i32, @as(i64, @bitCast(imm)))) |small| { + else => if (std.math.cast(i32, @as(i64, @bitCast(imm)))) |small| { try self.asmMemoryImmediate( .{ ._, .mov }, .{ .base = base, .mod = .{ .rm = .{ .size = Memory.Size.fromSize(abi_size), .disp = disp, } } }, - Immediate.s(small), + .s(small), ); } else { var offset: i32 = 0; @@ -15512,10 +15766,10 @@ fn genSetMem( .size = .dword, .disp = disp + offset, } } }, - if (ty.isSignedInt(zcu)) Immediate.s( - @truncate(@as(i64, @bitCast(imm)) >> (math.cast(u6, offset * 8) orelse 63)), - ) else Immediate.u( - @as(u32, @truncate(if (math.cast(u6, offset * 8)) |shift| imm >> shift else 0)), + if (ty.isSignedInt(zcu)) .s( + @truncate(@as(i64, @bitCast(imm)) >> (std.math.cast(u6, offset * 8) orelse 63)), + ) else .u( + @as(u32, @truncate(if (std.math.cast(u6, offset * 8)) |shift| imm >> shift else 0)), ), ); }, @@ -15542,7 +15796,9 @@ fn genSetMem( .general_purpose, .segment, .x87, .ip => @divExact(src_alias.bitSize(), 8), .mmx, .sse => abi_size, }); - const src_align = Alignment.fromNonzeroByteUnits(math.ceilPowerOfTwoAssert(u32, src_size)); + const src_align = InternPool.Alignment.fromNonzeroByteUnits( + std.math.ceilPowerOfTwoAssert(u32, src_size), + ); if (src_size > mem_size) { const frame_index = try self.allocFrameIndex(FrameAlloc.init(.{ .size = src_size, @@ -15755,7 +16011,7 @@ fn genLazySymbolRef( .base = .{ .reloc = sym_index }, .mod = .{ .rm = .{ .size = .qword } }, }), - .call => try self.asmImmediate(.{ ._, .call }, Immediate.rel(.{ .sym_index = sym_index })), + .call => try self.asmImmediate(.{ ._, .call }, .rel(.{ .sym_index = sym_index })), else => unreachable, } } else if (self.bin_file.cast(.plan9)) |p9_file| { @@ -15861,7 +16117,7 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void { const dst_mcv = if (dst_rc.supersetOf(src_rc) and dst_ty.abiSize(zcu) <= src_ty.abiSize(zcu) and self.reuseOperand(inst, ty_op.operand, 0, src_mcv)) src_mcv else dst: { const dst_mcv = try self.allocRegOrMem(inst, true); - try self.genCopy(switch (math.order(dst_ty.abiSize(zcu), src_ty.abiSize(zcu))) { + try self.genCopy(switch (std.math.order(dst_ty.abiSize(zcu), src_ty.abiSize(zcu))) { .lt => dst_ty, .eq => if (!dst_mcv.isMemory() or src_mcv.isMemory()) dst_ty else src_ty, .gt => src_ty, @@ -15878,7 +16134,7 @@ fn airBitCast(self: *Self, inst: Air.Inst.Index) !void { const bit_size = dst_ty.bitSize(zcu); if (abi_size * 8 <= bit_size or dst_ty.isVector(zcu)) break :result dst_mcv; - const dst_limbs_len = math.divCeil(i32, @intCast(bit_size), 64) catch unreachable; + const dst_limbs_len = std.math.divCeil(i32, @intCast(bit_size), 64) catch unreachable; const high_mcv: MCValue = switch (dst_mcv) { .register => |dst_reg| .{ .register = dst_reg }, .register_pair => |dst_regs| .{ .register = dst_regs[1] }, @@ -15940,7 +16196,7 @@ fn airFloatFromInt(self: *Self, inst: Air.Inst.Index) !void { const src_bits: u32 = @intCast(src_ty.bitSize(zcu)); const src_signedness = if (src_ty.isAbiInt(zcu)) src_ty.intInfo(zcu).signedness else .unsigned; - const src_size = math.divCeil(u32, @max(switch (src_signedness) { + const src_size = std.math.divCeil(u32, @max(switch (src_signedness) { .signed => src_bits, .unsigned => src_bits + 1, }, 32), 8) catch unreachable; @@ -16017,7 +16273,7 @@ fn airIntFromFloat(self: *Self, inst: Air.Inst.Index) !void { const dst_bits: u32 = @intCast(dst_ty.bitSize(zcu)); const dst_signedness = if (dst_ty.isAbiInt(zcu)) dst_ty.intInfo(zcu).signedness else .unsigned; - const dst_size = math.divCeil(u32, @max(switch (dst_signedness) { + const dst_size = std.math.divCeil(u32, @max(switch (dst_signedness) { .signed => dst_bits, .unsigned => dst_bits + 1, }, 32), 8) catch unreachable; @@ -16593,13 +16849,17 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void { const reg_locks = self.register_manager.lockRegsAssumeUnused(4, .{ .rax, .rdi, .rsi, .rcx }); defer for (reg_locks) |lock| self.register_manager.unlockReg(lock); - const dst_ptr = try self.resolveInst(bin_op.lhs); - const dst_ptr_ty = self.typeOf(bin_op.lhs); - const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, + const dst = try self.resolveInst(bin_op.lhs); + const dst_ty = self.typeOf(bin_op.lhs); + const dst_locks: [2]?RegisterLock = switch (dst) { + .register => |dst_reg| .{ self.register_manager.lockRegAssumeUnused(dst_reg), null }, + .register_pair => |dst_regs| .{ + self.register_manager.lockRegAssumeUnused(dst_regs[0]), + self.register_manager.lockRegAssumeUnused(dst_regs[1]), + }, + else => .{ null, null }, }; - defer if (dst_ptr_lock) |lock| self.register_manager.unlockReg(lock); + for (dst_locks) |dst_lock| if (dst_lock) |lock| self.register_manager.unlockReg(lock); const src_val = try self.resolveInst(bin_op.rhs); const elem_ty = self.typeOf(bin_op.rhs); @@ -16612,16 +16872,20 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void { const elem_abi_size: u31 = @intCast(elem_ty.abiSize(zcu)); if (elem_abi_size == 1) { - const ptr: MCValue = switch (dst_ptr_ty.ptrSize(zcu)) { - // TODO: this only handles slices stored in the stack - .slice => dst_ptr, - .one => dst_ptr, + const dst_ptr: MCValue = switch (dst_ty.ptrSize(zcu)) { + .slice => switch (dst) { + .register_pair => |dst_regs| .{ .register = dst_regs[0] }, + else => dst, + }, + .one => dst, .c, .many => unreachable, }; - const len: MCValue = switch (dst_ptr_ty.ptrSize(zcu)) { - // TODO: this only handles slices stored in the stack - .slice => dst_ptr.address().offset(8).deref(), - .one => .{ .immediate = dst_ptr_ty.childType(zcu).arrayLen(zcu) }, + const len: MCValue = switch (dst_ty.ptrSize(zcu)) { + .slice => switch (dst) { + .register_pair => |dst_regs| .{ .register = dst_regs[1] }, + else => dst.address().offset(8).deref(), + }, + .one => .{ .immediate = dst_ty.childType(zcu).arrayLen(zcu) }, .c, .many => unreachable, }; const len_lock: ?RegisterLock = switch (len) { @@ -16630,20 +16894,25 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void { }; defer if (len_lock) |lock| self.register_manager.unlockReg(lock); - try self.genInlineMemset(ptr, src_val, len, .{ .safety = safety }); + try self.genInlineMemset(dst_ptr, src_val, len, .{ .safety = safety }); break :result; } // Store the first element, and then rely on memcpy copying forwards. // Length zero requires a runtime check - so we handle arrays specially // here to elide it. - switch (dst_ptr_ty.ptrSize(zcu)) { + switch (dst_ty.ptrSize(zcu)) { .slice => { - const slice_ptr_ty = dst_ptr_ty.slicePtrFieldType(zcu); + const slice_ptr_ty = dst_ty.slicePtrFieldType(zcu); - // TODO: this only handles slices stored in the stack - const ptr = dst_ptr; - const len = dst_ptr.address().offset(8).deref(); + const dst_ptr: MCValue = switch (dst) { + .register_pair => |dst_regs| .{ .register = dst_regs[0] }, + else => dst, + }; + const len: MCValue = switch (dst) { + .register_pair => |dst_regs| .{ .register = dst_regs[1] }, + else => dst.address().offset(8).deref(), + }; // Used to store the number of elements for comparison. // After comparison, updated to store number of bytes needed to copy. @@ -16656,38 +16925,7 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void { try self.asmRegisterRegister(.{ ._, .@"test" }, len_reg, len_reg); const skip_reloc = try self.asmJccReloc(.z, undefined); - try self.store(slice_ptr_ty, ptr, src_val, .{ .safety = safety }); - - const second_elem_ptr_reg = - try self.register_manager.allocReg(null, abi.RegisterClass.gp); - const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg }; - const second_elem_ptr_lock = - self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg); - defer self.register_manager.unlockReg(second_elem_ptr_lock); - - try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{ - .reg = try self.copyToTmpRegister(Type.usize, ptr), - .off = elem_abi_size, - } }, .{}); - - try self.genBinOpMir(.{ ._, .sub }, Type.usize, len_mcv, .{ .immediate = 1 }); - try self.asmRegisterRegisterImmediate( - .{ .i_, .mul }, - len_reg, - len_reg, - Immediate.s(elem_abi_size), - ); - try self.genInlineMemcpy(second_elem_ptr_mcv, ptr, len_mcv); - - self.performReloc(skip_reloc); - }, - .one => { - const elem_ptr_ty = try pt.singleMutPtrType(elem_ty); - - const len = dst_ptr_ty.childType(zcu).arrayLen(zcu); - - assert(len != 0); // prevented by Sema - try self.store(elem_ptr_ty, dst_ptr, src_val, .{ .safety = safety }); + try self.store(slice_ptr_ty, dst_ptr, src_val, .{ .safety = safety }); const second_elem_ptr_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); @@ -16701,8 +16939,39 @@ fn airMemset(self: *Self, inst: Air.Inst.Index, safety: bool) !void { .off = elem_abi_size, } }, .{}); + try self.genBinOpMir(.{ ._, .sub }, Type.usize, len_mcv, .{ .immediate = 1 }); + try self.asmRegisterRegisterImmediate( + .{ .i_, .mul }, + len_reg, + len_reg, + .s(elem_abi_size), + ); + try self.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, len_mcv); + + self.performReloc(skip_reloc); + }, + .one => { + const elem_ptr_ty = try pt.singleMutPtrType(elem_ty); + + const len = dst_ty.childType(zcu).arrayLen(zcu); + + assert(len != 0); // prevented by Sema + try self.store(elem_ptr_ty, dst, src_val, .{ .safety = safety }); + + const second_elem_ptr_reg = + try self.register_manager.allocReg(null, abi.RegisterClass.gp); + const second_elem_ptr_mcv: MCValue = .{ .register = second_elem_ptr_reg }; + const second_elem_ptr_lock = + self.register_manager.lockRegAssumeUnused(second_elem_ptr_reg); + defer self.register_manager.unlockReg(second_elem_ptr_lock); + + try self.genSetReg(second_elem_ptr_reg, Type.usize, .{ .register_offset = .{ + .reg = try self.copyToTmpRegister(Type.usize, dst), + .off = elem_abi_size, + } }, .{}); + const bytes_to_copy: MCValue = .{ .immediate = elem_abi_size * (len - 1) }; - try self.genInlineMemcpy(second_elem_ptr_mcv, dst_ptr, bytes_to_copy); + try self.genInlineMemcpy(second_elem_ptr_mcv, dst, bytes_to_copy); }, .c, .many => unreachable, } @@ -16719,48 +16988,72 @@ fn airMemcpy(self: *Self, inst: Air.Inst.Index) !void { const reg_locks = self.register_manager.lockRegsAssumeUnused(3, .{ .rdi, .rsi, .rcx }); defer for (reg_locks) |lock| self.register_manager.unlockReg(lock); - const dst_ptr = try self.resolveInst(bin_op.lhs); - const dst_ptr_ty = self.typeOf(bin_op.lhs); - const dst_ptr_lock: ?RegisterLock = switch (dst_ptr) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, + const dst = try self.resolveInst(bin_op.lhs); + const dst_ty = self.typeOf(bin_op.lhs); + const dst_locks: [2]?RegisterLock = switch (dst) { + .register => |dst_reg| .{ self.register_manager.lockRegAssumeUnused(dst_reg), null }, + .register_pair => |dst_regs| .{ + self.register_manager.lockRegAssumeUnused(dst_regs[0]), + self.register_manager.lockReg(dst_regs[1]), + }, + else => .{ null, null }, }; - defer if (dst_ptr_lock) |lock| self.register_manager.unlockReg(lock); + for (dst_locks) |dst_lock| if (dst_lock) |lock| self.register_manager.unlockReg(lock); - const src_ptr = try self.resolveInst(bin_op.rhs); - const src_ptr_lock: ?RegisterLock = switch (src_ptr) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), - else => null, + const src = try self.resolveInst(bin_op.rhs); + const src_locks: [2]?RegisterLock = switch (src) { + .register => |src_reg| .{ self.register_manager.lockReg(src_reg), null }, + .register_pair => |src_regs| .{ + self.register_manager.lockRegAssumeUnused(src_regs[0]), + self.register_manager.lockRegAssumeUnused(src_regs[1]), + }, + else => .{ null, null }, }; - defer if (src_ptr_lock) |lock| self.register_manager.unlockReg(lock); + for (src_locks) |src_lock| if (src_lock) |lock| self.register_manager.unlockReg(lock); - const len: MCValue = switch (dst_ptr_ty.ptrSize(zcu)) { + const len: MCValue = switch (dst_ty.ptrSize(zcu)) { .slice => len: { const len_reg = try self.register_manager.allocReg(null, abi.RegisterClass.gp); const len_lock = self.register_manager.lockRegAssumeUnused(len_reg); defer self.register_manager.unlockReg(len_lock); - try self.asmRegisterMemoryImmediate( - .{ .i_, .mul }, - len_reg, - try dst_ptr.address().offset(8).deref().mem(self, .qword), - Immediate.s(@intCast(dst_ptr_ty.childType(zcu).abiSize(zcu))), - ); + switch (dst) { + .register_pair => |dst_regs| try self.asmRegisterRegisterImmediate( + .{ .i_, .mul }, + len_reg, + dst_regs[1], + .s(@intCast(dst_ty.childType(zcu).abiSize(zcu))), + ), + else => try self.asmRegisterMemoryImmediate( + .{ .i_, .mul }, + len_reg, + try dst.address().offset(8).deref().mem(self, .qword), + .s(@intCast(dst_ty.childType(zcu).abiSize(zcu))), + ), + } break :len .{ .register = len_reg }; }, .one => len: { - const array_ty = dst_ptr_ty.childType(zcu); + const array_ty = dst_ty.childType(zcu); break :len .{ .immediate = array_ty.arrayLen(zcu) * array_ty.childType(zcu).abiSize(zcu) }; }, .c, .many => unreachable, }; const len_lock: ?RegisterLock = switch (len) { - .register => |reg| self.register_manager.lockRegAssumeUnused(reg), + .register => |reg| self.register_manager.lockReg(reg), else => null, }; defer if (len_lock) |lock| self.register_manager.unlockReg(lock); - // TODO: dst_ptr and src_ptr could be slices rather than raw pointers + const dst_ptr: MCValue = switch (dst) { + .register_pair => |dst_regs| .{ .register = dst_regs[0] }, + else => dst, + }; + const src_ptr: MCValue = switch (src) { + .register_pair => |src_regs| .{ .register = src_regs[0] }, + else => src, + }; + try self.genInlineMemcpy(dst_ptr, src_ptr, len); return self.finishAir(inst, .unreach, .{ bin_op.lhs, bin_op.rhs, .none }); @@ -16930,27 +17223,23 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void { try self.genSetReg( regs[1], vector_ty, - .{ .immediate = @as(u64, math.maxInt(u64)) >> @intCast(64 - vector_len) }, + .{ .immediate = @as(u64, std.math.maxInt(u64)) >> @intCast(64 - vector_len) }, .{}, ); const src_mcv = try self.resolveInst(ty_op.operand); - const abi_size = @max(math.divCeil(u32, vector_len, 8) catch unreachable, 4); + const abi_size = @max(std.math.divCeil(u32, vector_len, 8) catch unreachable, 4); try self.asmCmovccRegisterRegister( switch (src_mcv) { .eflags => |cc| cc, .register => |src_reg| cc: { - try self.asmRegisterImmediate( - .{ ._, .@"test" }, - src_reg.to8(), - Immediate.u(1), - ); + try self.asmRegisterImmediate(.{ ._, .@"test" }, src_reg.to8(), .u(1)); break :cc .nz; }, else => cc: { try self.asmMemoryImmediate( .{ ._, .@"test" }, try src_mcv.mem(self, .byte), - Immediate.u(1), + .u(1), ); break :cc .nz; }, @@ -17037,7 +17326,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void { .{ if (self.hasFeature(.avx)) .vp_w else .p_w, .shufl }, dst_alias, dst_alias, - Immediate.u(0b00_00_00_00), + .u(0b00_00_00_00), ); if (switch (scalar_bits) { 1...8 => vector_len > 4, @@ -17049,7 +17338,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void { .{ if (self.hasFeature(.avx)) .vp_d else .p_d, .shuf }, dst_alias, dst_alias, - Immediate.u(if (scalar_bits <= 64) 0b00_00_00_00 else 0b01_00_01_00), + .u(if (scalar_bits <= 64) 0b00_00_00_00 else 0b01_00_01_00), ); break :result .{ .register = dst_reg }; }, @@ -17080,7 +17369,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void { dst_reg.to128(), src_reg.to128(), src_reg.to128(), - Immediate.u(0), + .u(0), ); } break :result .{ .register = dst_reg }; @@ -17095,7 +17384,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void { .{ ._ps, .shuf }, dst_reg.to128(), dst_reg.to128(), - Immediate.u(0), + .u(0), ); break :result dst_mcv; } @@ -17122,14 +17411,14 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void { dst_reg.to128(), src_reg.to128(), src_reg.to128(), - Immediate.u(0), + .u(0), ); try self.asmRegisterRegisterRegisterImmediate( .{ .v_f128, .insert }, dst_reg.to256(), dst_reg.to256(), dst_reg.to128(), - Immediate.u(1), + .u(1), ); } } @@ -17198,7 +17487,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void { dst_reg.to256(), dst_reg.to256(), dst_reg.to128(), - Immediate.u(1), + .u(1), ); } } @@ -17231,7 +17520,7 @@ fn airSplat(self: *Self, inst: Air.Inst.Index) !void { dst_reg.to256(), src_reg.to256(), src_reg.to128(), - Immediate.u(1), + .u(1), ); } break :result .{ .register = dst_reg }; @@ -17308,7 +17597,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void { mask_alias, mask_alias, mask_reg.to128(), - Immediate.u(1), + .u(1), ); break :broadcast; }, @@ -17362,7 +17651,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void { .{ if (has_avx) .vp_w else .p_w, .shufl }, mask_alias, mask_alias, - Immediate.u(0b00_00_00_00), + .u(0b00_00_00_00), ); if (abi_size <= 8) break :broadcast; } @@ -17370,7 +17659,7 @@ fn airSelect(self: *Self, inst: Air.Inst.Index) !void { .{ if (has_avx) .vp_d else .p_d, .shuf }, mask_alias, mask_alias, - Immediate.u(switch (elem_abi_size) { + .u(switch (elem_abi_size) { 1...2, 5...8 => 0b01_00_01_00, 3...4 => 0b00_00_00_00, else => unreachable, @@ -17649,7 +17938,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { for (mask_elems, 0..) |maybe_mask_elem, elem_index| { const mask_elem = maybe_mask_elem orelse continue; const mask_elem_index = - math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :unpck; + std.math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :unpck; const elem_byte = (elem_index >> 1) * elem_abi_size; if (mask_elem_index * elem_abi_size != (elem_byte & 0b0111) | @as(u4, switch (variant) { .unpckl => 0b0000, @@ -17746,10 +18035,10 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { } else sources[(elem_index & 0b010) >> 1] = source; const select_bit: u3 = @intCast((elem_index & 0b011) << 1); - const select = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit; + const select_mask = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit; if (elem_index & 0b100 == 0) - control |= select - else if (control & @as(u8, 0b11) << select_bit != select) break :pshufd; + control |= select_mask + else if (control & @as(u8, 0b11) << select_bit != select_mask) break :pshufd; } const operands = [2]Air.Inst.Ref{ extra.a, extra.b }; @@ -17767,7 +18056,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { .{ if (has_avx) .vp_d else .p_d, .shuf }, dst_alias, try src_mcv.mem(self, Memory.Size.fromSize(max_abi_size)), - Immediate.u(control), + .u(control), ) else try self.asmRegisterRegisterImmediate( .{ if (has_avx) .vp_d else .p_d, .shuf }, dst_alias, @@ -17775,7 +18064,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { src_mcv.getReg().? else try self.copyToTmpRegister(operand_tys[sources[0].?], src_mcv), max_abi_size), - Immediate.u(control), + .u(control), ); break :result .{ .register = dst_reg }; } @@ -17797,10 +18086,10 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { } else sources[(elem_index & 0b010) >> 1] = source; const select_bit: u3 = @intCast((elem_index & 0b011) << 1); - const select = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit; + const select_mask = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit; if (elem_index & 0b100 == 0) - control |= select - else if (control & @as(u8, 0b11) << select_bit != select) break :shufps; + control |= select_mask + else if (control & @as(u8, 0b11) << select_bit != select_mask) break :shufps; } if (sources[0] orelse break :shufps == sources[1] orelse break :shufps) break :shufps; @@ -17824,7 +18113,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { dst_alias, registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)), - Immediate.u(control), + .u(control), ) else try self.asmRegisterRegisterRegisterImmediate( .{ .v_ps, .shuf }, dst_alias, @@ -17833,12 +18122,12 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { rhs_mcv.getReg().? else try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), - Immediate.u(control), + .u(control), ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate( .{ ._ps, .shuf }, dst_alias, try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)), - Immediate.u(control), + .u(control), ) else try self.asmRegisterRegisterImmediate( .{ ._ps, .shuf }, dst_alias, @@ -17846,7 +18135,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { rhs_mcv.getReg().? else try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), - Immediate.u(control), + .u(control), ); break :result dst_mcv; } @@ -17891,7 +18180,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { dst_alias, registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)), - Immediate.u(control), + .u(control), ) else try self.asmRegisterRegisterRegisterImmediate( .{ .v_pd, .shuf }, dst_alias, @@ -17900,12 +18189,12 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { rhs_mcv.getReg().? else try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), - Immediate.u(control), + .u(control), ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate( .{ ._pd, .shuf }, dst_alias, try rhs_mcv.mem(self, Memory.Size.fromSize(max_abi_size)), - Immediate.u(control), + .u(control), ) else try self.asmRegisterRegisterImmediate( .{ ._pd, .shuf }, dst_alias, @@ -17913,7 +18202,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { rhs_mcv.getReg().? else try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), - Immediate.u(control), + .u(control), ); break :result dst_mcv; } @@ -17927,13 +18216,13 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { for (mask_elems, 0..) |maybe_mask_elem, elem_index| { const mask_elem = maybe_mask_elem orelse continue; const mask_elem_index = - math.cast(u4, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blend; + std.math.cast(u4, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blend; if (mask_elem_index != elem_index) break :blend; - const select = @as(u8, @intFromBool(mask_elem < 0)) << @truncate(elem_index); + const select_mask = @as(u8, @intFromBool(mask_elem < 0)) << @truncate(elem_index); if (elem_index & 0b1000 == 0) - control |= select - else if (control & @as(u8, 0b1) << @truncate(elem_index) != select) break :blend; + control |= select_mask + else if (control & @as(u8, 0b1) << @truncate(elem_index) != select_mask) break :blend; } if (!elem_ty.isRuntimeFloat() and self.hasFeature(.avx2)) vpblendd: { @@ -17961,7 +18250,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { registerAlias(dst_reg, dst_abi_size), registerAlias(lhs_reg, dst_abi_size), try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)), - Immediate.u(expanded_control), + .u(expanded_control), ) else try self.asmRegisterRegisterRegisterImmediate( .{ .vp_d, .blend }, registerAlias(dst_reg, dst_abi_size), @@ -17970,7 +18259,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { rhs_mcv.getReg().? else try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), - Immediate.u(expanded_control), + .u(expanded_control), ); break :result .{ .register = dst_reg }; } @@ -18016,7 +18305,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { else dst_reg, dst_abi_size), try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)), - Immediate.u(expanded_control), + .u(expanded_control), ) else try self.asmRegisterRegisterRegisterImmediate( .{ .vp_w, .blend }, registerAlias(dst_reg, dst_abi_size), @@ -18028,12 +18317,12 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { rhs_mcv.getReg().? else try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), - Immediate.u(expanded_control), + .u(expanded_control), ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate( .{ .p_w, .blend }, registerAlias(dst_reg, dst_abi_size), try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)), - Immediate.u(expanded_control), + .u(expanded_control), ) else try self.asmRegisterRegisterImmediate( .{ .p_w, .blend }, registerAlias(dst_reg, dst_abi_size), @@ -18041,7 +18330,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { rhs_mcv.getReg().? else try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), - Immediate.u(expanded_control), + .u(expanded_control), ); break :result .{ .register = dst_reg }; } @@ -18077,7 +18366,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { else dst_reg, dst_abi_size), try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)), - Immediate.u(expanded_control), + .u(expanded_control), ) else try self.asmRegisterRegisterRegisterImmediate( switch (elem_abi_size) { 4 => .{ .v_ps, .blend }, @@ -18093,7 +18382,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { rhs_mcv.getReg().? else try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), - Immediate.u(expanded_control), + .u(expanded_control), ) else if (rhs_mcv.isMemory()) try self.asmRegisterMemoryImmediate( switch (elem_abi_size) { 4 => .{ ._ps, .blend }, @@ -18102,7 +18391,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { }, registerAlias(dst_reg, dst_abi_size), try rhs_mcv.mem(self, Memory.Size.fromSize(dst_abi_size)), - Immediate.u(expanded_control), + .u(expanded_control), ) else try self.asmRegisterRegisterImmediate( switch (elem_abi_size) { 4 => .{ ._ps, .blend }, @@ -18114,7 +18403,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { rhs_mcv.getReg().? else try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), - Immediate.u(expanded_control), + .u(expanded_control), ); break :result .{ .register = dst_reg }; } @@ -18138,7 +18427,7 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { ) |*select_mask_elem, maybe_mask_elem, elem_index| { const mask_elem = maybe_mask_elem orelse continue; const mask_elem_index = - math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blendv; + std.math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blendv; if (mask_elem_index != elem_index) break :blendv; select_mask_elem.* = (if (mask_elem < 0) @@ -18380,7 +18669,9 @@ fn airShuffle(self: *Self, inst: Air.Inst.Index) !void { break :result null; }) orelse return self.fail("TODO implement airShuffle from {} and {} to {} with {}", .{ - lhs_ty.fmt(pt), rhs_ty.fmt(pt), dst_ty.fmt(pt), + lhs_ty.fmt(pt), + rhs_ty.fmt(pt), + dst_ty.fmt(pt), Value.fromInterned(extra.mask).fmtValue(pt), }); return self.finishAir(inst, result, .{ extra.a, extra.b, .none }); @@ -18397,7 +18688,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void { try self.spillEflagsIfOccupied(); const operand_mcv = try self.resolveInst(reduce.operand); - const mask_len = (math.cast(u6, operand_ty.vectorLen(zcu)) orelse + const mask_len = (std.math.cast(u6, operand_ty.vectorLen(zcu)) orelse return self.fail("TODO implement airReduce for {}", .{operand_ty.fmt(pt)})); const mask = (@as(u64, 1) << mask_len) - 1; const abi_size: u32 = @intCast(operand_ty.abiSize(zcu)); @@ -18406,7 +18697,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void { if (operand_mcv.isMemory()) try self.asmMemoryImmediate( .{ ._, .@"test" }, try operand_mcv.mem(self, Memory.Size.fromSize(abi_size)), - Immediate.u(mask), + .u(mask), ) else { const operand_reg = registerAlias(if (operand_mcv.isRegister()) operand_mcv.getReg().? @@ -18415,7 +18706,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void { if (mask_len < abi_size * 8) try self.asmRegisterImmediate( .{ ._, .@"test" }, operand_reg, - Immediate.u(mask), + .u(mask), ) else try self.asmRegisterRegister( .{ ._, .@"test" }, operand_reg, @@ -18431,7 +18722,7 @@ fn airReduce(self: *Self, inst: Air.Inst.Index) !void { try self.asmRegister(.{ ._, .not }, tmp_reg); if (mask_len < abi_size * 8) - try self.asmRegisterImmediate(.{ ._, .@"test" }, tmp_reg, Immediate.u(mask)) + try self.asmRegisterImmediate(.{ ._, .@"test" }, tmp_reg, .u(mask)) else try self.asmRegisterRegister(.{ ._, .@"test" }, tmp_reg, tmp_reg); break :result .{ .eflags = .z }; @@ -18579,12 +18870,12 @@ fn airAggregateInit(self: *Self, inst: Air.Inst.Index) !void { try self.asmRegisterImmediate( .{ ._, .@"and" }, registerAlias(elem_reg, @min(result_size, 4)), - Immediate.u(1), + .u(1), ); if (elem_i > 0) try self.asmRegisterImmediate( .{ ._l, .sh }, registerAlias(elem_reg, result_size), - Immediate.u(@intCast(elem_i)), + .u(@intCast(elem_i)), ); try self.asmRegisterRegister( .{ ._, .@"or" }, @@ -18748,8 +19039,8 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void { lock.* = self.register_manager.lockRegAssumeUnused(reg); } - const mir_tag = @as(?Mir.Inst.FixedTag, if (mem.eql(u2, &order, &.{ 1, 3, 2 }) or - mem.eql(u2, &order, &.{ 3, 1, 2 })) + const mir_tag = @as(?Mir.Inst.FixedTag, if (std.mem.eql(u2, &order, &.{ 1, 3, 2 }) or + std.mem.eql(u2, &order, &.{ 3, 1, 2 })) switch (ty.zigTypeTag(zcu)) { .float => switch (ty.floatBits(self.target.*)) { 32 => .{ .v_ss, .fmadd132 }, @@ -18776,7 +19067,7 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void { }, else => unreachable, } - else if (mem.eql(u2, &order, &.{ 2, 1, 3 }) or mem.eql(u2, &order, &.{ 1, 2, 3 })) + else if (std.mem.eql(u2, &order, &.{ 2, 1, 3 }) or std.mem.eql(u2, &order, &.{ 1, 2, 3 })) switch (ty.zigTypeTag(zcu)) { .float => switch (ty.floatBits(self.target.*)) { 32 => .{ .v_ss, .fmadd213 }, @@ -18803,7 +19094,7 @@ fn airMulAdd(self: *Self, inst: Air.Inst.Index) !void { }, else => unreachable, } - else if (mem.eql(u2, &order, &.{ 2, 3, 1 }) or mem.eql(u2, &order, &.{ 3, 2, 1 })) + else if (std.mem.eql(u2, &order, &.{ 2, 3, 1 }) or std.mem.eql(u2, &order, &.{ 3, 2, 1 })) switch (ty.zigTypeTag(zcu)) { .float => switch (ty.floatBits(self.target.*)) { 32 => .{ .v_ss, .fmadd231 }, @@ -18953,13 +19244,13 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void { const overflow_arg_area: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 8 } }; const reg_save_area: MCValue = .{ .indirect = .{ .reg = ptr_arg_list_reg, .off = 16 } }; - const classes = mem.sliceTo(&abi.classifySystemV(promote_ty, zcu, self.target.*, .arg), .none); + const classes = std.mem.sliceTo(&abi.classifySystemV(promote_ty, zcu, self.target.*, .arg), .none); switch (classes[0]) { .integer => { assert(classes.len == 1); try self.genSetReg(offset_reg, Type.c_uint, gp_offset, .{}); - try self.asmRegisterImmediate(.{ ._, .cmp }, offset_reg, Immediate.u( + try self.asmRegisterImmediate(.{ ._, .cmp }, offset_reg, .u( abi.SysV.c_abi_int_param_regs.len * 8, )); const mem_reloc = try self.asmJccReloc(.ae, undefined); @@ -19007,7 +19298,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void { assert(classes.len == 1); try self.genSetReg(offset_reg, Type.c_uint, fp_offset, .{}); - try self.asmRegisterImmediate(.{ ._, .cmp }, offset_reg, Immediate.u( + try self.asmRegisterImmediate(.{ ._, .cmp }, offset_reg, .u( abi.SysV.c_abi_int_param_regs.len * 8 + abi.SysV.c_abi_sse_param_regs.len * 16, )); const mem_reloc = try self.asmJccReloc(.ae, undefined); @@ -19055,9 +19346,7 @@ fn airVaArg(self: *Self, inst: Air.Inst.Index) !void { assert(classes.len == 1); unreachable; }, - else => return self.fail("TODO implement c_va_arg for {} on SysV", .{ - promote_ty.fmt(pt), - }), + else => return self.fail("TODO implement c_va_arg for {} on SysV", .{promote_ty.fmt(pt)}), } if (unused) break :result .unreach; @@ -19194,7 +19483,7 @@ fn limitImmediateType(self: *Self, operand: Air.Inst.Ref, comptime T: type) !MCV .immediate => |imm| { // This immediate is unsigned. const U = std.meta.Int(.unsigned, ti.bits - @intFromBool(ti.signedness == .signed)); - if (imm >= math.maxInt(U)) { + if (imm >= std.math.maxInt(U)) { return MCValue{ .register = try self.copyToTmpRegister(Type.usize, mcv) }; } }, @@ -19226,7 +19515,7 @@ const CallMCValues = struct { args: []MCValue, return_value: InstTracking, stack_byte_count: u31, - stack_align: Alignment, + stack_align: InternPool.Alignment, gp_count: u32, fp_count: u32, @@ -19303,7 +19592,7 @@ fn resolveCallingConventionValues( var ret_tracking_i: usize = 0; const classes = switch (resolved_cc) { - .x86_64_sysv => mem.sliceTo(&abi.classifySystemV(ret_ty, zcu, self.target.*, .ret), .none), + .x86_64_sysv => std.mem.sliceTo(&abi.classifySystemV(ret_ty, zcu, self.target.*, .ret), .none), .x86_64_win => &.{abi.classifyWindows(ret_ty, zcu)}, else => unreachable, }; @@ -19378,7 +19667,7 @@ fn resolveCallingConventionValues( var arg_mcv_i: usize = 0; const classes = switch (resolved_cc) { - .x86_64_sysv => mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .arg), .none), + .x86_64_sysv => std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .arg), .none), .x86_64_win => &.{abi.classifyWindows(ty, zcu)}, else => unreachable, }; @@ -19435,7 +19724,7 @@ fn resolveCallingConventionValues( const frame_elem_align = 8; const frame_elems_len = ty.vectorLen(zcu) - remaining_param_int_regs; - const frame_elem_size = mem.alignForward( + const frame_elem_size = std.mem.alignForward( u64, ty.childType(zcu).abiSize(zcu), frame_elem_align, @@ -19443,7 +19732,7 @@ fn resolveCallingConventionValues( const frame_size: u31 = @intCast(frame_elems_len * frame_elem_size); result.stack_byte_count = - mem.alignForward(u31, result.stack_byte_count, frame_elem_align); + std.mem.alignForward(u31, result.stack_byte_count, frame_elem_align); arg_mcv[arg_mcv_i] = .{ .elementwise_regs_then_frame = .{ .regs = remaining_param_int_regs, .frame_off = @intCast(result.stack_byte_count), @@ -19461,19 +19750,14 @@ fn resolveCallingConventionValues( continue; } - const param_size: u31 = @intCast(ty.abiSize(zcu)); const param_align = ty.abiAlignment(zcu).max(.@"8"); - result.stack_byte_count = mem.alignForward( - u31, - result.stack_byte_count, - @intCast(param_align.toByteUnits().?), - ); + result.stack_byte_count = @intCast(param_align.forward(result.stack_byte_count)); result.stack_align = result.stack_align.max(param_align); arg.* = .{ .load_frame = .{ .index = stack_frame_base, .off = result.stack_byte_count, } }; - result.stack_byte_count += param_size; + result.stack_byte_count += @intCast(ty.abiSize(zcu)); } assert(param_int_reg_i <= 6); result.gp_count = param_int_reg_i; @@ -19509,19 +19793,14 @@ fn resolveCallingConventionValues( arg.* = .none; continue; } - const param_size: u31 = @intCast(ty.abiSize(zcu)); const param_align = ty.abiAlignment(zcu); - result.stack_byte_count = mem.alignForward( - u31, - result.stack_byte_count, - @intCast(param_align.toByteUnits().?), - ); + result.stack_byte_count = @intCast(param_align.forward(result.stack_byte_count)); result.stack_align = result.stack_align.max(param_align); arg.* = .{ .load_frame = .{ .index = stack_frame_base, .off = result.stack_byte_count, } }; - result.stack_byte_count += param_size; + result.stack_byte_count += @intCast(ty.abiSize(zcu)); } }, else => return self.fail("TODO implement function parameters and return values for {} on x86_64", .{cc}), @@ -19541,7 +19820,7 @@ fn fail(self: *Self, comptime format: []const u8, args: anytype) error{ OutOfMem return error.CodegenFail; } -fn failMsg(self: *Self, msg: *ErrorMsg) error{ OutOfMemory, CodegenFail } { +fn failMsg(self: *Self, msg: *Zcu.ErrorMsg) error{ OutOfMemory, CodegenFail } { @branchHint(.cold); const zcu = self.pt.zcu; switch (self.owner) { @@ -19603,8 +19882,7 @@ fn registerAlias(reg: Register, size_bytes: u32) Register { } fn memSize(self: *Self, ty: Type) Memory.Size { - const pt = self.pt; - const zcu = pt.zcu; + const zcu = self.pt.zcu; return switch (ty.zigTypeTag(zcu)) { .float => Memory.Size.fromBitSize(ty.floatBits(self.target.*)), else => Memory.Size.fromSize(@intCast(ty.abiSize(zcu))), @@ -19614,7 +19892,7 @@ fn memSize(self: *Self, ty: Type) Memory.Size { fn splitType(self: *Self, ty: Type) ![2]Type { const pt = self.pt; const zcu = pt.zcu; - const classes = mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .other), .none); + const classes = std.mem.sliceTo(&abi.classifySystemV(ty, zcu, self.target.*, .other), .none); var parts: [2]Type = undefined; if (classes.len == 2) for (&parts, classes, 0..) |*part, class, part_i| { part.* = switch (class) { @@ -19648,7 +19926,7 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void { .signedness = .unsigned, .bits = @intCast(ty.bitSize(zcu)), }; - const shift = math.cast(u6, 64 - int_info.bits % 64) orelse return; + const shift = std.math.cast(u6, 64 - int_info.bits % 64) orelse return; try self.spillEflagsIfOccupied(); switch (int_info.signedness) { .signed => { @@ -19690,8 +19968,7 @@ fn truncateRegister(self: *Self, ty: Type, reg: Register) !void { } fn regBitSize(self: *Self, ty: Type) u64 { - const pt = self.pt; - const zcu = pt.zcu; + const zcu = self.pt.zcu; const abi_size = ty.abiSize(zcu); return switch (ty.zigTypeTag(zcu)) { else => switch (abi_size) { @@ -19713,14 +19990,14 @@ fn regExtraBits(self: *Self, ty: Type) u64 { return self.regBitSize(ty) - ty.bitSize(self.pt.zcu); } -fn hasFeature(self: *Self, feature: Target.x86.Feature) bool { - return Target.x86.featureSetHas(self.target.cpu.features, feature); +fn hasFeature(self: *Self, feature: std.Target.x86.Feature) bool { + return std.Target.x86.featureSetHas(self.target.cpu.features, feature); } fn hasAnyFeatures(self: *Self, features: anytype) bool { - return Target.x86.featureSetHasAny(self.target.cpu.features, features); + return std.Target.x86.featureSetHasAny(self.target.cpu.features, features); } fn hasAllFeatures(self: *Self, features: anytype) bool { - return Target.x86.featureSetHasAll(self.target.cpu.features, features); + return std.Target.x86.featureSetHasAll(self.target.cpu.features, features); } fn typeOf(self: *Self, inst: Air.Inst.Ref) Type { @@ -19732,9 +20009,13 @@ fn typeOf(self: *Self, inst: Air.Inst.Ref) Type { fn typeOfIndex(self: *Self, inst: Air.Inst.Index) Type { const pt = self.pt; const zcu = pt.zcu; - return switch (self.air.instructions.items(.tag)[@intFromEnum(inst)]) { - .loop_switch_br => self.typeOf(self.air.unwrapSwitch(inst).operand), - else => self.air.typeOfIndex(inst, &zcu.intern_pool), + const temp: Temp = .{ .index = inst }; + return switch (temp.unwrap(self)) { + .ref => switch (self.air.instructions.items(.tag)[@intFromEnum(inst)]) { + .loop_switch_br => self.typeOf(self.air.unwrapSwitch(inst).operand), + else => self.air.typeOfIndex(inst, &zcu.intern_pool), + }, + .temp => temp.typeOf(self), }; } @@ -19815,3 +20096,613 @@ fn promoteVarArg(self: *Self, ty: Type) Type { }, } } + +// ====================================== rewrite starts here ====================================== + +const Temp = struct { + index: Air.Inst.Index, + + fn unwrap(temp: Temp, self: *Self) union(enum) { + ref: Air.Inst.Ref, + temp: Index, + } { + switch (temp.index.unwrap()) { + .ref => |ref| return .{ .ref = ref }, + .target => |target_index| { + const temp_index: Index = @enumFromInt(target_index); + assert(temp_index.isValid(self)); + return .{ .temp = temp_index }; + }, + } + } + + fn typeOf(temp: Temp, self: *Self) Type { + return switch (temp.unwrap(self)) { + .ref => |ref| self.typeOf(ref), + .temp => |temp_index| temp_index.typeOf(self), + }; + } + + fn isMut(temp: Temp, self: *Self) bool { + return temp.unwrap(self) == .temp; + } + + fn tracking(temp: Temp, self: *Self) InstTracking { + return self.inst_tracking.get(temp.index).?; + } + + fn getOffset(temp: Temp, off: i32, self: *Self) !Temp { + const new_temp_index = self.next_temp_index; + self.temp_type[@intFromEnum(new_temp_index)] = Type.usize; + self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1); + switch (temp.tracking(self).short) { + else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }), + .register => |reg| { + const new_reg = + try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); + new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg }); + try self.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{ + .base = .{ .reg = reg.to64() }, + .mod = .{ .rm = .{ + .size = .qword, + .disp = off, + } }, + }); + }, + .register_offset => |reg_off| { + const new_reg = + try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); + new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg }); + try self.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{ + .base = .{ .reg = reg_off.reg.to64() }, + .mod = .{ .rm = .{ + .size = .qword, + .disp = reg_off.off + off, + } }, + }); + }, + .lea_symbol => |sym_off| new_temp_index.tracking(self).* = InstTracking.init(.{ .lea_symbol = .{ + .sym_index = sym_off.sym_index, + .off = sym_off.off + off, + } }), + .load_frame => |frame_addr| { + const new_reg = + try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); + new_temp_index.tracking(self).* = InstTracking.init(.{ .register_offset = .{ + .reg = new_reg, + .off = off, + } }); + try self.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{ + .base = .{ .frame = frame_addr.index }, + .mod = .{ .rm = .{ + .size = .qword, + .disp = frame_addr.off, + } }, + }); + }, + .lea_frame => |frame_addr| new_temp_index.tracking(self).* = InstTracking.init(.{ .lea_frame = .{ + .index = frame_addr.index, + .off = frame_addr.off + off, + } }), + } + return .{ .index = new_temp_index.toIndex() }; + } + + fn toOffset(temp: *Temp, off: i32, self: *Self) !void { + if (off == 0) return; + switch (temp.unwrap(self)) { + .ref => {}, + .temp => |temp_index| { + const temp_tracking = temp_index.tracking(self); + switch (temp_tracking.short) { + else => {}, + .register => |reg| { + try self.freeValue(temp_tracking.long); + temp_tracking.* = InstTracking.init(.{ .register_offset = .{ + .reg = reg, + .off = off, + } }); + return; + }, + .register_offset => |reg_off| { + try self.freeValue(temp_tracking.long); + temp_tracking.* = InstTracking.init(.{ .register_offset = .{ + .reg = reg_off.reg, + .off = reg_off.off + off, + } }); + return; + }, + .lea_symbol => |sym_off| { + assert(std.meta.eql(temp_tracking.long.lea_symbol, sym_off)); + temp_tracking.* = InstTracking.init(.{ .lea_symbol = .{ + .sym_index = sym_off.sym_index, + .off = sym_off.off + off, + } }); + return; + }, + .lea_frame => |frame_addr| { + assert(std.meta.eql(temp_tracking.long.lea_frame, frame_addr)); + temp_tracking.* = InstTracking.init(.{ .lea_frame = .{ + .index = frame_addr.index, + .off = frame_addr.off + off, + } }); + return; + }, + } + }, + } + const new_temp = try temp.getOffset(off, self); + try temp.die(self); + temp.* = new_temp; + } + + fn getLimb(temp: Temp, limb_index: u28, self: *Self) !Temp { + const new_temp_index = self.next_temp_index; + self.temp_type[@intFromEnum(new_temp_index)] = Type.usize; + switch (temp.tracking(self).short) { + else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }), + .immediate => |imm| { + assert(limb_index == 0); + new_temp_index.tracking(self).* = InstTracking.init(.{ .immediate = imm }); + }, + .register => |reg| { + assert(limb_index == 0); + const new_reg = + try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); + new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg }); + try self.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), reg.to64()); + }, + .register_pair => |regs| { + const new_reg = + try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); + new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg }); + try self.asmRegisterRegister(.{ ._, .mov }, new_reg.to64(), regs[limb_index].to64()); + }, + .register_offset => |reg_off| { + assert(limb_index == 0); + const new_reg = + try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); + new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg }); + try self.asmRegisterMemory(.{ ._, .lea }, new_reg.to64(), .{ + .base = .{ .reg = reg_off.reg.to64() }, + .mod = .{ .rm = .{ + .size = .qword, + .disp = reg_off.off + @as(u31, limb_index) * 8, + } }, + }); + }, + .load_symbol => |sym_off| { + const new_reg = + try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); + new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg }); + try self.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{ + .base = .{ .reloc = sym_off.sym_index }, + .mod = .{ .rm = .{ + .size = .qword, + .disp = sym_off.off + @as(u31, limb_index) * 8, + } }, + }); + }, + .lea_symbol => |sym_off| { + assert(limb_index == 0); + new_temp_index.tracking(self).* = InstTracking.init(.{ .lea_symbol = sym_off }); + }, + .load_frame => |frame_addr| { + const new_reg = + try self.register_manager.allocReg(new_temp_index.toIndex(), abi.RegisterClass.gp); + new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg }); + try self.asmRegisterMemory(.{ ._, .mov }, new_reg.to64(), .{ + .base = .{ .frame = frame_addr.index }, + .mod = .{ .rm = .{ + .size = .qword, + .disp = frame_addr.off + @as(u31, limb_index) * 8, + } }, + }); + }, + .lea_frame => |frame_addr| { + assert(limb_index == 0); + new_temp_index.tracking(self).* = InstTracking.init(.{ .lea_frame = frame_addr }); + }, + } + self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1); + return .{ .index = new_temp_index.toIndex() }; + } + + fn toLimb(temp: *Temp, limb_index: u28, self: *Self) !void { + switch (temp.unwrap(self)) { + .ref => {}, + .temp => |temp_index| { + const temp_tracking = temp_index.tracking(self); + switch (temp_tracking.short) { + else => {}, + .register, .lea_symbol, .lea_frame => { + assert(limb_index == 0); + self.temp_type[@intFromEnum(temp_index)] = Type.usize; + return; + }, + .register_pair => |regs| { + switch (temp_tracking.long) { + .none, .reserved_frame => {}, + else => temp_tracking.long = + temp_tracking.long.address().offset(@as(u31, limb_index) * 8).deref(), + } + for (regs, 0..) |reg, reg_index| if (reg_index != limb_index) + self.register_manager.freeReg(reg); + temp_tracking.* = InstTracking.init(.{ .register = regs[limb_index] }); + self.temp_type[@intFromEnum(temp_index)] = Type.usize; + return; + }, + .load_symbol => |sym_off| { + assert(std.meta.eql(temp_tracking.long.load_symbol, sym_off)); + temp_tracking.* = InstTracking.init(.{ .load_symbol = .{ + .sym_index = sym_off.sym_index, + .off = sym_off.off + @as(u31, limb_index) * 8, + } }); + self.temp_type[@intFromEnum(temp_index)] = Type.usize; + return; + }, + .load_frame => |frame_addr| if (!frame_addr.index.isNamed()) { + assert(std.meta.eql(temp_tracking.long.load_frame, frame_addr)); + temp_tracking.* = InstTracking.init(.{ .load_frame = .{ + .index = frame_addr.index, + .off = frame_addr.off + @as(u31, limb_index) * 8, + } }); + self.temp_type[@intFromEnum(temp_index)] = Type.usize; + return; + }, + } + }, + } + const new_temp = try temp.getLimb(limb_index, self); + try temp.die(self); + temp.* = new_temp; + } + + fn toReg(temp: *Temp, new_reg: Register, self: *Self) !bool { + const val, const ty = switch (temp.unwrap(self)) { + .ref => |ref| .{ temp.tracking(self).short, self.typeOf(ref) }, + .temp => |temp_index| val: { + const temp_tracking = temp_index.tracking(self); + if (temp_tracking.short == .register and + temp_tracking.short.register == new_reg) return false; + break :val .{ temp_tracking.short, temp_index.typeOf(self) }; + }, + }; + const new_temp_index = self.next_temp_index; + self.temp_type[@intFromEnum(new_temp_index)] = ty; + try self.genSetReg(new_reg, ty, val, .{}); + new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg }); + try temp.die(self); + self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1); + temp.* = .{ .index = new_temp_index.toIndex() }; + return true; + } + + fn toAnyReg(temp: *Temp, self: *Self) !bool { + const val, const ty = switch (temp.unwrap(self)) { + .ref => |ref| .{ temp.tracking(self).short, self.typeOf(ref) }, + .temp => |temp_index| val: { + const temp_tracking = temp_index.tracking(self); + if (temp_tracking.short == .register) return false; + break :val .{ temp_tracking.short, temp_index.typeOf(self) }; + }, + }; + const new_temp_index = self.next_temp_index; + self.temp_type[@intFromEnum(new_temp_index)] = ty; + const new_reg = + try self.register_manager.allocReg(new_temp_index.toIndex(), self.regClassForType(ty)); + try self.genSetReg(new_reg, ty, val, .{}); + new_temp_index.tracking(self).* = InstTracking.init(.{ .register = new_reg }); + try temp.die(self); + self.next_temp_index = @enumFromInt(@intFromEnum(new_temp_index) + 1); + temp.* = .{ .index = new_temp_index.toIndex() }; + return true; + } + + fn toPair(first_temp: *Temp, second_temp: *Temp, self: *Self) !void { + while (true) for ([_]*Temp{ first_temp, second_temp }) |part_temp| { + if (try part_temp.toAnyReg(self)) break; + } else break; + const first_temp_tracking = first_temp.unwrap(self).temp.tracking(self); + const second_temp_tracking = second_temp.unwrap(self).temp.tracking(self); + const result: MCValue = .{ .register_pair = .{ + first_temp_tracking.short.register, + second_temp_tracking.short.register, + } }; + const result_temp_index = self.next_temp_index; + const result_temp: Temp = .{ .index = result_temp_index.toIndex() }; + assert(self.reuseTemp(result_temp.index, first_temp.index, first_temp_tracking)); + assert(self.reuseTemp(result_temp.index, second_temp.index, second_temp_tracking)); + self.temp_type[@intFromEnum(result_temp_index)] = Type.slice_const_u8; + result_temp_index.tracking(self).* = InstTracking.init(result); + first_temp.* = result_temp; + } + + fn toLea(temp: *Temp, self: *Self) !bool { + switch (temp.tracking(self).short) { + .none, + .unreach, + .dead, + .undef, + .eflags, + .register_pair, + .register_overflow, + .elementwise_regs_then_frame, + .reserved_frame, + .air_ref, + => unreachable, // not a valid pointer + .immediate, + .register, + .register_offset, + .lea_direct, + .lea_got, + .lea_tlv, + .lea_frame, + => return false, + .memory, + .indirect, + .load_symbol, + .load_direct, + .load_got, + .load_tlv, + .load_frame, + => return temp.toAnyReg(self), + .lea_symbol => |sym_off| { + const off = sym_off.off; + if (off == 0) return false; + try temp.toOffset(-off, self); + while (try temp.toAnyReg(self)) {} + try temp.toOffset(off, self); + return true; + }, + } + } + + fn load(ptr: *Temp, val_ty: Type, self: *Self) !Temp { + const val_abi_size: u32 = @intCast(val_ty.abiSize(self.pt.zcu)); + const val = try self.tempAlloc(val_ty); + switch (val.tracking(self).short) { + else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }), + .register => |val_reg| { + while (try ptr.toLea(self)) {} + switch (val_reg.class()) { + .general_purpose => try self.asmRegisterMemory( + .{ ._, .mov }, + registerAlias(val_reg, val_abi_size), + try ptr.tracking(self).short.deref().mem(self, self.memSize(val_ty)), + ), + else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }), + } + }, + .load_frame => |val_frame_addr| { + var val_ptr = try self.tempFromValue(Type.usize, .{ .lea_frame = val_frame_addr }); + var len = try self.tempFromValue(Type.usize, .{ .immediate = val_abi_size }); + try val_ptr.memcpy(ptr, &len, self); + try val_ptr.die(self); + try len.die(self); + }, + } + return val; + } + + fn store(ptr: *Temp, val: *Temp, self: *Self) !void { + const val_ty = val.typeOf(self); + const val_abi_size: u32 = @intCast(val_ty.abiSize(self.pt.zcu)); + val: switch (val.tracking(self).short) { + else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }), + .immediate => |imm| if (std.math.cast(i32, imm)) |s| { + while (try ptr.toLea(self)) {} + try self.asmMemoryImmediate( + .{ ._, .mov }, + try ptr.tracking(self).short.deref().mem(self, self.memSize(val_ty)), + .s(s), + ); + } else continue :val .{ .register = undefined }, + .register => { + while (try ptr.toLea(self) or try val.toAnyReg(self)) {} + const val_reg = val.tracking(self).short.register; + switch (val_reg.class()) { + .general_purpose => try self.asmMemoryRegister( + .{ ._, .mov }, + try ptr.tracking(self).short.deref().mem(self, self.memSize(val_ty)), + registerAlias(val_reg, val_abi_size), + ), + else => |tag| std.debug.panic("{s}: {any}\n", .{ @src().fn_name, tag }), + } + }, + } + } + + fn memcpy(dst: *Temp, src: *Temp, len: *Temp, self: *Self) !void { + while (true) for ([_]*Temp{ dst, src, len }, [_]Register{ .rdi, .rsi, .rcx }) |temp, reg| { + if (try temp.toReg(reg, self)) break; + } else break; + try self.asmOpOnly(.{ .@"rep _sb", .mov }); + } + + fn moveTo(temp: Temp, inst: Air.Inst.Index, self: *Self) !void { + if (self.liveness.isUnused(inst)) try temp.die(self) else switch (temp.unwrap(self)) { + .ref => { + const result = try self.allocRegOrMem(inst, true); + try self.genCopy(self.typeOfIndex(inst), result, temp.tracking(self).short, .{}); + tracking_log.debug("{} => {} (birth)", .{ inst, result }); + self.inst_tracking.putAssumeCapacityNoClobber(inst, InstTracking.init(result)); + }, + .temp => |temp_index| { + const temp_tracking = temp_index.tracking(self); + tracking_log.debug("{} => {} (birth)", .{ inst, temp_tracking.short }); + self.inst_tracking.putAssumeCapacityNoClobber(inst, temp_tracking.*); + assert(self.reuseTemp(inst, temp_index.toIndex(), temp_tracking)); + }, + } + } + + fn die(temp: Temp, self: *Self) !void { + switch (temp.unwrap(self)) { + .ref => {}, + .temp => |temp_index| try temp_index.tracking(self).die(self, temp_index.toIndex()), + } + } + + const Index = enum(u4) { + _, + + fn toIndex(index: Index) Air.Inst.Index { + return Air.Inst.Index.fromTargetIndex(@intFromEnum(index)); + } + + fn fromIndex(index: Air.Inst.Index) Index { + return @enumFromInt(index.toTargetIndex()); + } + + fn tracking(index: Index, self: *Self) *InstTracking { + return &self.inst_tracking.values()[@intFromEnum(index)]; + } + + fn isValid(index: Index, self: *Self) bool { + return index.tracking(self).short != .dead; + } + + fn typeOf(index: Index, self: *Self) Type { + assert(index.isValid(self)); + return self.temp_type[@intFromEnum(index)]; + } + + const max = std.math.maxInt(@typeInfo(Index).@"enum".tag_type); + const Set = std.StaticBitSet(max); + const SafetySet = if (std.debug.runtime_safety) Set else struct { + inline fn initEmpty() @This() { + return .{}; + } + + inline fn isSet(_: @This(), index: usize) bool { + assert(index < max); + return true; + } + + inline fn set(_: @This(), index: usize) void { + assert(index < max); + } + + inline fn eql(_: @This(), _: @This()) bool { + return true; + } + }; + }; +}; + +fn resetTemps(self: *Self) void { + for (0..@intFromEnum(self.next_temp_index)) |temp_index| { + const temp: Temp.Index = @enumFromInt(temp_index); + assert(!temp.isValid(self)); + self.temp_type[temp_index] = undefined; + } + self.next_temp_index = @enumFromInt(0); +} + +fn reuseTemp( + self: *Self, + new_inst: Air.Inst.Index, + old_inst: Air.Inst.Index, + tracking: *InstTracking, +) bool { + switch (tracking.short) { + .register, + .register_pair, + .register_offset, + .register_overflow, + => for (tracking.short.getRegs()) |tracked_reg| { + if (RegisterManager.indexOfRegIntoTracked(tracked_reg)) |tracked_index| { + self.register_manager.registers[tracked_index] = new_inst; + } + }, + .load_frame => |frame_addr| if (frame_addr.index.isNamed()) return false, + else => {}, + } + switch (tracking.short) { + .eflags, .register_overflow => self.eflags_inst = new_inst, + else => {}, + } + tracking.reuse(self, new_inst, old_inst); + return true; +} + +fn tempAlloc(self: *Self, ty: Type) !Temp { + const temp_index = self.next_temp_index; + temp_index.tracking(self).* = InstTracking.init(try self.allocRegOrMemAdvanced(ty, temp_index.toIndex(), true)); + self.temp_type[@intFromEnum(temp_index)] = ty; + self.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); + return .{ .index = temp_index.toIndex() }; +} + +fn tempFromValue(self: *Self, ty: Type, value: MCValue) !Temp { + const temp_index = self.next_temp_index; + temp_index.tracking(self).* = InstTracking.init(value); + self.temp_type[@intFromEnum(temp_index)] = ty; + try self.getValue(value, temp_index.toIndex()); + self.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); + return .{ .index = temp_index.toIndex() }; +} + +fn tempFromOperand( + self: *Self, + inst: Air.Inst.Index, + op_index: Liveness.OperandInt, + op_ref: Air.Inst.Ref, +) !Temp { + const zcu = self.pt.zcu; + const ip = &zcu.intern_pool; + + if (!self.liveness.operandDies(inst, op_index)) { + if (op_ref.toIndex()) |op_inst| return .{ .index = op_inst }; + const val = op_ref.toInterned().?; + const gop = try self.const_tracking.getOrPut(self.gpa, val); + if (!gop.found_existing) gop.value_ptr.* = InstTracking.init(init: { + const const_mcv = try self.genTypedValue(Value.fromInterned(val)); + switch (const_mcv) { + .lea_tlv => |tlv_sym| switch (self.bin_file.tag) { + .elf, .macho => { + if (self.mod.pic) { + try self.spillRegisters(&.{ .rdi, .rax }); + } else { + try self.spillRegisters(&.{.rax}); + } + const frame_index = try self.allocFrameIndex(FrameAlloc.init(.{ + .size = 8, + .alignment = .@"8", + })); + try self.genSetMem( + .{ .frame = frame_index }, + 0, + Type.usize, + .{ .lea_symbol = .{ .sym_index = tlv_sym } }, + .{}, + ); + break :init .{ .load_frame = .{ .index = frame_index } }; + }, + else => break :init const_mcv, + }, + else => break :init const_mcv, + } + }); + return self.tempFromValue(Type.fromInterned(ip.typeOf(val)), gop.value_ptr.short); + } + + const temp_index = self.next_temp_index; + const temp: Temp = .{ .index = temp_index.toIndex() }; + const op_inst = op_ref.toIndex().?; + const tracking = self.getResolvedInstValue(op_inst); + temp_index.tracking(self).* = tracking.*; + if (!self.reuseTemp(temp.index, op_inst, tracking)) return .{ .index = op_ref.toIndex().? }; + self.temp_type[@intFromEnum(temp_index)] = self.typeOf(op_ref); + self.next_temp_index = @enumFromInt(@intFromEnum(temp_index) + 1); + return temp; +} + +inline fn tempsFromOperands(self: *Self, inst: Air.Inst.Index, op_refs: anytype) ![op_refs.len]Temp { + var temps: [op_refs.len]Temp = undefined; + inline for (&temps, 0.., op_refs) |*temp, op_index, op_ref| { + temp.* = try self.tempFromOperand(inst, op_index, op_ref); + } + return temps; +} diff --git a/src/arch/x86_64/abi.zig b/src/arch/x86_64/abi.zig index d825b70cf6..85ece4f93c 100644 --- a/src/arch/x86_64/abi.zig +++ b/src/arch/x86_64/abi.zig @@ -250,9 +250,8 @@ pub fn classifySystemV(ty: Type, zcu: *Zcu, target: std.Target, ctx: Context) [8 return memory_class; }, .optional => { - if (ty.isPtrLikeOptional(zcu)) { - result[0] = .integer; - return result; + if (ty.optionalReprIsPayload(zcu)) { + return classifySystemV(ty.optionalChild(zcu), zcu, target, ctx); } return memory_class; }, diff --git a/src/arch/x86_64/bits.zig b/src/arch/x86_64/bits.zig index 95397e8064..ac5181cb3e 100644 --- a/src/arch/x86_64/bits.zig +++ b/src/arch/x86_64/bits.zig @@ -547,7 +547,39 @@ pub const Memory = struct { } }; - pub const Scale = enum(u2) { @"1", @"2", @"4", @"8" }; + pub const Scale = enum(u2) { + @"1", + @"2", + @"4", + @"8", + + pub fn fromFactor(factor: u4) Scale { + return switch (factor) { + else => unreachable, + 1 => .@"1", + 2 => .@"2", + 4 => .@"4", + 8 => .@"8", + }; + } + + pub fn toFactor(scale: Scale) u4 { + return switch (scale) { + .@"1" => 1, + .@"2" => 2, + .@"4" => 4, + .@"8" => 8, + }; + } + + pub fn fromLog2(log2: u2) Scale { + return @enumFromInt(log2); + } + + pub fn toLog2(scale: Scale) u2 { + return @intFromEnum(scale); + } + }; }; pub const Immediate = union(enum) { diff --git a/src/print_air.zig b/src/print_air.zig index 280d05edfa..d99be7770d 100644 --- a/src/print_air.zig +++ b/src/print_air.zig @@ -96,8 +96,8 @@ const Writer = struct { fn writeInst(w: *Writer, s: anytype, inst: Air.Inst.Index) @TypeOf(s).Error!void { const tag = w.air.instructions.items(.tag)[@intFromEnum(inst)]; try s.writeByteNTimes(' ', w.indent); - try s.print("%{d}{c}= {s}(", .{ - @intFromEnum(inst), + try s.print("{}{c}= {s}(", .{ + inst, @as(u8, if (if (w.liveness) |liveness| liveness.isUnused(inst) else false) '!' else ' '), @tagName(tag), }); @@ -409,7 +409,7 @@ const Writer = struct { try s.writeAll("}"); for (liveness_block.deaths) |operand| { - try s.print(" %{d}!", .{@intFromEnum(operand)}); + try s.print(" {}!", .{operand}); } } @@ -728,7 +728,7 @@ const Writer = struct { try s.writeByteNTimes(' ', w.indent); for (liveness_condbr.else_deaths, 0..) |operand, i| { if (i != 0) try s.writeAll(" "); - try s.print("%{d}!", .{@intFromEnum(operand)}); + try s.print("{}!", .{operand}); } try s.writeAll("\n"); } @@ -739,7 +739,7 @@ const Writer = struct { try s.writeAll("}"); for (liveness_condbr.then_deaths) |operand| { - try s.print(" %{d}!", .{@intFromEnum(operand)}); + try s.print(" {}!", .{operand}); } } @@ -765,7 +765,7 @@ const Writer = struct { try s.writeByteNTimes(' ', w.indent); for (liveness_condbr.else_deaths, 0..) |operand, i| { if (i != 0) try s.writeAll(" "); - try s.print("%{d}!", .{@intFromEnum(operand)}); + try s.print("{}!", .{operand}); } try s.writeAll("\n"); } @@ -776,7 +776,7 @@ const Writer = struct { try s.writeAll("}"); for (liveness_condbr.then_deaths) |operand| { - try s.print(" %{d}!", .{@intFromEnum(operand)}); + try s.print(" {}!", .{operand}); } } @@ -807,7 +807,7 @@ const Writer = struct { try s.writeByteNTimes(' ', w.indent); for (liveness_condbr.then_deaths, 0..) |operand, i| { if (i != 0) try s.writeAll(" "); - try s.print("%{d}!", .{@intFromEnum(operand)}); + try s.print("{}!", .{operand}); } try s.writeAll("\n"); } @@ -827,7 +827,7 @@ const Writer = struct { try s.writeByteNTimes(' ', w.indent); for (liveness_condbr.else_deaths, 0..) |operand, i| { if (i != 0) try s.writeAll(" "); - try s.print("%{d}!", .{@intFromEnum(operand)}); + try s.print("{}!", .{operand}); } try s.writeAll("\n"); } @@ -884,7 +884,7 @@ const Writer = struct { try s.writeByteNTimes(' ', w.indent); for (deaths, 0..) |operand, i| { if (i != 0) try s.writeAll(" "); - try s.print("%{d}!", .{@intFromEnum(operand)}); + try s.print("{}!", .{operand}); } try s.writeAll("\n"); } @@ -910,7 +910,7 @@ const Writer = struct { try s.writeByteNTimes(' ', w.indent); for (deaths, 0..) |operand, i| { if (i != 0) try s.writeAll(" "); - try s.print("%{d}!", .{@intFromEnum(operand)}); + try s.print("{}!", .{operand}); } try s.writeAll("\n"); } @@ -994,7 +994,7 @@ const Writer = struct { dies: bool, ) @TypeOf(s).Error!void { _ = w; - try s.print("%{d}", .{@intFromEnum(inst)}); + try s.print("{}", .{inst}); if (dies) try s.writeByte('!'); } diff --git a/tools/lldb_pretty_printers.py b/tools/lldb_pretty_printers.py index fd7e9fa08a..d3ccf738e8 100644 --- a/tools/lldb_pretty_printers.py +++ b/tools/lldb_pretty_printers.py @@ -383,7 +383,7 @@ def InstRef_SummaryProvider(value, _=None): 'InternPool.Index(%d)' % value.unsigned if value.unsigned < 0x80000000 else 'instructions[%d]' % (value.unsigned - 0x80000000)) def InstIndex_SummaryProvider(value, _=None): - return 'instructions[%d]' % value.unsigned + return 'instructions[%d]' % value.unsigned if value.unsigned < 0x80000000 else 'temps[%d]' % (value.unsigned - 0x80000000) class zig_DeclIndex_SynthProvider: def __init__(self, value, _=None): self.value = value