diff --git a/src/Air.zig b/src/Air.zig index 1b394ca1c1..db5307f459 100644 --- a/src/Air.zig +++ b/src/Air.zig @@ -874,10 +874,6 @@ pub const Inst = struct { /// Uses the `ty_pl` field. save_err_return_trace_index, - /// Store an element to a vector pointer at an index. - /// Uses the `vector_store_elem` field. - vector_store_elem, - /// Compute a pointer to a `Nav` at runtime, always one of: /// /// * `threadlocal var` @@ -1220,11 +1216,6 @@ pub const Inst = struct { operand: Ref, operation: std.builtin.ReduceOp, }, - vector_store_elem: struct { - vector_ptr: Ref, - // Index into a different array. - payload: u32, - }, ty_nav: struct { ty: InternPool.Index, nav: InternPool.Nav.Index, @@ -1689,7 +1680,6 @@ pub fn typeOfIndex(air: *const Air, inst: Air.Inst.Index, ip: *const InternPool) .set_union_tag, .prefetch, .set_err_return_trace, - .vector_store_elem, .c_va_end, => return .void, @@ -1857,7 +1847,6 @@ pub fn mustLower(air: Air, inst: Air.Inst.Index, ip: *const InternPool) bool { .prefetch, .wasm_memory_grow, .set_err_return_trace, - .vector_store_elem, .c_va_arg, .c_va_copy, .c_va_end, diff --git a/src/Air/Liveness.zig b/src/Air/Liveness.zig index 58169730e8..ea170d0893 100644 --- a/src/Air/Liveness.zig +++ b/src/Air/Liveness.zig @@ -463,12 +463,6 @@ fn analyzeInst( return analyzeOperands(a, pass, data, inst, .{ o.lhs, o.rhs, .none }); }, - .vector_store_elem => { - const o = inst_datas[@intFromEnum(inst)].vector_store_elem; - const extra = a.air.extraData(Air.Bin, o.payload).data; - return analyzeOperands(a, pass, data, inst, .{ o.vector_ptr, extra.lhs, extra.rhs }); - }, - .arg, .alloc, .ret_ptr, diff --git a/src/Air/Liveness/Verify.zig b/src/Air/Liveness/Verify.zig index a1cce26a64..2f50937bbe 100644 --- a/src/Air/Liveness/Verify.zig +++ b/src/Air/Liveness/Verify.zig @@ -322,11 +322,6 @@ fn verifyBody(self: *Verify, body: []const Air.Inst.Index) Error!void { const extra = self.air.extraData(Air.Bin, pl_op.payload).data; try self.verifyInstOperands(inst, .{ extra.lhs, extra.rhs, pl_op.operand }); }, - .vector_store_elem => { - const vector_store_elem = data[@intFromEnum(inst)].vector_store_elem; - const extra = self.air.extraData(Air.Bin, vector_store_elem.payload).data; - try self.verifyInstOperands(inst, .{ vector_store_elem.vector_ptr, extra.lhs, extra.rhs }); - }, .cmpxchg_strong, .cmpxchg_weak, => { diff --git a/src/Air/print.zig b/src/Air/print.zig index 73cf2ed9b3..4b44af3206 100644 --- a/src/Air/print.zig +++ b/src/Air/print.zig @@ -330,7 +330,6 @@ const Writer = struct { .shuffle_two => try w.writeShuffleTwo(s, inst), .reduce, .reduce_optimized => try w.writeReduce(s, inst), .cmp_vector, .cmp_vector_optimized => try w.writeCmpVector(s, inst), - .vector_store_elem => try w.writeVectorStoreElem(s, inst), .runtime_nav_ptr => try w.writeRuntimeNavPtr(s, inst), .work_item_id, @@ -576,17 +575,6 @@ const Writer = struct { try w.writeOperand(s, inst, 1, extra.rhs); } - fn writeVectorStoreElem(w: *Writer, s: *std.Io.Writer, inst: Air.Inst.Index) Error!void { - const data = w.air.instructions.items(.data)[@intFromEnum(inst)].vector_store_elem; - const extra = w.air.extraData(Air.VectorCmp, data.payload).data; - - try w.writeOperand(s, inst, 0, data.vector_ptr); - try s.writeAll(", "); - try w.writeOperand(s, inst, 1, extra.lhs); - try s.writeAll(", "); - try w.writeOperand(s, inst, 2, extra.rhs); - } - fn writeRuntimeNavPtr(w: *Writer, s: *std.Io.Writer, inst: Air.Inst.Index) Error!void { const ip = &w.pt.zcu.intern_pool; const ty_nav = w.air.instructions.items(.data)[@intFromEnum(inst)].ty_nav; diff --git a/src/Air/types_resolved.zig b/src/Air/types_resolved.zig index 44669b82df..cac981cb00 100644 --- a/src/Air/types_resolved.zig +++ b/src/Air/types_resolved.zig @@ -316,13 +316,6 @@ fn checkBody(air: Air, body: []const Air.Inst.Index, zcu: *Zcu) bool { if (!checkRef(data.prefetch.ptr, zcu)) return false; }, - .vector_store_elem => { - const bin = air.extraData(Air.Bin, data.vector_store_elem.payload).data; - if (!checkRef(data.vector_store_elem.vector_ptr, zcu)) return false; - if (!checkRef(bin.lhs, zcu)) return false; - if (!checkRef(bin.rhs, zcu)) return false; - }, - .runtime_nav_ptr => { if (!checkType(.fromInterned(data.ty_nav.ty), zcu)) return false; }, diff --git a/src/InternPool.zig b/src/InternPool.zig index 3bee7a3f94..e53caf382f 100644 --- a/src/InternPool.zig +++ b/src/InternPool.zig @@ -2104,7 +2104,6 @@ pub const Key = union(enum) { pub const VectorIndex = enum(u16) { none = std.math.maxInt(u16), - runtime = std.math.maxInt(u16) - 1, _, }; @@ -3739,10 +3738,8 @@ pub const LoadedStructType = struct { return s.field_inits.get(ip)[i]; } - /// Returns `none` in the case the struct is a tuple. - pub fn fieldName(s: LoadedStructType, ip: *const InternPool, i: usize) OptionalNullTerminatedString { - if (s.field_names.len == 0) return .none; - return s.field_names.get(ip)[i].toOptional(); + pub fn fieldName(s: LoadedStructType, ip: *const InternPool, i: usize) NullTerminatedString { + return s.field_names.get(ip)[i]; } pub fn fieldIsComptime(s: LoadedStructType, ip: *const InternPool, i: usize) bool { diff --git a/src/Sema.zig b/src/Sema.zig index af342a0d04..8b5c7c5de2 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -15919,26 +15919,27 @@ fn zirOverflowArithmetic( }, .mul_with_overflow => { // If either of the arguments is zero, the result is zero and no overflow occured. - // If either of the arguments is one, the result is the other and no overflow occured. - // Otherwise, if either of the arguments is undefined, both results are undefined. - const scalar_one = try pt.intValue(dest_ty.scalarType(zcu), 1); if (maybe_lhs_val) |lhs_val| { - if (!lhs_val.isUndef(zcu)) { - if (try lhs_val.compareAllWithZeroSema(.eq, pt)) { - break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = lhs }; - } else if (try sema.compareAll(lhs_val, .eq, try sema.splat(dest_ty, scalar_one), dest_ty)) { - break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = rhs }; - } + if (!lhs_val.isUndef(zcu) and try lhs_val.compareAllWithZeroSema(.eq, pt)) { + break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = lhs }; } } - if (maybe_rhs_val) |rhs_val| { - if (!rhs_val.isUndef(zcu)) { - if (try rhs_val.compareAllWithZeroSema(.eq, pt)) { - break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = rhs }; - } else if (try sema.compareAll(rhs_val, .eq, try sema.splat(dest_ty, scalar_one), dest_ty)) { - break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = lhs }; - } + if (!rhs_val.isUndef(zcu) and try rhs_val.compareAllWithZeroSema(.eq, pt)) { + break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = rhs }; + } + } + // If either of the arguments is one, the result is the other and no overflow occured. + const scalar_one = try pt.intValue(dest_ty.scalarType(zcu), 1); + const vec_one = try sema.splat(dest_ty, scalar_one); + if (maybe_lhs_val) |lhs_val| { + if (!lhs_val.isUndef(zcu) and try sema.compareAll(lhs_val, .eq, vec_one, dest_ty)) { + break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = rhs }; + } + } + if (maybe_rhs_val) |rhs_val| { + if (!rhs_val.isUndef(zcu) and try sema.compareAll(rhs_val, .eq, vec_one, dest_ty)) { + break :result .{ .overflow_bit = try sema.splat(overflow_ty, .zero_u1), .inst = lhs }; } } @@ -15947,7 +15948,6 @@ fn zirOverflowArithmetic( if (lhs_val.isUndef(zcu) or rhs_val.isUndef(zcu)) { break :result .{ .overflow_bit = .undef, .wrapped = .undef }; } - const result = try arith.mulWithOverflow(sema, dest_ty, lhs_val, rhs_val); break :result .{ .overflow_bit = result.overflow_bit, .wrapped = result.wrapped_result }; } @@ -17751,10 +17751,7 @@ fn zirTypeInfo(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!Ai try ty.resolveStructFieldInits(pt); for (struct_field_vals, 0..) |*field_val, field_index| { - const field_name = if (struct_type.fieldName(ip, field_index).unwrap()) |field_name| - field_name - else - try ip.getOrPutStringFmt(gpa, pt.tid, "{d}", .{field_index}, .no_embedded_nulls); + const field_name = struct_type.fieldName(ip, field_index); const field_name_len = field_name.length(ip); const field_ty: Type = .fromInterned(struct_type.field_types.get(ip)[field_index]); const field_init = struct_type.fieldInit(ip, field_index); @@ -28347,6 +28344,10 @@ fn elemPtrArray( break :o index; } else null; + if (offset == null and array_ty.zigTypeTag(zcu) == .vector) { + return sema.fail(block, elem_index_src, "vector index not comptime known", .{}); + } + const elem_ptr_ty = try array_ptr_ty.elemPtrType(offset, pt); if (maybe_undef_array_ptr_val) |array_ptr_val| { @@ -28364,10 +28365,6 @@ fn elemPtrArray( try sema.validateRuntimeValue(block, array_ptr_src, array_ptr); } - if (offset == null and array_ty.zigTypeTag(zcu) == .vector) { - return sema.fail(block, elem_index_src, "vector index not comptime known", .{}); - } - // Runtime check is only needed if unable to comptime check. if (oob_safety and block.wantSafety() and offset == null) { const len_inst = try pt.intRef(.usize, array_len); @@ -30399,22 +30396,6 @@ fn storePtr2( const is_ret = air_tag == .ret_ptr; - // Detect if we are storing an array operand to a bitcasted vector pointer. - // If so, we instead reach through the bitcasted pointer to the vector pointer, - // bitcast the array operand to a vector, and then lower this as a store of - // a vector value to a vector pointer. This generally results in better code, - // as well as working around an LLVM bug: - // https://github.com/ziglang/zig/issues/11154 - if (sema.obtainBitCastedVectorPtr(ptr)) |vector_ptr| { - const vector_ty = sema.typeOf(vector_ptr).childType(zcu); - const vector = sema.coerceExtra(block, vector_ty, uncasted_operand, operand_src, .{ .is_ret = is_ret }) catch |err| switch (err) { - error.NotCoercible => unreachable, - else => |e| return e, - }; - try sema.storePtr2(block, src, vector_ptr, ptr_src, vector, operand_src, .store); - return; - } - const operand = sema.coerceExtra(block, elem_ty, uncasted_operand, operand_src, .{ .is_ret = is_ret }) catch |err| switch (err) { error.NotCoercible => unreachable, else => |e| return e, @@ -30447,29 +30428,6 @@ fn storePtr2( try sema.requireRuntimeBlock(block, src, runtime_src); - if (ptr_ty.ptrInfo(zcu).flags.vector_index == .runtime) { - const ptr_inst = ptr.toIndex().?; - const air_tags = sema.air_instructions.items(.tag); - if (air_tags[@intFromEnum(ptr_inst)] == .ptr_elem_ptr) { - const ty_pl = sema.air_instructions.items(.data)[@intFromEnum(ptr_inst)].ty_pl; - const bin_op = sema.getTmpAir().extraData(Air.Bin, ty_pl.payload).data; - _ = try block.addInst(.{ - .tag = .vector_store_elem, - .data = .{ .vector_store_elem = .{ - .vector_ptr = bin_op.lhs, - .payload = try block.sema.addExtra(Air.Bin{ - .lhs = bin_op.rhs, - .rhs = operand, - }), - } }, - }); - return; - } - return sema.fail(block, ptr_src, "unable to determine vector element index of type '{f}'", .{ - ptr_ty.fmt(pt), - }); - } - const store_inst = if (is_ret) try block.addBinOp(.store, ptr, operand) else @@ -30569,37 +30527,6 @@ fn markMaybeComptimeAllocRuntime(sema: *Sema, block: *Block, alloc_inst: Air.Ins } } -/// Traverse an arbitrary number of bitcasted pointers and return the underyling vector -/// pointer. Only if the final element type matches the vector element type, and the -/// lengths match. -fn obtainBitCastedVectorPtr(sema: *Sema, ptr: Air.Inst.Ref) ?Air.Inst.Ref { - const pt = sema.pt; - const zcu = pt.zcu; - const array_ty = sema.typeOf(ptr).childType(zcu); - if (array_ty.zigTypeTag(zcu) != .array) return null; - var ptr_ref = ptr; - var ptr_inst = ptr_ref.toIndex() orelse return null; - const air_datas = sema.air_instructions.items(.data); - const air_tags = sema.air_instructions.items(.tag); - const vector_ty = while (air_tags[@intFromEnum(ptr_inst)] == .bitcast) { - ptr_ref = air_datas[@intFromEnum(ptr_inst)].ty_op.operand; - if (!sema.isKnownZigType(ptr_ref, .pointer)) return null; - const child_ty = sema.typeOf(ptr_ref).childType(zcu); - if (child_ty.zigTypeTag(zcu) == .vector) break child_ty; - ptr_inst = ptr_ref.toIndex() orelse return null; - } else return null; - - // We have a pointer-to-array and a pointer-to-vector. If the elements and - // lengths match, return the result. - if (array_ty.childType(zcu).eql(vector_ty.childType(zcu), zcu) and - array_ty.arrayLen(zcu) == vector_ty.vectorLen(zcu)) - { - return ptr_ref; - } else { - return null; - } -} - /// Call when you have Value objects rather than Air instructions, and you want to /// assert the store must be done at comptime. fn storePtrVal( @@ -35579,8 +35506,13 @@ fn structFieldInits( const default_val = try sema.resolveConstValue(&block_scope, init_src, coerced, null); if (default_val.canMutateComptimeVarState(zcu)) { - const field_name = struct_type.fieldName(ip, field_i).unwrap().?; - return sema.failWithContainsReferenceToComptimeVar(&block_scope, init_src, field_name, "field default value", default_val); + return sema.failWithContainsReferenceToComptimeVar( + &block_scope, + init_src, + struct_type.fieldName(ip, field_i), + "field default value", + default_val, + ); } struct_type.field_inits.get(ip)[field_i] = default_val.toIntern(); } diff --git a/src/Sema/comptime_ptr_access.zig b/src/Sema/comptime_ptr_access.zig index 9441f8cf72..4e101ecd0f 100644 --- a/src/Sema/comptime_ptr_access.zig +++ b/src/Sema/comptime_ptr_access.zig @@ -24,7 +24,6 @@ pub fn loadComptimePtr(sema: *Sema, block: *Block, src: LazySrcLoc, ptr: Value) const child_bits = Type.fromInterned(ptr_info.child).bitSize(zcu); const bit_offset = ptr_info.packed_offset.bit_offset + switch (ptr_info.flags.vector_index) { .none => 0, - .runtime => return .runtime_load, else => |idx| switch (pt.zcu.getTarget().cpu.arch.endian()) { .little => child_bits * @intFromEnum(idx), .big => host_bits - child_bits * (@intFromEnum(idx) + 1), // element order reversed on big endian @@ -81,7 +80,6 @@ pub fn storeComptimePtr( }; const bit_offset = ptr_info.packed_offset.bit_offset + switch (ptr_info.flags.vector_index) { .none => 0, - .runtime => return .runtime_store, else => |idx| switch (zcu.getTarget().cpu.arch.endian()) { .little => Type.fromInterned(ptr_info.child).bitSize(zcu) * @intFromEnum(idx), .big => host_bits - Type.fromInterned(ptr_info.child).bitSize(zcu) * (@intFromEnum(idx) + 1), // element order reversed on big endian diff --git a/src/Type.zig b/src/Type.zig index f3f5c99491..d6e38420cb 100644 --- a/src/Type.zig +++ b/src/Type.zig @@ -198,9 +198,7 @@ pub fn print(ty: Type, writer: *std.Io.Writer, pt: Zcu.PerThread) std.Io.Writer. info.packed_offset.bit_offset, info.packed_offset.host_size, }); } - if (info.flags.vector_index == .runtime) { - try writer.writeAll(":?"); - } else if (info.flags.vector_index != .none) { + if (info.flags.vector_index != .none) { try writer.print(":{d}", .{@intFromEnum(info.flags.vector_index)}); } try writer.writeAll(") "); @@ -3113,7 +3111,7 @@ pub fn enumTagFieldIndex(ty: Type, enum_tag: Value, zcu: *const Zcu) ?u32 { pub fn structFieldName(ty: Type, index: usize, zcu: *const Zcu) InternPool.OptionalNullTerminatedString { const ip = &zcu.intern_pool; return switch (ip.indexToKey(ty.toIntern())) { - .struct_type => ip.loadStructType(ty.toIntern()).fieldName(ip, index), + .struct_type => ip.loadStructType(ty.toIntern()).fieldName(ip, index).toOptional(), .tuple_type => .none, else => unreachable, }; @@ -3985,7 +3983,7 @@ pub fn elemPtrType(ptr_ty: Type, offset: ?usize, pt: Zcu.PerThread) !Type { break :blk .{ .host_size = @intCast(parent_ty.arrayLen(zcu)), .alignment = parent_ty.abiAlignment(zcu), - .vector_index = if (offset) |some| @enumFromInt(some) else .runtime, + .vector_index = @enumFromInt(offset.?), }; } else .{}; diff --git a/src/Value.zig b/src/Value.zig index b72ee2f789..9ced6f1074 100644 --- a/src/Value.zig +++ b/src/Value.zig @@ -574,166 +574,37 @@ pub fn writeToPackedMemory( } } -/// Load a Value from the contents of `buffer`. +/// Load a Value from the contents of `buffer`, where `ty` is an unsigned integer type. /// /// Asserts that buffer.len >= ty.abiSize(). The buffer is allowed to extend past /// the end of the value in memory. -pub fn readFromMemory( +pub fn readUintFromMemory( ty: Type, pt: Zcu.PerThread, buffer: []const u8, arena: Allocator, -) error{ - IllDefinedMemoryLayout, - Unimplemented, - OutOfMemory, -}!Value { +) Allocator.Error!Value { const zcu = pt.zcu; - const ip = &zcu.intern_pool; - const target = zcu.getTarget(); - const endian = target.cpu.arch.endian(); - switch (ty.zigTypeTag(zcu)) { - .void => return Value.void, - .bool => { - if (buffer[0] == 0) { - return Value.false; - } else { - return Value.true; - } - }, - .int, .@"enum" => |ty_tag| { - const int_ty = switch (ty_tag) { - .int => ty, - .@"enum" => ty.intTagType(zcu), - else => unreachable, - }; - const int_info = int_ty.intInfo(zcu); - const bits = int_info.bits; - const byte_count: u16 = @intCast((@as(u17, bits) + 7) / 8); - if (bits == 0 or buffer.len == 0) return zcu.getCoerced(try zcu.intValue(int_ty, 0), ty); + const endian = zcu.getTarget().cpu.arch.endian(); - if (bits <= 64) switch (int_info.signedness) { // Fast path for integers <= u64 - .signed => { - const val = std.mem.readVarInt(i64, buffer[0..byte_count], endian); - const result = (val << @as(u6, @intCast(64 - bits))) >> @as(u6, @intCast(64 - bits)); - return zcu.getCoerced(try zcu.intValue(int_ty, result), ty); - }, - .unsigned => { - const val = std.mem.readVarInt(u64, buffer[0..byte_count], endian); - const result = (val << @as(u6, @intCast(64 - bits))) >> @as(u6, @intCast(64 - bits)); - return zcu.getCoerced(try zcu.intValue(int_ty, result), ty); - }, - } else { // Slow path, we have to construct a big-int - const Limb = std.math.big.Limb; - const limb_count = (byte_count + @sizeOf(Limb) - 1) / @sizeOf(Limb); - const limbs_buffer = try arena.alloc(Limb, limb_count); + assert(ty.isUnsignedInt(zcu)); + const bits = ty.intInfo(zcu).bits; + const byte_count: u16 = @intCast((@as(u17, bits) + 7) / 8); - var bigint = BigIntMutable.init(limbs_buffer, 0); - bigint.readTwosComplement(buffer[0..byte_count], bits, endian, int_info.signedness); - return zcu.getCoerced(try zcu.intValue_big(int_ty, bigint.toConst()), ty); - } - }, - .float => return Value.fromInterned(try pt.intern(.{ .float = .{ - .ty = ty.toIntern(), - .storage = switch (ty.floatBits(target)) { - 16 => .{ .f16 = @bitCast(std.mem.readInt(u16, buffer[0..2], endian)) }, - 32 => .{ .f32 = @bitCast(std.mem.readInt(u32, buffer[0..4], endian)) }, - 64 => .{ .f64 = @bitCast(std.mem.readInt(u64, buffer[0..8], endian)) }, - 80 => .{ .f80 = @bitCast(std.mem.readInt(u80, buffer[0..10], endian)) }, - 128 => .{ .f128 = @bitCast(std.mem.readInt(u128, buffer[0..16], endian)) }, - else => unreachable, - }, - } })), - .array => { - const elem_ty = ty.childType(zcu); - const elem_size = elem_ty.abiSize(zcu); - const elems = try arena.alloc(InternPool.Index, @intCast(ty.arrayLen(zcu))); - var offset: usize = 0; - for (elems) |*elem| { - elem.* = (try readFromMemory(elem_ty, zcu, buffer[offset..], arena)).toIntern(); - offset += @intCast(elem_size); - } - return pt.aggregateValue(ty, elems); - }, - .vector => { - // We use byte_count instead of abi_size here, so that any padding bytes - // follow the data bytes, on both big- and little-endian systems. - const byte_count = (@as(usize, @intCast(ty.bitSize(zcu))) + 7) / 8; - return readFromPackedMemory(ty, zcu, buffer[0..byte_count], 0, arena); - }, - .@"struct" => { - const struct_type = zcu.typeToStruct(ty).?; - switch (struct_type.layout) { - .auto => unreachable, // Sema is supposed to have emitted a compile error already - .@"extern" => { - const field_types = struct_type.field_types; - const field_vals = try arena.alloc(InternPool.Index, field_types.len); - for (field_vals, 0..) |*field_val, i| { - const field_ty = Type.fromInterned(field_types.get(ip)[i]); - const off: usize = @intCast(ty.structFieldOffset(i, zcu)); - const sz: usize = @intCast(field_ty.abiSize(zcu)); - field_val.* = (try readFromMemory(field_ty, zcu, buffer[off..(off + sz)], arena)).toIntern(); - } - return pt.aggregateValue(ty, field_vals); - }, - .@"packed" => { - const byte_count = (@as(usize, @intCast(ty.bitSize(zcu))) + 7) / 8; - return readFromPackedMemory(ty, zcu, buffer[0..byte_count], 0, arena); - }, - } - }, - .error_set => { - const bits = zcu.errorSetBits(); - const byte_count: u16 = @intCast((@as(u17, bits) + 7) / 8); - const int = std.mem.readVarInt(u64, buffer[0..byte_count], endian); - const index = (int << @as(u6, @intCast(64 - bits))) >> @as(u6, @intCast(64 - bits)); - const name = zcu.global_error_set.keys()[@intCast(index)]; + assert(buffer.len >= byte_count); - return Value.fromInterned(try pt.intern(.{ .err = .{ - .ty = ty.toIntern(), - .name = name, - } })); - }, - .@"union" => switch (ty.containerLayout(zcu)) { - .auto => return error.IllDefinedMemoryLayout, - .@"extern" => { - const union_size = ty.abiSize(zcu); - const array_ty = try zcu.arrayType(.{ .len = union_size, .child = .u8_type }); - const val = (try readFromMemory(array_ty, zcu, buffer, arena)).toIntern(); - return Value.fromInterned(try pt.internUnion(.{ - .ty = ty.toIntern(), - .tag = .none, - .val = val, - })); - }, - .@"packed" => { - const byte_count = (@as(usize, @intCast(ty.bitSize(zcu))) + 7) / 8; - return readFromPackedMemory(ty, zcu, buffer[0..byte_count], 0, arena); - }, - }, - .pointer => { - assert(!ty.isSlice(zcu)); // No well defined layout. - const int_val = try readFromMemory(Type.usize, zcu, buffer, arena); - return Value.fromInterned(try pt.intern(.{ .ptr = .{ - .ty = ty.toIntern(), - .base_addr = .int, - .byte_offset = int_val.toUnsignedInt(zcu), - } })); - }, - .optional => { - assert(ty.isPtrLikeOptional(zcu)); - const child_ty = ty.optionalChild(zcu); - const child_val = try readFromMemory(child_ty, zcu, buffer, arena); - return Value.fromInterned(try pt.intern(.{ .opt = .{ - .ty = ty.toIntern(), - .val = switch (child_val.orderAgainstZero(pt)) { - .lt => unreachable, - .eq => .none, - .gt => child_val.toIntern(), - }, - } })); - }, - else => return error.Unimplemented, + if (bits <= 64) { + const val = std.mem.readVarInt(u64, buffer[0..byte_count], endian); + const result = (val << @as(u6, @intCast(64 - bits))) >> @as(u6, @intCast(64 - bits)); + return pt.intValue(ty, result); + } else { + const Limb = std.math.big.Limb; + const limb_count = (byte_count + @sizeOf(Limb) - 1) / @sizeOf(Limb); + const limbs_buffer = try arena.alloc(Limb, limb_count); + + var bigint: BigIntMutable = .init(limbs_buffer, 0); + bigint.readTwosComplement(buffer[0..byte_count], bits, endian, .unsigned); + return pt.intValue_big(ty, bigint.toConst()); } } diff --git a/src/Zcu/PerThread.zig b/src/Zcu/PerThread.zig index 20aaa3d3c2..41b5a32f6e 100644 --- a/src/Zcu/PerThread.zig +++ b/src/Zcu/PerThread.zig @@ -3512,7 +3512,6 @@ pub fn ptrType(pt: Zcu.PerThread, info: InternPool.Key.PtrType) Allocator.Error! canon_info.packed_offset.host_size = 0; } }, - .runtime => {}, _ => assert(@intFromEnum(info.flags.vector_index) < info.packed_offset.host_size), } @@ -3663,21 +3662,40 @@ pub fn intRef(pt: Zcu.PerThread, ty: Type, x: anytype) Allocator.Error!Air.Inst. } pub fn intValue_big(pt: Zcu.PerThread, ty: Type, x: BigIntConst) Allocator.Error!Value { - return Value.fromInterned(try pt.intern(.{ .int = .{ + if (ty.toIntern() != .comptime_int_type) { + const int_info = ty.intInfo(pt.zcu); + assert(x.fitsInTwosComp(int_info.signedness, int_info.bits)); + } + return .fromInterned(try pt.intern(.{ .int = .{ .ty = ty.toIntern(), .storage = .{ .big_int = x }, } })); } pub fn intValue_u64(pt: Zcu.PerThread, ty: Type, x: u64) Allocator.Error!Value { - return Value.fromInterned(try pt.intern(.{ .int = .{ + if (ty.toIntern() != .comptime_int_type and x != 0) { + const int_info = ty.intInfo(pt.zcu); + const unsigned_bits = int_info.bits - @intFromBool(int_info.signedness == .signed); + assert(unsigned_bits >= std.math.log2(x) + 1); + } + return .fromInterned(try pt.intern(.{ .int = .{ .ty = ty.toIntern(), .storage = .{ .u64 = x }, } })); } pub fn intValue_i64(pt: Zcu.PerThread, ty: Type, x: i64) Allocator.Error!Value { - return Value.fromInterned(try pt.intern(.{ .int = .{ + if (ty.toIntern() != .comptime_int_type and x != 0) { + const int_info = ty.intInfo(pt.zcu); + const unsigned_bits = int_info.bits - @intFromBool(int_info.signedness == .signed); + if (x > 0) { + assert(unsigned_bits >= std.math.log2(x) + 1); + } else { + assert(int_info.signedness == .signed); + assert(unsigned_bits >= std.math.log2_int_ceil(u64, @abs(x))); + } + } + return .fromInterned(try pt.intern(.{ .int = .{ .ty = ty.toIntern(), .storage = .{ .i64 = x }, } })); diff --git a/src/codegen/aarch64/Select.zig b/src/codegen/aarch64/Select.zig index 4fe798271f..36ca69e589 100644 --- a/src/codegen/aarch64/Select.zig +++ b/src/codegen/aarch64/Select.zig @@ -826,18 +826,6 @@ pub fn analyze(isel: *Select, air_body: []const Air.Inst.Index) !void { try isel.analyzeUse(un_op); - air_body_index += 1; - air_inst_index = air_body[air_body_index]; - continue :air_tag air_tags[@intFromEnum(air_inst_index)]; - }, - .vector_store_elem => { - const vector_store_elem = air_data[@intFromEnum(air_inst_index)].vector_store_elem; - const bin_op = isel.air.extraData(Air.Bin, vector_store_elem.payload).data; - - try isel.analyzeUse(vector_store_elem.vector_ptr); - try isel.analyzeUse(bin_op.lhs); - try isel.analyzeUse(bin_op.rhs); - air_body_index += 1; air_inst_index = air_body[air_body_index]; continue :air_tag air_tags[@intFromEnum(air_inst_index)]; diff --git a/src/codegen/c.zig b/src/codegen/c.zig index 7341a9fd0b..0abea3d503 100644 --- a/src/codegen/c.zig +++ b/src/codegen/c.zig @@ -37,6 +37,7 @@ pub fn legalizeFeatures(_: *const std.Target) ?*const Air.Legalize.Features { .expand_packed_load = true, .expand_packed_store = true, .expand_packed_struct_field_val = true, + .expand_packed_aggregate_init = true, }), }; } @@ -1392,114 +1393,21 @@ pub const DeclGen = struct { try w.writeByte('}'); }, .@"packed" => { - const int_info = ty.intInfo(zcu); - - const bits = Type.smallestUnsignedBits(int_info.bits - 1); - const bit_offset_ty = try pt.intType(.unsigned, bits); - - var bit_offset: u64 = 0; - var eff_num_fields: usize = 0; - - for (0..loaded_struct.field_types.len) |field_index| { - const field_ty: Type = .fromInterned(loaded_struct.field_types.get(ip)[field_index]); - if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue; - eff_num_fields += 1; - } - - if (eff_num_fields == 0) { - try w.writeByte('('); - try dg.renderUndefValue(w, ty, location); - try w.writeByte(')'); - } else if (ty.bitSize(zcu) > 64) { - // zig_or_u128(zig_or_u128(zig_shl_u128(a, a_off), zig_shl_u128(b, b_off)), zig_shl_u128(c, c_off)) - var num_or = eff_num_fields - 1; - while (num_or > 0) : (num_or -= 1) { - try w.writeAll("zig_or_"); - try dg.renderTypeForBuiltinFnName(w, ty); - try w.writeByte('('); - } - - var eff_index: usize = 0; - var needs_closing_paren = false; - for (0..loaded_struct.field_types.len) |field_index| { - const field_ty: Type = .fromInterned(loaded_struct.field_types.get(ip)[field_index]); - if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue; - - const field_val = switch (ip.indexToKey(val.toIntern()).aggregate.storage) { - .bytes => |bytes| try pt.intern(.{ .int = .{ - .ty = field_ty.toIntern(), - .storage = .{ .u64 = bytes.at(field_index, ip) }, - } }), - .elems => |elems| elems[field_index], - .repeated_elem => |elem| elem, - }; - const cast_context = IntCastContext{ .value = .{ .value = Value.fromInterned(field_val) } }; - if (bit_offset != 0) { - try w.writeAll("zig_shl_"); - try dg.renderTypeForBuiltinFnName(w, ty); - try w.writeByte('('); - try dg.renderIntCast(w, ty, cast_context, field_ty, .FunctionArgument); - try w.writeAll(", "); - try dg.renderValue(w, try pt.intValue(bit_offset_ty, bit_offset), .FunctionArgument); - try w.writeByte(')'); - } else { - try dg.renderIntCast(w, ty, cast_context, field_ty, .FunctionArgument); - } - - if (needs_closing_paren) try w.writeByte(')'); - if (eff_index != eff_num_fields - 1) try w.writeAll(", "); - - bit_offset += field_ty.bitSize(zcu); - needs_closing_paren = true; - eff_index += 1; - } - } else { - try w.writeByte('('); - // a << a_off | b << b_off | c << c_off - var empty = true; - for (0..loaded_struct.field_types.len) |field_index| { - const field_ty: Type = .fromInterned(loaded_struct.field_types.get(ip)[field_index]); - if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue; - - if (!empty) try w.writeAll(" | "); - try w.writeByte('('); - try dg.renderCType(w, ctype); - try w.writeByte(')'); - - const field_val = switch (ip.indexToKey(val.toIntern()).aggregate.storage) { - .bytes => |bytes| try pt.intern(.{ .int = .{ - .ty = field_ty.toIntern(), - .storage = .{ .u64 = bytes.at(field_index, ip) }, - } }), - .elems => |elems| elems[field_index], - .repeated_elem => |elem| elem, - }; - - const field_int_info: std.builtin.Type.Int = if (field_ty.isAbiInt(zcu)) - field_ty.intInfo(zcu) - else - .{ .signedness = .unsigned, .bits = undefined }; - switch (field_int_info.signedness) { - .signed => { - try w.writeByte('('); - try dg.renderValue(w, Value.fromInterned(field_val), .Other); - try w.writeAll(" & "); - const field_uint_ty = try pt.intType(.unsigned, field_int_info.bits); - try dg.renderValue(w, try field_uint_ty.maxIntScalar(pt, field_uint_ty), .Other); - try w.writeByte(')'); - }, - .unsigned => try dg.renderValue(w, Value.fromInterned(field_val), .Other), - } - if (bit_offset != 0) { - try w.writeAll(" << "); - try dg.renderValue(w, try pt.intValue(bit_offset_ty, bit_offset), .FunctionArgument); - } - - bit_offset += field_ty.bitSize(zcu); - empty = false; - } - try w.writeByte(')'); - } + // https://github.com/ziglang/zig/issues/24657 will eliminate most of the + // following logic, leaving only the recursive `renderValue` call. Once + // that proposal is implemented, a `packed struct` will literally be + // represented in the InternPool by its comptime-known backing integer. + var arena: std.heap.ArenaAllocator = .init(zcu.gpa); + defer arena.deinit(); + const backing_ty: Type = .fromInterned(loaded_struct.backingIntTypeUnordered(ip)); + const buf = try arena.allocator().alloc(u8, @intCast(ty.abiSize(zcu))); + val.writeToMemory(pt, buf) catch |err| switch (err) { + error.IllDefinedMemoryLayout => unreachable, + error.OutOfMemory => |e| return e, + error.ReinterpretDeclRef, error.Unimplemented => return dg.fail("TODO: C backend: lower packed struct value", .{}), + }; + const backing_val: Value = try .readUintFromMemory(backing_ty, pt, buf, arena.allocator()); + return dg.renderValue(w, backing_val, location); }, } }, @@ -1507,33 +1415,38 @@ pub const DeclGen = struct { }, .un => |un| { const loaded_union = ip.loadUnionType(ty.toIntern()); + if (loaded_union.flagsUnordered(ip).layout == .@"packed") { + // https://github.com/ziglang/zig/issues/24657 will eliminate most of the + // following logic, leaving only the recursive `renderValue` call. Once + // that proposal is implemented, a `packed union` will literally be + // represented in the InternPool by its comptime-known backing integer. + var arena: std.heap.ArenaAllocator = .init(zcu.gpa); + defer arena.deinit(); + const backing_ty = try ty.unionBackingType(pt); + const buf = try arena.allocator().alloc(u8, @intCast(ty.abiSize(zcu))); + val.writeToMemory(pt, buf) catch |err| switch (err) { + error.IllDefinedMemoryLayout => unreachable, + error.OutOfMemory => |e| return e, + error.ReinterpretDeclRef, error.Unimplemented => return dg.fail("TODO: C backend: lower packed union value", .{}), + }; + const backing_val: Value = try .readUintFromMemory(backing_ty, pt, buf, arena.allocator()); + return dg.renderValue(w, backing_val, location); + } if (un.tag == .none) { const backing_ty = try ty.unionBackingType(pt); - switch (loaded_union.flagsUnordered(ip).layout) { - .@"packed" => { - if (!location.isInitializer()) { - try w.writeByte('('); - try dg.renderType(w, backing_ty); - try w.writeByte(')'); - } - try dg.renderValue(w, Value.fromInterned(un.val), location); - }, - .@"extern" => { - if (location == .StaticInitializer) { - return dg.fail("TODO: C backend: implement extern union backing type rendering in static initializers", .{}); - } - - const ptr_ty = try pt.singleConstPtrType(ty); - try w.writeAll("*(("); - try dg.renderType(w, ptr_ty); - try w.writeAll(")("); - try dg.renderType(w, backing_ty); - try w.writeAll("){"); - try dg.renderValue(w, Value.fromInterned(un.val), location); - try w.writeAll("})"); - }, - else => unreachable, + assert(loaded_union.flagsUnordered(ip).layout == .@"extern"); + if (location == .StaticInitializer) { + return dg.fail("TODO: C backend: implement extern union backing type rendering in static initializers", .{}); } + + const ptr_ty = try pt.singleConstPtrType(ty); + try w.writeAll("*(("); + try dg.renderType(w, ptr_ty); + try w.writeAll(")("); + try dg.renderType(w, backing_ty); + try w.writeAll("){"); + try dg.renderValue(w, Value.fromInterned(un.val), location); + try w.writeAll("})"); } else { if (!location.isInitializer()) { try w.writeByte('('); @@ -1544,21 +1457,6 @@ pub const DeclGen = struct { const field_index = zcu.unionTagFieldIndex(loaded_union, Value.fromInterned(un.tag)).?; const field_ty: Type = .fromInterned(loaded_union.field_types.get(ip)[field_index]); const field_name = loaded_union.loadTagType(ip).names.get(ip)[field_index]; - if (loaded_union.flagsUnordered(ip).layout == .@"packed") { - if (field_ty.hasRuntimeBits(zcu)) { - if (field_ty.isPtrAtRuntime(zcu)) { - try w.writeByte('('); - try dg.renderCType(w, ctype); - try w.writeByte(')'); - } else if (field_ty.zigTypeTag(zcu) == .float) { - try w.writeByte('('); - try dg.renderCType(w, ctype); - try w.writeByte(')'); - } - try dg.renderValue(w, Value.fromInterned(un.val), location); - } else try w.writeByte('0'); - return; - } const has_tag = loaded_union.hasTag(ip); if (has_tag) try w.writeByte('{'); @@ -1745,9 +1643,11 @@ pub const DeclGen = struct { } return w.writeByte('}'); }, - .@"packed" => return w.print("{f}", .{ - try dg.fmtIntLiteralHex(try pt.undefValue(ty), .Other), - }), + .@"packed" => return dg.renderUndefValue( + w, + .fromInterned(loaded_struct.backingIntTypeUnordered(ip)), + location, + ), } }, .tuple_type => |tuple_info| { @@ -1815,9 +1715,11 @@ pub const DeclGen = struct { } if (has_tag) try w.writeByte('}'); }, - .@"packed" => return w.print("{f}", .{ - try dg.fmtIntLiteralHex(try pt.undefValue(ty), .Other), - }), + .@"packed" => return dg.renderUndefValue( + w, + try ty.unionBackingType(pt), + location, + ), } }, .error_union_type => |error_union_type| switch (ctype.info(ctype_pool)) { @@ -2445,10 +2347,7 @@ pub const DeclGen = struct { const ty = val.typeOf(zcu); return .{ .data = .{ .dg = dg, - .int_info = if (ty.zigTypeTag(zcu) == .@"union" and ty.containerLayout(zcu) == .@"packed") - .{ .signedness = .unsigned, .bits = @intCast(ty.bitSize(zcu)) } - else - ty.intInfo(zcu), + .int_info = ty.intInfo(zcu), .kind = kind, .ctype = try dg.ctypeFromType(ty, kind), .val = val, @@ -3656,7 +3555,6 @@ fn genBodyInner(f: *Function, body: []const Air.Inst.Index) Error!void { .is_named_enum_value => return f.fail("TODO: C backend: implement is_named_enum_value", .{}), .error_set_has_value => return f.fail("TODO: C backend: implement error_set_has_value", .{}), - .vector_store_elem => return f.fail("TODO: C backend: implement vector_store_elem", .{}), .runtime_nav_ptr => try airRuntimeNavPtr(f, inst), @@ -3956,6 +3854,10 @@ fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue { const ptr_info = ptr_scalar_ty.ptrInfo(zcu); const src_ty: Type = .fromInterned(ptr_info.child); + // `Air.Legalize.Feature.expand_packed_load` should ensure that the only + // bit-pointers we see here are vector element pointers. + assert(ptr_info.packed_offset.host_size == 0 or ptr_info.flags.vector_index != .none); + if (!src_ty.hasRuntimeBitsIgnoreComptime(zcu)) { try reap(f, inst, &.{ty_op.operand}); return .none; @@ -3987,40 +3889,6 @@ fn airLoad(f: *Function, inst: Air.Inst.Index) !CValue { try w.writeAll(", sizeof("); try f.renderType(w, src_ty); try w.writeAll("))"); - } else if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) { - const host_bits: u16 = ptr_info.packed_offset.host_size * 8; - const host_ty = try pt.intType(.unsigned, host_bits); - - const bit_offset_ty = try pt.intType(.unsigned, Type.smallestUnsignedBits(host_bits - 1)); - const bit_offset_val = try pt.intValue(bit_offset_ty, ptr_info.packed_offset.bit_offset); - - const field_ty = try pt.intType(.unsigned, @as(u16, @intCast(src_ty.bitSize(zcu)))); - - try f.writeCValue(w, local, .Other); - try v.elem(f, w); - try w.writeAll(" = ("); - try f.renderType(w, src_ty); - try w.writeAll(")zig_wrap_"); - try f.object.dg.renderTypeForBuiltinFnName(w, field_ty); - try w.writeAll("(("); - try f.renderType(w, field_ty); - try w.writeByte(')'); - const cant_cast = host_ty.isInt(zcu) and host_ty.bitSize(zcu) > 64; - if (cant_cast) { - if (field_ty.bitSize(zcu) > 64) return f.fail("TODO: C backend: implement casting between types > 64 bits", .{}); - try w.writeAll("zig_lo_"); - try f.object.dg.renderTypeForBuiltinFnName(w, host_ty); - try w.writeByte('('); - } - try w.writeAll("zig_shr_"); - try f.object.dg.renderTypeForBuiltinFnName(w, host_ty); - try w.writeByte('('); - try f.writeCValueDeref(w, operand); - try v.elem(f, w); - try w.print(", {f})", .{try f.fmtIntLiteralDec(bit_offset_val)}); - if (cant_cast) try w.writeByte(')'); - try f.object.dg.renderBuiltinInfo(w, field_ty, .bits); - try w.writeByte(')'); } else { try f.writeCValue(w, local, .Other); try v.elem(f, w); @@ -4213,6 +4081,10 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue { const ptr_scalar_ty = ptr_ty.scalarType(zcu); const ptr_info = ptr_scalar_ty.ptrInfo(zcu); + // `Air.Legalize.Feature.expand_packed_store` should ensure that the only + // bit-pointers we see here are vector element pointers. + assert(ptr_info.packed_offset.host_size == 0 or ptr_info.flags.vector_index != .none); + const ptr_val = try f.resolveInst(bin_op.lhs); const src_ty = f.typeOf(bin_op.rhs); @@ -4277,66 +4149,6 @@ fn airStore(f: *Function, inst: Air.Inst.Index, safety: bool) !CValue { try w.writeByte(';'); try f.object.newline(); try v.end(f, inst, w); - } else if (ptr_info.packed_offset.host_size > 0 and ptr_info.flags.vector_index == .none) { - const host_bits = ptr_info.packed_offset.host_size * 8; - const host_ty = try pt.intType(.unsigned, host_bits); - - const bit_offset_ty = try pt.intType(.unsigned, Type.smallestUnsignedBits(host_bits - 1)); - const bit_offset_val = try pt.intValue(bit_offset_ty, ptr_info.packed_offset.bit_offset); - - const src_bits = src_ty.bitSize(zcu); - - const ExpectedContents = [BigInt.Managed.default_capacity]BigIntLimb; - var stack align(@alignOf(ExpectedContents)) = - std.heap.stackFallback(@sizeOf(ExpectedContents), f.object.dg.gpa); - - var mask = try BigInt.Managed.initCapacity(stack.get(), BigInt.calcTwosCompLimbCount(host_bits)); - defer mask.deinit(); - - try mask.setTwosCompIntLimit(.max, .unsigned, @intCast(src_bits)); - try mask.shiftLeft(&mask, ptr_info.packed_offset.bit_offset); - try mask.bitNotWrap(&mask, .unsigned, host_bits); - - const mask_val = try pt.intValue_big(host_ty, mask.toConst()); - - const v = try Vectorize.start(f, inst, w, ptr_ty); - const a = try Assignment.start(f, w, src_scalar_ctype); - try f.writeCValueDeref(w, ptr_val); - try v.elem(f, w); - try a.assign(f, w); - try w.writeAll("zig_or_"); - try f.object.dg.renderTypeForBuiltinFnName(w, host_ty); - try w.writeAll("(zig_and_"); - try f.object.dg.renderTypeForBuiltinFnName(w, host_ty); - try w.writeByte('('); - try f.writeCValueDeref(w, ptr_val); - try v.elem(f, w); - try w.print(", {f}), zig_shl_", .{try f.fmtIntLiteralHex(mask_val)}); - try f.object.dg.renderTypeForBuiltinFnName(w, host_ty); - try w.writeByte('('); - const cant_cast = host_ty.isInt(zcu) and host_ty.bitSize(zcu) > 64; - if (cant_cast) { - if (src_ty.bitSize(zcu) > 64) return f.fail("TODO: C backend: implement casting between types > 64 bits", .{}); - try w.writeAll("zig_make_"); - try f.object.dg.renderTypeForBuiltinFnName(w, host_ty); - try w.writeAll("(0, "); - } else { - try w.writeByte('('); - try f.renderType(w, host_ty); - try w.writeByte(')'); - } - - if (src_ty.isPtrAtRuntime(zcu)) { - try w.writeByte('('); - try f.renderType(w, .usize); - try w.writeByte(')'); - } - try f.writeCValue(w, src_val, .Other); - try v.elem(f, w); - if (cant_cast) try w.writeByte(')'); - try w.print(", {f}))", .{try f.fmtIntLiteralDec(bit_offset_val)}); - try a.end(f, w); - try v.end(f, inst, w); } else { switch (ptr_val) { .local_ref => |ptr_local_index| switch (src_val) { @@ -6015,10 +5827,7 @@ fn fieldLocation( else if (!field_ptr_ty.childType(zcu).hasRuntimeBitsIgnoreComptime(zcu)) .{ .byte_offset = loaded_struct.offsets.get(ip)[field_index] } else - .{ .field = if (loaded_struct.fieldName(ip, field_index).unwrap()) |field_name| - .{ .identifier = field_name.toSlice(ip) } - else - .{ .field = field_index } }, + .{ .field = .{ .identifier = loaded_struct.fieldName(ip, field_index).toSlice(ip) } }, .@"packed" => if (field_ptr_ty.ptrInfo(zcu).packed_offset.host_size == 0) .{ .byte_offset = @divExact(zcu.structPackedFieldBitOffset(loaded_struct, field_index) + container_ptr_ty.ptrInfo(zcu).packed_offset.bit_offset, 8) } @@ -6202,115 +6011,20 @@ fn airStructFieldVal(f: *Function, inst: Air.Inst.Index) !CValue { // Ensure complete type definition is visible before accessing fields. _ = try f.ctypeFromType(struct_ty, .complete); + assert(struct_ty.containerLayout(zcu) != .@"packed"); // `Air.Legalize.Feature.expand_packed_struct_field_val` handles this case const field_name: CValue = switch (ip.indexToKey(struct_ty.toIntern())) { - .struct_type => field_name: { - const loaded_struct = ip.loadStructType(struct_ty.toIntern()); - switch (loaded_struct.layout) { - .auto, .@"extern" => break :field_name if (loaded_struct.fieldName(ip, extra.field_index).unwrap()) |field_name| - .{ .identifier = field_name.toSlice(ip) } - else - .{ .field = extra.field_index }, - .@"packed" => { - const int_info = struct_ty.intInfo(zcu); - - const bit_offset_ty = try pt.intType(.unsigned, Type.smallestUnsignedBits(int_info.bits - 1)); - - const bit_offset = zcu.structPackedFieldBitOffset(loaded_struct, extra.field_index); - - const field_int_signedness = if (inst_ty.isAbiInt(zcu)) - inst_ty.intInfo(zcu).signedness - else - .unsigned; - const field_int_ty = try pt.intType(field_int_signedness, @as(u16, @intCast(inst_ty.bitSize(zcu)))); - - const temp_local = try f.allocLocal(inst, field_int_ty); - try f.writeCValue(w, temp_local, .Other); - try w.writeAll(" = zig_wrap_"); - try f.object.dg.renderTypeForBuiltinFnName(w, field_int_ty); - try w.writeAll("(("); - try f.renderType(w, field_int_ty); - try w.writeByte(')'); - const cant_cast = int_info.bits > 64; - if (cant_cast) { - if (field_int_ty.bitSize(zcu) > 64) return f.fail("TODO: C backend: implement casting between types > 64 bits", .{}); - try w.writeAll("zig_lo_"); - try f.object.dg.renderTypeForBuiltinFnName(w, struct_ty); - try w.writeByte('('); - } - if (bit_offset > 0) { - try w.writeAll("zig_shr_"); - try f.object.dg.renderTypeForBuiltinFnName(w, struct_ty); - try w.writeByte('('); - } - try f.writeCValue(w, struct_byval, .Other); - if (bit_offset > 0) try w.print(", {f})", .{ - try f.fmtIntLiteralDec(try pt.intValue(bit_offset_ty, bit_offset)), - }); - if (cant_cast) try w.writeByte(')'); - try f.object.dg.renderBuiltinInfo(w, field_int_ty, .bits); - try w.writeAll(");"); - try f.object.newline(); - if (inst_ty.eql(field_int_ty, zcu)) return temp_local; - - const local = try f.allocLocal(inst, inst_ty); - if (local.new_local != temp_local.new_local) { - try w.writeAll("memcpy("); - try f.writeCValue(w, .{ .local_ref = local.new_local }, .FunctionArgument); - try w.writeAll(", "); - try f.writeCValue(w, .{ .local_ref = temp_local.new_local }, .FunctionArgument); - try w.writeAll(", sizeof("); - try f.renderType(w, inst_ty); - try w.writeAll("));"); - try f.object.newline(); - } - try freeLocal(f, inst, temp_local.new_local, null); - return local; - }, + .struct_type => .{ .identifier = struct_ty.structFieldName(extra.field_index, zcu).unwrap().?.toSlice(ip) }, + .union_type => name: { + const union_type = ip.loadUnionType(struct_ty.toIntern()); + const enum_tag_ty: Type = .fromInterned(union_type.enum_tag_ty); + const field_name_str = enum_tag_ty.enumFieldName(extra.field_index, zcu).toSlice(ip); + if (union_type.hasTag(ip)) { + break :name .{ .payload_identifier = field_name_str }; + } else { + break :name .{ .identifier = field_name_str }; } }, .tuple_type => .{ .field = extra.field_index }, - .union_type => field_name: { - const loaded_union = ip.loadUnionType(struct_ty.toIntern()); - switch (loaded_union.flagsUnordered(ip).layout) { - .auto, .@"extern" => { - const name = loaded_union.loadTagType(ip).names.get(ip)[extra.field_index]; - break :field_name if (loaded_union.hasTag(ip)) - .{ .payload_identifier = name.toSlice(ip) } - else - .{ .identifier = name.toSlice(ip) }; - }, - .@"packed" => { - const operand_lval = if (struct_byval == .constant) blk: { - const operand_local = try f.allocLocal(inst, struct_ty); - try f.writeCValue(w, operand_local, .Other); - try w.writeAll(" = "); - try f.writeCValue(w, struct_byval, .Other); - try w.writeByte(';'); - try f.object.newline(); - break :blk operand_local; - } else struct_byval; - const local = try f.allocLocal(inst, inst_ty); - if (switch (local) { - .new_local, .local => |local_index| switch (operand_lval) { - .new_local, .local => |operand_local_index| local_index != operand_local_index, - else => true, - }, - else => true, - }) { - try w.writeAll("memcpy(&"); - try f.writeCValue(w, local, .Other); - try w.writeAll(", &"); - try f.writeCValue(w, operand_lval, .Other); - try w.writeAll(", sizeof("); - try f.renderType(w, inst_ty); - try w.writeAll("));"); - try f.object.newline(); - } - try f.freeCValue(inst, operand_lval); - return local; - }, - } - }, else => unreachable, }; @@ -7702,98 +7416,13 @@ fn airAggregateInit(f: *Function, inst: Air.Inst.Index) !CValue { if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue; const a = try Assignment.start(f, w, try f.ctypeFromType(field_ty, .complete)); - try f.writeCValueMember(w, local, if (loaded_struct.fieldName(ip, field_index).unwrap()) |field_name| - .{ .identifier = field_name.toSlice(ip) } - else - .{ .field = field_index }); + try f.writeCValueMember(w, local, .{ .identifier = loaded_struct.fieldName(ip, field_index).toSlice(ip) }); try a.assign(f, w); try f.writeCValue(w, resolved_elements[field_index], .Other); try a.end(f, w); } }, - .@"packed" => { - try f.writeCValue(w, local, .Other); - try w.writeAll(" = "); - - const backing_int_ty: Type = .fromInterned(loaded_struct.backingIntTypeUnordered(ip)); - const int_info = backing_int_ty.intInfo(zcu); - - const bit_offset_ty = try pt.intType(.unsigned, Type.smallestUnsignedBits(int_info.bits - 1)); - - var bit_offset: u64 = 0; - - var empty = true; - for (0..elements.len) |field_index| { - if (inst_ty.structFieldIsComptime(field_index, zcu)) continue; - const field_ty = inst_ty.fieldType(field_index, zcu); - if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue; - - if (!empty) { - try w.writeAll("zig_or_"); - try f.object.dg.renderTypeForBuiltinFnName(w, inst_ty); - try w.writeByte('('); - } - empty = false; - } - empty = true; - for (resolved_elements, 0..) |element, field_index| { - if (inst_ty.structFieldIsComptime(field_index, zcu)) continue; - const field_ty = inst_ty.fieldType(field_index, zcu); - if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue; - - if (!empty) try w.writeAll(", "); - // TODO: Skip this entire shift if val is 0? - try w.writeAll("zig_shlw_"); - try f.object.dg.renderTypeForBuiltinFnName(w, inst_ty); - try w.writeByte('('); - - if (field_ty.isAbiInt(zcu)) { - try w.writeAll("zig_and_"); - try f.object.dg.renderTypeForBuiltinFnName(w, inst_ty); - try w.writeByte('('); - } - - if (inst_ty.isAbiInt(zcu) and (field_ty.isAbiInt(zcu) or field_ty.isPtrAtRuntime(zcu))) { - try f.renderIntCast(w, inst_ty, element, .{}, field_ty, .FunctionArgument); - } else { - try w.writeByte('('); - try f.renderType(w, inst_ty); - try w.writeByte(')'); - if (field_ty.isPtrAtRuntime(zcu)) { - try w.writeByte('('); - try f.renderType(w, switch (int_info.signedness) { - .unsigned => .usize, - .signed => .isize, - }); - try w.writeByte(')'); - } - try f.writeCValue(w, element, .Other); - } - - if (field_ty.isAbiInt(zcu)) { - try w.writeAll(", "); - const field_int_info = field_ty.intInfo(zcu); - const field_mask = if (int_info.signedness == .signed and int_info.bits == field_int_info.bits) - try pt.intValue(backing_int_ty, -1) - else - try (try pt.intType(.unsigned, field_int_info.bits)).maxIntScalar(pt, backing_int_ty); - try f.object.dg.renderValue(w, field_mask, .FunctionArgument); - try w.writeByte(')'); - } - - try w.print(", {f}", .{ - try f.fmtIntLiteralDec(try pt.intValue(bit_offset_ty, bit_offset)), - }); - try f.object.dg.renderBuiltinInfo(w, inst_ty, .bits); - try w.writeByte(')'); - if (!empty) try w.writeByte(')'); - - bit_offset += field_ty.bitSize(zcu); - empty = false; - } - try w.writeByte(';'); - try f.object.newline(); - }, + .@"packed" => unreachable, // `Air.Legalize.Feature.expand_packed_struct_init` handles this case } }, .tuple_type => |tuple_info| for (0..tuple_info.types.len) |field_index| { @@ -7828,9 +7457,10 @@ fn airUnionInit(f: *Function, inst: Air.Inst.Index) !CValue { try reap(f, inst, &.{extra.init}); const w = &f.object.code.writer; - const local = try f.allocLocal(inst, union_ty); if (loaded_union.flagsUnordered(ip).layout == .@"packed") return f.moveCValue(inst, union_ty, payload); + const local = try f.allocLocal(inst, union_ty); + const field: CValue = if (union_ty.unionTagTypeSafety(zcu)) |tag_ty| field: { const layout = union_ty.unionGetLayout(zcu); if (layout.tag_size != 0) { diff --git a/src/codegen/c/Type.zig b/src/codegen/c/Type.zig index fa4db36a0c..ac535a47d1 100644 --- a/src/codegen/c/Type.zig +++ b/src/codegen/c/Type.zig @@ -2514,11 +2514,7 @@ pub const Pool = struct { kind.noParameter(), ); if (field_ctype.index == .void) continue; - const field_name = if (loaded_struct.fieldName(ip, field_index) - .unwrap()) |field_name| - try pool.string(allocator, field_name.toSlice(ip)) - else - String.fromUnnamed(@intCast(field_index)); + const field_name = try pool.string(allocator, loaded_struct.fieldName(ip, field_index).toSlice(ip)); const field_alignas = AlignAs.fromAlignment(.{ .@"align" = loaded_struct.fieldAlign(ip, field_index), .abi = field_type.abiAlignment(zcu), diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig index 10b90e25b8..1160c2958e 100644 --- a/src/codegen/llvm.zig +++ b/src/codegen/llvm.zig @@ -2411,8 +2411,7 @@ pub const Object = struct { const field_size = field_ty.abiSize(zcu); const field_align = ty.fieldAlignment(field_index, zcu); const field_offset = ty.structFieldOffset(field_index, zcu); - const field_name = struct_type.fieldName(ip, field_index).unwrap() orelse - try ip.getOrPutStringFmt(gpa, pt.tid, "{d}", .{field_index}, .no_embedded_nulls); + const field_name = struct_type.fieldName(ip, field_index); fields.appendAssumeCapacity(try o.builder.debugMemberType( try o.builder.metadataString(field_name.toSlice(ip)), null, // File @@ -5093,8 +5092,6 @@ pub const FuncGen = struct { .wasm_memory_size => try self.airWasmMemorySize(inst), .wasm_memory_grow => try self.airWasmMemoryGrow(inst), - .vector_store_elem => try self.airVectorStoreElem(inst), - .runtime_nav_ptr => try self.airRuntimeNavPtr(inst), .inferred_alloc, .inferred_alloc_comptime => unreachable, @@ -6873,16 +6870,14 @@ pub const FuncGen = struct { const array_llvm_ty = try o.lowerType(pt, array_ty); const elem_ty = array_ty.childType(zcu); if (isByRef(array_ty, zcu)) { - const indices: [2]Builder.Value = .{ - try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), rhs, - }; + const elem_ptr = try self.wip.gep(.inbounds, array_llvm_ty, array_llvm_val, &.{ + try o.builder.intValue(try o.lowerType(pt, Type.usize), 0), + rhs, + }, ""); if (isByRef(elem_ty, zcu)) { - const elem_ptr = try self.wip.gep(.inbounds, array_llvm_ty, array_llvm_val, &indices, ""); const elem_alignment = elem_ty.abiAlignment(zcu).toLlvm(); return self.loadByRef(elem_ptr, elem_ty, elem_alignment, .normal); } else { - const elem_ptr = - try self.wip.gep(.inbounds, array_llvm_ty, array_llvm_val, &indices, ""); return self.loadTruncate(.normal, elem_ty, elem_ptr, .default); } } @@ -8140,33 +8135,6 @@ pub const FuncGen = struct { }, ""); } - fn airVectorStoreElem(self: *FuncGen, inst: Air.Inst.Index) !Builder.Value { - const o = self.ng.object; - const pt = self.ng.pt; - const zcu = pt.zcu; - const data = self.air.instructions.items(.data)[@intFromEnum(inst)].vector_store_elem; - const extra = self.air.extraData(Air.Bin, data.payload).data; - - const vector_ptr = try self.resolveInst(data.vector_ptr); - const vector_ptr_ty = self.typeOf(data.vector_ptr); - const index = try self.resolveInst(extra.lhs); - const operand = try self.resolveInst(extra.rhs); - - self.maybeMarkAllowZeroAccess(vector_ptr_ty.ptrInfo(zcu)); - - // TODO: Emitting a load here is a violation of volatile semantics. Not fixable in general. - // https://github.com/ziglang/zig/issues/18652#issuecomment-2452844908 - const access_kind: Builder.MemoryAccessKind = - if (vector_ptr_ty.isVolatilePtr(zcu)) .@"volatile" else .normal; - const elem_llvm_ty = try o.lowerType(pt, vector_ptr_ty.childType(zcu)); - const alignment = vector_ptr_ty.ptrAlignment(zcu).toLlvm(); - const loaded = try self.wip.load(access_kind, elem_llvm_ty, vector_ptr, alignment, ""); - - const new_vector = try self.wip.insertElement(loaded, operand, index, ""); - _ = try self.store(vector_ptr, vector_ptr_ty, new_vector, .none); - return .none; - } - fn airRuntimeNavPtr(fg: *FuncGen, inst: Air.Inst.Index) !Builder.Value { const o = fg.ng.object; const pt = fg.ng.pt; @@ -8303,8 +8271,7 @@ pub const FuncGen = struct { const rhs = try self.resolveInst(bin_op.rhs); const inst_ty = self.typeOfIndex(inst); const scalar_ty = inst_ty.scalarType(zcu); - - if (scalar_ty.isAnyFloat()) return self.todo("saturating float add", .{}); + assert(scalar_ty.zigTypeTag(zcu) == .int); return self.wip.callIntrinsic( .normal, .none, @@ -8344,8 +8311,7 @@ pub const FuncGen = struct { const rhs = try self.resolveInst(bin_op.rhs); const inst_ty = self.typeOfIndex(inst); const scalar_ty = inst_ty.scalarType(zcu); - - if (scalar_ty.isAnyFloat()) return self.todo("saturating float sub", .{}); + assert(scalar_ty.zigTypeTag(zcu) == .int); return self.wip.callIntrinsic( .normal, .none, @@ -8385,8 +8351,7 @@ pub const FuncGen = struct { const rhs = try self.resolveInst(bin_op.rhs); const inst_ty = self.typeOfIndex(inst); const scalar_ty = inst_ty.scalarType(zcu); - - if (scalar_ty.isAnyFloat()) return self.todo("saturating float mul", .{}); + assert(scalar_ty.zigTypeTag(zcu) == .int); return self.wip.callIntrinsic( .normal, .none, @@ -11454,7 +11419,6 @@ pub const FuncGen = struct { const access_kind: Builder.MemoryAccessKind = if (info.flags.is_volatile) .@"volatile" else .normal; - assert(info.flags.vector_index != .runtime); if (info.flags.vector_index != .none) { const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index); const vec_elem_ty = try o.lowerType(pt, elem_ty); @@ -11524,7 +11488,6 @@ pub const FuncGen = struct { const access_kind: Builder.MemoryAccessKind = if (info.flags.is_volatile) .@"volatile" else .normal; - assert(info.flags.vector_index != .runtime); if (info.flags.vector_index != .none) { const index_u32 = try o.builder.intValue(.i32, info.flags.vector_index); const vec_elem_ty = try o.lowerType(pt, elem_ty); diff --git a/src/codegen/riscv64/CodeGen.zig b/src/codegen/riscv64/CodeGen.zig index fe40ba4bbb..bf5e5b6718 100644 --- a/src/codegen/riscv64/CodeGen.zig +++ b/src/codegen/riscv64/CodeGen.zig @@ -1633,7 +1633,6 @@ fn genBody(func: *Func, body: []const Air.Inst.Index) InnerError!void { .is_named_enum_value => return func.fail("TODO implement is_named_enum_value", .{}), .error_set_has_value => return func.fail("TODO implement error_set_has_value", .{}), - .vector_store_elem => return func.fail("TODO implement vector_store_elem", .{}), .c_va_arg => return func.fail("TODO implement c_va_arg", .{}), .c_va_copy => return func.fail("TODO implement c_va_copy", .{}), diff --git a/src/codegen/sparc64/CodeGen.zig b/src/codegen/sparc64/CodeGen.zig index 48d44e39f9..684bfcfabb 100644 --- a/src/codegen/sparc64/CodeGen.zig +++ b/src/codegen/sparc64/CodeGen.zig @@ -702,7 +702,6 @@ fn genBody(self: *Self, body: []const Air.Inst.Index) InnerError!void { .is_named_enum_value => @panic("TODO implement is_named_enum_value"), .error_set_has_value => @panic("TODO implement error_set_has_value"), - .vector_store_elem => @panic("TODO implement vector_store_elem"), .runtime_nav_ptr => @panic("TODO implement runtime_nav_ptr"), .c_va_arg => return self.fail("TODO implement c_va_arg", .{}), diff --git a/src/codegen/spirv/CodeGen.zig b/src/codegen/spirv/CodeGen.zig index c8956a274b..281504c7d2 100644 --- a/src/codegen/spirv/CodeGen.zig +++ b/src/codegen/spirv/CodeGen.zig @@ -1520,8 +1520,7 @@ fn resolveType(cg: *CodeGen, ty: Type, repr: Repr) Error!Id { const field_ty: Type = .fromInterned(struct_type.field_types.get(ip)[field_index]); if (!field_ty.hasRuntimeBitsIgnoreComptime(zcu)) continue; - const field_name = struct_type.fieldName(ip, field_index).unwrap() orelse - try ip.getOrPutStringFmt(zcu.gpa, pt.tid, "{d}", .{field_index}, .no_embedded_nulls); + const field_name = struct_type.fieldName(ip, field_index); try member_types.append(try cg.resolveType(field_ty, .indirect)); try member_names.append(field_name.toSlice(ip)); try member_offsets.append(@intCast(ty.structFieldOffset(field_index, zcu))); @@ -2726,8 +2725,6 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) Error!void { .ptr_elem_val => try cg.airPtrElemVal(inst), .array_elem_val => try cg.airArrayElemVal(inst), - .vector_store_elem => return cg.airVectorStoreElem(inst), - .set_union_tag => return cg.airSetUnionTag(inst), .get_union_tag => try cg.airGetUnionTag(inst), .union_init => try cg.airUnionInit(inst), @@ -4446,29 +4443,6 @@ fn airPtrElemVal(cg: *CodeGen, inst: Air.Inst.Index) !?Id { return try cg.load(elem_ty, elem_ptr_id, .{ .is_volatile = ptr_ty.isVolatilePtr(zcu) }); } -fn airVectorStoreElem(cg: *CodeGen, inst: Air.Inst.Index) !void { - const zcu = cg.module.zcu; - const data = cg.air.instructions.items(.data)[@intFromEnum(inst)].vector_store_elem; - const extra = cg.air.extraData(Air.Bin, data.payload).data; - - const vector_ptr_ty = cg.typeOf(data.vector_ptr); - const vector_ty = vector_ptr_ty.childType(zcu); - const scalar_ty = vector_ty.scalarType(zcu); - - const scalar_ty_id = try cg.resolveType(scalar_ty, .indirect); - const storage_class = cg.module.storageClass(vector_ptr_ty.ptrAddressSpace(zcu)); - const scalar_ptr_ty_id = try cg.module.ptrType(scalar_ty_id, storage_class); - - const vector_ptr = try cg.resolve(data.vector_ptr); - const index = try cg.resolve(extra.lhs); - const operand = try cg.resolve(extra.rhs); - - const elem_ptr_id = try cg.accessChainId(scalar_ptr_ty_id, vector_ptr, &.{index}); - try cg.store(scalar_ty, elem_ptr_id, operand, .{ - .is_volatile = vector_ptr_ty.isVolatilePtr(zcu), - }); -} - fn airSetUnionTag(cg: *CodeGen, inst: Air.Inst.Index) !void { const zcu = cg.module.zcu; const bin_op = cg.air.instructions.items(.data)[@intFromEnum(inst)].bin_op; diff --git a/src/codegen/wasm/CodeGen.zig b/src/codegen/wasm/CodeGen.zig index d8d8933cc3..b7f7aa151d 100644 --- a/src/codegen/wasm/CodeGen.zig +++ b/src/codegen/wasm/CodeGen.zig @@ -1978,7 +1978,6 @@ fn genInst(cg: *CodeGen, inst: Air.Inst.Index) InnerError!void { .save_err_return_trace_index, .is_named_enum_value, .addrspace_cast, - .vector_store_elem, .c_va_arg, .c_va_copy, .c_va_end, diff --git a/src/codegen/x86_64/CodeGen.zig b/src/codegen/x86_64/CodeGen.zig index 6434079800..8e54a3a8c0 100644 --- a/src/codegen/x86_64/CodeGen.zig +++ b/src/codegen/x86_64/CodeGen.zig @@ -854,12 +854,6 @@ const FrameAlloc = struct { } }; -const StackAllocation = struct { - inst: ?Air.Inst.Index, - /// TODO do we need size? should be determined by inst.ty.abiSize(zcu) - size: u32, -}; - const BlockData = struct { relocs: std.ArrayListUnmanaged(Mir.Inst.Index) = .empty, state: State, @@ -89326,7 +89320,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { error.SelectFailed => res[0] = try ops[0].load(val_ty, .{ .disp = switch (cg.typeOf(ty_op.operand).ptrInfo(zcu).flags.vector_index) { .none => 0, - .runtime => unreachable, else => |vector_index| @intCast(val_ty.abiSize(zcu) * @intFromEnum(vector_index)), }, }, cg), @@ -89569,7 +89562,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { error.SelectFailed => try ops[0].store(&ops[1], .{ .disp = switch (cg.typeOf(bin_op.lhs).ptrInfo(zcu).flags.vector_index) { .none => 0, - .runtime => unreachable, else => |vector_index| @intCast(cg.typeOf(bin_op.rhs).abiSize(zcu) * @intFromEnum(vector_index)), }, .safe = switch (air_tag) { @@ -171402,8 +171394,9 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { .aggregate_init => |air_tag| fallback: { const ty_pl = air_datas[@intFromEnum(inst)].ty_pl; const agg_ty = ty_pl.ty.toType(); - if ((agg_ty.isVector(zcu) and agg_ty.childType(zcu).toIntern() == .bool_type) or - (agg_ty.zigTypeTag(zcu) == .@"struct" and agg_ty.containerLayout(zcu) == .@"packed")) break :fallback try cg.airAggregateInit(inst); + if (agg_ty.isVector(zcu) and agg_ty.childType(zcu).toIntern() == .bool_type) { + break :fallback try cg.airAggregateInitBoolVec(inst); + } var res = try cg.tempAllocMem(agg_ty); const reset_index = cg.next_temp_index; var bt = cg.liveness.iterateBigTomb(inst); @@ -171441,10 +171434,7 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { } } }, - .@"packed" => return cg.fail("failed to select {s} {f}", .{ - @tagName(air_tag), - agg_ty.fmt(pt), - }), + .@"packed" => unreachable, } }, .tuple_type => |tuple_type| { @@ -173054,633 +173044,6 @@ fn genBody(cg: *CodeGen, body: []const Air.Inst.Index) InnerError!void { try ert.die(cg); try res.finish(inst, &.{}, &.{}, cg); }, - .vector_store_elem => { - const vector_store_elem = air_datas[@intFromEnum(inst)].vector_store_elem; - const bin_op = cg.air.extraData(Air.Bin, vector_store_elem.payload).data; - var ops = try cg.tempsFromOperands(inst, .{ vector_store_elem.vector_ptr, bin_op.lhs, bin_op.rhs }); - cg.select(&.{}, &.{}, &ops, comptime &.{ .{ - .src_constraints = .{ .{ .ptr_bool_vec = .byte }, .any, .bool }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 0 } } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ }, - .{ ._, ._r, .bt, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .lea(.src0b), .tmp0b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .ptr_bool_vec = .byte }, .any, .bool }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 1 } } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ }, - .{ ._, ._s, .bt, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._, .mov, .lea(.src0b), .tmp0b, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .ptr_bool_vec = .byte }, .any, .bool }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._r, .bt, .tmp1d, .src1d, ._, ._ }, - .{ ._, ._s, .bt, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, - .{ ._, ._z, .cmov, .tmp0d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .lea(.src0b), .tmp0b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .ptr_bool_vec = .byte }, .any, .bool }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, - }, - .extra_temps = .{ - .{ .type = .u8, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .lea(.src0b), ._, ._ }, - .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, - .{ ._, ._nz, .j, .@"0f", ._, ._, ._ }, - .{ ._, ._r, .bt, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._mp, .j, .@"1f", ._, ._, ._ }, - .{ .@"0:", ._s, .bt, .tmp0d, .src1d, ._, ._ }, - .{ .@"1:", ._, .mov, .lea(.src0b), .tmp0b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .ptr_bool_vec = .word }, .any, .bool }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 0 } } }, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .bt, .lea(.src0w), .src1w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .ptr_bool_vec = .word }, .any, .bool }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 1 } } }, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._s, .bt, .lea(.src0d), .src1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .ptr_bool_vec = .word }, .any, .bool }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, - }, - .extra_temps = .{ - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .movzx, .tmp0d, .lea(.src0w), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._r, .bt, .tmp1d, .src1d, ._, ._ }, - .{ ._, ._s, .bt, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, - .{ ._, ._z, .cmov, .tmp0d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .lea(.src0w), .tmp0w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .{ .ptr_bool_vec = .word }, .any, .bool }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .bt, .lea(.src0w), .src1w, ._, ._ }, - .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", ._s, .bt, .lea(.src0w), .src1w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .ptr_any_bool_vec, .any, .bool }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 0 } } }, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._r, .bt, .lea(.src0d), .src1d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .ptr_any_bool_vec, .any, .bool }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .{ .imm = 1 } } }, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._s, .bt, .lea(.src0d), .src1d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .{ .ptr_bool_vec = .dword }, .any, .bool }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .lea(.src0d), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .tmp0d, ._, ._ }, - .{ ._, ._r, .bt, .tmp1d, .src1d, ._, ._ }, - .{ ._, ._s, .bt, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, - .{ ._, ._z, .cmov, .tmp0d, .tmp1d, ._, ._ }, - .{ ._, ._, .mov, .lea(.src0d), .tmp0d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", .cmov, null, null }, - .src_constraints = .{ .{ .ptr_bool_vec = .qword }, .any, .bool }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, - }, - .extra_temps = .{ - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u64, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0q, .lea(.src0q), ._, ._ }, - .{ ._, ._, .mov, .tmp1q, .tmp0q, ._, ._ }, - .{ ._, ._r, .bt, .tmp1q, .src1q, ._, ._ }, - .{ ._, ._s, .bt, .tmp0q, .src1q, ._, ._ }, - .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, - .{ ._, ._z, .cmov, .tmp0q, .tmp1q, ._, ._ }, - .{ ._, ._, .mov, .lea(.src0q), .tmp0q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .cmov, null, null, null }, - .src_constraints = .{ .ptr_any_bool_vec, .any, .bool }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, - }, - .extra_temps = .{ - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .{ .type = .u32, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .tmp0d, .src1d, ._, ._ }, - .{ ._, ._r, .sh, .tmp0d, .ui(5), ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .leasi(.src0d, .@"4", .tmp0), ._, ._ }, - .{ ._, ._, .mov, .tmp2d, .tmp1d, ._, ._ }, - .{ ._, ._r, .bt, .tmp2d, .src1d, ._, ._ }, - .{ ._, ._s, .bt, .tmp1d, .src1d, ._, ._ }, - .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, - .{ ._, ._z, .cmov, .tmp1d, .tmp2d, ._, ._ }, - .{ ._, ._, .mov, .leasi(.src0d, .@"4", .tmp0), .tmp1d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .ptr_any_bool_vec, .any, .bool }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, - }, - .clobbers = .{ .eflags = true }, - .each = .{ .once = &.{ - .{ ._, ._, .@"test", .src2b, .si(1), ._, ._ }, - .{ ._, ._nz, .j, .@"1f", ._, ._, ._ }, - .{ ._, ._r, .bt, .lea(.src0d), .src1d, ._, ._ }, - .{ ._, ._mp, .j, .@"0f", ._, ._, ._ }, - .{ .@"1:", ._s, .bt, .lea(.src0d), .src1d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any, .any, .{ .int = .byte } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .imm8 } }, - .{ .src = .{ .to_gpr, .simm32, .to_gpr } }, - }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .leaa(.src0b, .add_src0_elem_size_mul_src1), .src2b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any, .any, .{ .int = .byte } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .imm8 } }, - .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, - }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .leai(.src0b, .src1), .src2b, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any, .any, .{ .int = .word } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .imm16 } }, - .{ .src = .{ .to_gpr, .simm32, .to_gpr } }, - }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .leaa(.src0w, .add_src0_elem_size_mul_src1), .src2w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any, .any, .{ .int = .word } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .imm16 } }, - .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, - }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .leasi(.src0w, .@"2", .src1), .src2w, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .word } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .to_sse } }, - }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .extr, .leaa(.src0w, .add_src0_elem_size_mul_src1), .src2x, .ui(0), ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .word } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .to_sse } }, - }, - .each = .{ .once = &.{ - .{ ._, .p_w, .extr, .leaa(.src0w, .add_src0_elem_size_mul_src1), .src2x, .ui(0), ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .word } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .to_sse } }, - }, - .extra_temps = .{ - .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .each = .{ .once = &.{ - .{ ._, .p_w, .extr, .tmp0d, .src2x, .ui(0), ._ }, - .{ ._, ._, .mov, .leaa(.src0w, .add_src0_elem_size_mul_src1), .tmp0w, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .word } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .to_sse } }, - }, - .extra_temps = .{ - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .each = .{ .once = &.{ - .{ ._, ._ss, .mov, .mem(.tmp1d), .src2x, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .mem(.tmp1d), ._, ._ }, - .{ ._, ._, .mov, .leaa(.src0w, .add_src0_elem_size_mul_src1), .tmp1w, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .word } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .to_sse } }, - }, - .each = .{ .once = &.{ - .{ ._, .vp_w, .extr, .leasi(.src0w, .@"2", .src1), .src2x, .ui(0), ._ }, - } }, - }, .{ - .required_features = .{ .sse4_1, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .word } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .to_sse } }, - }, - .each = .{ .once = &.{ - .{ ._, .p_w, .extr, .leasi(.src0w, .@"2", .src1), .src2x, .ui(0), ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .word } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .to_sse } }, - }, - .extra_temps = .{ - .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .each = .{ .once = &.{ - .{ ._, .p_w, .extr, .tmp0d, .src2x, .ui(0), ._ }, - .{ ._, ._, .mov, .leasi(.src0w, .@"2", .src1), .tmp0w, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .word } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .to_sse } }, - }, - .extra_temps = .{ - .{ .type = .f32, .kind = .mem }, - .{ .type = .f16, .kind = .{ .rc = .general_purpose } }, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - .unused, - }, - .each = .{ .once = &.{ - .{ ._, ._ss, .mov, .mem(.tmp1d), .src2x, ._, ._ }, - .{ ._, ._, .mov, .tmp1d, .mem(.tmp1d), ._, ._ }, - .{ ._, ._, .mov, .leasi(.src0w, .@"2", .src1), .tmp1w, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any, .any, .{ .int = .dword } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .imm32 } }, - .{ .src = .{ .to_gpr, .simm32, .to_gpr } }, - }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .leaa(.src0d, .add_src0_elem_size_mul_src1), .src2d, ._, ._ }, - } }, - }, .{ - .src_constraints = .{ .any, .any, .{ .int = .dword } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .imm32 } }, - .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, - }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .leasi(.src0d, .@"4", .src1), .src2d, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .dword } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .to_sse } }, - }, - .each = .{ .once = &.{ - .{ ._, .v_ss, .mov, .leaa(.src0d, .add_src0_elem_size_mul_src1), .src2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .dword } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .to_sse } }, - }, - .each = .{ .once = &.{ - .{ ._, ._ss, .mov, .leaa(.src0d, .add_src0_elem_size_mul_src1), .src2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .dword } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .to_sse } }, - }, - .each = .{ .once = &.{ - .{ ._, .v_ss, .mov, .leasi(.src0d, .@"4", .src1), .src2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .dword } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .to_sse } }, - }, - .each = .{ .once = &.{ - .{ ._, ._ss, .mov, .leasi(.src0d, .@"4", .src1), .src2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any, .any, .{ .int = .qword } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .simm32 } }, - .{ .src = .{ .to_gpr, .simm32, .to_gpr } }, - }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .leaa(.src0q, .add_src0_elem_size_mul_src1), .src2q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .@"64bit", null, null, null }, - .src_constraints = .{ .any, .any, .{ .int = .qword } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .simm32 } }, - .{ .src = .{ .to_gpr, .to_gpr, .to_gpr } }, - }, - .each = .{ .once = &.{ - .{ ._, ._, .mov, .leasi(.src0q, .@"8", .src1), .src2q, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .qword } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .to_sse } }, - }, - .each = .{ .once = &.{ - .{ ._, .v_sd, .mov, .leaa(.src0q, .add_src0_elem_size_mul_src1), .src2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .qword } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .to_sse } }, - }, - .each = .{ .once = &.{ - .{ ._, ._sd, .mov, .leaa(.src0q, .add_src0_elem_size_mul_src1), .src2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .qword } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .simm32, .to_sse } }, - }, - .each = .{ .once = &.{ - .{ ._, ._ps, .movl, .leaa(.src0q, .add_src0_elem_size_mul_src1), .src2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .avx, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .qword } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .to_sse } }, - }, - .each = .{ .once = &.{ - .{ ._, .v_sd, .mov, .leasi(.src0q, .@"8", .src1), .src2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse2, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .qword } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .to_sse } }, - }, - .each = .{ .once = &.{ - .{ ._, ._sd, .mov, .leasi(.src0q, .@"8", .src1), .src2x, ._, ._ }, - } }, - }, .{ - .required_features = .{ .sse, null, null, null }, - .src_constraints = .{ .any, .any, .{ .float = .qword } }, - .patterns = &.{ - .{ .src = .{ .to_gpr, .to_gpr, .to_sse } }, - }, - .each = .{ .once = &.{ - .{ ._, ._ps, .movl, .leasi(.src0q, .@"8", .src1), .src2x, ._, ._ }, - } }, - } }) catch |err| switch (err) { - error.SelectFailed => { - const elem_size = cg.typeOf(bin_op.rhs).abiSize(zcu); - while (try ops[0].toRegClass(true, .general_purpose, cg) or - try ops[1].toRegClass(true, .general_purpose, cg)) - {} - const base_reg = ops[0].tracking(cg).short.register.to64(); - const rhs_reg = ops[1].tracking(cg).short.register.to64(); - if (!std.math.isPowerOfTwo(elem_size)) { - try cg.spillEflagsIfOccupied(); - try cg.asmRegisterRegisterImmediate( - .{ .i_, .mul }, - rhs_reg, - rhs_reg, - .u(elem_size), - ); - try cg.asmRegisterMemory(.{ ._, .lea }, base_reg, .{ - .base = .{ .reg = base_reg }, - .mod = .{ .rm = .{ .index = rhs_reg } }, - }); - } else if (elem_size > 8) { - try cg.spillEflagsIfOccupied(); - try cg.asmRegisterImmediate( - .{ ._l, .sh }, - rhs_reg, - .u(std.math.log2_int(u64, elem_size)), - ); - try cg.asmRegisterMemory(.{ ._, .lea }, base_reg, .{ - .base = .{ .reg = base_reg }, - .mod = .{ .rm = .{ .index = rhs_reg } }, - }); - } else try cg.asmRegisterMemory(.{ ._, .lea }, base_reg, .{ - .base = .{ .reg = base_reg }, - .mod = .{ .rm = .{ - .index = rhs_reg, - .scale = .fromFactor(@intCast(elem_size)), - } }, - }); - try ops[0].store(&ops[2], .{}, cg); - }, - else => |e| return e, - }; - for (ops) |op| try op.die(cg); - }, .runtime_nav_ptr => { const ty_nav = air_datas[@intFromEnum(inst)].ty_nav; const nav = ip.getNav(ty_nav.nav); @@ -180646,944 +180009,55 @@ fn airSelect(self: *CodeGen, inst: Air.Inst.Index) !void { return self.finishAir(inst, result, .{ pl_op.operand, extra.lhs, extra.rhs }); } -fn airShuffle(self: *CodeGen, inst: Air.Inst.Index) !void { - const pt = self.pt; - const zcu = pt.zcu; - const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; - const extra = self.air.extraData(Air.Shuffle, ty_pl.payload).data; - - const dst_ty = self.typeOfIndex(inst); - const elem_ty = dst_ty.childType(zcu); - const elem_abi_size: u16 = @intCast(elem_ty.abiSize(zcu)); - const dst_abi_size: u32 = @intCast(dst_ty.abiSize(zcu)); - const lhs_ty = self.typeOf(extra.a); - const lhs_abi_size: u32 = @intCast(lhs_ty.abiSize(zcu)); - const rhs_ty = self.typeOf(extra.b); - const rhs_abi_size: u32 = @intCast(rhs_ty.abiSize(zcu)); - const max_abi_size = @max(dst_abi_size, lhs_abi_size, rhs_abi_size); - - const ExpectedContents = [32]?i32; - var stack align(@max(@alignOf(ExpectedContents), @alignOf(std.heap.StackFallbackAllocator(0)))) = - std.heap.stackFallback(@sizeOf(ExpectedContents), self.gpa); - const allocator = stack.get(); - - const mask_elems = try allocator.alloc(?i32, extra.mask_len); - defer allocator.free(mask_elems); - for (mask_elems, 0..) |*mask_elem, elem_index| { - const mask_elem_val = - Value.fromInterned(extra.mask).elemValue(pt, elem_index) catch unreachable; - mask_elem.* = if (mask_elem_val.isUndef(zcu)) - null - else - @intCast(mask_elem_val.toSignedInt(zcu)); - } - - const has_avx = self.hasFeature(.avx); - const result = @as(?MCValue, result: { - for (mask_elems) |mask_elem| { - if (mask_elem) |_| break; - } else break :result try self.allocRegOrMem(inst, true); - - for (mask_elems, 0..) |mask_elem, elem_index| { - if (mask_elem orelse continue != elem_index) break; - } else { - const lhs_mcv = try self.resolveInst(extra.a); - if (self.reuseOperand(inst, extra.a, 0, lhs_mcv)) break :result lhs_mcv; - const dst_mcv = try self.allocRegOrMem(inst, true); - try self.genCopy(dst_ty, dst_mcv, lhs_mcv, .{}); - break :result dst_mcv; - } - - for (mask_elems, 0..) |mask_elem, elem_index| { - if (~(mask_elem orelse continue) != elem_index) break; - } else { - const rhs_mcv = try self.resolveInst(extra.b); - if (self.reuseOperand(inst, extra.b, 1, rhs_mcv)) break :result rhs_mcv; - const dst_mcv = try self.allocRegOrMem(inst, true); - try self.genCopy(dst_ty, dst_mcv, rhs_mcv, .{}); - break :result dst_mcv; - } - - for ([_]Mir.Inst.Tag{ .unpckl, .unpckh }) |variant| unpck: { - if (elem_abi_size > 8) break :unpck; - if (dst_abi_size > self.vectorSize(if (elem_abi_size >= 4) .float else .int)) break :unpck; - - var sources: [2]?u1 = @splat(null); - for (mask_elems, 0..) |maybe_mask_elem, elem_index| { - const mask_elem = maybe_mask_elem orelse continue; - const mask_elem_index = - std.math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :unpck; - const elem_byte = (elem_index >> 1) * elem_abi_size; - if (mask_elem_index * elem_abi_size != (elem_byte & 0b0111) | @as(u4, switch (variant) { - .unpckl => 0b0000, - .unpckh => 0b1000, - else => unreachable, - }) | (elem_byte << 1 & 0b10000)) break :unpck; - - const source = @intFromBool(mask_elem < 0); - if (sources[elem_index & 0b00001]) |prev_source| { - if (source != prev_source) break :unpck; - } else sources[elem_index & 0b00001] = source; - } - if (sources[0] orelse break :unpck == sources[1] orelse break :unpck) break :unpck; - - const operands = [2]Air.Inst.Ref{ extra.a, extra.b }; - const operand_tys = [2]Type{ lhs_ty, rhs_ty }; - const lhs_mcv = try self.resolveInst(operands[sources[0].?]); - const rhs_mcv = try self.resolveInst(operands[sources[1].?]); - - const dst_mcv: MCValue = if (lhs_mcv.isRegister() and - self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv)) - lhs_mcv - else if (has_avx and lhs_mcv.isRegister()) - .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } - else - try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv); - const dst_reg = dst_mcv.getReg().?; - const dst_alias = registerAlias(dst_reg, max_abi_size); - - const mir_tag: Mir.Inst.FixedTag = if ((elem_abi_size >= 4 and elem_ty.isRuntimeFloat()) or - (dst_abi_size > 16 and !self.hasFeature(.avx2))) .{ switch (elem_abi_size) { - 4 => if (has_avx) .v_ps else ._ps, - 8 => if (has_avx) .v_pd else ._pd, - else => unreachable, - }, variant } else .{ if (has_avx) .vp_ else .p_, switch (variant) { - .unpckl => switch (elem_abi_size) { - 1 => .unpcklbw, - 2 => .unpcklwd, - 4 => .unpckldq, - 8 => .unpcklqdq, - else => unreachable, - }, - .unpckh => switch (elem_abi_size) { - 1 => .unpckhbw, - 2 => .unpckhwd, - 4 => .unpckhdq, - 8 => .unpckhqdq, - else => unreachable, - }, - else => unreachable, - } }; - if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemory( - mir_tag, - dst_alias, - registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), - try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), - ) else try self.asmRegisterRegisterRegister( - mir_tag, - dst_alias, - registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), - registerAlias(if (rhs_mcv.isRegister()) - rhs_mcv.getReg().? - else - try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), - ) else if (rhs_mcv.isBase()) try self.asmRegisterMemory( - mir_tag, - dst_alias, - try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), - ) else try self.asmRegisterRegister( - mir_tag, - dst_alias, - registerAlias(if (rhs_mcv.isRegister()) - rhs_mcv.getReg().? - else - try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), - ); - break :result dst_mcv; - } - - pshufd: { - if (elem_abi_size != 4) break :pshufd; - if (max_abi_size > self.vectorSize(.float)) break :pshufd; - - var control: u8 = 0b00_00_00_00; - var sources: [1]?u1 = @splat(null); - for (mask_elems, 0..) |maybe_mask_elem, elem_index| { - const mask_elem = maybe_mask_elem orelse continue; - const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem); - if (mask_elem_index & 0b100 != elem_index & 0b100) break :pshufd; - - const source = @intFromBool(mask_elem < 0); - if (sources[0]) |prev_source| { - if (source != prev_source) break :pshufd; - } else sources[(elem_index & 0b010) >> 1] = source; - - const select_bit: u3 = @intCast((elem_index & 0b011) << 1); - const select_mask = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit; - if (elem_index & 0b100 == 0) - control |= select_mask - else if (control & @as(u8, 0b11) << select_bit != select_mask) break :pshufd; - } - - const operands = [2]Air.Inst.Ref{ extra.a, extra.b }; - const operand_tys = [2]Type{ lhs_ty, rhs_ty }; - const src_mcv = try self.resolveInst(operands[sources[0] orelse break :pshufd]); - - const dst_reg = if (src_mcv.isRegister() and - self.reuseOperand(inst, operands[sources[0].?], sources[0].?, src_mcv)) - src_mcv.getReg().? - else - try self.register_manager.allocReg(inst, abi.RegisterClass.sse); - const dst_alias = registerAlias(dst_reg, max_abi_size); - - if (src_mcv.isBase()) try self.asmRegisterMemoryImmediate( - .{ if (has_avx) .vp_d else .p_d, .shuf }, - dst_alias, - try src_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), - .u(control), - ) else try self.asmRegisterRegisterImmediate( - .{ if (has_avx) .vp_d else .p_d, .shuf }, - dst_alias, - registerAlias(if (src_mcv.isRegister()) - src_mcv.getReg().? - else - try self.copyToTmpRegister(operand_tys[sources[0].?], src_mcv), max_abi_size), - .u(control), - ); - break :result .{ .register = dst_reg }; - } - - shufps: { - if (elem_abi_size != 4) break :shufps; - if (max_abi_size > self.vectorSize(.float)) break :shufps; - - var control: u8 = 0b00_00_00_00; - var sources: [2]?u1 = @splat(null); - for (mask_elems, 0..) |maybe_mask_elem, elem_index| { - const mask_elem = maybe_mask_elem orelse continue; - const mask_elem_index: u3 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem); - if (mask_elem_index & 0b100 != elem_index & 0b100) break :shufps; - - const source = @intFromBool(mask_elem < 0); - if (sources[(elem_index & 0b010) >> 1]) |prev_source| { - if (source != prev_source) break :shufps; - } else sources[(elem_index & 0b010) >> 1] = source; - - const select_bit: u3 = @intCast((elem_index & 0b011) << 1); - const select_mask = @as(u8, @intCast(mask_elem_index & 0b011)) << select_bit; - if (elem_index & 0b100 == 0) - control |= select_mask - else if (control & @as(u8, 0b11) << select_bit != select_mask) break :shufps; - } - if (sources[0] orelse break :shufps == sources[1] orelse break :shufps) break :shufps; - - const operands = [2]Air.Inst.Ref{ extra.a, extra.b }; - const operand_tys = [2]Type{ lhs_ty, rhs_ty }; - const lhs_mcv = try self.resolveInst(operands[sources[0].?]); - const rhs_mcv = try self.resolveInst(operands[sources[1].?]); - - const dst_mcv: MCValue = if (lhs_mcv.isRegister() and - self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv)) - lhs_mcv - else if (has_avx and lhs_mcv.isRegister()) - .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } - else - try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv); - const dst_reg = dst_mcv.getReg().?; - const dst_alias = registerAlias(dst_reg, max_abi_size); - - if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( - .{ .v_ps, .shuf }, - dst_alias, - registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), - try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), - .u(control), - ) else try self.asmRegisterRegisterRegisterImmediate( - .{ .v_ps, .shuf }, - dst_alias, - registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), - registerAlias(if (rhs_mcv.isRegister()) - rhs_mcv.getReg().? - else - try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), - .u(control), - ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate( - .{ ._ps, .shuf }, - dst_alias, - try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), - .u(control), - ) else try self.asmRegisterRegisterImmediate( - .{ ._ps, .shuf }, - dst_alias, - registerAlias(if (rhs_mcv.isRegister()) - rhs_mcv.getReg().? - else - try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), - .u(control), - ); - break :result dst_mcv; - } - - shufpd: { - if (elem_abi_size != 8) break :shufpd; - if (max_abi_size > self.vectorSize(.float)) break :shufpd; - - var control: u4 = 0b0_0_0_0; - var sources: [2]?u1 = @splat(null); - for (mask_elems, 0..) |maybe_mask_elem, elem_index| { - const mask_elem = maybe_mask_elem orelse continue; - const mask_elem_index: u2 = @intCast(if (mask_elem < 0) ~mask_elem else mask_elem); - if (mask_elem_index & 0b10 != elem_index & 0b10) break :shufpd; - - const source = @intFromBool(mask_elem < 0); - if (sources[elem_index & 0b01]) |prev_source| { - if (source != prev_source) break :shufpd; - } else sources[elem_index & 0b01] = source; - - control |= @as(u4, @intCast(mask_elem_index & 0b01)) << @intCast(elem_index); - } - if (sources[0] orelse break :shufpd == sources[1] orelse break :shufpd) break :shufpd; - - const operands: [2]Air.Inst.Ref = .{ extra.a, extra.b }; - const operand_tys: [2]Type = .{ lhs_ty, rhs_ty }; - const lhs_mcv = try self.resolveInst(operands[sources[0].?]); - const rhs_mcv = try self.resolveInst(operands[sources[1].?]); - - const dst_mcv: MCValue = if (lhs_mcv.isRegister() and - self.reuseOperand(inst, operands[sources[0].?], sources[0].?, lhs_mcv)) - lhs_mcv - else if (has_avx and lhs_mcv.isRegister()) - .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } - else - try self.copyToRegisterWithInstTracking(inst, operand_tys[sources[0].?], lhs_mcv); - const dst_reg = dst_mcv.getReg().?; - const dst_alias = registerAlias(dst_reg, max_abi_size); - - if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( - .{ .v_pd, .shuf }, - dst_alias, - registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), - try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), - .u(control), - ) else try self.asmRegisterRegisterRegisterImmediate( - .{ .v_pd, .shuf }, - dst_alias, - registerAlias(lhs_mcv.getReg() orelse dst_reg, max_abi_size), - registerAlias(if (rhs_mcv.isRegister()) - rhs_mcv.getReg().? - else - try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), - .u(control), - ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate( - .{ ._pd, .shuf }, - dst_alias, - try rhs_mcv.mem(self, .{ .size = .fromSize(max_abi_size) }), - .u(control), - ) else try self.asmRegisterRegisterImmediate( - .{ ._pd, .shuf }, - dst_alias, - registerAlias(if (rhs_mcv.isRegister()) - rhs_mcv.getReg().? - else - try self.copyToTmpRegister(operand_tys[sources[1].?], rhs_mcv), max_abi_size), - .u(control), - ); - break :result dst_mcv; - } - - blend: { - if (elem_abi_size < 2) break :blend; - if (dst_abi_size > self.vectorSize(.float)) break :blend; - if (!self.hasFeature(.sse4_1)) break :blend; - - var control: u8 = 0b0_0_0_0_0_0_0_0; - for (mask_elems, 0..) |maybe_mask_elem, elem_index| { - const mask_elem = maybe_mask_elem orelse continue; - const mask_elem_index = - std.math.cast(u4, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blend; - if (mask_elem_index != elem_index) break :blend; - - const select_mask = @as(u8, @intFromBool(mask_elem < 0)) << @truncate(elem_index); - if (elem_index & 0b1000 == 0) - control |= select_mask - else if (control & @as(u8, 0b1) << @truncate(elem_index) != select_mask) break :blend; - } - - if (!elem_ty.isRuntimeFloat() and self.hasFeature(.avx2)) vpblendd: { - const expanded_control = switch (elem_abi_size) { - 4 => control, - 8 => @as(u8, if (control & 0b0001 != 0) 0b00_00_00_11 else 0b00_00_00_00) | - @as(u8, if (control & 0b0010 != 0) 0b00_00_11_00 else 0b00_00_00_00) | - @as(u8, if (control & 0b0100 != 0) 0b00_11_00_00 else 0b00_00_00_00) | - @as(u8, if (control & 0b1000 != 0) 0b11_00_00_00 else 0b00_00_00_00), - else => break :vpblendd, - }; - - const lhs_mcv = try self.resolveInst(extra.a); - const lhs_reg = if (lhs_mcv.isRegister()) - lhs_mcv.getReg().? - else - try self.copyToTmpRegister(dst_ty, lhs_mcv); - const lhs_lock = self.register_manager.lockReg(lhs_reg); - defer if (lhs_lock) |lock| self.register_manager.unlockReg(lock); - - const rhs_mcv = try self.resolveInst(extra.b); - const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.sse); - if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( - .{ .vp_d, .blend }, - registerAlias(dst_reg, dst_abi_size), - registerAlias(lhs_reg, dst_abi_size), - try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), - .u(expanded_control), - ) else try self.asmRegisterRegisterRegisterImmediate( - .{ .vp_d, .blend }, - registerAlias(dst_reg, dst_abi_size), - registerAlias(lhs_reg, dst_abi_size), - registerAlias(if (rhs_mcv.isRegister()) - rhs_mcv.getReg().? - else - try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), - .u(expanded_control), - ); - break :result .{ .register = dst_reg }; - } - - if (!elem_ty.isRuntimeFloat() or elem_abi_size == 2) pblendw: { - const expanded_control = switch (elem_abi_size) { - 2 => control, - 4 => if (dst_abi_size <= 16 or - @as(u4, @intCast(control >> 4)) == @as(u4, @truncate(control >> 0))) - @as(u8, if (control & 0b0001 != 0) 0b00_00_00_11 else 0b00_00_00_00) | - @as(u8, if (control & 0b0010 != 0) 0b00_00_11_00 else 0b00_00_00_00) | - @as(u8, if (control & 0b0100 != 0) 0b00_11_00_00 else 0b00_00_00_00) | - @as(u8, if (control & 0b1000 != 0) 0b11_00_00_00 else 0b00_00_00_00) - else - break :pblendw, - 8 => if (dst_abi_size <= 16 or - @as(u2, @intCast(control >> 2)) == @as(u2, @truncate(control >> 0))) - @as(u8, if (control & 0b01 != 0) 0b0000_1111 else 0b0000_0000) | - @as(u8, if (control & 0b10 != 0) 0b1111_0000 else 0b0000_0000) - else - break :pblendw, - 16 => break :pblendw, - else => unreachable, - }; - - const lhs_mcv = try self.resolveInst(extra.a); - const rhs_mcv = try self.resolveInst(extra.b); - - const dst_mcv: MCValue = if (lhs_mcv.isRegister() and - self.reuseOperand(inst, extra.a, 0, lhs_mcv)) - lhs_mcv - else if (has_avx and lhs_mcv.isRegister()) - .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } - else - try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv); - const dst_reg = dst_mcv.getReg().?; - - if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( - .{ .vp_w, .blend }, - registerAlias(dst_reg, dst_abi_size), - registerAlias(if (lhs_mcv.isRegister()) - lhs_mcv.getReg().? - else - dst_reg, dst_abi_size), - try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), - .u(expanded_control), - ) else try self.asmRegisterRegisterRegisterImmediate( - .{ .vp_w, .blend }, - registerAlias(dst_reg, dst_abi_size), - registerAlias(if (lhs_mcv.isRegister()) - lhs_mcv.getReg().? - else - dst_reg, dst_abi_size), - registerAlias(if (rhs_mcv.isRegister()) - rhs_mcv.getReg().? - else - try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), - .u(expanded_control), - ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate( - .{ .p_w, .blend }, - registerAlias(dst_reg, dst_abi_size), - try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), - .u(expanded_control), - ) else try self.asmRegisterRegisterImmediate( - .{ .p_w, .blend }, - registerAlias(dst_reg, dst_abi_size), - registerAlias(if (rhs_mcv.isRegister()) - rhs_mcv.getReg().? - else - try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), - .u(expanded_control), - ); - break :result .{ .register = dst_reg }; - } - - const expanded_control = switch (elem_abi_size) { - 4, 8 => control, - 16 => @as(u4, if (control & 0b01 != 0) 0b00_11 else 0b00_00) | - @as(u4, if (control & 0b10 != 0) 0b11_00 else 0b00_00), - else => unreachable, - }; - - const lhs_mcv = try self.resolveInst(extra.a); - const rhs_mcv = try self.resolveInst(extra.b); - - const dst_mcv: MCValue = if (lhs_mcv.isRegister() and - self.reuseOperand(inst, extra.a, 0, lhs_mcv)) - lhs_mcv - else if (has_avx and lhs_mcv.isRegister()) - .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } - else - try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv); - const dst_reg = dst_mcv.getReg().?; - - if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryImmediate( - switch (elem_abi_size) { - 4 => .{ .v_ps, .blend }, - 8, 16 => .{ .v_pd, .blend }, - else => unreachable, - }, - registerAlias(dst_reg, dst_abi_size), - registerAlias(if (lhs_mcv.isRegister()) - lhs_mcv.getReg().? - else - dst_reg, dst_abi_size), - try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), - .u(expanded_control), - ) else try self.asmRegisterRegisterRegisterImmediate( - switch (elem_abi_size) { - 4 => .{ .v_ps, .blend }, - 8, 16 => .{ .v_pd, .blend }, - else => unreachable, - }, - registerAlias(dst_reg, dst_abi_size), - registerAlias(if (lhs_mcv.isRegister()) - lhs_mcv.getReg().? - else - dst_reg, dst_abi_size), - registerAlias(if (rhs_mcv.isRegister()) - rhs_mcv.getReg().? - else - try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), - .u(expanded_control), - ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryImmediate( - switch (elem_abi_size) { - 4 => .{ ._ps, .blend }, - 8, 16 => .{ ._pd, .blend }, - else => unreachable, - }, - registerAlias(dst_reg, dst_abi_size), - try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), - .u(expanded_control), - ) else try self.asmRegisterRegisterImmediate( - switch (elem_abi_size) { - 4 => .{ ._ps, .blend }, - 8, 16 => .{ ._pd, .blend }, - else => unreachable, - }, - registerAlias(dst_reg, dst_abi_size), - registerAlias(if (rhs_mcv.isRegister()) - rhs_mcv.getReg().? - else - try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), - .u(expanded_control), - ); - break :result .{ .register = dst_reg }; - } - - blendv: { - if (dst_abi_size > self.vectorSize(if (elem_abi_size >= 4) .float else .int)) break :blendv; - - const select_mask_elem_ty = try pt.intType(.unsigned, elem_abi_size * 8); - const select_mask_ty = try pt.vectorType(.{ - .len = @intCast(mask_elems.len), - .child = select_mask_elem_ty.toIntern(), - }); - var select_mask_elems: [32]InternPool.Index = undefined; - for ( - select_mask_elems[0..mask_elems.len], - mask_elems, - 0.., - ) |*select_mask_elem, maybe_mask_elem, elem_index| { - const mask_elem = maybe_mask_elem orelse continue; - const mask_elem_index = - std.math.cast(u5, if (mask_elem < 0) ~mask_elem else mask_elem) orelse break :blendv; - if (mask_elem_index != elem_index) break :blendv; - - select_mask_elem.* = (if (mask_elem < 0) - try select_mask_elem_ty.maxIntScalar(pt, select_mask_elem_ty) - else - try select_mask_elem_ty.minIntScalar(pt, select_mask_elem_ty)).toIntern(); - } - const select_mask_mcv = try self.lowerValue( - try pt.aggregateValue(select_mask_ty, select_mask_elems[0..mask_elems.len]), - ); - - if (self.hasFeature(.sse4_1)) { - const mir_tag: Mir.Inst.FixedTag = .{ - if ((elem_abi_size >= 4 and elem_ty.isRuntimeFloat()) or - (dst_abi_size > 16 and !self.hasFeature(.avx2))) switch (elem_abi_size) { - 4 => if (has_avx) .v_ps else ._ps, - 8 => if (has_avx) .v_pd else ._pd, - else => unreachable, - } else if (has_avx) .vp_b else .p_b, - .blendv, - }; - - const select_mask_reg = if (!has_avx) reg: { - try self.register_manager.getKnownReg(.xmm0, null); - try self.genSetReg(.xmm0, select_mask_elem_ty, select_mask_mcv, .{}); - break :reg .xmm0; - } else try self.copyToTmpRegister(select_mask_ty, select_mask_mcv); - const select_mask_alias = registerAlias(select_mask_reg, dst_abi_size); - const select_mask_lock = self.register_manager.lockRegAssumeUnused(select_mask_reg); - defer self.register_manager.unlockReg(select_mask_lock); - - const lhs_mcv = try self.resolveInst(extra.a); - const rhs_mcv = try self.resolveInst(extra.b); - - const dst_mcv: MCValue = if (lhs_mcv.isRegister() and - self.reuseOperand(inst, extra.a, 0, lhs_mcv)) - lhs_mcv - else if (has_avx and lhs_mcv.isRegister()) - .{ .register = try self.register_manager.allocReg(inst, abi.RegisterClass.sse) } - else - try self.copyToRegisterWithInstTracking(inst, dst_ty, lhs_mcv); - const dst_reg = dst_mcv.getReg().?; - const dst_alias = registerAlias(dst_reg, dst_abi_size); - - if (has_avx) if (rhs_mcv.isBase()) try self.asmRegisterRegisterMemoryRegister( - mir_tag, - dst_alias, - if (lhs_mcv.isRegister()) - registerAlias(lhs_mcv.getReg().?, dst_abi_size) - else - dst_alias, - try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), - select_mask_alias, - ) else try self.asmRegisterRegisterRegisterRegister( - mir_tag, - dst_alias, - if (lhs_mcv.isRegister()) - registerAlias(lhs_mcv.getReg().?, dst_abi_size) - else - dst_alias, - registerAlias(if (rhs_mcv.isRegister()) - rhs_mcv.getReg().? - else - try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), - select_mask_alias, - ) else if (rhs_mcv.isBase()) try self.asmRegisterMemoryRegister( - mir_tag, - dst_alias, - try rhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), - select_mask_alias, - ) else try self.asmRegisterRegisterRegister( - mir_tag, - dst_alias, - registerAlias(if (rhs_mcv.isRegister()) - rhs_mcv.getReg().? - else - try self.copyToTmpRegister(dst_ty, rhs_mcv), dst_abi_size), - select_mask_alias, - ); - break :result dst_mcv; - } - - const lhs_mcv = try self.resolveInst(extra.a); - const rhs_mcv = try self.resolveInst(extra.b); - - const dst_mcv: MCValue = if (rhs_mcv.isRegister() and - self.reuseOperand(inst, extra.b, 1, rhs_mcv)) - rhs_mcv - else - try self.copyToRegisterWithInstTracking(inst, dst_ty, rhs_mcv); - const dst_reg = dst_mcv.getReg().?; - const dst_alias = registerAlias(dst_reg, dst_abi_size); - - const mask_reg = try self.copyToTmpRegister(select_mask_ty, select_mask_mcv); - const mask_alias = registerAlias(mask_reg, dst_abi_size); - const mask_lock = self.register_manager.lockRegAssumeUnused(mask_reg); - defer self.register_manager.unlockReg(mask_lock); - - const mir_fixes: Mir.Inst.Fixes = if (elem_ty.isRuntimeFloat()) - switch (elem_ty.floatBits(self.target)) { - 16, 80, 128 => .p_, - 32 => ._ps, - 64 => ._pd, - else => unreachable, - } - else - .p_; - try self.asmRegisterRegister(.{ mir_fixes, .@"and" }, dst_alias, mask_alias); - if (lhs_mcv.isBase()) try self.asmRegisterMemory( - .{ mir_fixes, .andn }, - mask_alias, - try lhs_mcv.mem(self, .{ .size = .fromSize(dst_abi_size) }), - ) else try self.asmRegisterRegister( - .{ mir_fixes, .andn }, - mask_alias, - if (lhs_mcv.isRegister()) - lhs_mcv.getReg().? - else - try self.copyToTmpRegister(dst_ty, lhs_mcv), - ); - try self.asmRegisterRegister(.{ mir_fixes, .@"or" }, dst_alias, mask_alias); - break :result dst_mcv; - } - - pshufb: { - if (max_abi_size > 16) break :pshufb; - if (!self.hasFeature(.ssse3)) break :pshufb; - - const temp_regs = - try self.register_manager.allocRegs(2, .{ inst, null }, abi.RegisterClass.sse); - const temp_locks = self.register_manager.lockRegsAssumeUnused(2, temp_regs); - defer for (temp_locks) |lock| self.register_manager.unlockReg(lock); - - const lhs_temp_alias = registerAlias(temp_regs[0], max_abi_size); - try self.genSetReg(temp_regs[0], lhs_ty, .{ .air_ref = extra.a }, .{}); - - const rhs_temp_alias = registerAlias(temp_regs[1], max_abi_size); - try self.genSetReg(temp_regs[1], rhs_ty, .{ .air_ref = extra.b }, .{}); - - var lhs_mask_elems: [16]InternPool.Index = undefined; - for (lhs_mask_elems[0..max_abi_size], 0..) |*lhs_mask_elem, byte_index| { - const elem_index = byte_index / elem_abi_size; - lhs_mask_elem.* = (try pt.intValue(.u8, if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: { - const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000; - if (mask_elem < 0) break :elem 0b1_00_00000; - const mask_elem_index: u31 = @intCast(mask_elem); - const byte_off: u32 = @intCast(byte_index % elem_abi_size); - break :elem mask_elem_index * elem_abi_size + byte_off; - })).toIntern(); - } - const lhs_mask_ty = try pt.vectorType(.{ .len = max_abi_size, .child = .u8_type }); - const lhs_mask_mcv = try self.lowerValue( - try pt.aggregateValue(lhs_mask_ty, lhs_mask_elems[0..max_abi_size]), - ); - const lhs_mask_mem: Memory = .{ - .base = .{ .reg = try self.copyToTmpRegister(.usize, lhs_mask_mcv.address()) }, - .mod = .{ .rm = .{ .size = .fromSize(@max(max_abi_size, 16)) } }, - }; - if (has_avx) try self.asmRegisterRegisterMemory( - .{ .vp_b, .shuf }, - lhs_temp_alias, - lhs_temp_alias, - lhs_mask_mem, - ) else try self.asmRegisterMemory( - .{ .p_b, .shuf }, - lhs_temp_alias, - lhs_mask_mem, - ); - - var rhs_mask_elems: [16]InternPool.Index = undefined; - for (rhs_mask_elems[0..max_abi_size], 0..) |*rhs_mask_elem, byte_index| { - const elem_index = byte_index / elem_abi_size; - rhs_mask_elem.* = (try pt.intValue(.u8, if (elem_index >= mask_elems.len) 0b1_00_00000 else elem: { - const mask_elem = mask_elems[elem_index] orelse break :elem 0b1_00_00000; - if (mask_elem >= 0) break :elem 0b1_00_00000; - const mask_elem_index: u31 = @intCast(~mask_elem); - const byte_off: u32 = @intCast(byte_index % elem_abi_size); - break :elem mask_elem_index * elem_abi_size + byte_off; - })).toIntern(); - } - const rhs_mask_ty = try pt.vectorType(.{ .len = max_abi_size, .child = .u8_type }); - const rhs_mask_mcv = try self.lowerValue( - try pt.aggregateValue(rhs_mask_ty, rhs_mask_elems[0..max_abi_size]), - ); - const rhs_mask_mem: Memory = .{ - .base = .{ .reg = try self.copyToTmpRegister(.usize, rhs_mask_mcv.address()) }, - .mod = .{ .rm = .{ .size = .fromSize(@max(max_abi_size, 16)) } }, - }; - if (has_avx) try self.asmRegisterRegisterMemory( - .{ .vp_b, .shuf }, - rhs_temp_alias, - rhs_temp_alias, - rhs_mask_mem, - ) else try self.asmRegisterMemory( - .{ .p_b, .shuf }, - rhs_temp_alias, - rhs_mask_mem, - ); - - if (has_avx) try self.asmRegisterRegisterRegister( - .{ switch (elem_ty.zigTypeTag(zcu)) { - else => break :result null, - .int => .vp_, - .float => switch (elem_ty.floatBits(self.target)) { - 32 => .v_ps, - 64 => .v_pd, - 16, 80, 128 => break :result null, - else => unreachable, - }, - }, .@"or" }, - lhs_temp_alias, - lhs_temp_alias, - rhs_temp_alias, - ) else try self.asmRegisterRegister( - .{ switch (elem_ty.zigTypeTag(zcu)) { - else => break :result null, - .int => .p_, - .float => switch (elem_ty.floatBits(self.target)) { - 32 => ._ps, - 64 => ._pd, - 16, 80, 128 => break :result null, - else => unreachable, - }, - }, .@"or" }, - lhs_temp_alias, - rhs_temp_alias, - ); - break :result .{ .register = temp_regs[0] }; - } - - break :result null; - }) orelse return self.fail("TODO implement airShuffle from {f} and {f} to {f} with {f}", .{ - lhs_ty.fmt(pt), - rhs_ty.fmt(pt), - dst_ty.fmt(pt), - Value.fromInterned(extra.mask).fmtValue(pt), - }); - return self.finishAir(inst, result, .{ extra.a, extra.b, .none }); -} - -fn airAggregateInit(self: *CodeGen, inst: Air.Inst.Index) !void { +fn airAggregateInitBoolVec(self: *CodeGen, inst: Air.Inst.Index) !void { const pt = self.pt; const zcu = pt.zcu; const result_ty = self.typeOfIndex(inst); const len: usize = @intCast(result_ty.arrayLen(zcu)); const ty_pl = self.air.instructions.items(.data)[@intFromEnum(inst)].ty_pl; const elements: []const Air.Inst.Ref = @ptrCast(self.air.extra.items[ty_pl.payload..][0..len]); - const result: MCValue = result: { - switch (result_ty.zigTypeTag(zcu)) { - .@"struct" => { - if (result_ty.containerLayout(zcu) == .@"packed") return self.fail( - "TODO implement airAggregateInit for {f}", - .{result_ty.fmt(pt)}, - ); - const frame_index = try self.allocFrameIndex(.initSpill(result_ty, zcu)); - const loaded_struct = zcu.intern_pool.loadStructType(result_ty.toIntern()); - try self.genInlineMemset( - .{ .lea_frame = .{ .index = frame_index } }, - .{ .immediate = 0 }, - .{ .immediate = result_ty.abiSize(zcu) }, - .{}, - ); - for (elements, 0..) |elem, elem_i_usize| { - const elem_i: u32 = @intCast(elem_i_usize); - if ((try result_ty.structFieldValueComptime(pt, elem_i)) != null) continue; - const elem_ty = result_ty.fieldType(elem_i, zcu); - const elem_bit_size: u32 = @intCast(elem_ty.bitSize(zcu)); - if (elem_bit_size > 64) { - return self.fail( - "TODO airAggregateInit implement packed structs with large fields", - .{}, - ); - } - const elem_abi_size: u32 = @intCast(elem_ty.abiSize(zcu)); - const elem_abi_bits = elem_abi_size * 8; - const elem_off = zcu.structPackedFieldBitOffset(loaded_struct, elem_i); - const elem_byte_off: i32 = @intCast(elem_off / elem_abi_bits * elem_abi_size); - const elem_bit_off = elem_off % elem_abi_bits; - const elem_mcv = try self.resolveInst(elem); - const elem_lock = switch (elem_mcv) { - .register => |reg| self.register_manager.lockReg(reg), - .immediate => |imm| lock: { - if (imm == 0) continue; - break :lock null; - }, - else => null, - }; - defer if (elem_lock) |lock| self.register_manager.unlockReg(lock); + assert(result_ty.zigTypeTag(zcu) == .vector); + assert(result_ty.childType(zcu).toIntern() == .bool_type); - const elem_extra_bits = self.regExtraBits(elem_ty); - { - const temp_reg = try self.copyToTmpRegister(elem_ty, elem_mcv); - const temp_alias = registerAlias(temp_reg, elem_abi_size); - const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg); - defer self.register_manager.unlockReg(temp_lock); + const result_size = result_ty.abiSize(zcu); + if (result_size > 8) return self.fail("TODO airAggregateInitBoolVec over 8 bytes", .{}); - if (elem_bit_off < elem_extra_bits) { - try self.truncateRegister(elem_ty, temp_alias); - } - if (elem_bit_off > 0) try self.genShiftBinOpMir( - .{ ._l, .sh }, - elem_ty, - .{ .register = temp_alias }, - .u8, - .{ .immediate = elem_bit_off }, - ); - try self.genBinOpMir( - .{ ._, .@"or" }, - elem_ty, - .{ .load_frame = .{ .index = frame_index, .off = elem_byte_off } }, - .{ .register = temp_alias }, - ); - } - if (elem_bit_off > elem_extra_bits) { - const temp_reg = try self.copyToTmpRegister(elem_ty, elem_mcv); - const temp_alias = registerAlias(temp_reg, elem_abi_size); - const temp_lock = self.register_manager.lockRegAssumeUnused(temp_reg); - defer self.register_manager.unlockReg(temp_lock); + const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp); - if (elem_extra_bits > 0) { - try self.truncateRegister(elem_ty, temp_alias); - } - try self.genShiftBinOpMir( - .{ ._r, .sh }, - elem_ty, - .{ .register = temp_reg }, - .u8, - .{ .immediate = elem_abi_bits - elem_bit_off }, - ); - try self.genBinOpMir( - .{ ._, .@"or" }, - elem_ty, - .{ .load_frame = .{ - .index = frame_index, - .off = elem_byte_off + @as(i32, @intCast(elem_abi_size)), - } }, - .{ .register = temp_alias }, - ); - } - } - break :result .{ .load_frame = .{ .index = frame_index } }; - }, - .vector => { - const elem_ty = result_ty.childType(zcu); - if (elem_ty.toIntern() != .bool_type) return self.fail( - "TODO implement airAggregateInit for {f}", - .{result_ty.fmt(pt)}, - ); - const result_size: u32 = @intCast(result_ty.abiSize(zcu)); - const dst_reg = try self.register_manager.allocReg(inst, abi.RegisterClass.gp); - const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg); - defer self.register_manager.unlockReg(dst_lock); - try self.asmRegisterRegister( - .{ ._, .xor }, - registerAlias(dst_reg, @min(result_size, 4)), - registerAlias(dst_reg, @min(result_size, 4)), - ); + { + const dst_lock = self.register_manager.lockRegAssumeUnused(dst_reg); + defer self.register_manager.unlockReg(dst_lock); + try self.asmRegisterRegister( + .{ ._, .xor }, + registerAlias(dst_reg, @min(result_size, 4)), + registerAlias(dst_reg, @min(result_size, 4)), + ); - for (elements, 0..) |elem, elem_i| { - const elem_reg = try self.copyToTmpRegister(elem_ty, .{ .air_ref = elem }); - const elem_lock = self.register_manager.lockRegAssumeUnused(elem_reg); - defer self.register_manager.unlockReg(elem_lock); + for (elements, 0..) |elem, elem_i| { + const elem_reg = try self.copyToTmpRegister(.bool, .{ .air_ref = elem }); + const elem_lock = self.register_manager.lockRegAssumeUnused(elem_reg); + defer self.register_manager.unlockReg(elem_lock); - try self.asmRegisterImmediate( - .{ ._, .@"and" }, - registerAlias(elem_reg, @min(result_size, 4)), - .u(1), - ); - if (elem_i > 0) try self.asmRegisterImmediate( - .{ ._l, .sh }, - registerAlias(elem_reg, result_size), - .u(@intCast(elem_i)), - ); - try self.asmRegisterRegister( - .{ ._, .@"or" }, - registerAlias(dst_reg, result_size), - registerAlias(elem_reg, result_size), - ); - } - break :result .{ .register = dst_reg }; - }, - else => unreachable, + try self.asmRegisterImmediate( + .{ ._, .@"and" }, + registerAlias(elem_reg, @min(result_size, 4)), + .u(1), + ); + if (elem_i > 0) try self.asmRegisterImmediate( + .{ ._l, .sh }, + registerAlias(elem_reg, @intCast(result_size)), + .u(@intCast(elem_i)), + ); + try self.asmRegisterRegister( + .{ ._, .@"or" }, + registerAlias(dst_reg, @intCast(result_size)), + registerAlias(elem_reg, @intCast(result_size)), + ); } - }; + } + + const result: MCValue = .{ .register = dst_reg }; if (elements.len <= Air.Liveness.bpi - 1) { var buf: [Air.Liveness.bpi - 1]Air.Inst.Ref = @splat(.none); @@ -182269,15 +180743,6 @@ fn fail(cg: *CodeGen, comptime format: []const u8, args: anytype) error{ OutOfMe }; } -fn failMsg(cg: *CodeGen, msg: *Zcu.ErrorMsg) error{ OutOfMemory, CodegenFail } { - @branchHint(.cold); - const zcu = cg.pt.zcu; - return switch (cg.owner) { - .nav_index => |i| zcu.codegenFailMsg(i, msg), - .lazy_sym => |s| zcu.codegenFailTypeMsg(s.ty, msg), - }; -} - fn parseRegName(name: []const u8) ?Register { if (std.mem.startsWith(u8, name, "db")) return @enumFromInt( @intFromEnum(Register.dr0) + (std.fmt.parseInt(u4, name["db".len..], 0) catch return null), @@ -188819,7 +187284,6 @@ const Select = struct { const ptr_info = ty.ptrInfo(zcu); return switch (ptr_info.flags.vector_index) { .none => false, - .runtime => unreachable, else => ptr_info.child == .bool_type, }; }, @@ -188827,7 +187291,6 @@ const Select = struct { const ptr_info = ty.ptrInfo(zcu); return switch (ptr_info.flags.vector_index) { .none => false, - .runtime => unreachable, else => ptr_info.child == .bool_type and size.bitSize(cg.target) >= ptr_info.packed_offset.host_size, }; }, @@ -190814,7 +189277,7 @@ const Select = struct { .src0_elem_size_mul_src1 => @intCast(Select.Operand.Ref.src0.typeOf(s).elemType2(s.cg.pt.zcu).abiSize(s.cg.pt.zcu) * Select.Operand.Ref.src1.valueOf(s).immediate), .vector_index => switch (op.flags.base.ref.typeOf(s).ptrInfo(s.cg.pt.zcu).flags.vector_index) { - .none, .runtime => unreachable, + .none => unreachable, else => |vector_index| @intFromEnum(vector_index), }, .src1 => @intCast(Select.Operand.Ref.src1.valueOf(s).immediate), diff --git a/src/link/Dwarf.zig b/src/link/Dwarf.zig index e03517f97a..e8fea3c988 100644 --- a/src/link/Dwarf.zig +++ b/src/link/Dwarf.zig @@ -3158,11 +3158,7 @@ fn updateComptimeNavInner(dwarf: *Dwarf, pt: Zcu.PerThread, nav_index: InternPoo .struct_field else .struct_field); - if (loaded_struct.fieldName(ip, field_index).unwrap()) |field_name| try wip_nav.strp(field_name.toSlice(ip)) else { - var field_name_buf: [std.fmt.count("{d}", .{std.math.maxInt(u32)})]u8 = undefined; - const field_name = std.fmt.bufPrint(&field_name_buf, "{d}", .{field_index}) catch unreachable; - try wip_nav.strp(field_name); - } + try wip_nav.strp(loaded_struct.fieldName(ip, field_index).toSlice(ip)); try wip_nav.refType(field_type); if (!is_comptime) { try diw.writeUleb128(loaded_struct.offsets.get(ip)[field_index]); @@ -3187,7 +3183,7 @@ fn updateComptimeNavInner(dwarf: *Dwarf, pt: Zcu.PerThread, nav_index: InternPoo var field_bit_offset: u16 = 0; for (0..loaded_struct.field_types.len) |field_index| { try wip_nav.abbrevCode(.packed_struct_field); - try wip_nav.strp(loaded_struct.fieldName(ip, field_index).unwrap().?.toSlice(ip)); + try wip_nav.strp(loaded_struct.fieldName(ip, field_index).toSlice(ip)); const field_type: Type = .fromInterned(loaded_struct.field_types.get(ip)[field_index]); try wip_nav.refType(field_type); try diw.writeUleb128(field_bit_offset); @@ -4269,11 +4265,7 @@ fn updateLazyValue( .comptime_value_field_runtime_bits else continue); - if (loaded_struct_type.fieldName(ip, field_index).unwrap()) |field_name| try wip_nav.strp(field_name.toSlice(ip)) else { - var field_name_buf: [std.fmt.count("{d}", .{std.math.maxInt(u32)})]u8 = undefined; - const field_name = std.fmt.bufPrint(&field_name_buf, "{d}", .{field_index}) catch unreachable; - try wip_nav.strp(field_name); - } + try wip_nav.strp(loaded_struct_type.fieldName(ip, field_index).toSlice(ip)); const field_value: Value = .fromInterned(switch (aggregate.storage) { .bytes => unreachable, .elems => |elems| elems[field_index], @@ -4467,11 +4459,7 @@ fn updateContainerTypeWriterError( .struct_field else .struct_field); - if (loaded_struct.fieldName(ip, field_index).unwrap()) |field_name| try wip_nav.strp(field_name.toSlice(ip)) else { - var field_name_buf: [std.fmt.count("{d}", .{std.math.maxInt(u32)})]u8 = undefined; - const field_name = std.fmt.bufPrint(&field_name_buf, "{d}", .{field_index}) catch unreachable; - try wip_nav.strp(field_name); - } + try wip_nav.strp(loaded_struct.fieldName(ip, field_index).toSlice(ip)); try wip_nav.refType(field_type); if (!is_comptime) { try diw.writeUleb128(loaded_struct.offsets.get(ip)[field_index]); @@ -4573,11 +4561,7 @@ fn updateContainerTypeWriterError( .struct_field else .struct_field); - if (loaded_struct.fieldName(ip, field_index).unwrap()) |field_name| try wip_nav.strp(field_name.toSlice(ip)) else { - var field_name_buf: [std.fmt.count("{d}", .{std.math.maxInt(u32)})]u8 = undefined; - const field_name = std.fmt.bufPrint(&field_name_buf, "{d}", .{field_index}) catch unreachable; - try wip_nav.strp(field_name); - } + try wip_nav.strp(loaded_struct.fieldName(ip, field_index).toSlice(ip)); try wip_nav.refType(field_type); if (!is_comptime) { try diw.writeUleb128(loaded_struct.offsets.get(ip)[field_index]); @@ -4600,7 +4584,7 @@ fn updateContainerTypeWriterError( var field_bit_offset: u16 = 0; for (0..loaded_struct.field_types.len) |field_index| { try wip_nav.abbrevCode(.packed_struct_field); - try wip_nav.strp(loaded_struct.fieldName(ip, field_index).unwrap().?.toSlice(ip)); + try wip_nav.strp(loaded_struct.fieldName(ip, field_index).toSlice(ip)); const field_type: Type = .fromInterned(loaded_struct.field_types.get(ip)[field_index]); try wip_nav.refType(field_type); try diw.writeUleb128(field_bit_offset);