diff --git a/lib/std/atomic/int.zig b/lib/std/atomic/int.zig index 94985b914f..446059e7ef 100644 --- a/lib/std/atomic/int.zig +++ b/lib/std/atomic/int.zig @@ -1,6 +1,3 @@ -const builtin = @import("builtin"); -const AtomicOrder = builtin.AtomicOrder; - /// Thread-safe, lock-free integer pub fn Int(comptime T: type) type { return struct { @@ -14,16 +11,16 @@ pub fn Int(comptime T: type) type { /// Returns previous value pub fn incr(self: *Self) T { - return @atomicRmw(T, &self.unprotected_value, builtin.AtomicRmwOp.Add, 1, AtomicOrder.SeqCst); + return @atomicRmw(T, &self.unprotected_value, .Add, 1, .SeqCst); } /// Returns previous value pub fn decr(self: *Self) T { - return @atomicRmw(T, &self.unprotected_value, builtin.AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst); + return @atomicRmw(T, &self.unprotected_value, .Sub, 1, .SeqCst); } pub fn get(self: *Self) T { - return @atomicLoad(T, &self.unprotected_value, AtomicOrder.SeqCst); + return @atomicLoad(T, &self.unprotected_value, .SeqCst); } pub fn set(self: *Self, new_value: T) void { @@ -31,11 +28,11 @@ pub fn Int(comptime T: type) type { } pub fn xchg(self: *Self, new_value: T) T { - return @atomicRmw(T, &self.unprotected_value, builtin.AtomicRmwOp.Xchg, new_value, AtomicOrder.SeqCst); + return @atomicRmw(T, &self.unprotected_value, .Xchg, new_value, .SeqCst); } pub fn fetchAdd(self: *Self, op: T) T { - return @atomicRmw(T, &self.unprotected_value, builtin.AtomicRmwOp.Add, op, AtomicOrder.SeqCst); + return @atomicRmw(T, &self.unprotected_value, .Add, op, .SeqCst); } }; } diff --git a/src/all_types.hpp b/src/all_types.hpp index 149b7f4647..14b99228ca 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -3567,8 +3567,6 @@ struct IrInstGenCmpxchg { IrInstGen *cmp_value; IrInstGen *new_value; IrInstGen *result_loc; - // non null if operand needs widening and truncating - ZigType *actual_type; bool is_weak; }; @@ -4201,8 +4199,6 @@ struct IrInstGenAtomicRmw { IrInstGen *ptr; IrInstGen *operand; - // non null if operand needs widening and truncating - ZigType *actual_type; AtomicRmwOp op; AtomicOrder ordering; }; @@ -4219,8 +4215,6 @@ struct IrInstGenAtomicLoad { IrInstGen base; IrInstGen *ptr; - // non null if operand needs widening and truncating - ZigType *actual_type; AtomicOrder ordering; }; @@ -4238,8 +4232,6 @@ struct IrInstGenAtomicStore { IrInstGen *ptr; IrInstGen *value; - // non null if operand needs widening and truncating - ZigType *actual_type; AtomicOrder ordering; }; diff --git a/src/codegen.cpp b/src/codegen.cpp index 66cf919d88..55713c1b88 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -5219,18 +5219,48 @@ static enum ZigLLVM_AtomicRMWBinOp to_ZigLLVMAtomicRMWBinOp(AtomicRmwOp op, bool zig_unreachable(); } +static LLVMTypeRef get_atomic_abi_type(CodeGen *g, IrInstGen *instruction) { + // If the operand type of an atomic operation is not a power of two sized + // we need to widen it before using it and then truncate the result. + + ir_assert(instruction->value->type->id == ZigTypeIdPointer, instruction); + ZigType *operand_type = instruction->value->type->data.pointer.child_type; + if (operand_type->id == ZigTypeIdInt || operand_type->id == ZigTypeIdEnum) { + if (operand_type->id == ZigTypeIdEnum) { + operand_type = operand_type->data.enumeration.tag_int_type; + } + auto bit_count = operand_type->data.integral.bit_count; + bool is_signed = operand_type->data.integral.is_signed; + + ir_assert(bit_count != 0, instruction); + if (bit_count == 1 || !is_power_of_2(bit_count)) { + return get_llvm_type(g, get_int_type(g, is_signed, operand_type->abi_size * 8)); + } else { + return nullptr; + } + } else if (operand_type->id == ZigTypeIdFloat) { + return nullptr; + } else if (operand_type->id == ZigTypeIdBool) { + return g->builtin_types.entry_u8->llvm_type; + } else { + ir_assert(get_codegen_ptr_type_bail(g, operand_type) != nullptr, instruction); + return nullptr; + } +} + static LLVMValueRef ir_render_cmpxchg(CodeGen *g, IrExecutableGen *executable, IrInstGenCmpxchg *instruction) { LLVMValueRef ptr_val = ir_llvm_value(g, instruction->ptr); LLVMValueRef cmp_val = ir_llvm_value(g, instruction->cmp_value); LLVMValueRef new_val = ir_llvm_value(g, instruction->new_value); ZigType *operand_type = instruction->new_value->value->type; - if (instruction->actual_type != nullptr) { + LLVMTypeRef actual_abi_type = get_atomic_abi_type(g, instruction->ptr); + if (actual_abi_type != nullptr) { // operand needs widening and truncating ptr_val = LLVMBuildBitCast(g->builder, ptr_val, - LLVMPointerType(get_llvm_type(g, instruction->actual_type), 0), ""); - cmp_val = LLVMBuildZExt(g->builder, cmp_val, get_llvm_type(g, instruction->actual_type), ""); - new_val = LLVMBuildZExt(g->builder, new_val, get_llvm_type(g, instruction->actual_type), ""); + LLVMPointerType(actual_abi_type, 0), ""); + cmp_val = LLVMBuildZExt(g->builder, cmp_val, actual_abi_type, ""); + new_val = LLVMBuildZExt(g->builder, new_val, actual_abi_type, ""); } LLVMAtomicOrdering success_order = to_LLVMAtomicOrdering(instruction->success_order); @@ -5245,7 +5275,7 @@ static LLVMValueRef ir_render_cmpxchg(CodeGen *g, IrExecutableGen *executable, I if (!handle_is_ptr(g, optional_type)) { LLVMValueRef payload_val = LLVMBuildExtractValue(g->builder, result_val, 0, ""); - if (instruction->actual_type != nullptr) { + if (actual_abi_type != nullptr) { payload_val = LLVMBuildTrunc(g->builder, payload_val, get_llvm_type(g, operand_type), ""); } LLVMValueRef success_bit = LLVMBuildExtractValue(g->builder, result_val, 1, ""); @@ -5262,7 +5292,7 @@ static LLVMValueRef ir_render_cmpxchg(CodeGen *g, IrExecutableGen *executable, I ir_assert(type_has_bits(g, child_type), &instruction->base); LLVMValueRef payload_val = LLVMBuildExtractValue(g->builder, result_val, 0, ""); - if (instruction->actual_type != nullptr) { + if (actual_abi_type != nullptr) { payload_val = LLVMBuildTrunc(g->builder, payload_val, get_llvm_type(g, operand_type), ""); } LLVMValueRef val_ptr = LLVMBuildStructGEP(g->builder, result_loc, maybe_child_index, ""); @@ -5842,11 +5872,12 @@ static LLVMValueRef ir_render_atomic_rmw(CodeGen *g, IrExecutableGen *executable LLVMValueRef ptr = ir_llvm_value(g, instruction->ptr); LLVMValueRef operand = ir_llvm_value(g, instruction->operand); - if (instruction->actual_type != nullptr) { + LLVMTypeRef actual_abi_type = get_atomic_abi_type(g, instruction->ptr); + if (actual_abi_type != nullptr) { // operand needs widening and truncating LLVMValueRef casted_ptr = LLVMBuildBitCast(g->builder, ptr, - LLVMPointerType(get_llvm_type(g, instruction->actual_type), 0), ""); - LLVMValueRef casted_operand = LLVMBuildZExt(g->builder, operand, get_llvm_type(g, instruction->actual_type), ""); + LLVMPointerType(actual_abi_type, 0), ""); + LLVMValueRef casted_operand = LLVMBuildZExt(g->builder, operand, actual_abi_type, ""); LLVMValueRef uncasted_result = ZigLLVMBuildAtomicRMW(g->builder, op, casted_ptr, casted_operand, ordering, g->is_single_threaded); return LLVMBuildTrunc(g->builder, uncasted_result, get_llvm_type(g, operand_type), ""); @@ -5872,10 +5903,11 @@ static LLVMValueRef ir_render_atomic_load(CodeGen *g, IrExecutableGen *executabl LLVMValueRef ptr = ir_llvm_value(g, instruction->ptr); ZigType *operand_type = instruction->ptr->value->type->data.pointer.child_type; - if (instruction->actual_type != nullptr) { + LLVMTypeRef actual_abi_type = get_atomic_abi_type(g, instruction->ptr); + if (actual_abi_type != nullptr) { // operand needs widening and truncating ptr = LLVMBuildBitCast(g->builder, ptr, - LLVMPointerType(get_llvm_type(g, instruction->actual_type), 0), ""); + LLVMPointerType(actual_abi_type, 0), ""); LLVMValueRef load_inst = gen_load(g, ptr, instruction->ptr->value->type, ""); LLVMSetOrdering(load_inst, ordering); return LLVMBuildTrunc(g->builder, load_inst, get_llvm_type(g, operand_type), ""); @@ -5892,11 +5924,12 @@ static LLVMValueRef ir_render_atomic_store(CodeGen *g, IrExecutableGen *executab LLVMValueRef ptr = ir_llvm_value(g, instruction->ptr); LLVMValueRef value = ir_llvm_value(g, instruction->value); - if (instruction->actual_type != nullptr) { + LLVMTypeRef actual_abi_type = get_atomic_abi_type(g, instruction->ptr); + if (actual_abi_type != nullptr) { // operand needs widening ptr = LLVMBuildBitCast(g->builder, ptr, - LLVMPointerType(get_llvm_type(g, instruction->actual_type), 0), ""); - value = LLVMBuildZExt(g->builder, value, get_llvm_type(g, instruction->actual_type), ""); + LLVMPointerType(actual_abi_type, 0), ""); + value = LLVMBuildZExt(g->builder, value, actual_abi_type, ""); } LLVMValueRef store_inst = gen_store(g, value, ptr, instruction->ptr->value->type); LLVMSetOrdering(store_inst, ordering); diff --git a/src/ir.cpp b/src/ir.cpp index 338f803bbe..dc3e7941d6 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -227,7 +227,7 @@ static IrInstGen *ir_analyze_container_field_ptr(IrAnalyze *ira, Buf *field_name static void ir_assert(bool ok, IrInst* source_instruction); static void ir_assert_gen(bool ok, IrInstGen *source_instruction); static IrInstGen *ir_get_var_ptr(IrAnalyze *ira, IrInst *source_instr, ZigVar *var); -static ZigType *ir_resolve_atomic_operand_type(IrAnalyze *ira, IrInstGen *op, ZigType **actual_type); +static ZigType *ir_resolve_atomic_operand_type(IrAnalyze *ira, IrInstGen *op); static IrInstSrc *ir_lval_wrap(IrBuilderSrc *irb, Scope *scope, IrInstSrc *value, LVal lval, ResultLoc *result_loc); static IrInstSrc *ir_expr_wrap(IrBuilderSrc *irb, Scope *scope, IrInstSrc *inst, ResultLoc *result_loc); static ZigType *adjust_ptr_align(CodeGen *g, ZigType *ptr_type, uint32_t new_align); @@ -3406,7 +3406,7 @@ static IrInstSrc *ir_build_cmpxchg_src(IrBuilderSrc *irb, Scope *scope, AstNode static IrInstGen *ir_build_cmpxchg_gen(IrAnalyze *ira, IrInst *source_instruction, ZigType *result_type, IrInstGen *ptr, IrInstGen *cmp_value, IrInstGen *new_value, - AtomicOrder success_order, AtomicOrder failure_order, bool is_weak, IrInstGen *result_loc, ZigType *actual_type) + AtomicOrder success_order, AtomicOrder failure_order, bool is_weak, IrInstGen *result_loc) { IrInstGenCmpxchg *instruction = ir_build_inst_gen(&ira->new_irb, source_instruction->scope, source_instruction->source_node); @@ -3418,7 +3418,6 @@ static IrInstGen *ir_build_cmpxchg_gen(IrAnalyze *ira, IrInst *source_instructio instruction->failure_order = failure_order; instruction->is_weak = is_weak; instruction->result_loc = result_loc; - instruction->actual_type = actual_type; ir_ref_inst_gen(ptr, ira->new_irb.current_basic_block); ir_ref_inst_gen(cmp_value, ira->new_irb.current_basic_block); @@ -4555,7 +4554,7 @@ static IrInstSrc *ir_build_atomic_rmw_src(IrBuilderSrc *irb, Scope *scope, AstNo } static IrInstGen *ir_build_atomic_rmw_gen(IrAnalyze *ira, IrInst *source_instr, - IrInstGen *ptr, IrInstGen *operand, AtomicRmwOp op, AtomicOrder ordering, ZigType *operand_type, ZigType *actual_type) + IrInstGen *ptr, IrInstGen *operand, AtomicRmwOp op, AtomicOrder ordering, ZigType *operand_type) { IrInstGenAtomicRmw *instruction = ir_build_inst_gen(&ira->new_irb, source_instr->scope, source_instr->source_node); instruction->base.value->type = operand_type; @@ -4563,7 +4562,6 @@ static IrInstGen *ir_build_atomic_rmw_gen(IrAnalyze *ira, IrInst *source_instr, instruction->op = op; instruction->operand = operand; instruction->ordering = ordering; - instruction->actual_type = actual_type; ir_ref_inst_gen(ptr, ira->new_irb.current_basic_block); ir_ref_inst_gen(operand, ira->new_irb.current_basic_block); @@ -4587,14 +4585,13 @@ static IrInstSrc *ir_build_atomic_load_src(IrBuilderSrc *irb, Scope *scope, AstN } static IrInstGen *ir_build_atomic_load_gen(IrAnalyze *ira, IrInst *source_instr, - IrInstGen *ptr, AtomicOrder ordering, ZigType *operand_type, ZigType *actual_type) + IrInstGen *ptr, AtomicOrder ordering, ZigType *operand_type) { IrInstGenAtomicLoad *instruction = ir_build_inst_gen(&ira->new_irb, source_instr->scope, source_instr->source_node); instruction->base.value->type = operand_type; instruction->ptr = ptr; instruction->ordering = ordering; - instruction->actual_type = actual_type; ir_ref_inst_gen(ptr, ira->new_irb.current_basic_block); @@ -4619,14 +4616,13 @@ static IrInstSrc *ir_build_atomic_store_src(IrBuilderSrc *irb, Scope *scope, Ast } static IrInstGen *ir_build_atomic_store_gen(IrAnalyze *ira, IrInst *source_instr, - IrInstGen *ptr, IrInstGen *value, AtomicOrder ordering, ZigType *actual_type) + IrInstGen *ptr, IrInstGen *value, AtomicOrder ordering) { IrInstGenAtomicStore *instruction = ir_build_inst_void(&ira->new_irb, source_instr->scope, source_instr->source_node); instruction->ptr = ptr; instruction->value = value; instruction->ordering = ordering; - instruction->actual_type = actual_type; ir_ref_inst_gen(ptr, ira->new_irb.current_basic_block); ir_ref_inst_gen(value, ira->new_irb.current_basic_block); @@ -25125,8 +25121,7 @@ static IrInstGen *ir_analyze_instruction_embed_file(IrAnalyze *ira, IrInstSrcEmb } static IrInstGen *ir_analyze_instruction_cmpxchg(IrAnalyze *ira, IrInstSrcCmpxchg *instruction) { - ZigType *actual_type; - ZigType *operand_type = ir_resolve_atomic_operand_type(ira, instruction->type_value->child, &actual_type); + ZigType *operand_type = ir_resolve_atomic_operand_type(ira, instruction->type_value->child); if (type_is_invalid(operand_type)) return ira->codegen->invalid_inst_gen; @@ -25203,29 +25198,34 @@ static IrInstGen *ir_analyze_instruction_cmpxchg(IrAnalyze *ira, IrInstSrcCmpxch // special case zero bit types if (type_has_one_possible_value(ira->codegen, operand_type) == OnePossibleValueYes) { - ZigValue *val = ira->codegen->pass1_arena->allocate(1); - val->special = ConstValSpecialStatic; - val->type = result_type; - set_optional_value_to_null(val); - return ir_const_move(ira, &instruction->base.base, val); + IrInstGen *result = ir_const(ira, &instruction->base.base, result_type); + set_optional_value_to_null(result->value); + return result; } if (instr_is_comptime(casted_ptr) && casted_ptr->value->data.x_ptr.mut != ConstPtrMutRuntimeVar && instr_is_comptime(casted_cmp_value) && instr_is_comptime(casted_new_value)) { - IrInstGen *result = ir_get_deref(ira, &instruction->base.base, casted_ptr, nullptr); - ZigValue *op1_val = ir_resolve_const(ira, result, UndefBad); + ZigValue *ptr_val = ir_resolve_const(ira, casted_ptr, UndefBad); + if (ptr_val == nullptr) + return ira->codegen->invalid_inst_gen; + + ZigValue *op1_val = const_ptr_pointee(ira, ira->codegen, ptr_val, instruction->base.base.source_node); + if (op1_val == nullptr) + return ira->codegen->invalid_inst_gen; + ZigValue *op2_val = ir_resolve_const(ira, casted_cmp_value, UndefBad); + if (op2_val == nullptr) + return ira->codegen->invalid_inst_gen; + bool eql = const_values_equal(ira->codegen, op1_val, op2_val); - ZigValue *val = ira->codegen->pass1_arena->allocate(1); - val->special = ConstValSpecialStatic; - val->type = result_type; + IrInstGen *result = ir_const(ira, &instruction->base.base, result_type); if (eql) { ir_analyze_store_ptr(ira, &instruction->base.base, casted_ptr, casted_new_value, false); - set_optional_value_to_null(val); + set_optional_value_to_null(result->value); } else { - set_optional_payload(val, op1_val); + set_optional_payload(result->value, op1_val); } - return ir_const_move(ira, &instruction->base.base, val); + return result; } IrInstGen *result_loc; @@ -25241,7 +25241,7 @@ static IrInstGen *ir_analyze_instruction_cmpxchg(IrAnalyze *ira, IrInstSrcCmpxch return ir_build_cmpxchg_gen(ira, &instruction->base.base, result_type, casted_ptr, casted_cmp_value, casted_new_value, - success_order, failure_order, instruction->is_weak, result_loc, actual_type); + success_order, failure_order, instruction->is_weak, result_loc); } static IrInstGen *ir_analyze_instruction_fence(IrAnalyze *ira, IrInstSrcFence *instruction) { @@ -28333,12 +28333,11 @@ static IrInstGen *ir_analyze_instruction_tag_type(IrAnalyze *ira, IrInstSrcTagTy } } -static ZigType *ir_resolve_atomic_operand_type(IrAnalyze *ira, IrInstGen *op, ZigType **actual_type) { +static ZigType *ir_resolve_atomic_operand_type(IrAnalyze *ira, IrInstGen *op) { ZigType *operand_type = ir_resolve_type(ira, op); if (type_is_invalid(operand_type)) return ira->codegen->builtin_types.entry_invalid; - *actual_type = nullptr; if (operand_type->id == ZigTypeIdInt || operand_type->id == ZigTypeIdEnum) { ZigType *int_type; if (operand_type->id == ZigTypeIdEnum) { @@ -28355,10 +28354,6 @@ static ZigType *ir_resolve_atomic_operand_type(IrAnalyze *ira, IrInstGen *op, Zi max_atomic_bits, bit_count)); return ira->codegen->builtin_types.entry_invalid; } - - if (bit_count == 1 || !is_power_of_2(bit_count)) { - *actual_type = get_int_type(ira->codegen, int_type->data.integral.is_signed, int_type->abi_size * 8); - } } else if (operand_type->id == ZigTypeIdFloat) { uint32_t max_atomic_bits = target_arch_largest_atomic_bits(ira->codegen->zig_target->arch); if (operand_type->data.floating.bit_count > max_atomic_bits) { @@ -28369,7 +28364,6 @@ static ZigType *ir_resolve_atomic_operand_type(IrAnalyze *ira, IrInstGen *op, Zi } } else if (operand_type->id == ZigTypeIdBool) { // will be treated as u8 - *actual_type = ira->codegen->builtin_types.entry_u8; } else { Error err; ZigType *operand_ptr_type; @@ -28387,8 +28381,7 @@ static ZigType *ir_resolve_atomic_operand_type(IrAnalyze *ira, IrInstGen *op, Zi } static IrInstGen *ir_analyze_instruction_atomic_rmw(IrAnalyze *ira, IrInstSrcAtomicRmw *instruction) { - ZigType *actual_type; - ZigType *operand_type = ir_resolve_atomic_operand_type(ira, instruction->operand_type->child, &actual_type); + ZigType *operand_type = ir_resolve_atomic_operand_type(ira, instruction->operand_type->child); if (type_is_invalid(operand_type)) return ira->codegen->invalid_inst_gen; @@ -28451,12 +28444,11 @@ static IrInstGen *ir_analyze_instruction_atomic_rmw(IrAnalyze *ira, IrInstSrcAto } return ir_build_atomic_rmw_gen(ira, &instruction->base.base, casted_ptr, casted_operand, op, - ordering, operand_type, actual_type); + ordering, operand_type); } static IrInstGen *ir_analyze_instruction_atomic_load(IrAnalyze *ira, IrInstSrcAtomicLoad *instruction) { - ZigType *actual_type; - ZigType *operand_type = ir_resolve_atomic_operand_type(ira, instruction->operand_type->child, &actual_type); + ZigType *operand_type = ir_resolve_atomic_operand_type(ira, instruction->operand_type->child); if (type_is_invalid(operand_type)) return ira->codegen->invalid_inst_gen; @@ -28486,12 +28478,11 @@ static IrInstGen *ir_analyze_instruction_atomic_load(IrAnalyze *ira, IrInstSrcAt return result; } - return ir_build_atomic_load_gen(ira, &instruction->base.base, casted_ptr, ordering, operand_type, actual_type); + return ir_build_atomic_load_gen(ira, &instruction->base.base, casted_ptr, ordering, operand_type); } static IrInstGen *ir_analyze_instruction_atomic_store(IrAnalyze *ira, IrInstSrcAtomicStore *instruction) { - ZigType *actual_type; - ZigType *operand_type = ir_resolve_atomic_operand_type(ira, instruction->operand_type->child, &actual_type); + ZigType *operand_type = ir_resolve_atomic_operand_type(ira, instruction->operand_type->child); if (type_is_invalid(operand_type)) return ira->codegen->invalid_inst_gen; @@ -28535,7 +28526,7 @@ static IrInstGen *ir_analyze_instruction_atomic_store(IrAnalyze *ira, IrInstSrcA return result; } - return ir_build_atomic_store_gen(ira, &instruction->base.base, casted_ptr, casted_value, ordering, actual_type); + return ir_build_atomic_store_gen(ira, &instruction->base.base, casted_ptr, casted_value, ordering); } static IrInstGen *ir_analyze_instruction_save_err_ret_addr(IrAnalyze *ira, IrInstSrcSaveErrRetAddr *instruction) {