From d98974e826791e4546a72599a1a5fb24692d5acf Mon Sep 17 00:00:00 2001 From: Jacob Young Date: Wed, 19 Apr 2023 22:51:15 -0400 Subject: [PATCH] cbe: fix issues with atomic floats Since the Zig language documentation claims support for `.Min` and `.Max` in `@atomicRmw` with floats, allow in Sema and implement for both the llvm and C backends. --- lib/zig.h | 195 +++++++++++----- src/Sema.zig | 4 +- src/codegen/c.zig | 217 +++++++++++++----- src/codegen/llvm.zig | 8 +- src/codegen/llvm/bindings.zig | 2 + test/behavior/atomics.zig | 22 +- ...n_functions_returning_void_or_noreturn.zig | 1 - 7 files changed, 305 insertions(+), 144 deletions(-) diff --git a/lib/zig.h b/lib/zig.h index d9b97a38d4..52acaa1ee2 100644 --- a/lib/zig.h +++ b/lib/zig.h @@ -255,42 +255,90 @@ typedef char bool; #if __STDC_VERSION__ >= 201112L && !defined(__STDC_NO_ATOMICS__) #include -#define zig_atomic(type) _Atomic(type) -#define zig_cmpxchg_strong(obj, expected, desired, succ, fail, type) atomic_compare_exchange_strong_explicit(obj, &(expected), desired, succ, fail) -#define zig_cmpxchg_weak(obj, expected, desired, succ, fail, type) atomic_compare_exchange_weak_explicit (obj, &(expected), desired, succ, fail) -#define zig_atomicrmw_xchg(obj, arg, order, type) atomic_exchange_explicit (obj, arg, order) -#define zig_atomicrmw_add(obj, arg, order, type) atomic_fetch_add_explicit (obj, arg, order) -#define zig_atomicrmw_sub(obj, arg, order, type) atomic_fetch_sub_explicit (obj, arg, order) -#define zig_atomicrmw_or(obj, arg, order, type) atomic_fetch_or_explicit (obj, arg, order) -#define zig_atomicrmw_xor(obj, arg, order, type) atomic_fetch_xor_explicit (obj, arg, order) -#define zig_atomicrmw_and(obj, arg, order, type) atomic_fetch_and_explicit (obj, arg, order) -#define zig_atomicrmw_nand(obj, arg, order, type) __atomic_fetch_nand (obj, arg, order) -#define zig_atomicrmw_min(obj, arg, order, type) __atomic_fetch_min (obj, arg, order) -#define zig_atomicrmw_max(obj, arg, order, type) __atomic_fetch_max (obj, arg, order) -#define zig_atomic_store(obj, arg, order, type) atomic_store_explicit (obj, arg, order) -#define zig_atomic_load(obj, order, type) atomic_load_explicit (obj, order) +typedef enum memory_order zig_memory_order; +#define zig_atomic(Type) _Atomic(Type) +#define zig_cmpxchg_strong( obj, expected, desired, succ, fail, Type, ReprType) atomic_compare_exchange_strong_explicit(obj, &(expected), desired, succ, fail) +#define zig_cmpxchg_weak( obj, expected, desired, succ, fail, Type, ReprType) atomic_compare_exchange_weak_explicit (obj, &(expected), desired, succ, fail) +#define zig_atomicrmw_xchg(res, obj, arg, order, Type, ReprType) res = atomic_exchange_explicit (obj, arg, order) +#define zig_atomicrmw_add(res, obj, arg, order, Type, ReprType) res = atomic_fetch_add_explicit (obj, arg, order) +#define zig_atomicrmw_sub(res, obj, arg, order, Type, ReprType) res = atomic_fetch_sub_explicit (obj, arg, order) +#define zig_atomicrmw_or(res, obj, arg, order, Type, ReprType) res = atomic_fetch_or_explicit (obj, arg, order) +#define zig_atomicrmw_xor(res, obj, arg, order, Type, ReprType) res = atomic_fetch_xor_explicit (obj, arg, order) +#define zig_atomicrmw_and(res, obj, arg, order, Type, ReprType) res = atomic_fetch_and_explicit (obj, arg, order) +#define zig_atomicrmw_nand(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_nand(obj, arg, order) +#define zig_atomicrmw_min(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_min (obj, arg, order) +#define zig_atomicrmw_max(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_max (obj, arg, order) +#define zig_atomic_store( obj, arg, order, Type, ReprType) atomic_store_explicit (obj, arg, order) +#define zig_atomic_load(res, obj, order, Type, ReprType) res = atomic_load_explicit (obj, order) +#define zig_atomicrmw_xchg_float zig_atomicrmw_xchg +#define zig_atomicrmw_add_float zig_atomicrmw_add +#define zig_atomicrmw_sub_float zig_atomicrmw_sub +#define zig_atomicrmw_min_float(res, obj, arg, order, Type, ReprType) do { \ + zig_##Type zig_atomicrmw_desired; \ + zig_atomic_load(res, obj, order, Type, ReprType); \ + do { \ + zig_atomicrmw_desired = zig_libc_name_##Type(fmin)(res, arg); \ + } while (!zig_cmpxchg_weak(obj, res, zig_atomicrmw_desired, order, memory_order_relaxed, Type, ReprType)); \ +} while (0) +#define zig_atomicrmw_max_float(res, obj, arg, order, Type, ReprType) do { \ + zig_##Type zig_atomicrmw_desired; \ + zig_atomic_load(res, obj, order, Type, ReprType); \ + do { \ + zig_atomicrmw_desired = zig_libc_name_##Type(fmax)(res, arg); \ + } while (!zig_cmpxchg_weak(obj, res, zig_atomicrmw_desired, order, memory_order_relaxed, Type, ReprType)); \ +} while (0) #define zig_fence(order) atomic_thread_fence(order) #elif defined(__GNUC__) +typedef int zig_memory_order; #define memory_order_relaxed __ATOMIC_RELAXED #define memory_order_consume __ATOMIC_CONSUME #define memory_order_acquire __ATOMIC_ACQUIRE #define memory_order_release __ATOMIC_RELEASE #define memory_order_acq_rel __ATOMIC_ACQ_REL #define memory_order_seq_cst __ATOMIC_SEQ_CST -#define zig_atomic(type) type -#define zig_cmpxchg_strong(obj, expected, desired, succ, fail, type) __atomic_compare_exchange_n(obj, &(expected), desired, false, succ, fail) -#define zig_cmpxchg_weak(obj, expected, desired, succ, fail, type) __atomic_compare_exchange_n(obj, &(expected), desired, true , succ, fail) -#define zig_atomicrmw_xchg(obj, arg, order, type) __atomic_exchange_n(obj, arg, order) -#define zig_atomicrmw_add(obj, arg, order, type) __atomic_fetch_add (obj, arg, order) -#define zig_atomicrmw_sub(obj, arg, order, type) __atomic_fetch_sub (obj, arg, order) -#define zig_atomicrmw_or(obj, arg, order, type) __atomic_fetch_or (obj, arg, order) -#define zig_atomicrmw_xor(obj, arg, order, type) __atomic_fetch_xor (obj, arg, order) -#define zig_atomicrmw_and(obj, arg, order, type) __atomic_fetch_and (obj, arg, order) -#define zig_atomicrmw_nand(obj, arg, order, type) __atomic_fetch_nand(obj, arg, order) -#define zig_atomicrmw_min(obj, arg, order, type) __atomic_fetch_min (obj, arg, order) -#define zig_atomicrmw_max(obj, arg, order, type) __atomic_fetch_max (obj, arg, order) -#define zig_atomic_store(obj, arg, order, type) __atomic_store_n (obj, arg, order) -#define zig_atomic_load(obj, order, type) __atomic_load_n (obj, order) +#define zig_atomic(Type) Type +#define zig_cmpxchg_strong( obj, expected, desired, succ, fail, Type, ReprType) __atomic_compare_exchange(obj, &(expected), &(desired), false, succ, fail) +#define zig_cmpxchg_weak( obj, expected, desired, succ, fail, Type, ReprType) __atomic_compare_exchange(obj, &(expected), &(desired), true, succ, fail) +#define zig_atomicrmw_xchg(res, obj, arg, order, Type, ReprType) __atomic_exchange(obj, &(arg), &(res), order) +#define zig_atomicrmw_add(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_add (obj, arg, order) +#define zig_atomicrmw_sub(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_sub (obj, arg, order) +#define zig_atomicrmw_or(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_or (obj, arg, order) +#define zig_atomicrmw_xor(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_xor (obj, arg, order) +#define zig_atomicrmw_and(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_and (obj, arg, order) +#define zig_atomicrmw_nand(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_nand(obj, arg, order) +#define zig_atomicrmw_min(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_min (obj, arg, order) +#define zig_atomicrmw_max(res, obj, arg, order, Type, ReprType) res = __atomic_fetch_max (obj, arg, order) +#define zig_atomic_store( obj, arg, order, Type, ReprType) __atomic_store (obj, &(arg), order) +#define zig_atomic_load(res, obj, order, Type, ReprType) __atomic_load (obj, &(res), order) +#define zig_atomicrmw_xchg_float zig_atomicrmw_xchg +#define zig_atomicrmw_add_float(res, obj, arg, order, Type, ReprType) do { \ + zig_##Type zig_atomicrmw_desired; \ + zig_atomic_load(res, obj, order, Type, ReprType); \ + do { \ + zig_atomicrmw_desired = (res) + (arg); \ + } while (!zig_cmpxchg_weak(obj, res, zig_atomicrmw_desired, order, memory_order_relaxed, Type, ReprType)); \ +} while (0) +#define zig_atomicrmw_sub_float(res, obj, arg, order, Type, ReprType) do { \ + zig_##Type zig_atomicrmw_desired; \ + zig_atomic_load(res, obj, order, Type, ReprType); \ + do { \ + zig_atomicrmw_desired = (res) - (arg); \ + } while (!zig_cmpxchg_weak(obj, res, zig_atomicrmw_desired, order, memory_order_relaxed, Type, ReprType)); \ +} while (0) +#define zig_atomicrmw_min_float(res, obj, arg, order, Type, ReprType) do { \ + zig_##Type zig_atomicrmw_desired; \ + zig_atomic_load(res, obj, order, Type, ReprType); \ + do { \ + zig_atomicrmw_desired = zig_libc_name_##Type(fmin)(res, arg); \ + } while (!zig_cmpxchg_weak(obj, res, zig_atomicrmw_desired, order, memory_order_relaxed, Type, ReprType)); \ +} while (0) +#define zig_atomicrmw_max_float(res, obj, arg, order, Type, ReprType) do { \ + zig_##Type zig_atomicrmw_desired; \ + zig_atomic_load(res, obj, order, Type, ReprType); \ + do { \ + zig_atomicrmw_desired = zig_libc_name_##Type(fmax)(res, arg); \ + } while (!zig_cmpxchg_weak(obj, res, zig_atomicrmw_desired, order, memory_order_relaxed, Type, ReprType)); \ +} while (0) #define zig_fence(order) __atomic_thread_fence(order) #elif _MSC_VER && (_M_IX86 || _M_X64) #define memory_order_relaxed 0 @@ -299,20 +347,25 @@ typedef char bool; #define memory_order_release 3 #define memory_order_acq_rel 4 #define memory_order_seq_cst 5 -#define zig_atomic(type) type -#define zig_cmpxchg_strong(obj, expected, desired, succ, fail, type) zig_expand_concat(zig_msvc_cmpxchg_, type)(obj, &(expected), desired) -#define zig_cmpxchg_weak(obj, expected, desired, succ, fail, type) zig_cmpxchg_strong(obj, expected, desired, succ, fail, type) -#define zig_atomicrmw_xchg(obj, arg, order, type) zig_expand_concat(zig_msvc_atomicrmw_xchg_, type)(obj, arg) -#define zig_atomicrmw_add(obj, arg, order, type) zig_expand_concat(zig_msvc_atomicrmw_add_, type)(obj, arg) -#define zig_atomicrmw_sub(obj, arg, order, type) zig_expand_concat(zig_msvc_atomicrmw_sub_, type)(obj, arg) -#define zig_atomicrmw_or(obj, arg, order, type) zig_expand_concat(zig_msvc_atomicrmw_or_, type)(obj, arg) -#define zig_atomicrmw_xor(obj, arg, order, type) zig_expand_concat(zig_msvc_atomicrmw_xor_, type)(obj, arg) -#define zig_atomicrmw_and(obj, arg, order, type) zig_expand_concat(zig_msvc_atomicrmw_and_, type)(obj, arg) -#define zig_atomicrmw_nand(obj, arg, order, type) zig_expand_concat(zig_msvc_atomicrmw_nand_, type)(obj, arg) -#define zig_atomicrmw_min(obj, arg, order, type) zig_expand_concat(zig_msvc_atomicrmw_min_, type)(obj, arg) -#define zig_atomicrmw_max(obj, arg, order, type) zig_expand_concat(zig_msvc_atomicrmw_max_, type)(obj, arg) -#define zig_atomic_store(obj, arg, order, type) zig_expand_concat(zig_msvc_atomic_store_, type)(obj, arg) -#define zig_atomic_load(obj, order, type) zig_expand_concat(zig_msvc_atomic_load_, type)(obj) +#define zig_atomic(Type) Type +#define zig_cmpxchg_strong( obj, expected, desired, succ, fail, Type, ReprType) res = zig_msvc_cmpxchg_##Type(obj, &(expected), desired) +#define zig_cmpxchg_weak( obj, expected, desired, succ, fail, Type, ReprType) zig_cmpxchg_strong(res, obj, expected, desired, succ, fail, Type) +#define zig_atomicrmw_xchg(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_xchg_##Type(obj, arg) +#define zig_atomicrmw_add(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_add_ ##Type(obj, arg) +#define zig_atomicrmw_sub(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_sub_ ##Type(obj, arg) +#define zig_atomicrmw_or(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_or_ ##Type(obj, arg) +#define zig_atomicrmw_xor(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_xor_ ##Type(obj, arg) +#define zig_atomicrmw_and(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_and_ ##Type(obj, arg) +#define zig_atomicrmw_nand(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_nand_##Type(obj, arg) +#define zig_atomicrmw_min(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_min_ ##Type(obj, arg) +#define zig_atomicrmw_max(res, obj, arg, order, Type, ReprType) res = zig_msvc_atomicrmw_max_ ##Type(obj, arg) +#define zig_atomic_store( obj, arg, order, Type, ReprType) zig_msvc_atomic_store_ ##Type(obj, arg) +#define zig_atomic_load(res, obj, order, Type, ReprType) res = zig_msvc_atomic_load_ ##Type(obj) +#define zig_atomicrmw_xchg_float zig_atomicrmw_xchg +#define zig_atomicrmw_add_float zig_atomicrmw_add +#define zig_atomicrmw_sub_float zig_atomicrmw_sub +#define zig_atomicrmw_min_float zig_atomicrmw_min +#define zig_atomicrmw_max_float zig_atomicrmw_max #if _M_X64 #define zig_fence(order) __faststorefence() #else @@ -327,21 +380,25 @@ typedef char bool; #define memory_order_release 3 #define memory_order_acq_rel 4 #define memory_order_seq_cst 5 -#define zig_atomic(type) type -#define zig_cmpxchg_strong(obj, expected, desired, succ, fail, type) zig_unimplemented() -#define zig_cmpxchg_weak(obj, expected, desired, succ, fail, type) zig_unimplemented() -#define zig_atomicrmw_xchg(obj, arg, order, type) zig_unimplemented() -#define zig_atomicrmw_add(obj, arg, order, type) zig_unimplemented() -#define zig_atomicrmw_sub(obj, arg, order, type) zig_unimplemented() -#define zig_atomicrmw_or(obj, arg, order, type) zig_unimplemented() -#define zig_atomicrmw_xor(obj, arg, order, type) zig_unimplemented() -#define zig_atomicrmw_and(obj, arg, order, type) zig_unimplemented() -#define zig_atomicrmw_nand(obj, arg, order, type) zig_unimplemented() -#define zig_atomicrmw_min(obj, arg, order, type) zig_unimplemented() -#define zig_atomicrmw_max(obj, arg, order, type) zig_unimplemented() -#define zig_atomic_store(obj, arg, order, type) zig_unimplemented() -#define zig_atomic_load(obj, order, type) zig_unimplemented() -#define zig_fence(order) zig_unimplemented() +#define zig_atomic(Type) Type +#define zig_cmpxchg_strong( obj, expected, desired, succ, fail, Type, ReprType) zig_atomics_unavailable +#define zig_cmpxchg_weak( obj, expected, desired, succ, fail, Type, ReprType) zig_atomics_unavailable +#define zig_atomicrmw_xchg(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable +#define zig_atomicrmw_add(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable +#define zig_atomicrmw_sub(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable +#define zig_atomicrmw_or(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable +#define zig_atomicrmw_xor(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable +#define zig_atomicrmw_and(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable +#define zig_atomicrmw_nand(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable +#define zig_atomicrmw_min(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable +#define zig_atomicrmw_max(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable +#define zig_atomic_store( obj, arg, order, Type, ReprType) zig_atomics_unavailable +#define zig_atomic_load(res, obj, order, Type, ReprType) zig_atomics_unavailable +#define zig_atomicrmw_add_float(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable +#define zig_atomicrmw_sub_float(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable +#define zig_atomicrmw_min_float(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable +#define zig_atomicrmw_max_float(res, obj, arg, order, Type, ReprType) zig_atomics_unavailable +#define zig_fence(order) zig_fence_unavailable #endif #if __STDC_VERSION__ >= 201112L @@ -3461,6 +3518,8 @@ zig_float_builtins(64) zig_float_builtins(80) zig_float_builtins(128) +/* ============================ Atomics Support ============================= */ + #if _MSC_VER && (_M_IX86 || _M_X64) // TODO: zig_msvc_atomic_load should load 32 bit without interlocked on x86, and load 64 bit without interlocked on x64 @@ -3596,6 +3655,28 @@ zig_msvc_atomics(i64, int64_t, __int64, 64) desired = expected - value; \ } while (!zig_msvc_cmpxchg_##Type(obj, &expected, desired)); \ return expected; \ + } \ + static inline zig_##Type zig_msvc_atomicrmw_min_##Type(zig_##Type volatile* obj, zig_##Type value) { \ + ReprType repr; \ + zig_##Type expected; \ + zig_##Type desired; \ + repr = *(ReprType volatile*)obj; \ + memcpy(&expected, &repr, sizeof(expected)); \ + do { \ + desired = zig_libc_name_##Type(fmin)(expected, value); \ + } while (!zig_msvc_cmpxchg_##Type(obj, &expected, desired)); \ + return expected; \ + } \ + static inline zig_##Type zig_msvc_atomicrmw_max_##Type(zig_##Type volatile* obj, zig_##Type value) { \ + ReprType repr; \ + zig_##Type expected; \ + zig_##Type desired; \ + repr = *(ReprType volatile*)obj; \ + memcpy(&expected, &repr, sizeof(expected)); \ + do { \ + desired = zig_libc_name_##Type(fmax)(expected, value); \ + } while (!zig_msvc_cmpxchg_##Type(obj, &expected, desired)); \ + return expected; \ } zig_msvc_flt_atomics(f32, long, ) diff --git a/src/Sema.zig b/src/Sema.zig index b8afce2cdd..7137b78392 100644 --- a/src/Sema.zig +++ b/src/Sema.zig @@ -21319,8 +21319,8 @@ fn zirAtomicRmw(sema: *Sema, block: *Block, inst: Zir.Inst.Index) CompileError!A return sema.fail(block, op_src, "@atomicRmw with bool only allowed with .Xchg", .{}); }, .Float => switch (op) { - .Xchg, .Add, .Sub => {}, - else => return sema.fail(block, op_src, "@atomicRmw with float only allowed with .Xchg, .Add, and .Sub", .{}), + .Xchg, .Add, .Sub, .Max, .Min => {}, + else => return sema.fail(block, op_src, "@atomicRmw with float only allowed with .Xchg, .Add, .Sub, .Max, and .Min", .{}), }, else => {}, } diff --git a/src/codegen/c.zig b/src/codegen/c.zig index 31717ae5b5..88185f32d5 100644 --- a/src/codegen/c.zig +++ b/src/codegen/c.zig @@ -45,9 +45,6 @@ pub const CValue = union(enum) { identifier: []const u8, /// Render the slice as an payload.identifier (using fmtIdent) payload_identifier: []const u8, - /// Render these bytes literally. - /// TODO make this a [*:0]const u8 to save memory - bytes: []const u8, }; const BlockData = struct { @@ -1770,7 +1767,6 @@ pub const DeclGen = struct { fmtIdent("payload"), fmtIdent(ident), }), - .bytes => |bytes| return w.writeAll(bytes), } } @@ -1799,11 +1795,6 @@ pub const DeclGen = struct { fmtIdent("payload"), fmtIdent(ident), }), - .bytes => |bytes| { - try w.writeAll("(*"); - try w.writeAll(bytes); - return w.writeByte(')'); - }, } } @@ -1816,7 +1807,7 @@ pub const DeclGen = struct { fn writeCValueDerefMember(dg: *DeclGen, writer: anytype, c_value: CValue, member: CValue) !void { switch (c_value) { .none, .constant, .field, .undef => unreachable, - .new_local, .local, .arg, .arg_array, .decl, .identifier, .payload_identifier, .bytes => { + .new_local, .local, .arg, .arg_array, .decl, .identifier, .payload_identifier => { try dg.writeCValue(writer, c_value); try writer.writeAll("->"); }, @@ -5959,15 +5950,29 @@ fn airCmpxchg(f: *Function, inst: Air.Inst.Index, flavor: [*:0]const u8) !CValue try reap(f, inst, &.{ extra.ptr, extra.expected_value, extra.new_value }); const writer = f.object.writer(); const ptr_ty = f.air.typeOf(extra.ptr); + const ty = ptr_ty.childType(); + + const target = f.object.dg.module.getTarget(); + var repr_pl = Type.Payload.Bits{ + .base = .{ .tag = .int_unsigned }, + .data = @intCast(u16, ty.abiSize(target) * 8), + }; + const repr_ty = if (ty.isRuntimeFloat()) Type.initPayload(&repr_pl.base) else ty; + + const new_value_mat = try Materialize.start(f, inst, writer, ty, new_value); const local = try f.allocLocal(inst, inst_ty); if (inst_ty.isPtrLikeOptional()) { - try f.writeCValue(writer, local, .Other); - try writer.writeAll(" = "); - try f.writeCValue(writer, expected_value, .Initializer); - try writer.writeAll(";\n"); + { + const a = try Assignment.start(f, writer, ty); + try f.writeCValue(writer, local, .Other); + try a.assign(f, writer); + try f.writeCValue(writer, expected_value, .Other); + try a.end(f, writer); + } + try writer.writeAll("if ("); try writer.print("zig_cmpxchg_{s}((zig_atomic(", .{flavor}); - try f.renderType(writer, ptr_ty.childType()); + try f.renderType(writer, ty); try writer.writeByte(')'); if (ptr_ty.isVolatilePtr()) try writer.writeAll(" volatile"); try writer.writeAll(" *)"); @@ -5975,45 +5980,62 @@ fn airCmpxchg(f: *Function, inst: Air.Inst.Index, flavor: [*:0]const u8) !CValue try writer.writeAll(", "); try f.writeCValue(writer, local, .FunctionArgument); try writer.writeAll(", "); - try f.writeCValue(writer, new_value, .FunctionArgument); + try new_value_mat.mat(f, writer); try writer.writeAll(", "); try writeMemoryOrder(writer, extra.successOrder()); try writer.writeAll(", "); try writeMemoryOrder(writer, extra.failureOrder()); try writer.writeAll(", "); - try f.object.dg.renderTypeForBuiltinFnName(writer, ptr_ty.childType()); + try f.object.dg.renderTypeForBuiltinFnName(writer, ty); + try writer.writeAll(", "); + try f.object.dg.renderType(writer, repr_ty); try writer.writeByte(')'); try writer.writeAll(") {\n"); f.object.indent_writer.pushIndent(); - try f.writeCValue(writer, local, .Other); - try writer.writeAll(" = NULL;\n"); + { + const a = try Assignment.start(f, writer, ty); + try f.writeCValue(writer, local, .Other); + try a.assign(f, writer); + try writer.writeAll("NULL"); + try a.end(f, writer); + } f.object.indent_writer.popIndent(); try writer.writeAll("}\n"); } else { - try f.writeCValue(writer, local, .Other); - try writer.writeAll(".payload = "); - try f.writeCValue(writer, expected_value, .Other); - try writer.writeAll(";\n"); - try f.writeCValue(writer, local, .Other); - try writer.print(".is_null = zig_cmpxchg_{s}((zig_atomic(", .{flavor}); - try f.renderType(writer, ptr_ty.childType()); - try writer.writeByte(')'); - if (ptr_ty.isVolatilePtr()) try writer.writeAll(" volatile"); - try writer.writeAll(" *)"); - try f.writeCValue(writer, ptr, .Other); - try writer.writeAll(", "); - try f.writeCValueMember(writer, local, .{ .identifier = "payload" }); - try writer.writeAll(", "); - try f.writeCValue(writer, new_value, .FunctionArgument); - try writer.writeAll(", "); - try writeMemoryOrder(writer, extra.successOrder()); - try writer.writeAll(", "); - try writeMemoryOrder(writer, extra.failureOrder()); - try writer.writeAll(", "); - try f.object.dg.renderTypeForBuiltinFnName(writer, ptr_ty.childType()); - try writer.writeByte(')'); - try writer.writeAll(";\n"); + { + const a = try Assignment.start(f, writer, ty); + try f.writeCValueMember(writer, local, .{ .identifier = "payload" }); + try a.assign(f, writer); + try f.writeCValue(writer, expected_value, .Other); + try a.end(f, writer); + } + { + const a = try Assignment.start(f, writer, Type.bool); + try f.writeCValueMember(writer, local, .{ .identifier = "is_null" }); + try a.assign(f, writer); + try writer.print("zig_cmpxchg_{s}((zig_atomic(", .{flavor}); + try f.renderType(writer, ty); + try writer.writeByte(')'); + if (ptr_ty.isVolatilePtr()) try writer.writeAll(" volatile"); + try writer.writeAll(" *)"); + try f.writeCValue(writer, ptr, .Other); + try writer.writeAll(", "); + try f.writeCValueMember(writer, local, .{ .identifier = "payload" }); + try writer.writeAll(", "); + try new_value_mat.mat(f, writer); + try writer.writeAll(", "); + try writeMemoryOrder(writer, extra.successOrder()); + try writer.writeAll(", "); + try writeMemoryOrder(writer, extra.failureOrder()); + try writer.writeAll(", "); + try f.object.dg.renderTypeForBuiltinFnName(writer, ty); + try writer.writeAll(", "); + try f.object.dg.renderType(writer, repr_ty); + try writer.writeByte(')'); + try a.end(f, writer); + } } + try new_value_mat.end(f, inst); if (f.liveness.isUnused(inst)) { try freeLocal(f, inst, local.new_local, 0); @@ -6028,35 +6050,42 @@ fn airAtomicRmw(f: *Function, inst: Air.Inst.Index) !CValue { const extra = f.air.extraData(Air.AtomicRmw, pl_op.payload).data; const inst_ty = f.air.typeOfIndex(inst); const ptr_ty = f.air.typeOf(pl_op.operand); + const ty = ptr_ty.childType(); const ptr = try f.resolveInst(pl_op.operand); const operand = try f.resolveInst(extra.operand); try reap(f, inst, &.{ pl_op.operand, extra.operand }); const writer = f.object.writer(); const local = try f.allocLocal(inst, inst_ty); + const target = f.object.dg.module.getTarget(); + var repr_pl = Type.Payload.Bits{ + .base = .{ .tag = .int_unsigned }, + .data = @intCast(u16, ty.abiSize(target) * 8), + }; + const is_float = ty.isRuntimeFloat(); + const repr_ty = if (is_float) Type.initPayload(&repr_pl.base) else ty; + + const operand_mat = try Materialize.start(f, inst, writer, ty, operand); + try writer.print("zig_atomicrmw_{s}", .{toAtomicRmwSuffix(extra.op())}); + if (is_float) try writer.writeAll("_float"); + try writer.writeByte('('); try f.writeCValue(writer, local, .Other); - try writer.print(" = zig_atomicrmw_{s}((", .{toAtomicRmwSuffix(extra.op())}); - switch (extra.op()) { - else => { - try writer.writeAll("zig_atomic("); - try f.renderType(writer, ptr_ty.elemType()); - try writer.writeByte(')'); - }, - .Nand, .Min, .Max => { - // These are missing from stdatomic.h, so no atomic types for now. - try f.renderType(writer, ptr_ty.elemType()); - }, - } + try writer.writeAll(", (zig_atomic("); + try f.renderType(writer, ty); + try writer.writeByte(')'); if (ptr_ty.isVolatilePtr()) try writer.writeAll(" volatile"); try writer.writeAll(" *)"); try f.writeCValue(writer, ptr, .Other); try writer.writeAll(", "); - try f.writeCValue(writer, operand, .FunctionArgument); + try operand_mat.mat(f, writer); try writer.writeAll(", "); try writeMemoryOrder(writer, extra.ordering()); try writer.writeAll(", "); - try f.object.dg.renderTypeForBuiltinFnName(writer, ptr_ty.childType()); + try f.object.dg.renderTypeForBuiltinFnName(writer, ty); + try writer.writeAll(", "); + try f.object.dg.renderType(writer, repr_ty); try writer.writeAll(");\n"); + try operand_mat.end(f, inst); if (f.liveness.isUnused(inst)) { try freeLocal(f, inst, local.new_local, 0); @@ -6071,14 +6100,23 @@ fn airAtomicLoad(f: *Function, inst: Air.Inst.Index) !CValue { const ptr = try f.resolveInst(atomic_load.ptr); try reap(f, inst, &.{atomic_load.ptr}); const ptr_ty = f.air.typeOf(atomic_load.ptr); + const ty = ptr_ty.childType(); + + const target = f.object.dg.module.getTarget(); + var repr_pl = Type.Payload.Bits{ + .base = .{ .tag = .int_unsigned }, + .data = @intCast(u16, ty.abiSize(target) * 8), + }; + const repr_ty = if (ty.isRuntimeFloat()) Type.initPayload(&repr_pl.base) else ty; const inst_ty = f.air.typeOfIndex(inst); const writer = f.object.writer(); const local = try f.allocLocal(inst, inst_ty); - try f.writeCValue(writer, local, .Other); - try writer.writeAll(" = zig_atomic_load((zig_atomic("); - try f.renderType(writer, ptr_ty.elemType()); + try writer.writeAll("zig_atomic_load("); + try f.writeCValue(writer, local, .Other); + try writer.writeAll(", (zig_atomic("); + try f.renderType(writer, ty); try writer.writeByte(')'); if (ptr_ty.isVolatilePtr()) try writer.writeAll(" volatile"); try writer.writeAll(" *)"); @@ -6086,7 +6124,9 @@ fn airAtomicLoad(f: *Function, inst: Air.Inst.Index) !CValue { try writer.writeAll(", "); try writeMemoryOrder(writer, atomic_load.order); try writer.writeAll(", "); - try f.object.dg.renderTypeForBuiltinFnName(writer, ptr_ty.childType()); + try f.object.dg.renderTypeForBuiltinFnName(writer, ty); + try writer.writeAll(", "); + try f.object.dg.renderType(writer, repr_ty); try writer.writeAll(");\n"); return local; @@ -6095,22 +6135,34 @@ fn airAtomicLoad(f: *Function, inst: Air.Inst.Index) !CValue { fn airAtomicStore(f: *Function, inst: Air.Inst.Index, order: [*:0]const u8) !CValue { const bin_op = f.air.instructions.items(.data)[inst].bin_op; const ptr_ty = f.air.typeOf(bin_op.lhs); + const ty = ptr_ty.childType(); const ptr = try f.resolveInst(bin_op.lhs); const element = try f.resolveInst(bin_op.rhs); try reap(f, inst, &.{ bin_op.lhs, bin_op.rhs }); const writer = f.object.writer(); + const target = f.object.dg.module.getTarget(); + var repr_pl = Type.Payload.Bits{ + .base = .{ .tag = .int_unsigned }, + .data = @intCast(u16, ty.abiSize(target) * 8), + }; + const repr_ty = if (ty.isRuntimeFloat()) Type.initPayload(&repr_pl.base) else ty; + + const element_mat = try Materialize.start(f, inst, writer, ty, element); try writer.writeAll("zig_atomic_store((zig_atomic("); - try f.renderType(writer, ptr_ty.elemType()); + try f.renderType(writer, ty); try writer.writeByte(')'); if (ptr_ty.isVolatilePtr()) try writer.writeAll(" volatile"); try writer.writeAll(" *)"); try f.writeCValue(writer, ptr, .Other); try writer.writeAll(", "); - try f.writeCValue(writer, element, .FunctionArgument); + try element_mat.mat(f, writer); try writer.print(", {s}, ", .{order}); - try f.object.dg.renderTypeForBuiltinFnName(writer, ptr_ty.childType()); + try f.object.dg.renderTypeForBuiltinFnName(writer, ty); + try writer.writeAll(", "); + try f.object.dg.renderType(writer, repr_ty); try writer.writeAll(");\n"); + try element_mat.end(f, inst); return .none; } @@ -7370,6 +7422,45 @@ fn formatIntLiteral( try data.cty.renderLiteralSuffix(writer); } +const Materialize = struct { + local: CValue, + + pub fn start( + f: *Function, + inst: Air.Inst.Index, + writer: anytype, + ty: Type, + value: CValue, + ) !Materialize { + switch (value) { + .local_ref, .constant, .decl_ref, .undef => { + const local = try f.allocLocal(inst, ty); + + const a = try Assignment.start(f, writer, ty); + try f.writeCValue(writer, local, .Other); + try a.assign(f, writer); + try f.writeCValue(writer, value, .Other); + try a.end(f, writer); + + return .{ .local = local }; + }, + .new_local => |local| return .{ .local = .{ .local = local } }, + else => return .{ .local = value }, + } + } + + pub fn mat(self: Materialize, f: *Function, writer: anytype) !void { + try f.writeCValue(writer, self.local, .Other); + } + + pub fn end(self: Materialize, f: *Function, inst: Air.Inst.Index) !void { + switch (self.local) { + .new_local => |local| try freeLocal(f, inst, local, 0), + else => {}, + } + } +}; + const Assignment = struct { cty: CType.Index, diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig index 897a582952..76dabd3e9b 100644 --- a/src/codegen/llvm.zig +++ b/src/codegen/llvm.zig @@ -10135,14 +10135,14 @@ fn toLlvmAtomicRmwBinOp( ) llvm.AtomicRMWBinOp { return switch (op) { .Xchg => .Xchg, - .Add => if (is_float) llvm.AtomicRMWBinOp.FAdd else return .Add, - .Sub => if (is_float) llvm.AtomicRMWBinOp.FSub else return .Sub, + .Add => if (is_float) .FAdd else return .Add, + .Sub => if (is_float) .FSub else return .Sub, .And => .And, .Nand => .Nand, .Or => .Or, .Xor => .Xor, - .Max => if (is_signed) llvm.AtomicRMWBinOp.Max else return .UMax, - .Min => if (is_signed) llvm.AtomicRMWBinOp.Min else return .UMin, + .Max => if (is_float) .FMax else if (is_signed) .Max else return .UMax, + .Min => if (is_float) .FMin else if (is_signed) .Min else return .UMin, }; } diff --git a/src/codegen/llvm/bindings.zig b/src/codegen/llvm/bindings.zig index 55f0ba8963..0e5a2d0a7d 100644 --- a/src/codegen/llvm/bindings.zig +++ b/src/codegen/llvm/bindings.zig @@ -1436,6 +1436,8 @@ pub const AtomicRMWBinOp = enum(c_int) { UMin, FAdd, FSub, + FMax, + FMin, }; pub const TypeKind = enum(c_int) { diff --git a/test/behavior/atomics.zig b/test/behavior/atomics.zig index 6b6e7c8430..c7098a4c4d 100644 --- a/test/behavior/atomics.zig +++ b/test/behavior/atomics.zig @@ -209,15 +209,7 @@ test "atomicrmw with floats" { if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_c) { - // TODO: test.c:34929:7: error: address argument to atomic operation must be a pointer to integer or pointer ('zig_f32 *' (aka 'float *') invalid - // when compiling with -std=c99 -pedantic - return error.SkipZigTest; - } - - if ((builtin.zig_backend == .stage2_llvm or builtin.zig_backend == .stage2_c) and - builtin.cpu.arch == .aarch64) - { + if (builtin.zig_backend == .stage2_llvm and builtin.cpu.arch == .aarch64) { // https://github.com/ziglang/zig/issues/10627 return error.SkipZigTest; } @@ -234,6 +226,10 @@ fn testAtomicRmwFloat() !void { try expect(x == 6); _ = @atomicRmw(f32, &x, .Sub, 2, .SeqCst); try expect(x == 4); + _ = @atomicRmw(f32, &x, .Max, 13, .SeqCst); + try expect(x == 13); + _ = @atomicRmw(f32, &x, .Min, 42, .SeqCst); + try expect(x == 13); } test "atomicrmw with ints" { @@ -242,10 +238,6 @@ test "atomicrmw with ints" { if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_c and builtin.cpu.arch == .aarch64) { - return error.SkipZigTest; - } - try testAtomicRmwInts(); comptime try testAtomicRmwInts(); } @@ -390,10 +382,6 @@ test "atomics with different types" { if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_c and builtin.cpu.arch == .aarch64) { - return error.SkipZigTest; - } - try testAtomicsWithType(bool, true, false); try testAtomicsWithType(u1, 0, 1); diff --git a/test/behavior/builtin_functions_returning_void_or_noreturn.zig b/test/behavior/builtin_functions_returning_void_or_noreturn.zig index 3caf01542c..0d4598bd09 100644 --- a/test/behavior/builtin_functions_returning_void_or_noreturn.zig +++ b/test/behavior/builtin_functions_returning_void_or_noreturn.zig @@ -7,7 +7,6 @@ var x: u8 = 1; // This excludes builtin functions that return void or noreturn that cannot be tested. test { if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO - if (builtin.zig_backend == .stage2_c) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO