From 5bbd482286930a17d37ef46396c419edee98573c Mon Sep 17 00:00:00 2001 From: Luuk de Gram Date: Tue, 18 Apr 2023 21:06:03 +0200 Subject: [PATCH 1/8] wasm: implement `cmpxchg{weak/strong}` --- src/arch/wasm/CodeGen.zig | 41 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/src/arch/wasm/CodeGen.zig b/src/arch/wasm/CodeGen.zig index 9bf39b73f1..cdeaa09b5e 100644 --- a/src/arch/wasm/CodeGen.zig +++ b/src/arch/wasm/CodeGen.zig @@ -1956,8 +1956,6 @@ fn genInst(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { .is_err_ptr, .is_non_err_ptr, - .cmpxchg_weak, - .cmpxchg_strong, .fence, .atomic_load, .atomic_store_unordered, @@ -1977,6 +1975,9 @@ fn genInst(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { .c_va_start, => |tag| return func.fail("TODO: Implement wasm inst: {s}", .{@tagName(tag)}), + .cmpxchg_weak => func.airCmpxchg(inst), + .cmpxchg_strong => func.airCmpxchg(inst), + .add_optimized, .addwrap_optimized, .sub_optimized, @@ -6597,3 +6598,39 @@ fn airErrorSetHasValue(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { return func.finishAir(inst, result, &.{ty_op.operand}); } + +fn airCmpxchg(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const ty_pl = func.air.instructions.items(.data)[inst].ty_pl; + const extra = func.air.extraData(Air.Cmpxchg, ty_pl.payload).data; + + const ptr_ty = func.air.typeOf(extra.ptr); + const ty = ptr_ty.childType(); + const result_ty = func.air.typeOfIndex(inst); + + const ptr_operand = try func.resolveInst(extra.ptr); + const expected_val = try func.resolveInst(extra.expected_value); + const new_val = try func.resolveInst(extra.new_value); + + const ptr_val = try WValue.toLocal(try func.load(ptr_operand, ty, 0), func, ty); + + try func.lowerToStack(ptr_operand); + try func.emitWValue(new_val); + try func.emitWValue(ptr_val); + const cmp_tmp = try func.cmp(ptr_val, expected_val, ty, .eq); + const cmp_result = try cmp_tmp.toLocal(func, Type.bool); + try func.emitWValue(cmp_result); + try func.addTag(.select); + try func.store(.stack, .stack, ty, 0); + try func.addImm32(-1); + try func.emitWValue(cmp_result); + try func.addTag(.i32_xor); + try func.addImm32(1); + try func.addTag(.i32_and); + const and_result = try WValue.toLocal(.stack, func, Type.bool); + + const result_ptr = try func.allocStack(result_ty); + try func.store(result_ptr, and_result, Type.bool, @intCast(u32, ty.abiSize(func.target))); + try func.store(result_ptr, ptr_val, ty, 0); + + return func.finishAir(inst, result_ptr, &.{ extra.ptr, extra.new_value, extra.expected_value }); +} From 650976b2262a9ce715ebeb1e5949b7b453b75f2f Mon Sep 17 00:00:00 2001 From: Luuk de Gram Date: Wed, 19 Apr 2023 19:46:53 +0200 Subject: [PATCH 2/8] wasm: use atomic feature for `@cmpxchg` when enabled When the user passes the cpu feature `atomics` to the target triple, the backend will lower the AIR instruction using opcodes from the atomics feature instead of manually lowering it. --- src/arch/wasm/CodeGen.zig | 50 +++++++++++++++++++++++++++++++-------- src/arch/wasm/Emit.zig | 16 +++++++++++-- 2 files changed, 54 insertions(+), 12 deletions(-) diff --git a/src/arch/wasm/CodeGen.zig b/src/arch/wasm/CodeGen.zig index cdeaa09b5e..786b3fea76 100644 --- a/src/arch/wasm/CodeGen.zig +++ b/src/arch/wasm/CodeGen.zig @@ -940,6 +940,14 @@ fn addMemArg(func: *CodeGen, tag: Mir.Inst.Tag, mem_arg: Mir.MemArg) error{OutOf try func.addInst(.{ .tag = tag, .data = .{ .payload = extra_index } }); } +/// Inserts an instruction from the 'atomics' feature which accesses wasm's linear memory dependent on the +/// given `tag`. +fn addAtomicMemArg(func: *CodeGen, tag: wasm.AtomicsOpcode, mem_arg: Mir.MemArg) error{OutOfMemory}!void { + const extra_index = try func.addExtra(@as(struct { val: u32 }, .{ .val = wasm.atomicsOpcode(tag) })); + _ = try func.addExtra(mem_arg); + try func.addInst(.{ .tag = .atomics_prefix, .data = .{ .payload = extra_index } }); +} + /// Appends entries to `mir_extra` based on the type of `extra`. /// Returns the index into `mir_extra` fn addExtra(func: *CodeGen, extra: anytype) error{OutOfMemory}!u32 { @@ -6518,6 +6526,7 @@ fn getTagNameFunction(func: *CodeGen, enum_ty: Type) InnerError!u32 { return func.bin_file.createFunction(func_name, func_type, &body_list, &relocs); } +<<<<<<< HEAD fn airErrorSetHasValue(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { const ty_op = func.air.instructions.items(.data)[inst].ty_op; @@ -6599,6 +6608,10 @@ fn airErrorSetHasValue(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { return func.finishAir(inst, result, &.{ty_op.operand}); } +inline fn useAtomicFeature(func: *const CodeGen) bool { + return std.Target.wasm.featureSetHas(func.target.cpu.features, .atomics); +} + fn airCmpxchg(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { const ty_pl = func.air.instructions.items(.data)[inst].ty_pl; const extra = func.air.extraData(Air.Cmpxchg, ty_pl.payload).data; @@ -6611,18 +6624,35 @@ fn airCmpxchg(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { const expected_val = try func.resolveInst(extra.expected_value); const new_val = try func.resolveInst(extra.new_value); - const ptr_val = try WValue.toLocal(try func.load(ptr_operand, ty, 0), func, ty); + const ptr_val = if (func.useAtomicFeature()) val: { + const val_local = try func.allocLocal(ty); + try func.emitWValue(ptr_operand); + try func.emitWValue(expected_val); + try func.emitWValue(new_val); + try func.addAtomicMemArg(.i32_atomic_rmw_cmpxchg, .{ + .offset = ptr_operand.offset(), + .alignment = ty.abiAlignment(func.target), + }); + try func.addLabel(.local_tee, val_local.local.value); + _ = try func.cmp(.stack, expected_val, ty, .eq); + break :val val_local; + } else val: { + const ptr_val = try WValue.toLocal(try func.load(ptr_operand, ty, 0), func, ty); + + try func.lowerToStack(ptr_operand); + try func.emitWValue(new_val); + try func.emitWValue(ptr_val); + const cmp_tmp = try func.cmp(ptr_val, expected_val, ty, .eq); + const cmp_result = try cmp_tmp.toLocal(func, Type.bool); + try func.emitWValue(cmp_result); + try func.addTag(.select); + try func.store(.stack, .stack, ty, 0); + try func.emitWValue(cmp_result); + + break :val ptr_val; + }; - try func.lowerToStack(ptr_operand); - try func.emitWValue(new_val); - try func.emitWValue(ptr_val); - const cmp_tmp = try func.cmp(ptr_val, expected_val, ty, .eq); - const cmp_result = try cmp_tmp.toLocal(func, Type.bool); - try func.emitWValue(cmp_result); - try func.addTag(.select); - try func.store(.stack, .stack, ty, 0); try func.addImm32(-1); - try func.emitWValue(cmp_result); try func.addTag(.i32_xor); try func.addImm32(1); try func.addTag(.i32_and); diff --git a/src/arch/wasm/Emit.zig b/src/arch/wasm/Emit.zig index 1d039f7495..173a2ac672 100644 --- a/src/arch/wasm/Emit.zig +++ b/src/arch/wasm/Emit.zig @@ -521,8 +521,20 @@ fn emitSimd(emit: *Emit, inst: Mir.Inst.Index) !void { } fn emitAtomic(emit: *Emit, inst: Mir.Inst.Index) !void { - _ = inst; - return emit.fail("TODO: Implement atomics instructions", .{}); + const extra_index = emit.mir.instructions.items(.data)[inst].payload; + const opcode = emit.mir.extra[extra_index]; + const writer = emit.code.writer(); + try emit.code.append(std.wasm.opcode(.atomics_prefix)); + try leb128.writeULEB128(writer, opcode); + switch (@intToEnum(std.wasm.AtomicsOpcode, opcode)) { + .i32_atomic_rmw_cmpxchg, + .i64_atomic_rmw_cmpxchg, + => { + const mem_arg = emit.mir.extraData(Mir.MemArg, extra_index + 1).data; + try encodeMemArg(mem_arg, writer); + }, + else => |tag| return emit.fail("TODO: Implement atomic instruction: {s}", .{@tagName(tag)}), + } } fn emitMemFill(emit: *Emit) !void { From fd47eddc862b0a5bd90949fe21cf87a716c5464c Mon Sep 17 00:00:00 2001 From: Luuk de Gram Date: Fri, 21 Apr 2023 16:27:24 +0200 Subject: [PATCH 3/8] wasm: implement `@atomicLoad` Uses the atomic instructions when cpu feature is enabled, otherwise lowers it down to a regular load. --- src/arch/wasm/CodeGen.zig | 28 +++++++++++++++++++++++++++- src/arch/wasm/Emit.zig | 7 +++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/src/arch/wasm/CodeGen.zig b/src/arch/wasm/CodeGen.zig index 786b3fea76..fb27f8f343 100644 --- a/src/arch/wasm/CodeGen.zig +++ b/src/arch/wasm/CodeGen.zig @@ -1965,7 +1965,6 @@ fn genInst(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { .is_non_err_ptr, .fence, - .atomic_load, .atomic_store_unordered, .atomic_store_monotonic, .atomic_store_release, @@ -1983,6 +1982,7 @@ fn genInst(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { .c_va_start, => |tag| return func.fail("TODO: Implement wasm inst: {s}", .{@tagName(tag)}), + .atomic_load => func.airAtomicLoad(inst), .cmpxchg_weak => func.airCmpxchg(inst), .cmpxchg_strong => func.airCmpxchg(inst), @@ -6664,3 +6664,29 @@ fn airCmpxchg(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { return func.finishAir(inst, result_ptr, &.{ extra.ptr, extra.new_value, extra.expected_value }); } + +fn airAtomicLoad(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const atomic_load = func.air.instructions.items(.data)[inst].atomic_load; + const ptr = try func.resolveInst(atomic_load.ptr); + const ty = func.air.typeOfIndex(inst); + + if (func.useAtomicFeature()) { + const tag: wasm.AtomicsOpcode = switch (ty.abiSize(func.target)) { + 1 => .i32_atomic_load8_u, + 2 => .i32_atomic_load16_u, + 4 => .i32_atomic_load, + 8 => .i64_atomic_load, + else => |size| return func.fail("TODO: @atomicLoad for integers with abi size {d}", .{size}), + }; + try func.emitWValue(ptr); + try func.addAtomicMemArg(tag, .{ + .offset = ptr.offset(), + .alignment = ty.abiAlignment(func.target), + }); + } else { + _ = try func.load(ptr, ty, 0); + } + + const result = try WValue.toLocal(.stack, func, ty); + return func.finishAir(inst, result, &.{atomic_load.ptr}); +} diff --git a/src/arch/wasm/Emit.zig b/src/arch/wasm/Emit.zig index 173a2ac672..420f0d7606 100644 --- a/src/arch/wasm/Emit.zig +++ b/src/arch/wasm/Emit.zig @@ -529,6 +529,13 @@ fn emitAtomic(emit: *Emit, inst: Mir.Inst.Index) !void { switch (@intToEnum(std.wasm.AtomicsOpcode, opcode)) { .i32_atomic_rmw_cmpxchg, .i64_atomic_rmw_cmpxchg, + .i32_atomic_load, + .i64_atomic_load, + .i32_atomic_load8_u, + .i32_atomic_load16_u, + .i64_atomic_load8_u, + .i64_atomic_load16_u, + .i64_atomic_load32_u, => { const mem_arg = emit.mir.extraData(Mir.MemArg, extra_index + 1).data; try encodeMemArg(mem_arg, writer); From b19c258f045aa1653b4e38ba2ff96302d914eea2 Mon Sep 17 00:00:00 2001 From: Luuk de Gram Date: Sat, 22 Apr 2023 15:40:53 +0200 Subject: [PATCH 4/8] wasm: implement `@atomicRmw` Implements the lowering of the `@atomicRmw` builtin. Uses the atomic opcodes when the cpu feature `atomics` is enabled. Otherwise lowers it to regular instructions. For the operations that do not lower to a direct atomic opcode, we use a loop in combiantion with a cmpxchg to ensure the swapping of values is doing atomically. --- src/arch/wasm/CodeGen.zig | 179 +++++++++++++++++++++++++++++++++++++- src/arch/wasm/Emit.zig | 59 ++++++++++++- 2 files changed, 233 insertions(+), 5 deletions(-) diff --git a/src/arch/wasm/CodeGen.zig b/src/arch/wasm/CodeGen.zig index fb27f8f343..d617ad9789 100644 --- a/src/arch/wasm/CodeGen.zig +++ b/src/arch/wasm/CodeGen.zig @@ -1969,7 +1969,6 @@ fn genInst(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { .atomic_store_monotonic, .atomic_store_release, .atomic_store_seq_cst, - .atomic_rmw, .err_return_trace, .set_err_return_trace, .save_err_return_trace_index, @@ -1983,6 +1982,7 @@ fn genInst(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { => |tag| return func.fail("TODO: Implement wasm inst: {s}", .{@tagName(tag)}), .atomic_load => func.airAtomicLoad(inst), + .atomic_rmw => func.airAtomicRmw(inst), .cmpxchg_weak => func.airCmpxchg(inst), .cmpxchg_strong => func.airCmpxchg(inst), @@ -6526,7 +6526,6 @@ fn getTagNameFunction(func: *CodeGen, enum_ty: Type) InnerError!u32 { return func.bin_file.createFunction(func_name, func_type, &body_list, &relocs); } -<<<<<<< HEAD fn airErrorSetHasValue(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { const ty_op = func.air.instructions.items(.data)[inst].ty_op; @@ -6676,7 +6675,7 @@ fn airAtomicLoad(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { 2 => .i32_atomic_load16_u, 4 => .i32_atomic_load, 8 => .i64_atomic_load, - else => |size| return func.fail("TODO: @atomicLoad for integers with abi size {d}", .{size}), + else => |size| return func.fail("TODO: @atomicLoad for types with abi size {d}", .{size}), }; try func.emitWValue(ptr); try func.addAtomicMemArg(tag, .{ @@ -6690,3 +6689,177 @@ fn airAtomicLoad(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { const result = try WValue.toLocal(.stack, func, ty); return func.finishAir(inst, result, &.{atomic_load.ptr}); } + +fn airAtomicRmw(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const pl_op = func.air.instructions.items(.data)[inst].pl_op; + const extra = func.air.extraData(Air.AtomicRmw, pl_op.payload).data; + + const ptr = try func.resolveInst(pl_op.operand); + const operand = try func.resolveInst(extra.operand); + const ty = func.air.typeOfIndex(inst); + const op: std.builtin.AtomicRmwOp = extra.op(); + + if (func.useAtomicFeature()) { + switch (op) { + .Max, + .Min, + .Nand, + => { + const tmp = try func.load(ptr, ty, 0); + const value = try tmp.toLocal(func, ty); + + // create a loop to cmpxchg the new value + try func.startBlock(.loop, wasm.block_empty); + + try func.emitWValue(ptr); + try func.emitWValue(value); + if (op == .Nand) { + const wasm_bits = toWasmBits(@intCast(u16, ty.bitSize(func.target))).?; + + const and_res = try func.binOp(value, operand, ty, .@"and"); + if (wasm_bits == 32) + try func.addImm32(-1) + else if (wasm_bits == 64) + try func.addImm64(@bitCast(u64, @as(i64, -1))) + else + return func.fail("TODO: `@atomicRmw` with operator `Nand` for types larger than 64 bits", .{}); + _ = try func.binOp(and_res, .stack, ty, .xor); + } else { + try func.emitWValue(value); + try func.emitWValue(operand); + _ = try func.cmp(value, operand, ty, if (op == .Max) .gt else .lt); + try func.addTag(.select); + } + try func.addAtomicMemArg( + switch (ty.abiSize(func.target)) { + 1 => .i32_atomic_rmw8_cmpxchg_u, + 2 => .i32_atomic_rmw16_cmpxchg_u, + 4 => .i32_atomic_rmw_cmpxchg, + 8 => .i64_atomic_rmw_cmpxchg, + else => return func.fail("TODO: implement `@atomicRmw` with operation `{s}` for types larger than 64 bits", .{@tagName(op)}), + }, + .{ + .offset = ptr.offset(), + .alignment = ty.abiAlignment(func.target), + }, + ); + const select_res = try func.allocLocal(ty); + try func.addLabel(.local_tee, select_res.local.value); + _ = try func.cmp(.stack, value, ty, .neq); // leave on stack so we can use it for br_if + + try func.emitWValue(select_res); + try func.addLabel(.local_set, value.local.value); + + try func.addLabel(.br_if, 0); + try func.endBlock(); + return func.finishAir(inst, value, &.{ pl_op.operand, extra.operand }); + }, + + // the other operations have their own instructions for Wasm. + else => { + try func.emitWValue(ptr); + try func.emitWValue(operand); + const tag: wasm.AtomicsOpcode = switch (ty.abiSize(func.target)) { + 1 => switch (op) { + .Xchg => .i32_atomic_rmw8_xchg_u, + .Add => .i32_atomic_rmw8_add_u, + .Sub => .i32_atomic_rmw8_sub_u, + .And => .i32_atomic_rmw8_and_u, + .Or => .i32_atomic_rmw8_or_u, + .Xor => .i32_atomic_rmw8_xor_u, + else => unreachable, + }, + 2 => switch (op) { + .Xchg => .i32_atomic_rmw16_xchg_u, + .Add => .i32_atomic_rmw16_add_u, + .Sub => .i32_atomic_rmw16_sub_u, + .And => .i32_atomic_rmw16_and_u, + .Or => .i32_atomic_rmw16_or_u, + .Xor => .i32_atomic_rmw16_xor_u, + else => unreachable, + }, + 4 => switch (op) { + .Xchg => .i32_atomic_rmw_xchg, + .Add => .i32_atomic_rmw_add, + .Sub => .i32_atomic_rmw_sub, + .And => .i32_atomic_rmw_and, + .Or => .i32_atomic_rmw_or, + .Xor => .i32_atomic_rmw_xor, + else => unreachable, + }, + 8 => switch (op) { + .Xchg => .i64_atomic_rmw_xchg, + .Add => .i64_atomic_rmw_add, + .Sub => .i64_atomic_rmw_sub, + .And => .i64_atomic_rmw_and, + .Or => .i64_atomic_rmw_or, + .Xor => .i64_atomic_rmw_xor, + else => unreachable, + }, + else => |size| return func.fail("TODO: Implement `@atomicRmw` for types with abi size {d}", .{size}), + }; + try func.addAtomicMemArg(tag, .{ + .offset = ptr.offset(), + .alignment = ty.abiAlignment(func.target), + }); + const result = try WValue.toLocal(.stack, func, ty); + return func.finishAir(inst, result, &.{ pl_op.operand, extra.operand }); + }, + } + } else { + const loaded = try func.load(ptr, ty, 0); + const result = try loaded.toLocal(func, ty); + + switch (op) { + .Xchg => { + try func.store(ptr, operand, ty, 0); + }, + .Add, + .Sub, + .And, + .Or, + .Xor, + => { + try func.emitWValue(ptr); + _ = try func.binOp(result, operand, ty, switch (op) { + .Add => .add, + .Sub => .sub, + .And => .@"and", + .Or => .@"or", + .Xor => .xor, + else => unreachable, + }); + if (ty.isInt() and (op == .Add or op == .Sub)) { + _ = try func.wrapOperand(.stack, ty); + } + try func.store(.stack, .stack, ty, ptr.offset()); + }, + .Max, + .Min, + => { + try func.emitWValue(ptr); + try func.emitWValue(result); + try func.emitWValue(operand); + _ = try func.cmp(result, operand, ty, if (op == .Max) .gt else .lt); + try func.addTag(.select); + try func.store(.stack, .stack, ty, ptr.offset()); + }, + .Nand => { + const wasm_bits = toWasmBits(@intCast(u16, ty.bitSize(func.target))).?; + + try func.emitWValue(ptr); + const and_res = try func.binOp(result, operand, ty, .@"and"); + if (wasm_bits == 32) + try func.addImm32(-1) + else if (wasm_bits == 64) + try func.addImm64(@bitCast(u64, @as(i64, -1))) + else + return func.fail("TODO: `@atomicRmw` with operator `Nand` for types larger than 64 bits", .{}); + _ = try func.binOp(and_res, .stack, ty, .xor); + try func.store(.stack, .stack, ty, ptr.offset()); + }, + } + + return func.finishAir(inst, result, &.{ pl_op.operand, extra.operand }); + } +} diff --git a/src/arch/wasm/Emit.zig b/src/arch/wasm/Emit.zig index 420f0d7606..ec5d9d9880 100644 --- a/src/arch/wasm/Emit.zig +++ b/src/arch/wasm/Emit.zig @@ -527,8 +527,6 @@ fn emitAtomic(emit: *Emit, inst: Mir.Inst.Index) !void { try emit.code.append(std.wasm.opcode(.atomics_prefix)); try leb128.writeULEB128(writer, opcode); switch (@intToEnum(std.wasm.AtomicsOpcode, opcode)) { - .i32_atomic_rmw_cmpxchg, - .i64_atomic_rmw_cmpxchg, .i32_atomic_load, .i64_atomic_load, .i32_atomic_load8_u, @@ -536,6 +534,63 @@ fn emitAtomic(emit: *Emit, inst: Mir.Inst.Index) !void { .i64_atomic_load8_u, .i64_atomic_load16_u, .i64_atomic_load32_u, + .i32_atomic_store, + .i64_atomic_store, + .i32_atomic_store8, + .i32_atomic_store16, + .i64_atomic_store8, + .i64_atomic_store16, + .i64_atomic_store32, + .i32_atomic_rmw_add, + .i64_atomic_rmw_add, + .i32_atomic_rmw8_add_u, + .i32_atomic_rmw16_add_u, + .i64_atomic_rmw8_add_u, + .i64_atomic_rmw16_add_u, + .i64_atomic_rmw32_add_u, + .i32_atomic_rmw_sub, + .i64_atomic_rmw_sub, + .i32_atomic_rmw8_sub_u, + .i32_atomic_rmw16_sub_u, + .i64_atomic_rmw8_sub_u, + .i64_atomic_rmw16_sub_u, + .i64_atomic_rmw32_sub_u, + .i32_atomic_rmw_and, + .i64_atomic_rmw_and, + .i32_atomic_rmw8_and_u, + .i32_atomic_rmw16_and_u, + .i64_atomic_rmw8_and_u, + .i64_atomic_rmw16_and_u, + .i64_atomic_rmw32_and_u, + .i32_atomic_rmw_or, + .i64_atomic_rmw_or, + .i32_atomic_rmw8_or_u, + .i32_atomic_rmw16_or_u, + .i64_atomic_rmw8_or_u, + .i64_atomic_rmw16_or_u, + .i64_atomic_rmw32_or_u, + .i32_atomic_rmw_xor, + .i64_atomic_rmw_xor, + .i32_atomic_rmw8_xor_u, + .i32_atomic_rmw16_xor_u, + .i64_atomic_rmw8_xor_u, + .i64_atomic_rmw16_xor_u, + .i64_atomic_rmw32_xor_u, + .i32_atomic_rmw_xchg, + .i64_atomic_rmw_xchg, + .i32_atomic_rmw8_xchg_u, + .i32_atomic_rmw16_xchg_u, + .i64_atomic_rmw8_xchg_u, + .i64_atomic_rmw16_xchg_u, + .i64_atomic_rmw32_xchg_u, + + .i32_atomic_rmw_cmpxchg, + .i64_atomic_rmw_cmpxchg, + .i32_atomic_rmw8_cmpxchg_u, + .i32_atomic_rmw16_cmpxchg_u, + .i64_atomic_rmw8_cmpxchg_u, + .i64_atomic_rmw16_cmpxchg_u, + .i64_atomic_rmw32_cmpxchg_u, => { const mem_arg = emit.mir.extraData(Mir.MemArg, extra_index + 1).data; try encodeMemArg(mem_arg, writer); From 0e3303ccd9985f4336ccde779b2f1b90f130d7a2 Mon Sep 17 00:00:00 2001 From: Luuk de Gram Date: Mon, 24 Apr 2023 18:42:28 +0200 Subject: [PATCH 5/8] wasm: implement `@fence` Uses the `atomic.fence` instruction for multi-thread-enabled builds where the `atomics` feature is enabled for the wasm32 target. In all other cases, this lowers to a nop. --- src/arch/wasm/CodeGen.zig | 19 ++++++++++++++++++- src/arch/wasm/Emit.zig | 6 ++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/arch/wasm/CodeGen.zig b/src/arch/wasm/CodeGen.zig index d617ad9789..38b96efb09 100644 --- a/src/arch/wasm/CodeGen.zig +++ b/src/arch/wasm/CodeGen.zig @@ -948,6 +948,12 @@ fn addAtomicMemArg(func: *CodeGen, tag: wasm.AtomicsOpcode, mem_arg: Mir.MemArg) try func.addInst(.{ .tag = .atomics_prefix, .data = .{ .payload = extra_index } }); } +/// Helper function to emit atomic mir opcodes. +fn addAtomicTag(func: *CodeGen, tag: wasm.AtomicsOpcode) error{OutOfMemory}!void { + const extra_index = try func.addExtra(@as(struct { val: u32 }, .{ .val = wasm.atomicsOpcode(tag) })); + try func.addInst(.{ .tag = .atomics_prefix, .data = .{ .payload = extra_index } }); +} + /// Appends entries to `mir_extra` based on the type of `extra`. /// Returns the index into `mir_extra` fn addExtra(func: *CodeGen, extra: anytype) error{OutOfMemory}!u32 { @@ -1964,7 +1970,6 @@ fn genInst(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { .is_err_ptr, .is_non_err_ptr, - .fence, .atomic_store_unordered, .atomic_store_monotonic, .atomic_store_release, @@ -1985,6 +1990,7 @@ fn genInst(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { .atomic_rmw => func.airAtomicRmw(inst), .cmpxchg_weak => func.airCmpxchg(inst), .cmpxchg_strong => func.airCmpxchg(inst), + .fence => func.airFence(inst), .add_optimized, .addwrap_optimized, @@ -6863,3 +6869,14 @@ fn airAtomicRmw(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { return func.finishAir(inst, result, &.{ pl_op.operand, extra.operand }); } } + +fn airFence(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { + // Only when the atomic feature is enabled, and we're not building + // for a single-threaded build, can we emit the `fence` instruction. + // In all other cases, we emit no instructions for a fence. + if (func.useAtomicFeature() and !func.bin_file.base.options.single_threaded) { + try func.addAtomicTag(.atomic_fence); + } + + return func.finishAir(inst, .none, &.{}); +} diff --git a/src/arch/wasm/Emit.zig b/src/arch/wasm/Emit.zig index ec5d9d9880..bfa5324dc6 100644 --- a/src/arch/wasm/Emit.zig +++ b/src/arch/wasm/Emit.zig @@ -595,6 +595,12 @@ fn emitAtomic(emit: *Emit, inst: Mir.Inst.Index) !void { const mem_arg = emit.mir.extraData(Mir.MemArg, extra_index + 1).data; try encodeMemArg(mem_arg, writer); }, + .atomic_fence => { + // TODO: When multi-memory proposal is accepted and implemented in the compiler, + // change this to (user-)specified index, rather than hardcode it to memory index 0. + const memory_index: u32 = 0; + try leb128.writeULEB128(writer, memory_index); + }, else => |tag| return emit.fail("TODO: Implement atomic instruction: {s}", .{@tagName(tag)}), } } From 7c09e09457fdd04fa3c28d32ecc91ea4454c09aa Mon Sep 17 00:00:00 2001 From: Luuk de Gram Date: Mon, 24 Apr 2023 19:45:06 +0200 Subject: [PATCH 6/8] wasm: implement atomic stores --- src/arch/wasm/CodeGen.zig | 47 ++++++++++++++++++++++++++++++++++----- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/src/arch/wasm/CodeGen.zig b/src/arch/wasm/CodeGen.zig index 38b96efb09..ca8f8f0734 100644 --- a/src/arch/wasm/CodeGen.zig +++ b/src/arch/wasm/CodeGen.zig @@ -1970,10 +1970,6 @@ fn genInst(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { .is_err_ptr, .is_non_err_ptr, - .atomic_store_unordered, - .atomic_store_monotonic, - .atomic_store_release, - .atomic_store_seq_cst, .err_return_trace, .set_err_return_trace, .save_err_return_trace_index, @@ -1987,6 +1983,12 @@ fn genInst(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { => |tag| return func.fail("TODO: Implement wasm inst: {s}", .{@tagName(tag)}), .atomic_load => func.airAtomicLoad(inst), + .atomic_store_unordered, + .atomic_store_monotonic, + .atomic_store_release, + .atomic_store_seq_cst, + // in WebAssembly, all atomic instructions are sequentially ordered. + => func.airAtomicStore(inst), .atomic_rmw => func.airAtomicRmw(inst), .cmpxchg_weak => func.airCmpxchg(inst), .cmpxchg_strong => func.airCmpxchg(inst), @@ -6634,7 +6636,13 @@ fn airCmpxchg(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { try func.emitWValue(ptr_operand); try func.emitWValue(expected_val); try func.emitWValue(new_val); - try func.addAtomicMemArg(.i32_atomic_rmw_cmpxchg, .{ + try func.addAtomicMemArg(switch (ty.abiSize(func.target)) { + 1 => .i32_atomic_rmw8_cmpxchg_u, + 2 => .i32_atomic_rmw16_cmpxchg_u, + 4 => .i32_atomic_rmw_cmpxchg, + 8 => .i32_atomic_rmw_cmpxchg, + else => |size| return func.fail("TODO: implement `@cmpxchg` for types with abi size '{d}'", .{size}), + }, .{ .offset = ptr_operand.offset(), .alignment = ty.abiAlignment(func.target), }); @@ -6880,3 +6888,32 @@ fn airFence(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { return func.finishAir(inst, .none, &.{}); } + +fn airAtomicStore(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { + const bin_op = func.air.instructions.items(.data)[inst].bin_op; + + const ptr = try func.resolveInst(bin_op.lhs); + const operand = try func.resolveInst(bin_op.rhs); + const ptr_ty = func.air.typeOf(bin_op.lhs); + const ty = ptr_ty.childType(); + + if (func.useAtomicFeature()) { + const tag: wasm.AtomicsOpcode = switch (ty.abiSize(func.target)) { + 1 => .i32_atomic_store8, + 2 => .i32_atomic_store16, + 4 => .i32_atomic_store, + 8 => .i64_atomic_store, + else => |size| return func.fail("TODO: @atomicLoad for types with abi size {d}", .{size}), + }; + try func.emitWValue(ptr); + try func.lowerToStack(operand); + try func.addAtomicMemArg(tag, .{ + .offset = ptr.offset(), + .alignment = ty.abiAlignment(func.target), + }); + } else { + try func.store(ptr, operand, ty, 0); + } + + return func.finishAir(inst, .none, &.{ bin_op.lhs, bin_op.rhs }); +} From b26a46d05b9a22b35f385948bc24059600213751 Mon Sep 17 00:00:00 2001 From: Luuk de Gram Date: Tue, 25 Apr 2023 19:52:36 +0200 Subject: [PATCH 7/8] wasm: support pointers in `cmpxchg` --- src/arch/wasm/CodeGen.zig | 45 +++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 16 deletions(-) diff --git a/src/arch/wasm/CodeGen.zig b/src/arch/wasm/CodeGen.zig index ca8f8f0734..f2daa621e8 100644 --- a/src/arch/wasm/CodeGen.zig +++ b/src/arch/wasm/CodeGen.zig @@ -6631,11 +6631,13 @@ fn airCmpxchg(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { const expected_val = try func.resolveInst(extra.expected_value); const new_val = try func.resolveInst(extra.new_value); + const cmp_result = try func.allocLocal(Type.bool); + const ptr_val = if (func.useAtomicFeature()) val: { const val_local = try func.allocLocal(ty); try func.emitWValue(ptr_operand); - try func.emitWValue(expected_val); - try func.emitWValue(new_val); + try func.lowerToStack(expected_val); + try func.lowerToStack(new_val); try func.addAtomicMemArg(switch (ty.abiSize(func.target)) { 1 => .i32_atomic_rmw8_cmpxchg_u, 2 => .i32_atomic_rmw16_cmpxchg_u, @@ -6648,32 +6650,43 @@ fn airCmpxchg(func: *CodeGen, inst: Air.Inst.Index) InnerError!void { }); try func.addLabel(.local_tee, val_local.local.value); _ = try func.cmp(.stack, expected_val, ty, .eq); + try func.addLabel(.local_set, cmp_result.local.value); break :val val_local; } else val: { + if (ty.abiSize(func.target) > 8) { + return func.fail("TODO: Implement `@cmpxchg` for types larger than abi size of 8 bytes", .{}); + } const ptr_val = try WValue.toLocal(try func.load(ptr_operand, ty, 0), func, ty); try func.lowerToStack(ptr_operand); - try func.emitWValue(new_val); + try func.lowerToStack(new_val); try func.emitWValue(ptr_val); - const cmp_tmp = try func.cmp(ptr_val, expected_val, ty, .eq); - const cmp_result = try cmp_tmp.toLocal(func, Type.bool); - try func.emitWValue(cmp_result); + _ = try func.cmp(ptr_val, expected_val, ty, .eq); + try func.addLabel(.local_tee, cmp_result.local.value); try func.addTag(.select); try func.store(.stack, .stack, ty, 0); - try func.emitWValue(cmp_result); break :val ptr_val; }; - try func.addImm32(-1); - try func.addTag(.i32_xor); - try func.addImm32(1); - try func.addTag(.i32_and); - const and_result = try WValue.toLocal(.stack, func, Type.bool); - - const result_ptr = try func.allocStack(result_ty); - try func.store(result_ptr, and_result, Type.bool, @intCast(u32, ty.abiSize(func.target))); - try func.store(result_ptr, ptr_val, ty, 0); + const result_ptr = if (isByRef(result_ty, func.target)) val: { + try func.emitWValue(cmp_result); + try func.addImm32(-1); + try func.addTag(.i32_xor); + try func.addImm32(1); + try func.addTag(.i32_and); + const and_result = try WValue.toLocal(.stack, func, Type.bool); + const result_ptr = try func.allocStack(result_ty); + try func.store(result_ptr, and_result, Type.bool, @intCast(u32, ty.abiSize(func.target))); + try func.store(result_ptr, ptr_val, ty, 0); + break :val result_ptr; + } else val: { + try func.addImm32(0); + try func.emitWValue(ptr_val); + try func.emitWValue(cmp_result); + try func.addTag(.select); + break :val try WValue.toLocal(.stack, func, result_ty); + }; return func.finishAir(inst, result_ptr, &.{ extra.ptr, extra.new_value, extra.expected_value }); } From 1d971817031f046990d60d1808ca48e73b838929 Mon Sep 17 00:00:00 2001 From: Luuk de Gram Date: Tue, 25 Apr 2023 19:53:06 +0200 Subject: [PATCH 8/8] wasm: enable atomics behavior tests --- test/behavior/atomics.zig | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/test/behavior/atomics.zig b/test/behavior/atomics.zig index 656eee78f9..19afa79683 100644 --- a/test/behavior/atomics.zig +++ b/test/behavior/atomics.zig @@ -11,7 +11,6 @@ const supports_128_bit_atomics = switch (builtin.cpu.arch) { }; test "cmpxchg" { - if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO @@ -38,7 +37,6 @@ fn testCmpxchg() !void { } test "fence" { - if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO @@ -48,7 +46,6 @@ test "fence" { } test "atomicrmw and atomicload" { - if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO @@ -76,7 +73,6 @@ fn testAtomicLoad(ptr: *u8) !void { } test "cmpxchg with ptr" { - if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO @@ -101,7 +97,6 @@ test "cmpxchg with ptr" { } test "cmpxchg with ignored result" { - if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO @@ -146,7 +141,6 @@ fn test_u128_cmpxchg() !void { var a_global_variable = @as(u32, 1234); test "cmpxchg on a global variable" { - if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO @@ -161,7 +155,6 @@ test "cmpxchg on a global variable" { } test "atomic load and rmw with enum" { - if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO @@ -178,7 +171,6 @@ test "atomic load and rmw with enum" { } test "atomic store" { - if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO @@ -191,7 +183,6 @@ test "atomic store" { } test "atomic store comptime" { - if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO @@ -209,7 +200,6 @@ fn testAtomicStore() !void { } test "atomicrmw with floats" { - if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_x86_64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO @@ -239,7 +229,6 @@ fn testAtomicRmwFloat() !void { } test "atomicrmw with ints" { - if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO @@ -413,7 +402,6 @@ fn testAtomicsWithType(comptime T: type, a: T, b: T) !void { } test "return @atomicStore, using it as a void value" { - if (builtin.zig_backend == .stage2_wasm) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_aarch64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_arm) return error.SkipZigTest; // TODO