riscv: implement lr/sr loop logic for non-native atomics

lib/std/start.zig

@@ -478,10 +478,8 @@ inline fn callMainWithArgs(argc: usize, argv: [*][*:0]u8, envp: [][*:0]u8) u8 {
     std.os.argv = argv[0..argc];
     std.os.environ = envp;
 
-    if (builtin.zig_backend != .stage2_riscv64) {
     std.debug.maybeEnableSegfaultHandler();
     maybeIgnoreSigpipe();
-    }
 
     return callMain();
 }

src/arch/riscv64/CodeGen.zig

@@ -4717,14 +4717,11 @@ fn airFence(func: *Func, inst: Air.Inst.Index) !void {
     };
 
     _ = try func.addInst(.{
-        .tag = .pseudo_fence,
+        .tag = if (order == .acq_rel) .fencetso else .fence,
-        .data = .{
+        .data = .{ .fence = .{
-            .fence = .{
             .pred = pred,
             .succ = succ,
-                .fm = if (order == .acq_rel) .tso else .none,
+        } },
-            },
-        },
     });
     return func.finishAirBookkeeping();
 }
@@ -5278,12 +5275,12 @@ fn isNull(func: *Func, inst: Air.Inst.Index, opt_ty: Type, opt_mcv: MCValue) !MC
         .dead,
         .undef,
         .immediate,
-        .register_pair,
         .register_offset,
         .lea_frame,
         .lea_symbol,
         .reserved_frame,
         .air_ref,
+        .register_pair,
         => unreachable,
 
         .register => |opt_reg| {
@@ -7109,6 +7106,7 @@ fn airCmpxchg(func: *Func, inst: Air.Inst.Index) !void {
 
 fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void {
     const pt = func.pt;
+    const zcu = pt.zcu;
     const pl_op = func.air.instructions.items(.data)[@intFromEnum(inst)].pl_op;
     const extra = func.air.extraData(Air.AtomicRmw, pl_op.payload).data;
 
@@ -7131,11 +7129,11 @@ fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void {
         else => unreachable,
     }
 
-    switch (val_size) {
+    const method: enum { amo, loop } = switch (val_size) {
-        1, 2 => return func.fail("TODO: airAtomicRmw {s} Int {}", .{ @tagName(op), val_size }),
+        1, 2 => .loop,
-        4, 8 => {},
+        4, 8 => .amo,
         else => unreachable,
-    }
+    };
 
     const ptr_register, const ptr_lock = try func.promoteReg(ptr_ty, ptr_mcv);
     defer if (ptr_lock) |lock| func.register_manager.unlockReg(lock);
@@ -7145,6 +7143,7 @@ fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void {
 
     const result_mcv = try func.allocRegOrMem(val_ty, inst, true);
     assert(result_mcv == .register); // should fit into 8 bytes
+    const result_reg = result_mcv.register;
 
     const aq, const rl = switch (order) {
         .unordered => unreachable,
@@ -7155,28 +7154,96 @@ fn airAtomicRmw(func: *Func, inst: Air.Inst.Index) !void {
         .seq_cst => .{ true, true },
     };
 
+    switch (method) {
+        .amo => {
+            const is_d = val_ty.abiSize(pt) == 8;
+            const is_un = val_ty.isUnsignedInt(zcu);
+
+            const mnem: Mnemonic = switch (op) {
+                // zig fmt: off
+                .Xchg => if (is_d) .amoswapd  else .amoswapw,
+                .Add  => if (is_d) .amoaddd   else .amoaddw,
+                .And  => if (is_d) .amoandd   else .amoandw,
+                .Or   => if (is_d) .amoord    else .amoorw,
+                .Xor  => if (is_d) .amoxord   else .amoxorw,
+                .Max  => if (is_d) if (is_un) .amomaxud else .amomaxd else if (is_un) .amomaxuw else .amomaxw,
+                .Min  => if (is_d) if (is_un) .amominud else .amomind else if (is_un) .amominuw else .amominw,
+                else => return func.fail("TODO: airAtomicRmw amo {s}", .{@tagName(op)}),
+                // zig fmt: on
+            };
+
             _ = try func.addInst(.{
-        .tag = .pseudo_amo,
+                .tag = mnem,
                 .data = .{ .amo = .{
-            .rd = result_mcv.register,
+                    .rd = result_reg,
                     .rs1 = ptr_register,
                     .rs2 = val_register,
                     .aq = if (aq) .aq else .none,
                     .rl = if (rl) .rl else .none,
-            .op = switch (op) {
-                .Xchg => .SWAP,
-                .Add => .ADD,
-                .Sub => return func.fail("TODO: airAtomicRmw SUB", .{}),
-                .And => .AND,
-                .Nand => return func.fail("TODO: airAtomicRmw NAND", .{}),
-                .Or => .OR,
-                .Xor => .XOR,
-                .Max => .MAX,
-                .Min => .MIN,
-            },
-            .ty = val_ty,
                 } },
             });
+        },
+        .loop => {
+            // where we'll jump back when the sc fails
+            const jump_back = try func.addInst(.{
+                .tag = .lrw,
+                .data = .{ .amo = .{
+                    .rd = result_reg,
+                    .rs1 = ptr_register,
+                    .rs2 = .zero,
+                    .aq = if (aq) .aq else .none,
+                    .rl = if (rl) .rl else .none,
+                } },
+            });
+
+            const after_reg, const after_lock = try func.allocReg(.int);
+            defer func.register_manager.unlockReg(after_lock);
+
+            switch (op) {
+                .Add => {
+                    _ = try func.genBinOp(
+                        .add,
+                        .{ .register = result_reg },
+                        val_ty,
+                        .{ .register = val_register },
+                        val_ty,
+                        after_reg,
+                    );
+                },
+                .Sub => {
+                    _ = try func.genBinOp(
+                        .sub,
+                        .{ .register = result_reg },
+                        val_ty,
+                        .{ .register = val_register },
+                        val_ty,
+                        after_reg,
+                    );
+                },
+                else => return func.fail("TODO: airAtomicRmw loop {s}", .{@tagName(op)}),
+            }
+
+            _ = try func.addInst(.{
+                .tag = .scw,
+                .data = .{ .amo = .{
+                    .rd = after_reg,
+                    .rs1 = ptr_register,
+                    .rs2 = after_reg,
+                    .aq = if (aq) .aq else .none,
+                    .rl = if (rl) .rl else .none,
+                } },
+            });
+
+            _ = try func.addInst(.{
+                .tag = .bne,
+                .data = .{ .b_type = .{
+                    .inst = jump_back,
+                    .rs1 = after_reg,
+                    .rs2 = .zero,
+                } },
+            });
+        },
+    }
 
     return func.finishAir(inst, result_mcv, .{ pl_op.operand, extra.operand, .none });
 }
@@ -7199,11 +7266,10 @@ fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void {
 
     if (order == .seq_cst) {
         _ = try func.addInst(.{
-            .tag = .pseudo_fence,
+            .tag = .fence,
             .data = .{ .fence = .{
                 .pred = .rw,
                 .succ = .rw,
-                .fm = .none,
             } },
         });
     }
@@ -7217,14 +7283,11 @@ fn airAtomicLoad(func: *Func, inst: Air.Inst.Index) !void {
         // Make sure all previous reads happen before any reading or writing accurs.
         .seq_cst, .acquire => {
             _ = try func.addInst(.{
-                .tag = .pseudo_fence,
+                .tag = .fence,
-                .data = .{
+                .data = .{ .fence = .{
-                    .fence = .{
                     .pred = .r,
                     .succ = .rw,
-                        .fm = .none,
+                } },
-                    },
-                },
             });
         },
         else => unreachable,
@@ -7249,14 +7312,11 @@ fn airAtomicStore(func: *Func, inst: Air.Inst.Index, order: std.builtin.AtomicOr
         .unordered, .monotonic => {},
         .release, .seq_cst => {
             _ = try func.addInst(.{
-                .tag = .pseudo_fence,
+                .tag = .fence,
-                .data = .{
+                .data = .{ .fence = .{
-                    .fence = .{
                     .pred = .rw,
                     .succ = .w,
-                        .fm = .none,
+                } },
-                    },
-                },
             });
         },
         else => unreachable,

src/arch/riscv64/Lower.zig

@@ -446,44 +446,6 @@ pub fn lowerMir(lower: *Lower, index: Mir.Inst.Index, options: struct {
                 .{ .imm = Immediate.s(0) },
             });
         },
-
-        .pseudo_amo => {
-            const amo = inst.data.amo;
-            const is_d = amo.ty.abiSize(pt) == 8;
-            const is_un = amo.ty.isUnsignedInt(pt.zcu);
-
-            const mnem: Mnemonic = switch (amo.op) {
-                // zig fmt: off
-                .SWAP => if (is_d) .amoswapd  else .amoswapw,
-                .ADD  => if (is_d) .amoaddd   else .amoaddw,
-                .AND  => if (is_d) .amoandd   else .amoandw,
-                .OR   => if (is_d) .amoord    else .amoorw,
-                .XOR  => if (is_d) .amoxord   else .amoxorw,
-                .MAX  => if (is_d) if (is_un) .amomaxud else .amomaxd else if (is_un) .amomaxuw else .amomaxw,
-                .MIN  => if (is_d) if (is_un) .amominud else .amomind else if (is_un) .amominuw else .amominw,
-                // zig fmt: on
-            };
-
-            try lower.emit(mnem, &.{
-                .{ .reg = inst.data.amo.rd },
-                .{ .reg = inst.data.amo.rs1 },
-                .{ .reg = inst.data.amo.rs2 },
-                .{ .barrier = inst.data.amo.rl },
-                .{ .barrier = inst.data.amo.aq },
-            });
-        },
-
-        .pseudo_fence => {
-            const fence = inst.data.fence;
-
-            try lower.emit(switch (fence.fm) {
-                .tso => .fencetso,
-                .none => .fence,
-            }, &.{
-                .{ .barrier = fence.succ },
-                .{ .barrier = fence.pred },
-            });
-        },
     }
 
     return .{
@@ -524,6 +486,17 @@ fn generic(lower: *Lower, inst: Mir.Inst) Error!void {
             .{ .reg = csr.rs1 },
             .{ .reg = csr.rd },
         },
+        .amo => |amo| &.{
+            .{ .reg = amo.rd },
+            .{ .reg = amo.rs1 },
+            .{ .reg = amo.rs2 },
+            .{ .barrier = amo.rl },
+            .{ .barrier = amo.aq },
+        },
+        .fence => |fence| &.{
+            .{ .barrier = fence.succ },
+            .{ .barrier = fence.pred },
+        },
         else => return lower.fail("TODO: generic lower {s}", .{@tagName(inst.data)}),
     });
 }

src/arch/riscv64/Mir.zig

@@ -73,10 +73,6 @@ pub const Inst = struct {
         fence: struct {
             pred: Barrier,
             succ: Barrier,
-            fm: enum {
-                none,
-                tso,
-            },
         },
         amo: struct {
             rd: Register,
@@ -84,8 +80,6 @@ pub const Inst = struct {
             rs2: Register,
             aq: Barrier,
             rl: Barrier,
-            op: AmoOp,
-            ty: Type,
         },
         csr: struct {
             csr: CSR,

src/arch/riscv64/abi.zig

@@ -125,10 +125,7 @@ pub fn classifySystem(ty: Type, pt: Zcu.PerThread) [8]SystemClass {
                 result[0] = .integer;
                 return result;
             }
-            result[0] = .integer;
+            return memory_class;
-            if (ty.optionalChild(zcu).abiSize(pt) == 0) return result;
-            result[1] = .integer;
-            return result;
         },
         .Int, .Enum, .ErrorSet => {
             const int_bits = ty.intInfo(pt.zcu).bits;

src/arch/riscv64/encoding.zig

@@ -353,6 +353,7 @@ pub const Lir = struct {
             // BRANCH
 
             .beq     => .{ .opcode = .BRANCH, .format = .B, .data = .{ .f = .{ .funct3 = 0b000 } } },
+            .bne     => .{ .opcode = .BRANCH, .format = .B, .data = .{ .f = .{ .funct3 = 0b001 } } },
 
 
             // SYSTEM
@@ -378,8 +379,8 @@ pub const Lir = struct {
 
             .amoaddw   => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b00000 } } },
             .amoswapw  => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b00001 } } },
-            // LR.W
+            .lrw       => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b00010 } } }, 
-            // SC.W
+            .scw       => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b00011 } } }, 
             .amoxorw   => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b00100 } } },
             .amoandw   => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b01100 } } },
             .amoorw    => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b01000 } } },
@@ -388,10 +389,11 @@ pub const Lir = struct {
             .amominuw  => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b11000 } } },
             .amomaxuw  => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .W, .funct5 = 0b11100 } } },
 
+        
             .amoaddd   => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b00000 } } },
             .amoswapd  => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b00001 } } },
-            // LR.D
+            .lrd       => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b00010 } } }, 
-            // SC.D
+            .scd       => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b00011 } } }, 
             .amoxord   => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b00100 } } },
             .amoandd   => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b01100 } } },
             .amoord    => .{ .opcode = .AMO, .format = .R, .data = .{ .amo = .{ .width = .D, .funct5 = 0b01000 } } },
@@ -434,8 +436,6 @@ pub const Lir = struct {
             .pseudo_compare,
             .pseudo_not,
             .pseudo_extern_fn_reloc,
-            .pseudo_fence,
-            .pseudo_amo,
             .nop,
             => std.debug.panic("lir: didn't catch pseudo {s}", .{@tagName(mnem)}),
             // zig fmt: on

src/arch/riscv64/mnem.zig

@@ -40,6 +40,7 @@ pub const Mnemonic = enum(u16) {
     jal,
 
     beq,
+    bne,
 
     // Memory
     lui,
@@ -178,6 +179,8 @@ pub const Mnemonic = enum(u16) {
     fence,
     fencetso,
 
+    lrw,
+    scw,
     amoswapw,
     amoaddw,
     amoandw,
@@ -188,6 +191,8 @@ pub const Mnemonic = enum(u16) {
     amomaxuw,
     amominuw,
 
+    lrd,
+    scd,
     amoswapd,
     amoaddd,
     amoandd,
@@ -237,8 +242,6 @@ pub const Mnemonic = enum(u16) {
     pseudo_compare,
     pseudo_not,
     pseudo_extern_fn_reloc,
-    pseudo_fence,
-    pseudo_amo,
 };
 
 pub const Pseudo = enum(u8) {