mirror/zig
1
0
Fork 0
mirror of https://github.com/ziglang/zig.git synced 2025-12-06 14:23:09 +00:00
Code Issues Packages Projects Releases Wiki Activity
zig/lib/std/atomic/Atomic.zig
Andrew Kelley d5e21a4f1a std: remove meta.trait 
In general, I don't like the idea of std.meta.trait, and so I am
providing some guidance by deleting the entire namespace from the
standard library and compiler codebase.

My main criticism is that it's overcomplicated machinery that bloats
compile times and is ultimately unnecessary given the existence of Zig's
strong type system and reference traces.

Users who want this can create a third party package that provides this
functionality.

closes #18051
2023-11-22 13:24:27 -05:00

620 lines
24 KiB
Zig
Raw Blame History

const std = @import("../std.zig");
const builtin = @import("builtin");
const testing = std.testing;
const Ordering = std.atomic.Ordering;
pub fn Atomic(comptime T: type) type {
return extern struct {
value: T,
const Self = @This();
pub fn init(value: T) Self {
return .{ .value = value };
}
/// Perform an atomic fence which uses the atomic value as a hint for the modification order.
/// Use this when you want to imply a fence on an atomic variable without necessarily performing a memory access.
///
/// Example:
/// ```
/// const RefCount = struct {
/// count: Atomic(usize),
/// dropFn: *const fn (*RefCount) void,
///
/// fn ref(self: *RefCount) void {
/// _ = self.count.fetchAdd(1, .Monotonic); // no ordering necessary, just updating a counter
/// }
///
/// fn unref(self: *RefCount) void {
/// // Release ensures code before unref() happens-before the count is decremented as dropFn could be called by then.
/// if (self.count.fetchSub(1, .Release)) {
/// // Acquire ensures count decrement and code before previous unrefs()s happens-before we call dropFn below.
/// // NOTE: another alternative is to use .AcqRel on the fetchSub count decrement but it's extra barrier in possibly hot path.
/// self.count.fence(.Acquire);
/// (self.dropFn)(self);
/// }
/// }
/// };
/// ```
pub inline fn fence(self: *Self, comptime ordering: Ordering) void {
// LLVM's ThreadSanitizer doesn't support the normal fences so we specialize for it.
if (builtin.sanitize_thread) {
const tsan = struct {
extern "c" fn __tsan_acquire(addr: *anyopaque) void;
extern "c" fn __tsan_release(addr: *anyopaque) void;
};
const addr: *anyopaque = self;
return switch (ordering) {
.Unordered, .Monotonic => @compileError(@tagName(ordering) ++ " only applies to atomic loads and stores"),
.Acquire => tsan.__tsan_acquire(addr),
.Release => tsan.__tsan_release(addr),
.AcqRel, .SeqCst => {
tsan.__tsan_acquire(addr);
tsan.__tsan_release(addr);
},
};
}
return std.atomic.fence(ordering);
}
/// Non-atomically load from the atomic value without synchronization.
/// Care must be taken to avoid data-races when interacting with other atomic operations.
pub inline fn loadUnchecked(self: Self) T {
return self.value;
}
/// Non-atomically store to the atomic value without synchronization.
/// Care must be taken to avoid data-races when interacting with other atomic operations.
pub inline fn storeUnchecked(self: *Self, value: T) void {
self.value = value;
}
pub inline fn load(self: *const Self, comptime ordering: Ordering) T {
return switch (ordering) {
.AcqRel => @compileError(@tagName(ordering) ++ " implies " ++ @tagName(Ordering.Release) ++ " which is only allowed on atomic stores"),
.Release => @compileError(@tagName(ordering) ++ " is only allowed on atomic stores"),
else => @atomicLoad(T, &self.value, ordering),
};
}
pub inline fn store(self: *Self, value: T, comptime ordering: Ordering) void {
switch (ordering) {
.AcqRel => @compileError(@tagName(ordering) ++ " implies " ++ @tagName(Ordering.Acquire) ++ " which is only allowed on atomic loads"),
.Acquire => @compileError(@tagName(ordering) ++ " is only allowed on atomic loads"),
else => @atomicStore(T, &self.value, value, ordering),
}
}
pub inline fn swap(self: *Self, value: T, comptime ordering: Ordering) T {
return self.rmw(.Xchg, value, ordering);
}
pub inline fn compareAndSwap(
self: *Self,
compare: T,
exchange: T,
comptime success: Ordering,
comptime failure: Ordering,
) ?T {
return self.cmpxchg(true, compare, exchange, success, failure);
}
pub inline fn tryCompareAndSwap(
self: *Self,
compare: T,
exchange: T,
comptime success: Ordering,
comptime failure: Ordering,
) ?T {
return self.cmpxchg(false, compare, exchange, success, failure);
}
inline fn cmpxchg(
self: *Self,
comptime is_strong: bool,
compare: T,
exchange: T,
comptime success: Ordering,
comptime failure: Ordering,
) ?T {
if (success == .Unordered or failure == .Unordered) {
@compileError(@tagName(Ordering.Unordered) ++ " is only allowed on atomic loads and stores");
}
const success_is_stronger = switch (failure) {
.SeqCst => success == .SeqCst,
.AcqRel => @compileError(@tagName(failure) ++ " implies " ++ @tagName(Ordering.Release) ++ " which is only allowed on success"),
.Acquire => success == .SeqCst or success == .AcqRel or success == .Acquire,
.Release => @compileError(@tagName(failure) ++ " is only allowed on success"),
.Monotonic => true,
.Unordered => unreachable,
};
if (!success_is_stronger) {
@compileError(@tagName(success) ++ " must be stronger than " ++ @tagName(failure));
}
return switch (is_strong) {
true => @cmpxchgStrong(T, &self.value, compare, exchange, success, failure),
false => @cmpxchgWeak(T, &self.value, compare, exchange, success, failure),
};
}
inline fn rmw(
self: *Self,
comptime op: std.builtin.AtomicRmwOp,
value: T,
comptime ordering: Ordering,
) T {
return @atomicRmw(T, &self.value, op, value, ordering);
}
pub inline fn fetchAdd(self: *Self, value: T, comptime ordering: Ordering) T {
return self.rmw(.Add, value, ordering);
}
pub inline fn fetchSub(self: *Self, value: T, comptime ordering: Ordering) T {
return self.rmw(.Sub, value, ordering);
}
pub inline fn fetchMin(self: *Self, value: T, comptime ordering: Ordering) T {
return self.rmw(.Min, value, ordering);
}
pub inline fn fetchMax(self: *Self, value: T, comptime ordering: Ordering) T {
return self.rmw(.Max, value, ordering);
}
pub inline fn fetchAnd(self: *Self, value: T, comptime ordering: Ordering) T {
return self.rmw(.And, value, ordering);
}
pub inline fn fetchNand(self: *Self, value: T, comptime ordering: Ordering) T {
return self.rmw(.Nand, value, ordering);
}
pub inline fn fetchOr(self: *Self, value: T, comptime ordering: Ordering) T {
return self.rmw(.Or, value, ordering);
}
pub inline fn fetchXor(self: *Self, value: T, comptime ordering: Ordering) T {
return self.rmw(.Xor, value, ordering);
}
const Bit = std.math.Log2Int(T);
const BitRmwOp = enum {
Set,
Reset,
Toggle,
};
pub inline fn bitSet(self: *Self, bit: Bit, comptime ordering: Ordering) u1 {
return bitRmw(self, .Set, bit, ordering);
}
pub inline fn bitReset(self: *Self, bit: Bit, comptime ordering: Ordering) u1 {
return bitRmw(self, .Reset, bit, ordering);
}
pub inline fn bitToggle(self: *Self, bit: Bit, comptime ordering: Ordering) u1 {
return bitRmw(self, .Toggle, bit, ordering);
}
inline fn bitRmw(self: *Self, comptime op: BitRmwOp, bit: Bit, comptime ordering: Ordering) u1 {
// x86 supports dedicated bitwise instructions
if (comptime builtin.target.cpu.arch.isX86() and @sizeOf(T) >= 2 and @sizeOf(T) <= 8) {
// TODO: this causes std lib test failures when enabled
if (false) {
return x86BitRmw(self, op, bit, ordering);
}
}
const mask = @as(T, 1) << bit;
const value = switch (op) {
.Set => self.fetchOr(mask, ordering),
.Reset => self.fetchAnd(~mask, ordering),
.Toggle => self.fetchXor(mask, ordering),
};
return @intFromBool(value & mask != 0);
}
inline fn x86BitRmw(self: *Self, comptime op: BitRmwOp, bit: Bit, comptime ordering: Ordering) u1 {
const old_bit: u8 = switch (@sizeOf(T)) {
2 => switch (op) {
.Set => asm volatile ("lock btsw %[bit], %[ptr]"
// LLVM doesn't support u1 flag register return values
: [result] "={@ccc}" (-> u8),
: [ptr] "*m" (&self.value),
[bit] "X" (@as(T, bit)),
: "cc", "memory"
),
.Reset => asm volatile ("lock btrw %[bit], %[ptr]"
// LLVM doesn't support u1 flag register return values
: [result] "={@ccc}" (-> u8),
: [ptr] "*m" (&self.value),
[bit] "X" (@as(T, bit)),
: "cc", "memory"
),
.Toggle => asm volatile ("lock btcw %[bit], %[ptr]"
// LLVM doesn't support u1 flag register return values
: [result] "={@ccc}" (-> u8),
: [ptr] "*m" (&self.value),
[bit] "X" (@as(T, bit)),
: "cc", "memory"
),
},
4 => switch (op) {
.Set => asm volatile ("lock btsl %[bit], %[ptr]"
// LLVM doesn't support u1 flag register return values
: [result] "={@ccc}" (-> u8),
: [ptr] "*m" (&self.value),
[bit] "X" (@as(T, bit)),
: "cc", "memory"
),
.Reset => asm volatile ("lock btrl %[bit], %[ptr]"
// LLVM doesn't support u1 flag register return values
: [result] "={@ccc}" (-> u8),
: [ptr] "*m" (&self.value),
[bit] "X" (@as(T, bit)),
: "cc", "memory"
),
.Toggle => asm volatile ("lock btcl %[bit], %[ptr]"
// LLVM doesn't support u1 flag register return values
: [result] "={@ccc}" (-> u8),
: [ptr] "*m" (&self.value),
[bit] "X" (@as(T, bit)),
: "cc", "memory"
),
},
8 => switch (op) {
.Set => asm volatile ("lock btsq %[bit], %[ptr]"
// LLVM doesn't support u1 flag register return values
: [result] "={@ccc}" (-> u8),
: [ptr] "*m" (&self.value),
[bit] "X" (@as(T, bit)),
: "cc", "memory"
),
.Reset => asm volatile ("lock btrq %[bit], %[ptr]"
// LLVM doesn't support u1 flag register return values
: [result] "={@ccc}" (-> u8),
: [ptr] "*m" (&self.value),
[bit] "X" (@as(T, bit)),
: "cc", "memory"
),
.Toggle => asm volatile ("lock btcq %[bit], %[ptr]"
// LLVM doesn't support u1 flag register return values
: [result] "={@ccc}" (-> u8),
: [ptr] "*m" (&self.value),
[bit] "X" (@as(T, bit)),
: "cc", "memory"
),
},
else => @compileError("Invalid atomic type " ++ @typeName(T)),
};
// TODO: emit appropriate tsan fence if compiling with tsan
_ = ordering;
return @intCast(old_bit);
}
};
}
test "Atomic.fence" {
inline for (.{ .Acquire, .Release, .AcqRel, .SeqCst }) |ordering| {
var x = Atomic(usize).init(0);
x.fence(ordering);
}
}
fn atomicIntTypes() []const type {
comptime var bytes = 1;
comptime var types: []const type = &[_]type{};
inline while (bytes <= @sizeOf(usize)) : (bytes *= 2) {
types = types ++ &[_]type{std.meta.Int(.unsigned, bytes * 8)};
}
return types;
}
test "Atomic.loadUnchecked" {
inline for (atomicIntTypes()) |Int| {
var x = Atomic(Int).init(5);
try testing.expectEqual(x.loadUnchecked(), 5);
}
}
test "Atomic.storeUnchecked" {
inline for (atomicIntTypes()) |Int| {
_ = Int;
var x = Atomic(usize).init(5);
x.storeUnchecked(10);
try testing.expectEqual(x.loadUnchecked(), 10);
}
}
test "Atomic.load" {
inline for (atomicIntTypes()) |Int| {
inline for (.{ .Unordered, .Monotonic, .Acquire, .SeqCst }) |ordering| {
var x = Atomic(Int).init(5);
try testing.expectEqual(x.load(ordering), 5);
}
}
}
test "Atomic.store" {
inline for (atomicIntTypes()) |Int| {
inline for (.{ .Unordered, .Monotonic, .Release, .SeqCst }) |ordering| {
_ = Int;
var x = Atomic(usize).init(5);
x.store(10, ordering);
try testing.expectEqual(x.load(.SeqCst), 10);
}
}
}
const atomic_rmw_orderings = [_]Ordering{
.Monotonic,
.Acquire,
.Release,
.AcqRel,
.SeqCst,
};
test "Atomic.swap" {
inline for (atomic_rmw_orderings) |ordering| {
var x = Atomic(usize).init(5);
try testing.expectEqual(x.swap(10, ordering), 5);
try testing.expectEqual(x.load(.SeqCst), 10);
var y = Atomic(enum(usize) { a, b, c }).init(.c);
try testing.expectEqual(y.swap(.a, ordering), .c);
try testing.expectEqual(y.load(.SeqCst), .a);
var z = Atomic(f32).init(5.0);
try testing.expectEqual(z.swap(10.0, ordering), 5.0);
try testing.expectEqual(z.load(.SeqCst), 10.0);
var a = Atomic(bool).init(false);
try testing.expectEqual(a.swap(true, ordering), false);
try testing.expectEqual(a.load(.SeqCst), true);
var b = Atomic(?*u8).init(null);
try testing.expectEqual(b.swap(@as(?*u8, @ptrFromInt(@alignOf(u8))), ordering), null);
try testing.expectEqual(b.load(.SeqCst), @as(?*u8, @ptrFromInt(@alignOf(u8))));
}
}
const atomic_cmpxchg_orderings = [_][2]Ordering{
.{ .Monotonic, .Monotonic },
.{ .Acquire, .Monotonic },
.{ .Acquire, .Acquire },
.{ .Release, .Monotonic },
// Although accepted by LLVM, acquire failure implies AcqRel success
// .{ .Release, .Acquire },
.{ .AcqRel, .Monotonic },
.{ .AcqRel, .Acquire },
.{ .SeqCst, .Monotonic },
.{ .SeqCst, .Acquire },
.{ .SeqCst, .SeqCst },
};
test "Atomic.compareAndSwap" {
inline for (atomicIntTypes()) |Int| {
inline for (atomic_cmpxchg_orderings) |ordering| {
var x = Atomic(Int).init(0);
try testing.expectEqual(x.compareAndSwap(1, 0, ordering[0], ordering[1]), 0);
try testing.expectEqual(x.load(.SeqCst), 0);
try testing.expectEqual(x.compareAndSwap(0, 1, ordering[0], ordering[1]), null);
try testing.expectEqual(x.load(.SeqCst), 1);
try testing.expectEqual(x.compareAndSwap(1, 0, ordering[0], ordering[1]), null);
try testing.expectEqual(x.load(.SeqCst), 0);
}
}
}
test "Atomic.tryCompareAndSwap" {
inline for (atomicIntTypes()) |Int| {
inline for (atomic_cmpxchg_orderings) |ordering| {
var x = Atomic(Int).init(0);
try testing.expectEqual(x.tryCompareAndSwap(1, 0, ordering[0], ordering[1]), 0);
try testing.expectEqual(x.load(.SeqCst), 0);
while (x.tryCompareAndSwap(0, 1, ordering[0], ordering[1])) |_| {}
try testing.expectEqual(x.load(.SeqCst), 1);
while (x.tryCompareAndSwap(1, 0, ordering[0], ordering[1])) |_| {}
try testing.expectEqual(x.load(.SeqCst), 0);
}
}
}
test "Atomic.fetchAdd" {
inline for (atomicIntTypes()) |Int| {
inline for (atomic_rmw_orderings) |ordering| {
var x = Atomic(Int).init(5);
try testing.expectEqual(x.fetchAdd(5, ordering), 5);
try testing.expectEqual(x.load(.SeqCst), 10);
try testing.expectEqual(x.fetchAdd(std.math.maxInt(Int), ordering), 10);
try testing.expectEqual(x.load(.SeqCst), 9);
}
}
}
test "Atomic.fetchSub" {
inline for (atomicIntTypes()) |Int| {
inline for (atomic_rmw_orderings) |ordering| {
var x = Atomic(Int).init(5);
try testing.expectEqual(x.fetchSub(5, ordering), 5);
try testing.expectEqual(x.load(.SeqCst), 0);
try testing.expectEqual(x.fetchSub(1, ordering), 0);
try testing.expectEqual(x.load(.SeqCst), std.math.maxInt(Int));
}
}
}
test "Atomic.fetchMin" {
inline for (atomicIntTypes()) |Int| {
inline for (atomic_rmw_orderings) |ordering| {
var x = Atomic(Int).init(5);
try testing.expectEqual(x.fetchMin(0, ordering), 5);
try testing.expectEqual(x.load(.SeqCst), 0);
try testing.expectEqual(x.fetchMin(10, ordering), 0);
try testing.expectEqual(x.load(.SeqCst), 0);
}
}
}
test "Atomic.fetchMax" {
inline for (atomicIntTypes()) |Int| {
inline for (atomic_rmw_orderings) |ordering| {
var x = Atomic(Int).init(5);
try testing.expectEqual(x.fetchMax(10, ordering), 5);
try testing.expectEqual(x.load(.SeqCst), 10);
try testing.expectEqual(x.fetchMax(5, ordering), 10);
try testing.expectEqual(x.load(.SeqCst), 10);
}
}
}
test "Atomic.fetchAnd" {
inline for (atomicIntTypes()) |Int| {
inline for (atomic_rmw_orderings) |ordering| {
var x = Atomic(Int).init(0b11);
try testing.expectEqual(x.fetchAnd(0b10, ordering), 0b11);
try testing.expectEqual(x.load(.SeqCst), 0b10);
try testing.expectEqual(x.fetchAnd(0b00, ordering), 0b10);
try testing.expectEqual(x.load(.SeqCst), 0b00);
}
}
}
test "Atomic.fetchNand" {
inline for (atomicIntTypes()) |Int| {
inline for (atomic_rmw_orderings) |ordering| {
var x = Atomic(Int).init(0b11);
try testing.expectEqual(x.fetchNand(0b10, ordering), 0b11);
try testing.expectEqual(x.load(.SeqCst), ~@as(Int, 0b10));
try testing.expectEqual(x.fetchNand(0b00, ordering), ~@as(Int, 0b10));
try testing.expectEqual(x.load(.SeqCst), ~@as(Int, 0b00));
}
}
}
test "Atomic.fetchOr" {
inline for (atomicIntTypes()) |Int| {
inline for (atomic_rmw_orderings) |ordering| {
var x = Atomic(Int).init(0b11);
try testing.expectEqual(x.fetchOr(0b100, ordering), 0b11);
try testing.expectEqual(x.load(.SeqCst), 0b111);
try testing.expectEqual(x.fetchOr(0b010, ordering), 0b111);
try testing.expectEqual(x.load(.SeqCst), 0b111);
}
}
}
test "Atomic.fetchXor" {
inline for (atomicIntTypes()) |Int| {
inline for (atomic_rmw_orderings) |ordering| {
var x = Atomic(Int).init(0b11);
try testing.expectEqual(x.fetchXor(0b10, ordering), 0b11);
try testing.expectEqual(x.load(.SeqCst), 0b01);
try testing.expectEqual(x.fetchXor(0b01, ordering), 0b01);
try testing.expectEqual(x.load(.SeqCst), 0b00);
}
}
}
test "Atomic.bitSet" {
inline for (atomicIntTypes()) |Int| {
inline for (atomic_rmw_orderings) |ordering| {
var x = Atomic(Int).init(0);
for (0..@bitSizeOf(Int)) |bit_index| {
const bit = @as(std.math.Log2Int(Int), @intCast(bit_index));
const mask = @as(Int, 1) << bit;
// setting the bit should change the bit
try testing.expect(x.load(.SeqCst) & mask == 0);
try testing.expectEqual(x.bitSet(bit, ordering), 0);
try testing.expect(x.load(.SeqCst) & mask != 0);
// setting it again shouldn't change the bit
try testing.expectEqual(x.bitSet(bit, ordering), 1);
try testing.expect(x.load(.SeqCst) & mask != 0);
// all the previous bits should have not changed (still be set)
for (0..bit_index) |prev_bit_index| {
const prev_bit = @as(std.math.Log2Int(Int), @intCast(prev_bit_index));
const prev_mask = @as(Int, 1) << prev_bit;
try testing.expect(x.load(.SeqCst) & prev_mask != 0);
}
}
}
}
}
test "Atomic.bitReset" {
inline for (atomicIntTypes()) |Int| {
inline for (atomic_rmw_orderings) |ordering| {
var x = Atomic(Int).init(0);
for (0..@bitSizeOf(Int)) |bit_index| {
const bit = @as(std.math.Log2Int(Int), @intCast(bit_index));
const mask = @as(Int, 1) << bit;
x.storeUnchecked(x.loadUnchecked() | mask);
// unsetting the bit should change the bit
try testing.expect(x.load(.SeqCst) & mask != 0);
try testing.expectEqual(x.bitReset(bit, ordering), 1);
try testing.expect(x.load(.SeqCst) & mask == 0);
// unsetting it again shouldn't change the bit
try testing.expectEqual(x.bitReset(bit, ordering), 0);
try testing.expect(x.load(.SeqCst) & mask == 0);
// all the previous bits should have not changed (still be reset)
for (0..bit_index) |prev_bit_index| {
const prev_bit = @as(std.math.Log2Int(Int), @intCast(prev_bit_index));
const prev_mask = @as(Int, 1) << prev_bit;
try testing.expect(x.load(.SeqCst) & prev_mask == 0);
}
}
}
}
}
test "Atomic.bitToggle" {
inline for (atomicIntTypes()) |Int| {
inline for (atomic_rmw_orderings) |ordering| {
var x = Atomic(Int).init(0);
for (0..@bitSizeOf(Int)) |bit_index| {
const bit = @as(std.math.Log2Int(Int), @intCast(bit_index));
const mask = @as(Int, 1) << bit;
// toggling the bit should change the bit
try testing.expect(x.load(.SeqCst) & mask == 0);
try testing.expectEqual(x.bitToggle(bit, ordering), 0);
try testing.expect(x.load(.SeqCst) & mask != 0);
// toggling it again *should* change the bit
try testing.expectEqual(x.bitToggle(bit, ordering), 1);
try testing.expect(x.load(.SeqCst) & mask == 0);
// all the previous bits should have not changed (still be toggled back)
for (0..bit_index) |prev_bit_index| {
const prev_bit = @as(std.math.Log2Int(Int), @intCast(prev_bit_index));
const prev_mask = @as(Int, 1) << prev_bit;
try testing.expect(x.load(.SeqCst) & prev_mask == 0);
}
}
}
}
}
Reference in New Issue View Git Blame Copy Permalink