From bb6ad1a6c21694b4ea9340c552962b5ea60712e4 Mon Sep 17 00:00:00 2001 From: kprotty Date: Sun, 3 Nov 2019 21:09:51 -0600 Subject: [PATCH 01/18] Remove StaticallyInitializedMutex --- lib/std/statically_initialized_mutex.zig | 105 ----------------------- lib/std/std.zig | 1 - 2 files changed, 106 deletions(-) delete mode 100644 lib/std/statically_initialized_mutex.zig diff --git a/lib/std/statically_initialized_mutex.zig b/lib/std/statically_initialized_mutex.zig deleted file mode 100644 index 2ad47b5d91..0000000000 --- a/lib/std/statically_initialized_mutex.zig +++ /dev/null @@ -1,105 +0,0 @@ -const std = @import("std.zig"); -const builtin = @import("builtin"); -const AtomicOrder = builtin.AtomicOrder; -const AtomicRmwOp = builtin.AtomicRmwOp; -const assert = std.debug.assert; -const expect = std.testing.expect; -const windows = std.os.windows; - -/// Lock may be held only once. If the same thread -/// tries to acquire the same mutex twice, it deadlocks. -/// This type is intended to be initialized statically. If you don't -/// require static initialization, use std.Mutex. -/// On Windows, this mutex allocates resources when it is -/// first used, and the resources cannot be freed. -/// On Linux, this is an alias of std.Mutex. -pub const StaticallyInitializedMutex = switch (builtin.os) { - builtin.Os.linux => std.Mutex, - builtin.Os.windows => struct { - lock: windows.CRITICAL_SECTION, - init_once: windows.RTL_RUN_ONCE, - - pub const Held = struct { - mutex: *StaticallyInitializedMutex, - - pub fn release(self: Held) void { - windows.kernel32.LeaveCriticalSection(&self.mutex.lock); - } - }; - - pub fn init() StaticallyInitializedMutex { - return StaticallyInitializedMutex{ - .lock = undefined, - .init_once = windows.INIT_ONCE_STATIC_INIT, - }; - } - - extern fn initCriticalSection( - InitOnce: *windows.RTL_RUN_ONCE, - Parameter: ?*c_void, - Context: ?*c_void, - ) windows.BOOL { - const lock = @ptrCast(*windows.CRITICAL_SECTION, @alignCast(@alignOf(windows.CRITICAL_SECTION), Parameter)); - windows.kernel32.InitializeCriticalSection(lock); - return windows.TRUE; - } - - /// TODO: once https://github.com/ziglang/zig/issues/287 is solved and std.Mutex has a better - /// implementation of a runtime initialized mutex, remove this function. - pub fn deinit(self: *StaticallyInitializedMutex) void { - windows.InitOnceExecuteOnce(&self.init_once, initCriticalSection, &self.lock, null); - windows.kernel32.DeleteCriticalSection(&self.lock); - } - - pub fn acquire(self: *StaticallyInitializedMutex) Held { - windows.InitOnceExecuteOnce(&self.init_once, initCriticalSection, &self.lock, null); - windows.kernel32.EnterCriticalSection(&self.lock); - return Held{ .mutex = self }; - } - }, - else => std.Mutex, -}; - -test "std.StaticallyInitializedMutex" { - const TestContext = struct { - data: i128, - - const TestContext = @This(); - const incr_count = 10000; - - var mutex = StaticallyInitializedMutex.init(); - - fn worker(ctx: *TestContext) void { - var i: usize = 0; - while (i != TestContext.incr_count) : (i += 1) { - const held = mutex.acquire(); - defer held.release(); - - ctx.data += 1; - } - } - }; - - var plenty_of_memory = try std.heap.direct_allocator.alloc(u8, 300 * 1024); - defer std.heap.direct_allocator.free(plenty_of_memory); - - var fixed_buffer_allocator = std.heap.ThreadSafeFixedBufferAllocator.init(plenty_of_memory); - var a = &fixed_buffer_allocator.allocator; - - var context = TestContext{ .data = 0 }; - - if (builtin.single_threaded) { - TestContext.worker(&context); - expect(context.data == TestContext.incr_count); - } else { - const thread_count = 10; - var threads: [thread_count]*std.Thread = undefined; - for (threads) |*t| { - t.* = try std.Thread.spawn(&context, TestContext.worker); - } - for (threads) |t| - t.wait(); - - expect(context.data == thread_count * TestContext.incr_count); - } -} diff --git a/lib/std/std.zig b/lib/std/std.zig index ea6257009f..8ba67e79a2 100644 --- a/lib/std/std.zig +++ b/lib/std/std.zig @@ -19,7 +19,6 @@ pub const Progress = @import("progress.zig").Progress; pub const SegmentedList = @import("segmented_list.zig").SegmentedList; pub const SinglyLinkedList = @import("linked_list.zig").SinglyLinkedList; pub const SpinLock = @import("spinlock.zig").SpinLock; -pub const StaticallyInitializedMutex = @import("statically_initialized_mutex.zig").StaticallyInitializedMutex; pub const StringHashMap = @import("hash_map.zig").StringHashMap; pub const TailQueue = @import("linked_list.zig").TailQueue; pub const Target = @import("target.zig").Target; From 0d4f4aad9e44043b329de0f2e556c3b61cdaafa9 Mon Sep 17 00:00:00 2001 From: kprotty Date: Tue, 5 Nov 2019 08:16:08 -0600 Subject: [PATCH 02/18] adaptive SpinLock --- lib/std/os/windows/kernel32.zig | 2 ++ lib/std/spinlock.zig | 41 +++++++++++++++++++++++++++++---- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/lib/std/os/windows/kernel32.zig b/lib/std/os/windows/kernel32.zig index 736d81ae58..d466f9946d 100644 --- a/lib/std/os/windows/kernel32.zig +++ b/lib/std/os/windows/kernel32.zig @@ -184,6 +184,8 @@ pub extern "kernel32" stdcallcc fn SetHandleInformation(hObject: HANDLE, dwMask: pub extern "kernel32" stdcallcc fn Sleep(dwMilliseconds: DWORD) void; +pub extern "kernel32" stdcallcc fn SwitchToThread() BOOL; + pub extern "kernel32" stdcallcc fn TerminateProcess(hProcess: HANDLE, uExitCode: UINT) BOOL; pub extern "kernel32" stdcallcc fn TlsAlloc() DWORD; diff --git a/lib/std/spinlock.zig b/lib/std/spinlock.zig index 905460a2d0..894acd0fc4 100644 --- a/lib/std/spinlock.zig +++ b/lib/std/spinlock.zig @@ -1,8 +1,9 @@ const std = @import("std.zig"); const builtin = @import("builtin"); -const AtomicOrder = builtin.AtomicOrder; -const AtomicRmwOp = builtin.AtomicRmwOp; const assert = std.debug.assert; +const time = std.time; +const linux = std.os.linux; +const windows = std.os.windows; pub const SpinLock = struct { lock: u8, // TODO use a bool or enum @@ -11,7 +12,8 @@ pub const SpinLock = struct { spinlock: *SpinLock, pub fn release(self: Held) void { - assert(@atomicRmw(u8, &self.spinlock.lock, builtin.AtomicRmwOp.Xchg, 0, AtomicOrder.SeqCst) == 1); + // TODO: @atomicStore() https://github.com/ziglang/zig/issues/2995 + assert(@atomicRmw(u8, &self.spinlock.lock, .Xchg, 0, .Release) == 1); } }; @@ -19,10 +21,41 @@ pub const SpinLock = struct { return SpinLock{ .lock = 0 }; } + // Hybrid spinning from + // http://www.1024cores.net/home/lock-free-algorithms/tricks/spinning pub fn acquire(self: *SpinLock) Held { - while (@atomicRmw(u8, &self.lock, builtin.AtomicRmwOp.Xchg, 1, AtomicOrder.SeqCst) != 0) {} + var backoff: usize = 0; + while (@atomicRmw(u8, &self.lock, .Xchg, 1, .Acquire) != 0) : (backoff +%= 1) { + if (backoff < 10) { + yieldCpu(); + } else if (backoff < 20) { + for (([30]void)(undefined)) |_| yieldCpu(); + } else if (backoff < 24) { + yieldThread(); + } else if (backoff < 26) { + time.sleep(1 * time.millisecond); + } else { + time.sleep(10 * time.millisecond); + } + } return Held{ .spinlock = self }; } + + fn yieldCpu() void { + switch (builtin.arch) { + .i386, .x86_64 => asm volatile("pause" ::: "memory"), + .arm, .aarch64 => asm volatile("yield"), + else => time.sleep(0), + } + } + + fn yieldThread() void { + switch (builtin.os) { + .linux => assert(linux.syscall0(linux.SYS_sched_yield) == 0), + .windows => _ = windows.kernel32.SwitchToThread(), + else => time.sleep(1 * time.microsecond), + } + } }; test "spinlock" { From 84e370d0b2ce2240356ebd0a6007aec8620dfa10 Mon Sep 17 00:00:00 2001 From: kprotty Date: Tue, 5 Nov 2019 09:31:29 -0600 Subject: [PATCH 03/18] Move hybrid yielding into its own struct --- lib/std/spinlock.zig | 44 ++++++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/lib/std/spinlock.zig b/lib/std/spinlock.zig index 894acd0fc4..8a485ef0fe 100644 --- a/lib/std/spinlock.zig +++ b/lib/std/spinlock.zig @@ -21,23 +21,10 @@ pub const SpinLock = struct { return SpinLock{ .lock = 0 }; } - // Hybrid spinning from - // http://www.1024cores.net/home/lock-free-algorithms/tricks/spinning pub fn acquire(self: *SpinLock) Held { - var backoff: usize = 0; - while (@atomicRmw(u8, &self.lock, .Xchg, 1, .Acquire) != 0) : (backoff +%= 1) { - if (backoff < 10) { - yieldCpu(); - } else if (backoff < 20) { - for (([30]void)(undefined)) |_| yieldCpu(); - } else if (backoff < 24) { - yieldThread(); - } else if (backoff < 26) { - time.sleep(1 * time.millisecond); - } else { - time.sleep(10 * time.millisecond); - } - } + var backoff = Backoff.init(); + while (@atomicRmw(u8, &self.lock, .Xchg, 1, .Acquire) != 0) + backoff.yield(); return Held{ .spinlock = self }; } @@ -56,6 +43,31 @@ pub const SpinLock = struct { else => time.sleep(1 * time.microsecond), } } + + const Backoff = struct { + iteration: usize, + + fn init() @This() { + return @This(){ .iteration = 0 }; + } + + // Hybrid yielding from + // http://www.1024cores.net/home/lock-free-algorithms/tricks/spinning + fn yield(self: *@This()) void { + defer self.iteration +%= 1; + if (self.iteration < 10) { + yieldCpu(); + } else if (self.iteration < 20) { + for (([30]void)(undefined)) |_| yieldCpu(); + } else if (self.iteration < 24) { + yieldThread(); + } else if (self.iteration < 26) { + time.sleep(1 * time.millisecond); + } else { + time.sleep(10 * time.millisecond); + } + } + }; }; test "spinlock" { From 14209cf615f2c218319e0c7a567484caf328dfe6 Mon Sep 17 00:00:00 2001 From: kprotty Date: Tue, 5 Nov 2019 13:43:17 -0600 Subject: [PATCH 04/18] make SpinLock.Backoff pub --- lib/std/spinlock.zig | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/std/spinlock.zig b/lib/std/spinlock.zig index 8a485ef0fe..c45778cec1 100644 --- a/lib/std/spinlock.zig +++ b/lib/std/spinlock.zig @@ -44,16 +44,17 @@ pub const SpinLock = struct { } } - const Backoff = struct { + /// Provides a method to incrementally yield longer each time its called. + pub const Backoff = struct { iteration: usize, - fn init() @This() { + pub fn init() @This() { return @This(){ .iteration = 0 }; } - // Hybrid yielding from - // http://www.1024cores.net/home/lock-free-algorithms/tricks/spinning - fn yield(self: *@This()) void { + /// Hybrid yielding from + /// http://www.1024cores.net/home/lock-free-algorithms/tricks/spinning + pub fn yield(self: *@This()) void { defer self.iteration +%= 1; if (self.iteration < 10) { yieldCpu(); From 465ebf494df070ddb7eab9ba7787aacec394b1cc Mon Sep 17 00:00:00 2001 From: kprotty Date: Tue, 5 Nov 2019 15:58:58 -0600 Subject: [PATCH 05/18] ThreadParker implementation --- lib/std/os/windows/kernel32.zig | 4 + lib/std/os/windows/ntdll.zig | 18 ++ lib/std/parker.zig | 340 ++++++++++++++++++++++++++++++++ lib/std/std.zig | 1 + 4 files changed, 363 insertions(+) create mode 100644 lib/std/parker.zig diff --git a/lib/std/os/windows/kernel32.zig b/lib/std/os/windows/kernel32.zig index d466f9946d..97c5a5b695 100644 --- a/lib/std/os/windows/kernel32.zig +++ b/lib/std/os/windows/kernel32.zig @@ -82,6 +82,8 @@ pub extern "kernel32" stdcallcc fn GetFileAttributesW(lpFileName: [*]const WCHAR pub extern "kernel32" stdcallcc fn GetModuleFileNameW(hModule: ?HMODULE, lpFilename: [*]u16, nSize: DWORD) DWORD; +pub extern "kernel32" stdcallcc fn GetModuleHandleA(lpModuleName: ?[*]const CHAR) ?HMODULE; + pub extern "kernel32" stdcallcc fn GetModuleHandleW(lpModuleName: ?[*]const WCHAR) HMODULE; pub extern "kernel32" stdcallcc fn GetLastError() DWORD; @@ -204,6 +206,8 @@ pub extern "kernel32" stdcallcc fn WriteFile( pub extern "kernel32" stdcallcc fn WriteFileEx(hFile: HANDLE, lpBuffer: [*]const u8, nNumberOfBytesToWrite: DWORD, lpOverlapped: LPOVERLAPPED, lpCompletionRoutine: LPOVERLAPPED_COMPLETION_ROUTINE) BOOL; +pub extern "kernel32" stdcallcc fn LoadLibraryA(lpLibFileName: [*]const u8) ?HMODULE; + pub extern "kernel32" stdcallcc fn LoadLibraryW(lpLibFileName: [*]const u16) ?HMODULE; pub extern "kernel32" stdcallcc fn GetProcAddress(hModule: HMODULE, lpProcName: [*]const u8) ?FARPROC; diff --git a/lib/std/os/windows/ntdll.zig b/lib/std/os/windows/ntdll.zig index f914920093..293891123e 100644 --- a/lib/std/os/windows/ntdll.zig +++ b/lib/std/os/windows/ntdll.zig @@ -43,3 +43,21 @@ pub extern "NtDll" stdcallcc fn NtQueryDirectoryFile( FileName: ?*UNICODE_STRING, RestartScan: BOOLEAN, ) NTSTATUS; +pub extern "NtDll" stdcallcc fn NtCreateKeyedEvent( + KeyedEventHandle: *HANDLE, + DesiredAccess: ACCESS_MASK, + ObjectAttributes: ?PVOID, + Flags: ULONG, +) NTSTATUS; +pub extern "NtDll" stdcallcc fn NtReleaseKeyedEvent( + EventHandle: HANDLE, + Key: *const c_void, + Alertable: BOOLEAN, + Timeout: ?*LARGE_INTEGER, +) NTSTATUS; +pub extern "NtDll" stdcallcc fn NtWaitForKeyedEvent( + EventHandle: HANDLE, + Key: *const c_void, + Alertable: BOOLEAN, + Timeout: ?*LARGE_INTEGER, +) NTSTATUS; diff --git a/lib/std/parker.zig b/lib/std/parker.zig new file mode 100644 index 0000000000..b6945fa000 --- /dev/null +++ b/lib/std/parker.zig @@ -0,0 +1,340 @@ +const std = @import("std.zig"); +const builtin = @import("builtin"); +const testing = std.testing; +const assert = std.debug.assert; +const SpinLock = std.SpinLock; +const linux = std.os.linux; +const windows = std.os.windows; + +pub const ThreadParker = switch (builtin.os) { + .macosx, + .tvos, + .ios, + .watchos, + .netbsd, + .openbsd, + .freebsd, + .kfreebsd, + .dragonfly, + .haiku, + .hermit, + .solaris, + .minix, + .fuchsia, + .emscripten => if (builtin.link_libc) PosixParker else SpinParker, + .linux => if (builtin.link_libc) PosixParker else LinuxParker, + .windows => WindowsParker, + else => SpinParker, +}; + +const SpinParker = struct { + pub fn init() SpinParker { + return SpinParker{}; + } + pub fn deinit(self: *SpinParker) void {} + + pub fn unpark(self: *SpinParker, ptr: *const u32) void {} + + pub fn park(self: *SpinParker, ptr: *const u32, expected: u32) void { + var backoff = SpinLock.Backoff.init(); + while (@atomicLoad(u32, ptr, .Acquire) == expected) + backoff.yield(); + } +}; + +const LinuxParker = struct { + pub fn init() LinuxParker { + return LinuxParker{}; + } + pub fn deinit(self: *LinuxParker) void {} + + pub fn unpark(self: *LinuxParker, ptr: *const u32) void { + const rc = linux.futex_wake(@ptrCast(*const i32, ptr), linux.FUTEX_WAKE | linux.FUTEX_PRIVATE_FLAG, 1); + assert(linux.getErrno(rc) == 0); + } + + pub fn park(self: *LinuxParker, ptr: *const u32, expected: u32) void { + const value = @intCast(i32, expected); + while (@atomicLoad(u32, ptr, .Acquire) == expected) { + const rc = linux.futex_wait(@ptrCast(*const i32, ptr), linux.FUTEX_WAIT | linux.FUTEX_PRIVATE_FLAG, value, null); + switch (linux.getErrno(rc)) { + 0, linux.EAGAIN => return, + linux.EINTR => continue, + linux.EINVAL => unreachable, + else => unreachable, + } + } + } +}; + +const WindowsParker = struct { + waiters: u32, + + pub fn init() WindowsParker { + return WindowsParker{ .waiters = 0 }; + } + pub fn deinit(self: *WindowsParker) void {} + + pub fn unpark(self: *WindowsParker, ptr: *const u32) void { + switch (Backend.get().*) { + .WaitAddress => |*backend| backend.unpark(ptr, &self.waiters), + .KeyedEvent => |*backend| backend.unpark(ptr, &self.waiters), + } + } + + pub fn park(self: *WindowsParker, ptr: *const u32, expected: u32) void { + switch (Backend.get().*) { + .WaitAddress => |*backend| backend.park(ptr, expected, &self.waiters), + .KeyedEvent => |*backend| backend.park(ptr, expected, &self.waiters), + } + } + + const Backend = union(enum) { + WaitAddress: WaitAddress, + KeyedEvent: KeyedEvent, + + var backend = std.lazyInit(Backend); + + fn get() *const Backend { + return backend.get() orelse { + // Statically linking to the KeyedEvent functions should mean its supported. + // TODO: Maybe add a CreateSemaphore backend for systems older than Windows XP ? + backend.data = WaitAddress.init() + orelse KeyedEvent.init() + orelse unreachable; + backend.resolve(); + return &backend.data; + }; + } + + const WaitAddress = struct { + WakeByAddressSingle: stdcallcc fn(Address: *const c_void) void, + WaitOnAddress: stdcallcc fn ( + Address: *const c_void, + CompareAddress: *const c_void, + AddressSize: windows.SIZE_T, + dwMilliseconds: windows.DWORD, + ) windows.BOOL, + + fn init() ?Backend { + const dll_name = c"api-ms-win-core-synch-l1-2-0"; + const dll = windows.kernel32.GetModuleHandleA(dll_name) + orelse windows.kernel32.LoadLibraryA(dll_name) + orelse return null; + + var self: WaitAddress = undefined; + const WaitOnAddress = windows.kernel32.GetProcAddress(dll, c"WaitOnAddress") orelse return null; + self.WaitOnAddress = @intToPtr(@typeOf(self.WaitOnAddress), @ptrToInt(WaitOnAddress)); + const WakeByAddressSingle = windows.kernel32.GetProcAddress(dll, c"WakeByAddressSingle") orelse return null; + self.WakeByAddressSingle = @intToPtr(@typeOf(self.WakeByAddressSingle), @ptrToInt(WakeByAddressSingle)); + return Backend{ .WaitAddress = self }; + } + + fn unpark(self: WaitAddress, ptr: *const u32, waiters: *u32) void { + const addr = @ptrCast(*const c_void, ptr); + self.WakeByAddressSingle(addr); + } + + fn park(self: WaitAddress, ptr: *const u32, expected: u32, waiters: *u32) void { + var compare = expected; + const addr = @ptrCast(*const c_void, ptr); + const cmp = @ptrCast(*const c_void, &compare); + while (@atomicLoad(u32, ptr, .Acquire) == expected) + _ = self.WaitOnAddress(addr, cmp, @sizeOf(u32), windows.INFINITE); + } + }; + + const KeyedEvent = struct { + handle: windows.HANDLE, + + fn init() ?Backend { + var self: KeyedEvent = undefined; + const access_mask = windows.GENERIC_READ | windows.GENERIC_WRITE; + if (windows.ntdll.NtCreateKeyedEvent(&self.handle, access_mask, null, 0) != 0) + return null; + return Backend{ .KeyedEvent = self }; + } + + fn unpark(self: KeyedEvent, ptr: *const u32, waiters: *u32) void { + const key = @ptrCast(*const c_void, ptr); + var waiting = @atomicLoad(u32, waiters, .Acquire); + while (waiting != 0) { + waiting = @cmpxchgWeak(u32, waiters, waiting, waiting - 1, .AcqRel, .Monotonic) orelse { + const rc = windows.ntdll.NtReleaseKeyedEvent(self.handle, key, windows.FALSE, null); + assert(rc == 0); + return; + }; + } + } + + fn park(self: KeyedEvent, ptr: *const u32, expected: u32, waiters: *u32) void { + const key = @ptrCast(*const c_void, ptr); + while (@atomicLoad(u32, ptr, .Acquire) == expected) { + _ = @atomicRmw(u32, waiters, .Add, 1, .Release); + const rc = windows.ntdll.NtWaitForKeyedEvent(self.handle, key, windows.FALSE, null); + assert(rc == 0); + } + } + }; + }; +}; + +const PosixParker = struct { + cond: pthread_cond_t, + mutex: pthread_mutex_t, + + pub fn init() PosixParker { + return PosixParker{ + .cond = PTHREAD_COND_INITIALIZER, + .mutex = PTHREAD_MUTEX_INITIALIZER, + }; + } + + pub fn deinit(self: *PosixParker) void { + // On dragonfly, the destroy functions return EINVAL if they were initialized statically. + const retm = pthread_mutex_destroy(&self.mutex); + assert(retm == 0 or retm == (if (builtin.os == .dragonfly) os.EINVAL else 0)); + const retc = pthread_cond_destroy(&self.cond); + assert(retc == 0 or retc == (if (builtin.os == .dragonfly) os.EINVAL else 0)); + } + + pub fn unpark(self: *PosixParker, ptr: *const u32) void { + assert(pthread_cond_signal(&self.cond) == 0); + } + + pub fn park(self: *PosixParker, ptr: *const u32, expected: u32) void { + assert(pthread_mutex_lock(&self.mutex) == 0); + defer assert(pthread_mutex_unlock(&self.mutex) == 0); + while (@atomicLoad(u32, ptr, .Acquire) == expected) + assert(pthread_cond_wait(&self.cond, &self.mutex) == 0); + } + + const PTHREAD_MUTEX_INITIALIZER = pthread_mutex_t{}; + extern "c" fn pthread_mutex_lock(mutex: *pthread_mutex_t) c_int; + extern "c" fn pthread_mutex_unlock(mutex: *pthread_mutex_t) c_int; + extern "c" fn pthread_mutex_destroy(mutex: *pthread_mutex_t) c_int; + + const PTHREAD_COND_INITIALIZER = pthread_cond_t{}; + extern "c" fn pthread_cond_wait(noalias cond: *pthread_cond_t, noalias mutex: *pthread_mutex_t) c_int; + extern "c" fn pthread_cond_signal(cond: *pthread_cond_t) c_int; + extern "c" fn pthread_cond_destroy(cond: *pthread_cond_t) c_int; + + // https://github.com/rust-lang/libc + usingnamespace switch (builtin.os) { + .macosx, .tvos, .ios, .watchos => struct { + pub const pthread_mutex_t = extern struct { + __sig: c_long = 0x32AAABA7, + __opaque: [__PTHREAD_MUTEX_SIZE__]u8 = [_]u8{0} ** __PTHREAD_MUTEX_SIZE__, + }; + pub const pthread_cond_t = extern struct { + __sig: c_long = 0x3CB0B1BB, + __opaque: [__PTHREAD_COND_SIZE__]u8 = [_]u8{0} ** __PTHREAD_COND_SIZE__, + }; + const __PTHREAD_MUTEX_SIZE__ = if (@sizeOf(usize) == 8) 56 else 40; + const __PTHREAD_COND_SIZE__ = if (@sizeOf(usize) == 8) 40 else 24; + }, + .netbsd => struct { + pub const pthread_mutex_t = extern struct { + ptm_magic: c_uint = 0x33330003, + ptm_errorcheck: padded_spin_t = 0, + ptm_unused: padded_spin_t = 0, + ptm_owner: usize = 0, + ptm_waiters: ?*u8 = null, + ptm_recursed: c_uint = 0, + ptm_spare2: ?*c_void = null, + }; + pub const pthread_cond_t = extern struct { + ptc_magic: c_uint = 0x55550005, + ptc_lock: pthread_spin_t = 0, + ptc_waiters_first: ?*u8 = null, + ptc_waiters_last: ?*u8 = null, + ptc_mutex: ?*pthread_mutex_t = null, + ptc_private: ?*c_void = null, + }; + const pthread_spin_t = if (builtin.arch == .arm or .arch == .powerpc) c_int else u8; + const padded_spin_t = switch (builtin.arch) { + .sparc, .sparcel, .sparcv9, .i386, .x86_64, .le64 => u32, + else => spin_t, + }; + }, + .openbsd, .freebsd, .kfreebsd, .dragonfly => struct { + pub const pthread_mutex_t = extern struct { + inner: ?*c_void = null, + }; + pub const pthread_cond_t = extern struct { + inner: ?*c_void = null, + }; + }, + .haiku => struct { + pub const pthread_mutex_t = extern struct { + flags: u32 = 0, + lock: i32 = 0, + unused: i32 = -42, + owner: i32 = -1, + owner_count: i32 = 0, + }; + pub const pthread_cond_t = extern struct { + flags: u32 = 0, + unused: i32 = -42, + mutex: ?*c_void = null, + waiter_count: i32 = 0, + lock: i32 = 0, + }; + }, + .hermit => struct { + pub const pthread_mutex_t = extern struct { + inner: usize = ~usize(0), + }; + pub const pthread_cond_t = extern struct { + inner: usize = ~usize(0), + }; + }, + .solaris => struct { + pub const pthread_mutex_t = extern struct { + __pthread_mutex_flag1: u16 = 0, + __pthread_mutex_flag2: u8 = 0, + __pthread_mutex_ceiling: u8 = 0, + __pthread_mutex_type: u16 = 0, + __pthread_mutex_magic: u16 = 0x4d58, + __pthread_mutex_lock: u64 = 0, + __pthread_mutex_data: u64 = 0, + }; + pub const pthread_cond_t = extern struct { + __pthread_cond_flag: u32 = 0, + __pthread_cond_type: u16 = 0, + __pthread_cond_magic: u16 = 0x4356, + __pthread_cond_data: u64 = 0, + }; + }, + .fuchsia, .minix, .linux => struct { + pub const pthread_mutex_t = extern struct { + size: [__SIZEOF_PTHREAD_MUTEX_T]u8 align(@alignOf(usize)) = [_]u8{0} ** __SIZEOF_PTHREAD_MUTEX_T, + }; + pub const pthread_cond_t = extern struct { + size: [__SIZEOF_PTHREAD_COND_T]u8 align(@alignOf(usize)) = [_]u8{0} ** __SIZEOF_PTHREAD_COND_T, + }; + const __SIZEOF_PTHREAD_COND_T = 48; + const __SIZEOF_PTHREAD_MUTEX_T = if (builtin.os == .fuchsia) 40 else switch (builtin.abi) { + .musl, .musleabi, .musleabihf => if (@sizeOf(usize) == 8) 40 else 24, + .gnu, .gnuabin32, .gnuabi64, .gnueabi, .gnueabihf, .gnux32 => switch (builtin.arch) { + .aarch64 => 48, + .x86_64 => if (builtin.abi == .gnux32) 40 else 32, + .mips64, .powerpc64, .powerpc64le, .sparcv9 => 40, + else => if (@sizeOf(usize) == 8) 40 else 24, + }, + else => unreachable, + }; + }, + .emscripten => struct { + pub const pthread_mutex_t = extern struct { + size: [__SIZEOF_PTHREAD_MUTEX_T]u8 align(4) = [_]u8{0} ** __SIZEOF_PTHREAD_MUTEX_T, + }; + pub const pthread_cond_t = extern struct { + size: [__SIZEOF_PTHREAD_COND_T]u8 align(@alignOf(usize)) = [_]u8{0} ** __SIZEOF_PTHREAD_COND_T, + }; + const __SIZEOF_PTHREAD_COND_T = 48; + const __SIZEOF_PTHREAD_MUTEX_T = 28; + }, + else => unreachable, + }; +}; diff --git a/lib/std/std.zig b/lib/std/std.zig index 8ba67e79a2..78b7cc5b12 100644 --- a/lib/std/std.zig +++ b/lib/std/std.zig @@ -23,6 +23,7 @@ pub const StringHashMap = @import("hash_map.zig").StringHashMap; pub const TailQueue = @import("linked_list.zig").TailQueue; pub const Target = @import("target.zig").Target; pub const Thread = @import("thread.zig").Thread; +pub const ThreadParker = @import("parker.zig").ThreadParker; pub const atomic = @import("atomic.zig"); pub const base64 = @import("base64.zig"); From c9db420a09ad894c1e521d1bdf1c99a3424aa9a6 Mon Sep 17 00:00:00 2001 From: kprotty Date: Tue, 5 Nov 2019 16:14:43 -0600 Subject: [PATCH 06/18] Adaptive Mutex: - uses std.ThreadParker - supports static initialization (deprecates StaticallyInitializedMutex) --- lib/std/mutex.zig | 143 ++++++++++++++++++++----------------------- lib/std/parker.zig | 38 +++++++++++- lib/std/spinlock.zig | 4 +- 3 files changed, 105 insertions(+), 80 deletions(-) diff --git a/lib/std/mutex.zig b/lib/std/mutex.zig index 5f3b9272d9..26f0142c7c 100644 --- a/lib/std/mutex.zig +++ b/lib/std/mutex.zig @@ -1,19 +1,13 @@ const std = @import("std.zig"); const builtin = @import("builtin"); -const AtomicOrder = builtin.AtomicOrder; -const AtomicRmwOp = builtin.AtomicRmwOp; const testing = std.testing; const SpinLock = std.SpinLock; -const linux = std.os.linux; -const windows = std.os.windows; +const ThreadParker = std.ThreadParker; /// Lock may be held only once. If the same thread /// tries to acquire the same mutex twice, it deadlocks. -/// This type must be initialized at runtime, and then deinitialized when no -/// longer needed, to free resources. -/// If you need static initialization, use std.StaticallyInitializedMutex. -/// The Linux implementation is based on mutex3 from -/// https://www.akkadia.org/drepper/futex.pdf +/// This type supports static initialization and is based off of Golang 1.13 runtime.lock_futex: +/// https://github.com/golang/go/blob/master/src/runtime/lock_futex.go /// When an application is built in single threaded release mode, all the functions are /// no-ops. In single threaded debug mode, there is deadlock detection. pub const Mutex = if (builtin.single_threaded) @@ -43,83 +37,78 @@ pub const Mutex = if (builtin.single_threaded) return Held{ .mutex = self }; } } -else switch (builtin.os) { - builtin.Os.linux => struct { - /// 0: unlocked - /// 1: locked, no waiters - /// 2: locked, one or more waiters - lock: i32, +else struct { + state: u32, // TODO: make this an enum + parker: ThreadParker, - pub const Held = struct { - mutex: *Mutex, + const Unlocked = 0; + const Sleeping = 1; + const Locked = 2; - pub fn release(self: Held) void { - const c = @atomicRmw(i32, &self.mutex.lock, AtomicRmwOp.Sub, 1, AtomicOrder.Release); - if (c != 1) { - _ = @atomicRmw(i32, &self.mutex.lock, AtomicRmwOp.Xchg, 0, AtomicOrder.Release); - const rc = linux.futex_wake(&self.mutex.lock, linux.FUTEX_WAKE | linux.FUTEX_PRIVATE_FLAG, 1); - switch (linux.getErrno(rc)) { - 0 => {}, - linux.EINVAL => unreachable, - else => unreachable, - } - } - } + /// number of iterations to spin yielding the cpu + const SpinCpu = 4; + /// number of iterations to perform in the cpu yield loop + const SpinCpuCount = 30; + /// number of iterations to spin yielding the thread + const SpinThread = 1; + + pub fn init() Mutex { + return Mutex{ + .state = Unlocked, + .parker = ThreadParker.init(), }; + } - pub fn init() Mutex { - return Mutex{ .lock = 0 }; + pub fn deinit(self: *Mutex) void { + self.parker.deinit(); + } + + pub const Held = struct { + mutex: *Mutex, + + pub fn release(self: Held) void { + switch (@atomicRmw(u32, &self.mutex.state, .Xchg, Unlocked, .Release)) { + Locked => {}, + Sleeping => self.mutex.parker.unpark(&self.mutex.state), + Unlocked => unreachable, // unlocking an unlocked mutex + else => unreachable, // should never be anything else + } } + }; - pub fn deinit(self: *Mutex) void {} + pub fn acquire(self: *Mutex) Held { + // Try and speculatively grab the lock. + // If it fails, the state is either Locked or Sleeping + // depending on if theres a thread stuck sleeping below. + var state = @atomicRmw(u32, &self.state, .Xchg, Locked, .Acquire); + if (state == Unlocked) + return Held{ .mutex = self }; - pub fn acquire(self: *Mutex) Held { - var c = @cmpxchgWeak(i32, &self.lock, 0, 1, AtomicOrder.Acquire, AtomicOrder.Monotonic) orelse + while (true) { + // try and acquire the lock using cpu spinning on failure + for (([SpinCpu]void)(undefined)) |_| { + var value = @atomicLoad(u32, &self.state, .Monotonic); + while (value == Unlocked) + value = @cmpxchgWeak(u32, &self.state, Unlocked, state, .Acquire, .Monotonic) orelse return Held{ .mutex = self }; + for (([SpinCpuCount]void)(undefined)) |_| + SpinLock.yieldCpu(); + } + + // try and acquire the lock using thread rescheduling on failure + for (([SpinThread]void)(undefined)) |_| { + var value = @atomicLoad(u32, &self.state, .Monotonic); + while (value == Unlocked) + value = @cmpxchgWeak(u32, &self.state, Unlocked, state, .Acquire, .Monotonic) orelse return Held{ .mutex = self }; + SpinLock.yieldThread(); + } + + // failed to acquire the lock, go to sleep until woken up by `Held.release()` + if (@atomicRmw(u32, &self.state, .Xchg, Sleeping, .Acquire) == Unlocked) return Held{ .mutex = self }; - if (c != 2) - c = @atomicRmw(i32, &self.lock, AtomicRmwOp.Xchg, 2, AtomicOrder.Acquire); - while (c != 0) { - const rc = linux.futex_wait(&self.lock, linux.FUTEX_WAIT | linux.FUTEX_PRIVATE_FLAG, 2, null); - switch (linux.getErrno(rc)) { - 0, linux.EINTR, linux.EAGAIN => {}, - linux.EINVAL => unreachable, - else => unreachable, - } - c = @atomicRmw(i32, &self.lock, AtomicRmwOp.Xchg, 2, AtomicOrder.Acquire); - } - return Held{ .mutex = self }; + state = Sleeping; + self.parker.park(&self.state, Sleeping); } - }, - // TODO once https://github.com/ziglang/zig/issues/287 (copy elision) is solved, we can make a - // better implementation of this. The problem is we need the init() function to have access to - // the address of the CRITICAL_SECTION, and then have it not move. - builtin.Os.windows => std.StaticallyInitializedMutex, - else => struct { - /// TODO better implementation than spin lock. - /// When changing this, one must also change the corresponding - /// std.StaticallyInitializedMutex code, since it aliases this type, - /// under the assumption that it works both statically and at runtime. - lock: SpinLock, - - pub const Held = struct { - mutex: *Mutex, - - pub fn release(self: Held) void { - SpinLock.Held.release(SpinLock.Held{ .spinlock = &self.mutex.lock }); - } - }; - - pub fn init() Mutex { - return Mutex{ .lock = SpinLock.init() }; - } - - pub fn deinit(self: *Mutex) void {} - - pub fn acquire(self: *Mutex) Held { - _ = self.lock.acquire(); - return Held{ .mutex = self }; - } - }, + } }; const TestContext = struct { diff --git a/lib/std/parker.zig b/lib/std/parker.zig index b6945fa000..8571c89423 100644 --- a/lib/std/parker.zig +++ b/lib/std/parker.zig @@ -159,7 +159,7 @@ const WindowsParker = struct { const key = @ptrCast(*const c_void, ptr); var waiting = @atomicLoad(u32, waiters, .Acquire); while (waiting != 0) { - waiting = @cmpxchgWeak(u32, waiters, waiting, waiting - 1, .AcqRel, .Monotonic) orelse { + waiting = @cmpxchgWeak(u32, waiters, waiting, waiting - 1, .Acquire, .Monotonic) orelse { const rc = windows.ntdll.NtReleaseKeyedEvent(self.handle, key, windows.FALSE, null); assert(rc == 0); return; @@ -338,3 +338,39 @@ const PosixParker = struct { else => unreachable, }; }; + +test "std.ThreadParker" { + const Context = struct { + parker: ThreadParker, + data: u32, + + fn receiver(self: *@This()) void { + self.parker.park(&self.data, 0); // receives 1 + assert(@atomicRmw(u32, &self.data, .Xchg, 2, .SeqCst) == 1); // sends 2 + self.parker.unpark(&self.data); // wakes up waiters on 2 + self.parker.park(&self.data, 2); // receives 3 + assert(@atomicRmw(u32, &self.data, .Xchg, 4, .SeqCst) == 3); // sends 4 + self.parker.unpark(&self.data); // wakes up waiters on 4 + } + + fn sender(self: *@This()) void { + assert(@atomicRmw(u32, &self.data, .Xchg, 1, .SeqCst) == 0); // sends 1 + self.parker.unpark(&self.data); // wakes up waiters on 1 + self.parker.park(&self.data, 1); // receives 2 + assert(@atomicRmw(u32, &self.data, .Xchg, 3, .SeqCst) == 2); // sends 3 + self.parker.unpark(&self.data); // wakes up waiters on 3 + self.parker.park(&self.data, 3); // receives 4 + } + }; + + var context = Context{ + .parker = ThreadParker.init(), + .data = 0, + }; + defer context.parker.deinit(); + + var receiver = try std.Thread.spawn(&context, Context.receiver); + defer receiver.wait(); + + context.sender(); +} \ No newline at end of file diff --git a/lib/std/spinlock.zig b/lib/std/spinlock.zig index c45778cec1..3fd73ab8a3 100644 --- a/lib/std/spinlock.zig +++ b/lib/std/spinlock.zig @@ -28,7 +28,7 @@ pub const SpinLock = struct { return Held{ .spinlock = self }; } - fn yieldCpu() void { + pub fn yieldCpu() void { switch (builtin.arch) { .i386, .x86_64 => asm volatile("pause" ::: "memory"), .arm, .aarch64 => asm volatile("yield"), @@ -36,7 +36,7 @@ pub const SpinLock = struct { } } - fn yieldThread() void { + pub fn yieldThread() void { switch (builtin.os) { .linux => assert(linux.syscall0(linux.SYS_sched_yield) == 0), .windows => _ = windows.kernel32.SwitchToThread(), From 4dbfc48ab3adee68ccb354b80f787a4d42a72c24 Mon Sep 17 00:00:00 2001 From: kprotty Date: Tue, 5 Nov 2019 16:16:03 -0600 Subject: [PATCH 07/18] Disable asm("yield") for arm in SpinLock.yieldCpu() because of CI --- lib/std/spinlock.zig | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/std/spinlock.zig b/lib/std/spinlock.zig index 3fd73ab8a3..dadf96cef5 100644 --- a/lib/std/spinlock.zig +++ b/lib/std/spinlock.zig @@ -31,7 +31,10 @@ pub const SpinLock = struct { pub fn yieldCpu() void { switch (builtin.arch) { .i386, .x86_64 => asm volatile("pause" ::: "memory"), - .arm, .aarch64 => asm volatile("yield"), + // .arm, .aarch64 => asm volatile("yield"), + // + // Causes CI to fail + // See: https://github.com/ziglang/zig/pull/3585#issuecomment-549962765 else => time.sleep(0), } } From 7045f1e875eba3ec9cd5bd051aee059d34776bed Mon Sep 17 00:00:00 2001 From: kprotty Date: Wed, 6 Nov 2019 17:23:44 -0600 Subject: [PATCH 08/18] skip std.ThreadParker test on single-threaded --- lib/std/parker.zig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/std/parker.zig b/lib/std/parker.zig index 8571c89423..2dc38d2b38 100644 --- a/lib/std/parker.zig +++ b/lib/std/parker.zig @@ -340,6 +340,9 @@ const PosixParker = struct { }; test "std.ThreadParker" { + if (builtin.single_threaded) + return error.SkipZigTest; + const Context = struct { parker: ThreadParker, data: u32, From 92dac89d019fcd91aa043a116d6a6e166da8c4b2 Mon Sep 17 00:00:00 2001 From: kprotty Date: Thu, 7 Nov 2019 09:23:02 -0600 Subject: [PATCH 09/18] lock the mutex on pthread_cond_signal() --- lib/std/parker.zig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/std/parker.zig b/lib/std/parker.zig index 2dc38d2b38..8df6dcaa92 100644 --- a/lib/std/parker.zig +++ b/lib/std/parker.zig @@ -199,6 +199,8 @@ const PosixParker = struct { } pub fn unpark(self: *PosixParker, ptr: *const u32) void { + assert(pthread_mutex_lock(&self.mutex) == 0); + defer assert(pthread_mutex_unlock(&self.mutex) == 0); assert(pthread_cond_signal(&self.cond) == 0); } From b535e86cc0e86e76471aebd731566ae140098a07 Mon Sep 17 00:00:00 2001 From: kprotty Date: Thu, 7 Nov 2019 15:32:20 -0600 Subject: [PATCH 10/18] move SpinLock definitions around --- lib/std/c.zig | 1 + lib/std/mutex.zig | 17 +++++++++-------- lib/std/os.zig | 10 ++++++++++ lib/std/os/linux.zig | 4 ++++ lib/std/spinlock.zig | 37 +++++++++++++++---------------------- 5 files changed, 39 insertions(+), 30 deletions(-) diff --git a/lib/std/c.zig b/lib/std/c.zig index 71e111a61d..201aa6b103 100644 --- a/lib/std/c.zig +++ b/lib/std/c.zig @@ -158,6 +158,7 @@ pub extern "c" fn pthread_attr_init(attr: *pthread_attr_t) c_int; pub extern "c" fn pthread_attr_setstack(attr: *pthread_attr_t, stackaddr: *c_void, stacksize: usize) c_int; pub extern "c" fn pthread_attr_destroy(attr: *pthread_attr_t) c_int; pub extern "c" fn pthread_self() pthread_t; +pub extern "c" fn pthread_yield() c_int; pub extern "c" fn pthread_join(thread: pthread_t, arg_return: ?*?*c_void) c_int; pub extern "c" fn kqueue() c_int; diff --git a/lib/std/mutex.zig b/lib/std/mutex.zig index 26f0142c7c..110f581f5e 100644 --- a/lib/std/mutex.zig +++ b/lib/std/mutex.zig @@ -46,11 +46,11 @@ else struct { const Locked = 2; /// number of iterations to spin yielding the cpu - const SpinCpu = 4; + const SPIN_CPU = 4; /// number of iterations to perform in the cpu yield loop - const SpinCpuCount = 30; + const SPIN_CPU_COUNT = 30; /// number of iterations to spin yielding the thread - const SpinThread = 1; + const SPIN_THREAD = 1; pub fn init() Mutex { return Mutex{ @@ -86,20 +86,21 @@ else struct { while (true) { // try and acquire the lock using cpu spinning on failure - for (([SpinCpu]void)(undefined)) |_| { + var spin: usize = 0; + while (spin < SPIN_CPU) : (spin += 1) { var value = @atomicLoad(u32, &self.state, .Monotonic); while (value == Unlocked) value = @cmpxchgWeak(u32, &self.state, Unlocked, state, .Acquire, .Monotonic) orelse return Held{ .mutex = self }; - for (([SpinCpuCount]void)(undefined)) |_| - SpinLock.yieldCpu(); + SpinLock.yield(SPIN_CPU_COUNT); } // try and acquire the lock using thread rescheduling on failure - for (([SpinThread]void)(undefined)) |_| { + spin = 0; + while (spin < SPIN_THREAD) : (spin += 1) { var value = @atomicLoad(u32, &self.state, .Monotonic); while (value == Unlocked) value = @cmpxchgWeak(u32, &self.state, Unlocked, state, .Acquire, .Monotonic) orelse return Held{ .mutex = self }; - SpinLock.yieldThread(); + std.os.yield(); } // failed to acquire the lock, go to sleep until woken up by `Held.release()` diff --git a/lib/std/os.zig b/lib/std/os.zig index efddc9aafb..72dd2a7196 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -3169,3 +3169,13 @@ pub fn dn_expand( } return error.InvalidDnsPacket; } + +pub fn yield() void { + switch (builtin.os) { + .windows => _ = windows.kernel32.SwitchToThread(), + .linux => _ = assert(linux.sched_yield() == 0), + else => if (builtin.link_libc) { + assert(std.c.pthread_yield() == 0); + }, + } +} diff --git a/lib/std/os/linux.zig b/lib/std/os/linux.zig index d9f1f4e311..a7dedefb43 100644 --- a/lib/std/os/linux.zig +++ b/lib/std/os/linux.zig @@ -954,6 +954,10 @@ pub fn fremovexattr(fd: usize, name: [*]const u8) usize { return syscall2(SYS_fremovexattr, fd, @ptrToInt(name)); } +pub fn sched_yield() usize { + return syscall0(SYS_sched_yield); +} + pub fn sched_getaffinity(pid: i32, size: usize, set: *cpu_set_t) usize { const rc = syscall3(SYS_sched_getaffinity, @bitCast(usize, isize(pid)), size, @ptrToInt(set)); if (@bitCast(isize, rc) < 0) return rc; diff --git a/lib/std/spinlock.zig b/lib/std/spinlock.zig index dadf96cef5..55b92f51d1 100644 --- a/lib/std/spinlock.zig +++ b/lib/std/spinlock.zig @@ -28,22 +28,17 @@ pub const SpinLock = struct { return Held{ .spinlock = self }; } - pub fn yieldCpu() void { - switch (builtin.arch) { - .i386, .x86_64 => asm volatile("pause" ::: "memory"), - // .arm, .aarch64 => asm volatile("yield"), - // - // Causes CI to fail - // See: https://github.com/ziglang/zig/pull/3585#issuecomment-549962765 - else => time.sleep(0), - } - } - - pub fn yieldThread() void { - switch (builtin.os) { - .linux => assert(linux.syscall0(linux.SYS_sched_yield) == 0), - .windows => _ = windows.kernel32.SwitchToThread(), - else => time.sleep(1 * time.microsecond), + pub fn yield(iterations: usize) void { + var i = iterations; + while (i != 0) : (i -= 1) { + switch (builtin.arch) { + .i386, .x86_64 => asm volatile("pause" ::: "memory"), + // .arm, .aarch64 => asm volatile("yield"), + // + // Causes CI to fail + // See: https://github.com/ziglang/zig/pull/3585#issuecomment-549962765 + else => time.sleep(0), + } } } @@ -55,16 +50,14 @@ pub const SpinLock = struct { return @This(){ .iteration = 0 }; } - /// Hybrid yielding from + /// Modified hybrid yielding from /// http://www.1024cores.net/home/lock-free-algorithms/tricks/spinning pub fn yield(self: *@This()) void { defer self.iteration +%= 1; - if (self.iteration < 10) { - yieldCpu(); - } else if (self.iteration < 20) { - for (([30]void)(undefined)) |_| yieldCpu(); + if (self.iteration < 20) { + SpinLock.yield(self.iteration); } else if (self.iteration < 24) { - yieldThread(); + os.yield(); } else if (self.iteration < 26) { time.sleep(1 * time.millisecond); } else { From b5d84635f20a891cea55174771b630099bb3860d Mon Sep 17 00:00:00 2001 From: kprotty Date: Thu, 7 Nov 2019 15:46:57 -0600 Subject: [PATCH 11/18] remove WaitOnAddress backend in std.ThreadParker --- lib/std/os/windows/kernel32.zig | 4 -- lib/std/parker.zig | 116 +++++++------------------------- 2 files changed, 24 insertions(+), 96 deletions(-) diff --git a/lib/std/os/windows/kernel32.zig b/lib/std/os/windows/kernel32.zig index 97c5a5b695..d466f9946d 100644 --- a/lib/std/os/windows/kernel32.zig +++ b/lib/std/os/windows/kernel32.zig @@ -82,8 +82,6 @@ pub extern "kernel32" stdcallcc fn GetFileAttributesW(lpFileName: [*]const WCHAR pub extern "kernel32" stdcallcc fn GetModuleFileNameW(hModule: ?HMODULE, lpFilename: [*]u16, nSize: DWORD) DWORD; -pub extern "kernel32" stdcallcc fn GetModuleHandleA(lpModuleName: ?[*]const CHAR) ?HMODULE; - pub extern "kernel32" stdcallcc fn GetModuleHandleW(lpModuleName: ?[*]const WCHAR) HMODULE; pub extern "kernel32" stdcallcc fn GetLastError() DWORD; @@ -206,8 +204,6 @@ pub extern "kernel32" stdcallcc fn WriteFile( pub extern "kernel32" stdcallcc fn WriteFileEx(hFile: HANDLE, lpBuffer: [*]const u8, nNumberOfBytesToWrite: DWORD, lpOverlapped: LPOVERLAPPED, lpCompletionRoutine: LPOVERLAPPED_COMPLETION_ROUTINE) BOOL; -pub extern "kernel32" stdcallcc fn LoadLibraryA(lpLibFileName: [*]const u8) ?HMODULE; - pub extern "kernel32" stdcallcc fn LoadLibraryW(lpLibFileName: [*]const u16) ?HMODULE; pub extern "kernel32" stdcallcc fn GetProcAddress(hModule: HMODULE, lpProcName: [*]const u8) ?FARPROC; diff --git a/lib/std/parker.zig b/lib/std/parker.zig index 8df6dcaa92..7c0a7a65f8 100644 --- a/lib/std/parker.zig +++ b/lib/std/parker.zig @@ -76,107 +76,39 @@ const WindowsParker = struct { pub fn deinit(self: *WindowsParker) void {} pub fn unpark(self: *WindowsParker, ptr: *const u32) void { - switch (Backend.get().*) { - .WaitAddress => |*backend| backend.unpark(ptr, &self.waiters), - .KeyedEvent => |*backend| backend.unpark(ptr, &self.waiters), + const handle = getEventHandlePtr().*; + const key = @ptrCast(*const c_void, ptr); + var waiting = @atomicLoad(u32, &self.waiters, .Monotonic); + while (waiting != 0) { + waiting = @cmpxchgWeak(u32, &self.waiters, waiting, waiting - 1, .Acquire, .Monotonic) orelse { + const rc = windows.ntdll.NtReleaseKeyedEvent(handle, key, windows.FALSE, null); + assert(rc == 0); + return; + }; } } pub fn park(self: *WindowsParker, ptr: *const u32, expected: u32) void { - switch (Backend.get().*) { - .WaitAddress => |*backend| backend.park(ptr, expected, &self.waiters), - .KeyedEvent => |*backend| backend.park(ptr, expected, &self.waiters), + const handle = getEventHandlePtr().*; + const key = @ptrCast(*const c_void, ptr); + while (@atomicLoad(u32, ptr, .Acquire) == expected) { + _ = @atomicRmw(u32, &self.waiters, .Add, 1, .Release); + const rc = windows.ntdll.NtWaitForKeyedEvent(handle, key, windows.FALSE, null); + assert(rc == 0); } } - const Backend = union(enum) { - WaitAddress: WaitAddress, - KeyedEvent: KeyedEvent, + var event_handle = std.lazyInit(windows.HANDLE); - var backend = std.lazyInit(Backend); - - fn get() *const Backend { - return backend.get() orelse { - // Statically linking to the KeyedEvent functions should mean its supported. - // TODO: Maybe add a CreateSemaphore backend for systems older than Windows XP ? - backend.data = WaitAddress.init() - orelse KeyedEvent.init() - orelse unreachable; - backend.resolve(); - return &backend.data; - }; - } - - const WaitAddress = struct { - WakeByAddressSingle: stdcallcc fn(Address: *const c_void) void, - WaitOnAddress: stdcallcc fn ( - Address: *const c_void, - CompareAddress: *const c_void, - AddressSize: windows.SIZE_T, - dwMilliseconds: windows.DWORD, - ) windows.BOOL, - - fn init() ?Backend { - const dll_name = c"api-ms-win-core-synch-l1-2-0"; - const dll = windows.kernel32.GetModuleHandleA(dll_name) - orelse windows.kernel32.LoadLibraryA(dll_name) - orelse return null; - - var self: WaitAddress = undefined; - const WaitOnAddress = windows.kernel32.GetProcAddress(dll, c"WaitOnAddress") orelse return null; - self.WaitOnAddress = @intToPtr(@typeOf(self.WaitOnAddress), @ptrToInt(WaitOnAddress)); - const WakeByAddressSingle = windows.kernel32.GetProcAddress(dll, c"WakeByAddressSingle") orelse return null; - self.WakeByAddressSingle = @intToPtr(@typeOf(self.WakeByAddressSingle), @ptrToInt(WakeByAddressSingle)); - return Backend{ .WaitAddress = self }; - } - - fn unpark(self: WaitAddress, ptr: *const u32, waiters: *u32) void { - const addr = @ptrCast(*const c_void, ptr); - self.WakeByAddressSingle(addr); - } - - fn park(self: WaitAddress, ptr: *const u32, expected: u32, waiters: *u32) void { - var compare = expected; - const addr = @ptrCast(*const c_void, ptr); - const cmp = @ptrCast(*const c_void, &compare); - while (@atomicLoad(u32, ptr, .Acquire) == expected) - _ = self.WaitOnAddress(addr, cmp, @sizeOf(u32), windows.INFINITE); - } + fn getEventHandlePtr() *const windows.HANDLE { + return event_handle.get() orelse { + const access_mask = windows.GENERIC_READ | windows.GENERIC_WRITE; + if (windows.ntdll.NtCreateKeyedEvent(&event_handle.data, access_mask, null, 0) != 0) + @panic("Failed to setup an NT Keyed Event handle for the process"); + event_handle.resolve(); + return &event_handle.data; }; - - const KeyedEvent = struct { - handle: windows.HANDLE, - - fn init() ?Backend { - var self: KeyedEvent = undefined; - const access_mask = windows.GENERIC_READ | windows.GENERIC_WRITE; - if (windows.ntdll.NtCreateKeyedEvent(&self.handle, access_mask, null, 0) != 0) - return null; - return Backend{ .KeyedEvent = self }; - } - - fn unpark(self: KeyedEvent, ptr: *const u32, waiters: *u32) void { - const key = @ptrCast(*const c_void, ptr); - var waiting = @atomicLoad(u32, waiters, .Acquire); - while (waiting != 0) { - waiting = @cmpxchgWeak(u32, waiters, waiting, waiting - 1, .Acquire, .Monotonic) orelse { - const rc = windows.ntdll.NtReleaseKeyedEvent(self.handle, key, windows.FALSE, null); - assert(rc == 0); - return; - }; - } - } - - fn park(self: KeyedEvent, ptr: *const u32, expected: u32, waiters: *u32) void { - const key = @ptrCast(*const c_void, ptr); - while (@atomicLoad(u32, ptr, .Acquire) == expected) { - _ = @atomicRmw(u32, waiters, .Add, 1, .Release); - const rc = windows.ntdll.NtWaitForKeyedEvent(self.handle, key, windows.FALSE, null); - assert(rc == 0); - } - } - }; - }; + } }; const PosixParker = struct { From f41e58d015eec5fa595b4df8af3cf6a2b598bb02 Mon Sep 17 00:00:00 2001 From: kprotty Date: Thu, 7 Nov 2019 15:51:20 -0600 Subject: [PATCH 12/18] fix SpinLock.yield for pull/3626 --- lib/std/spinlock.zig | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/lib/std/spinlock.zig b/lib/std/spinlock.zig index 55b92f51d1..f69aae1bec 100644 --- a/lib/std/spinlock.zig +++ b/lib/std/spinlock.zig @@ -32,11 +32,8 @@ pub const SpinLock = struct { var i = iterations; while (i != 0) : (i -= 1) { switch (builtin.arch) { - .i386, .x86_64 => asm volatile("pause" ::: "memory"), - // .arm, .aarch64 => asm volatile("yield"), - // - // Causes CI to fail - // See: https://github.com/ziglang/zig/pull/3585#issuecomment-549962765 + .i386, .x86_64 => asm volatile("pause"), + .arm, .aarch64 => asm volatile("yield"), else => time.sleep(0), } } From 12e68cbeb636c920475b2420314c23de37689509 Mon Sep 17 00:00:00 2001 From: kprotty Date: Thu, 7 Nov 2019 16:33:25 -0600 Subject: [PATCH 13/18] pthread_sched_yield -> sched_yield --- lib/std/c.zig | 3 ++- lib/std/mutex.zig | 2 +- lib/std/os.zig | 14 +++++++------- lib/std/spinlock.zig | 2 +- 4 files changed, 11 insertions(+), 10 deletions(-) diff --git a/lib/std/c.zig b/lib/std/c.zig index 201aa6b103..26fa71fe68 100644 --- a/lib/std/c.zig +++ b/lib/std/c.zig @@ -158,7 +158,6 @@ pub extern "c" fn pthread_attr_init(attr: *pthread_attr_t) c_int; pub extern "c" fn pthread_attr_setstack(attr: *pthread_attr_t, stackaddr: *c_void, stacksize: usize) c_int; pub extern "c" fn pthread_attr_destroy(attr: *pthread_attr_t) c_int; pub extern "c" fn pthread_self() pthread_t; -pub extern "c" fn pthread_yield() c_int; pub extern "c" fn pthread_join(thread: pthread_t, arg_return: ?*?*c_void) c_int; pub extern "c" fn kqueue() c_int; @@ -201,3 +200,5 @@ pub extern "c" fn dn_expand( exp_dn: [*]u8, length: c_int, ) c_int; + +pub extern "c" fn sched_yield() c_int; diff --git a/lib/std/mutex.zig b/lib/std/mutex.zig index 110f581f5e..5387bd4b15 100644 --- a/lib/std/mutex.zig +++ b/lib/std/mutex.zig @@ -100,7 +100,7 @@ else struct { var value = @atomicLoad(u32, &self.state, .Monotonic); while (value == Unlocked) value = @cmpxchgWeak(u32, &self.state, Unlocked, state, .Acquire, .Monotonic) orelse return Held{ .mutex = self }; - std.os.yield(); + std.os.sched_yield(); } // failed to acquire the lock, go to sleep until woken up by `Held.release()` diff --git a/lib/std/os.zig b/lib/std/os.zig index 72dd2a7196..2f19ee2a7d 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -3170,12 +3170,12 @@ pub fn dn_expand( return error.InvalidDnsPacket; } -pub fn yield() void { - switch (builtin.os) { - .windows => _ = windows.kernel32.SwitchToThread(), - .linux => _ = assert(linux.sched_yield() == 0), - else => if (builtin.link_libc) { - assert(std.c.pthread_yield() == 0); - }, +pub fn sched_yield() void { + if (builtin.os == .windows) { + _ = windows.kernel32.SwitchToThread(); + } else if (builtin.os == .linux and !builtin.link_libc) { + assert(linux.sched_yield() == 0); + } else if (builtin.link_libc) { + assert(std.c.sched_yield() == 0); } } diff --git a/lib/std/spinlock.zig b/lib/std/spinlock.zig index f69aae1bec..f2f2833875 100644 --- a/lib/std/spinlock.zig +++ b/lib/std/spinlock.zig @@ -54,7 +54,7 @@ pub const SpinLock = struct { if (self.iteration < 20) { SpinLock.yield(self.iteration); } else if (self.iteration < 24) { - os.yield(); + os.sched_yield(); } else if (self.iteration < 26) { time.sleep(1 * time.millisecond); } else { From bb4abfdc78b527b315950ce9f233518cdaa6a72a Mon Sep 17 00:00:00 2001 From: kprotty Date: Thu, 7 Nov 2019 16:36:33 -0600 Subject: [PATCH 14/18] Use `system` instead of `builtin.link_libc` --- lib/std/os.zig | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/lib/std/os.zig b/lib/std/os.zig index 2f19ee2a7d..d9355490f7 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -3171,11 +3171,8 @@ pub fn dn_expand( } pub fn sched_yield() void { - if (builtin.os == .windows) { - _ = windows.kernel32.SwitchToThread(); - } else if (builtin.os == .linux and !builtin.link_libc) { - assert(linux.sched_yield() == 0); - } else if (builtin.link_libc) { - assert(std.c.sched_yield() == 0); + switch (builtin.os) { + .windows => _ = windows.kernel32.SwitchToThread(), + else => assert(system.sched_yield() == 0), } } From fe8c1cf804c3c39e57ca291baf005b862fc3ba94 Mon Sep 17 00:00:00 2001 From: kprotty Date: Thu, 7 Nov 2019 17:14:08 -0600 Subject: [PATCH 15/18] missing `os` declaration --- lib/std/spinlock.zig | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/std/spinlock.zig b/lib/std/spinlock.zig index f2f2833875..b374eba3eb 100644 --- a/lib/std/spinlock.zig +++ b/lib/std/spinlock.zig @@ -2,8 +2,7 @@ const std = @import("std.zig"); const builtin = @import("builtin"); const assert = std.debug.assert; const time = std.time; -const linux = std.os.linux; -const windows = std.os.windows; +const os = std.os; pub const SpinLock = struct { lock: u8, // TODO use a bool or enum From 43900011f7379e61564196141a687a2b8352d27f Mon Sep 17 00:00:00 2001 From: kprotty Date: Thu, 7 Nov 2019 18:56:43 -0600 Subject: [PATCH 16/18] handle NtCreateKeyedEvent failure by spinning to ensure correctness --- lib/std/parker.zig | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/lib/std/parker.zig b/lib/std/parker.zig index 7c0a7a65f8..11d3f749de 100644 --- a/lib/std/parker.zig +++ b/lib/std/parker.zig @@ -1,5 +1,6 @@ const std = @import("std.zig"); const builtin = @import("builtin"); +const time = std.time; const testing = std.testing; const assert = std.debug.assert; const SpinLock = std.SpinLock; @@ -76,12 +77,13 @@ const WindowsParker = struct { pub fn deinit(self: *WindowsParker) void {} pub fn unpark(self: *WindowsParker, ptr: *const u32) void { - const handle = getEventHandlePtr().*; const key = @ptrCast(*const c_void, ptr); + const handle_ptr = getEventHandlePtr() orelse return; + var waiting = @atomicLoad(u32, &self.waiters, .Monotonic); while (waiting != 0) { waiting = @cmpxchgWeak(u32, &self.waiters, waiting, waiting - 1, .Acquire, .Monotonic) orelse { - const rc = windows.ntdll.NtReleaseKeyedEvent(handle, key, windows.FALSE, null); + const rc = windows.ntdll.NtReleaseKeyedEvent(handle_ptr.*, key, windows.FALSE, null); assert(rc == 0); return; }; @@ -89,22 +91,27 @@ const WindowsParker = struct { } pub fn park(self: *WindowsParker, ptr: *const u32, expected: u32) void { - const handle = getEventHandlePtr().*; + var spin = SpinLock.Backoff.init(); const key = @ptrCast(*const c_void, ptr); + while (@atomicLoad(u32, ptr, .Acquire) == expected) { - _ = @atomicRmw(u32, &self.waiters, .Add, 1, .Release); - const rc = windows.ntdll.NtWaitForKeyedEvent(handle, key, windows.FALSE, null); - assert(rc == 0); + if (getEventHandlePtr()) |handle_ptr| { + _ = @atomicRmw(u32, &self.waiters, .Add, 1, .Release); + const rc = windows.ntdll.NtWaitForKeyedEvent(handle_ptr.*, key, windows.FALSE, null); + assert(rc == 0); + } else { + spin.yield(); + } } } var event_handle = std.lazyInit(windows.HANDLE); - fn getEventHandlePtr() *const windows.HANDLE { + fn getEventHandlePtr() ?*const windows.HANDLE { return event_handle.get() orelse { const access_mask = windows.GENERIC_READ | windows.GENERIC_WRITE; if (windows.ntdll.NtCreateKeyedEvent(&event_handle.data, access_mask, null, 0) != 0) - @panic("Failed to setup an NT Keyed Event handle for the process"); + return null; event_handle.resolve(); return &event_handle.data; }; From e2189b6e5d5a5644eb937b682cdfe4e658fe27e8 Mon Sep 17 00:00:00 2001 From: kprotty Date: Thu, 7 Nov 2019 19:37:53 -0600 Subject: [PATCH 17/18] cleanup getEventHandle --- lib/std/parker.zig | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/lib/std/parker.zig b/lib/std/parker.zig index 11d3f749de..036fe160ce 100644 --- a/lib/std/parker.zig +++ b/lib/std/parker.zig @@ -78,12 +78,12 @@ const WindowsParker = struct { pub fn unpark(self: *WindowsParker, ptr: *const u32) void { const key = @ptrCast(*const c_void, ptr); - const handle_ptr = getEventHandlePtr() orelse return; + const handle = getEventHandle() orelse return; var waiting = @atomicLoad(u32, &self.waiters, .Monotonic); while (waiting != 0) { waiting = @cmpxchgWeak(u32, &self.waiters, waiting, waiting - 1, .Acquire, .Monotonic) orelse { - const rc = windows.ntdll.NtReleaseKeyedEvent(handle_ptr.*, key, windows.FALSE, null); + const rc = windows.ntdll.NtReleaseKeyedEvent(handle, key, windows.FALSE, null); assert(rc == 0); return; }; @@ -92,12 +92,13 @@ const WindowsParker = struct { pub fn park(self: *WindowsParker, ptr: *const u32, expected: u32) void { var spin = SpinLock.Backoff.init(); + const ev_handle = getEventHandle(); const key = @ptrCast(*const c_void, ptr); - while (@atomicLoad(u32, ptr, .Acquire) == expected) { - if (getEventHandlePtr()) |handle_ptr| { + while (@atomicLoad(u32, ptr, .Monotonic) == expected) { + if (ev_handle) |handle| { _ = @atomicRmw(u32, &self.waiters, .Add, 1, .Release); - const rc = windows.ntdll.NtWaitForKeyedEvent(handle_ptr.*, key, windows.FALSE, null); + const rc = windows.ntdll.NtWaitForKeyedEvent(handle, key, windows.FALSE, null); assert(rc == 0); } else { spin.yield(); @@ -107,14 +108,15 @@ const WindowsParker = struct { var event_handle = std.lazyInit(windows.HANDLE); - fn getEventHandlePtr() ?*const windows.HANDLE { - return event_handle.get() orelse { - const access_mask = windows.GENERIC_READ | windows.GENERIC_WRITE; - if (windows.ntdll.NtCreateKeyedEvent(&event_handle.data, access_mask, null, 0) != 0) - return null; - event_handle.resolve(); - return &event_handle.data; - }; + fn getEventHandle() ?windows.HANDLE { + if (event_handle.get()) |handle_ptr| + return handle_ptr.*; + defer event_handle.resolve(); + + const access_mask = windows.GENERIC_READ | windows.GENERIC_WRITE; + if (windows.ntdll.NtCreateKeyedEvent(&event_handle.data, access_mask, null, 0) != 0) + return null; + return event_handle.data; } }; From 2723ffc2b2d837aec8185f86814a6e6aba45c34d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 8 Nov 2019 01:52:23 -0500 Subject: [PATCH 18/18] fixups and zig fmt --- lib/std/c.zig | 22 +++- lib/std/c/darwin.zig | 16 +++ lib/std/c/dragonfly.zig | 13 ++- lib/std/c/emscripten.zig | 8 ++ lib/std/c/freebsd.zig | 12 ++ lib/std/c/fuchsia.zig | 8 ++ lib/std/c/haiku.zig | 14 +++ lib/std/c/hermit.zig | 6 + lib/std/c/linux.zig | 23 ++++ lib/std/c/minix.zig | 18 +++ lib/std/c/netbsd.zig | 29 +++++ lib/std/c/openbsd.zig | 6 + lib/std/c/solaris.zig | 15 +++ lib/std/mutex.zig | 131 ++++++++++----------- lib/std/os.zig | 20 +++- lib/std/os/bits.zig | 3 - lib/std/os/bits/darwin.zig | 5 - lib/std/os/bits/dragonfly.zig | 11 +- lib/std/os/bits/freebsd.zig | 5 - lib/std/os/bits/linux.zig | 5 - lib/std/os/bits/netbsd.zig | 6 - lib/std/parker.zig | 206 ++++++---------------------------- lib/std/spinlock.zig | 6 +- 23 files changed, 309 insertions(+), 279 deletions(-) create mode 100644 lib/std/c/emscripten.zig create mode 100644 lib/std/c/fuchsia.zig create mode 100644 lib/std/c/haiku.zig create mode 100644 lib/std/c/hermit.zig create mode 100644 lib/std/c/minix.zig create mode 100644 lib/std/c/openbsd.zig create mode 100644 lib/std/c/solaris.zig diff --git a/lib/std/c.zig b/lib/std/c.zig index f70c7e30c4..3fd39b26f5 100644 --- a/lib/std/c.zig +++ b/lib/std/c.zig @@ -8,9 +8,16 @@ pub usingnamespace switch (builtin.os) { .linux => @import("c/linux.zig"), .windows => @import("c/windows.zig"), .macosx, .ios, .tvos, .watchos => @import("c/darwin.zig"), - .freebsd => @import("c/freebsd.zig"), + .freebsd, .kfreebsd => @import("c/freebsd.zig"), .netbsd => @import("c/netbsd.zig"), .dragonfly => @import("c/dragonfly.zig"), + .openbsd => @import("c/openbsd.zig"), + .haiku => @import("c/haiku.zig"), + .hermit => @import("c/hermit.zig"), + .solaris => @import("c/solaris.zig"), + .fuchsia => @import("c/fuchsia.zig"), + .minix => @import("c/minix.zig"), + .emscripten => @import("c/emscripten.zig"), else => struct {}, }; @@ -205,3 +212,16 @@ pub extern "c" fn dn_expand( ) c_int; pub extern "c" fn sched_yield() c_int; + +pub const PTHREAD_MUTEX_INITIALIZER = pthread_mutex_t{}; +pub extern "c" fn pthread_mutex_lock(mutex: *pthread_mutex_t) c_int; +pub extern "c" fn pthread_mutex_unlock(mutex: *pthread_mutex_t) c_int; +pub extern "c" fn pthread_mutex_destroy(mutex: *pthread_mutex_t) c_int; + +pub const PTHREAD_COND_INITIALIZER = pthread_cond_t{}; +pub extern "c" fn pthread_cond_wait(noalias cond: *pthread_cond_t, noalias mutex: *pthread_mutex_t) c_int; +pub extern "c" fn pthread_cond_signal(cond: *pthread_cond_t) c_int; +pub extern "c" fn pthread_cond_destroy(cond: *pthread_cond_t) c_int; + +pub const pthread_t = *@OpaqueType(); +pub const FILE = @OpaqueType(); diff --git a/lib/std/c/darwin.zig b/lib/std/c/darwin.zig index f3cd77ec6f..eaccb1dcfd 100644 --- a/lib/std/c/darwin.zig +++ b/lib/std/c/darwin.zig @@ -112,3 +112,19 @@ pub const EAI_PROTOCOL = 13; /// argument buffer overflow pub const EAI_OVERFLOW = 14; pub const EAI_MAX = 15; + +pub const pthread_mutex_t = extern struct { + __sig: c_long = 0x32AAABA7, + __opaque: [__PTHREAD_MUTEX_SIZE__]u8 = [_]u8{0} ** __PTHREAD_MUTEX_SIZE__, +}; +pub const pthread_cond_t = extern struct { + __sig: c_long = 0x3CB0B1BB, + __opaque: [__PTHREAD_COND_SIZE__]u8 = [_]u8{0} ** __PTHREAD_COND_SIZE__, +}; +const __PTHREAD_MUTEX_SIZE__ = if (@sizeOf(usize) == 8) 56 else 40; +const __PTHREAD_COND_SIZE__ = if (@sizeOf(usize) == 8) 40 else 24; + +pub const pthread_attr_t = extern struct { + __sig: c_long, + __opaque: [56]u8, +}; diff --git a/lib/std/c/dragonfly.zig b/lib/std/c/dragonfly.zig index bd44a25514..a271b2e869 100644 --- a/lib/std/c/dragonfly.zig +++ b/lib/std/c/dragonfly.zig @@ -1,6 +1,5 @@ const std = @import("../std.zig"); usingnamespace std.c; - extern "c" threadlocal var errno: c_int; pub fn _errno() *c_int { return &errno; @@ -12,3 +11,15 @@ pub extern "c" fn getrandom(buf_ptr: [*]u8, buf_len: usize, flags: c_uint) isize pub const dl_iterate_phdr_callback = extern fn (info: *dl_phdr_info, size: usize, data: ?*c_void) c_int; pub extern "c" fn dl_iterate_phdr(callback: dl_iterate_phdr_callback, data: ?*c_void) c_int; + +pub const pthread_mutex_t = extern struct { + inner: ?*c_void = null, +}; +pub const pthread_cond_t = extern struct { + inner: ?*c_void = null, +}; + +pub const pthread_attr_t = extern struct { // copied from freebsd + __size: [56]u8, + __align: c_long, +}; diff --git a/lib/std/c/emscripten.zig b/lib/std/c/emscripten.zig new file mode 100644 index 0000000000..e91e1421c7 --- /dev/null +++ b/lib/std/c/emscripten.zig @@ -0,0 +1,8 @@ +pub const pthread_mutex_t = extern struct { + size: [__SIZEOF_PTHREAD_MUTEX_T]u8 align(4) = [_]u8{0} ** __SIZEOF_PTHREAD_MUTEX_T, +}; +pub const pthread_cond_t = extern struct { + size: [__SIZEOF_PTHREAD_COND_T]u8 align(@alignOf(usize)) = [_]u8{0} ** __SIZEOF_PTHREAD_COND_T, +}; +const __SIZEOF_PTHREAD_COND_T = 48; +const __SIZEOF_PTHREAD_MUTEX_T = 28; diff --git a/lib/std/c/freebsd.zig b/lib/std/c/freebsd.zig index 550b5a59b1..4f18df15fd 100644 --- a/lib/std/c/freebsd.zig +++ b/lib/std/c/freebsd.zig @@ -10,3 +10,15 @@ pub extern "c" fn getrandom(buf_ptr: [*]u8, buf_len: usize, flags: c_uint) isize pub const dl_iterate_phdr_callback = extern fn (info: *dl_phdr_info, size: usize, data: ?*c_void) c_int; pub extern "c" fn dl_iterate_phdr(callback: dl_iterate_phdr_callback, data: ?*c_void) c_int; + +pub const pthread_mutex_t = extern struct { + inner: ?*c_void = null, +}; +pub const pthread_cond_t = extern struct { + inner: ?*c_void = null, +}; + +pub const pthread_attr_t = extern struct { + __size: [56]u8, + __align: c_long, +}; diff --git a/lib/std/c/fuchsia.zig b/lib/std/c/fuchsia.zig new file mode 100644 index 0000000000..4f52b8900d --- /dev/null +++ b/lib/std/c/fuchsia.zig @@ -0,0 +1,8 @@ +pub const pthread_mutex_t = extern struct { + size: [__SIZEOF_PTHREAD_MUTEX_T]u8 align(@alignOf(usize)) = [_]u8{0} ** __SIZEOF_PTHREAD_MUTEX_T, +}; +pub const pthread_cond_t = extern struct { + size: [__SIZEOF_PTHREAD_COND_T]u8 align(@alignOf(usize)) = [_]u8{0} ** __SIZEOF_PTHREAD_COND_T, +}; +const __SIZEOF_PTHREAD_COND_T = 48; +const __SIZEOF_PTHREAD_MUTEX_T = 40; diff --git a/lib/std/c/haiku.zig b/lib/std/c/haiku.zig new file mode 100644 index 0000000000..eb1e52a618 --- /dev/null +++ b/lib/std/c/haiku.zig @@ -0,0 +1,14 @@ +pub const pthread_mutex_t = extern struct { + flags: u32 = 0, + lock: i32 = 0, + unused: i32 = -42, + owner: i32 = -1, + owner_count: i32 = 0, +}; +pub const pthread_cond_t = extern struct { + flags: u32 = 0, + unused: i32 = -42, + mutex: ?*c_void = null, + waiter_count: i32 = 0, + lock: i32 = 0, +}; diff --git a/lib/std/c/hermit.zig b/lib/std/c/hermit.zig new file mode 100644 index 0000000000..b9be76c3ba --- /dev/null +++ b/lib/std/c/hermit.zig @@ -0,0 +1,6 @@ +pub const pthread_mutex_t = extern struct { + inner: usize = ~usize(0), +}; +pub const pthread_cond_t = extern struct { + inner: usize = ~usize(0), +}; diff --git a/lib/std/c/linux.zig b/lib/std/c/linux.zig index a1db162e20..07fdceec9a 100644 --- a/lib/std/c/linux.zig +++ b/lib/std/c/linux.zig @@ -75,3 +75,26 @@ pub const dl_iterate_phdr_callback = extern fn (info: *dl_phdr_info, size: usize pub extern "c" fn dl_iterate_phdr(callback: dl_iterate_phdr_callback, data: ?*c_void) c_int; pub extern "c" fn sigaltstack(ss: ?*stack_t, old_ss: ?*stack_t) c_int; + +pub const pthread_attr_t = extern struct { + __size: [56]u8, + __align: c_long, +}; + +pub const pthread_mutex_t = extern struct { + size: [__SIZEOF_PTHREAD_MUTEX_T]u8 align(@alignOf(usize)) = [_]u8{0} ** __SIZEOF_PTHREAD_MUTEX_T, +}; +pub const pthread_cond_t = extern struct { + size: [__SIZEOF_PTHREAD_COND_T]u8 align(@alignOf(usize)) = [_]u8{0} ** __SIZEOF_PTHREAD_COND_T, +}; +const __SIZEOF_PTHREAD_COND_T = 48; +const __SIZEOF_PTHREAD_MUTEX_T = if (builtin.os == .fuchsia) 40 else switch (builtin.abi) { + .musl, .musleabi, .musleabihf => if (@sizeOf(usize) == 8) 40 else 24, + .gnu, .gnuabin32, .gnuabi64, .gnueabi, .gnueabihf, .gnux32 => switch (builtin.arch) { + .aarch64 => 48, + .x86_64 => if (builtin.abi == .gnux32) 40 else 32, + .mips64, .powerpc64, .powerpc64le, .sparcv9 => 40, + else => if (@sizeOf(usize) == 8) 40 else 24, + }, + else => unreachable, +}; diff --git a/lib/std/c/minix.zig b/lib/std/c/minix.zig new file mode 100644 index 0000000000..98ec087a93 --- /dev/null +++ b/lib/std/c/minix.zig @@ -0,0 +1,18 @@ +const builtin = @import("builtin"); +pub const pthread_mutex_t = extern struct { + size: [__SIZEOF_PTHREAD_MUTEX_T]u8 align(@alignOf(usize)) = [_]u8{0} ** __SIZEOF_PTHREAD_MUTEX_T, +}; +pub const pthread_cond_t = extern struct { + size: [__SIZEOF_PTHREAD_COND_T]u8 align(@alignOf(usize)) = [_]u8{0} ** __SIZEOF_PTHREAD_COND_T, +}; +const __SIZEOF_PTHREAD_COND_T = 48; +const __SIZEOF_PTHREAD_MUTEX_T = switch (builtin.abi) { + .musl, .musleabi, .musleabihf => if (@sizeOf(usize) == 8) 40 else 24, + .gnu, .gnuabin32, .gnuabi64, .gnueabi, .gnueabihf, .gnux32 => switch (builtin.arch) { + .aarch64 => 48, + .x86_64 => if (builtin.abi == .gnux32) 40 else 32, + .mips64, .powerpc64, .powerpc64le, .sparcv9 => 40, + else => if (@sizeOf(usize) == 8) 40 else 24, + }, + else => unreachable, +}; diff --git a/lib/std/c/netbsd.zig b/lib/std/c/netbsd.zig index 417c78db69..c253362ac1 100644 --- a/lib/std/c/netbsd.zig +++ b/lib/std/c/netbsd.zig @@ -6,3 +6,32 @@ pub const _errno = __errno; pub extern "c" fn getdents(fd: c_int, buf_ptr: [*]u8, nbytes: usize) usize; pub extern "c" fn sigaltstack(ss: ?*stack_t, old_ss: ?*stack_t) c_int; + +pub const pthread_mutex_t = extern struct { + ptm_magic: c_uint = 0x33330003, + ptm_errorcheck: padded_spin_t = 0, + ptm_unused: padded_spin_t = 0, + ptm_owner: usize = 0, + ptm_waiters: ?*u8 = null, + ptm_recursed: c_uint = 0, + ptm_spare2: ?*c_void = null, +}; +pub const pthread_cond_t = extern struct { + ptc_magic: c_uint = 0x55550005, + ptc_lock: pthread_spin_t = 0, + ptc_waiters_first: ?*u8 = null, + ptc_waiters_last: ?*u8 = null, + ptc_mutex: ?*pthread_mutex_t = null, + ptc_private: ?*c_void = null, +}; +const pthread_spin_t = if (builtin.arch == .arm or .arch == .powerpc) c_int else u8; +const padded_spin_t = switch (builtin.arch) { + .sparc, .sparcel, .sparcv9, .i386, .x86_64, .le64 => u32, + else => spin_t, +}; + +pub const pthread_attr_t = extern struct { + pta_magic: u32, + pta_flags: c_int, + pta_private: *c_void, +}; diff --git a/lib/std/c/openbsd.zig b/lib/std/c/openbsd.zig new file mode 100644 index 0000000000..22963b08c3 --- /dev/null +++ b/lib/std/c/openbsd.zig @@ -0,0 +1,6 @@ +pub const pthread_mutex_t = extern struct { + inner: ?*c_void = null, +}; +pub const pthread_cond_t = extern struct { + inner: ?*c_void = null, +}; diff --git a/lib/std/c/solaris.zig b/lib/std/c/solaris.zig new file mode 100644 index 0000000000..7c70a01fc4 --- /dev/null +++ b/lib/std/c/solaris.zig @@ -0,0 +1,15 @@ +pub const pthread_mutex_t = extern struct { + __pthread_mutex_flag1: u16 = 0, + __pthread_mutex_flag2: u8 = 0, + __pthread_mutex_ceiling: u8 = 0, + __pthread_mutex_type: u16 = 0, + __pthread_mutex_magic: u16 = 0x4d58, + __pthread_mutex_lock: u64 = 0, + __pthread_mutex_data: u64 = 0, +}; +pub const pthread_cond_t = extern struct { + __pthread_cond_flag: u32 = 0, + __pthread_cond_type: u16 = 0, + __pthread_cond_magic: u16 = 0x4356, + __pthread_cond_data: u64 = 0, +}; diff --git a/lib/std/mutex.zig b/lib/std/mutex.zig index 5387bd4b15..9b44e2edc3 100644 --- a/lib/std/mutex.zig +++ b/lib/std/mutex.zig @@ -37,81 +37,84 @@ pub const Mutex = if (builtin.single_threaded) return Held{ .mutex = self }; } } -else struct { - state: u32, // TODO: make this an enum - parker: ThreadParker, +else + struct { + state: u32, // TODO: make this an enum + parker: ThreadParker, - const Unlocked = 0; - const Sleeping = 1; - const Locked = 2; + const Unlocked = 0; + const Sleeping = 1; + const Locked = 2; - /// number of iterations to spin yielding the cpu - const SPIN_CPU = 4; - /// number of iterations to perform in the cpu yield loop - const SPIN_CPU_COUNT = 30; - /// number of iterations to spin yielding the thread - const SPIN_THREAD = 1; + /// number of iterations to spin yielding the cpu + const SPIN_CPU = 4; - pub fn init() Mutex { - return Mutex{ - .state = Unlocked, - .parker = ThreadParker.init(), + /// number of iterations to perform in the cpu yield loop + const SPIN_CPU_COUNT = 30; + + /// number of iterations to spin yielding the thread + const SPIN_THREAD = 1; + + pub fn init() Mutex { + return Mutex{ + .state = Unlocked, + .parker = ThreadParker.init(), + }; + } + + pub fn deinit(self: *Mutex) void { + self.parker.deinit(); + } + + pub const Held = struct { + mutex: *Mutex, + + pub fn release(self: Held) void { + switch (@atomicRmw(u32, &self.mutex.state, .Xchg, Unlocked, .Release)) { + Locked => {}, + Sleeping => self.mutex.parker.unpark(&self.mutex.state), + Unlocked => unreachable, // unlocking an unlocked mutex + else => unreachable, // should never be anything else + } + } }; - } - pub fn deinit(self: *Mutex) void { - self.parker.deinit(); - } + pub fn acquire(self: *Mutex) Held { + // Try and speculatively grab the lock. + // If it fails, the state is either Locked or Sleeping + // depending on if theres a thread stuck sleeping below. + var state = @atomicRmw(u32, &self.state, .Xchg, Locked, .Acquire); + if (state == Unlocked) + return Held{ .mutex = self }; - pub const Held = struct { - mutex: *Mutex, + while (true) { + // try and acquire the lock using cpu spinning on failure + var spin: usize = 0; + while (spin < SPIN_CPU) : (spin += 1) { + var value = @atomicLoad(u32, &self.state, .Monotonic); + while (value == Unlocked) + value = @cmpxchgWeak(u32, &self.state, Unlocked, state, .Acquire, .Monotonic) orelse return Held{ .mutex = self }; + SpinLock.yield(SPIN_CPU_COUNT); + } - pub fn release(self: Held) void { - switch (@atomicRmw(u32, &self.mutex.state, .Xchg, Unlocked, .Release)) { - Locked => {}, - Sleeping => self.mutex.parker.unpark(&self.mutex.state), - Unlocked => unreachable, // unlocking an unlocked mutex - else => unreachable, // should never be anything else + // try and acquire the lock using thread rescheduling on failure + spin = 0; + while (spin < SPIN_THREAD) : (spin += 1) { + var value = @atomicLoad(u32, &self.state, .Monotonic); + while (value == Unlocked) + value = @cmpxchgWeak(u32, &self.state, Unlocked, state, .Acquire, .Monotonic) orelse return Held{ .mutex = self }; + std.os.sched_yield() catch std.time.sleep(1); + } + + // failed to acquire the lock, go to sleep until woken up by `Held.release()` + if (@atomicRmw(u32, &self.state, .Xchg, Sleeping, .Acquire) == Unlocked) + return Held{ .mutex = self }; + state = Sleeping; + self.parker.park(&self.state, Sleeping); } } }; - pub fn acquire(self: *Mutex) Held { - // Try and speculatively grab the lock. - // If it fails, the state is either Locked or Sleeping - // depending on if theres a thread stuck sleeping below. - var state = @atomicRmw(u32, &self.state, .Xchg, Locked, .Acquire); - if (state == Unlocked) - return Held{ .mutex = self }; - - while (true) { - // try and acquire the lock using cpu spinning on failure - var spin: usize = 0; - while (spin < SPIN_CPU) : (spin += 1) { - var value = @atomicLoad(u32, &self.state, .Monotonic); - while (value == Unlocked) - value = @cmpxchgWeak(u32, &self.state, Unlocked, state, .Acquire, .Monotonic) orelse return Held{ .mutex = self }; - SpinLock.yield(SPIN_CPU_COUNT); - } - - // try and acquire the lock using thread rescheduling on failure - spin = 0; - while (spin < SPIN_THREAD) : (spin += 1) { - var value = @atomicLoad(u32, &self.state, .Monotonic); - while (value == Unlocked) - value = @cmpxchgWeak(u32, &self.state, Unlocked, state, .Acquire, .Monotonic) orelse return Held{ .mutex = self }; - std.os.sched_yield(); - } - - // failed to acquire the lock, go to sleep until woken up by `Held.release()` - if (@atomicRmw(u32, &self.state, .Xchg, Sleeping, .Acquire) == Unlocked) - return Held{ .mutex = self }; - state = Sleeping; - self.parker.park(&self.state, Sleeping); - } - } -}; - const TestContext = struct { mutex: *Mutex, data: i128, diff --git a/lib/std/os.zig b/lib/std/os.zig index cf97a5ee45..6803006bf1 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -3172,9 +3172,21 @@ pub fn dn_expand( return error.InvalidDnsPacket; } -pub fn sched_yield() void { - switch (builtin.os) { - .windows => _ = windows.kernel32.SwitchToThread(), - else => assert(system.sched_yield() == 0), +pub const SchedYieldError = error{ + /// The system is not configured to allow yielding + SystemCannotYield, +}; + +pub fn sched_yield() SchedYieldError!void { + if (builtin.os == .windows) { + // The return value has to do with how many other threads there are; it is not + // an error condition on Windows. + _ = windows.kernel32.SwitchToThread(); + return; + } + switch (errno(system.sched_yield())) { + 0 => return, + ENOSYS => return error.SystemCannotYield, + else => return error.SystemCannotYield, } } diff --git a/lib/std/os/bits.zig b/lib/std/os/bits.zig index 6d6163b8fa..7dd968e473 100644 --- a/lib/std/os/bits.zig +++ b/lib/std/os/bits.zig @@ -14,9 +14,6 @@ pub usingnamespace switch (builtin.os) { else => struct {}, }; -pub const pthread_t = *@OpaqueType(); -pub const FILE = @OpaqueType(); - pub const iovec = extern struct { iov_base: [*]u8, iov_len: usize, diff --git a/lib/std/os/bits/darwin.zig b/lib/std/os/bits/darwin.zig index ca7f663661..335f296dd6 100644 --- a/lib/std/os/bits/darwin.zig +++ b/lib/std/os/bits/darwin.zig @@ -133,11 +133,6 @@ pub const dirent = extern struct { } }; -pub const pthread_attr_t = extern struct { - __sig: c_long, - __opaque: [56]u8, -}; - /// Renamed from `kevent` to `Kevent` to avoid conflict with function name. pub const Kevent = extern struct { ident: usize, diff --git a/lib/std/os/bits/dragonfly.zig b/lib/std/os/bits/dragonfly.zig index d1b65707f4..87d89df7d8 100644 --- a/lib/std/os/bits/dragonfly.zig +++ b/lib/std/os/bits/dragonfly.zig @@ -360,11 +360,6 @@ pub const Kevent = extern struct { udata: usize, }; -pub const pthread_attr_t = extern struct { // copied from freebsd - __size: [56]u8, - __align: c_long, -}; - pub const EVFILT_FS = -10; pub const EVFILT_USER = -9; pub const EVFILT_EXCEPT = -8; @@ -515,13 +510,13 @@ pub const sigset_t = extern struct { pub const sig_atomic_t = c_int; pub const Sigaction = extern struct { __sigaction_u: extern union { - __sa_handler: ?extern fn(c_int) void, - __sa_sigaction: ?extern fn(c_int, [*c]siginfo_t, ?*c_void) void, + __sa_handler: ?extern fn (c_int) void, + __sa_sigaction: ?extern fn (c_int, [*c]siginfo_t, ?*c_void) void, }, sa_flags: c_int, sa_mask: sigset_t, }; -pub const sig_t = [*c]extern fn(c_int) void; +pub const sig_t = [*c]extern fn (c_int) void; pub const sigvec = extern struct { sv_handler: [*c]__sighandler_t, diff --git a/lib/std/os/bits/freebsd.zig b/lib/std/os/bits/freebsd.zig index b0506ad0e1..a7a2536133 100644 --- a/lib/std/os/bits/freebsd.zig +++ b/lib/std/os/bits/freebsd.zig @@ -15,11 +15,6 @@ pub const Kevent = extern struct { // TODO ext }; -pub const pthread_attr_t = extern struct { - __size: [56]u8, - __align: c_long, -}; - pub const dl_phdr_info = extern struct { dlpi_addr: usize, dlpi_name: ?[*]const u8, diff --git a/lib/std/os/bits/linux.zig b/lib/std/os/bits/linux.zig index 0cf30920e4..0fd528b0c0 100644 --- a/lib/std/os/bits/linux.zig +++ b/lib/std/os/bits/linux.zig @@ -1000,11 +1000,6 @@ pub const dl_phdr_info = extern struct { dlpi_phnum: u16, }; -pub const pthread_attr_t = extern struct { - __size: [56]u8, - __align: c_long, -}; - pub const CPU_SETSIZE = 128; pub const cpu_set_t = [CPU_SETSIZE / @sizeOf(usize)]usize; pub const cpu_count_t = @IntType(false, std.math.log2(CPU_SETSIZE * 8)); diff --git a/lib/std/os/bits/netbsd.zig b/lib/std/os/bits/netbsd.zig index f93bbbf106..497c4a0344 100644 --- a/lib/std/os/bits/netbsd.zig +++ b/lib/std/os/bits/netbsd.zig @@ -14,12 +14,6 @@ pub const Kevent = extern struct { udata: usize, }; -pub const pthread_attr_t = extern struct { - pta_magic: u32, - pta_flags: c_int, - pta_private: *c_void, -}; - pub const dl_phdr_info = extern struct { dlpi_addr: usize, dlpi_name: ?[*]const u8, diff --git a/lib/std/parker.zig b/lib/std/parker.zig index 036fe160ce..4ba0100b9e 100644 --- a/lib/std/parker.zig +++ b/lib/std/parker.zig @@ -8,24 +8,9 @@ const linux = std.os.linux; const windows = std.os.windows; pub const ThreadParker = switch (builtin.os) { - .macosx, - .tvos, - .ios, - .watchos, - .netbsd, - .openbsd, - .freebsd, - .kfreebsd, - .dragonfly, - .haiku, - .hermit, - .solaris, - .minix, - .fuchsia, - .emscripten => if (builtin.link_libc) PosixParker else SpinParker, .linux => if (builtin.link_libc) PosixParker else LinuxParker, .windows => WindowsParker, - else => SpinParker, + else => if (builtin.link_libc) PosixParker else SpinParker, }; const SpinParker = struct { @@ -62,7 +47,7 @@ const LinuxParker = struct { 0, linux.EAGAIN => return, linux.EINTR => continue, linux.EINVAL => unreachable, - else => unreachable, + else => continue, } } } @@ -94,7 +79,7 @@ const WindowsParker = struct { var spin = SpinLock.Backoff.init(); const ev_handle = getEventHandle(); const key = @ptrCast(*const c_void, ptr); - + while (@atomicLoad(u32, ptr, .Monotonic) == expected) { if (ev_handle) |handle| { _ = @atomicRmw(u32, &self.waiters, .Add, 1, .Release); @@ -121,191 +106,64 @@ const WindowsParker = struct { }; const PosixParker = struct { - cond: pthread_cond_t, - mutex: pthread_mutex_t, + cond: c.pthread_cond_t, + mutex: c.pthread_mutex_t, + + const c = std.c; pub fn init() PosixParker { return PosixParker{ - .cond = PTHREAD_COND_INITIALIZER, - .mutex = PTHREAD_MUTEX_INITIALIZER, + .cond = c.PTHREAD_COND_INITIALIZER, + .mutex = c.PTHREAD_MUTEX_INITIALIZER, }; } pub fn deinit(self: *PosixParker) void { // On dragonfly, the destroy functions return EINVAL if they were initialized statically. - const retm = pthread_mutex_destroy(&self.mutex); + const retm = c.pthread_mutex_destroy(&self.mutex); assert(retm == 0 or retm == (if (builtin.os == .dragonfly) os.EINVAL else 0)); - const retc = pthread_cond_destroy(&self.cond); + const retc = c.pthread_cond_destroy(&self.cond); assert(retc == 0 or retc == (if (builtin.os == .dragonfly) os.EINVAL else 0)); } pub fn unpark(self: *PosixParker, ptr: *const u32) void { - assert(pthread_mutex_lock(&self.mutex) == 0); - defer assert(pthread_mutex_unlock(&self.mutex) == 0); - assert(pthread_cond_signal(&self.cond) == 0); + assert(c.pthread_mutex_lock(&self.mutex) == 0); + defer assert(c.pthread_mutex_unlock(&self.mutex) == 0); + assert(c.pthread_cond_signal(&self.cond) == 0); } pub fn park(self: *PosixParker, ptr: *const u32, expected: u32) void { - assert(pthread_mutex_lock(&self.mutex) == 0); - defer assert(pthread_mutex_unlock(&self.mutex) == 0); + assert(c.pthread_mutex_lock(&self.mutex) == 0); + defer assert(c.pthread_mutex_unlock(&self.mutex) == 0); while (@atomicLoad(u32, ptr, .Acquire) == expected) - assert(pthread_cond_wait(&self.cond, &self.mutex) == 0); + assert(c.pthread_cond_wait(&self.cond, &self.mutex) == 0); } - - const PTHREAD_MUTEX_INITIALIZER = pthread_mutex_t{}; - extern "c" fn pthread_mutex_lock(mutex: *pthread_mutex_t) c_int; - extern "c" fn pthread_mutex_unlock(mutex: *pthread_mutex_t) c_int; - extern "c" fn pthread_mutex_destroy(mutex: *pthread_mutex_t) c_int; - - const PTHREAD_COND_INITIALIZER = pthread_cond_t{}; - extern "c" fn pthread_cond_wait(noalias cond: *pthread_cond_t, noalias mutex: *pthread_mutex_t) c_int; - extern "c" fn pthread_cond_signal(cond: *pthread_cond_t) c_int; - extern "c" fn pthread_cond_destroy(cond: *pthread_cond_t) c_int; - - // https://github.com/rust-lang/libc - usingnamespace switch (builtin.os) { - .macosx, .tvos, .ios, .watchos => struct { - pub const pthread_mutex_t = extern struct { - __sig: c_long = 0x32AAABA7, - __opaque: [__PTHREAD_MUTEX_SIZE__]u8 = [_]u8{0} ** __PTHREAD_MUTEX_SIZE__, - }; - pub const pthread_cond_t = extern struct { - __sig: c_long = 0x3CB0B1BB, - __opaque: [__PTHREAD_COND_SIZE__]u8 = [_]u8{0} ** __PTHREAD_COND_SIZE__, - }; - const __PTHREAD_MUTEX_SIZE__ = if (@sizeOf(usize) == 8) 56 else 40; - const __PTHREAD_COND_SIZE__ = if (@sizeOf(usize) == 8) 40 else 24; - }, - .netbsd => struct { - pub const pthread_mutex_t = extern struct { - ptm_magic: c_uint = 0x33330003, - ptm_errorcheck: padded_spin_t = 0, - ptm_unused: padded_spin_t = 0, - ptm_owner: usize = 0, - ptm_waiters: ?*u8 = null, - ptm_recursed: c_uint = 0, - ptm_spare2: ?*c_void = null, - }; - pub const pthread_cond_t = extern struct { - ptc_magic: c_uint = 0x55550005, - ptc_lock: pthread_spin_t = 0, - ptc_waiters_first: ?*u8 = null, - ptc_waiters_last: ?*u8 = null, - ptc_mutex: ?*pthread_mutex_t = null, - ptc_private: ?*c_void = null, - }; - const pthread_spin_t = if (builtin.arch == .arm or .arch == .powerpc) c_int else u8; - const padded_spin_t = switch (builtin.arch) { - .sparc, .sparcel, .sparcv9, .i386, .x86_64, .le64 => u32, - else => spin_t, - }; - }, - .openbsd, .freebsd, .kfreebsd, .dragonfly => struct { - pub const pthread_mutex_t = extern struct { - inner: ?*c_void = null, - }; - pub const pthread_cond_t = extern struct { - inner: ?*c_void = null, - }; - }, - .haiku => struct { - pub const pthread_mutex_t = extern struct { - flags: u32 = 0, - lock: i32 = 0, - unused: i32 = -42, - owner: i32 = -1, - owner_count: i32 = 0, - }; - pub const pthread_cond_t = extern struct { - flags: u32 = 0, - unused: i32 = -42, - mutex: ?*c_void = null, - waiter_count: i32 = 0, - lock: i32 = 0, - }; - }, - .hermit => struct { - pub const pthread_mutex_t = extern struct { - inner: usize = ~usize(0), - }; - pub const pthread_cond_t = extern struct { - inner: usize = ~usize(0), - }; - }, - .solaris => struct { - pub const pthread_mutex_t = extern struct { - __pthread_mutex_flag1: u16 = 0, - __pthread_mutex_flag2: u8 = 0, - __pthread_mutex_ceiling: u8 = 0, - __pthread_mutex_type: u16 = 0, - __pthread_mutex_magic: u16 = 0x4d58, - __pthread_mutex_lock: u64 = 0, - __pthread_mutex_data: u64 = 0, - }; - pub const pthread_cond_t = extern struct { - __pthread_cond_flag: u32 = 0, - __pthread_cond_type: u16 = 0, - __pthread_cond_magic: u16 = 0x4356, - __pthread_cond_data: u64 = 0, - }; - }, - .fuchsia, .minix, .linux => struct { - pub const pthread_mutex_t = extern struct { - size: [__SIZEOF_PTHREAD_MUTEX_T]u8 align(@alignOf(usize)) = [_]u8{0} ** __SIZEOF_PTHREAD_MUTEX_T, - }; - pub const pthread_cond_t = extern struct { - size: [__SIZEOF_PTHREAD_COND_T]u8 align(@alignOf(usize)) = [_]u8{0} ** __SIZEOF_PTHREAD_COND_T, - }; - const __SIZEOF_PTHREAD_COND_T = 48; - const __SIZEOF_PTHREAD_MUTEX_T = if (builtin.os == .fuchsia) 40 else switch (builtin.abi) { - .musl, .musleabi, .musleabihf => if (@sizeOf(usize) == 8) 40 else 24, - .gnu, .gnuabin32, .gnuabi64, .gnueabi, .gnueabihf, .gnux32 => switch (builtin.arch) { - .aarch64 => 48, - .x86_64 => if (builtin.abi == .gnux32) 40 else 32, - .mips64, .powerpc64, .powerpc64le, .sparcv9 => 40, - else => if (@sizeOf(usize) == 8) 40 else 24, - }, - else => unreachable, - }; - }, - .emscripten => struct { - pub const pthread_mutex_t = extern struct { - size: [__SIZEOF_PTHREAD_MUTEX_T]u8 align(4) = [_]u8{0} ** __SIZEOF_PTHREAD_MUTEX_T, - }; - pub const pthread_cond_t = extern struct { - size: [__SIZEOF_PTHREAD_COND_T]u8 align(@alignOf(usize)) = [_]u8{0} ** __SIZEOF_PTHREAD_COND_T, - }; - const __SIZEOF_PTHREAD_COND_T = 48; - const __SIZEOF_PTHREAD_MUTEX_T = 28; - }, - else => unreachable, - }; }; test "std.ThreadParker" { if (builtin.single_threaded) return error.SkipZigTest; - + const Context = struct { parker: ThreadParker, data: u32, fn receiver(self: *@This()) void { - self.parker.park(&self.data, 0); // receives 1 - assert(@atomicRmw(u32, &self.data, .Xchg, 2, .SeqCst) == 1); // sends 2 - self.parker.unpark(&self.data); // wakes up waiters on 2 - self.parker.park(&self.data, 2); // receives 3 - assert(@atomicRmw(u32, &self.data, .Xchg, 4, .SeqCst) == 3); // sends 4 - self.parker.unpark(&self.data); // wakes up waiters on 4 + self.parker.park(&self.data, 0); // receives 1 + assert(@atomicRmw(u32, &self.data, .Xchg, 2, .SeqCst) == 1); // sends 2 + self.parker.unpark(&self.data); // wakes up waiters on 2 + self.parker.park(&self.data, 2); // receives 3 + assert(@atomicRmw(u32, &self.data, .Xchg, 4, .SeqCst) == 3); // sends 4 + self.parker.unpark(&self.data); // wakes up waiters on 4 } fn sender(self: *@This()) void { - assert(@atomicRmw(u32, &self.data, .Xchg, 1, .SeqCst) == 0); // sends 1 - self.parker.unpark(&self.data); // wakes up waiters on 1 - self.parker.park(&self.data, 1); // receives 2 - assert(@atomicRmw(u32, &self.data, .Xchg, 3, .SeqCst) == 2); // sends 3 - self.parker.unpark(&self.data); // wakes up waiters on 3 - self.parker.park(&self.data, 3); // receives 4 + assert(@atomicRmw(u32, &self.data, .Xchg, 1, .SeqCst) == 0); // sends 1 + self.parker.unpark(&self.data); // wakes up waiters on 1 + self.parker.park(&self.data, 1); // receives 2 + assert(@atomicRmw(u32, &self.data, .Xchg, 3, .SeqCst) == 2); // sends 3 + self.parker.unpark(&self.data); // wakes up waiters on 3 + self.parker.park(&self.data, 3); // receives 4 } }; @@ -314,9 +172,9 @@ test "std.ThreadParker" { .data = 0, }; defer context.parker.deinit(); - + var receiver = try std.Thread.spawn(&context, Context.receiver); defer receiver.wait(); context.sender(); -} \ No newline at end of file +} diff --git a/lib/std/spinlock.zig b/lib/std/spinlock.zig index b374eba3eb..3bed3d3891 100644 --- a/lib/std/spinlock.zig +++ b/lib/std/spinlock.zig @@ -31,8 +31,8 @@ pub const SpinLock = struct { var i = iterations; while (i != 0) : (i -= 1) { switch (builtin.arch) { - .i386, .x86_64 => asm volatile("pause"), - .arm, .aarch64 => asm volatile("yield"), + .i386, .x86_64 => asm volatile ("pause"), + .arm, .aarch64 => asm volatile ("yield"), else => time.sleep(0), } } @@ -53,7 +53,7 @@ pub const SpinLock = struct { if (self.iteration < 20) { SpinLock.yield(self.iteration); } else if (self.iteration < 24) { - os.sched_yield(); + os.sched_yield() catch time.sleep(1); } else if (self.iteration < 26) { time.sleep(1 * time.millisecond); } else {