From 7703f4c60a501c4f4bc74c7d814619aa818d601b Mon Sep 17 00:00:00 2001 From: xackus <14938807+xackus@users.noreply.github.com> Date: Sun, 1 Nov 2020 23:23:05 +0100 Subject: [PATCH 1/4] stage2: ask for more file descriptors --- src/main.zig | 44 +++++++++++++++++++------------------------- src/stage1/os.cpp | 23 ----------------------- 2 files changed, 19 insertions(+), 48 deletions(-) diff --git a/src/main.zig b/src/main.zig index 45fcb5ce61..77fa2bc205 100644 --- a/src/main.zig +++ b/src/main.zig @@ -2982,35 +2982,29 @@ fn parseCodeModel(arg: []const u8) std.builtin.CodeModel { /// garbage collector to run concurrently to zig processes, and to allow multiple /// zig processes to run concurrently with each other, without clobbering each other. fn gimmeMoreOfThoseSweetSweetFileDescriptors() void { - switch (std.Target.current.os.tag) { - .windows, .wasi, .uefi, .other, .freestanding => return, - // std lib is missing getrlimit/setrlimit. - // https://github.com/ziglang/zig/issues/6361 - //else => {}, - else => return, - } + if (!@hasDecl(std.os, "rlimit")) return; const posix = std.os; - var lim = posix.getrlimit(posix.RLIMIT_NOFILE, &lim) catch return; // Oh well; we tried. + + var lim = posix.getrlimit(.NOFILE) catch return; // Oh well; we tried. if (lim.cur == lim.max) return; + + // Do a binary search for the limit. + var min: posix.rlim_t = lim.cur; + var max: posix.rlim_t = 1 << 20; + // But if there's a defined upper bound, don't search, just set it. + if (lim.max != posix.RLIM_INFINITY) { + min = lim.max; + max = lim.max; + } + while (true) { - // Do a binary search for the limit. - var min: posix.rlim_t = lim.cur; - var max: posix.rlim_t = 1 << 20; - // But if there's a defined upper bound, don't search, just set it. - if (lim.max != posix.RLIM_INFINITY) { - min = lim.max; - max = lim.max; - } - while (true) { - lim.cur = min + (max - min) / 2; - if (posix.setrlimit(posix.RLIMIT_NOFILE, lim)) |_| { - min = lim.cur; - } else |_| { - max = lim.cur; - } - if (min + 1 < max) continue; - return; + lim.cur = min + (max - min) / 2; + if (posix.setrlimit(.NOFILE, lim)) |_| { + min = lim.cur; + } else |_| { + max = lim.cur; } + if (min + 1 >= max) break; } } diff --git a/src/stage1/os.cpp b/src/stage1/os.cpp index c9dd49071c..1cb015c1af 100644 --- a/src/stage1/os.cpp +++ b/src/stage1/os.cpp @@ -977,29 +977,6 @@ int os_init(void) { #elif defined(__MACH__) host_get_clock_service(mach_host_self(), SYSTEM_CLOCK, &macos_monotonic_clock); host_get_clock_service(mach_host_self(), CALENDAR_CLOCK, &macos_calendar_clock); -#endif -#if defined(ZIG_OS_POSIX) - // Raise the open file descriptor limit. - // Code lifted from node.js - struct rlimit lim; - if (getrlimit(RLIMIT_NOFILE, &lim) == 0 && lim.rlim_cur != lim.rlim_max) { - // Do a binary search for the limit. - rlim_t min = lim.rlim_cur; - rlim_t max = 1 << 20; - // But if there's a defined upper bound, don't search, just set it. - if (lim.rlim_max != RLIM_INFINITY) { - min = lim.rlim_max; - max = lim.rlim_max; - } - do { - lim.rlim_cur = min + (max - min) / 2; - if (setrlimit(RLIMIT_NOFILE, &lim)) { - max = lim.rlim_cur; - } else { - min = lim.rlim_cur; - } - } while (min + 1 < max); - } #endif return 0; } From 06b4526a3e15d776437cdcf45d76ed34c6c39272 Mon Sep 17 00:00:00 2001 From: xackus <14938807+xackus@users.noreply.github.com> Date: Mon, 2 Nov 2020 23:18:55 +0100 Subject: [PATCH 2/4] fix freebsd compilation --- src/main.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/main.zig b/src/main.zig index 77fa2bc205..551f168683 100644 --- a/src/main.zig +++ b/src/main.zig @@ -2998,7 +2998,7 @@ fn gimmeMoreOfThoseSweetSweetFileDescriptors() void { } while (true) { - lim.cur = min + (max - min) / 2; + lim.cur = min + @divTrunc(max - min, 2); // on freebsd rlim_t is signed if (posix.setrlimit(.NOFILE, lim)) |_| { min = lim.cur; } else |_| { From 8dda64fa3e1d3249baa948dc0558e3f65d71e6df Mon Sep 17 00:00:00 2001 From: Jakub Konka Date: Tue, 3 Nov 2020 00:36:28 +0100 Subject: [PATCH 3/4] Fix Darwin codepath On Darwin, according to the man pages for setrlimit(), when adjusting max number of open fds, the reported hard max by getrlimit() is only theoretical, while the actual maximum, set in the kernel, is hardcoded in the header file. Therefore, the reported max has to be adjusted as `min(OPEN_MAX, lim.max)`. Signed-off-by: Jakub Konka --- lib/std/os/bits/darwin.zig | 3 +++ src/main.zig | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/lib/std/os/bits/darwin.zig b/lib/std/os/bits/darwin.zig index a30d11fcd1..3e4149decd 100644 --- a/lib/std/os/bits/darwin.zig +++ b/lib/std/os/bits/darwin.zig @@ -1475,6 +1475,9 @@ pub const CLOCK_UPTIME_RAW_APPROX = 9; pub const CLOCK_PROCESS_CPUTIME_ID = 12; pub const CLOCK_THREAD_CPUTIME_ID = 16; +/// Max open files per process +/// https://opensource.apple.com/source/xnu/xnu-4903.221.2/bsd/sys/syslimits.h.auto.html +pub const OPEN_MAX = 10240; pub const RUSAGE_SELF = 0; pub const RUSAGE_CHILDREN = -1; diff --git a/src/main.zig b/src/main.zig index 551f168683..90a0aba460 100644 --- a/src/main.zig +++ b/src/main.zig @@ -2986,6 +2986,14 @@ fn gimmeMoreOfThoseSweetSweetFileDescriptors() void { const posix = std.os; var lim = posix.getrlimit(.NOFILE) catch return; // Oh well; we tried. + if (std.Target.current.isDarwin()) { + // On Darwin, `NOFILE` is bounded by a hardcoded value `OPEN_MAX`. + // According to the man pages for setrlimit(): + // setrlimit() now returns with errno set to EINVAL in places that historically succeeded. + // It no longer accepts "rlim_cur = RLIM_INFINITY" for RLIM_NOFILE. + // Use "rlim_cur = min(OPEN_MAX, rlim_max)". + lim.max = std.math.min(posix.darwin.OPEN_MAX, lim.max); + } if (lim.cur == lim.max) return; // Do a binary search for the limit. From e023a5fe5d8e0c2849c160ffab993ec1e378681c Mon Sep 17 00:00:00 2001 From: xackus <14938807+xackus@users.noreply.github.com> Date: Tue, 3 Nov 2020 00:58:35 +0100 Subject: [PATCH 4/4] force comptime on isDarwin --- src/main.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main.zig b/src/main.zig index 90a0aba460..7df8cb1eda 100644 --- a/src/main.zig +++ b/src/main.zig @@ -2986,13 +2986,13 @@ fn gimmeMoreOfThoseSweetSweetFileDescriptors() void { const posix = std.os; var lim = posix.getrlimit(.NOFILE) catch return; // Oh well; we tried. - if (std.Target.current.isDarwin()) { + if (comptime std.Target.current.isDarwin()) { // On Darwin, `NOFILE` is bounded by a hardcoded value `OPEN_MAX`. // According to the man pages for setrlimit(): // setrlimit() now returns with errno set to EINVAL in places that historically succeeded. // It no longer accepts "rlim_cur = RLIM_INFINITY" for RLIM_NOFILE. // Use "rlim_cur = min(OPEN_MAX, rlim_max)". - lim.max = std.math.min(posix.darwin.OPEN_MAX, lim.max); + lim.max = std.math.min(std.os.darwin.OPEN_MAX, lim.max); } if (lim.cur == lim.max) return;