From 997812e8fb9146360aa5200f0adbcd4863183287 Mon Sep 17 00:00:00 2001 From: Luna Date: Wed, 13 Nov 2019 21:25:19 -0300 Subject: [PATCH 1/6] os: add memfd_create currently only linux is supported --- lib/std/io/test.zig | 13 +++++++++++++ lib/std/os.zig | 32 ++++++++++++++++++++++++++++++++ lib/std/os/bits/linux.zig | 36 ++++++++++++++++++++++++++++++++++++ lib/std/os/linux.zig | 4 ++++ 4 files changed, 85 insertions(+) diff --git a/lib/std/io/test.zig b/lib/std/io/test.zig index 92259fd6e9..d6de912240 100644 --- a/lib/std/io/test.zig +++ b/lib/std/io/test.zig @@ -649,3 +649,16 @@ test "updateTimes" { std.testing.expect(stat_new.atime < stat_old.atime); std.testing.expect(stat_new.mtime < stat_old.mtime); } + +test "memfd_create" { + if (builtin.os != .linux) return error.SkipZigTest; + + const fd = try std.os.memfd_create("test", 0); + try std.os.write(fd, "test"); + try std.os.lseek_SET(fd, 0); + + var buf: [10]u8 = undefined; + const bytes_read = try std.os.read(fd, &buf); + expect(bytes_read == 4); + expect(mem.eql(u8, buf[0..4], "test")); +} diff --git a/lib/std/os.zig b/lib/std/os.zig index f7426faeda..37043f19bb 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -3281,3 +3281,35 @@ pub fn setsockopt(fd: fd_t, level: u32, optname: u32, opt: []const u8) SetSockOp else => |err| return unexpectedErrno(err), } } + +// TODO support for non-null terminated strings? +pub fn memfd_createC(name: [*:0]const u8, flags: usize) !fd_t { + if (builtin.os != .linux) @compileError("memfd_create() not implemented for this target"); + + const rc = linux.memfd_create(name, flags); + switch (linux.getErrno(rc)) { + 0 => return @intCast(fd_t, rc), + EFAULT => unreachable, // name has invalid memory + EINVAL => unreachable, // name/flags are faulty + ENFILE => return error.SystemFdQuotaExceeded, + EMFILE => return error.ProcessFdQuotaExceeded, + ENOMEM => return error.OutOfMemory, + else => |err| return unexpectedErrno(err), + } +} + +pub const MFD_NAME_PREFIX = "memfd:"; +pub const MFD_MAX_NAME_LEN = NAME_MAX - MFD_NAME_PREFIX.len; +fn toMemFdPath(name: []const u8) ![MFD_MAX_NAME_LEN:0]u8 { + var path_with_null: [MFD_MAX_NAME_LEN:0]u8 = undefined; + // >= rather than > to make room for the null byte + if (name.len >= MFD_MAX_NAME_LEN) return error.NameTooLong; + mem.copy(u8, &path_with_null, name); + path_with_null[name.len] = 0; + return path_with_null; +} + +pub fn memfd_create(name: []const u8, flags: usize) !fd_t { + const name_t = try toMemFdPath(name); + return try memfd_createC(&name_t, flags); +} diff --git a/lib/std/os/bits/linux.zig b/lib/std/os/bits/linux.zig index 8033f27c2b..3fb160e3ec 100644 --- a/lib/std/os/bits/linux.zig +++ b/lib/std/os/bits/linux.zig @@ -26,6 +26,7 @@ pub const uid_t = i32; pub const gid_t = u32; pub const clock_t = isize; +pub const NAME_MAX = 255; pub const PATH_MAX = 4096; pub const IOV_MAX = 1024; @@ -1417,3 +1418,38 @@ pub const POLLHUP = 0x010; pub const POLLNVAL = 0x020; pub const POLLRDNORM = 0x040; pub const POLLRDBAND = 0x080; + +pub const MFD_CLOEXEC = 0x0001; +pub const MFD_ALLOW_SEALING = 0x0002; +pub const MFD_HUGETLB = 0x0004; +pub const MFD_ALL_FLAGS = MFD_CLOEXEC | MFD_ALLOW_SEALING | MFD_HUGETLB; + +pub const HUGETLB_FLAG_ENCODE_SHIFT = 26; +pub const HUGETLB_FLAG_ENCODE_MASK = 0x3f; +pub const HUGETLB_FLAG_ENCODE_64KB = 16 << HUGETLB_FLAG_ENCODE_SHIFT; +pub const HUGETLB_FLAG_ENCODE_512KB = 19 << HUGETLB_FLAG_ENCODE_SHIFT; +pub const HUGETLB_FLAG_ENCODE_1MB = 20 << HUGETLB_FLAG_ENCODE_SHIFT; +pub const HUGETLB_FLAG_ENCODE_2MB = 21 << HUGETLB_FLAG_ENCODE_SHIFT; +pub const HUGETLB_FLAG_ENCODE_8MB = 23 << HUGETLB_FLAG_ENCODE_SHIFT; +pub const HUGETLB_FLAG_ENCODE_16MB = 24 << HUGETLB_FLAG_ENCODE_SHIFT; +pub const HUGETLB_FLAG_ENCODE_32MB = 25 << HUGETLB_FLAG_ENCODE_SHIFT; +pub const HUGETLB_FLAG_ENCODE_256MB = 28 << HUGETLB_FLAG_ENCODE_SHIFT; +pub const HUGETLB_FLAG_ENCODE_512MB = 29 << HUGETLB_FLAG_ENCODE_SHIFT; +pub const HUGETLB_FLAG_ENCODE_1GB = 30 << HUGETLB_FLAG_ENCODE_SHIFT; +pub const HUGETLB_FLAG_ENCODE_2GB = 31 << HUGETLB_FLAG_ENCODE_SHIFT; +pub const HUGETLB_FLAG_ENCODE_16GB = 34 << HUGETLB_FLAG_ENCODE_SHIFT; + +pub const MFD_HUGE_SHIFT = HUGETLB_FLAG_ENCODE_SHIFT; +pub const MFD_HUGE_MASK = HUGETLB_FLAG_ENCODE_MASK; +pub const MFD_HUGE_64KB = HUGETLB_FLAG_ENCODE_64KB; +pub const MFD_HUGE_512KB = HUGETLB_FLAG_ENCODE_512KB; +pub const MFD_HUGE_1MB = HUGETLB_FLAG_ENCODE_1MB; +pub const MFD_HUGE_2MB = HUGETLB_FLAG_ENCODE_2MB; +pub const MFD_HUGE_8MB = HUGETLB_FLAG_ENCODE_8MB; +pub const MFD_HUGE_16MB = HUGETLB_FLAG_ENCODE_16MB; +pub const MFD_HUGE_32MB = HUGETLB_FLAG_ENCODE_32MB; +pub const MFD_HUGE_256MB = HUGETLB_FLAG_ENCODE_256MB; +pub const MFD_HUGE_512MB = HUGETLB_FLAG_ENCODE_512MB; +pub const MFD_HUGE_1GB = HUGETLB_FLAG_ENCODE_1GB; +pub const MFD_HUGE_2GB = HUGETLB_FLAG_ENCODE_2GB; +pub const MFD_HUGE_16GB = HUGETLB_FLAG_ENCODE_16GB; diff --git a/lib/std/os/linux.zig b/lib/std/os/linux.zig index d5bab64493..69e5b25f86 100644 --- a/lib/std/os/linux.zig +++ b/lib/std/os/linux.zig @@ -1110,6 +1110,10 @@ pub fn io_uring_register(fd: i32, opcode: u32, arg: ?*const c_void, nr_args: u32 return syscall4(SYS_io_uring_register, @bitCast(usize, @as(isize, fd)), opcode, @ptrToInt(arg), nr_args); } +pub fn memfd_create(name: [*:0]const u8, flags: usize) usize { + return syscall2(SYS_memfd_create, @ptrToInt(name), flags); +} + test "" { if (builtin.os == .linux) { _ = @import("linux/test.zig"); From f0cbf63e1ae38d9c13cb07a151567b4890287ae9 Mon Sep 17 00:00:00 2001 From: Luna Date: Tue, 31 Dec 2019 15:33:23 -0300 Subject: [PATCH 2/6] os: use system for memfd_create - os: update flags type for memfd_create --- lib/std/os.zig | 11 ++++------- lib/std/os/linux.zig | 2 +- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/lib/std/os.zig b/lib/std/os.zig index 37043f19bb..31212e9b8b 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -3282,12 +3282,9 @@ pub fn setsockopt(fd: fd_t, level: u32, optname: u32, opt: []const u8) SetSockOp } } -// TODO support for non-null terminated strings? -pub fn memfd_createC(name: [*:0]const u8, flags: usize) !fd_t { - if (builtin.os != .linux) @compileError("memfd_create() not implemented for this target"); - - const rc = linux.memfd_create(name, flags); - switch (linux.getErrno(rc)) { +pub fn memfd_createC(name: [*:0]const u8, flags: u32) !fd_t { + const rc = system.memfd_create(name, flags); + switch (errno(rc)) { 0 => return @intCast(fd_t, rc), EFAULT => unreachable, // name has invalid memory EINVAL => unreachable, // name/flags are faulty @@ -3309,7 +3306,7 @@ fn toMemFdPath(name: []const u8) ![MFD_MAX_NAME_LEN:0]u8 { return path_with_null; } -pub fn memfd_create(name: []const u8, flags: usize) !fd_t { +pub fn memfd_create(name: []const u8, flags: u32) !fd_t { const name_t = try toMemFdPath(name); return try memfd_createC(&name_t, flags); } diff --git a/lib/std/os/linux.zig b/lib/std/os/linux.zig index 69e5b25f86..b6030fe2fe 100644 --- a/lib/std/os/linux.zig +++ b/lib/std/os/linux.zig @@ -1110,7 +1110,7 @@ pub fn io_uring_register(fd: i32, opcode: u32, arg: ?*const c_void, nr_args: u32 return syscall4(SYS_io_uring_register, @bitCast(usize, @as(isize, fd)), opcode, @ptrToInt(arg), nr_args); } -pub fn memfd_create(name: [*:0]const u8, flags: usize) usize { +pub fn memfd_create(name: [*:0]const u8, flags: u32) usize { return syscall2(SYS_memfd_create, @ptrToInt(name), flags); } From 13c9faaa2dbd88b45b3f19734e6ca72f87ddaaaa Mon Sep 17 00:00:00 2001 From: Luna Date: Tue, 31 Dec 2019 15:35:07 -0300 Subject: [PATCH 3/6] c.linux: add memfd_create --- lib/std/c/linux.zig | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/std/c/linux.zig b/lib/std/c/linux.zig index 5b45b49332..9969474097 100644 --- a/lib/std/c/linux.zig +++ b/lib/std/c/linux.zig @@ -76,6 +76,8 @@ pub extern "c" fn dl_iterate_phdr(callback: dl_iterate_phdr_callback, data: ?*c_ pub extern "c" fn sigaltstack(ss: ?*stack_t, old_ss: ?*stack_t) c_int; +pub extern "c" fn memfd_create(name: [*:0]const u8, flags: c_uint) c_int; + pub const pthread_attr_t = extern struct { __size: [56]u8, __align: c_long, From a153a972adbe783bc8a0d08df0942a3bf0552188 Mon Sep 17 00:00:00 2001 From: Luna Date: Tue, 31 Dec 2019 15:35:17 -0300 Subject: [PATCH 4/6] io.test: close memfd at end of test --- lib/std/io/test.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/std/io/test.zig b/lib/std/io/test.zig index d6de912240..b572a7bf92 100644 --- a/lib/std/io/test.zig +++ b/lib/std/io/test.zig @@ -654,6 +654,7 @@ test "memfd_create" { if (builtin.os != .linux) return error.SkipZigTest; const fd = try std.os.memfd_create("test", 0); + defer std.os.close(fd); try std.os.write(fd, "test"); try std.os.lseek_SET(fd, 0); From 8186211404552a31eaf2a7bc182a46a7e40eacda Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 31 Dec 2019 16:34:14 -0500 Subject: [PATCH 5/6] improvements to memfd_create * move test from std/io/test.zig to std/os/test.zig * do glibc version check, and make direct system call if glibc is too old * disable test when not linking libc, to avoid not working with outdated qemu version on the CI server. see #4019 --- lib/std/io/test.zig | 14 -------------- lib/std/os.zig | 16 +++++++++++++--- lib/std/os/test.zig | 21 +++++++++++++++++++++ 3 files changed, 34 insertions(+), 17 deletions(-) diff --git a/lib/std/io/test.zig b/lib/std/io/test.zig index b572a7bf92..92259fd6e9 100644 --- a/lib/std/io/test.zig +++ b/lib/std/io/test.zig @@ -649,17 +649,3 @@ test "updateTimes" { std.testing.expect(stat_new.atime < stat_old.atime); std.testing.expect(stat_new.mtime < stat_old.mtime); } - -test "memfd_create" { - if (builtin.os != .linux) return error.SkipZigTest; - - const fd = try std.os.memfd_create("test", 0); - defer std.os.close(fd); - try std.os.write(fd, "test"); - try std.os.lseek_SET(fd, 0); - - var buf: [10]u8 = undefined; - const bytes_read = try std.os.read(fd, &buf); - expect(bytes_read == 4); - expect(mem.eql(u8, buf[0..4], "test")); -} diff --git a/lib/std/os.zig b/lib/std/os.zig index 31212e9b8b..b51dbe7d2a 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -3282,9 +3282,19 @@ pub fn setsockopt(fd: fd_t, level: u32, optname: u32, opt: []const u8) SetSockOp } } +pub const MemFdCreateError = error{ + SystemFdQuotaExceeded, + ProcessFdQuotaExceeded, + OutOfMemory, +} || UnexpectedError; + pub fn memfd_createC(name: [*:0]const u8, flags: u32) !fd_t { - const rc = system.memfd_create(name, flags); - switch (errno(rc)) { + // memfd_create is available only in glibc versions starting with 2.27. + const use_c = std.c.versionCheck(.{ .major = 2, .minor = 27, .patch = 0 }).ok; + const sys = if (use_c) std.c else linux; + const getErrno = if (use_c) std.c.getErrno else linux.getErrno; + const rc = sys.memfd_create(name, flags); + switch (getErrno(rc)) { 0 => return @intCast(fd_t, rc), EFAULT => unreachable, // name has invalid memory EINVAL => unreachable, // name/flags are faulty @@ -3308,5 +3318,5 @@ fn toMemFdPath(name: []const u8) ![MFD_MAX_NAME_LEN:0]u8 { pub fn memfd_create(name: []const u8, flags: u32) !fd_t { const name_t = try toMemFdPath(name); - return try memfd_createC(&name_t, flags); + return memfd_createC(&name_t, flags); } diff --git a/lib/std/os/test.zig b/lib/std/os/test.zig index f61e8b4657..16946660e4 100644 --- a/lib/std/os/test.zig +++ b/lib/std/os/test.zig @@ -237,3 +237,24 @@ test "argsAlloc" { var args = try std.process.argsAlloc(std.heap.page_allocator); std.process.argsFree(std.heap.page_allocator, args); } + +test "memfd_create" { + // memfd_create is linux specific. + if (builtin.os != .linux) return error.SkipZigTest; + // Zig's CI testing infrastructure uses QEMU. Currently the version is + // qemu 2.11 from Ubuntu 18.04, which does not have memfd_create support. + // memfd_create support is introduced in qemu 4.2. To avoid + // "invalid syscall" errors from qemu, we disable the test when not linking libc. + // https://github.com/ziglang/zig/issues/4019 + if (!builtin.link_libc) return error.SkipZigTest; + + const fd = try std.os.memfd_create("test", 0); + defer std.os.close(fd); + try std.os.write(fd, "test"); + try std.os.lseek_SET(fd, 0); + + var buf: [10]u8 = undefined; + const bytes_read = try std.os.read(fd, &buf); + expect(bytes_read == 4); + expect(mem.eql(u8, buf[0..4], "test")); +} From 0c8ec369f051d616434cff309fe737c52fd98199 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 31 Dec 2019 17:57:20 -0500 Subject: [PATCH 6/6] std.os.memfd_create: add error.SystemOutdated --- lib/std/os.zig | 7 ++++++- lib/std/os/test.zig | 13 +++++-------- lib/std/target.zig | 7 +++++++ 3 files changed, 18 insertions(+), 9 deletions(-) diff --git a/lib/std/os.zig b/lib/std/os.zig index b51dbe7d2a..878d1a57e7 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -3286,9 +3286,13 @@ pub const MemFdCreateError = error{ SystemFdQuotaExceeded, ProcessFdQuotaExceeded, OutOfMemory, + + /// memfd_create is available in Linux 3.17 and later. This error is returned + /// for older kernel versions. + SystemOutdated, } || UnexpectedError; -pub fn memfd_createC(name: [*:0]const u8, flags: u32) !fd_t { +pub fn memfd_createC(name: [*:0]const u8, flags: u32) MemFdCreateError!fd_t { // memfd_create is available only in glibc versions starting with 2.27. const use_c = std.c.versionCheck(.{ .major = 2, .minor = 27, .patch = 0 }).ok; const sys = if (use_c) std.c else linux; @@ -3301,6 +3305,7 @@ pub fn memfd_createC(name: [*:0]const u8, flags: u32) !fd_t { ENFILE => return error.SystemFdQuotaExceeded, EMFILE => return error.ProcessFdQuotaExceeded, ENOMEM => return error.OutOfMemory, + ENOSYS => return error.SystemOutdated, else => |err| return unexpectedErrno(err), } } diff --git a/lib/std/os/test.zig b/lib/std/os/test.zig index 16946660e4..09e8c24899 100644 --- a/lib/std/os/test.zig +++ b/lib/std/os/test.zig @@ -241,14 +241,11 @@ test "argsAlloc" { test "memfd_create" { // memfd_create is linux specific. if (builtin.os != .linux) return error.SkipZigTest; - // Zig's CI testing infrastructure uses QEMU. Currently the version is - // qemu 2.11 from Ubuntu 18.04, which does not have memfd_create support. - // memfd_create support is introduced in qemu 4.2. To avoid - // "invalid syscall" errors from qemu, we disable the test when not linking libc. - // https://github.com/ziglang/zig/issues/4019 - if (!builtin.link_libc) return error.SkipZigTest; - - const fd = try std.os.memfd_create("test", 0); + const fd = std.os.memfd_create("test", 0) catch |err| switch (err) { + // Related: https://github.com/ziglang/zig/issues/4019 + error.SystemOutdated => return error.SkipZigTest, + else => |e| return e, + }; defer std.os.close(fd); try std.os.write(fd, "test"); try std.os.lseek_SET(fd, 0); diff --git a/lib/std/target.zig b/lib/std/target.zig index 50c749d30b..22fea691c4 100644 --- a/lib/std/target.zig +++ b/lib/std/target.zig @@ -597,6 +597,13 @@ pub const Target = union(enum) { }; } + pub fn isMusl(self: Target) bool { + return switch (self.getAbi()) { + .musl, .musleabi, .musleabihf => true, + else => false, + }; + } + pub fn isDarwin(self: Target) bool { return switch (self.getOs()) { .ios, .macosx, .watchos, .tvos => true,