From 28a9e4c4aaff3a186c344d98d2a2bf918d20a13c Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Fri, 7 May 2021 09:39:55 +0200 Subject: [PATCH 01/10] libc: Add workaround for #4926 This change was cherry-picked from an updated version of the sysdep folder contents, we're still shipping an outdated and incomplete set of files. --- lib/libc/glibc/sysdeps/i386/sysdep.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/libc/glibc/sysdeps/i386/sysdep.h b/lib/libc/glibc/sysdeps/i386/sysdep.h index b4bcd8fb6c..b338b0dc36 100644 --- a/lib/libc/glibc/sysdeps/i386/sysdep.h +++ b/lib/libc/glibc/sysdeps/i386/sysdep.h @@ -61,7 +61,7 @@ lose: SYSCALL_PIC_SETUP \ # define SETUP_PIC_REG(reg) \ .ifndef GET_PC_THUNK(reg); \ - .section .gnu.linkonce.t.GET_PC_THUNK(reg),"ax",@progbits; \ + .section .text.GET_PC_THUNK(reg),"axG",@progbits,GET_PC_THUNK(reg),comdat; \ .globl GET_PC_THUNK(reg); \ .hidden GET_PC_THUNK(reg); \ .p2align 4; \ @@ -97,8 +97,9 @@ GET_PC_THUNK(reg): \ # define SETUP_PIC_REG_STR(reg) \ ".ifndef " GET_PC_THUNK_STR (reg) "\n" \ - ".section .gnu.linkonce.t." GET_PC_THUNK_STR (reg) ",\"ax\",@progbits\n" \ + "section .text." GET_PC_THUNK_STR (reg) ",\"axG\",@progbits," \ ".globl " GET_PC_THUNK_STR (reg) "\n" \ + GET_PC_THUNK_STR (reg) ",comdat\n" \ ".hidden " GET_PC_THUNK_STR (reg) "\n" \ ".p2align 4\n" \ ".type " GET_PC_THUNK_STR (reg) ",@function\n" \ From a909db6aea38f65ffe996d61a62a4f1bd2cbc44c Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Fri, 7 May 2021 09:41:26 +0200 Subject: [PATCH 02/10] std: Prefer 64bit libc functions where possible While musl decided to hard-wire off_t to a 64bit quantity, glibc is much older and defaults to 32bit offsets and offers some -64 suffixed versions of I/O functions. There's a weird mix-up of types: sometimes off_t is used, sometimes not, sometimes it's defined as a signed quantity and sometimes as an unsigned one, but we'll sort this problem later. --- lib/std/c/linux.zig | 19 +++++++- lib/std/os.zig | 114 ++++++++++++++++++++++++++++++++++---------- 2 files changed, 107 insertions(+), 26 deletions(-) diff --git a/lib/std/c/linux.zig b/lib/std/c/linux.zig index d2018f6f79..f7a91f4e1b 100644 --- a/lib/std/c/linux.zig +++ b/lib/std/c/linux.zig @@ -61,6 +61,23 @@ pub const EAI = extern enum(c_int) { _, }; +pub extern "c" fn fallocate64(fd: fd_t, mode: c_int, offset: i64, len: i64) c_int; +pub extern "c" fn fopen64(noalias filename: [*:0]const u8, noalias modes: [*:0]const u8) ?*FILE; +pub extern "c" fn fstat64(fd: fd_t, buf: *libc_stat) c_int; +pub extern "c" fn fstatat64(dirfd: fd_t, path: [*:0]const u8, stat_buf: *libc_stat, flags: u32) c_int; +pub extern "c" fn ftruncate64(fd: c_int, length: i64) c_int; +pub extern "c" fn getrlimit64(resource: rlimit_resource, rlim: *rlimit) c_int; +pub extern "c" fn lseek64(fd: fd_t, offset: u64, whence: c_int) u64; +pub extern "c" fn mmap64(addr: ?*align(std.mem.page_size) c_void, len: usize, prot: c_uint, flags: c_uint, fd: fd_t, offset: u64) *c_void; +pub extern "c" fn open64(path: [*:0]const u8, oflag: c_uint, ...) c_int; +pub extern "c" fn openat64(fd: c_int, path: [*:0]const u8, oflag: c_uint, ...) c_int; +pub extern "c" fn pread64(fd: fd_t, buf: [*]u8, nbyte: usize, offset: u64) isize; +pub extern "c" fn preadv64(fd: c_int, iov: [*]const iovec, iovcnt: c_uint, offset: u64) isize; +pub extern "c" fn pwrite64(fd: fd_t, buf: [*]const u8, nbyte: usize, offset: u64) isize; +pub extern "c" fn pwritev64(fd: c_int, iov: [*]const iovec_const, iovcnt: c_uint, offset: u64) isize; +pub extern "c" fn sendfile64(out_fd: fd_t, in_fd: fd_t, offset: ?*i64, count: usize) isize; +pub extern "c" fn setrlimit64(resource: rlimit_resource, rlim: *const rlimit) c_int; + pub extern "c" fn getrandom(buf_ptr: [*]u8, buf_len: usize, flags: c_uint) isize; pub extern "c" fn sched_getaffinity(pid: c_int, size: usize, set: *cpu_set_t) c_int; pub extern "c" fn eventfd(initval: c_uint, flags: c_uint) c_int; @@ -90,8 +107,6 @@ pub extern "c" fn pipe2(fds: *[2]fd_t, flags: u32) c_int; pub extern "c" fn fallocate(fd: fd_t, mode: c_int, offset: off_t, len: off_t) c_int; -pub extern "c" fn ftruncate64(fd: c_int, length: off_t) c_int; - pub extern "c" fn sendfile( out_fd: fd_t, in_fd: fd_t, diff --git a/lib/std/os.zig b/lib/std/os.zig index 8bfebb362c..db030f0e2b 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -497,8 +497,13 @@ pub fn pread(fd: fd_t, buf: []u8, offset: u64) PReadError!usize { }; const adjusted_len = math.min(max_count, buf.len); + const pread_sym = if (builtin.os.tag == .linux and builtin.link_libc) + system.pread64 + else + system.pread; + while (true) { - const rc = system.pread(fd, buf.ptr, adjusted_len, offset); + const rc = pread_sym(fd, buf.ptr, adjusted_len, offset); switch (errno(rc)) { 0 => return @intCast(usize, rc), EINTR => continue, @@ -567,15 +572,12 @@ pub fn ftruncate(fd: fd_t, length: u64) TruncateError!void { } while (true) { - const rc = if (builtin.link_libc) - if (std.Target.current.os.tag == .linux) - system.ftruncate64(fd, @bitCast(off_t, length)) - else - system.ftruncate(fd, @bitCast(off_t, length)) + const ftruncate_sym = if (builtin.os.tag == .linux and builtin.link_libc) + system.ftruncate64 else - system.ftruncate(fd, length); + system.ftruncate; - switch (errno(rc)) { + switch (errno(ftruncate_sym(fd, @bitCast(off_t, length)))) { 0 => return, EINTR => continue, EFBIG => return error.FileTooBig, @@ -637,8 +639,13 @@ pub fn preadv(fd: fd_t, iov: []const iovec, offset: u64) PReadError!usize { const iov_count = math.cast(u31, iov.len) catch math.maxInt(u31); + const preadv_sym = if (builtin.os.tag == .linux and builtin.link_libc) + system.preadv64 + else + system.preadv; + while (true) { - const rc = system.preadv(fd, iov.ptr, iov_count, offset); + const rc = preadv_sym(fd, iov.ptr, iov_count, offset); switch (errno(rc)) { 0 => return @bitCast(usize, rc), EINTR => continue, @@ -895,8 +902,13 @@ pub fn pwrite(fd: fd_t, bytes: []const u8, offset: u64) PWriteError!usize { }; const adjusted_len = math.min(max_count, bytes.len); + const pwrite_sym = if (builtin.os.tag == .linux and builtin.link_libc) + system.pwrite64 + else + system.pwrite; + while (true) { - const rc = system.pwrite(fd, bytes.ptr, adjusted_len, offset); + const rc = pwrite_sym(fd, bytes.ptr, adjusted_len, offset); switch (errno(rc)) { 0 => return @intCast(usize, rc), EINTR => continue, @@ -977,9 +989,14 @@ pub fn pwritev(fd: fd_t, iov: []const iovec_const, offset: u64) PWriteError!usiz } } + const pwritev_sym = if (builtin.os.tag == .linux and builtin.link_libc) + system.pwritev64 + else + system.pwritev; + const iov_count = math.cast(u31, iov.len) catch math.maxInt(u31); while (true) { - const rc = system.pwritev(fd, iov.ptr, iov_count, offset); + const rc = pwritev_sym(fd, iov.ptr, iov_count, offset); switch (errno(rc)) { 0 => return @intCast(usize, rc), EINTR => continue, @@ -1068,8 +1085,14 @@ pub fn openZ(file_path: [*:0]const u8, flags: u32, perm: mode_t) OpenError!fd_t const file_path_w = try windows.cStrToPrefixedFileW(file_path); return openW(file_path_w.span(), flags, perm); } + + const open_sym = if (builtin.os.tag == .linux and builtin.link_libc) + system.open64 + else + system.open; + while (true) { - const rc = system.open(file_path, flags, perm); + const rc = open_sym(file_path, flags, perm); switch (errno(rc)) { 0 => return @intCast(fd_t, rc), EINTR => continue, @@ -1202,8 +1225,14 @@ pub fn openatZ(dir_fd: fd_t, file_path: [*:0]const u8, flags: u32, mode: mode_t) const file_path_w = try windows.cStrToPrefixedFileW(file_path); return openatW(dir_fd, file_path_w.span(), flags, mode); } + + const openat_sym = if (builtin.os.tag == .linux and builtin.link_libc) + system.openat64 + else + system.openat; + while (true) { - const rc = system.openat(dir_fd, file_path, flags, mode); + const rc = openat_sym(dir_fd, file_path, flags, mode); switch (errno(rc)) { 0 => return @intCast(fd_t, rc), EINTR => continue, @@ -3437,8 +3466,13 @@ pub fn fstat(fd: fd_t) FStatError!Stat { @compileError("fstat is not yet implemented on Windows"); } + const fstat_sym = if (builtin.os.tag == .linux and builtin.link_libc) + system.fstat64 + else + system.fstat; + var stat = mem.zeroes(Stat); - switch (errno(system.fstat(fd, &stat))) { + switch (errno(fstat_sym(fd, &stat))) { 0 => return stat, EINVAL => unreachable, EBADF => unreachable, // Always a race condition. @@ -3488,8 +3522,13 @@ pub fn fstatatWasi(dirfd: fd_t, pathname: []const u8, flags: u32) FStatAtError!S /// Same as `fstatat` but `pathname` is null-terminated. /// See also `fstatat`. pub fn fstatatZ(dirfd: fd_t, pathname: [*:0]const u8, flags: u32) FStatAtError!Stat { + const fstatat_sym = if (builtin.os.tag == .linux and builtin.link_libc) + system.fstatat64 + else + system.fstatat; + var stat = mem.zeroes(Stat); - switch (errno(system.fstatat(dirfd, pathname, &stat, flags))) { + switch (errno(fstatat_sym(dirfd, pathname, &stat, flags))) { 0 => return stat, EINVAL => unreachable, EBADF => unreachable, // Always a race condition. @@ -3701,12 +3740,16 @@ pub fn mmap( fd: fd_t, offset: u64, ) MMapError![]align(mem.page_size) u8 { + const mmap_sym = if (builtin.os.tag == .linux and builtin.link_libc) + system.mmap64 + else + system.mmap; + + const rc = mmap_sym(ptr, length, prot, flags, fd, offset); const err = if (builtin.link_libc) blk: { - const rc = std.c.mmap(ptr, length, prot, flags, fd, offset); if (rc != std.c.MAP_FAILED) return @ptrCast([*]align(mem.page_size) u8, @alignCast(mem.page_size, rc))[0..length]; break :blk system._errno().*; } else blk: { - const rc = system.mmap(ptr, length, prot, flags, fd, offset); const err = errno(rc); if (err == 0) return @intToPtr([*]align(mem.page_size) u8, rc)[0..length]; break :blk err; @@ -4139,7 +4182,12 @@ pub fn lseek_END(fd: fd_t, offset: i64) SeekError!void { else => |err| return unexpectedErrno(err), } } - switch (errno(system.lseek(fd, offset, SEEK_END))) { + const lseek_sym = if (builtin.os.tag == .linux and builtin.link_libc) + system.lseek64 + else + system.lseek; + + switch (errno(lseek_sym(fd, offset, SEEK_END))) { 0 => return, EBADF => unreachable, // always a race condition EINVAL => return error.Unseekable, @@ -4180,7 +4228,12 @@ pub fn lseek_CUR_get(fd: fd_t) SeekError!u64 { else => |err| return unexpectedErrno(err), } } - const rc = system.lseek(fd, 0, SEEK_CUR); + const lseek_sym = if (builtin.os.tag == .linux and builtin.link_libc) + system.lseek64 + else + system.lseek; + + const rc = lseek_sym(fd, 0, SEEK_CUR); switch (errno(rc)) { 0 => return @bitCast(u64, rc), EBADF => unreachable, // always a race condition @@ -5198,9 +5251,14 @@ pub fn sendfile( // Here we match BSD behavior, making a zero count value send as many bytes as possible. const adjusted_count = if (in_len == 0) max_count else math.min(in_len, @as(size_t, max_count)); + const sendfile_sym = if (builtin.link_libc) + system.sendfile64 + else + system.sendfile; + while (true) { var offset: off_t = @bitCast(off_t, in_offset); - const rc = system.sendfile(out_fd, in_fd, &offset, adjusted_count); + const rc = sendfile_sym(out_fd, in_fd, &offset, adjusted_count); switch (errno(rc)) { 0 => { const amt = @bitCast(usize, rc); @@ -6018,9 +6076,13 @@ pub fn prctl(option: PR, args: anytype) PrctlError!u31 { pub const GetrlimitError = UnexpectedError; pub fn getrlimit(resource: rlimit_resource) GetrlimitError!rlimit { + const getrlimit_sym = if (builtin.os.tag == .linux and builtin.link_libc) + system.getrlimit64 + else + system.getrlimit; + var limits: rlimit = undefined; - const rc = system.getrlimit(resource, &limits); - switch (errno(rc)) { + switch (errno(getrlimit_sym(resource, &limits))) { 0 => return limits, EFAULT => unreachable, // bogus pointer EINVAL => unreachable, @@ -6031,8 +6093,12 @@ pub fn getrlimit(resource: rlimit_resource) GetrlimitError!rlimit { pub const SetrlimitError = error{PermissionDenied} || UnexpectedError; pub fn setrlimit(resource: rlimit_resource, limits: rlimit) SetrlimitError!void { - const rc = system.setrlimit(resource, &limits); - switch (errno(rc)) { + const setrlimit_sym = if (builtin.os.tag == .linux and builtin.link_libc) + system.setrlimit64 + else + system.setrlimit; + + switch (errno(setrlimit_sym(resource, &limits))) { 0 => return, EFAULT => unreachable, // bogus pointer EINVAL => unreachable, From 679876ba7266e6a9eac73705280a39158808bd52 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Fri, 7 May 2021 09:44:33 +0200 Subject: [PATCH 03/10] c: Fix prototypes for bcmp and memcmp They return c_int and not isize. --- lib/std/special/c.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/std/special/c.zig b/lib/std/special/c.zig index db0e107def..13e8459b19 100644 --- a/lib/std/special/c.zig +++ b/lib/std/special/c.zig @@ -225,7 +225,7 @@ export fn memmove(dest: ?[*]u8, src: ?[*]const u8, n: usize) callconv(.C) ?[*]u8 return dest; } -export fn memcmp(vl: ?[*]const u8, vr: ?[*]const u8, n: usize) callconv(.C) isize { +export fn memcmp(vl: ?[*]const u8, vr: ?[*]const u8, n: usize) callconv(.C) c_int { @setRuntimeSafety(false); var index: usize = 0; @@ -250,7 +250,7 @@ test "memcmp" { try std.testing.expect(memcmp(base_arr[0..], arr3[0..], base_arr.len) < 0); } -export fn bcmp(vl: [*]allowzero const u8, vr: [*]allowzero const u8, n: usize) callconv(.C) isize { +export fn bcmp(vl: [*]allowzero const u8, vr: [*]allowzero const u8, n: usize) callconv(.C) c_int { @setRuntimeSafety(false); var index: usize = 0; From 93f48189f18c9e2a74c8385497a31cbec3059a79 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Fri, 7 May 2021 09:45:37 +0200 Subject: [PATCH 04/10] test: Enable i386-linux-gnu test This is needed to catch any possible problem with executables linking to 32bit glibc. --- test/tests.zig | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/test/tests.zig b/test/tests.zig index 7cdf83f2cd..8bd1f3528e 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -98,15 +98,14 @@ const test_targets = blk: { }, .link_libc = true, }, - // https://github.com/ziglang/zig/issues/4926 - //TestTarget{ - // .target = .{ - // .cpu_arch = .i386, - // .os_tag = .linux, - // .abi = .gnu, - // }, - // .link_libc = true, - //}, + TestTarget{ + .target = .{ + .cpu_arch = .i386, + .os_tag = .linux, + .abi = .gnu, + }, + .link_libc = true, + }, TestTarget{ .target = .{ From f4a05286d34d2360da5a76cf0c77697775b571eb Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Fri, 7 May 2021 11:33:48 +0200 Subject: [PATCH 05/10] std: Fix cast that's sometimes invalid --- lib/std/c/linux.zig | 4 ++-- lib/std/os.zig | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/lib/std/c/linux.zig b/lib/std/c/linux.zig index f7a91f4e1b..d2d5e610c3 100644 --- a/lib/std/c/linux.zig +++ b/lib/std/c/linux.zig @@ -61,11 +61,11 @@ pub const EAI = extern enum(c_int) { _, }; -pub extern "c" fn fallocate64(fd: fd_t, mode: c_int, offset: i64, len: i64) c_int; +pub extern "c" fn fallocate64(fd: fd_t, mode: c_int, offset: off_t, len: off_t) c_int; pub extern "c" fn fopen64(noalias filename: [*:0]const u8, noalias modes: [*:0]const u8) ?*FILE; pub extern "c" fn fstat64(fd: fd_t, buf: *libc_stat) c_int; pub extern "c" fn fstatat64(dirfd: fd_t, path: [*:0]const u8, stat_buf: *libc_stat, flags: u32) c_int; -pub extern "c" fn ftruncate64(fd: c_int, length: i64) c_int; +pub extern "c" fn ftruncate64(fd: c_int, length: off_t) c_int; pub extern "c" fn getrlimit64(resource: rlimit_resource, rlim: *rlimit) c_int; pub extern "c" fn lseek64(fd: fd_t, offset: u64, whence: c_int) u64; pub extern "c" fn mmap64(addr: ?*align(std.mem.page_size) c_void, len: usize, prot: c_uint, flags: c_uint, fd: fd_t, offset: u64) *c_void; diff --git a/lib/std/os.zig b/lib/std/os.zig index db030f0e2b..b78141d20b 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -577,7 +577,12 @@ pub fn ftruncate(fd: fd_t, length: u64) TruncateError!void { else system.ftruncate; - switch (errno(ftruncate_sym(fd, @bitCast(off_t, length)))) { + // XXX Pick a side and avoid this cast madness. + const casted_length = if (builtin.link_libc) + @bitCast(off_t, length) + else + length; + switch (errno(ftruncate_sym(fd, casted_length))) { 0 => return, EINTR => continue, EFBIG => return error.FileTooBig, From 31f1cc9a0d80e87efb1f414352ff9d6abf2067ca Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Fri, 7 May 2021 14:42:10 +0200 Subject: [PATCH 06/10] std: Harmonize use of off_t between libc and Zig impls Let's follow the libc/kernel convention and use an i64 for offsets, we bitcast as needed and avoid the useless conditional casts. --- lib/std/c.zig | 10 +++++----- lib/std/c/linux.zig | 12 ++++++------ lib/std/os.zig | 38 +++++++++++++++++++++++++------------- lib/std/os/linux.zig | 37 +++++++++++++++++++------------------ 4 files changed, 55 insertions(+), 42 deletions(-) diff --git a/lib/std/c.zig b/lib/std/c.zig index 58ebaaa804..acfc6b34f7 100644 --- a/lib/std/c.zig +++ b/lib/std/c.zig @@ -89,13 +89,13 @@ pub extern "c" fn ftruncate(fd: c_int, length: off_t) c_int; pub extern "c" fn raise(sig: c_int) c_int; pub extern "c" fn read(fd: fd_t, buf: [*]u8, nbyte: usize) isize; pub extern "c" fn readv(fd: c_int, iov: [*]const iovec, iovcnt: c_uint) isize; -pub extern "c" fn pread(fd: fd_t, buf: [*]u8, nbyte: usize, offset: u64) isize; -pub extern "c" fn preadv(fd: c_int, iov: [*]const iovec, iovcnt: c_uint, offset: u64) isize; +pub extern "c" fn pread(fd: fd_t, buf: [*]u8, nbyte: usize, offset: off_t) isize; +pub extern "c" fn preadv(fd: c_int, iov: [*]const iovec, iovcnt: c_uint, offset: off_t) isize; pub extern "c" fn writev(fd: c_int, iov: [*]const iovec_const, iovcnt: c_uint) isize; -pub extern "c" fn pwritev(fd: c_int, iov: [*]const iovec_const, iovcnt: c_uint, offset: u64) isize; +pub extern "c" fn pwritev(fd: c_int, iov: [*]const iovec_const, iovcnt: c_uint, offset: off_t) isize; pub extern "c" fn write(fd: fd_t, buf: [*]const u8, nbyte: usize) isize; -pub extern "c" fn pwrite(fd: fd_t, buf: [*]const u8, nbyte: usize, offset: u64) isize; -pub extern "c" fn mmap(addr: ?*align(page_size) c_void, len: usize, prot: c_uint, flags: c_uint, fd: fd_t, offset: u64) *c_void; +pub extern "c" fn pwrite(fd: fd_t, buf: [*]const u8, nbyte: usize, offset: off_t) isize; +pub extern "c" fn mmap(addr: ?*align(page_size) c_void, len: usize, prot: c_uint, flags: c_uint, fd: fd_t, offset: off_t) *c_void; pub extern "c" fn munmap(addr: *align(page_size) c_void, len: usize) c_int; pub extern "c" fn mprotect(addr: *align(page_size) c_void, len: usize, prot: c_uint) c_int; pub extern "c" fn link(oldpath: [*:0]const u8, newpath: [*:0]const u8, flags: c_int) c_int; diff --git a/lib/std/c/linux.zig b/lib/std/c/linux.zig index d2d5e610c3..a8c8bbc34d 100644 --- a/lib/std/c/linux.zig +++ b/lib/std/c/linux.zig @@ -67,14 +67,14 @@ pub extern "c" fn fstat64(fd: fd_t, buf: *libc_stat) c_int; pub extern "c" fn fstatat64(dirfd: fd_t, path: [*:0]const u8, stat_buf: *libc_stat, flags: u32) c_int; pub extern "c" fn ftruncate64(fd: c_int, length: off_t) c_int; pub extern "c" fn getrlimit64(resource: rlimit_resource, rlim: *rlimit) c_int; -pub extern "c" fn lseek64(fd: fd_t, offset: u64, whence: c_int) u64; -pub extern "c" fn mmap64(addr: ?*align(std.mem.page_size) c_void, len: usize, prot: c_uint, flags: c_uint, fd: fd_t, offset: u64) *c_void; +pub extern "c" fn lseek64(fd: fd_t, offset: i64, whence: c_int) i64; +pub extern "c" fn mmap64(addr: ?*align(std.mem.page_size) c_void, len: usize, prot: c_uint, flags: c_uint, fd: fd_t, offset: i64) *c_void; pub extern "c" fn open64(path: [*:0]const u8, oflag: c_uint, ...) c_int; pub extern "c" fn openat64(fd: c_int, path: [*:0]const u8, oflag: c_uint, ...) c_int; -pub extern "c" fn pread64(fd: fd_t, buf: [*]u8, nbyte: usize, offset: u64) isize; -pub extern "c" fn preadv64(fd: c_int, iov: [*]const iovec, iovcnt: c_uint, offset: u64) isize; -pub extern "c" fn pwrite64(fd: fd_t, buf: [*]const u8, nbyte: usize, offset: u64) isize; -pub extern "c" fn pwritev64(fd: c_int, iov: [*]const iovec_const, iovcnt: c_uint, offset: u64) isize; +pub extern "c" fn pread64(fd: fd_t, buf: [*]u8, nbyte: usize, offset: i64) isize; +pub extern "c" fn preadv64(fd: c_int, iov: [*]const iovec, iovcnt: c_uint, offset: i64) isize; +pub extern "c" fn pwrite64(fd: fd_t, buf: [*]const u8, nbyte: usize, offset: i64) isize; +pub extern "c" fn pwritev64(fd: c_int, iov: [*]const iovec_const, iovcnt: c_uint, offset: i64) isize; pub extern "c" fn sendfile64(out_fd: fd_t, in_fd: fd_t, offset: ?*i64, count: usize) isize; pub extern "c" fn setrlimit64(resource: rlimit_resource, rlim: *const rlimit) c_int; diff --git a/lib/std/os.zig b/lib/std/os.zig index b78141d20b..358290a347 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -502,8 +502,9 @@ pub fn pread(fd: fd_t, buf: []u8, offset: u64) PReadError!usize { else system.pread; + const ioffset = @bitCast(i64, offset); // the OS treats this as unsigned while (true) { - const rc = pread_sym(fd, buf.ptr, adjusted_len, offset); + const rc = pread_sym(fd, buf.ptr, adjusted_len, ioffset); switch (errno(rc)) { 0 => return @intCast(usize, rc), EINTR => continue, @@ -577,12 +578,8 @@ pub fn ftruncate(fd: fd_t, length: u64) TruncateError!void { else system.ftruncate; - // XXX Pick a side and avoid this cast madness. - const casted_length = if (builtin.link_libc) - @bitCast(off_t, length) - else - length; - switch (errno(ftruncate_sym(fd, casted_length))) { + const ilen = @bitCast(i64, length); // the OS treats this as unsigned + switch (errno(ftruncate_sym(fd, ilen))) { 0 => return, EINTR => continue, EFBIG => return error.FileTooBig, @@ -912,8 +909,9 @@ pub fn pwrite(fd: fd_t, bytes: []const u8, offset: u64) PWriteError!usize { else system.pwrite; + const ioffset = @bitCast(i64, offset); // the OS treats this as unsigned while (true) { - const rc = pwrite_sym(fd, bytes.ptr, adjusted_len, offset); + const rc = pwrite_sym(fd, bytes.ptr, adjusted_len, ioffset); switch (errno(rc)) { 0 => return @intCast(usize, rc), EINTR => continue, @@ -3750,7 +3748,8 @@ pub fn mmap( else system.mmap; - const rc = mmap_sym(ptr, length, prot, flags, fd, offset); + const ioffset = @bitCast(i64, offset); // the OS treats this as unsigned + const rc = mmap_sym(ptr, length, prot, flags, fd, ioffset); const err = if (builtin.link_libc) blk: { if (rc != std.c.MAP_FAILED) return @ptrCast([*]align(mem.page_size) u8, @alignCast(mem.page_size, rc))[0..length]; break :blk system._errno().*; @@ -4104,8 +4103,14 @@ pub fn lseek_SET(fd: fd_t, offset: u64) SeekError!void { else => |err| return unexpectedErrno(err), } } - const ipos = @bitCast(i64, offset); // the OS treats this as unsigned - switch (errno(system.lseek(fd, ipos, SEEK_SET))) { + + const lseek_sym = if (builtin.os.tag == .linux and builtin.link_libc) + system.lseek64 + else + system.lseek; + + const ioffset = @bitCast(i64, offset); // the OS treats this as unsigned + switch (errno(lseek_sym(fd, ioffset, SEEK_SET))) { 0 => return, EBADF => unreachable, // always a race condition EINVAL => return error.Unseekable, @@ -4146,7 +4151,13 @@ pub fn lseek_CUR(fd: fd_t, offset: i64) SeekError!void { else => |err| return unexpectedErrno(err), } } - switch (errno(system.lseek(fd, offset, SEEK_CUR))) { + const lseek_sym = if (builtin.os.tag == .linux and builtin.link_libc) + system.lseek64 + else + system.lseek; + + const ioffset = @bitCast(i64, offset); // the OS treats this as unsigned + switch (errno(lseek_sym(fd, ioffset, SEEK_CUR))) { 0 => return, EBADF => unreachable, // always a race condition EINVAL => return error.Unseekable, @@ -4192,7 +4203,8 @@ pub fn lseek_END(fd: fd_t, offset: i64) SeekError!void { else system.lseek; - switch (errno(lseek_sym(fd, offset, SEEK_END))) { + const ioffset = @bitCast(i64, offset); // the OS treats this as unsigned + switch (errno(lseek_sym(fd, ioffset, SEEK_END))) { 0 => return, EBADF => unreachable, // always a race condition EINVAL => return error.Unseekable, diff --git a/lib/std/os/linux.zig b/lib/std/os/linux.zig index 6c88d9eae1..912dd3a6f7 100644 --- a/lib/std/os/linux.zig +++ b/lib/std/os/linux.zig @@ -59,15 +59,16 @@ const require_aligned_register_pair = std.Target.current.cpu.arch.isThumb(); // Split a 64bit value into a {LSB,MSB} pair. -fn splitValue64(val: u64) [2]u32 { +fn splitValue64(val: i64) [2]u32 { + const u = @bitCast(u64, val); switch (builtin.endian) { .Little => return [2]u32{ - @truncate(u32, val), - @truncate(u32, val >> 32), + @truncate(u32, u), + @truncate(u32, u >> 32), }, .Big => return [2]u32{ - @truncate(u32, val >> 32), - @truncate(u32, val), + @truncate(u32, u >> 32), + @truncate(u32, u), }, } } @@ -243,7 +244,7 @@ pub fn umount2(special: [*:0]const u8, flags: u32) usize { return syscall2(.umount2, @ptrToInt(special), flags); } -pub fn mmap(address: ?[*]u8, length: usize, prot: usize, flags: u32, fd: i32, offset: u64) usize { +pub fn mmap(address: ?[*]u8, length: usize, prot: usize, flags: u32, fd: i32, offset: i64) usize { if (@hasField(SYS, "mmap2")) { // Make sure the offset is also specified in multiples of page size if ((offset & (MMAP2_UNIT - 1)) != 0) @@ -256,7 +257,7 @@ pub fn mmap(address: ?[*]u8, length: usize, prot: usize, flags: u32, fd: i32, of prot, flags, @bitCast(usize, @as(isize, fd)), - @truncate(usize, offset / MMAP2_UNIT), + @truncate(usize, @bitCast(u64, offset) / MMAP2_UNIT), ); } else { return syscall6( @@ -266,7 +267,7 @@ pub fn mmap(address: ?[*]u8, length: usize, prot: usize, flags: u32, fd: i32, of prot, flags, @bitCast(usize, @as(isize, fd)), - offset, + @bitCast(u64, offset), ); } } @@ -308,7 +309,7 @@ pub fn read(fd: i32, buf: [*]u8, count: usize) usize { return syscall3(.read, @bitCast(usize, @as(isize, fd)), @ptrToInt(buf), count); } -pub fn preadv(fd: i32, iov: [*]const iovec, count: usize, offset: u64) usize { +pub fn preadv(fd: i32, iov: [*]const iovec, count: usize, offset: i64) usize { const offset_halves = splitValue64(offset); return syscall5( .preadv, @@ -320,7 +321,7 @@ pub fn preadv(fd: i32, iov: [*]const iovec, count: usize, offset: u64) usize { ); } -pub fn preadv2(fd: i32, iov: [*]const iovec, count: usize, offset: u64, flags: kernel_rwf) usize { +pub fn preadv2(fd: i32, iov: [*]const iovec, count: usize, offset: i64, flags: kernel_rwf) usize { const offset_halves = splitValue64(offset); return syscall6( .preadv2, @@ -341,7 +342,7 @@ pub fn writev(fd: i32, iov: [*]const iovec_const, count: usize) usize { return syscall3(.writev, @bitCast(usize, @as(isize, fd)), @ptrToInt(iov), count); } -pub fn pwritev(fd: i32, iov: [*]const iovec_const, count: usize, offset: u64) usize { +pub fn pwritev(fd: i32, iov: [*]const iovec_const, count: usize, offset: i64) usize { const offset_halves = splitValue64(offset); return syscall5( .pwritev, @@ -353,7 +354,7 @@ pub fn pwritev(fd: i32, iov: [*]const iovec_const, count: usize, offset: u64) us ); } -pub fn pwritev2(fd: i32, iov: [*]const iovec_const, count: usize, offset: u64, flags: kernel_rwf) usize { +pub fn pwritev2(fd: i32, iov: [*]const iovec_const, count: usize, offset: i64, flags: kernel_rwf) usize { const offset_halves = splitValue64(offset); return syscall6( .pwritev2, @@ -386,7 +387,7 @@ pub fn symlinkat(existing: [*:0]const u8, newfd: i32, newpath: [*:0]const u8) us return syscall3(.symlinkat, @ptrToInt(existing), @bitCast(usize, @as(isize, newfd)), @ptrToInt(newpath)); } -pub fn pread(fd: i32, buf: [*]u8, count: usize, offset: u64) usize { +pub fn pread(fd: i32, buf: [*]u8, count: usize, offset: i64) usize { if (@hasField(SYS, "pread64") and usize_bits < 64) { const offset_halves = splitValue64(offset); if (require_aligned_register_pair) { @@ -417,7 +418,7 @@ pub fn pread(fd: i32, buf: [*]u8, count: usize, offset: u64) usize { @bitCast(usize, @as(isize, fd)), @ptrToInt(buf), count, - offset, + @bitCast(u64, offset), ); } } @@ -452,7 +453,7 @@ pub fn write(fd: i32, buf: [*]const u8, count: usize) usize { return syscall3(.write, @bitCast(usize, @as(isize, fd)), @ptrToInt(buf), count); } -pub fn ftruncate(fd: i32, length: u64) usize { +pub fn ftruncate(fd: i32, length: i64) usize { if (@hasField(SYS, "ftruncate64") and usize_bits < 64) { const length_halves = splitValue64(length); if (require_aligned_register_pair) { @@ -475,12 +476,12 @@ pub fn ftruncate(fd: i32, length: u64) usize { return syscall2( .ftruncate, @bitCast(usize, @as(isize, fd)), - @truncate(usize, length), + @bitCast(usize, length), ); } } -pub fn pwrite(fd: i32, buf: [*]const u8, count: usize, offset: u64) usize { +pub fn pwrite(fd: i32, buf: [*]const u8, count: usize, offset: i64) usize { if (@hasField(SYS, "pwrite64") and usize_bits < 64) { const offset_halves = splitValue64(offset); @@ -512,7 +513,7 @@ pub fn pwrite(fd: i32, buf: [*]const u8, count: usize, offset: u64) usize { @bitCast(usize, @as(isize, fd)), @ptrToInt(buf), count, - offset, + @bitCast(u64, offset), ); } } From c065e12dbefcde1a48926405bd4f401eadecaca2 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Fri, 7 May 2021 15:24:37 +0200 Subject: [PATCH 07/10] std: Add more tests, fix broken code It turns out that nothing in the test suite was exercising preadv/pwritev and so the previous commits silently broke them. Adding tests revealed readvAll and preadvAll were also broken and not covered by any test. --- lib/std/fs/file.zig | 6 ++-- lib/std/fs/test.zig | 83 +++++++++++++++++++++++++++++++++++++++++++++ lib/std/os.zig | 6 ++-- 3 files changed, 90 insertions(+), 5 deletions(-) diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index 3a8fd9e9a3..bc202c7b89 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -482,7 +482,7 @@ pub const File = struct { /// order to handle partial reads from the underlying OS layer. /// See https://github.com/ziglang/zig/issues/7699 pub fn readvAll(self: File, iovecs: []os.iovec) ReadError!usize { - if (iovecs.len == 0) return; + if (iovecs.len == 0) return 0; var i: usize = 0; var off: usize = 0; @@ -524,8 +524,8 @@ pub const File = struct { /// The `iovecs` parameter is mutable because this function needs to mutate the fields in /// order to handle partial reads from the underlying OS layer. /// See https://github.com/ziglang/zig/issues/7699 - pub fn preadvAll(self: File, iovecs: []const os.iovec, offset: u64) PReadError!void { - if (iovecs.len == 0) return; + pub fn preadvAll(self: File, iovecs: []os.iovec, offset: u64) PReadError!usize { + if (iovecs.len == 0) return 0; var i: usize = 0; var off: usize = 0; diff --git a/lib/std/fs/test.zig b/lib/std/fs/test.zig index 3417e782c0..f586c50b6a 100644 --- a/lib/std/fs/test.zig +++ b/lib/std/fs/test.zig @@ -520,6 +520,89 @@ test "makePath, put some files in it, deleteTree" { } } +test "writev, readv" { + var tmp = tmpDir(.{}); + defer tmp.cleanup(); + + const line1 = "line1\n"; + const line2 = "line2\n"; + + var buf1: [line1.len]u8 = undefined; + var buf2: [line2.len]u8 = undefined; + var write_vecs = [_]std.os.iovec_const{ + .{ + .iov_base = line1, + .iov_len = line1.len, + }, + .{ + .iov_base = line2, + .iov_len = line2.len, + }, + }; + var read_vecs = [_]std.os.iovec{ + .{ + .iov_base = &buf2, + .iov_len = buf2.len, + }, + .{ + .iov_base = &buf1, + .iov_len = buf1.len, + }, + }; + + var src_file = try tmp.dir.createFile("test.txt", .{ .read = true }); + defer src_file.close(); + + try src_file.writevAll(&write_vecs); + try testing.expectEqual(@as(u64, line1.len + line2.len), try src_file.getEndPos()); + try src_file.seekTo(0); + const read = try src_file.readvAll(&read_vecs); + try testing.expectEqual(@as(usize, line1.len + line2.len), read); + try testing.expectEqualStrings(&buf1, "line2\n"); + try testing.expectEqualStrings(&buf2, "line1\n"); +} + +test "pwritev, preadv" { + var tmp = tmpDir(.{}); + defer tmp.cleanup(); + + const line1 = "line1\n"; + const line2 = "line2\n"; + + var buf1: [line1.len]u8 = undefined; + var buf2: [line2.len]u8 = undefined; + var write_vecs = [_]std.os.iovec_const{ + .{ + .iov_base = line1, + .iov_len = line1.len, + }, + .{ + .iov_base = line2, + .iov_len = line2.len, + }, + }; + var read_vecs = [_]std.os.iovec{ + .{ + .iov_base = &buf2, + .iov_len = buf2.len, + }, + .{ + .iov_base = &buf1, + .iov_len = buf1.len, + }, + }; + + var src_file = try tmp.dir.createFile("test.txt", .{ .read = true }); + defer src_file.close(); + + try src_file.pwritevAll(&write_vecs, 16); + try testing.expectEqual(@as(u64, 16 + line1.len + line2.len), try src_file.getEndPos()); + const read = try src_file.preadvAll(&read_vecs, 16); + try testing.expectEqual(@as(usize, line1.len + line2.len), read); + try testing.expectEqualStrings(&buf1, "line2\n"); + try testing.expectEqualStrings(&buf2, "line1\n"); +} + test "access file" { if (builtin.os.tag == .wasi) return error.SkipZigTest; diff --git a/lib/std/os.zig b/lib/std/os.zig index 358290a347..0ee20d7c8e 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -646,8 +646,9 @@ pub fn preadv(fd: fd_t, iov: []const iovec, offset: u64) PReadError!usize { else system.preadv; + const ioffset = @bitCast(i64, offset); // the OS treats this as unsigned while (true) { - const rc = preadv_sym(fd, iov.ptr, iov_count, offset); + const rc = preadv_sym(fd, iov.ptr, iov_count, ioffset); switch (errno(rc)) { 0 => return @bitCast(usize, rc), EINTR => continue, @@ -998,8 +999,9 @@ pub fn pwritev(fd: fd_t, iov: []const iovec_const, offset: u64) PWriteError!usiz system.pwritev; const iov_count = math.cast(u31, iov.len) catch math.maxInt(u31); + const ioffset = @bitCast(i64, offset); // the OS treats this as unsigned while (true) { - const rc = pwritev_sym(fd, iov.ptr, iov_count, offset); + const rc = pwritev_sym(fd, iov.ptr, iov_count, ioffset); switch (errno(rc)) { 0 => return @intCast(usize, rc), EINTR => continue, From 780f510ac0137512ad2923835e4ecdb22a81028e Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Fri, 7 May 2021 18:09:52 +0200 Subject: [PATCH 08/10] std: Fix fallocate offset type --- lib/std/os/linux.zig | 8 ++++---- lib/std/os/linux/test.zig | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/std/os/linux.zig b/lib/std/os/linux.zig index 912dd3a6f7..d5e25f2ab7 100644 --- a/lib/std/os/linux.zig +++ b/lib/std/os/linux.zig @@ -142,8 +142,8 @@ pub fn utimensat(dirfd: i32, path: ?[*:0]const u8, times: *const [2]timespec, fl return syscall4(.utimensat, @bitCast(usize, @as(isize, dirfd)), @ptrToInt(path), @ptrToInt(times), flags); } -pub fn fallocate(fd: i32, mode: i32, offset: u64, length: u64) usize { - if (@sizeOf(usize) == 4) { +pub fn fallocate(fd: i32, mode: i32, offset: i64, length: i64) usize { + if (usize_bits < 64) { const offset_halves = splitValue64(offset); const length_halves = splitValue64(length); return syscall6( @@ -160,8 +160,8 @@ pub fn fallocate(fd: i32, mode: i32, offset: u64, length: u64) usize { .fallocate, @bitCast(usize, @as(isize, fd)), @bitCast(usize, @as(isize, mode)), - offset, - length, + @bitCast(u64, offset), + @bitCast(u64, length), ); } } diff --git a/lib/std/os/linux/test.zig b/lib/std/os/linux/test.zig index 5a37713247..739f0bb82a 100644 --- a/lib/std/os/linux/test.zig +++ b/lib/std/os/linux/test.zig @@ -20,7 +20,7 @@ test "fallocate" { try expect((try file.stat()).size == 0); - const len: u64 = 65536; + const len: i64 = 65536; switch (linux.getErrno(linux.fallocate(file.handle, 0, 0, len))) { 0 => {}, linux.ENOSYS => return error.SkipZigTest, From 2bb8e1ff550ef2b2676e4f7ec3fdaf17aee8d715 Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Tue, 11 May 2021 12:43:58 +0200 Subject: [PATCH 09/10] stage2: Change libc components' linking order Use the same order as Clang (and, by extension, GCC) for the three most important libc components: lm comes first, followed by lpthread and then lc. --- src/glibc.zig | 3 ++- src/link/Elf.zig | 15 +++++---------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/src/glibc.zig b/src/glibc.zig index a1f02e2f11..7600f19503 100644 --- a/src/glibc.zig +++ b/src/glibc.zig @@ -40,10 +40,11 @@ pub const ABI = struct { } }; +// The order of the elements in this array defines the linking order. pub const libs = [_]Lib{ - .{ .name = "c", .sover = 6 }, .{ .name = "m", .sover = 6 }, .{ .name = "pthread", .sover = 0 }, + .{ .name = "c", .sover = 6 }, .{ .name = "dl", .sover = 2 }, .{ .name = "rt", .sover = 1 }, .{ .name = "ld", .sover = 2 }, diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 4375661f64..c31a094093 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -1648,17 +1648,12 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { // libc dep if (self.base.options.link_libc) { if (self.base.options.libc_installation != null) { - if (self.base.options.link_mode == .Static) { - try argv.append("--start-group"); - try argv.append("-lc"); - try argv.append("-lm"); - try argv.append("--end-group"); - } else { - try argv.append("-lc"); - try argv.append("-lm"); - } - + const needs_grouping = self.base.options.link_mode == .Static; + if (needs_grouping) try argv.append("--start-group"); + try argv.append("-lm"); try argv.append("-lpthread"); + try argv.append("-lc"); + if (needs_grouping) try argv.append("--end-group"); } else if (target.isGnuLibC()) { try argv.append(comp.libunwind_static_lib.?.full_object_path); for (glibc.libs) |lib| { From 8bb58fb4286ad03a243e32e31d67dd2bb5c117ef Mon Sep 17 00:00:00 2001 From: LemonBoy Date: Tue, 11 May 2021 15:49:53 +0200 Subject: [PATCH 10/10] std: Fix offset param splitting for preadv/pwritev The kernel interface expects the offset as a low+high pair even on BE systems. --- lib/std/os/linux.zig | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/lib/std/os/linux.zig b/lib/std/os/linux.zig index d5e25f2ab7..fbb44ba7bd 100644 --- a/lib/std/os/linux.zig +++ b/lib/std/os/linux.zig @@ -59,6 +59,20 @@ const require_aligned_register_pair = std.Target.current.cpu.arch.isThumb(); // Split a 64bit value into a {LSB,MSB} pair. +// The LE/BE variants specify the endianness to assume. +fn splitValueLE64(val: i64) [2]u32 { + const u = @bitCast(u64, val); + return [2]u32{ + @truncate(u32, u), + @truncate(u32, u >> 32), + }; +} +fn splitValueBE64(val: i64) [2]u32 { + return [2]u32{ + @truncate(u32, u >> 32), + @truncate(u32, u), + }; +} fn splitValue64(val: i64) [2]u32 { const u = @bitCast(u64, val); switch (builtin.endian) { @@ -310,7 +324,7 @@ pub fn read(fd: i32, buf: [*]u8, count: usize) usize { } pub fn preadv(fd: i32, iov: [*]const iovec, count: usize, offset: i64) usize { - const offset_halves = splitValue64(offset); + const offset_halves = splitValueLE64(offset); return syscall5( .preadv, @bitCast(usize, @as(isize, fd)), @@ -343,7 +357,7 @@ pub fn writev(fd: i32, iov: [*]const iovec_const, count: usize) usize { } pub fn pwritev(fd: i32, iov: [*]const iovec_const, count: usize, offset: i64) usize { - const offset_halves = splitValue64(offset); + const offset_halves = splitValueLE64(offset); return syscall5( .pwritev, @bitCast(usize, @as(isize, fd)),