From f65cdef7c8bfe7a47e1c30cfa3903ca2d53495cf Mon Sep 17 00:00:00 2001 From: Philippe Pittoli Date: Fri, 6 May 2022 01:03:03 +0200 Subject: [PATCH 1/6] std.fs.Dir.statFile: use fstatat This avoids extra syscalls. --- lib/std/fs.zig | 24 +++++++++++++++------ lib/std/fs/file.zig | 51 +++++++++++++++++++++++++++++++++++++++++++++ src/Module.zig | 17 +++++++++------ 3 files changed, 80 insertions(+), 12 deletions(-) diff --git a/lib/std/fs.zig b/lib/std/fs.zig index e253aaff9e..f47f06a2d9 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -2598,14 +2598,26 @@ pub const Dir = struct { return file.stat(); } - pub const StatFileError = File.OpenError || StatError; + pub const StatFileError = File.OpenError || File.StatError || os.FStatAtError; - // TODO: improve this to use the fstatat syscall instead of making 2 syscalls here. - pub fn statFile(self: Dir, sub_path: []const u8) StatFileError!File.Stat { - var file = try self.openFile(sub_path, .{}); - defer file.close(); + /// Provides info on a file (File.Stat) for any file in the opened directory, + /// with a single syscall (fstatat), except on Windows. + /// Currently on Windows, files are opened then closed (implying several syscalls, unfortunately). + /// Symlinks are not followed on linux, haiku, solaris and *BSDs. + /// Other OSs have a default behavior (they currently lack an os.AT.SYMLINK_NOFOLLOW flag). + pub fn statFile(self: Dir, sub_path: []const u8) StatFileError!Stat { + if (builtin.os.tag == .windows) { + var file = try self.openFile(sub_path, .{}); + defer file.close(); + return file.stat(); + } - return file.stat(); + const flags = switch (builtin.os.tag) { + .linux, .haiku, .solaris, .freebsd, .netbsd, .dragonfly, .openbsd => os.AT.SYMLINK_NOFOLLOW, + else => 0, // TODO: correct flags not yet implemented + }; + + return Stat.fromSystemStat(try os.fstatat(self.fd, sub_path, flags)); } const Permissions = File.Permissions; diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index a1e81c9b94..4dd40fe0a2 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -313,6 +313,57 @@ pub const File = struct { mtime: i128, /// Creation time in nanoseconds, relative to UTC 1970-01-01. ctime: i128, + + pub fn systemStatKindToFsKind(st: os.system.Stat) Kind { + const kind: File.Kind = if (builtin.os.tag == .wasi and !builtin.link_libc) + switch (st.filetype) { + .BLOCK_DEVICE => Kind.BlockDevice, + .CHARACTER_DEVICE => Kind.CharacterDevice, + .DIRECTORY => Kind.Directory, + .SYMBOLIC_LINK => Kind.SymLink, + .REGULAR_FILE => Kind.File, + .SOCKET_STREAM, .SOCKET_DGRAM => Kind.UnixDomainSocket, + else => Kind.Unknown, + } + else blk: { + const m = st.mode & os.S.IFMT; + switch (m) { + os.S.IFBLK => break :blk Kind.BlockDevice, + os.S.IFCHR => break :blk Kind.CharacterDevice, + os.S.IFDIR => break :blk Kind.Directory, + os.S.IFIFO => break :blk Kind.NamedPipe, + os.S.IFLNK => break :blk Kind.SymLink, + os.S.IFREG => break :blk Kind.File, + os.S.IFSOCK => break :blk Kind.UnixDomainSocket, + else => {}, + } + if (builtin.os.tag == .solaris) switch (m) { + os.S.IFDOOR => break :blk Kind.Door, + os.S.IFPORT => break :blk Kind.EventPort, + else => {}, + }; + + break :blk .Unknown; + }; + return kind; + } + + pub fn fromSystemStat(st: os.system.Stat) File.StatError!Stat { + const atime = st.atime(); + const mtime = st.mtime(); + const ctime = st.ctime(); + const kind = systemStatKindToFsKind(st); + + return Stat{ + .inode = st.ino, + .size = @bitCast(u64, st.size), + .mode = st.mode, + .kind = kind, + .atime = @as(i128, atime.tv_sec) * std.time.ns_per_s + atime.tv_nsec, + .mtime = @as(i128, mtime.tv_sec) * std.time.ns_per_s + mtime.tv_nsec, + .ctime = @as(i128, ctime.tv_sec) * std.time.ns_per_s + ctime.tv_nsec, + }; + } }; pub const StatError = os.FStatError; diff --git a/src/Module.zig b/src/Module.zig index 83294f3068..d4af06f896 100644 --- a/src/Module.zig +++ b/src/Module.zig @@ -4137,14 +4137,19 @@ pub fn populateBuiltinFile(mod: *Module) !void { }; } } else |err| switch (err) { - error.BadPathName => unreachable, // it's always "builtin.zig" error.NameTooLong => unreachable, // it's always "builtin.zig" - error.PipeBusy => unreachable, // it's not a pipe - error.WouldBlock => unreachable, // not asking for non-blocking I/O - error.FileNotFound => try writeBuiltinFile(file, builtin_pkg), - - else => |e| return e, + else => |e| { + if (builtin.os.tag == .windows) { + switch (e) { + error.BadPathName => unreachable, // it's always "builtin.zig" + error.PipeBusy => unreachable, // it's not a pipe + error.WouldBlock => unreachable, // not asking for non-blocking I/O + else => return e, + } + } + return e; + }, } file.tree = try std.zig.parse(gpa, file.source); From a62c8d36d530dc78f88dc0f41ce3f244c3c273bd Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 9 Sep 2022 09:55:09 -0700 Subject: [PATCH 2/6] std.fs.Dir.statFile rework * revert changes to Module because the error set is consistent across operating systems. * remove duplicated Stat.fromSystem code and use a less redundant name. * make fs.Dir.statFile follow symlinks, and avoid pointless control flow through the posix layer. --- lib/std/fs.zig | 38 +++++++++++-------- lib/std/fs/file.zig | 90 ++++++++++++--------------------------------- src/Module.zig | 17 +++------ 3 files changed, 51 insertions(+), 94 deletions(-) diff --git a/lib/std/fs.zig b/lib/std/fs.zig index f47f06a2d9..31f118c190 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -2600,24 +2600,30 @@ pub const Dir = struct { pub const StatFileError = File.OpenError || File.StatError || os.FStatAtError; - /// Provides info on a file (File.Stat) for any file in the opened directory, - /// with a single syscall (fstatat), except on Windows. - /// Currently on Windows, files are opened then closed (implying several syscalls, unfortunately). - /// Symlinks are not followed on linux, haiku, solaris and *BSDs. - /// Other OSs have a default behavior (they currently lack an os.AT.SYMLINK_NOFOLLOW flag). + /// Returns metadata for a file inside the directory. + /// + /// On Windows, this requires three syscalls. On other operating systems, it + /// only takes one. + /// + /// Symlinks are followed. + /// + /// `sub_path` may be absolute, in which case `self` is ignored. pub fn statFile(self: Dir, sub_path: []const u8) StatFileError!Stat { - if (builtin.os.tag == .windows) { - var file = try self.openFile(sub_path, .{}); - defer file.close(); - return file.stat(); + switch (builtin.os.tag) { + .windows => { + var file = try self.openFile(sub_path, .{}); + defer file.close(); + return file.stat(); + }, + .wasi => { + const st = try os.fstatatWasi(self.fd, sub_path, os.wasi.LOOKUP_SYMLINK_FOLLOW); + return Stat.fromSystem(st); + }, + else => { + const st = try os.fstatat(self.fd, sub_path, 0); + return Stat.fromSystem(st); + }, } - - const flags = switch (builtin.os.tag) { - .linux, .haiku, .solaris, .freebsd, .netbsd, .dragonfly, .openbsd => os.AT.SYMLINK_NOFOLLOW, - else => 0, // TODO: correct flags not yet implemented - }; - - return Stat.fromSystemStat(try os.fstatat(self.fd, sub_path, flags)); } const Permissions = File.Permissions; diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index 4dd40fe0a2..cb048adb30 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -294,14 +294,17 @@ pub const File = struct { } pub const Stat = struct { - /// A number that the system uses to point to the file metadata. This number is not guaranteed to be - /// unique across time, as some file systems may reuse an inode after its file has been deleted. - /// Some systems may change the inode of a file over time. + /// A number that the system uses to point to the file metadata. This + /// number is not guaranteed to be unique across time, as some file + /// systems may reuse an inode after its file has been deleted. Some + /// systems may change the inode of a file over time. /// - /// On Linux, the inode is a structure that stores the metadata, and the inode _number_ is what - /// you see here: the index number of the inode. + /// On Linux, the inode is a structure that stores the metadata, and + /// the inode _number_ is what you see here: the index number of the + /// inode. /// - /// The FileIndex on Windows is similar. It is a number for a file that is unique to each filesystem. + /// The FileIndex on Windows is similar. It is a number for a file that + /// is unique to each filesystem. inode: INode, size: u64, mode: Mode, @@ -314,18 +317,19 @@ pub const File = struct { /// Creation time in nanoseconds, relative to UTC 1970-01-01. ctime: i128, - pub fn systemStatKindToFsKind(st: os.system.Stat) Kind { - const kind: File.Kind = if (builtin.os.tag == .wasi and !builtin.link_libc) - switch (st.filetype) { - .BLOCK_DEVICE => Kind.BlockDevice, - .CHARACTER_DEVICE => Kind.CharacterDevice, - .DIRECTORY => Kind.Directory, - .SYMBOLIC_LINK => Kind.SymLink, - .REGULAR_FILE => Kind.File, - .SOCKET_STREAM, .SOCKET_DGRAM => Kind.UnixDomainSocket, - else => Kind.Unknown, - } - else blk: { + pub fn fromSystem(st: os.system.Stat) Stat { + const atime = st.atime(); + const mtime = st.mtime(); + const ctime = st.ctime(); + const kind: Kind = if (builtin.os.tag == .wasi and !builtin.link_libc) switch (st.filetype) { + .BLOCK_DEVICE => Kind.BlockDevice, + .CHARACTER_DEVICE => Kind.CharacterDevice, + .DIRECTORY => Kind.Directory, + .SYMBOLIC_LINK => Kind.SymLink, + .REGULAR_FILE => Kind.File, + .SOCKET_STREAM, .SOCKET_DGRAM => Kind.UnixDomainSocket, + else => Kind.Unknown, + } else blk: { const m = st.mode & os.S.IFMT; switch (m) { os.S.IFBLK => break :blk Kind.BlockDevice, @@ -345,14 +349,6 @@ pub const File = struct { break :blk .Unknown; }; - return kind; - } - - pub fn fromSystemStat(st: os.system.Stat) File.StatError!Stat { - const atime = st.atime(); - const mtime = st.mtime(); - const ctime = st.ctime(); - const kind = systemStatKindToFsKind(st); return Stat{ .inode = st.ino, @@ -393,47 +389,7 @@ pub const File = struct { } const st = try os.fstat(self.handle); - const atime = st.atime(); - const mtime = st.mtime(); - const ctime = st.ctime(); - const kind: Kind = if (builtin.os.tag == .wasi and !builtin.link_libc) switch (st.filetype) { - .BLOCK_DEVICE => Kind.BlockDevice, - .CHARACTER_DEVICE => Kind.CharacterDevice, - .DIRECTORY => Kind.Directory, - .SYMBOLIC_LINK => Kind.SymLink, - .REGULAR_FILE => Kind.File, - .SOCKET_STREAM, .SOCKET_DGRAM => Kind.UnixDomainSocket, - else => Kind.Unknown, - } else blk: { - const m = st.mode & os.S.IFMT; - switch (m) { - os.S.IFBLK => break :blk Kind.BlockDevice, - os.S.IFCHR => break :blk Kind.CharacterDevice, - os.S.IFDIR => break :blk Kind.Directory, - os.S.IFIFO => break :blk Kind.NamedPipe, - os.S.IFLNK => break :blk Kind.SymLink, - os.S.IFREG => break :blk Kind.File, - os.S.IFSOCK => break :blk Kind.UnixDomainSocket, - else => {}, - } - if (builtin.os.tag == .solaris) switch (m) { - os.S.IFDOOR => break :blk Kind.Door, - os.S.IFPORT => break :blk Kind.EventPort, - else => {}, - }; - - break :blk .Unknown; - }; - - return Stat{ - .inode = st.ino, - .size = @bitCast(u64, st.size), - .mode = st.mode, - .kind = kind, - .atime = @as(i128, atime.tv_sec) * std.time.ns_per_s + atime.tv_nsec, - .mtime = @as(i128, mtime.tv_sec) * std.time.ns_per_s + mtime.tv_nsec, - .ctime = @as(i128, ctime.tv_sec) * std.time.ns_per_s + ctime.tv_nsec, - }; + return Stat.fromSystem(st); } pub const ChmodError = std.os.FChmodError; diff --git a/src/Module.zig b/src/Module.zig index d4af06f896..83294f3068 100644 --- a/src/Module.zig +++ b/src/Module.zig @@ -4137,19 +4137,14 @@ pub fn populateBuiltinFile(mod: *Module) !void { }; } } else |err| switch (err) { + error.BadPathName => unreachable, // it's always "builtin.zig" error.NameTooLong => unreachable, // it's always "builtin.zig" + error.PipeBusy => unreachable, // it's not a pipe + error.WouldBlock => unreachable, // not asking for non-blocking I/O + error.FileNotFound => try writeBuiltinFile(file, builtin_pkg), - else => |e| { - if (builtin.os.tag == .windows) { - switch (e) { - error.BadPathName => unreachable, // it's always "builtin.zig" - error.PipeBusy => unreachable, // it's not a pipe - error.WouldBlock => unreachable, // not asking for non-blocking I/O - else => return e, - } - } - return e; - }, + + else => |e| return e, } file.tree = try std.zig.parse(gpa, file.source); From 353121d9d57d9dfcd734cd3b4b65d826f7eb00ee Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 9 Sep 2022 11:47:14 -0700 Subject: [PATCH 3/6] std.c.fstatat64: add noalias attributes --- lib/std/c/linux.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/c/linux.zig b/lib/std/c/linux.zig index cfdf5dfc45..d72fd15a57 100644 --- a/lib/std/c/linux.zig +++ b/lib/std/c/linux.zig @@ -229,7 +229,7 @@ pub const EAI = enum(c_int) { pub extern "c" fn fallocate64(fd: fd_t, mode: c_int, offset: off_t, len: off_t) c_int; pub extern "c" fn fopen64(noalias filename: [*:0]const u8, noalias modes: [*:0]const u8) ?*FILE; pub extern "c" fn fstat64(fd: fd_t, buf: *Stat) c_int; -pub extern "c" fn fstatat64(dirfd: fd_t, path: [*:0]const u8, stat_buf: *Stat, flags: u32) c_int; +pub extern "c" fn fstatat64(dirfd: fd_t, noalias path: [*:0]const u8, noalias stat_buf: *Stat, flags: u32) c_int; pub extern "c" fn ftruncate64(fd: c_int, length: off_t) c_int; pub extern "c" fn getrlimit64(resource: rlimit_resource, rlim: *rlimit) c_int; pub extern "c" fn lseek64(fd: fd_t, offset: i64, whence: c_int) i64; From bac4a5c1969ccb8b1d5cbac429462d2ad878fbe5 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 9 Sep 2022 11:47:36 -0700 Subject: [PATCH 4/6] std: remove a solved TODO comment --- lib/std/fs.zig | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 31f118c190..d2df596b3e 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -150,7 +150,6 @@ pub fn copyFileAbsolute(source_path: []const u8, dest_path: []const u8, args: Co return Dir.copyFile(my_cwd, source_path, my_cwd, dest_path, args); } -/// TODO update this API to avoid a getrandom syscall for every operation. pub const AtomicFile = struct { file: File, // TODO either replace this with rand_buf or use []u16 on Windows From 32d76f0e4a56f3331392f4eb9cc78a89cf11b109 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 9 Sep 2022 11:48:05 -0700 Subject: [PATCH 5/6] stage2: remove `pub` from a private function --- src/Module.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Module.zig b/src/Module.zig index 83294f3068..3347280f59 100644 --- a/src/Module.zig +++ b/src/Module.zig @@ -4156,7 +4156,7 @@ pub fn populateBuiltinFile(mod: *Module) !void { file.status = .success_zir; } -pub fn writeBuiltinFile(file: *File, builtin_pkg: *Package) !void { +fn writeBuiltinFile(file: *File, builtin_pkg: *Package) !void { var af = try builtin_pkg.root_src_directory.handle.atomicFile(builtin_pkg.root_src_path, .{}); defer af.deinit(); try af.file.writeAll(file.source); From b911b54aeb45d2d2d7a8d616af5df04ad230bcc3 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 14 Dec 2022 14:24:45 -0700 Subject: [PATCH 6/6] CI: fix path to langref when running tidy --- ci/aarch64-linux-debug.sh | 2 +- ci/aarch64-linux-release.sh | 2 +- ci/x86_64-linux-debug.sh | 2 +- ci/x86_64-linux-release.sh | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/aarch64-linux-debug.sh b/ci/aarch64-linux-debug.sh index 0452b7d7dd..758085c759 100644 --- a/ci/aarch64-linux-debug.sh +++ b/ci/aarch64-linux-debug.sh @@ -67,7 +67,7 @@ stage3-debug/bin/zig build test docs \ --zig-lib-dir "$(pwd)/../lib" # Look for HTML errors. -tidy --drop-empty-elements no -qe ../zig-cache/langref.html +tidy --drop-empty-elements no -qe "$ZIG_LOCAL_CACHE_DIR/langref.html" # Produce the experimental std lib documentation. stage3-debug/bin/zig test ../lib/std/std.zig -femit-docs -fno-emit-bin --zig-lib-dir ../lib diff --git a/ci/aarch64-linux-release.sh b/ci/aarch64-linux-release.sh index ef24accda4..59b7d7f9b9 100644 --- a/ci/aarch64-linux-release.sh +++ b/ci/aarch64-linux-release.sh @@ -67,7 +67,7 @@ stage3-release/bin/zig build test docs \ --zig-lib-dir "$(pwd)/../lib" # Look for HTML errors. -tidy --drop-empty-elements no -qe ../zig-cache/langref.html +tidy --drop-empty-elements no -qe "$ZIG_LOCAL_CACHE_DIR/langref.html" # Produce the experimental std lib documentation. stage3-release/bin/zig test ../lib/std/std.zig -femit-docs -fno-emit-bin --zig-lib-dir ../lib diff --git a/ci/x86_64-linux-debug.sh b/ci/x86_64-linux-debug.sh index 6ee2ce9b89..069dc78657 100755 --- a/ci/x86_64-linux-debug.sh +++ b/ci/x86_64-linux-debug.sh @@ -66,7 +66,7 @@ stage3-debug/bin/zig build test docs \ --zig-lib-dir "$(pwd)/../lib" # Look for HTML errors. -tidy --drop-empty-elements no -qe ../zig-cache/langref.html +tidy --drop-empty-elements no -qe "$ZIG_LOCAL_CACHE_DIR/langref.html" # Produce the experimental std lib documentation. stage3-debug/bin/zig test ../lib/std/std.zig -femit-docs -fno-emit-bin --zig-lib-dir ../lib diff --git a/ci/x86_64-linux-release.sh b/ci/x86_64-linux-release.sh index d9c1172b04..06f9e48c66 100755 --- a/ci/x86_64-linux-release.sh +++ b/ci/x86_64-linux-release.sh @@ -67,7 +67,7 @@ stage3-release/bin/zig build test docs \ --zig-lib-dir "$(pwd)/../lib" # Look for HTML errors. -tidy --drop-empty-elements no -qe ../zig-cache/langref.html +tidy --drop-empty-elements no -qe "$ZIG_LOCAL_CACHE_DIR/langref.html" # Produce the experimental std lib documentation. stage3-release/bin/zig test ../lib/std/std.zig -femit-docs -fno-emit-bin --zig-lib-dir ../lib