From e78d3750c58d26bac0e24c40eb89c2f4796bc15c Mon Sep 17 00:00:00 2001 From: stratact Date: Wed, 18 Sep 2019 23:56:45 -0700 Subject: [PATCH 01/10] Use 8192 sized buffers and remove allocator parameters --- doc/docgen.zig | 2 +- lib/std/build.zig | 4 +-- lib/std/event/fs.zig | 2 +- lib/std/fs.zig | 61 +++++++++++--------------------------- lib/std/os/test.zig | 6 ++-- src-self-hosted/main.zig | 2 +- src-self-hosted/stage1.zig | 2 +- src-self-hosted/test.zig | 4 +-- test/cli.zig | 2 +- tools/process_headers.zig | 2 +- 10 files changed, 31 insertions(+), 56 deletions(-) diff --git a/doc/docgen.zig b/doc/docgen.zig index 73289e5edc..bdc2604213 100644 --- a/doc/docgen.zig +++ b/doc/docgen.zig @@ -51,7 +51,7 @@ pub fn main() !void { var toc = try genToc(allocator, &tokenizer); try fs.makePath(allocator, tmp_dir_name); - defer fs.deleteTree(allocator, tmp_dir_name) catch {}; + defer fs.deleteTree(tmp_dir_name) catch {}; try genHtml(allocator, &tokenizer, &toc, &buffered_out_stream.stream, zig_exe); try buffered_out_stream.flush(); diff --git a/lib/std/build.zig b/lib/std/build.zig index 68d3f2a315..32974da92a 100644 --- a/lib/std/build.zig +++ b/lib/std/build.zig @@ -331,7 +331,7 @@ pub const Builder = struct { if (self.verbose) { warn("rm {}\n", full_path); } - fs.deleteTree(self.allocator, full_path) catch {}; + fs.deleteTree(full_path) catch {}; } // TODO remove empty directories @@ -2671,7 +2671,7 @@ pub const RemoveDirStep = struct { const self = @fieldParentPtr(RemoveDirStep, "step", step); const full_path = self.builder.pathFromRoot(self.dir_path); - fs.deleteTree(self.builder.allocator, full_path) catch |err| { + fs.deleteTree(full_path) catch |err| { warn("Unable to remove {}: {}\n", full_path, @errorName(err)); return err; }; diff --git a/lib/std/event/fs.zig b/lib/std/event/fs.zig index 4490e1deae..81fd950e44 100644 --- a/lib/std/event/fs.zig +++ b/lib/std/event/fs.zig @@ -1312,7 +1312,7 @@ const test_tmp_dir = "std_event_fs_test"; // // // TODO move this into event loop too // try os.makePath(allocator, test_tmp_dir); -// defer os.deleteTree(allocator, test_tmp_dir) catch {}; +// defer os.deleteTree(test_tmp_dir) catch {}; // // var loop: Loop = undefined; // try loop.initMultiThreaded(allocator); diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 287c0316b7..1552f65ecd 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -37,6 +37,8 @@ pub const MAX_PATH_BYTES = switch (builtin.os) { else => @compileError("Unsupported OS"), }; +pub const MAX_BUF_BYTES: usize = 8192; + // here we replace the standard +/ with -_ so that it can be used in a file name const b64_fs_encoder = base64.Base64Encoder.init("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_", base64.standard_pad_char); @@ -371,7 +373,7 @@ const DeleteTreeError = error{ /// this function recursively removes its entries and then tries again. /// TODO determine if we can remove the allocator requirement /// https://github.com/ziglang/zig/issues/2886 -pub fn deleteTree(allocator: *Allocator, full_path: []const u8) DeleteTreeError!void { +pub fn deleteTree(full_path: []const u8) DeleteTreeError!void { start_over: while (true) { var got_access_denied = false; // First, try deleting the item as a file. This way we don't follow sym links. @@ -395,7 +397,7 @@ pub fn deleteTree(allocator: *Allocator, full_path: []const u8) DeleteTreeError! => return err, } { - var dir = Dir.open(allocator, full_path) catch |err| switch (err) { + var dir = Dir.open(full_path) catch |err| switch (err) { error.NotDir => { if (got_access_denied) { return error.AccessDenied; @@ -424,17 +426,14 @@ pub fn deleteTree(allocator: *Allocator, full_path: []const u8) DeleteTreeError! }; defer dir.close(); - var full_entry_buf = std.ArrayList(u8).init(allocator); - defer full_entry_buf.deinit(); - while (try dir.next()) |entry| { - try full_entry_buf.resize(full_path.len + entry.name.len + 1); - const full_entry_path = full_entry_buf.toSlice(); + var full_entry_buf: [MAX_BUF_BYTES]u8 = undefined; + const full_entry_path = full_entry_buf[0..]; mem.copy(u8, full_entry_path, full_path); full_entry_path[full_path.len] = path.sep; mem.copy(u8, full_entry_path[full_path.len + 1 ..], entry.name); - try deleteTree(allocator, full_entry_path); + try deleteTree(full_entry_path[0..full_path.len + entry.name.len + 1]); } } return deleteDir(full_path); @@ -445,19 +444,18 @@ pub fn deleteTree(allocator: *Allocator, full_path: []const u8) DeleteTreeError! /// files, and into the one that reads files from an open directory handle. pub const Dir = struct { handle: Handle, - allocator: *Allocator, pub const Handle = switch (builtin.os) { .macosx, .ios, .freebsd, .netbsd => struct { fd: i32, seek: i64, - buf: []u8, + buf: [MAX_BUF_BYTES]u8, index: usize, end_index: usize, }, .linux => struct { fd: i32, - buf: []u8, + buf: [MAX_BUF_BYTES]u8, index: usize, end_index: usize, }, @@ -512,9 +510,8 @@ pub const Dir = struct { /// Call close when done. /// TODO remove the allocator requirement from this API /// https://github.com/ziglang/zig/issues/2885 - pub fn open(allocator: *Allocator, dir_path: []const u8) OpenError!Dir { + pub fn open(dir_path: []const u8) OpenError!Dir { return Dir{ - .allocator = allocator, .handle = switch (builtin.os) { .windows => blk: { var find_file_data: os.windows.WIN32_FIND_DATAW = undefined; @@ -548,7 +545,6 @@ pub const Dir = struct { if (os.windows.is_the_target) { return os.windows.FindClose(self.handle.handle); } - self.allocator.free(self.handle.buf); os.close(self.handle.fd); } @@ -579,14 +575,10 @@ pub const Dir = struct { fn nextDarwin(self: *Dir) !?Entry { start_over: while (true) { if (self.handle.index >= self.handle.end_index) { - if (self.handle.buf.len == 0) { - self.handle.buf = try self.allocator.alloc(u8, mem.page_size); - } - while (true) { const rc = os.system.__getdirentries64( self.handle.fd, - self.handle.buf.ptr, + self.handle.buf[0..].ptr, self.handle.buf.len, &self.handle.seek, ); @@ -596,10 +588,7 @@ pub const Dir = struct { os.EBADF => unreachable, os.EFAULT => unreachable, os.ENOTDIR => unreachable, - os.EINVAL => { - self.handle.buf = try self.allocator.realloc(self.handle.buf, self.handle.buf.len * 2); - continue; - }, + os.EINVAL => unreachable, else => |err| return os.unexpectedErrno(err), } } @@ -666,21 +655,14 @@ pub const Dir = struct { fn nextLinux(self: *Dir) !?Entry { start_over: while (true) { if (self.handle.index >= self.handle.end_index) { - if (self.handle.buf.len == 0) { - self.handle.buf = try self.allocator.alloc(u8, mem.page_size); - } - while (true) { - const rc = os.linux.getdents64(self.handle.fd, self.handle.buf.ptr, self.handle.buf.len); + const rc = os.linux.getdents64(self.handle.fd, self.handle.buf[0..].ptr, self.handle.buf.len); switch (os.linux.getErrno(rc)) { 0 => {}, os.EBADF => unreachable, os.EFAULT => unreachable, os.ENOTDIR => unreachable, - os.EINVAL => { - self.handle.buf = try self.allocator.realloc(self.handle.buf, self.handle.buf.len * 2); - continue; - }, + os.EINVAL => unreachable, else => |err| return os.unexpectedErrno(err), } if (rc == 0) return null; @@ -720,14 +702,10 @@ pub const Dir = struct { fn nextBsd(self: *Dir) !?Entry { start_over: while (true) { if (self.handle.index >= self.handle.end_index) { - if (self.handle.buf.len == 0) { - self.handle.buf = try self.allocator.alloc(u8, mem.page_size); - } - while (true) { const rc = os.system.getdirentries( self.handle.fd, - self.handle.buf.ptr, + self.handle.buf[0..].ptr, self.handle.buf.len, &self.handle.seek, ); @@ -736,10 +714,7 @@ pub const Dir = struct { os.EBADF => unreachable, os.EFAULT => unreachable, os.ENOTDIR => unreachable, - os.EINVAL => { - self.handle.buf = try self.allocator.realloc(self.handle.buf, self.handle.buf.len * 2); - continue; - }, + os.EINVAL => unreachable, else => |err| return os.unexpectedErrno(err), } if (rc == 0) return null; @@ -807,7 +782,7 @@ pub const Walker = struct { try self.name_buffer.append(base.name); if (base.kind == .Directory) { // TODO https://github.com/ziglang/zig/issues/2888 - var new_dir = try Dir.open(self.stack.allocator, self.name_buffer.toSliceConst()); + var new_dir = try Dir.open(self.name_buffer.toSliceConst()); { errdefer new_dir.close(); try self.stack.append(StackItem{ @@ -841,7 +816,7 @@ pub const Walker = struct { pub fn walkPath(allocator: *Allocator, dir_path: []const u8) !Walker { assert(!mem.endsWith(u8, dir_path, path.sep_str)); - var dir_it = try Dir.open(allocator, dir_path); + var dir_it = try Dir.open(dir_path); errdefer dir_it.close(); var name_buffer = try std.Buffer.init(allocator, dir_path); diff --git a/lib/std/os/test.zig b/lib/std/os/test.zig index 8d6b437ff3..b6e6728142 100644 --- a/lib/std/os/test.zig +++ b/lib/std/os/test.zig @@ -19,8 +19,8 @@ test "makePath, put some files in it, deleteTree" { try fs.makePath(a, "os_test_tmp" ++ fs.path.sep_str ++ "b" ++ fs.path.sep_str ++ "c"); try io.writeFile("os_test_tmp" ++ fs.path.sep_str ++ "b" ++ fs.path.sep_str ++ "c" ++ fs.path.sep_str ++ "file.txt", "nonsense"); try io.writeFile("os_test_tmp" ++ fs.path.sep_str ++ "b" ++ fs.path.sep_str ++ "file2.txt", "blah"); - try fs.deleteTree(a, "os_test_tmp"); - if (fs.Dir.open(a, "os_test_tmp")) |dir| { + try fs.deleteTree("os_test_tmp"); + if (fs.Dir.open("os_test_tmp")) |dir| { @panic("expected error"); } else |err| { expect(err == error.FileNotFound); @@ -37,7 +37,7 @@ test "access file" { try io.writeFile("os_test_tmp" ++ fs.path.sep_str ++ "file.txt", ""); try os.access("os_test_tmp" ++ fs.path.sep_str ++ "file.txt", os.F_OK); - try fs.deleteTree(a, "os_test_tmp"); + try fs.deleteTree("os_test_tmp"); } fn testThreadIdFn(thread_id: *Thread.Id) void { diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 52eda5824a..a2e77394ce 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -747,7 +747,7 @@ async fn fmtPath(fmt: *Fmt, file_path_ref: []const u8, check_mode: bool) FmtErro )) catch |err| switch (err) { error.IsDir, error.AccessDenied => { // TODO make event based (and dir.next()) - var dir = try fs.Dir.open(fmt.loop.allocator, file_path); + var dir = try fs.Dir.open(file_path); defer dir.close(); var group = event.Group(FmtError!void).init(fmt.loop); diff --git a/src-self-hosted/stage1.zig b/src-self-hosted/stage1.zig index 48b14ebcac..d36073abf3 100644 --- a/src-self-hosted/stage1.zig +++ b/src-self-hosted/stage1.zig @@ -283,7 +283,7 @@ fn fmtPath(fmt: *Fmt, file_path_ref: []const u8, check_mode: bool) FmtError!void const source_code = io.readFileAlloc(fmt.allocator, file_path) catch |err| switch (err) { error.IsDir, error.AccessDenied => { // TODO make event based (and dir.next()) - var dir = try fs.Dir.open(fmt.allocator, file_path); + var dir = try fs.Dir.open(file_path); defer dir.close(); while (try dir.next()) |entry| { diff --git a/src-self-hosted/test.zig b/src-self-hosted/test.zig index 526518ca47..ce51de9a7b 100644 --- a/src-self-hosted/test.zig +++ b/src-self-hosted/test.zig @@ -56,11 +56,11 @@ pub const TestContext = struct { errdefer allocator.free(self.zig_lib_dir); try std.fs.makePath(allocator, tmp_dir_name); - errdefer std.fs.deleteTree(allocator, tmp_dir_name) catch {}; + errdefer std.fs.deleteTree(tmp_dir_name) catch {}; } fn deinit(self: *TestContext) void { - std.fs.deleteTree(allocator, tmp_dir_name) catch {}; + std.fs.deleteTree(tmp_dir_name) catch {}; allocator.free(self.zig_lib_dir); self.zig_compiler.deinit(); self.loop.deinit(); diff --git a/test/cli.zig b/test/cli.zig index 63a116a811..2999df14b0 100644 --- a/test/cli.zig +++ b/test/cli.zig @@ -37,7 +37,7 @@ pub fn main() !void { testMissingOutputPath, }; for (test_fns) |testFn| { - try fs.deleteTree(a, dir_path); + try fs.deleteTree(dir_path); try fs.makeDir(dir_path); try testFn(zig_exe, dir_path); } diff --git a/tools/process_headers.zig b/tools/process_headers.zig index 420c118cb9..667da5f7c9 100644 --- a/tools/process_headers.zig +++ b/tools/process_headers.zig @@ -340,7 +340,7 @@ pub fn main() !void { try dir_stack.append(target_include_dir); while (dir_stack.popOrNull()) |full_dir_name| { - var dir = std.fs.Dir.open(allocator, full_dir_name) catch |err| switch (err) { + var dir = std.fs.Dir.open(full_dir_name) catch |err| switch (err) { error.FileNotFound => continue :search, error.AccessDenied => continue :search, else => return err, From 5b1a492012241276a4b7539ca6664234f0629c79 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 20 Oct 2019 21:48:23 -0400 Subject: [PATCH 02/10] breaking: improve std.fs directory handling API * Added `std.c.unlinkat` and `std.os.unlinkat`. * Removed `std.fs.MAX_BUF_BYTES` (this declaration never made it to master branch) * Added `std.fs.Dir.deleteTree` to be used on an open directory handle. * `std.fs.deleteTree` has better behavior for both relative and absolute paths. For absolute paths, it opens the base directory and uses that handle for subsequent operations. For relative paths, it does a similar strategy, using the cwd handle. * The error set of `std.fs.deleteTree` is improved to no longer have these possible errors: - OutOfMemory - FileTooBig - IsDir - DirNotEmpty - PathAlreadyExists - NoSpaceLeft * Added `std.fs.Dir.posix_cwd` which is a statically initialized directory representing the current working directory. * The error set of `std.Dir.open` is improved to no longer have these possible errors: - FileTooBig - IsDir - NoSpaceLeft - PathAlreadyExists - OutOfMemory * Added more alternative functions to `std.fs` for when the path parameter is a null terminated string. This can sometimes be more effecient on systems which have an ABI based on null terminated strings. * Added `std.fs.Dir.openDir`, `std.fs.Dir.deleteFile`, and `std.fs.Dir.deleteDir` which all operate on an open directory handle. * `std.fs.Walker.Entry` now has a `dir` field, which can be used to do operations directly on `std.fs.Walker.Entry.basename`, avoiding `error.NameTooLong` for deeply nested paths. * Added more docs to `std.os.OpenError` This commit does the POSIX components for these changes. I plan to follow up shortly with a commit for Windows. --- lib/std/c.zig | 1 + lib/std/fs.zig | 869 +++++++++++++++++++++---------------- lib/std/os.zig | 56 ++- src-self-hosted/stage1.zig | 6 +- 4 files changed, 563 insertions(+), 369 deletions(-) diff --git a/lib/std/c.zig b/lib/std/c.zig index 9c7ec2f0e5..f591481d04 100644 --- a/lib/std/c.zig +++ b/lib/std/c.zig @@ -80,6 +80,7 @@ pub extern "c" fn mmap(addr: ?*align(page_size) c_void, len: usize, prot: c_uint pub extern "c" fn munmap(addr: *align(page_size) c_void, len: usize) c_int; pub extern "c" fn mprotect(addr: *align(page_size) c_void, len: usize, prot: c_uint) c_int; pub extern "c" fn unlink(path: [*]const u8) c_int; +pub extern "c" fn unlinkat(dirfd: fd_t, path: [*]const u8, flags: c_uint) c_int; pub extern "c" fn getcwd(buf: [*]u8, size: usize) ?[*]u8; pub extern "c" fn waitpid(pid: c_int, stat_loc: *c_uint, options: c_uint) c_int; pub extern "c" fn fork() c_int; diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 1552f65ecd..89071b407f 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -37,8 +37,6 @@ pub const MAX_PATH_BYTES = switch (builtin.os) { else => @compileError("Unsupported OS"), }; -pub const MAX_BUF_BYTES: usize = 8192; - // here we replace the standard +/ with -_ so that it can be used in a file name const b64_fs_encoder = base64.Base64Encoder.init("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_", base64.standard_pad_char); @@ -337,136 +335,35 @@ pub fn deleteDirW(dir_path: [*]const u16) !void { return os.rmdirW(dir_path); } -const DeleteTreeError = error{ - OutOfMemory, - AccessDenied, - FileTooBig, - IsDir, - SymLinkLoop, - ProcessFdQuotaExceeded, - NameTooLong, - SystemFdQuotaExceeded, - NoDevice, - SystemResources, - NoSpaceLeft, - PathAlreadyExists, - ReadOnlyFileSystem, - NotDir, - FileNotFound, - FileSystem, - FileBusy, - DirNotEmpty, - DeviceBusy, +/// Removes a symlink, file, or directory. +/// If `full_path` is relative, this is equivalent to `Dir.deleteTree` with the +/// current working directory as the open directory handle. +/// If `full_path` is absolute, this is equivalent to `Dir.deleteTree` with the +/// base directory. +pub fn deleteTree(full_path: []const u8) !void { + if (path.isAbsolute(full_path)) { + const dirname = path.dirname(full_path) orelse return error{ + /// Attempt to remove the root file system path. + /// This error is unreachable if `full_path` is relative. + CannotDeleteRootDirectory, + }.CannotDeleteRootDirectory; - /// On Windows, file paths must be valid Unicode. - InvalidUtf8, + var dir = try Dir.open(dirname); + defer dir.close(); - /// On Windows, file paths cannot contain these characters: - /// '/', '*', '?', '"', '<', '>', '|' - BadPathName, - - Unexpected, -}; - -/// Whether `full_path` describes a symlink, file, or directory, this function -/// removes it. If it cannot be removed because it is a non-empty directory, -/// this function recursively removes its entries and then tries again. -/// TODO determine if we can remove the allocator requirement -/// https://github.com/ziglang/zig/issues/2886 -pub fn deleteTree(full_path: []const u8) DeleteTreeError!void { - start_over: while (true) { - var got_access_denied = false; - // First, try deleting the item as a file. This way we don't follow sym links. - if (deleteFile(full_path)) { - return; - } else |err| switch (err) { - error.FileNotFound => return, - error.IsDir => {}, - error.AccessDenied => got_access_denied = true, - - error.InvalidUtf8, - error.SymLinkLoop, - error.NameTooLong, - error.SystemResources, - error.ReadOnlyFileSystem, - error.NotDir, - error.FileSystem, - error.FileBusy, - error.BadPathName, - error.Unexpected, - => return err, - } - { - var dir = Dir.open(full_path) catch |err| switch (err) { - error.NotDir => { - if (got_access_denied) { - return error.AccessDenied; - } - continue :start_over; - }, - - error.OutOfMemory, - error.AccessDenied, - error.FileTooBig, - error.IsDir, - error.SymLinkLoop, - error.ProcessFdQuotaExceeded, - error.NameTooLong, - error.SystemFdQuotaExceeded, - error.NoDevice, - error.FileNotFound, - error.SystemResources, - error.NoSpaceLeft, - error.PathAlreadyExists, - error.Unexpected, - error.InvalidUtf8, - error.BadPathName, - error.DeviceBusy, - => return err, - }; - defer dir.close(); - - while (try dir.next()) |entry| { - var full_entry_buf: [MAX_BUF_BYTES]u8 = undefined; - const full_entry_path = full_entry_buf[0..]; - mem.copy(u8, full_entry_path, full_path); - full_entry_path[full_path.len] = path.sep; - mem.copy(u8, full_entry_path[full_path.len + 1 ..], entry.name); - - try deleteTree(full_entry_path[0..full_path.len + entry.name.len + 1]); - } - } - return deleteDir(full_path); + return dir.deleteTree(path.basename(full_path)); + } else { + return Dir.posix_cwd.deleteTree(full_path); } } -/// TODO: separate this API into the one that opens directory handles to then subsequently open -/// files, and into the one that reads files from an open directory handle. pub const Dir = struct { - handle: Handle, + fd: os.fd_t, - pub const Handle = switch (builtin.os) { - .macosx, .ios, .freebsd, .netbsd => struct { - fd: i32, - seek: i64, - buf: [MAX_BUF_BYTES]u8, - index: usize, - end_index: usize, - }, - .linux => struct { - fd: i32, - buf: [MAX_BUF_BYTES]u8, - index: usize, - end_index: usize, - }, - .windows => struct { - handle: os.windows.HANDLE, - find_file_data: os.windows.WIN32_FIND_DATAW, - first: bool, - name_data: [256]u8, - }, - else => @compileError("unimplemented"), - }; + /// An open handle to the current working directory. + /// Closing this directory is safety-checked illegal behavior. + /// Not available on Windows. + pub const posix_cwd = Dir{ .fd = os.AT_FDCWD }; pub const Entry = struct { name: []const u8, @@ -485,269 +382,504 @@ pub const Dir = struct { }; }; + pub const Iterator = switch (builtin.os) { + .macosx, .ios, .freebsd, .netbsd => struct { + dir: Dir, + seek: i64, + buf: [buffer_len]u8, + index: usize, + end_index: usize, + + pub const buffer_len = 8192; + + const Self = @This(); + + /// Memory such as file names referenced in this returned entry becomes invalid + /// with subsequent calls to `next`, as well as when this `Dir` is deinitialized. + pub fn next(self: *Self) !?Entry { + switch (builtin.os) { + .macosx, .ios => return self.nextDarwin(), + .freebsd, .netbsd => return self.nextBsd(), + else => @compileError("unimplemented"), + } + } + + fn nextDarwin(self: *Self) !?Entry { + start_over: while (true) { + if (self.index >= self.end_index) { + while (true) { + const rc = os.system.__getdirentries64( + self.dir.fd, + &self.buf, + self.buf.len, + &self.seek, + ); + if (rc == 0) return null; + if (rc < 0) { + switch (os.errno(rc)) { + os.EBADF => unreachable, + os.EFAULT => unreachable, + os.ENOTDIR => unreachable, + os.EINVAL => unreachable, + else => |err| return os.unexpectedErrno(err), + } + } + self.index = 0; + self.end_index = @intCast(usize, rc); + break; + } + } + const darwin_entry = @ptrCast(*align(1) os.dirent, &self.buf[self.index]); + const next_index = self.index + darwin_entry.d_reclen; + self.index = next_index; + + const name = @ptrCast([*]u8, &darwin_entry.d_name)[0..darwin_entry.d_namlen]; + + if (mem.eql(u8, name, ".") or mem.eql(u8, name, "..")) { + continue :start_over; + } + + const entry_kind = switch (darwin_entry.d_type) { + os.DT_BLK => Entry.Kind.BlockDevice, + os.DT_CHR => Entry.Kind.CharacterDevice, + os.DT_DIR => Entry.Kind.Directory, + os.DT_FIFO => Entry.Kind.NamedPipe, + os.DT_LNK => Entry.Kind.SymLink, + os.DT_REG => Entry.Kind.File, + os.DT_SOCK => Entry.Kind.UnixDomainSocket, + os.DT_WHT => Entry.Kind.Whiteout, + else => Entry.Kind.Unknown, + }; + return Entry{ + .name = name, + .kind = entry_kind, + }; + } + } + + fn nextBsd(self: *Self) !?Entry { + start_over: while (true) { + if (self.index >= self.end_index) { + while (true) { + const rc = os.system.getdirentries( + self.dir.fd, + self.buf[0..].ptr, + self.buf.len, + &self.seek, + ); + switch (os.errno(rc)) { + 0 => {}, + os.EBADF => unreachable, + os.EFAULT => unreachable, + os.ENOTDIR => unreachable, + os.EINVAL => unreachable, + else => |err| return os.unexpectedErrno(err), + } + if (rc == 0) return null; + self.index = 0; + self.end_index = @intCast(usize, rc); + break; + } + } + const freebsd_entry = @ptrCast(*align(1) os.dirent, &self.buf[self.index]); + const next_index = self.index + freebsd_entry.d_reclen; + self.index = next_index; + + const name = @ptrCast([*]u8, &freebsd_entry.d_name)[0..freebsd_entry.d_namlen]; + + if (mem.eql(u8, name, ".") or mem.eql(u8, name, "..")) { + continue :start_over; + } + + const entry_kind = switch (freebsd_entry.d_type) { + os.DT_BLK => Entry.Kind.BlockDevice, + os.DT_CHR => Entry.Kind.CharacterDevice, + os.DT_DIR => Entry.Kind.Directory, + os.DT_FIFO => Entry.Kind.NamedPipe, + os.DT_LNK => Entry.Kind.SymLink, + os.DT_REG => Entry.Kind.File, + os.DT_SOCK => Entry.Kind.UnixDomainSocket, + os.DT_WHT => Entry.Kind.Whiteout, + else => Entry.Kind.Unknown, + }; + return Entry{ + .name = name, + .kind = entry_kind, + }; + } + } + }, + .linux => struct { + dir: Dir, + buf: [buffer_len]u8, + index: usize, + end_index: usize, + + pub const buffer_len = 8192; + + const Self = @This(); + + /// Memory such as file names referenced in this returned entry becomes invalid + /// with subsequent calls to `next`, as well as when this `Dir` is deinitialized. + pub fn next(self: *Self) !?Entry { + start_over: while (true) { + if (self.index >= self.end_index) { + while (true) { + const rc = os.linux.getdents64(self.dir.fd, &self.buf, self.buf.len); + switch (os.linux.getErrno(rc)) { + 0 => {}, + os.EBADF => unreachable, + os.EFAULT => unreachable, + os.ENOTDIR => unreachable, + os.EINVAL => unreachable, + else => |err| return os.unexpectedErrno(err), + } + if (rc == 0) return null; + self.index = 0; + self.end_index = rc; + break; + } + } + const linux_entry = @ptrCast(*align(1) os.dirent64, &self.buf[self.index]); + const next_index = self.index + linux_entry.d_reclen; + self.index = next_index; + + const name = mem.toSlice(u8, @ptrCast([*]u8, &linux_entry.d_name)); + + // skip . and .. entries + if (mem.eql(u8, name, ".") or mem.eql(u8, name, "..")) { + continue :start_over; + } + + const entry_kind = switch (linux_entry.d_type) { + os.DT_BLK => Entry.Kind.BlockDevice, + os.DT_CHR => Entry.Kind.CharacterDevice, + os.DT_DIR => Entry.Kind.Directory, + os.DT_FIFO => Entry.Kind.NamedPipe, + os.DT_LNK => Entry.Kind.SymLink, + os.DT_REG => Entry.Kind.File, + os.DT_SOCK => Entry.Kind.UnixDomainSocket, + else => Entry.Kind.Unknown, + }; + return Entry{ + .name = name, + .kind = entry_kind, + }; + } + } + }, + .windows => struct { + dir: Dir, + find_file_data: os.windows.WIN32_FIND_DATAW, + first: bool, + name_data: [256]u8, + }, + else => @compileError("unimplemented"), + }; + pub const OpenError = error{ FileNotFound, NotDir, AccessDenied, - FileTooBig, - IsDir, SymLinkLoop, ProcessFdQuotaExceeded, NameTooLong, SystemFdQuotaExceeded, NoDevice, SystemResources, - NoSpaceLeft, - PathAlreadyExists, - OutOfMemory, InvalidUtf8, BadPathName, DeviceBusy, + } || os.UnexpectedError; - Unexpected, - }; - - /// Call close when done. - /// TODO remove the allocator requirement from this API - /// https://github.com/ziglang/zig/issues/2885 + /// Call `close` to free the directory handle. pub fn open(dir_path: []const u8) OpenError!Dir { - return Dir{ - .handle = switch (builtin.os) { - .windows => blk: { - var find_file_data: os.windows.WIN32_FIND_DATAW = undefined; - const handle = try os.windows.FindFirstFile(dir_path, &find_file_data); - break :blk Handle{ - .handle = handle, - .find_file_data = find_file_data, // TODO guaranteed copy elision - .first = true, - .name_data = undefined, - }; - }, - .macosx, .ios, .freebsd, .netbsd => Handle{ - .fd = try os.open(dir_path, os.O_RDONLY | os.O_NONBLOCK | os.O_DIRECTORY | os.O_CLOEXEC, 0), - .seek = 0, - .index = 0, - .end_index = 0, - .buf = [_]u8{}, - }, - .linux => Handle{ - .fd = try os.open(dir_path, os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC, 0), - .index = 0, - .end_index = 0, - .buf = [_]u8{}, - }, - else => @compileError("unimplemented"), - }, - }; + return posix_cwd.openDir(dir_path); + } + + /// Same as `open` except the parameter is null-terminated. + pub fn openC(dir_path_c: [*]const u8) OpenError!Dir { + return posix_cwd.openDirC(dir_path_c); } pub fn close(self: *Dir) void { if (os.windows.is_the_target) { - return os.windows.FindClose(self.handle.handle); + @panic("TODO"); } - os.close(self.handle.fd); + os.close(self.fd); + self.* = undefined; } - /// Memory such as file names referenced in this returned entry becomes invalid - /// with subsequent calls to next, as well as when this `Dir` is deinitialized. - pub fn next(self: *Dir) !?Entry { + /// Call `File.close` on the result when done. + pub fn openRead(self: Dir, sub_path: []const u8) File.OpenError!File { + const path_c = try os.toPosixPath(sub_path); + return self.openReadC(&path_c); + } + + /// Call `File.close` on the result when done. + pub fn openReadC(self: Dir, sub_path: [*]const u8) File.OpenError!File { + const flags = os.O_LARGEFILE | os.O_RDONLY; + const fd = try os.openatC(self.fd, sub_path, flags, 0); + return File.openHandle(fd); + } + + /// Call `close` on the result when done. + pub fn openDir(self: Dir, sub_path: []const u8) OpenError!Dir { + const sub_path_c = try os.toPosixPath(sub_path); + return self.openDirC(&sub_path_c); + } + + /// Call `close` on the result when done. + pub fn openDirC(self: Dir, sub_path: [*]const u8) OpenError!Dir { + const flags = os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC; + const fd = os.openatC(self.fd, sub_path, flags, 0) catch |err| switch (err) { + error.FileTooBig => unreachable, // can't happen for directories + error.IsDir => unreachable, // we're providing O_DIRECTORY + error.NoSpaceLeft => unreachable, // not providing O_CREAT + error.PathAlreadyExists => unreachable, // not providing O_CREAT + else => |e| return e, + }; + return Dir{ .fd = fd }; + } + + pub const DeleteFileError = os.UnlinkError; + + /// Delete a file name and possibly the file it refers to, based on an open directory handle. + pub fn deleteFile(self: Dir, sub_path: []const u8) DeleteFileError!void { + const sub_path_c = try os.toPosixPath(sub_path); + return self.deleteFileC(&sub_path_c); + } + + /// Same as `deleteFile` except the parameter is null-terminated. + pub fn deleteFileC(self: Dir, sub_path_c: [*]const u8) DeleteFileError!void { + os.unlinkatC(self.fd, sub_path_c, 0) catch |err| switch (err) { + error.DirNotEmpty => unreachable, // not passing AT_REMOVEDIR + else => |e| return e, + }; + } + + pub const DeleteDirError = error{ + DirNotEmpty, + FileNotFound, + AccessDenied, + FileBusy, + FileSystem, + SymLinkLoop, + NameTooLong, + NotDir, + SystemResources, + ReadOnlyFileSystem, + InvalidUtf8, + BadPathName, + Unexpected, + }; + + /// Returns `error.DirNotEmpty` if the directory is not empty. + /// To delete a directory recursively, see `deleteTree`. + pub fn deleteDir(self: Dir, sub_path: []const u8) DeleteDirError!void { + const sub_path_c = try os.toPosixPath(sub_path); + return self.deleteDirC(&sub_path_c); + } + + /// Same as `deleteDir` except the parameter is null-terminated. + pub fn deleteDirC(self: Dir, sub_path_c: [*]const u8) DeleteDirError!void { + os.unlinkatC(self.fd, sub_path_c, os.AT_REMOVEDIR) catch |err| switch (err) { + error.IsDir => unreachable, // not possible since we pass AT_REMOVEDIR + else => |e| return e, + }; + } + + pub fn iterate(self: Dir) Iterator { switch (builtin.os) { - .linux => return self.nextLinux(), - .macosx, .ios => return self.nextDarwin(), - .windows => return self.nextWindows(), - .freebsd => return self.nextBsd(), - .netbsd => return self.nextBsd(), + .macosx, .ios, .freebsd, .netbsd => return Iterator{ + .dir = self, + .seek = 0, + .index = 0, + .end_index = 0, + .buf = undefined, + }, + .linux => return Iterator{ + .dir = self, + .index = 0, + .end_index = 0, + .buf = undefined, + }, + .windows => @panic("TODO"), else => @compileError("unimplemented"), } } - pub fn openRead(self: Dir, file_path: []const u8) os.OpenError!File { - const path_c = try os.toPosixPath(file_path); - return self.openReadC(&path_c); - } + pub const DeleteTreeError = error{ + AccessDenied, + FileTooBig, + SymLinkLoop, + ProcessFdQuotaExceeded, + NameTooLong, + SystemFdQuotaExceeded, + NoDevice, + SystemResources, + ReadOnlyFileSystem, + FileSystem, + FileBusy, + DeviceBusy, - pub fn openReadC(self: Dir, file_path: [*]const u8) OpenError!File { - const flags = os.O_LARGEFILE | os.O_RDONLY; - const fd = try os.openatC(self.handle.fd, file_path, flags, 0); - return File.openHandle(fd); - } + /// One of the path components was not a directory. + /// This error is unreachable if `sub_path` does not contain a path separator. + NotDir, - fn nextDarwin(self: *Dir) !?Entry { + /// On Windows, file paths must be valid Unicode. + InvalidUtf8, + + /// On Windows, file paths cannot contain these characters: + /// '/', '*', '?', '"', '<', '>', '|' + BadPathName, + } || os.UnexpectedError; + + /// Whether `full_path` describes a symlink, file, or directory, this function + /// removes it. If it cannot be removed because it is a non-empty directory, + /// this function recursively removes its entries and then tries again. + /// This operation is not atomic on most file systems. + pub fn deleteTree(self: Dir, sub_path: []const u8) DeleteTreeError!void { start_over: while (true) { - if (self.handle.index >= self.handle.end_index) { - while (true) { - const rc = os.system.__getdirentries64( - self.handle.fd, - self.handle.buf[0..].ptr, - self.handle.buf.len, - &self.handle.seek, - ); - if (rc == 0) return null; - if (rc < 0) { - switch (os.errno(rc)) { - os.EBADF => unreachable, - os.EFAULT => unreachable, - os.ENOTDIR => unreachable, - os.EINVAL => unreachable, - else => |err| return os.unexpectedErrno(err), - } + var got_access_denied = false; + // First, try deleting the item as a file. This way we don't follow sym links. + if (self.deleteFile(sub_path)) { + return; + } else |err| switch (err) { + error.FileNotFound => return, + error.IsDir => {}, + error.AccessDenied => got_access_denied = true, + + error.InvalidUtf8, + error.SymLinkLoop, + error.NameTooLong, + error.SystemResources, + error.ReadOnlyFileSystem, + error.NotDir, + error.FileSystem, + error.FileBusy, + error.BadPathName, + error.Unexpected, + => |e| return e, + } + var dir = self.openDir(sub_path) catch |err| switch (err) { + error.NotDir => { + if (got_access_denied) { + return error.AccessDenied; } - self.handle.index = 0; - self.handle.end_index = @intCast(usize, rc); - break; + continue :start_over; + }, + error.FileNotFound => { + // That's fine, we were trying to remove this directory anyway. + continue :start_over; + }, + + error.AccessDenied, + error.SymLinkLoop, + error.ProcessFdQuotaExceeded, + error.NameTooLong, + error.SystemFdQuotaExceeded, + error.NoDevice, + error.SystemResources, + error.Unexpected, + error.InvalidUtf8, + error.BadPathName, + error.DeviceBusy, + => |e| return e, + }; + var cleanup_dir_parent: ?Dir = null; + defer if (cleanup_dir_parent) |*d| d.close(); + + var cleanup_dir = true; + defer if (cleanup_dir) dir.close(); + + var dir_name_buf: [MAX_PATH_BYTES]u8 = undefined; + var dir_name: []const u8 = sub_path; + var parent_dir = self; + + // Here we must avoid recursion, in order to provide O(1) memory guarantee of this function. + // Go through each entry and if it is not a directory, delete it. If it is a directory, + // open it, and close the original directory. Repeat. Then start the entire operation over. + + scan_dir: while (true) { + var dir_it = dir.iterate(); + while (try dir_it.next()) |entry| { + if (dir.deleteFile(entry.name)) { + continue; + } else |err| switch (err) { + error.FileNotFound => continue, + + // Impossible because we do not pass any path separators. + error.NotDir => unreachable, + + error.IsDir => {}, + error.AccessDenied => got_access_denied = true, + + error.InvalidUtf8, + error.SymLinkLoop, + error.NameTooLong, + error.SystemResources, + error.ReadOnlyFileSystem, + error.FileSystem, + error.FileBusy, + error.BadPathName, + error.Unexpected, + => |e| return e, + } + + const new_dir = dir.openDir(entry.name) catch |err| switch (err) { + error.NotDir => { + if (got_access_denied) { + return error.AccessDenied; + } + continue :scan_dir; + }, + error.FileNotFound => { + // That's fine, we were trying to remove this directory anyway. + continue :scan_dir; + }, + + error.AccessDenied, + error.SymLinkLoop, + error.ProcessFdQuotaExceeded, + error.NameTooLong, + error.SystemFdQuotaExceeded, + error.NoDevice, + error.SystemResources, + error.Unexpected, + error.InvalidUtf8, + error.BadPathName, + error.DeviceBusy, + => |e| return e, + }; + if (cleanup_dir_parent) |*d| d.close(); + cleanup_dir_parent = dir; + dir = new_dir; + mem.copy(u8, &dir_name_buf, entry.name); + dir_name = dir_name_buf[0..entry.name.len]; + continue :scan_dir; + } + // Reached the end of the directory entries, which means we successfully deleted all of them. + // Now to remove the directory itself. + dir.close(); + cleanup_dir = false; + + if (cleanup_dir_parent) |d| { + d.deleteDir(dir_name) catch |err| switch (err) { + // These two things can happen due to file system race conditions. + error.FileNotFound, error.DirNotEmpty => continue :start_over, + else => |e| return e, + }; + continue :start_over; + } else { + self.deleteDir(sub_path) catch |err| switch (err) { + error.FileNotFound => return, + error.DirNotEmpty => continue :start_over, + else => |e| return e, + }; + return; } } - const darwin_entry = @ptrCast(*align(1) os.dirent, &self.handle.buf[self.handle.index]); - const next_index = self.handle.index + darwin_entry.d_reclen; - self.handle.index = next_index; - - const name = @ptrCast([*]u8, &darwin_entry.d_name)[0..darwin_entry.d_namlen]; - - if (mem.eql(u8, name, ".") or mem.eql(u8, name, "..")) { - continue :start_over; - } - - const entry_kind = switch (darwin_entry.d_type) { - os.DT_BLK => Entry.Kind.BlockDevice, - os.DT_CHR => Entry.Kind.CharacterDevice, - os.DT_DIR => Entry.Kind.Directory, - os.DT_FIFO => Entry.Kind.NamedPipe, - os.DT_LNK => Entry.Kind.SymLink, - os.DT_REG => Entry.Kind.File, - os.DT_SOCK => Entry.Kind.UnixDomainSocket, - os.DT_WHT => Entry.Kind.Whiteout, - else => Entry.Kind.Unknown, - }; - return Entry{ - .name = name, - .kind = entry_kind, - }; - } - } - - fn nextWindows(self: *Dir) !?Entry { - while (true) { - if (self.handle.first) { - self.handle.first = false; - } else { - if (!try os.windows.FindNextFile(self.handle.handle, &self.handle.find_file_data)) - return null; - } - const name_utf16le = mem.toSlice(u16, self.handle.find_file_data.cFileName[0..].ptr); - if (mem.eql(u16, name_utf16le, [_]u16{'.'}) or mem.eql(u16, name_utf16le, [_]u16{ '.', '.' })) - continue; - // Trust that Windows gives us valid UTF-16LE - const name_utf8_len = std.unicode.utf16leToUtf8(self.handle.name_data[0..], name_utf16le) catch unreachable; - const name_utf8 = self.handle.name_data[0..name_utf8_len]; - const kind = blk: { - const attrs = self.handle.find_file_data.dwFileAttributes; - if (attrs & os.windows.FILE_ATTRIBUTE_DIRECTORY != 0) break :blk Entry.Kind.Directory; - if (attrs & os.windows.FILE_ATTRIBUTE_REPARSE_POINT != 0) break :blk Entry.Kind.SymLink; - break :blk Entry.Kind.File; - }; - return Entry{ - .name = name_utf8, - .kind = kind, - }; - } - } - - fn nextLinux(self: *Dir) !?Entry { - start_over: while (true) { - if (self.handle.index >= self.handle.end_index) { - while (true) { - const rc = os.linux.getdents64(self.handle.fd, self.handle.buf[0..].ptr, self.handle.buf.len); - switch (os.linux.getErrno(rc)) { - 0 => {}, - os.EBADF => unreachable, - os.EFAULT => unreachable, - os.ENOTDIR => unreachable, - os.EINVAL => unreachable, - else => |err| return os.unexpectedErrno(err), - } - if (rc == 0) return null; - self.handle.index = 0; - self.handle.end_index = rc; - break; - } - } - const linux_entry = @ptrCast(*align(1) os.dirent64, &self.handle.buf[self.handle.index]); - const next_index = self.handle.index + linux_entry.d_reclen; - self.handle.index = next_index; - - const name = mem.toSlice(u8, @ptrCast([*]u8, &linux_entry.d_name)); - - // skip . and .. entries - if (mem.eql(u8, name, ".") or mem.eql(u8, name, "..")) { - continue :start_over; - } - - const entry_kind = switch (linux_entry.d_type) { - os.DT_BLK => Entry.Kind.BlockDevice, - os.DT_CHR => Entry.Kind.CharacterDevice, - os.DT_DIR => Entry.Kind.Directory, - os.DT_FIFO => Entry.Kind.NamedPipe, - os.DT_LNK => Entry.Kind.SymLink, - os.DT_REG => Entry.Kind.File, - os.DT_SOCK => Entry.Kind.UnixDomainSocket, - else => Entry.Kind.Unknown, - }; - return Entry{ - .name = name, - .kind = entry_kind, - }; - } - } - - fn nextBsd(self: *Dir) !?Entry { - start_over: while (true) { - if (self.handle.index >= self.handle.end_index) { - while (true) { - const rc = os.system.getdirentries( - self.handle.fd, - self.handle.buf[0..].ptr, - self.handle.buf.len, - &self.handle.seek, - ); - switch (os.errno(rc)) { - 0 => {}, - os.EBADF => unreachable, - os.EFAULT => unreachable, - os.ENOTDIR => unreachable, - os.EINVAL => unreachable, - else => |err| return os.unexpectedErrno(err), - } - if (rc == 0) return null; - self.handle.index = 0; - self.handle.end_index = @intCast(usize, rc); - break; - } - } - const freebsd_entry = @ptrCast(*align(1) os.dirent, &self.handle.buf[self.handle.index]); - const next_index = self.handle.index + freebsd_entry.d_reclen; - self.handle.index = next_index; - - const name = @ptrCast([*]u8, &freebsd_entry.d_name)[0..freebsd_entry.d_namlen]; - - if (mem.eql(u8, name, ".") or mem.eql(u8, name, "..")) { - continue :start_over; - } - - const entry_kind = switch (freebsd_entry.d_type) { - os.DT_BLK => Entry.Kind.BlockDevice, - os.DT_CHR => Entry.Kind.CharacterDevice, - os.DT_DIR => Entry.Kind.Directory, - os.DT_FIFO => Entry.Kind.NamedPipe, - os.DT_LNK => Entry.Kind.SymLink, - os.DT_REG => Entry.Kind.File, - os.DT_SOCK => Entry.Kind.UnixDomainSocket, - os.DT_WHT => Entry.Kind.Whiteout, - else => Entry.Kind.Unknown, - }; - return Entry{ - .name = name, - .kind = entry_kind, - }; } } }; @@ -757,13 +889,18 @@ pub const Walker = struct { name_buffer: std.Buffer, pub const Entry = struct { - path: []const u8, + /// The containing directory. This can be used to operate directly on `basename` + /// rather than `path`, avoiding `error.NameTooLong` for deeply nested paths. + /// The directory remains open until `next` or `deinit` is called. + dir: Dir, basename: []const u8, + + path: []const u8, kind: Dir.Entry.Kind, }; const StackItem = struct { - dir_it: Dir, + dir_it: Dir.Iterator, dirname_len: usize, }; @@ -781,23 +918,26 @@ pub const Walker = struct { try self.name_buffer.appendByte(path.sep); try self.name_buffer.append(base.name); if (base.kind == .Directory) { - // TODO https://github.com/ziglang/zig/issues/2888 - var new_dir = try Dir.open(self.name_buffer.toSliceConst()); + var new_dir = top.dir_it.dir.openDir(base.name) catch |err| switch (err) { + error.NameTooLong => unreachable, // no path sep in base.name + else => |e| return e, + }; { errdefer new_dir.close(); try self.stack.append(StackItem{ - .dir_it = new_dir, + .dir_it = new_dir.iterate(), .dirname_len = self.name_buffer.len(), }); } } return Entry{ + .dir = top.dir_it.dir, .basename = self.name_buffer.toSliceConst()[dirname_len + 1 ..], .path = self.name_buffer.toSliceConst(), .kind = base.kind, }; } else { - self.stack.pop().dir_it.close(); + self.stack.pop().dir_it.dir.close(); } } } @@ -812,12 +952,13 @@ pub const Walker = struct { /// Recursively iterates over a directory. /// Must call `Walker.deinit` when done. /// `dir_path` must not end in a path separator. +/// The order of returned file system entries is undefined. /// TODO: https://github.com/ziglang/zig/issues/2888 pub fn walkPath(allocator: *Allocator, dir_path: []const u8) !Walker { assert(!mem.endsWith(u8, dir_path, path.sep_str)); - var dir_it = try Dir.open(dir_path); - errdefer dir_it.close(); + var dir = try Dir.open(dir_path); + errdefer dir.close(); var name_buffer = try std.Buffer.init(allocator, dir_path); errdefer name_buffer.deinit(); @@ -828,7 +969,7 @@ pub fn walkPath(allocator: *Allocator, dir_path: []const u8) !Walker { }; try walker.stack.append(Walker.StackItem{ - .dir_it = dir_it, + .dir_it = dir.iterate(), .dirname_len = dir_path.len, }); diff --git a/lib/std/os.zig b/lib/std/os.zig index de01da2fa5..e7319095cb 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -529,22 +529,36 @@ pub fn pwritev(fd: fd_t, iov: []const iovec_const, offset: u64) WriteError!void pub const OpenError = error{ AccessDenied, - FileTooBig, - IsDir, SymLinkLoop, ProcessFdQuotaExceeded, - NameTooLong, SystemFdQuotaExceeded, NoDevice, FileNotFound, + /// The path exceeded `MAX_PATH_BYTES` bytes. + NameTooLong, + /// Insufficient kernel memory was available, or /// the named file is a FIFO and per-user hard limit on /// memory allocation for pipes has been reached. SystemResources, + /// The file is too large to be opened. This error is unreachable + /// for 64-bit targets, as well as when opening directories. + FileTooBig, + + /// The path refers to directory but the `O_DIRECTORY` flag was not provided. + IsDir, + + /// A new path cannot be created because the device has no room for the new file. + /// This error is only reachable when the `O_CREAT` flag is provided. NoSpaceLeft, + + /// A component used as a directory in the path was not, in fact, a directory, or + /// `O_DIRECTORY` was specified and the path was not a directory. NotDir, + + /// The path already exists and the `O_CREAT` and `O_EXCL` flags were provided. PathAlreadyExists, DeviceBusy, } || UnexpectedError; @@ -978,6 +992,42 @@ pub fn unlinkC(file_path: [*]const u8) UnlinkError!void { } } +pub const UnlinkatError = UnlinkError || error{ + /// When passing `AT_REMOVEDIR`, this error occurs when the named directory is not empty. + DirNotEmpty, +}; + +/// Delete a file name and possibly the file it refers to, based on an open directory handle. +pub fn unlinkat(dirfd: fd_t, file_path: []const u8, flags: u32) UnlinkatError!void { + const file_path_c = try toPosixPath(file_path); + return unlinkatC(dirfd, &file_path_c, flags); +} + +/// Same as `unlinkat` but `file_path` is a null-terminated string. +pub fn unlinkatC(dirfd: fd_t, file_path_c: [*]const u8, flags: u32) UnlinkatError!void { + switch (errno(system.unlinkat(dirfd, file_path_c, flags))) { + 0 => return, + EACCES => return error.AccessDenied, + EPERM => return error.AccessDenied, + EBUSY => return error.FileBusy, + EFAULT => unreachable, + EIO => return error.FileSystem, + EISDIR => return error.IsDir, + ELOOP => return error.SymLinkLoop, + ENAMETOOLONG => return error.NameTooLong, + ENOENT => return error.FileNotFound, + ENOTDIR => return error.NotDir, + ENOMEM => return error.SystemResources, + EROFS => return error.ReadOnlyFileSystem, + ENOTEMPTY => return error.DirNotEmpty, + + EINVAL => unreachable, // invalid flags, or pathname has . as last component + EBADF => unreachable, // always a race condition + + else => |err| return unexpectedErrno(err), + } +} + const RenameError = error{ AccessDenied, FileBusy, diff --git a/src-self-hosted/stage1.zig b/src-self-hosted/stage1.zig index d36073abf3..5a0294fa7d 100644 --- a/src-self-hosted/stage1.zig +++ b/src-self-hosted/stage1.zig @@ -286,8 +286,10 @@ fn fmtPath(fmt: *Fmt, file_path_ref: []const u8, check_mode: bool) FmtError!void var dir = try fs.Dir.open(file_path); defer dir.close(); - while (try dir.next()) |entry| { - if (entry.kind == fs.Dir.Entry.Kind.Directory or mem.endsWith(u8, entry.name, ".zig")) { + var dir_it = dir.iterate(); + + while (try dir_it.next()) |entry| { + if (entry.kind == .Directory or mem.endsWith(u8, entry.name, ".zig")) { const full_path = try fs.path.join(fmt.allocator, [_][]const u8{ file_path, entry.name }); try fmtPath(fmt, full_path, check_mode); } From ef67c497856fb97f7d2854991a72237ad4332810 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Oct 2019 14:18:01 -0400 Subject: [PATCH 03/10] [wip] use NtDll APIs on Windows to implement std.fs.Dir --- lib/std/fs.zig | 365 +++++++++++++++++++++++++--------- lib/std/fs/file.zig | 1 + lib/std/fs/path.zig | 19 ++ lib/std/io.zig | 5 +- lib/std/os.zig | 79 ++++++++ lib/std/os/bits/windows.zig | 3 + lib/std/os/windows.zig | 23 ++- lib/std/os/windows/bits.zig | 146 +++++++++++++- lib/std/os/windows/ntdll.zig | 25 ++- lib/std/os/windows/status.zig | 6 + 10 files changed, 566 insertions(+), 106 deletions(-) diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 89071b407f..51623d3057 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -353,18 +353,13 @@ pub fn deleteTree(full_path: []const u8) !void { return dir.deleteTree(path.basename(full_path)); } else { - return Dir.posix_cwd.deleteTree(full_path); + return Dir.cwd().deleteTree(full_path); } } pub const Dir = struct { fd: os.fd_t, - /// An open handle to the current working directory. - /// Closing this directory is safety-checked illegal behavior. - /// Not available on Windows. - pub const posix_cwd = Dir{ .fd = os.AT_FDCWD }; - pub const Entry = struct { name: []const u8, kind: Kind, @@ -386,12 +381,10 @@ pub const Dir = struct { .macosx, .ios, .freebsd, .netbsd => struct { dir: Dir, seek: i64, - buf: [buffer_len]u8, + buf: [8192]u8, // TODO align(@alignOf(os.dirent)), index: usize, end_index: usize, - pub const buffer_len = 8192; - const Self = @This(); /// Memory such as file names referenced in this returned entry becomes invalid @@ -407,27 +400,24 @@ pub const Dir = struct { fn nextDarwin(self: *Self) !?Entry { start_over: while (true) { if (self.index >= self.end_index) { - while (true) { - const rc = os.system.__getdirentries64( - self.dir.fd, - &self.buf, - self.buf.len, - &self.seek, - ); - if (rc == 0) return null; - if (rc < 0) { - switch (os.errno(rc)) { - os.EBADF => unreachable, - os.EFAULT => unreachable, - os.ENOTDIR => unreachable, - os.EINVAL => unreachable, - else => |err| return os.unexpectedErrno(err), - } + const rc = os.system.__getdirentries64( + self.dir.fd, + &self.buf, + self.buf.len, + &self.seek, + ); + if (rc == 0) return null; + if (rc < 0) { + switch (os.errno(rc)) { + os.EBADF => unreachable, + os.EFAULT => unreachable, + os.ENOTDIR => unreachable, + os.EINVAL => unreachable, + else => |err| return os.unexpectedErrno(err), } - self.index = 0; - self.end_index = @intCast(usize, rc); - break; } + self.index = 0; + self.end_index = @intCast(usize, rc); } const darwin_entry = @ptrCast(*align(1) os.dirent, &self.buf[self.index]); const next_index = self.index + darwin_entry.d_reclen; @@ -460,26 +450,23 @@ pub const Dir = struct { fn nextBsd(self: *Self) !?Entry { start_over: while (true) { if (self.index >= self.end_index) { - while (true) { - const rc = os.system.getdirentries( - self.dir.fd, - self.buf[0..].ptr, - self.buf.len, - &self.seek, - ); - switch (os.errno(rc)) { - 0 => {}, - os.EBADF => unreachable, - os.EFAULT => unreachable, - os.ENOTDIR => unreachable, - os.EINVAL => unreachable, - else => |err| return os.unexpectedErrno(err), - } - if (rc == 0) return null; - self.index = 0; - self.end_index = @intCast(usize, rc); - break; + const rc = os.system.getdirentries( + self.dir.fd, + self.buf[0..].ptr, + self.buf.len, + &self.seek, + ); + switch (os.errno(rc)) { + 0 => {}, + os.EBADF => unreachable, + os.EFAULT => unreachable, + os.ENOTDIR => unreachable, + os.EINVAL => unreachable, + else => |err| return os.unexpectedErrno(err), } + if (rc == 0) return null; + self.index = 0; + self.end_index = @intCast(usize, rc); } const freebsd_entry = @ptrCast(*align(1) os.dirent, &self.buf[self.index]); const next_index = self.index + freebsd_entry.d_reclen; @@ -511,12 +498,10 @@ pub const Dir = struct { }, .linux => struct { dir: Dir, - buf: [buffer_len]u8, + buf: [8192]u8, // TODO align(@alignOf(os.dirent64)), index: usize, end_index: usize, - pub const buffer_len = 8192; - const Self = @This(); /// Memory such as file names referenced in this returned entry becomes invalid @@ -524,21 +509,18 @@ pub const Dir = struct { pub fn next(self: *Self) !?Entry { start_over: while (true) { if (self.index >= self.end_index) { - while (true) { - const rc = os.linux.getdents64(self.dir.fd, &self.buf, self.buf.len); - switch (os.linux.getErrno(rc)) { - 0 => {}, - os.EBADF => unreachable, - os.EFAULT => unreachable, - os.ENOTDIR => unreachable, - os.EINVAL => unreachable, - else => |err| return os.unexpectedErrno(err), - } - if (rc == 0) return null; - self.index = 0; - self.end_index = rc; - break; + const rc = os.linux.getdents64(self.dir.fd, &self.buf, self.buf.len); + switch (os.linux.getErrno(rc)) { + 0 => {}, + os.EBADF => unreachable, + os.EFAULT => unreachable, + os.ENOTDIR => unreachable, + os.EINVAL => unreachable, + else => |err| return os.unexpectedErrno(err), } + if (rc == 0) return null; + self.index = 0; + self.end_index = rc; } const linux_entry = @ptrCast(*align(1) os.dirent64, &self.buf[self.index]); const next_index = self.index + linux_entry.d_reclen; @@ -570,13 +552,117 @@ pub const Dir = struct { }, .windows => struct { dir: Dir, - find_file_data: os.windows.WIN32_FIND_DATAW, + buf: [8192]u8 align(@alignOf(os.windows.FILE_BOTH_DIR_INFORMATION)), + index: usize, + end_index: usize, first: bool, name_data: [256]u8, + + const Self = @This(); + + pub fn next(self: *Self) !?Entry { + start_over: while (true) { + const w = os.windows; + if (self.index >= self.end_index) { + var io: w.IO_STATUS_BLOCK = undefined; + //var mask_buf = [2]u16{ 'a', 0 }; + //var mask = w.UNICODE_STRING{ + // .Length = 2, + // .MaximumLength = 2, + // .Buffer = &mask_buf, + //}; + const rc = w.ntdll.NtQueryDirectoryFile( + self.dir.fd, + null, + null, + null, + &io, + &self.buf, + self.buf.len, + .FileBothDirectoryInformation, + w.FALSE, + null, + if (self.first) w.BOOLEAN(w.TRUE) else w.BOOLEAN(w.FALSE), + ); + self.first = false; + if (io.Information == 0) return null; + self.index = 0; + self.end_index = io.Information; + switch (rc) { + w.STATUS.SUCCESS => {}, + else => return w.unexpectedStatus(rc), + } + } + + const aligned_ptr = @alignCast(@alignOf(w.FILE_BOTH_DIR_INFORMATION), &self.buf[self.index]); + const dir_info = @ptrCast(*w.FILE_BOTH_DIR_INFORMATION, aligned_ptr); + if (dir_info.NextEntryOffset != 0) { + self.index += dir_info.NextEntryOffset; + } else { + self.index = self.buf.len; + } + + const name_utf16le = @ptrCast([*]u16, &dir_info.FileName)[0 .. dir_info.FileNameLength / 2]; + + if (mem.eql(u16, name_utf16le, [_]u16{'.'}) or mem.eql(u16, name_utf16le, [_]u16{ '.', '.' })) + continue; + // Trust that Windows gives us valid UTF-16LE + const name_utf8_len = std.unicode.utf16leToUtf8(self.name_data[0..], name_utf16le) catch unreachable; + const name_utf8 = self.name_data[0..name_utf8_len]; + const kind = blk: { + const attrs = dir_info.FileAttributes; + if (attrs & w.FILE_ATTRIBUTE_DIRECTORY != 0) break :blk Entry.Kind.Directory; + if (attrs & w.FILE_ATTRIBUTE_REPARSE_POINT != 0) break :blk Entry.Kind.SymLink; + break :blk Entry.Kind.File; + }; + return Entry{ + .name = name_utf8, + .kind = kind, + }; + } + } }, else => @compileError("unimplemented"), }; + pub fn iterate(self: Dir) Iterator { + switch (builtin.os) { + .macosx, .ios, .freebsd, .netbsd => return Iterator{ + .dir = self, + .seek = 0, + .index = 0, + .end_index = 0, + .buf = undefined, + }, + .linux => return Iterator{ + .dir = self, + .index = 0, + .end_index = 0, + .buf = undefined, + }, + .windows => return Iterator{ + .dir = self, + .index = 0, + .end_index = 0, + .first = true, + .buf = undefined, + .name_data = undefined, + }, + else => @compileError("unimplemented"), + } + } + + /// Returns an open handle to the current working directory. + /// Closing the returned `Dir` is checked illegal behavior. + /// On POSIX targets, this function is comptime-callable. + pub fn cwd() Dir { + if (os.windows.is_the_target) { + return Dir{ .fd = os.windows.peb().ProcessParameters.CurrentDirectory.Handle }; + } else { + return Dir{ .fd = os.AT_FDCWD }; + } + } + pub const OpenError = error{ FileNotFound, NotDir, @@ -594,18 +680,15 @@ pub const Dir = struct { /// Call `close` to free the directory handle. pub fn open(dir_path: []const u8) OpenError!Dir { - return posix_cwd.openDir(dir_path); + return cwd().openDir(dir_path); } /// Same as `open` except the parameter is null-terminated. pub fn openC(dir_path_c: [*]const u8) OpenError!Dir { - return posix_cwd.openDirC(dir_path_c); + return cwd().openDirC(dir_path_c); } pub fn close(self: *Dir) void { - if (os.windows.is_the_target) { - @panic("TODO"); - } os.close(self.fd); self.* = undefined; } @@ -625,14 +708,25 @@ pub const Dir = struct { /// Call `close` on the result when done. pub fn openDir(self: Dir, sub_path: []const u8) OpenError!Dir { + std.debug.warn("openDir {}\n", sub_path); + if (os.windows.is_the_target) { + const sub_path_w = try os.windows.sliceToPrefixedFileW(sub_path); + return self.openDirW(&sub_path_w); + } + const sub_path_c = try os.toPosixPath(sub_path); return self.openDirC(&sub_path_c); } - /// Call `close` on the result when done. - pub fn openDirC(self: Dir, sub_path: [*]const u8) OpenError!Dir { + /// Same as `openDir` except the parameter is null-terminated. + pub fn openDirC(self: Dir, sub_path_c: [*]const u8) OpenError!Dir { + if (os.windows.is_the_target) { + const sub_path_w = try os.windows.cStrToPrefixedFileW(sub_path_c); + return self.openDirW(&sub_path_w); + } + const flags = os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC; - const fd = os.openatC(self.fd, sub_path, flags, 0) catch |err| switch (err) { + const fd = os.openatC(self.fd, sub_path_c, flags, 0) catch |err| switch (err) { error.FileTooBig => unreachable, // can't happen for directories error.IsDir => unreachable, // we're providing O_DIRECTORY error.NoSpaceLeft => unreachable, // not providing O_CREAT @@ -642,6 +736,79 @@ pub const Dir = struct { return Dir{ .fd = fd }; } + /// Same as `openDir` except the path parameter is UTF16LE, NT-prefixed. + /// This function is Windows-only. + pub fn openDirW(self: Dir, sub_path_w: [*]const u16) OpenError!Dir { + const w = os.windows; + var result = Dir{ + .fd = undefined, + }; + //var mask: ?[*]const u16 = undefined; + //var nt_name: w.UNICODE_STRING = undefined; + //if (w.ntdll.RtlDosPathNameToNtPathName_U(sub_path_w, &nt_name, null, null) == 0) { + // return error.FileNotFound; + //} + //defer w.ntdll.RtlFreeUnicodeString(&nt_name); + //if (mask) |m| { + // if (m[0] == 0) { + // return error.FileNotFound; + // } else { + // nt_name.Length = @intCast(u16, @ptrToInt(mask) - @ptrToInt(nt_name.Buffer)); + // } + //} else { + // return error.FileNotFound; + //} + + const path_len_bytes = @intCast(u16, mem.toSliceConst(u16, sub_path_w).len * 2); + std.debug.warn("path_len_bytes = {}\n", path_len_bytes); + var nt_name = w.UNICODE_STRING{ + .Length = path_len_bytes, + .MaximumLength = path_len_bytes, + .Buffer = @intToPtr([*]u16, @ptrToInt(sub_path_w)), + }; + var attr = w.OBJECT_ATTRIBUTES{ + .Length = @sizeOf(w.OBJECT_ATTRIBUTES), + .RootDirectory = if (path.isAbsoluteW(sub_path_w)) null else self.fd, + .Attributes = 0, // Note we do not use OBJ_CASE_INSENSITIVE here. + .ObjectName = &nt_name, + .SecurityDescriptor = null, + .SecurityQualityOfService = null, + }; + std.debug.warn("RootDirectory = {}\n", attr.RootDirectory); + var io: w.IO_STATUS_BLOCK = undefined; + const wide_slice = nt_name.Buffer[0 .. nt_name.Length / 2]; + //const wide_slice2 = std.mem.toSliceConst(u16, mask.?); + var buf: [200]u8 = undefined; + //var buf2: [200]u8 = undefined; + const len = std.unicode.utf16leToUtf8(&buf, wide_slice) catch unreachable; + //const len2 = std.unicode.utf16leToUtf8(&buf2, wide_slice2) catch unreachable; + std.debug.warn("path: {}\n", buf[0..len]); + //std.debug.warn("path: {}\nmask: {}\n", buf[0..len], buf2[0..len2]); + const rc = w.ntdll.NtCreateFile( + &result.fd, + w.GENERIC_READ | w.SYNCHRONIZE, + &attr, + &io, + null, + 0, + w.FILE_SHARE_READ | w.FILE_SHARE_WRITE, + w.FILE_OPEN, + w.FILE_DIRECTORY_FILE | w.FILE_SYNCHRONOUS_IO_NONALERT | w.FILE_OPEN_FOR_BACKUP_INTENT, + null, + 0, + ); + std.debug.warn("result.fd = {}\n", result.fd); + switch (rc) { + w.STATUS.SUCCESS => return result, + w.STATUS.OBJECT_NAME_INVALID => @panic("openDirW invalid object name"), + w.STATUS.OBJECT_NAME_NOT_FOUND => return error.FileNotFound, + w.STATUS.INVALID_PARAMETER => { + @panic("invalid parameter"); + }, + else => return w.unexpectedStatus(rc), + } + } + pub const DeleteFileError = os.UnlinkError; /// Delete a file name and possibly the file it refers to, based on an open directory handle. @@ -677,6 +844,10 @@ pub const Dir = struct { /// Returns `error.DirNotEmpty` if the directory is not empty. /// To delete a directory recursively, see `deleteTree`. pub fn deleteDir(self: Dir, sub_path: []const u8) DeleteDirError!void { + if (os.windows.is_the_target) { + const sub_path_w = try os.windows.sliceToPrefixedFileW(sub_path); + return self.deleteDirW(&sub_path_w); + } const sub_path_c = try os.toPosixPath(sub_path); return self.deleteDirC(&sub_path_c); } @@ -689,24 +860,25 @@ pub const Dir = struct { }; } - pub fn iterate(self: Dir) Iterator { - switch (builtin.os) { - .macosx, .ios, .freebsd, .netbsd => return Iterator{ - .dir = self, - .seek = 0, - .index = 0, - .end_index = 0, - .buf = undefined, - }, - .linux => return Iterator{ - .dir = self, - .index = 0, - .end_index = 0, - .buf = undefined, - }, - .windows => @panic("TODO"), - else => @compileError("unimplemented"), - } + /// Same as `deleteDir` except the parameter is UTF16LE, NT prefixed. + /// This function is Windows-only. + pub fn deleteDirW(self: Dir, sub_path_w: [*]const u16) DeleteDirError!void { + os.unlinkatW(self.fd, sub_path_w, os.AT_REMOVEDIR) catch |err| switch (err) { + error.IsDir => unreachable, // not possible since we pass AT_REMOVEDIR + else => |e| return e, + }; + } + + /// Read value of a symbolic link. + /// The return value is a slice of `buffer`, from index `0`. + pub fn readLink(self: Dir, sub_path: []const u8, buffer: *[MAX_PATH_BYTES]u8) ![]u8 { + const sub_path_c = try os.toPosixPath(sub_path); + return self.readLinkC(&sub_path_c, buffer); + } + + /// Same as `readLink`, except the `pathname` parameter is null-terminated. + pub fn readLinkC(self: Dir, sub_path_c: [*]const u8, buffer: *[MAX_PATH_BYTES]u8) ![]u8 { + return os.readlinkatC(self.fd, sub_path_c, buffer); } pub const DeleteTreeError = error{ @@ -953,7 +1125,6 @@ pub const Walker = struct { /// Must call `Walker.deinit` when done. /// `dir_path` must not end in a path separator. /// The order of returned file system entries is undefined. -/// TODO: https://github.com/ziglang/zig/issues/2888 pub fn walkPath(allocator: *Allocator, dir_path: []const u8) !Walker { assert(!mem.endsWith(u8, dir_path, path.sep_str)); @@ -978,15 +1149,13 @@ pub fn walkPath(allocator: *Allocator, dir_path: []const u8) !Walker { /// Read value of a symbolic link. /// The return value is a slice of buffer, from index `0`. -/// TODO https://github.com/ziglang/zig/issues/2888 -pub fn readLink(pathname: []const u8, buffer: *[os.PATH_MAX]u8) ![]u8 { +pub fn readLink(pathname: []const u8, buffer: *[MAX_PATH_BYTES]u8) ![]u8 { return os.readlink(pathname, buffer); } -/// Same as `readLink`, except the `pathname` parameter is null-terminated. -/// TODO https://github.com/ziglang/zig/issues/2888 -pub fn readLinkC(pathname: [*]const u8, buffer: *[os.PATH_MAX]u8) ![]u8 { - return os.readlinkC(pathname, buffer); +/// Same as `readLink`, except the parameter is null-terminated. +pub fn readLinkC(pathname_c: [*]const u8, buffer: *[MAX_PATH_BYTES]u8) ![]u8 { + return os.readlinkC(pathname_c, buffer); } pub const OpenSelfExeError = os.OpenError || os.windows.CreateFileError || SelfExePathError; diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index 32c221d063..a9212d69d4 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -243,6 +243,7 @@ pub const File = struct { switch (rc) { windows.STATUS.SUCCESS => {}, windows.STATUS.BUFFER_OVERFLOW => {}, + windows.STATUS.INVALID_PARAMETER => unreachable, else => return windows.unexpectedStatus(rc), } return Stat{ diff --git a/lib/std/fs/path.zig b/lib/std/fs/path.zig index 2bb23f04ce..560d9b58f3 100644 --- a/lib/std/fs/path.zig +++ b/lib/std/fs/path.zig @@ -136,6 +136,25 @@ pub fn isAbsolute(path: []const u8) bool { } } +pub fn isAbsoluteW(path_w: [*]const u16) bool { + if (path_w[0] == '/') + return true; + + if (path_w[0] == '\\') { + return true; + } + if (path_w[0] == 0 or path_w[1] == 0 or path_w[2] == 0) { + return false; + } + if (path_w[1] == ':') { + if (path_w[2] == '/') + return true; + if (path_w[2] == '\\') + return true; + } + return false; +} + pub fn isAbsoluteWindows(path: []const u8) bool { if (path[0] == '/') return true; diff --git a/lib/std/io.zig b/lib/std/io.zig index 4407e6b2b3..19623d93ee 100644 --- a/lib/std/io.zig +++ b/lib/std/io.zig @@ -127,6 +127,7 @@ pub fn OutStream(comptime WriteError: type) type { }; } +/// TODO move this to `std.fs` and add a version to `std.fs.Dir`. pub fn writeFile(path: []const u8, data: []const u8) !void { var file = try File.openWrite(path); defer file.close(); @@ -134,11 +135,13 @@ pub fn writeFile(path: []const u8, data: []const u8) !void { } /// On success, caller owns returned buffer. +/// TODO move this to `std.fs` and add a version to `std.fs.Dir`. pub fn readFileAlloc(allocator: *mem.Allocator, path: []const u8) ![]u8 { return readFileAllocAligned(allocator, path, @alignOf(u8)); } /// On success, caller owns returned buffer. +/// TODO move this to `std.fs` and add a version to `std.fs.Dir`. pub fn readFileAllocAligned(allocator: *mem.Allocator, path: []const u8, comptime A: u29) ![]align(A) u8 { var file = try File.openRead(path); defer file.close(); @@ -1084,7 +1087,7 @@ pub fn Deserializer(comptime endian: builtin.Endian, comptime packing: Packing, // safety. If it is bad, it will be caught anyway. const TagInt = @TagType(TagType); const tag = try self.deserializeInt(TagInt); - + inline for (info.fields) |field_info| { if (field_info.enum_field.?.value == tag) { const name = field_info.name; diff --git a/lib/std/os.zig b/lib/std/os.zig index e7319095cb..2d8ddb0abc 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -999,12 +999,20 @@ pub const UnlinkatError = UnlinkError || error{ /// Delete a file name and possibly the file it refers to, based on an open directory handle. pub fn unlinkat(dirfd: fd_t, file_path: []const u8, flags: u32) UnlinkatError!void { + if (windows.is_the_target) { + const file_path_w = try windows.sliceToPrefixedFileW(file_path); + return unlinkatW(dirfd, &file_path_w, flags); + } const file_path_c = try toPosixPath(file_path); return unlinkatC(dirfd, &file_path_c, flags); } /// Same as `unlinkat` but `file_path` is a null-terminated string. pub fn unlinkatC(dirfd: fd_t, file_path_c: [*]const u8, flags: u32) UnlinkatError!void { + if (windows.is_the_target) { + const file_path_w = try windows.cStrToPrefixedFileW(file_path_c); + return unlinkatW(dirfd, &file_path_w, flags); + } switch (errno(system.unlinkat(dirfd, file_path_c, flags))) { 0 => return, EACCES => return error.AccessDenied, @@ -1028,6 +1036,56 @@ pub fn unlinkatC(dirfd: fd_t, file_path_c: [*]const u8, flags: u32) UnlinkatErro } } +/// Same as `unlinkat` but `sub_path_w` is UTF16LE, NT prefixed. Windows only. +pub fn unlinkatW(dirfd: fd_t, sub_path_w: [*]const u16, flags: u32) UnlinkatError!void { + const w = windows; + + const want_rmdir_behavior = (flags & AT_REMOVEDIR) != 0; + const create_options_flags = if (want_rmdir_behavior) + w.ULONG(w.FILE_DELETE_ON_CLOSE) + else + w.ULONG(w.FILE_DELETE_ON_CLOSE | w.FILE_NON_DIRECTORY_FILE); + var nt_name: w.UNICODE_STRING = undefined; + if (w.ntdll.RtlDosPathNameToNtPathName_U(sub_path_w, &nt_name, null, null) == 0) { + return error.FileNotFound; + } + defer w.ntdll.RtlFreeUnicodeString(&nt_name); + + var attr = w.OBJECT_ATTRIBUTES{ + .Length = @sizeOf(w.OBJECT_ATTRIBUTES), + .RootDirectory = dirfd, + .Attributes = 0, // Note we do not use OBJ_CASE_INSENSITIVE here. + .ObjectName = &nt_name, + .SecurityDescriptor = null, + .SecurityQualityOfService = null, + }; + var io: w.IO_STATUS_BLOCK = undefined; + var tmp_handle: w.HANDLE = undefined; + var rc = w.ntdll.NtCreateFile( + &tmp_handle, + w.SYNCHRONIZE | w.DELETE, + &attr, + &io, + null, + 0, + w.FILE_SHARE_READ | w.FILE_SHARE_WRITE | w.FILE_SHARE_DELETE, + w.FILE_OPEN, + create_options_flags, + null, + 0, + ); + if (rc == w.STATUS.SUCCESS) { + rc = w.ntdll.NtClose(tmp_handle); + } + switch (rc) { + w.STATUS.SUCCESS => return, + w.STATUS.OBJECT_NAME_INVALID => unreachable, + w.STATUS.OBJECT_NAME_NOT_FOUND => return error.FileNotFound, + w.STATUS.INVALID_PARAMETER => unreachable, + else => return w.unexpectedStatus(rc), + } +} + const RenameError = error{ AccessDenied, FileBusy, @@ -1287,6 +1345,27 @@ pub fn readlinkC(file_path: [*]const u8, out_buffer: []u8) ReadLinkError![]u8 { } } +pub fn readlinkatC(dirfd: fd_t, file_path: [*]const u8, out_buffer: []u8) ReadLinkError![]u8 { + if (windows.is_the_target) { + const file_path_w = try windows.cStrToPrefixedFileW(file_path); + @compileError("TODO implement readlink for Windows"); + } + const rc = system.readlinkat(dirfd, file_path, out_buffer.ptr, out_buffer.len); + switch (errno(rc)) { + 0 => return out_buffer[0..@bitCast(usize, rc)], + EACCES => return error.AccessDenied, + EFAULT => unreachable, + EINVAL => unreachable, + EIO => return error.FileSystem, + ELOOP => return error.SymLinkLoop, + ENAMETOOLONG => return error.NameTooLong, + ENOENT => return error.FileNotFound, + ENOMEM => return error.SystemResources, + ENOTDIR => return error.NotDir, + else => |err| return unexpectedErrno(err), + } +} + pub const SetIdError = error{ ResourceLimitReached, InvalidUserId, diff --git a/lib/std/os/bits/windows.zig b/lib/std/os/bits/windows.zig index fc148d812f..c261b52bed 100644 --- a/lib/std/os/bits/windows.zig +++ b/lib/std/os/bits/windows.zig @@ -158,3 +158,6 @@ pub const EWOULDBLOCK = 140; pub const EDQUOT = 10069; pub const F_OK = 0; + +/// Remove directory instead of unlinking file +pub const AT_REMOVEDIR = 0x200; diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig index 7c1761a4b8..6a5d03b529 100644 --- a/lib/std/os/windows.zig +++ b/lib/std/os/windows.zig @@ -792,6 +792,25 @@ pub fn SetFileTime( } } +pub fn peb() *PEB { + switch (builtin.arch) { + .i386 => { + return asm ( + \\ mov %%fs:0x18, %[ptr] + \\ mov %%ds:0x30(%[ptr]), %[ptr] + : [ptr] "=r" (-> *PEB) + ); + }, + .x86_64 => { + return asm ( + \\ mov %%gs:0x60, %[ptr] + : [ptr] "=r" (-> *PEB) + ); + }, + else => @compileError("unsupported architecture"), + } +} + /// A file time is a 64-bit value that represents the number of 100-nanosecond /// intervals that have elapsed since 12:00 A.M. January 1, 1601 Coordinated /// Universal Time (UTC). @@ -844,8 +863,8 @@ pub fn sliceToPrefixedSuffixedFileW(s: []const u8, comptime suffix: []const u16) else => {}, } } - const start_index = if (mem.startsWith(u8, s, "\\\\") or !std.fs.path.isAbsolute(s)) 0 else blk: { - const prefix = [_]u16{ '\\', '\\', '?', '\\' }; + const start_index = if (mem.startsWith(u8, s, "\\?") or !std.fs.path.isAbsolute(s)) 0 else blk: { + const prefix = [_]u16{ '\\', '?', '?', '\\' }; mem.copy(u16, result[0..], prefix); break :blk prefix.len; }; diff --git a/lib/std/os/windows/bits.zig b/lib/std/os/windows/bits.zig index ddfdd27e1b..68d5ab6b87 100644 --- a/lib/std/os/windows/bits.zig +++ b/lib/std/os/windows/bits.zig @@ -300,6 +300,44 @@ pub const FILE_SHARE_DELETE = 0x00000004; pub const FILE_SHARE_READ = 0x00000001; pub const FILE_SHARE_WRITE = 0x00000002; +pub const DELETE = 0x00010000; +pub const READ_CONTROL = 0x00020000; +pub const WRITE_DAC = 0x00040000; +pub const WRITE_OWNER = 0x00080000; +pub const SYNCHRONIZE = 0x00100000; +pub const STANDARD_RIGHTS_REQUIRED = 0x000f0000; + +// disposition for NtCreateFile +pub const FILE_SUPERSEDE = 0; +pub const FILE_OPEN = 1; +pub const FILE_CREATE = 2; +pub const FILE_OPEN_IF = 3; +pub const FILE_OVERWRITE = 4; +pub const FILE_OVERWRITE_IF = 5; +pub const FILE_MAXIMUM_DISPOSITION = 5; + +// flags for NtCreateFile and NtOpenFile +pub const FILE_DIRECTORY_FILE = 0x00000001; +pub const FILE_WRITE_THROUGH = 0x00000002; +pub const FILE_SEQUENTIAL_ONLY = 0x00000004; +pub const FILE_NO_INTERMEDIATE_BUFFERING = 0x00000008; +pub const FILE_SYNCHRONOUS_IO_ALERT = 0x00000010; +pub const FILE_SYNCHRONOUS_IO_NONALERT = 0x00000020; +pub const FILE_NON_DIRECTORY_FILE = 0x00000040; +pub const FILE_CREATE_TREE_CONNECTION = 0x00000080; +pub const FILE_COMPLETE_IF_OPLOCKED = 0x00000100; +pub const FILE_NO_EA_KNOWLEDGE = 0x00000200; +pub const FILE_OPEN_FOR_RECOVERY = 0x00000400; +pub const FILE_RANDOM_ACCESS = 0x00000800; +pub const FILE_DELETE_ON_CLOSE = 0x00001000; +pub const FILE_OPEN_BY_FILE_ID = 0x00002000; +pub const FILE_OPEN_FOR_BACKUP_INTENT = 0x00004000; +pub const FILE_NO_COMPRESSION = 0x00008000; +pub const FILE_RESERVE_OPFILTER = 0x00100000; +pub const FILE_TRANSACTED_MODE = 0x00200000; +pub const FILE_OPEN_OFFLINE_FILE = 0x00400000; +pub const FILE_OPEN_FOR_FREE_SPACE_QUERY = 0x00800000; + pub const CREATE_ALWAYS = 2; pub const CREATE_NEW = 1; pub const OPEN_ALWAYS = 4; @@ -720,15 +758,117 @@ pub const VECTORED_EXCEPTION_HANDLER = stdcallcc fn (ExceptionInfo: *EXCEPTION_P pub const OBJECT_ATTRIBUTES = extern struct { Length: ULONG, - RootDirectory: HANDLE, + RootDirectory: ?HANDLE, ObjectName: *UNICODE_STRING, Attributes: ULONG, SecurityDescriptor: ?*c_void, SecurityQualityOfService: ?*c_void, }; +pub const OBJ_INHERIT = 0x00000002; +pub const OBJ_PERMANENT = 0x00000010; +pub const OBJ_EXCLUSIVE = 0x00000020; +pub const OBJ_CASE_INSENSITIVE = 0x00000040; +pub const OBJ_OPENIF = 0x00000080; +pub const OBJ_OPENLINK = 0x00000100; +pub const OBJ_KERNEL_HANDLE = 0x00000200; +pub const OBJ_VALID_ATTRIBUTES = 0x000003F2; + pub const UNICODE_STRING = extern struct { - Length: USHORT, - MaximumLength: USHORT, + Length: c_ushort, + MaximumLength: c_ushort, Buffer: [*]WCHAR, }; + +pub const PEB = extern struct { + Reserved1: [2]BYTE, + BeingDebugged: BYTE, + Reserved2: [1]BYTE, + Reserved3: [2]PVOID, + Ldr: *PEB_LDR_DATA, + ProcessParameters: *RTL_USER_PROCESS_PARAMETERS, + Reserved4: [3]PVOID, + AtlThunkSListPtr: PVOID, + Reserved5: PVOID, + Reserved6: ULONG, + Reserved7: PVOID, + Reserved8: ULONG, + AtlThunkSListPtr32: ULONG, + Reserved9: [45]PVOID, + Reserved10: [96]BYTE, + PostProcessInitRoutine: PPS_POST_PROCESS_INIT_ROUTINE, + Reserved11: [128]BYTE, + Reserved12: [1]PVOID, + SessionId: ULONG, +}; + +pub const PEB_LDR_DATA = extern struct { + Reserved1: [8]BYTE, + Reserved2: [3]PVOID, + InMemoryOrderModuleList: LIST_ENTRY, +}; + +pub const RTL_USER_PROCESS_PARAMETERS = extern struct { + AllocationSize: ULONG, + Size: ULONG, + Flags: ULONG, + DebugFlags: ULONG, + ConsoleHandle: HANDLE, + ConsoleFlags: ULONG, + hStdInput: HANDLE, + hStdOutput: HANDLE, + hStdError: HANDLE, + CurrentDirectory: CURDIR, + DllPath: UNICODE_STRING, + ImagePathName: UNICODE_STRING, + CommandLine: UNICODE_STRING, + Environment: [*]WCHAR, + dwX: ULONG, + dwY: ULONG, + dwXSize: ULONG, + dwYSize: ULONG, + dwXCountChars: ULONG, + dwYCountChars: ULONG, + dwFillAttribute: ULONG, + dwFlags: ULONG, + dwShowWindow: ULONG, + WindowTitle: UNICODE_STRING, + Desktop: UNICODE_STRING, + ShellInfo: UNICODE_STRING, + RuntimeInfo: UNICODE_STRING, + DLCurrentDirectory: [0x20]RTL_DRIVE_LETTER_CURDIR, +}; + +pub const RTL_DRIVE_LETTER_CURDIR = extern struct { + Flags: c_ushort, + Length: c_ushort, + TimeStamp: ULONG, + DosPath: UNICODE_STRING, +}; + +pub const PPS_POST_PROCESS_INIT_ROUTINE = ?extern fn () void; + +pub const FILE_BOTH_DIR_INFORMATION = extern struct { + NextEntryOffset: ULONG, + FileIndex: ULONG, + CreationTime: LARGE_INTEGER, + LastAccessTime: LARGE_INTEGER, + LastWriteTime: LARGE_INTEGER, + ChangeTime: LARGE_INTEGER, + EndOfFile: LARGE_INTEGER, + AllocationSize: LARGE_INTEGER, + FileAttributes: ULONG, + FileNameLength: ULONG, + EaSize: ULONG, + ShortNameLength: CHAR, + ShortName: [12]WCHAR, + FileName: [1]WCHAR, +}; +pub const FILE_BOTH_DIRECTORY_INFORMATION = FILE_BOTH_DIR_INFORMATION; + +pub const IO_APC_ROUTINE = extern fn (PVOID, *IO_STATUS_BLOCK, ULONG) void; + +pub const CURDIR = extern struct { + DosPath: UNICODE_STRING, + Handle: HANDLE, +}; diff --git a/lib/std/os/windows/ntdll.zig b/lib/std/os/windows/ntdll.zig index bfc98aba8a..f914920093 100644 --- a/lib/std/os/windows/ntdll.zig +++ b/lib/std/os/windows/ntdll.zig @@ -13,12 +13,33 @@ pub extern "NtDll" stdcallcc fn NtCreateFile( DesiredAccess: ACCESS_MASK, ObjectAttributes: *OBJECT_ATTRIBUTES, IoStatusBlock: *IO_STATUS_BLOCK, - AllocationSize: *LARGE_INTEGER, + AllocationSize: ?*LARGE_INTEGER, FileAttributes: ULONG, ShareAccess: ULONG, CreateDisposition: ULONG, CreateOptions: ULONG, - EaBuffer: *c_void, + EaBuffer: ?*c_void, EaLength: ULONG, ) NTSTATUS; pub extern "NtDll" stdcallcc fn NtClose(Handle: HANDLE) NTSTATUS; +pub extern "NtDll" stdcallcc fn RtlDosPathNameToNtPathName_U( + DosPathName: [*]const u16, + NtPathName: *UNICODE_STRING, + NtFileNamePart: ?*?[*]const u16, + DirectoryInfo: ?*CURDIR, +) BOOL; +pub extern "NtDll" stdcallcc fn RtlFreeUnicodeString(UnicodeString: *UNICODE_STRING) void; + +pub extern "NtDll" stdcallcc fn NtQueryDirectoryFile( + FileHandle: HANDLE, + Event: ?HANDLE, + ApcRoutine: ?IO_APC_ROUTINE, + ApcContext: ?*c_void, + IoStatusBlock: *IO_STATUS_BLOCK, + FileInformation: *c_void, + Length: ULONG, + FileInformationClass: FILE_INFORMATION_CLASS, + ReturnSingleEntry: BOOLEAN, + FileName: ?*UNICODE_STRING, + RestartScan: BOOLEAN, +) NTSTATUS; diff --git a/lib/std/os/windows/status.zig b/lib/std/os/windows/status.zig index b9fd2b495f..d381e3b2e1 100644 --- a/lib/std/os/windows/status.zig +++ b/lib/std/os/windows/status.zig @@ -3,3 +3,9 @@ pub const SUCCESS = 0x00000000; /// The data was too large to fit into the specified buffer. pub const BUFFER_OVERFLOW = 0x80000005; + +pub const INVALID_PARAMETER = 0xC000000D; +pub const ACCESS_DENIED = 0xC0000022; +pub const OBJECT_NAME_INVALID = 0xC0000033; +pub const OBJECT_NAME_NOT_FOUND = 0xC0000034; +pub const OBJECT_PATH_SYNTAX_BAD = 0xC000003B; From 87f632b08a04f4a03facfa220df1cbc466a5de39 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Oct 2019 19:17:33 -0400 Subject: [PATCH 04/10] fs.Dir.openDir: use empty object name for "." on Windows --- CMakeLists.txt | 3 +++ lib/std/fs.zig | 35 +++++++++++++++++++++++++++------ lib/std/os/windows.zig | 4 +++- lib/std/os/windows/bits.zig | 2 ++ lib/std/os/windows/kernel32.zig | 2 ++ lib/std/os/windows/status.zig | 1 + 6 files changed, 40 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3a34d034dd..cd1ba9d3f8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -630,5 +630,8 @@ set_target_properties(zig PROPERTIES LINK_FLAGS ${EXE_LDFLAGS} ) target_link_libraries(zig compiler "${LIBUSERLAND}") +if(MSVC) + target_link_libraries(zig ntdll.lib) +endif() add_dependencies(zig zig_build_libuserland) install(TARGETS zig DESTINATION bin) diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 51623d3057..8f9f8c17d9 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -590,6 +590,7 @@ pub const Dir = struct { self.end_index = io.Information; switch (rc) { w.STATUS.SUCCESS => {}, + w.STATUS.ACCESS_DENIED => return error.AccessDenied, else => return w.unexpectedStatus(rc), } } @@ -708,7 +709,7 @@ pub const Dir = struct { /// Call `close` on the result when done. pub fn openDir(self: Dir, sub_path: []const u8) OpenError!Dir { - std.debug.warn("openDir {}\n", sub_path); + // std.debug.warn("openDir {}\n", sub_path); if (os.windows.is_the_target) { const sub_path_w = try os.windows.sliceToPrefixedFileW(sub_path); return self.openDirW(&sub_path_w); @@ -740,9 +741,26 @@ pub const Dir = struct { /// This function is Windows-only. pub fn openDirW(self: Dir, sub_path_w: [*]const u16) OpenError!Dir { const w = os.windows; + var result = Dir{ .fd = undefined, }; + + const desired_access = w.GENERIC_READ | w.SYNCHRONIZE; + + // if (sub_path_w[0] == '.' and sub_path_w[1] == 0) { + // // Windows gives me STATUS_OBJECT_NAME_INVALID with "." as the object name. + + // if (w.kernel32.DuplicateHandle(w.self_process_handle, self.fd, w.self_process_handle, &result.fd, desired_access, w.TRUE, 0) == 0) { + // switch (w.kernel32.GetLastError()) { + // else => |err| return w.unexpectedError(err), + // } + + // @panic("handle DuplicateHandle error"); + // } + // return result; + // } + //var mask: ?[*]const u16 = undefined; //var nt_name: w.UNICODE_STRING = undefined; //if (w.ntdll.RtlDosPathNameToNtPathName_U(sub_path_w, &nt_name, null, null) == 0) { @@ -760,7 +778,7 @@ pub const Dir = struct { //} const path_len_bytes = @intCast(u16, mem.toSliceConst(u16, sub_path_w).len * 2); - std.debug.warn("path_len_bytes = {}\n", path_len_bytes); + // std.debug.warn("path_len_bytes = {}\n", path_len_bytes); var nt_name = w.UNICODE_STRING{ .Length = path_len_bytes, .MaximumLength = path_len_bytes, @@ -774,7 +792,11 @@ pub const Dir = struct { .SecurityDescriptor = null, .SecurityQualityOfService = null, }; - std.debug.warn("RootDirectory = {}\n", attr.RootDirectory); + if (sub_path_w[0] == '.' and sub_path_w[1] == 0) { + // Windows does not recognize this, but it does work with empty string. + nt_name.Length = 0; + } + // std.debug.warn("RootDirectory = {}\n", attr.RootDirectory); var io: w.IO_STATUS_BLOCK = undefined; const wide_slice = nt_name.Buffer[0 .. nt_name.Length / 2]; //const wide_slice2 = std.mem.toSliceConst(u16, mask.?); @@ -782,11 +804,11 @@ pub const Dir = struct { //var buf2: [200]u8 = undefined; const len = std.unicode.utf16leToUtf8(&buf, wide_slice) catch unreachable; //const len2 = std.unicode.utf16leToUtf8(&buf2, wide_slice2) catch unreachable; - std.debug.warn("path: {}\n", buf[0..len]); + // std.debug.warn("path: {}\n", buf[0..len]); //std.debug.warn("path: {}\nmask: {}\n", buf[0..len], buf2[0..len2]); const rc = w.ntdll.NtCreateFile( &result.fd, - w.GENERIC_READ | w.SYNCHRONIZE, + desired_access, &attr, &io, null, @@ -797,11 +819,12 @@ pub const Dir = struct { null, 0, ); - std.debug.warn("result.fd = {}\n", result.fd); + // std.debug.warn("result.fd = {}\n", result.fd); switch (rc) { w.STATUS.SUCCESS => return result, w.STATUS.OBJECT_NAME_INVALID => @panic("openDirW invalid object name"), w.STATUS.OBJECT_NAME_NOT_FOUND => return error.FileNotFound, + w.STATUS.OBJECT_PATH_NOT_FOUND => return error.FileNotFound, w.STATUS.INVALID_PARAMETER => { @panic("invalid parameter"); }, diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig index 6a5d03b529..93a5a65a92 100644 --- a/lib/std/os/windows.zig +++ b/lib/std/os/windows.zig @@ -20,6 +20,8 @@ pub const shell32 = @import("windows/shell32.zig"); pub usingnamespace @import("windows/bits.zig"); +pub const self_process_handle = @intToPtr(HANDLE, maxInt(usize)); + /// `builtin` is missing `subsystem` when the subsystem is automatically detected, /// so Zig standard library has the subsystem detection logic here. This should generally be /// used rather than `builtin.subsystem`. @@ -898,7 +900,7 @@ pub fn unexpectedError(err: DWORD) std.os.UnexpectedError { /// and you get an unexpected status. pub fn unexpectedStatus(status: NTSTATUS) std.os.UnexpectedError { if (std.os.unexpected_error_tracing) { - std.debug.warn("error.Unexpected NTSTATUS={}\n", status); + std.debug.warn("error.Unexpected NTSTATUS=0x{x}\n", status); std.debug.dumpCurrentStackTrace(null); } return error.Unexpected; diff --git a/lib/std/os/windows/bits.zig b/lib/std/os/windows/bits.zig index 68d5ab6b87..214f75186f 100644 --- a/lib/std/os/windows/bits.zig +++ b/lib/std/os/windows/bits.zig @@ -872,3 +872,5 @@ pub const CURDIR = extern struct { DosPath: UNICODE_STRING, Handle: HANDLE, }; + +pub const DUPLICATE_SAME_ACCESS = 2; \ No newline at end of file diff --git a/lib/std/os/windows/kernel32.zig b/lib/std/os/windows/kernel32.zig index 2ae73ad45a..736d81ae58 100644 --- a/lib/std/os/windows/kernel32.zig +++ b/lib/std/os/windows/kernel32.zig @@ -47,6 +47,8 @@ pub extern "kernel32" stdcallcc fn CreateThread(lpThreadAttributes: ?LPSECURITY_ pub extern "kernel32" stdcallcc fn DeleteFileW(lpFileName: [*]const u16) BOOL; +pub extern "kernel32" stdcallcc fn DuplicateHandle(hSourceProcessHandle: HANDLE, hSourceHandle: HANDLE, hTargetProcessHandle: HANDLE, lpTargetHandle: *HANDLE, dwDesiredAccess: DWORD, bInheritHandle: BOOL, dwOptions: DWORD) BOOL; + pub extern "kernel32" stdcallcc fn ExitProcess(exit_code: UINT) noreturn; pub extern "kernel32" stdcallcc fn FindFirstFileW(lpFileName: [*]const u16, lpFindFileData: *WIN32_FIND_DATAW) HANDLE; diff --git a/lib/std/os/windows/status.zig b/lib/std/os/windows/status.zig index d381e3b2e1..424cae5e29 100644 --- a/lib/std/os/windows/status.zig +++ b/lib/std/os/windows/status.zig @@ -8,4 +8,5 @@ pub const INVALID_PARAMETER = 0xC000000D; pub const ACCESS_DENIED = 0xC0000022; pub const OBJECT_NAME_INVALID = 0xC0000033; pub const OBJECT_NAME_NOT_FOUND = 0xC0000034; +pub const OBJECT_PATH_NOT_FOUND = 0xC000003A; pub const OBJECT_PATH_SYNTAX_BAD = 0xC000003B; From 98c4365b66e6184aa867ec7d5d1d01435ec99798 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Oct 2019 19:39:30 -0400 Subject: [PATCH 05/10] darwin: add AT_* bits --- lib/std/os/bits/darwin.zig | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/std/os/bits/darwin.zig b/lib/std/os/bits/darwin.zig index b076f95884..fe3156bb90 100644 --- a/lib/std/os/bits/darwin.zig +++ b/lib/std/os/bits/darwin.zig @@ -1178,3 +1178,17 @@ pub fn S_IWHT(m: u32) bool { return m & S_IFMT == S_IFWHT; } pub const HOST_NAME_MAX = 72; + +pub const AT_FDCWD = -2; + +/// Use effective ids in access check +pub const AT_EACCESS = 0x0010; + +/// Act on the symlink itself not the target +pub const AT_SYMLINK_NOFOLLOW = 0x0020; + +/// Act on target of symlink +pub const AT_SYMLINK_FOLLOW = 0x0040; + +/// Path refers to directory +pub const AT_REMOVEDIR = 0x0080; From 5917572e59ff68e33ae7fb0ec7b8d592e2915463 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Oct 2019 20:16:43 -0400 Subject: [PATCH 06/10] cleanup --- lib/std/fs.zig | 76 +++++++++++++------------------------------------- 1 file changed, 19 insertions(+), 57 deletions(-) diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 8f9f8c17d9..3fdd175e77 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -377,6 +377,8 @@ pub const Dir = struct { }; }; + const IteratorError = error{AccessDenied} || os.UnexpectedError; + pub const Iterator = switch (builtin.os) { .macosx, .ios, .freebsd, .netbsd => struct { dir: Dir, @@ -387,9 +389,11 @@ pub const Dir = struct { const Self = @This(); + pub const Error = IteratorError; + /// Memory such as file names referenced in this returned entry becomes invalid /// with subsequent calls to `next`, as well as when this `Dir` is deinitialized. - pub fn next(self: *Self) !?Entry { + pub fn next(self: *Self) Error!?Entry { switch (builtin.os) { .macosx, .ios => return self.nextDarwin(), .freebsd, .netbsd => return self.nextBsd(), @@ -504,9 +508,11 @@ pub const Dir = struct { const Self = @This(); + pub const Error = IteratorError; + /// Memory such as file names referenced in this returned entry becomes invalid /// with subsequent calls to `next`, as well as when this `Dir` is deinitialized. - pub fn next(self: *Self) !?Entry { + pub fn next(self: *Self) Error!?Entry { start_over: while (true) { if (self.index >= self.end_index) { const rc = os.linux.getdents64(self.dir.fd, &self.buf, self.buf.len); @@ -560,17 +566,13 @@ pub const Dir = struct { const Self = @This(); - pub fn next(self: *Self) !?Entry { + pub const Error = IteratorError; + + pub fn next(self: *Self) Error!?Entry { start_over: while (true) { const w = os.windows; if (self.index >= self.end_index) { var io: w.IO_STATUS_BLOCK = undefined; - //var mask_buf = [2]u16{ 'a', 0 }; - //var mask = w.UNICODE_STRING{ - // .Length = 2, - // .MaximumLength = 2, - // .Buffer = &mask_buf, - //}; const rc = w.ntdll.NtQueryDirectoryFile( self.dir.fd, null, @@ -709,7 +711,6 @@ pub const Dir = struct { /// Call `close` on the result when done. pub fn openDir(self: Dir, sub_path: []const u8) OpenError!Dir { - // std.debug.warn("openDir {}\n", sub_path); if (os.windows.is_the_target) { const sub_path_w = try os.windows.sliceToPrefixedFileW(sub_path); return self.openDirW(&sub_path_w); @@ -746,39 +747,7 @@ pub const Dir = struct { .fd = undefined, }; - const desired_access = w.GENERIC_READ | w.SYNCHRONIZE; - - // if (sub_path_w[0] == '.' and sub_path_w[1] == 0) { - // // Windows gives me STATUS_OBJECT_NAME_INVALID with "." as the object name. - - // if (w.kernel32.DuplicateHandle(w.self_process_handle, self.fd, w.self_process_handle, &result.fd, desired_access, w.TRUE, 0) == 0) { - // switch (w.kernel32.GetLastError()) { - // else => |err| return w.unexpectedError(err), - // } - - // @panic("handle DuplicateHandle error"); - // } - // return result; - // } - - //var mask: ?[*]const u16 = undefined; - //var nt_name: w.UNICODE_STRING = undefined; - //if (w.ntdll.RtlDosPathNameToNtPathName_U(sub_path_w, &nt_name, null, null) == 0) { - // return error.FileNotFound; - //} - //defer w.ntdll.RtlFreeUnicodeString(&nt_name); - //if (mask) |m| { - // if (m[0] == 0) { - // return error.FileNotFound; - // } else { - // nt_name.Length = @intCast(u16, @ptrToInt(mask) - @ptrToInt(nt_name.Buffer)); - // } - //} else { - // return error.FileNotFound; - //} - const path_len_bytes = @intCast(u16, mem.toSliceConst(u16, sub_path_w).len * 2); - // std.debug.warn("path_len_bytes = {}\n", path_len_bytes); var nt_name = w.UNICODE_STRING{ .Length = path_len_bytes, .MaximumLength = path_len_bytes, @@ -796,19 +765,15 @@ pub const Dir = struct { // Windows does not recognize this, but it does work with empty string. nt_name.Length = 0; } - // std.debug.warn("RootDirectory = {}\n", attr.RootDirectory); + if (sub_path_w[0] == '.' and sub_path_w[1] == '.' and sub_path_w[2] == 0) { + // If you're looking to contribute to zig and fix this, see here for an example of how to + // implement this: https://git.midipix.org/ntapi/tree/src/fs/ntapi_tt_open_physical_parent_directory.c + @panic("TODO opening '..' with a relative directory handle is not yet implemented on Windows"); + } var io: w.IO_STATUS_BLOCK = undefined; - const wide_slice = nt_name.Buffer[0 .. nt_name.Length / 2]; - //const wide_slice2 = std.mem.toSliceConst(u16, mask.?); - var buf: [200]u8 = undefined; - //var buf2: [200]u8 = undefined; - const len = std.unicode.utf16leToUtf8(&buf, wide_slice) catch unreachable; - //const len2 = std.unicode.utf16leToUtf8(&buf2, wide_slice2) catch unreachable; - // std.debug.warn("path: {}\n", buf[0..len]); - //std.debug.warn("path: {}\nmask: {}\n", buf[0..len], buf2[0..len2]); const rc = w.ntdll.NtCreateFile( &result.fd, - desired_access, + w.GENERIC_READ | w.SYNCHRONIZE, &attr, &io, null, @@ -819,15 +784,12 @@ pub const Dir = struct { null, 0, ); - // std.debug.warn("result.fd = {}\n", result.fd); switch (rc) { w.STATUS.SUCCESS => return result, - w.STATUS.OBJECT_NAME_INVALID => @panic("openDirW invalid object name"), + w.STATUS.OBJECT_NAME_INVALID => unreachable, w.STATUS.OBJECT_NAME_NOT_FOUND => return error.FileNotFound, w.STATUS.OBJECT_PATH_NOT_FOUND => return error.FileNotFound, - w.STATUS.INVALID_PARAMETER => { - @panic("invalid parameter"); - }, + w.STATUS.INVALID_PARAMETER => unreachable, else => return w.unexpectedStatus(rc), } } From 9a82f00a4fb897acf50446a725768c3aaa562513 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Oct 2019 20:16:52 -0400 Subject: [PATCH 07/10] add freebsd AT_* bits --- lib/std/os/bits/freebsd.zig | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/lib/std/os/bits/freebsd.zig b/lib/std/os/bits/freebsd.zig index 3d07e92e01..4427acb334 100644 --- a/lib/std/os/bits/freebsd.zig +++ b/lib/std/os/bits/freebsd.zig @@ -939,3 +939,23 @@ pub fn S_IWHT(m: u32) bool { } pub const HOST_NAME_MAX = 255; + +/// Magic value that specify the use of the current working directory +/// to determine the target of relative file paths in the openat() and +/// similar syscalls. +pub const AT_FDCWD = -100; + +/// Check access using effective user and group ID +pub const AT_EACCESS = 0x0100; + +/// Do not follow symbolic links +pub const AT_SYMLINK_NOFOLLOW = 0x0200; + +/// Follow symbolic link +pub const AT_SYMLINK_FOLLOW = 0x0400; + +/// Remove directory instead of file +pub const AT_REMOVEDIR = 0x0800; + +/// Fail if not under dirfd +pub const AT_BENEATH = 0x1000; From 65165554d0c6cf39ff091ace5ab5d48106a935fd Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Oct 2019 20:49:35 -0400 Subject: [PATCH 08/10] test runner: restore previous behavior when... ...stderr does not support ansi escape codes --- lib/std/special/test_runner.zig | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/std/special/test_runner.zig b/lib/std/special/test_runner.zig index 8fb1b5842d..78db63e4d6 100644 --- a/lib/std/special/test_runner.zig +++ b/lib/std/special/test_runner.zig @@ -15,14 +15,17 @@ pub fn main() anyerror!void { for (test_fn_list) |test_fn, i| { var test_node = root_node.start(test_fn.name, null); test_node.activate(); + if (progress.terminal == null) std.debug.warn("{}/{} {}...", i + 1, test_fn_list.len, test_fn.name); if (test_fn.func()) |_| { ok_count += 1; test_node.end(); + if (progress.terminal == null) std.debug.warn("OK\n"); } else |err| switch (err) { error.SkipZigTest => { skip_count += 1; test_node.end(); progress.log("{}...SKIP\n", test_fn.name); + if (progress.terminal == null) std.debug.warn("SKIP\n"); }, else => return err, } @@ -30,5 +33,6 @@ pub fn main() anyerror!void { root_node.end(); if (ok_count != test_fn_list.len) { progress.log("{} passed; {} skipped.\n", ok_count, skip_count); + if (progress.terminal == null) std.debug.warn("{} passed; {} skipped.\n", ok_count, skip_count); } } From fc6f84f3f08aca0d4e2352091cc2db2ca5779c7d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Oct 2019 20:58:53 -0400 Subject: [PATCH 09/10] fix os.unlinkatW implementation --- lib/std/os.zig | 22 ++++++++++++++++++---- lib/std/os/windows/status.zig | 1 + 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/lib/std/os.zig b/lib/std/os.zig index 2d8ddb0abc..7a5eeda82c 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -1045,11 +1045,24 @@ pub fn unlinkatW(dirfd: fd_t, sub_path_w: [*]const u16, flags: u32) UnlinkatErro w.ULONG(w.FILE_DELETE_ON_CLOSE) else w.ULONG(w.FILE_DELETE_ON_CLOSE | w.FILE_NON_DIRECTORY_FILE); - var nt_name: w.UNICODE_STRING = undefined; - if (w.ntdll.RtlDosPathNameToNtPathName_U(sub_path_w, &nt_name, null, null) == 0) { - return error.FileNotFound; + + const path_len_bytes = @intCast(u16, mem.toSliceConst(u16, sub_path_w).len * 2); + var nt_name = w.UNICODE_STRING{ + .Length = path_len_bytes, + .MaximumLength = path_len_bytes, + // The Windows API makes this mutable, but it will not mutate here. + .Buffer = @intToPtr([*]u16, @ptrToInt(sub_path_w)), + }; + + if (sub_path_w[0] == '.' and sub_path_w[1] == 0) { + // Windows does not recognize this, but it does work with empty string. + nt_name.Length = 0; } - defer w.ntdll.RtlFreeUnicodeString(&nt_name); + if (sub_path_w[0] == '.' and sub_path_w[1] == '.' and sub_path_w[2] == 0) { + // Can't remove the parent directory with an open handle. + return error.FileBusy; + } + var attr = w.OBJECT_ATTRIBUTES{ .Length = @sizeOf(w.OBJECT_ATTRIBUTES), @@ -1082,6 +1095,7 @@ pub fn unlinkatW(dirfd: fd_t, sub_path_w: [*]const u16, flags: u32) UnlinkatErro w.STATUS.OBJECT_NAME_INVALID => unreachable, w.STATUS.OBJECT_NAME_NOT_FOUND => return error.FileNotFound, w.STATUS.INVALID_PARAMETER => unreachable, + w.STATUS.FILE_IS_A_DIRECTORY => return error.IsDir, else => return w.unexpectedStatus(rc), } } diff --git a/lib/std/os/windows/status.zig b/lib/std/os/windows/status.zig index 424cae5e29..5f38948620 100644 --- a/lib/std/os/windows/status.zig +++ b/lib/std/os/windows/status.zig @@ -10,3 +10,4 @@ pub const OBJECT_NAME_INVALID = 0xC0000033; pub const OBJECT_NAME_NOT_FOUND = 0xC0000034; pub const OBJECT_PATH_NOT_FOUND = 0xC000003A; pub const OBJECT_PATH_SYNTAX_BAD = 0xC000003B; +pub const FILE_IS_A_DIRECTORY = 0xC00000BA; From 064377be9aaf409bf483f512354ef22f656bf213 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 21 Oct 2019 22:20:45 -0400 Subject: [PATCH 10/10] test runner: restore previous behavior of printing all tests passed --- lib/std/special/test_runner.zig | 7 ++++--- test/cli.zig | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/lib/std/special/test_runner.zig b/lib/std/special/test_runner.zig index 78db63e4d6..f835401e1c 100644 --- a/lib/std/special/test_runner.zig +++ b/lib/std/special/test_runner.zig @@ -31,8 +31,9 @@ pub fn main() anyerror!void { } } root_node.end(); - if (ok_count != test_fn_list.len) { - progress.log("{} passed; {} skipped.\n", ok_count, skip_count); - if (progress.terminal == null) std.debug.warn("{} passed; {} skipped.\n", ok_count, skip_count); + if (ok_count == test_fn_list.len) { + std.debug.warn("All tests passed.\n"); + } else { + std.debug.warn("{} passed; {} skipped.\n", ok_count, skip_count); } } diff --git a/test/cli.zig b/test/cli.zig index 2999df14b0..6095536422 100644 --- a/test/cli.zig +++ b/test/cli.zig @@ -87,7 +87,7 @@ fn exec(cwd: []const u8, argv: []const []const u8) !ChildProcess.ExecResult { fn testZigInitLib(zig_exe: []const u8, dir_path: []const u8) !void { _ = try exec(dir_path, [_][]const u8{ zig_exe, "init-lib" }); const test_result = try exec(dir_path, [_][]const u8{ zig_exe, "build", "test" }); - testing.expect(std.mem.eql(u8, test_result.stderr, "")); + testing.expect(std.mem.endsWith(u8, test_result.stderr, "All tests passed.\n")); } fn testZigInitExe(zig_exe: []const u8, dir_path: []const u8) !void {