From 666584067a97da6d39867ff0c57fcd3be2d50e96 Mon Sep 17 00:00:00 2001 From: Jonathan Marler Date: Sun, 3 Jan 2021 04:19:24 -0700 Subject: [PATCH 1/4] implement nt path conversion for windows --- lib/std/fs.zig | 9 ---- lib/std/fs/test.zig | 13 ++++- lib/std/mem.zig | 43 +++++++++++++++++ lib/std/os/windows.zig | 95 ++++++++++++++++++++++++++++++++----- lib/std/os/windows/test.zig | 70 +++++++++++++++++++++++++++ 5 files changed, 208 insertions(+), 22 deletions(-) create mode 100644 lib/std/os/windows/test.zig diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 2e3d4fca05..a280f6ccaa 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -1365,15 +1365,6 @@ pub const Dir = struct { .SecurityDescriptor = null, .SecurityQualityOfService = null, }; - if (sub_path_w[0] == '.' and sub_path_w[1] == 0) { - // Windows does not recognize this, but it does work with empty string. - nt_name.Length = 0; - } - if (sub_path_w[0] == '.' and sub_path_w[1] == '.' and sub_path_w[2] == 0) { - // If you're looking to contribute to zig and fix this, see here for an example of how to - // implement this: https://git.midipix.org/ntapi/tree/src/fs/ntapi_tt_open_physical_parent_directory.c - @panic("TODO opening '..' with a relative directory handle is not yet implemented on Windows"); - } const open_reparse_point: w.DWORD = if (no_follow) w.FILE_OPEN_REPARSE_POINT else 0x0; var io: w.IO_STATUS_BLOCK = undefined; const rc = w.ntdll.NtCreateFile( diff --git a/lib/std/fs/test.zig b/lib/std/fs/test.zig index f586c50b6a..1354b10c80 100644 --- a/lib/std/fs/test.zig +++ b/lib/std/fs/test.zig @@ -79,8 +79,17 @@ test "openDirAbsolute" { break :blk try fs.realpathAlloc(&arena.allocator, relative_path); }; - var dir = try fs.openDirAbsolute(base_path, .{}); - defer dir.close(); + { + var dir = try fs.openDirAbsolute(base_path, .{}); + defer dir.close(); + } + + for ([_][]const u8{ ".", ".." }) |sub_path| { + const dir_path = try fs.path.join(&arena.allocator, &[_][]const u8{ base_path, sub_path }); + defer arena.allocator.free(dir_path); + var dir = try fs.openDirAbsolute(dir_path, .{}); + defer dir.close(); + } } test "readLinkAbsolute" { diff --git a/lib/std/mem.zig b/lib/std/mem.zig index 044d73413a..2828da2c07 100644 --- a/lib/std/mem.zig +++ b/lib/std/mem.zig @@ -2120,6 +2120,49 @@ test "replace" { try testing.expectEqualStrings(expected, output[0..expected.len]); } +/// Replace all occurences of `needle` with `replacement`. +pub fn replaceScalar(comptime T: type, slice: []T, needle: T, replacement: T) void { + for (slice) |e, i| { + if (e == needle) { + slice[i] = replacement; + } + } +} + +/// Collapse consecutive duplicate elements into one entry. +pub fn collapseRepeats(comptime T: type, slice: []T, elem: T) usize { + if (slice.len == 0) return 0; + var write_idx: usize = 1; + var read_idx: usize = 1; + while (read_idx < slice.len) : (read_idx += 1) { + if (slice[read_idx - 1] != elem or slice[read_idx] != elem) { + slice[write_idx] = slice[read_idx]; + write_idx += 1; + } + } + return write_idx; +} + +fn testCollapseRepeats(str: []const u8, elem: u8, expected: []const u8) !void { + const mutable = try std.testing.allocator.dupe(u8, str); + defer std.testing.allocator.free(mutable); + const actual = mutable[0..collapseRepeats(u8, mutable, elem)]; + testing.expect(std.mem.eql(u8, actual, expected)); +} +test "collapseRepeats" { + try testCollapseRepeats("", '/', ""); + try testCollapseRepeats("a", '/', "a"); + try testCollapseRepeats("/", '/', "/"); + try testCollapseRepeats("//", '/', "/"); + try testCollapseRepeats("/a", '/', "/a"); + try testCollapseRepeats("//a", '/', "/a"); + try testCollapseRepeats("a/", '/', "a/"); + try testCollapseRepeats("a//", '/', "a/"); + try testCollapseRepeats("a/a", '/', "a/a"); + try testCollapseRepeats("a//a", '/', "a/a"); + try testCollapseRepeats("//a///a////", '/', "/a/a/"); +} + /// Calculate the size needed in an output buffer to perform a replacement. /// The needle must not be empty. pub fn replacementSize(comptime T: type, input: []const T, needle: []const T, replacement: []const T) usize { diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig index 9d898dadaf..eed0311eed 100644 --- a/lib/std/os/windows.zig +++ b/lib/std/os/windows.zig @@ -1723,6 +1723,81 @@ pub const PathSpace = struct { } }; +/// The error type for `removeDotDirsSanitized` +pub const RemoveDotDirsError = error{TooManyParentDirs}; + +/// Removes '.' and '..' path components from a "sanitized relative path". +/// A "sanitized path" is one where: +/// 1) all forward slashes have been replaced with back slashes +/// 2) all repeating back slashes have been collapsed +/// 3) the path is a relative one (does not start with a back slash) +pub fn removeDotDirsSanitized(comptime T: type, path: []T) RemoveDotDirsError!usize { + std.debug.assert(path.len == 0 or path[0] != '\\'); + + var write_idx: usize = 0; + var read_idx: usize = 0; + while (read_idx < path.len) { + if (path[read_idx] == '.') { + if (read_idx + 1 == path.len) + return write_idx; + + const after_dot = path[read_idx + 1]; + if (after_dot == '\\') { + read_idx += 2; + continue; + } + if (after_dot == '.' and (read_idx + 2 == path.len or path[read_idx + 2] == '\\')) { + if (write_idx == 0) return error.TooManyParentDirs; + std.debug.assert(write_idx >= 2); + write_idx -= 1; + while (true) { + write_idx -= 1; + if (write_idx == 0) break; + if (path[write_idx] == '\\') { + write_idx += 1; + break; + } + } + if (read_idx + 2 == path.len) + return write_idx; + read_idx += 3; + continue; + } + } + + // skip to the next path separator + while (true) : (read_idx += 1) { + if (read_idx == path.len) + return write_idx; + path[write_idx] = path[read_idx]; + write_idx += 1; + if (path[read_idx] == '\\') + break; + } + read_idx += 1; + } + return write_idx; +} + +/// Normalizes a Windows path with the following steps: +/// 1) convert all forward slashes to back slashes +/// 2) collapse duplicate back slashes +/// 3) remove '.' and '..' directory parts +/// Returns the length of the new path. +pub fn normalizePath(comptime T: type, path: []T) RemoveDotDirsError!usize { + mem.replaceScalar(T, path, '/', '\\'); + const new_len = mem.collapseRepeats(T, path, '\\'); + + const prefix_len: usize = init: { + if (new_len >= 1 and path[0] == '\\') break :init 1; + if (new_len >= 2 and path[1] == ':') + break :init if (new_len >= 3 and path[2] == '\\') @as(usize, 3) else @as(usize, 2); + break :init 0; + }; + + return prefix_len + try removeDotDirsSanitized(T, path[prefix_len..new_len]); +} + /// Same as `sliceToPrefixedFileW` but accepts a pointer /// to a null-terminated path. pub fn cStrToPrefixedFileW(s: [*:0]const u8) !PathSpace { @@ -1749,17 +1824,9 @@ pub fn sliceToPrefixedFileW(s: []const u8) !PathSpace { }; path_space.len = start_index + try std.unicode.utf8ToUtf16Le(path_space.data[start_index..], s); if (path_space.len > path_space.data.len) return error.NameTooLong; - // > File I/O functions in the Windows API convert "/" to "\" as part of - // > converting the name to an NT-style name, except when using the "\\?\" - // > prefix as detailed in the following sections. - // from https://docs.microsoft.com/en-us/windows/desktop/FileIO/naming-a-file#maximum-path-length-limitation - // Because we want the larger maximum path length for absolute paths, we - // convert forward slashes to backward slashes here. - for (path_space.data[0..path_space.len]) |*elem| { - if (elem.* == '/') { - elem.* = '\\'; - } - } + path_space.len = start_index + (normalizePath(u16, path_space.data[start_index..path_space.len]) catch |err| switch (err) { + error.TooManyParentDirs => return error.BadPathName, + }); path_space.data[path_space.len] = 0; return path_space; } @@ -1864,3 +1931,9 @@ pub fn unexpectedStatus(status: NTSTATUS) std.os.UnexpectedError { } return error.Unexpected; } + +test "" { + if (builtin.os.tag == .windows) { + _ = @import("windows/test.zig"); + } +} diff --git a/lib/std/os/windows/test.zig b/lib/std/os/windows/test.zig new file mode 100644 index 0000000000..ceffc17bc2 --- /dev/null +++ b/lib/std/os/windows/test.zig @@ -0,0 +1,70 @@ +// SPDX-License-Identifier: MIT +// Copyright (c) 2015-2020 Zig Contributors +// This file is part of [zig](https://ziglang.org/), which is MIT licensed. +// The MIT license requires this copyright notice to be included in all copies +// and substantial portions of the software. +const std = @import("../../std.zig"); +const builtin = @import("builtin"); +const windows = std.os.windows; +const mem = std.mem; +const testing = std.testing; +const expect = testing.expect; + +fn testRemoveDotDirs(str: []const u8, expected: []const u8) !void { + const mutable = try testing.allocator.dupe(u8, str); + defer testing.allocator.free(mutable); + const actual = mutable[0..try windows.removeDotDirsSanitized(u8, mutable)]; + testing.expect(mem.eql(u8, actual, expected)); +} +fn testRemoveDotDirsError(err: anyerror, str: []const u8) !void { + const mutable = try testing.allocator.dupe(u8, str); + defer testing.allocator.free(mutable); + testing.expectError(err, windows.removeDotDirsSanitized(u8, mutable)); +} +test "removeDotDirs" { + try testRemoveDotDirs("", ""); + try testRemoveDotDirs(".", ""); + try testRemoveDotDirs(".\\", ""); + try testRemoveDotDirs(".\\.", ""); + try testRemoveDotDirs(".\\.\\", ""); + try testRemoveDotDirs(".\\.\\.", ""); + + try testRemoveDotDirs("a", "a"); + try testRemoveDotDirs("a\\", "a\\"); + try testRemoveDotDirs("a\\b", "a\\b"); + try testRemoveDotDirs("a\\.", "a\\"); + try testRemoveDotDirs("a\\b\\.", "a\\b\\"); + try testRemoveDotDirs("a\\.\\b", "a\\b"); + + try testRemoveDotDirs(".a", ".a"); + try testRemoveDotDirs(".a\\", ".a\\"); + try testRemoveDotDirs(".a\\.b", ".a\\.b"); + try testRemoveDotDirs(".a\\.", ".a\\"); + try testRemoveDotDirs(".a\\.\\.", ".a\\"); + try testRemoveDotDirs(".a\\.\\.\\.b", ".a\\.b"); + try testRemoveDotDirs(".a\\.\\.\\.b\\", ".a\\.b\\"); + + try testRemoveDotDirsError(error.TooManyParentDirs, ".."); + try testRemoveDotDirsError(error.TooManyParentDirs, "..\\"); + try testRemoveDotDirsError(error.TooManyParentDirs, ".\\..\\"); + try testRemoveDotDirsError(error.TooManyParentDirs, ".\\.\\..\\"); + + try testRemoveDotDirs("a\\..", ""); + try testRemoveDotDirs("a\\..\\", ""); + try testRemoveDotDirs("a\\..\\.", ""); + try testRemoveDotDirs("a\\..\\.\\", ""); + try testRemoveDotDirs("a\\..\\.\\.", ""); + try testRemoveDotDirsError(error.TooManyParentDirs, "a\\..\\.\\.\\.."); + + try testRemoveDotDirs("a\\..\\.\\.\\b", "b"); + try testRemoveDotDirs("a\\..\\.\\.\\b\\", "b\\"); + try testRemoveDotDirs("a\\..\\.\\.\\b\\.", "b\\"); + try testRemoveDotDirs("a\\..\\.\\.\\b\\.\\", "b\\"); + try testRemoveDotDirs("a\\..\\.\\.\\b\\.\\..", ""); + try testRemoveDotDirs("a\\..\\.\\.\\b\\.\\..\\", ""); + try testRemoveDotDirs("a\\..\\.\\.\\b\\.\\..\\.", ""); + try testRemoveDotDirsError(error.TooManyParentDirs, "a\\..\\.\\.\\b\\.\\..\\.\\.."); + + try testRemoveDotDirs("a\\b\\..\\", "a\\"); + try testRemoveDotDirs("a\\b\\..\\c", "a\\c"); +} From a72ad61cd4b718dd1ce2bf8ba4a3404f8304648c Mon Sep 17 00:00:00 2001 From: Jonathan Marler Date: Wed, 6 Jan 2021 23:55:33 -0700 Subject: [PATCH 2/4] have collapseRepeats return slice intead of just len --- lib/std/mem.zig | 10 +++++++--- lib/std/os/windows.zig | 2 +- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/lib/std/mem.zig b/lib/std/mem.zig index 2828da2c07..e57bd2fe49 100644 --- a/lib/std/mem.zig +++ b/lib/std/mem.zig @@ -2130,7 +2130,7 @@ pub fn replaceScalar(comptime T: type, slice: []T, needle: T, replacement: T) vo } /// Collapse consecutive duplicate elements into one entry. -pub fn collapseRepeats(comptime T: type, slice: []T, elem: T) usize { +pub fn collapseRepeatsLen(comptime T: type, slice: []T, elem: T) usize { if (slice.len == 0) return 0; var write_idx: usize = 1; var read_idx: usize = 1; @@ -2143,11 +2143,15 @@ pub fn collapseRepeats(comptime T: type, slice: []T, elem: T) usize { return write_idx; } +/// Collapse consecutive duplicate elements into one entry. +pub fn collapseRepeats(comptime T: type, slice: []T, elem: T) []T { + return slice[0 .. collapseRepeatsLen(T, slice, elem)]; +} + fn testCollapseRepeats(str: []const u8, elem: u8, expected: []const u8) !void { const mutable = try std.testing.allocator.dupe(u8, str); defer std.testing.allocator.free(mutable); - const actual = mutable[0..collapseRepeats(u8, mutable, elem)]; - testing.expect(std.mem.eql(u8, actual, expected)); + testing.expect(std.mem.eql(u8, collapseRepeats(u8, mutable, elem), expected)); } test "collapseRepeats" { try testCollapseRepeats("", '/', ""); diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig index eed0311eed..aa33159b74 100644 --- a/lib/std/os/windows.zig +++ b/lib/std/os/windows.zig @@ -1786,7 +1786,7 @@ pub fn removeDotDirsSanitized(comptime T: type, path: []T) RemoveDotDirsError!us /// Returns the length of the new path. pub fn normalizePath(comptime T: type, path: []T) RemoveDotDirsError!usize { mem.replaceScalar(T, path, '/', '\\'); - const new_len = mem.collapseRepeats(T, path, '\\'); + const new_len = mem.collapseRepeatsLen(T, path, '\\'); const prefix_len: usize = init: { if (new_len >= 1 and path[0] == '\\') break :init 1; From 59de5d0350f76933549e5402f090785351d9498d Mon Sep 17 00:00:00 2001 From: Jonathan Marler Date: Fri, 2 Apr 2021 13:17:06 -0600 Subject: [PATCH 3/4] add the openDir cwd parent test --- lib/std/fs/test.zig | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/std/fs/test.zig b/lib/std/fs/test.zig index 1354b10c80..87e0096bb1 100644 --- a/lib/std/fs/test.zig +++ b/lib/std/fs/test.zig @@ -92,6 +92,13 @@ test "openDirAbsolute" { } } +test "openDir cwd parent .." { + if (builtin.os.tag == .wasi) return error.SkipZigTest; + + var cwd = try fs.cwd().openDir("..", .{}); + defer cwd.close(); +} + test "readLinkAbsolute" { if (builtin.os.tag == .wasi) return error.SkipZigTest; From b0116afd8adbc73d820fee418ec74104f43bff6e Mon Sep 17 00:00:00 2001 From: Jonathan Marler Date: Thu, 20 May 2021 23:23:10 -0600 Subject: [PATCH 4/4] handle relative paths with too many ".." --- lib/std/fs/test.zig | 4 ++-- lib/std/mem.zig | 2 +- lib/std/os/windows.zig | 26 ++++++++++++++++++++++++-- lib/std/os/windows/kernel32.zig | 7 +++++++ lib/std/os/windows/test.zig | 4 ++-- 5 files changed, 36 insertions(+), 7 deletions(-) diff --git a/lib/std/fs/test.zig b/lib/std/fs/test.zig index 87e0096bb1..bcf4bf5c97 100644 --- a/lib/std/fs/test.zig +++ b/lib/std/fs/test.zig @@ -95,8 +95,8 @@ test "openDirAbsolute" { test "openDir cwd parent .." { if (builtin.os.tag == .wasi) return error.SkipZigTest; - var cwd = try fs.cwd().openDir("..", .{}); - defer cwd.close(); + var dir = try fs.cwd().openDir("..", .{}); + defer dir.close(); } test "readLinkAbsolute" { diff --git a/lib/std/mem.zig b/lib/std/mem.zig index e57bd2fe49..56391bcfe1 100644 --- a/lib/std/mem.zig +++ b/lib/std/mem.zig @@ -2151,7 +2151,7 @@ pub fn collapseRepeats(comptime T: type, slice: []T, elem: T) []T { fn testCollapseRepeats(str: []const u8, elem: u8, expected: []const u8) !void { const mutable = try std.testing.allocator.dupe(u8, str); defer std.testing.allocator.free(mutable); - testing.expect(std.mem.eql(u8, collapseRepeats(u8, mutable, elem), expected)); + try testing.expect(std.mem.eql(u8, collapseRepeats(u8, mutable, elem), expected)); } test "collapseRepeats" { try testCollapseRepeats("", '/', ""); diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig index aa33159b74..e02cf4e280 100644 --- a/lib/std/os/windows.zig +++ b/lib/std/os/windows.zig @@ -1817,20 +1817,42 @@ pub fn sliceToPrefixedFileW(s: []const u8) !PathSpace { else => {}, } } + const prefix_u16 = [_]u16{ '\\', '?', '?', '\\' }; const start_index = if (prefix_index > 0 or !std.fs.path.isAbsolute(s)) 0 else blk: { - const prefix_u16 = [_]u16{ '\\', '?', '?', '\\' }; mem.copy(u16, path_space.data[0..], prefix_u16[0..]); break :blk prefix_u16.len; }; path_space.len = start_index + try std.unicode.utf8ToUtf16Le(path_space.data[start_index..], s); if (path_space.len > path_space.data.len) return error.NameTooLong; path_space.len = start_index + (normalizePath(u16, path_space.data[start_index..path_space.len]) catch |err| switch (err) { - error.TooManyParentDirs => return error.BadPathName, + error.TooManyParentDirs => { + if (!std.fs.path.isAbsolute(s)) { + var temp_path: PathSpace = undefined; + temp_path.len = try std.unicode.utf8ToUtf16Le(&temp_path.data, s); + std.debug.assert(temp_path.len == path_space.len); + temp_path.data[path_space.len] = 0; + path_space.len = prefix_u16.len + try getFullPathNameW(&temp_path.data, path_space.data[prefix_u16.len..]); + mem.copy(u16, &path_space.data, &prefix_u16); + std.debug.assert(path_space.data[path_space.len] == 0); + return path_space; + } + return error.BadPathName; + }, }); path_space.data[path_space.len] = 0; return path_space; } +fn getFullPathNameW(path: [*:0]const u16, out: []u16) !usize { + const result= kernel32.GetFullPathNameW(path, @intCast(u32, out.len), std.meta.assumeSentinel(out.ptr, 0), null); + if (result == 0) { + switch (kernel32.GetLastError()) { + else => |err| return unexpectedError(err), + } + } + return result; +} + /// Assumes an absolute path. pub fn wToPrefixedFileW(s: []const u16) !PathSpace { // TODO https://github.com/ziglang/zig/issues/2765 diff --git a/lib/std/os/windows/kernel32.zig b/lib/std/os/windows/kernel32.zig index e117f362eb..f2e8b87d74 100644 --- a/lib/std/os/windows/kernel32.zig +++ b/lib/std/os/windows/kernel32.zig @@ -136,6 +136,13 @@ pub extern "kernel32" fn GetFinalPathNameByHandleW( dwFlags: DWORD, ) callconv(WINAPI) DWORD; +pub extern "kernel32" fn GetFullPathNameW( + lpFileName: [*:0]const u16, + nBufferLength: u32, + lpBuffer: ?[*:0]u16, + lpFilePart: ?*?[*:0]u16, +) callconv(@import("std").os.windows.WINAPI) u32; + pub extern "kernel32" fn GetOverlappedResult(hFile: HANDLE, lpOverlapped: *OVERLAPPED, lpNumberOfBytesTransferred: *DWORD, bWait: BOOL) callconv(WINAPI) BOOL; pub extern "kernel32" fn GetProcessHeap() callconv(WINAPI) ?HANDLE; diff --git a/lib/std/os/windows/test.zig b/lib/std/os/windows/test.zig index ceffc17bc2..8c18d413ca 100644 --- a/lib/std/os/windows/test.zig +++ b/lib/std/os/windows/test.zig @@ -14,12 +14,12 @@ fn testRemoveDotDirs(str: []const u8, expected: []const u8) !void { const mutable = try testing.allocator.dupe(u8, str); defer testing.allocator.free(mutable); const actual = mutable[0..try windows.removeDotDirsSanitized(u8, mutable)]; - testing.expect(mem.eql(u8, actual, expected)); + try testing.expect(mem.eql(u8, actual, expected)); } fn testRemoveDotDirsError(err: anyerror, str: []const u8) !void { const mutable = try testing.allocator.dupe(u8, str); defer testing.allocator.free(mutable); - testing.expectError(err, windows.removeDotDirsSanitized(u8, mutable)); + try testing.expectError(err, windows.removeDotDirsSanitized(u8, mutable)); } test "removeDotDirs" { try testRemoveDotDirs("", "");