diff --git a/CMakeLists.txt b/CMakeLists.txt index fbc34f8051..cd1ba9d3f8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -46,6 +46,7 @@ message("Configuring zig version ${ZIG_VERSION}") set(ZIG_STATIC off CACHE BOOL "Attempt to build a static zig executable (not compatible with glibc)") set(ZIG_STATIC_LLVM off CACHE BOOL "Prefer linking against static LLVM libraries") set(ZIG_SKIP_INSTALL_LIB_FILES off CACHE BOOL "Disable copying lib/ files to install prefix") +set(ZIG_ENABLE_MEM_PROFILE off CACHE BOOL "Activate memory usage instrumentation") if(ZIG_STATIC) set(ZIG_STATIC_LLVM "on") @@ -455,6 +456,7 @@ set(ZIG_SOURCES "${CMAKE_SOURCE_DIR}/src/ir_print.cpp" "${CMAKE_SOURCE_DIR}/src/libc_installation.cpp" "${CMAKE_SOURCE_DIR}/src/link.cpp" + "${CMAKE_SOURCE_DIR}/src/memory_profiling.cpp" "${CMAKE_SOURCE_DIR}/src/os.cpp" "${CMAKE_SOURCE_DIR}/src/parser.cpp" "${CMAKE_SOURCE_DIR}/src/range_set.cpp" @@ -628,5 +630,8 @@ set_target_properties(zig PROPERTIES LINK_FLAGS ${EXE_LDFLAGS} ) target_link_libraries(zig compiler "${LIBUSERLAND}") +if(MSVC) + target_link_libraries(zig ntdll.lib) +endif() add_dependencies(zig zig_build_libuserland) install(TARGETS zig DESTINATION bin) diff --git a/doc/docgen.zig b/doc/docgen.zig index 73289e5edc..bdc2604213 100644 --- a/doc/docgen.zig +++ b/doc/docgen.zig @@ -51,7 +51,7 @@ pub fn main() !void { var toc = try genToc(allocator, &tokenizer); try fs.makePath(allocator, tmp_dir_name); - defer fs.deleteTree(allocator, tmp_dir_name) catch {}; + defer fs.deleteTree(tmp_dir_name) catch {}; try genHtml(allocator, &tokenizer, &toc, &buffered_out_stream.stream, zig_exe); try buffered_out_stream.flush(); diff --git a/doc/langref.html.in b/doc/langref.html.in index a30670b0c7..17792d6470 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -10086,8 +10086,8 @@ ContainerMembers <- TestDecl ContainerMembers / TopLevelComptime ContainerMembers / KEYWORD_pub? TopLevelDecl ContainerMembers - / KEYWORD_pub? ContainerField COMMA ContainerMembers - / KEYWORD_pub? ContainerField + / ContainerField COMMA ContainerMembers + / ContainerField / TestDecl <- KEYWORD_test STRINGLITERAL Block diff --git a/lib/std/build.zig b/lib/std/build.zig index 68d3f2a315..dd602ba922 100644 --- a/lib/std/build.zig +++ b/lib/std/build.zig @@ -331,7 +331,7 @@ pub const Builder = struct { if (self.verbose) { warn("rm {}\n", full_path); } - fs.deleteTree(self.allocator, full_path) catch {}; + fs.deleteTree(full_path) catch {}; } // TODO remove empty directories @@ -1491,6 +1491,8 @@ pub const LibExeObjStep = struct { /// Position Independent Code force_pic: ?bool = null, + subsystem: ?builtin.SubSystem = null, + const LinkObject = union(enum) { StaticPath: []const u8, OtherStep: *LibExeObjStep, @@ -2325,6 +2327,20 @@ pub const LibExeObjStep = struct { } } + if (self.subsystem) |subsystem| { + try zig_args.append("--subsystem"); + try zig_args.append(switch (subsystem) { + .Console => "console", + .Windows => "windows", + .Posix => "posix", + .Native => "native", + .EfiApplication => "efi_application", + .EfiBootServiceDriver => "efi_boot_service_driver", + .EfiRom => "efi_rom", + .EfiRuntimeDriver => "efi_runtime_driver", + }); + } + if (self.kind == Kind.Test) { try builder.spawnChild(zig_args.toSliceConst()); } else { @@ -2671,7 +2687,7 @@ pub const RemoveDirStep = struct { const self = @fieldParentPtr(RemoveDirStep, "step", step); const full_path = self.builder.pathFromRoot(self.dir_path); - fs.deleteTree(self.builder.allocator, full_path) catch |err| { + fs.deleteTree(full_path) catch |err| { warn("Unable to remove {}: {}\n", full_path, @errorName(err)); return err; }; diff --git a/lib/std/c.zig b/lib/std/c.zig index 9c7ec2f0e5..f591481d04 100644 --- a/lib/std/c.zig +++ b/lib/std/c.zig @@ -80,6 +80,7 @@ pub extern "c" fn mmap(addr: ?*align(page_size) c_void, len: usize, prot: c_uint pub extern "c" fn munmap(addr: *align(page_size) c_void, len: usize) c_int; pub extern "c" fn mprotect(addr: *align(page_size) c_void, len: usize, prot: c_uint) c_int; pub extern "c" fn unlink(path: [*]const u8) c_int; +pub extern "c" fn unlinkat(dirfd: fd_t, path: [*]const u8, flags: c_uint) c_int; pub extern "c" fn getcwd(buf: [*]u8, size: usize) ?[*]u8; pub extern "c" fn waitpid(pid: c_int, stat_loc: *c_uint, options: c_uint) c_int; pub extern "c" fn fork() c_int; diff --git a/lib/std/child_process.zig b/lib/std/child_process.zig index 7fb663ac28..d786c3f4de 100644 --- a/lib/std/child_process.zig +++ b/lib/std/child_process.zig @@ -17,35 +17,35 @@ const TailQueue = std.TailQueue; const maxInt = std.math.maxInt; pub const ChildProcess = struct { - pub pid: if (os.windows.is_the_target) void else i32, - pub handle: if (os.windows.is_the_target) windows.HANDLE else void, - pub thread_handle: if (os.windows.is_the_target) windows.HANDLE else void, + pid: if (os.windows.is_the_target) void else i32, + handle: if (os.windows.is_the_target) windows.HANDLE else void, + thread_handle: if (os.windows.is_the_target) windows.HANDLE else void, - pub allocator: *mem.Allocator, + allocator: *mem.Allocator, - pub stdin: ?File, - pub stdout: ?File, - pub stderr: ?File, + stdin: ?File, + stdout: ?File, + stderr: ?File, - pub term: ?(SpawnError!Term), + term: ?(SpawnError!Term), - pub argv: []const []const u8, + argv: []const []const u8, /// Leave as null to use the current env map using the supplied allocator. - pub env_map: ?*const BufMap, + env_map: ?*const BufMap, - pub stdin_behavior: StdIo, - pub stdout_behavior: StdIo, - pub stderr_behavior: StdIo, + stdin_behavior: StdIo, + stdout_behavior: StdIo, + stderr_behavior: StdIo, /// Set to change the user id when spawning the child process. - pub uid: if (os.windows.is_the_target) void else ?u32, + uid: if (os.windows.is_the_target) void else ?u32, /// Set to change the group id when spawning the child process. - pub gid: if (os.windows.is_the_target) void else ?u32, + gid: if (os.windows.is_the_target) void else ?u32, /// Set to change the current working directory when spawning the child process. - pub cwd: ?[]const u8, + cwd: ?[]const u8, err_pipe: if (os.windows.is_the_target) void else [2]os.fd_t, llnode: if (os.windows.is_the_target) void else TailQueue(*ChildProcess).Node, diff --git a/lib/std/event/fs.zig b/lib/std/event/fs.zig index 4490e1deae..81fd950e44 100644 --- a/lib/std/event/fs.zig +++ b/lib/std/event/fs.zig @@ -1312,7 +1312,7 @@ const test_tmp_dir = "std_event_fs_test"; // // // TODO move this into event loop too // try os.makePath(allocator, test_tmp_dir); -// defer os.deleteTree(allocator, test_tmp_dir) catch {}; +// defer os.deleteTree(test_tmp_dir) catch {}; // // var loop: Loop = undefined; // try loop.initMultiThreaded(allocator); diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 287c0316b7..3fdd175e77 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -335,140 +335,30 @@ pub fn deleteDirW(dir_path: [*]const u16) !void { return os.rmdirW(dir_path); } -const DeleteTreeError = error{ - OutOfMemory, - AccessDenied, - FileTooBig, - IsDir, - SymLinkLoop, - ProcessFdQuotaExceeded, - NameTooLong, - SystemFdQuotaExceeded, - NoDevice, - SystemResources, - NoSpaceLeft, - PathAlreadyExists, - ReadOnlyFileSystem, - NotDir, - FileNotFound, - FileSystem, - FileBusy, - DirNotEmpty, - DeviceBusy, +/// Removes a symlink, file, or directory. +/// If `full_path` is relative, this is equivalent to `Dir.deleteTree` with the +/// current working directory as the open directory handle. +/// If `full_path` is absolute, this is equivalent to `Dir.deleteTree` with the +/// base directory. +pub fn deleteTree(full_path: []const u8) !void { + if (path.isAbsolute(full_path)) { + const dirname = path.dirname(full_path) orelse return error{ + /// Attempt to remove the root file system path. + /// This error is unreachable if `full_path` is relative. + CannotDeleteRootDirectory, + }.CannotDeleteRootDirectory; - /// On Windows, file paths must be valid Unicode. - InvalidUtf8, + var dir = try Dir.open(dirname); + defer dir.close(); - /// On Windows, file paths cannot contain these characters: - /// '/', '*', '?', '"', '<', '>', '|' - BadPathName, - - Unexpected, -}; - -/// Whether `full_path` describes a symlink, file, or directory, this function -/// removes it. If it cannot be removed because it is a non-empty directory, -/// this function recursively removes its entries and then tries again. -/// TODO determine if we can remove the allocator requirement -/// https://github.com/ziglang/zig/issues/2886 -pub fn deleteTree(allocator: *Allocator, full_path: []const u8) DeleteTreeError!void { - start_over: while (true) { - var got_access_denied = false; - // First, try deleting the item as a file. This way we don't follow sym links. - if (deleteFile(full_path)) { - return; - } else |err| switch (err) { - error.FileNotFound => return, - error.IsDir => {}, - error.AccessDenied => got_access_denied = true, - - error.InvalidUtf8, - error.SymLinkLoop, - error.NameTooLong, - error.SystemResources, - error.ReadOnlyFileSystem, - error.NotDir, - error.FileSystem, - error.FileBusy, - error.BadPathName, - error.Unexpected, - => return err, - } - { - var dir = Dir.open(allocator, full_path) catch |err| switch (err) { - error.NotDir => { - if (got_access_denied) { - return error.AccessDenied; - } - continue :start_over; - }, - - error.OutOfMemory, - error.AccessDenied, - error.FileTooBig, - error.IsDir, - error.SymLinkLoop, - error.ProcessFdQuotaExceeded, - error.NameTooLong, - error.SystemFdQuotaExceeded, - error.NoDevice, - error.FileNotFound, - error.SystemResources, - error.NoSpaceLeft, - error.PathAlreadyExists, - error.Unexpected, - error.InvalidUtf8, - error.BadPathName, - error.DeviceBusy, - => return err, - }; - defer dir.close(); - - var full_entry_buf = std.ArrayList(u8).init(allocator); - defer full_entry_buf.deinit(); - - while (try dir.next()) |entry| { - try full_entry_buf.resize(full_path.len + entry.name.len + 1); - const full_entry_path = full_entry_buf.toSlice(); - mem.copy(u8, full_entry_path, full_path); - full_entry_path[full_path.len] = path.sep; - mem.copy(u8, full_entry_path[full_path.len + 1 ..], entry.name); - - try deleteTree(allocator, full_entry_path); - } - } - return deleteDir(full_path); + return dir.deleteTree(path.basename(full_path)); + } else { + return Dir.cwd().deleteTree(full_path); } } -/// TODO: separate this API into the one that opens directory handles to then subsequently open -/// files, and into the one that reads files from an open directory handle. pub const Dir = struct { - handle: Handle, - allocator: *Allocator, - - pub const Handle = switch (builtin.os) { - .macosx, .ios, .freebsd, .netbsd => struct { - fd: i32, - seek: i64, - buf: []u8, - index: usize, - end_index: usize, - }, - .linux => struct { - fd: i32, - buf: []u8, - index: usize, - end_index: usize, - }, - .windows => struct { - handle: os.windows.HANDLE, - find_file_data: os.windows.WIN32_FIND_DATAW, - first: bool, - name_data: [256]u8, - }, - else => @compileError("unimplemented"), - }; + fd: os.fd_t, pub const Entry = struct { name: []const u8, @@ -487,292 +377,666 @@ pub const Dir = struct { }; }; + const IteratorError = error{AccessDenied} || os.UnexpectedError; + + pub const Iterator = switch (builtin.os) { + .macosx, .ios, .freebsd, .netbsd => struct { + dir: Dir, + seek: i64, + buf: [8192]u8, // TODO align(@alignOf(os.dirent)), + index: usize, + end_index: usize, + + const Self = @This(); + + pub const Error = IteratorError; + + /// Memory such as file names referenced in this returned entry becomes invalid + /// with subsequent calls to `next`, as well as when this `Dir` is deinitialized. + pub fn next(self: *Self) Error!?Entry { + switch (builtin.os) { + .macosx, .ios => return self.nextDarwin(), + .freebsd, .netbsd => return self.nextBsd(), + else => @compileError("unimplemented"), + } + } + + fn nextDarwin(self: *Self) !?Entry { + start_over: while (true) { + if (self.index >= self.end_index) { + const rc = os.system.__getdirentries64( + self.dir.fd, + &self.buf, + self.buf.len, + &self.seek, + ); + if (rc == 0) return null; + if (rc < 0) { + switch (os.errno(rc)) { + os.EBADF => unreachable, + os.EFAULT => unreachable, + os.ENOTDIR => unreachable, + os.EINVAL => unreachable, + else => |err| return os.unexpectedErrno(err), + } + } + self.index = 0; + self.end_index = @intCast(usize, rc); + } + const darwin_entry = @ptrCast(*align(1) os.dirent, &self.buf[self.index]); + const next_index = self.index + darwin_entry.d_reclen; + self.index = next_index; + + const name = @ptrCast([*]u8, &darwin_entry.d_name)[0..darwin_entry.d_namlen]; + + if (mem.eql(u8, name, ".") or mem.eql(u8, name, "..")) { + continue :start_over; + } + + const entry_kind = switch (darwin_entry.d_type) { + os.DT_BLK => Entry.Kind.BlockDevice, + os.DT_CHR => Entry.Kind.CharacterDevice, + os.DT_DIR => Entry.Kind.Directory, + os.DT_FIFO => Entry.Kind.NamedPipe, + os.DT_LNK => Entry.Kind.SymLink, + os.DT_REG => Entry.Kind.File, + os.DT_SOCK => Entry.Kind.UnixDomainSocket, + os.DT_WHT => Entry.Kind.Whiteout, + else => Entry.Kind.Unknown, + }; + return Entry{ + .name = name, + .kind = entry_kind, + }; + } + } + + fn nextBsd(self: *Self) !?Entry { + start_over: while (true) { + if (self.index >= self.end_index) { + const rc = os.system.getdirentries( + self.dir.fd, + self.buf[0..].ptr, + self.buf.len, + &self.seek, + ); + switch (os.errno(rc)) { + 0 => {}, + os.EBADF => unreachable, + os.EFAULT => unreachable, + os.ENOTDIR => unreachable, + os.EINVAL => unreachable, + else => |err| return os.unexpectedErrno(err), + } + if (rc == 0) return null; + self.index = 0; + self.end_index = @intCast(usize, rc); + } + const freebsd_entry = @ptrCast(*align(1) os.dirent, &self.buf[self.index]); + const next_index = self.index + freebsd_entry.d_reclen; + self.index = next_index; + + const name = @ptrCast([*]u8, &freebsd_entry.d_name)[0..freebsd_entry.d_namlen]; + + if (mem.eql(u8, name, ".") or mem.eql(u8, name, "..")) { + continue :start_over; + } + + const entry_kind = switch (freebsd_entry.d_type) { + os.DT_BLK => Entry.Kind.BlockDevice, + os.DT_CHR => Entry.Kind.CharacterDevice, + os.DT_DIR => Entry.Kind.Directory, + os.DT_FIFO => Entry.Kind.NamedPipe, + os.DT_LNK => Entry.Kind.SymLink, + os.DT_REG => Entry.Kind.File, + os.DT_SOCK => Entry.Kind.UnixDomainSocket, + os.DT_WHT => Entry.Kind.Whiteout, + else => Entry.Kind.Unknown, + }; + return Entry{ + .name = name, + .kind = entry_kind, + }; + } + } + }, + .linux => struct { + dir: Dir, + buf: [8192]u8, // TODO align(@alignOf(os.dirent64)), + index: usize, + end_index: usize, + + const Self = @This(); + + pub const Error = IteratorError; + + /// Memory such as file names referenced in this returned entry becomes invalid + /// with subsequent calls to `next`, as well as when this `Dir` is deinitialized. + pub fn next(self: *Self) Error!?Entry { + start_over: while (true) { + if (self.index >= self.end_index) { + const rc = os.linux.getdents64(self.dir.fd, &self.buf, self.buf.len); + switch (os.linux.getErrno(rc)) { + 0 => {}, + os.EBADF => unreachable, + os.EFAULT => unreachable, + os.ENOTDIR => unreachable, + os.EINVAL => unreachable, + else => |err| return os.unexpectedErrno(err), + } + if (rc == 0) return null; + self.index = 0; + self.end_index = rc; + } + const linux_entry = @ptrCast(*align(1) os.dirent64, &self.buf[self.index]); + const next_index = self.index + linux_entry.d_reclen; + self.index = next_index; + + const name = mem.toSlice(u8, @ptrCast([*]u8, &linux_entry.d_name)); + + // skip . and .. entries + if (mem.eql(u8, name, ".") or mem.eql(u8, name, "..")) { + continue :start_over; + } + + const entry_kind = switch (linux_entry.d_type) { + os.DT_BLK => Entry.Kind.BlockDevice, + os.DT_CHR => Entry.Kind.CharacterDevice, + os.DT_DIR => Entry.Kind.Directory, + os.DT_FIFO => Entry.Kind.NamedPipe, + os.DT_LNK => Entry.Kind.SymLink, + os.DT_REG => Entry.Kind.File, + os.DT_SOCK => Entry.Kind.UnixDomainSocket, + else => Entry.Kind.Unknown, + }; + return Entry{ + .name = name, + .kind = entry_kind, + }; + } + } + }, + .windows => struct { + dir: Dir, + buf: [8192]u8 align(@alignOf(os.windows.FILE_BOTH_DIR_INFORMATION)), + index: usize, + end_index: usize, + first: bool, + name_data: [256]u8, + + const Self = @This(); + + pub const Error = IteratorError; + + pub fn next(self: *Self) Error!?Entry { + start_over: while (true) { + const w = os.windows; + if (self.index >= self.end_index) { + var io: w.IO_STATUS_BLOCK = undefined; + const rc = w.ntdll.NtQueryDirectoryFile( + self.dir.fd, + null, + null, + null, + &io, + &self.buf, + self.buf.len, + .FileBothDirectoryInformation, + w.FALSE, + null, + if (self.first) w.BOOLEAN(w.TRUE) else w.BOOLEAN(w.FALSE), + ); + self.first = false; + if (io.Information == 0) return null; + self.index = 0; + self.end_index = io.Information; + switch (rc) { + w.STATUS.SUCCESS => {}, + w.STATUS.ACCESS_DENIED => return error.AccessDenied, + else => return w.unexpectedStatus(rc), + } + } + + const aligned_ptr = @alignCast(@alignOf(w.FILE_BOTH_DIR_INFORMATION), &self.buf[self.index]); + const dir_info = @ptrCast(*w.FILE_BOTH_DIR_INFORMATION, aligned_ptr); + if (dir_info.NextEntryOffset != 0) { + self.index += dir_info.NextEntryOffset; + } else { + self.index = self.buf.len; + } + + const name_utf16le = @ptrCast([*]u16, &dir_info.FileName)[0 .. dir_info.FileNameLength / 2]; + + if (mem.eql(u16, name_utf16le, [_]u16{'.'}) or mem.eql(u16, name_utf16le, [_]u16{ '.', '.' })) + continue; + // Trust that Windows gives us valid UTF-16LE + const name_utf8_len = std.unicode.utf16leToUtf8(self.name_data[0..], name_utf16le) catch unreachable; + const name_utf8 = self.name_data[0..name_utf8_len]; + const kind = blk: { + const attrs = dir_info.FileAttributes; + if (attrs & w.FILE_ATTRIBUTE_DIRECTORY != 0) break :blk Entry.Kind.Directory; + if (attrs & w.FILE_ATTRIBUTE_REPARSE_POINT != 0) break :blk Entry.Kind.SymLink; + break :blk Entry.Kind.File; + }; + return Entry{ + .name = name_utf8, + .kind = kind, + }; + } + } + }, + else => @compileError("unimplemented"), + }; + + pub fn iterate(self: Dir) Iterator { + switch (builtin.os) { + .macosx, .ios, .freebsd, .netbsd => return Iterator{ + .dir = self, + .seek = 0, + .index = 0, + .end_index = 0, + .buf = undefined, + }, + .linux => return Iterator{ + .dir = self, + .index = 0, + .end_index = 0, + .buf = undefined, + }, + .windows => return Iterator{ + .dir = self, + .index = 0, + .end_index = 0, + .first = true, + .buf = undefined, + .name_data = undefined, + }, + else => @compileError("unimplemented"), + } + } + + /// Returns an open handle to the current working directory. + /// Closing the returned `Dir` is checked illegal behavior. + /// On POSIX targets, this function is comptime-callable. + pub fn cwd() Dir { + if (os.windows.is_the_target) { + return Dir{ .fd = os.windows.peb().ProcessParameters.CurrentDirectory.Handle }; + } else { + return Dir{ .fd = os.AT_FDCWD }; + } + } + pub const OpenError = error{ FileNotFound, NotDir, AccessDenied, - FileTooBig, - IsDir, SymLinkLoop, ProcessFdQuotaExceeded, NameTooLong, SystemFdQuotaExceeded, NoDevice, SystemResources, - NoSpaceLeft, - PathAlreadyExists, - OutOfMemory, InvalidUtf8, BadPathName, DeviceBusy, + } || os.UnexpectedError; - Unexpected, - }; + /// Call `close` to free the directory handle. + pub fn open(dir_path: []const u8) OpenError!Dir { + return cwd().openDir(dir_path); + } - /// Call close when done. - /// TODO remove the allocator requirement from this API - /// https://github.com/ziglang/zig/issues/2885 - pub fn open(allocator: *Allocator, dir_path: []const u8) OpenError!Dir { - return Dir{ - .allocator = allocator, - .handle = switch (builtin.os) { - .windows => blk: { - var find_file_data: os.windows.WIN32_FIND_DATAW = undefined; - const handle = try os.windows.FindFirstFile(dir_path, &find_file_data); - break :blk Handle{ - .handle = handle, - .find_file_data = find_file_data, // TODO guaranteed copy elision - .first = true, - .name_data = undefined, - }; - }, - .macosx, .ios, .freebsd, .netbsd => Handle{ - .fd = try os.open(dir_path, os.O_RDONLY | os.O_NONBLOCK | os.O_DIRECTORY | os.O_CLOEXEC, 0), - .seek = 0, - .index = 0, - .end_index = 0, - .buf = [_]u8{}, - }, - .linux => Handle{ - .fd = try os.open(dir_path, os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC, 0), - .index = 0, - .end_index = 0, - .buf = [_]u8{}, - }, - else => @compileError("unimplemented"), - }, - }; + /// Same as `open` except the parameter is null-terminated. + pub fn openC(dir_path_c: [*]const u8) OpenError!Dir { + return cwd().openDirC(dir_path_c); } pub fn close(self: *Dir) void { - if (os.windows.is_the_target) { - return os.windows.FindClose(self.handle.handle); - } - self.allocator.free(self.handle.buf); - os.close(self.handle.fd); + os.close(self.fd); + self.* = undefined; } - /// Memory such as file names referenced in this returned entry becomes invalid - /// with subsequent calls to next, as well as when this `Dir` is deinitialized. - pub fn next(self: *Dir) !?Entry { - switch (builtin.os) { - .linux => return self.nextLinux(), - .macosx, .ios => return self.nextDarwin(), - .windows => return self.nextWindows(), - .freebsd => return self.nextBsd(), - .netbsd => return self.nextBsd(), - else => @compileError("unimplemented"), - } - } - - pub fn openRead(self: Dir, file_path: []const u8) os.OpenError!File { - const path_c = try os.toPosixPath(file_path); + /// Call `File.close` on the result when done. + pub fn openRead(self: Dir, sub_path: []const u8) File.OpenError!File { + const path_c = try os.toPosixPath(sub_path); return self.openReadC(&path_c); } - pub fn openReadC(self: Dir, file_path: [*]const u8) OpenError!File { + /// Call `File.close` on the result when done. + pub fn openReadC(self: Dir, sub_path: [*]const u8) File.OpenError!File { const flags = os.O_LARGEFILE | os.O_RDONLY; - const fd = try os.openatC(self.handle.fd, file_path, flags, 0); + const fd = try os.openatC(self.fd, sub_path, flags, 0); return File.openHandle(fd); } - fn nextDarwin(self: *Dir) !?Entry { - start_over: while (true) { - if (self.handle.index >= self.handle.end_index) { - if (self.handle.buf.len == 0) { - self.handle.buf = try self.allocator.alloc(u8, mem.page_size); - } + /// Call `close` on the result when done. + pub fn openDir(self: Dir, sub_path: []const u8) OpenError!Dir { + if (os.windows.is_the_target) { + const sub_path_w = try os.windows.sliceToPrefixedFileW(sub_path); + return self.openDirW(&sub_path_w); + } - while (true) { - const rc = os.system.__getdirentries64( - self.handle.fd, - self.handle.buf.ptr, - self.handle.buf.len, - &self.handle.seek, - ); - if (rc == 0) return null; - if (rc < 0) { - switch (os.errno(rc)) { - os.EBADF => unreachable, - os.EFAULT => unreachable, - os.ENOTDIR => unreachable, - os.EINVAL => { - self.handle.buf = try self.allocator.realloc(self.handle.buf, self.handle.buf.len * 2); - continue; - }, - else => |err| return os.unexpectedErrno(err), - } + const sub_path_c = try os.toPosixPath(sub_path); + return self.openDirC(&sub_path_c); + } + + /// Same as `openDir` except the parameter is null-terminated. + pub fn openDirC(self: Dir, sub_path_c: [*]const u8) OpenError!Dir { + if (os.windows.is_the_target) { + const sub_path_w = try os.windows.cStrToPrefixedFileW(sub_path_c); + return self.openDirW(&sub_path_w); + } + + const flags = os.O_RDONLY | os.O_DIRECTORY | os.O_CLOEXEC; + const fd = os.openatC(self.fd, sub_path_c, flags, 0) catch |err| switch (err) { + error.FileTooBig => unreachable, // can't happen for directories + error.IsDir => unreachable, // we're providing O_DIRECTORY + error.NoSpaceLeft => unreachable, // not providing O_CREAT + error.PathAlreadyExists => unreachable, // not providing O_CREAT + else => |e| return e, + }; + return Dir{ .fd = fd }; + } + + /// Same as `openDir` except the path parameter is UTF16LE, NT-prefixed. + /// This function is Windows-only. + pub fn openDirW(self: Dir, sub_path_w: [*]const u16) OpenError!Dir { + const w = os.windows; + + var result = Dir{ + .fd = undefined, + }; + + const path_len_bytes = @intCast(u16, mem.toSliceConst(u16, sub_path_w).len * 2); + var nt_name = w.UNICODE_STRING{ + .Length = path_len_bytes, + .MaximumLength = path_len_bytes, + .Buffer = @intToPtr([*]u16, @ptrToInt(sub_path_w)), + }; + var attr = w.OBJECT_ATTRIBUTES{ + .Length = @sizeOf(w.OBJECT_ATTRIBUTES), + .RootDirectory = if (path.isAbsoluteW(sub_path_w)) null else self.fd, + .Attributes = 0, // Note we do not use OBJ_CASE_INSENSITIVE here. + .ObjectName = &nt_name, + .SecurityDescriptor = null, + .SecurityQualityOfService = null, + }; + if (sub_path_w[0] == '.' and sub_path_w[1] == 0) { + // Windows does not recognize this, but it does work with empty string. + nt_name.Length = 0; + } + if (sub_path_w[0] == '.' and sub_path_w[1] == '.' and sub_path_w[2] == 0) { + // If you're looking to contribute to zig and fix this, see here for an example of how to + // implement this: https://git.midipix.org/ntapi/tree/src/fs/ntapi_tt_open_physical_parent_directory.c + @panic("TODO opening '..' with a relative directory handle is not yet implemented on Windows"); + } + var io: w.IO_STATUS_BLOCK = undefined; + const rc = w.ntdll.NtCreateFile( + &result.fd, + w.GENERIC_READ | w.SYNCHRONIZE, + &attr, + &io, + null, + 0, + w.FILE_SHARE_READ | w.FILE_SHARE_WRITE, + w.FILE_OPEN, + w.FILE_DIRECTORY_FILE | w.FILE_SYNCHRONOUS_IO_NONALERT | w.FILE_OPEN_FOR_BACKUP_INTENT, + null, + 0, + ); + switch (rc) { + w.STATUS.SUCCESS => return result, + w.STATUS.OBJECT_NAME_INVALID => unreachable, + w.STATUS.OBJECT_NAME_NOT_FOUND => return error.FileNotFound, + w.STATUS.OBJECT_PATH_NOT_FOUND => return error.FileNotFound, + w.STATUS.INVALID_PARAMETER => unreachable, + else => return w.unexpectedStatus(rc), + } + } + + pub const DeleteFileError = os.UnlinkError; + + /// Delete a file name and possibly the file it refers to, based on an open directory handle. + pub fn deleteFile(self: Dir, sub_path: []const u8) DeleteFileError!void { + const sub_path_c = try os.toPosixPath(sub_path); + return self.deleteFileC(&sub_path_c); + } + + /// Same as `deleteFile` except the parameter is null-terminated. + pub fn deleteFileC(self: Dir, sub_path_c: [*]const u8) DeleteFileError!void { + os.unlinkatC(self.fd, sub_path_c, 0) catch |err| switch (err) { + error.DirNotEmpty => unreachable, // not passing AT_REMOVEDIR + else => |e| return e, + }; + } + + pub const DeleteDirError = error{ + DirNotEmpty, + FileNotFound, + AccessDenied, + FileBusy, + FileSystem, + SymLinkLoop, + NameTooLong, + NotDir, + SystemResources, + ReadOnlyFileSystem, + InvalidUtf8, + BadPathName, + Unexpected, + }; + + /// Returns `error.DirNotEmpty` if the directory is not empty. + /// To delete a directory recursively, see `deleteTree`. + pub fn deleteDir(self: Dir, sub_path: []const u8) DeleteDirError!void { + if (os.windows.is_the_target) { + const sub_path_w = try os.windows.sliceToPrefixedFileW(sub_path); + return self.deleteDirW(&sub_path_w); + } + const sub_path_c = try os.toPosixPath(sub_path); + return self.deleteDirC(&sub_path_c); + } + + /// Same as `deleteDir` except the parameter is null-terminated. + pub fn deleteDirC(self: Dir, sub_path_c: [*]const u8) DeleteDirError!void { + os.unlinkatC(self.fd, sub_path_c, os.AT_REMOVEDIR) catch |err| switch (err) { + error.IsDir => unreachable, // not possible since we pass AT_REMOVEDIR + else => |e| return e, + }; + } + + /// Same as `deleteDir` except the parameter is UTF16LE, NT prefixed. + /// This function is Windows-only. + pub fn deleteDirW(self: Dir, sub_path_w: [*]const u16) DeleteDirError!void { + os.unlinkatW(self.fd, sub_path_w, os.AT_REMOVEDIR) catch |err| switch (err) { + error.IsDir => unreachable, // not possible since we pass AT_REMOVEDIR + else => |e| return e, + }; + } + + /// Read value of a symbolic link. + /// The return value is a slice of `buffer`, from index `0`. + pub fn readLink(self: Dir, sub_path: []const u8, buffer: *[MAX_PATH_BYTES]u8) ![]u8 { + const sub_path_c = try os.toPosixPath(sub_path); + return self.readLinkC(&sub_path_c, buffer); + } + + /// Same as `readLink`, except the `pathname` parameter is null-terminated. + pub fn readLinkC(self: Dir, sub_path_c: [*]const u8, buffer: *[MAX_PATH_BYTES]u8) ![]u8 { + return os.readlinkatC(self.fd, sub_path_c, buffer); + } + + pub const DeleteTreeError = error{ + AccessDenied, + FileTooBig, + SymLinkLoop, + ProcessFdQuotaExceeded, + NameTooLong, + SystemFdQuotaExceeded, + NoDevice, + SystemResources, + ReadOnlyFileSystem, + FileSystem, + FileBusy, + DeviceBusy, + + /// One of the path components was not a directory. + /// This error is unreachable if `sub_path` does not contain a path separator. + NotDir, + + /// On Windows, file paths must be valid Unicode. + InvalidUtf8, + + /// On Windows, file paths cannot contain these characters: + /// '/', '*', '?', '"', '<', '>', '|' + BadPathName, + } || os.UnexpectedError; + + /// Whether `full_path` describes a symlink, file, or directory, this function + /// removes it. If it cannot be removed because it is a non-empty directory, + /// this function recursively removes its entries and then tries again. + /// This operation is not atomic on most file systems. + pub fn deleteTree(self: Dir, sub_path: []const u8) DeleteTreeError!void { + start_over: while (true) { + var got_access_denied = false; + // First, try deleting the item as a file. This way we don't follow sym links. + if (self.deleteFile(sub_path)) { + return; + } else |err| switch (err) { + error.FileNotFound => return, + error.IsDir => {}, + error.AccessDenied => got_access_denied = true, + + error.InvalidUtf8, + error.SymLinkLoop, + error.NameTooLong, + error.SystemResources, + error.ReadOnlyFileSystem, + error.NotDir, + error.FileSystem, + error.FileBusy, + error.BadPathName, + error.Unexpected, + => |e| return e, + } + var dir = self.openDir(sub_path) catch |err| switch (err) { + error.NotDir => { + if (got_access_denied) { + return error.AccessDenied; } - self.handle.index = 0; - self.handle.end_index = @intCast(usize, rc); - break; - } - } - const darwin_entry = @ptrCast(*align(1) os.dirent, &self.handle.buf[self.handle.index]); - const next_index = self.handle.index + darwin_entry.d_reclen; - self.handle.index = next_index; + continue :start_over; + }, + error.FileNotFound => { + // That's fine, we were trying to remove this directory anyway. + continue :start_over; + }, - const name = @ptrCast([*]u8, &darwin_entry.d_name)[0..darwin_entry.d_namlen]; - - if (mem.eql(u8, name, ".") or mem.eql(u8, name, "..")) { - continue :start_over; - } - - const entry_kind = switch (darwin_entry.d_type) { - os.DT_BLK => Entry.Kind.BlockDevice, - os.DT_CHR => Entry.Kind.CharacterDevice, - os.DT_DIR => Entry.Kind.Directory, - os.DT_FIFO => Entry.Kind.NamedPipe, - os.DT_LNK => Entry.Kind.SymLink, - os.DT_REG => Entry.Kind.File, - os.DT_SOCK => Entry.Kind.UnixDomainSocket, - os.DT_WHT => Entry.Kind.Whiteout, - else => Entry.Kind.Unknown, + error.AccessDenied, + error.SymLinkLoop, + error.ProcessFdQuotaExceeded, + error.NameTooLong, + error.SystemFdQuotaExceeded, + error.NoDevice, + error.SystemResources, + error.Unexpected, + error.InvalidUtf8, + error.BadPathName, + error.DeviceBusy, + => |e| return e, }; - return Entry{ - .name = name, - .kind = entry_kind, - }; - } - } + var cleanup_dir_parent: ?Dir = null; + defer if (cleanup_dir_parent) |*d| d.close(); - fn nextWindows(self: *Dir) !?Entry { - while (true) { - if (self.handle.first) { - self.handle.first = false; - } else { - if (!try os.windows.FindNextFile(self.handle.handle, &self.handle.find_file_data)) - return null; - } - const name_utf16le = mem.toSlice(u16, self.handle.find_file_data.cFileName[0..].ptr); - if (mem.eql(u16, name_utf16le, [_]u16{'.'}) or mem.eql(u16, name_utf16le, [_]u16{ '.', '.' })) - continue; - // Trust that Windows gives us valid UTF-16LE - const name_utf8_len = std.unicode.utf16leToUtf8(self.handle.name_data[0..], name_utf16le) catch unreachable; - const name_utf8 = self.handle.name_data[0..name_utf8_len]; - const kind = blk: { - const attrs = self.handle.find_file_data.dwFileAttributes; - if (attrs & os.windows.FILE_ATTRIBUTE_DIRECTORY != 0) break :blk Entry.Kind.Directory; - if (attrs & os.windows.FILE_ATTRIBUTE_REPARSE_POINT != 0) break :blk Entry.Kind.SymLink; - break :blk Entry.Kind.File; - }; - return Entry{ - .name = name_utf8, - .kind = kind, - }; - } - } + var cleanup_dir = true; + defer if (cleanup_dir) dir.close(); - fn nextLinux(self: *Dir) !?Entry { - start_over: while (true) { - if (self.handle.index >= self.handle.end_index) { - if (self.handle.buf.len == 0) { - self.handle.buf = try self.allocator.alloc(u8, mem.page_size); - } + var dir_name_buf: [MAX_PATH_BYTES]u8 = undefined; + var dir_name: []const u8 = sub_path; + var parent_dir = self; - while (true) { - const rc = os.linux.getdents64(self.handle.fd, self.handle.buf.ptr, self.handle.buf.len); - switch (os.linux.getErrno(rc)) { - 0 => {}, - os.EBADF => unreachable, - os.EFAULT => unreachable, - os.ENOTDIR => unreachable, - os.EINVAL => { - self.handle.buf = try self.allocator.realloc(self.handle.buf, self.handle.buf.len * 2); - continue; + // Here we must avoid recursion, in order to provide O(1) memory guarantee of this function. + // Go through each entry and if it is not a directory, delete it. If it is a directory, + // open it, and close the original directory. Repeat. Then start the entire operation over. + + scan_dir: while (true) { + var dir_it = dir.iterate(); + while (try dir_it.next()) |entry| { + if (dir.deleteFile(entry.name)) { + continue; + } else |err| switch (err) { + error.FileNotFound => continue, + + // Impossible because we do not pass any path separators. + error.NotDir => unreachable, + + error.IsDir => {}, + error.AccessDenied => got_access_denied = true, + + error.InvalidUtf8, + error.SymLinkLoop, + error.NameTooLong, + error.SystemResources, + error.ReadOnlyFileSystem, + error.FileSystem, + error.FileBusy, + error.BadPathName, + error.Unexpected, + => |e| return e, + } + + const new_dir = dir.openDir(entry.name) catch |err| switch (err) { + error.NotDir => { + if (got_access_denied) { + return error.AccessDenied; + } + continue :scan_dir; }, - else => |err| return os.unexpectedErrno(err), - } - if (rc == 0) return null; - self.handle.index = 0; - self.handle.end_index = rc; - break; - } - } - const linux_entry = @ptrCast(*align(1) os.dirent64, &self.handle.buf[self.handle.index]); - const next_index = self.handle.index + linux_entry.d_reclen; - self.handle.index = next_index; - - const name = mem.toSlice(u8, @ptrCast([*]u8, &linux_entry.d_name)); - - // skip . and .. entries - if (mem.eql(u8, name, ".") or mem.eql(u8, name, "..")) { - continue :start_over; - } - - const entry_kind = switch (linux_entry.d_type) { - os.DT_BLK => Entry.Kind.BlockDevice, - os.DT_CHR => Entry.Kind.CharacterDevice, - os.DT_DIR => Entry.Kind.Directory, - os.DT_FIFO => Entry.Kind.NamedPipe, - os.DT_LNK => Entry.Kind.SymLink, - os.DT_REG => Entry.Kind.File, - os.DT_SOCK => Entry.Kind.UnixDomainSocket, - else => Entry.Kind.Unknown, - }; - return Entry{ - .name = name, - .kind = entry_kind, - }; - } - } - - fn nextBsd(self: *Dir) !?Entry { - start_over: while (true) { - if (self.handle.index >= self.handle.end_index) { - if (self.handle.buf.len == 0) { - self.handle.buf = try self.allocator.alloc(u8, mem.page_size); - } - - while (true) { - const rc = os.system.getdirentries( - self.handle.fd, - self.handle.buf.ptr, - self.handle.buf.len, - &self.handle.seek, - ); - switch (os.errno(rc)) { - 0 => {}, - os.EBADF => unreachable, - os.EFAULT => unreachable, - os.ENOTDIR => unreachable, - os.EINVAL => { - self.handle.buf = try self.allocator.realloc(self.handle.buf, self.handle.buf.len * 2); - continue; + error.FileNotFound => { + // That's fine, we were trying to remove this directory anyway. + continue :scan_dir; }, - else => |err| return os.unexpectedErrno(err), - } - if (rc == 0) return null; - self.handle.index = 0; - self.handle.end_index = @intCast(usize, rc); - break; + + error.AccessDenied, + error.SymLinkLoop, + error.ProcessFdQuotaExceeded, + error.NameTooLong, + error.SystemFdQuotaExceeded, + error.NoDevice, + error.SystemResources, + error.Unexpected, + error.InvalidUtf8, + error.BadPathName, + error.DeviceBusy, + => |e| return e, + }; + if (cleanup_dir_parent) |*d| d.close(); + cleanup_dir_parent = dir; + dir = new_dir; + mem.copy(u8, &dir_name_buf, entry.name); + dir_name = dir_name_buf[0..entry.name.len]; + continue :scan_dir; + } + // Reached the end of the directory entries, which means we successfully deleted all of them. + // Now to remove the directory itself. + dir.close(); + cleanup_dir = false; + + if (cleanup_dir_parent) |d| { + d.deleteDir(dir_name) catch |err| switch (err) { + // These two things can happen due to file system race conditions. + error.FileNotFound, error.DirNotEmpty => continue :start_over, + else => |e| return e, + }; + continue :start_over; + } else { + self.deleteDir(sub_path) catch |err| switch (err) { + error.FileNotFound => return, + error.DirNotEmpty => continue :start_over, + else => |e| return e, + }; + return; } } - const freebsd_entry = @ptrCast(*align(1) os.dirent, &self.handle.buf[self.handle.index]); - const next_index = self.handle.index + freebsd_entry.d_reclen; - self.handle.index = next_index; - - const name = @ptrCast([*]u8, &freebsd_entry.d_name)[0..freebsd_entry.d_namlen]; - - if (mem.eql(u8, name, ".") or mem.eql(u8, name, "..")) { - continue :start_over; - } - - const entry_kind = switch (freebsd_entry.d_type) { - os.DT_BLK => Entry.Kind.BlockDevice, - os.DT_CHR => Entry.Kind.CharacterDevice, - os.DT_DIR => Entry.Kind.Directory, - os.DT_FIFO => Entry.Kind.NamedPipe, - os.DT_LNK => Entry.Kind.SymLink, - os.DT_REG => Entry.Kind.File, - os.DT_SOCK => Entry.Kind.UnixDomainSocket, - os.DT_WHT => Entry.Kind.Whiteout, - else => Entry.Kind.Unknown, - }; - return Entry{ - .name = name, - .kind = entry_kind, - }; } } }; @@ -782,13 +1046,18 @@ pub const Walker = struct { name_buffer: std.Buffer, pub const Entry = struct { - path: []const u8, + /// The containing directory. This can be used to operate directly on `basename` + /// rather than `path`, avoiding `error.NameTooLong` for deeply nested paths. + /// The directory remains open until `next` or `deinit` is called. + dir: Dir, basename: []const u8, + + path: []const u8, kind: Dir.Entry.Kind, }; const StackItem = struct { - dir_it: Dir, + dir_it: Dir.Iterator, dirname_len: usize, }; @@ -806,23 +1075,26 @@ pub const Walker = struct { try self.name_buffer.appendByte(path.sep); try self.name_buffer.append(base.name); if (base.kind == .Directory) { - // TODO https://github.com/ziglang/zig/issues/2888 - var new_dir = try Dir.open(self.stack.allocator, self.name_buffer.toSliceConst()); + var new_dir = top.dir_it.dir.openDir(base.name) catch |err| switch (err) { + error.NameTooLong => unreachable, // no path sep in base.name + else => |e| return e, + }; { errdefer new_dir.close(); try self.stack.append(StackItem{ - .dir_it = new_dir, + .dir_it = new_dir.iterate(), .dirname_len = self.name_buffer.len(), }); } } return Entry{ + .dir = top.dir_it.dir, .basename = self.name_buffer.toSliceConst()[dirname_len + 1 ..], .path = self.name_buffer.toSliceConst(), .kind = base.kind, }; } else { - self.stack.pop().dir_it.close(); + self.stack.pop().dir_it.dir.close(); } } } @@ -837,12 +1109,12 @@ pub const Walker = struct { /// Recursively iterates over a directory. /// Must call `Walker.deinit` when done. /// `dir_path` must not end in a path separator. -/// TODO: https://github.com/ziglang/zig/issues/2888 +/// The order of returned file system entries is undefined. pub fn walkPath(allocator: *Allocator, dir_path: []const u8) !Walker { assert(!mem.endsWith(u8, dir_path, path.sep_str)); - var dir_it = try Dir.open(allocator, dir_path); - errdefer dir_it.close(); + var dir = try Dir.open(dir_path); + errdefer dir.close(); var name_buffer = try std.Buffer.init(allocator, dir_path); errdefer name_buffer.deinit(); @@ -853,7 +1125,7 @@ pub fn walkPath(allocator: *Allocator, dir_path: []const u8) !Walker { }; try walker.stack.append(Walker.StackItem{ - .dir_it = dir_it, + .dir_it = dir.iterate(), .dirname_len = dir_path.len, }); @@ -862,15 +1134,13 @@ pub fn walkPath(allocator: *Allocator, dir_path: []const u8) !Walker { /// Read value of a symbolic link. /// The return value is a slice of buffer, from index `0`. -/// TODO https://github.com/ziglang/zig/issues/2888 -pub fn readLink(pathname: []const u8, buffer: *[os.PATH_MAX]u8) ![]u8 { +pub fn readLink(pathname: []const u8, buffer: *[MAX_PATH_BYTES]u8) ![]u8 { return os.readlink(pathname, buffer); } -/// Same as `readLink`, except the `pathname` parameter is null-terminated. -/// TODO https://github.com/ziglang/zig/issues/2888 -pub fn readLinkC(pathname: [*]const u8, buffer: *[os.PATH_MAX]u8) ![]u8 { - return os.readlinkC(pathname, buffer); +/// Same as `readLink`, except the parameter is null-terminated. +pub fn readLinkC(pathname_c: [*]const u8, buffer: *[MAX_PATH_BYTES]u8) ![]u8 { + return os.readlinkC(pathname_c, buffer); } pub const OpenSelfExeError = os.OpenError || os.windows.CreateFileError || SelfExePathError; diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index 32c221d063..a9212d69d4 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -243,6 +243,7 @@ pub const File = struct { switch (rc) { windows.STATUS.SUCCESS => {}, windows.STATUS.BUFFER_OVERFLOW => {}, + windows.STATUS.INVALID_PARAMETER => unreachable, else => return windows.unexpectedStatus(rc), } return Stat{ diff --git a/lib/std/fs/path.zig b/lib/std/fs/path.zig index 2bb23f04ce..560d9b58f3 100644 --- a/lib/std/fs/path.zig +++ b/lib/std/fs/path.zig @@ -136,6 +136,25 @@ pub fn isAbsolute(path: []const u8) bool { } } +pub fn isAbsoluteW(path_w: [*]const u16) bool { + if (path_w[0] == '/') + return true; + + if (path_w[0] == '\\') { + return true; + } + if (path_w[0] == 0 or path_w[1] == 0 or path_w[2] == 0) { + return false; + } + if (path_w[1] == ':') { + if (path_w[2] == '/') + return true; + if (path_w[2] == '\\') + return true; + } + return false; +} + pub fn isAbsoluteWindows(path: []const u8) bool { if (path[0] == '/') return true; diff --git a/lib/std/heap.zig b/lib/std/heap.zig index b968b6242f..f3a0d457d2 100644 --- a/lib/std/heap.zig +++ b/lib/std/heap.zig @@ -338,7 +338,7 @@ pub const HeapAllocator = switch (builtin.os) { /// This allocator takes an existing allocator, wraps it, and provides an interface /// where you can allocate without freeing, and then free it all together. pub const ArenaAllocator = struct { - pub allocator: Allocator, + allocator: Allocator, child_allocator: *Allocator, buffer_list: std.SinglyLinkedList([]u8), diff --git a/lib/std/http/headers.zig b/lib/std/http/headers.zig index a8dfa68629..7ee035ce80 100644 --- a/lib/std/http/headers.zig +++ b/lib/std/http/headers.zig @@ -28,9 +28,9 @@ fn never_index_default(name: []const u8) bool { const HeaderEntry = struct { allocator: *Allocator, - pub name: []const u8, - pub value: []u8, - pub never_index: bool, + name: []const u8, + value: []u8, + never_index: bool, const Self = @This(); diff --git a/lib/std/io.zig b/lib/std/io.zig index 4407e6b2b3..7502601650 100644 --- a/lib/std/io.zig +++ b/lib/std/io.zig @@ -127,6 +127,7 @@ pub fn OutStream(comptime WriteError: type) type { }; } +/// TODO move this to `std.fs` and add a version to `std.fs.Dir`. pub fn writeFile(path: []const u8, data: []const u8) !void { var file = try File.openWrite(path); defer file.close(); @@ -134,11 +135,13 @@ pub fn writeFile(path: []const u8, data: []const u8) !void { } /// On success, caller owns returned buffer. +/// TODO move this to `std.fs` and add a version to `std.fs.Dir`. pub fn readFileAlloc(allocator: *mem.Allocator, path: []const u8) ![]u8 { return readFileAllocAligned(allocator, path, @alignOf(u8)); } /// On success, caller owns returned buffer. +/// TODO move this to `std.fs` and add a version to `std.fs.Dir`. pub fn readFileAllocAligned(allocator: *mem.Allocator, path: []const u8, comptime A: u29) ![]align(A) u8 { var file = try File.openRead(path); defer file.close(); @@ -161,7 +164,7 @@ pub fn BufferedInStreamCustom(comptime buffer_size: usize, comptime Error: type) const Self = @This(); const Stream = InStream(Error); - pub stream: Stream, + stream: Stream, unbuffered_in_stream: *Stream, @@ -273,7 +276,7 @@ pub fn PeekStream(comptime buffer_size: usize, comptime InStreamError: type) typ pub const Error = InStreamError; pub const Stream = InStream(Error); - pub stream: Stream, + stream: Stream, base: *Stream, // Right now the look-ahead space is statically allocated, but a version with dynamic allocation @@ -336,7 +339,7 @@ pub const SliceInStream = struct { pub const Error = error{}; pub const Stream = InStream(Error); - pub stream: Stream, + stream: Stream, pos: usize, slice: []const u8, @@ -514,9 +517,9 @@ pub const SliceOutStream = struct { pub const Error = error{OutOfSpace}; pub const Stream = OutStream(Error); - pub stream: Stream, + stream: Stream, - pub pos: usize, + pos: usize, slice: []u8, pub fn init(slice: []u8) SliceOutStream { @@ -571,7 +574,7 @@ pub const NullOutStream = struct { pub const Error = error{}; pub const Stream = OutStream(Error); - pub stream: Stream, + stream: Stream, pub fn init() NullOutStream { return NullOutStream{ @@ -595,8 +598,8 @@ pub fn CountingOutStream(comptime OutStreamError: type) type { pub const Stream = OutStream(Error); pub const Error = OutStreamError; - pub stream: Stream, - pub bytes_written: u64, + stream: Stream, + bytes_written: u64, child_stream: *Stream, pub fn init(child_stream: *Stream) Self { @@ -635,7 +638,7 @@ pub fn BufferedOutStreamCustom(comptime buffer_size: usize, comptime OutStreamEr pub const Stream = OutStream(Error); pub const Error = OutStreamError; - pub stream: Stream, + stream: Stream, unbuffered_out_stream: *Stream, @@ -1084,7 +1087,7 @@ pub fn Deserializer(comptime endian: builtin.Endian, comptime packing: Packing, // safety. If it is bad, it will be caught anyway. const TagInt = @TagType(TagType); const tag = try self.deserializeInt(TagInt); - + inline for (info.fields) |field_info| { if (field_info.enum_field.?.value == tag) { const name = field_info.name; diff --git a/lib/std/io/seekable_stream.zig b/lib/std/io/seekable_stream.zig index 86f76d8c14..48dc31b785 100644 --- a/lib/std/io/seekable_stream.zig +++ b/lib/std/io/seekable_stream.zig @@ -39,8 +39,8 @@ pub const SliceSeekableInStream = struct { pub const Stream = InStream(Error); pub const SeekableInStream = SeekableStream(SeekError, GetSeekPosError); - pub stream: Stream, - pub seekable_stream: SeekableInStream, + stream: Stream, + seekable_stream: SeekableInStream, pos: usize, slice: []const u8, diff --git a/lib/std/os.zig b/lib/std/os.zig index de01da2fa5..3f8daf0290 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -529,22 +529,36 @@ pub fn pwritev(fd: fd_t, iov: []const iovec_const, offset: u64) WriteError!void pub const OpenError = error{ AccessDenied, - FileTooBig, - IsDir, SymLinkLoop, ProcessFdQuotaExceeded, - NameTooLong, SystemFdQuotaExceeded, NoDevice, FileNotFound, + /// The path exceeded `MAX_PATH_BYTES` bytes. + NameTooLong, + /// Insufficient kernel memory was available, or /// the named file is a FIFO and per-user hard limit on /// memory allocation for pipes has been reached. SystemResources, + /// The file is too large to be opened. This error is unreachable + /// for 64-bit targets, as well as when opening directories. + FileTooBig, + + /// The path refers to directory but the `O_DIRECTORY` flag was not provided. + IsDir, + + /// A new path cannot be created because the device has no room for the new file. + /// This error is only reachable when the `O_CREAT` flag is provided. NoSpaceLeft, + + /// A component used as a directory in the path was not, in fact, a directory, or + /// `O_DIRECTORY` was specified and the path was not a directory. NotDir, + + /// The path already exists and the `O_CREAT` and `O_EXCL` flags were provided. PathAlreadyExists, DeviceBusy, } || UnexpectedError; @@ -978,6 +992,114 @@ pub fn unlinkC(file_path: [*]const u8) UnlinkError!void { } } +pub const UnlinkatError = UnlinkError || error{ + /// When passing `AT_REMOVEDIR`, this error occurs when the named directory is not empty. + DirNotEmpty, +}; + +/// Delete a file name and possibly the file it refers to, based on an open directory handle. +pub fn unlinkat(dirfd: fd_t, file_path: []const u8, flags: u32) UnlinkatError!void { + if (windows.is_the_target) { + const file_path_w = try windows.sliceToPrefixedFileW(file_path); + return unlinkatW(dirfd, &file_path_w, flags); + } + const file_path_c = try toPosixPath(file_path); + return unlinkatC(dirfd, &file_path_c, flags); +} + +/// Same as `unlinkat` but `file_path` is a null-terminated string. +pub fn unlinkatC(dirfd: fd_t, file_path_c: [*]const u8, flags: u32) UnlinkatError!void { + if (windows.is_the_target) { + const file_path_w = try windows.cStrToPrefixedFileW(file_path_c); + return unlinkatW(dirfd, &file_path_w, flags); + } + switch (errno(system.unlinkat(dirfd, file_path_c, flags))) { + 0 => return, + EACCES => return error.AccessDenied, + EPERM => return error.AccessDenied, + EBUSY => return error.FileBusy, + EFAULT => unreachable, + EIO => return error.FileSystem, + EISDIR => return error.IsDir, + ELOOP => return error.SymLinkLoop, + ENAMETOOLONG => return error.NameTooLong, + ENOENT => return error.FileNotFound, + ENOTDIR => return error.NotDir, + ENOMEM => return error.SystemResources, + EROFS => return error.ReadOnlyFileSystem, + ENOTEMPTY => return error.DirNotEmpty, + + EINVAL => unreachable, // invalid flags, or pathname has . as last component + EBADF => unreachable, // always a race condition + + else => |err| return unexpectedErrno(err), + } +} + +/// Same as `unlinkat` but `sub_path_w` is UTF16LE, NT prefixed. Windows only. +pub fn unlinkatW(dirfd: fd_t, sub_path_w: [*]const u16, flags: u32) UnlinkatError!void { + const w = windows; + + const want_rmdir_behavior = (flags & AT_REMOVEDIR) != 0; + const create_options_flags = if (want_rmdir_behavior) + w.ULONG(w.FILE_DELETE_ON_CLOSE) + else + w.ULONG(w.FILE_DELETE_ON_CLOSE | w.FILE_NON_DIRECTORY_FILE); + + const path_len_bytes = @intCast(u16, mem.toSliceConst(u16, sub_path_w).len * 2); + var nt_name = w.UNICODE_STRING{ + .Length = path_len_bytes, + .MaximumLength = path_len_bytes, + // The Windows API makes this mutable, but it will not mutate here. + .Buffer = @intToPtr([*]u16, @ptrToInt(sub_path_w)), + }; + + if (sub_path_w[0] == '.' and sub_path_w[1] == 0) { + // Windows does not recognize this, but it does work with empty string. + nt_name.Length = 0; + } + if (sub_path_w[0] == '.' and sub_path_w[1] == '.' and sub_path_w[2] == 0) { + // Can't remove the parent directory with an open handle. + return error.FileBusy; + } + + + var attr = w.OBJECT_ATTRIBUTES{ + .Length = @sizeOf(w.OBJECT_ATTRIBUTES), + .RootDirectory = dirfd, + .Attributes = 0, // Note we do not use OBJ_CASE_INSENSITIVE here. + .ObjectName = &nt_name, + .SecurityDescriptor = null, + .SecurityQualityOfService = null, + }; + var io: w.IO_STATUS_BLOCK = undefined; + var tmp_handle: w.HANDLE = undefined; + var rc = w.ntdll.NtCreateFile( + &tmp_handle, + w.SYNCHRONIZE | w.DELETE, + &attr, + &io, + null, + 0, + w.FILE_SHARE_READ | w.FILE_SHARE_WRITE | w.FILE_SHARE_DELETE, + w.FILE_OPEN, + create_options_flags, + null, + 0, + ); + if (rc == w.STATUS.SUCCESS) { + rc = w.ntdll.NtClose(tmp_handle); + } + switch (rc) { + w.STATUS.SUCCESS => return, + w.STATUS.OBJECT_NAME_INVALID => unreachable, + w.STATUS.OBJECT_NAME_NOT_FOUND => return error.FileNotFound, + w.STATUS.INVALID_PARAMETER => unreachable, + w.STATUS.FILE_IS_A_DIRECTORY => return error.IsDir, + else => return w.unexpectedStatus(rc), + } +} + const RenameError = error{ AccessDenied, FileBusy, @@ -1237,6 +1359,27 @@ pub fn readlinkC(file_path: [*]const u8, out_buffer: []u8) ReadLinkError![]u8 { } } +pub fn readlinkatC(dirfd: fd_t, file_path: [*]const u8, out_buffer: []u8) ReadLinkError![]u8 { + if (windows.is_the_target) { + const file_path_w = try windows.cStrToPrefixedFileW(file_path); + @compileError("TODO implement readlink for Windows"); + } + const rc = system.readlinkat(dirfd, file_path, out_buffer.ptr, out_buffer.len); + switch (errno(rc)) { + 0 => return out_buffer[0..@bitCast(usize, rc)], + EACCES => return error.AccessDenied, + EFAULT => unreachable, + EINVAL => unreachable, + EIO => return error.FileSystem, + ELOOP => return error.SymLinkLoop, + ENAMETOOLONG => return error.NameTooLong, + ENOENT => return error.FileNotFound, + ENOMEM => return error.SystemResources, + ENOTDIR => return error.NotDir, + else => |err| return unexpectedErrno(err), + } +} + pub const SetIdError = error{ ResourceLimitReached, InvalidUserId, @@ -1476,18 +1619,46 @@ pub const AcceptError = error{ BlockedByFirewall, } || UnexpectedError; -/// Accept a connection on a socket. `fd` must be opened in blocking mode. -/// See also `accept4_async`. -pub fn accept4(fd: i32, addr: *sockaddr, flags: u32) AcceptError!i32 { +/// Accept a connection on a socket. +/// If the application has a global event loop enabled, EAGAIN is handled +/// via the event loop. Otherwise EAGAIN results in error.WouldBlock. +pub fn accept4( + /// This argument is a socket that has been created with `socket`, bound to a local address + /// with `bind`, and is listening for connections after a `listen`. + sockfd: i32, + /// This argument is a pointer to a sockaddr structure. This structure is filled in with the + /// address of the peer socket, as known to the communications layer. The exact format of the + /// address returned addr is determined by the socket's address family (see `socket` and the + /// respective protocol man pages). + addr: *sockaddr, + /// This argument is a value-result argument: the caller must initialize it to contain the + /// size (in bytes) of the structure pointed to by addr; on return it will contain the actual size + /// of the peer address. + /// + /// The returned address is truncated if the buffer provided is too small; in this case, `addr_size` + /// will return a value greater than was supplied to the call. + addr_size: *usize, + /// If flags is 0, then `accept4` is the same as `accept`. The following values can be bitwise + /// ORed in flags to obtain different behavior: + /// * `SOCK_NONBLOCK` - Set the `O_NONBLOCK` file status flag on the open file description (see `open`) + /// referred to by the new file descriptor. Using this flag saves extra calls to `fcntl` to achieve + /// the same result. + /// * `SOCK_CLOEXEC` - Set the close-on-exec (`FD_CLOEXEC`) flag on the new file descriptor. See the + /// description of the `O_CLOEXEC` flag in `open` for reasons why this may be useful. + flags: u32, +) AcceptError!i32 { while (true) { - var sockaddr_size = u32(@sizeOf(sockaddr)); - const rc = system.accept4(fd, addr, &sockaddr_size, flags); + const rc = system.accept4(sockfd, addr, addr_size, flags); switch (errno(rc)) { 0 => return @intCast(i32, rc), EINTR => continue, - else => |err| return unexpectedErrno(err), - EAGAIN => unreachable, // This function is for blocking only. + EAGAIN => if (std.event.Loop.instance) |loop| { + loop.waitUntilFdReadable(sockfd) catch return error.WouldBlock; + continue; + } else { + return error.WouldBlock; + }, EBADF => unreachable, // always a race condition ECONNABORTED => return error.ConnectionAborted, EFAULT => unreachable, @@ -1500,34 +1671,8 @@ pub fn accept4(fd: i32, addr: *sockaddr, flags: u32) AcceptError!i32 { EOPNOTSUPP => return error.OperationNotSupported, EPROTO => return error.ProtocolFailure, EPERM => return error.BlockedByFirewall, - } - } -} -/// This is the same as `accept4` except `fd` is expected to be non-blocking. -/// Returns -1 if would block. -pub fn accept4_async(fd: i32, addr: *sockaddr, flags: u32) AcceptError!i32 { - while (true) { - var sockaddr_size = u32(@sizeOf(sockaddr)); - const rc = system.accept4(fd, addr, &sockaddr_size, flags); - switch (errno(rc)) { - 0 => return @intCast(i32, rc), - EINTR => continue, else => |err| return unexpectedErrno(err), - - EAGAIN => return -1, - EBADF => unreachable, // always a race condition - ECONNABORTED => return error.ConnectionAborted, - EFAULT => unreachable, - EINVAL => unreachable, - EMFILE => return error.ProcessFdQuotaExceeded, - ENFILE => return error.SystemFdQuotaExceeded, - ENOBUFS => return error.SystemResources, - ENOMEM => return error.SystemResources, - ENOTSOCK => return error.FileDescriptorNotASocket, - EOPNOTSUPP => return error.OperationNotSupported, - EPROTO => return error.ProtocolFailure, - EPERM => return error.BlockedByFirewall, } } } diff --git a/lib/std/os/bits/darwin.zig b/lib/std/os/bits/darwin.zig index b076f95884..fe3156bb90 100644 --- a/lib/std/os/bits/darwin.zig +++ b/lib/std/os/bits/darwin.zig @@ -1178,3 +1178,17 @@ pub fn S_IWHT(m: u32) bool { return m & S_IFMT == S_IFWHT; } pub const HOST_NAME_MAX = 72; + +pub const AT_FDCWD = -2; + +/// Use effective ids in access check +pub const AT_EACCESS = 0x0010; + +/// Act on the symlink itself not the target +pub const AT_SYMLINK_NOFOLLOW = 0x0020; + +/// Act on target of symlink +pub const AT_SYMLINK_FOLLOW = 0x0040; + +/// Path refers to directory +pub const AT_REMOVEDIR = 0x0080; diff --git a/lib/std/os/bits/freebsd.zig b/lib/std/os/bits/freebsd.zig index 3d07e92e01..4427acb334 100644 --- a/lib/std/os/bits/freebsd.zig +++ b/lib/std/os/bits/freebsd.zig @@ -939,3 +939,23 @@ pub fn S_IWHT(m: u32) bool { } pub const HOST_NAME_MAX = 255; + +/// Magic value that specify the use of the current working directory +/// to determine the target of relative file paths in the openat() and +/// similar syscalls. +pub const AT_FDCWD = -100; + +/// Check access using effective user and group ID +pub const AT_EACCESS = 0x0100; + +/// Do not follow symbolic links +pub const AT_SYMLINK_NOFOLLOW = 0x0200; + +/// Follow symbolic link +pub const AT_SYMLINK_FOLLOW = 0x0400; + +/// Remove directory instead of file +pub const AT_REMOVEDIR = 0x0800; + +/// Fail if not under dirfd +pub const AT_BENEATH = 0x1000; diff --git a/lib/std/os/bits/windows.zig b/lib/std/os/bits/windows.zig index fc148d812f..c261b52bed 100644 --- a/lib/std/os/bits/windows.zig +++ b/lib/std/os/bits/windows.zig @@ -158,3 +158,6 @@ pub const EWOULDBLOCK = 140; pub const EDQUOT = 10069; pub const F_OK = 0; + +/// Remove directory instead of unlinking file +pub const AT_REMOVEDIR = 0x200; diff --git a/lib/std/os/test.zig b/lib/std/os/test.zig index 8d6b437ff3..b6e6728142 100644 --- a/lib/std/os/test.zig +++ b/lib/std/os/test.zig @@ -19,8 +19,8 @@ test "makePath, put some files in it, deleteTree" { try fs.makePath(a, "os_test_tmp" ++ fs.path.sep_str ++ "b" ++ fs.path.sep_str ++ "c"); try io.writeFile("os_test_tmp" ++ fs.path.sep_str ++ "b" ++ fs.path.sep_str ++ "c" ++ fs.path.sep_str ++ "file.txt", "nonsense"); try io.writeFile("os_test_tmp" ++ fs.path.sep_str ++ "b" ++ fs.path.sep_str ++ "file2.txt", "blah"); - try fs.deleteTree(a, "os_test_tmp"); - if (fs.Dir.open(a, "os_test_tmp")) |dir| { + try fs.deleteTree("os_test_tmp"); + if (fs.Dir.open("os_test_tmp")) |dir| { @panic("expected error"); } else |err| { expect(err == error.FileNotFound); @@ -37,7 +37,7 @@ test "access file" { try io.writeFile("os_test_tmp" ++ fs.path.sep_str ++ "file.txt", ""); try os.access("os_test_tmp" ++ fs.path.sep_str ++ "file.txt", os.F_OK); - try fs.deleteTree(a, "os_test_tmp"); + try fs.deleteTree("os_test_tmp"); } fn testThreadIdFn(thread_id: *Thread.Id) void { diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig index 7c1761a4b8..6c4aeb4cef 100644 --- a/lib/std/os/windows.zig +++ b/lib/std/os/windows.zig @@ -20,6 +20,8 @@ pub const shell32 = @import("windows/shell32.zig"); pub usingnamespace @import("windows/bits.zig"); +pub const self_process_handle = @intToPtr(HANDLE, maxInt(usize)); + /// `builtin` is missing `subsystem` when the subsystem is automatically detected, /// so Zig standard library has the subsystem detection logic here. This should generally be /// used rather than `builtin.subsystem`. @@ -42,7 +44,6 @@ pub const subsystem: ?builtin.SubSystem = blk: { break :blk builtin.SubSystem.Console; } }, - .uefi => break :blk builtin.SubSystem.EfiApplication, else => break :blk null, } }; @@ -792,6 +793,25 @@ pub fn SetFileTime( } } +pub fn peb() *PEB { + switch (builtin.arch) { + .i386 => { + return asm ( + \\ mov %%fs:0x18, %[ptr] + \\ mov %%ds:0x30(%[ptr]), %[ptr] + : [ptr] "=r" (-> *PEB) + ); + }, + .x86_64 => { + return asm ( + \\ mov %%gs:0x60, %[ptr] + : [ptr] "=r" (-> *PEB) + ); + }, + else => @compileError("unsupported architecture"), + } +} + /// A file time is a 64-bit value that represents the number of 100-nanosecond /// intervals that have elapsed since 12:00 A.M. January 1, 1601 Coordinated /// Universal Time (UTC). @@ -844,8 +864,8 @@ pub fn sliceToPrefixedSuffixedFileW(s: []const u8, comptime suffix: []const u16) else => {}, } } - const start_index = if (mem.startsWith(u8, s, "\\\\") or !std.fs.path.isAbsolute(s)) 0 else blk: { - const prefix = [_]u16{ '\\', '\\', '?', '\\' }; + const start_index = if (mem.startsWith(u8, s, "\\?") or !std.fs.path.isAbsolute(s)) 0 else blk: { + const prefix = [_]u16{ '\\', '?', '?', '\\' }; mem.copy(u16, result[0..], prefix); break :blk prefix.len; }; @@ -879,7 +899,7 @@ pub fn unexpectedError(err: DWORD) std.os.UnexpectedError { /// and you get an unexpected status. pub fn unexpectedStatus(status: NTSTATUS) std.os.UnexpectedError { if (std.os.unexpected_error_tracing) { - std.debug.warn("error.Unexpected NTSTATUS={}\n", status); + std.debug.warn("error.Unexpected NTSTATUS=0x{x}\n", status); std.debug.dumpCurrentStackTrace(null); } return error.Unexpected; diff --git a/lib/std/os/windows/bits.zig b/lib/std/os/windows/bits.zig index ddfdd27e1b..214f75186f 100644 --- a/lib/std/os/windows/bits.zig +++ b/lib/std/os/windows/bits.zig @@ -300,6 +300,44 @@ pub const FILE_SHARE_DELETE = 0x00000004; pub const FILE_SHARE_READ = 0x00000001; pub const FILE_SHARE_WRITE = 0x00000002; +pub const DELETE = 0x00010000; +pub const READ_CONTROL = 0x00020000; +pub const WRITE_DAC = 0x00040000; +pub const WRITE_OWNER = 0x00080000; +pub const SYNCHRONIZE = 0x00100000; +pub const STANDARD_RIGHTS_REQUIRED = 0x000f0000; + +// disposition for NtCreateFile +pub const FILE_SUPERSEDE = 0; +pub const FILE_OPEN = 1; +pub const FILE_CREATE = 2; +pub const FILE_OPEN_IF = 3; +pub const FILE_OVERWRITE = 4; +pub const FILE_OVERWRITE_IF = 5; +pub const FILE_MAXIMUM_DISPOSITION = 5; + +// flags for NtCreateFile and NtOpenFile +pub const FILE_DIRECTORY_FILE = 0x00000001; +pub const FILE_WRITE_THROUGH = 0x00000002; +pub const FILE_SEQUENTIAL_ONLY = 0x00000004; +pub const FILE_NO_INTERMEDIATE_BUFFERING = 0x00000008; +pub const FILE_SYNCHRONOUS_IO_ALERT = 0x00000010; +pub const FILE_SYNCHRONOUS_IO_NONALERT = 0x00000020; +pub const FILE_NON_DIRECTORY_FILE = 0x00000040; +pub const FILE_CREATE_TREE_CONNECTION = 0x00000080; +pub const FILE_COMPLETE_IF_OPLOCKED = 0x00000100; +pub const FILE_NO_EA_KNOWLEDGE = 0x00000200; +pub const FILE_OPEN_FOR_RECOVERY = 0x00000400; +pub const FILE_RANDOM_ACCESS = 0x00000800; +pub const FILE_DELETE_ON_CLOSE = 0x00001000; +pub const FILE_OPEN_BY_FILE_ID = 0x00002000; +pub const FILE_OPEN_FOR_BACKUP_INTENT = 0x00004000; +pub const FILE_NO_COMPRESSION = 0x00008000; +pub const FILE_RESERVE_OPFILTER = 0x00100000; +pub const FILE_TRANSACTED_MODE = 0x00200000; +pub const FILE_OPEN_OFFLINE_FILE = 0x00400000; +pub const FILE_OPEN_FOR_FREE_SPACE_QUERY = 0x00800000; + pub const CREATE_ALWAYS = 2; pub const CREATE_NEW = 1; pub const OPEN_ALWAYS = 4; @@ -720,15 +758,119 @@ pub const VECTORED_EXCEPTION_HANDLER = stdcallcc fn (ExceptionInfo: *EXCEPTION_P pub const OBJECT_ATTRIBUTES = extern struct { Length: ULONG, - RootDirectory: HANDLE, + RootDirectory: ?HANDLE, ObjectName: *UNICODE_STRING, Attributes: ULONG, SecurityDescriptor: ?*c_void, SecurityQualityOfService: ?*c_void, }; +pub const OBJ_INHERIT = 0x00000002; +pub const OBJ_PERMANENT = 0x00000010; +pub const OBJ_EXCLUSIVE = 0x00000020; +pub const OBJ_CASE_INSENSITIVE = 0x00000040; +pub const OBJ_OPENIF = 0x00000080; +pub const OBJ_OPENLINK = 0x00000100; +pub const OBJ_KERNEL_HANDLE = 0x00000200; +pub const OBJ_VALID_ATTRIBUTES = 0x000003F2; + pub const UNICODE_STRING = extern struct { - Length: USHORT, - MaximumLength: USHORT, + Length: c_ushort, + MaximumLength: c_ushort, Buffer: [*]WCHAR, }; + +pub const PEB = extern struct { + Reserved1: [2]BYTE, + BeingDebugged: BYTE, + Reserved2: [1]BYTE, + Reserved3: [2]PVOID, + Ldr: *PEB_LDR_DATA, + ProcessParameters: *RTL_USER_PROCESS_PARAMETERS, + Reserved4: [3]PVOID, + AtlThunkSListPtr: PVOID, + Reserved5: PVOID, + Reserved6: ULONG, + Reserved7: PVOID, + Reserved8: ULONG, + AtlThunkSListPtr32: ULONG, + Reserved9: [45]PVOID, + Reserved10: [96]BYTE, + PostProcessInitRoutine: PPS_POST_PROCESS_INIT_ROUTINE, + Reserved11: [128]BYTE, + Reserved12: [1]PVOID, + SessionId: ULONG, +}; + +pub const PEB_LDR_DATA = extern struct { + Reserved1: [8]BYTE, + Reserved2: [3]PVOID, + InMemoryOrderModuleList: LIST_ENTRY, +}; + +pub const RTL_USER_PROCESS_PARAMETERS = extern struct { + AllocationSize: ULONG, + Size: ULONG, + Flags: ULONG, + DebugFlags: ULONG, + ConsoleHandle: HANDLE, + ConsoleFlags: ULONG, + hStdInput: HANDLE, + hStdOutput: HANDLE, + hStdError: HANDLE, + CurrentDirectory: CURDIR, + DllPath: UNICODE_STRING, + ImagePathName: UNICODE_STRING, + CommandLine: UNICODE_STRING, + Environment: [*]WCHAR, + dwX: ULONG, + dwY: ULONG, + dwXSize: ULONG, + dwYSize: ULONG, + dwXCountChars: ULONG, + dwYCountChars: ULONG, + dwFillAttribute: ULONG, + dwFlags: ULONG, + dwShowWindow: ULONG, + WindowTitle: UNICODE_STRING, + Desktop: UNICODE_STRING, + ShellInfo: UNICODE_STRING, + RuntimeInfo: UNICODE_STRING, + DLCurrentDirectory: [0x20]RTL_DRIVE_LETTER_CURDIR, +}; + +pub const RTL_DRIVE_LETTER_CURDIR = extern struct { + Flags: c_ushort, + Length: c_ushort, + TimeStamp: ULONG, + DosPath: UNICODE_STRING, +}; + +pub const PPS_POST_PROCESS_INIT_ROUTINE = ?extern fn () void; + +pub const FILE_BOTH_DIR_INFORMATION = extern struct { + NextEntryOffset: ULONG, + FileIndex: ULONG, + CreationTime: LARGE_INTEGER, + LastAccessTime: LARGE_INTEGER, + LastWriteTime: LARGE_INTEGER, + ChangeTime: LARGE_INTEGER, + EndOfFile: LARGE_INTEGER, + AllocationSize: LARGE_INTEGER, + FileAttributes: ULONG, + FileNameLength: ULONG, + EaSize: ULONG, + ShortNameLength: CHAR, + ShortName: [12]WCHAR, + FileName: [1]WCHAR, +}; +pub const FILE_BOTH_DIRECTORY_INFORMATION = FILE_BOTH_DIR_INFORMATION; + +pub const IO_APC_ROUTINE = extern fn (PVOID, *IO_STATUS_BLOCK, ULONG) void; + +pub const CURDIR = extern struct { + DosPath: UNICODE_STRING, + Handle: HANDLE, +}; + +pub const DUPLICATE_SAME_ACCESS = 2; \ No newline at end of file diff --git a/lib/std/os/windows/kernel32.zig b/lib/std/os/windows/kernel32.zig index 2ae73ad45a..736d81ae58 100644 --- a/lib/std/os/windows/kernel32.zig +++ b/lib/std/os/windows/kernel32.zig @@ -47,6 +47,8 @@ pub extern "kernel32" stdcallcc fn CreateThread(lpThreadAttributes: ?LPSECURITY_ pub extern "kernel32" stdcallcc fn DeleteFileW(lpFileName: [*]const u16) BOOL; +pub extern "kernel32" stdcallcc fn DuplicateHandle(hSourceProcessHandle: HANDLE, hSourceHandle: HANDLE, hTargetProcessHandle: HANDLE, lpTargetHandle: *HANDLE, dwDesiredAccess: DWORD, bInheritHandle: BOOL, dwOptions: DWORD) BOOL; + pub extern "kernel32" stdcallcc fn ExitProcess(exit_code: UINT) noreturn; pub extern "kernel32" stdcallcc fn FindFirstFileW(lpFileName: [*]const u16, lpFindFileData: *WIN32_FIND_DATAW) HANDLE; diff --git a/lib/std/os/windows/ntdll.zig b/lib/std/os/windows/ntdll.zig index bfc98aba8a..f914920093 100644 --- a/lib/std/os/windows/ntdll.zig +++ b/lib/std/os/windows/ntdll.zig @@ -13,12 +13,33 @@ pub extern "NtDll" stdcallcc fn NtCreateFile( DesiredAccess: ACCESS_MASK, ObjectAttributes: *OBJECT_ATTRIBUTES, IoStatusBlock: *IO_STATUS_BLOCK, - AllocationSize: *LARGE_INTEGER, + AllocationSize: ?*LARGE_INTEGER, FileAttributes: ULONG, ShareAccess: ULONG, CreateDisposition: ULONG, CreateOptions: ULONG, - EaBuffer: *c_void, + EaBuffer: ?*c_void, EaLength: ULONG, ) NTSTATUS; pub extern "NtDll" stdcallcc fn NtClose(Handle: HANDLE) NTSTATUS; +pub extern "NtDll" stdcallcc fn RtlDosPathNameToNtPathName_U( + DosPathName: [*]const u16, + NtPathName: *UNICODE_STRING, + NtFileNamePart: ?*?[*]const u16, + DirectoryInfo: ?*CURDIR, +) BOOL; +pub extern "NtDll" stdcallcc fn RtlFreeUnicodeString(UnicodeString: *UNICODE_STRING) void; + +pub extern "NtDll" stdcallcc fn NtQueryDirectoryFile( + FileHandle: HANDLE, + Event: ?HANDLE, + ApcRoutine: ?IO_APC_ROUTINE, + ApcContext: ?*c_void, + IoStatusBlock: *IO_STATUS_BLOCK, + FileInformation: *c_void, + Length: ULONG, + FileInformationClass: FILE_INFORMATION_CLASS, + ReturnSingleEntry: BOOLEAN, + FileName: ?*UNICODE_STRING, + RestartScan: BOOLEAN, +) NTSTATUS; diff --git a/lib/std/os/windows/status.zig b/lib/std/os/windows/status.zig index b9fd2b495f..5f38948620 100644 --- a/lib/std/os/windows/status.zig +++ b/lib/std/os/windows/status.zig @@ -3,3 +3,11 @@ pub const SUCCESS = 0x00000000; /// The data was too large to fit into the specified buffer. pub const BUFFER_OVERFLOW = 0x80000005; + +pub const INVALID_PARAMETER = 0xC000000D; +pub const ACCESS_DENIED = 0xC0000022; +pub const OBJECT_NAME_INVALID = 0xC0000033; +pub const OBJECT_NAME_NOT_FOUND = 0xC0000034; +pub const OBJECT_PATH_NOT_FOUND = 0xC000003A; +pub const OBJECT_PATH_SYNTAX_BAD = 0xC000003B; +pub const FILE_IS_A_DIRECTORY = 0xC00000BA; diff --git a/lib/std/progress.zig b/lib/std/progress.zig index 1d29763c10..fba77092b3 100644 --- a/lib/std/progress.zig +++ b/lib/std/progress.zig @@ -155,11 +155,11 @@ pub const Progress = struct { } if (node.estimated_total_items) |total| { if (need_ellipse) self.bufWrite(&end, " "); - self.bufWrite(&end, "[{}/{}] ", node.completed_items, total); + self.bufWrite(&end, "[{}/{}] ", node.completed_items + 1, total); need_ellipse = false; } else if (node.completed_items != 0) { if (need_ellipse) self.bufWrite(&end, " "); - self.bufWrite(&end, "[{}] ", node.completed_items); + self.bufWrite(&end, "[{}] ", node.completed_items + 1); need_ellipse = false; } } diff --git a/lib/std/special/docs/index.html b/lib/std/special/docs/index.html index bf71d6c04c..8ce1033235 100644 --- a/lib/std/special/docs/index.html +++ b/lib/std/special/docs/index.html @@ -5,62 +5,228 @@ Documentation - Zig - - -
- -

Loading...

- - -

- - - - - - -
diff --git a/lib/std/special/test_runner.zig b/lib/std/special/test_runner.zig index 8fb1b5842d..2733775001 100644 --- a/lib/std/special/test_runner.zig +++ b/lib/std/special/test_runner.zig @@ -15,20 +15,29 @@ pub fn main() anyerror!void { for (test_fn_list) |test_fn, i| { var test_node = root_node.start(test_fn.name, null); test_node.activate(); + progress.refresh(); + if (progress.terminal == null) std.debug.warn("{}/{} {}...", i + 1, test_fn_list.len, test_fn.name); if (test_fn.func()) |_| { ok_count += 1; test_node.end(); + if (progress.terminal == null) std.debug.warn("OK\n"); } else |err| switch (err) { error.SkipZigTest => { skip_count += 1; test_node.end(); progress.log("{}...SKIP\n", test_fn.name); + if (progress.terminal == null) std.debug.warn("SKIP\n"); + }, + else => { + progress.log(""); + return err; }, - else => return err, } } root_node.end(); - if (ok_count != test_fn_list.len) { - progress.log("{} passed; {} skipped.\n", ok_count, skip_count); + if (ok_count == test_fn_list.len) { + std.debug.warn("All tests passed.\n"); + } else { + std.debug.warn("{} passed; {} skipped.\n", ok_count, skip_count); } } diff --git a/lib/std/zig/ast.zig b/lib/std/zig/ast.zig index 79e6647144..9f32c6e54b 100644 --- a/lib/std/zig/ast.zig +++ b/lib/std/zig/ast.zig @@ -290,7 +290,7 @@ pub const Error = union(enum) { pub const ExpectedSuffixOp = SingleTokenError("Expected pointer dereference, optional unwrap, or field access, found '{}'"); pub const ExpectedParamType = SimpleError("Expected parameter type"); - pub const ExpectedPubItem = SimpleError("Pub must be followed by fn decl, var decl, or container member"); + pub const ExpectedPubItem = SimpleError("Expected function or variable declaration after pub"); pub const UnattachedDocComment = SimpleError("Unattached documentation comment"); pub const ExtraAlignQualifier = SimpleError("Extra align qualifier"); pub const ExtraConstQualifier = SimpleError("Extra const qualifier"); @@ -757,7 +757,6 @@ pub const Node = struct { pub const ContainerField = struct { base: Node, doc_comments: ?*DocComment, - visib_token: ?TokenIndex, name_token: TokenIndex, type_expr: ?*Node, value_expr: ?*Node, @@ -780,7 +779,6 @@ pub const Node = struct { } pub fn firstToken(self: *const ContainerField) TokenIndex { - if (self.visib_token) |visib_token| return visib_token; return self.name_token; } diff --git a/lib/std/zig/parse.zig b/lib/std/zig/parse.zig index 151784284d..71fa376361 100644 --- a/lib/std/zig/parse.zig +++ b/lib/std/zig/parse.zig @@ -138,9 +138,15 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No continue; } + if (visib_token != null) { + try tree.errors.push(AstError{ + .ExpectedPubItem = AstError.ExpectedPubItem{ .token = it.index }, + }); + return error.ParseError; + } + if (try parseContainerField(arena, it, tree)) |node| { const field = node.cast(Node.ContainerField).?; - field.visib_token = visib_token; field.doc_comments = doc_comments; try list.push(node); const comma = eatToken(it, .Comma) orelse break; @@ -149,13 +155,6 @@ fn parseContainerMembers(arena: *Allocator, it: *TokenIterator, tree: *Tree) !No continue; } - // Dangling pub - if (visib_token != null) { - try tree.errors.push(AstError{ - .ExpectedPubItem = AstError.ExpectedPubItem{ .token = it.index }, - }); - } - break; } @@ -407,7 +406,6 @@ fn parseContainerField(arena: *Allocator, it: *TokenIterator, tree: *Tree) !?*No node.* = Node.ContainerField{ .base = Node{ .id = .ContainerField }, .doc_comments = null, - .visib_token = null, .name_token = name_token, .type_expr = type_expr, .value_expr = value_expr, diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index 78f80786c5..28bd287c8a 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -1766,7 +1766,7 @@ test "zig fmt: struct declaration" { \\const S = struct { \\ const Self = @This(); \\ f1: u8, - \\ pub f3: u8, + \\ f3: u8, \\ \\ fn method(self: *Self) Self { \\ return self.*; @@ -1777,14 +1777,14 @@ test "zig fmt: struct declaration" { \\ \\const Ps = packed struct { \\ a: u8, - \\ pub b: u8, + \\ b: u8, \\ \\ c: u8, \\}; \\ \\const Es = extern struct { \\ a: u8, - \\ pub b: u8, + \\ b: u8, \\ \\ c: u8, \\}; diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index 1be4410f38..e5875fd04f 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -254,10 +254,6 @@ fn renderTopLevelDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree, i try renderDocComments(tree, stream, field, indent, start_col); - if (field.visib_token) |visib_token| { - try renderToken(tree, stream, visib_token, indent, start_col, Space.Space); // pub - } - if (field.type_expr == null and field.value_expr == null) { return renderToken(tree, stream, field.name_token, indent, start_col, Space.Comma); // name, } else if (field.type_expr != null and field.value_expr == null) { @@ -2206,8 +2202,8 @@ const FindByteOutStream = struct { pub const Error = error{}; pub const Stream = std.io.OutStream(Error); - pub stream: Stream, - pub byte_found: bool, + stream: Stream, + byte_found: bool, byte: u8, pub fn init(byte: u8) Self { diff --git a/src-self-hosted/main.zig b/src-self-hosted/main.zig index 52eda5824a..a2e77394ce 100644 --- a/src-self-hosted/main.zig +++ b/src-self-hosted/main.zig @@ -747,7 +747,7 @@ async fn fmtPath(fmt: *Fmt, file_path_ref: []const u8, check_mode: bool) FmtErro )) catch |err| switch (err) { error.IsDir, error.AccessDenied => { // TODO make event based (and dir.next()) - var dir = try fs.Dir.open(fmt.loop.allocator, file_path); + var dir = try fs.Dir.open(file_path); defer dir.close(); var group = event.Group(FmtError!void).init(fmt.loop); diff --git a/src-self-hosted/stage1.zig b/src-self-hosted/stage1.zig index 48b14ebcac..5a0294fa7d 100644 --- a/src-self-hosted/stage1.zig +++ b/src-self-hosted/stage1.zig @@ -283,11 +283,13 @@ fn fmtPath(fmt: *Fmt, file_path_ref: []const u8, check_mode: bool) FmtError!void const source_code = io.readFileAlloc(fmt.allocator, file_path) catch |err| switch (err) { error.IsDir, error.AccessDenied => { // TODO make event based (and dir.next()) - var dir = try fs.Dir.open(fmt.allocator, file_path); + var dir = try fs.Dir.open(file_path); defer dir.close(); - while (try dir.next()) |entry| { - if (entry.kind == fs.Dir.Entry.Kind.Directory or mem.endsWith(u8, entry.name, ".zig")) { + var dir_it = dir.iterate(); + + while (try dir_it.next()) |entry| { + if (entry.kind == .Directory or mem.endsWith(u8, entry.name, ".zig")) { const full_path = try fs.path.join(fmt.allocator, [_][]const u8{ file_path, entry.name }); try fmtPath(fmt, full_path, check_mode); } diff --git a/src-self-hosted/test.zig b/src-self-hosted/test.zig index 526518ca47..ce51de9a7b 100644 --- a/src-self-hosted/test.zig +++ b/src-self-hosted/test.zig @@ -56,11 +56,11 @@ pub const TestContext = struct { errdefer allocator.free(self.zig_lib_dir); try std.fs.makePath(allocator, tmp_dir_name); - errdefer std.fs.deleteTree(allocator, tmp_dir_name) catch {}; + errdefer std.fs.deleteTree(tmp_dir_name) catch {}; } fn deinit(self: *TestContext) void { - std.fs.deleteTree(allocator, tmp_dir_name) catch {}; + std.fs.deleteTree(tmp_dir_name) catch {}; allocator.free(self.zig_lib_dir); self.zig_compiler.deinit(); self.loop.deinit(); diff --git a/src/all_types.hpp b/src/all_types.hpp index f2e614f192..267c5479be 100644 --- a/src/all_types.hpp +++ b/src/all_types.hpp @@ -990,8 +990,6 @@ struct AstNodeStructField { // populated if the "align(A)" is present AstNode *align_expr; Buf doc_comments; - - VisibMod visib_mod; }; struct AstNodeStringLiteral { @@ -2569,12 +2567,12 @@ enum IrInstructionId { struct IrInstruction { Scope *scope; AstNode *source_node; - ConstExprValue value; - size_t debug_id; LLVMValueRef llvm_value; + ConstExprValue value; + uint32_t debug_id; // if ref_count is zero and the instruction has no side effects, // the instruction can be omitted in codegen - size_t ref_count; + uint32_t ref_count; // When analyzing IR, instructions that point to this instruction in the "old ir" // can find the instruction that corresponds to this value in the "new ir" // with this child field. diff --git a/src/analyze.cpp b/src/analyze.cpp index 92b221a2ec..ed4547957b 100644 --- a/src/analyze.cpp +++ b/src/analyze.cpp @@ -5783,8 +5783,8 @@ ConstExprValue *create_const_arg_tuple(CodeGen *g, size_t arg_index_start, size_ ConstExprValue *create_const_vals(size_t count) { - ConstGlobalRefs *global_refs = allocate(count); - ConstExprValue *vals = allocate(count); + ConstGlobalRefs *global_refs = allocate(count, "ConstGlobalRefs"); + ConstExprValue *vals = allocate(count, "ConstExprValue"); for (size_t i = 0; i < count; i += 1) { vals[i].global_refs = &global_refs[i]; } diff --git a/src/codegen.cpp b/src/codegen.cpp index 18af74afd7..82d2480eb9 100644 --- a/src/codegen.cpp +++ b/src/codegen.cpp @@ -6355,12 +6355,17 @@ static LLVMValueRef gen_const_ptr_union_recursive(CodeGen *g, ConstExprValue *un ConstParent *parent = &union_const_val->parent; LLVMValueRef base_ptr = gen_parent_ptr(g, union_const_val, parent); + // Slot in the structure where the payload is stored, if equal to SIZE_MAX + // the union has no tag and a single field and is collapsed into the field + // itself + size_t union_payload_index = union_const_val->type->data.unionation.gen_union_index; + ZigType *u32 = g->builtin_types.entry_u32; LLVMValueRef indices[] = { LLVMConstNull(get_llvm_type(g, u32)), - LLVMConstInt(get_llvm_type(g, u32), 0, false), // TODO test const union with more aligned tag type than payload + LLVMConstInt(get_llvm_type(g, u32), union_payload_index, false), }; - return LLVMConstInBoundsGEP(base_ptr, indices, 2); + return LLVMConstInBoundsGEP(base_ptr, indices, (union_payload_index != SIZE_MAX) ? 2 : 1); } static LLVMValueRef pack_const_int(CodeGen *g, LLVMTypeRef big_int_type_ref, ConstExprValue *const_val) { diff --git a/src/config.h.in b/src/config.h.in index a99aab0d72..7a0ea3536a 100644 --- a/src/config.h.in +++ b/src/config.h.in @@ -13,9 +13,6 @@ #define ZIG_VERSION_PATCH @ZIG_VERSION_PATCH@ #define ZIG_VERSION_STRING "@ZIG_VERSION@" -// Only used for running tests before installing. -#define ZIG_TEST_DIR "@CMAKE_SOURCE_DIR@/test" - // Used for communicating build information to self hosted build. #define ZIG_CMAKE_BINARY_DIR "@CMAKE_BINARY_DIR@" #define ZIG_CXX_COMPILER "@CMAKE_CXX_COMPILER@" @@ -24,4 +21,6 @@ #define ZIG_LLVM_CONFIG_EXE "@LLVM_CONFIG_EXE@" #define ZIG_DIA_GUIDS_LIB "@ZIG_DIA_GUIDS_LIB_ESCAPED@" +#cmakedefine ZIG_ENABLE_MEM_PROFILE + #endif diff --git a/src/dump_analysis.cpp b/src/dump_analysis.cpp index d5b109fbcb..9703d3e57d 100644 --- a/src/dump_analysis.cpp +++ b/src/dump_analysis.cpp @@ -240,23 +240,6 @@ static void jw_string(JsonWriter *jw, const char *s) { static void tree_print(FILE *f, ZigType *ty, size_t indent); -static void pretty_print_bytes(FILE *f, double n) { - if (n > 1024.0 * 1024.0 * 1024.0) { - fprintf(f, "%.02f GiB", n / 1024.0 / 1024.0 / 1024.0); - return; - } - if (n > 1024.0 * 1024.0) { - fprintf(f, "%.02f MiB", n / 1024.0 / 1024.0); - return; - } - if (n > 1024.0) { - fprintf(f, "%.02f KiB", n / 1024.0); - return; - } - fprintf(f, "%.02f bytes", n ); - return; -} - static int compare_type_abi_sizes_desc(const void *a, const void *b) { uint64_t size_a = (*(ZigType * const*)(a))->abi_size; uint64_t size_b = (*(ZigType * const*)(b))->abi_size; @@ -322,7 +305,7 @@ static void tree_print(FILE *f, ZigType *ty, size_t indent) { start_peer(f, indent); fprintf(f, "\"sizef\": \""); - pretty_print_bytes(f, ty->abi_size); + zig_pretty_print_bytes(f, ty->abi_size); fprintf(f, "\""); start_peer(f, indent); diff --git a/src/ir.cpp b/src/ir.cpp index 7dd141423c..4cce464a4c 100644 --- a/src/ir.cpp +++ b/src/ir.cpp @@ -413,7 +413,7 @@ ZigType *ir_analyze_type_expr(IrAnalyze *ira, Scope *scope, AstNode *node) { } static IrBasicBlock *ir_create_basic_block(IrBuilder *irb, Scope *scope, const char *name_hint) { - IrBasicBlock *result = allocate(1); + IrBasicBlock *result = allocate(1, "IrBasicBlock"); result->scope = scope; result->name_hint = name_hint; result->debug_id = exec_next_debug_id(irb->exec); @@ -1085,13 +1085,18 @@ static constexpr IrInstructionId ir_instruction_id(IrInstructionSpillEnd *) { template static T *ir_create_instruction(IrBuilder *irb, Scope *scope, AstNode *source_node) { - T *special_instruction = allocate(1); + const char *name = nullptr; +#ifdef ZIG_ENABLE_MEM_PROFILE + T *dummy = nullptr; + name = ir_instruction_type_str(ir_instruction_id(dummy)); +#endif + T *special_instruction = allocate(1, name); special_instruction->base.id = ir_instruction_id(special_instruction); special_instruction->base.scope = scope; special_instruction->base.source_node = source_node; special_instruction->base.debug_id = exec_next_debug_id(irb->exec); special_instruction->base.owner_bb = irb->current_basic_block; - special_instruction->base.value.global_refs = allocate(1); + special_instruction->base.value.global_refs = allocate(1, "ConstGlobalRefs"); return special_instruction; } @@ -3569,7 +3574,7 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node, switch (node->data.return_expr.kind) { case ReturnKindUnconditional: { - ResultLocReturn *result_loc_ret = allocate(1); + ResultLocReturn *result_loc_ret = allocate(1, "ResultLocReturn"); result_loc_ret->base.id = ResultLocIdReturn; ir_build_reset_result(irb, scope, node, &result_loc_ret->base); @@ -3664,7 +3669,7 @@ static IrInstruction *ir_gen_return(IrBuilder *irb, Scope *scope, AstNode *node, ir_mark_gen(ir_build_add_implicit_return_type(irb, scope, node, err_val, nullptr)); IrInstructionSpillBegin *spill_begin = ir_build_spill_begin(irb, scope, node, err_val, SpillIdRetErrCode); - ResultLocReturn *result_loc_ret = allocate(1); + ResultLocReturn *result_loc_ret = allocate(1, "ResultLocReturn"); result_loc_ret->base.id = ResultLocIdReturn; ir_build_reset_result(irb, scope, node, &result_loc_ret->base); ir_build_end_expr(irb, scope, node, err_val, &result_loc_ret->base); @@ -3692,7 +3697,7 @@ static ZigVar *create_local_var(CodeGen *codegen, AstNode *node, Scope *parent_s Buf *name, bool src_is_const, bool gen_is_const, bool is_shadowable, IrInstruction *is_comptime, bool skip_name_check) { - ZigVar *variable_entry = allocate(1); + ZigVar *variable_entry = allocate(1, "ZigVar"); variable_entry->parent_scope = parent_scope; variable_entry->shadowable = is_shadowable; variable_entry->mem_slot_index = SIZE_MAX; @@ -3767,7 +3772,7 @@ static ZigVar *ir_create_var(IrBuilder *irb, AstNode *node, Scope *scope, Buf *n } static ResultLocPeer *create_peer_result(ResultLocPeerParent *peer_parent) { - ResultLocPeer *result = allocate(1); + ResultLocPeer *result = allocate(1, "ResultLocPeer"); result->base.id = ResultLocIdPeer; result->base.source_instruction = peer_parent->base.source_instruction; result->parent = peer_parent; @@ -3806,7 +3811,7 @@ static IrInstruction *ir_gen_block(IrBuilder *irb, Scope *parent_scope, AstNode scope_block->is_comptime = ir_build_const_bool(irb, parent_scope, block_node, ir_should_inline(irb->exec, parent_scope)); - scope_block->peer_parent = allocate(1); + scope_block->peer_parent = allocate(1, "ResultLocPeerParent"); scope_block->peer_parent->base.id = ResultLocIdPeerParent; scope_block->peer_parent->base.source_instruction = scope_block->is_comptime; scope_block->peer_parent->end_bb = scope_block->end_block; @@ -3933,7 +3938,7 @@ static IrInstruction *ir_gen_assign(IrBuilder *irb, Scope *scope, AstNode *node) if (lvalue == irb->codegen->invalid_instruction) return irb->codegen->invalid_instruction; - ResultLocInstruction *result_loc_inst = allocate(1); + ResultLocInstruction *result_loc_inst = allocate(1, "ResultLocInstruction"); result_loc_inst->base.id = ResultLocIdInstruction; result_loc_inst->base.source_instruction = lvalue; ir_ref_instruction(lvalue, irb->current_basic_block); @@ -4005,10 +4010,10 @@ static IrInstruction *ir_gen_bool_or(IrBuilder *irb, Scope *scope, AstNode *node ir_set_cursor_at_end_and_append_block(irb, true_block); - IrInstruction **incoming_values = allocate(2); + IrInstruction **incoming_values = allocate(2, "IrInstruction *"); incoming_values[0] = val1; incoming_values[1] = val2; - IrBasicBlock **incoming_blocks = allocate(2); + IrBasicBlock **incoming_blocks = allocate(2, "IrBasicBlock *"); incoming_blocks[0] = post_val1_block; incoming_blocks[1] = post_val2_block; @@ -8017,7 +8022,8 @@ static IrInstruction *ir_gen_err_set_decl(IrBuilder *irb, Scope *parent_scope, A err_set_type->abi_size = irb->codegen->builtin_types.entry_global_error_set->abi_size; err_set_type->data.error_set.errors = allocate(err_count); - ErrorTableEntry **errors = allocate(irb->codegen->errors_by_index.length + err_count); + size_t errors_count = irb->codegen->errors_by_index.length + err_count; + ErrorTableEntry **errors = allocate(errors_count, "ErrorTableEntry *"); for (uint32_t i = 0; i < err_count; i += 1) { AstNode *field_node = node->data.err_set_decl.decls.at(i); @@ -8048,7 +8054,7 @@ static IrInstruction *ir_gen_err_set_decl(IrBuilder *irb, Scope *parent_scope, A } errors[err->value] = err; } - free(errors); + deallocate(errors, errors_count, "ErrorTableEntry *"); return ir_build_const_type(irb, parent_scope, node, err_set_type); } @@ -9574,7 +9580,8 @@ static ZigType *get_error_set_intersection(IrAnalyze *ira, ZigType *set1, ZigTyp if (type_is_global_error_set(set2)) { return set1; } - ErrorTableEntry **errors = allocate(ira->codegen->errors_by_index.length); + size_t errors_count = ira->codegen->errors_by_index.length; + ErrorTableEntry **errors = allocate(errors_count, "ErrorTableEntry *"); populate_error_set_table(errors, set1); ZigList intersection_list = {}; @@ -9595,7 +9602,7 @@ static ZigType *get_error_set_intersection(IrAnalyze *ira, ZigType *set1, ZigTyp buf_appendf(&err_set_type->name, "%s%s", comma, buf_ptr(&existing_entry_with_docs->name)); } } - free(errors); + deallocate(errors, errors_count, "ErrorTableEntry *"); err_set_type->data.error_set.err_count = intersection_list.length; err_set_type->data.error_set.errors = intersection_list.items; @@ -9792,7 +9799,8 @@ static ConstCastOnly types_match_const_cast_only(IrAnalyze *ira, ZigType *wanted return result; } - ErrorTableEntry **errors = allocate(g->errors_by_index.length); + size_t errors_count = g->errors_by_index.length; + ErrorTableEntry **errors = allocate(errors_count, "ErrorTableEntry *"); for (uint32_t i = 0; i < container_set->data.error_set.err_count; i += 1) { ErrorTableEntry *error_entry = container_set->data.error_set.errors[i]; assert(errors[error_entry->value] == nullptr); @@ -9809,7 +9817,7 @@ static ConstCastOnly types_match_const_cast_only(IrAnalyze *ira, ZigType *wanted result.data.error_set_mismatch->missing_errors.append(contained_error_entry); } } - free(errors); + deallocate(errors, errors_count, "ErrorTableEntry *"); return result; } @@ -10112,6 +10120,18 @@ static ZigType *ir_resolve_peer_types(IrAnalyze *ira, AstNode *source_node, ZigT } else { err_set_type = cur_type; } + + if (!resolve_inferred_error_set(ira->codegen, err_set_type, cur_inst->source_node)) { + return ira->codegen->builtin_types.entry_invalid; + } + + if (type_is_global_error_set(err_set_type)) { + err_set_type = ira->codegen->builtin_types.entry_global_error_set; + continue; + } + + update_errors_helper(ira->codegen, &errors, &errors_count); + for (uint32_t i = 0; i < err_set_type->data.error_set.err_count; i += 1) { ErrorTableEntry *error_entry = err_set_type->data.error_set.errors[i]; assert(errors[error_entry->value] == nullptr); @@ -10814,7 +10834,8 @@ static IrInstruction *ira_suspend(IrAnalyze *ira, IrInstruction *old_instruction IrSuspendPosition *suspend_pos) { if (ira->codegen->verbose_ir) { - fprintf(stderr, "suspend %s_%zu %s_%zu #%zu (%zu,%zu)\n", ira->old_irb.current_basic_block->name_hint, + fprintf(stderr, "suspend %s_%zu %s_%zu #%" PRIu32 " (%zu,%zu)\n", + ira->old_irb.current_basic_block->name_hint, ira->old_irb.current_basic_block->debug_id, ira->old_irb.exec->basic_block_list.at(ira->old_bb_index)->name_hint, ira->old_irb.exec->basic_block_list.at(ira->old_bb_index)->debug_id, @@ -10852,7 +10873,7 @@ static IrInstruction *ira_resume(IrAnalyze *ira) { ira->instruction_index = pos.instruction_index; assert(pos.instruction_index < ira->old_irb.current_basic_block->instruction_list.length); if (ira->codegen->verbose_ir) { - fprintf(stderr, "%s_%zu #%zu\n", ira->old_irb.current_basic_block->name_hint, + fprintf(stderr, "%s_%zu #%" PRIu32 "\n", ira->old_irb.current_basic_block->name_hint, ira->old_irb.current_basic_block->debug_id, ira->old_irb.current_basic_block->instruction_list.at(pos.instruction_index)->debug_id); } @@ -14686,14 +14707,15 @@ static IrInstruction *ir_analyze_instruction_merge_err_sets(IrAnalyze *ira, return ira->codegen->invalid_instruction; } - ErrorTableEntry **errors = allocate(ira->codegen->errors_by_index.length); + size_t errors_count = ira->codegen->errors_by_index.length; + ErrorTableEntry **errors = allocate(errors_count, "ErrorTableEntry *"); for (uint32_t i = 0, count = op1_type->data.error_set.err_count; i < count; i += 1) { ErrorTableEntry *error_entry = op1_type->data.error_set.errors[i]; assert(errors[error_entry->value] == nullptr); errors[error_entry->value] = error_entry; } ZigType *result_type = get_error_set_union(ira->codegen, errors, op1_type, op2_type, instruction->type_name); - free(errors); + deallocate(errors, errors_count, "ErrorTableEntry *"); return ir_const_type(ira, &instruction->base, result_type); } @@ -19241,7 +19263,7 @@ static IrInstruction *ir_analyze_instruction_switch_target(IrAnalyze *ira, ZigType *target_type = target_value_ptr->value.type->data.pointer.child_type; ConstExprValue *pointee_val = nullptr; - if (instr_is_comptime(target_value_ptr)) { + if (instr_is_comptime(target_value_ptr) && target_value_ptr->value.data.x_ptr.mut != ConstPtrMutRuntimeVar) { pointee_val = const_ptr_pointee(ira, ira->codegen, &target_value_ptr->value, target_value_ptr->source_node); if (pointee_val == nullptr) return ira->codegen->invalid_instruction; @@ -23074,17 +23096,22 @@ static IrInstruction *ir_analyze_instruction_slice(IrAnalyze *ira, IrInstruction zig_unreachable(); } - uint64_t start_scalar = bigint_as_u64(&casted_start->value.data.x_bigint); + ConstExprValue *start_val = ir_resolve_const(ira, casted_start, UndefBad); + if (!start_val) + return ira->codegen->invalid_instruction; + + uint64_t start_scalar = bigint_as_u64(&start_val->data.x_bigint); if (!ptr_is_undef && start_scalar > rel_end) { ir_add_error(ira, &instruction->base, buf_sprintf("out of bounds slice")); return ira->codegen->invalid_instruction; } - uint64_t end_scalar; + uint64_t end_scalar = rel_end; if (end) { - end_scalar = bigint_as_u64(&end->value.data.x_bigint); - } else { - end_scalar = rel_end; + ConstExprValue *end_val = ir_resolve_const(ira, end, UndefBad); + if (!end_val) + return ira->codegen->invalid_instruction; + end_scalar = bigint_as_u64(&end_val->data.x_bigint); } if (!ptr_is_undef) { if (end_scalar > rel_end) { @@ -24034,7 +24061,8 @@ static IrInstruction *ir_analyze_instruction_check_switch_prongs(IrAnalyze *ira, return ira->codegen->invalid_instruction; } - AstNode **field_prev_uses = allocate(ira->codegen->errors_by_index.length); + size_t field_prev_uses_count = ira->codegen->errors_by_index.length; + AstNode **field_prev_uses = allocate(field_prev_uses_count, "AstNode *"); for (size_t range_i = 0; range_i < instruction->range_count; range_i += 1) { IrInstructionCheckSwitchProngsRange *range = &instruction->ranges[range_i]; @@ -24091,7 +24119,7 @@ static IrInstruction *ir_analyze_instruction_check_switch_prongs(IrAnalyze *ira, } } - free(field_prev_uses); + deallocate(field_prev_uses, field_prev_uses_count, "AstNode *"); } else if (switch_type->id == ZigTypeIdInt) { RangeSet rs = {0}; for (size_t range_i = 0; range_i < instruction->range_count; range_i += 1) { @@ -26318,7 +26346,7 @@ ZigType *ir_analyze(CodeGen *codegen, IrExecutable *old_exec, IrExecutable *new_ } if (ira->codegen->verbose_ir) { - fprintf(stderr, "analyze #%zu\n", old_instruction->debug_id); + fprintf(stderr, "analyze #%" PRIu32 "\n", old_instruction->debug_id); } IrInstruction *new_instruction = ir_analyze_instruction_base(ira, old_instruction); if (new_instruction != nullptr) { diff --git a/src/ir_print.cpp b/src/ir_print.cpp index ecd8248d69..d3c77f3638 100644 --- a/src/ir_print.cpp +++ b/src/ir_print.cpp @@ -38,8 +38,8 @@ struct IrPrint { static void ir_print_other_instruction(IrPrint *irp, IrInstruction *instruction); -static const char* ir_instruction_type_str(IrInstruction* instruction) { - switch (instruction->id) { +const char* ir_instruction_type_str(IrInstructionId id) { + switch (id) { case IrInstructionIdInvalid: return "Invalid"; case IrInstructionIdShuffleVector: @@ -385,9 +385,9 @@ static void ir_print_prefix(IrPrint *irp, IrInstruction *instruction, bool trail const char mark = trailing ? ':' : '#'; const char *type_name = instruction->value.type ? buf_ptr(&instruction->value.type->name) : "(unknown)"; const char *ref_count = ir_has_side_effects(instruction) ? - "-" : buf_ptr(buf_sprintf("%" ZIG_PRI_usize "", instruction->ref_count)); - fprintf(irp->f, "%c%-3zu| %-22s| %-12s| %-2s| ", mark, instruction->debug_id, - ir_instruction_type_str(instruction), type_name, ref_count); + "-" : buf_ptr(buf_sprintf("%" PRIu32 "", instruction->ref_count)); + fprintf(irp->f, "%c%-3" PRIu32 "| %-22s| %-12s| %-2s| ", mark, instruction->debug_id, + ir_instruction_type_str(instruction->id), type_name, ref_count); } static void ir_print_const_value(IrPrint *irp, ConstExprValue *const_val) { @@ -398,7 +398,7 @@ static void ir_print_const_value(IrPrint *irp, ConstExprValue *const_val) { } static void ir_print_var_instruction(IrPrint *irp, IrInstruction *instruction) { - fprintf(irp->f, "#%" ZIG_PRI_usize "", instruction->debug_id); + fprintf(irp->f, "#%" PRIu32 "", instruction->debug_id); if (irp->pass != IrPassSrc && irp->printed.maybe_get(instruction) == nullptr) { irp->printed.put(instruction, 0); irp->pending.append(instruction); diff --git a/src/ir_print.hpp b/src/ir_print.hpp index 0960af4e6f..e3947077c8 100644 --- a/src/ir_print.hpp +++ b/src/ir_print.hpp @@ -15,4 +15,6 @@ void ir_print(CodeGen *codegen, FILE *f, IrExecutable *executable, int indent_size, IrPass pass); void ir_print_instruction(CodeGen *codegen, FILE *f, IrInstruction *instruction, int indent_size, IrPass pass); +const char* ir_instruction_type_str(IrInstructionId id); + #endif diff --git a/src/main.cpp b/src/main.cpp index b398eec991..4709035859 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -64,6 +64,9 @@ static int print_full_usage(const char *arg0, FILE *file, int return_code) { " -fno-PIC disable Position Independent Code\n" " -ftime-report print timing diagnostics\n" " -fstack-report print stack size diagnostics\n" +#ifdef ZIG_ENABLE_MEM_PROFILE + " -fmem-report print memory usage diagnostics\n" +#endif " -fdump-analysis write analysis.json file with type information\n" " -femit-docs create a docs/ dir with html documentation\n" " -fno-emit-bin skip emitting machine code\n" @@ -306,9 +309,29 @@ static int zig_error_no_build_file(void) { extern "C" int ZigClang_main(int argc, char **argv); +#ifdef ZIG_ENABLE_MEM_PROFILE +bool mem_report = false; +#endif + +int main_exit(Stage2ProgressNode *root_progress_node, int exit_code) { + if (root_progress_node != nullptr) { + stage2_progress_end(root_progress_node); + } +#ifdef ZIG_ENABLE_MEM_PROFILE + if (mem_report) { + memprof_dump_stats(stderr); + } +#endif + return exit_code; +} + int main(int argc, char **argv) { stage2_attach_segfault_handler(); +#ifdef ZIG_ENABLE_MEM_PROFILE + memprof_init(); +#endif + char *arg0 = argv[0]; Error err; @@ -670,6 +693,13 @@ int main(int argc, char **argv) { timing_info = true; } else if (strcmp(arg, "-fstack-report") == 0) { stack_report = true; + } else if (strcmp(arg, "-fmem-report") == 0) { +#ifdef ZIG_ENABLE_MEM_PROFILE + mem_report = true; +#else + fprintf(stderr, "-fmem-report requires configuring with -DZIG_ENABLE_MEM_PROFILE=ON\n"); + return print_error_usage(arg0); +#endif } else if (strcmp(arg, "-fdump-analysis") == 0) { enable_dump_analysis = true; } else if (strcmp(arg, "-femit-docs") == 0) { @@ -1038,16 +1068,14 @@ int main(int argc, char **argv) { if (in_file) { ZigLibCInstallation libc; if ((err = zig_libc_parse(&libc, buf_create_from_str(in_file), &target, true))) - return EXIT_FAILURE; - stage2_progress_end(root_progress_node); - return EXIT_SUCCESS; + return main_exit(root_progress_node, EXIT_FAILURE); + return main_exit(root_progress_node, EXIT_SUCCESS); } ZigLibCInstallation libc; if ((err = zig_libc_find_native(&libc, true))) - return EXIT_FAILURE; + return main_exit(root_progress_node, EXIT_FAILURE); zig_libc_render(&libc, stdout); - stage2_progress_end(root_progress_node); - return EXIT_SUCCESS; + return main_exit(root_progress_node, EXIT_SUCCESS); } case CmdBuiltin: { CodeGen *g = codegen_create(main_pkg_path, nullptr, &target, @@ -1065,10 +1093,9 @@ int main(int argc, char **argv) { Buf *builtin_source = codegen_generate_builtin_source(g); if (fwrite(buf_ptr(builtin_source), 1, buf_len(builtin_source), stdout) != buf_len(builtin_source)) { fprintf(stderr, "unable to write to stdout: %s\n", strerror(ferror(stdout))); - return EXIT_FAILURE; + return main_exit(root_progress_node, EXIT_FAILURE); } - stage2_progress_end(root_progress_node); - return EXIT_SUCCESS; + return main_exit(root_progress_node, EXIT_SUCCESS); } case CmdRun: case CmdBuild: @@ -1142,7 +1169,7 @@ int main(int argc, char **argv) { libc = allocate(1); if ((err = zig_libc_parse(libc, buf_create_from_str(libc_txt), &target, true))) { fprintf(stderr, "Unable to parse --libc text file: %s\n", err_str(err)); - return EXIT_FAILURE; + return main_exit(root_progress_node, EXIT_FAILURE); } } Buf *cache_dir_buf; @@ -1219,7 +1246,7 @@ int main(int argc, char **argv) { codegen_set_rdynamic(g, rdynamic); if (mmacosx_version_min && mios_version_min) { fprintf(stderr, "-mmacosx-version-min and -mios-version-min options not allowed together\n"); - return EXIT_FAILURE; + return main_exit(root_progress_node, EXIT_FAILURE); } if (mmacosx_version_min) { @@ -1259,6 +1286,11 @@ int main(int argc, char **argv) { zig_print_stack_report(g, stdout); if (cmd == CmdRun) { + stage2_progress_end(root_progress_node); +#ifdef ZIG_ENABLE_MEM_PROFILE + memprof_dump_stats(stderr); +#endif + const char *exec_path = buf_ptr(&g->output_file_path); ZigList args = {0}; @@ -1282,10 +1314,9 @@ int main(int argc, char **argv) { buf_replace(&g->output_file_path, '/', '\\'); #endif if (printf("%s\n", buf_ptr(&g->output_file_path)) < 0) - return EXIT_FAILURE; + return main_exit(root_progress_node, EXIT_FAILURE); } - stage2_progress_end(root_progress_node); - return EXIT_SUCCESS; + return main_exit(root_progress_node, EXIT_SUCCESS); } else { zig_unreachable(); } @@ -1293,8 +1324,7 @@ int main(int argc, char **argv) { codegen_translate_c(g, in_file_buf, stdout, cmd == CmdTranslateCUserland); if (timing_info) codegen_print_timing_report(g, stderr); - stage2_progress_end(root_progress_node); - return EXIT_SUCCESS; + return main_exit(root_progress_node, EXIT_SUCCESS); } else if (cmd == CmdTest) { codegen_set_emit_file_type(g, emit_file_type); @@ -1314,7 +1344,7 @@ int main(int argc, char **argv) { if (g->disable_bin_generation) { fprintf(stderr, "Semantic analysis complete. No binary produced due to -fno-emit-bin.\n"); - return 0; + return main_exit(root_progress_node, EXIT_SUCCESS); } Buf *test_exe_path_unresolved = &g->output_file_path; @@ -1324,7 +1354,7 @@ int main(int argc, char **argv) { if (emit_file_type != EmitFileTypeBinary) { fprintf(stderr, "Created %s but skipping execution because it is non executable.\n", buf_ptr(test_exe_path)); - return 0; + return main_exit(root_progress_node, EXIT_SUCCESS); } for (size_t i = 0; i < test_exec_args.length; i += 1) { @@ -1336,7 +1366,7 @@ int main(int argc, char **argv) { if (!target_can_exec(&native, &target) && test_exec_args.length == 0) { fprintf(stderr, "Created %s but skipping execution because it is non-native.\n", buf_ptr(test_exe_path)); - return 0; + return main_exit(root_progress_node, EXIT_SUCCESS); } Termination term; @@ -1348,21 +1378,20 @@ int main(int argc, char **argv) { fprintf(stderr, "\nTests failed. Use the following command to reproduce the failure:\n"); fprintf(stderr, "%s\n", buf_ptr(test_exe_path)); } - stage2_progress_end(root_progress_node); - return (term.how == TerminationIdClean) ? term.code : -1; + return main_exit(root_progress_node, (term.how == TerminationIdClean) ? term.code : -1); } else { zig_unreachable(); } } case CmdVersion: printf("%s\n", ZIG_VERSION_STRING); - return EXIT_SUCCESS; + return main_exit(root_progress_node, EXIT_SUCCESS); case CmdZen: { const char *ptr; size_t len; stage2_zen(&ptr, &len); fwrite(ptr, len, 1, stdout); - return EXIT_SUCCESS; + return main_exit(root_progress_node, EXIT_SUCCESS); } case CmdTargets: return print_target_list(stdout); diff --git a/src/memory_profiling.cpp b/src/memory_profiling.cpp new file mode 100644 index 0000000000..494ffae117 --- /dev/null +++ b/src/memory_profiling.cpp @@ -0,0 +1,139 @@ +#include "memory_profiling.hpp" +#include "hash_map.hpp" +#include "list.hpp" +#include "util.hpp" +#include + +#ifdef ZIG_ENABLE_MEM_PROFILE + +static bool str_eql_str(const char *a, const char *b) { + return strcmp(a, b) == 0; +} + +static uint32_t str_hash(const char *s) { + // FNV 32-bit hash + uint32_t h = 2166136261; + for (; *s; s += 1) { + h = h ^ *s; + h = h * 16777619; + } + return h; +} + +struct CountAndSize { + size_t item_count; + size_t type_size; +}; + +ZigList unknown_names = {}; +HashMap usage_table = {}; +bool table_active = false; + + +static const char *get_default_name(const char *name_or_null, size_t type_size) { + if (name_or_null != nullptr) return name_or_null; + if (type_size >= unknown_names.length) { + table_active = false; + unknown_names.resize(type_size + 1); + table_active = true; + } + if (unknown_names.at(type_size) == nullptr) { + char buf[100]; + sprintf(buf, "Unknown_%zu%c", type_size, 0); + unknown_names.at(type_size) = strdup(buf); + } + return unknown_names.at(type_size); +} + +void memprof_alloc(const char *name, size_t count, size_t type_size) { + if (!table_active) return; + if (count == 0) return; + // temporarily disable during table put + table_active = false; + name = get_default_name(name, type_size); + auto existing_entry = usage_table.put_unique(name, {count, type_size}); + if (existing_entry != nullptr) { + assert(existing_entry->value.type_size == type_size); // allocated name does not match type + existing_entry->value.item_count += count; + } + table_active = true; +} + +void memprof_dealloc(const char *name, size_t count, size_t type_size) { + if (!table_active) return; + if (count == 0) return; + name = get_default_name(name, type_size); + auto existing_entry = usage_table.maybe_get(name); + if (existing_entry == nullptr) { + zig_panic("deallocated more than allocated; compromised memory usage stats"); + } + if (existing_entry->value.type_size != type_size) { + zig_panic("deallocated name '%s' does not match expected type size %zu", name, type_size); + } + existing_entry->value.item_count -= count; +} + +void memprof_init(void) { + usage_table.init(1024); + table_active = true; +} + +struct MemItem { + const char *type_name; + CountAndSize count_and_size; +}; + +static size_t get_bytes(const MemItem *item) { + return item->count_and_size.item_count * item->count_and_size.type_size; +} + +static int compare_bytes_desc(const void *a, const void *b) { + size_t size_a = get_bytes((const MemItem *)(a)); + size_t size_b = get_bytes((const MemItem *)(b)); + if (size_a > size_b) + return -1; + if (size_a < size_b) + return 1; + return 0; +} + +void memprof_dump_stats(FILE *file) { + assert(table_active); + // disable modifications from this function + table_active = false; + + ZigList list = {}; + + auto it = usage_table.entry_iterator(); + for (;;) { + auto *entry = it.next(); + if (!entry) + break; + + list.append({entry->key, entry->value}); + } + + qsort(list.items, list.length, sizeof(MemItem), compare_bytes_desc); + + size_t total_bytes_used = 0; + + for (size_t i = 0; i < list.length; i += 1) { + const MemItem *item = &list.at(i); + fprintf(file, "%s: %zu items, %zu bytes each, total ", item->type_name, + item->count_and_size.item_count, item->count_and_size.type_size); + size_t bytes = get_bytes(item); + zig_pretty_print_bytes(file, bytes); + fprintf(file, "\n"); + + total_bytes_used += bytes; + } + + fprintf(stderr, "Total bytes used: "); + zig_pretty_print_bytes(file, total_bytes_used); + fprintf(file, "\n"); + + list.deinit(); + table_active = true; +} + +#endif diff --git a/src/memory_profiling.hpp b/src/memory_profiling.hpp new file mode 100644 index 0000000000..6d43d81e3c --- /dev/null +++ b/src/memory_profiling.hpp @@ -0,0 +1,22 @@ +/* + * Copyright (c) 2019 Andrew Kelley + * + * This file is part of zig, which is MIT licensed. + * See http://opensource.org/licenses/MIT + */ + +#ifndef ZIG_MEMORY_PROFILING_HPP +#define ZIG_MEMORY_PROFILING_HPP + +#include "config.h" + +#include +#include + +void memprof_init(void); + +void memprof_alloc(const char *name, size_t item_count, size_t type_size); +void memprof_dealloc(const char *name, size_t item_count, size_t type_size); + +void memprof_dump_stats(FILE *file); +#endif diff --git a/src/parser.cpp b/src/parser.cpp index 9fcf233e2d..b6742ef1f1 100644 --- a/src/parser.cpp +++ b/src/parser.cpp @@ -518,8 +518,8 @@ static Token *ast_parse_doc_comments(ParseContext *pc, Buf *buf) { // <- TestDecl ContainerMembers // / TopLevelComptime ContainerMembers // / KEYWORD_pub? TopLevelDecl ContainerMembers -// / KEYWORD_pub? ContainerField COMMA ContainerMembers -// / KEYWORD_pub? ContainerField +// / ContainerField COMMA ContainerMembers +// / ContainerField // / static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) { AstNodeContainerDecl res = {}; @@ -548,10 +548,13 @@ static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) { continue; } + if (visib_token != nullptr) { + ast_error(pc, peek_token(pc), "expected function or variable declaration after pub"); + } + AstNode *container_field = ast_parse_container_field(pc); if (container_field != nullptr) { assert(container_field->type == NodeTypeStructField); - container_field->data.struct_field.visib_mod = visib_mod; container_field->data.struct_field.doc_comments = doc_comment_buf; res.fields.append(container_field); if (eat_token_if(pc, TokenIdComma) != nullptr) { @@ -561,12 +564,7 @@ static AstNodeContainerDecl ast_parse_container_members(ParseContext *pc) { } } - // We visib_token wasn't eaten, then we haven't consumed the first token in this rule yet. - // It is therefore safe to return and let the caller continue parsing. - if (visib_token == nullptr) - break; - - ast_invalid_token_error(pc, peek_token(pc)); + break; } return res; diff --git a/src/util.cpp b/src/util.cpp index 13bfbbcd47..055d572010 100644 --- a/src/util.cpp +++ b/src/util.cpp @@ -119,3 +119,21 @@ Slice SplitIterator_rest(SplitIterator *self) { SplitIterator memSplit(Slice buffer, Slice split_bytes) { return SplitIterator{0, buffer, split_bytes}; } + +void zig_pretty_print_bytes(FILE *f, double n) { + if (n > 1024.0 * 1024.0 * 1024.0) { + fprintf(f, "%.02f GiB", n / 1024.0 / 1024.0 / 1024.0); + return; + } + if (n > 1024.0 * 1024.0) { + fprintf(f, "%.02f MiB", n / 1024.0 / 1024.0); + return; + } + if (n > 1024.0) { + fprintf(f, "%.02f KiB", n / 1024.0); + return; + } + fprintf(f, "%.02f bytes", n ); + return; +} + diff --git a/src/util.hpp b/src/util.hpp index 8abcef32ce..79bebd3355 100644 --- a/src/util.hpp +++ b/src/util.hpp @@ -8,6 +8,8 @@ #ifndef ZIG_UTIL_HPP #define ZIG_UTIL_HPP +#include "memory_profiling.hpp" + #include #include #include @@ -96,7 +98,10 @@ static inline int ctzll(unsigned long long mask) { template -ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate_nonzero(size_t count) { +ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate_nonzero(size_t count, const char *name = nullptr) { +#ifdef ZIG_ENABLE_MEM_PROFILE + memprof_alloc(name, count, sizeof(T)); +#endif #ifndef NDEBUG // make behavior when size == 0 portable if (count == 0) @@ -109,7 +114,10 @@ ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate_nonzero(size_t count) { } template -ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate(size_t count) { +ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate(size_t count, const char *name = nullptr) { +#ifdef ZIG_ENABLE_MEM_PROFILE + memprof_alloc(name, count, sizeof(T)); +#endif #ifndef NDEBUG // make behavior when size == 0 portable if (count == 0) @@ -122,7 +130,7 @@ ATTRIBUTE_RETURNS_NOALIAS static inline T *allocate(size_t count) { } template -static inline T *reallocate(T *old, size_t old_count, size_t new_count) { +static inline T *reallocate(T *old, size_t old_count, size_t new_count, const char *name = nullptr) { T *ptr = reallocate_nonzero(old, old_count, new_count); if (new_count > old_count) { memset(&ptr[old_count], 0, (new_count - old_count) * sizeof(T)); @@ -131,7 +139,11 @@ static inline T *reallocate(T *old, size_t old_count, size_t new_count) { } template -static inline T *reallocate_nonzero(T *old, size_t old_count, size_t new_count) { +static inline T *reallocate_nonzero(T *old, size_t old_count, size_t new_count, const char *name = nullptr) { +#ifdef ZIG_ENABLE_MEM_PROFILE + memprof_dealloc(name, old_count, sizeof(T)); + memprof_alloc(name, new_count, sizeof(T)); +#endif #ifndef NDEBUG // make behavior when size == 0 portable if (new_count == 0 && old == nullptr) @@ -143,6 +155,19 @@ static inline T *reallocate_nonzero(T *old, size_t old_count, size_t new_count) return ptr; } +template +static inline void deallocate(T *old, size_t count, const char *name = nullptr) { +#ifdef ZIG_ENABLE_MEM_PROFILE + memprof_dealloc(name, count, sizeof(T)); +#endif + free(old); +} + +template +static inline void destroy(T *old, const char *name = nullptr) { + return deallocate(old, 1); +} + template constexpr size_t array_length(const T (&)[n]) { return n; @@ -225,6 +250,8 @@ static inline double zig_f16_to_double(float16_t x) { return z; } +void zig_pretty_print_bytes(FILE *f, double n); + template struct Optional { T value; diff --git a/test/cli.zig b/test/cli.zig index 63a116a811..6095536422 100644 --- a/test/cli.zig +++ b/test/cli.zig @@ -37,7 +37,7 @@ pub fn main() !void { testMissingOutputPath, }; for (test_fns) |testFn| { - try fs.deleteTree(a, dir_path); + try fs.deleteTree(dir_path); try fs.makeDir(dir_path); try testFn(zig_exe, dir_path); } @@ -87,7 +87,7 @@ fn exec(cwd: []const u8, argv: []const []const u8) !ChildProcess.ExecResult { fn testZigInitLib(zig_exe: []const u8, dir_path: []const u8) !void { _ = try exec(dir_path, [_][]const u8{ zig_exe, "init-lib" }); const test_result = try exec(dir_path, [_][]const u8{ zig_exe, "build", "test" }); - testing.expect(std.mem.eql(u8, test_result.stderr, "")); + testing.expect(std.mem.endsWith(u8, test_result.stderr, "All tests passed.\n")); } fn testZigInitExe(zig_exe: []const u8, dir_path: []const u8) !void { diff --git a/test/compile_errors.zig b/test/compile_errors.zig index 288fd501cf..35c6ded4a4 100644 --- a/test/compile_errors.zig +++ b/test/compile_errors.zig @@ -2,6 +2,16 @@ const tests = @import("tests.zig"); const builtin = @import("builtin"); pub fn addCases(cases: *tests.CompileErrorContext) void { + cases.add( + "comparison with error union and error value", + \\export fn entry() void { + \\ var number_or_error: anyerror!i32 = error.SomethingAwful; + \\ _ = number_or_error == error.SomethingAwful; + \\} + , + "tmp.zig:3:25: error: operator not allowed for type 'anyerror!i32'", + ); + cases.add( "switch with overlapping case ranges", \\export fn entry() void { diff --git a/test/stage1/behavior/switch.zig b/test/stage1/behavior/switch.zig index bc1b4a7a09..936dbed786 100644 --- a/test/stage1/behavior/switch.zig +++ b/test/stage1/behavior/switch.zig @@ -434,3 +434,21 @@ test "switch with disjoint range" { 126...126 => {}, } } + +var state: u32 = 0; +fn poll() void { + switch (state) { + 0 => { + state = 1; + }, + else => { + state += 1; + }, + } +} + +test "switch on global mutable var isn't constant-folded" { + while (state < 2) { + poll(); + } +} diff --git a/test/stage1/behavior/union.zig b/test/stage1/behavior/union.zig index f5e9ec09a8..40adf601c4 100644 --- a/test/stage1/behavior/union.zig +++ b/test/stage1/behavior/union.zig @@ -521,3 +521,17 @@ test "extern union doesn't trigger field check at comptime" { const x = U{ .x = 0x55AAAA55 }; comptime expect(x.y == 0x55); } + +const Foo1 = union(enum) { + f: struct { + x: usize, + }, +}; +var glbl: Foo1 = undefined; + +test "global union with single field is correctly initialized" { + glbl = Foo1{ + .f = @memberType(Foo1, 0){ .x = 123 }, + }; + expect(glbl.f.x == 123); +} diff --git a/tools/process_headers.zig b/tools/process_headers.zig index 420c118cb9..667da5f7c9 100644 --- a/tools/process_headers.zig +++ b/tools/process_headers.zig @@ -340,7 +340,7 @@ pub fn main() !void { try dir_stack.append(target_include_dir); while (dir_stack.popOrNull()) |full_dir_name| { - var dir = std.fs.Dir.open(allocator, full_dir_name) catch |err| switch (err) { + var dir = std.fs.Dir.open(full_dir_name) catch |err| switch (err) { error.FileNotFound => continue :search, error.AccessDenied => continue :search, else => return err,