From 67e51311c3352ab4b2a381bd90dc386032254058 Mon Sep 17 00:00:00 2001 From: Timon Kruiper Date: Thu, 26 Mar 2020 13:41:53 +0100 Subject: [PATCH 1/8] fix behavior test with --test-evented-io on windows also make simple file operations work asynchronously on windows --- lib/std/debug.zig | 536 +++++++++++++++++++++-------------------- lib/std/fs.zig | 9 +- lib/std/fs/file.zig | 16 ++ lib/std/io.zig | 4 + lib/std/os.zig | 10 +- lib/std/os/windows.zig | 42 ++-- lib/std/pdb.zig | 2 +- 7 files changed, 326 insertions(+), 293 deletions(-) diff --git a/lib/std/debug.zig b/lib/std/debug.zig index 94475515e7..4525efe326 100644 --- a/lib/std/debug.zig +++ b/lib/std/debug.zig @@ -666,158 +666,160 @@ pub fn openSelfDebugInfo(allocator: *mem.Allocator) anyerror!DebugInfo { /// TODO resources https://github.com/ziglang/zig/issues/4353 fn openCoffDebugInfo(allocator: *mem.Allocator, coff_file_path: [:0]const u16) !ModuleDebugInfo { - const coff_file = try std.fs.openFileAbsoluteW(coff_file_path.ptr, .{}); - errdefer coff_file.close(); + noasync { + const coff_file = try std.fs.openFileAbsoluteW(coff_file_path.ptr, .{ .always_blocking = true }); + errdefer coff_file.close(); - const coff_obj = try allocator.create(coff.Coff); - coff_obj.* = coff.Coff.init(allocator, coff_file); + const coff_obj = try allocator.create(coff.Coff); + coff_obj.* = coff.Coff.init(allocator, coff_file); - var di = ModuleDebugInfo{ - .base_address = undefined, - .coff = coff_obj, - .pdb = undefined, - .sect_contribs = undefined, - .modules = undefined, - }; - - try di.coff.loadHeader(); - - var path_buf: [windows.MAX_PATH]u8 = undefined; - const len = try di.coff.getPdbPath(path_buf[0..]); - const raw_path = path_buf[0..len]; - - const path = try fs.path.resolve(allocator, &[_][]const u8{raw_path}); - - try di.pdb.openFile(di.coff, path); - - var pdb_stream = di.pdb.getStream(pdb.StreamType.Pdb) orelse return error.InvalidDebugInfo; - const version = try pdb_stream.inStream().readIntLittle(u32); - const signature = try pdb_stream.inStream().readIntLittle(u32); - const age = try pdb_stream.inStream().readIntLittle(u32); - var guid: [16]u8 = undefined; - try pdb_stream.inStream().readNoEof(&guid); - if (version != 20000404) // VC70, only value observed by LLVM team - return error.UnknownPDBVersion; - if (!mem.eql(u8, &di.coff.guid, &guid) or di.coff.age != age) - return error.PDBMismatch; - // We validated the executable and pdb match. - - const string_table_index = str_tab_index: { - const name_bytes_len = try pdb_stream.inStream().readIntLittle(u32); - const name_bytes = try allocator.alloc(u8, name_bytes_len); - try pdb_stream.inStream().readNoEof(name_bytes); - - const HashTableHeader = packed struct { - Size: u32, - Capacity: u32, - - fn maxLoad(cap: u32) u32 { - return cap * 2 / 3 + 1; - } + var di = ModuleDebugInfo{ + .base_address = undefined, + .coff = coff_obj, + .pdb = undefined, + .sect_contribs = undefined, + .modules = undefined, }; - const hash_tbl_hdr = try pdb_stream.inStream().readStruct(HashTableHeader); - if (hash_tbl_hdr.Capacity == 0) - return error.InvalidDebugInfo; - if (hash_tbl_hdr.Size > HashTableHeader.maxLoad(hash_tbl_hdr.Capacity)) - return error.InvalidDebugInfo; + try di.coff.loadHeader(); - const present = try readSparseBitVector(&pdb_stream.inStream(), allocator); - if (present.len != hash_tbl_hdr.Size) - return error.InvalidDebugInfo; - const deleted = try readSparseBitVector(&pdb_stream.inStream(), allocator); + var path_buf: [windows.MAX_PATH]u8 = undefined; + const len = try di.coff.getPdbPath(path_buf[0..]); + const raw_path = path_buf[0..len]; - const Bucket = struct { - first: u32, - second: u32, - }; - const bucket_list = try allocator.alloc(Bucket, present.len); - for (present) |_| { - const name_offset = try pdb_stream.inStream().readIntLittle(u32); - const name_index = try pdb_stream.inStream().readIntLittle(u32); - const name = mem.toSlice(u8, @ptrCast([*:0]u8, name_bytes.ptr + name_offset)); - if (mem.eql(u8, name, "/names")) { - break :str_tab_index name_index; + const path = try fs.path.resolve(allocator, &[_][]const u8{raw_path}); + + try di.pdb.openFile(di.coff, path); + + var pdb_stream = di.pdb.getStream(pdb.StreamType.Pdb) orelse return error.InvalidDebugInfo; + const version = try pdb_stream.inStream().readIntLittle(u32); + const signature = try pdb_stream.inStream().readIntLittle(u32); + const age = try pdb_stream.inStream().readIntLittle(u32); + var guid: [16]u8 = undefined; + try pdb_stream.inStream().readNoEof(&guid); + if (version != 20000404) // VC70, only value observed by LLVM team + return error.UnknownPDBVersion; + if (!mem.eql(u8, &di.coff.guid, &guid) or di.coff.age != age) + return error.PDBMismatch; + // We validated the executable and pdb match. + + const string_table_index = str_tab_index: { + const name_bytes_len = try pdb_stream.inStream().readIntLittle(u32); + const name_bytes = try allocator.alloc(u8, name_bytes_len); + try pdb_stream.inStream().readNoEof(name_bytes); + + const HashTableHeader = packed struct { + Size: u32, + Capacity: u32, + + fn maxLoad(cap: u32) u32 { + return cap * 2 / 3 + 1; + } + }; + const hash_tbl_hdr = try pdb_stream.inStream().readStruct(HashTableHeader); + if (hash_tbl_hdr.Capacity == 0) + return error.InvalidDebugInfo; + + if (hash_tbl_hdr.Size > HashTableHeader.maxLoad(hash_tbl_hdr.Capacity)) + return error.InvalidDebugInfo; + + const present = try readSparseBitVector(&pdb_stream.inStream(), allocator); + if (present.len != hash_tbl_hdr.Size) + return error.InvalidDebugInfo; + const deleted = try readSparseBitVector(&pdb_stream.inStream(), allocator); + + const Bucket = struct { + first: u32, + second: u32, + }; + const bucket_list = try allocator.alloc(Bucket, present.len); + for (present) |_| { + const name_offset = try pdb_stream.inStream().readIntLittle(u32); + const name_index = try pdb_stream.inStream().readIntLittle(u32); + const name = mem.toSlice(u8, @ptrCast([*:0]u8, name_bytes.ptr + name_offset)); + if (mem.eql(u8, name, "/names")) { + break :str_tab_index name_index; + } } - } - return error.MissingDebugInfo; - }; + return error.MissingDebugInfo; + }; - di.pdb.string_table = di.pdb.getStreamById(string_table_index) orelse return error.MissingDebugInfo; - di.pdb.dbi = di.pdb.getStream(pdb.StreamType.Dbi) orelse return error.MissingDebugInfo; + di.pdb.string_table = di.pdb.getStreamById(string_table_index) orelse return error.MissingDebugInfo; + di.pdb.dbi = di.pdb.getStream(pdb.StreamType.Dbi) orelse return error.MissingDebugInfo; - const dbi = di.pdb.dbi; + const dbi = di.pdb.dbi; - // Dbi Header - const dbi_stream_header = try dbi.inStream().readStruct(pdb.DbiStreamHeader); - if (dbi_stream_header.VersionHeader != 19990903) // V70, only value observed by LLVM team - return error.UnknownPDBVersion; - if (dbi_stream_header.Age != age) - return error.UnmatchingPDB; + // Dbi Header + const dbi_stream_header = try dbi.inStream().readStruct(pdb.DbiStreamHeader); + if (dbi_stream_header.VersionHeader != 19990903) // V70, only value observed by LLVM team + return error.UnknownPDBVersion; + if (dbi_stream_header.Age != age) + return error.UnmatchingPDB; - const mod_info_size = dbi_stream_header.ModInfoSize; - const section_contrib_size = dbi_stream_header.SectionContributionSize; + const mod_info_size = dbi_stream_header.ModInfoSize; + const section_contrib_size = dbi_stream_header.SectionContributionSize; - var modules = ArrayList(Module).init(allocator); + var modules = ArrayList(Module).init(allocator); - // Module Info Substream - var mod_info_offset: usize = 0; - while (mod_info_offset != mod_info_size) { - const mod_info = try dbi.inStream().readStruct(pdb.ModInfo); - var this_record_len: usize = @sizeOf(pdb.ModInfo); + // Module Info Substream + var mod_info_offset: usize = 0; + while (mod_info_offset != mod_info_size) { + const mod_info = try dbi.inStream().readStruct(pdb.ModInfo); + var this_record_len: usize = @sizeOf(pdb.ModInfo); - const module_name = try dbi.readNullTermString(allocator); - this_record_len += module_name.len + 1; + const module_name = try dbi.readNullTermString(allocator); + this_record_len += module_name.len + 1; - const obj_file_name = try dbi.readNullTermString(allocator); - this_record_len += obj_file_name.len + 1; + const obj_file_name = try dbi.readNullTermString(allocator); + this_record_len += obj_file_name.len + 1; - if (this_record_len % 4 != 0) { - const round_to_next_4 = (this_record_len | 0x3) + 1; - const march_forward_bytes = round_to_next_4 - this_record_len; - try dbi.seekBy(@intCast(isize, march_forward_bytes)); - this_record_len += march_forward_bytes; + if (this_record_len % 4 != 0) { + const round_to_next_4 = (this_record_len | 0x3) + 1; + const march_forward_bytes = round_to_next_4 - this_record_len; + try dbi.seekBy(@intCast(isize, march_forward_bytes)); + this_record_len += march_forward_bytes; + } + + try modules.append(Module{ + .mod_info = mod_info, + .module_name = module_name, + .obj_file_name = obj_file_name, + + .populated = false, + .symbols = undefined, + .subsect_info = undefined, + .checksum_offset = null, + }); + + mod_info_offset += this_record_len; + if (mod_info_offset > mod_info_size) + return error.InvalidDebugInfo; } - try modules.append(Module{ - .mod_info = mod_info, - .module_name = module_name, - .obj_file_name = obj_file_name, + di.modules = modules.toOwnedSlice(); - .populated = false, - .symbols = undefined, - .subsect_info = undefined, - .checksum_offset = null, - }); + // Section Contribution Substream + var sect_contribs = ArrayList(pdb.SectionContribEntry).init(allocator); + var sect_cont_offset: usize = 0; + if (section_contrib_size != 0) { + const ver = @intToEnum(pdb.SectionContrSubstreamVersion, try dbi.inStream().readIntLittle(u32)); + if (ver != pdb.SectionContrSubstreamVersion.Ver60) + return error.InvalidDebugInfo; + sect_cont_offset += @sizeOf(u32); + } + while (sect_cont_offset != section_contrib_size) { + const entry = try sect_contribs.addOne(); + entry.* = try dbi.inStream().readStruct(pdb.SectionContribEntry); + sect_cont_offset += @sizeOf(pdb.SectionContribEntry); - mod_info_offset += this_record_len; - if (mod_info_offset > mod_info_size) - return error.InvalidDebugInfo; + if (sect_cont_offset > section_contrib_size) + return error.InvalidDebugInfo; + } + + di.sect_contribs = sect_contribs.toOwnedSlice(); + + return di; } - - di.modules = modules.toOwnedSlice(); - - // Section Contribution Substream - var sect_contribs = ArrayList(pdb.SectionContribEntry).init(allocator); - var sect_cont_offset: usize = 0; - if (section_contrib_size != 0) { - const ver = @intToEnum(pdb.SectionContrSubstreamVersion, try dbi.inStream().readIntLittle(u32)); - if (ver != pdb.SectionContrSubstreamVersion.Ver60) - return error.InvalidDebugInfo; - sect_cont_offset += @sizeOf(u32); - } - while (sect_cont_offset != section_contrib_size) { - const entry = try sect_contribs.addOne(); - entry.* = try dbi.inStream().readStruct(pdb.SectionContribEntry); - sect_cont_offset += @sizeOf(pdb.SectionContribEntry); - - if (sect_cont_offset > section_contrib_size) - return error.InvalidDebugInfo; - } - - di.sect_contribs = sect_contribs.toOwnedSlice(); - - return di; } fn readSparseBitVector(stream: var, allocator: *mem.Allocator) ![]usize { @@ -1476,151 +1478,153 @@ pub const ModuleDebugInfo = switch (builtin.os.tag) { } fn getSymbolAtAddress(self: *@This(), address: usize) !SymbolInfo { - // Translate the VA into an address into this object - const relocated_address = address - self.base_address; + noasync { + // Translate the VA into an address into this object + const relocated_address = address - self.base_address; - var coff_section: *coff.Section = undefined; - const mod_index = for (self.sect_contribs) |sect_contrib| { - if (sect_contrib.Section > self.coff.sections.len) continue; - // Remember that SectionContribEntry.Section is 1-based. - coff_section = &self.coff.sections.toSlice()[sect_contrib.Section - 1]; + var coff_section: *coff.Section = undefined; + const mod_index = for (self.sect_contribs) |sect_contrib| { + if (sect_contrib.Section > self.coff.sections.len) continue; + // Remember that SectionContribEntry.Section is 1-based. + coff_section = &self.coff.sections.toSlice()[sect_contrib.Section - 1]; - const vaddr_start = coff_section.header.virtual_address + sect_contrib.Offset; - const vaddr_end = vaddr_start + sect_contrib.Size; - if (relocated_address >= vaddr_start and relocated_address < vaddr_end) { - break sect_contrib.ModuleIndex; - } - } else { - // we have no information to add to the address - return SymbolInfo{}; - }; + const vaddr_start = coff_section.header.virtual_address + sect_contrib.Offset; + const vaddr_end = vaddr_start + sect_contrib.Size; + if (relocated_address >= vaddr_start and relocated_address < vaddr_end) { + break sect_contrib.ModuleIndex; + } + } else { + // we have no information to add to the address + return SymbolInfo{}; + }; - const mod = &self.modules[mod_index]; - try populateModule(self, mod); - const obj_basename = fs.path.basename(mod.obj_file_name); + const mod = &self.modules[mod_index]; + try populateModule(self, mod); + const obj_basename = fs.path.basename(mod.obj_file_name); - var symbol_i: usize = 0; - const symbol_name = if (!mod.populated) "???" else while (symbol_i != mod.symbols.len) { - const prefix = @ptrCast(*pdb.RecordPrefix, &mod.symbols[symbol_i]); - if (prefix.RecordLen < 2) - return error.InvalidDebugInfo; - switch (prefix.RecordKind) { - .S_LPROC32, .S_GPROC32 => { - const proc_sym = @ptrCast(*pdb.ProcSym, &mod.symbols[symbol_i + @sizeOf(pdb.RecordPrefix)]); - const vaddr_start = coff_section.header.virtual_address + proc_sym.CodeOffset; - const vaddr_end = vaddr_start + proc_sym.CodeSize; - if (relocated_address >= vaddr_start and relocated_address < vaddr_end) { - break mem.toSliceConst(u8, @ptrCast([*:0]u8, proc_sym) + @sizeOf(pdb.ProcSym)); - } - }, - else => {}, - } - symbol_i += prefix.RecordLen + @sizeOf(u16); - if (symbol_i > mod.symbols.len) - return error.InvalidDebugInfo; - } else "???"; - - const subsect_info = mod.subsect_info; - - var sect_offset: usize = 0; - var skip_len: usize = undefined; - const opt_line_info = subsections: { - const checksum_offset = mod.checksum_offset orelse break :subsections null; - while (sect_offset != subsect_info.len) : (sect_offset += skip_len) { - const subsect_hdr = @ptrCast(*pdb.DebugSubsectionHeader, &subsect_info[sect_offset]); - skip_len = subsect_hdr.Length; - sect_offset += @sizeOf(pdb.DebugSubsectionHeader); - - switch (subsect_hdr.Kind) { - .Lines => { - var line_index = sect_offset; - - const line_hdr = @ptrCast(*pdb.LineFragmentHeader, &subsect_info[line_index]); - if (line_hdr.RelocSegment == 0) - return error.MissingDebugInfo; - line_index += @sizeOf(pdb.LineFragmentHeader); - const frag_vaddr_start = coff_section.header.virtual_address + line_hdr.RelocOffset; - const frag_vaddr_end = frag_vaddr_start + line_hdr.CodeSize; - - if (relocated_address >= frag_vaddr_start and relocated_address < frag_vaddr_end) { - // There is an unknown number of LineBlockFragmentHeaders (and their accompanying line and column records) - // from now on. We will iterate through them, and eventually find a LineInfo that we're interested in, - // breaking out to :subsections. If not, we will make sure to not read anything outside of this subsection. - const subsection_end_index = sect_offset + subsect_hdr.Length; - - while (line_index < subsection_end_index) { - const block_hdr = @ptrCast(*pdb.LineBlockFragmentHeader, &subsect_info[line_index]); - line_index += @sizeOf(pdb.LineBlockFragmentHeader); - const start_line_index = line_index; - - const has_column = line_hdr.Flags.LF_HaveColumns; - - // All line entries are stored inside their line block by ascending start address. - // Heuristic: we want to find the last line entry - // that has a vaddr_start <= relocated_address. - // This is done with a simple linear search. - var line_i: u32 = 0; - while (line_i < block_hdr.NumLines) : (line_i += 1) { - const line_num_entry = @ptrCast(*pdb.LineNumberEntry, &subsect_info[line_index]); - line_index += @sizeOf(pdb.LineNumberEntry); - - const vaddr_start = frag_vaddr_start + line_num_entry.Offset; - if (relocated_address < vaddr_start) { - break; - } - } - - // line_i == 0 would mean that no matching LineNumberEntry was found. - if (line_i > 0) { - const subsect_index = checksum_offset + block_hdr.NameIndex; - const chksum_hdr = @ptrCast(*pdb.FileChecksumEntryHeader, &mod.subsect_info[subsect_index]); - const strtab_offset = @sizeOf(pdb.PDBStringTableHeader) + chksum_hdr.FileNameOffset; - try self.pdb.string_table.seekTo(strtab_offset); - const source_file_name = try self.pdb.string_table.readNullTermString(self.allocator()); - - const line_entry_idx = line_i - 1; - - const column = if (has_column) blk: { - const start_col_index = start_line_index + @sizeOf(pdb.LineNumberEntry) * block_hdr.NumLines; - const col_index = start_col_index + @sizeOf(pdb.ColumnNumberEntry) * line_entry_idx; - const col_num_entry = @ptrCast(*pdb.ColumnNumberEntry, &subsect_info[col_index]); - break :blk col_num_entry.StartColumn; - } else 0; - - const found_line_index = start_line_index + line_entry_idx * @sizeOf(pdb.LineNumberEntry); - const line_num_entry = @ptrCast(*pdb.LineNumberEntry, &subsect_info[found_line_index]); - const flags = @ptrCast(*pdb.LineNumberEntry.Flags, &line_num_entry.Flags); - - break :subsections LineInfo{ - .allocator = self.allocator(), - .file_name = source_file_name, - .line = flags.Start, - .column = column, - }; - } - } - - // Checking that we are not reading garbage after the (possibly) multiple block fragments. - if (line_index != subsection_end_index) { - return error.InvalidDebugInfo; - } + var symbol_i: usize = 0; + const symbol_name = if (!mod.populated) "???" else while (symbol_i != mod.symbols.len) { + const prefix = @ptrCast(*pdb.RecordPrefix, &mod.symbols[symbol_i]); + if (prefix.RecordLen < 2) + return error.InvalidDebugInfo; + switch (prefix.RecordKind) { + .S_LPROC32, .S_GPROC32 => { + const proc_sym = @ptrCast(*pdb.ProcSym, &mod.symbols[symbol_i + @sizeOf(pdb.RecordPrefix)]); + const vaddr_start = coff_section.header.virtual_address + proc_sym.CodeOffset; + const vaddr_end = vaddr_start + proc_sym.CodeSize; + if (relocated_address >= vaddr_start and relocated_address < vaddr_end) { + break mem.toSliceConst(u8, @ptrCast([*:0]u8, proc_sym) + @sizeOf(pdb.ProcSym)); } }, else => {}, } - - if (sect_offset > subsect_info.len) + symbol_i += prefix.RecordLen + @sizeOf(u16); + if (symbol_i > mod.symbols.len) return error.InvalidDebugInfo; - } else { - break :subsections null; - } - }; + } else "???"; - return SymbolInfo{ - .symbol_name = symbol_name, - .compile_unit_name = obj_basename, - .line_info = opt_line_info, - }; + const subsect_info = mod.subsect_info; + + var sect_offset: usize = 0; + var skip_len: usize = undefined; + const opt_line_info = subsections: { + const checksum_offset = mod.checksum_offset orelse break :subsections null; + while (sect_offset != subsect_info.len) : (sect_offset += skip_len) { + const subsect_hdr = @ptrCast(*pdb.DebugSubsectionHeader, &subsect_info[sect_offset]); + skip_len = subsect_hdr.Length; + sect_offset += @sizeOf(pdb.DebugSubsectionHeader); + + switch (subsect_hdr.Kind) { + .Lines => { + var line_index = sect_offset; + + const line_hdr = @ptrCast(*pdb.LineFragmentHeader, &subsect_info[line_index]); + if (line_hdr.RelocSegment == 0) + return error.MissingDebugInfo; + line_index += @sizeOf(pdb.LineFragmentHeader); + const frag_vaddr_start = coff_section.header.virtual_address + line_hdr.RelocOffset; + const frag_vaddr_end = frag_vaddr_start + line_hdr.CodeSize; + + if (relocated_address >= frag_vaddr_start and relocated_address < frag_vaddr_end) { + // There is an unknown number of LineBlockFragmentHeaders (and their accompanying line and column records) + // from now on. We will iterate through them, and eventually find a LineInfo that we're interested in, + // breaking out to :subsections. If not, we will make sure to not read anything outside of this subsection. + const subsection_end_index = sect_offset + subsect_hdr.Length; + + while (line_index < subsection_end_index) { + const block_hdr = @ptrCast(*pdb.LineBlockFragmentHeader, &subsect_info[line_index]); + line_index += @sizeOf(pdb.LineBlockFragmentHeader); + const start_line_index = line_index; + + const has_column = line_hdr.Flags.LF_HaveColumns; + + // All line entries are stored inside their line block by ascending start address. + // Heuristic: we want to find the last line entry + // that has a vaddr_start <= relocated_address. + // This is done with a simple linear search. + var line_i: u32 = 0; + while (line_i < block_hdr.NumLines) : (line_i += 1) { + const line_num_entry = @ptrCast(*pdb.LineNumberEntry, &subsect_info[line_index]); + line_index += @sizeOf(pdb.LineNumberEntry); + + const vaddr_start = frag_vaddr_start + line_num_entry.Offset; + if (relocated_address < vaddr_start) { + break; + } + } + + // line_i == 0 would mean that no matching LineNumberEntry was found. + if (line_i > 0) { + const subsect_index = checksum_offset + block_hdr.NameIndex; + const chksum_hdr = @ptrCast(*pdb.FileChecksumEntryHeader, &mod.subsect_info[subsect_index]); + const strtab_offset = @sizeOf(pdb.PDBStringTableHeader) + chksum_hdr.FileNameOffset; + try self.pdb.string_table.seekTo(strtab_offset); + const source_file_name = try self.pdb.string_table.readNullTermString(self.allocator()); + + const line_entry_idx = line_i - 1; + + const column = if (has_column) blk: { + const start_col_index = start_line_index + @sizeOf(pdb.LineNumberEntry) * block_hdr.NumLines; + const col_index = start_col_index + @sizeOf(pdb.ColumnNumberEntry) * line_entry_idx; + const col_num_entry = @ptrCast(*pdb.ColumnNumberEntry, &subsect_info[col_index]); + break :blk col_num_entry.StartColumn; + } else 0; + + const found_line_index = start_line_index + line_entry_idx * @sizeOf(pdb.LineNumberEntry); + const line_num_entry = @ptrCast(*pdb.LineNumberEntry, &subsect_info[found_line_index]); + const flags = @ptrCast(*pdb.LineNumberEntry.Flags, &line_num_entry.Flags); + + break :subsections LineInfo{ + .allocator = self.allocator(), + .file_name = source_file_name, + .line = flags.Start, + .column = column, + }; + } + } + + // Checking that we are not reading garbage after the (possibly) multiple block fragments. + if (line_index != subsection_end_index) { + return error.InvalidDebugInfo; + } + } + }, + else => {}, + } + + if (sect_offset > subsect_info.len) + return error.InvalidDebugInfo; + } else { + break :subsections null; + } + }; + + return SymbolInfo{ + .symbol_name = symbol_name, + .compile_unit_name = obj_basename, + .line_info = opt_line_info, + }; + } } }, .linux, .netbsd, .freebsd, .dragonfly => struct { diff --git a/lib/std/fs.zig b/lib/std/fs.zig index e705966c97..099e4354dd 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -613,9 +613,14 @@ pub const Dir = struct { const access_mask = w.SYNCHRONIZE | (if (flags.read) @as(u32, w.GENERIC_READ) else 0) | (if (flags.write) @as(u32, w.GENERIC_WRITE) else 0); + const enable_async_io = std.io.is_async and !flags.always_blocking; return @as(File, .{ - .handle = try os.windows.OpenFileW(self.fd, sub_path_w, null, access_mask, w.FILE_OPEN), + .handle = try os.windows.OpenFileW(self.fd, sub_path_w, null, access_mask, w.FILE_OPEN, enable_async_io), .io_mode = .blocking, + .async_block_allowed = if (flags.always_blocking) + File.async_block_allowed_yes + else + File.async_block_allowed_no, }); } @@ -662,7 +667,7 @@ pub const Dir = struct { else @as(u32, w.FILE_OPEN_IF); return @as(File, .{ - .handle = try os.windows.OpenFileW(self.fd, sub_path_w, null, access_mask, creation), + .handle = try os.windows.OpenFileW(self.fd, sub_path_w, null, access_mask, creation, std.io.is_async), .io_mode = .blocking, }); } diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index c4991a81ff..6161c2348c 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -251,6 +251,10 @@ pub const File = struct { pub const PReadError = os.PReadError; pub fn read(self: File, buffer: []u8) ReadError!usize { + if (builtin.os.tag == .windows) { + const enable_async_io = std.io.is_async and !self.async_block_allowed; + return windows.ReadFile(self.handle, buffer, null, enable_async_io); + } if (need_async_thread and self.io_mode == .blocking and !self.async_block_allowed) { return std.event.Loop.instance.?.read(self.handle, buffer); } else { @@ -271,6 +275,10 @@ pub const File = struct { } pub fn pread(self: File, buffer: []u8, offset: u64) PReadError!usize { + if (builtin.os.tag == .windows) { + const enable_async_io = std.io.is_async and !self.async_block_allowed; + return windows.ReadFile(self.handle, buffer, offset, enable_async_io); + } if (need_async_thread and self.io_mode == .blocking and !self.async_block_allowed) { return std.event.Loop.instance.?.pread(self.handle, buffer, offset); } else { @@ -362,6 +370,10 @@ pub const File = struct { pub const PWriteError = os.PWriteError; pub fn write(self: File, bytes: []const u8) WriteError!usize { + if (builtin.os.tag == .windows) { + const enable_async_io = std.io.is_async and !self.async_block_allowed; + return windows.WriteFile(self.handle, bytes, null, enable_async_io); + } if (need_async_thread and self.io_mode == .blocking and !self.async_block_allowed) { return std.event.Loop.instance.?.write(self.handle, bytes); } else { @@ -377,6 +389,10 @@ pub const File = struct { } pub fn pwrite(self: File, bytes: []const u8, offset: u64) PWriteError!usize { + if (builtin.os.tag == .windows) { + const enable_async_io = std.io.is_async and !self.async_block_allowed; + return windows.WriteFile(self.handle, bytes, offset, enable_async_io); + } if (need_async_thread and self.io_mode == .blocking and !self.async_block_allowed) { return std.event.Loop.instance.?.pwrite(self.handle, bytes, offset); } else { diff --git a/lib/std/io.zig b/lib/std/io.zig index 243dd5ca95..da29d389c4 100644 --- a/lib/std/io.zig +++ b/lib/std/io.zig @@ -42,10 +42,12 @@ fn getStdOutHandle() os.fd_t { return os.STDOUT_FILENO; } +// TODO: async stdout on windows (https://github.com/ziglang/zig/pull/4816#issuecomment-604521023) pub fn getStdOut() File { return File{ .handle = getStdOutHandle(), .io_mode = .blocking, + .async_block_allowed = if (builtin.os.tag == .windows) File.async_block_allowed_yes else File.async_block_allowed_no, }; } @@ -81,10 +83,12 @@ fn getStdInHandle() os.fd_t { return os.STDIN_FILENO; } +// TODO: async stdin on windows (https://github.com/ziglang/zig/pull/4816#issuecomment-604521023) pub fn getStdIn() File { return File{ .handle = getStdInHandle(), .io_mode = .blocking, + .async_block_allowed = if (builtin.os.tag == .windows) File.async_block_allowed_yes else File.async_block_allowed_no, }; } diff --git a/lib/std/os.zig b/lib/std/os.zig index dc032badea..f2de3dc487 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -305,7 +305,7 @@ pub const ReadError = error{ /// For POSIX the limit is `math.maxInt(isize)`. pub fn read(fd: fd_t, buf: []u8) ReadError!usize { if (builtin.os.tag == .windows) { - return windows.ReadFile(fd, buf, null); + return windows.ReadFile(fd, buf, null, false); } if (builtin.os.tag == .wasi and !builtin.link_libc) { @@ -407,7 +407,7 @@ pub const PReadError = ReadError || error{Unseekable}; /// used to perform the I/O. `error.WouldBlock` is not possible on Windows. pub fn pread(fd: fd_t, buf: []u8, offset: u64) PReadError!usize { if (builtin.os.tag == .windows) { - return windows.ReadFile(fd, buf, offset); + return windows.ReadFile(fd, buf, offset, false); } while (true) { @@ -583,7 +583,7 @@ pub const WriteError = error{ /// The corresponding POSIX limit is `math.maxInt(isize)`. pub fn write(fd: fd_t, bytes: []const u8) WriteError!usize { if (builtin.os.tag == .windows) { - return windows.WriteFile(fd, bytes, null); + return windows.WriteFile(fd, bytes, null, false); } if (builtin.os.tag == .wasi and !builtin.link_libc) { @@ -708,7 +708,7 @@ pub const PWriteError = WriteError || error{Unseekable}; /// The corresponding POSIX limit is `math.maxInt(isize)`. pub fn pwrite(fd: fd_t, bytes: []const u8, offset: u64) PWriteError!usize { if (std.Target.current.os.tag == .windows) { - return windows.WriteFile(fd, bytes, offset); + return windows.WriteFile(fd, bytes, offset, false); } // Prevent EINVAL. @@ -1651,7 +1651,7 @@ pub fn renameatW( ReplaceIfExists: windows.BOOLEAN, ) RenameError!void { const access_mask = windows.SYNCHRONIZE | windows.GENERIC_WRITE | windows.DELETE; - const src_fd = try windows.OpenFileW(old_dir_fd, old_path, null, access_mask, windows.FILE_OPEN); + const src_fd = try windows.OpenFileW(old_dir_fd, old_path, null, access_mask, windows.FILE_OPEN, false); defer windows.CloseHandle(src_fd); const struct_buf_len = @sizeOf(windows.FILE_RENAME_INFORMATION) + (MAX_PATH_BYTES - 1); diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig index 813a77c275..11668c39cc 100644 --- a/lib/std/os/windows.zig +++ b/lib/std/os/windows.zig @@ -108,6 +108,7 @@ pub fn OpenFileW( sa: ?*SECURITY_ATTRIBUTES, access_mask: ACCESS_MASK, creation: ULONG, + enable_async_io: bool, ) OpenError!HANDLE { if (sub_path_w[0] == '.' and sub_path_w[1] == 0) { return error.IsDir; @@ -135,6 +136,7 @@ pub fn OpenFileW( .SecurityQualityOfService = null, }; var io: IO_STATUS_BLOCK = undefined; + const blocking_flag = if (!enable_async_io) FILE_SYNCHRONOUS_IO_NONALERT else @as(ULONG, 0); const rc = ntdll.NtCreateFile( &result, access_mask, @@ -144,7 +146,7 @@ pub fn OpenFileW( FILE_ATTRIBUTE_NORMAL, FILE_SHARE_WRITE | FILE_SHARE_READ | FILE_SHARE_DELETE, creation, - FILE_NON_DIRECTORY_FILE | FILE_SYNCHRONOUS_IO_NONALERT, + FILE_NON_DIRECTORY_FILE | blocking_flag, null, 0, ); @@ -428,10 +430,11 @@ pub const ReadFileError = error{ /// If buffer's length exceeds what a Windows DWORD integer can hold, it will be broken into /// multiple non-atomic reads. -pub fn ReadFile(in_hFile: HANDLE, buffer: []u8, offset: ?u64) ReadFileError!usize { - if (std.event.Loop.instance) |loop| { +pub fn ReadFile(in_hFile: HANDLE, buffer: []u8, offset: ?u64, enable_async_io: bool) ReadFileError!usize { + if (std.event.Loop.instance != null and enable_async_io) { + const loop = std.event.Loop.instance.?; // TODO support async ReadFile with no offset - const off = offset.?; + const off = if (offset == null) 0 else offset.?; var resume_node = std.event.Loop.ResumeNode.Basic{ .base = .{ .id = .Basic, @@ -446,20 +449,20 @@ pub fn ReadFile(in_hFile: HANDLE, buffer: []u8, offset: ?u64) ReadFileError!usiz }, }; // TODO only call create io completion port once per fd - _ = windows.CreateIoCompletionPort(fd, loop.os_data.io_port, undefined, undefined) catch undefined; + _ = CreateIoCompletionPort(in_hFile, loop.os_data.io_port, undefined, undefined) catch undefined; loop.beginOneEvent(); suspend { // TODO handle buffer bigger than DWORD can hold - _ = windows.kernel32.ReadFile(fd, buffer.ptr, @intCast(windows.DWORD, buffer.len), null, &resume_node.base.overlapped); + _ = kernel32.ReadFile(in_hFile, buffer.ptr, @intCast(DWORD, buffer.len), null, &resume_node.base.overlapped); } - var bytes_transferred: windows.DWORD = undefined; - if (windows.kernel32.GetOverlappedResult(fd, &resume_node.base.overlapped, &bytes_transferred, windows.FALSE) == 0) { - switch (windows.kernel32.GetLastError()) { + var bytes_transferred: DWORD = undefined; + if (kernel32.GetOverlappedResult(in_hFile, &resume_node.base.overlapped, &bytes_transferred, FALSE) == 0) { + switch (kernel32.GetLastError()) { .IO_PENDING => unreachable, .OPERATION_ABORTED => return error.OperationAborted, .BROKEN_PIPE => return error.BrokenPipe, .HANDLE_EOF => return @as(usize, bytes_transferred), - else => |err| return windows.unexpectedError(err), + else => |err| return unexpectedError(err), } } return @as(usize, bytes_transferred); @@ -501,10 +504,11 @@ pub const WriteFileError = error{ Unexpected, }; -pub fn WriteFile(handle: HANDLE, bytes: []const u8, offset: ?u64) WriteFileError!usize { - if (std.event.Loop.instance) |loop| { +pub fn WriteFile(handle: HANDLE, bytes: []const u8, offset: ?u64, enable_async_io: bool) WriteFileError!usize { + if (std.event.Loop.instance != null and enable_async_io) { + const loop = std.event.Loop.instance.?; // TODO support async WriteFile with no offset - const off = offset.?; + const off = if (offset == null) 0 else offset.?; var resume_node = std.event.Loop.ResumeNode.Basic{ .base = .{ .id = .Basic, @@ -519,14 +523,14 @@ pub fn WriteFile(handle: HANDLE, bytes: []const u8, offset: ?u64) WriteFileError }, }; // TODO only call create io completion port once per fd - _ = CreateIoCompletionPort(fd, loop.os_data.io_port, undefined, undefined); + _ = CreateIoCompletionPort(handle, loop.os_data.io_port, undefined, undefined) catch undefined; loop.beginOneEvent(); suspend { - const adjusted_len = math.cast(windows.DWORD, bytes.len) catch maxInt(windows.DWORD); - _ = kernel32.WriteFile(fd, bytes.ptr, adjusted_len, null, &resume_node.base.overlapped); + const adjusted_len = math.cast(DWORD, bytes.len) catch maxInt(DWORD); + _ = kernel32.WriteFile(handle, bytes.ptr, adjusted_len, null, &resume_node.base.overlapped); } - var bytes_transferred: windows.DWORD = undefined; - if (kernel32.GetOverlappedResult(fd, &resume_node.base.overlapped, &bytes_transferred, FALSE) == 0) { + var bytes_transferred: DWORD = undefined; + if (kernel32.GetOverlappedResult(handle, &resume_node.base.overlapped, &bytes_transferred, FALSE) == 0) { switch (kernel32.GetLastError()) { .IO_PENDING => unreachable, .INVALID_USER_BUFFER => return error.SystemResources, @@ -534,7 +538,7 @@ pub fn WriteFile(handle: HANDLE, bytes: []const u8, offset: ?u64) WriteFileError .OPERATION_ABORTED => return error.OperationAborted, .NOT_ENOUGH_QUOTA => return error.SystemResources, .BROKEN_PIPE => return error.BrokenPipe, - else => |err| return windows.unexpectedError(err), + else => |err| return unexpectedError(err), } } return bytes_transferred; diff --git a/lib/std/pdb.zig b/lib/std/pdb.zig index 93fef0b16d..db988fa178 100644 --- a/lib/std/pdb.zig +++ b/lib/std/pdb.zig @@ -470,7 +470,7 @@ pub const Pdb = struct { msf: Msf, pub fn openFile(self: *Pdb, coff_ptr: *coff.Coff, file_name: []u8) !void { - self.in_file = try fs.cwd().openFile(file_name, .{}); + self.in_file = try fs.cwd().openFile(file_name, .{ .always_blocking = true }); self.allocator = coff_ptr.allocator; self.coff = coff_ptr; From 45bce27b8fecda4fba1c22dd191030af29ccbc6f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 1 May 2020 23:17:15 -0400 Subject: [PATCH 2/8] cleanup and fixes. behavior tests passing with evented I/O --- lib/std/child_process.zig | 23 ++++-------- lib/std/debug.zig | 6 +-- lib/std/fs.zig | 57 ++++++++++++++++------------- lib/std/fs/file.zig | 77 +++++++++++++++++---------------------- lib/std/fs/test.zig | 12 +++++- lib/std/io.zig | 25 +++++++++---- lib/std/net.zig | 7 +--- lib/std/os.zig | 12 +++--- lib/std/os/windows.zig | 35 +++++++++++++----- lib/std/pdb.zig | 2 +- 10 files changed, 137 insertions(+), 119 deletions(-) diff --git a/lib/std/child_process.zig b/lib/std/child_process.zig index 12b76143ee..267c32a860 100644 --- a/lib/std/child_process.zig +++ b/lib/std/child_process.zig @@ -433,26 +433,17 @@ pub const ChildProcess = struct { // we are the parent const pid = @intCast(i32, pid_result); if (self.stdin_behavior == StdIo.Pipe) { - self.stdin = File{ - .handle = stdin_pipe[1], - .io_mode = std.io.mode, - }; + self.stdin = File{ .handle = stdin_pipe[1] }; } else { self.stdin = null; } if (self.stdout_behavior == StdIo.Pipe) { - self.stdout = File{ - .handle = stdout_pipe[0], - .io_mode = std.io.mode, - }; + self.stdout = File{ .handle = stdout_pipe[0] }; } else { self.stdout = null; } if (self.stderr_behavior == StdIo.Pipe) { - self.stderr = File{ - .handle = stderr_pipe[0], - .io_mode = std.io.mode, - }; + self.stderr = File{ .handle = stderr_pipe[0] }; } else { self.stderr = null; } @@ -835,8 +826,8 @@ const ErrInt = std.meta.Int(false, @sizeOf(anyerror) * 8); fn writeIntFd(fd: i32, value: ErrInt) !void { const file = File{ .handle = fd, - .io_mode = .blocking, - .async_block_allowed = File.async_block_allowed_yes, + .capable_io_mode = .blocking, + .intended_io_mode = .blocking, }; file.outStream().writeIntNative(u64, @intCast(u64, value)) catch return error.SystemResources; } @@ -844,8 +835,8 @@ fn writeIntFd(fd: i32, value: ErrInt) !void { fn readIntFd(fd: i32) !ErrInt { const file = File{ .handle = fd, - .io_mode = .blocking, - .async_block_allowed = File.async_block_allowed_yes, + .capable_io_mode = .blocking, + .intended_io_mode = .blocking, }; return @intCast(ErrInt, file.inStream().readIntNative(u64) catch return error.SystemResources); } diff --git a/lib/std/debug.zig b/lib/std/debug.zig index d3497b4dc8..97aa2b6e97 100644 --- a/lib/std/debug.zig +++ b/lib/std/debug.zig @@ -667,7 +667,7 @@ pub fn openSelfDebugInfo(allocator: *mem.Allocator) anyerror!DebugInfo { /// TODO resources https://github.com/ziglang/zig/issues/4353 fn openCoffDebugInfo(allocator: *mem.Allocator, coff_file_path: [:0]const u16) !ModuleDebugInfo { noasync { - const coff_file = try std.fs.openFileAbsoluteW(coff_file_path.ptr, .{}); + const coff_file = try std.fs.openFileAbsoluteW(coff_file_path, .{ .intended_io_mode = .blocking }); errdefer coff_file.close(); const coff_obj = try allocator.create(coff.Coff); @@ -1003,7 +1003,7 @@ fn openMachODebugInfo(allocator: *mem.Allocator, macho_file_path: []const u8) !M fn printLineFromFileAnyOs(out_stream: var, line_info: LineInfo) !void { // Need this to always block even in async I/O mode, because this could potentially // be called from e.g. the event loop code crashing. - var f = try fs.cwd().openFile(line_info.file_name, .{ .always_blocking = true }); + var f = try fs.cwd().openFile(line_info.file_name, .{ .intended_io_mode = .blocking }); defer f.close(); // TODO fstat and make sure that the file has the correct size @@ -1051,7 +1051,7 @@ const MachoSymbol = struct { fn mapWholeFile(path: []const u8) ![]align(mem.page_size) const u8 { noasync { - const file = try fs.cwd().openFile(path, .{ .always_blocking = true }); + const file = try fs.cwd().openFile(path, .{ .intended_io_mode = .blocking }); defer file.close(); const file_len = try math.cast(usize, try file.getEndPos()); diff --git a/lib/std/fs.zig b/lib/std/fs.zig index fc8a584a68..99b43a66e3 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -8,6 +8,8 @@ const Allocator = std.mem.Allocator; const assert = std.debug.assert; const math = std.math; +const is_darwin = std.Target.current.os.tag.isDarwin(); + pub const path = @import("fs/path.zig"); pub const File = @import("fs/file.zig").File; @@ -597,8 +599,11 @@ pub const Dir = struct { // Use the O_ locking flags if the os supports them // (Or if it's darwin, as darwin's `open` doesn't support the O_SYNC flag) - const has_flock_open_flags = @hasDecl(os, "O_EXLOCK") and !builtin.os.tag.isDarwin(); - const nonblocking_lock_flag = if (has_flock_open_flags and flags.lock_nonblocking) (os.O_NONBLOCK | os.O_SYNC) else @as(u32, 0); + const has_flock_open_flags = @hasDecl(os, "O_EXLOCK") and !is_darwin; + const nonblocking_lock_flag = if (has_flock_open_flags and flags.lock_nonblocking) + os.O_NONBLOCK | os.O_SYNC + else + @as(u32, 0); const lock_flag: u32 = if (has_flock_open_flags) switch (flags.lock) { .None => @as(u32, 0), .Shared => os.O_SHLOCK | nonblocking_lock_flag, @@ -612,7 +617,7 @@ pub const Dir = struct { @as(u32, os.O_WRONLY) else @as(u32, os.O_RDONLY); - const fd = if (need_async_thread and !flags.always_blocking) + const fd = if (flags.intended_io_mode != .blocking) try std.event.Loop.instance.?.openatZ(self.fd, sub_path, os_flags, 0) else try os.openatZ(self.fd, sub_path, os_flags, 0); @@ -629,11 +634,8 @@ pub const Dir = struct { return File{ .handle = fd, - .io_mode = .blocking, - .async_block_allowed = if (flags.always_blocking) - File.async_block_allowed_yes - else - File.async_block_allowed_no, + .capable_io_mode = .blocking, + .intended_io_mode = flags.intended_io_mode, }; } @@ -648,19 +650,16 @@ pub const Dir = struct { (if (flags.read) @as(u32, w.GENERIC_READ) else 0) | (if (flags.write) @as(u32, w.GENERIC_WRITE) else 0), .share_access = switch (flags.lock) { - .None => @as(?w.ULONG, null), + .None => w.FILE_SHARE_WRITE | w.FILE_SHARE_READ | w.FILE_SHARE_DELETE, .Shared => w.FILE_SHARE_READ | w.FILE_SHARE_DELETE, .Exclusive => w.FILE_SHARE_DELETE, }, .share_access_nonblocking = flags.lock_nonblocking, .creation = w.FILE_OPEN, - .enable_async_io = std.io.is_async and !flags.always_blocking, + .io_mode = flags.intended_io_mode, }), - .io_mode = .blocking, - .async_block_allowed = if (flags.always_blocking) - File.async_block_allowed_yes - else - File.async_block_allowed_no, + .capable_io_mode = std.io.default_mode, + .intended_io_mode = flags.intended_io_mode, }); } @@ -687,8 +686,11 @@ pub const Dir = struct { // Use the O_ locking flags if the os supports them // (Or if it's darwin, as darwin's `open` doesn't support the O_SYNC flag) - const has_flock_open_flags = @hasDecl(os, "O_EXLOCK") and !builtin.os.tag.isDarwin(); - const nonblocking_lock_flag = if (has_flock_open_flags and flags.lock_nonblocking) (os.O_NONBLOCK | os.O_SYNC) else @as(u32, 0); + const has_flock_open_flags = @hasDecl(os, "O_EXLOCK") and !is_darwin; + const nonblocking_lock_flag: u32 = if (has_flock_open_flags and flags.lock_nonblocking) + os.O_NONBLOCK | os.O_SYNC + else + 0; const lock_flag: u32 = if (has_flock_open_flags) switch (flags.lock) { .None => @as(u32, 0), .Shared => os.O_SHLOCK, @@ -700,7 +702,7 @@ pub const Dir = struct { (if (flags.truncate) @as(u32, os.O_TRUNC) else 0) | (if (flags.read) @as(u32, os.O_RDWR) else os.O_WRONLY) | (if (flags.exclusive) @as(u32, os.O_EXCL) else 0); - const fd = if (need_async_thread) + const fd = if (flags.intended_io_mode != .blocking) try std.event.Loop.instance.?.openatZ(self.fd, sub_path_c, os_flags, flags.mode) else try os.openatZ(self.fd, sub_path_c, os_flags, flags.mode); @@ -715,7 +717,11 @@ pub const Dir = struct { }); } - return File{ .handle = fd, .io_mode = .blocking }; + return File{ + .handle = fd, + .capable_io_mode = .blocking, + .intended_io_mode = flags.intended_io_mode, + }; } /// Same as `createFile` but Windows-only and the path parameter is @@ -739,9 +745,10 @@ pub const Dir = struct { @as(u32, w.FILE_OVERWRITE_IF) else @as(u32, w.FILE_OPEN_IF), - .enable_async_io = std.io.is_async, + .io_mode = flags.intended_io_mode, }), - .io_mode = .blocking, + .capable_io_mode = std.io.default_mode, + .intended_io_mode = flags.intended_io_mode, }); } @@ -1257,7 +1264,7 @@ pub const Dir = struct { @as(u32, os.W_OK) else @as(u32, os.F_OK); - const result = if (need_async_thread) + const result = if (need_async_thread and flags.intended_io_mode != .blocking) std.event.Loop.instance.?.faccessatZ(self.fd, sub_path, os_mode, 0) else os.faccessatZ(self.fd, sub_path, os_mode, 0); @@ -1399,8 +1406,8 @@ pub fn openFileAbsoluteZ(absolute_path_c: [*:0]const u8, flags: File.OpenFlags) } /// Same as `openFileAbsolute` but the path parameter is WTF-16 encoded. -pub fn openFileAbsoluteW(absolute_path_w: [*:0]const u16, flags: File.OpenFlags) File.OpenError!File { - assert(path.isAbsoluteWindowsW(absolute_path_w)); +pub fn openFileAbsoluteW(absolute_path_w: []const u16, flags: File.OpenFlags) File.OpenError!File { + assert(path.isAbsoluteWindowsWTF16(absolute_path_w)); return cwd().openFileW(absolute_path_w, flags); } @@ -1617,7 +1624,7 @@ pub fn selfExePathAlloc(allocator: *Allocator) ![]u8 { /// been deleted, the file path looks something like `/a/b/c/exe (deleted)`. /// TODO make the return type of this a null terminated pointer pub fn selfExePath(out_buffer: *[MAX_PATH_BYTES]u8) SelfExePathError![]u8 { - if (comptime std.Target.current.isDarwin()) { + if (is_darwin) { var u32_len: u32 = out_buffer.len; const rc = std.c._NSGetExecutablePath(out_buffer, &u32_len); if (rc != 0) return error.NameTooLong; diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index 4ff8d47287..08ae1bffca 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -8,7 +8,6 @@ const assert = std.debug.assert; const windows = os.windows; const Os = builtin.Os; const maxInt = std.math.maxInt; -const need_async_thread = std.fs.need_async_thread; pub const File = struct { /// The OS-specific file descriptor or file handle. @@ -17,15 +16,14 @@ pub const File = struct { /// On some systems, such as Linux, file system file descriptors are incapable of non-blocking I/O. /// This forces us to perform asynchronous I/O on a dedicated thread, to achieve non-blocking /// file-system I/O. To do this, `File` must be aware of whether it is a file system file descriptor, - /// or, more specifically, whether the I/O is blocking. - io_mode: io.Mode, + /// or, more specifically, whether the I/O is always blocking. + capable_io_mode: io.ModeOverride = io.default_mode, - /// Even when 'std.io.mode' is async, it is still sometimes desirable to perform blocking I/O, although - /// not by default. For example, when printing a stack trace to stderr. - async_block_allowed: @TypeOf(async_block_allowed_no) = async_block_allowed_no, - - pub const async_block_allowed_yes = if (io.is_async) true else {}; - pub const async_block_allowed_no = if (io.is_async) false else {}; + /// Furthermore, even when `std.io.mode` is async, it is still sometimes desirable to perform blocking I/O, + /// although not by default. For example, when printing a stack trace to stderr. + /// This field tracks both by acting as an overriding I/O mode. When not building in async I/O mode, + /// the type only has the `.blocking` tag, making it a zero-bit type. + intended_io_mode: io.ModeOverride = io.default_mode, pub const Mode = os.mode_t; @@ -36,9 +34,7 @@ pub const File = struct { pub const OpenError = windows.CreateFileError || os.OpenError || os.FlockError; - pub const Lock = enum { - None, Shared, Exclusive - }; + pub const Lock = enum { None, Shared, Exclusive }; /// TODO https://github.com/ziglang/zig/issues/3802 pub const OpenFlags = struct { @@ -62,15 +58,16 @@ pub const File = struct { /// Sets whether or not to wait until the file is locked to return. If set to true, /// `error.WouldBlock` will be returned. Otherwise, the file will wait until the file - /// is available to proceed. In async I/O mode, non-blocking at the OS level is always - /// used, and `true` means `error.WouldBlock` is returned, and `false` means - /// `error.WouldBlock` is handled by the event loop. + /// is available to proceed. + /// In async I/O mode, non-blocking at the OS level is + /// determined by `intended_io_mode`, and `true` means `error.WouldBlock` is returned, + /// and `false` means `error.WouldBlock` is handled by the event loop. lock_nonblocking: bool = false, - /// This prevents `O_NONBLOCK` from being passed even if `std.io.is_async`. - /// It allows the use of `noasync` when calling functions related to opening - /// the file, reading, writing, as well as locking functionality. - always_blocking: bool = false, + /// Setting this to `.blocking` prevents `O_NONBLOCK` from being passed even + /// if `std.io.is_async`. It allows the use of `noasync` when calling functions + /// related to opening the file, reading, writing, and locking. + intended_io_mode: io.ModeOverride = io.default_mode, }; /// TODO https://github.com/ziglang/zig/issues/3802 @@ -104,17 +101,25 @@ pub const File = struct { /// Sets whether or not to wait until the file is locked to return. If set to true, /// `error.WouldBlock` will be returned. Otherwise, the file will wait until the file /// is available to proceed. + /// In async I/O mode, non-blocking at the OS level is + /// determined by `intended_io_mode`, and `true` means `error.WouldBlock` is returned, + /// and `false` means `error.WouldBlock` is handled by the event loop. lock_nonblocking: bool = false, /// For POSIX systems this is the file system mode the file will /// be created with. mode: Mode = default_mode, + + /// Setting this to `.blocking` prevents `O_NONBLOCK` from being passed even + /// if `std.io.is_async`. It allows the use of `noasync` when calling functions + /// related to opening the file, reading, writing, and locking. + intended_io_mode: io.ModeOverride = io.default_mode, }; /// Upon success, the stream is in an uninitialized state. To continue using it, /// you must use the open() function. pub fn close(self: File) void { - if (need_async_thread and self.io_mode == .blocking and !self.async_block_allowed) { + if (self.capable_io_mode != self.intended_io_mode) { std.event.Loop.instance.?.close(self.handle); } else { os.close(self.handle); @@ -297,11 +302,7 @@ pub const File = struct { pub const PReadError = os.PReadError; pub fn read(self: File, buffer: []u8) ReadError!usize { - if (builtin.os.tag == .windows) { - const enable_async_io = std.io.is_async and !self.async_block_allowed; - return windows.ReadFile(self.handle, buffer, null, enable_async_io); - } - if (need_async_thread and self.io_mode == .blocking and !self.async_block_allowed) { + if (self.capable_io_mode != self.intended_io_mode) { return std.event.Loop.instance.?.read(self.handle, buffer); } else { return os.read(self.handle, buffer); @@ -321,11 +322,7 @@ pub const File = struct { } pub fn pread(self: File, buffer: []u8, offset: u64) PReadError!usize { - if (builtin.os.tag == .windows) { - const enable_async_io = std.io.is_async and !self.async_block_allowed; - return windows.ReadFile(self.handle, buffer, offset, enable_async_io); - } - if (need_async_thread and self.io_mode == .blocking and !self.async_block_allowed) { + if (self.capable_io_mode != self.intended_io_mode) { return std.event.Loop.instance.?.pread(self.handle, buffer, offset); } else { return os.pread(self.handle, buffer, offset); @@ -345,7 +342,7 @@ pub const File = struct { } pub fn readv(self: File, iovecs: []const os.iovec) ReadError!usize { - if (need_async_thread and self.io_mode == .blocking and !self.async_block_allowed) { + if (self.capable_io_mode != self.intended_io_mode) { return std.event.Loop.instance.?.readv(self.handle, iovecs); } else { return os.readv(self.handle, iovecs); @@ -379,7 +376,7 @@ pub const File = struct { } pub fn preadv(self: File, iovecs: []const os.iovec, offset: u64) PReadError!usize { - if (need_async_thread and self.io_mode == .blocking and !self.async_block_allowed) { + if (self.capable_io_mode != self.intended_io_mode) { return std.event.Loop.instance.?.preadv(self.handle, iovecs, offset); } else { return os.preadv(self.handle, iovecs, offset); @@ -416,11 +413,7 @@ pub const File = struct { pub const PWriteError = os.PWriteError; pub fn write(self: File, bytes: []const u8) WriteError!usize { - if (builtin.os.tag == .windows) { - const enable_async_io = std.io.is_async and !self.async_block_allowed; - return windows.WriteFile(self.handle, bytes, null, enable_async_io); - } - if (need_async_thread and self.io_mode == .blocking and !self.async_block_allowed) { + if (self.capable_io_mode != self.intended_io_mode) { return std.event.Loop.instance.?.write(self.handle, bytes); } else { return os.write(self.handle, bytes); @@ -435,11 +428,7 @@ pub const File = struct { } pub fn pwrite(self: File, bytes: []const u8, offset: u64) PWriteError!usize { - if (builtin.os.tag == .windows) { - const enable_async_io = std.io.is_async and !self.async_block_allowed; - return windows.WriteFile(self.handle, bytes, offset, enable_async_io); - } - if (need_async_thread and self.io_mode == .blocking and !self.async_block_allowed) { + if (self.capable_io_mode != self.intended_io_mode) { return std.event.Loop.instance.?.pwrite(self.handle, bytes, offset); } else { return os.pwrite(self.handle, bytes, offset); @@ -454,7 +443,7 @@ pub const File = struct { } pub fn writev(self: File, iovecs: []const os.iovec_const) WriteError!usize { - if (need_async_thread and self.io_mode == .blocking and !self.async_block_allowed) { + if (self.capable_io_mode != self.intended_io_mode) { return std.event.Loop.instance.?.writev(self.handle, iovecs); } else { return os.writev(self.handle, iovecs); @@ -480,7 +469,7 @@ pub const File = struct { } pub fn pwritev(self: File, iovecs: []os.iovec_const, offset: usize) PWriteError!usize { - if (need_async_thread and self.io_mode == .blocking and !self.async_block_allowed) { + if (self.capable_io_mode != self.intended_io_mode) { return std.event.Loop.instance.?.pwritev(self.handle, iovecs, offset); } else { return os.pwritev(self.handle, iovecs, offset); diff --git a/lib/std/fs/test.zig b/lib/std/fs/test.zig index da2d2c0d0c..77c4018a6a 100644 --- a/lib/std/fs/test.zig +++ b/lib/std/fs/test.zig @@ -27,7 +27,12 @@ test "open file with exclusive nonblocking lock twice" { } test "open file with lock twice, make sure it wasn't open at the same time" { - if (builtin.single_threaded) return; + if (builtin.single_threaded) return error.SkipZigTest; + + if (std.io.is_async) { + // This test starts its own threads and is not compatible with async I/O. + return error.SkipZigTest; + } const filename = "file_lock_test.txt"; @@ -58,6 +63,11 @@ test "open file with lock twice, make sure it wasn't open at the same time" { test "create file, lock and read from multiple process at once" { if (builtin.single_threaded) return error.SkipZigTest; + if (std.io.is_async) { + // This test starts its own threads and is not compatible with async I/O. + return error.SkipZigTest; + } + if (true) { // https://github.com/ziglang/zig/issues/5006 return error.SkipZigTest; diff --git a/lib/std/io.zig b/lib/std/io.zig index b9ab3ecc8e..f6c3325716 100644 --- a/lib/std/io.zig +++ b/lib/std/io.zig @@ -30,6 +30,11 @@ else Mode.blocking; pub const is_async = mode != .blocking; +/// This is an enum value to use for I/O mode at runtime, since it takes up zero bytes at runtime, +/// and makes expressions comptime-known when `is_async` is `false`. +pub const ModeOverride = if (is_async) Mode else enum { blocking }; +pub const default_mode: ModeOverride = if (is_async) Mode.evented else .blocking; + fn getStdOutHandle() os.fd_t { if (builtin.os.tag == .windows) { return os.windows.peb().ProcessParameters.hStdOutput; @@ -42,12 +47,13 @@ fn getStdOutHandle() os.fd_t { return os.STDOUT_FILENO; } -// TODO: async stdout on windows (https://github.com/ziglang/zig/pull/4816#issuecomment-604521023) +/// TODO: async stdout on windows without a dedicated thread. +/// https://github.com/ziglang/zig/pull/4816#issuecomment-604521023 pub fn getStdOut() File { return File{ .handle = getStdOutHandle(), - .io_mode = .blocking, - .async_block_allowed = if (builtin.os.tag == .windows) File.async_block_allowed_yes else File.async_block_allowed_no, + .capable_io_mode = .blocking, + .intended_io_mode = default_mode, }; } @@ -63,11 +69,13 @@ fn getStdErrHandle() os.fd_t { return os.STDERR_FILENO; } +/// This returns a `File` that is configured to block with every write, in order +/// to facilitate better debugging. This can be changed by modifying the `intended_io_mode` field. pub fn getStdErr() File { return File{ .handle = getStdErrHandle(), - .io_mode = .blocking, - .async_block_allowed = File.async_block_allowed_yes, + .capable_io_mode = .blocking, + .intended_io_mode = .blocking, }; } @@ -83,12 +91,13 @@ fn getStdInHandle() os.fd_t { return os.STDIN_FILENO; } -// TODO: async stdin on windows (https://github.com/ziglang/zig/pull/4816#issuecomment-604521023) +/// TODO: async stdin on windows without a dedicated thread. +/// https://github.com/ziglang/zig/pull/4816#issuecomment-604521023 pub fn getStdIn() File { return File{ .handle = getStdInHandle(), - .io_mode = .blocking, - .async_block_allowed = if (builtin.os.tag == .windows) File.async_block_allowed_yes else File.async_block_allowed_no, + .capable_io_mode = .blocking, + .intended_io_mode = default_mode, }; } diff --git a/lib/std/net.zig b/lib/std/net.zig index 9e57aa500b..1c7355bcb2 100644 --- a/lib/std/net.zig +++ b/lib/std/net.zig @@ -412,7 +412,7 @@ pub fn tcpConnectToAddress(address: Address) !fs.File { errdefer os.close(sockfd); try os.connect(sockfd, &address.any, address.getOsSockLen()); - return fs.File{ .handle = sockfd, .io_mode = std.io.mode }; + return fs.File{ .handle = sockfd }; } /// Call `AddressList.deinit` on the result. @@ -1381,10 +1381,7 @@ pub const StreamServer = struct { var adr_len: os.socklen_t = @sizeOf(Address); if (os.accept4(self.sockfd.?, &accepted_addr.any, &adr_len, accept_flags)) |fd| { return Connection{ - .file = fs.File{ - .handle = fd, - .io_mode = std.io.mode, - }, + .file = fs.File{ .handle = fd }, .address = accepted_addr, }; } else |err| switch (err) { diff --git a/lib/std/os.zig b/lib/std/os.zig index 489afe43b7..05d8b60a2d 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -173,8 +173,8 @@ fn getRandomBytesDevURandom(buf: []u8) !void { const file = std.fs.File{ .handle = fd, - .io_mode = .blocking, - .async_block_allowed = std.fs.File.async_block_allowed_yes, + .capable_io_mode = .blocking, + .intended_io_mode = .blocking, }; const stream = file.inStream(); stream.readNoEof(buf) catch return error.Unexpected; @@ -305,7 +305,7 @@ pub const ReadError = error{ /// For POSIX the limit is `math.maxInt(isize)`. pub fn read(fd: fd_t, buf: []u8) ReadError!usize { if (builtin.os.tag == .windows) { - return windows.ReadFile(fd, buf, null, false); + return windows.ReadFile(fd, buf, null, std.io.default_mode); } if (builtin.os.tag == .wasi and !builtin.link_libc) { @@ -408,7 +408,7 @@ pub const PReadError = ReadError || error{Unseekable}; /// used to perform the I/O. `error.WouldBlock` is not possible on Windows. pub fn pread(fd: fd_t, buf: []u8, offset: u64) PReadError!usize { if (builtin.os.tag == .windows) { - return windows.ReadFile(fd, buf, offset, false); + return windows.ReadFile(fd, buf, offset, std.io.default_mode); } while (true) { @@ -584,7 +584,7 @@ pub const WriteError = error{ /// The corresponding POSIX limit is `math.maxInt(isize)`. pub fn write(fd: fd_t, bytes: []const u8) WriteError!usize { if (builtin.os.tag == .windows) { - return windows.WriteFile(fd, bytes, null, false); + return windows.WriteFile(fd, bytes, null, std.io.default_mode); } if (builtin.os.tag == .wasi and !builtin.link_libc) { @@ -709,7 +709,7 @@ pub const PWriteError = WriteError || error{Unseekable}; /// The corresponding POSIX limit is `math.maxInt(isize)`. pub fn pwrite(fd: fd_t, bytes: []const u8, offset: u64) PWriteError!usize { if (std.Target.current.os.tag == .windows) { - return windows.WriteFile(fd, bytes, offset, false); + return windows.WriteFile(fd, bytes, offset, std.io.default_mode); } // Prevent EINVAL. diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig index 7f273c141a..3e55a825de 100644 --- a/lib/std/os/windows.zig +++ b/lib/std/os/windows.zig @@ -110,7 +110,7 @@ pub const OpenFileOptions = struct { share_access: ULONG = FILE_SHARE_WRITE | FILE_SHARE_READ | FILE_SHARE_DELETE, share_access_nonblocking: bool = false, creation: ULONG, - enable_async_io: bool = std.io.is_async, + io_mode: std.io.ModeOverride, }; /// TODO when share_access_nonblocking is false, this implementation uses @@ -145,7 +145,7 @@ pub fn OpenFile(sub_path_w: []const u16, options: OpenFileOptions) OpenError!HAN var delay: usize = 1; while (true) { - const blocking_flag: ULONG = if (!options.enable_async_io) FILE_SYNCHRONOUS_IO_NONALERT else 0; + const blocking_flag: ULONG = if (options.io_mode == .blocking) FILE_SYNCHRONOUS_IO_NONALERT else 0; const rc = ntdll.NtCreateFile( &result, options.access_mask, @@ -451,11 +451,11 @@ pub const ReadFileError = error{ /// If buffer's length exceeds what a Windows DWORD integer can hold, it will be broken into /// multiple non-atomic reads. -pub fn ReadFile(in_hFile: HANDLE, buffer: []u8, offset: ?u64, enable_async_io: bool) ReadFileError!usize { - if (std.event.Loop.instance != null and enable_async_io) { +pub fn ReadFile(in_hFile: HANDLE, buffer: []u8, offset: ?u64, io_mode: std.io.ModeOverride) ReadFileError!usize { + if (io_mode != .blocking) { const loop = std.event.Loop.instance.?; - // TODO support async ReadFile with no offset - const off = if (offset == null) 0 else offset.?; + // TODO make getting the file position non-blocking + const off = if (offset) |o| o else try SetFilePointerEx_CURRENT_get(in_hFile); var resume_node = std.event.Loop.ResumeNode.Basic{ .base = .{ .id = .Basic, @@ -486,6 +486,11 @@ pub fn ReadFile(in_hFile: HANDLE, buffer: []u8, offset: ?u64, enable_async_io: b else => |err| return unexpectedError(err), } } + if (offset == null) { + // TODO make setting the file position non-blocking + const new_off = off + bytes_transferred; + try SetFilePointerEx_CURRENT(in_hFile, @bitCast(i64, new_off)); + } return @as(usize, bytes_transferred); } else { var index: usize = 0; @@ -525,11 +530,16 @@ pub const WriteFileError = error{ Unexpected, }; -pub fn WriteFile(handle: HANDLE, bytes: []const u8, offset: ?u64, enable_async_io: bool) WriteFileError!usize { - if (std.event.Loop.instance != null and enable_async_io) { +pub fn WriteFile( + handle: HANDLE, + bytes: []const u8, + offset: ?u64, + io_mode: std.io.ModeOverride, +) WriteFileError!usize { + if (std.event.Loop.instance != null and io_mode != .blocking) { const loop = std.event.Loop.instance.?; - // TODO support async WriteFile with no offset - const off = if (offset == null) 0 else offset.?; + // TODO make getting the file position non-blocking + const off = if (offset) |o| o else try SetFilePointerEx_CURRENT_get(handle); var resume_node = std.event.Loop.ResumeNode.Basic{ .base = .{ .id = .Basic, @@ -562,6 +572,11 @@ pub fn WriteFile(handle: HANDLE, bytes: []const u8, offset: ?u64, enable_async_i else => |err| return unexpectedError(err), } } + if (offset == null) { + // TODO make setting the file position non-blocking + const new_off = off + bytes_transferred; + try SetFilePointerEx_CURRENT(handle, @bitCast(i64, new_off)); + } return bytes_transferred; } else { var bytes_written: DWORD = undefined; diff --git a/lib/std/pdb.zig b/lib/std/pdb.zig index 0943bc44f3..75589b71ff 100644 --- a/lib/std/pdb.zig +++ b/lib/std/pdb.zig @@ -470,7 +470,7 @@ pub const Pdb = struct { msf: Msf, pub fn openFile(self: *Pdb, coff_ptr: *coff.Coff, file_name: []u8) !void { - self.in_file = try fs.cwd().openFile(file_name, .{ .always_blocking = true }); + self.in_file = try fs.cwd().openFile(file_name, .{ .intended_io_mode = .blocking }); self.allocator = coff_ptr.allocator; self.coff = coff_ptr; From 2272a07ca0661547a6889aa7f1b6262fef5dad85 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 2 May 2020 00:41:19 -0400 Subject: [PATCH 3/8] std.event.Loop: promote the fs thread to be available for all OS's --- lib/std/debug.zig | 90 +++++++++-------- lib/std/event/loop.zig | 213 +++++++++++++--------------------------- lib/std/fs.zig | 2 +- lib/std/os.zig | 19 +++- lib/std/os/windows.zig | 4 +- lib/std/reset_event.zig | 2 +- 6 files changed, 137 insertions(+), 193 deletions(-) diff --git a/lib/std/debug.zig b/lib/std/debug.zig index 97aa2b6e97..df84a8bbcb 100644 --- a/lib/std/debug.zig +++ b/lib/std/debug.zig @@ -112,39 +112,43 @@ pub fn detectTTYConfig() TTY.Config { /// Tries to print the current stack trace to stderr, unbuffered, and ignores any error returned. /// TODO multithreaded awareness pub fn dumpCurrentStackTrace(start_addr: ?usize) void { - const stderr = getStderrStream(); - if (builtin.strip_debug_info) { - noasync stderr.print("Unable to dump stack trace: debug info stripped\n", .{}) catch return; - return; + noasync { + const stderr = getStderrStream(); + if (builtin.strip_debug_info) { + stderr.print("Unable to dump stack trace: debug info stripped\n", .{}) catch return; + return; + } + const debug_info = getSelfDebugInfo() catch |err| { + stderr.print("Unable to dump stack trace: Unable to open debug info: {}\n", .{@errorName(err)}) catch return; + return; + }; + writeCurrentStackTrace(stderr, debug_info, detectTTYConfig(), start_addr) catch |err| { + stderr.print("Unable to dump stack trace: {}\n", .{@errorName(err)}) catch return; + return; + }; } - const debug_info = getSelfDebugInfo() catch |err| { - noasync stderr.print("Unable to dump stack trace: Unable to open debug info: {}\n", .{@errorName(err)}) catch return; - return; - }; - writeCurrentStackTrace(stderr, debug_info, detectTTYConfig(), start_addr) catch |err| { - noasync stderr.print("Unable to dump stack trace: {}\n", .{@errorName(err)}) catch return; - return; - }; } /// Tries to print the stack trace starting from the supplied base pointer to stderr, /// unbuffered, and ignores any error returned. /// TODO multithreaded awareness pub fn dumpStackTraceFromBase(bp: usize, ip: usize) void { - const stderr = getStderrStream(); - if (builtin.strip_debug_info) { - noasync stderr.print("Unable to dump stack trace: debug info stripped\n", .{}) catch return; - return; - } - const debug_info = getSelfDebugInfo() catch |err| { - noasync stderr.print("Unable to dump stack trace: Unable to open debug info: {}\n", .{@errorName(err)}) catch return; - return; - }; - const tty_config = detectTTYConfig(); - printSourceAtAddress(debug_info, stderr, ip, tty_config) catch return; - var it = StackIterator.init(null, bp); - while (it.next()) |return_address| { - printSourceAtAddress(debug_info, stderr, return_address - 1, tty_config) catch return; + noasync { + const stderr = getStderrStream(); + if (builtin.strip_debug_info) { + stderr.print("Unable to dump stack trace: debug info stripped\n", .{}) catch return; + return; + } + const debug_info = getSelfDebugInfo() catch |err| { + stderr.print("Unable to dump stack trace: Unable to open debug info: {}\n", .{@errorName(err)}) catch return; + return; + }; + const tty_config = detectTTYConfig(); + printSourceAtAddress(debug_info, stderr, ip, tty_config) catch return; + var it = StackIterator.init(null, bp); + while (it.next()) |return_address| { + printSourceAtAddress(debug_info, stderr, return_address - 1, tty_config) catch return; + } } } @@ -199,19 +203,21 @@ pub fn captureStackTrace(first_address: ?usize, stack_trace: *builtin.StackTrace /// Tries to print a stack trace to stderr, unbuffered, and ignores any error returned. /// TODO multithreaded awareness pub fn dumpStackTrace(stack_trace: builtin.StackTrace) void { - const stderr = getStderrStream(); - if (builtin.strip_debug_info) { - noasync stderr.print("Unable to dump stack trace: debug info stripped\n", .{}) catch return; - return; + noasync { + const stderr = getStderrStream(); + if (builtin.strip_debug_info) { + stderr.print("Unable to dump stack trace: debug info stripped\n", .{}) catch return; + return; + } + const debug_info = getSelfDebugInfo() catch |err| { + stderr.print("Unable to dump stack trace: Unable to open debug info: {}\n", .{@errorName(err)}) catch return; + return; + }; + writeStackTrace(stack_trace, stderr, getDebugInfoAllocator(), debug_info, detectTTYConfig()) catch |err| { + stderr.print("Unable to dump stack trace: {}\n", .{@errorName(err)}) catch return; + return; + }; } - const debug_info = getSelfDebugInfo() catch |err| { - noasync stderr.print("Unable to dump stack trace: Unable to open debug info: {}\n", .{@errorName(err)}) catch return; - return; - }; - writeStackTrace(stack_trace, stderr, getDebugInfoAllocator(), debug_info, detectTTYConfig()) catch |err| { - noasync stderr.print("Unable to dump stack trace: {}\n", .{@errorName(err)}) catch return; - return; - }; } /// This function invokes undefined behavior when `ok` is `false`. @@ -255,7 +261,7 @@ pub fn panicExtra(trace: ?*const builtin.StackTrace, first_trace_addr: ?usize, c resetSegfaultHandler(); } - switch (panic_stage) { + noasync switch (panic_stage) { 0 => { panic_stage = 1; @@ -267,7 +273,7 @@ pub fn panicExtra(trace: ?*const builtin.StackTrace, first_trace_addr: ?usize, c defer held.release(); const stderr = getStderrStream(); - noasync stderr.print(format ++ "\n", args) catch os.abort(); + stderr.print(format ++ "\n", args) catch os.abort(); if (trace) |t| { dumpStackTrace(t.*); } @@ -292,12 +298,12 @@ pub fn panicExtra(trace: ?*const builtin.StackTrace, first_trace_addr: ?usize, c // we're still holding the mutex but that's fine as we're going to // call abort() const stderr = getStderrStream(); - noasync stderr.print("Panicked during a panic. Aborting.\n", .{}) catch os.abort(); + stderr.print("Panicked during a panic. Aborting.\n", .{}) catch os.abort(); }, else => { // Panicked while printing "Panicked during a panic." }, - } + }; os.abort(); } diff --git a/lib/std/event/loop.zig b/lib/std/event/loop.zig index f0ac85d4f0..6337911cb6 100644 --- a/lib/std/event/loop.zig +++ b/lib/std/event/loop.zig @@ -4,19 +4,27 @@ const root = @import("root"); const assert = std.debug.assert; const testing = std.testing; const mem = std.mem; -const AtomicRmwOp = builtin.AtomicRmwOp; -const AtomicOrder = builtin.AtomicOrder; const os = std.os; const windows = os.windows; const maxInt = std.math.maxInt; const Thread = std.Thread; +const is_windows = std.Target.current.os.tag == .windows; + pub const Loop = struct { next_tick_queue: std.atomic.Queue(anyframe), os_data: OsData, final_resume_node: ResumeNode, pending_event_count: usize, extra_threads: []*Thread, + /// TODO change this to a pool of configurable number of threads + /// and rename it to be not file-system-specific. it will become + /// a thread pool for turning non-CPU-bound blocking things into + /// async things. A fallback for any missing OS-specific API. + fs_thread: *Thread, + fs_queue: std.atomic.Queue(Request), + fs_end_request: Request.Node, + fs_thread_wakeup: std.ResetEvent, /// For resources that have the same lifetime as the `Loop`. /// This is only used by `Loop` for the thread pool and associated resources. @@ -143,7 +151,12 @@ pub const Loop = struct { .handle = undefined, .overlapped = ResumeNode.overlapped_init, }, + .fs_end_request = .{ .data = .{ .msg = .end, .finish = .NoAction } }, + .fs_queue = std.atomic.Queue(Request).init(), + .fs_thread = undefined, + .fs_thread_wakeup = std.ResetEvent.init(), }; + errdefer self.fs_thread_wakeup.deinit(); errdefer self.arena.deinit(); // We need at least one of these in case the fs thread wants to use onNextTick @@ -158,10 +171,19 @@ pub const Loop = struct { try self.initOsData(extra_thread_count); errdefer self.deinitOsData(); + + if (!builtin.single_threaded) { + self.fs_thread = try Thread.spawn(self, posixFsRun); + } + errdefer if (!builtin.single_threaded) { + self.posixFsRequest(&self.fs_end_request); + self.fs_thread.wait(); + }; } pub fn deinit(self: *Loop) void { self.deinitOsData(); + self.fs_thread_wakeup.deinit(); self.arena.deinit(); self.* = undefined; } @@ -173,21 +195,10 @@ pub const Loop = struct { const wakeup_bytes = [_]u8{0x1} ** 8; fn initOsData(self: *Loop, extra_thread_count: usize) InitOsDataError!void { - switch (builtin.os.tag) { + noasync switch (builtin.os.tag) { .linux => { - self.os_data.fs_queue = std.atomic.Queue(Request).init(); - self.os_data.fs_queue_item = 0; - // we need another thread for the file system because Linux does not have an async - // file system I/O API. - self.os_data.fs_end_request = Request.Node{ - .data = Request{ - .msg = .end, - .finish = .NoAction, - }, - }; - errdefer { - while (self.available_eventfd_resume_nodes.pop()) |node| noasync os.close(node.data.eventfd); + while (self.available_eventfd_resume_nodes.pop()) |node| os.close(node.data.eventfd); } for (self.eventfd_resume_nodes) |*eventfd_node| { eventfd_node.* = std.atomic.Stack(ResumeNode.EventFd).Node{ @@ -206,10 +217,10 @@ pub const Loop = struct { } self.os_data.epollfd = try os.epoll_create1(os.EPOLL_CLOEXEC); - errdefer noasync os.close(self.os_data.epollfd); + errdefer os.close(self.os_data.epollfd); self.os_data.final_eventfd = try os.eventfd(0, os.EFD_CLOEXEC | os.EFD_NONBLOCK); - errdefer noasync os.close(self.os_data.final_eventfd); + errdefer os.close(self.os_data.final_eventfd); self.os_data.final_eventfd_event = os.epoll_event{ .events = os.EPOLLIN, @@ -222,12 +233,6 @@ pub const Loop = struct { &self.os_data.final_eventfd_event, ); - self.os_data.fs_thread = try Thread.spawn(self, posixFsRun); - errdefer { - self.posixFsRequest(&self.os_data.fs_end_request); - self.os_data.fs_thread.wait(); - } - if (builtin.single_threaded) { assert(extra_thread_count == 0); return; @@ -236,7 +241,7 @@ pub const Loop = struct { var extra_thread_index: usize = 0; errdefer { // writing 8 bytes to an eventfd cannot fail - const amt = noasync os.write(self.os_data.final_eventfd, &wakeup_bytes) catch unreachable; + const amt = os.write(self.os_data.final_eventfd, &wakeup_bytes) catch unreachable; assert(amt == wakeup_bytes.len); while (extra_thread_index != 0) { extra_thread_index -= 1; @@ -249,22 +254,7 @@ pub const Loop = struct { }, .macosx, .freebsd, .netbsd, .dragonfly => { self.os_data.kqfd = try os.kqueue(); - errdefer noasync os.close(self.os_data.kqfd); - - self.os_data.fs_kqfd = try os.kqueue(); - errdefer noasync os.close(self.os_data.fs_kqfd); - - self.os_data.fs_queue = std.atomic.Queue(Request).init(); - // we need another thread for the file system because Darwin does not have an async - // file system I/O API. - self.os_data.fs_end_request = Request.Node{ - .prev = undefined, - .next = undefined, - .data = Request{ - .msg = .end, - .finish = .NoAction, - }, - }; + errdefer os.close(self.os_data.kqfd); const empty_kevs = &[0]os.Kevent{}; @@ -310,30 +300,6 @@ pub const Loop = struct { self.os_data.final_kevent.flags = os.EV_ENABLE; self.os_data.final_kevent.fflags = os.NOTE_TRIGGER; - self.os_data.fs_kevent_wake = os.Kevent{ - .ident = 0, - .filter = os.EVFILT_USER, - .flags = os.EV_ADD | os.EV_ENABLE, - .fflags = os.NOTE_TRIGGER, - .data = 0, - .udata = undefined, - }; - - self.os_data.fs_kevent_wait = os.Kevent{ - .ident = 0, - .filter = os.EVFILT_USER, - .flags = os.EV_ADD | os.EV_CLEAR, - .fflags = 0, - .data = 0, - .udata = undefined, - }; - - self.os_data.fs_thread = try Thread.spawn(self, posixFsRun); - errdefer { - self.posixFsRequest(&self.os_data.fs_end_request); - self.os_data.fs_thread.wait(); - } - if (builtin.single_threaded) { assert(extra_thread_count == 0); return; @@ -401,25 +367,24 @@ pub const Loop = struct { } }, else => {}, - } + }; } fn deinitOsData(self: *Loop) void { - switch (builtin.os.tag) { + noasync switch (builtin.os.tag) { .linux => { - noasync os.close(self.os_data.final_eventfd); - while (self.available_eventfd_resume_nodes.pop()) |node| noasync os.close(node.data.eventfd); - noasync os.close(self.os_data.epollfd); + os.close(self.os_data.final_eventfd); + while (self.available_eventfd_resume_nodes.pop()) |node| os.close(node.data.eventfd); + os.close(self.os_data.epollfd); }, .macosx, .freebsd, .netbsd, .dragonfly => { - noasync os.close(self.os_data.kqfd); - noasync os.close(self.os_data.fs_kqfd); + os.close(self.os_data.kqfd); }, .windows => { windows.CloseHandle(self.os_data.io_port); }, else => {}, - } + }; } /// resume_node must live longer than the anyframe that it holds a reference to. @@ -635,7 +600,7 @@ pub const Loop = struct { .freebsd, .netbsd, .dragonfly, - => self.os_data.fs_thread.wait(), + => self.fs_thread.wait(), else => {}, } @@ -672,23 +637,25 @@ pub const Loop = struct { /// call finishOneEvent when done pub fn beginOneEvent(self: *Loop) void { - _ = @atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Add, 1, AtomicOrder.SeqCst); + _ = @atomicRmw(usize, &self.pending_event_count, .Add, 1, .SeqCst); } pub fn finishOneEvent(self: *Loop) void { - const prev = @atomicRmw(usize, &self.pending_event_count, AtomicRmwOp.Sub, 1, AtomicOrder.SeqCst); - if (prev == 1) { + noasync { + const prev = @atomicRmw(usize, &self.pending_event_count, .Sub, 1, .SeqCst); + if (prev != 1) return; + // cause all the threads to stop + self.posixFsRequest(&self.fs_end_request); + switch (builtin.os.tag) { .linux => { - self.posixFsRequest(&self.os_data.fs_end_request); // writing 8 bytes to an eventfd cannot fail - const amt = noasync os.write(self.os_data.final_eventfd, &wakeup_bytes) catch unreachable; + const amt = os.write(self.os_data.final_eventfd, &wakeup_bytes) catch unreachable; assert(amt == wakeup_bytes.len); return; }, .macosx, .freebsd, .netbsd, .dragonfly => { - self.posixFsRequest(&self.os_data.fs_end_request); const final_kevent = @as(*const [1]os.Kevent, &self.os_data.final_kevent); const empty_kevs = &[0]os.Kevent{}; // cannot fail because we already added it and this just enables it @@ -1041,73 +1008,55 @@ pub const Loop = struct { fn posixFsRequest(self: *Loop, request_node: *Request.Node) void { self.beginOneEvent(); // finished in posixFsRun after processing the msg - self.os_data.fs_queue.put(request_node); - switch (builtin.os.tag) { - .macosx, .freebsd, .netbsd, .dragonfly => { - const fs_kevs = @as(*const [1]os.Kevent, &self.os_data.fs_kevent_wake); - const empty_kevs = &[0]os.Kevent{}; - _ = os.kevent(self.os_data.fs_kqfd, fs_kevs, empty_kevs, null) catch unreachable; - }, - .linux => { - @atomicStore(i32, &self.os_data.fs_queue_item, 1, AtomicOrder.SeqCst); - const rc = os.linux.futex_wake(&self.os_data.fs_queue_item, os.linux.FUTEX_WAKE, 1); - switch (os.linux.getErrno(rc)) { - 0 => {}, - os.EINVAL => unreachable, - else => unreachable, - } - }, - else => @compileError("Unsupported OS"), - } + self.fs_queue.put(request_node); + self.fs_thread_wakeup.set(); } fn posixFsCancel(self: *Loop, request_node: *Request.Node) void { - if (self.os_data.fs_queue.remove(request_node)) { + if (self.fs_queue.remove(request_node)) { self.finishOneEvent(); } } - // TODO make this whole function noasync - // https://github.com/ziglang/zig/issues/3157 fn posixFsRun(self: *Loop) void { - while (true) { - if (builtin.os.tag == .linux) { - @atomicStore(i32, &self.os_data.fs_queue_item, 0, .SeqCst); - } - while (self.os_data.fs_queue.get()) |node| { + noasync while (true) { + self.fs_thread_wakeup.reset(); + while (self.fs_queue.get()) |node| { switch (node.data.msg) { .end => return, .read => |*msg| { - msg.result = noasync os.read(msg.fd, msg.buf); + msg.result = os.read(msg.fd, msg.buf); }, .readv => |*msg| { - msg.result = noasync os.readv(msg.fd, msg.iov); + msg.result = os.readv(msg.fd, msg.iov); }, .write => |*msg| { - msg.result = noasync os.write(msg.fd, msg.bytes); + msg.result = os.write(msg.fd, msg.bytes); }, .writev => |*msg| { - msg.result = noasync os.writev(msg.fd, msg.iov); + msg.result = os.writev(msg.fd, msg.iov); }, .pwritev => |*msg| { - msg.result = noasync os.pwritev(msg.fd, msg.iov, msg.offset); + msg.result = os.pwritev(msg.fd, msg.iov, msg.offset); }, .pread => |*msg| { - msg.result = noasync os.pread(msg.fd, msg.buf, msg.offset); + msg.result = os.pread(msg.fd, msg.buf, msg.offset); }, .preadv => |*msg| { - msg.result = noasync os.preadv(msg.fd, msg.iov, msg.offset); + msg.result = os.preadv(msg.fd, msg.iov, msg.offset); }, .open => |*msg| { - msg.result = noasync os.openZ(msg.path, msg.flags, msg.mode); + if (is_windows) unreachable; // TODO + msg.result = os.openZ(msg.path, msg.flags, msg.mode); }, .openat => |*msg| { - msg.result = noasync os.openatZ(msg.fd, msg.path, msg.flags, msg.mode); + if (is_windows) unreachable; // TODO + msg.result = os.openatZ(msg.fd, msg.path, msg.flags, msg.mode); }, .faccessat => |*msg| { - msg.result = noasync os.faccessatZ(msg.dirfd, msg.path, msg.mode, msg.flags); + msg.result = os.faccessatZ(msg.dirfd, msg.path, msg.mode, msg.flags); }, - .close => |*msg| noasync os.close(msg.fd), + .close => |*msg| os.close(msg.fd), } switch (node.data.finish) { .TickNode => |*tick_node| self.onNextTick(tick_node), @@ -1115,22 +1064,8 @@ pub const Loop = struct { } self.finishOneEvent(); } - switch (builtin.os.tag) { - .linux => { - const rc = os.linux.futex_wait(&self.os_data.fs_queue_item, os.linux.FUTEX_WAIT, 0, null); - switch (os.linux.getErrno(rc)) { - 0, os.EINTR, os.EAGAIN => continue, - else => unreachable, - } - }, - .macosx, .freebsd, .netbsd, .dragonfly => { - const fs_kevs = @as(*const [1]os.Kevent, &self.os_data.fs_kevent_wait); - var out_kevs: [1]os.Kevent = undefined; - _ = os.kevent(self.os_data.fs_kqfd, fs_kevs, out_kevs[0..], null) catch unreachable; - }, - else => @compileError("Unsupported OS"), - } - } + self.fs_thread_wakeup.wait(); + }; } const OsData = switch (builtin.os.tag) { @@ -1146,22 +1081,12 @@ pub const Loop = struct { const KEventData = struct { kqfd: i32, final_kevent: os.Kevent, - fs_kevent_wake: os.Kevent, - fs_kevent_wait: os.Kevent, - fs_thread: *Thread, - fs_kqfd: i32, - fs_queue: std.atomic.Queue(Request), - fs_end_request: Request.Node, }; const LinuxOsData = struct { epollfd: i32, final_eventfd: i32, final_eventfd_event: os.linux.epoll_event, - fs_thread: *Thread, - fs_queue_item: i32, - fs_queue: std.atomic.Queue(Request), - fs_end_request: Request.Node, }; pub const Request = struct { @@ -1302,11 +1227,11 @@ test "std.event.Loop - basic" { loop.run(); } -async fn testEventLoop() i32 { +fn testEventLoop() i32 { return 1234; } -async fn testEventLoop2(h: anyframe->i32, did_it: *bool) void { +fn testEventLoop2(h: anyframe->i32, did_it: *bool) void { const value = await h; testing.expect(value == 1234); did_it.* = true; diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 99b43a66e3..7dcfc9261c 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -734,7 +734,7 @@ pub const Dir = struct { .dir = self.fd, .access_mask = w.SYNCHRONIZE | w.GENERIC_WRITE | read_flag, .share_access = switch (flags.lock) { - .None => @as(?w.ULONG, null), + .None => w.FILE_SHARE_WRITE | w.FILE_SHARE_READ | w.FILE_SHARE_DELETE, .Shared => w.FILE_SHARE_READ | w.FILE_SHARE_DELETE, .Exclusive => w.FILE_SHARE_DELETE, }, diff --git a/lib/std/os.zig b/lib/std/os.zig index 05d8b60a2d..39f5d36ddf 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -854,8 +854,11 @@ pub const OpenError = error{ /// Open and possibly create a file. Keeps trying if it gets interrupted. /// See also `openC`. -/// TODO support windows pub fn open(file_path: []const u8, flags: u32, perm: usize) OpenError!fd_t { + if (std.Target.current.os.tag == .windows) { + const file_path_w = try windows.sliceToPrefixedFileW(file_path); + return openW(&file_path_w, flags, perm); + } const file_path_c = try toPosixPath(file_path); return openZ(&file_path_c, flags, perm); } @@ -864,8 +867,11 @@ pub const openC = @compileError("deprecated: renamed to openZ"); /// Open and possibly create a file. Keeps trying if it gets interrupted. /// See also `open`. -/// TODO support windows pub fn openZ(file_path: [*:0]const u8, flags: u32, perm: usize) OpenError!fd_t { + if (std.Target.current.os.tag == .windows) { + const file_path_w = try windows.cStrToPrefixedFileW(file_path); + return openW(&file_path_w, flags, perm); + } while (true) { const rc = system.open(file_path, flags, perm); switch (errno(rc)) { @@ -895,6 +901,13 @@ pub fn openZ(file_path: [*:0]const u8, flags: u32, perm: usize) OpenError!fd_t { } } +/// Windows-only. The path parameter is +/// [WTF-16](https://simonsapin.github.io/wtf-8/#potentially-ill-formed-utf-16) encoded. +/// Translates the POSIX open API call to a Windows API call. +pub fn openW(file_path_w: []const u16, flags: u32, perm: usize) OpenError!fd_t { + @compileError("TODO implement openW for windows"); +} + /// Open and possibly create a file. Keeps trying if it gets interrupted. /// `file_path` is relative to the open directory handle `dir_fd`. /// See also `openatC`. @@ -1683,7 +1696,7 @@ pub fn renameatW( .dir = old_dir_fd, .access_mask = windows.SYNCHRONIZE | windows.GENERIC_WRITE | windows.DELETE, .creation = windows.FILE_OPEN, - .enable_async_io = false, + .io_mode = .blocking, }) catch |err| switch (err) { error.WouldBlock => unreachable, // Not possible without `.share_access_nonblocking = true`. else => |e| return e, diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig index 3e55a825de..3ef3b63086 100644 --- a/lib/std/os/windows.zig +++ b/lib/std/os/windows.zig @@ -116,10 +116,10 @@ pub const OpenFileOptions = struct { /// TODO when share_access_nonblocking is false, this implementation uses /// untinterruptible sleep() to block. This is not the final iteration of the API. pub fn OpenFile(sub_path_w: []const u16, options: OpenFileOptions) OpenError!HANDLE { - if (sub_path_w[0] == '.' and sub_path_w[1] == 0) { + if (mem.eql(u16, sub_path_w, ".")) { return error.IsDir; } - if (sub_path_w[0] == '.' and sub_path_w[1] == '.' and sub_path_w[2] == 0) { + if (mem.eql(u16, sub_path_w, "..")) { return error.IsDir; } diff --git a/lib/std/reset_event.zig b/lib/std/reset_event.zig index c28db809ca..860f494f3f 100644 --- a/lib/std/reset_event.zig +++ b/lib/std/reset_event.zig @@ -52,7 +52,7 @@ pub const ResetEvent = struct { /// Wait for the event to be set by blocking the current thread. /// A timeout in nanoseconds can be provided as a hint for how - /// long the thread should block on the unset event before throwind error.TimedOut. + /// long the thread should block on the unset event before throwing error.TimedOut. pub fn timedWait(self: *ResetEvent, timeout_ns: u64) !void { return self.os_event.wait(timeout_ns); } From 43f7856bac5fd1a93970d7cebd20289de2367691 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 2 May 2020 01:25:22 -0400 Subject: [PATCH 4/8] fix regressions in windows std lib tests --- lib/std/dynamic_library.zig | 4 +- lib/std/fs.zig | 22 +++---- lib/std/os.zig | 46 ++++++------- lib/std/os/windows.zig | 127 ++++++++++++++---------------------- 4 files changed, 86 insertions(+), 113 deletions(-) diff --git a/lib/std/dynamic_library.zig b/lib/std/dynamic_library.zig index 110d476b10..70e26cc71c 100644 --- a/lib/std/dynamic_library.zig +++ b/lib/std/dynamic_library.zig @@ -328,14 +328,14 @@ pub const WindowsDynLib = struct { pub fn open(path: []const u8) !WindowsDynLib { const path_w = try windows.sliceToPrefixedFileW(path); - return openW(&path_w); + return openW(path_w.span().ptr); } pub const openC = @compileError("deprecated: renamed to openZ"); pub fn openZ(path_c: [*:0]const u8) !WindowsDynLib { const path_w = try windows.cStrToPrefixedFileW(path_c); - return openW(&path_w); + return openW(path_w.span().ptr); } pub fn openW(path_w: [*:0]const u16) !WindowsDynLib { diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 7dcfc9261c..efcad99f40 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -199,7 +199,7 @@ pub const AtomicFile = struct { if (std.Target.current.os.tag == .windows) { const dest_path_w = try os.windows.sliceToPrefixedFileW(self.dest_basename); const tmp_path_w = try os.windows.cStrToPrefixedFileW(&self.tmp_path_buf); - try os.renameatW(self.dir.fd, &tmp_path_w, self.dir.fd, &dest_path_w, os.windows.TRUE); + try os.renameatW(self.dir.fd, tmp_path_w.span(), self.dir.fd, dest_path_w.span(), os.windows.TRUE); self.file_exists = false; } else { const dest_path_c = try os.toPosixPath(self.dest_basename); @@ -582,7 +582,7 @@ pub const Dir = struct { pub fn openFile(self: Dir, sub_path: []const u8, flags: File.OpenFlags) File.OpenError!File { if (builtin.os.tag == .windows) { const path_w = try os.windows.sliceToPrefixedFileW(sub_path); - return self.openFileW(&path_w, flags); + return self.openFileW(path_w.span(), flags); } const path_c = try os.toPosixPath(sub_path); return self.openFileZ(&path_c, flags); @@ -594,7 +594,7 @@ pub const Dir = struct { pub fn openFileZ(self: Dir, sub_path: [*:0]const u8, flags: File.OpenFlags) File.OpenError!File { if (builtin.os.tag == .windows) { const path_w = try os.windows.cStrToPrefixedFileW(sub_path); - return self.openFileW(&path_w, flags); + return self.openFileW(path_w.span(), flags); } // Use the O_ locking flags if the os supports them @@ -669,7 +669,7 @@ pub const Dir = struct { pub fn createFile(self: Dir, sub_path: []const u8, flags: File.CreateFlags) File.OpenError!File { if (builtin.os.tag == .windows) { const path_w = try os.windows.sliceToPrefixedFileW(sub_path); - return self.createFileW(&path_w, flags); + return self.createFileW(path_w.span(), flags); } const path_c = try os.toPosixPath(sub_path); return self.createFileZ(&path_c, flags); @@ -681,7 +681,7 @@ pub const Dir = struct { pub fn createFileZ(self: Dir, sub_path_c: [*:0]const u8, flags: File.CreateFlags) File.OpenError!File { if (builtin.os.tag == .windows) { const path_w = try os.windows.cStrToPrefixedFileW(sub_path_c); - return self.createFileW(&path_w, flags); + return self.createFileW(path_w.span(), flags); } // Use the O_ locking flags if the os supports them @@ -832,7 +832,7 @@ pub const Dir = struct { pub fn openDir(self: Dir, sub_path: []const u8, args: OpenDirOptions) OpenError!Dir { if (builtin.os.tag == .windows) { const sub_path_w = try os.windows.sliceToPrefixedFileW(sub_path); - return self.openDirW(&sub_path_w, args); + return self.openDirW(sub_path_w.span().ptr, args); } else { const sub_path_c = try os.toPosixPath(sub_path); return self.openDirZ(&sub_path_c, args); @@ -845,7 +845,7 @@ pub const Dir = struct { pub fn openDirZ(self: Dir, sub_path_c: [*:0]const u8, args: OpenDirOptions) OpenError!Dir { if (builtin.os.tag == .windows) { const sub_path_w = try os.windows.cStrToPrefixedFileW(sub_path_c); - return self.openDirW(&sub_path_w, args); + return self.openDirW(sub_path_w.span().ptr, args); } else if (!args.iterate) { const O_PATH = if (@hasDecl(os, "O_PATH")) os.O_PATH else 0; return self.openDirFlagsZ(sub_path_c, os.O_DIRECTORY | os.O_RDONLY | os.O_CLOEXEC | O_PATH); @@ -987,7 +987,7 @@ pub const Dir = struct { pub fn deleteDir(self: Dir, sub_path: []const u8) DeleteDirError!void { if (builtin.os.tag == .windows) { const sub_path_w = try os.windows.sliceToPrefixedFileW(sub_path); - return self.deleteDirW(&sub_path_w); + return self.deleteDirW(sub_path_w.span().ptr); } const sub_path_c = try os.toPosixPath(sub_path); return self.deleteDirZ(&sub_path_c); @@ -1246,7 +1246,7 @@ pub const Dir = struct { pub fn access(self: Dir, sub_path: []const u8, flags: File.OpenFlags) AccessError!void { if (builtin.os.tag == .windows) { const sub_path_w = try os.windows.sliceToPrefixedFileW(sub_path); - return self.accessW(&sub_path_w, flags); + return self.accessW(sub_path_w.span().ptr, flags); } const path_c = try os.toPosixPath(sub_path); return self.accessZ(&path_c, flags); @@ -1256,7 +1256,7 @@ pub const Dir = struct { pub fn accessZ(self: Dir, sub_path: [*:0]const u8, flags: File.OpenFlags) AccessError!void { if (builtin.os.tag == .windows) { const sub_path_w = try os.windows.cStrToPrefixedFileW(sub_path); - return self.accessW(&sub_path_w, flags); + return self.accessW(sub_path_w.span().ptr, flags); } const os_mode = if (flags.write and flags.read) @as(u32, os.R_OK | os.W_OK) @@ -1596,7 +1596,7 @@ pub fn openSelfExe() OpenSelfExeError!File { if (builtin.os.tag == .windows) { const wide_slice = selfExePathW(); const prefixed_path_w = try os.windows.wToPrefixedFileW(wide_slice); - return cwd().openFileW(&prefixed_path_w, .{}); + return cwd().openFileW(prefixed_path_w.span(), .{}); } var buf: [MAX_PATH_BYTES]u8 = undefined; const self_exe_path = try selfExePath(&buf); diff --git a/lib/std/os.zig b/lib/std/os.zig index 39f5d36ddf..a1b0dc1991 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -857,7 +857,7 @@ pub const OpenError = error{ pub fn open(file_path: []const u8, flags: u32, perm: usize) OpenError!fd_t { if (std.Target.current.os.tag == .windows) { const file_path_w = try windows.sliceToPrefixedFileW(file_path); - return openW(&file_path_w, flags, perm); + return openW(file_path_w.span(), flags, perm); } const file_path_c = try toPosixPath(file_path); return openZ(&file_path_c, flags, perm); @@ -870,7 +870,7 @@ pub const openC = @compileError("deprecated: renamed to openZ"); pub fn openZ(file_path: [*:0]const u8, flags: u32, perm: usize) OpenError!fd_t { if (std.Target.current.os.tag == .windows) { const file_path_w = try windows.cStrToPrefixedFileW(file_path); - return openW(&file_path_w, flags, perm); + return openW(file_path_w.span(), flags, perm); } while (true) { const rc = system.open(file_path, flags, perm); @@ -1317,7 +1317,7 @@ pub fn symlink(target_path: []const u8, sym_link_path: []const u8) SymLinkError! if (builtin.os.tag == .windows) { const target_path_w = try windows.sliceToPrefixedFileW(target_path); const sym_link_path_w = try windows.sliceToPrefixedFileW(sym_link_path); - return windows.CreateSymbolicLinkW(&sym_link_path_w, &target_path_w, 0); + return windows.CreateSymbolicLinkW(sym_link_path_w.span().ptr, target_path_w.span().ptr, 0); } else { const target_path_c = try toPosixPath(target_path); const sym_link_path_c = try toPosixPath(sym_link_path); @@ -1333,7 +1333,7 @@ pub fn symlinkZ(target_path: [*:0]const u8, sym_link_path: [*:0]const u8) SymLin if (builtin.os.tag == .windows) { const target_path_w = try windows.cStrToPrefixedFileW(target_path); const sym_link_path_w = try windows.cStrToPrefixedFileW(sym_link_path); - return windows.CreateSymbolicLinkW(&sym_link_path_w, &target_path_w, 0); + return windows.CreateSymbolicLinkW(sym_link_path_w.span().ptr, target_path_w.span().ptr, 0); } switch (errno(system.symlink(target_path, sym_link_path))) { 0 => return, @@ -1409,7 +1409,7 @@ pub const UnlinkError = error{ pub fn unlink(file_path: []const u8) UnlinkError!void { if (builtin.os.tag == .windows) { const file_path_w = try windows.sliceToPrefixedFileW(file_path); - return windows.DeleteFileW(&file_path_w); + return windows.DeleteFileW(file_path_w.span().ptr); } else { const file_path_c = try toPosixPath(file_path); return unlinkZ(&file_path_c); @@ -1422,7 +1422,7 @@ pub const unlinkC = @compileError("deprecated: renamed to unlinkZ"); pub fn unlinkZ(file_path: [*:0]const u8) UnlinkError!void { if (builtin.os.tag == .windows) { const file_path_w = try windows.cStrToPrefixedFileW(file_path); - return windows.DeleteFileW(&file_path_w); + return windows.DeleteFileW(file_path_w.span().ptr); } switch (errno(system.unlink(file_path))) { 0 => return, @@ -1453,7 +1453,7 @@ pub const UnlinkatError = UnlinkError || error{ pub fn unlinkat(dirfd: fd_t, file_path: []const u8, flags: u32) UnlinkatError!void { if (builtin.os.tag == .windows) { const file_path_w = try windows.sliceToPrefixedFileW(file_path); - return unlinkatW(dirfd, &file_path_w, flags); + return unlinkatW(dirfd, file_path_w.span().ptr, flags); } const file_path_c = try toPosixPath(file_path); return unlinkatZ(dirfd, &file_path_c, flags); @@ -1465,7 +1465,7 @@ pub const unlinkatC = @compileError("deprecated: renamed to unlinkatZ"); pub fn unlinkatZ(dirfd: fd_t, file_path_c: [*:0]const u8, flags: u32) UnlinkatError!void { if (builtin.os.tag == .windows) { const file_path_w = try windows.cStrToPrefixedFileW(file_path_c); - return unlinkatW(dirfd, &file_path_w, flags); + return unlinkatW(dirfd, file_path_w.span().ptr, flags); } switch (errno(system.unlinkat(dirfd, file_path_c, flags))) { 0 => return, @@ -1580,7 +1580,7 @@ pub fn rename(old_path: []const u8, new_path: []const u8) RenameError!void { if (builtin.os.tag == .windows) { const old_path_w = try windows.sliceToPrefixedFileW(old_path); const new_path_w = try windows.sliceToPrefixedFileW(new_path); - return renameW(&old_path_w, &new_path_w); + return renameW(old_path_w.span().ptr, new_path_w.span().ptr); } else { const old_path_c = try toPosixPath(old_path); const new_path_c = try toPosixPath(new_path); @@ -1595,7 +1595,7 @@ pub fn renameZ(old_path: [*:0]const u8, new_path: [*:0]const u8) RenameError!voi if (builtin.os.tag == .windows) { const old_path_w = try windows.cStrToPrefixedFileW(old_path); const new_path_w = try windows.cStrToPrefixedFileW(new_path); - return renameW(&old_path_w, &new_path_w); + return renameW(old_path_w.span().ptr, new_path_w.span().ptr); } switch (errno(system.rename(old_path, new_path))) { 0 => return, @@ -1638,7 +1638,7 @@ pub fn renameat( if (builtin.os.tag == .windows) { const old_path_w = try windows.sliceToPrefixedFileW(old_path); const new_path_w = try windows.sliceToPrefixedFileW(new_path); - return renameatW(old_dir_fd, &old_path_w, new_dir_fd, &new_path_w, windows.TRUE); + return renameatW(old_dir_fd, old_path_w.span(), new_dir_fd, new_path_w.span(), windows.TRUE); } else { const old_path_c = try toPosixPath(old_path); const new_path_c = try toPosixPath(new_path); @@ -1656,7 +1656,7 @@ pub fn renameatZ( if (builtin.os.tag == .windows) { const old_path_w = try windows.cStrToPrefixedFileW(old_path); const new_path_w = try windows.cStrToPrefixedFileW(new_path); - return renameatW(old_dir_fd, &old_path_w, new_dir_fd, &new_path_w, windows.TRUE); + return renameatW(old_dir_fd, old_path_w.span(), new_dir_fd, new_path_w.span(), windows.TRUE); } switch (errno(system.renameat(old_dir_fd, old_path, new_dir_fd, new_path))) { @@ -1760,7 +1760,7 @@ pub const MakeDirError = error{ pub fn mkdirat(dir_fd: fd_t, sub_dir_path: []const u8, mode: u32) MakeDirError!void { if (builtin.os.tag == .windows) { const sub_dir_path_w = try windows.sliceToPrefixedFileW(sub_dir_path); - return mkdiratW(dir_fd, &sub_dir_path_w, mode); + return mkdiratW(dir_fd, sub_dir_path_w.span().ptr, mode); } else { const sub_dir_path_c = try toPosixPath(sub_dir_path); return mkdiratZ(dir_fd, &sub_dir_path_c, mode); @@ -1772,7 +1772,7 @@ pub const mkdiratC = @compileError("deprecated: renamed to mkdiratZ"); pub fn mkdiratZ(dir_fd: fd_t, sub_dir_path: [*:0]const u8, mode: u32) MakeDirError!void { if (builtin.os.tag == .windows) { const sub_dir_path_w = try windows.cStrToPrefixedFileW(sub_dir_path); - return mkdiratW(dir_fd, &sub_dir_path_w, mode); + return mkdiratW(dir_fd, sub_dir_path_w.span().ptr, mode); } switch (errno(system.mkdirat(dir_fd, sub_dir_path, mode))) { 0 => return, @@ -1816,7 +1816,7 @@ pub fn mkdir(dir_path: []const u8, mode: u32) MakeDirError!void { pub fn mkdirZ(dir_path: [*:0]const u8, mode: u32) MakeDirError!void { if (builtin.os.tag == .windows) { const dir_path_w = try windows.cStrToPrefixedFileW(dir_path); - const sub_dir_handle = try windows.CreateDirectoryW(null, &dir_path_w, null); + const sub_dir_handle = try windows.CreateDirectoryW(null, dir_path_w.span().ptr, null); windows.CloseHandle(sub_dir_handle); return; } @@ -1857,7 +1857,7 @@ pub const DeleteDirError = error{ pub fn rmdir(dir_path: []const u8) DeleteDirError!void { if (builtin.os.tag == .windows) { const dir_path_w = try windows.sliceToPrefixedFileW(dir_path); - return windows.RemoveDirectoryW(&dir_path_w); + return windows.RemoveDirectoryW(dir_path_w.span().ptr); } else { const dir_path_c = try toPosixPath(dir_path); return rmdirZ(&dir_path_c); @@ -1870,7 +1870,7 @@ pub const rmdirC = @compileError("deprecated: renamed to rmdirZ"); pub fn rmdirZ(dir_path: [*:0]const u8) DeleteDirError!void { if (builtin.os.tag == .windows) { const dir_path_w = try windows.cStrToPrefixedFileW(dir_path); - return windows.RemoveDirectoryW(&dir_path_w); + return windows.RemoveDirectoryW(dir_path_w.span().ptr); } switch (errno(system.rmdir(dir_path))) { 0 => return, @@ -2880,7 +2880,7 @@ pub const AccessError = error{ pub fn access(path: []const u8, mode: u32) AccessError!void { if (builtin.os.tag == .windows) { const path_w = try windows.sliceToPrefixedFileW(path); - _ = try windows.GetFileAttributesW(&path_w); + _ = try windows.GetFileAttributesW(path_w.span().ptr); return; } const path_c = try toPosixPath(path); @@ -2893,7 +2893,7 @@ pub const accessC = @compileError("Deprecated in favor of `accessZ`"); pub fn accessZ(path: [*:0]const u8, mode: u32) AccessError!void { if (builtin.os.tag == .windows) { const path_w = try windows.cStrToPrefixedFileW(path); - _ = try windows.GetFileAttributesW(&path_w); + _ = try windows.GetFileAttributesW(path_w.span().ptr); return; } switch (errno(system.access(path, mode))) { @@ -2934,7 +2934,7 @@ pub fn accessW(path: [*:0]const u16, mode: u32) windows.GetFileAttributesError!v pub fn faccessat(dirfd: fd_t, path: []const u8, mode: u32, flags: u32) AccessError!void { if (builtin.os.tag == .windows) { const path_w = try windows.sliceToPrefixedFileW(path); - return faccessatW(dirfd, &path_w, mode, flags); + return faccessatW(dirfd, path_w.span().ptr, mode, flags); } const path_c = try toPosixPath(path); return faccessatZ(dirfd, &path_c, mode, flags); @@ -2944,7 +2944,7 @@ pub fn faccessat(dirfd: fd_t, path: []const u8, mode: u32, flags: u32) AccessErr pub fn faccessatZ(dirfd: fd_t, path: [*:0]const u8, mode: u32, flags: u32) AccessError!void { if (builtin.os.tag == .windows) { const path_w = try windows.cStrToPrefixedFileW(path); - return faccessatW(dirfd, &path_w, mode, flags); + return faccessatW(dirfd, path_w.span().ptr, mode, flags); } switch (errno(system.faccessat(dirfd, path, mode, flags))) { 0 => return, @@ -3299,7 +3299,7 @@ pub const RealPathError = error{ pub fn realpath(pathname: []const u8, out_buffer: *[MAX_PATH_BYTES]u8) RealPathError![]u8 { if (builtin.os.tag == .windows) { const pathname_w = try windows.sliceToPrefixedFileW(pathname); - return realpathW(&pathname_w, out_buffer); + return realpathW(pathname_w.span().ptr, out_buffer); } const pathname_c = try toPosixPath(pathname); return realpathZ(&pathname_c, out_buffer); @@ -3311,7 +3311,7 @@ pub const realpathC = @compileError("deprecated: renamed realpathZ"); pub fn realpathZ(pathname: [*:0]const u8, out_buffer: *[MAX_PATH_BYTES]u8) RealPathError![]u8 { if (builtin.os.tag == .windows) { const pathname_w = try windows.cStrToPrefixedFileW(pathname); - return realpathW(&pathname_w, out_buffer); + return realpathW(pathname_w.span().ptr, out_buffer); } if (builtin.os.tag == .linux and !builtin.link_libc) { const fd = openZ(pathname, linux.O_PATH | linux.O_NONBLOCK | linux.O_CLOEXEC, 0) catch |err| switch (err) { diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig index 3ef3b63086..ff10c08865 100644 --- a/lib/std/os/windows.zig +++ b/lib/std/os/windows.zig @@ -59,7 +59,7 @@ pub fn CreateFile( hTemplateFile: ?HANDLE, ) CreateFileError!HANDLE { const file_path_w = try sliceToPrefixedFileW(file_path); - return CreateFileW(&file_path_w, desired_access, share_mode, lpSecurityAttributes, creation_disposition, flags_and_attrs, hTemplateFile); + return CreateFileW(file_path_w.span().ptr, desired_access, share_mode, lpSecurityAttributes, creation_disposition, flags_and_attrs, hTemplateFile); } pub fn CreateFileW( @@ -116,10 +116,10 @@ pub const OpenFileOptions = struct { /// TODO when share_access_nonblocking is false, this implementation uses /// untinterruptible sleep() to block. This is not the final iteration of the API. pub fn OpenFile(sub_path_w: []const u16, options: OpenFileOptions) OpenError!HANDLE { - if (mem.eql(u16, sub_path_w, ".")) { + if (mem.eql(u16, sub_path_w, &[_]u16{'.'})) { return error.IsDir; } - if (mem.eql(u16, sub_path_w, "..")) { + if (mem.eql(u16, sub_path_w, &[_]u16{ '.', '.' })) { return error.IsDir; } @@ -199,7 +199,7 @@ pub fn CreatePipe(rd: *HANDLE, wr: *HANDLE, sattr: *const SECURITY_ATTRIBUTES) C pub fn CreateEventEx(attributes: ?*SECURITY_ATTRIBUTES, name: []const u8, flags: DWORD, desired_access: DWORD) !HANDLE { const nameW = try sliceToPrefixedFileW(name); - return CreateEventExW(attributes, &nameW, flags, desired_access); + return CreateEventExW(attributes, nameW.span().ptr, flags, desired_access); } pub fn CreateEventExW(attributes: ?*SECURITY_ATTRIBUTES, nameW: [*:0]const u16, flags: DWORD, desired_access: DWORD) !HANDLE { @@ -332,42 +332,6 @@ pub fn WaitForMultipleObjectsEx(handles: []const HANDLE, waitAll: bool, millisec } } -pub const FindFirstFileError = error{ - FileNotFound, - InvalidUtf8, - BadPathName, - NameTooLong, - Unexpected, -}; - -pub fn FindFirstFile(dir_path: []const u8, find_file_data: *WIN32_FIND_DATAW) FindFirstFileError!HANDLE { - const dir_path_w = try sliceToPrefixedSuffixedFileW(dir_path, [_]u16{ '\\', '*' }); - const handle = kernel32.FindFirstFileW(&dir_path_w, find_file_data); - - if (handle == INVALID_HANDLE_VALUE) { - switch (kernel32.GetLastError()) { - .FILE_NOT_FOUND => return error.FileNotFound, - .PATH_NOT_FOUND => return error.FileNotFound, - else => |err| return unexpectedError(err), - } - } - - return handle; -} - -pub const FindNextFileError = error{Unexpected}; - -/// Returns `true` if there was another file, `false` otherwise. -pub fn FindNextFile(handle: HANDLE, find_file_data: *WIN32_FIND_DATAW) FindNextFileError!bool { - if (kernel32.FindNextFileW(handle, find_file_data) == 0) { - switch (kernel32.GetLastError()) { - .NO_MORE_FILES => return false, - else => |err| return unexpectedError(err), - } - } - return true; -} - pub const CreateIoCompletionPortError = error{Unexpected}; pub fn CreateIoCompletionPort( @@ -644,7 +608,7 @@ pub fn CreateSymbolicLink( ) CreateSymbolicLinkError!void { const sym_link_path_w = try sliceToPrefixedFileW(sym_link_path); const target_path_w = try sliceToPrefixedFileW(target_path); - return CreateSymbolicLinkW(&sym_link_path_w, &target_path_w, flags); + return CreateSymbolicLinkW(sym_link_path_w.span().ptr, target_path_w.span().ptr, flags); } pub fn CreateSymbolicLinkW( @@ -669,7 +633,7 @@ pub const DeleteFileError = error{ pub fn DeleteFile(filename: []const u8) DeleteFileError!void { const filename_w = try sliceToPrefixedFileW(filename); - return DeleteFileW(&filename_w); + return DeleteFileW(filename_w.span().ptr); } pub fn DeleteFileW(filename: [*:0]const u16) DeleteFileError!void { @@ -691,7 +655,7 @@ pub const MoveFileError = error{Unexpected}; pub fn MoveFileEx(old_path: []const u8, new_path: []const u8, flags: DWORD) MoveFileError!void { const old_path_w = try sliceToPrefixedFileW(old_path); const new_path_w = try sliceToPrefixedFileW(new_path); - return MoveFileExW(&old_path_w, &new_path_w, flags); + return MoveFileExW(old_path_w.span().ptr, new_path_w.span().ptr, flags); } pub fn MoveFileExW(old_path: [*:0]const u16, new_path: [*:0]const u16, flags: DWORD) MoveFileError!void { @@ -716,7 +680,7 @@ pub const CreateDirectoryError = error{ /// Returns an open directory handle which the caller is responsible for closing with `CloseHandle`. pub fn CreateDirectory(dir: ?HANDLE, pathname: []const u8, sa: ?*SECURITY_ATTRIBUTES) CreateDirectoryError!HANDLE { const pathname_w = try sliceToPrefixedFileW(pathname); - return CreateDirectoryW(dir, &pathname_w, sa); + return CreateDirectoryW(dir, pathname_w.span().ptr, sa); } /// Same as `CreateDirectory` except takes a WTF-16 encoded path. @@ -784,7 +748,7 @@ pub const RemoveDirectoryError = error{ pub fn RemoveDirectory(dir_path: []const u8) RemoveDirectoryError!void { const dir_path_w = try sliceToPrefixedFileW(dir_path); - return RemoveDirectoryW(&dir_path_w); + return RemoveDirectoryW(dir_path_w.span().ptr); } pub fn RemoveDirectoryW(dir_path_w: [*:0]const u16) RemoveDirectoryError!void { @@ -913,7 +877,7 @@ pub const GetFileAttributesError = error{ pub fn GetFileAttributes(filename: []const u8) GetFileAttributesError!DWORD { const filename_w = try sliceToPrefixedFileW(filename); - return GetFileAttributesW(&filename_w); + return GetFileAttributesW(filename_w.span().ptr); } pub fn GetFileAttributesW(lpFileName: [*:0]const u16) GetFileAttributesError!DWORD { @@ -1253,34 +1217,22 @@ pub fn nanoSecondsToFileTime(ns: i64) FILETIME { }; } -pub fn cStrToPrefixedFileW(s: [*:0]const u8) ![PATH_MAX_WIDE:0]u16 { +pub const PathSpace = struct { + data: [PATH_MAX_WIDE:0]u16, + len: usize, + + pub fn span(self: PathSpace) [:0]const u16 { + return self.data[0..self.len :0]; + } +}; + +pub fn cStrToPrefixedFileW(s: [*:0]const u8) !PathSpace { return sliceToPrefixedFileW(mem.spanZ(s)); } -pub fn sliceToPrefixedFileW(s: []const u8) ![PATH_MAX_WIDE:0]u16 { - return sliceToPrefixedSuffixedFileW(s, &[_]u16{}); -} - -/// Assumes an absolute path. -pub fn wToPrefixedFileW(s: []const u16) ![PATH_MAX_WIDE:0]u16 { +pub fn sliceToPrefixedFileW(s: []const u8) !PathSpace { // TODO https://github.com/ziglang/zig/issues/2765 - var result: [PATH_MAX_WIDE:0]u16 = undefined; - - const start_index = if (mem.startsWith(u16, s, &[_]u16{ '\\', '?' })) 0 else blk: { - const prefix = [_]u16{ '\\', '?', '?', '\\' }; - mem.copy(u16, result[0..], &prefix); - break :blk prefix.len; - }; - const end_index = start_index + s.len; - if (end_index + 1 > result.len) return error.NameTooLong; - mem.copy(u16, result[start_index..], s); - result[end_index] = 0; - return result; -} - -pub fn sliceToPrefixedSuffixedFileW(s: []const u8, comptime suffix: []const u16) ![PATH_MAX_WIDE + suffix.len:0]u16 { - // TODO https://github.com/ziglang/zig/issues/2765 - var result: [PATH_MAX_WIDE + suffix.len:0]u16 = undefined; + var path_space: PathSpace = undefined; for (s) |byte| { switch (byte) { '*', '?', '"', '<', '>', '|' => return error.BadPathName, @@ -1289,25 +1241,46 @@ pub fn sliceToPrefixedSuffixedFileW(s: []const u8, comptime suffix: []const u16) } const start_index = if (mem.startsWith(u8, s, "\\?") or !std.fs.path.isAbsolute(s)) 0 else blk: { const prefix = [_]u16{ '\\', '?', '?', '\\' }; - mem.copy(u16, result[0..], &prefix); + mem.copy(u16, path_space.data[0..], &prefix); break :blk prefix.len; }; - const end_index = start_index + try std.unicode.utf8ToUtf16Le(result[start_index..], s); - if (end_index + suffix.len > result.len) return error.NameTooLong; + path_space.len = start_index + try std.unicode.utf8ToUtf16Le(path_space.data[start_index..], s); + if (path_space.len > path_space.data.len) return error.NameTooLong; // > File I/O functions in the Windows API convert "/" to "\" as part of // > converting the name to an NT-style name, except when using the "\\?\" // > prefix as detailed in the following sections. // from https://docs.microsoft.com/en-us/windows/desktop/FileIO/naming-a-file#maximum-path-length-limitation // Because we want the larger maximum path length for absolute paths, we // convert forward slashes to backward slashes here. - for (result[0..end_index]) |*elem| { + for (path_space.data[0..path_space.len]) |*elem| { if (elem.* == '/') { elem.* = '\\'; } } - mem.copy(u16, result[end_index..], suffix); - result[end_index + suffix.len] = 0; - return result; + path_space.data[path_space.len] = 0; + return path_space; +} + +/// Assumes an absolute path. +pub fn wToPrefixedFileW(s: []const u16) !PathSpace { + // TODO https://github.com/ziglang/zig/issues/2765 + var path_space: PathSpace = undefined; + + const start_index = if (mem.startsWith(u16, s, &[_]u16{ '\\', '?' })) 0 else blk: { + const prefix = [_]u16{ '\\', '?', '?', '\\' }; + mem.copy(u16, path_space.data[0..], &prefix); + break :blk prefix.len; + }; + path_space.len = start_index + s.len; + if (path_space.len > path_space.data.len) return error.NameTooLong; + mem.copy(u16, path_space.data[start_index..], s); + for (path_space.data[0..path_space.len]) |*elem| { + if (elem.* == '/') { + elem.* = '\\'; + } + } + path_space.data[path_space.len] = 0; + return path_space; } inline fn MAKELANGID(p: c_ushort, s: c_ushort) LANGID { From 92f3e9c92acd6cba98a8a8f46341d138f89aae53 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 2 May 2020 01:54:36 -0400 Subject: [PATCH 5/8] remove last use of share_with_child_process --- src-self-hosted/test.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src-self-hosted/test.zig b/src-self-hosted/test.zig index 5868f9383d..8186f1f4d8 100644 --- a/src-self-hosted/test.zig +++ b/src-self-hosted/test.zig @@ -96,7 +96,7 @@ pub const TestContext = struct { case: ZIRCompareOutputCase, target: std.Target, ) !void { - var tmp = std.testing.tmpDir(.{ .share_with_child_process = true }); + var tmp = std.testing.tmpDir(.{}); defer tmp.cleanup(); var prg_node = root_node.start(case.name, 4); From 6546c74825ec36cf6f065f1adb366073c00433ca Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 2 May 2020 03:38:05 -0400 Subject: [PATCH 6/8] child process: no need to remove O_CLOEXEC before execve --- lib/std/child_process.zig | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/lib/std/child_process.zig b/lib/std/child_process.zig index 6cf10664c6..3e56618857 100644 --- a/lib/std/child_process.zig +++ b/lib/std/child_process.zig @@ -421,16 +421,6 @@ pub const ChildProcess = struct { } if (self.cwd_dir) |cwd| { - // Remove the O_CLOEXEC flag. This is the only safe time to do it, between fork() and execve(). - var flags = os.fcntl(cwd.fd, os.F_GETFD, 0) catch |err| switch (err) { - error.Locked => unreachable, - else => |e| forkChildErrReport(err_pipe[1], e), - }; - flags &= ~@as(u32, os.O_CLOEXEC); - _ = os.fcntl(cwd.fd, os.F_SETFD, flags) catch |err| switch (err) { - error.Locked => unreachable, - else => |e| forkChildErrReport(err_pipe[1], e), - }; os.fchdir(cwd.fd) catch |err| forkChildErrReport(err_pipe[1], err); } else if (self.cwd) |cwd| { os.chdir(cwd) catch |err| forkChildErrReport(err_pipe[1], err); From 9dac8a5be9f5e439ea857b22867aaf2dc5b70c62 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 2 May 2020 04:31:26 -0400 Subject: [PATCH 7/8] update windows impl of child process to new File API --- lib/std/child_process.zig | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/lib/std/child_process.zig b/lib/std/child_process.zig index 3e56618857..57c1dfb945 100644 --- a/lib/std/child_process.zig +++ b/lib/std/child_process.zig @@ -675,26 +675,17 @@ pub const ChildProcess = struct { }; if (g_hChildStd_IN_Wr) |h| { - self.stdin = File{ - .handle = h, - .io_mode = io.mode, - }; + self.stdin = File{ .handle = h }; } else { self.stdin = null; } if (g_hChildStd_OUT_Rd) |h| { - self.stdout = File{ - .handle = h, - .io_mode = io.mode, - }; + self.stdout = File{ .handle = h }; } else { self.stdout = null; } if (g_hChildStd_ERR_Rd) |h| { - self.stderr = File{ - .handle = h, - .io_mode = io.mode, - }; + self.stderr = File{ .handle = h }; } else { self.stderr = null; } From 5656f5090d8646e076db50da03cfc6ae686eb76b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 2 May 2020 14:08:59 -0400 Subject: [PATCH 8/8] fs.File: improve handling async I/O on Windows Before it was possible for .intended_io_mode = .blocking, .capable_io_mode = .evented, and then the implementation would put a request on the fs thread, which is the wrong behavior. Now it always calls the appropriate WriteFile/ReadFile function, passing the intended io mode directly as a parameter. This makes the behavior tests pass on Windows with --test-evented-io. --- lib/std/fs/file.zig | 49 ++++++++++++++++++++++++++++++++++++--------- lib/std/os.zig | 4 ++-- 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index 08ae1bffca..b7c575a04a 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -8,6 +8,7 @@ const assert = std.debug.assert; const windows = os.windows; const Os = builtin.Os; const maxInt = std.math.maxInt; +const is_windows = std.Target.current.os.tag == .windows; pub const File = struct { /// The OS-specific file descriptor or file handle. @@ -119,7 +120,9 @@ pub const File = struct { /// Upon success, the stream is in an uninitialized state. To continue using it, /// you must use the open() function. pub fn close(self: File) void { - if (self.capable_io_mode != self.intended_io_mode) { + if (is_windows) { + windows.CloseHandle(self.handle); + } else if (self.capable_io_mode != self.intended_io_mode) { std.event.Loop.instance.?.close(self.handle); } else { os.close(self.handle); @@ -302,7 +305,9 @@ pub const File = struct { pub const PReadError = os.PReadError; pub fn read(self: File, buffer: []u8) ReadError!usize { - if (self.capable_io_mode != self.intended_io_mode) { + if (is_windows) { + return windows.ReadFile(self.handle, buffer, null, self.intended_io_mode); + } else if (self.capable_io_mode != self.intended_io_mode) { return std.event.Loop.instance.?.read(self.handle, buffer); } else { return os.read(self.handle, buffer); @@ -322,7 +327,9 @@ pub const File = struct { } pub fn pread(self: File, buffer: []u8, offset: u64) PReadError!usize { - if (self.capable_io_mode != self.intended_io_mode) { + if (is_windows) { + return windows.ReadFile(self.handle, buffer, offset, self.intended_io_mode); + } else if (self.capable_io_mode != self.intended_io_mode) { return std.event.Loop.instance.?.pread(self.handle, buffer, offset); } else { return os.pread(self.handle, buffer, offset); @@ -342,7 +349,12 @@ pub const File = struct { } pub fn readv(self: File, iovecs: []const os.iovec) ReadError!usize { - if (self.capable_io_mode != self.intended_io_mode) { + if (is_windows) { + // TODO improve this to use ReadFileScatter + if (iovecs.len == 0) return @as(usize, 0); + const first = iovecs[0]; + return windows.ReadFile(self.handle, first.iov_base[0..first.iov_len], null, self.intended_io_mode); + } else if (self.capable_io_mode != self.intended_io_mode) { return std.event.Loop.instance.?.readv(self.handle, iovecs); } else { return os.readv(self.handle, iovecs); @@ -376,7 +388,12 @@ pub const File = struct { } pub fn preadv(self: File, iovecs: []const os.iovec, offset: u64) PReadError!usize { - if (self.capable_io_mode != self.intended_io_mode) { + if (is_windows) { + // TODO improve this to use ReadFileScatter + if (iovecs.len == 0) return @as(usize, 0); + const first = iovecs[0]; + return windows.ReadFile(self.handle, first.iov_base[0..first.iov_len], offset, self.intended_io_mode); + } else if (self.capable_io_mode != self.intended_io_mode) { return std.event.Loop.instance.?.preadv(self.handle, iovecs, offset); } else { return os.preadv(self.handle, iovecs, offset); @@ -413,7 +430,9 @@ pub const File = struct { pub const PWriteError = os.PWriteError; pub fn write(self: File, bytes: []const u8) WriteError!usize { - if (self.capable_io_mode != self.intended_io_mode) { + if (is_windows) { + return windows.WriteFile(self.handle, bytes, null, self.intended_io_mode); + } else if (self.capable_io_mode != self.intended_io_mode) { return std.event.Loop.instance.?.write(self.handle, bytes); } else { return os.write(self.handle, bytes); @@ -428,7 +447,9 @@ pub const File = struct { } pub fn pwrite(self: File, bytes: []const u8, offset: u64) PWriteError!usize { - if (self.capable_io_mode != self.intended_io_mode) { + if (is_windows) { + return windows.WriteFile(self.handle, bytes, offset, self.intended_io_mode); + } else if (self.capable_io_mode != self.intended_io_mode) { return std.event.Loop.instance.?.pwrite(self.handle, bytes, offset); } else { return os.pwrite(self.handle, bytes, offset); @@ -443,7 +464,12 @@ pub const File = struct { } pub fn writev(self: File, iovecs: []const os.iovec_const) WriteError!usize { - if (self.capable_io_mode != self.intended_io_mode) { + if (is_windows) { + // TODO improve this to use WriteFileScatter + if (iovecs.len == 0) return @as(usize, 0); + const first = iovecs[0]; + return windows.WriteFile(self.handle, first.iov_base[0..first.iov_len], null, self.intended_io_mode); + } else if (self.capable_io_mode != self.intended_io_mode) { return std.event.Loop.instance.?.writev(self.handle, iovecs); } else { return os.writev(self.handle, iovecs); @@ -469,7 +495,12 @@ pub const File = struct { } pub fn pwritev(self: File, iovecs: []os.iovec_const, offset: usize) PWriteError!usize { - if (self.capable_io_mode != self.intended_io_mode) { + if (is_windows) { + // TODO improve this to use WriteFileScatter + if (iovecs.len == 0) return @as(usize, 0); + const first = iovecs[0]; + return windows.WriteFile(self.handle, first.iov_base[0..first.iov_len], offset, self.intended_io_mode); + } else if (self.capable_io_mode != self.intended_io_mode) { return std.event.Loop.instance.?.pwritev(self.handle, iovecs, offset); } else { return os.pwritev(self.handle, iovecs, offset); diff --git a/lib/std/os.zig b/lib/std/os.zig index a1b0dc1991..d605c0eed3 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -365,7 +365,7 @@ pub fn read(fd: fd_t, buf: []u8) ReadError!usize { /// On these systems, the read races with concurrent writes to the same file descriptor. pub fn readv(fd: fd_t, iov: []const iovec) ReadError!usize { if (std.Target.current.os.tag == .windows) { - // TODO does Windows have a way to read an io vector? + // TODO improve this to use ReadFileScatter if (iov.len == 0) return @as(usize, 0); const first = iov[0]; return read(fd, first.iov_base[0..first.iov_len]); @@ -651,7 +651,7 @@ pub fn write(fd: fd_t, bytes: []const u8) WriteError!usize { /// If `iov.len` is larger than will fit in a `u31`, a partial write will occur. pub fn writev(fd: fd_t, iov: []const iovec_const) WriteError!usize { if (std.Target.current.os.tag == .windows) { - // TODO does Windows have a way to write an io vector? + // TODO improve this to use WriteFileScatter if (iov.len == 0) return @as(usize, 0); const first = iov[0]; return write(fd, first.iov_base[0..first.iov_len]);