diff --git a/lib/compiler/build_runner.zig b/lib/compiler/build_runner.zig index 523ef98824..b6d771302c 100644 --- a/lib/compiler/build_runner.zig +++ b/lib/compiler/build_runner.zig @@ -1,5 +1,8 @@ -const std = @import("std"); +const runner = @This(); const builtin = @import("builtin"); + +const std = @import("std"); +const Io = std.Io; const assert = std.debug.assert; const fmt = std.fmt; const mem = std.mem; @@ -11,7 +14,6 @@ const WebServer = std.Build.WebServer; const Allocator = std.mem.Allocator; const fatal = std.process.fatal; const Writer = std.Io.Writer; -const runner = @This(); const tty = std.Io.tty; pub const root = @import("@build"); @@ -75,6 +77,7 @@ pub fn main() !void { .io = io, .arena = arena, .cache = .{ + .io = io, .gpa = arena, .manifest_dir = try local_cache_directory.handle.makeOpenPath("h", .{}), }, @@ -84,7 +87,7 @@ pub fn main() !void { .zig_lib_directory = zig_lib_directory, .host = .{ .query = .{}, - .result = try std.zig.system.resolveTargetQuery(.{}), + .result = try std.zig.system.resolveTargetQuery(io, .{}), }, .time_report = false, }; @@ -121,7 +124,7 @@ pub fn main() !void { var watch = false; var fuzz: ?std.Build.Fuzz.Mode = null; var debounce_interval_ms: u16 = 50; - var webui_listen: ?std.net.Address = null; + var webui_listen: ?Io.net.IpAddress = null; if (try std.zig.EnvVar.ZIG_BUILD_ERROR_STYLE.get(arena)) |str| { if (std.meta.stringToEnum(ErrorStyle, str)) |style| { @@ -288,11 +291,11 @@ pub fn main() !void { }); }; } else if (mem.eql(u8, arg, "--webui")) { - webui_listen = std.net.Address.parseIp("::1", 0) catch unreachable; + if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) }; } else if (mem.startsWith(u8, arg, "--webui=")) { const addr_str = arg["--webui=".len..]; if (std.mem.eql(u8, addr_str, "-")) fatal("web interface cannot listen on stdio", .{}); - webui_listen = std.net.Address.parseIpAndPort(addr_str) catch |err| { + webui_listen = Io.net.IpAddress.parseLiteral(addr_str) catch |err| { fatal("invalid web UI address '{s}': {s}", .{ addr_str, @errorName(err) }); }; } else if (mem.eql(u8, arg, "--debug-log")) { @@ -334,14 +337,10 @@ pub fn main() !void { watch = true; } else if (mem.eql(u8, arg, "--time-report")) { graph.time_report = true; - if (webui_listen == null) { - webui_listen = std.net.Address.parseIp("::1", 0) catch unreachable; - } + if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) }; } else if (mem.eql(u8, arg, "--fuzz")) { fuzz = .{ .forever = undefined }; - if (webui_listen == null) { - webui_listen = std.net.Address.parseIp("::1", 0) catch unreachable; - } + if (webui_listen == null) webui_listen = .{ .ip6 = .loopback(0) }; } else if (mem.startsWith(u8, arg, "--fuzz=")) { const value = arg["--fuzz=".len..]; if (value.len == 0) fatal("missing argument to --fuzz", .{}); @@ -550,13 +549,15 @@ pub fn main() !void { var w: Watch = w: { if (!watch) break :w undefined; - if (!Watch.have_impl) fatal("--watch not yet implemented for {s}", .{@tagName(builtin.os.tag)}); + if (!Watch.have_impl) fatal("--watch not yet implemented for {t}", .{builtin.os.tag}); break :w try .init(); }; try run.thread_pool.init(thread_pool_options); defer run.thread_pool.deinit(); + const now = Io.Timestamp.now(io, .awake) catch |err| fatal("failed to collect timestamp: {t}", .{err}); + run.web_server = if (webui_listen) |listen_address| ws: { if (builtin.single_threaded) unreachable; // `fatal` above break :ws .init(.{ @@ -568,11 +569,12 @@ pub fn main() !void { .root_prog_node = main_progress_node, .watch = watch, .listen_address = listen_address, + .base_timestamp = now, }); } else null; if (run.web_server) |*ws| { - ws.start() catch |err| fatal("failed to start web server: {s}", .{@errorName(err)}); + ws.start() catch |err| fatal("failed to start web server: {t}", .{err}); } rebuild: while (true) : (if (run.error_style.clearOnUpdate()) { @@ -755,6 +757,7 @@ fn runStepNames( fuzz: ?std.Build.Fuzz.Mode, ) !void { const gpa = run.gpa; + const io = b.graph.io; const step_stack = &run.step_stack; const thread_pool = &run.thread_pool; @@ -858,6 +861,7 @@ fn runStepNames( assert(mode == .limit); var f = std.Build.Fuzz.init( gpa, + io, thread_pool, step_stack.keys(), parent_prog_node, diff --git a/lib/compiler/test_runner.zig b/lib/compiler/test_runner.zig index 40f52fbd39..0d6f451947 100644 --- a/lib/compiler/test_runner.zig +++ b/lib/compiler/test_runner.zig @@ -2,6 +2,7 @@ const builtin = @import("builtin"); const std = @import("std"); +const Io = std.Io; const fatal = std.process.fatal; const testing = std.testing; const assert = std.debug.assert; @@ -16,6 +17,7 @@ var fba: std.heap.FixedBufferAllocator = .init(&fba_buffer); var fba_buffer: [8192]u8 = undefined; var stdin_buffer: [4096]u8 = undefined; var stdout_buffer: [4096]u8 = undefined; +var runner_threaded_io: Io.Threaded = .init_single_threaded; /// Keep in sync with logic in `std.Build.addRunArtifact` which decides whether /// the test runner will communicate with the build runner via `std.zig.Server`. @@ -63,8 +65,6 @@ pub fn main() void { fuzz_abi.fuzzer_init(.fromSlice(cache_dir)); } - fba.reset(); - if (listen) { return mainServer() catch @panic("internal test runner failure"); } else { @@ -74,7 +74,7 @@ pub fn main() void { fn mainServer() !void { @disableInstrumentation(); - var stdin_reader = std.fs.File.stdin().readerStreaming(&stdin_buffer); + var stdin_reader = std.fs.File.stdin().readerStreaming(runner_threaded_io.io(), &stdin_buffer); var stdout_writer = std.fs.File.stdout().writerStreaming(&stdout_buffer); var server = try std.zig.Server.init(.{ .in = &stdin_reader.interface, @@ -131,7 +131,7 @@ fn mainServer() !void { .run_test => { testing.allocator_instance = .{}; - testing.io_instance = .init(fba.allocator()); + testing.io_instance = .init(testing.allocator); log_err_count = 0; const index = try server.receiveBody_u32(); const test_fn = builtin.test_functions[index]; @@ -154,7 +154,6 @@ fn mainServer() !void { }, }; testing.io_instance.deinit(); - fba.reset(); const leak_count = testing.allocator_instance.detectLeaks(); testing.allocator_instance.deinitWithoutLeakChecks(); try server.serveTestResults(.{ @@ -234,10 +233,10 @@ fn mainTerminal() void { var leaks: usize = 0; for (test_fn_list, 0..) |test_fn, i| { testing.allocator_instance = .{}; - testing.io_instance = .init(fba.allocator()); + testing.io_instance = .init(testing.allocator); defer { - if (testing.allocator_instance.deinit() == .leak) leaks += 1; testing.io_instance.deinit(); + if (testing.allocator_instance.deinit() == .leak) leaks += 1; } testing.log_level = .warn; @@ -324,7 +323,7 @@ pub fn mainSimple() anyerror!void { .stage2_aarch64, .stage2_riscv64 => true, else => false, }; - // is the backend capable of calling `std.Io.Writer.print`? + // is the backend capable of calling `Io.Writer.print`? const enable_print = switch (builtin.zig_backend) { .stage2_aarch64, .stage2_riscv64 => true, else => false, diff --git a/lib/std/Build.zig b/lib/std/Build.zig index d3df0c0d39..e9d2e81fba 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -1837,6 +1837,8 @@ pub fn runAllowFail( if (!process.can_spawn) return error.ExecNotSupported; + const io = b.graph.io; + const max_output_size = 400 * 1024; var child = std.process.Child.init(argv, b.allocator); child.stdin_behavior = .Ignore; @@ -1847,7 +1849,7 @@ pub fn runAllowFail( try Step.handleVerbose2(b, null, child.env_map, argv); try child.spawn(); - var stdout_reader = child.stdout.?.readerStreaming(&.{}); + var stdout_reader = child.stdout.?.readerStreaming(io, &.{}); const stdout = stdout_reader.interface.allocRemaining(b.allocator, .limited(max_output_size)) catch { return error.ReadFailure; }; diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index fe9714296d..8202c4dd15 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -3,8 +3,10 @@ //! not to withstand attacks using specially-crafted input. const Cache = @This(); -const std = @import("std"); const builtin = @import("builtin"); + +const std = @import("std"); +const Io = std.Io; const crypto = std.crypto; const fs = std.fs; const assert = std.debug.assert; @@ -15,6 +17,7 @@ const Allocator = std.mem.Allocator; const log = std.log.scoped(.cache); gpa: Allocator, +io: Io, manifest_dir: fs.Dir, hash: HashHelper = .{}, /// This value is accessed from multiple threads, protected by mutex. @@ -661,9 +664,10 @@ pub const Manifest = struct { }, } { const gpa = self.cache.gpa; + const io = self.cache.io; const input_file_count = self.files.entries.len; var tiny_buffer: [1]u8 = undefined; // allows allocRemaining to detect limit exceeded - var manifest_reader = self.manifest_file.?.reader(&tiny_buffer); // Reads positionally from zero. + var manifest_reader = self.manifest_file.?.reader(io, &tiny_buffer); // Reads positionally from zero. const limit: std.Io.Limit = .limited(manifest_file_size_max); const file_contents = manifest_reader.interface.allocRemaining(gpa, limit) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, @@ -1337,7 +1341,8 @@ test "cache file and then recall it" { var digest2: HexDigest = undefined; { - var cache = Cache{ + var cache: Cache = .{ + .io = io, .gpa = testing.allocator, .manifest_dir = try tmp.dir.makeOpenPath(temp_manifest_dir, .{}), }; @@ -1402,7 +1407,8 @@ test "check that changing a file makes cache fail" { var digest2: HexDigest = undefined; { - var cache = Cache{ + var cache: Cache = .{ + .io = io, .gpa = testing.allocator, .manifest_dir = try tmp.dir.makeOpenPath(temp_manifest_dir, .{}), }; @@ -1451,6 +1457,8 @@ test "check that changing a file makes cache fail" { } test "no file inputs" { + const io = testing.io; + var tmp = testing.tmpDir(.{}); defer tmp.cleanup(); @@ -1459,7 +1467,8 @@ test "no file inputs" { var digest1: HexDigest = undefined; var digest2: HexDigest = undefined; - var cache = Cache{ + var cache: Cache = .{ + .io = io, .gpa = testing.allocator, .manifest_dir = try tmp.dir.makeOpenPath(temp_manifest_dir, .{}), }; @@ -1517,7 +1526,8 @@ test "Manifest with files added after initial hash work" { var digest3: HexDigest = undefined; { - var cache = Cache{ + var cache: Cache = .{ + .io = io, .gpa = testing.allocator, .manifest_dir = try tmp.dir.makeOpenPath(temp_manifest_dir, .{}), }; diff --git a/lib/std/Build/Fuzz.zig b/lib/std/Build/Fuzz.zig index 8281fc4726..d342628871 100644 --- a/lib/std/Build/Fuzz.zig +++ b/lib/std/Build/Fuzz.zig @@ -1,4 +1,5 @@ const std = @import("../std.zig"); +const Io = std.Io; const Build = std.Build; const Cache = Build.Cache; const Step = std.Build.Step; @@ -14,6 +15,7 @@ const Fuzz = @This(); const build_runner = @import("root"); gpa: Allocator, +io: Io, mode: Mode, /// Allocated into `gpa`. @@ -75,6 +77,7 @@ const CoverageMap = struct { pub fn init( gpa: Allocator, + io: Io, thread_pool: *std.Thread.Pool, all_steps: []const *Build.Step, root_prog_node: std.Progress.Node, @@ -111,6 +114,7 @@ pub fn init( return .{ .gpa = gpa, + .io = io, .mode = mode, .run_steps = run_steps, .wait_group = .{}, @@ -484,6 +488,7 @@ fn addEntryPoint(fuzz: *Fuzz, coverage_id: u64, addr: u64) error{ AlreadyReporte pub fn waitAndPrintReport(fuzz: *Fuzz) void { assert(fuzz.mode == .limit); + const io = fuzz.io; fuzz.wait_group.wait(); fuzz.wait_group.reset(); @@ -506,7 +511,7 @@ pub fn waitAndPrintReport(fuzz: *Fuzz) void { const fuzz_abi = std.Build.abi.fuzz; var rbuf: [0x1000]u8 = undefined; - var r = coverage_file.reader(&rbuf); + var r = coverage_file.reader(io, &rbuf); var header: fuzz_abi.SeenPcsHeader = undefined; r.interface.readSliceAll(std.mem.asBytes(&header)) catch |err| { diff --git a/lib/std/Build/Step.zig b/lib/std/Build/Step.zig index 72eb66e530..aa922ff37b 100644 --- a/lib/std/Build/Step.zig +++ b/lib/std/Build/Step.zig @@ -1,9 +1,11 @@ const Step = @This(); +const builtin = @import("builtin"); + const std = @import("../std.zig"); +const Io = std.Io; const Build = std.Build; const Allocator = std.mem.Allocator; const assert = std.debug.assert; -const builtin = @import("builtin"); const Cache = Build.Cache; const Path = Cache.Path; const ArrayList = std.ArrayList; @@ -327,7 +329,7 @@ pub fn cast(step: *Step, comptime T: type) ?*T { } /// For debugging purposes, prints identifying information about this Step. -pub fn dump(step: *Step, w: *std.Io.Writer, tty_config: std.Io.tty.Config) void { +pub fn dump(step: *Step, w: *Io.Writer, tty_config: Io.tty.Config) void { if (step.debug_stack_trace.instruction_addresses.len > 0) { w.print("name: '{s}'. creation stack trace:\n", .{step.name}) catch {}; std.debug.writeStackTrace(&step.debug_stack_trace, w, tty_config) catch {}; @@ -382,7 +384,7 @@ pub fn addError(step: *Step, comptime fmt: []const u8, args: anytype) error{OutO pub const ZigProcess = struct { child: std.process.Child, - poller: std.Io.Poller(StreamEnum), + poller: Io.Poller(StreamEnum), progress_ipc_fd: if (std.Progress.have_ipc) ?std.posix.fd_t else void, pub const StreamEnum = enum { stdout, stderr }; @@ -458,7 +460,7 @@ pub fn evalZigProcess( const zp = try gpa.create(ZigProcess); zp.* = .{ .child = child, - .poller = std.Io.poll(gpa, ZigProcess.StreamEnum, .{ + .poller = Io.poll(gpa, ZigProcess.StreamEnum, .{ .stdout = child.stdout.?, .stderr = child.stderr.?, }), @@ -505,11 +507,12 @@ pub fn evalZigProcess( } /// Wrapper around `std.fs.Dir.updateFile` that handles verbose and error output. -pub fn installFile(s: *Step, src_lazy_path: Build.LazyPath, dest_path: []const u8) !std.fs.Dir.PrevStatus { +pub fn installFile(s: *Step, src_lazy_path: Build.LazyPath, dest_path: []const u8) !Io.Dir.PrevStatus { const b = s.owner; + const io = b.graph.io; const src_path = src_lazy_path.getPath3(b, s); try handleVerbose(b, null, &.{ "install", "-C", b.fmt("{f}", .{src_path}), dest_path }); - return src_path.root_dir.handle.updateFile(src_path.sub_path, std.fs.cwd(), dest_path, .{}) catch |err| { + return Io.Dir.updateFile(src_path.root_dir.handle.adaptToNewApi(), io, src_path.sub_path, .cwd(), dest_path, .{}) catch |err| { return s.fail("unable to update file from '{f}' to '{s}': {s}", .{ src_path, dest_path, @errorName(err), }); @@ -738,7 +741,7 @@ pub fn allocPrintCmd2( argv: []const []const u8, ) Allocator.Error![]u8 { const shell = struct { - fn escape(writer: *std.Io.Writer, string: []const u8, is_argv0: bool) !void { + fn escape(writer: *Io.Writer, string: []const u8, is_argv0: bool) !void { for (string) |c| { if (switch (c) { else => true, @@ -772,7 +775,7 @@ pub fn allocPrintCmd2( } }; - var aw: std.Io.Writer.Allocating = .init(gpa); + var aw: Io.Writer.Allocating = .init(gpa); defer aw.deinit(); const writer = &aw.writer; if (opt_cwd) |cwd| writer.print("cd {s} && ", .{cwd}) catch return error.OutOfMemory; diff --git a/lib/std/Build/Step/Options.zig b/lib/std/Build/Step/Options.zig index d738b8edaa..8590fa9608 100644 --- a/lib/std/Build/Step/Options.zig +++ b/lib/std/Build/Step/Options.zig @@ -538,8 +538,10 @@ test Options { defer arena.deinit(); var graph: std.Build.Graph = .{ + .io = io, .arena = arena.allocator(), .cache = .{ + .io = io, .gpa = arena.allocator(), .manifest_dir = std.fs.cwd(), }, diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index cd29262612..314862e201 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -761,6 +761,7 @@ const IndexedOutput = struct { }; fn make(step: *Step, options: Step.MakeOptions) !void { const b = step.owner; + const io = b.graph.io; const arena = b.allocator; const run: *Run = @fieldParentPtr("step", step); const has_side_effects = run.hasSideEffects(); @@ -834,7 +835,7 @@ fn make(step: *Step, options: Step.MakeOptions) !void { defer file.close(); var buf: [1024]u8 = undefined; - var file_reader = file.reader(&buf); + var file_reader = file.reader(io, &buf); _ = file_reader.interface.streamRemaining(&result.writer) catch |err| switch (err) { error.ReadFailed => return step.fail( "failed to read from '{f}': {t}", @@ -1067,6 +1068,7 @@ pub fn rerunInFuzzMode( ) !void { const step = &run.step; const b = step.owner; + const io = b.graph.io; const arena = b.allocator; var argv_list: std.ArrayList([]const u8) = .empty; for (run.argv.items) |arg| { @@ -1093,7 +1095,7 @@ pub fn rerunInFuzzMode( defer file.close(); var buf: [1024]u8 = undefined; - var file_reader = file.reader(&buf); + var file_reader = file.reader(io, &buf); _ = file_reader.interface.streamRemaining(&result.writer) catch |err| switch (err) { error.ReadFailed => return file_reader.err.?, error.WriteFailed => return error.OutOfMemory, @@ -2090,6 +2092,7 @@ fn sendRunFuzzTestMessage( fn evalGeneric(run: *Run, child: *std.process.Child) !EvalGenericResult { const b = run.step.owner; + const io = b.graph.io; const arena = b.allocator; try child.spawn(); @@ -2113,7 +2116,7 @@ fn evalGeneric(run: *Run, child: *std.process.Child) !EvalGenericResult { defer file.close(); // TODO https://github.com/ziglang/zig/issues/23955 var read_buffer: [1024]u8 = undefined; - var file_reader = file.reader(&read_buffer); + var file_reader = file.reader(io, &read_buffer); var write_buffer: [1024]u8 = undefined; var stdin_writer = child.stdin.?.writer(&write_buffer); _ = stdin_writer.interface.sendFileAll(&file_reader, .unlimited) catch |err| switch (err) { @@ -2159,7 +2162,7 @@ fn evalGeneric(run: *Run, child: *std.process.Child) !EvalGenericResult { stdout_bytes = try poller.toOwnedSlice(.stdout); stderr_bytes = try poller.toOwnedSlice(.stderr); } else { - var stdout_reader = stdout.readerStreaming(&.{}); + var stdout_reader = stdout.readerStreaming(io, &.{}); stdout_bytes = stdout_reader.interface.allocRemaining(arena, run.stdio_limit) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, error.ReadFailed => return stdout_reader.err.?, @@ -2167,7 +2170,7 @@ fn evalGeneric(run: *Run, child: *std.process.Child) !EvalGenericResult { }; } } else if (child.stderr) |stderr| { - var stderr_reader = stderr.readerStreaming(&.{}); + var stderr_reader = stderr.readerStreaming(io, &.{}); stderr_bytes = stderr_reader.interface.allocRemaining(arena, run.stdio_limit) catch |err| switch (err) { error.OutOfMemory => return error.OutOfMemory, error.ReadFailed => return stderr_reader.err.?, diff --git a/lib/std/Build/Step/UpdateSourceFiles.zig b/lib/std/Build/Step/UpdateSourceFiles.zig index 674e2a01c6..6f1559bd68 100644 --- a/lib/std/Build/Step/UpdateSourceFiles.zig +++ b/lib/std/Build/Step/UpdateSourceFiles.zig @@ -3,11 +3,13 @@ //! not be used during the normal build process, but as a utility run by a //! developer with intention to update source files, which will then be //! committed to version control. +const UpdateSourceFiles = @This(); + const std = @import("std"); +const Io = std.Io; const Step = std.Build.Step; const fs = std.fs; const ArrayList = std.ArrayList; -const UpdateSourceFiles = @This(); step: Step, output_source_files: std.ArrayListUnmanaged(OutputSourceFile), @@ -70,22 +72,21 @@ pub fn addBytesToSource(usf: *UpdateSourceFiles, bytes: []const u8, sub_path: [] fn make(step: *Step, options: Step.MakeOptions) !void { _ = options; const b = step.owner; + const io = b.graph.io; const usf: *UpdateSourceFiles = @fieldParentPtr("step", step); var any_miss = false; for (usf.output_source_files.items) |output_source_file| { if (fs.path.dirname(output_source_file.sub_path)) |dirname| { b.build_root.handle.makePath(dirname) catch |err| { - return step.fail("unable to make path '{f}{s}': {s}", .{ - b.build_root, dirname, @errorName(err), - }); + return step.fail("unable to make path '{f}{s}': {t}", .{ b.build_root, dirname, err }); }; } switch (output_source_file.contents) { .bytes => |bytes| { b.build_root.handle.writeFile(.{ .sub_path = output_source_file.sub_path, .data = bytes }) catch |err| { - return step.fail("unable to write file '{f}{s}': {s}", .{ - b.build_root, output_source_file.sub_path, @errorName(err), + return step.fail("unable to write file '{f}{s}': {t}", .{ + b.build_root, output_source_file.sub_path, err, }); }; any_miss = true; @@ -94,15 +95,16 @@ fn make(step: *Step, options: Step.MakeOptions) !void { if (!step.inputs.populated()) try step.addWatchInput(file_source); const source_path = file_source.getPath2(b, step); - const prev_status = fs.Dir.updateFile( - fs.cwd(), + const prev_status = Io.Dir.updateFile( + .cwd(), + io, source_path, - b.build_root.handle, + b.build_root.handle.adaptToNewApi(), output_source_file.sub_path, .{}, ) catch |err| { - return step.fail("unable to update file from '{s}' to '{f}{s}': {s}", .{ - source_path, b.build_root, output_source_file.sub_path, @errorName(err), + return step.fail("unable to update file from '{s}' to '{f}{s}': {t}", .{ + source_path, b.build_root, output_source_file.sub_path, err, }); }; any_miss = any_miss or prev_status == .stale; diff --git a/lib/std/Build/Step/WriteFile.zig b/lib/std/Build/Step/WriteFile.zig index b1cfb3b42a..1a531604e3 100644 --- a/lib/std/Build/Step/WriteFile.zig +++ b/lib/std/Build/Step/WriteFile.zig @@ -2,6 +2,7 @@ //! the local cache which has a set of files that have either been generated //! during the build, or are copied from the source package. const std = @import("std"); +const Io = std.Io; const Step = std.Build.Step; const fs = std.fs; const ArrayList = std.ArrayList; @@ -174,6 +175,7 @@ fn maybeUpdateName(write_file: *WriteFile) void { fn make(step: *Step, options: Step.MakeOptions) !void { _ = options; const b = step.owner; + const io = b.graph.io; const arena = b.allocator; const gpa = arena; const write_file: *WriteFile = @fieldParentPtr("step", step); @@ -264,40 +266,27 @@ fn make(step: *Step, options: Step.MakeOptions) !void { }; defer cache_dir.close(); - const cwd = fs.cwd(); - for (write_file.files.items) |file| { if (fs.path.dirname(file.sub_path)) |dirname| { cache_dir.makePath(dirname) catch |err| { - return step.fail("unable to make path '{f}{s}{c}{s}': {s}", .{ - b.cache_root, cache_path, fs.path.sep, dirname, @errorName(err), + return step.fail("unable to make path '{f}{s}{c}{s}': {t}", .{ + b.cache_root, cache_path, fs.path.sep, dirname, err, }); }; } switch (file.contents) { .bytes => |bytes| { cache_dir.writeFile(.{ .sub_path = file.sub_path, .data = bytes }) catch |err| { - return step.fail("unable to write file '{f}{s}{c}{s}': {s}", .{ - b.cache_root, cache_path, fs.path.sep, file.sub_path, @errorName(err), + return step.fail("unable to write file '{f}{s}{c}{s}': {t}", .{ + b.cache_root, cache_path, fs.path.sep, file.sub_path, err, }); }; }, .copy => |file_source| { const source_path = file_source.getPath2(b, step); - const prev_status = fs.Dir.updateFile( - cwd, - source_path, - cache_dir, - file.sub_path, - .{}, - ) catch |err| { - return step.fail("unable to update file from '{s}' to '{f}{s}{c}{s}': {s}", .{ - source_path, - b.cache_root, - cache_path, - fs.path.sep, - file.sub_path, - @errorName(err), + const prev_status = Io.Dir.updateFile(.cwd(), io, source_path, cache_dir.adaptToNewApi(), file.sub_path, .{}) catch |err| { + return step.fail("unable to update file from '{s}' to '{f}{s}{c}{s}': {t}", .{ + source_path, b.cache_root, cache_path, fs.path.sep, file.sub_path, err, }); }; // At this point we already will mark the step as a cache miss. @@ -331,10 +320,11 @@ fn make(step: *Step, options: Step.MakeOptions) !void { switch (entry.kind) { .directory => try cache_dir.makePath(dest_path), .file => { - const prev_status = fs.Dir.updateFile( - src_entry_path.root_dir.handle, + const prev_status = Io.Dir.updateFile( + src_entry_path.root_dir.handle.adaptToNewApi(), + io, src_entry_path.sub_path, - cache_dir, + cache_dir.adaptToNewApi(), dest_path, .{}, ) catch |err| { diff --git a/lib/std/Build/WebServer.zig b/lib/std/Build/WebServer.zig index 135fbe7f0a..95338c9f08 100644 --- a/lib/std/Build/WebServer.zig +++ b/lib/std/Build/WebServer.zig @@ -3,14 +3,15 @@ thread_pool: *std.Thread.Pool, graph: *const Build.Graph, all_steps: []const *Build.Step, listen_address: net.IpAddress, -ttyconf: std.Io.tty.Config, +ttyconf: Io.tty.Config, root_prog_node: std.Progress.Node, watch: bool, tcp_server: ?net.Server, serve_thread: ?std.Thread, -base_timestamp: i128, +/// Uses `Io.Clock.awake`. +base_timestamp: i96, /// The "step name" data which trails `abi.Hello`, for the steps in `all_steps`. step_names_trailing: []u8, @@ -53,15 +54,17 @@ pub const Options = struct { thread_pool: *std.Thread.Pool, graph: *const std.Build.Graph, all_steps: []const *Build.Step, - ttyconf: std.Io.tty.Config, + ttyconf: Io.tty.Config, root_prog_node: std.Progress.Node, watch: bool, listen_address: net.IpAddress, + base_timestamp: Io.Timestamp, }; pub fn init(opts: Options) WebServer { - // The upcoming `std.Io` interface should allow us to use `Io.async` and `Io.concurrent` + // The upcoming `Io` interface should allow us to use `Io.async` and `Io.concurrent` // instead of threads, so that the web server can function in single-threaded builds. comptime assert(!builtin.single_threaded); + assert(opts.base_timestamp.clock == .awake); const all_steps = opts.all_steps; @@ -106,7 +109,7 @@ pub fn init(opts: Options) WebServer { .tcp_server = null, .serve_thread = null, - .base_timestamp = std.time.nanoTimestamp(), + .base_timestamp = opts.base_timestamp.nanoseconds, .step_names_trailing = step_names_trailing, .step_status_bits = step_status_bits, @@ -147,32 +150,34 @@ pub fn deinit(ws: *WebServer) void { pub fn start(ws: *WebServer) error{AlreadyReported}!void { assert(ws.tcp_server == null); assert(ws.serve_thread == null); + const io = ws.graph.io; - ws.tcp_server = ws.listen_address.listen(.{ .reuse_address = true }) catch |err| { + ws.tcp_server = ws.listen_address.listen(io, .{ .reuse_address = true }) catch |err| { log.err("failed to listen to port {d}: {s}", .{ ws.listen_address.getPort(), @errorName(err) }); return error.AlreadyReported; }; ws.serve_thread = std.Thread.spawn(.{}, serve, .{ws}) catch |err| { log.err("unable to spawn web server thread: {s}", .{@errorName(err)}); - ws.tcp_server.?.deinit(); + ws.tcp_server.?.deinit(io); ws.tcp_server = null; return error.AlreadyReported; }; - log.info("web interface listening at http://{f}/", .{ws.tcp_server.?.listen_address}); + log.info("web interface listening at http://{f}/", .{ws.tcp_server.?.socket.address}); if (ws.listen_address.getPort() == 0) { - log.info("hint: pass '--webui={f}' to use the same port next time", .{ws.tcp_server.?.listen_address}); + log.info("hint: pass '--webui={f}' to use the same port next time", .{ws.tcp_server.?.socket.address}); } } fn serve(ws: *WebServer) void { + const io = ws.graph.io; while (true) { - const connection = ws.tcp_server.?.accept() catch |err| { + var stream = ws.tcp_server.?.accept(io) catch |err| { log.err("failed to accept connection: {s}", .{@errorName(err)}); return; }; - _ = std.Thread.spawn(.{}, accept, .{ ws, connection }) catch |err| { + _ = std.Thread.spawn(.{}, accept, .{ ws, stream }) catch |err| { log.err("unable to spawn connection thread: {s}", .{@errorName(err)}); - connection.stream.close(); + stream.close(io); continue; }; } @@ -227,6 +232,7 @@ pub fn finishBuild(ws: *WebServer, opts: struct { ws.fuzz = Fuzz.init( ws.gpa, + ws.graph.io, ws.thread_pool, ws.all_steps, ws.root_prog_node, @@ -241,17 +247,25 @@ pub fn finishBuild(ws: *WebServer, opts: struct { } pub fn now(s: *const WebServer) i64 { - return @intCast(std.time.nanoTimestamp() - s.base_timestamp); + const io = s.graph.io; + const base: Io.Timestamp = .{ .nanoseconds = s.base_timestamp, .clock = .awake }; + const ts = Io.Timestamp.now(io, base.clock) catch base; + return @intCast(base.durationTo(ts).toNanoseconds()); } -fn accept(ws: *WebServer, connection: net.Server.Connection) void { - defer connection.stream.close(); - +fn accept(ws: *WebServer, stream: net.Stream) void { + const io = ws.graph.io; + defer { + // `net.Stream.close` wants to helpfully overwrite `stream` with + // `undefined`, but it cannot do so since it is an immutable parameter. + var copy = stream; + copy.close(io); + } var send_buffer: [4096]u8 = undefined; var recv_buffer: [4096]u8 = undefined; - var connection_reader = connection.stream.reader(&recv_buffer); - var connection_writer = connection.stream.writer(&send_buffer); - var server: http.Server = .init(connection_reader.interface(), &connection_writer.interface); + var connection_reader = stream.reader(io, &recv_buffer); + var connection_writer = stream.writer(io, &send_buffer); + var server: http.Server = .init(&connection_reader.interface, &connection_writer.interface); while (true) { var request = server.receiveHead() catch |err| switch (err) { @@ -466,12 +480,9 @@ pub fn serveFile( }, }); } -pub fn serveTarFile( - ws: *WebServer, - request: *http.Server.Request, - paths: []const Cache.Path, -) !void { +pub fn serveTarFile(ws: *WebServer, request: *http.Server.Request, paths: []const Cache.Path) !void { const gpa = ws.gpa; + const io = ws.graph.io; var send_buffer: [0x4000]u8 = undefined; var response = try request.respondStreaming(&send_buffer, .{ @@ -496,7 +507,7 @@ pub fn serveTarFile( defer file.close(); const stat = try file.stat(); var read_buffer: [1024]u8 = undefined; - var file_reader: std.fs.File.Reader = .initSize(file, &read_buffer, stat.size); + var file_reader: Io.File.Reader = .initSize(file.adaptToNewApi(), io, &read_buffer, stat.size); // TODO: this logic is completely bogus -- obviously so, because `path.root_dir.path` can // be cwd-relative. This is also related to why linkification doesn't work in the fuzzer UI: @@ -566,7 +577,7 @@ fn buildClientWasm(ws: *WebServer, arena: Allocator, optimize: std.builtin.Optim child.stderr_behavior = .Pipe; try child.spawn(); - var poller = std.Io.poll(gpa, enum { stdout, stderr }, .{ + var poller = Io.poll(gpa, enum { stdout, stderr }, .{ .stdout = child.stdout.?, .stderr = child.stderr.?, }); @@ -842,7 +853,10 @@ const cache_control_header: http.Header = .{ }; const builtin = @import("builtin"); + const std = @import("std"); +const Io = std.Io; +const net = std.Io.net; const assert = std.debug.assert; const mem = std.mem; const log = std.log.scoped(.web_server); @@ -852,6 +866,5 @@ const Cache = Build.Cache; const Fuzz = Build.Fuzz; const abi = Build.abi; const http = std.http; -const net = std.Io.net; const WebServer = @This(); diff --git a/lib/std/Io.zig b/lib/std/Io.zig index 5cdb9b0f01..e569c4ec94 100644 --- a/lib/std/Io.zig +++ b/lib/std/Io.zig @@ -654,6 +654,10 @@ pub const VTable = struct { conditionWait: *const fn (?*anyopaque, cond: *Condition, mutex: *Mutex) Cancelable!void, conditionWake: *const fn (?*anyopaque, cond: *Condition, wake: Condition.Wake) void, + dirMake: *const fn (?*anyopaque, dir: Dir, sub_path: []const u8, mode: Dir.Mode) Dir.MakeError!void, + dirStat: *const fn (?*anyopaque, dir: Dir) Dir.StatError!Dir.Stat, + dirStatPath: *const fn (?*anyopaque, dir: Dir, sub_path: []const u8) Dir.StatError!File.Stat, + fileStat: *const fn (?*anyopaque, file: File) File.StatError!File.Stat, createFile: *const fn (?*anyopaque, dir: Dir, sub_path: []const u8, flags: File.CreateFlags) File.OpenError!File, fileOpen: *const fn (?*anyopaque, dir: Dir, sub_path: []const u8, flags: File.OpenFlags) File.OpenError!File, fileClose: *const fn (?*anyopaque, File) void, @@ -804,6 +808,10 @@ pub const Timestamp = struct { assert(lhs.clock == rhs.clock); return std.math.compare(lhs.nanoseconds, op, rhs.nanoseconds); } + + pub fn toSeconds(t: Timestamp) i64 { + return @intCast(@divTrunc(t.nanoseconds, std.time.ns_per_s)); + } }; pub const Duration = struct { @@ -831,6 +839,10 @@ pub const Duration = struct { return @intCast(@divTrunc(d.nanoseconds, std.time.ns_per_s)); } + pub fn toNanoseconds(d: Duration) i96 { + return d.nanoseconds; + } + pub fn sleep(duration: Duration, io: Io) SleepError!void { return io.vtable.sleep(io.userdata, .{ .duration = .{ .duration = duration, .clock = .awake } }); } diff --git a/lib/std/Io/Dir.zig b/lib/std/Io/Dir.zig index a2ae96210e..3c9772076b 100644 --- a/lib/std/Io/Dir.zig +++ b/lib/std/Io/Dir.zig @@ -6,6 +6,9 @@ const File = Io.File; handle: Handle, +pub const Mode = Io.File.Mode; +pub const default_mode: Mode = 0o755; + pub fn cwd() Dir { return .{ .handle = std.fs.cwd().fd }; } @@ -47,8 +50,9 @@ pub const UpdateFileError = File.OpenError; /// Check the file size, mtime, and mode of `source_path` and `dest_path`. If /// they are equal, does nothing. Otherwise, atomically copies `source_path` to -/// `dest_path`. The destination file gains the mtime, atime, and mode of the -/// source file so that the next call to `updateFile` will not need a copy. +/// `dest_path`, creating the parent directory hierarchy as needed. The +/// destination file gains the mtime, atime, and mode of the source file so +/// that the next call to `updateFile` will not need a copy. /// /// Returns the previous status of the file before updating. /// @@ -65,7 +69,7 @@ pub fn updateFile( options: std.fs.Dir.CopyFileOptions, ) !PrevStatus { var src_file = try source_dir.openFile(io, source_path, .{}); - defer src_file.close(); + defer src_file.close(io); const src_stat = try src_file.stat(io); const actual_mode = options.override_mode orelse src_stat.mode; @@ -93,13 +97,13 @@ pub fn updateFile( } var buffer: [1000]u8 = undefined; // Used only when direct fd-to-fd is not available. - var atomic_file = try dest_dir.atomicFile(io, dest_path, .{ + var atomic_file = try std.fs.Dir.atomicFile(.adaptFromNewApi(dest_dir), dest_path, .{ .mode = actual_mode, .write_buffer = &buffer, }); defer atomic_file.deinit(); - var src_reader: File.Reader = .initSize(io, src_file, &.{}, src_stat.size); + var src_reader: File.Reader = .initSize(src_file, io, &.{}, src_stat.size); const dest_writer = &atomic_file.file_writer.interface; _ = dest_writer.sendFileAll(&src_reader, .unlimited) catch |err| switch (err) { @@ -111,3 +115,154 @@ pub fn updateFile( try atomic_file.renameIntoPlace(); return .stale; } + +pub const ReadFileError = File.OpenError || File.Reader.Error; + +/// Read all of file contents using a preallocated buffer. +/// +/// The returned slice has the same pointer as `buffer`. If the length matches `buffer.len` +/// the situation is ambiguous. It could either mean that the entire file was read, and +/// it exactly fits the buffer, or it could mean the buffer was not big enough for the +/// entire file. +/// +/// * On Windows, `file_path` should be encoded as [WTF-8](https://simonsapin.github.io/wtf-8/). +/// * On WASI, `file_path` should be encoded as valid UTF-8. +/// * On other platforms, `file_path` is an opaque sequence of bytes with no particular encoding. +pub fn readFile(dir: Dir, io: Io, file_path: []const u8, buffer: []u8) ReadFileError![]u8 { + var file = try dir.openFile(io, file_path, .{}); + defer file.close(io); + + var reader = file.reader(io, &.{}); + const n = reader.interface.readSliceShort(buffer) catch |err| switch (err) { + error.ReadFailed => return reader.err.?, + }; + + return buffer[0..n]; +} + +pub const MakeError = error{ + /// In WASI, this error may occur when the file descriptor does + /// not hold the required rights to create a new directory relative to it. + AccessDenied, + PermissionDenied, + DiskQuota, + PathAlreadyExists, + SymLinkLoop, + LinkQuotaExceeded, + NameTooLong, + FileNotFound, + SystemResources, + NoSpaceLeft, + NotDir, + ReadOnlyFileSystem, + /// WASI-only; file paths must be valid UTF-8. + InvalidUtf8, + /// Windows-only; file paths provided by the user must be valid WTF-8. + /// https://simonsapin.github.io/wtf-8/ + InvalidWtf8, + BadPathName, + NoDevice, + /// On Windows, `\\server` or `\\server\share` was not found. + NetworkNotFound, +} || Io.Cancelable || Io.UnexpectedError; + +/// Creates a single directory with a relative or absolute path. +/// +/// * On Windows, `sub_path` should be encoded as [WTF-8](https://simonsapin.github.io/wtf-8/). +/// * On WASI, `sub_path` should be encoded as valid UTF-8. +/// * On other platforms, `sub_path` is an opaque sequence of bytes with no particular encoding. +/// +/// Related: +/// * `makePath` +/// * `makeDirAbsolute` +pub fn makeDir(dir: Dir, io: Io, sub_path: []const u8) MakeError!void { + return io.vtable.dirMake(io.userdata, dir, sub_path, default_mode); +} + +pub const MakePathError = MakeError || StatPathError; + +/// Calls makeDir iteratively to make an entire path, creating any parent +/// directories that do not exist. +/// +/// Returns success if the path already exists and is a directory. +/// +/// This function is not atomic, and if it returns an error, the file system +/// may have been modified regardless. +/// +/// Fails on an empty path with `error.BadPathName` as that is not a path that +/// can be created. +/// +/// On Windows, `sub_path` should be encoded as [WTF-8](https://simonsapin.github.io/wtf-8/). +/// On WASI, `sub_path` should be encoded as valid UTF-8. +/// On other platforms, `sub_path` is an opaque sequence of bytes with no particular encoding. +/// +/// Paths containing `..` components are handled differently depending on the platform: +/// - On Windows, `..` are resolved before the path is passed to NtCreateFile, meaning +/// a `sub_path` like "first/../second" will resolve to "second" and only a +/// `./second` directory will be created. +/// - On other platforms, `..` are not resolved before the path is passed to `mkdirat`, +/// meaning a `sub_path` like "first/../second" will create both a `./first` +/// and a `./second` directory. +pub fn makePath(dir: Dir, io: Io, sub_path: []const u8) MakePathError!void { + _ = try makePathStatus(dir, io, sub_path); +} + +pub const MakePathStatus = enum { existed, created }; + +/// Same as `makePath` except returns whether the path already existed or was +/// successfully created. +pub fn makePathStatus(dir: Dir, io: Io, sub_path: []const u8) MakePathError!MakePathStatus { + var it = try std.fs.path.componentIterator(sub_path); + var status: MakePathStatus = .existed; + var component = it.last() orelse return error.BadPathName; + while (true) { + if (makeDir(dir, io, component.path)) |_| { + status = .created; + } else |err| switch (err) { + error.PathAlreadyExists => { + // stat the file and return an error if it's not a directory + // this is important because otherwise a dangling symlink + // could cause an infinite loop + check_dir: { + // workaround for windows, see https://github.com/ziglang/zig/issues/16738 + const fstat = statPath(dir, io, component.path) catch |stat_err| switch (stat_err) { + error.IsDir => break :check_dir, + else => |e| return e, + }; + if (fstat.kind != .directory) return error.NotDir; + } + }, + error.FileNotFound => |e| { + component = it.previous() orelse return e; + continue; + }, + else => |e| return e, + } + component = it.next() orelse return status; + } +} + +pub const Stat = File.Stat; +pub const StatError = File.StatError; + +pub fn stat(dir: Dir, io: Io) StatError!Stat { + return io.vtable.dirStat(io.userdata, dir); +} + +pub const StatPathError = File.OpenError || File.StatError; + +/// Returns metadata for a file inside the directory. +/// +/// On Windows, this requires three syscalls. On other operating systems, it +/// only takes one. +/// +/// Symlinks are followed. +/// +/// `sub_path` may be absolute, in which case `self` is ignored. +/// +/// * On Windows, `sub_path` should be encoded as [WTF-8](https://simonsapin.github.io/wtf-8/). +/// * On WASI, `sub_path` should be encoded as valid UTF-8. +/// * On other platforms, `sub_path` is an opaque sequence of bytes with no particular encoding. +pub fn statPath(dir: Dir, io: Io, sub_path: []const u8) StatPathError!File.Stat { + return io.vtable.dirStatPath(io.userdata, dir, sub_path); +} diff --git a/lib/std/Io/File.zig b/lib/std/Io/File.zig index 242d96d52f..7d87fab973 100644 --- a/lib/std/Io/File.zig +++ b/lib/std/Io/File.zig @@ -446,7 +446,11 @@ pub const Reader = struct { fn stream(io_reader: *Io.Reader, w: *Io.Writer, limit: Io.Limit) Io.Reader.StreamError!usize { const r: *Reader = @alignCast(@fieldParentPtr("interface", io_reader)); - switch (r.mode) { + return streamMode(r, w, limit, r.mode); + } + + pub fn streamMode(r: *Reader, w: *Io.Writer, limit: Io.Limit, mode: Reader.Mode) Io.Reader.StreamError!usize { + switch (mode) { .positional, .streaming => return w.sendFile(r, limit) catch |write_err| switch (write_err) { error.Unimplemented => { r.mode = r.mode.toReading(); diff --git a/lib/std/Io/Threaded.zig b/lib/std/Io/Threaded.zig index 0727730b93..0f217c7f80 100644 --- a/lib/std/Io/Threaded.zig +++ b/lib/std/Io/Threaded.zig @@ -63,7 +63,17 @@ const Closure = struct { pub const InitError = std.Thread.CpuCountError || Allocator.Error; -pub fn init(gpa: Allocator) Pool { +/// Related: +/// * `init_single_threaded` +pub fn init( + /// Must be threadsafe. Only used for the following functions: + /// * `Io.VTable.async` + /// * `Io.VTable.concurrent` + /// * `Io.VTable.groupAsync` + /// If these functions are avoided, then `Allocator.failing` may be passed + /// here. + gpa: Allocator, +) Pool { var pool: Pool = .{ .allocator = gpa, .threads = .empty, @@ -77,6 +87,20 @@ pub fn init(gpa: Allocator) Pool { return pool; } +/// Statically initialize such that any call to the following functions will +/// fail with `error.OutOfMemory`: +/// * `Io.VTable.async` +/// * `Io.VTable.concurrent` +/// * `Io.VTable.groupAsync` +/// When initialized this way, `deinit` is safe, but unnecessary to call. +pub const init_single_threaded: Pool = .{ + .allocator = .failing, + .threads = .empty, + .stack_size = std.Thread.SpawnConfig.default_stack_size, + .cpu_count = 1, + .concurrent_count = 0, +}; + pub fn deinit(pool: *Pool) void { const gpa = pool.allocator; pool.join(); @@ -136,6 +160,10 @@ pub fn io(pool: *Pool) Io { .conditionWait = conditionWait, .conditionWake = conditionWake, + .dirMake = dirMake, + .dirStat = dirStat, + .dirStatPath = dirStatPath, + .fileStat = fileStat, .createFile = createFile, .fileOpen = fileOpen, .fileClose = fileClose, @@ -520,10 +548,11 @@ fn groupAsync( fn groupWait(userdata: ?*anyopaque, group: *Io.Group, token: *anyopaque) void { const pool: *Pool = @ptrCast(@alignCast(userdata)); - _ = pool; + const gpa = pool.allocator; if (builtin.single_threaded) return; + // TODO these primitives are too high level, need to check cancel on EINTR const group_state: *std.atomic.Value(usize) = @ptrCast(&group.state); const reset_event: *ResetEvent = @ptrCast(&group.context); std.Thread.WaitGroup.waitStateless(group_state, reset_event); @@ -531,8 +560,9 @@ fn groupWait(userdata: ?*anyopaque, group: *Io.Group, token: *anyopaque) void { var node: *std.SinglyLinkedList.Node = @ptrCast(@alignCast(token)); while (true) { const gc: *GroupClosure = @fieldParentPtr("node", node); - gc.closure.requestCancel(); - node = node.next orelse break; + const node_next = node.next; + gc.free(gpa); + node = node_next orelse break; } } @@ -724,6 +754,41 @@ fn conditionWake(userdata: ?*anyopaque, cond: *Io.Condition, wake: Io.Condition. } } +fn dirMake(userdata: ?*anyopaque, dir: Io.Dir, sub_path: []const u8, mode: Io.Dir.Mode) Io.Dir.MakeError!void { + const pool: *Pool = @ptrCast(@alignCast(userdata)); + try pool.checkCancel(); + + _ = dir; + _ = sub_path; + _ = mode; + @panic("TODO"); +} + +fn dirStat(userdata: ?*anyopaque, dir: Io.Dir) Io.Dir.StatError!Io.Dir.Stat { + const pool: *Pool = @ptrCast(@alignCast(userdata)); + try pool.checkCancel(); + + _ = dir; + @panic("TODO"); +} + +fn dirStatPath(userdata: ?*anyopaque, dir: Io.Dir, sub_path: []const u8) Io.Dir.StatError!Io.File.Stat { + const pool: *Pool = @ptrCast(@alignCast(userdata)); + try pool.checkCancel(); + + _ = dir; + _ = sub_path; + @panic("TODO"); +} + +fn fileStat(userdata: ?*anyopaque, file: Io.File) Io.File.StatError!Io.File.Stat { + const pool: *Pool = @ptrCast(@alignCast(userdata)); + try pool.checkCancel(); + + _ = file; + @panic("TODO"); +} + fn createFile( userdata: ?*anyopaque, dir: Io.Dir, diff --git a/lib/std/Io/Writer.zig b/lib/std/Io/Writer.zig index 2bb3c12b15..32aae1673e 100644 --- a/lib/std/Io/Writer.zig +++ b/lib/std/Io/Writer.zig @@ -2827,6 +2827,8 @@ pub const Allocating = struct { }; test "discarding sendFile" { + const io = testing.io; + var tmp_dir = testing.tmpDir(.{}); defer tmp_dir.cleanup(); @@ -2837,7 +2839,7 @@ test "discarding sendFile" { try file_writer.interface.writeByte('h'); try file_writer.interface.flush(); - var file_reader = file_writer.moveToReader(); + var file_reader = file_writer.moveToReader(io); try file_reader.seekTo(0); var w_buffer: [256]u8 = undefined; @@ -2847,6 +2849,8 @@ test "discarding sendFile" { } test "allocating sendFile" { + const io = testing.io; + var tmp_dir = testing.tmpDir(.{}); defer tmp_dir.cleanup(); @@ -2857,7 +2861,7 @@ test "allocating sendFile" { try file_writer.interface.writeAll("abcd"); try file_writer.interface.flush(); - var file_reader = file_writer.moveToReader(); + var file_reader = file_writer.moveToReader(io); try file_reader.seekTo(0); try file_reader.interface.fill(2); diff --git a/lib/std/Io/net.zig b/lib/std/Io/net.zig index e305145575..dc1080ab38 100644 --- a/lib/std/Io/net.zig +++ b/lib/std/Io/net.zig @@ -57,6 +57,39 @@ pub const IpAddress = union(enum) { pub const Family = @typeInfo(IpAddress).@"union".tag_type.?; + pub const ParseLiteralError = error{ InvalidAddress, InvalidPort }; + + /// Parse an IP address which may include a port. + /// + /// For IPv4, this is written `address:port`. + /// + /// For IPv6, RFC 3986 defines this as an "IP literal", and the port is + /// differentiated from the address by surrounding the address part in + /// brackets "[addr]:port". Even if the port is not given, the brackets are + /// mandatory. + pub fn parseLiteral(text: []const u8) ParseLiteralError!IpAddress { + if (text.len == 0) return error.InvalidAddress; + if (text[0] == '[') { + const addr_end = std.mem.indexOfScalar(u8, text, ']') orelse + return error.InvalidAddress; + const addr_text = text[1..addr_end]; + const port: u16 = p: { + if (addr_end == text.len - 1) break :p 0; + if (text[addr_end + 1] != ':') return error.InvalidAddress; + break :p std.fmt.parseInt(u16, text[addr_end + 2 ..], 10) catch return error.InvalidPort; + }; + return parseIp6(addr_text, port) catch error.InvalidAddress; + } + if (std.mem.indexOfScalar(u8, text, ':')) |i| { + const addr = Ip4Address.parse(text[0..i], 0) catch return error.InvalidAddress; + return .{ .ip4 = .{ + .bytes = addr.bytes, + .port = std.fmt.parseInt(u16, text[i + 1 ..], 10) catch return error.InvalidPort, + } }; + } + return parseIp4(text, 0) catch error.InvalidAddress; + } + /// Parse the given IP address string into an `IpAddress` value. /// /// This is a pure function but it cannot handle IPv6 addresses that have diff --git a/lib/std/Io/net/HostName.zig b/lib/std/Io/net/HostName.zig index d8c3a3a903..72b5d39038 100644 --- a/lib/std/Io/net/HostName.zig +++ b/lib/std/Io/net/HostName.zig @@ -77,7 +77,9 @@ pub const LookupError = error{ InvalidDnsAAAARecord, InvalidDnsCnameRecord, NameServerFailure, -} || Io.Timestamp.Error || IpAddress.BindError || Io.File.OpenError || Io.File.Reader.Error || Io.Cancelable; + /// Failed to open or read "/etc/hosts" or "/etc/resolv.conf". + DetectingNetworkConfigurationFailed, +} || Io.Timestamp.Error || IpAddress.BindError || Io.Cancelable; pub const LookupResult = struct { /// How many `LookupOptions.addresses_buffer` elements are populated. @@ -428,14 +430,25 @@ fn lookupHosts(host_name: HostName, io: Io, options: LookupOptions) !LookupResul error.AccessDenied, => return .empty, - else => |e| return e, + error.Canceled => |e| return e, + + else => { + // TODO populate optional diagnostic struct + return error.DetectingNetworkConfigurationFailed; + }, }; defer file.close(io); var line_buf: [512]u8 = undefined; var file_reader = file.reader(io, &line_buf); return lookupHostsReader(host_name, options, &file_reader.interface) catch |err| switch (err) { - error.ReadFailed => return file_reader.err.?, + error.ReadFailed => switch (file_reader.err.?) { + error.Canceled => |e| return e, + else => { + // TODO populate optional diagnostic struct + return error.DetectingNetworkConfigurationFailed; + }, + }, }; } diff --git a/lib/std/Io/net/test.zig b/lib/std/Io/net/test.zig index 01c4b213a8..f1fb6cd57c 100644 --- a/lib/std/Io/net/test.zig +++ b/lib/std/Io/net/test.zig @@ -211,11 +211,11 @@ test "listen on a port, send bytes, receive bytes" { const t = try std.Thread.spawn(.{}, S.clientFn, .{server.socket.address}); defer t.join(); - var client = try server.accept(io); - defer client.stream.close(io); + var stream = try server.accept(io); + defer stream.close(io); var buf: [16]u8 = undefined; - var stream_reader = client.stream.reader(io, &.{}); - const n = try stream_reader.interface().readSliceShort(&buf); + var stream_reader = stream.reader(io, &.{}); + const n = try stream_reader.interface.readSliceShort(&buf); try testing.expectEqual(@as(usize, 12), n); try testing.expectEqualSlices(u8, "Hello world!", buf[0..n]); @@ -267,10 +267,9 @@ fn testServer(server: *net.Server) anyerror!void { const io = testing.io; - var client = try server.accept(io); - - const stream = client.stream.writer(io); - try stream.print("hello from server\n", .{}); + var stream = try server.accept(io); + var writer = stream.writer(io, &.{}); + try writer.interface.print("hello from server\n", .{}); } test "listen on a unix socket, send bytes, receive bytes" { @@ -310,11 +309,11 @@ test "listen on a unix socket, send bytes, receive bytes" { const t = try std.Thread.spawn(.{}, S.clientFn, .{socket_path}); defer t.join(); - var client = try server.accept(io); - defer client.stream.close(io); + var stream = try server.accept(io); + defer stream.close(io); var buf: [16]u8 = undefined; - var stream_reader = client.stream.reader(io, &.{}); - const n = try stream_reader.interface().readSliceShort(&buf); + var stream_reader = stream.reader(io, &.{}); + const n = try stream_reader.interface.readSliceShort(&buf); try testing.expectEqual(@as(usize, 12), n); try testing.expectEqualSlices(u8, "Hello world!", buf[0..n]); @@ -366,10 +365,10 @@ test "non-blocking tcp server" { const socket_file = try net.tcpConnectToAddress(server.socket.address); defer socket_file.close(); - var client = try server.accept(io); - defer client.stream.close(io); - const stream = client.stream.writer(io); - try stream.print("hello from server\n", .{}); + var stream = try server.accept(io); + defer stream.close(io); + var writer = stream.writer(io, .{}); + try writer.interface.print("hello from server\n", .{}); var buf: [100]u8 = undefined; const len = try socket_file.read(&buf); diff --git a/lib/std/crypto/tls/Client.zig b/lib/std/crypto/tls/Client.zig index b697d624fa..f6e334af8e 100644 --- a/lib/std/crypto/tls/Client.zig +++ b/lib/std/crypto/tls/Client.zig @@ -105,6 +105,14 @@ pub const Options = struct { /// Verify that the server certificate is authorized by a given ca bundle. bundle: Certificate.Bundle, }, + write_buffer: []u8, + read_buffer: []u8, + /// Cryptographically secure random bytes. The pointer is not captured; data is only + /// read during `init`. + entropy: *const [176]u8, + /// Current time according to the wall clock / calendar, in seconds. + realtime_now_seconds: i64, + /// If non-null, ssl secrets are logged to this stream. Creating such a log file allows /// other programs with access to that file to decrypt all traffic over this connection. /// @@ -120,8 +128,6 @@ pub const Options = struct { /// application layer itself verifies that the amount of data received equals /// the amount of data expected, such as HTTP with the Content-Length header. allow_truncation_attacks: bool = false, - write_buffer: []u8, - read_buffer: []u8, /// Populated when `error.TlsAlert` is returned from `init`. alert: ?*tls.Alert = null, }; @@ -189,14 +195,12 @@ pub fn init(input: *Reader, output: *Writer, options: Options) InitError!Client }; const host_len: u16 = @intCast(host.len); - var random_buffer: [176]u8 = undefined; - crypto.random.bytes(&random_buffer); - const client_hello_rand = random_buffer[0..32].*; + const client_hello_rand = options.entropy[0..32].*; var key_seq: u64 = 0; var server_hello_rand: [32]u8 = undefined; - const legacy_session_id = random_buffer[32..64].*; + const legacy_session_id = options.entropy[32..64].*; - var key_share = KeyShare.init(random_buffer[64..176].*) catch |err| switch (err) { + var key_share = KeyShare.init(options.entropy[64..176].*) catch |err| switch (err) { // Only possible to happen if the seed is all zeroes. error.IdentityElement => return error.InsufficientEntropy, }; @@ -321,7 +325,7 @@ pub fn init(input: *Reader, output: *Writer, options: Options) InitError!Client var handshake_cipher: tls.HandshakeCipher = undefined; var main_cert_pub_key: CertificatePublicKey = undefined; var tls12_negotiated_group: ?tls.NamedGroup = null; - const now_sec = std.time.timestamp(); + const now_sec = options.realtime_now_seconds; var cleartext_fragment_start: usize = 0; var cleartext_fragment_end: usize = 0; diff --git a/lib/std/debug/SelfInfo/Windows.zig b/lib/std/debug/SelfInfo/Windows.zig index f84836a6d4..ea2fa96199 100644 --- a/lib/std/debug/SelfInfo/Windows.zig +++ b/lib/std/debug/SelfInfo/Windows.zig @@ -434,7 +434,7 @@ const Module = struct { }; errdefer pdb_file.close(); - const pdb_reader = try arena.create(std.fs.File.Reader); + const pdb_reader = try arena.create(Io.File.Reader); pdb_reader.* = pdb_file.reader(try arena.alloc(u8, 4096)); var pdb = Pdb.init(gpa, pdb_reader) catch |err| switch (err) { @@ -544,6 +544,7 @@ fn findModule(si: *SelfInfo, gpa: Allocator, address: usize) error{ MissingDebug } const std = @import("std"); +const Io = std.Io; const Allocator = std.mem.Allocator; const Dwarf = std.debug.Dwarf; const Pdb = std.debug.Pdb; diff --git a/lib/std/elf.zig b/lib/std/elf.zig index 746d24f61a..c5f8a7ea80 100644 --- a/lib/std/elf.zig +++ b/lib/std/elf.zig @@ -710,7 +710,7 @@ pub const ProgramHeaderIterator = struct { const offset = it.phoff + size * it.index; try it.file_reader.seekTo(offset); - return takeProgramHeader(&it.file_reader.interface, it.is_64, it.endian); + return try takeProgramHeader(&it.file_reader.interface, it.is_64, it.endian); } }; @@ -731,7 +731,7 @@ pub const ProgramHeaderBufferIterator = struct { const offset = it.phoff + size * it.index; var reader = Io.Reader.fixed(it.buf[offset..]); - return takeProgramHeader(&reader, it.is_64, it.endian); + return try takeProgramHeader(&reader, it.is_64, it.endian); } }; @@ -771,7 +771,7 @@ pub const SectionHeaderIterator = struct { const offset = it.shoff + size * it.index; try it.file_reader.seekTo(offset); - return takeSectionHeader(&it.file_reader.interface, it.is_64, it.endian); + return try takeSectionHeader(&it.file_reader.interface, it.is_64, it.endian); } }; @@ -793,7 +793,7 @@ pub const SectionHeaderBufferIterator = struct { if (offset > it.buf.len) return error.EndOfStream; var reader = Io.Reader.fixed(it.buf[@intCast(offset)..]); - return takeSectionHeader(&reader, it.is_64, it.endian); + return try takeSectionHeader(&reader, it.is_64, it.endian); } }; @@ -826,12 +826,12 @@ pub const DynamicSectionIterator = struct { file_reader: *Io.File.Reader, - pub fn next(it: *SectionHeaderIterator) !?Elf64_Dyn { + pub fn next(it: *DynamicSectionIterator) !?Elf64_Dyn { if (it.offset >= it.end_offset) return null; const size: u64 = if (it.is_64) @sizeOf(Elf64_Dyn) else @sizeOf(Elf32_Dyn); defer it.offset += size; try it.file_reader.seekTo(it.offset); - return takeDynamicSection(&it.file_reader.interface, it.is_64, it.endian); + return try takeDynamicSection(&it.file_reader.interface, it.is_64, it.endian); } }; diff --git a/lib/std/fs/Dir.zig b/lib/std/fs/Dir.zig index 3e3577bbf0..c2ddf98627 100644 --- a/lib/std/fs/Dir.zig +++ b/lib/std/fs/Dir.zig @@ -1,6 +1,11 @@ +//! Deprecated in favor of `Io.Dir`. const Dir = @This(); + const builtin = @import("builtin"); +const native_os = builtin.os.tag; + const std = @import("../std.zig"); +const Io = std.Io; const File = std.fs.File; const AtomicFile = std.fs.AtomicFile; const base64_encoder = fs.base64_encoder; @@ -12,7 +17,6 @@ const Allocator = std.mem.Allocator; const assert = std.debug.assert; const linux = std.os.linux; const windows = std.os.windows; -const native_os = builtin.os.tag; const have_flock = @TypeOf(posix.system.flock) != void; fd: Handle, @@ -1189,84 +1193,41 @@ pub fn createFileW(self: Dir, sub_path_w: []const u16, flags: File.CreateFlags) return file; } -pub const MakeError = posix.MakeDirError; +/// Deprecated in favor of `Io.Dir.MakeError`. +pub const MakeError = Io.Dir.MakeError; -/// Creates a single directory with a relative or absolute path. -/// To create multiple directories to make an entire path, see `makePath`. -/// To operate on only absolute paths, see `makeDirAbsolute`. -/// On Windows, `sub_path` should be encoded as [WTF-8](https://wtf-8.codeberg.page/). -/// On WASI, `sub_path` should be encoded as valid UTF-8. -/// On other platforms, `sub_path` is an opaque sequence of bytes with no particular encoding. +/// Deprecated in favor of `Io.Dir.makeDir`. pub fn makeDir(self: Dir, sub_path: []const u8) MakeError!void { - try posix.mkdirat(self.fd, sub_path, default_mode); + var threaded: Io.Threaded = .init_single_threaded; + const io = threaded.io(); + return Io.Dir.makeDir(.{ .handle = self.fd }, io, sub_path); } -/// Same as `makeDir`, but `sub_path` is null-terminated. -/// To create multiple directories to make an entire path, see `makePath`. -/// To operate on only absolute paths, see `makeDirAbsoluteZ`. +/// Deprecated in favor of `Io.Dir.makeDir`. pub fn makeDirZ(self: Dir, sub_path: [*:0]const u8) MakeError!void { try posix.mkdiratZ(self.fd, sub_path, default_mode); } -/// Creates a single directory with a relative or absolute null-terminated WTF-16 LE-encoded path. -/// To create multiple directories to make an entire path, see `makePath`. -/// To operate on only absolute paths, see `makeDirAbsoluteW`. +/// Deprecated in favor of `Io.Dir.makeDir`. pub fn makeDirW(self: Dir, sub_path: [*:0]const u16) MakeError!void { try posix.mkdiratW(self.fd, mem.span(sub_path), default_mode); } -/// Calls makeDir iteratively to make an entire path -/// (i.e. creating any parent directories that do not exist). -/// Returns success if the path already exists and is a directory. -/// This function is not atomic, and if it returns an error, the file system may -/// have been modified regardless. -/// On Windows, `sub_path` should be encoded as [WTF-8](https://wtf-8.codeberg.page/). -/// On WASI, `sub_path` should be encoded as valid UTF-8. -/// On other platforms, `sub_path` is an opaque sequence of bytes with no particular encoding. -/// Fails on an empty path with `error.BadPathName` as that is not a path that can be created. -/// -/// Paths containing `..` components are handled differently depending on the platform: -/// - On Windows, `..` are resolved before the path is passed to NtCreateFile, meaning -/// a `sub_path` like "first/../second" will resolve to "second" and only a -/// `./second` directory will be created. -/// - On other platforms, `..` are not resolved before the path is passed to `mkdirat`, -/// meaning a `sub_path` like "first/../second" will create both a `./first` -/// and a `./second` directory. -pub fn makePath(self: Dir, sub_path: []const u8) (MakeError || StatFileError)!void { +/// Deprecated in favor of `Io.Dir.makePath`. +pub fn makePath(self: Dir, sub_path: []const u8) MakePathError!void { _ = try self.makePathStatus(sub_path); } -pub const MakePathStatus = enum { existed, created }; -/// Same as `makePath` except returns whether the path already existed or was successfully created. -pub fn makePathStatus(self: Dir, sub_path: []const u8) (MakeError || StatFileError)!MakePathStatus { - var it = try fs.path.componentIterator(sub_path); - var status: MakePathStatus = .existed; - var component = it.last() orelse return error.BadPathName; - while (true) { - if (self.makeDir(component.path)) |_| { - status = .created; - } else |err| switch (err) { - error.PathAlreadyExists => { - // stat the file and return an error if it's not a directory - // this is important because otherwise a dangling symlink - // could cause an infinite loop - check_dir: { - // workaround for windows, see https://github.com/ziglang/zig/issues/16738 - const fstat = self.statFile(component.path) catch |stat_err| switch (stat_err) { - error.IsDir => break :check_dir, - else => |e| return e, - }; - if (fstat.kind != .directory) return error.NotDir; - } - }, - error.FileNotFound => |e| { - component = it.previous() orelse return e; - continue; - }, - else => |e| return e, - } - component = it.next() orelse return status; - } +/// Deprecated in favor of `Io.Dir.MakePathStatus`. +pub const MakePathStatus = Io.Dir.MakePathStatus; +/// Deprecated in favor of `Io.Dir.MakePathError`. +pub const MakePathError = Io.Dir.MakePathError; + +/// Deprecated in favor of `Io.Dir.makePathStatus`. +pub fn makePathStatus(self: Dir, sub_path: []const u8) MakePathError!MakePathStatus { + var threaded: Io.Threaded = .init_single_threaded; + const io = threaded.io(); + return Io.Dir.makePathStatus(.{ .handle = self.fd }, io, sub_path); } /// Windows only. Calls makeOpenDirAccessMaskW iteratively to make an entire path @@ -2052,20 +2013,11 @@ pub fn readLinkW(self: Dir, sub_path_w: []const u16, buffer: []u8) ![]u8 { return windows.ReadLink(self.fd, sub_path_w, buffer); } -/// Read all of file contents using a preallocated buffer. -/// The returned slice has the same pointer as `buffer`. If the length matches `buffer.len` -/// the situation is ambiguous. It could either mean that the entire file was read, and -/// it exactly fits the buffer, or it could mean the buffer was not big enough for the -/// entire file. -/// On Windows, `file_path` should be encoded as [WTF-8](https://wtf-8.codeberg.page/). -/// On WASI, `file_path` should be encoded as valid UTF-8. -/// On other platforms, `file_path` is an opaque sequence of bytes with no particular encoding. +/// Deprecated in favor of `Io.Dir.readFile`. pub fn readFile(self: Dir, file_path: []const u8, buffer: []u8) ![]u8 { - var file = try self.openFile(file_path, .{}); - defer file.close(); - - const end_index = try file.readAll(buffer); - return buffer[0..end_index]; + var threaded: Io.Threaded = .init_single_threaded; + const io = threaded.io(); + return Io.Dir.readFile(.{ .handle = self.fd }, io, file_path, buffer); } pub const ReadFileAllocError = File.OpenError || File.ReadError || Allocator.Error || error{ @@ -2091,7 +2043,7 @@ pub fn readFileAlloc( /// Used to allocate the result. gpa: Allocator, /// If reached or exceeded, `error.StreamTooLong` is returned instead. - limit: std.Io.Limit, + limit: Io.Limit, ) ReadFileAllocError![]u8 { return readFileAllocOptions(dir, sub_path, gpa, limit, .of(u8), null); } @@ -2101,6 +2053,8 @@ pub fn readFileAlloc( /// /// If the file size is already known, a better alternative is to initialize a /// `File.Reader`. +/// +/// TODO move this function to Io.Dir pub fn readFileAllocOptions( dir: Dir, /// On Windows, should be encoded as [WTF-8](https://wtf-8.codeberg.page/). @@ -2110,13 +2064,16 @@ pub fn readFileAllocOptions( /// Used to allocate the result. gpa: Allocator, /// If reached or exceeded, `error.StreamTooLong` is returned instead. - limit: std.Io.Limit, + limit: Io.Limit, comptime alignment: std.mem.Alignment, comptime sentinel: ?u8, ) ReadFileAllocError!(if (sentinel) |s| [:s]align(alignment.toByteUnits()) u8 else []align(alignment.toByteUnits()) u8) { + var threaded: Io.Threaded = .init_single_threaded; + const io = threaded.io(); + var file = try dir.openFile(sub_path, .{}); defer file.close(); - var file_reader = file.reader(&.{}); + var file_reader = file.reader(io, &.{}); return file_reader.interface.allocRemainingAlignedSentinel(gpa, limit, alignment, sentinel) catch |err| switch (err) { error.ReadFailed => return file_reader.err.?, error.OutOfMemory, error.StreamTooLong => |e| return e, @@ -2647,6 +2604,8 @@ pub const CopyFileError = File.OpenError || File.StatError || /// [WTF-8](https://wtf-8.codeberg.page/). On WASI, both paths should be /// encoded as valid UTF-8. On other platforms, both paths are an opaque /// sequence of bytes with no particular encoding. +/// +/// TODO move this function to Io.Dir pub fn copyFile( source_dir: Dir, source_path: []const u8, @@ -2654,11 +2613,15 @@ pub fn copyFile( dest_path: []const u8, options: CopyFileOptions, ) CopyFileError!void { - var file_reader: File.Reader = .init(try source_dir.openFile(source_path, .{}), &.{}); - defer file_reader.file.close(); + var threaded: Io.Threaded = .init_single_threaded; + const io = threaded.io(); + + const file = try source_dir.openFile(source_path, .{}); + var file_reader: File.Reader = .init(.{ .handle = file.handle }, io, &.{}); + defer file_reader.file.close(io); const mode = options.override_mode orelse blk: { - const st = try file_reader.file.stat(); + const st = try file_reader.file.stat(io); file_reader.size = st.size; break :blk st.mode; }; @@ -2708,6 +2671,7 @@ pub fn atomicFile(self: Dir, dest_path: []const u8, options: AtomicFileOptions) pub const Stat = File.Stat; pub const StatError = File.StatError; +/// Deprecated in favor of `Io.Dir.stat`. pub fn stat(self: Dir) StatError!Stat { const file: File = .{ .handle = self.fd }; return file.stat(); @@ -2715,17 +2679,7 @@ pub fn stat(self: Dir) StatError!Stat { pub const StatFileError = File.OpenError || File.StatError || posix.FStatAtError; -/// Returns metadata for a file inside the directory. -/// -/// On Windows, this requires three syscalls. On other operating systems, it -/// only takes one. -/// -/// Symlinks are followed. -/// -/// `sub_path` may be absolute, in which case `self` is ignored. -/// On Windows, `sub_path` should be encoded as [WTF-8](https://wtf-8.codeberg.page/). -/// On WASI, `sub_path` should be encoded as valid UTF-8. -/// On other platforms, `sub_path` is an opaque sequence of bytes with no particular encoding. +/// Deprecated in favor of `Io.Dir.statPath`. pub fn statFile(self: Dir, sub_path: []const u8) StatFileError!Stat { if (native_os == .windows) { var file = try self.openFile(sub_path, .{}); @@ -2799,3 +2753,11 @@ pub fn setPermissions(self: Dir, permissions: Permissions) SetPermissionsError!v const file: File = .{ .handle = self.fd }; try file.setPermissions(permissions); } + +pub fn adaptToNewApi(dir: Dir) Io.Dir { + return .{ .handle = dir.fd }; +} + +pub fn adaptFromNewApi(dir: Io.Dir) Dir { + return .{ .fd = dir.handle }; +} diff --git a/lib/std/fs/File.zig b/lib/std/fs/File.zig index 7c94fc1df6..3de4d9bcb4 100644 --- a/lib/std/fs/File.zig +++ b/lib/std/fs/File.zig @@ -858,10 +858,12 @@ pub const Writer = struct { }; } - pub fn moveToReader(w: *Writer) Reader { + /// TODO when this logic moves from fs.File to Io.File the io parameter should be deleted + pub fn moveToReader(w: *Writer, io: std.Io) Reader { defer w.* = undefined; return .{ - .file = w.file, + .io = io, + .file = .{ .handle = w.file.handle }, .mode = w.mode, .pos = w.pos, .interface = Reader.initInterface(w.interface.buffer), @@ -1350,15 +1352,15 @@ pub const Writer = struct { /// /// Positional is more threadsafe, since the global seek position is not /// affected. -pub fn reader(file: File, buffer: []u8) Reader { - return .init(file, buffer); +pub fn reader(file: File, io: std.Io, buffer: []u8) Reader { + return .init(.{ .handle = file.handle }, io, buffer); } /// Positional is more threadsafe, since the global seek position is not /// affected, but when such syscalls are not available, preemptively /// initializing in streaming mode skips a failed syscall. -pub fn readerStreaming(file: File, buffer: []u8) Reader { - return .initStreaming(file, buffer); +pub fn readerStreaming(file: File, io: std.Io, buffer: []u8) Reader { + return .initStreaming(.{ .handle = file.handle }, io, buffer); } /// Defaults to positional reading; falls back to streaming. @@ -1538,3 +1540,11 @@ pub fn downgradeLock(file: File) LockError!void { }; } } + +pub fn adaptToNewApi(file: File) std.Io.File { + return .{ .handle = file.handle }; +} + +pub fn adaptFromNewApi(file: std.Io.File) File { + return .{ .handle = file.handle }; +} diff --git a/lib/std/fs/test.zig b/lib/std/fs/test.zig index e9c341d39a..e60e3c7911 100644 --- a/lib/std/fs/test.zig +++ b/lib/std/fs/test.zig @@ -1,10 +1,12 @@ -const std = @import("../std.zig"); const builtin = @import("builtin"); +const native_os = builtin.os.tag; + +const std = @import("../std.zig"); +const Io = std.Io; const testing = std.testing; const fs = std.fs; const mem = std.mem; const wasi = std.os.wasi; -const native_os = builtin.os.tag; const windows = std.os.windows; const posix = std.posix; @@ -73,6 +75,7 @@ const PathType = enum { }; const TestContext = struct { + io: Io, path_type: PathType, path_sep: u8, arena: ArenaAllocator, @@ -83,6 +86,7 @@ const TestContext = struct { pub fn init(path_type: PathType, path_sep: u8, allocator: mem.Allocator, transform_fn: *const PathType.TransformFn) TestContext { const tmp = tmpDir(.{ .iterate = true }); return .{ + .io = testing.io, .path_type = path_type, .path_sep = path_sep, .arena = ArenaAllocator.init(allocator), @@ -1319,6 +1323,8 @@ test "max file name component lengths" { } test "writev, readv" { + const io = testing.io; + var tmp = tmpDir(.{}); defer tmp.cleanup(); @@ -1327,78 +1333,55 @@ test "writev, readv" { var buf1: [line1.len]u8 = undefined; var buf2: [line2.len]u8 = undefined; - var write_vecs = [_]posix.iovec_const{ - .{ - .base = line1, - .len = line1.len, - }, - .{ - .base = line2, - .len = line2.len, - }, - }; - var read_vecs = [_]posix.iovec{ - .{ - .base = &buf2, - .len = buf2.len, - }, - .{ - .base = &buf1, - .len = buf1.len, - }, - }; + var write_vecs: [2][]const u8 = .{ line1, line2 }; + var read_vecs: [2][]u8 = .{ &buf2, &buf1 }; var src_file = try tmp.dir.createFile("test.txt", .{ .read = true }); defer src_file.close(); - try src_file.writevAll(&write_vecs); + var writer = src_file.writerStreaming(&.{}); + + try writer.interface.writeVecAll(&write_vecs); + try writer.interface.flush(); try testing.expectEqual(@as(u64, line1.len + line2.len), try src_file.getEndPos()); - try src_file.seekTo(0); - const read = try src_file.readvAll(&read_vecs); - try testing.expectEqual(@as(usize, line1.len + line2.len), read); + + var reader = writer.moveToReader(io); + try reader.seekTo(0); + try reader.interface.readVecAll(&read_vecs); try testing.expectEqualStrings(&buf1, "line2\n"); try testing.expectEqualStrings(&buf2, "line1\n"); + try testing.expectError(error.EndOfStream, reader.interface.readSliceAll(&buf1)); } test "pwritev, preadv" { + const io = testing.io; + var tmp = tmpDir(.{}); defer tmp.cleanup(); const line1 = "line1\n"; const line2 = "line2\n"; - + var lines: [2][]const u8 = .{ line1, line2 }; var buf1: [line1.len]u8 = undefined; var buf2: [line2.len]u8 = undefined; - var write_vecs = [_]posix.iovec_const{ - .{ - .base = line1, - .len = line1.len, - }, - .{ - .base = line2, - .len = line2.len, - }, - }; - var read_vecs = [_]posix.iovec{ - .{ - .base = &buf2, - .len = buf2.len, - }, - .{ - .base = &buf1, - .len = buf1.len, - }, - }; + var read_vecs: [2][]u8 = .{ &buf2, &buf1 }; var src_file = try tmp.dir.createFile("test.txt", .{ .read = true }); defer src_file.close(); - try src_file.pwritevAll(&write_vecs, 16); + var writer = src_file.writer(&.{}); + + try writer.seekTo(16); + try writer.interface.writeVecAll(&lines); + try writer.interface.flush(); try testing.expectEqual(@as(u64, 16 + line1.len + line2.len), try src_file.getEndPos()); - const read = try src_file.preadvAll(&read_vecs, 16); - try testing.expectEqual(@as(usize, line1.len + line2.len), read); + + var reader = writer.moveToReader(io); + try reader.seekTo(16); + try reader.interface.readVecAll(&read_vecs); try testing.expectEqualStrings(&buf1, "line2\n"); try testing.expectEqualStrings(&buf2, "line1\n"); + try testing.expectError(error.EndOfStream, reader.interface.readSliceAll(&buf1)); } test "setEndPos" { @@ -1406,6 +1389,8 @@ test "setEndPos" { if (native_os == .wasi and builtin.link_libc) return error.SkipZigTest; if (builtin.cpu.arch.isMIPS64() and (builtin.abi == .gnuabin32 or builtin.abi == .muslabin32)) return error.SkipZigTest; // https://github.com/ziglang/zig/issues/23806 + const io = testing.io; + var tmp = tmpDir(.{}); defer tmp.cleanup(); @@ -1416,11 +1401,13 @@ test "setEndPos" { const initial_size = try f.getEndPos(); var buffer: [32]u8 = undefined; + var reader = f.reader(io, &.{}); { try f.setEndPos(initial_size); try testing.expectEqual(initial_size, try f.getEndPos()); - try testing.expectEqual(initial_size, try f.preadAll(&buffer, 0)); + try reader.seekTo(0); + try testing.expectEqual(initial_size, reader.interface.readSliceShort(&buffer)); try testing.expectEqualStrings("ninebytes", buffer[0..@intCast(initial_size)]); } @@ -1428,7 +1415,8 @@ test "setEndPos" { const larger = initial_size + 4; try f.setEndPos(larger); try testing.expectEqual(larger, try f.getEndPos()); - try testing.expectEqual(larger, try f.preadAll(&buffer, 0)); + try reader.seekTo(0); + try testing.expectEqual(larger, reader.interface.readSliceShort(&buffer)); try testing.expectEqualStrings("ninebytes\x00\x00\x00\x00", buffer[0..@intCast(larger)]); } @@ -1436,25 +1424,21 @@ test "setEndPos" { const smaller = initial_size - 5; try f.setEndPos(smaller); try testing.expectEqual(smaller, try f.getEndPos()); - try testing.expectEqual(smaller, try f.preadAll(&buffer, 0)); + try reader.seekTo(0); + try testing.expectEqual(smaller, try reader.interface.readSliceShort(&buffer)); try testing.expectEqualStrings("nine", buffer[0..@intCast(smaller)]); } try f.setEndPos(0); try testing.expectEqual(0, try f.getEndPos()); - try testing.expectEqual(0, try f.preadAll(&buffer, 0)); + try reader.seekTo(0); + try testing.expectEqual(0, try reader.interface.readSliceShort(&buffer)); // Invalid file length should error gracefully. Actual limit is host // and file-system dependent, but 1PB should fail on filesystems like // EXT4 and NTFS. But XFS or Btrfs support up to 8EiB files. - f.setEndPos(0x4_0000_0000_0000) catch |err| if (err != error.FileTooBig) { - return err; - }; - - f.setEndPos(std.math.maxInt(u63)) catch |err| if (err != error.FileTooBig) { - return err; - }; - + try testing.expectError(error.FileTooBig, f.setEndPos(0x4_0000_0000_0000)); + try testing.expectError(error.FileTooBig, f.setEndPos(std.math.maxInt(u63))); try testing.expectError(error.FileTooBig, f.setEndPos(std.math.maxInt(u63) + 1)); try testing.expectError(error.FileTooBig, f.setEndPos(std.math.maxInt(u64))); } @@ -1560,31 +1544,6 @@ test "sendfile with buffered data" { try std.testing.expectEqualSlices(u8, "AAAA", written_buf[0..amt]); } -test "copyRangeAll" { - var tmp = tmpDir(.{}); - defer tmp.cleanup(); - - try tmp.dir.makePath("os_test_tmp"); - - var dir = try tmp.dir.openDir("os_test_tmp", .{}); - defer dir.close(); - - var src_file = try dir.createFile("file1.txt", .{ .read = true }); - defer src_file.close(); - - const data = "u6wj+JmdF3qHsFPE BUlH2g4gJCmEz0PP"; - try src_file.writeAll(data); - - var dest_file = try dir.createFile("file2.txt", .{ .read = true }); - defer dest_file.close(); - - var written_buf: [100]u8 = undefined; - _ = try src_file.copyRangeAll(0, dest_file, 0, data.len); - - const amt = try dest_file.preadAll(&written_buf, 0); - try testing.expectEqualStrings(data, written_buf[0..amt]); -} - test "copyFile" { try testWithAllSupportedPathTypes(struct { fn impl(ctx: *TestContext) !void { @@ -1708,8 +1667,8 @@ test "open file with exclusive lock twice, make sure second lock waits" { } }; - var started = std.Thread.ResetEvent{}; - var locked = std.Thread.ResetEvent{}; + var started: std.Thread.ResetEvent = .unset; + var locked: std.Thread.ResetEvent = .unset; const t = try std.Thread.spawn(.{}, S.checkFn, .{ &ctx.dir, @@ -1773,7 +1732,7 @@ test "read from locked file" { const f = try ctx.dir.createFile(filename, .{ .read = true }); defer f.close(); var buffer: [1]u8 = undefined; - _ = try f.readAll(&buffer); + _ = try f.read(&buffer); } { const f = try ctx.dir.createFile(filename, .{ @@ -1785,9 +1744,9 @@ test "read from locked file" { defer f2.close(); var buffer: [1]u8 = undefined; if (builtin.os.tag == .windows) { - try std.testing.expectError(error.LockViolation, f2.readAll(&buffer)); + try std.testing.expectError(error.LockViolation, f2.read(&buffer)); } else { - try std.testing.expectEqual(0, f2.readAll(&buffer)); + try std.testing.expectEqual(0, f2.read(&buffer)); } } } @@ -1944,6 +1903,7 @@ test "'.' and '..' in fs.Dir functions" { try testWithAllSupportedPathTypes(struct { fn impl(ctx: *TestContext) !void { + const io = ctx.io; const subdir_path = try ctx.transformPath("./subdir"); const file_path = try ctx.transformPath("./subdir/../file"); const copy_path = try ctx.transformPath("./subdir/../copy"); @@ -1966,7 +1926,8 @@ test "'.' and '..' in fs.Dir functions" { try ctx.dir.deleteFile(rename_path); try ctx.dir.writeFile(.{ .sub_path = update_path, .data = "something" }); - const prev_status = try ctx.dir.updateFile(file_path, ctx.dir, update_path, .{}); + var dir = ctx.dir.adaptToNewApi(); + const prev_status = try dir.updateFile(io, file_path, dir, update_path, .{}); try testing.expectEqual(fs.Dir.PrevStatus.stale, prev_status); try ctx.dir.deleteDir(subdir_path); @@ -2005,13 +1966,6 @@ test "'.' and '..' in absolute functions" { renamed_file.close(); try fs.deleteFileAbsolute(renamed_file_path); - const update_file_path = try fs.path.join(allocator, &.{ subdir_path, "../update" }); - const update_file = try fs.createFileAbsolute(update_file_path, .{}); - try update_file.writeAll("something"); - update_file.close(); - const prev_status = try fs.updateFileAbsolute(created_file_path, update_file_path, .{}); - try testing.expectEqual(fs.Dir.PrevStatus.stale, prev_status); - try fs.deleteDirAbsolute(subdir_path); } @@ -2079,6 +2033,7 @@ test "invalid UTF-8/WTF-8 paths" { try testWithAllSupportedPathTypes(struct { fn impl(ctx: *TestContext) !void { + const io = ctx.io; // This is both invalid UTF-8 and WTF-8, since \xFF is an invalid start byte const invalid_path = try ctx.transformPath("\xFF"); @@ -2129,7 +2084,8 @@ test "invalid UTF-8/WTF-8 paths" { try testing.expectError(expected_err, ctx.dir.access(invalid_path, .{})); try testing.expectError(expected_err, ctx.dir.accessZ(invalid_path, .{})); - try testing.expectError(expected_err, ctx.dir.updateFile(invalid_path, ctx.dir, invalid_path, .{})); + var dir = ctx.dir.adaptToNewApi(); + try testing.expectError(expected_err, dir.updateFile(io, invalid_path, dir, invalid_path, .{})); try testing.expectError(expected_err, ctx.dir.copyFile(invalid_path, ctx.dir, invalid_path, .{})); try testing.expectError(expected_err, ctx.dir.statFile(invalid_path)); @@ -2144,7 +2100,6 @@ test "invalid UTF-8/WTF-8 paths" { try testing.expectError(expected_err, fs.renameZ(ctx.dir, invalid_path, ctx.dir, invalid_path)); if (native_os != .wasi and ctx.path_type != .relative) { - try testing.expectError(expected_err, fs.updateFileAbsolute(invalid_path, invalid_path, .{})); try testing.expectError(expected_err, fs.copyFileAbsolute(invalid_path, invalid_path, .{})); try testing.expectError(expected_err, fs.makeDirAbsolute(invalid_path)); try testing.expectError(expected_err, fs.makeDirAbsoluteZ(invalid_path)); @@ -2175,6 +2130,8 @@ test "invalid UTF-8/WTF-8 paths" { } test "read file non vectored" { + const io = std.testing.io; + var tmp_dir = testing.tmpDir(.{}); defer tmp_dir.cleanup(); @@ -2188,7 +2145,7 @@ test "read file non vectored" { try file_writer.interface.flush(); } - var file_reader: std.fs.File.Reader = .init(file, &.{}); + var file_reader: std.Io.File.Reader = .initAdapted(file, io, &.{}); var write_buffer: [100]u8 = undefined; var w: std.Io.Writer = .fixed(&write_buffer); @@ -2205,6 +2162,8 @@ test "read file non vectored" { } test "seek keeping partial buffer" { + const io = std.testing.io; + var tmp_dir = testing.tmpDir(.{}); defer tmp_dir.cleanup(); @@ -2219,7 +2178,7 @@ test "seek keeping partial buffer" { } var read_buffer: [3]u8 = undefined; - var file_reader: std.fs.File.Reader = .init(file, &read_buffer); + var file_reader: Io.File.Reader = .initAdapted(file, io, &read_buffer); try testing.expectEqual(0, file_reader.logicalPos()); @@ -2246,13 +2205,15 @@ test "seek keeping partial buffer" { } test "seekBy" { + const io = testing.io; + var tmp_dir = testing.tmpDir(.{}); defer tmp_dir.cleanup(); try tmp_dir.dir.writeFile(.{ .sub_path = "blah.txt", .data = "let's test seekBy" }); const f = try tmp_dir.dir.openFile("blah.txt", .{ .mode = .read_only }); defer f.close(); - var reader = f.readerStreaming(&.{}); + var reader = f.readerStreaming(io, &.{}); try reader.seekBy(2); var buffer: [20]u8 = undefined; diff --git a/lib/std/http/Client.zig b/lib/std/http/Client.zig index a701c09a90..dbd547611f 100644 --- a/lib/std/http/Client.zig +++ b/lib/std/http/Client.zig @@ -247,6 +247,7 @@ pub const Connection = struct { port: u16, stream: Io.net.Stream, ) error{OutOfMemory}!*Plain { + const io = client.io; const gpa = client.allocator; const alloc_len = allocLen(client, remote_host.bytes.len); const base = try gpa.alignedAlloc(u8, .of(Plain), alloc_len); @@ -260,8 +261,8 @@ pub const Connection = struct { plain.* = .{ .connection = .{ .client = client, - .stream_writer = stream.writer(socket_write_buffer), - .stream_reader = stream.reader(socket_read_buffer), + .stream_writer = stream.writer(io, socket_write_buffer), + .stream_reader = stream.reader(io, socket_read_buffer), .pool_node = .{}, .port = port, .host_len = @intCast(remote_host.bytes.len), @@ -300,6 +301,7 @@ pub const Connection = struct { port: u16, stream: Io.net.Stream, ) error{ OutOfMemory, TlsInitializationFailed }!*Tls { + const io = client.io; const gpa = client.allocator; const alloc_len = allocLen(client, remote_host.bytes.len); const base = try gpa.alignedAlloc(u8, .of(Tls), alloc_len); @@ -316,11 +318,14 @@ pub const Connection = struct { assert(base.ptr + alloc_len == socket_read_buffer.ptr + socket_read_buffer.len); @memcpy(host_buffer, remote_host.bytes); const tls: *Tls = @ptrCast(base); + var random_buffer: [176]u8 = undefined; + std.crypto.random.bytes(&random_buffer); + const now_ts = if (Io.Timestamp.now(io, .real)) |ts| ts.toSeconds() else |_| return error.TlsInitializationFailed; tls.* = .{ .connection = .{ .client = client, - .stream_writer = stream.writer(tls_write_buffer), - .stream_reader = stream.reader(socket_read_buffer), + .stream_writer = stream.writer(io, tls_write_buffer), + .stream_reader = stream.reader(io, socket_read_buffer), .pool_node = .{}, .port = port, .host_len = @intCast(remote_host.bytes.len), @@ -338,6 +343,8 @@ pub const Connection = struct { .ssl_key_log = client.ssl_key_log, .read_buffer = tls_read_buffer, .write_buffer = socket_write_buffer, + .entropy = &random_buffer, + .realtime_now_seconds = now_ts, // This is appropriate for HTTPS because the HTTP headers contain // the content length which is used to detect truncation attacks. .allow_truncation_attacks = true, @@ -1390,16 +1397,8 @@ pub const basic_authorization = struct { }; pub const ConnectTcpError = error{ - ConnectionRefused, - NetworkUnreachable, - ConnectionTimedOut, - ConnectionResetByPeer, - TemporaryNameServerFailure, - NameServerFailure, - UnknownHostName, - UnexpectedConnectFailure, TlsInitializationFailed, -} || Allocator.Error || Io.Cancelable; +} || Allocator.Error || HostName.ConnectError; /// Reuses a `Connection` if one matching `host` and `port` is already open. /// @@ -1424,6 +1423,7 @@ pub const ConnectTcpOptions = struct { }; pub fn connectTcpOptions(client: *Client, options: ConnectTcpOptions) ConnectTcpError!*Connection { + const io = client.io; const host = options.host; const port = options.port; const protocol = options.protocol; @@ -1437,22 +1437,17 @@ pub fn connectTcpOptions(client: *Client, options: ConnectTcpOptions) ConnectTcp .protocol = protocol, })) |conn| return conn; - const stream = host.connect(client.io, port, .{ .mode = .stream }) catch |err| switch (err) { - error.ConnectionRefused => return error.ConnectionRefused, - error.NetworkUnreachable => return error.NetworkUnreachable, - error.ConnectionTimedOut => return error.ConnectionTimedOut, - error.ConnectionResetByPeer => return error.ConnectionResetByPeer, - error.NameServerFailure => return error.NameServerFailure, - error.UnknownHostName => return error.UnknownHostName, - error.Canceled => return error.Canceled, - //else => return error.UnexpectedConnectFailure, - }; - errdefer stream.close(); + var stream = try host.connect(io, port, .{ .mode = .stream }); + errdefer stream.close(io); switch (protocol) { .tls => { if (disable_tls) return error.TlsInitializationFailed; - const tc = try Connection.Tls.create(client, proxied_host, proxied_port, stream); + const tc = Connection.Tls.create(client, proxied_host, proxied_port, stream) catch |err| switch (err) { + error.OutOfMemory => |e| return e, + error.Unexpected => |e| return e, + error.UnsupportedClock => return error.TlsInitializationFailed, + }; client.connection_pool.addUsed(&tc.connection); return &tc.connection; }, diff --git a/lib/std/os/linux/IoUring.zig b/lib/std/os/linux/IoUring.zig index 25d4d88fd0..eaaa7643a9 100644 --- a/lib/std/os/linux/IoUring.zig +++ b/lib/std/os/linux/IoUring.zig @@ -3,7 +3,7 @@ const std = @import("std"); const builtin = @import("builtin"); const assert = std.debug.assert; const mem = std.mem; -const net = std.net; +const net = std.Io.net; const posix = std.posix; const linux = std.os.linux; const testing = std.testing; @@ -2361,19 +2361,22 @@ test "sendmsg/recvmsg" { }; defer ring.deinit(); - var address_server = try net.Address.parseIp4("127.0.0.1", 0); + var address_server: linux.sockaddr.in = .{ + .port = 0, + .addr = @bitCast([4]u8{ 127, 0, 0, 1 }), + }; - const server = try posix.socket(address_server.any.family, posix.SOCK.DGRAM, 0); + const server = try posix.socket(address_server.family, posix.SOCK.DGRAM, 0); defer posix.close(server); try posix.setsockopt(server, posix.SOL.SOCKET, posix.SO.REUSEPORT, &mem.toBytes(@as(c_int, 1))); try posix.setsockopt(server, posix.SOL.SOCKET, posix.SO.REUSEADDR, &mem.toBytes(@as(c_int, 1))); - try posix.bind(server, &address_server.any, address_server.getOsSockLen()); + try posix.bind(server, addrAny(&address_server), @sizeOf(linux.sockaddr.in)); // set address_server to the OS-chosen IP/port. - var slen: posix.socklen_t = address_server.getOsSockLen(); - try posix.getsockname(server, &address_server.any, &slen); + var slen: posix.socklen_t = @sizeOf(linux.sockaddr.in); + try posix.getsockname(server, addrAny(&address_server), &slen); - const client = try posix.socket(address_server.any.family, posix.SOCK.DGRAM, 0); + const client = try posix.socket(address_server.family, posix.SOCK.DGRAM, 0); defer posix.close(client); const buffer_send = [_]u8{42} ** 128; @@ -2381,8 +2384,8 @@ test "sendmsg/recvmsg" { posix.iovec_const{ .base = &buffer_send, .len = buffer_send.len }, }; const msg_send: posix.msghdr_const = .{ - .name = &address_server.any, - .namelen = address_server.getOsSockLen(), + .name = addrAny(&address_server), + .namelen = @sizeOf(linux.sockaddr.in), .iov = &iovecs_send, .iovlen = 1, .control = null, @@ -2398,11 +2401,13 @@ test "sendmsg/recvmsg" { var iovecs_recv = [_]posix.iovec{ posix.iovec{ .base = &buffer_recv, .len = buffer_recv.len }, }; - const addr = [_]u8{0} ** 4; - var address_recv = net.Address.initIp4(addr, 0); + var address_recv: linux.sockaddr.in = .{ + .port = 0, + .addr = 0, + }; var msg_recv: posix.msghdr = .{ - .name = &address_recv.any, - .namelen = address_recv.getOsSockLen(), + .name = addrAny(&address_recv), + .namelen = @sizeOf(linux.sockaddr.in), .iov = &iovecs_recv, .iovlen = 1, .control = null, @@ -2441,6 +2446,8 @@ test "sendmsg/recvmsg" { test "timeout (after a relative time)" { if (!is_linux) return error.SkipZigTest; + const io = testing.io; + var ring = IoUring.init(1, 0) catch |err| switch (err) { error.SystemOutdated => return error.SkipZigTest, error.PermissionDenied => return error.SkipZigTest, @@ -2452,12 +2459,12 @@ test "timeout (after a relative time)" { const margin = 5; const ts: linux.kernel_timespec = .{ .sec = 0, .nsec = ms * 1000000 }; - const started = std.time.milliTimestamp(); + const started = try std.Io.Timestamp.now(io, .awake); const sqe = try ring.timeout(0x55555555, &ts, 0, 0); try testing.expectEqual(linux.IORING_OP.TIMEOUT, sqe.opcode); try testing.expectEqual(@as(u32, 1), try ring.submit()); const cqe = try ring.copy_cqe(); - const stopped = std.time.milliTimestamp(); + const stopped = try std.Io.Timestamp.now(io, .awake); try testing.expectEqual(linux.io_uring_cqe{ .user_data = 0x55555555, @@ -2466,7 +2473,8 @@ test "timeout (after a relative time)" { }, cqe); // Tests should not depend on timings: skip test if outside margin. - if (!std.math.approxEqAbs(f64, ms, @as(f64, @floatFromInt(stopped - started)), margin)) return error.SkipZigTest; + const ms_elapsed = started.durationTo(stopped).toMilliseconds(); + if (ms_elapsed > margin) return error.SkipZigTest; } test "timeout (after a number of completions)" { @@ -2861,19 +2869,22 @@ test "shutdown" { }; defer ring.deinit(); - var address = try net.Address.parseIp4("127.0.0.1", 0); + var address: linux.sockaddr.in = .{ + .port = 0, + .addr = @bitCast([4]u8{ 127, 0, 0, 1 }), + }; // Socket bound, expect shutdown to work { - const server = try posix.socket(address.any.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0); + const server = try posix.socket(address.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0); defer posix.close(server); try posix.setsockopt(server, posix.SOL.SOCKET, posix.SO.REUSEADDR, &mem.toBytes(@as(c_int, 1))); - try posix.bind(server, &address.any, address.getOsSockLen()); + try posix.bind(server, addrAny(&address), @sizeOf(linux.sockaddr.in)); try posix.listen(server, 1); // set address to the OS-chosen IP/port. - var slen: posix.socklen_t = address.getOsSockLen(); - try posix.getsockname(server, &address.any, &slen); + var slen: posix.socklen_t = @sizeOf(linux.sockaddr.in); + try posix.getsockname(server, addrAny(&address), &slen); const shutdown_sqe = try ring.shutdown(0x445445445, server, linux.SHUT.RD); try testing.expectEqual(linux.IORING_OP.SHUTDOWN, shutdown_sqe.opcode); @@ -2898,7 +2909,7 @@ test "shutdown" { // Socket not bound, expect to fail with ENOTCONN { - const server = try posix.socket(address.any.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0); + const server = try posix.socket(address.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0); defer posix.close(server); const shutdown_sqe = ring.shutdown(0x445445445, server, linux.SHUT.RD) catch |err| switch (err) { @@ -2966,22 +2977,11 @@ test "renameat" { }, cqe); // Validate that the old file doesn't exist anymore - { - _ = tmp.dir.openFile(old_path, .{}) catch |err| switch (err) { - error.FileNotFound => {}, - else => std.debug.panic("unexpected error: {}", .{err}), - }; - } + try testing.expectError(error.FileNotFound, tmp.dir.openFile(old_path, .{})); // Validate that the new file exists with the proper content - { - const new_file = try tmp.dir.openFile(new_path, .{}); - defer new_file.close(); - - var new_file_data: [16]u8 = undefined; - const bytes_read = try new_file.readAll(&new_file_data); - try testing.expectEqualStrings("hello", new_file_data[0..bytes_read]); - } + var new_file_data: [16]u8 = undefined; + try testing.expectEqualStrings("hello", try tmp.dir.readFile(new_path, &new_file_data)); } test "unlinkat" { @@ -3179,12 +3179,8 @@ test "linkat" { }, cqe); // Validate the second file - const second_file = try tmp.dir.openFile(second_path, .{}); - defer second_file.close(); - var second_file_data: [16]u8 = undefined; - const bytes_read = try second_file.readAll(&second_file_data); - try testing.expectEqualStrings("hello", second_file_data[0..bytes_read]); + try testing.expectEqualStrings("hello", try tmp.dir.readFile(second_path, &second_file_data)); } test "provide_buffers: read" { @@ -3588,7 +3584,10 @@ const SocketTestHarness = struct { fn createSocketTestHarness(ring: *IoUring) !SocketTestHarness { // Create a TCP server socket - var address = try net.Address.parseIp4("127.0.0.1", 0); + var address: linux.sockaddr.in = .{ + .port = 0, + .addr = @bitCast([4]u8{ 127, 0, 0, 1 }), + }; const listener_socket = try createListenerSocket(&address); errdefer posix.close(listener_socket); @@ -3598,9 +3597,9 @@ fn createSocketTestHarness(ring: *IoUring) !SocketTestHarness { _ = try ring.accept(0xaaaaaaaa, listener_socket, &accept_addr, &accept_addr_len, 0); // Create a TCP client socket - const client = try posix.socket(address.any.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0); + const client = try posix.socket(address.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0); errdefer posix.close(client); - _ = try ring.connect(0xcccccccc, client, &address.any, address.getOsSockLen()); + _ = try ring.connect(0xcccccccc, client, addrAny(&address), @sizeOf(linux.sockaddr.in)); try testing.expectEqual(@as(u32, 2), try ring.submit()); @@ -3636,18 +3635,18 @@ fn createSocketTestHarness(ring: *IoUring) !SocketTestHarness { }; } -fn createListenerSocket(address: *net.Address) !posix.socket_t { +fn createListenerSocket(address: *linux.sockaddr.in) !posix.socket_t { const kernel_backlog = 1; - const listener_socket = try posix.socket(address.any.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0); + const listener_socket = try posix.socket(address.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0); errdefer posix.close(listener_socket); try posix.setsockopt(listener_socket, posix.SOL.SOCKET, posix.SO.REUSEADDR, &mem.toBytes(@as(c_int, 1))); - try posix.bind(listener_socket, &address.any, address.getOsSockLen()); + try posix.bind(listener_socket, addrAny(address), @sizeOf(linux.sockaddr.in)); try posix.listen(listener_socket, kernel_backlog); // set address to the OS-chosen IP/port. - var slen: posix.socklen_t = address.getOsSockLen(); - try posix.getsockname(listener_socket, &address.any, &slen); + var slen: posix.socklen_t = @sizeOf(linux.sockaddr.in); + try posix.getsockname(listener_socket, addrAny(address), &slen); return listener_socket; } @@ -3662,7 +3661,10 @@ test "accept multishot" { }; defer ring.deinit(); - var address = try net.Address.parseIp4("127.0.0.1", 0); + var address: linux.sockaddr.in = .{ + .port = 0, + .addr = @bitCast([4]u8{ 127, 0, 0, 1 }), + }; const listener_socket = try createListenerSocket(&address); defer posix.close(listener_socket); @@ -3676,9 +3678,9 @@ test "accept multishot" { var nr: usize = 4; // number of clients to connect while (nr > 0) : (nr -= 1) { // connect client - const client = try posix.socket(address.any.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0); + const client = try posix.socket(address.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0); errdefer posix.close(client); - try posix.connect(client, &address.any, address.getOsSockLen()); + try posix.connect(client, addrAny(&address), @sizeOf(linux.sockaddr.in)); // test accept completion var cqe = try ring.copy_cqe(); @@ -3756,7 +3758,10 @@ test "accept_direct" { else => return err, }; defer ring.deinit(); - var address = try net.Address.parseIp4("127.0.0.1", 0); + var address: linux.sockaddr.in = .{ + .port = 0, + .addr = @bitCast([4]u8{ 127, 0, 0, 1 }), + }; // register direct file descriptors var registered_fds = [_]posix.fd_t{-1} ** 2; @@ -3779,8 +3784,8 @@ test "accept_direct" { try testing.expectEqual(@as(u32, 1), try ring.submit()); // connect - const client = try posix.socket(address.any.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0); - try posix.connect(client, &address.any, address.getOsSockLen()); + const client = try posix.socket(address.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0); + try posix.connect(client, addrAny(&address), @sizeOf(linux.sockaddr.in)); defer posix.close(client); // accept completion @@ -3813,8 +3818,8 @@ test "accept_direct" { _ = try ring.accept_direct(accept_userdata, listener_socket, null, null, 0); try testing.expectEqual(@as(u32, 1), try ring.submit()); // connect - const client = try posix.socket(address.any.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0); - try posix.connect(client, &address.any, address.getOsSockLen()); + const client = try posix.socket(address.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0); + try posix.connect(client, addrAny(&address), @sizeOf(linux.sockaddr.in)); defer posix.close(client); // completion with error const cqe_accept = try ring.copy_cqe(); @@ -3837,7 +3842,10 @@ test "accept_multishot_direct" { }; defer ring.deinit(); - var address = try net.Address.parseIp4("127.0.0.1", 0); + var address: linux.sockaddr.in = .{ + .port = 0, + .addr = @bitCast([4]u8{ 127, 0, 0, 1 }), + }; var registered_fds = [_]posix.fd_t{-1} ** 2; try ring.register_files(registered_fds[0..]); @@ -3855,8 +3863,8 @@ test "accept_multishot_direct" { for (registered_fds) |_| { // connect - const client = try posix.socket(address.any.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0); - try posix.connect(client, &address.any, address.getOsSockLen()); + const client = try posix.socket(address.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0); + try posix.connect(client, addrAny(&address), @sizeOf(linux.sockaddr.in)); defer posix.close(client); // accept completion @@ -3870,8 +3878,8 @@ test "accept_multishot_direct" { // Multishot is terminated (more flag is not set). { // connect - const client = try posix.socket(address.any.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0); - try posix.connect(client, &address.any, address.getOsSockLen()); + const client = try posix.socket(address.family, posix.SOCK.STREAM | posix.SOCK.CLOEXEC, 0); + try posix.connect(client, addrAny(&address), @sizeOf(linux.sockaddr.in)); defer posix.close(client); // completion with error const cqe_accept = try ring.copy_cqe(); @@ -3944,7 +3952,10 @@ test "socket_direct/socket_direct_alloc/close_direct" { try testing.expect(cqe_socket.res == 2); // returns registered file index // use sockets from registered_fds in connect operation - var address = try net.Address.parseIp4("127.0.0.1", 0); + var address: linux.sockaddr.in = .{ + .port = 0, + .addr = @bitCast([4]u8{ 127, 0, 0, 1 }), + }; const listener_socket = try createListenerSocket(&address); defer posix.close(listener_socket); const accept_userdata: u64 = 0xaaaaaaaa; @@ -3954,7 +3965,7 @@ test "socket_direct/socket_direct_alloc/close_direct" { // prepare accept _ = try ring.accept(accept_userdata, listener_socket, null, null, 0); // prepare connect with fixed socket - const connect_sqe = try ring.connect(connect_userdata, @intCast(fd_index), &address.any, address.getOsSockLen()); + const connect_sqe = try ring.connect(connect_userdata, @intCast(fd_index), addrAny(&address), @sizeOf(linux.sockaddr.in)); connect_sqe.flags |= linux.IOSQE_FIXED_FILE; // fd is fixed file index // submit both try testing.expectEqual(@as(u32, 2), try ring.submit()); @@ -4483,12 +4494,15 @@ test "bind/listen/connect" { // LISTEN is higher required operation if (!probe.is_supported(.LISTEN)) return error.SkipZigTest; - var addr = net.Address.initIp4([4]u8{ 127, 0, 0, 1 }, 0); - const proto: u32 = if (addr.any.family == linux.AF.UNIX) 0 else linux.IPPROTO.TCP; + var addr: linux.sockaddr.in = .{ + .port = 0, + .addr = @bitCast([4]u8{ 127, 0, 0, 1 }), + }; + const proto: u32 = if (addr.family == linux.AF.UNIX) 0 else linux.IPPROTO.TCP; const listen_fd = brk: { // Create socket - _ = try ring.socket(1, addr.any.family, linux.SOCK.STREAM | linux.SOCK.CLOEXEC, proto, 0); + _ = try ring.socket(1, addr.family, linux.SOCK.STREAM | linux.SOCK.CLOEXEC, proto, 0); try testing.expectEqual(1, try ring.submit()); var cqe = try ring.copy_cqe(); try testing.expectEqual(1, cqe.user_data); @@ -4500,7 +4514,7 @@ test "bind/listen/connect" { var optval: u32 = 1; (try ring.setsockopt(2, listen_fd, linux.SOL.SOCKET, linux.SO.REUSEADDR, mem.asBytes(&optval))).link_next(); (try ring.setsockopt(3, listen_fd, linux.SOL.SOCKET, linux.SO.REUSEPORT, mem.asBytes(&optval))).link_next(); - (try ring.bind(4, listen_fd, &addr.any, addr.getOsSockLen(), 0)).link_next(); + (try ring.bind(4, listen_fd, addrAny(&addr), @sizeOf(linux.sockaddr.in), 0)).link_next(); _ = try ring.listen(5, listen_fd, 1, 0); // Submit 4 operations try testing.expectEqual(4, try ring.submit()); @@ -4521,15 +4535,15 @@ test "bind/listen/connect" { try testing.expectEqual(1, optval); // Read system assigned port into addr - var addr_len: posix.socklen_t = addr.getOsSockLen(); - try posix.getsockname(listen_fd, &addr.any, &addr_len); + var addr_len: posix.socklen_t = @sizeOf(linux.sockaddr.in); + try posix.getsockname(listen_fd, addrAny(&addr), &addr_len); break :brk listen_fd; }; const connect_fd = brk: { // Create connect socket - _ = try ring.socket(6, addr.any.family, linux.SOCK.STREAM | linux.SOCK.CLOEXEC, proto, 0); + _ = try ring.socket(6, addr.family, linux.SOCK.STREAM | linux.SOCK.CLOEXEC, proto, 0); try testing.expectEqual(1, try ring.submit()); const cqe = try ring.copy_cqe(); try testing.expectEqual(6, cqe.user_data); @@ -4542,7 +4556,7 @@ test "bind/listen/connect" { // Prepare accept/connect operations _ = try ring.accept(7, listen_fd, null, null, 0); - _ = try ring.connect(8, connect_fd, &addr.any, addr.getOsSockLen()); + _ = try ring.connect(8, connect_fd, addrAny(&addr), @sizeOf(linux.sockaddr.in)); try testing.expectEqual(2, try ring.submit()); // Get listener accepted socket var accept_fd: posix.socket_t = 0; @@ -4604,3 +4618,7 @@ fn testSendRecv(ring: *IoUring, send_fd: posix.socket_t, recv_fd: posix.socket_t try testing.expectEqualSlices(u8, buffer_send, buffer_recv[0..buffer_send.len]); try testing.expectEqualSlices(u8, buffer_send, buffer_recv[buffer_send.len..]); } + +fn addrAny(addr: *linux.sockaddr.in) *linux.sockaddr { + return @ptrCast(addr); +} diff --git a/lib/std/posix.zig b/lib/std/posix.zig index 93676f2a74..c5e61749b2 100644 --- a/lib/std/posix.zig +++ b/lib/std/posix.zig @@ -3000,31 +3000,7 @@ pub fn mkdiratW(dir_fd: fd_t, sub_path_w: []const u16, mode: mode_t) MakeDirErro windows.CloseHandle(sub_dir_handle); } -pub const MakeDirError = error{ - /// In WASI, this error may occur when the file descriptor does - /// not hold the required rights to create a new directory relative to it. - AccessDenied, - PermissionDenied, - DiskQuota, - PathAlreadyExists, - SymLinkLoop, - LinkQuotaExceeded, - NameTooLong, - FileNotFound, - SystemResources, - NoSpaceLeft, - NotDir, - ReadOnlyFileSystem, - /// WASI-only; file paths must be valid UTF-8. - InvalidUtf8, - /// Windows-only; file paths provided by the user must be valid WTF-8. - /// https://wtf-8.codeberg.page/ - InvalidWtf8, - BadPathName, - NoDevice, - /// On Windows, `\\server` or `\\server\share` was not found. - NetworkNotFound, -} || UnexpectedError; +pub const MakeDirError = std.Io.Dir.MakeError; /// Create a directory. /// `mode` is ignored on Windows and WASI. diff --git a/lib/std/posix/test.zig b/lib/std/posix/test.zig index f8206b8a5a..b5ce476441 100644 --- a/lib/std/posix/test.zig +++ b/lib/std/posix/test.zig @@ -731,11 +731,8 @@ test "dup & dup2" { try dup2ed.writeAll("dup2"); } - var file = try tmp.dir.openFile("os_dup_test", .{}); - defer file.close(); - - var buf: [7]u8 = undefined; - try testing.expectEqualStrings("dupdup2", buf[0..try file.readAll(&buf)]); + var buffer: [8]u8 = undefined; + try testing.expectEqualStrings("dupdup2", try tmp.dir.readFile("os_dup_test", &buffer)); } test "writev longer than IOV_MAX" { diff --git a/lib/std/process/Child.zig b/lib/std/process/Child.zig index 50157d52d9..c018d77424 100644 --- a/lib/std/process/Child.zig +++ b/lib/std/process/Child.zig @@ -1,5 +1,9 @@ -const std = @import("../std.zig"); +const ChildProcess = @This(); + const builtin = @import("builtin"); +const native_os = builtin.os.tag; + +const std = @import("../std.zig"); const unicode = std.unicode; const fs = std.fs; const process = std.process; @@ -11,9 +15,7 @@ const mem = std.mem; const EnvMap = std.process.EnvMap; const maxInt = std.math.maxInt; const assert = std.debug.assert; -const native_os = builtin.os.tag; const Allocator = std.mem.Allocator; -const ChildProcess = @This(); const ArrayList = std.ArrayList; pub const Id = switch (native_os) { @@ -317,16 +319,23 @@ pub fn waitForSpawn(self: *ChildProcess) SpawnError!void { const err_pipe = self.err_pipe orelse return; self.err_pipe = null; - // Wait for the child to report any errors in or before `execvpe`. - if (readIntFd(err_pipe)) |child_err_int| { - posix.close(err_pipe); + const report = readIntFd(err_pipe); + posix.close(err_pipe); + if (report) |child_err_int| { const child_err: SpawnError = @errorCast(@errorFromInt(child_err_int)); self.term = child_err; return child_err; - } else |_| { - // Write end closed by CLOEXEC at the time of the `execvpe` call, indicating success! - posix.close(err_pipe); + } else |read_err| switch (read_err) { + error.EndOfStream => { + // Write end closed by CLOEXEC at the time of the `execvpe` call, + // indicating success. + }, + else => { + // Problem reading the error from the error reporting pipe. We + // don't know if the child is alive or dead. Better to assume it is + // alive so the resource does not risk being leaked. + }, } } @@ -1014,8 +1023,14 @@ fn writeIntFd(fd: i32, value: ErrInt) !void { fn readIntFd(fd: i32) !ErrInt { var buffer: [8]u8 = undefined; - var fr: std.fs.File.Reader = .initStreaming(.{ .handle = fd }, &buffer); - return @intCast(fr.interface.takeInt(u64, .little) catch return error.SystemResources); + var i: usize = 0; + while (i < buffer.len) { + const n = try std.posix.read(fd, buffer[i..]); + if (n == 0) return error.EndOfStream; + i += n; + } + const int = mem.readInt(u64, &buffer, .little); + return @intCast(int); } const ErrInt = std.meta.Int(.unsigned, @sizeOf(anyerror) * 8); diff --git a/lib/std/tar/Writer.zig b/lib/std/tar/Writer.zig index bffdb8ee7c..a48e8cc407 100644 --- a/lib/std/tar/Writer.zig +++ b/lib/std/tar/Writer.zig @@ -1,7 +1,9 @@ +const Writer = @This(); + const std = @import("std"); +const Io = std.Io; const assert = std.debug.assert; const testing = std.testing; -const Writer = @This(); const block_size = @sizeOf(Header); @@ -14,7 +16,7 @@ pub const Options = struct { mtime: u64 = 0, }; -underlying_writer: *std.Io.Writer, +underlying_writer: *Io.Writer, prefix: []const u8 = "", mtime_now: u64 = 0, @@ -36,12 +38,12 @@ pub fn writeDir(w: *Writer, sub_path: []const u8, options: Options) Error!void { try w.writeHeader(.directory, sub_path, "", 0, options); } -pub const WriteFileError = std.Io.Writer.FileError || Error || std.fs.File.Reader.SizeError; +pub const WriteFileError = Io.Writer.FileError || Error || Io.File.Reader.SizeError; pub fn writeFile( w: *Writer, sub_path: []const u8, - file_reader: *std.fs.File.Reader, + file_reader: *Io.File.Reader, stat_mtime: i128, ) WriteFileError!void { const size = try file_reader.getSize(); @@ -58,7 +60,7 @@ pub fn writeFile( try w.writePadding64(size); } -pub const WriteFileStreamError = Error || std.Io.Reader.StreamError; +pub const WriteFileStreamError = Error || Io.Reader.StreamError; /// Writes file reading file content from `reader`. Reads exactly `size` bytes /// from `reader`, or returns `error.EndOfStream`. @@ -66,7 +68,7 @@ pub fn writeFileStream( w: *Writer, sub_path: []const u8, size: u64, - reader: *std.Io.Reader, + reader: *Io.Reader, options: Options, ) WriteFileStreamError!void { try w.writeHeader(.regular, sub_path, "", size, options); @@ -136,15 +138,15 @@ fn writeExtendedHeader(w: *Writer, typeflag: Header.FileType, buffers: []const [ try w.writePadding(len); } -fn writePadding(w: *Writer, bytes: usize) std.Io.Writer.Error!void { +fn writePadding(w: *Writer, bytes: usize) Io.Writer.Error!void { return writePaddingPos(w, bytes % block_size); } -fn writePadding64(w: *Writer, bytes: u64) std.Io.Writer.Error!void { +fn writePadding64(w: *Writer, bytes: u64) Io.Writer.Error!void { return writePaddingPos(w, @intCast(bytes % block_size)); } -fn writePaddingPos(w: *Writer, pos: usize) std.Io.Writer.Error!void { +fn writePaddingPos(w: *Writer, pos: usize) Io.Writer.Error!void { if (pos == 0) return; try w.underlying_writer.splatByteAll(0, block_size - pos); } @@ -153,7 +155,7 @@ fn writePaddingPos(w: *Writer, pos: usize) std.Io.Writer.Error!void { /// "reasonable system must not assume that such a block exists when reading an /// archive". Therefore, the Zig standard library recommends to not call this /// function. -pub fn finishPedantically(w: *Writer) std.Io.Writer.Error!void { +pub fn finishPedantically(w: *Writer) Io.Writer.Error!void { try w.underlying_writer.splatByteAll(0, block_size * 2); } @@ -248,7 +250,7 @@ pub const Header = extern struct { try octal(&w.checksum, checksum); } - pub fn write(h: *Header, bw: *std.Io.Writer) error{ OctalOverflow, WriteFailed }!void { + pub fn write(h: *Header, bw: *Io.Writer) error{ OctalOverflow, WriteFailed }!void { try h.updateChecksum(); try bw.writeAll(std.mem.asBytes(h)); } @@ -396,14 +398,14 @@ test "write files" { { const root = "root"; - var output: std.Io.Writer.Allocating = .init(testing.allocator); + var output: Io.Writer.Allocating = .init(testing.allocator); var w: Writer = .{ .underlying_writer = &output.writer }; defer output.deinit(); try w.setRoot(root); for (files) |file| try w.writeFileBytes(file.path, file.content, .{}); - var input: std.Io.Reader = .fixed(output.written()); + var input: Io.Reader = .fixed(output.written()); var it: std.tar.Iterator = .init(&input, .{ .file_name_buffer = &file_name_buffer, .link_name_buffer = &link_name_buffer, @@ -424,7 +426,7 @@ test "write files" { try testing.expectEqual('/', actual.name[root.len..][0]); try testing.expectEqualStrings(expected.path, actual.name[root.len + 1 ..]); - var content: std.Io.Writer.Allocating = .init(testing.allocator); + var content: Io.Writer.Allocating = .init(testing.allocator); defer content.deinit(); try it.streamRemaining(actual, &content.writer); try testing.expectEqualSlices(u8, expected.content, content.written()); @@ -432,15 +434,15 @@ test "write files" { } // without root { - var output: std.Io.Writer.Allocating = .init(testing.allocator); + var output: Io.Writer.Allocating = .init(testing.allocator); var w: Writer = .{ .underlying_writer = &output.writer }; defer output.deinit(); for (files) |file| { - var content: std.Io.Reader = .fixed(file.content); + var content: Io.Reader = .fixed(file.content); try w.writeFileStream(file.path, file.content.len, &content, .{}); } - var input: std.Io.Reader = .fixed(output.written()); + var input: Io.Reader = .fixed(output.written()); var it: std.tar.Iterator = .init(&input, .{ .file_name_buffer = &file_name_buffer, .link_name_buffer = &link_name_buffer, @@ -452,7 +454,7 @@ test "write files" { const expected = files[i]; try testing.expectEqualStrings(expected.path, actual.name); - var content: std.Io.Writer.Allocating = .init(testing.allocator); + var content: Io.Writer.Allocating = .init(testing.allocator); defer content.deinit(); try it.streamRemaining(actual, &content.writer); try testing.expectEqualSlices(u8, expected.content, content.written()); diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 04e5c2b221..6d3ccc9b22 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -559,7 +559,7 @@ test isUnderscore { /// If the source can be UTF-16LE encoded, this function asserts that `gpa` /// will align a byte-sized allocation to at least 2. Allocators that don't do /// this are rare. -pub fn readSourceFileToEndAlloc(gpa: Allocator, file_reader: *std.fs.File.Reader) ![:0]u8 { +pub fn readSourceFileToEndAlloc(gpa: Allocator, file_reader: *Io.File.Reader) ![:0]u8 { var buffer: std.ArrayList(u8) = .empty; defer buffer.deinit(gpa); diff --git a/lib/std/zig/system.zig b/lib/std/zig/system.zig index 43d64205a7..7318612151 100644 --- a/lib/std/zig/system.zig +++ b/lib/std/zig/system.zig @@ -442,6 +442,7 @@ pub fn resolveTargetQuery(io: Io, query: Target.Query) DetectError!Target { error.DeviceBusy, error.InputOutput, error.LockViolation, + error.FileSystem, error.UnableToOpenElfFile, error.UnhelpfulFile, @@ -542,16 +543,15 @@ fn detectNativeCpuAndFeatures(cpu_arch: Target.Cpu.Arch, os: Target.Os, query: T return null; } -pub const AbiAndDynamicLinkerFromFileError = error{}; - -pub fn abiAndDynamicLinkerFromFile( +fn abiAndDynamicLinkerFromFile( file_reader: *Io.File.Reader, header: *const elf.Header, cpu: Target.Cpu, os: Target.Os, ld_info_list: []const LdInfo, query: Target.Query, -) AbiAndDynamicLinkerFromFileError!Target { +) !Target { + const io = file_reader.io; var result: Target = .{ .cpu = cpu, .os = os, @@ -623,8 +623,8 @@ pub fn abiAndDynamicLinkerFromFile( try file_reader.seekTo(shstr.sh_offset); try file_reader.interface.readSliceAll(shstrtab); const dynstr: ?struct { offset: u64, size: u64 } = find_dyn_str: { - var it = header.iterateSectionHeaders(&file_reader.interface); - while (it.next()) |shdr| { + var it = header.iterateSectionHeaders(file_reader); + while (try it.next()) |shdr| { const end = mem.findScalarPos(u8, shstrtab, shdr.sh_name, 0) orelse continue; const sh_name = shstrtab[shdr.sh_name..end :0]; if (mem.eql(u8, sh_name, ".dynstr")) break :find_dyn_str .{ @@ -645,7 +645,7 @@ pub fn abiAndDynamicLinkerFromFile( var it = mem.tokenizeScalar(u8, rpath_list, ':'); while (it.next()) |rpath| { - if (glibcVerFromRPath(rpath)) |ver| { + if (glibcVerFromRPath(io, rpath)) |ver| { result.os.version_range.linux.glibc = ver; return result; } else |err| switch (err) { @@ -660,7 +660,7 @@ pub fn abiAndDynamicLinkerFromFile( // There is no DT_RUNPATH so we try to find libc.so.6 inside the same // directory as the dynamic linker. if (fs.path.dirname(dl_path)) |rpath| { - if (glibcVerFromRPath(rpath)) |ver| { + if (glibcVerFromRPath(io, rpath)) |ver| { result.os.version_range.linux.glibc = ver; return result; } else |err| switch (err) { @@ -725,7 +725,7 @@ pub fn abiAndDynamicLinkerFromFile( @memcpy(path_buf[index..][0..abi.len], abi); index += abi.len; const rpath = path_buf[0..index]; - if (glibcVerFromRPath(rpath)) |ver| { + if (glibcVerFromRPath(io, rpath)) |ver| { result.os.version_range.linux.glibc = ver; return result; } else |err| switch (err) { @@ -842,18 +842,13 @@ fn glibcVerFromRPath(io: Io, rpath: []const u8) !std.SemanticVersion { error.InvalidElfMagic, error.InvalidElfEndian, error.InvalidElfClass, - error.InvalidElfFile, error.InvalidElfVersion, error.InvalidGnuLibCVersion, error.EndOfStream, => return error.GLibCNotFound, - error.SystemResources, - error.UnableToReadElfFile, - error.Unexpected, - error.FileSystem, - error.ProcessNotFound, - => |e| return e, + error.ReadFailed => return file_reader.err.?, + else => |e| return e, }; } @@ -867,8 +862,8 @@ fn glibcVerFromSoFile(file_reader: *Io.File.Reader) !std.SemanticVersion { try file_reader.seekTo(shstr.sh_offset); try file_reader.interface.readSliceAll(shstrtab); const dynstr: struct { offset: u64, size: u64 } = find_dyn_str: { - var it = header.iterateSectionHeaders(&file_reader.interface); - while (it.next()) |shdr| { + var it = header.iterateSectionHeaders(file_reader); + while (try it.next()) |shdr| { const end = mem.findScalarPos(u8, shstrtab, shdr.sh_name, 0) orelse continue; const sh_name = shstrtab[shdr.sh_name..end :0]; if (mem.eql(u8, sh_name, ".dynstr")) break :find_dyn_str .{ @@ -882,19 +877,25 @@ fn glibcVerFromSoFile(file_reader: *Io.File.Reader) !std.SemanticVersion { // strings that start with "GLIBC_2." indicate the existence of such a glibc version, // and furthermore, that the system-installed glibc is at minimum that version. var max_ver: std.SemanticVersion = .{ .major = 2, .minor = 2, .patch = 5 }; - + var offset: u64 = 0; try file_reader.seekTo(dynstr.offset); - while (file_reader.interface.takeSentinel(0)) |s| { - if (mem.startsWith(u8, s, "GLIBC_2.")) { - const chopped = s["GLIBC_".len..]; - const ver = Target.Query.parseVersion(chopped) catch |err| switch (err) { - error.Overflow => return error.InvalidGnuLibCVersion, - error.InvalidVersion => return error.InvalidGnuLibCVersion, - }; - switch (ver.order(max_ver)) { - .gt => max_ver = ver, - .lt, .eq => continue, + while (offset < dynstr.size) { + if (file_reader.interface.takeSentinel(0)) |s| { + if (mem.startsWith(u8, s, "GLIBC_2.")) { + const chopped = s["GLIBC_".len..]; + const ver = Target.Query.parseVersion(chopped) catch |err| switch (err) { + error.Overflow => return error.InvalidGnuLibCVersion, + error.InvalidVersion => return error.InvalidGnuLibCVersion, + }; + switch (ver.order(max_ver)) { + .gt => max_ver = ver, + .lt, .eq => continue, + } } + offset += s.len + 1; + } else |err| switch (err) { + error.EndOfStream, error.StreamTooLong => break, + error.ReadFailed => |e| return e, } } @@ -1091,22 +1092,12 @@ fn detectAbiAndDynamicLinker(io: Io, cpu: Target.Cpu, os: Target.Os, query: Targ error.ProcessFdQuotaExceeded, error.SystemFdQuotaExceeded, error.ProcessNotFound, + error.Canceled, => |e| return e, error.ReadFailed => return file_reader.err.?, - error.UnableToReadElfFile, - error.InvalidElfClass, - error.InvalidElfVersion, - error.InvalidElfEndian, - error.InvalidElfFile, - error.InvalidElfMagic, - error.Unexpected, - error.EndOfStream, - error.NameTooLong, - error.StaticElfFile, - // Finally, we fall back on the standard path. - => |e| { + else => |e| { std.log.warn("encountered {t}; falling back to default ABI and dynamic linker", .{e}); return defaultAbiAndDynamicLinker(cpu, os, query); }, diff --git a/test/src/Cases.zig b/test/src/Cases.zig index 7edb66aede..9f134075b4 100644 --- a/test/src/Cases.zig +++ b/test/src/Cases.zig @@ -455,8 +455,7 @@ pub fn lowerToBuildSteps( parent_step: *std.Build.Step, options: CaseTestOptions, ) void { - const host = std.zig.system.resolveTargetQuery(.{}) catch |err| - std.debug.panic("unable to detect native host: {s}\n", .{@errorName(err)}); + const host = b.resolveTargetQuery(.{}); const cases_dir_path = b.build_root.join(b.allocator, &.{ "test", "cases" }) catch @panic("OOM"); for (self.cases.items) |case| { @@ -587,7 +586,7 @@ pub fn lowerToBuildSteps( }, .Execution => |expected_stdout| no_exec: { const run = if (case.target.result.ofmt == .c) run_step: { - if (getExternalExecutor(&host, &case.target.result, .{ .link_libc = true }) != .native) { + if (getExternalExecutor(&host.result, &case.target.result, .{ .link_libc = true }) != .native) { // We wouldn't be able to run the compiled C code. break :no_exec; } @@ -972,14 +971,6 @@ const TestManifest = struct { } }; -fn resolveTargetQuery(query: std.Target.Query) std.Build.ResolvedTarget { - return .{ - .query = query, - .target = std.zig.system.resolveTargetQuery(query) catch - @panic("unable to resolve target query"), - }; -} - fn knownFileExtension(filename: []const u8) bool { // List taken from `Compilation.classifyFileExt` in the compiler. for ([_][]const u8{