From ba0e3be5cfa2f60f2f9d2a4eb319408f972796c2 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 10 Mar 2020 15:27:45 -0400 Subject: [PATCH 01/15] (breaking) rework stream abstractions The main goal here is to make the function pointers comptime, so that we don't have to do the crazy stuff with async function frames. Since InStream, OutStream, and SeekableStream are already generic across error sets, it's not really worse to make them generic across the vtable as well. See #764 for the open issue acknowledging that using generics for these abstractions is a design flaw. See #130 for the efforts to make these abstractions non-generic. This commit also changes the OutStream API so that `write` returns number of bytes written, and `writeAll` is the one that loops until the whole buffer is written. --- lib/std/buffer.zig | 11 ++ lib/std/child_process.zig | 10 +- lib/std/debug.zig | 223 ++++++++++++++------------ lib/std/dwarf.zig | 161 ++++++++++--------- lib/std/elf.zig | 48 ++++++ lib/std/fs.zig | 2 +- lib/std/fs/file.zig | 94 +++-------- lib/std/io.zig | 219 ++++++------------------- lib/std/io/buffered_atomic_file.zig | 50 ++++++ lib/std/io/buffered_out_stream.zig | 56 +++++++ lib/std/io/c_out_stream.zig | 43 ----- lib/std/io/counting_out_stream.zig | 42 +++++ lib/std/io/fixed_buffer_stream.zig | 66 ++++++++ lib/std/io/in_stream.zig | 83 ++++------ lib/std/io/out_stream.zig | 75 ++++----- lib/std/io/seekable_stream.zig | 105 +++--------- lib/std/json/write_stream.zig | 32 ++-- lib/std/os.zig | 2 +- lib/std/pdb.zig | 10 +- lib/std/zig/ast.zig | 2 +- lib/std/zig/render.zig | 139 ++++++++-------- src-self-hosted/libc_installation.zig | 4 +- src-self-hosted/print_targets.zig | 13 +- src-self-hosted/stage2.zig | 30 ++-- 24 files changed, 748 insertions(+), 772 deletions(-) create mode 100644 lib/std/io/buffered_atomic_file.zig create mode 100644 lib/std/io/buffered_out_stream.zig delete mode 100644 lib/std/io/c_out_stream.zig create mode 100644 lib/std/io/counting_out_stream.zig create mode 100644 lib/std/io/fixed_buffer_stream.zig diff --git a/lib/std/buffer.zig b/lib/std/buffer.zig index 9bf024191e..a33670b6d3 100644 --- a/lib/std/buffer.zig +++ b/lib/std/buffer.zig @@ -157,6 +157,17 @@ pub const Buffer = struct { pub fn print(self: *Buffer, comptime fmt: []const u8, args: var) !void { return std.fmt.format(self, error{OutOfMemory}, Buffer.append, fmt, args); } + + pub fn outStream(self: *Buffer) std.io.OutStream(*Buffer, error{OutOfMemory}, appendWrite) { + return .{ .context = self }; + } + + /// Same as `append` except it returns the number of bytes written, which is always the same + /// as `m.len`. The purpose of this function existing is to match `std.io.OutStream` API. + pub fn appendWrite(self: *Buffer, m: []const u8) !usize { + try self.append(m); + return m.len; + } }; test "simple Buffer" { diff --git a/lib/std/child_process.zig b/lib/std/child_process.zig index 4f5fc2b496..95619384b1 100644 --- a/lib/std/child_process.zig +++ b/lib/std/child_process.zig @@ -221,9 +221,9 @@ pub const ChildProcess = struct { var stderr_file_in_stream = child.stderr.?.inStream(); // TODO need to poll to read these streams to prevent a deadlock (or rely on evented I/O). - const stdout = try stdout_file_in_stream.stream.readAllAlloc(args.allocator, args.max_output_bytes); + const stdout = try stdout_file_in_stream.readAllAlloc(args.allocator, args.max_output_bytes); errdefer args.allocator.free(stdout); - const stderr = try stderr_file_in_stream.stream.readAllAlloc(args.allocator, args.max_output_bytes); + const stderr = try stderr_file_in_stream.readAllAlloc(args.allocator, args.max_output_bytes); errdefer args.allocator.free(stderr); return ExecResult{ @@ -857,8 +857,7 @@ fn writeIntFd(fd: i32, value: ErrInt) !void { .io_mode = .blocking, .async_block_allowed = File.async_block_allowed_yes, }; - const stream = &file.outStream().stream; - stream.writeIntNative(u64, @intCast(u64, value)) catch return error.SystemResources; + file.outStream().writeIntNative(u64, @intCast(u64, value)) catch return error.SystemResources; } fn readIntFd(fd: i32) !ErrInt { @@ -867,8 +866,7 @@ fn readIntFd(fd: i32) !ErrInt { .io_mode = .blocking, .async_block_allowed = File.async_block_allowed_yes, }; - const stream = &file.inStream().stream; - return @intCast(ErrInt, stream.readIntNative(u64) catch return error.SystemResources); + return @intCast(ErrInt, file.inStream().readIntNative(u64) catch return error.SystemResources); } /// Caller must free result. diff --git a/lib/std/debug.zig b/lib/std/debug.zig index 558b7e0513..5c56490519 100644 --- a/lib/std/debug.zig +++ b/lib/std/debug.zig @@ -55,7 +55,7 @@ pub const LineInfo = struct { var stderr_file: File = undefined; var stderr_file_out_stream: File.OutStream = undefined; -var stderr_stream: ?*io.OutStream(File.WriteError) = null; +var stderr_stream: ?*File.OutStream = null; var stderr_mutex = std.Mutex.init(); pub fn warn(comptime fmt: []const u8, args: var) void { @@ -65,13 +65,13 @@ pub fn warn(comptime fmt: []const u8, args: var) void { noasync stderr.print(fmt, args) catch return; } -pub fn getStderrStream() *io.OutStream(File.WriteError) { +pub fn getStderrStream() *File.OutStream { if (stderr_stream) |st| { return st; } else { stderr_file = io.getStdErr(); stderr_file_out_stream = stderr_file.outStream(); - const st = &stderr_file_out_stream.stream; + const st = &stderr_file_out_stream; stderr_stream = st; return st; } @@ -408,15 +408,15 @@ pub const TTY = struct { windows_api, fn setColor(conf: Config, out_stream: var, color: Color) void { - switch (conf) { + noasync switch (conf) { .no_color => return, .escape_codes => switch (color) { - .Red => noasync out_stream.write(RED) catch return, - .Green => noasync out_stream.write(GREEN) catch return, - .Cyan => noasync out_stream.write(CYAN) catch return, - .White, .Bold => noasync out_stream.write(WHITE) catch return, - .Dim => noasync out_stream.write(DIM) catch return, - .Reset => noasync out_stream.write(RESET) catch return, + .Red => out_stream.writeAll(RED) catch return, + .Green => out_stream.writeAll(GREEN) catch return, + .Cyan => out_stream.writeAll(CYAN) catch return, + .White, .Bold => out_stream.writeAll(WHITE) catch return, + .Dim => out_stream.writeAll(DIM) catch return, + .Reset => out_stream.writeAll(RESET) catch return, }, .windows_api => if (builtin.os.tag == .windows) { const S = struct { @@ -455,7 +455,7 @@ pub const TTY = struct { } else { unreachable; }, - } + }; } }; }; @@ -565,38 +565,40 @@ fn printLineInfo( tty_config: TTY.Config, comptime printLineFromFile: var, ) !void { - tty_config.setColor(out_stream, .White); + noasync { + tty_config.setColor(out_stream, .White); - if (line_info) |*li| { - try noasync out_stream.print("{}:{}:{}", .{ li.file_name, li.line, li.column }); - } else { - try noasync out_stream.write("???:?:?"); - } + if (line_info) |*li| { + try out_stream.print("{}:{}:{}", .{ li.file_name, li.line, li.column }); + } else { + try out_stream.writeAll("???:?:?"); + } - tty_config.setColor(out_stream, .Reset); - try noasync out_stream.write(": "); - tty_config.setColor(out_stream, .Dim); - try noasync out_stream.print("0x{x} in {} ({})", .{ address, symbol_name, compile_unit_name }); - tty_config.setColor(out_stream, .Reset); - try noasync out_stream.write("\n"); + tty_config.setColor(out_stream, .Reset); + try out_stream.writeAll(": "); + tty_config.setColor(out_stream, .Dim); + try out_stream.print("0x{x} in {} ({})", .{ address, symbol_name, compile_unit_name }); + tty_config.setColor(out_stream, .Reset); + try out_stream.writeAll("\n"); - // Show the matching source code line if possible - if (line_info) |li| { - if (noasync printLineFromFile(out_stream, li)) { - if (li.column > 0) { - // The caret already takes one char - const space_needed = @intCast(usize, li.column - 1); + // Show the matching source code line if possible + if (line_info) |li| { + if (printLineFromFile(out_stream, li)) { + if (li.column > 0) { + // The caret already takes one char + const space_needed = @intCast(usize, li.column - 1); - try noasync out_stream.writeByteNTimes(' ', space_needed); - tty_config.setColor(out_stream, .Green); - try noasync out_stream.write("^"); - tty_config.setColor(out_stream, .Reset); + try out_stream.writeByteNTimes(' ', space_needed); + tty_config.setColor(out_stream, .Green); + try out_stream.writeAll("^"); + tty_config.setColor(out_stream, .Reset); + } + try out_stream.writeAll("\n"); + } else |err| switch (err) { + error.EndOfFile, error.FileNotFound => {}, + error.BadPathName => {}, + else => return err, } - try noasync out_stream.write("\n"); - } else |err| switch (err) { - error.EndOfFile, error.FileNotFound => {}, - error.BadPathName => {}, - else => return err, } } } @@ -609,21 +611,21 @@ pub const OpenSelfDebugInfoError = error{ }; /// TODO resources https://github.com/ziglang/zig/issues/4353 -/// TODO once https://github.com/ziglang/zig/issues/3157 is fully implemented, -/// make this `noasync fn` and remove the individual noasync calls. pub fn openSelfDebugInfo(allocator: *mem.Allocator) anyerror!DebugInfo { - if (builtin.strip_debug_info) - return error.MissingDebugInfo; - if (@hasDecl(root, "os") and @hasDecl(root.os, "debug") and @hasDecl(root.os.debug, "openSelfDebugInfo")) { - return noasync root.os.debug.openSelfDebugInfo(allocator); - } - switch (builtin.os.tag) { - .linux, - .freebsd, - .macosx, - .windows, - => return DebugInfo.init(allocator), - else => @compileError("openSelfDebugInfo unsupported for this platform"), + noasync { + if (builtin.strip_debug_info) + return error.MissingDebugInfo; + if (@hasDecl(root, "os") and @hasDecl(root.os, "debug") and @hasDecl(root.os.debug, "openSelfDebugInfo")) { + return root.os.debug.openSelfDebugInfo(allocator); + } + switch (builtin.os.tag) { + .linux, + .freebsd, + .macosx, + .windows, + => return DebugInfo.init(allocator), + else => @compileError("openSelfDebugInfo unsupported for this platform"), + } } } @@ -808,45 +810,64 @@ fn chopSlice(ptr: []const u8, offset: u64, size: u64) ![]const u8 { /// TODO resources https://github.com/ziglang/zig/issues/4353 pub fn openElfDebugInfo(allocator: *mem.Allocator, elf_file_path: []const u8) !ModuleDebugInfo { - const mapped_mem = try mapWholeFile(elf_file_path); + noasync { + const mapped_mem = try mapWholeFile(elf_file_path); + const hdr = @ptrCast(*const elf.Ehdr, &mapped_mem[0]); + if (!mem.eql(u8, hdr.e_ident[0..4], "\x7fELF")) return error.InvalidElfMagic; + if (hdr.e_ident[elf.EI_VERSION] != 1) return error.InvalidElfVersion; - var seekable_stream = io.SliceSeekableInStream.init(mapped_mem); - var efile = try noasync elf.Elf.openStream( - allocator, - @ptrCast(*DW.DwarfSeekableStream, &seekable_stream.seekable_stream), - @ptrCast(*DW.DwarfInStream, &seekable_stream.stream), - ); - defer noasync efile.close(); + const endian: builtin.Endian = switch (hdr.e_ident[elf.EI_DATA]) { + elf.ELFDATA2LSB => .Little, + elf.ELFDATA2MSB => .Big, + else => return error.InvalidElfEndian, + }; + assert(endian == std.builtin.endian); // this is our own debug info - const debug_info = (try noasync efile.findSection(".debug_info")) orelse - return error.MissingDebugInfo; - const debug_abbrev = (try noasync efile.findSection(".debug_abbrev")) orelse - return error.MissingDebugInfo; - const debug_str = (try noasync efile.findSection(".debug_str")) orelse - return error.MissingDebugInfo; - const debug_line = (try noasync efile.findSection(".debug_line")) orelse - return error.MissingDebugInfo; - const opt_debug_ranges = try noasync efile.findSection(".debug_ranges"); + const shoff = hdr.e_shoff; + const str_section_off = shoff + @as(u64, hdr.e_shentsize) * @as(u64, hdr.e_shstrndx); + const header_strings = mapped_mem[str_section_off..str_section_off + hdr.e_shentsize]; + const shdrs = @ptrCast([*]const elf.Shdr, @alignCast(@alignOf(elf.Shdr), &mapped_mem[shoff]))[0..hdr.e_shnum]; - var di = DW.DwarfInfo{ - .endian = efile.endian, - .debug_info = try chopSlice(mapped_mem, debug_info.sh_offset, debug_info.sh_size), - .debug_abbrev = try chopSlice(mapped_mem, debug_abbrev.sh_offset, debug_abbrev.sh_size), - .debug_str = try chopSlice(mapped_mem, debug_str.sh_offset, debug_str.sh_size), - .debug_line = try chopSlice(mapped_mem, debug_line.sh_offset, debug_line.sh_size), - .debug_ranges = if (opt_debug_ranges) |debug_ranges| - try chopSlice(mapped_mem, debug_ranges.sh_offset, debug_ranges.sh_size) - else - null, - }; + var opt_debug_info: ?[]const u8 = null; + var opt_debug_abbrev: ?[]const u8 = null; + var opt_debug_str: ?[]const u8 = null; + var opt_debug_line: ?[]const u8 = null; + var opt_debug_ranges: ?[]const u8 = null; - try noasync DW.openDwarfDebugInfo(&di, allocator); + for (shdrs) |*shdr| { + if (shdr.sh_type == elf.SHT_NULL) continue; - return ModuleDebugInfo{ - .base_address = undefined, - .dwarf = di, - .mapped_memory = mapped_mem, - }; + const name = std.mem.span(@ptrCast([*:0]const u8, header_strings[shdr.sh_name..].ptr)); + if (mem.eql(u8, name, ".debug_info")) { + opt_debug_info = try chopSlice(mapped_mem, shdr.sh_offset, shdr.sh_size); + } else if (mem.eql(u8, name, ".debug_abbrev")) { + opt_debug_abbrev = try chopSlice(mapped_mem, shdr.sh_offset, shdr.sh_size); + } else if (mem.eql(u8, name, ".debug_str")) { + opt_debug_str = try chopSlice(mapped_mem, shdr.sh_offset, shdr.sh_size); + } else if (mem.eql(u8, name, ".debug_line")) { + opt_debug_line = try chopSlice(mapped_mem, shdr.sh_offset, shdr.sh_size); + } else if (mem.eql(u8, name, ".debug_ranges")) { + opt_debug_ranges = try chopSlice(mapped_mem, shdr.sh_offset, shdr.sh_size); + } + } + + var di = DW.DwarfInfo{ + .endian = endian, + .debug_info = opt_debug_info orelse return error.MissingDebugInfo, + .debug_abbrev = opt_debug_abbrev orelse return error.MissingDebugInfo, + .debug_str = opt_debug_str orelse return error.MissingDebugInfo, + .debug_line = opt_debug_line orelse return error.MissingDebugInfo, + .debug_ranges = opt_debug_ranges, + }; + + try DW.openDwarfDebugInfo(&di, allocator); + + return ModuleDebugInfo{ + .base_address = undefined, + .dwarf = di, + .mapped_memory = mapped_mem, + }; + } } /// TODO resources https://github.com/ziglang/zig/issues/4353 @@ -982,22 +1003,24 @@ const MachoSymbol = struct { } }; -fn mapWholeFile(path: []const u8) ![]const u8 { - const file = try noasync fs.openFileAbsolute(path, .{ .always_blocking = true }); - defer noasync file.close(); +fn mapWholeFile(path: []const u8) ![]align(mem.page_size) const u8 { + noasync { + const file = try fs.openFileAbsolute(path, .{ .always_blocking = true }); + defer file.close(); - const file_len = try math.cast(usize, try file.getEndPos()); - const mapped_mem = try os.mmap( - null, - file_len, - os.PROT_READ, - os.MAP_SHARED, - file.handle, - 0, - ); - errdefer os.munmap(mapped_mem); + const file_len = try math.cast(usize, try file.getEndPos()); + const mapped_mem = try os.mmap( + null, + file_len, + os.PROT_READ, + os.MAP_SHARED, + file.handle, + 0, + ); + errdefer os.munmap(mapped_mem); - return mapped_mem; + return mapped_mem; + } } pub const DebugInfo = struct { diff --git a/lib/std/dwarf.zig b/lib/std/dwarf.zig index 32a49b68e0..769f349e33 100644 --- a/lib/std/dwarf.zig +++ b/lib/std/dwarf.zig @@ -11,9 +11,6 @@ const ArrayList = std.ArrayList; usingnamespace @import("dwarf_bits.zig"); -pub const DwarfSeekableStream = io.SeekableStream(anyerror, anyerror); -pub const DwarfInStream = io.InStream(anyerror); - const PcRange = struct { start: u64, end: u64, @@ -239,7 +236,7 @@ const LineNumberProgram = struct { } }; -fn readInitialLength(comptime E: type, in_stream: *io.InStream(E), is_64: *bool) !u64 { +fn readInitialLength(in_stream: var, is_64: *bool) !u64 { const first_32_bits = try in_stream.readIntLittle(u32); is_64.* = (first_32_bits == 0xffffffff); if (is_64.*) { @@ -414,40 +411,42 @@ pub const DwarfInfo = struct { } fn scanAllFunctions(di: *DwarfInfo) !void { - var s = io.SliceSeekableInStream.init(di.debug_info); + var stream = io.fixedBufferStream(di.debug_info); + const in = &stream.inStream(); + const seekable = &stream.seekableStream(); var this_unit_offset: u64 = 0; - while (this_unit_offset < try s.seekable_stream.getEndPos()) { - s.seekable_stream.seekTo(this_unit_offset) catch |err| switch (err) { + while (this_unit_offset < try seekable.getEndPos()) { + seekable.seekTo(this_unit_offset) catch |err| switch (err) { error.EndOfStream => unreachable, else => return err, }; var is_64: bool = undefined; - const unit_length = try readInitialLength(@TypeOf(s.stream.readFn).ReturnType.ErrorSet, &s.stream, &is_64); + const unit_length = try readInitialLength(in, &is_64); if (unit_length == 0) return; const next_offset = unit_length + (if (is_64) @as(usize, 12) else @as(usize, 4)); - const version = try s.stream.readInt(u16, di.endian); + const version = try in.readInt(u16, di.endian); if (version < 2 or version > 5) return error.InvalidDebugInfo; - const debug_abbrev_offset = if (is_64) try s.stream.readInt(u64, di.endian) else try s.stream.readInt(u32, di.endian); + const debug_abbrev_offset = if (is_64) try in.readInt(u64, di.endian) else try in.readInt(u32, di.endian); - const address_size = try s.stream.readByte(); + const address_size = try in.readByte(); if (address_size != @sizeOf(usize)) return error.InvalidDebugInfo; - const compile_unit_pos = try s.seekable_stream.getPos(); + const compile_unit_pos = try seekable.getPos(); const abbrev_table = try di.getAbbrevTable(debug_abbrev_offset); - try s.seekable_stream.seekTo(compile_unit_pos); + try seekable.seekTo(compile_unit_pos); const next_unit_pos = this_unit_offset + next_offset; - while ((try s.seekable_stream.getPos()) < next_unit_pos) { - const die_obj = (try di.parseDie(&s.stream, abbrev_table, is_64)) orelse continue; + while ((try seekable.getPos()) < next_unit_pos) { + const die_obj = (try di.parseDie(in, abbrev_table, is_64)) orelse continue; defer die_obj.attrs.deinit(); - const after_die_offset = try s.seekable_stream.getPos(); + const after_die_offset = try seekable.getPos(); switch (die_obj.tag_id) { TAG_subprogram, TAG_inlined_subroutine, TAG_subroutine, TAG_entry_point => { @@ -463,14 +462,14 @@ pub const DwarfInfo = struct { // Follow the DIE it points to and repeat const ref_offset = try this_die_obj.getAttrRef(AT_abstract_origin); if (ref_offset > next_offset) return error.InvalidDebugInfo; - try s.seekable_stream.seekTo(this_unit_offset + ref_offset); - this_die_obj = (try di.parseDie(&s.stream, abbrev_table, is_64)) orelse return error.InvalidDebugInfo; + try seekable.seekTo(this_unit_offset + ref_offset); + this_die_obj = (try di.parseDie(in, abbrev_table, is_64)) orelse return error.InvalidDebugInfo; } else if (this_die_obj.getAttr(AT_specification)) |ref| { // Follow the DIE it points to and repeat const ref_offset = try this_die_obj.getAttrRef(AT_specification); if (ref_offset > next_offset) return error.InvalidDebugInfo; - try s.seekable_stream.seekTo(this_unit_offset + ref_offset); - this_die_obj = (try di.parseDie(&s.stream, abbrev_table, is_64)) orelse return error.InvalidDebugInfo; + try seekable.seekTo(this_unit_offset + ref_offset); + this_die_obj = (try di.parseDie(in, abbrev_table, is_64)) orelse return error.InvalidDebugInfo; } else { break :x null; } @@ -511,7 +510,7 @@ pub const DwarfInfo = struct { else => {}, } - try s.seekable_stream.seekTo(after_die_offset); + try seekable.seekTo(after_die_offset); } this_unit_offset += next_offset; @@ -519,35 +518,37 @@ pub const DwarfInfo = struct { } fn scanAllCompileUnits(di: *DwarfInfo) !void { - var s = io.SliceSeekableInStream.init(di.debug_info); + var stream = io.fixedBufferStream(di.debug_info); + const in = &stream.inStream(); + const seekable = &stream.seekableStream(); var this_unit_offset: u64 = 0; - while (this_unit_offset < try s.seekable_stream.getEndPos()) { - s.seekable_stream.seekTo(this_unit_offset) catch |err| switch (err) { + while (this_unit_offset < try seekable.getEndPos()) { + seekable.seekTo(this_unit_offset) catch |err| switch (err) { error.EndOfStream => unreachable, else => return err, }; var is_64: bool = undefined; - const unit_length = try readInitialLength(@TypeOf(s.stream.readFn).ReturnType.ErrorSet, &s.stream, &is_64); + const unit_length = try readInitialLength(in, &is_64); if (unit_length == 0) return; const next_offset = unit_length + (if (is_64) @as(usize, 12) else @as(usize, 4)); - const version = try s.stream.readInt(u16, di.endian); + const version = try in.readInt(u16, di.endian); if (version < 2 or version > 5) return error.InvalidDebugInfo; - const debug_abbrev_offset = if (is_64) try s.stream.readInt(u64, di.endian) else try s.stream.readInt(u32, di.endian); + const debug_abbrev_offset = if (is_64) try in.readInt(u64, di.endian) else try in.readInt(u32, di.endian); - const address_size = try s.stream.readByte(); + const address_size = try in.readByte(); if (address_size != @sizeOf(usize)) return error.InvalidDebugInfo; - const compile_unit_pos = try s.seekable_stream.getPos(); + const compile_unit_pos = try seekable.getPos(); const abbrev_table = try di.getAbbrevTable(debug_abbrev_offset); - try s.seekable_stream.seekTo(compile_unit_pos); + try seekable.seekTo(compile_unit_pos); const compile_unit_die = try di.allocator().create(Die); - compile_unit_die.* = (try di.parseDie(&s.stream, abbrev_table, is_64)) orelse return error.InvalidDebugInfo; + compile_unit_die.* = (try di.parseDie(in, abbrev_table, is_64)) orelse return error.InvalidDebugInfo; if (compile_unit_die.tag_id != TAG_compile_unit) return error.InvalidDebugInfo; @@ -593,7 +594,9 @@ pub const DwarfInfo = struct { } if (di.debug_ranges) |debug_ranges| { if (compile_unit.die.getAttrSecOffset(AT_ranges)) |ranges_offset| { - var s = io.SliceSeekableInStream.init(debug_ranges); + var stream = io.fixedBufferStream(debug_ranges); + const in = &stream.inStream(); + const seekable = &stream.seekableStream(); // All the addresses in the list are relative to the value // specified by DW_AT_low_pc or to some other value encoded @@ -604,11 +607,11 @@ pub const DwarfInfo = struct { else => return err, }; - try s.seekable_stream.seekTo(ranges_offset); + try seekable.seekTo(ranges_offset); while (true) { - const begin_addr = try s.stream.readIntLittle(usize); - const end_addr = try s.stream.readIntLittle(usize); + const begin_addr = try in.readIntLittle(usize); + const end_addr = try in.readIntLittle(usize); if (begin_addr == 0 and end_addr == 0) { break; } @@ -646,25 +649,27 @@ pub const DwarfInfo = struct { } fn parseAbbrevTable(di: *DwarfInfo, offset: u64) !AbbrevTable { - var s = io.SliceSeekableInStream.init(di.debug_abbrev); + var stream = io.fixedBufferStream(di.debug_abbrev); + const in = &stream.inStream(); + const seekable = &stream.seekableStream(); - try s.seekable_stream.seekTo(offset); + try seekable.seekTo(offset); var result = AbbrevTable.init(di.allocator()); errdefer result.deinit(); while (true) { - const abbrev_code = try leb.readULEB128(u64, &s.stream); + const abbrev_code = try leb.readULEB128(u64, in); if (abbrev_code == 0) return result; try result.append(AbbrevTableEntry{ .abbrev_code = abbrev_code, - .tag_id = try leb.readULEB128(u64, &s.stream), - .has_children = (try s.stream.readByte()) == CHILDREN_yes, + .tag_id = try leb.readULEB128(u64, in), + .has_children = (try in.readByte()) == CHILDREN_yes, .attrs = ArrayList(AbbrevAttr).init(di.allocator()), }); const attrs = &result.items[result.len - 1].attrs; while (true) { - const attr_id = try leb.readULEB128(u64, &s.stream); - const form_id = try leb.readULEB128(u64, &s.stream); + const attr_id = try leb.readULEB128(u64, in); + const form_id = try leb.readULEB128(u64, in); if (attr_id == 0 and form_id == 0) break; try attrs.append(AbbrevAttr{ .attr_id = attr_id, @@ -695,42 +700,44 @@ pub const DwarfInfo = struct { } fn getLineNumberInfo(di: *DwarfInfo, compile_unit: CompileUnit, target_address: usize) !debug.LineInfo { - var s = io.SliceSeekableInStream.init(di.debug_line); + var stream = io.fixedBufferStream(di.debug_line); + const in = &stream.inStream(); + const seekable = &stream.seekableStream(); const compile_unit_cwd = try compile_unit.die.getAttrString(di, AT_comp_dir); const line_info_offset = try compile_unit.die.getAttrSecOffset(AT_stmt_list); - try s.seekable_stream.seekTo(line_info_offset); + try seekable.seekTo(line_info_offset); var is_64: bool = undefined; - const unit_length = try readInitialLength(@TypeOf(s.stream.readFn).ReturnType.ErrorSet, &s.stream, &is_64); + const unit_length = try readInitialLength(in, &is_64); if (unit_length == 0) { return error.MissingDebugInfo; } const next_offset = unit_length + (if (is_64) @as(usize, 12) else @as(usize, 4)); - const version = try s.stream.readInt(u16, di.endian); + const version = try in.readInt(u16, di.endian); // TODO support 3 and 5 if (version != 2 and version != 4) return error.InvalidDebugInfo; - const prologue_length = if (is_64) try s.stream.readInt(u64, di.endian) else try s.stream.readInt(u32, di.endian); - const prog_start_offset = (try s.seekable_stream.getPos()) + prologue_length; + const prologue_length = if (is_64) try in.readInt(u64, di.endian) else try in.readInt(u32, di.endian); + const prog_start_offset = (try seekable.getPos()) + prologue_length; - const minimum_instruction_length = try s.stream.readByte(); + const minimum_instruction_length = try in.readByte(); if (minimum_instruction_length == 0) return error.InvalidDebugInfo; if (version >= 4) { // maximum_operations_per_instruction - _ = try s.stream.readByte(); + _ = try in.readByte(); } - const default_is_stmt = (try s.stream.readByte()) != 0; - const line_base = try s.stream.readByteSigned(); + const default_is_stmt = (try in.readByte()) != 0; + const line_base = try in.readByteSigned(); - const line_range = try s.stream.readByte(); + const line_range = try in.readByte(); if (line_range == 0) return error.InvalidDebugInfo; - const opcode_base = try s.stream.readByte(); + const opcode_base = try in.readByte(); const standard_opcode_lengths = try di.allocator().alloc(u8, opcode_base - 1); defer di.allocator().free(standard_opcode_lengths); @@ -738,14 +745,14 @@ pub const DwarfInfo = struct { { var i: usize = 0; while (i < opcode_base - 1) : (i += 1) { - standard_opcode_lengths[i] = try s.stream.readByte(); + standard_opcode_lengths[i] = try in.readByte(); } } var include_directories = ArrayList([]const u8).init(di.allocator()); try include_directories.append(compile_unit_cwd); while (true) { - const dir = try s.stream.readUntilDelimiterAlloc(di.allocator(), 0, math.maxInt(usize)); + const dir = try in.readUntilDelimiterAlloc(di.allocator(), 0, math.maxInt(usize)); if (dir.len == 0) break; try include_directories.append(dir); } @@ -754,11 +761,11 @@ pub const DwarfInfo = struct { var prog = LineNumberProgram.init(default_is_stmt, include_directories.toSliceConst(), &file_entries, target_address); while (true) { - const file_name = try s.stream.readUntilDelimiterAlloc(di.allocator(), 0, math.maxInt(usize)); + const file_name = try in.readUntilDelimiterAlloc(di.allocator(), 0, math.maxInt(usize)); if (file_name.len == 0) break; - const dir_index = try leb.readULEB128(usize, &s.stream); - const mtime = try leb.readULEB128(usize, &s.stream); - const len_bytes = try leb.readULEB128(usize, &s.stream); + const dir_index = try leb.readULEB128(usize, in); + const mtime = try leb.readULEB128(usize, in); + const len_bytes = try leb.readULEB128(usize, in); try file_entries.append(FileEntry{ .file_name = file_name, .dir_index = dir_index, @@ -767,17 +774,17 @@ pub const DwarfInfo = struct { }); } - try s.seekable_stream.seekTo(prog_start_offset); + try seekable.seekTo(prog_start_offset); const next_unit_pos = line_info_offset + next_offset; - while ((try s.seekable_stream.getPos()) < next_unit_pos) { - const opcode = try s.stream.readByte(); + while ((try seekable.getPos()) < next_unit_pos) { + const opcode = try in.readByte(); if (opcode == LNS_extended_op) { - const op_size = try leb.readULEB128(u64, &s.stream); + const op_size = try leb.readULEB128(u64, in); if (op_size < 1) return error.InvalidDebugInfo; - var sub_op = try s.stream.readByte(); + var sub_op = try in.readByte(); switch (sub_op) { LNE_end_sequence => { prog.end_sequence = true; @@ -785,14 +792,14 @@ pub const DwarfInfo = struct { prog.reset(); }, LNE_set_address => { - const addr = try s.stream.readInt(usize, di.endian); + const addr = try in.readInt(usize, di.endian); prog.address = addr; }, LNE_define_file => { - const file_name = try s.stream.readUntilDelimiterAlloc(di.allocator(), 0, math.maxInt(usize)); - const dir_index = try leb.readULEB128(usize, &s.stream); - const mtime = try leb.readULEB128(usize, &s.stream); - const len_bytes = try leb.readULEB128(usize, &s.stream); + const file_name = try in.readUntilDelimiterAlloc(di.allocator(), 0, math.maxInt(usize)); + const dir_index = try leb.readULEB128(usize, in); + const mtime = try leb.readULEB128(usize, in); + const len_bytes = try leb.readULEB128(usize, in); try file_entries.append(FileEntry{ .file_name = file_name, .dir_index = dir_index, @@ -802,7 +809,7 @@ pub const DwarfInfo = struct { }, else => { const fwd_amt = math.cast(isize, op_size - 1) catch return error.InvalidDebugInfo; - try s.seekable_stream.seekBy(fwd_amt); + try seekable.seekBy(fwd_amt); }, } } else if (opcode >= opcode_base) { @@ -821,19 +828,19 @@ pub const DwarfInfo = struct { prog.basic_block = false; }, LNS_advance_pc => { - const arg = try leb.readULEB128(usize, &s.stream); + const arg = try leb.readULEB128(usize, in); prog.address += arg * minimum_instruction_length; }, LNS_advance_line => { - const arg = try leb.readILEB128(i64, &s.stream); + const arg = try leb.readILEB128(i64, in); prog.line += arg; }, LNS_set_file => { - const arg = try leb.readULEB128(usize, &s.stream); + const arg = try leb.readULEB128(usize, in); prog.file = arg; }, LNS_set_column => { - const arg = try leb.readULEB128(u64, &s.stream); + const arg = try leb.readULEB128(u64, in); prog.column = arg; }, LNS_negate_stmt => { @@ -847,14 +854,14 @@ pub const DwarfInfo = struct { prog.address += inc_addr; }, LNS_fixed_advance_pc => { - const arg = try s.stream.readInt(u16, di.endian); + const arg = try in.readInt(u16, di.endian); prog.address += arg; }, LNS_set_prologue_end => {}, else => { if (opcode - 1 >= standard_opcode_lengths.len) return error.InvalidDebugInfo; const len_bytes = standard_opcode_lengths[opcode - 1]; - try s.seekable_stream.seekBy(len_bytes); + try seekable.seekBy(len_bytes); }, } } diff --git a/lib/std/elf.zig b/lib/std/elf.zig index fc57db7c98..efaa4d6f06 100644 --- a/lib/std/elf.zig +++ b/lib/std/elf.zig @@ -333,6 +333,54 @@ pub const ET = extern enum(u16) { pub const SectionHeader = Elf64_Shdr; pub const ProgramHeader = Elf64_Phdr; +const Header = struct { + endian: builtin.Endian, + is_64: bool, + entry: u64, + phoff: u64, + shoff: u64, + phentsize: u16, + phnum: u16, + shentsize: u16, + shnum: u16, + shstrndx: u16, +}; + +pub fn readHeader(in_stream: var) !Header { + var hdr_buf: [@sizeOf(Elf64_Ehdr)]u8 align(@alignOf(Elf64_Ehdr)) = undefined; + try in_stream.readAll(&hdr_buf); + const hdr32 = @ptrCast(*elf.Elf32_Ehdr, &hdr_buf); + const hdr64 = @ptrCast(*elf.Elf64_Ehdr, &hdr_buf); + if (!mem.eql(u8, hdr32.e_ident[0..4], "\x7fELF")) return error.InvalidElfMagic; + if (hdr32.e_ident[EI_VERSION] != 1) return error.InvalidElfVersion; + + const endian = switch (hdr32.e_ident[elf.EI_DATA]) { + ELFDATA2LSB => .Little, + ELFDATA2MSB => .Big, + else => return error.InvalidElfEndian, + }; + const need_bswap = endian != std.builtin.endian; + + const is_64 = switch (hdr32.e_ident[EI_CLASS]) { + ELFCLASS32 => false, + ELFCLASS64 => true, + else => return error.InvalidElfClass, + }; + + return @as(Header, .{ + .endian = endian, + .is_64 = is_64, + .entry = int(is_64, need_bswap, hdr32.e_entry, hdr64.e_entry), + .phoff = int(is_64, need_bswap, hdr32.e_phoff, hdr64.e_phoff), + .shoff = int(is_64, need_bswap, hdr32.e_shoff, hdr64.e_shoff), + .phentsize = int(is_64, need_bswap, hdr32.e_phentsize, hdr64.e_phentsize), + .phnum = int(is_64, need_bswap, hdr32.e_phnum, hdr64.e_phnum), + .shentsize = int(is_64, need_bswap, hdr32.e_shentsize, hdr64.e_shentsize), + .shnum = int(is_64, need_bswap, hdr32.e_shnum, hdr64.e_shnum), + .shstrndx = int(is_64, need_bswap, hdr32.e_shstrndx, hdr64.e_shstrndx), + }); +} + pub const Elf = struct { seekable_stream: *io.SeekableStream(anyerror, anyerror), in_stream: *io.InStream(anyerror), diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 1f20a14f74..892b977291 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -1150,7 +1150,7 @@ pub const Dir = struct { const buf = try allocator.alignedAlloc(u8, A, size); errdefer allocator.free(buf); - try file.inStream().stream.readNoEof(buf); + try file.inStream().readNoEof(buf); return buf; } diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index 8f7a46ad3e..845e4bc75e 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -71,7 +71,7 @@ pub const File = struct { if (need_async_thread and self.io_mode == .blocking and !self.async_block_allowed) { std.event.Loop.instance.?.close(self.handle); } else { - return os.close(self.handle); + os.close(self.handle); } } @@ -496,85 +496,29 @@ pub const File = struct { } } - pub fn inStream(file: File) InStream { - return InStream{ - .file = file, - .stream = InStream.Stream{ .readFn = InStream.readFn }, - }; + pub const InStream = io.InStream(File, ReadError, read); + + pub fn inStream(file: File) io.InStream(File, ReadError, read) { + return .{ .context = file }; } + pub const OutStream = io.OutStream(File, WriteError, write); + pub fn outStream(file: File) OutStream { - return OutStream{ - .file = file, - .stream = OutStream.Stream{ .writeFn = OutStream.writeFn }, - }; + return .{ .context = file }; } + pub const SeekableStream = io.SeekableStream( + File, + SeekError, + GetPosError, + seekTo, + seekBy, + getPos, + getEndPos, + ); + pub fn seekableStream(file: File) SeekableStream { - return SeekableStream{ - .file = file, - .stream = SeekableStream.Stream{ - .seekToFn = SeekableStream.seekToFn, - .seekByFn = SeekableStream.seekByFn, - .getPosFn = SeekableStream.getPosFn, - .getEndPosFn = SeekableStream.getEndPosFn, - }, - }; + return .{ .context = file }; } - - /// Implementation of io.InStream trait for File - pub const InStream = struct { - file: File, - stream: Stream, - - pub const Error = ReadError; - pub const Stream = io.InStream(Error); - - fn readFn(in_stream: *Stream, buffer: []u8) Error!usize { - const self = @fieldParentPtr(InStream, "stream", in_stream); - return self.file.read(buffer); - } - }; - - /// Implementation of io.OutStream trait for File - pub const OutStream = struct { - file: File, - stream: Stream, - - pub const Error = WriteError; - pub const Stream = io.OutStream(Error); - - fn writeFn(out_stream: *Stream, bytes: []const u8) Error!usize { - const self = @fieldParentPtr(OutStream, "stream", out_stream); - return self.file.write(bytes); - } - }; - - /// Implementation of io.SeekableStream trait for File - pub const SeekableStream = struct { - file: File, - stream: Stream, - - pub const Stream = io.SeekableStream(SeekError, GetPosError); - - pub fn seekToFn(seekable_stream: *Stream, pos: u64) SeekError!void { - const self = @fieldParentPtr(SeekableStream, "stream", seekable_stream); - return self.file.seekTo(pos); - } - - pub fn seekByFn(seekable_stream: *Stream, amt: i64) SeekError!void { - const self = @fieldParentPtr(SeekableStream, "stream", seekable_stream); - return self.file.seekBy(amt); - } - - pub fn getEndPosFn(seekable_stream: *Stream) GetPosError!u64 { - const self = @fieldParentPtr(SeekableStream, "stream", seekable_stream); - return self.file.getEndPos(); - } - - pub fn getPosFn(seekable_stream: *Stream) GetPosError!u64 { - const self = @fieldParentPtr(SeekableStream, "stream", seekable_stream); - return self.file.getPos(); - } - }; }; diff --git a/lib/std/io.zig b/lib/std/io.zig index 99e9391f1d..dcb0551dc9 100644 --- a/lib/std/io.zig +++ b/lib/std/io.zig @@ -93,10 +93,46 @@ pub fn getStdIn() File { } pub const SeekableStream = @import("io/seekable_stream.zig").SeekableStream; -pub const SliceSeekableInStream = @import("io/seekable_stream.zig").SliceSeekableInStream; -pub const COutStream = @import("io/c_out_stream.zig").COutStream; pub const InStream = @import("io/in_stream.zig").InStream; pub const OutStream = @import("io/out_stream.zig").OutStream; +pub const FixedBufferInStream = @import("io/fixed_buffer_stream.zig").FixedBufferInStream; +pub const BufferedAtomicFile = @import("io/buffered_atomic_file.zig").BufferedAtomicFile; + +pub const BufferedOutStream = @import("io/buffered_out_stream.zig").BufferedOutStream; +pub const BufferedOutStreamCustom = @import("io/buffered_out_stream.zig").BufferedOutStreamCustom; +pub const bufferedOutStream = @import("io/buffered_out_stream.zig").bufferedOutStream; + +pub const CountingOutStream = @import("io/counting_out_stream.zig").CountingOutStream; + +pub fn fixedBufferStream(bytes: []const u8) FixedBufferInStream { + return (FixedBufferInStream{ .bytes = bytes, .pos = 0 }); +} + +pub fn cOutStream(c_file: *std.c.FILE) COutStream { + return .{ .context = c_file }; +} + +pub const COutStream = OutStream(*std.c.FILE, std.fs.File.WriteError, cOutStreamWrite); + +pub fn cOutStreamWrite(c_file: *std.c.FILE, bytes: []const u8) std.fs.File.WriteError!usize { + const amt_written = std.c.fwrite(bytes.ptr, 1, bytes.len, c_file); + if (amt_written >= 0) return amt_written; + switch (std.c._errno().*) { + 0 => unreachable, + os.EINVAL => unreachable, + os.EFAULT => unreachable, + os.EAGAIN => unreachable, // this is a blocking API + os.EBADF => unreachable, // always a race condition + os.EDESTADDRREQ => unreachable, // connect was never called + os.EDQUOT => return error.DiskQuota, + os.EFBIG => return error.FileTooBig, + os.EIO => return error.InputOutput, + os.ENOSPC => return error.NoSpaceLeft, + os.EPERM => return error.AccessDenied, + os.EPIPE => return error.BrokenPipe, + else => |err| return os.unexpectedErrno(@intCast(usize, err)), + } +} /// Deprecated; use `std.fs.Dir.writeFile`. pub fn writeFile(path: []const u8, data: []const u8) !void { @@ -495,139 +531,18 @@ test "io.SliceOutStream" { testing.expectEqualSlices(u8, "HelloWorld!", slice_stream.getWritten()); } -var null_out_stream_state = NullOutStream.init(); -pub const null_out_stream = &null_out_stream_state.stream; - /// An OutStream that doesn't write to anything. -pub const NullOutStream = struct { - pub const Error = error{}; - pub const Stream = OutStream(Error); +pub const null_out_stream = @as(NullOutStream, .{ .context = {} }); - stream: Stream, - - pub fn init() NullOutStream { - return NullOutStream{ - .stream = Stream{ .writeFn = writeFn }, - }; - } - - fn writeFn(out_stream: *Stream, bytes: []const u8) Error!usize { - return bytes.len; - } -}; - -test "io.NullOutStream" { - var null_stream = NullOutStream.init(); - const stream = &null_stream.stream; - stream.write("yay" ** 10000) catch unreachable; +const NullOutStream = OutStream(void, error{}, dummyWrite); +fn dummyWrite(context: void, data: []const u8) error{}!usize { + return data.len; } -/// An OutStream that counts how many bytes has been written to it. -pub fn CountingOutStream(comptime OutStreamError: type) type { - return struct { - const Self = @This(); - pub const Stream = OutStream(Error); - pub const Error = OutStreamError; - - stream: Stream, - bytes_written: u64, - child_stream: *Stream, - - pub fn init(child_stream: *Stream) Self { - return Self{ - .stream = Stream{ .writeFn = writeFn }, - .bytes_written = 0, - .child_stream = child_stream, - }; - } - - fn writeFn(out_stream: *Stream, bytes: []const u8) OutStreamError!usize { - const self = @fieldParentPtr(Self, "stream", out_stream); - try self.child_stream.write(bytes); - self.bytes_written += bytes.len; - return bytes.len; - } - }; +test "null_out_stream" { + null_out_stream.writeAll("yay" ** 1000) catch |err| switch (err) {}; } -test "io.CountingOutStream" { - var null_stream = NullOutStream.init(); - var counting_stream = CountingOutStream(NullOutStream.Error).init(&null_stream.stream); - const stream = &counting_stream.stream; - - const bytes = "yay" ** 10000; - stream.write(bytes) catch unreachable; - testing.expect(counting_stream.bytes_written == bytes.len); -} - -pub fn BufferedOutStream(comptime Error: type) type { - return BufferedOutStreamCustom(mem.page_size, Error); -} - -pub fn BufferedOutStreamCustom(comptime buffer_size: usize, comptime OutStreamError: type) type { - return struct { - const Self = @This(); - pub const Stream = OutStream(Error); - pub const Error = OutStreamError; - - stream: Stream, - - unbuffered_out_stream: *Stream, - - const FifoType = std.fifo.LinearFifo(u8, std.fifo.LinearFifoBufferType{ .Static = buffer_size }); - fifo: FifoType, - - pub fn init(unbuffered_out_stream: *Stream) Self { - return Self{ - .unbuffered_out_stream = unbuffered_out_stream, - .fifo = FifoType.init(), - .stream = Stream{ .writeFn = writeFn }, - }; - } - - pub fn flush(self: *Self) !void { - while (true) { - const slice = self.fifo.readableSlice(0); - if (slice.len == 0) break; - try self.unbuffered_out_stream.write(slice); - self.fifo.discard(slice.len); - } - } - - fn writeFn(out_stream: *Stream, bytes: []const u8) Error!usize { - const self = @fieldParentPtr(Self, "stream", out_stream); - if (bytes.len >= self.fifo.writableLength()) { - try self.flush(); - return self.unbuffered_out_stream.writeOnce(bytes); - } - self.fifo.writeAssumeCapacity(bytes); - return bytes.len; - } - }; -} - -/// Implementation of OutStream trait for Buffer -pub const BufferOutStream = struct { - buffer: *Buffer, - stream: Stream, - - pub const Error = error{OutOfMemory}; - pub const Stream = OutStream(Error); - - pub fn init(buffer: *Buffer) BufferOutStream { - return BufferOutStream{ - .buffer = buffer, - .stream = Stream{ .writeFn = writeFn }, - }; - } - - fn writeFn(out_stream: *Stream, bytes: []const u8) !usize { - const self = @fieldParentPtr(BufferOutStream, "stream", out_stream); - try self.buffer.append(bytes); - return bytes.len; - } -}; - /// Creates a stream which allows for writing bit fields to another stream pub fn BitOutStream(endian: builtin.Endian, comptime Error: type) type { return struct { @@ -752,52 +667,11 @@ pub fn BitOutStream(endian: builtin.Endian, comptime Error: type) type { return buffer.len; } - return self.out_stream.writeOnce(buffer); + return self.out_stream.write(buffer); } }; } -pub const BufferedAtomicFile = struct { - atomic_file: fs.AtomicFile, - file_stream: File.OutStream, - buffered_stream: BufferedOutStream(File.WriteError), - allocator: *mem.Allocator, - - pub fn create(allocator: *mem.Allocator, dest_path: []const u8) !*BufferedAtomicFile { - // TODO with well defined copy elision we don't need this allocation - var self = try allocator.create(BufferedAtomicFile); - self.* = BufferedAtomicFile{ - .atomic_file = undefined, - .file_stream = undefined, - .buffered_stream = undefined, - .allocator = allocator, - }; - errdefer allocator.destroy(self); - - self.atomic_file = try fs.AtomicFile.init(dest_path, File.default_mode); - errdefer self.atomic_file.deinit(); - - self.file_stream = self.atomic_file.file.outStream(); - self.buffered_stream = BufferedOutStream(File.WriteError).init(&self.file_stream.stream); - return self; - } - - /// always call destroy, even after successful finish() - pub fn destroy(self: *BufferedAtomicFile) void { - self.atomic_file.deinit(); - self.allocator.destroy(self); - } - - pub fn finish(self: *BufferedAtomicFile) !void { - try self.buffered_stream.flush(); - try self.atomic_file.finish(); - } - - pub fn stream(self: *BufferedAtomicFile) *OutStream(File.WriteError) { - return &self.buffered_stream.stream; - } -}; - pub const Packing = enum { /// Pack data to byte alignment Byte, @@ -1129,8 +1003,9 @@ pub fn Serializer(comptime endian: builtin.Endian, comptime packing: Packing, co }; } -test "import io tests" { +test "" { comptime { _ = @import("io/test.zig"); } + std.meta.refAllDecls(@This()); } diff --git a/lib/std/io/buffered_atomic_file.zig b/lib/std/io/buffered_atomic_file.zig new file mode 100644 index 0000000000..48c550c90f --- /dev/null +++ b/lib/std/io/buffered_atomic_file.zig @@ -0,0 +1,50 @@ +const std = @import("../std.zig"); +const mem = std.mem; +const fs = std.fs; +const File = std.fs.File; + +pub const BufferedAtomicFile = struct { + atomic_file: fs.AtomicFile, + file_stream: File.OutStream, + buffered_stream: BufferedOutStream, + allocator: *mem.Allocator, + + pub const buffer_size = 4096; + pub const BufferedOutStream = std.io.BufferedOutStreamCustom(buffer_size, File.OutStream); + pub const OutStream = std.io.OutStream(*BufferedOutStream, BufferedOutStream.Error, BufferedOutStream.write); + + /// TODO when https://github.com/ziglang/zig/issues/2761 is solved + /// this API will not need an allocator + pub fn create(allocator: *mem.Allocator, dest_path: []const u8) !*BufferedAtomicFile { + var self = try allocator.create(BufferedAtomicFile); + self.* = BufferedAtomicFile{ + .atomic_file = undefined, + .file_stream = undefined, + .buffered_stream = undefined, + .allocator = allocator, + }; + errdefer allocator.destroy(self); + + self.atomic_file = try fs.AtomicFile.init(dest_path, File.default_mode); + errdefer self.atomic_file.deinit(); + + self.file_stream = self.atomic_file.file.outStream(); + self.buffered_stream = std.io.bufferedOutStream(buffer_size, self.file_stream); + return self; + } + + /// always call destroy, even after successful finish() + pub fn destroy(self: *BufferedAtomicFile) void { + self.atomic_file.deinit(); + self.allocator.destroy(self); + } + + pub fn finish(self: *BufferedAtomicFile) !void { + try self.buffered_stream.flush(); + try self.atomic_file.finish(); + } + + pub fn stream(self: *BufferedAtomicFile) OutStream { + return .{ .context = &self.buffered_stream }; + } +}; diff --git a/lib/std/io/buffered_out_stream.zig b/lib/std/io/buffered_out_stream.zig new file mode 100644 index 0000000000..3650022b29 --- /dev/null +++ b/lib/std/io/buffered_out_stream.zig @@ -0,0 +1,56 @@ +const std = @import("../std.zig"); +const io = std.io; + +pub fn BufferedOutStream(comptime OutStreamType: type) type { + return BufferedOutStreamCustom(4096, OutStreamType); +} + +pub fn BufferedOutStreamCustom(comptime buffer_size: usize, comptime OutStreamType: type) type { + return struct { + unbuffered_out_stream: OutStreamType, + fifo: FifoType, + + pub const Error = OutStreamType.Error; + pub const OutStream = io.OutStream(*Self, Error, write); + + const Self = @This(); + const FifoType = std.fifo.LinearFifo(u8, std.fifo.LinearFifoBufferType{ .Static = buffer_size }); + + pub fn init(unbuffered_out_stream: OutStreamType) Self { + return Self{ + .unbuffered_out_stream = unbuffered_out_stream, + .fifo = FifoType.init(), + }; + } + + pub fn flush(self: *Self) !void { + while (true) { + const slice = self.fifo.readableSlice(0); + if (slice.len == 0) break; + try self.unbuffered_out_stream.writeAll(slice); + self.fifo.discard(slice.len); + } + } + + pub fn outStream(self: *Self) OutStream { + return .{ .context = self }; + } + + pub fn write(self: *Self, bytes: []const u8) Error!usize { + if (bytes.len >= self.fifo.writableLength()) { + try self.flush(); + return self.unbuffered_out_stream.write(bytes); + } + self.fifo.writeAssumeCapacity(bytes); + return bytes.len; + } + }; +} + +pub fn bufferedOutStream( + comptime buffer_size: usize, + underlying_stream: var, +) BufferedOutStreamCustom(buffer_size, @TypeOf(underlying_stream)) { + return BufferedOutStreamCustom(buffer_size, @TypeOf(underlying_stream)).init(underlying_stream); +} + diff --git a/lib/std/io/c_out_stream.zig b/lib/std/io/c_out_stream.zig deleted file mode 100644 index adaa3fcbaf..0000000000 --- a/lib/std/io/c_out_stream.zig +++ /dev/null @@ -1,43 +0,0 @@ -const std = @import("../std.zig"); -const os = std.os; -const OutStream = std.io.OutStream; -const builtin = @import("builtin"); - -/// TODO make a proposal to make `std.fs.File` use *FILE when linking libc and this just becomes -/// std.io.FileOutStream because std.fs.File.write would do this when linking -/// libc. -pub const COutStream = struct { - pub const Error = std.fs.File.WriteError; - pub const Stream = OutStream(Error); - - stream: Stream, - c_file: *std.c.FILE, - - pub fn init(c_file: *std.c.FILE) COutStream { - return COutStream{ - .c_file = c_file, - .stream = Stream{ .writeFn = writeFn }, - }; - } - - fn writeFn(out_stream: *Stream, bytes: []const u8) Error!usize { - const self = @fieldParentPtr(COutStream, "stream", out_stream); - const amt_written = std.c.fwrite(bytes.ptr, 1, bytes.len, self.c_file); - if (amt_written >= 0) return amt_written; - switch (std.c._errno().*) { - 0 => unreachable, - os.EINVAL => unreachable, - os.EFAULT => unreachable, - os.EAGAIN => unreachable, // this is a blocking API - os.EBADF => unreachable, // always a race condition - os.EDESTADDRREQ => unreachable, // connect was never called - os.EDQUOT => return error.DiskQuota, - os.EFBIG => return error.FileTooBig, - os.EIO => return error.InputOutput, - os.ENOSPC => return error.NoSpaceLeft, - os.EPERM => return error.AccessDenied, - os.EPIPE => return error.BrokenPipe, - else => |err| return os.unexpectedErrno(@intCast(usize, err)), - } - } -}; diff --git a/lib/std/io/counting_out_stream.zig b/lib/std/io/counting_out_stream.zig new file mode 100644 index 0000000000..2b0cba29fc --- /dev/null +++ b/lib/std/io/counting_out_stream.zig @@ -0,0 +1,42 @@ +const std = @import("../std.zig"); +const io = std.io; + +/// An OutStream that counts how many bytes has been written to it. +pub fn CountingOutStream(comptime OutStreamType: type) type { + return struct { + bytes_written: u64, + child_stream: OutStreamType, + + pub const Error = OutStreamType.Error; + pub const OutStream = io.OutStream(*Self, Error, write); + + const Self = @This(); + + pub fn init(child_stream: OutStreamType) Self { + return Self{ + .bytes_written = 0, + .child_stream = child_stream, + }; + } + + pub fn write(self: *Self, bytes: []const u8) Error!usize { + const amt = try self.child_stream.write(bytes); + self.bytes_written += amt; + return amt; + } + + pub fn outStream(self: *Self) OutStream { + return .{ .context = self }; + } + }; +} + +test "io.CountingOutStream" { + var counting_stream = CountingOutStream(NullOutStream.Error).init(std.io.null_out_stream); + const stream = &counting_stream.stream; + + const bytes = "yay" ** 10000; + stream.write(bytes) catch unreachable; + testing.expect(counting_stream.bytes_written == bytes.len); +} + diff --git a/lib/std/io/fixed_buffer_stream.zig b/lib/std/io/fixed_buffer_stream.zig new file mode 100644 index 0000000000..847feeae03 --- /dev/null +++ b/lib/std/io/fixed_buffer_stream.zig @@ -0,0 +1,66 @@ +const std = @import("../std.zig"); +const io = std.io; + +pub const FixedBufferInStream = struct { + bytes: []const u8, + pos: usize, + + pub const SeekError = error{EndOfStream}; + pub const GetSeekPosError = error{}; + + pub const InStream = io.InStream(*FixedBufferInStream, error{}, read); + + pub fn inStream(self: *FixedBufferInStream) InStream { + return .{ .context = self }; + } + + pub const SeekableStream = io.SeekableStream( + *FixedBufferInStream, + SeekError, + GetSeekPosError, + seekTo, + seekBy, + getPos, + getEndPos, + ); + + pub fn seekableStream(self: *FixedBufferInStream) SeekableStream { + return .{ .context = self }; + } + + pub fn read(self: *FixedBufferInStream, dest: []u8) error{}!usize { + const size = std.math.min(dest.len, self.bytes.len - self.pos); + const end = self.pos + size; + + std.mem.copy(u8, dest[0..size], self.bytes[self.pos..end]); + self.pos = end; + + return size; + } + + pub fn seekTo(self: *FixedBufferInStream, pos: u64) SeekError!void { + const usize_pos = std.math.cast(usize, pos) catch return error.EndOfStream; + if (usize_pos > self.bytes.len) return error.EndOfStream; + self.pos = usize_pos; + } + + pub fn seekBy(self: *FixedBufferInStream, amt: i64) SeekError!void { + if (amt < 0) { + const abs_amt = std.math.cast(usize, -amt) catch return error.EndOfStream; + if (abs_amt > self.pos) return error.EndOfStream; + self.pos -= abs_amt; + } else { + const usize_amt = std.math.cast(usize, amt) catch return error.EndOfStream; + if (self.pos + usize_amt > self.bytes.len) return error.EndOfStream; + self.pos += usize_amt; + } + } + + pub fn getEndPos(self: *FixedBufferInStream) GetSeekPosError!u64 { + return self.bytes.len; + } + + pub fn getPos(self: *FixedBufferInStream) GetSeekPosError!u64 { + return self.pos; + } +}; diff --git a/lib/std/io/in_stream.zig b/lib/std/io/in_stream.zig index e234182771..d181f4df43 100644 --- a/lib/std/io/in_stream.zig +++ b/lib/std/io/in_stream.zig @@ -1,44 +1,31 @@ const std = @import("../std.zig"); -const builtin = @import("builtin"); -const root = @import("root"); +const builtin = std.builtin; const math = std.math; const assert = std.debug.assert; const mem = std.mem; const Buffer = std.Buffer; const testing = std.testing; -pub const default_stack_size = 1 * 1024 * 1024; -pub const stack_size: usize = if (@hasDecl(root, "stack_size_std_io_InStream")) - root.stack_size_std_io_InStream -else - default_stack_size; - -pub fn InStream(comptime ReadError: type) type { +pub fn InStream( + comptime Context: type, + comptime ReadError: type, + /// Returns the number of bytes read. It may be less than buffer.len. + /// If the number of bytes read is 0, it means end of stream. + /// End of stream is not an error condition. + comptime readFn: fn (context: Context, buffer: []u8) ReadError!usize, +) type { return struct { - const Self = @This(); pub const Error = ReadError; - pub const ReadFn = if (std.io.is_async) - async fn (self: *Self, buffer: []u8) Error!usize - else - fn (self: *Self, buffer: []u8) Error!usize; + + context: Context, + + const Self = @This(); /// Returns the number of bytes read. It may be less than buffer.len. /// If the number of bytes read is 0, it means end of stream. /// End of stream is not an error condition. - readFn: ReadFn, - - /// Returns the number of bytes read. It may be less than buffer.len. - /// If the number of bytes read is 0, it means end of stream. - /// End of stream is not an error condition. - pub fn read(self: *Self, buffer: []u8) Error!usize { - if (std.io.is_async) { - // Let's not be writing 0xaa in safe modes for upwards of 4 MiB for every stream read. - @setRuntimeSafety(false); - var stack_frame: [stack_size]u8 align(std.Target.stack_align) = undefined; - return await @asyncCall(&stack_frame, {}, self.readFn, self, buffer); - } else { - return self.readFn(self, buffer); - } + pub fn read(self: Self, buffer: []u8) Error!usize { + return readFn(self.context, buffer); } /// Deprecated: use `readAll`. @@ -47,7 +34,7 @@ pub fn InStream(comptime ReadError: type) type { /// Returns the number of bytes read. If the number read is smaller than buf.len, it /// means the stream reached the end. Reaching the end of a stream is not an error /// condition. - pub fn readAll(self: *Self, buffer: []u8) Error!usize { + pub fn readAll(self: Self, buffer: []u8) Error!usize { var index: usize = 0; while (index != buffer.len) { const amt = try self.read(buffer[index..]); @@ -59,13 +46,13 @@ pub fn InStream(comptime ReadError: type) type { /// Returns the number of bytes read. If the number read would be smaller than buf.len, /// error.EndOfStream is returned instead. - pub fn readNoEof(self: *Self, buf: []u8) !void { + pub fn readNoEof(self: Self, buf: []u8) !void { const amt_read = try self.readAll(buf); if (amt_read < buf.len) return error.EndOfStream; } /// Deprecated: use `readAllArrayList`. - pub fn readAllBuffer(self: *Self, buffer: *Buffer, max_size: usize) !void { + pub fn readAllBuffer(self: Self, buffer: *Buffer, max_size: usize) !void { buffer.list.shrink(0); try self.readAllArrayList(&buffer.list, max_size); errdefer buffer.shrink(0); @@ -75,7 +62,7 @@ pub fn InStream(comptime ReadError: type) type { /// Appends to the `std.ArrayList` contents by reading from the stream until end of stream is found. /// If the number of bytes appended would exceed `max_append_size`, `error.StreamTooLong` is returned /// and the `std.ArrayList` has exactly `max_append_size` bytes appended. - pub fn readAllArrayList(self: *Self, array_list: *std.ArrayList(u8), max_append_size: usize) !void { + pub fn readAllArrayList(self: Self, array_list: *std.ArrayList(u8), max_append_size: usize) !void { try array_list.ensureCapacity(math.min(max_append_size, 4096)); const original_len = array_list.len; var start_index: usize = original_len; @@ -104,7 +91,7 @@ pub fn InStream(comptime ReadError: type) type { /// memory would be greater than `max_size`, returns `error.StreamTooLong`. /// Caller owns returned memory. /// If this function returns an error, the contents from the stream read so far are lost. - pub fn readAllAlloc(self: *Self, allocator: *mem.Allocator, max_size: usize) ![]u8 { + pub fn readAllAlloc(self: Self, allocator: *mem.Allocator, max_size: usize) ![]u8 { var array_list = std.ArrayList(u8).init(allocator); defer array_list.deinit(); try self.readAllArrayList(&array_list, max_size); @@ -116,7 +103,7 @@ pub fn InStream(comptime ReadError: type) type { /// If the `std.ArrayList` length would exceed `max_size`, `error.StreamTooLong` is returned and the /// `std.ArrayList` is populated with `max_size` bytes from the stream. pub fn readUntilDelimiterArrayList( - self: *Self, + self: Self, array_list: *std.ArrayList(u8), delimiter: u8, max_size: usize, @@ -142,7 +129,7 @@ pub fn InStream(comptime ReadError: type) type { /// Caller owns returned memory. /// If this function returns an error, the contents from the stream read so far are lost. pub fn readUntilDelimiterAlloc( - self: *Self, + self: Self, allocator: *mem.Allocator, delimiter: u8, max_size: usize, @@ -159,7 +146,7 @@ pub fn InStream(comptime ReadError: type) type { /// function is called again after that, returns null. /// Returns a slice of the stream data, with ptr equal to `buf.ptr`. The /// delimiter byte is not included in the returned slice. - pub fn readUntilDelimiterOrEof(self: *Self, buf: []u8, delimiter: u8) !?[]u8 { + pub fn readUntilDelimiterOrEof(self: Self, buf: []u8, delimiter: u8) !?[]u8 { var index: usize = 0; while (true) { const byte = self.readByte() catch |err| switch (err) { @@ -184,7 +171,7 @@ pub fn InStream(comptime ReadError: type) type { /// Reads from the stream until specified byte is found, discarding all data, /// including the delimiter. /// If end-of-stream is found, this function succeeds. - pub fn skipUntilDelimiterOrEof(self: *Self, delimiter: u8) !void { + pub fn skipUntilDelimiterOrEof(self: Self, delimiter: u8) !void { while (true) { const byte = self.readByte() catch |err| switch (err) { error.EndOfStream => return, @@ -195,7 +182,7 @@ pub fn InStream(comptime ReadError: type) type { } /// Reads 1 byte from the stream or returns `error.EndOfStream`. - pub fn readByte(self: *Self) !u8 { + pub fn readByte(self: Self) !u8 { var result: [1]u8 = undefined; const amt_read = try self.read(result[0..]); if (amt_read < 1) return error.EndOfStream; @@ -203,43 +190,43 @@ pub fn InStream(comptime ReadError: type) type { } /// Same as `readByte` except the returned byte is signed. - pub fn readByteSigned(self: *Self) !i8 { + pub fn readByteSigned(self: Self) !i8 { return @bitCast(i8, try self.readByte()); } /// Reads a native-endian integer - pub fn readIntNative(self: *Self, comptime T: type) !T { + pub fn readIntNative(self: Self, comptime T: type) !T { var bytes: [(T.bit_count + 7) / 8]u8 = undefined; try self.readNoEof(bytes[0..]); return mem.readIntNative(T, &bytes); } /// Reads a foreign-endian integer - pub fn readIntForeign(self: *Self, comptime T: type) !T { + pub fn readIntForeign(self: Self, comptime T: type) !T { var bytes: [(T.bit_count + 7) / 8]u8 = undefined; try self.readNoEof(bytes[0..]); return mem.readIntForeign(T, &bytes); } - pub fn readIntLittle(self: *Self, comptime T: type) !T { + pub fn readIntLittle(self: Self, comptime T: type) !T { var bytes: [(T.bit_count + 7) / 8]u8 = undefined; try self.readNoEof(bytes[0..]); return mem.readIntLittle(T, &bytes); } - pub fn readIntBig(self: *Self, comptime T: type) !T { + pub fn readIntBig(self: Self, comptime T: type) !T { var bytes: [(T.bit_count + 7) / 8]u8 = undefined; try self.readNoEof(bytes[0..]); return mem.readIntBig(T, &bytes); } - pub fn readInt(self: *Self, comptime T: type, endian: builtin.Endian) !T { + pub fn readInt(self: Self, comptime T: type, endian: builtin.Endian) !T { var bytes: [(T.bit_count + 7) / 8]u8 = undefined; try self.readNoEof(bytes[0..]); return mem.readInt(T, &bytes, endian); } - pub fn readVarInt(self: *Self, comptime ReturnType: type, endian: builtin.Endian, size: usize) !ReturnType { + pub fn readVarInt(self: Self, comptime ReturnType: type, endian: builtin.Endian, size: usize) !ReturnType { assert(size <= @sizeOf(ReturnType)); var bytes_buf: [@sizeOf(ReturnType)]u8 = undefined; const bytes = bytes_buf[0..size]; @@ -247,14 +234,14 @@ pub fn InStream(comptime ReadError: type) type { return mem.readVarInt(ReturnType, bytes, endian); } - pub fn skipBytes(self: *Self, num_bytes: u64) !void { + pub fn skipBytes(self: Self, num_bytes: u64) !void { var i: u64 = 0; while (i < num_bytes) : (i += 1) { _ = try self.readByte(); } } - pub fn readStruct(self: *Self, comptime T: type) !T { + pub fn readStruct(self: Self, comptime T: type) !T { // Only extern and packed structs have defined in-memory layout. comptime assert(@typeInfo(T).Struct.layout != builtin.TypeInfo.ContainerLayout.Auto); var res: [1]T = undefined; @@ -265,7 +252,7 @@ pub fn InStream(comptime ReadError: type) type { /// Reads an integer with the same size as the given enum's tag type. If the integer matches /// an enum tag, casts the integer to the enum tag and returns it. Otherwise, returns an error. /// TODO optimization taking advantage of most fields being in order - pub fn readEnum(self: *Self, comptime Enum: type, endian: builtin.Endian) !Enum { + pub fn readEnum(self: Self, comptime Enum: type, endian: builtin.Endian) !Enum { const E = error{ /// An integer was read, but it did not match any of the tags in the supplied enum. InvalidValue, diff --git a/lib/std/io/out_stream.zig b/lib/std/io/out_stream.zig index cb75b27bf1..03901f6672 100644 --- a/lib/std/io/out_stream.zig +++ b/lib/std/io/out_stream.zig @@ -1,94 +1,85 @@ const std = @import("../std.zig"); -const builtin = @import("builtin"); -const root = @import("root"); +const builtin = std.builtin; const mem = std.mem; -pub const default_stack_size = 1 * 1024 * 1024; -pub const stack_size: usize = if (@hasDecl(root, "stack_size_std_io_OutStream")) - root.stack_size_std_io_OutStream -else - default_stack_size; - -pub fn OutStream(comptime WriteError: type) type { +pub fn OutStream( + comptime Context: type, + comptime WriteError: type, + comptime writeFn: fn (context: Context, bytes: []const u8) WriteError!usize, +) type { return struct { + context: Context, + const Self = @This(); pub const Error = WriteError; - pub const WriteFn = if (std.io.is_async) - async fn (self: *Self, bytes: []const u8) Error!usize - else - fn (self: *Self, bytes: []const u8) Error!usize; - writeFn: WriteFn, - - pub fn writeOnce(self: *Self, bytes: []const u8) Error!usize { - if (std.io.is_async) { - // Let's not be writing 0xaa in safe modes for upwards of 4 MiB for every stream write. - @setRuntimeSafety(false); - var stack_frame: [stack_size]u8 align(std.Target.stack_align) = undefined; - return await @asyncCall(&stack_frame, {}, self.writeFn, self, bytes); - } else { - return self.writeFn(self, bytes); - } + pub fn write(self: Self, bytes: []const u8) Error!usize { + return writeFn(self.context, bytes); } - pub fn write(self: *Self, bytes: []const u8) Error!void { + pub fn writeAll(self: Self, bytes: []const u8) Error!void { var index: usize = 0; while (index != bytes.len) { - index += try self.writeOnce(bytes[index..]); + index += try self.write(bytes[index..]); } } - pub fn print(self: *Self, comptime format: []const u8, args: var) Error!void { - return std.fmt.format(self, Error, write, format, args); + pub fn print(self: Self, comptime format: []const u8, args: var) Error!void { + return std.fmt.format(self, Error, writeAll, format, args); } - pub fn writeByte(self: *Self, byte: u8) Error!void { + pub fn writeByte(self: Self, byte: u8) Error!void { const array = [1]u8{byte}; - return self.write(&array); + return self.writeAll(&array); } - pub fn writeByteNTimes(self: *Self, byte: u8, n: usize) Error!void { + pub fn writeByteNTimes(self: Self, byte: u8, n: usize) Error!void { var bytes: [256]u8 = undefined; mem.set(u8, bytes[0..], byte); var remaining: usize = n; while (remaining > 0) { const to_write = std.math.min(remaining, bytes.len); - try self.write(bytes[0..to_write]); + try self.writeAll(bytes[0..to_write]); remaining -= to_write; } } /// Write a native-endian integer. - pub fn writeIntNative(self: *Self, comptime T: type, value: T) Error!void { + /// TODO audit non-power-of-two int sizes + pub fn writeIntNative(self: Self, comptime T: type, value: T) Error!void { var bytes: [(T.bit_count + 7) / 8]u8 = undefined; mem.writeIntNative(T, &bytes, value); - return self.write(&bytes); + return self.writeAll(&bytes); } /// Write a foreign-endian integer. - pub fn writeIntForeign(self: *Self, comptime T: type, value: T) Error!void { + /// TODO audit non-power-of-two int sizes + pub fn writeIntForeign(self: Self, comptime T: type, value: T) Error!void { var bytes: [(T.bit_count + 7) / 8]u8 = undefined; mem.writeIntForeign(T, &bytes, value); - return self.write(&bytes); + return self.writeAll(&bytes); } - pub fn writeIntLittle(self: *Self, comptime T: type, value: T) Error!void { + /// TODO audit non-power-of-two int sizes + pub fn writeIntLittle(self: Self, comptime T: type, value: T) Error!void { var bytes: [(T.bit_count + 7) / 8]u8 = undefined; mem.writeIntLittle(T, &bytes, value); - return self.write(&bytes); + return self.writeAll(&bytes); } - pub fn writeIntBig(self: *Self, comptime T: type, value: T) Error!void { + /// TODO audit non-power-of-two int sizes + pub fn writeIntBig(self: Self, comptime T: type, value: T) Error!void { var bytes: [(T.bit_count + 7) / 8]u8 = undefined; mem.writeIntBig(T, &bytes, value); - return self.write(&bytes); + return self.writeAll(&bytes); } - pub fn writeInt(self: *Self, comptime T: type, value: T, endian: builtin.Endian) Error!void { + /// TODO audit non-power-of-two int sizes + pub fn writeInt(self: Self, comptime T: type, value: T, endian: builtin.Endian) Error!void { var bytes: [(T.bit_count + 7) / 8]u8 = undefined; mem.writeInt(T, &bytes, value, endian); - return self.write(&bytes); + return self.writeAll(&bytes); } }; } diff --git a/lib/std/io/seekable_stream.zig b/lib/std/io/seekable_stream.zig index 052abbc856..ed410b7aa7 100644 --- a/lib/std/io/seekable_stream.zig +++ b/lib/std/io/seekable_stream.zig @@ -1,103 +1,36 @@ const std = @import("../std.zig"); const InStream = std.io.InStream; -pub fn SeekableStream(comptime SeekErrorType: type, comptime GetSeekPosErrorType: type) type { +pub fn SeekableStream( + comptime Context: type, + comptime SeekErrorType: type, + comptime GetSeekPosErrorType: type, + comptime seekToFn: fn (context: Context, pos: u64) SeekErrorType!void, + comptime seekByFn: fn (context: Context, pos: i64) SeekErrorType!void, + comptime getPosFn: fn (context: Context) GetSeekPosErrorType!u64, + comptime getEndPosFn: fn (context: Context) GetSeekPosErrorType!u64, +) type { return struct { + context: Context, + const Self = @This(); pub const SeekError = SeekErrorType; pub const GetSeekPosError = GetSeekPosErrorType; - seekToFn: fn (self: *Self, pos: u64) SeekError!void, - seekByFn: fn (self: *Self, pos: i64) SeekError!void, - - getPosFn: fn (self: *Self) GetSeekPosError!u64, - getEndPosFn: fn (self: *Self) GetSeekPosError!u64, - - pub fn seekTo(self: *Self, pos: u64) SeekError!void { - return self.seekToFn(self, pos); + pub fn seekTo(self: Self, pos: u64) SeekError!void { + return seekToFn(self.context, pos); } - pub fn seekBy(self: *Self, amt: i64) SeekError!void { - return self.seekByFn(self, amt); + pub fn seekBy(self: Self, amt: i64) SeekError!void { + return seekByFn(self.context, amt); } - pub fn getEndPos(self: *Self) GetSeekPosError!u64 { - return self.getEndPosFn(self); + pub fn getEndPos(self: Self) GetSeekPosError!u64 { + return getEndPosFn(self.context); } - pub fn getPos(self: *Self) GetSeekPosError!u64 { - return self.getPosFn(self); + pub fn getPos(self: Self) GetSeekPosError!u64 { + return getPosFn(self.context); } }; } - -pub const SliceSeekableInStream = struct { - const Self = @This(); - pub const Error = error{}; - pub const SeekError = error{EndOfStream}; - pub const GetSeekPosError = error{}; - pub const Stream = InStream(Error); - pub const SeekableInStream = SeekableStream(SeekError, GetSeekPosError); - - stream: Stream, - seekable_stream: SeekableInStream, - - pos: usize, - slice: []const u8, - - pub fn init(slice: []const u8) Self { - return Self{ - .slice = slice, - .pos = 0, - .stream = Stream{ .readFn = readFn }, - .seekable_stream = SeekableInStream{ - .seekToFn = seekToFn, - .seekByFn = seekByFn, - .getEndPosFn = getEndPosFn, - .getPosFn = getPosFn, - }, - }; - } - - fn readFn(in_stream: *Stream, dest: []u8) Error!usize { - const self = @fieldParentPtr(Self, "stream", in_stream); - const size = std.math.min(dest.len, self.slice.len - self.pos); - const end = self.pos + size; - - std.mem.copy(u8, dest[0..size], self.slice[self.pos..end]); - self.pos = end; - - return size; - } - - fn seekToFn(in_stream: *SeekableInStream, pos: u64) SeekError!void { - const self = @fieldParentPtr(Self, "seekable_stream", in_stream); - const usize_pos = @intCast(usize, pos); - if (usize_pos > self.slice.len) return error.EndOfStream; - self.pos = usize_pos; - } - - fn seekByFn(in_stream: *SeekableInStream, amt: i64) SeekError!void { - const self = @fieldParentPtr(Self, "seekable_stream", in_stream); - - if (amt < 0) { - const abs_amt = @intCast(usize, -amt); - if (abs_amt > self.pos) return error.EndOfStream; - self.pos -= abs_amt; - } else { - const usize_amt = @intCast(usize, amt); - if (self.pos + usize_amt > self.slice.len) return error.EndOfStream; - self.pos += usize_amt; - } - } - - fn getEndPosFn(in_stream: *SeekableInStream) GetSeekPosError!u64 { - const self = @fieldParentPtr(Self, "seekable_stream", in_stream); - return @intCast(u64, self.slice.len); - } - - fn getPosFn(in_stream: *SeekableInStream) GetSeekPosError!u64 { - const self = @fieldParentPtr(Self, "seekable_stream", in_stream); - return @intCast(u64, self.pos); - } -}; diff --git a/lib/std/json/write_stream.zig b/lib/std/json/write_stream.zig index 213b6768d4..6d88c8b6da 100644 --- a/lib/std/json/write_stream.zig +++ b/lib/std/json/write_stream.zig @@ -30,11 +30,11 @@ pub fn WriteStream(comptime OutStream: type, comptime max_depth: usize) type { /// The string used as spacing. space: []const u8 = " ", - stream: *OutStream, + stream: OutStream, state_index: usize, state: [max_depth]State, - pub fn init(stream: *OutStream) Self { + pub fn init(stream: OutStream) Self { var self = Self{ .stream = stream, .state_index = 1, @@ -90,8 +90,8 @@ pub fn WriteStream(comptime OutStream: type, comptime max_depth: usize) type { self.pushState(.Value); try self.indent(); try self.writeEscapedString(name); - try self.stream.write(":"); - try self.stream.write(self.space); + try self.stream.writeAll(":"); + try self.stream.writeAll(self.space); }, } } @@ -134,16 +134,16 @@ pub fn WriteStream(comptime OutStream: type, comptime max_depth: usize) type { pub fn emitNull(self: *Self) !void { assert(self.state[self.state_index] == State.Value); - try self.stream.write("null"); + try self.stream.writeAll("null"); self.popState(); } pub fn emitBool(self: *Self, value: bool) !void { assert(self.state[self.state_index] == State.Value); if (value) { - try self.stream.write("true"); + try self.stream.writeAll("true"); } else { - try self.stream.write("false"); + try self.stream.writeAll("false"); } self.popState(); } @@ -188,13 +188,13 @@ pub fn WriteStream(comptime OutStream: type, comptime max_depth: usize) type { try self.stream.writeByte('"'); for (string) |s| { switch (s) { - '"' => try self.stream.write("\\\""), - '\t' => try self.stream.write("\\t"), - '\r' => try self.stream.write("\\r"), - '\n' => try self.stream.write("\\n"), - 8 => try self.stream.write("\\b"), - 12 => try self.stream.write("\\f"), - '\\' => try self.stream.write("\\\\"), + '"' => try self.stream.writeAll("\\\""), + '\t' => try self.stream.writeAll("\\t"), + '\r' => try self.stream.writeAll("\\r"), + '\n' => try self.stream.writeAll("\\n"), + 8 => try self.stream.writeAll("\\b"), + 12 => try self.stream.writeAll("\\f"), + '\\' => try self.stream.writeAll("\\\\"), else => try self.stream.writeByte(s), } } @@ -231,10 +231,10 @@ pub fn WriteStream(comptime OutStream: type, comptime max_depth: usize) type { fn indent(self: *Self) !void { assert(self.state_index >= 1); - try self.stream.write(self.newline); + try self.stream.writeAll(self.newline); var i: usize = 0; while (i < self.state_index - 1) : (i += 1) { - try self.stream.write(self.one_indent); + try self.stream.writeAll(self.one_indent); } } diff --git a/lib/std/os.zig b/lib/std/os.zig index 460f0bee02..76a5dc2be5 100644 --- a/lib/std/os.zig +++ b/lib/std/os.zig @@ -176,7 +176,7 @@ fn getRandomBytesDevURandom(buf: []u8) !void { .io_mode = .blocking, .async_block_allowed = std.fs.File.async_block_allowed_yes, }; - const stream = &file.inStream().stream; + const stream = file.inStream(); stream.readNoEof(buf) catch return error.Unexpected; } diff --git a/lib/std/pdb.zig b/lib/std/pdb.zig index db60e60494..7c48cf7b7c 100644 --- a/lib/std/pdb.zig +++ b/lib/std/pdb.zig @@ -632,11 +632,7 @@ const MsfStream = struct { blocks: []u32 = undefined, block_size: u32 = undefined, - /// Implementation of InStream trait for Pdb.MsfStream - stream: Stream = undefined, - pub const Error = @TypeOf(read).ReturnType.ErrorSet; - pub const Stream = io.InStream(Error); fn init(block_size: u32, file: File, blocks: []u32) MsfStream { const stream = MsfStream{ @@ -644,7 +640,6 @@ const MsfStream = struct { .pos = 0, .blocks = blocks, .block_size = block_size, - .stream = Stream{ .readFn = readFn }, }; return stream; @@ -715,8 +710,7 @@ const MsfStream = struct { return block * self.block_size + offset; } - fn readFn(in_stream: *Stream, buffer: []u8) Error!usize { - const self = @fieldParentPtr(MsfStream, "stream", in_stream); - return self.read(buffer); + fn inStream(self: *MsfStream) std.io.InStream(*MsfStream, Error, read) { + return .{ .context = self }; } }; diff --git a/lib/std/zig/ast.zig b/lib/std/zig/ast.zig index 95a772376d..8ae219a998 100644 --- a/lib/std/zig/ast.zig +++ b/lib/std/zig/ast.zig @@ -375,7 +375,7 @@ pub const Error = union(enum) { token: TokenIndex, pub fn render(self: *const ThisError, tokens: *Tree.TokenList, stream: var) !void { - return stream.write(msg); + return stream.writeAll(msg); } }; } diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index ce9049e35b..9600de21df 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -12,64 +12,58 @@ pub const Error = error{ }; /// Returns whether anything changed -pub fn render(allocator: *mem.Allocator, stream: var, tree: *ast.Tree) (@TypeOf(stream).Child.Error || Error)!bool { - comptime assert(@typeInfo(@TypeOf(stream)) == .Pointer); - - var anything_changed: bool = false; - +pub fn render(allocator: *mem.Allocator, stream: var, tree: *ast.Tree) (@TypeOf(stream).Error || Error)!bool { // make a passthrough stream that checks whether something changed const MyStream = struct { const MyStream = @This(); - const StreamError = @TypeOf(stream).Child.Error; - const Stream = std.io.OutStream(StreamError); + const StreamError = @TypeOf(stream).Error; - anything_changed_ptr: *bool, child_stream: @TypeOf(stream), - stream: Stream, + anything_changed: bool, source_index: usize, source: []const u8, - fn write(iface_stream: *Stream, bytes: []const u8) StreamError!usize { - const self = @fieldParentPtr(MyStream, "stream", iface_stream); - - if (!self.anything_changed_ptr.*) { + fn write(self: *MyStream, bytes: []const u8) StreamError!usize { + if (!self.anything_changed) { const end = self.source_index + bytes.len; if (end > self.source.len) { - self.anything_changed_ptr.* = true; + self.anything_changed = true; } else { const src_slice = self.source[self.source_index..end]; self.source_index += bytes.len; if (!mem.eql(u8, bytes, src_slice)) { - self.anything_changed_ptr.* = true; + self.anything_changed = true; } } } - return self.child_stream.writeOnce(bytes); + return self.child_stream.write(bytes); } }; var my_stream = MyStream{ - .stream = MyStream.Stream{ .writeFn = MyStream.write }, .child_stream = stream, - .anything_changed_ptr = &anything_changed, + .anything_changed = false, .source_index = 0, .source = tree.source, }; + const my_stream_stream: std.io.OutStream(*MyStream, MyStream.StreamError, MyStream.write) = .{ + .context = &my_stream, + }; - try renderRoot(allocator, &my_stream.stream, tree); + try renderRoot(allocator, my_stream_stream, tree); - if (!anything_changed and my_stream.source_index != my_stream.source.len) { - anything_changed = true; + if (my_stream.source_index != my_stream.source.len) { + my_stream.anything_changed = true; } - return anything_changed; + return my_stream.anything_changed; } fn renderRoot( allocator: *mem.Allocator, stream: var, tree: *ast.Tree, -) (@TypeOf(stream).Child.Error || Error)!void { +) (@TypeOf(stream).Error || Error)!void { var tok_it = tree.tokens.iterator(0); // render all the line comments at the beginning of the file @@ -189,7 +183,7 @@ fn renderRoot( } } -fn renderExtraNewline(tree: *ast.Tree, stream: var, start_col: *usize, node: *ast.Node) @TypeOf(stream).Child.Error!void { +fn renderExtraNewline(tree: *ast.Tree, stream: var, start_col: *usize, node: *ast.Node) @TypeOf(stream).Error!void { const first_token = node.firstToken(); var prev_token = first_token; if (prev_token == 0) return; @@ -204,11 +198,11 @@ fn renderExtraNewline(tree: *ast.Tree, stream: var, start_col: *usize, node: *as } } -fn renderTopLevelDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree, indent: usize, start_col: *usize, decl: *ast.Node) (@TypeOf(stream).Child.Error || Error)!void { +fn renderTopLevelDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree, indent: usize, start_col: *usize, decl: *ast.Node) (@TypeOf(stream).Error || Error)!void { try renderContainerDecl(allocator, stream, tree, indent, start_col, decl, .Newline); } -fn renderContainerDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree, indent: usize, start_col: *usize, decl: *ast.Node, space: Space) (@TypeOf(stream).Child.Error || Error)!void { +fn renderContainerDecl(allocator: *mem.Allocator, stream: var, tree: *ast.Tree, indent: usize, start_col: *usize, decl: *ast.Node, space: Space) (@TypeOf(stream).Error || Error)!void { switch (decl.id) { .FnProto => { const fn_proto = @fieldParentPtr(ast.Node.FnProto, "base", decl); @@ -343,7 +337,7 @@ fn renderExpression( start_col: *usize, base: *ast.Node, space: Space, -) (@TypeOf(stream).Child.Error || Error)!void { +) (@TypeOf(stream).Error || Error)!void { switch (base.id) { .Identifier => { const identifier = @fieldParentPtr(ast.Node.Identifier, "base", base); @@ -449,9 +443,9 @@ fn renderExpression( switch (op_tok_id) { .Asterisk, .AsteriskAsterisk => try stream.writeByte('*'), .LBracket => if (tree.tokens.at(prefix_op_node.op_token + 2).id == .Identifier) - try stream.write("[*c") + try stream.writeAll("[*c") else - try stream.write("[*"), + try stream.writeAll("[*"), else => unreachable, } if (ptr_info.sentinel) |sentinel| { @@ -757,7 +751,7 @@ fn renderExpression( while (it.next()) |field_init| { var find_stream = FindByteOutStream.init('\n'); var dummy_col: usize = 0; - try renderExpression(allocator, &find_stream.stream, tree, 0, &dummy_col, field_init.*, Space.None); + try renderExpression(allocator, find_stream.outStream(), tree, 0, &dummy_col, field_init.*, Space.None); if (find_stream.byte_found) break :blk false; } break :blk true; @@ -909,8 +903,7 @@ fn renderExpression( var column_widths = widths[widths.len - row_size ..]; // Null stream for counting the printed length of each expression - var null_stream = std.io.NullOutStream.init(); - var counting_stream = std.io.CountingOutStream(std.io.NullOutStream.Error).init(&null_stream.stream); + var counting_stream = std.io.CountingOutStream(@TypeOf(std.io.null_out_stream)).init(std.io.null_out_stream); var it = exprs.iterator(0); var i: usize = 0; @@ -918,7 +911,7 @@ fn renderExpression( while (it.next()) |expr| : (i += 1) { counting_stream.bytes_written = 0; var dummy_col: usize = 0; - try renderExpression(allocator, &counting_stream.stream, tree, indent, &dummy_col, expr.*, Space.None); + try renderExpression(allocator, counting_stream.outStream(), tree, indent, &dummy_col, expr.*, Space.None); const width = @intCast(usize, counting_stream.bytes_written); const col = i % row_size; column_widths[col] = std.math.max(column_widths[col], width); @@ -1336,7 +1329,7 @@ fn renderExpression( // TODO: Remove condition after deprecating 'typeOf'. See https://github.com/ziglang/zig/issues/1348 if (mem.eql(u8, tree.tokenSlicePtr(tree.tokens.at(builtin_call.builtin_token)), "@typeOf")) { - try stream.write("@TypeOf"); + try stream.writeAll("@TypeOf"); } else { try renderToken(tree, stream, builtin_call.builtin_token, indent, start_col, Space.None); // @name } @@ -1505,9 +1498,9 @@ fn renderExpression( try renderExpression(allocator, stream, tree, indent, start_col, callconv_expr, Space.None); try renderToken(tree, stream, callconv_rparen, indent, start_col, Space.Space); // ) } else if (cc_rewrite_str) |str| { - try stream.write("callconv("); - try stream.write(mem.toSliceConst(u8, str)); - try stream.write(") "); + try stream.writeAll("callconv("); + try stream.writeAll(mem.toSliceConst(u8, str)); + try stream.writeAll(") "); } switch (fn_proto.return_type) { @@ -1997,11 +1990,11 @@ fn renderExpression( .AsmInput => { const asm_input = @fieldParentPtr(ast.Node.AsmInput, "base", base); - try stream.write("["); + try stream.writeAll("["); try renderExpression(allocator, stream, tree, indent, start_col, asm_input.symbolic_name, Space.None); - try stream.write("] "); + try stream.writeAll("] "); try renderExpression(allocator, stream, tree, indent, start_col, asm_input.constraint, Space.None); - try stream.write(" ("); + try stream.writeAll(" ("); try renderExpression(allocator, stream, tree, indent, start_col, asm_input.expr, Space.None); return renderToken(tree, stream, asm_input.lastToken(), indent, start_col, space); // ) }, @@ -2009,18 +2002,18 @@ fn renderExpression( .AsmOutput => { const asm_output = @fieldParentPtr(ast.Node.AsmOutput, "base", base); - try stream.write("["); + try stream.writeAll("["); try renderExpression(allocator, stream, tree, indent, start_col, asm_output.symbolic_name, Space.None); - try stream.write("] "); + try stream.writeAll("] "); try renderExpression(allocator, stream, tree, indent, start_col, asm_output.constraint, Space.None); - try stream.write(" ("); + try stream.writeAll(" ("); switch (asm_output.kind) { ast.Node.AsmOutput.Kind.Variable => |variable_name| { try renderExpression(allocator, stream, tree, indent, start_col, &variable_name.base, Space.None); }, ast.Node.AsmOutput.Kind.Return => |return_type| { - try stream.write("-> "); + try stream.writeAll("-> "); try renderExpression(allocator, stream, tree, indent, start_col, return_type, Space.None); }, } @@ -2052,7 +2045,7 @@ fn renderVarDecl( indent: usize, start_col: *usize, var_decl: *ast.Node.VarDecl, -) (@TypeOf(stream).Child.Error || Error)!void { +) (@TypeOf(stream).Error || Error)!void { if (var_decl.visib_token) |visib_token| { try renderToken(tree, stream, visib_token, indent, start_col, Space.Space); // pub } @@ -2125,7 +2118,7 @@ fn renderParamDecl( start_col: *usize, base: *ast.Node, space: Space, -) (@TypeOf(stream).Child.Error || Error)!void { +) (@TypeOf(stream).Error || Error)!void { const param_decl = @fieldParentPtr(ast.Node.ParamDecl, "base", base); try renderDocComments(tree, stream, param_decl, indent, start_col); @@ -2154,7 +2147,7 @@ fn renderStatement( indent: usize, start_col: *usize, base: *ast.Node, -) (@TypeOf(stream).Child.Error || Error)!void { +) (@TypeOf(stream).Error || Error)!void { switch (base.id) { .VarDecl => { const var_decl = @fieldParentPtr(ast.Node.VarDecl, "base", base); @@ -2193,7 +2186,7 @@ fn renderTokenOffset( start_col: *usize, space: Space, token_skip_bytes: usize, -) (@TypeOf(stream).Child.Error || Error)!void { +) (@TypeOf(stream).Error || Error)!void { if (space == Space.BlockStart) { if (start_col.* < indent + indent_delta) return renderToken(tree, stream, token_index, indent, start_col, Space.Space); @@ -2204,7 +2197,7 @@ fn renderTokenOffset( } var token = tree.tokens.at(token_index); - try stream.write(mem.trimRight(u8, tree.tokenSlicePtr(token)[token_skip_bytes..], " ")); + try stream.writeAll(mem.trimRight(u8, tree.tokenSlicePtr(token)[token_skip_bytes..], " ")); if (space == Space.NoComment) return; @@ -2214,15 +2207,15 @@ fn renderTokenOffset( if (space == Space.Comma) switch (next_token.id) { .Comma => return renderToken(tree, stream, token_index + 1, indent, start_col, Space.Newline), .LineComment => { - try stream.write(", "); + try stream.writeAll(", "); return renderToken(tree, stream, token_index + 1, indent, start_col, Space.Newline); }, else => { if (token_index + 2 < tree.tokens.len and tree.tokens.at(token_index + 2).id == .MultilineStringLiteralLine) { - try stream.write(","); + try stream.writeAll(","); return; } else { - try stream.write(",\n"); + try stream.writeAll(",\n"); start_col.* = 0; return; } @@ -2246,7 +2239,7 @@ fn renderTokenOffset( if (next_token.id == .MultilineStringLiteralLine) { return; } else { - try stream.write("\n"); + try stream.writeAll("\n"); start_col.* = 0; return; } @@ -2309,7 +2302,7 @@ fn renderTokenOffset( if (next_token.id == .MultilineStringLiteralLine) { return; } else { - try stream.write("\n"); + try stream.writeAll("\n"); start_col.* = 0; return; } @@ -2327,7 +2320,7 @@ fn renderTokenOffset( const newline_count = if (loc.line == 1) @as(u8, 1) else @as(u8, 2); try stream.writeByteNTimes('\n', newline_count); try stream.writeByteNTimes(' ', indent); - try stream.write(mem.trimRight(u8, tree.tokenSlicePtr(next_token), " ")); + try stream.writeAll(mem.trimRight(u8, tree.tokenSlicePtr(next_token), " ")); offset += 1; token = next_token; @@ -2338,7 +2331,7 @@ fn renderTokenOffset( if (next_token.id == .MultilineStringLiteralLine) { return; } else { - try stream.write("\n"); + try stream.writeAll("\n"); start_col.* = 0; return; } @@ -2381,7 +2374,7 @@ fn renderToken( indent: usize, start_col: *usize, space: Space, -) (@TypeOf(stream).Child.Error || Error)!void { +) (@TypeOf(stream).Error || Error)!void { return renderTokenOffset(tree, stream, token_index, indent, start_col, space, 0); } @@ -2391,7 +2384,7 @@ fn renderDocComments( node: var, indent: usize, start_col: *usize, -) (@TypeOf(stream).Child.Error || Error)!void { +) (@TypeOf(stream).Error || Error)!void { const comment = node.doc_comments orelse return; var it = comment.lines.iterator(0); const first_token = node.firstToken(); @@ -2401,7 +2394,7 @@ fn renderDocComments( try stream.writeByteNTimes(' ', indent); } else { try renderToken(tree, stream, line_token_index.*, indent, start_col, Space.NoComment); - try stream.write("\n"); + try stream.writeAll("\n"); try stream.writeByteNTimes(' ', indent); } } @@ -2427,27 +2420,23 @@ fn nodeCausesSliceOpSpace(base: *ast.Node) bool { }; } -// An OutStream that returns whether the given character has been written to it. -// The contents are not written to anything. +/// A `std.io.OutStream` that returns whether the given character has been written to it. +/// The contents are not written to anything. const FindByteOutStream = struct { - const Self = FindByteOutStream; - pub const Error = error{}; - pub const Stream = std.io.OutStream(Error); - - stream: Stream, byte_found: bool, byte: u8, - pub fn init(byte: u8) Self { - return Self{ - .stream = Stream{ .writeFn = writeFn }, + pub const Error = error{}; + pub const OutStream = std.io.OutStream(*FindByteOutStream, Error, write); + + pub fn init(byte: u8) FindByteOutStream { + return FindByteOutStream{ .byte = byte, .byte_found = false, }; } - fn writeFn(out_stream: *Stream, bytes: []const u8) Error!usize { - const self = @fieldParentPtr(Self, "stream", out_stream); + pub fn write(self: *FindByteOutStream, bytes: []const u8) Error!usize { if (self.byte_found) return bytes.len; self.byte_found = blk: { for (bytes) |b| @@ -2456,11 +2445,15 @@ const FindByteOutStream = struct { }; return bytes.len; } + + pub fn outStream(self: *FindByteOutStream) OutStream { + return .{ .context = self }; + } }; -fn copyFixingWhitespace(stream: var, slice: []const u8) @TypeOf(stream).Child.Error!void { +fn copyFixingWhitespace(stream: var, slice: []const u8) @TypeOf(stream).Error!void { for (slice) |byte| switch (byte) { - '\t' => try stream.write(" "), + '\t' => try stream.writeAll(" "), '\r' => {}, else => try stream.writeByte(byte), }; diff --git a/src-self-hosted/libc_installation.zig b/src-self-hosted/libc_installation.zig index 41def38126..78aafeff28 100644 --- a/src-self-hosted/libc_installation.zig +++ b/src-self-hosted/libc_installation.zig @@ -38,7 +38,7 @@ pub const LibCInstallation = struct { pub fn parse( allocator: *Allocator, libc_file: []const u8, - stderr: *std.io.OutStream(fs.File.WriteError), + stderr: var, ) !LibCInstallation { var self: LibCInstallation = .{}; @@ -123,7 +123,7 @@ pub const LibCInstallation = struct { return self; } - pub fn render(self: LibCInstallation, out: *std.io.OutStream(fs.File.WriteError)) !void { + pub fn render(self: LibCInstallation, out: var) !void { @setEvalBranchQuota(4000); const include_dir = self.include_dir orelse ""; const sys_include_dir = self.sys_include_dir orelse ""; diff --git a/src-self-hosted/print_targets.zig b/src-self-hosted/print_targets.zig index bb3841ba24..473e3dfa51 100644 --- a/src-self-hosted/print_targets.zig +++ b/src-self-hosted/print_targets.zig @@ -52,7 +52,7 @@ const available_libcs = [_][]const u8{ "sparc-linux-gnu", "sparcv9-linux-gnu", "wasm32-freestanding-musl", - "x86_64-linux-gnu (native)", + "x86_64-linux-gnu", "x86_64-linux-gnux32", "x86_64-linux-musl", "x86_64-windows-gnu", @@ -61,7 +61,8 @@ const available_libcs = [_][]const u8{ pub fn cmdTargets( allocator: *Allocator, args: []const []const u8, - stdout: *io.OutStream(fs.File.WriteError), + /// Output stream + stdout: var, native_target: Target, ) !void { const available_glibcs = blk: { @@ -92,9 +93,9 @@ pub fn cmdTargets( }; defer allocator.free(available_glibcs); - const BOS = io.BufferedOutStream(fs.File.WriteError); - var bos = BOS.init(stdout); - var jws = std.json.WriteStream(BOS.Stream, 6).init(&bos.stream); + var bos = io.bufferedOutStream(4096, stdout); + const bos_stream = bos.outStream(); + var jws = std.json.WriteStream(@TypeOf(bos_stream), 6).init(bos_stream); try jws.beginObject(); @@ -219,6 +220,6 @@ pub fn cmdTargets( try jws.endObject(); - try bos.stream.writeByte('\n'); + try bos_stream.writeByte('\n'); return bos.flush(); } diff --git a/src-self-hosted/stage2.zig b/src-self-hosted/stage2.zig index ee6789ab86..8f2656de44 100644 --- a/src-self-hosted/stage2.zig +++ b/src-self-hosted/stage2.zig @@ -18,8 +18,8 @@ const assert = std.debug.assert; const LibCInstallation = @import("libc_installation.zig").LibCInstallation; var stderr_file: fs.File = undefined; -var stderr: *io.OutStream(fs.File.WriteError) = undefined; -var stdout: *io.OutStream(fs.File.WriteError) = undefined; +var stderr: fs.File.OutStream = undefined; +var stdout: fs.File.OutStream = undefined; comptime { _ = @import("dep_tokenizer.zig"); @@ -146,7 +146,7 @@ export fn stage2_free_clang_errors(errors_ptr: [*]translate_c.ClangErrMsg, error } export fn stage2_render_ast(tree: *ast.Tree, output_file: *FILE) Error { - const c_out_stream = &std.io.COutStream.init(output_file).stream; + const c_out_stream = std.io.cOutStream(output_file); _ = std.zig.render(std.heap.c_allocator, c_out_stream, tree) catch |e| switch (e) { error.WouldBlock => unreachable, // stage1 opens stuff in exclusively blocking mode error.SystemResources => return .SystemResources, @@ -186,9 +186,9 @@ fn fmtMain(argc: c_int, argv: [*]const [*:0]const u8) !void { try args_list.append(mem.toSliceConst(u8, argv[arg_i])); } - stdout = &std.io.getStdOut().outStream().stream; + stdout = std.io.getStdOut().outStream(); stderr_file = std.io.getStdErr(); - stderr = &stderr_file.outStream().stream; + stderr = stderr_file.outStream(); const args = args_list.toSliceConst()[2..]; @@ -203,11 +203,11 @@ fn fmtMain(argc: c_int, argv: [*]const [*:0]const u8) !void { const arg = args[i]; if (mem.startsWith(u8, arg, "-")) { if (mem.eql(u8, arg, "--help")) { - try stdout.write(self_hosted_main.usage_fmt); + try stdout.writeAll(self_hosted_main.usage_fmt); process.exit(0); } else if (mem.eql(u8, arg, "--color")) { if (i + 1 >= args.len) { - try stderr.write("expected [auto|on|off] after --color\n"); + try stderr.writeAll("expected [auto|on|off] after --color\n"); process.exit(1); } i += 1; @@ -238,14 +238,14 @@ fn fmtMain(argc: c_int, argv: [*]const [*:0]const u8) !void { if (stdin_flag) { if (input_files.len != 0) { - try stderr.write("cannot use --stdin with positional arguments\n"); + try stderr.writeAll("cannot use --stdin with positional arguments\n"); process.exit(1); } const stdin_file = io.getStdIn(); var stdin = stdin_file.inStream(); - const source_code = try stdin.stream.readAllAlloc(allocator, self_hosted_main.max_src_size); + const source_code = try stdin.readAllAlloc(allocator, self_hosted_main.max_src_size); defer allocator.free(source_code); const tree = std.zig.parse(allocator, source_code) catch |err| { @@ -272,7 +272,7 @@ fn fmtMain(argc: c_int, argv: [*]const [*:0]const u8) !void { } if (input_files.len == 0) { - try stderr.write("expected at least one source file argument\n"); + try stderr.writeAll("expected at least one source file argument\n"); process.exit(1); } @@ -409,11 +409,11 @@ fn printErrMsgToFile( const end_loc = tree.tokenLocationPtr(first_token.end, last_token); var text_buf = try std.Buffer.initSize(allocator, 0); - var out_stream = &std.io.BufferOutStream.init(&text_buf).stream; + const out_stream = &text_buf.outStream(); try parse_error.render(&tree.tokens, out_stream); const text = text_buf.toOwnedSlice(); - const stream = &file.outStream().stream; + const stream = &file.outStream(); try stream.print("{}:{}:{}: error: {}\n", .{ path, start_loc.line + 1, start_loc.column + 1, text }); if (!color_on) return; @@ -641,7 +641,7 @@ fn cmdTargets(zig_triple: [*:0]const u8) !void { return @import("print_targets.zig").cmdTargets( std.heap.c_allocator, &[0][]u8{}, - &std.io.getStdOut().outStream().stream, + std.io.getStdOut().outStream(), target, ); } @@ -808,7 +808,7 @@ const Stage2LibCInstallation = extern struct { // ABI warning export fn stage2_libc_parse(stage1_libc: *Stage2LibCInstallation, libc_file_z: [*:0]const u8) Error { stderr_file = std.io.getStdErr(); - stderr = &stderr_file.outStream().stream; + stderr = stderr_file.outStream(); const libc_file = mem.toSliceConst(u8, libc_file_z); var libc = LibCInstallation.parse(std.heap.c_allocator, libc_file, stderr) catch |err| switch (err) { error.ParseError => return .SemanticAnalyzeFail, @@ -870,7 +870,7 @@ export fn stage2_libc_find_native(stage1_libc: *Stage2LibCInstallation) Error { // ABI warning export fn stage2_libc_render(stage1_libc: *Stage2LibCInstallation, output_file: *FILE) Error { var libc = stage1_libc.toStage2(); - const c_out_stream = &std.io.COutStream.init(output_file).stream; + const c_out_stream = std.io.cOutStream(output_file); libc.render(c_out_stream) catch |err| switch (err) { error.WouldBlock => unreachable, // stage1 opens stuff in exclusively blocking mode error.SystemResources => return .SystemResources, From b6fbd524f122449e6e2bb4d73ce3f59b01286f50 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 10 Mar 2020 16:31:04 -0400 Subject: [PATCH 02/15] (breaking) improve and simplify fixed buffer streams API --- lib/std/io.zig | 156 +------------------------ lib/std/io/buffered_atomic_file.zig | 4 +- lib/std/io/buffered_in_stream.zig | 84 ++++++++++++++ lib/std/io/buffered_out_stream.zig | 23 +--- lib/std/io/fixed_buffer_stream.zig | 173 +++++++++++++++++++--------- lib/std/json.zig | 4 +- lib/std/progress.zig | 2 +- lib/std/std.zig | 1 - src-self-hosted/print_targets.zig | 2 +- 9 files changed, 218 insertions(+), 231 deletions(-) create mode 100644 lib/std/io/buffered_in_stream.zig diff --git a/lib/std/io.zig b/lib/std/io.zig index dcb0551dc9..1283461271 100644 --- a/lib/std/io.zig +++ b/lib/std/io.zig @@ -95,18 +95,18 @@ pub fn getStdIn() File { pub const SeekableStream = @import("io/seekable_stream.zig").SeekableStream; pub const InStream = @import("io/in_stream.zig").InStream; pub const OutStream = @import("io/out_stream.zig").OutStream; -pub const FixedBufferInStream = @import("io/fixed_buffer_stream.zig").FixedBufferInStream; pub const BufferedAtomicFile = @import("io/buffered_atomic_file.zig").BufferedAtomicFile; pub const BufferedOutStream = @import("io/buffered_out_stream.zig").BufferedOutStream; -pub const BufferedOutStreamCustom = @import("io/buffered_out_stream.zig").BufferedOutStreamCustom; pub const bufferedOutStream = @import("io/buffered_out_stream.zig").bufferedOutStream; -pub const CountingOutStream = @import("io/counting_out_stream.zig").CountingOutStream; +pub const BufferedInStream = @import("io/buffered_in_stream.zig").BufferedInStream; +pub const bufferedInStream = @import("io/buffered_in_stream.zig").bufferedInStream; -pub fn fixedBufferStream(bytes: []const u8) FixedBufferInStream { - return (FixedBufferInStream{ .bytes = bytes, .pos = 0 }); -} +pub const FixedBufferStream = @import("io/fixed_buffer_stream.zig").FixedBufferStream; +pub const fixedBufferStream = @import("io/fixed_buffer_stream.zig").fixedBufferStream; + +pub const CountingOutStream = @import("io/counting_out_stream.zig").CountingOutStream; pub fn cOutStream(c_file: *std.c.FILE) COutStream { return .{ .context = c_file }; @@ -144,92 +144,6 @@ pub fn readFileAlloc(allocator: *mem.Allocator, path: []const u8) ![]u8 { return fs.cwd().readFileAlloc(allocator, path, math.maxInt(usize)); } -pub fn BufferedInStream(comptime Error: type) type { - return BufferedInStreamCustom(mem.page_size, Error); -} - -pub fn BufferedInStreamCustom(comptime buffer_size: usize, comptime Error: type) type { - return struct { - const Self = @This(); - const Stream = InStream(Error); - - stream: Stream, - - unbuffered_in_stream: *Stream, - - const FifoType = std.fifo.LinearFifo(u8, std.fifo.LinearFifoBufferType{ .Static = buffer_size }); - fifo: FifoType, - - pub fn init(unbuffered_in_stream: *Stream) Self { - return Self{ - .unbuffered_in_stream = unbuffered_in_stream, - .fifo = FifoType.init(), - .stream = Stream{ .readFn = readFn }, - }; - } - - fn readFn(in_stream: *Stream, dest: []u8) !usize { - const self = @fieldParentPtr(Self, "stream", in_stream); - var dest_index: usize = 0; - while (dest_index < dest.len) { - const written = self.fifo.read(dest[dest_index..]); - if (written == 0) { - // fifo empty, fill it - const writable = self.fifo.writableSlice(0); - assert(writable.len > 0); - const n = try self.unbuffered_in_stream.read(writable); - if (n == 0) { - // reading from the unbuffered stream returned nothing - // so we have nothing left to read. - return dest_index; - } - self.fifo.update(n); - } - dest_index += written; - } - return dest.len; - } - }; -} - -test "io.BufferedInStream" { - const OneByteReadInStream = struct { - const Error = error{NoError}; - const Stream = InStream(Error); - - stream: Stream, - str: []const u8, - curr: usize, - - fn init(str: []const u8) @This() { - return @This(){ - .stream = Stream{ .readFn = readFn }, - .str = str, - .curr = 0, - }; - } - - fn readFn(in_stream: *Stream, dest: []u8) Error!usize { - const self = @fieldParentPtr(@This(), "stream", in_stream); - if (self.str.len <= self.curr or dest.len == 0) - return 0; - - dest[0] = self.str[self.curr]; - self.curr += 1; - return 1; - } - }; - - const str = "This is a test"; - var one_byte_stream = OneByteReadInStream.init(str); - var buf_in_stream = BufferedInStream(OneByteReadInStream.Error).init(&one_byte_stream.stream); - const stream = &buf_in_stream.stream; - - const res = try stream.readAllAlloc(testing.allocator, str.len + 1); - defer testing.allocator.free(res); - testing.expectEqualSlices(u8, str, res); -} - /// Creates a stream which supports 'un-reading' data, so that it can be read again. /// This makes look-ahead style parsing much easier. pub fn PeekStream(comptime buffer_type: std.fifo.LinearFifoBufferType, comptime InStreamError: type) type { @@ -473,64 +387,6 @@ pub fn BitInStream(endian: builtin.Endian, comptime Error: type) type { }; } -/// This is a simple OutStream that writes to a fixed buffer. If the returned number -/// of bytes written is less than requested, the buffer is full. -/// Returns error.OutOfMemory when no bytes would be written. -pub const SliceOutStream = struct { - pub const Error = error{OutOfMemory}; - pub const Stream = OutStream(Error); - - stream: Stream, - - pos: usize, - slice: []u8, - - pub fn init(slice: []u8) SliceOutStream { - return SliceOutStream{ - .slice = slice, - .pos = 0, - .stream = Stream{ .writeFn = writeFn }, - }; - } - - pub fn getWritten(self: *const SliceOutStream) []const u8 { - return self.slice[0..self.pos]; - } - - pub fn reset(self: *SliceOutStream) void { - self.pos = 0; - } - - fn writeFn(out_stream: *Stream, bytes: []const u8) Error!usize { - const self = @fieldParentPtr(SliceOutStream, "stream", out_stream); - - if (bytes.len == 0) return 0; - - assert(self.pos <= self.slice.len); - - const n = if (self.pos + bytes.len <= self.slice.len) - bytes.len - else - self.slice.len - self.pos; - - std.mem.copy(u8, self.slice[self.pos .. self.pos + n], bytes[0..n]); - self.pos += n; - - if (n == 0) return error.OutOfMemory; - - return n; - } -}; - -test "io.SliceOutStream" { - var buf: [255]u8 = undefined; - var slice_stream = SliceOutStream.init(buf[0..]); - const stream = &slice_stream.stream; - - try stream.print("{}{}!", .{ "Hello", "World" }); - testing.expectEqualSlices(u8, "HelloWorld!", slice_stream.getWritten()); -} - /// An OutStream that doesn't write to anything. pub const null_out_stream = @as(NullOutStream, .{ .context = {} }); diff --git a/lib/std/io/buffered_atomic_file.zig b/lib/std/io/buffered_atomic_file.zig index 48c550c90f..e2f8c75af0 100644 --- a/lib/std/io/buffered_atomic_file.zig +++ b/lib/std/io/buffered_atomic_file.zig @@ -10,7 +10,7 @@ pub const BufferedAtomicFile = struct { allocator: *mem.Allocator, pub const buffer_size = 4096; - pub const BufferedOutStream = std.io.BufferedOutStreamCustom(buffer_size, File.OutStream); + pub const BufferedOutStream = std.io.BufferedOutStream(buffer_size, File.OutStream); pub const OutStream = std.io.OutStream(*BufferedOutStream, BufferedOutStream.Error, BufferedOutStream.write); /// TODO when https://github.com/ziglang/zig/issues/2761 is solved @@ -29,7 +29,7 @@ pub const BufferedAtomicFile = struct { errdefer self.atomic_file.deinit(); self.file_stream = self.atomic_file.file.outStream(); - self.buffered_stream = std.io.bufferedOutStream(buffer_size, self.file_stream); + self.buffered_stream = .{ .unbuffered_out_stream = self.file_stream }; return self; } diff --git a/lib/std/io/buffered_in_stream.zig b/lib/std/io/buffered_in_stream.zig new file mode 100644 index 0000000000..72f1b30b76 --- /dev/null +++ b/lib/std/io/buffered_in_stream.zig @@ -0,0 +1,84 @@ +const std = @import("../std.zig"); +const io = std.io; + +pub fn BufferedInStream(comptime buffer_size: usize, comptime InStreamType) type { + return struct { + unbuffered_in_stream: InStreamType, + fifo: FifoType = FifoType.init(), + + pub const Error = InStreamType.Error; + pub const InStream = io.InStream(*Self, Error, read); + + const Self = @This(); + const FifoType = std.fifo.LinearFifo(u8, std.fifo.LinearFifoBufferType{ .Static = buffer_size }); + + pub fn read(self: *Self, dest: []u8) Error!usize { + var dest_index: usize = 0; + while (dest_index < dest.len) { + const written = self.fifo.read(dest[dest_index..]); + if (written == 0) { + // fifo empty, fill it + const writable = self.fifo.writableSlice(0); + assert(writable.len > 0); + const n = try self.unbuffered_in_stream.read(writable); + if (n == 0) { + // reading from the unbuffered stream returned nothing + // so we have nothing left to read. + return dest_index; + } + self.fifo.update(n); + } + dest_index += written; + } + return dest.len; + } + + pub fn inStream(self: *Self) InStream { + return .{ .context = self }; + } + }; +} + +pub fn bufferedInStream(underlying_stream: var) BufferedInStream(4096, @TypeOf(underlying_stream)) { + return .{ .unbuffered_in_stream = underlying_stream }; +} + +test "io.BufferedInStream" { + const OneByteReadInStream = struct { + str: []const u8, + curr: usize, + + const Error = error{NoError}; + const Self = @This(); + const InStream = io.InStream(*Self, Error, read); + + fn init(str: []const u8) Self { + return Self{ + .str = str, + .curr = 0, + }; + } + + fn read(self: *Self, dest: []u8) Error!usize { + if (self.str.len <= self.curr or dest.len == 0) + return 0; + + dest[0] = self.str[self.curr]; + self.curr += 1; + return 1; + } + + fn inStream(self: *Self) InStream { + return .{ .context = self }; + } + }; + + const str = "This is a test"; + var one_byte_stream = OneByteReadInStream.init(str); + var buf_in_stream = bufferedInStream(one_byte_stream.inStream()); + const stream = buf_in_stream.inStream(); + + const res = try stream.readAllAlloc(testing.allocator, str.len + 1); + defer testing.allocator.free(res); + testing.expectEqualSlices(u8, str, res); +} diff --git a/lib/std/io/buffered_out_stream.zig b/lib/std/io/buffered_out_stream.zig index 3650022b29..43af83c537 100644 --- a/lib/std/io/buffered_out_stream.zig +++ b/lib/std/io/buffered_out_stream.zig @@ -1,14 +1,10 @@ const std = @import("../std.zig"); const io = std.io; -pub fn BufferedOutStream(comptime OutStreamType: type) type { - return BufferedOutStreamCustom(4096, OutStreamType); -} - -pub fn BufferedOutStreamCustom(comptime buffer_size: usize, comptime OutStreamType: type) type { +pub fn BufferedOutStream(comptime buffer_size: usize, comptime OutStreamType: type) type { return struct { unbuffered_out_stream: OutStreamType, - fifo: FifoType, + fifo: FifoType = FifoType.init(), pub const Error = OutStreamType.Error; pub const OutStream = io.OutStream(*Self, Error, write); @@ -16,13 +12,6 @@ pub fn BufferedOutStreamCustom(comptime buffer_size: usize, comptime OutStreamTy const Self = @This(); const FifoType = std.fifo.LinearFifo(u8, std.fifo.LinearFifoBufferType{ .Static = buffer_size }); - pub fn init(unbuffered_out_stream: OutStreamType) Self { - return Self{ - .unbuffered_out_stream = unbuffered_out_stream, - .fifo = FifoType.init(), - }; - } - pub fn flush(self: *Self) !void { while (true) { const slice = self.fifo.readableSlice(0); @@ -47,10 +36,6 @@ pub fn BufferedOutStreamCustom(comptime buffer_size: usize, comptime OutStreamTy }; } -pub fn bufferedOutStream( - comptime buffer_size: usize, - underlying_stream: var, -) BufferedOutStreamCustom(buffer_size, @TypeOf(underlying_stream)) { - return BufferedOutStreamCustom(buffer_size, @TypeOf(underlying_stream)).init(underlying_stream); +pub fn bufferedOutStream(underlying_stream: var) BufferedOutStream(4096, @TypeOf(underlying_stream)) { + return .{ .unbuffered_out_stream = underlying_stream }; } - diff --git a/lib/std/io/fixed_buffer_stream.zig b/lib/std/io/fixed_buffer_stream.zig index 847feeae03..75a337d3de 100644 --- a/lib/std/io/fixed_buffer_stream.zig +++ b/lib/std/io/fixed_buffer_stream.zig @@ -1,66 +1,129 @@ const std = @import("../std.zig"); const io = std.io; +const testing = std.testing; -pub const FixedBufferInStream = struct { - bytes: []const u8, - pos: usize, +/// This turns a slice into an `io.OutStream`, `io.InStream`, or `io.SeekableStream`. +/// If the supplied slice is const, then `io.OutStream` is not available. +pub fn FixedBufferStream(comptime Buffer: type) type { + return struct { + /// `Buffer` is either a `[]u8` or `[]const u8`. + buffer: Buffer, + pos: usize, - pub const SeekError = error{EndOfStream}; - pub const GetSeekPosError = error{}; + pub const ReadError = error{EndOfStream}; + pub const WriteError = error{OutOfMemory}; + pub const SeekError = error{EndOfStream}; + pub const GetSeekPosError = error{}; - pub const InStream = io.InStream(*FixedBufferInStream, error{}, read); + pub const InStream = io.InStream(*Self, ReadError, read); + pub const OutStream = io.OutStream(*Self, WriteError, write); - pub fn inStream(self: *FixedBufferInStream) InStream { - return .{ .context = self }; - } + pub const SeekableStream = io.SeekableStream( + *Self, + SeekError, + GetSeekPosError, + seekTo, + seekBy, + getPos, + getEndPos, + ); - pub const SeekableStream = io.SeekableStream( - *FixedBufferInStream, - SeekError, - GetSeekPosError, - seekTo, - seekBy, - getPos, - getEndPos, - ); + const Self = @This(); - pub fn seekableStream(self: *FixedBufferInStream) SeekableStream { - return .{ .context = self }; - } - - pub fn read(self: *FixedBufferInStream, dest: []u8) error{}!usize { - const size = std.math.min(dest.len, self.bytes.len - self.pos); - const end = self.pos + size; - - std.mem.copy(u8, dest[0..size], self.bytes[self.pos..end]); - self.pos = end; - - return size; - } - - pub fn seekTo(self: *FixedBufferInStream, pos: u64) SeekError!void { - const usize_pos = std.math.cast(usize, pos) catch return error.EndOfStream; - if (usize_pos > self.bytes.len) return error.EndOfStream; - self.pos = usize_pos; - } - - pub fn seekBy(self: *FixedBufferInStream, amt: i64) SeekError!void { - if (amt < 0) { - const abs_amt = std.math.cast(usize, -amt) catch return error.EndOfStream; - if (abs_amt > self.pos) return error.EndOfStream; - self.pos -= abs_amt; - } else { - const usize_amt = std.math.cast(usize, amt) catch return error.EndOfStream; - if (self.pos + usize_amt > self.bytes.len) return error.EndOfStream; - self.pos += usize_amt; + pub fn inStream(self: *Self) InStream { + return .{ .context = self }; } - } - pub fn getEndPos(self: *FixedBufferInStream) GetSeekPosError!u64 { - return self.bytes.len; - } + pub fn outStream(self: *Self) OutStream { + return .{ .context = self }; + } - pub fn getPos(self: *FixedBufferInStream) GetSeekPosError!u64 { - return self.pos; - } -}; + pub fn seekableStream(self: *Self) SeekableStream { + return .{ .context = self }; + } + + pub fn read(self: *Self, dest: []u8) ReadError!usize { + const size = std.math.min(dest.len, self.buffer.len - self.pos); + const end = self.pos + size; + + std.mem.copy(u8, dest[0..size], self.buffer[self.pos..end]); + self.pos = end; + + if (size == 0) return error.EndOfStream; + return size; + } + + /// If the returned number of bytes written is less than requested, the + /// buffer is full. Returns `error.OutOfMemory` when no bytes would be written. + pub fn write(self: *Self, bytes: []const u8) WriteError!usize { + if (bytes.len == 0) return 0; + + assert(self.pos <= self.buffer.len); + + const n = if (self.pos + bytes.len <= self.buffer.len) + bytes.len + else + self.buffer.len - self.pos; + + std.mem.copy(u8, self.buffer[self.pos .. self.pos + n], bytes[0..n]); + self.pos += n; + + if (n == 0) return error.OutOfMemory; + + return n; + } + + pub fn seekTo(self: *Self, pos: u64) SeekError!void { + const usize_pos = std.math.cast(usize, pos) catch return error.EndOfStream; + if (usize_pos > self.buffer.len) return error.EndOfStream; + self.pos = usize_pos; + } + + pub fn seekBy(self: *Self, amt: i64) SeekError!void { + if (amt < 0) { + const abs_amt = std.math.cast(usize, -amt) catch return error.EndOfStream; + if (abs_amt > self.pos) return error.EndOfStream; + self.pos -= abs_amt; + } else { + const usize_amt = std.math.cast(usize, amt) catch return error.EndOfStream; + if (self.pos + usize_amt > self.buffer.len) return error.EndOfStream; + self.pos += usize_amt; + } + } + + pub fn getEndPos(self: *Self) GetSeekPosError!u64 { + return self.buffer.len; + } + + pub fn getPos(self: *Self) GetSeekPosError!u64 { + return self.pos; + } + + pub fn getWritten(self: Self) []const u8 { + return self.slice[0..self.pos]; + } + + pub fn reset(self: *Self) void { + self.pos = 0; + } + }; +} + +pub fn fixedBufferStream(buffer: var) FixedBufferStream(NonSentinelSpan(@TypeOf(buffer))) { + return .{ .buffer = std.mem.span(buffer), .pos = 0 }; +} + +fn NonSentinelSpan(comptime T: type) type { + var ptr_info = @typeInfo(std.mem.Span(T)).Pointer; + ptr_info.sentinel = null; + return @Type(std.builtin.TypeInfo{ .Pointer = ptr_info }); +} + +test "FixedBufferStream" { + var buf: [255]u8 = undefined; + var fbs = fixedBufferStream(&buf); + const stream = fbs.outStream(); + + try stream.print("{}{}!", .{ "Hello", "World" }); + testing.expectEqualSlices(u8, "HelloWorld!", fbs.getWritten()); +} diff --git a/lib/std/json.zig b/lib/std/json.zig index bb59b4e0f3..f2cf68d7eb 100644 --- a/lib/std/json.zig +++ b/lib/std/json.zig @@ -2107,8 +2107,8 @@ test "import more json tests" { test "write json then parse it" { var out_buffer: [1000]u8 = undefined; - var slice_out_stream = std.io.SliceOutStream.init(&out_buffer); - const out_stream = &slice_out_stream.stream; + var fixed_buffer_stream = std.io.fixedBufferStream(&out_buffer); + const out_stream = fixed_buffer_stream.outStream(); var jw = WriteStream(@TypeOf(out_stream).Child, 4).init(out_stream); try jw.beginObject(); diff --git a/lib/std/progress.zig b/lib/std/progress.zig index 1c5ecd261a..0264d99c91 100644 --- a/lib/std/progress.zig +++ b/lib/std/progress.zig @@ -177,7 +177,7 @@ pub const Progress = struct { pub fn log(self: *Progress, comptime format: []const u8, args: var) void { const file = self.terminal orelse return; self.refresh(); - file.outStream().stream.print(format, args) catch { + file.outStream().print(format, args) catch { self.terminal = null; return; }; diff --git a/lib/std/std.zig b/lib/std/std.zig index dd4d968efb..9277370ca6 100644 --- a/lib/std/std.zig +++ b/lib/std/std.zig @@ -5,7 +5,6 @@ pub const BloomFilter = @import("bloom_filter.zig").BloomFilter; pub const BufMap = @import("buf_map.zig").BufMap; pub const BufSet = @import("buf_set.zig").BufSet; pub const Buffer = @import("buffer.zig").Buffer; -pub const BufferOutStream = @import("io.zig").BufferOutStream; pub const ChildProcess = @import("child_process.zig").ChildProcess; pub const DynLib = @import("dynamic_library.zig").DynLib; pub const HashMap = @import("hash_map.zig").HashMap; diff --git a/src-self-hosted/print_targets.zig b/src-self-hosted/print_targets.zig index 473e3dfa51..ad506425d2 100644 --- a/src-self-hosted/print_targets.zig +++ b/src-self-hosted/print_targets.zig @@ -93,7 +93,7 @@ pub fn cmdTargets( }; defer allocator.free(available_glibcs); - var bos = io.bufferedOutStream(4096, stdout); + var bos = io.bufferedOutStream(stdout); const bos_stream = bos.outStream(); var jws = std.json.WriteStream(@TypeOf(bos_stream), 6).init(bos_stream); From 18f1fef1426cb0405c733890b2e1d8d48627e4fe Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 10 Mar 2020 18:44:30 -0400 Subject: [PATCH 03/15] update standard library to new I/O streams API --- lib/std/atomic/queue.zig | 33 +- lib/std/buffer.zig | 12 + lib/std/debug/leb128.zig | 24 +- lib/std/heap.zig | 1 + lib/std/heap/logging_allocator.zig | 106 ++-- lib/std/io.zig | 762 +---------------------------- lib/std/io/bit_in_stream.zig | 237 +++++++++ lib/std/io/bit_out_stream.zig | 197 ++++++++ lib/std/io/buffered_in_stream.zig | 4 +- lib/std/io/c_out_stream.zig | 44 ++ lib/std/io/counting_out_stream.zig | 25 +- lib/std/io/fixed_buffer_stream.zig | 56 ++- lib/std/io/in_stream.zig | 3 +- lib/std/io/peek_stream.zig | 112 +++++ lib/std/io/serialization.zig | 606 +++++++++++++++++++++++ lib/std/io/test.zig | 534 +------------------- lib/std/json.zig | 5 +- lib/std/json/write_stream.zig | 13 +- lib/std/net.zig | 4 +- lib/std/os/test.zig | 11 +- lib/std/zig/parser_test.zig | 11 +- lib/std/zig/render.zig | 2 +- 22 files changed, 1417 insertions(+), 1385 deletions(-) create mode 100644 lib/std/io/bit_in_stream.zig create mode 100644 lib/std/io/bit_out_stream.zig create mode 100644 lib/std/io/c_out_stream.zig create mode 100644 lib/std/io/peek_stream.zig create mode 100644 lib/std/io/serialization.zig diff --git a/lib/std/atomic/queue.zig b/lib/std/atomic/queue.zig index 1969587f30..1a0f39587e 100644 --- a/lib/std/atomic/queue.zig +++ b/lib/std/atomic/queue.zig @@ -104,21 +104,17 @@ pub fn Queue(comptime T: type) type { } pub fn dump(self: *Self) void { - var stderr_file = std.io.getStdErr() catch return; - const stderr = &stderr_file.outStream().stream; - const Error = @typeInfo(@TypeOf(stderr)).Pointer.child.Error; - - self.dumpToStream(Error, stderr) catch return; + self.dumpToStream(std.io.getStdErr().outStream()) catch return; } - pub fn dumpToStream(self: *Self, comptime Error: type, stream: *std.io.OutStream(Error)) Error!void { + pub fn dumpToStream(self: *Self, stream: var) !void { const S = struct { fn dumpRecursive( - s: *std.io.OutStream(Error), + s: var, optional_node: ?*Node, indent: usize, comptime depth: comptime_int, - ) Error!void { + ) !void { try s.writeByteNTimes(' ', indent); if (optional_node) |node| { try s.print("0x{x}={}\n", .{ @ptrToInt(node), node.data }); @@ -326,17 +322,16 @@ test "std.atomic.Queue single-threaded" { test "std.atomic.Queue dump" { const mem = std.mem; - const SliceOutStream = std.io.SliceOutStream; var buffer: [1024]u8 = undefined; var expected_buffer: [1024]u8 = undefined; - var sos = SliceOutStream.init(buffer[0..]); + var fbs = std.io.fixedBufferStream(&buffer); var queue = Queue(i32).init(); // Test empty stream - sos.reset(); - try queue.dumpToStream(SliceOutStream.Error, &sos.stream); - expect(mem.eql(u8, buffer[0..sos.pos], + fbs.reset(); + try queue.dumpToStream(fbs.outStream()); + expect(mem.eql(u8, buffer[0..fbs.pos], \\head: (null) \\tail: (null) \\ @@ -350,8 +345,8 @@ test "std.atomic.Queue dump" { }; queue.put(&node_0); - sos.reset(); - try queue.dumpToStream(SliceOutStream.Error, &sos.stream); + fbs.reset(); + try queue.dumpToStream(fbs.outStream()); var expected = try std.fmt.bufPrint(expected_buffer[0..], \\head: 0x{x}=1 @@ -360,7 +355,7 @@ test "std.atomic.Queue dump" { \\ (null) \\ , .{ @ptrToInt(queue.head), @ptrToInt(queue.tail) }); - expect(mem.eql(u8, buffer[0..sos.pos], expected)); + expect(mem.eql(u8, buffer[0..fbs.pos], expected)); // Test a stream with two elements var node_1 = Queue(i32).Node{ @@ -370,8 +365,8 @@ test "std.atomic.Queue dump" { }; queue.put(&node_1); - sos.reset(); - try queue.dumpToStream(SliceOutStream.Error, &sos.stream); + fbs.reset(); + try queue.dumpToStream(fbs.outStream()); expected = try std.fmt.bufPrint(expected_buffer[0..], \\head: 0x{x}=1 @@ -381,5 +376,5 @@ test "std.atomic.Queue dump" { \\ (null) \\ , .{ @ptrToInt(queue.head), @ptrToInt(queue.head.?.next), @ptrToInt(queue.tail) }); - expect(mem.eql(u8, buffer[0..sos.pos], expected)); + expect(mem.eql(u8, buffer[0..fbs.pos], expected)); } diff --git a/lib/std/buffer.zig b/lib/std/buffer.zig index a33670b6d3..28ce2a5610 100644 --- a/lib/std/buffer.zig +++ b/lib/std/buffer.zig @@ -219,3 +219,15 @@ test "Buffer.print" { try buf.print("Hello {} the {}", .{ 2, "world" }); testing.expect(buf.eql("Hello 2 the world")); } + +test "Buffer.outStream" { + var buffer = try Buffer.initSize(testing.allocator, 0); + defer buffer.deinit(); + const buf_stream = buffer.outStream(); + + const x: i32 = 42; + const y: i32 = 1234; + try buf_stream.print("x: {}\ny: {}\n", .{ x, y }); + + testing.expect(mem.eql(u8, buffer.toSlice(), "x: 42\ny: 1234\n")); +} diff --git a/lib/std/debug/leb128.zig b/lib/std/debug/leb128.zig index 1d81b9390a..e2157335b5 100644 --- a/lib/std/debug/leb128.zig +++ b/lib/std/debug/leb128.zig @@ -121,18 +121,18 @@ pub fn readILEB128Mem(comptime T: type, ptr: *[*]const u8) !T { } fn test_read_stream_ileb128(comptime T: type, encoded: []const u8) !T { - var in_stream = std.io.SliceInStream.init(encoded); - return try readILEB128(T, &in_stream.stream); + var in_stream = std.io.fixedBufferStream(encoded); + return try readILEB128(T, in_stream.inStream()); } fn test_read_stream_uleb128(comptime T: type, encoded: []const u8) !T { - var in_stream = std.io.SliceInStream.init(encoded); - return try readULEB128(T, &in_stream.stream); + var in_stream = std.io.fixedBufferStream(encoded); + return try readULEB128(T, in_stream.inStream()); } fn test_read_ileb128(comptime T: type, encoded: []const u8) !T { - var in_stream = std.io.SliceInStream.init(encoded); - const v1 = readILEB128(T, &in_stream.stream); + var in_stream = std.io.fixedBufferStream(encoded); + const v1 = readILEB128(T, in_stream.inStream()); var in_ptr = encoded.ptr; const v2 = readILEB128Mem(T, &in_ptr); testing.expectEqual(v1, v2); @@ -140,8 +140,8 @@ fn test_read_ileb128(comptime T: type, encoded: []const u8) !T { } fn test_read_uleb128(comptime T: type, encoded: []const u8) !T { - var in_stream = std.io.SliceInStream.init(encoded); - const v1 = readULEB128(T, &in_stream.stream); + var in_stream = std.io.fixedBufferStream(encoded); + const v1 = readULEB128(T, in_stream.inStream()); var in_ptr = encoded.ptr; const v2 = readULEB128Mem(T, &in_ptr); testing.expectEqual(v1, v2); @@ -149,22 +149,22 @@ fn test_read_uleb128(comptime T: type, encoded: []const u8) !T { } fn test_read_ileb128_seq(comptime T: type, comptime N: usize, encoded: []const u8) void { - var in_stream = std.io.SliceInStream.init(encoded); + var in_stream = std.io.fixedBufferStream(encoded); var in_ptr = encoded.ptr; var i: usize = 0; while (i < N) : (i += 1) { - const v1 = readILEB128(T, &in_stream.stream); + const v1 = readILEB128(T, in_stream.inStream()); const v2 = readILEB128Mem(T, &in_ptr); testing.expectEqual(v1, v2); } } fn test_read_uleb128_seq(comptime T: type, comptime N: usize, encoded: []const u8) void { - var in_stream = std.io.SliceInStream.init(encoded); + var in_stream = std.io.fixedBufferStream(encoded); var in_ptr = encoded.ptr; var i: usize = 0; while (i < N) : (i += 1) { - const v1 = readULEB128(T, &in_stream.stream); + const v1 = readULEB128(T, in_stream.inStream()); const v2 = readULEB128Mem(T, &in_ptr); testing.expectEqual(v1, v2); } diff --git a/lib/std/heap.zig b/lib/std/heap.zig index 65809e97b4..8c79249d4b 100644 --- a/lib/std/heap.zig +++ b/lib/std/heap.zig @@ -10,6 +10,7 @@ const c = std.c; const maxInt = std.math.maxInt; pub const LoggingAllocator = @import("heap/logging_allocator.zig").LoggingAllocator; +pub const loggingAllocator = @import("heap/logging_allocator.zig").loggingAllocator; const Allocator = mem.Allocator; diff --git a/lib/std/heap/logging_allocator.zig b/lib/std/heap/logging_allocator.zig index 7dce7bc20a..0d15986a76 100644 --- a/lib/std/heap/logging_allocator.zig +++ b/lib/std/heap/logging_allocator.zig @@ -1,63 +1,69 @@ const std = @import("../std.zig"); const Allocator = std.mem.Allocator; -const AnyErrorOutStream = std.io.OutStream(anyerror); - /// This allocator is used in front of another allocator and logs to the provided stream /// on every call to the allocator. Stream errors are ignored. /// If https://github.com/ziglang/zig/issues/2586 is implemented, this API can be improved. -pub const LoggingAllocator = struct { - allocator: Allocator, +pub fn LoggingAllocator(comptime OutStreamType: type) type { + return struct { + allocator: Allocator, + parent_allocator: *Allocator, + out_stream: OutStreamType, + + const Self = @This(); + + pub fn init(parent_allocator: *Allocator, out_stream: OutStreamType) Self { + return Self{ + .allocator = Allocator{ + .reallocFn = realloc, + .shrinkFn = shrink, + }, + .parent_allocator = parent_allocator, + .out_stream = out_stream, + }; + } + + fn realloc(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 { + const self = @fieldParentPtr(Self, "allocator", allocator); + if (old_mem.len == 0) { + self.out_stream.print("allocation of {} ", .{new_size}) catch {}; + } else { + self.out_stream.print("resize from {} to {} ", .{ old_mem.len, new_size }) catch {}; + } + const result = self.parent_allocator.reallocFn(self.parent_allocator, old_mem, old_align, new_size, new_align); + if (result) |buff| { + self.out_stream.print("success!\n", .{}) catch {}; + } else |err| { + self.out_stream.print("failure!\n", .{}) catch {}; + } + return result; + } + + fn shrink(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 { + const self = @fieldParentPtr(Self, "allocator", allocator); + const result = self.parent_allocator.shrinkFn(self.parent_allocator, old_mem, old_align, new_size, new_align); + if (new_size == 0) { + self.out_stream.print("free of {} bytes success!\n", .{old_mem.len}) catch {}; + } else { + self.out_stream.print("shrink from {} bytes to {} bytes success!\n", .{ old_mem.len, new_size }) catch {}; + } + return result; + } + }; +} + +pub fn loggingAllocator( parent_allocator: *Allocator, - out_stream: *AnyErrorOutStream, - - const Self = @This(); - - pub fn init(parent_allocator: *Allocator, out_stream: *AnyErrorOutStream) Self { - return Self{ - .allocator = Allocator{ - .reallocFn = realloc, - .shrinkFn = shrink, - }, - .parent_allocator = parent_allocator, - .out_stream = out_stream, - }; - } - - fn realloc(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) ![]u8 { - const self = @fieldParentPtr(Self, "allocator", allocator); - if (old_mem.len == 0) { - self.out_stream.print("allocation of {} ", .{new_size}) catch {}; - } else { - self.out_stream.print("resize from {} to {} ", .{ old_mem.len, new_size }) catch {}; - } - const result = self.parent_allocator.reallocFn(self.parent_allocator, old_mem, old_align, new_size, new_align); - if (result) |buff| { - self.out_stream.print("success!\n", .{}) catch {}; - } else |err| { - self.out_stream.print("failure!\n", .{}) catch {}; - } - return result; - } - - fn shrink(allocator: *Allocator, old_mem: []u8, old_align: u29, new_size: usize, new_align: u29) []u8 { - const self = @fieldParentPtr(Self, "allocator", allocator); - const result = self.parent_allocator.shrinkFn(self.parent_allocator, old_mem, old_align, new_size, new_align); - if (new_size == 0) { - self.out_stream.print("free of {} bytes success!\n", .{old_mem.len}) catch {}; - } else { - self.out_stream.print("shrink from {} bytes to {} bytes success!\n", .{ old_mem.len, new_size }) catch {}; - } - return result; - } -}; + out_stream: var, +) LoggingAllocator(@TypeOf(out_stream)) { + return LoggingAllocator(@TypeOf(out_stream)).init(parent_allocator, out_stream); +} test "LoggingAllocator" { var buf: [255]u8 = undefined; - var slice_stream = std.io.SliceOutStream.init(buf[0..]); - const stream = &slice_stream.stream; + var fbs = std.io.fixedBufferStream(&buf); - const allocator = &LoggingAllocator.init(std.testing.allocator, @ptrCast(*AnyErrorOutStream, stream)).allocator; + const allocator = &loggingAllocator(std.testing.allocator, fbs.outStream()).allocator; const ptr = try allocator.alloc(u8, 10); allocator.free(ptr); @@ -66,5 +72,5 @@ test "LoggingAllocator" { \\allocation of 10 success! \\free of 10 bytes success! \\ - , slice_stream.getWritten()); + , fbs.getWritten()); } diff --git a/lib/std/io.zig b/lib/std/io.zig index 1283461271..22af5c06b1 100644 --- a/lib/std/io.zig +++ b/lib/std/io.zig @@ -4,17 +4,13 @@ const root = @import("root"); const c = std.c; const math = std.math; -const debug = std.debug; -const assert = debug.assert; +const assert = std.debug.assert; const os = std.os; const fs = std.fs; const mem = std.mem; const meta = std.meta; const trait = meta.trait; -const Buffer = std.Buffer; -const fmt = std.fmt; const File = std.fs.File; -const testing = std.testing; pub const Mode = enum { /// I/O operates normally, waiting for the operating system syscalls to complete. @@ -92,10 +88,9 @@ pub fn getStdIn() File { }; } -pub const SeekableStream = @import("io/seekable_stream.zig").SeekableStream; pub const InStream = @import("io/in_stream.zig").InStream; pub const OutStream = @import("io/out_stream.zig").OutStream; -pub const BufferedAtomicFile = @import("io/buffered_atomic_file.zig").BufferedAtomicFile; +pub const SeekableStream = @import("io/seekable_stream.zig").SeekableStream; pub const BufferedOutStream = @import("io/buffered_out_stream.zig").BufferedOutStream; pub const bufferedOutStream = @import("io/buffered_out_stream.zig").bufferedOutStream; @@ -103,36 +98,33 @@ pub const bufferedOutStream = @import("io/buffered_out_stream.zig").bufferedOutS pub const BufferedInStream = @import("io/buffered_in_stream.zig").BufferedInStream; pub const bufferedInStream = @import("io/buffered_in_stream.zig").bufferedInStream; +pub const PeekStream = @import("io/peek_stream.zig").PeekStream; +pub const peekStream = @import("io/peek_stream.zig").peekStream; + pub const FixedBufferStream = @import("io/fixed_buffer_stream.zig").FixedBufferStream; pub const fixedBufferStream = @import("io/fixed_buffer_stream.zig").fixedBufferStream; +pub const COutStream = @import("io/c_out_stream.zig").COutStream; +pub const cOutStream = @import("io/c_out_stream.zig").cOutStream; + pub const CountingOutStream = @import("io/counting_out_stream.zig").CountingOutStream; +pub const countingOutStream = @import("io/counting_out_stream.zig").countingOutStream; -pub fn cOutStream(c_file: *std.c.FILE) COutStream { - return .{ .context = c_file }; -} +pub const BitInStream = @import("io/bit_in_stream.zig").BitInStream; +pub const bitInStream = @import("io/bit_in_stream.zig").bitInStream; -pub const COutStream = OutStream(*std.c.FILE, std.fs.File.WriteError, cOutStreamWrite); +pub const BitOutStream = @import("io/bit_out_stream.zig").BitOutStream; +pub const bitOutStream = @import("io/bit_out_stream.zig").bitOutStream; -pub fn cOutStreamWrite(c_file: *std.c.FILE, bytes: []const u8) std.fs.File.WriteError!usize { - const amt_written = std.c.fwrite(bytes.ptr, 1, bytes.len, c_file); - if (amt_written >= 0) return amt_written; - switch (std.c._errno().*) { - 0 => unreachable, - os.EINVAL => unreachable, - os.EFAULT => unreachable, - os.EAGAIN => unreachable, // this is a blocking API - os.EBADF => unreachable, // always a race condition - os.EDESTADDRREQ => unreachable, // connect was never called - os.EDQUOT => return error.DiskQuota, - os.EFBIG => return error.FileTooBig, - os.EIO => return error.InputOutput, - os.ENOSPC => return error.NoSpaceLeft, - os.EPERM => return error.AccessDenied, - os.EPIPE => return error.BrokenPipe, - else => |err| return os.unexpectedErrno(@intCast(usize, err)), - } -} +pub const Packing = @import("io/serialization.zig").Packing; + +pub const Serializer = @import("io/serialization.zig").Serializer; +pub const serializer = @import("io/serialization.zig").serializer; + +pub const Deserializer = @import("io/serialization.zig").Deserializer; +pub const deserializer = @import("io/serialization.zig").deserializer; + +pub const BufferedAtomicFile = @import("io/buffered_atomic_file.zig").BufferedAtomicFile; /// Deprecated; use `std.fs.Dir.writeFile`. pub fn writeFile(path: []const u8, data: []const u8) !void { @@ -144,249 +136,6 @@ pub fn readFileAlloc(allocator: *mem.Allocator, path: []const u8) ![]u8 { return fs.cwd().readFileAlloc(allocator, path, math.maxInt(usize)); } -/// Creates a stream which supports 'un-reading' data, so that it can be read again. -/// This makes look-ahead style parsing much easier. -pub fn PeekStream(comptime buffer_type: std.fifo.LinearFifoBufferType, comptime InStreamError: type) type { - return struct { - const Self = @This(); - pub const Error = InStreamError; - pub const Stream = InStream(Error); - - stream: Stream, - base: *Stream, - - const FifoType = std.fifo.LinearFifo(u8, buffer_type); - fifo: FifoType, - - pub usingnamespace switch (buffer_type) { - .Static => struct { - pub fn init(base: *Stream) Self { - return .{ - .base = base, - .fifo = FifoType.init(), - .stream = Stream{ .readFn = readFn }, - }; - } - }, - .Slice => struct { - pub fn init(base: *Stream, buf: []u8) Self { - return .{ - .base = base, - .fifo = FifoType.init(buf), - .stream = Stream{ .readFn = readFn }, - }; - } - }, - .Dynamic => struct { - pub fn init(base: *Stream, allocator: *mem.Allocator) Self { - return .{ - .base = base, - .fifo = FifoType.init(allocator), - .stream = Stream{ .readFn = readFn }, - }; - } - }, - }; - - pub fn putBackByte(self: *Self, byte: u8) !void { - try self.putBack(&[_]u8{byte}); - } - - pub fn putBack(self: *Self, bytes: []const u8) !void { - try self.fifo.unget(bytes); - } - - fn readFn(in_stream: *Stream, dest: []u8) Error!usize { - const self = @fieldParentPtr(Self, "stream", in_stream); - - // copy over anything putBack()'d - var dest_index = self.fifo.read(dest); - if (dest_index == dest.len) return dest_index; - - // ask the backing stream for more - dest_index += try self.base.read(dest[dest_index..]); - return dest_index; - } - }; -} - -pub const SliceInStream = struct { - const Self = @This(); - pub const Error = error{}; - pub const Stream = InStream(Error); - - stream: Stream, - - pos: usize, - slice: []const u8, - - pub fn init(slice: []const u8) Self { - return Self{ - .slice = slice, - .pos = 0, - .stream = Stream{ .readFn = readFn }, - }; - } - - fn readFn(in_stream: *Stream, dest: []u8) Error!usize { - const self = @fieldParentPtr(Self, "stream", in_stream); - const size = math.min(dest.len, self.slice.len - self.pos); - const end = self.pos + size; - - mem.copy(u8, dest[0..size], self.slice[self.pos..end]); - self.pos = end; - - return size; - } -}; - -/// Creates a stream which allows for reading bit fields from another stream -pub fn BitInStream(endian: builtin.Endian, comptime Error: type) type { - return struct { - const Self = @This(); - - in_stream: *Stream, - bit_buffer: u7, - bit_count: u3, - stream: Stream, - - pub const Stream = InStream(Error); - const u8_bit_count = comptime meta.bitCount(u8); - const u7_bit_count = comptime meta.bitCount(u7); - const u4_bit_count = comptime meta.bitCount(u4); - - pub fn init(in_stream: *Stream) Self { - return Self{ - .in_stream = in_stream, - .bit_buffer = 0, - .bit_count = 0, - .stream = Stream{ .readFn = read }, - }; - } - - /// Reads `bits` bits from the stream and returns a specified unsigned int type - /// containing them in the least significant end, returning an error if the - /// specified number of bits could not be read. - pub fn readBitsNoEof(self: *Self, comptime U: type, bits: usize) !U { - var n: usize = undefined; - const result = try self.readBits(U, bits, &n); - if (n < bits) return error.EndOfStream; - return result; - } - - /// Reads `bits` bits from the stream and returns a specified unsigned int type - /// containing them in the least significant end. The number of bits successfully - /// read is placed in `out_bits`, as reaching the end of the stream is not an error. - pub fn readBits(self: *Self, comptime U: type, bits: usize, out_bits: *usize) Error!U { - comptime assert(trait.isUnsignedInt(U)); - - //by extending the buffer to a minimum of u8 we can cover a number of edge cases - // related to shifting and casting. - const u_bit_count = comptime meta.bitCount(U); - const buf_bit_count = bc: { - assert(u_bit_count >= bits); - break :bc if (u_bit_count <= u8_bit_count) u8_bit_count else u_bit_count; - }; - const Buf = std.meta.IntType(false, buf_bit_count); - const BufShift = math.Log2Int(Buf); - - out_bits.* = @as(usize, 0); - if (U == u0 or bits == 0) return 0; - var out_buffer = @as(Buf, 0); - - if (self.bit_count > 0) { - const n = if (self.bit_count >= bits) @intCast(u3, bits) else self.bit_count; - const shift = u7_bit_count - n; - switch (endian) { - .Big => { - out_buffer = @as(Buf, self.bit_buffer >> shift); - self.bit_buffer <<= n; - }, - .Little => { - const value = (self.bit_buffer << shift) >> shift; - out_buffer = @as(Buf, value); - self.bit_buffer >>= n; - }, - } - self.bit_count -= n; - out_bits.* = n; - } - //at this point we know bit_buffer is empty - - //copy bytes until we have enough bits, then leave the rest in bit_buffer - while (out_bits.* < bits) { - const n = bits - out_bits.*; - const next_byte = self.in_stream.readByte() catch |err| { - if (err == error.EndOfStream) { - return @intCast(U, out_buffer); - } - //@BUG: See #1810. Not sure if the bug is that I have to do this for some - // streams, or that I don't for streams with emtpy errorsets. - return @errSetCast(Error, err); - }; - - switch (endian) { - .Big => { - if (n >= u8_bit_count) { - out_buffer <<= @intCast(u3, u8_bit_count - 1); - out_buffer <<= 1; - out_buffer |= @as(Buf, next_byte); - out_bits.* += u8_bit_count; - continue; - } - - const shift = @intCast(u3, u8_bit_count - n); - out_buffer <<= @intCast(BufShift, n); - out_buffer |= @as(Buf, next_byte >> shift); - out_bits.* += n; - self.bit_buffer = @truncate(u7, next_byte << @intCast(u3, n - 1)); - self.bit_count = shift; - }, - .Little => { - if (n >= u8_bit_count) { - out_buffer |= @as(Buf, next_byte) << @intCast(BufShift, out_bits.*); - out_bits.* += u8_bit_count; - continue; - } - - const shift = @intCast(u3, u8_bit_count - n); - const value = (next_byte << shift) >> shift; - out_buffer |= @as(Buf, value) << @intCast(BufShift, out_bits.*); - out_bits.* += n; - self.bit_buffer = @truncate(u7, next_byte >> @intCast(u3, n)); - self.bit_count = shift; - }, - } - } - - return @intCast(U, out_buffer); - } - - pub fn alignToByte(self: *Self) void { - self.bit_buffer = 0; - self.bit_count = 0; - } - - pub fn read(self_stream: *Stream, buffer: []u8) Error!usize { - var self = @fieldParentPtr(Self, "stream", self_stream); - - var out_bits: usize = undefined; - var out_bits_total = @as(usize, 0); - //@NOTE: I'm not sure this is a good idea, maybe alignToByte should be forced - if (self.bit_count > 0) { - for (buffer) |*b, i| { - b.* = try self.readBits(u8, u8_bit_count, &out_bits); - out_bits_total += out_bits; - } - const incomplete_byte = @boolToInt(out_bits_total % u8_bit_count > 0); - return (out_bits_total / u8_bit_count) + incomplete_byte; - } - - return self.in_stream.read(buffer); - } - }; -} - /// An OutStream that doesn't write to anything. pub const null_out_stream = @as(NullOutStream, .{ .context = {} }); @@ -396,472 +145,9 @@ fn dummyWrite(context: void, data: []const u8) error{}!usize { } test "null_out_stream" { - null_out_stream.writeAll("yay" ** 1000) catch |err| switch (err) {}; -} - -/// Creates a stream which allows for writing bit fields to another stream -pub fn BitOutStream(endian: builtin.Endian, comptime Error: type) type { - return struct { - const Self = @This(); - - out_stream: *Stream, - bit_buffer: u8, - bit_count: u4, - stream: Stream, - - pub const Stream = OutStream(Error); - const u8_bit_count = comptime meta.bitCount(u8); - const u4_bit_count = comptime meta.bitCount(u4); - - pub fn init(out_stream: *Stream) Self { - return Self{ - .out_stream = out_stream, - .bit_buffer = 0, - .bit_count = 0, - .stream = Stream{ .writeFn = write }, - }; - } - - /// Write the specified number of bits to the stream from the least significant bits of - /// the specified unsigned int value. Bits will only be written to the stream when there - /// are enough to fill a byte. - pub fn writeBits(self: *Self, value: var, bits: usize) Error!void { - if (bits == 0) return; - - const U = @TypeOf(value); - comptime assert(trait.isUnsignedInt(U)); - - //by extending the buffer to a minimum of u8 we can cover a number of edge cases - // related to shifting and casting. - const u_bit_count = comptime meta.bitCount(U); - const buf_bit_count = bc: { - assert(u_bit_count >= bits); - break :bc if (u_bit_count <= u8_bit_count) u8_bit_count else u_bit_count; - }; - const Buf = std.meta.IntType(false, buf_bit_count); - const BufShift = math.Log2Int(Buf); - - const buf_value = @intCast(Buf, value); - - const high_byte_shift = @intCast(BufShift, buf_bit_count - u8_bit_count); - var in_buffer = switch (endian) { - .Big => buf_value << @intCast(BufShift, buf_bit_count - bits), - .Little => buf_value, - }; - var in_bits = bits; - - if (self.bit_count > 0) { - const bits_remaining = u8_bit_count - self.bit_count; - const n = @intCast(u3, if (bits_remaining > bits) bits else bits_remaining); - switch (endian) { - .Big => { - const shift = @intCast(BufShift, high_byte_shift + self.bit_count); - const v = @intCast(u8, in_buffer >> shift); - self.bit_buffer |= v; - in_buffer <<= n; - }, - .Little => { - const v = @truncate(u8, in_buffer) << @intCast(u3, self.bit_count); - self.bit_buffer |= v; - in_buffer >>= n; - }, - } - self.bit_count += n; - in_bits -= n; - - //if we didn't fill the buffer, it's because bits < bits_remaining; - if (self.bit_count != u8_bit_count) return; - try self.out_stream.writeByte(self.bit_buffer); - self.bit_buffer = 0; - self.bit_count = 0; - } - //at this point we know bit_buffer is empty - - //copy bytes until we can't fill one anymore, then leave the rest in bit_buffer - while (in_bits >= u8_bit_count) { - switch (endian) { - .Big => { - const v = @intCast(u8, in_buffer >> high_byte_shift); - try self.out_stream.writeByte(v); - in_buffer <<= @intCast(u3, u8_bit_count - 1); - in_buffer <<= 1; - }, - .Little => { - const v = @truncate(u8, in_buffer); - try self.out_stream.writeByte(v); - in_buffer >>= @intCast(u3, u8_bit_count - 1); - in_buffer >>= 1; - }, - } - in_bits -= u8_bit_count; - } - - if (in_bits > 0) { - self.bit_count = @intCast(u4, in_bits); - self.bit_buffer = switch (endian) { - .Big => @truncate(u8, in_buffer >> high_byte_shift), - .Little => @truncate(u8, in_buffer), - }; - } - } - - /// Flush any remaining bits to the stream. - pub fn flushBits(self: *Self) Error!void { - if (self.bit_count == 0) return; - try self.out_stream.writeByte(self.bit_buffer); - self.bit_buffer = 0; - self.bit_count = 0; - } - - pub fn write(self_stream: *Stream, buffer: []const u8) Error!usize { - var self = @fieldParentPtr(Self, "stream", self_stream); - - // TODO: I'm not sure this is a good idea, maybe flushBits should be forced - if (self.bit_count > 0) { - for (buffer) |b, i| - try self.writeBits(b, u8_bit_count); - return buffer.len; - } - - return self.out_stream.write(buffer); - } - }; -} - -pub const Packing = enum { - /// Pack data to byte alignment - Byte, - - /// Pack data to bit alignment - Bit, -}; - -/// Creates a deserializer that deserializes types from any stream. -/// If `is_packed` is true, the data stream is treated as bit-packed, -/// otherwise data is expected to be packed to the smallest byte. -/// Types may implement a custom deserialization routine with a -/// function named `deserialize` in the form of: -/// pub fn deserialize(self: *Self, deserializer: var) !void -/// which will be called when the deserializer is used to deserialize -/// that type. It will pass a pointer to the type instance to deserialize -/// into and a pointer to the deserializer struct. -pub fn Deserializer(comptime endian: builtin.Endian, comptime packing: Packing, comptime Error: type) type { - return struct { - const Self = @This(); - - in_stream: if (packing == .Bit) BitInStream(endian, Stream.Error) else *Stream, - - pub const Stream = InStream(Error); - - pub fn init(in_stream: *Stream) Self { - return Self{ - .in_stream = switch (packing) { - .Bit => BitInStream(endian, Stream.Error).init(in_stream), - .Byte => in_stream, - }, - }; - } - - pub fn alignToByte(self: *Self) void { - if (packing == .Byte) return; - self.in_stream.alignToByte(); - } - - //@BUG: inferred error issue. See: #1386 - fn deserializeInt(self: *Self, comptime T: type) (Error || error{EndOfStream})!T { - comptime assert(trait.is(.Int)(T) or trait.is(.Float)(T)); - - const u8_bit_count = 8; - const t_bit_count = comptime meta.bitCount(T); - - const U = std.meta.IntType(false, t_bit_count); - const Log2U = math.Log2Int(U); - const int_size = (U.bit_count + 7) / 8; - - if (packing == .Bit) { - const result = try self.in_stream.readBitsNoEof(U, t_bit_count); - return @bitCast(T, result); - } - - var buffer: [int_size]u8 = undefined; - const read_size = try self.in_stream.read(buffer[0..]); - if (read_size < int_size) return error.EndOfStream; - - if (int_size == 1) { - if (t_bit_count == 8) return @bitCast(T, buffer[0]); - const PossiblySignedByte = std.meta.IntType(T.is_signed, 8); - return @truncate(T, @bitCast(PossiblySignedByte, buffer[0])); - } - - var result = @as(U, 0); - for (buffer) |byte, i| { - switch (endian) { - .Big => { - result = (result << u8_bit_count) | byte; - }, - .Little => { - result |= @as(U, byte) << @intCast(Log2U, u8_bit_count * i); - }, - } - } - - return @bitCast(T, result); - } - - /// Deserializes and returns data of the specified type from the stream - pub fn deserialize(self: *Self, comptime T: type) !T { - var value: T = undefined; - try self.deserializeInto(&value); - return value; - } - - /// Deserializes data into the type pointed to by `ptr` - pub fn deserializeInto(self: *Self, ptr: var) !void { - const T = @TypeOf(ptr); - comptime assert(trait.is(.Pointer)(T)); - - if (comptime trait.isSlice(T) or comptime trait.isPtrTo(.Array)(T)) { - for (ptr) |*v| - try self.deserializeInto(v); - return; - } - - comptime assert(trait.isSingleItemPtr(T)); - - const C = comptime meta.Child(T); - const child_type_id = @typeInfo(C); - - //custom deserializer: fn(self: *Self, deserializer: var) !void - if (comptime trait.hasFn("deserialize")(C)) return C.deserialize(ptr, self); - - if (comptime trait.isPacked(C) and packing != .Bit) { - var packed_deserializer = Deserializer(endian, .Bit, Error).init(self.in_stream); - return packed_deserializer.deserializeInto(ptr); - } - - switch (child_type_id) { - .Void => return, - .Bool => ptr.* = (try self.deserializeInt(u1)) > 0, - .Float, .Int => ptr.* = try self.deserializeInt(C), - .Struct => { - const info = @typeInfo(C).Struct; - - inline for (info.fields) |*field_info| { - const name = field_info.name; - const FieldType = field_info.field_type; - - if (FieldType == void or FieldType == u0) continue; - - //it doesn't make any sense to read pointers - if (comptime trait.is(.Pointer)(FieldType)) { - @compileError("Will not " ++ "read field " ++ name ++ " of struct " ++ - @typeName(C) ++ " because it " ++ "is of pointer-type " ++ - @typeName(FieldType) ++ "."); - } - - try self.deserializeInto(&@field(ptr, name)); - } - }, - .Union => { - const info = @typeInfo(C).Union; - if (info.tag_type) |TagType| { - //we avoid duplicate iteration over the enum tags - // by getting the int directly and casting it without - // safety. If it is bad, it will be caught anyway. - const TagInt = @TagType(TagType); - const tag = try self.deserializeInt(TagInt); - - inline for (info.fields) |field_info| { - if (field_info.enum_field.?.value == tag) { - const name = field_info.name; - const FieldType = field_info.field_type; - ptr.* = @unionInit(C, name, undefined); - try self.deserializeInto(&@field(ptr, name)); - return; - } - } - //This is reachable if the enum data is bad - return error.InvalidEnumTag; - } - @compileError("Cannot meaningfully deserialize " ++ @typeName(C) ++ - " because it is an untagged union. Use a custom deserialize()."); - }, - .Optional => { - const OC = comptime meta.Child(C); - const exists = (try self.deserializeInt(u1)) > 0; - if (!exists) { - ptr.* = null; - return; - } - - ptr.* = @as(OC, undefined); //make it non-null so the following .? is guaranteed safe - const val_ptr = &ptr.*.?; - try self.deserializeInto(val_ptr); - }, - .Enum => { - var value = try self.deserializeInt(@TagType(C)); - ptr.* = try meta.intToEnum(C, value); - }, - else => { - @compileError("Cannot deserialize " ++ @tagName(child_type_id) ++ " types (unimplemented)."); - }, - } - } - }; -} - -/// Creates a serializer that serializes types to any stream. -/// If `is_packed` is true, the data will be bit-packed into the stream. -/// Note that the you must call `serializer.flush()` when you are done -/// writing bit-packed data in order ensure any unwritten bits are committed. -/// If `is_packed` is false, data is packed to the smallest byte. In the case -/// of packed structs, the struct will written bit-packed and with the specified -/// endianess, after which data will resume being written at the next byte boundary. -/// Types may implement a custom serialization routine with a -/// function named `serialize` in the form of: -/// pub fn serialize(self: Self, serializer: var) !void -/// which will be called when the serializer is used to serialize that type. It will -/// pass a const pointer to the type instance to be serialized and a pointer -/// to the serializer struct. -pub fn Serializer(comptime endian: builtin.Endian, comptime packing: Packing, comptime Error: type) type { - return struct { - const Self = @This(); - - out_stream: if (packing == .Bit) BitOutStream(endian, Stream.Error) else *Stream, - - pub const Stream = OutStream(Error); - - pub fn init(out_stream: *Stream) Self { - return Self{ - .out_stream = switch (packing) { - .Bit => BitOutStream(endian, Stream.Error).init(out_stream), - .Byte => out_stream, - }, - }; - } - - /// Flushes any unwritten bits to the stream - pub fn flush(self: *Self) Error!void { - if (packing == .Bit) return self.out_stream.flushBits(); - } - - fn serializeInt(self: *Self, value: var) Error!void { - const T = @TypeOf(value); - comptime assert(trait.is(.Int)(T) or trait.is(.Float)(T)); - - const t_bit_count = comptime meta.bitCount(T); - const u8_bit_count = comptime meta.bitCount(u8); - - const U = std.meta.IntType(false, t_bit_count); - const Log2U = math.Log2Int(U); - const int_size = (U.bit_count + 7) / 8; - - const u_value = @bitCast(U, value); - - if (packing == .Bit) return self.out_stream.writeBits(u_value, t_bit_count); - - var buffer: [int_size]u8 = undefined; - if (int_size == 1) buffer[0] = u_value; - - for (buffer) |*byte, i| { - const idx = switch (endian) { - .Big => int_size - i - 1, - .Little => i, - }; - const shift = @intCast(Log2U, idx * u8_bit_count); - const v = u_value >> shift; - byte.* = if (t_bit_count < u8_bit_count) v else @truncate(u8, v); - } - - try self.out_stream.write(&buffer); - } - - /// Serializes the passed value into the stream - pub fn serialize(self: *Self, value: var) Error!void { - const T = comptime @TypeOf(value); - - if (comptime trait.isIndexable(T)) { - for (value) |v| - try self.serialize(v); - return; - } - - //custom serializer: fn(self: Self, serializer: var) !void - if (comptime trait.hasFn("serialize")(T)) return T.serialize(value, self); - - if (comptime trait.isPacked(T) and packing != .Bit) { - var packed_serializer = Serializer(endian, .Bit, Error).init(self.out_stream); - try packed_serializer.serialize(value); - try packed_serializer.flush(); - return; - } - - switch (@typeInfo(T)) { - .Void => return, - .Bool => try self.serializeInt(@as(u1, @boolToInt(value))), - .Float, .Int => try self.serializeInt(value), - .Struct => { - const info = @typeInfo(T); - - inline for (info.Struct.fields) |*field_info| { - const name = field_info.name; - const FieldType = field_info.field_type; - - if (FieldType == void or FieldType == u0) continue; - - //It doesn't make sense to write pointers - if (comptime trait.is(.Pointer)(FieldType)) { - @compileError("Will not " ++ "serialize field " ++ name ++ - " of struct " ++ @typeName(T) ++ " because it " ++ - "is of pointer-type " ++ @typeName(FieldType) ++ "."); - } - try self.serialize(@field(value, name)); - } - }, - .Union => { - const info = @typeInfo(T).Union; - if (info.tag_type) |TagType| { - const active_tag = meta.activeTag(value); - try self.serialize(active_tag); - //This inline loop is necessary because active_tag is a runtime - // value, but @field requires a comptime value. Our alternative - // is to check each field for a match - inline for (info.fields) |field_info| { - if (field_info.enum_field.?.value == @enumToInt(active_tag)) { - const name = field_info.name; - const FieldType = field_info.field_type; - try self.serialize(@field(value, name)); - return; - } - } - unreachable; - } - @compileError("Cannot meaningfully serialize " ++ @typeName(T) ++ - " because it is an untagged union. Use a custom serialize()."); - }, - .Optional => { - if (value == null) { - try self.serializeInt(@as(u1, @boolToInt(false))); - return; - } - try self.serializeInt(@as(u1, @boolToInt(true))); - - const OC = comptime meta.Child(T); - const val_ptr = &value.?; - try self.serialize(val_ptr.*); - }, - .Enum => { - try self.serializeInt(@enumToInt(value)); - }, - else => @compileError("Cannot serialize " ++ @tagName(@typeInfo(T)) ++ " types (unimplemented)."), - } - } - }; + null_out_stream.writeAll("yay" ** 10) catch |err| switch (err) {}; } test "" { - comptime { - _ = @import("io/test.zig"); - } - std.meta.refAllDecls(@This()); + _ = @import("io/test.zig"); } diff --git a/lib/std/io/bit_in_stream.zig b/lib/std/io/bit_in_stream.zig new file mode 100644 index 0000000000..8766213433 --- /dev/null +++ b/lib/std/io/bit_in_stream.zig @@ -0,0 +1,237 @@ +const std = @import("../std.zig"); +const builtin = std.builtin; +const io = std.io; +const assert = std.debug.assert; +const testing = std.testing; +const trait = std.meta.trait; +const meta = std.meta; +const math = std.math; + +/// Creates a stream which allows for reading bit fields from another stream +pub fn BitInStream(endian: builtin.Endian, comptime InStreamType: type) type { + return struct { + in_stream: InStreamType, + bit_buffer: u7, + bit_count: u3, + + pub const Error = InStreamType.Error; + pub const InStream = io.InStream(*Self, Error, read); + + const Self = @This(); + const u8_bit_count = comptime meta.bitCount(u8); + const u7_bit_count = comptime meta.bitCount(u7); + const u4_bit_count = comptime meta.bitCount(u4); + + pub fn init(in_stream: InStreamType) Self { + return Self{ + .in_stream = in_stream, + .bit_buffer = 0, + .bit_count = 0, + }; + } + + /// Reads `bits` bits from the stream and returns a specified unsigned int type + /// containing them in the least significant end, returning an error if the + /// specified number of bits could not be read. + pub fn readBitsNoEof(self: *Self, comptime U: type, bits: usize) !U { + var n: usize = undefined; + const result = try self.readBits(U, bits, &n); + if (n < bits) return error.EndOfStream; + return result; + } + + /// Reads `bits` bits from the stream and returns a specified unsigned int type + /// containing them in the least significant end. The number of bits successfully + /// read is placed in `out_bits`, as reaching the end of the stream is not an error. + pub fn readBits(self: *Self, comptime U: type, bits: usize, out_bits: *usize) Error!U { + comptime assert(trait.isUnsignedInt(U)); + + //by extending the buffer to a minimum of u8 we can cover a number of edge cases + // related to shifting and casting. + const u_bit_count = comptime meta.bitCount(U); + const buf_bit_count = bc: { + assert(u_bit_count >= bits); + break :bc if (u_bit_count <= u8_bit_count) u8_bit_count else u_bit_count; + }; + const Buf = std.meta.IntType(false, buf_bit_count); + const BufShift = math.Log2Int(Buf); + + out_bits.* = @as(usize, 0); + if (U == u0 or bits == 0) return 0; + var out_buffer = @as(Buf, 0); + + if (self.bit_count > 0) { + const n = if (self.bit_count >= bits) @intCast(u3, bits) else self.bit_count; + const shift = u7_bit_count - n; + switch (endian) { + .Big => { + out_buffer = @as(Buf, self.bit_buffer >> shift); + self.bit_buffer <<= n; + }, + .Little => { + const value = (self.bit_buffer << shift) >> shift; + out_buffer = @as(Buf, value); + self.bit_buffer >>= n; + }, + } + self.bit_count -= n; + out_bits.* = n; + } + //at this point we know bit_buffer is empty + + //copy bytes until we have enough bits, then leave the rest in bit_buffer + while (out_bits.* < bits) { + const n = bits - out_bits.*; + const next_byte = self.in_stream.readByte() catch |err| { + if (err == error.EndOfStream) { + return @intCast(U, out_buffer); + } + //@BUG: See #1810. Not sure if the bug is that I have to do this for some + // streams, or that I don't for streams with emtpy errorsets. + return @errSetCast(Error, err); + }; + + switch (endian) { + .Big => { + if (n >= u8_bit_count) { + out_buffer <<= @intCast(u3, u8_bit_count - 1); + out_buffer <<= 1; + out_buffer |= @as(Buf, next_byte); + out_bits.* += u8_bit_count; + continue; + } + + const shift = @intCast(u3, u8_bit_count - n); + out_buffer <<= @intCast(BufShift, n); + out_buffer |= @as(Buf, next_byte >> shift); + out_bits.* += n; + self.bit_buffer = @truncate(u7, next_byte << @intCast(u3, n - 1)); + self.bit_count = shift; + }, + .Little => { + if (n >= u8_bit_count) { + out_buffer |= @as(Buf, next_byte) << @intCast(BufShift, out_bits.*); + out_bits.* += u8_bit_count; + continue; + } + + const shift = @intCast(u3, u8_bit_count - n); + const value = (next_byte << shift) >> shift; + out_buffer |= @as(Buf, value) << @intCast(BufShift, out_bits.*); + out_bits.* += n; + self.bit_buffer = @truncate(u7, next_byte >> @intCast(u3, n)); + self.bit_count = shift; + }, + } + } + + return @intCast(U, out_buffer); + } + + pub fn alignToByte(self: *Self) void { + self.bit_buffer = 0; + self.bit_count = 0; + } + + pub fn read(self: *Self, buffer: []u8) Error!usize { + var out_bits: usize = undefined; + var out_bits_total = @as(usize, 0); + //@NOTE: I'm not sure this is a good idea, maybe alignToByte should be forced + if (self.bit_count > 0) { + for (buffer) |*b, i| { + b.* = try self.readBits(u8, u8_bit_count, &out_bits); + out_bits_total += out_bits; + } + const incomplete_byte = @boolToInt(out_bits_total % u8_bit_count > 0); + return (out_bits_total / u8_bit_count) + incomplete_byte; + } + + return self.in_stream.read(buffer); + } + + pub fn inStream(self: *Self) InStream { + return .{ .context = self }; + } + }; +} + +pub fn bitInStream( + comptime endian: builtin.Endian, + underlying_stream: var, +) BitInStream(endian, @TypeOf(underlying_stream)) { + return BitInStream(endian, @TypeOf(underlying_stream)).init(underlying_stream); +} + +test "api coverage" { + const mem_be = [_]u8{ 0b11001101, 0b00001011 }; + const mem_le = [_]u8{ 0b00011101, 0b10010101 }; + + var mem_in_be = io.fixedBufferStream(&mem_be); + var bit_stream_be = bitInStream(.Big, mem_in_be.inStream()); + + var out_bits: usize = undefined; + + const expect = testing.expect; + const expectError = testing.expectError; + + expect(1 == try bit_stream_be.readBits(u2, 1, &out_bits)); + expect(out_bits == 1); + expect(2 == try bit_stream_be.readBits(u5, 2, &out_bits)); + expect(out_bits == 2); + expect(3 == try bit_stream_be.readBits(u128, 3, &out_bits)); + expect(out_bits == 3); + expect(4 == try bit_stream_be.readBits(u8, 4, &out_bits)); + expect(out_bits == 4); + expect(5 == try bit_stream_be.readBits(u9, 5, &out_bits)); + expect(out_bits == 5); + expect(1 == try bit_stream_be.readBits(u1, 1, &out_bits)); + expect(out_bits == 1); + + mem_in_be.pos = 0; + bit_stream_be.bit_count = 0; + expect(0b110011010000101 == try bit_stream_be.readBits(u15, 15, &out_bits)); + expect(out_bits == 15); + + mem_in_be.pos = 0; + bit_stream_be.bit_count = 0; + expect(0b1100110100001011 == try bit_stream_be.readBits(u16, 16, &out_bits)); + expect(out_bits == 16); + + _ = try bit_stream_be.readBits(u0, 0, &out_bits); + + expect(0 == try bit_stream_be.readBits(u1, 1, &out_bits)); + expect(out_bits == 0); + expectError(error.EndOfStream, bit_stream_be.readBitsNoEof(u1, 1)); + + var mem_in_le = io.fixedBufferStream(&mem_le); + var bit_stream_le = bitInStream(.Little, mem_in_le.inStream()); + + expect(1 == try bit_stream_le.readBits(u2, 1, &out_bits)); + expect(out_bits == 1); + expect(2 == try bit_stream_le.readBits(u5, 2, &out_bits)); + expect(out_bits == 2); + expect(3 == try bit_stream_le.readBits(u128, 3, &out_bits)); + expect(out_bits == 3); + expect(4 == try bit_stream_le.readBits(u8, 4, &out_bits)); + expect(out_bits == 4); + expect(5 == try bit_stream_le.readBits(u9, 5, &out_bits)); + expect(out_bits == 5); + expect(1 == try bit_stream_le.readBits(u1, 1, &out_bits)); + expect(out_bits == 1); + + mem_in_le.pos = 0; + bit_stream_le.bit_count = 0; + expect(0b001010100011101 == try bit_stream_le.readBits(u15, 15, &out_bits)); + expect(out_bits == 15); + + mem_in_le.pos = 0; + bit_stream_le.bit_count = 0; + expect(0b1001010100011101 == try bit_stream_le.readBits(u16, 16, &out_bits)); + expect(out_bits == 16); + + _ = try bit_stream_le.readBits(u0, 0, &out_bits); + + expect(0 == try bit_stream_le.readBits(u1, 1, &out_bits)); + expect(out_bits == 0); + expectError(error.EndOfStream, bit_stream_le.readBitsNoEof(u1, 1)); +} diff --git a/lib/std/io/bit_out_stream.zig b/lib/std/io/bit_out_stream.zig new file mode 100644 index 0000000000..1a2bb62e7c --- /dev/null +++ b/lib/std/io/bit_out_stream.zig @@ -0,0 +1,197 @@ +const std = @import("../std.zig"); +const builtin = std.builtin; +const io = std.io; +const testing = std.testing; +const assert = std.debug.assert; +const trait = std.meta.trait; +const meta = std.meta; +const math = std.math; + +/// Creates a stream which allows for writing bit fields to another stream +pub fn BitOutStream(endian: builtin.Endian, comptime OutStreamType: type) type { + return struct { + out_stream: OutStreamType, + bit_buffer: u8, + bit_count: u4, + + pub const Error = OutStreamType.Error; + pub const OutStream = io.OutStream(*Self, Error, write); + + const Self = @This(); + const u8_bit_count = comptime meta.bitCount(u8); + const u4_bit_count = comptime meta.bitCount(u4); + + pub fn init(out_stream: OutStreamType) Self { + return Self{ + .out_stream = out_stream, + .bit_buffer = 0, + .bit_count = 0, + }; + } + + /// Write the specified number of bits to the stream from the least significant bits of + /// the specified unsigned int value. Bits will only be written to the stream when there + /// are enough to fill a byte. + pub fn writeBits(self: *Self, value: var, bits: usize) Error!void { + if (bits == 0) return; + + const U = @TypeOf(value); + comptime assert(trait.isUnsignedInt(U)); + + //by extending the buffer to a minimum of u8 we can cover a number of edge cases + // related to shifting and casting. + const u_bit_count = comptime meta.bitCount(U); + const buf_bit_count = bc: { + assert(u_bit_count >= bits); + break :bc if (u_bit_count <= u8_bit_count) u8_bit_count else u_bit_count; + }; + const Buf = std.meta.IntType(false, buf_bit_count); + const BufShift = math.Log2Int(Buf); + + const buf_value = @intCast(Buf, value); + + const high_byte_shift = @intCast(BufShift, buf_bit_count - u8_bit_count); + var in_buffer = switch (endian) { + .Big => buf_value << @intCast(BufShift, buf_bit_count - bits), + .Little => buf_value, + }; + var in_bits = bits; + + if (self.bit_count > 0) { + const bits_remaining = u8_bit_count - self.bit_count; + const n = @intCast(u3, if (bits_remaining > bits) bits else bits_remaining); + switch (endian) { + .Big => { + const shift = @intCast(BufShift, high_byte_shift + self.bit_count); + const v = @intCast(u8, in_buffer >> shift); + self.bit_buffer |= v; + in_buffer <<= n; + }, + .Little => { + const v = @truncate(u8, in_buffer) << @intCast(u3, self.bit_count); + self.bit_buffer |= v; + in_buffer >>= n; + }, + } + self.bit_count += n; + in_bits -= n; + + //if we didn't fill the buffer, it's because bits < bits_remaining; + if (self.bit_count != u8_bit_count) return; + try self.out_stream.writeByte(self.bit_buffer); + self.bit_buffer = 0; + self.bit_count = 0; + } + //at this point we know bit_buffer is empty + + //copy bytes until we can't fill one anymore, then leave the rest in bit_buffer + while (in_bits >= u8_bit_count) { + switch (endian) { + .Big => { + const v = @intCast(u8, in_buffer >> high_byte_shift); + try self.out_stream.writeByte(v); + in_buffer <<= @intCast(u3, u8_bit_count - 1); + in_buffer <<= 1; + }, + .Little => { + const v = @truncate(u8, in_buffer); + try self.out_stream.writeByte(v); + in_buffer >>= @intCast(u3, u8_bit_count - 1); + in_buffer >>= 1; + }, + } + in_bits -= u8_bit_count; + } + + if (in_bits > 0) { + self.bit_count = @intCast(u4, in_bits); + self.bit_buffer = switch (endian) { + .Big => @truncate(u8, in_buffer >> high_byte_shift), + .Little => @truncate(u8, in_buffer), + }; + } + } + + /// Flush any remaining bits to the stream. + pub fn flushBits(self: *Self) Error!void { + if (self.bit_count == 0) return; + try self.out_stream.writeByte(self.bit_buffer); + self.bit_buffer = 0; + self.bit_count = 0; + } + + pub fn write(self: *Self, buffer: []const u8) Error!usize { + // TODO: I'm not sure this is a good idea, maybe flushBits should be forced + if (self.bit_count > 0) { + for (buffer) |b, i| + try self.writeBits(b, u8_bit_count); + return buffer.len; + } + + return self.out_stream.write(buffer); + } + + pub fn outStream(self: *Self) OutStream { + return .{ .context = self }; + } + }; +} + +pub fn bitOutStream( + comptime endian: builtin.Endian, + underlying_stream: var, +) BitOutStream(endian, @TypeOf(underlying_stream)) { + return BitOutStream(endian, @TypeOf(underlying_stream)).init(underlying_stream); +} + +test "api coverage" { + var mem_be = [_]u8{0} ** 2; + var mem_le = [_]u8{0} ** 2; + + var mem_out_be = io.fixedBufferStream(&mem_be); + var bit_stream_be = bitOutStream(.Big, mem_out_be.outStream()); + + try bit_stream_be.writeBits(@as(u2, 1), 1); + try bit_stream_be.writeBits(@as(u5, 2), 2); + try bit_stream_be.writeBits(@as(u128, 3), 3); + try bit_stream_be.writeBits(@as(u8, 4), 4); + try bit_stream_be.writeBits(@as(u9, 5), 5); + try bit_stream_be.writeBits(@as(u1, 1), 1); + + testing.expect(mem_be[0] == 0b11001101 and mem_be[1] == 0b00001011); + + mem_out_be.pos = 0; + + try bit_stream_be.writeBits(@as(u15, 0b110011010000101), 15); + try bit_stream_be.flushBits(); + testing.expect(mem_be[0] == 0b11001101 and mem_be[1] == 0b00001010); + + mem_out_be.pos = 0; + try bit_stream_be.writeBits(@as(u32, 0b110011010000101), 16); + testing.expect(mem_be[0] == 0b01100110 and mem_be[1] == 0b10000101); + + try bit_stream_be.writeBits(@as(u0, 0), 0); + + var mem_out_le = io.fixedBufferStream(&mem_le); + var bit_stream_le = bitOutStream(.Little, mem_out_le.outStream()); + + try bit_stream_le.writeBits(@as(u2, 1), 1); + try bit_stream_le.writeBits(@as(u5, 2), 2); + try bit_stream_le.writeBits(@as(u128, 3), 3); + try bit_stream_le.writeBits(@as(u8, 4), 4); + try bit_stream_le.writeBits(@as(u9, 5), 5); + try bit_stream_le.writeBits(@as(u1, 1), 1); + + testing.expect(mem_le[0] == 0b00011101 and mem_le[1] == 0b10010101); + + mem_out_le.pos = 0; + try bit_stream_le.writeBits(@as(u15, 0b110011010000101), 15); + try bit_stream_le.flushBits(); + testing.expect(mem_le[0] == 0b10000101 and mem_le[1] == 0b01100110); + + mem_out_le.pos = 0; + try bit_stream_le.writeBits(@as(u32, 0b1100110100001011), 16); + testing.expect(mem_le[0] == 0b00001011 and mem_le[1] == 0b11001101); + + try bit_stream_le.writeBits(@as(u0, 0), 0); +} diff --git a/lib/std/io/buffered_in_stream.zig b/lib/std/io/buffered_in_stream.zig index 72f1b30b76..7f76c8c576 100644 --- a/lib/std/io/buffered_in_stream.zig +++ b/lib/std/io/buffered_in_stream.zig @@ -1,7 +1,9 @@ const std = @import("../std.zig"); const io = std.io; +const assert = std.debug.assert; +const testing = std.testing; -pub fn BufferedInStream(comptime buffer_size: usize, comptime InStreamType) type { +pub fn BufferedInStream(comptime buffer_size: usize, comptime InStreamType: type) type { return struct { unbuffered_in_stream: InStreamType, fifo: FifoType = FifoType.init(), diff --git a/lib/std/io/c_out_stream.zig b/lib/std/io/c_out_stream.zig new file mode 100644 index 0000000000..106fc601a2 --- /dev/null +++ b/lib/std/io/c_out_stream.zig @@ -0,0 +1,44 @@ +const std = @import("../std.zig"); +const builtin = std.builtin; +const io = std.io; +const testing = std.testing; + +pub const COutStream = io.OutStream(*std.c.FILE, std.fs.File.WriteError, cOutStreamWrite); + +pub fn cOutStream(c_file: *std.c.FILE) COutStream { + return .{ .context = c_file }; +} + +fn cOutStreamWrite(c_file: *std.c.FILE, bytes: []const u8) std.fs.File.WriteError!usize { + const amt_written = std.c.fwrite(bytes.ptr, 1, bytes.len, c_file); + if (amt_written >= 0) return amt_written; + switch (std.c._errno().*) { + 0 => unreachable, + os.EINVAL => unreachable, + os.EFAULT => unreachable, + os.EAGAIN => unreachable, // this is a blocking API + os.EBADF => unreachable, // always a race condition + os.EDESTADDRREQ => unreachable, // connect was never called + os.EDQUOT => return error.DiskQuota, + os.EFBIG => return error.FileTooBig, + os.EIO => return error.InputOutput, + os.ENOSPC => return error.NoSpaceLeft, + os.EPERM => return error.AccessDenied, + os.EPIPE => return error.BrokenPipe, + else => |err| return os.unexpectedErrno(@intCast(usize, err)), + } +} + +test "" { + if (!builtin.link_libc) return error.SkipZigTest; + + const filename = "tmp_io_test_file.txt"; + const out_file = std.c.fopen(filename, "w") orelse return error.UnableToOpenTestFile; + defer { + _ = std.c.fclose(out_file); + fs.cwd().deleteFileC(filename) catch {}; + } + + const out_stream = &io.COutStream.init(out_file).stream; + try out_stream.print("hi: {}\n", .{@as(i32, 123)}); +} diff --git a/lib/std/io/counting_out_stream.zig b/lib/std/io/counting_out_stream.zig index 2b0cba29fc..f5bd6634f3 100644 --- a/lib/std/io/counting_out_stream.zig +++ b/lib/std/io/counting_out_stream.zig @@ -1,5 +1,6 @@ const std = @import("../std.zig"); const io = std.io; +const testing = std.testing; /// An OutStream that counts how many bytes has been written to it. pub fn CountingOutStream(comptime OutStreamType: type) type { @@ -12,13 +13,6 @@ pub fn CountingOutStream(comptime OutStreamType: type) type { const Self = @This(); - pub fn init(child_stream: OutStreamType) Self { - return Self{ - .bytes_written = 0, - .child_stream = child_stream, - }; - } - pub fn write(self: *Self, bytes: []const u8) Error!usize { const amt = try self.child_stream.write(bytes); self.bytes_written += amt; @@ -31,12 +25,15 @@ pub fn CountingOutStream(comptime OutStreamType: type) type { }; } -test "io.CountingOutStream" { - var counting_stream = CountingOutStream(NullOutStream.Error).init(std.io.null_out_stream); - const stream = &counting_stream.stream; - - const bytes = "yay" ** 10000; - stream.write(bytes) catch unreachable; - testing.expect(counting_stream.bytes_written == bytes.len); +pub fn countingOutStream(child_stream: var) CountingOutStream(@TypeOf(child_stream)) { + return .{ .bytes_written = 0, .child_stream = child_stream }; } +test "io.CountingOutStream" { + var counting_stream = countingOutStream(std.io.null_out_stream); + const stream = counting_stream.outStream(); + + const bytes = "yay" ** 100; + stream.writeAll(bytes) catch unreachable; + testing.expect(counting_stream.bytes_written == bytes.len); +} diff --git a/lib/std/io/fixed_buffer_stream.zig b/lib/std/io/fixed_buffer_stream.zig index 75a337d3de..9cdad7dc57 100644 --- a/lib/std/io/fixed_buffer_stream.zig +++ b/lib/std/io/fixed_buffer_stream.zig @@ -1,9 +1,11 @@ const std = @import("../std.zig"); const io = std.io; const testing = std.testing; +const mem = std.mem; +const assert = std.debug.assert; -/// This turns a slice into an `io.OutStream`, `io.InStream`, or `io.SeekableStream`. -/// If the supplied slice is const, then `io.OutStream` is not available. +/// This turns a byte buffer into an `io.OutStream`, `io.InStream`, or `io.SeekableStream`. +/// If the supplied byte buffer is const, then `io.OutStream` is not available. pub fn FixedBufferStream(comptime Buffer: type) type { return struct { /// `Buffer` is either a `[]u8` or `[]const u8`. @@ -46,7 +48,7 @@ pub fn FixedBufferStream(comptime Buffer: type) type { const size = std.math.min(dest.len, self.buffer.len - self.pos); const end = self.pos + size; - std.mem.copy(u8, dest[0..size], self.buffer[self.pos..end]); + mem.copy(u8, dest[0..size], self.buffer[self.pos..end]); self.pos = end; if (size == 0) return error.EndOfStream; @@ -65,7 +67,7 @@ pub fn FixedBufferStream(comptime Buffer: type) type { else self.buffer.len - self.pos; - std.mem.copy(u8, self.buffer[self.pos .. self.pos + n], bytes[0..n]); + mem.copy(u8, self.buffer[self.pos .. self.pos + n], bytes[0..n]); self.pos += n; if (n == 0) return error.OutOfMemory; @@ -100,7 +102,7 @@ pub fn FixedBufferStream(comptime Buffer: type) type { } pub fn getWritten(self: Self) []const u8 { - return self.slice[0..self.pos]; + return self.buffer[0..self.pos]; } pub fn reset(self: *Self) void { @@ -110,16 +112,16 @@ pub fn FixedBufferStream(comptime Buffer: type) type { } pub fn fixedBufferStream(buffer: var) FixedBufferStream(NonSentinelSpan(@TypeOf(buffer))) { - return .{ .buffer = std.mem.span(buffer), .pos = 0 }; + return .{ .buffer = mem.span(buffer), .pos = 0 }; } fn NonSentinelSpan(comptime T: type) type { - var ptr_info = @typeInfo(std.mem.Span(T)).Pointer; + var ptr_info = @typeInfo(mem.Span(T)).Pointer; ptr_info.sentinel = null; return @Type(std.builtin.TypeInfo{ .Pointer = ptr_info }); } -test "FixedBufferStream" { +test "FixedBufferStream output" { var buf: [255]u8 = undefined; var fbs = fixedBufferStream(&buf); const stream = fbs.outStream(); @@ -127,3 +129,41 @@ test "FixedBufferStream" { try stream.print("{}{}!", .{ "Hello", "World" }); testing.expectEqualSlices(u8, "HelloWorld!", fbs.getWritten()); } + +test "FixedBufferStream output 2" { + var buffer: [10]u8 = undefined; + var fbs = fixedBufferStream(&buffer); + + try fbs.outStream().writeAll("Hello"); + testing.expect(mem.eql(u8, fbs.getWritten(), "Hello")); + + try fbs.outStream().writeAll("world"); + testing.expect(mem.eql(u8, fbs.getWritten(), "Helloworld")); + + testing.expectError(error.OutOfMemory, fbs.outStream().writeAll("!")); + testing.expect(mem.eql(u8, fbs.getWritten(), "Helloworld")); + + fbs.reset(); + testing.expect(fbs.getWritten().len == 0); + + testing.expectError(error.OutOfMemory, fbs.outStream().writeAll("Hello world!")); + testing.expect(mem.eql(u8, fbs.getWritten(), "Hello worl")); +} + +test "FixedBufferStream input" { + const bytes = [_]u8{ 1, 2, 3, 4, 5, 6, 7 }; + var fbs = fixedBufferStream(&bytes); + + var dest: [4]u8 = undefined; + + var read = try fbs.inStream().read(dest[0..4]); + testing.expect(read == 4); + testing.expect(mem.eql(u8, dest[0..4], bytes[0..4])); + + read = try fbs.inStream().read(dest[0..4]); + testing.expect(read == 3); + testing.expect(mem.eql(u8, dest[0..3], bytes[4..7])); + + read = try fbs.inStream().read(dest[0..4]); + testing.expect(read == 0); +} diff --git a/lib/std/io/in_stream.zig b/lib/std/io/in_stream.zig index d181f4df43..2830582482 100644 --- a/lib/std/io/in_stream.zig +++ b/lib/std/io/in_stream.zig @@ -273,8 +273,7 @@ pub fn InStream( test "InStream" { var buf = "a\x02".*; - var slice_stream = std.io.SliceInStream.init(&buf); - const in_stream = &slice_stream.stream; + const in_stream = std.io.fixedBufferStream(&buf).inStream(); testing.expect((try in_stream.readByte()) == 'a'); testing.expect((try in_stream.readEnum(enum(u8) { a = 0, diff --git a/lib/std/io/peek_stream.zig b/lib/std/io/peek_stream.zig new file mode 100644 index 0000000000..5ee30ce273 --- /dev/null +++ b/lib/std/io/peek_stream.zig @@ -0,0 +1,112 @@ +const std = @import("../std.zig"); +const io = std.io; +const mem = std.mem; +const testing = std.testing; + +/// Creates a stream which supports 'un-reading' data, so that it can be read again. +/// This makes look-ahead style parsing much easier. +/// TODO merge this with `std.io.BufferedInStream`: https://github.com/ziglang/zig/issues/4501 +pub fn PeekStream( + comptime buffer_type: std.fifo.LinearFifoBufferType, + comptime InStreamType: type, +) type { + return struct { + unbuffered_in_stream: InStreamType, + fifo: FifoType, + + pub const Error = InStreamType.Error; + pub const InStream = io.InStream(*Self, Error, read); + + const Self = @This(); + const FifoType = std.fifo.LinearFifo(u8, buffer_type); + + pub usingnamespace switch (buffer_type) { + .Static => struct { + pub fn init(base: InStreamType) Self { + return .{ + .base = base, + .fifo = FifoType.init(), + }; + } + }, + .Slice => struct { + pub fn init(base: InStreamType, buf: []u8) Self { + return .{ + .base = base, + .fifo = FifoType.init(buf), + }; + } + }, + .Dynamic => struct { + pub fn init(base: InStreamType, allocator: *mem.Allocator) Self { + return .{ + .base = base, + .fifo = FifoType.init(allocator), + }; + } + }, + }; + + pub fn putBackByte(self: *Self, byte: u8) !void { + try self.putBack(&[_]u8{byte}); + } + + pub fn putBack(self: *Self, bytes: []const u8) !void { + try self.fifo.unget(bytes); + } + + pub fn read(self: *Self, dest: []u8) Error!usize { + // copy over anything putBack()'d + var dest_index = self.fifo.read(dest); + if (dest_index == dest.len) return dest_index; + + // ask the backing stream for more + dest_index += try self.base.read(dest[dest_index..]); + return dest_index; + } + + pub fn inStream(self: *Self) InStream { + return .{ .context = self }; + } + }; +} + +pub fn peekStream( + comptime lookahead: comptime_int, + underlying_stream: var, +) PeekStream(.{ .Static = lookahead }, @TypeOf(underlying_stream)) { + return PeekStream(.{ .Static = lookahead }, @TypeOf(underlying_stream)).init(underlying_stream); +} + +test "PeekStream" { + const bytes = [_]u8{ 1, 2, 3, 4, 5, 6, 7, 8 }; + var fbs = io.fixedBufferStream(&bytes); + var ps = peekStream(2, fbs.inStream()); + + var dest: [4]u8 = undefined; + + try ps.putBackByte(9); + try ps.putBackByte(10); + + var read = try ps.inStream().read(dest[0..4]); + testing.expect(read == 4); + testing.expect(dest[0] == 10); + testing.expect(dest[1] == 9); + testing.expect(mem.eql(u8, dest[2..4], bytes[0..2])); + + read = try ps.inStream().read(dest[0..4]); + testing.expect(read == 4); + testing.expect(mem.eql(u8, dest[0..4], bytes[2..6])); + + read = try ps.inStream().read(dest[0..4]); + testing.expect(read == 2); + testing.expect(mem.eql(u8, dest[0..2], bytes[6..8])); + + try ps.putBackByte(11); + try ps.putBackByte(12); + + read = try ps.inStream().read(dest[0..4]); + testing.expect(read == 2); + testing.expect(dest[0] == 12); + testing.expect(dest[1] == 11); +} diff --git a/lib/std/io/serialization.zig b/lib/std/io/serialization.zig new file mode 100644 index 0000000000..4aa462aab8 --- /dev/null +++ b/lib/std/io/serialization.zig @@ -0,0 +1,606 @@ +const std = @import("../std.zig"); +const builtin = std.builtin; +const io = std.io; + +pub const Packing = enum { + /// Pack data to byte alignment + Byte, + + /// Pack data to bit alignment + Bit, +}; + +/// Creates a deserializer that deserializes types from any stream. +/// If `is_packed` is true, the data stream is treated as bit-packed, +/// otherwise data is expected to be packed to the smallest byte. +/// Types may implement a custom deserialization routine with a +/// function named `deserialize` in the form of: +/// pub fn deserialize(self: *Self, deserializer: var) !void +/// which will be called when the deserializer is used to deserialize +/// that type. It will pass a pointer to the type instance to deserialize +/// into and a pointer to the deserializer struct. +pub fn Deserializer(comptime endian: builtin.Endian, comptime packing: Packing, comptime InStreamType: type) type { + return struct { + in_stream: if (packing == .Bit) io.BitInStream(endian, InStreamType) else InStreamType, + + const Self = @This(); + + pub fn init(in_stream: InStreamType) Self { + return Self{ + .in_stream = switch (packing) { + .Bit => io.bitInStream(endian, in_stream), + .Byte => in_stream, + }, + }; + } + + pub fn alignToByte(self: *Self) void { + if (packing == .Byte) return; + self.in_stream.alignToByte(); + } + + //@BUG: inferred error issue. See: #1386 + fn deserializeInt(self: *Self, comptime T: type) (InStreamType.Error || error{EndOfStream})!T { + comptime assert(trait.is(.Int)(T) or trait.is(.Float)(T)); + + const u8_bit_count = 8; + const t_bit_count = comptime meta.bitCount(T); + + const U = std.meta.IntType(false, t_bit_count); + const Log2U = math.Log2Int(U); + const int_size = (U.bit_count + 7) / 8; + + if (packing == .Bit) { + const result = try self.in_stream.readBitsNoEof(U, t_bit_count); + return @bitCast(T, result); + } + + var buffer: [int_size]u8 = undefined; + const read_size = try self.in_stream.read(buffer[0..]); + if (read_size < int_size) return error.EndOfStream; + + if (int_size == 1) { + if (t_bit_count == 8) return @bitCast(T, buffer[0]); + const PossiblySignedByte = std.meta.IntType(T.is_signed, 8); + return @truncate(T, @bitCast(PossiblySignedByte, buffer[0])); + } + + var result = @as(U, 0); + for (buffer) |byte, i| { + switch (endian) { + .Big => { + result = (result << u8_bit_count) | byte; + }, + .Little => { + result |= @as(U, byte) << @intCast(Log2U, u8_bit_count * i); + }, + } + } + + return @bitCast(T, result); + } + + /// Deserializes and returns data of the specified type from the stream + pub fn deserialize(self: *Self, comptime T: type) !T { + var value: T = undefined; + try self.deserializeInto(&value); + return value; + } + + /// Deserializes data into the type pointed to by `ptr` + pub fn deserializeInto(self: *Self, ptr: var) !void { + const T = @TypeOf(ptr); + comptime assert(trait.is(.Pointer)(T)); + + if (comptime trait.isSlice(T) or comptime trait.isPtrTo(.Array)(T)) { + for (ptr) |*v| + try self.deserializeInto(v); + return; + } + + comptime assert(trait.isSingleItemPtr(T)); + + const C = comptime meta.Child(T); + const child_type_id = @typeInfo(C); + + //custom deserializer: fn(self: *Self, deserializer: var) !void + if (comptime trait.hasFn("deserialize")(C)) return C.deserialize(ptr, self); + + if (comptime trait.isPacked(C) and packing != .Bit) { + var packed_deserializer = deserializer(endian, .Bit, self.in_stream); + return packed_deserializer.deserializeInto(ptr); + } + + switch (child_type_id) { + .Void => return, + .Bool => ptr.* = (try self.deserializeInt(u1)) > 0, + .Float, .Int => ptr.* = try self.deserializeInt(C), + .Struct => { + const info = @typeInfo(C).Struct; + + inline for (info.fields) |*field_info| { + const name = field_info.name; + const FieldType = field_info.field_type; + + if (FieldType == void or FieldType == u0) continue; + + //it doesn't make any sense to read pointers + if (comptime trait.is(.Pointer)(FieldType)) { + @compileError("Will not " ++ "read field " ++ name ++ " of struct " ++ + @typeName(C) ++ " because it " ++ "is of pointer-type " ++ + @typeName(FieldType) ++ "."); + } + + try self.deserializeInto(&@field(ptr, name)); + } + }, + .Union => { + const info = @typeInfo(C).Union; + if (info.tag_type) |TagType| { + //we avoid duplicate iteration over the enum tags + // by getting the int directly and casting it without + // safety. If it is bad, it will be caught anyway. + const TagInt = @TagType(TagType); + const tag = try self.deserializeInt(TagInt); + + inline for (info.fields) |field_info| { + if (field_info.enum_field.?.value == tag) { + const name = field_info.name; + const FieldType = field_info.field_type; + ptr.* = @unionInit(C, name, undefined); + try self.deserializeInto(&@field(ptr, name)); + return; + } + } + //This is reachable if the enum data is bad + return error.InvalidEnumTag; + } + @compileError("Cannot meaningfully deserialize " ++ @typeName(C) ++ + " because it is an untagged union. Use a custom deserialize()."); + }, + .Optional => { + const OC = comptime meta.Child(C); + const exists = (try self.deserializeInt(u1)) > 0; + if (!exists) { + ptr.* = null; + return; + } + + ptr.* = @as(OC, undefined); //make it non-null so the following .? is guaranteed safe + const val_ptr = &ptr.*.?; + try self.deserializeInto(val_ptr); + }, + .Enum => { + var value = try self.deserializeInt(@TagType(C)); + ptr.* = try meta.intToEnum(C, value); + }, + else => { + @compileError("Cannot deserialize " ++ @tagName(child_type_id) ++ " types (unimplemented)."); + }, + } + } + }; +} + +pub fn deserializer( + comptime endian: builtin.Endian, + comptime packing: Packing, + in_stream: var, +) Deserializer(endian, packing, @TypeOf(in_stream)) { + return Deserializer(endian, packing, @TypeOf(in_stream)).init(in_stream); +} + +/// Creates a serializer that serializes types to any stream. +/// If `is_packed` is true, the data will be bit-packed into the stream. +/// Note that the you must call `serializer.flush()` when you are done +/// writing bit-packed data in order ensure any unwritten bits are committed. +/// If `is_packed` is false, data is packed to the smallest byte. In the case +/// of packed structs, the struct will written bit-packed and with the specified +/// endianess, after which data will resume being written at the next byte boundary. +/// Types may implement a custom serialization routine with a +/// function named `serialize` in the form of: +/// pub fn serialize(self: Self, serializer: var) !void +/// which will be called when the serializer is used to serialize that type. It will +/// pass a const pointer to the type instance to be serialized and a pointer +/// to the serializer struct. +pub fn Serializer(comptime endian: builtin.Endian, comptime packing: Packing, comptime OutStreamType: type) type { + return struct { + out_stream: if (packing == .Bit) BitOutStream(endian, OutStreamType) else OutStreamType, + + const Self = @This(); + pub const Error = OutStreamType.Error; + + pub fn init(out_stream: OutStreamType) Self { + return Self{ + .out_stream = switch (packing) { + .Bit => io.bitOutStream(endian, out_stream), + .Byte => out_stream, + }, + }; + } + + /// Flushes any unwritten bits to the stream + pub fn flush(self: *Self) Error!void { + if (packing == .Bit) return self.out_stream.flushBits(); + } + + fn serializeInt(self: *Self, value: var) Error!void { + const T = @TypeOf(value); + comptime assert(trait.is(.Int)(T) or trait.is(.Float)(T)); + + const t_bit_count = comptime meta.bitCount(T); + const u8_bit_count = comptime meta.bitCount(u8); + + const U = std.meta.IntType(false, t_bit_count); + const Log2U = math.Log2Int(U); + const int_size = (U.bit_count + 7) / 8; + + const u_value = @bitCast(U, value); + + if (packing == .Bit) return self.out_stream.writeBits(u_value, t_bit_count); + + var buffer: [int_size]u8 = undefined; + if (int_size == 1) buffer[0] = u_value; + + for (buffer) |*byte, i| { + const idx = switch (endian) { + .Big => int_size - i - 1, + .Little => i, + }; + const shift = @intCast(Log2U, idx * u8_bit_count); + const v = u_value >> shift; + byte.* = if (t_bit_count < u8_bit_count) v else @truncate(u8, v); + } + + try self.out_stream.write(&buffer); + } + + /// Serializes the passed value into the stream + pub fn serialize(self: *Self, value: var) Error!void { + const T = comptime @TypeOf(value); + + if (comptime trait.isIndexable(T)) { + for (value) |v| + try self.serialize(v); + return; + } + + //custom serializer: fn(self: Self, serializer: var) !void + if (comptime trait.hasFn("serialize")(T)) return T.serialize(value, self); + + if (comptime trait.isPacked(T) and packing != .Bit) { + var packed_serializer = Serializer(endian, .Bit, Error).init(self.out_stream); + try packed_serializer.serialize(value); + try packed_serializer.flush(); + return; + } + + switch (@typeInfo(T)) { + .Void => return, + .Bool => try self.serializeInt(@as(u1, @boolToInt(value))), + .Float, .Int => try self.serializeInt(value), + .Struct => { + const info = @typeInfo(T); + + inline for (info.Struct.fields) |*field_info| { + const name = field_info.name; + const FieldType = field_info.field_type; + + if (FieldType == void or FieldType == u0) continue; + + //It doesn't make sense to write pointers + if (comptime trait.is(.Pointer)(FieldType)) { + @compileError("Will not " ++ "serialize field " ++ name ++ + " of struct " ++ @typeName(T) ++ " because it " ++ + "is of pointer-type " ++ @typeName(FieldType) ++ "."); + } + try self.serialize(@field(value, name)); + } + }, + .Union => { + const info = @typeInfo(T).Union; + if (info.tag_type) |TagType| { + const active_tag = meta.activeTag(value); + try self.serialize(active_tag); + //This inline loop is necessary because active_tag is a runtime + // value, but @field requires a comptime value. Our alternative + // is to check each field for a match + inline for (info.fields) |field_info| { + if (field_info.enum_field.?.value == @enumToInt(active_tag)) { + const name = field_info.name; + const FieldType = field_info.field_type; + try self.serialize(@field(value, name)); + return; + } + } + unreachable; + } + @compileError("Cannot meaningfully serialize " ++ @typeName(T) ++ + " because it is an untagged union. Use a custom serialize()."); + }, + .Optional => { + if (value == null) { + try self.serializeInt(@as(u1, @boolToInt(false))); + return; + } + try self.serializeInt(@as(u1, @boolToInt(true))); + + const OC = comptime meta.Child(T); + const val_ptr = &value.?; + try self.serialize(val_ptr.*); + }, + .Enum => { + try self.serializeInt(@enumToInt(value)); + }, + else => @compileError("Cannot serialize " ++ @tagName(@typeInfo(T)) ++ " types (unimplemented)."), + } + } + }; +} + +pub fn serializer( + comptime endian: builtin.Endian, + comptime packing: Packing, + out_stream: var, +) Serializer(endian, packing, @TypeOf(out_stream)) { + return Serializer(endian, packing, @TypeOf(out_stream)).init(out_stream); +} + +fn testIntSerializerDeserializer(comptime endian: builtin.Endian, comptime packing: io.Packing) !void { + @setEvalBranchQuota(1500); + //@NOTE: if this test is taking too long, reduce the maximum tested bitsize + const max_test_bitsize = 128; + + const total_bytes = comptime blk: { + var bytes = 0; + comptime var i = 0; + while (i <= max_test_bitsize) : (i += 1) bytes += (i / 8) + @boolToInt(i % 8 > 0); + break :blk bytes * 2; + }; + + var data_mem: [total_bytes]u8 = undefined; + var out = io.fixedBufferStream(&data_mem); + var serializer = serializer(endian, packing, out.outStream()); + + var in = io.fixedBufferStream(&data_mem); + var deserializer = Deserializer(endian, packing, in.inStream()); + + comptime var i = 0; + inline while (i <= max_test_bitsize) : (i += 1) { + const U = std.meta.IntType(false, i); + const S = std.meta.IntType(true, i); + try serializer.serializeInt(@as(U, i)); + if (i != 0) try serializer.serializeInt(@as(S, -1)) else try serializer.serialize(@as(S, 0)); + } + try serializer.flush(); + + i = 0; + inline while (i <= max_test_bitsize) : (i += 1) { + const U = std.meta.IntType(false, i); + const S = std.meta.IntType(true, i); + const x = try deserializer.deserializeInt(U); + const y = try deserializer.deserializeInt(S); + expect(x == @as(U, i)); + if (i != 0) expect(y == @as(S, -1)) else expect(y == 0); + } + + const u8_bit_count = comptime meta.bitCount(u8); + //0 + 1 + 2 + ... n = (n * (n + 1)) / 2 + //and we have each for unsigned and signed, so * 2 + const total_bits = (max_test_bitsize * (max_test_bitsize + 1)); + const extra_packed_byte = @boolToInt(total_bits % u8_bit_count > 0); + const total_packed_bytes = (total_bits / u8_bit_count) + extra_packed_byte; + + expect(in.pos == if (packing == .Bit) total_packed_bytes else total_bytes); + + //Verify that empty error set works with serializer. + //deserializer is covered by FixedBufferStream + var null_serializer = io.serializer(endian, packing, std.io.null_out_stream); + try null_serializer.serialize(data_mem[0..]); + try null_serializer.flush(); +} + +test "Serializer/Deserializer Int" { + try testIntSerializerDeserializer(.Big, .Byte); + try testIntSerializerDeserializer(.Little, .Byte); + // TODO these tests are disabled due to tripping an LLVM assertion + // https://github.com/ziglang/zig/issues/2019 + //try testIntSerializerDeserializer(builtin.Endian.Big, true); + //try testIntSerializerDeserializer(builtin.Endian.Little, true); +} + +fn testIntSerializerDeserializerInfNaN( + comptime endian: builtin.Endian, + comptime packing: io.Packing, +) !void { + const mem_size = (16 * 2 + 32 * 2 + 64 * 2 + 128 * 2) / comptime meta.bitCount(u8); + var data_mem: [mem_size]u8 = undefined; + + var out = io.fixedBufferStream(&data_mem); + var serializer = serializer(endian, packing, out.outStream()); + + var in = io.fixedBufferStream(&data_mem); + var deserializer = deserializer(endian, packing, in.inStream()); + + //@TODO: isInf/isNan not currently implemented for f128. + try serializer.serialize(std.math.nan(f16)); + try serializer.serialize(std.math.inf(f16)); + try serializer.serialize(std.math.nan(f32)); + try serializer.serialize(std.math.inf(f32)); + try serializer.serialize(std.math.nan(f64)); + try serializer.serialize(std.math.inf(f64)); + //try serializer.serialize(std.math.nan(f128)); + //try serializer.serialize(std.math.inf(f128)); + const nan_check_f16 = try deserializer.deserialize(f16); + const inf_check_f16 = try deserializer.deserialize(f16); + const nan_check_f32 = try deserializer.deserialize(f32); + deserializer.alignToByte(); + const inf_check_f32 = try deserializer.deserialize(f32); + const nan_check_f64 = try deserializer.deserialize(f64); + const inf_check_f64 = try deserializer.deserialize(f64); + //const nan_check_f128 = try deserializer.deserialize(f128); + //const inf_check_f128 = try deserializer.deserialize(f128); + expect(std.math.isNan(nan_check_f16)); + expect(std.math.isInf(inf_check_f16)); + expect(std.math.isNan(nan_check_f32)); + expect(std.math.isInf(inf_check_f32)); + expect(std.math.isNan(nan_check_f64)); + expect(std.math.isInf(inf_check_f64)); + //expect(std.math.isNan(nan_check_f128)); + //expect(std.math.isInf(inf_check_f128)); +} + +test "Serializer/Deserializer Int: Inf/NaN" { + try testIntSerializerDeserializerInfNaN(.Big, .Byte); + try testIntSerializerDeserializerInfNaN(.Little, .Byte); + try testIntSerializerDeserializerInfNaN(.Big, .Bit); + try testIntSerializerDeserializerInfNaN(.Little, .Bit); +} + +fn testAlternateSerializer(self: var, serializer: var) !void { + try serializer.serialize(self.f_f16); +} + +fn testSerializerDeserializer(comptime endian: builtin.Endian, comptime packing: io.Packing) !void { + const ColorType = enum(u4) { + RGB8 = 1, + RA16 = 2, + R32 = 3, + }; + + const TagAlign = union(enum(u32)) { + A: u8, + B: u8, + C: u8, + }; + + const Color = union(ColorType) { + RGB8: struct { + r: u8, + g: u8, + b: u8, + a: u8, + }, + RA16: struct { + r: u16, + a: u16, + }, + R32: u32, + }; + + const PackedStruct = packed struct { + f_i3: i3, + f_u2: u2, + }; + + //to test custom serialization + const Custom = struct { + f_f16: f16, + f_unused_u32: u32, + + pub fn deserialize(self: *@This(), deserializer: var) !void { + try deserializer.deserializeInto(&self.f_f16); + self.f_unused_u32 = 47; + } + + pub const serialize = testAlternateSerializer; + }; + + const MyStruct = struct { + f_i3: i3, + f_u8: u8, + f_tag_align: TagAlign, + f_u24: u24, + f_i19: i19, + f_void: void, + f_f32: f32, + f_f128: f128, + f_packed_0: PackedStruct, + f_i7arr: [10]i7, + f_of64n: ?f64, + f_of64v: ?f64, + f_color_type: ColorType, + f_packed_1: PackedStruct, + f_custom: Custom, + f_color: Color, + }; + + const my_inst = MyStruct{ + .f_i3 = -1, + .f_u8 = 8, + .f_tag_align = TagAlign{ .B = 148 }, + .f_u24 = 24, + .f_i19 = 19, + .f_void = {}, + .f_f32 = 32.32, + .f_f128 = 128.128, + .f_packed_0 = PackedStruct{ .f_i3 = -1, .f_u2 = 2 }, + .f_i7arr = [10]i7{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, + .f_of64n = null, + .f_of64v = 64.64, + .f_color_type = ColorType.R32, + .f_packed_1 = PackedStruct{ .f_i3 = 1, .f_u2 = 1 }, + .f_custom = Custom{ .f_f16 = 38.63, .f_unused_u32 = 47 }, + .f_color = Color{ .R32 = 123822 }, + }; + + var data_mem: [@sizeOf(MyStruct)]u8 = undefined; + var out = io.fixedBufferStream(&data_mem); + var serializer = serializer(endian, packing, out.outStream()); + + var in = io.fixedBufferStream(&data_mem); + var deserializer = deserializer(endian, packing, in.inStream()); + + try serializer.serialize(my_inst); + + const my_copy = try deserializer.deserialize(MyStruct); + expect(meta.eql(my_copy, my_inst)); +} + +test "Serializer/Deserializer generic" { + if (std.Target.current.os.tag == .windows) { + // TODO https://github.com/ziglang/zig/issues/508 + return error.SkipZigTest; + } + try testSerializerDeserializer(builtin.Endian.Big, .Byte); + try testSerializerDeserializer(builtin.Endian.Little, .Byte); + try testSerializerDeserializer(builtin.Endian.Big, .Bit); + try testSerializerDeserializer(builtin.Endian.Little, .Bit); +} + +fn testBadData(comptime endian: builtin.Endian, comptime packing: io.Packing) !void { + const E = enum(u14) { + One = 1, + Two = 2, + }; + + const A = struct { + e: E, + }; + + const C = union(E) { + One: u14, + Two: f16, + }; + + var data_mem: [4]u8 = undefined; + var out = io.fixedBufferStream.init(&data_mem); + var serializer = serializer(endian, packing, out.outStream()); + + var in = io.fixedBufferStream(&data_mem); + var deserializer = deserializer(endian, packing, in.inStream()); + + try serializer.serialize(@as(u14, 3)); + expectError(error.InvalidEnumTag, deserializer.deserialize(A)); + out.pos = 0; + try serializer.serialize(@as(u14, 3)); + try serializer.serialize(@as(u14, 88)); + expectError(error.InvalidEnumTag, deserializer.deserialize(C)); +} + +test "Deserializer bad data" { + try testBadData(.Big, .Byte); + try testBadData(.Little, .Byte); + try testBadData(.Big, .Bit); + try testBadData(.Little, .Bit); +} diff --git a/lib/std/io/test.zig b/lib/std/io/test.zig index 1ab0f82313..38dd2bb67a 100644 --- a/lib/std/io/test.zig +++ b/lib/std/io/test.zig @@ -22,11 +22,10 @@ test "write a file, read it, then delete it" { var file = try cwd.createFile(tmp_file_name, .{}); defer file.close(); - var file_out_stream = file.outStream(); - var buf_stream = io.BufferedOutStream(File.WriteError).init(&file_out_stream.stream); - const st = &buf_stream.stream; + var buf_stream = io.bufferedOutStream(file.outStream()); + const st = buf_stream.outStream(); try st.print("begin", .{}); - try st.write(data[0..]); + try st.writeAll(data[0..]); try st.print("end", .{}); try buf_stream.flush(); } @@ -48,9 +47,8 @@ test "write a file, read it, then delete it" { const expected_file_size: u64 = "begin".len + data.len + "end".len; expectEqual(expected_file_size, file_size); - var file_in_stream = file.inStream(); - var buf_stream = io.BufferedInStream(File.ReadError).init(&file_in_stream.stream); - const st = &buf_stream.stream; + var buf_stream = io.bufferedInStream(file.inStream()); + const st = buf_stream.inStream(); const contents = try st.readAllAlloc(std.testing.allocator, 2 * 1024); defer std.testing.allocator.free(contents); @@ -61,224 +59,13 @@ test "write a file, read it, then delete it" { try cwd.deleteFile(tmp_file_name); } -test "BufferOutStream" { - var buffer = try std.Buffer.initSize(std.testing.allocator, 0); - defer buffer.deinit(); - var buf_stream = &std.io.BufferOutStream.init(&buffer).stream; - - const x: i32 = 42; - const y: i32 = 1234; - try buf_stream.print("x: {}\ny: {}\n", .{ x, y }); - - expect(mem.eql(u8, buffer.toSlice(), "x: 42\ny: 1234\n")); -} - -test "SliceInStream" { - const bytes = [_]u8{ 1, 2, 3, 4, 5, 6, 7 }; - var ss = io.SliceInStream.init(&bytes); - - var dest: [4]u8 = undefined; - - var read = try ss.stream.read(dest[0..4]); - expect(read == 4); - expect(mem.eql(u8, dest[0..4], bytes[0..4])); - - read = try ss.stream.read(dest[0..4]); - expect(read == 3); - expect(mem.eql(u8, dest[0..3], bytes[4..7])); - - read = try ss.stream.read(dest[0..4]); - expect(read == 0); -} - -test "PeekStream" { - const bytes = [_]u8{ 1, 2, 3, 4, 5, 6, 7, 8 }; - var ss = io.SliceInStream.init(&bytes); - var ps = io.PeekStream(.{ .Static = 2 }, io.SliceInStream.Error).init(&ss.stream); - - var dest: [4]u8 = undefined; - - try ps.putBackByte(9); - try ps.putBackByte(10); - - var read = try ps.stream.read(dest[0..4]); - expect(read == 4); - expect(dest[0] == 10); - expect(dest[1] == 9); - expect(mem.eql(u8, dest[2..4], bytes[0..2])); - - read = try ps.stream.read(dest[0..4]); - expect(read == 4); - expect(mem.eql(u8, dest[0..4], bytes[2..6])); - - read = try ps.stream.read(dest[0..4]); - expect(read == 2); - expect(mem.eql(u8, dest[0..2], bytes[6..8])); - - try ps.putBackByte(11); - try ps.putBackByte(12); - - read = try ps.stream.read(dest[0..4]); - expect(read == 2); - expect(dest[0] == 12); - expect(dest[1] == 11); -} - -test "SliceOutStream" { - var buffer: [10]u8 = undefined; - var ss = io.SliceOutStream.init(buffer[0..]); - - try ss.stream.write("Hello"); - expect(mem.eql(u8, ss.getWritten(), "Hello")); - - try ss.stream.write("world"); - expect(mem.eql(u8, ss.getWritten(), "Helloworld")); - - expectError(error.OutOfMemory, ss.stream.write("!")); - expect(mem.eql(u8, ss.getWritten(), "Helloworld")); - - ss.reset(); - expect(ss.getWritten().len == 0); - - expectError(error.OutOfMemory, ss.stream.write("Hello world!")); - expect(mem.eql(u8, ss.getWritten(), "Hello worl")); -} - -test "BitInStream" { - const mem_be = [_]u8{ 0b11001101, 0b00001011 }; - const mem_le = [_]u8{ 0b00011101, 0b10010101 }; - - var mem_in_be = io.SliceInStream.init(mem_be[0..]); - const InError = io.SliceInStream.Error; - var bit_stream_be = io.BitInStream(builtin.Endian.Big, InError).init(&mem_in_be.stream); - - var out_bits: usize = undefined; - - expect(1 == try bit_stream_be.readBits(u2, 1, &out_bits)); - expect(out_bits == 1); - expect(2 == try bit_stream_be.readBits(u5, 2, &out_bits)); - expect(out_bits == 2); - expect(3 == try bit_stream_be.readBits(u128, 3, &out_bits)); - expect(out_bits == 3); - expect(4 == try bit_stream_be.readBits(u8, 4, &out_bits)); - expect(out_bits == 4); - expect(5 == try bit_stream_be.readBits(u9, 5, &out_bits)); - expect(out_bits == 5); - expect(1 == try bit_stream_be.readBits(u1, 1, &out_bits)); - expect(out_bits == 1); - - mem_in_be.pos = 0; - bit_stream_be.bit_count = 0; - expect(0b110011010000101 == try bit_stream_be.readBits(u15, 15, &out_bits)); - expect(out_bits == 15); - - mem_in_be.pos = 0; - bit_stream_be.bit_count = 0; - expect(0b1100110100001011 == try bit_stream_be.readBits(u16, 16, &out_bits)); - expect(out_bits == 16); - - _ = try bit_stream_be.readBits(u0, 0, &out_bits); - - expect(0 == try bit_stream_be.readBits(u1, 1, &out_bits)); - expect(out_bits == 0); - expectError(error.EndOfStream, bit_stream_be.readBitsNoEof(u1, 1)); - - var mem_in_le = io.SliceInStream.init(mem_le[0..]); - var bit_stream_le = io.BitInStream(builtin.Endian.Little, InError).init(&mem_in_le.stream); - - expect(1 == try bit_stream_le.readBits(u2, 1, &out_bits)); - expect(out_bits == 1); - expect(2 == try bit_stream_le.readBits(u5, 2, &out_bits)); - expect(out_bits == 2); - expect(3 == try bit_stream_le.readBits(u128, 3, &out_bits)); - expect(out_bits == 3); - expect(4 == try bit_stream_le.readBits(u8, 4, &out_bits)); - expect(out_bits == 4); - expect(5 == try bit_stream_le.readBits(u9, 5, &out_bits)); - expect(out_bits == 5); - expect(1 == try bit_stream_le.readBits(u1, 1, &out_bits)); - expect(out_bits == 1); - - mem_in_le.pos = 0; - bit_stream_le.bit_count = 0; - expect(0b001010100011101 == try bit_stream_le.readBits(u15, 15, &out_bits)); - expect(out_bits == 15); - - mem_in_le.pos = 0; - bit_stream_le.bit_count = 0; - expect(0b1001010100011101 == try bit_stream_le.readBits(u16, 16, &out_bits)); - expect(out_bits == 16); - - _ = try bit_stream_le.readBits(u0, 0, &out_bits); - - expect(0 == try bit_stream_le.readBits(u1, 1, &out_bits)); - expect(out_bits == 0); - expectError(error.EndOfStream, bit_stream_le.readBitsNoEof(u1, 1)); -} - -test "BitOutStream" { - var mem_be = [_]u8{0} ** 2; - var mem_le = [_]u8{0} ** 2; - - var mem_out_be = io.SliceOutStream.init(mem_be[0..]); - const OutError = io.SliceOutStream.Error; - var bit_stream_be = io.BitOutStream(builtin.Endian.Big, OutError).init(&mem_out_be.stream); - - try bit_stream_be.writeBits(@as(u2, 1), 1); - try bit_stream_be.writeBits(@as(u5, 2), 2); - try bit_stream_be.writeBits(@as(u128, 3), 3); - try bit_stream_be.writeBits(@as(u8, 4), 4); - try bit_stream_be.writeBits(@as(u9, 5), 5); - try bit_stream_be.writeBits(@as(u1, 1), 1); - - expect(mem_be[0] == 0b11001101 and mem_be[1] == 0b00001011); - - mem_out_be.pos = 0; - - try bit_stream_be.writeBits(@as(u15, 0b110011010000101), 15); - try bit_stream_be.flushBits(); - expect(mem_be[0] == 0b11001101 and mem_be[1] == 0b00001010); - - mem_out_be.pos = 0; - try bit_stream_be.writeBits(@as(u32, 0b110011010000101), 16); - expect(mem_be[0] == 0b01100110 and mem_be[1] == 0b10000101); - - try bit_stream_be.writeBits(@as(u0, 0), 0); - - var mem_out_le = io.SliceOutStream.init(mem_le[0..]); - var bit_stream_le = io.BitOutStream(builtin.Endian.Little, OutError).init(&mem_out_le.stream); - - try bit_stream_le.writeBits(@as(u2, 1), 1); - try bit_stream_le.writeBits(@as(u5, 2), 2); - try bit_stream_le.writeBits(@as(u128, 3), 3); - try bit_stream_le.writeBits(@as(u8, 4), 4); - try bit_stream_le.writeBits(@as(u9, 5), 5); - try bit_stream_le.writeBits(@as(u1, 1), 1); - - expect(mem_le[0] == 0b00011101 and mem_le[1] == 0b10010101); - - mem_out_le.pos = 0; - try bit_stream_le.writeBits(@as(u15, 0b110011010000101), 15); - try bit_stream_le.flushBits(); - expect(mem_le[0] == 0b10000101 and mem_le[1] == 0b01100110); - - mem_out_le.pos = 0; - try bit_stream_le.writeBits(@as(u32, 0b1100110100001011), 16); - expect(mem_le[0] == 0b00001011 and mem_le[1] == 0b11001101); - - try bit_stream_le.writeBits(@as(u0, 0), 0); -} - test "BitStreams with File Stream" { const tmp_file_name = "temp_test_file.txt"; { var file = try fs.cwd().createFile(tmp_file_name, .{}); defer file.close(); - var file_out = file.outStream(); - var file_out_stream = &file_out.stream; - const OutError = File.WriteError; - var bit_stream = io.BitOutStream(builtin.endian, OutError).init(file_out_stream); + var bit_stream = io.bitOutStream(builtin.endian, file.outStream()); try bit_stream.writeBits(@as(u2, 1), 1); try bit_stream.writeBits(@as(u5, 2), 2); @@ -292,10 +79,7 @@ test "BitStreams with File Stream" { var file = try fs.cwd().openFile(tmp_file_name, .{}); defer file.close(); - var file_in = file.inStream(); - var file_in_stream = &file_in.stream; - const InError = File.ReadError; - var bit_stream = io.BitInStream(builtin.endian, InError).init(file_in_stream); + var bit_stream = io.bitInStream(builtin.endian, file.inStream()); var out_bits: usize = undefined; @@ -317,298 +101,6 @@ test "BitStreams with File Stream" { try fs.cwd().deleteFile(tmp_file_name); } -fn testIntSerializerDeserializer(comptime endian: builtin.Endian, comptime packing: io.Packing) !void { - @setEvalBranchQuota(1500); - //@NOTE: if this test is taking too long, reduce the maximum tested bitsize - const max_test_bitsize = 128; - - const total_bytes = comptime blk: { - var bytes = 0; - comptime var i = 0; - while (i <= max_test_bitsize) : (i += 1) bytes += (i / 8) + @boolToInt(i % 8 > 0); - break :blk bytes * 2; - }; - - var data_mem: [total_bytes]u8 = undefined; - var out = io.SliceOutStream.init(data_mem[0..]); - const OutError = io.SliceOutStream.Error; - var out_stream = &out.stream; - var serializer = io.Serializer(endian, packing, OutError).init(out_stream); - - var in = io.SliceInStream.init(data_mem[0..]); - const InError = io.SliceInStream.Error; - var in_stream = &in.stream; - var deserializer = io.Deserializer(endian, packing, InError).init(in_stream); - - comptime var i = 0; - inline while (i <= max_test_bitsize) : (i += 1) { - const U = std.meta.IntType(false, i); - const S = std.meta.IntType(true, i); - try serializer.serializeInt(@as(U, i)); - if (i != 0) try serializer.serializeInt(@as(S, -1)) else try serializer.serialize(@as(S, 0)); - } - try serializer.flush(); - - i = 0; - inline while (i <= max_test_bitsize) : (i += 1) { - const U = std.meta.IntType(false, i); - const S = std.meta.IntType(true, i); - const x = try deserializer.deserializeInt(U); - const y = try deserializer.deserializeInt(S); - expect(x == @as(U, i)); - if (i != 0) expect(y == @as(S, -1)) else expect(y == 0); - } - - const u8_bit_count = comptime meta.bitCount(u8); - //0 + 1 + 2 + ... n = (n * (n + 1)) / 2 - //and we have each for unsigned and signed, so * 2 - const total_bits = (max_test_bitsize * (max_test_bitsize + 1)); - const extra_packed_byte = @boolToInt(total_bits % u8_bit_count > 0); - const total_packed_bytes = (total_bits / u8_bit_count) + extra_packed_byte; - - expect(in.pos == if (packing == .Bit) total_packed_bytes else total_bytes); - - //Verify that empty error set works with serializer. - //deserializer is covered by SliceInStream - const NullError = io.NullOutStream.Error; - var null_out = io.NullOutStream.init(); - var null_out_stream = &null_out.stream; - var null_serializer = io.Serializer(endian, packing, NullError).init(null_out_stream); - try null_serializer.serialize(data_mem[0..]); - try null_serializer.flush(); -} - -test "Serializer/Deserializer Int" { - try testIntSerializerDeserializer(.Big, .Byte); - try testIntSerializerDeserializer(.Little, .Byte); - // TODO these tests are disabled due to tripping an LLVM assertion - // https://github.com/ziglang/zig/issues/2019 - //try testIntSerializerDeserializer(builtin.Endian.Big, true); - //try testIntSerializerDeserializer(builtin.Endian.Little, true); -} - -fn testIntSerializerDeserializerInfNaN( - comptime endian: builtin.Endian, - comptime packing: io.Packing, -) !void { - const mem_size = (16 * 2 + 32 * 2 + 64 * 2 + 128 * 2) / comptime meta.bitCount(u8); - var data_mem: [mem_size]u8 = undefined; - - var out = io.SliceOutStream.init(data_mem[0..]); - const OutError = io.SliceOutStream.Error; - var out_stream = &out.stream; - var serializer = io.Serializer(endian, packing, OutError).init(out_stream); - - var in = io.SliceInStream.init(data_mem[0..]); - const InError = io.SliceInStream.Error; - var in_stream = &in.stream; - var deserializer = io.Deserializer(endian, packing, InError).init(in_stream); - - //@TODO: isInf/isNan not currently implemented for f128. - try serializer.serialize(std.math.nan(f16)); - try serializer.serialize(std.math.inf(f16)); - try serializer.serialize(std.math.nan(f32)); - try serializer.serialize(std.math.inf(f32)); - try serializer.serialize(std.math.nan(f64)); - try serializer.serialize(std.math.inf(f64)); - //try serializer.serialize(std.math.nan(f128)); - //try serializer.serialize(std.math.inf(f128)); - const nan_check_f16 = try deserializer.deserialize(f16); - const inf_check_f16 = try deserializer.deserialize(f16); - const nan_check_f32 = try deserializer.deserialize(f32); - deserializer.alignToByte(); - const inf_check_f32 = try deserializer.deserialize(f32); - const nan_check_f64 = try deserializer.deserialize(f64); - const inf_check_f64 = try deserializer.deserialize(f64); - //const nan_check_f128 = try deserializer.deserialize(f128); - //const inf_check_f128 = try deserializer.deserialize(f128); - expect(std.math.isNan(nan_check_f16)); - expect(std.math.isInf(inf_check_f16)); - expect(std.math.isNan(nan_check_f32)); - expect(std.math.isInf(inf_check_f32)); - expect(std.math.isNan(nan_check_f64)); - expect(std.math.isInf(inf_check_f64)); - //expect(std.math.isNan(nan_check_f128)); - //expect(std.math.isInf(inf_check_f128)); -} - -test "Serializer/Deserializer Int: Inf/NaN" { - try testIntSerializerDeserializerInfNaN(.Big, .Byte); - try testIntSerializerDeserializerInfNaN(.Little, .Byte); - try testIntSerializerDeserializerInfNaN(.Big, .Bit); - try testIntSerializerDeserializerInfNaN(.Little, .Bit); -} - -fn testAlternateSerializer(self: var, serializer: var) !void { - try serializer.serialize(self.f_f16); -} - -fn testSerializerDeserializer(comptime endian: builtin.Endian, comptime packing: io.Packing) !void { - const ColorType = enum(u4) { - RGB8 = 1, - RA16 = 2, - R32 = 3, - }; - - const TagAlign = union(enum(u32)) { - A: u8, - B: u8, - C: u8, - }; - - const Color = union(ColorType) { - RGB8: struct { - r: u8, - g: u8, - b: u8, - a: u8, - }, - RA16: struct { - r: u16, - a: u16, - }, - R32: u32, - }; - - const PackedStruct = packed struct { - f_i3: i3, - f_u2: u2, - }; - - //to test custom serialization - const Custom = struct { - f_f16: f16, - f_unused_u32: u32, - - pub fn deserialize(self: *@This(), deserializer: var) !void { - try deserializer.deserializeInto(&self.f_f16); - self.f_unused_u32 = 47; - } - - pub const serialize = testAlternateSerializer; - }; - - const MyStruct = struct { - f_i3: i3, - f_u8: u8, - f_tag_align: TagAlign, - f_u24: u24, - f_i19: i19, - f_void: void, - f_f32: f32, - f_f128: f128, - f_packed_0: PackedStruct, - f_i7arr: [10]i7, - f_of64n: ?f64, - f_of64v: ?f64, - f_color_type: ColorType, - f_packed_1: PackedStruct, - f_custom: Custom, - f_color: Color, - }; - - const my_inst = MyStruct{ - .f_i3 = -1, - .f_u8 = 8, - .f_tag_align = TagAlign{ .B = 148 }, - .f_u24 = 24, - .f_i19 = 19, - .f_void = {}, - .f_f32 = 32.32, - .f_f128 = 128.128, - .f_packed_0 = PackedStruct{ .f_i3 = -1, .f_u2 = 2 }, - .f_i7arr = [10]i7{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, - .f_of64n = null, - .f_of64v = 64.64, - .f_color_type = ColorType.R32, - .f_packed_1 = PackedStruct{ .f_i3 = 1, .f_u2 = 1 }, - .f_custom = Custom{ .f_f16 = 38.63, .f_unused_u32 = 47 }, - .f_color = Color{ .R32 = 123822 }, - }; - - var data_mem: [@sizeOf(MyStruct)]u8 = undefined; - var out = io.SliceOutStream.init(data_mem[0..]); - const OutError = io.SliceOutStream.Error; - var out_stream = &out.stream; - var serializer = io.Serializer(endian, packing, OutError).init(out_stream); - - var in = io.SliceInStream.init(data_mem[0..]); - const InError = io.SliceInStream.Error; - var in_stream = &in.stream; - var deserializer = io.Deserializer(endian, packing, InError).init(in_stream); - - try serializer.serialize(my_inst); - - const my_copy = try deserializer.deserialize(MyStruct); - expect(meta.eql(my_copy, my_inst)); -} - -test "Serializer/Deserializer generic" { - if (std.Target.current.os.tag == .windows) { - // TODO https://github.com/ziglang/zig/issues/508 - return error.SkipZigTest; - } - try testSerializerDeserializer(builtin.Endian.Big, .Byte); - try testSerializerDeserializer(builtin.Endian.Little, .Byte); - try testSerializerDeserializer(builtin.Endian.Big, .Bit); - try testSerializerDeserializer(builtin.Endian.Little, .Bit); -} - -fn testBadData(comptime endian: builtin.Endian, comptime packing: io.Packing) !void { - const E = enum(u14) { - One = 1, - Two = 2, - }; - - const A = struct { - e: E, - }; - - const C = union(E) { - One: u14, - Two: f16, - }; - - var data_mem: [4]u8 = undefined; - var out = io.SliceOutStream.init(data_mem[0..]); - const OutError = io.SliceOutStream.Error; - var out_stream = &out.stream; - var serializer = io.Serializer(endian, packing, OutError).init(out_stream); - - var in = io.SliceInStream.init(data_mem[0..]); - const InError = io.SliceInStream.Error; - var in_stream = &in.stream; - var deserializer = io.Deserializer(endian, packing, InError).init(in_stream); - - try serializer.serialize(@as(u14, 3)); - expectError(error.InvalidEnumTag, deserializer.deserialize(A)); - out.pos = 0; - try serializer.serialize(@as(u14, 3)); - try serializer.serialize(@as(u14, 88)); - expectError(error.InvalidEnumTag, deserializer.deserialize(C)); -} - -test "Deserializer bad data" { - try testBadData(.Big, .Byte); - try testBadData(.Little, .Byte); - try testBadData(.Big, .Bit); - try testBadData(.Little, .Bit); -} - -test "c out stream" { - if (!builtin.link_libc) return error.SkipZigTest; - - const filename = "tmp_io_test_file.txt"; - const out_file = std.c.fopen(filename, "w") orelse return error.UnableToOpenTestFile; - defer { - _ = std.c.fclose(out_file); - fs.cwd().deleteFileC(filename) catch {}; - } - - const out_stream = &io.COutStream.init(out_file).stream; - try out_stream.print("hi: {}\n", .{@as(i32, 123)}); -} - test "File seek ops" { const tmp_file_name = "temp_test_file.txt"; var file = try fs.cwd().createFile(tmp_file_name, .{}); @@ -621,16 +113,16 @@ test "File seek ops" { // Seek to the end try file.seekFromEnd(0); - std.testing.expect((try file.getPos()) == try file.getEndPos()); + expect((try file.getPos()) == try file.getEndPos()); // Negative delta try file.seekBy(-4096); - std.testing.expect((try file.getPos()) == 4096); + expect((try file.getPos()) == 4096); // Positive delta try file.seekBy(10); - std.testing.expect((try file.getPos()) == 4106); + expect((try file.getPos()) == 4106); // Absolute position try file.seekTo(1234); - std.testing.expect((try file.getPos()) == 1234); + expect((try file.getPos()) == 1234); } test "updateTimes" { @@ -647,6 +139,6 @@ test "updateTimes" { stat_old.mtime - 5 * std.time.ns_per_s, ); var stat_new = try file.stat(); - std.testing.expect(stat_new.atime < stat_old.atime); - std.testing.expect(stat_new.mtime < stat_old.mtime); + expect(stat_new.atime < stat_old.atime); + expect(stat_new.mtime < stat_old.mtime); } diff --git a/lib/std/json.zig b/lib/std/json.zig index f2cf68d7eb..4e2440d4e9 100644 --- a/lib/std/json.zig +++ b/lib/std/json.zig @@ -10,6 +10,7 @@ const mem = std.mem; const maxInt = std.math.maxInt; pub const WriteStream = @import("json/write_stream.zig").WriteStream; +pub const writeStream = @import("json/write_stream.zig").writeStream; const StringEscapes = union(enum) { None, @@ -2109,7 +2110,7 @@ test "write json then parse it" { var fixed_buffer_stream = std.io.fixedBufferStream(&out_buffer); const out_stream = fixed_buffer_stream.outStream(); - var jw = WriteStream(@TypeOf(out_stream).Child, 4).init(out_stream); + var jw = writeStream(out_stream, 4); try jw.beginObject(); @@ -2140,7 +2141,7 @@ test "write json then parse it" { var parser = Parser.init(testing.allocator, false); defer parser.deinit(); - var tree = try parser.parse(slice_out_stream.getWritten()); + var tree = try parser.parse(fixed_buffer_stream.getWritten()); defer tree.deinit(); testing.expect(tree.root.Object.get("f").?.value.Bool == false); diff --git a/lib/std/json/write_stream.zig b/lib/std/json/write_stream.zig index 6d88c8b6da..f4d171011c 100644 --- a/lib/std/json/write_stream.zig +++ b/lib/std/json/write_stream.zig @@ -249,15 +249,22 @@ pub fn WriteStream(comptime OutStream: type, comptime max_depth: usize) type { }; } +pub fn writeStream( + out_stream: var, + comptime max_depth: usize, +) WriteStream(@TypeOf(out_stream), max_depth) { + return WriteStream(@TypeOf(out_stream), max_depth).init(out_stream); +} + test "json write stream" { var out_buf: [1024]u8 = undefined; - var slice_stream = std.io.SliceOutStream.init(&out_buf); - const out = &slice_stream.stream; + var slice_stream = std.io.fixedBufferStream(&out_buf); + const out = slice_stream.outStream(); var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena_allocator.deinit(); - var w = std.json.WriteStream(@TypeOf(out).Child, 10).init(out); + var w = std.json.writeStream(out, 10); try w.emitJson(try getJson(&arena_allocator.allocator)); const result = slice_stream.getWritten(); diff --git a/lib/std/net.zig b/lib/std/net.zig index 6d0daefdc0..de10a17640 100644 --- a/lib/std/net.zig +++ b/lib/std/net.zig @@ -816,7 +816,7 @@ fn linuxLookupNameFromHosts( }; defer file.close(); - const stream = &std.io.BufferedInStream(fs.File.ReadError).init(&file.inStream().stream).stream; + const stream = std.io.bufferedInStream(file.inStream()).inStream(); var line_buf: [512]u8 = undefined; while (stream.readUntilDelimiterOrEof(&line_buf, '\n') catch |err| switch (err) { error.StreamTooLong => blk: { @@ -1010,7 +1010,7 @@ fn getResolvConf(allocator: *mem.Allocator, rc: *ResolvConf) !void { }; defer file.close(); - const stream = &std.io.BufferedInStream(fs.File.ReadError).init(&file.inStream().stream).stream; + const stream = std.io.bufferedInStream(file.inStream()).inStream(); var line_buf: [512]u8 = undefined; while (stream.readUntilDelimiterOrEof(&line_buf, '\n') catch |err| switch (err) { error.StreamTooLong => blk: { diff --git a/lib/std/os/test.zig b/lib/std/os/test.zig index 5f97597537..75a6d5db91 100644 --- a/lib/std/os/test.zig +++ b/lib/std/os/test.zig @@ -354,8 +354,7 @@ test "mmap" { const file = try fs.cwd().createFile(test_out_file, .{}); defer file.close(); - var out_stream = file.outStream(); - const stream = &out_stream.stream; + const stream = file.outStream(); var i: u32 = 0; while (i < alloc_size / @sizeOf(u32)) : (i += 1) { @@ -378,8 +377,8 @@ test "mmap" { ); defer os.munmap(data); - var mem_stream = io.SliceInStream.init(data); - const stream = &mem_stream.stream; + var mem_stream = io.fixedBufferStream(data); + const stream = mem_stream.inStream(); var i: u32 = 0; while (i < alloc_size / @sizeOf(u32)) : (i += 1) { @@ -402,8 +401,8 @@ test "mmap" { ); defer os.munmap(data); - var mem_stream = io.SliceInStream.init(data); - const stream = &mem_stream.stream; + var mem_stream = io.fixedBufferStream(data); + const stream = mem_stream.inStream(); var i: u32 = alloc_size / 2 / @sizeOf(u32); while (i < alloc_size / @sizeOf(u32)) : (i += 1) { diff --git a/lib/std/zig/parser_test.zig b/lib/std/zig/parser_test.zig index ddea0fc57c..d00568e49f 100644 --- a/lib/std/zig/parser_test.zig +++ b/lib/std/zig/parser_test.zig @@ -2809,7 +2809,7 @@ const maxInt = std.math.maxInt; var fixed_buffer_mem: [100 * 1024]u8 = undefined; fn testParse(source: []const u8, allocator: *mem.Allocator, anything_changed: *bool) ![]u8 { - const stderr = &io.getStdErr().outStream().stream; + const stderr = io.getStdErr().outStream(); const tree = try std.zig.parse(allocator, source); defer tree.deinit(); @@ -2824,17 +2824,17 @@ fn testParse(source: []const u8, allocator: *mem.Allocator, anything_changed: *b { var i: usize = 0; while (i < loc.column) : (i += 1) { - try stderr.write(" "); + try stderr.writeAll(" "); } } { const caret_count = token.end - token.start; var i: usize = 0; while (i < caret_count) : (i += 1) { - try stderr.write("~"); + try stderr.writeAll("~"); } } - try stderr.write("\n"); + try stderr.writeAll("\n"); } if (tree.errors.len != 0) { return error.ParseError; @@ -2843,8 +2843,7 @@ fn testParse(source: []const u8, allocator: *mem.Allocator, anything_changed: *b var buffer = try std.Buffer.initSize(allocator, 0); errdefer buffer.deinit(); - var buffer_out_stream = io.BufferOutStream.init(&buffer); - anything_changed.* = try std.zig.render(allocator, &buffer_out_stream.stream, tree); + anything_changed.* = try std.zig.render(allocator, buffer.outStream(), tree); return buffer.toOwnedSlice(); } diff --git a/lib/std/zig/render.zig b/lib/std/zig/render.zig index 9600de21df..1a221bd5b3 100644 --- a/lib/std/zig/render.zig +++ b/lib/std/zig/render.zig @@ -903,7 +903,7 @@ fn renderExpression( var column_widths = widths[widths.len - row_size ..]; // Null stream for counting the printed length of each expression - var counting_stream = std.io.CountingOutStream(@TypeOf(std.io.null_out_stream)).init(std.io.null_out_stream); + var counting_stream = std.io.countingOutStream(std.io.null_out_stream); var it = exprs.iterator(0); var i: usize = 0; From cd26d3b0bb2b32f1daf1d3d46bb53731dd54afec Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 10 Mar 2020 18:54:24 -0400 Subject: [PATCH 04/15] fix regressions caused earlier in this branch --- lib/std/debug.zig | 11 +++++++++-- lib/std/io/fixed_buffer_stream.zig | 3 +-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/lib/std/debug.zig b/lib/std/debug.zig index 5c56490519..b27b1caa68 100644 --- a/lib/std/debug.zig +++ b/lib/std/debug.zig @@ -825,8 +825,15 @@ pub fn openElfDebugInfo(allocator: *mem.Allocator, elf_file_path: []const u8) !M const shoff = hdr.e_shoff; const str_section_off = shoff + @as(u64, hdr.e_shentsize) * @as(u64, hdr.e_shstrndx); - const header_strings = mapped_mem[str_section_off..str_section_off + hdr.e_shentsize]; - const shdrs = @ptrCast([*]const elf.Shdr, @alignCast(@alignOf(elf.Shdr), &mapped_mem[shoff]))[0..hdr.e_shnum]; + const str_shdr = @ptrCast( + *const elf.Shdr, + @alignCast(@alignOf(elf.Shdr), &mapped_mem[str_section_off]), + ); + const header_strings = mapped_mem[str_shdr.sh_offset .. str_shdr.sh_offset + str_shdr.sh_size]; + const shdrs = @ptrCast( + [*]const elf.Shdr, + @alignCast(@alignOf(elf.Shdr), &mapped_mem[shoff]), + )[0..hdr.e_shnum]; var opt_debug_info: ?[]const u8 = null; var opt_debug_abbrev: ?[]const u8 = null; diff --git a/lib/std/io/fixed_buffer_stream.zig b/lib/std/io/fixed_buffer_stream.zig index 9cdad7dc57..0f2b542201 100644 --- a/lib/std/io/fixed_buffer_stream.zig +++ b/lib/std/io/fixed_buffer_stream.zig @@ -164,6 +164,5 @@ test "FixedBufferStream input" { testing.expect(read == 3); testing.expect(mem.eql(u8, dest[0..3], bytes[4..7])); - read = try fbs.inStream().read(dest[0..4]); - testing.expect(read == 0); + testing.expectError(error.EndOfStream, fbs.inStream().read(dest[0..4])); } From 9abee660dce3b43b9ac3b8260fbf269532d6c7f5 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 10 Mar 2020 19:28:05 -0400 Subject: [PATCH 05/15] fix stack trace code not opening files in forced blocking mode --- lib/std/debug.zig | 4 +++- lib/std/event/group.zig | 4 +++- lib/std/event/lock.zig | 3 +++ lib/std/net/test.zig | 2 +- 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/std/debug.zig b/lib/std/debug.zig index b27b1caa68..e849d72bcf 100644 --- a/lib/std/debug.zig +++ b/lib/std/debug.zig @@ -964,7 +964,9 @@ fn openMachODebugInfo(allocator: *mem.Allocator, macho_file_path: []const u8) !M } fn printLineFromFileAnyOs(out_stream: var, line_info: LineInfo) !void { - var f = try fs.cwd().openFile(line_info.file_name, .{}); + // Need this to always block even in async I/O mode, because this could potentially + // be called from e.g. the event loop code crashing. + var f = try fs.cwd().openFile(line_info.file_name, .{ .always_blocking = true }); defer f.close(); // TODO fstat and make sure that the file has the correct size diff --git a/lib/std/event/group.zig b/lib/std/event/group.zig index ac1bf68245..5eebb7ffbc 100644 --- a/lib/std/event/group.zig +++ b/lib/std/event/group.zig @@ -120,9 +120,11 @@ test "std.event.Group" { // https://github.com/ziglang/zig/issues/1908 if (builtin.single_threaded) return error.SkipZigTest; - // TODO provide a way to run tests in evented I/O mode if (!std.io.is_async) return error.SkipZigTest; + // TODO this file has bit-rotted. repair it + if (true) return error.SkipZigTest; + const handle = async testGroup(std.heap.page_allocator); } diff --git a/lib/std/event/lock.zig b/lib/std/event/lock.zig index b9cbb5d95f..1bb51261d7 100644 --- a/lib/std/event/lock.zig +++ b/lib/std/event/lock.zig @@ -125,6 +125,9 @@ test "std.event.Lock" { // TODO https://github.com/ziglang/zig/issues/3251 if (builtin.os.tag == .freebsd) return error.SkipZigTest; + // TODO this file has bit-rotted. repair it + if (true) return error.SkipZigTest; + var lock = Lock.init(); defer lock.deinit(); diff --git a/lib/std/net/test.zig b/lib/std/net/test.zig index 4f3d955f30..087f965c4e 100644 --- a/lib/std/net/test.zig +++ b/lib/std/net/test.zig @@ -113,6 +113,6 @@ fn testClient(addr: net.Address) anyerror!void { fn testServer(server: *net.StreamServer) anyerror!void { var client = try server.accept(); - const stream = &client.file.outStream().stream; + const stream = client.file.outStream(); try stream.print("hello from server\n", .{}); } From 2bff0dda795cd84e4da59652736b7cfa7e1c964c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 10 Mar 2020 20:22:30 -0400 Subject: [PATCH 06/15] fix regressions found by test suite --- doc/docgen.zig | 95 +++++++++++++-------------- doc/langref.html.in | 2 +- lib/std/build.zig | 3 +- lib/std/build/run.zig | 6 +- lib/std/coff.zig | 15 ++--- lib/std/debug.zig | 38 +++++------ lib/std/pdb.zig | 14 ++-- lib/std/special/build_runner.zig | 8 +-- test/compare_output.zig | 34 +++++----- test/standalone/guess_number/main.zig | 2 +- test/tests.zig | 14 ++-- 11 files changed, 103 insertions(+), 128 deletions(-) diff --git a/doc/docgen.zig b/doc/docgen.zig index 319d9e0035..4d2625f54f 100644 --- a/doc/docgen.zig +++ b/doc/docgen.zig @@ -40,12 +40,9 @@ pub fn main() !void { var out_file = try fs.cwd().createFile(out_file_name, .{}); defer out_file.close(); - var file_in_stream = in_file.inStream(); + const input_file_bytes = try in_file.inStream().readAllAlloc(allocator, max_doc_file_size); - const input_file_bytes = try file_in_stream.stream.readAllAlloc(allocator, max_doc_file_size); - - var file_out_stream = out_file.outStream(); - var buffered_out_stream = io.BufferedOutStream(fs.File.WriteError).init(&file_out_stream.stream); + var buffered_out_stream = io.bufferedOutStream(out_file.outStream()); var tokenizer = Tokenizer.init(in_file_name, input_file_bytes); var toc = try genToc(allocator, &tokenizer); @@ -53,7 +50,7 @@ pub fn main() !void { try fs.cwd().makePath(tmp_dir_name); defer fs.deleteTree(tmp_dir_name) catch {}; - try genHtml(allocator, &tokenizer, &toc, &buffered_out_stream.stream, zig_exe); + try genHtml(allocator, &tokenizer, &toc, buffered_out_stream.outStream(), zig_exe); try buffered_out_stream.flush(); } @@ -327,8 +324,7 @@ fn genToc(allocator: *mem.Allocator, tokenizer: *Tokenizer) !Toc { var toc_buf = try std.Buffer.initSize(allocator, 0); defer toc_buf.deinit(); - var toc_buf_adapter = io.BufferOutStream.init(&toc_buf); - var toc = &toc_buf_adapter.stream; + var toc = toc_buf.outStream(); var nodes = std.ArrayList(Node).init(allocator); defer nodes.deinit(); @@ -342,7 +338,7 @@ fn genToc(allocator: *mem.Allocator, tokenizer: *Tokenizer) !Toc { if (header_stack_size != 0) { return parseError(tokenizer, token, "unbalanced headers", .{}); } - try toc.write(" \n"); + try toc.writeAll(" \n"); break; }, Token.Id.Content => { @@ -407,7 +403,7 @@ fn genToc(allocator: *mem.Allocator, tokenizer: *Tokenizer) !Toc { if (last_columns) |n| { try toc.print("
    \n", .{n}); } else { - try toc.write("
      \n"); + try toc.writeAll("
        \n"); } } else { last_action = Action.Open; @@ -424,9 +420,9 @@ fn genToc(allocator: *mem.Allocator, tokenizer: *Tokenizer) !Toc { if (last_action == Action.Close) { try toc.writeByteNTimes(' ', 8 + header_stack_size * 4); - try toc.write("
      \n"); + try toc.writeAll("
    \n"); } else { - try toc.write("\n"); + try toc.writeAll("\n"); last_action = Action.Close; } } else if (mem.eql(u8, tag_name, "see_also")) { @@ -614,8 +610,7 @@ fn urlize(allocator: *mem.Allocator, input: []const u8) ![]u8 { var buf = try std.Buffer.initSize(allocator, 0); defer buf.deinit(); - var buf_adapter = io.BufferOutStream.init(&buf); - var out = &buf_adapter.stream; + const out = buf.outStream(); for (input) |c| { switch (c) { 'a'...'z', 'A'...'Z', '_', '-', '0'...'9' => { @@ -634,8 +629,7 @@ fn escapeHtml(allocator: *mem.Allocator, input: []const u8) ![]u8 { var buf = try std.Buffer.initSize(allocator, 0); defer buf.deinit(); - var buf_adapter = io.BufferOutStream.init(&buf); - var out = &buf_adapter.stream; + const out = buf.outStream(); try writeEscaped(out, input); return buf.toOwnedSlice(); } @@ -643,10 +637,10 @@ fn escapeHtml(allocator: *mem.Allocator, input: []const u8) ![]u8 { fn writeEscaped(out: var, input: []const u8) !void { for (input) |c| { try switch (c) { - '&' => out.write("&"), - '<' => out.write("<"), - '>' => out.write(">"), - '"' => out.write("""), + '&' => out.writeAll("&"), + '<' => out.writeAll("<"), + '>' => out.writeAll(">"), + '"' => out.writeAll("""), else => out.writeByte(c), }; } @@ -681,8 +675,7 @@ fn termColor(allocator: *mem.Allocator, input: []const u8) ![]u8 { var buf = try std.Buffer.initSize(allocator, 0); defer buf.deinit(); - var buf_adapter = io.BufferOutStream.init(&buf); - var out = &buf_adapter.stream; + var out = buf.outStream(); var number_start_index: usize = undefined; var first_number: usize = undefined; var second_number: usize = undefined; @@ -743,7 +736,7 @@ fn termColor(allocator: *mem.Allocator, input: []const u8) ![]u8 { 'm' => { state = TermState.Start; while (open_span_count != 0) : (open_span_count -= 1) { - try out.write(""); + try out.writeAll(""); } if (first_number != 0 or second_number != 0) { try out.print("", .{ first_number, second_number }); @@ -774,7 +767,7 @@ fn isType(name: []const u8) bool { fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Token, raw_src: []const u8) !void { const src = mem.trim(u8, raw_src, " \n"); - try out.write(""); + try out.writeAll(""); var tokenizer = std.zig.Tokenizer.init(src); var index: usize = 0; var next_tok_is_fn = false; @@ -835,15 +828,15 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Tok .Keyword_allowzero, .Keyword_while, => { - try out.write(""); + try out.writeAll(""); try writeEscaped(out, src[token.start..token.end]); - try out.write(""); + try out.writeAll(""); }, .Keyword_fn => { - try out.write(""); + try out.writeAll(""); try writeEscaped(out, src[token.start..token.end]); - try out.write(""); + try out.writeAll(""); next_tok_is_fn = true; }, @@ -852,24 +845,24 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Tok .Keyword_true, .Keyword_false, => { - try out.write(""); + try out.writeAll(""); try writeEscaped(out, src[token.start..token.end]); - try out.write(""); + try out.writeAll(""); }, .StringLiteral, .MultilineStringLiteralLine, .CharLiteral, => { - try out.write(""); + try out.writeAll(""); try writeEscaped(out, src[token.start..token.end]); - try out.write(""); + try out.writeAll(""); }, .Builtin => { - try out.write(""); + try out.writeAll(""); try writeEscaped(out, src[token.start..token.end]); - try out.write(""); + try out.writeAll(""); }, .LineComment, @@ -877,16 +870,16 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Tok .ContainerDocComment, .ShebangLine, => { - try out.write(""); + try out.writeAll(""); try writeEscaped(out, src[token.start..token.end]); - try out.write(""); + try out.writeAll(""); }, .Identifier => { if (prev_tok_was_fn) { - try out.write(""); + try out.writeAll(""); try writeEscaped(out, src[token.start..token.end]); - try out.write(""); + try out.writeAll(""); } else { const is_int = blk: { if (src[token.start] != 'i' and src[token.start] != 'u') @@ -901,9 +894,9 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Tok break :blk true; }; if (is_int or isType(src[token.start..token.end])) { - try out.write(""); + try out.writeAll(""); try writeEscaped(out, src[token.start..token.end]); - try out.write(""); + try out.writeAll(""); } else { try writeEscaped(out, src[token.start..token.end]); } @@ -913,9 +906,9 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Tok .IntegerLiteral, .FloatLiteral, => { - try out.write(""); + try out.writeAll(""); try writeEscaped(out, src[token.start..token.end]); - try out.write(""); + try out.writeAll(""); }, .Bang, @@ -983,7 +976,7 @@ fn tokenizeAndPrintRaw(docgen_tokenizer: *Tokenizer, out: var, source_token: Tok } index = token.end; } - try out.write(""); + try out.writeAll(""); } fn tokenizeAndPrint(docgen_tokenizer: *Tokenizer, out: var, source_token: Token) !void { @@ -1002,7 +995,7 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: var for (toc.nodes) |node| { switch (node) { .Content => |data| { - try out.write(data); + try out.writeAll(data); }, .Link => |info| { if (!toc.urls.contains(info.url)) { @@ -1011,12 +1004,12 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: var try out.print("{}", .{ info.url, info.name }); }, .Nav => { - try out.write(toc.toc); + try out.writeAll(toc.toc); }, .Builtin => |tok| { - try out.write("
    ");
+                try out.writeAll("
    ");
                 try tokenizeAndPrintRaw(tokenizer, out, tok, builtin_code);
-                try out.write("
    ");
+                try out.writeAll("
    "); }, .HeaderOpen => |info| { try out.print( @@ -1025,7 +1018,7 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: var ); }, .SeeAlso => |items| { - try out.write("

    See also:

      \n"); + try out.writeAll("

      See also:

        \n"); for (items) |item| { const url = try urlize(allocator, item.name); if (!toc.urls.contains(url)) { @@ -1033,7 +1026,7 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: var } try out.print("
      • {}
        • \n", .{ url, item.name }); } - try out.write("
      \n"); + try out.writeAll("
    \n"); }, .Syntax => |content_tok| { try tokenizeAndPrint(tokenizer, out, content_tok); @@ -1047,9 +1040,9 @@ fn genHtml(allocator: *mem.Allocator, tokenizer: *Tokenizer, toc: *Toc, out: var if (!code.is_inline) { try out.print("

    {}.zig

    ", .{code.name}); } - try out.write("
    ");
+                try out.writeAll("
    ");
                 try tokenizeAndPrint(tokenizer, out, code.source_token);
-                try out.write("
    ");
+                try out.writeAll("
    "); const name_plus_ext = try std.fmt.allocPrint(allocator, "{}.zig", .{code.name}); const tmp_source_file_name = try fs.path.join( allocator, diff --git a/doc/langref.html.in b/doc/langref.html.in index 447d545975..baba472e88 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -230,7 +230,7 @@ const std = @import("std"); pub fn main() !void { - const stdout = &std.io.getStdOut().outStream().stream; + const stdout = std.io.getStdOut().outStream(); try stdout.print("Hello, {}!\n", .{"world"}); } {#code_end#} diff --git a/lib/std/build.zig b/lib/std/build.zig index bc8b8acba0..e8484e9d1c 100644 --- a/lib/std/build.zig +++ b/lib/std/build.zig @@ -926,8 +926,7 @@ pub const Builder = struct { try child.spawn(); - var stdout_file_in_stream = child.stdout.?.inStream(); - const stdout = try stdout_file_in_stream.stream.readAllAlloc(self.allocator, max_output_size); + const stdout = try child.stdout.?.inStream().readAllAlloc(self.allocator, max_output_size); errdefer self.allocator.free(stdout); const term = try child.wait(); diff --git a/lib/std/build/run.zig b/lib/std/build/run.zig index 41417572ca..91ecd56c3a 100644 --- a/lib/std/build/run.zig +++ b/lib/std/build/run.zig @@ -175,8 +175,7 @@ pub const RunStep = struct { switch (self.stdout_action) { .expect_exact, .expect_matches => { - var stdout_file_in_stream = child.stdout.?.inStream(); - stdout = stdout_file_in_stream.stream.readAllAlloc(self.builder.allocator, max_stdout_size) catch unreachable; + stdout = child.stdout.?.inStream().readAllAlloc(self.builder.allocator, max_stdout_size) catch unreachable; }, .inherit, .ignore => {}, } @@ -186,8 +185,7 @@ pub const RunStep = struct { switch (self.stderr_action) { .expect_exact, .expect_matches => { - var stderr_file_in_stream = child.stderr.?.inStream(); - stderr = stderr_file_in_stream.stream.readAllAlloc(self.builder.allocator, max_stdout_size) catch unreachable; + stderr = child.stderr.?.inStream().readAllAlloc(self.builder.allocator, max_stdout_size) catch unreachable; }, .inherit, .ignore => {}, } diff --git a/lib/std/coff.zig b/lib/std/coff.zig index b1be21c4d3..2e17f46454 100644 --- a/lib/std/coff.zig +++ b/lib/std/coff.zig @@ -56,8 +56,7 @@ pub const Coff = struct { pub fn loadHeader(self: *Coff) !void { const pe_pointer_offset = 0x3C; - var file_stream = self.in_file.inStream(); - const in = &file_stream.stream; + const in = self.in_file.inStream(); var magic: [2]u8 = undefined; try in.readNoEof(magic[0..]); @@ -89,11 +88,11 @@ pub const Coff = struct { else => return error.InvalidMachine, } - try self.loadOptionalHeader(&file_stream); + try self.loadOptionalHeader(); } - fn loadOptionalHeader(self: *Coff, file_stream: *File.InStream) !void { - const in = &file_stream.stream; + fn loadOptionalHeader(self: *Coff) !void { + const in = self.in_file.inStream(); self.pe_header.magic = try in.readIntLittle(u16); // For now we're only interested in finding the reference to the .pdb, // so we'll skip most of this header, which size is different in 32 @@ -136,8 +135,7 @@ pub const Coff = struct { const debug_dir = &self.pe_header.data_directory[DEBUG_DIRECTORY]; const file_offset = debug_dir.virtual_address - header.virtual_address + header.pointer_to_raw_data; - var file_stream = self.in_file.inStream(); - const in = &file_stream.stream; + const in = self.in_file.inStream(); try self.in_file.seekTo(file_offset); // Find the correct DebugDirectoryEntry, and where its data is stored. @@ -188,8 +186,7 @@ pub const Coff = struct { try self.sections.ensureCapacity(self.coff_header.number_of_sections); - var file_stream = self.in_file.inStream(); - const in = &file_stream.stream; + const in = self.in_file.inStream(); var name: [8]u8 = undefined; diff --git a/lib/std/debug.zig b/lib/std/debug.zig index e849d72bcf..0a7a0dee7e 100644 --- a/lib/std/debug.zig +++ b/lib/std/debug.zig @@ -475,15 +475,15 @@ fn populateModule(di: *ModuleDebugInfo, mod: *Module) !void { const modi = di.pdb.getStreamById(mod.mod_info.ModuleSymStream) orelse return error.MissingDebugInfo; - const signature = try modi.stream.readIntLittle(u32); + const signature = try modi.inStream().readIntLittle(u32); if (signature != 4) return error.InvalidDebugInfo; mod.symbols = try allocator.alloc(u8, mod.mod_info.SymByteSize - 4); - try modi.stream.readNoEof(mod.symbols); + try modi.inStream().readNoEof(mod.symbols); mod.subsect_info = try allocator.alloc(u8, mod.mod_info.C13ByteSize); - try modi.stream.readNoEof(mod.subsect_info); + try modi.inStream().readNoEof(mod.subsect_info); var sect_offset: usize = 0; var skip_len: usize = undefined; @@ -656,11 +656,11 @@ fn openCoffDebugInfo(allocator: *mem.Allocator, coff_file_path: [:0]const u16) ! try di.pdb.openFile(di.coff, path); var pdb_stream = di.pdb.getStream(pdb.StreamType.Pdb) orelse return error.InvalidDebugInfo; - const version = try pdb_stream.stream.readIntLittle(u32); - const signature = try pdb_stream.stream.readIntLittle(u32); - const age = try pdb_stream.stream.readIntLittle(u32); + const version = try pdb_stream.inStream().readIntLittle(u32); + const signature = try pdb_stream.inStream().readIntLittle(u32); + const age = try pdb_stream.inStream().readIntLittle(u32); var guid: [16]u8 = undefined; - try pdb_stream.stream.readNoEof(&guid); + try pdb_stream.inStream().readNoEof(&guid); if (version != 20000404) // VC70, only value observed by LLVM team return error.UnknownPDBVersion; if (!mem.eql(u8, &di.coff.guid, &guid) or di.coff.age != age) @@ -668,9 +668,9 @@ fn openCoffDebugInfo(allocator: *mem.Allocator, coff_file_path: [:0]const u16) ! // We validated the executable and pdb match. const string_table_index = str_tab_index: { - const name_bytes_len = try pdb_stream.stream.readIntLittle(u32); + const name_bytes_len = try pdb_stream.inStream().readIntLittle(u32); const name_bytes = try allocator.alloc(u8, name_bytes_len); - try pdb_stream.stream.readNoEof(name_bytes); + try pdb_stream.inStream().readNoEof(name_bytes); const HashTableHeader = packed struct { Size: u32, @@ -680,17 +680,17 @@ fn openCoffDebugInfo(allocator: *mem.Allocator, coff_file_path: [:0]const u16) ! return cap * 2 / 3 + 1; } }; - const hash_tbl_hdr = try pdb_stream.stream.readStruct(HashTableHeader); + const hash_tbl_hdr = try pdb_stream.inStream().readStruct(HashTableHeader); if (hash_tbl_hdr.Capacity == 0) return error.InvalidDebugInfo; if (hash_tbl_hdr.Size > HashTableHeader.maxLoad(hash_tbl_hdr.Capacity)) return error.InvalidDebugInfo; - const present = try readSparseBitVector(&pdb_stream.stream, allocator); + const present = try readSparseBitVector(&pdb_stream.inStream(), allocator); if (present.len != hash_tbl_hdr.Size) return error.InvalidDebugInfo; - const deleted = try readSparseBitVector(&pdb_stream.stream, allocator); + const deleted = try readSparseBitVector(&pdb_stream.inStream(), allocator); const Bucket = struct { first: u32, @@ -698,8 +698,8 @@ fn openCoffDebugInfo(allocator: *mem.Allocator, coff_file_path: [:0]const u16) ! }; const bucket_list = try allocator.alloc(Bucket, present.len); for (present) |_| { - const name_offset = try pdb_stream.stream.readIntLittle(u32); - const name_index = try pdb_stream.stream.readIntLittle(u32); + const name_offset = try pdb_stream.inStream().readIntLittle(u32); + const name_index = try pdb_stream.inStream().readIntLittle(u32); const name = mem.toSlice(u8, @ptrCast([*:0]u8, name_bytes.ptr + name_offset)); if (mem.eql(u8, name, "/names")) { break :str_tab_index name_index; @@ -714,7 +714,7 @@ fn openCoffDebugInfo(allocator: *mem.Allocator, coff_file_path: [:0]const u16) ! const dbi = di.pdb.dbi; // Dbi Header - const dbi_stream_header = try dbi.stream.readStruct(pdb.DbiStreamHeader); + const dbi_stream_header = try dbi.inStream().readStruct(pdb.DbiStreamHeader); if (dbi_stream_header.VersionHeader != 19990903) // V70, only value observed by LLVM team return error.UnknownPDBVersion; if (dbi_stream_header.Age != age) @@ -728,7 +728,7 @@ fn openCoffDebugInfo(allocator: *mem.Allocator, coff_file_path: [:0]const u16) ! // Module Info Substream var mod_info_offset: usize = 0; while (mod_info_offset != mod_info_size) { - const mod_info = try dbi.stream.readStruct(pdb.ModInfo); + const mod_info = try dbi.inStream().readStruct(pdb.ModInfo); var this_record_len: usize = @sizeOf(pdb.ModInfo); const module_name = try dbi.readNullTermString(allocator); @@ -766,14 +766,14 @@ fn openCoffDebugInfo(allocator: *mem.Allocator, coff_file_path: [:0]const u16) ! var sect_contribs = ArrayList(pdb.SectionContribEntry).init(allocator); var sect_cont_offset: usize = 0; if (section_contrib_size != 0) { - const ver = @intToEnum(pdb.SectionContrSubstreamVersion, try dbi.stream.readIntLittle(u32)); + const ver = @intToEnum(pdb.SectionContrSubstreamVersion, try dbi.inStream().readIntLittle(u32)); if (ver != pdb.SectionContrSubstreamVersion.Ver60) return error.InvalidDebugInfo; sect_cont_offset += @sizeOf(u32); } while (sect_cont_offset != section_contrib_size) { const entry = try sect_contribs.addOne(); - entry.* = try dbi.stream.readStruct(pdb.SectionContribEntry); + entry.* = try dbi.inStream().readStruct(pdb.SectionContribEntry); sect_cont_offset += @sizeOf(pdb.SectionContribEntry); if (sect_cont_offset > section_contrib_size) @@ -827,7 +827,7 @@ pub fn openElfDebugInfo(allocator: *mem.Allocator, elf_file_path: []const u8) !M const str_section_off = shoff + @as(u64, hdr.e_shentsize) * @as(u64, hdr.e_shstrndx); const str_shdr = @ptrCast( *const elf.Shdr, - @alignCast(@alignOf(elf.Shdr), &mapped_mem[str_section_off]), + @alignCast(@alignOf(elf.Shdr), &mapped_mem[try math.cast(usize, str_section_off)]), ); const header_strings = mapped_mem[str_shdr.sh_offset .. str_shdr.sh_offset + str_shdr.sh_size]; const shdrs = @ptrCast( diff --git a/lib/std/pdb.zig b/lib/std/pdb.zig index 7c48cf7b7c..93fef0b16d 100644 --- a/lib/std/pdb.zig +++ b/lib/std/pdb.zig @@ -495,8 +495,7 @@ const Msf = struct { streams: []MsfStream, fn openFile(self: *Msf, allocator: *mem.Allocator, file: File) !void { - var file_stream = file.inStream(); - const in = &file_stream.stream; + const in = file.inStream(); const superblock = try in.readStruct(SuperBlock); @@ -529,7 +528,7 @@ const Msf = struct { ); const begin = self.directory.pos; - const stream_count = try self.directory.stream.readIntLittle(u32); + const stream_count = try self.directory.inStream().readIntLittle(u32); const stream_sizes = try allocator.alloc(u32, stream_count); defer allocator.free(stream_sizes); @@ -538,7 +537,7 @@ const Msf = struct { // and must be taken into account when resolving stream indices. const Nil = 0xFFFFFFFF; for (stream_sizes) |*s, i| { - const size = try self.directory.stream.readIntLittle(u32); + const size = try self.directory.inStream().readIntLittle(u32); s.* = if (size == Nil) 0 else blockCountFromSize(size, superblock.BlockSize); } @@ -553,7 +552,7 @@ const Msf = struct { var blocks = try allocator.alloc(u32, size); var j: u32 = 0; while (j < size) : (j += 1) { - const block_id = try self.directory.stream.readIntLittle(u32); + const block_id = try self.directory.inStream().readIntLittle(u32); const n = (block_id % superblock.BlockSize); // 0 is for SuperBlock, 1 and 2 for FPMs. if (block_id == 0 or n == 1 or n == 2 or block_id * superblock.BlockSize > try file.getEndPos()) @@ -648,7 +647,7 @@ const MsfStream = struct { fn readNullTermString(self: *MsfStream, allocator: *mem.Allocator) ![]u8 { var list = ArrayList(u8).init(allocator); while (true) { - const byte = try self.stream.readByte(); + const byte = try self.inStream().readByte(); if (byte == 0) { return list.toSlice(); } @@ -662,8 +661,7 @@ const MsfStream = struct { var offset = self.pos % self.block_size; try self.in_file.seekTo(block * self.block_size + offset); - var file_stream = self.in_file.inStream(); - const in = &file_stream.stream; + const in = self.in_file.inStream(); var size: usize = 0; var rem_buffer = buffer; diff --git a/lib/std/special/build_runner.zig b/lib/std/special/build_runner.zig index baf21570ec..974247e2a1 100644 --- a/lib/std/special/build_runner.zig +++ b/lib/std/special/build_runner.zig @@ -42,8 +42,8 @@ pub fn main() !void { var targets = ArrayList([]const u8).init(allocator); - const stderr_stream = &io.getStdErr().outStream().stream; - const stdout_stream = &io.getStdOut().outStream().stream; + const stderr_stream = io.getStdErr().outStream(); + const stdout_stream = io.getStdOut().outStream(); while (nextArg(args, &arg_idx)) |arg| { if (mem.startsWith(u8, arg, "-D")) { @@ -159,7 +159,7 @@ fn usage(builder: *Builder, already_ran_build: bool, out_stream: var) !void { try out_stream.print(" {s:22} {}\n", .{ name, top_level_step.description }); } - try out_stream.write( + try out_stream.writeAll( \\ \\General Options: \\ --help Print this help and exit @@ -184,7 +184,7 @@ fn usage(builder: *Builder, already_ran_build: bool, out_stream: var) !void { } } - try out_stream.write( + try out_stream.writeAll( \\ \\Advanced Options: \\ --build-file [file] Override path to build.zig diff --git a/test/compare_output.zig b/test/compare_output.zig index ec89af35f8..1a0179c4c2 100644 --- a/test/compare_output.zig +++ b/test/compare_output.zig @@ -22,7 +22,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void { \\ \\pub fn main() void { \\ privateFunction(); - \\ const stdout = &getStdOut().outStream().stream; + \\ const stdout = getStdOut().outStream(); \\ stdout.print("OK 2\n", .{}) catch unreachable; \\} \\ @@ -37,7 +37,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void { \\// purposefully conflicting function with main.zig \\// but it's private so it should be OK \\fn privateFunction() void { - \\ const stdout = &getStdOut().outStream().stream; + \\ const stdout = getStdOut().outStream(); \\ stdout.print("OK 1\n", .{}) catch unreachable; \\} \\ @@ -63,7 +63,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void { tc.addSourceFile("foo.zig", \\usingnamespace @import("std").io; \\pub fn foo_function() void { - \\ const stdout = &getStdOut().outStream().stream; + \\ const stdout = getStdOut().outStream(); \\ stdout.print("OK\n", .{}) catch unreachable; \\} ); @@ -74,7 +74,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void { \\ \\pub fn bar_function() void { \\ if (foo_function()) { - \\ const stdout = &getStdOut().outStream().stream; + \\ const stdout = getStdOut().outStream(); \\ stdout.print("OK\n", .{}) catch unreachable; \\ } \\} @@ -106,7 +106,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void { \\pub const a_text = "OK\n"; \\ \\pub fn ok() void { - \\ const stdout = &io.getStdOut().outStream().stream; + \\ const stdout = io.getStdOut().outStream(); \\ stdout.print(b_text, .{}) catch unreachable; \\} ); @@ -124,7 +124,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void { \\const io = @import("std").io; \\ \\pub fn main() void { - \\ const stdout = &io.getStdOut().outStream().stream; + \\ const stdout = io.getStdOut().outStream(); \\ stdout.print("Hello, world!\n{d:4} {x:3} {c}\n", .{@as(u32, 12), @as(u16, 0x12), @as(u8, 'a')}) catch unreachable; \\} , "Hello, world!\n 12 12 a\n"); @@ -267,7 +267,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void { \\ var x_local : i32 = print_ok(x); \\} \\fn print_ok(val: @TypeOf(x)) @TypeOf(foo) { - \\ const stdout = &io.getStdOut().outStream().stream; + \\ const stdout = io.getStdOut().outStream(); \\ stdout.print("OK\n", .{}) catch unreachable; \\ return 0; \\} @@ -349,7 +349,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void { \\pub fn main() void { \\ const bar = Bar {.field2 = 13,}; \\ const foo = Foo {.field1 = bar,}; - \\ const stdout = &io.getStdOut().outStream().stream; + \\ const stdout = io.getStdOut().outStream(); \\ if (!foo.method()) { \\ stdout.print("BAD\n", .{}) catch unreachable; \\ } @@ -363,7 +363,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void { cases.add("defer with only fallthrough", \\const io = @import("std").io; \\pub fn main() void { - \\ const stdout = &io.getStdOut().outStream().stream; + \\ const stdout = io.getStdOut().outStream(); \\ stdout.print("before\n", .{}) catch unreachable; \\ defer stdout.print("defer1\n", .{}) catch unreachable; \\ defer stdout.print("defer2\n", .{}) catch unreachable; @@ -376,7 +376,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void { \\const io = @import("std").io; \\const os = @import("std").os; \\pub fn main() void { - \\ const stdout = &io.getStdOut().outStream().stream; + \\ const stdout = io.getStdOut().outStream(); \\ stdout.print("before\n", .{}) catch unreachable; \\ defer stdout.print("defer1\n", .{}) catch unreachable; \\ defer stdout.print("defer2\n", .{}) catch unreachable; @@ -393,7 +393,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void { \\ do_test() catch return; \\} \\fn do_test() !void { - \\ const stdout = &io.getStdOut().outStream().stream; + \\ const stdout = io.getStdOut().outStream(); \\ stdout.print("before\n", .{}) catch unreachable; \\ defer stdout.print("defer1\n", .{}) catch unreachable; \\ errdefer stdout.print("deferErr\n", .{}) catch unreachable; @@ -412,7 +412,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void { \\ do_test() catch return; \\} \\fn do_test() !void { - \\ const stdout = &io.getStdOut().outStream().stream; + \\ const stdout = io.getStdOut().outStream(); \\ stdout.print("before\n", .{}) catch unreachable; \\ defer stdout.print("defer1\n", .{}) catch unreachable; \\ errdefer stdout.print("deferErr\n", .{}) catch unreachable; @@ -429,7 +429,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void { \\const io = @import("std").io; \\ \\pub fn main() void { - \\ const stdout = &io.getStdOut().outStream().stream; + \\ const stdout = io.getStdOut().outStream(); \\ stdout.print(foo_txt, .{}) catch unreachable; \\} , "1234\nabcd\n"); @@ -448,9 +448,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void { \\ \\pub fn main() !void { \\ var args_it = std.process.args(); - \\ var stdout_file = io.getStdOut(); - \\ var stdout_adapter = stdout_file.outStream(); - \\ const stdout = &stdout_adapter.stream; + \\ const stdout = io.getStdOut().outStream(); \\ var index: usize = 0; \\ _ = args_it.skip(); \\ while (args_it.next(allocator)) |arg_or_err| : (index += 1) { @@ -489,9 +487,7 @@ pub fn addCases(cases: *tests.CompareOutputContext) void { \\ \\pub fn main() !void { \\ var args_it = std.process.args(); - \\ var stdout_file = io.getStdOut(); - \\ var stdout_adapter = stdout_file.outStream(); - \\ const stdout = &stdout_adapter.stream; + \\ const stdout = io.getStdOut().outStream(); \\ var index: usize = 0; \\ _ = args_it.skip(); \\ while (args_it.next(allocator)) |arg_or_err| : (index += 1) { diff --git a/test/standalone/guess_number/main.zig b/test/standalone/guess_number/main.zig index f5b3b36993..14babcd145 100644 --- a/test/standalone/guess_number/main.zig +++ b/test/standalone/guess_number/main.zig @@ -4,7 +4,7 @@ const io = std.io; const fmt = std.fmt; pub fn main() !void { - const stdout = &io.getStdOut().outStream().stream; + const stdout = io.getStdOut().outStream(); const stdin = io.getStdIn(); try stdout.print("Welcome to the Guess Number Game in Zig.\n", .{}); diff --git a/test/tests.zig b/test/tests.zig index e324902579..22dad10e3e 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -566,12 +566,9 @@ pub const StackTracesContext = struct { } child.spawn() catch |err| debug.panic("Unable to spawn {}: {}\n", .{ full_exe_path, @errorName(err) }); - var stdout_file_in_stream = child.stdout.?.inStream(); - var stderr_file_in_stream = child.stderr.?.inStream(); - - const stdout = stdout_file_in_stream.stream.readAllAlloc(b.allocator, max_stdout_size) catch unreachable; + const stdout = child.stdout.?.inStream().readAllAlloc(b.allocator, max_stdout_size) catch unreachable; defer b.allocator.free(stdout); - const stderr = stderr_file_in_stream.stream.readAllAlloc(b.allocator, max_stdout_size) catch unreachable; + const stderr = child.stderr.?.inStream().readAllAlloc(b.allocator, max_stdout_size) catch unreachable; defer b.allocator.free(stderr); const term = child.wait() catch |err| { @@ -798,11 +795,8 @@ pub const CompileErrorContext = struct { var stdout_buf = Buffer.initNull(b.allocator); var stderr_buf = Buffer.initNull(b.allocator); - var stdout_file_in_stream = child.stdout.?.inStream(); - var stderr_file_in_stream = child.stderr.?.inStream(); - - stdout_file_in_stream.stream.readAllBuffer(&stdout_buf, max_stdout_size) catch unreachable; - stderr_file_in_stream.stream.readAllBuffer(&stderr_buf, max_stdout_size) catch unreachable; + child.stdout.?.inStream().readAllBuffer(&stdout_buf, max_stdout_size) catch unreachable; + child.stderr.?.inStream().readAllBuffer(&stderr_buf, max_stdout_size) catch unreachable; const term = child.wait() catch |err| { debug.panic("Unable to spawn {}: {}\n", .{ zig_args.items[0], @errorName(err) }); From d882a305875d5e0ab5d7877c3763b5aaf0ff61ce Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 10 Mar 2020 20:51:30 -0400 Subject: [PATCH 07/15] fix stage2 lib on windows --- src-self-hosted/libc_installation.zig | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src-self-hosted/libc_installation.zig b/src-self-hosted/libc_installation.zig index 78aafeff28..0f9736456a 100644 --- a/src-self-hosted/libc_installation.zig +++ b/src-self-hosted/libc_installation.zig @@ -348,7 +348,7 @@ pub const LibCInstallation = struct { for (searches) |search| { result_buf.shrink(0); - const stream = &std.io.BufferOutStream.init(&result_buf).stream; + const stream = result_buf.outStream(); try stream.print("{}\\Include\\{}\\ucrt", .{ search.path, search.version }); var dir = fs.cwd().openDirList(result_buf.toSliceConst()) catch |err| switch (err) { @@ -395,7 +395,7 @@ pub const LibCInstallation = struct { for (searches) |search| { result_buf.shrink(0); - const stream = &std.io.BufferOutStream.init(&result_buf).stream; + const stream = result_buf.outStream(); try stream.print("{}\\Lib\\{}\\ucrt\\{}", .{ search.path, search.version, arch_sub_dir }); var dir = fs.cwd().openDirList(result_buf.toSliceConst()) catch |err| switch (err) { @@ -459,7 +459,7 @@ pub const LibCInstallation = struct { for (searches) |search| { result_buf.shrink(0); - const stream = &std.io.BufferOutStream.init(&result_buf).stream; + const stream = result_buf.outStream(); try stream.print("{}\\Lib\\{}\\um\\{}", .{ search.path, search.version, arch_sub_dir }); var dir = fs.cwd().openDirList(result_buf.toSliceConst()) catch |err| switch (err) { From bd14a81e301401b5b2a8b849d17d6ae0750272b8 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 10 Mar 2020 21:09:49 -0400 Subject: [PATCH 08/15] fix std.ChildProcess on Windows --- lib/std/child_process.zig | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/std/child_process.zig b/lib/std/child_process.zig index 95619384b1..008fc34ff6 100644 --- a/lib/std/child_process.zig +++ b/lib/std/child_process.zig @@ -217,13 +217,13 @@ pub const ChildProcess = struct { try child.spawn(); - var stdout_file_in_stream = child.stdout.?.inStream(); - var stderr_file_in_stream = child.stderr.?.inStream(); + const stdout_in = child.stdout.?.inStream(); + const stderr_in = child.stderr.?.inStream(); // TODO need to poll to read these streams to prevent a deadlock (or rely on evented I/O). - const stdout = try stdout_file_in_stream.readAllAlloc(args.allocator, args.max_output_bytes); + const stdout = try stdout_in.readAllAlloc(args.allocator, args.max_output_bytes); errdefer args.allocator.free(stdout); - const stderr = try stderr_file_in_stream.readAllAlloc(args.allocator, args.max_output_bytes); + const stderr = try stderr_in.readAllAlloc(args.allocator, args.max_output_bytes); errdefer args.allocator.free(stderr); return ExecResult{ @@ -780,7 +780,7 @@ fn windowsCreateCommandLine(allocator: *mem.Allocator, argv: []const []const u8) var buf = try Buffer.initSize(allocator, 0); defer buf.deinit(); - var buf_stream = &io.BufferOutStream.init(&buf).stream; + var buf_stream = buf.outStream(); for (argv) |arg, arg_i| { if (arg_i != 0) try buf.appendByte(' '); From ed13cffca4c42718cc7239faf7aab642ac67ef77 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 10 Mar 2020 22:01:58 -0400 Subject: [PATCH 09/15] rework some old ELF parsing code and start to fix emitRaw --- lib/std/build/emit_raw.zig | 112 +++++--------- lib/std/elf.zig | 292 ++++++++++--------------------------- lib/std/zig/system.zig | 35 ++--- 3 files changed, 128 insertions(+), 311 deletions(-) diff --git a/lib/std/build/emit_raw.zig b/lib/std/build/emit_raw.zig index 44e0227b6e..63c2329947 100644 --- a/lib/std/build/emit_raw.zig +++ b/lib/std/build/emit_raw.zig @@ -14,11 +14,6 @@ const io = std.io; const sort = std.sort; const warn = std.debug.warn; -const BinOutStream = io.OutStream(anyerror); -const BinSeekStream = io.SeekableStream(anyerror, anyerror); -const ElfSeekStream = io.SeekableStream(anyerror, anyerror); -const ElfInStream = io.InStream(anyerror); - const BinaryElfSection = struct { elfOffset: u64, binaryOffset: u64, @@ -41,22 +36,21 @@ const BinaryElfOutput = struct { const Self = @This(); - pub fn init(allocator: *Allocator) Self { - return Self{ - .segments = ArrayList(*BinaryElfSegment).init(allocator), - .sections = ArrayList(*BinaryElfSection).init(allocator), - }; - } - pub fn deinit(self: *Self) void { self.sections.deinit(); self.segments.deinit(); } - pub fn parseElf(self: *Self, elfFile: elf.Elf) !void { - const allocator = self.segments.allocator; + pub fn parse(allocator: *Allocator, elf_file: File) !Self { + var self: Self = .{ + .segments = ArrayList(*BinaryElfSegment).init(allocator), + .sections = ArrayList(*BinaryElfSection).init(allocator), + }; + const elf_hdrs = try std.elf.readAllHeaders(allocator, elf_file); - for (elfFile.section_headers) |section, i| { + var binaryElfOutput = BinaryElfOutput.init(arena_allocator); + + for (elf_hdrs.section_headers) |section, i| { if (sectionValidForOutput(section)) { const newSection = try allocator.create(BinaryElfSection); @@ -69,19 +63,19 @@ const BinaryElfOutput = struct { } } - for (elfFile.program_headers) |programHeader, i| { - if (programHeader.p_type == elf.PT_LOAD) { + for (elf_hdrs.program_headers) |phdr, i| { + if (phdr.p_type == elf.PT_LOAD) { const newSegment = try allocator.create(BinaryElfSegment); - newSegment.physicalAddress = if (programHeader.p_paddr != 0) programHeader.p_paddr else programHeader.p_vaddr; - newSegment.virtualAddress = programHeader.p_vaddr; - newSegment.fileSize = @intCast(usize, programHeader.p_filesz); - newSegment.elfOffset = programHeader.p_offset; + newSegment.physicalAddress = if (phdr.p_paddr != 0) phdr.p_paddr else phdr.p_vaddr; + newSegment.virtualAddress = phdr.p_vaddr; + newSegment.fileSize = @intCast(usize, phdr.p_filesz); + newSegment.elfOffset = phdr.p_offset; newSegment.binaryOffset = 0; newSegment.firstSection = null; for (self.sections.toSlice()) |section| { - if (sectionWithinSegment(section, programHeader)) { + if (sectionWithinSegment(section, phdr)) { if (section.segment) |sectionSegment| { if (sectionSegment.elfOffset > newSegment.elfOffset) { section.segment = newSegment; @@ -126,14 +120,17 @@ const BinaryElfOutput = struct { } sort.sort(*BinaryElfSection, self.sections.toSlice(), sectionSortCompare); + + return self; } - fn sectionWithinSegment(section: *BinaryElfSection, segment: elf.ProgramHeader) bool { + fn sectionWithinSegment(section: *BinaryElfSection, segment: elf.Elf64_Phdr) bool { return segment.p_offset <= section.elfOffset and (segment.p_offset + segment.p_filesz) >= (section.elfOffset + section.fileSize); } - fn sectionValidForOutput(section: elf.SectionHeader) bool { - return section.sh_size > 0 and section.sh_type != elf.SHT_NOBITS and ((section.sh_flags & elf.SHF_ALLOC) == elf.SHF_ALLOC); + fn sectionValidForOutput(shdr: var) bool { + return shdr.sh_size > 0 and shdr.sh_type != elf.SHT_NOBITS and + ((shdr.sh_flags & elf.SHF_ALLOC) == elf.SHF_ALLOC); } fn segmentSortCompare(left: *BinaryElfSegment, right: *BinaryElfSegment) bool { @@ -151,60 +148,27 @@ const BinaryElfOutput = struct { } }; -const WriteContext = struct { - inStream: *ElfInStream, - inSeekStream: *ElfSeekStream, - outStream: *BinOutStream, - outSeekStream: *BinSeekStream, -}; +fn writeBinaryElfSection(elf_file: File, out_file: File, section: *BinaryElfSection) !void { + try out_file.seekTo(section.binaryOffset); -fn writeBinaryElfSection(allocator: *Allocator, context: WriteContext, section: *BinaryElfSection) !void { - var readBuffer = try allocator.alloc(u8, section.fileSize); - defer allocator.free(readBuffer); - - try context.inSeekStream.seekTo(section.elfOffset); - _ = try context.inStream.read(readBuffer); - - try context.outSeekStream.seekTo(section.binaryOffset); - try context.outStream.write(readBuffer); + try out_file.writeFileAll(elf_file, .{ + .in_offset = section.elfOffset, + .in_len = section.fileSize, + }); } -fn emit_raw(allocator: *Allocator, elf_path: []const u8, raw_path: []const u8) !void { - var arenaAlloc = ArenaAllocator.init(allocator); - errdefer arenaAlloc.deinit(); - var arena_allocator = &arenaAlloc.allocator; +fn emitRaw(allocator: *Allocator, elf_path: []const u8, raw_path: []const u8) !void { + var elf_file = try fs.cwd().openFile(elf_path, .{}); + defer elf_file.close(); - const currentDir = fs.cwd(); + var out_file = try fs.cwd().createFile(raw_path, .{}); + defer out_file.close(); - var file = try currentDir.openFile(elf_path, File.OpenFlags{}); - defer file.close(); + const binary_elf_output = BinaryElfOutput.parse(allocator, elf_file); + defer binary_elf_output.deinit(); - var fileInStream = file.inStream(); - var fileSeekStream = file.seekableStream(); - - var elfFile = try elf.Elf.openStream(allocator, @ptrCast(*ElfSeekStream, &fileSeekStream.stream), @ptrCast(*ElfInStream, &fileInStream.stream)); - defer elfFile.close(); - - var outFile = try currentDir.createFile(raw_path, File.CreateFlags{}); - defer outFile.close(); - - var outFileOutStream = outFile.outStream(); - var outFileSeekStream = outFile.seekableStream(); - - const writeContext = WriteContext{ - .inStream = @ptrCast(*ElfInStream, &fileInStream.stream), - .inSeekStream = @ptrCast(*ElfSeekStream, &fileSeekStream.stream), - .outStream = @ptrCast(*BinOutStream, &outFileOutStream.stream), - .outSeekStream = @ptrCast(*BinSeekStream, &outFileSeekStream.stream), - }; - - var binaryElfOutput = BinaryElfOutput.init(arena_allocator); - defer binaryElfOutput.deinit(); - - try binaryElfOutput.parseElf(elfFile); - - for (binaryElfOutput.sections.toSlice()) |section| { - try writeBinaryElfSection(allocator, writeContext, section); + for (binary_elf_output.sections.toSlice()) |section| { + try writeBinaryElfSection(elf_file, out_file, section); } } @@ -250,6 +214,6 @@ pub const InstallRawStep = struct { const full_dest_path = builder.getInstallPath(self.dest_dir, self.dest_filename); fs.cwd().makePath(builder.getInstallPath(self.dest_dir, "")) catch unreachable; - try emit_raw(builder.allocator, full_src_path, full_dest_path); + try emitRaw(builder.allocator, full_src_path, full_dest_path); } }; diff --git a/lib/std/elf.zig b/lib/std/elf.zig index efaa4d6f06..3aefd87f09 100644 --- a/lib/std/elf.zig +++ b/lib/std/elf.zig @@ -330,9 +330,7 @@ pub const ET = extern enum(u16) { pub const HIPROC = 0xffff; }; -pub const SectionHeader = Elf64_Shdr; -pub const ProgramHeader = Elf64_Phdr; - +/// All integers are native endian. const Header = struct { endian: builtin.Endian, is_64: bool, @@ -346,9 +344,9 @@ const Header = struct { shstrndx: u16, }; -pub fn readHeader(in_stream: var) !Header { +pub fn readHeader(file: File) !Header { var hdr_buf: [@sizeOf(Elf64_Ehdr)]u8 align(@alignOf(Elf64_Ehdr)) = undefined; - try in_stream.readAll(&hdr_buf); + try in_stream.preadAll(&hdr_buf, 0); const hdr32 = @ptrCast(*elf.Elf32_Ehdr, &hdr_buf); const hdr64 = @ptrCast(*elf.Elf64_Ehdr, &hdr_buf); if (!mem.eql(u8, hdr32.e_ident[0..4], "\x7fELF")) return error.InvalidElfMagic; @@ -381,216 +379,86 @@ pub fn readHeader(in_stream: var) !Header { }); } -pub const Elf = struct { - seekable_stream: *io.SeekableStream(anyerror, anyerror), - in_stream: *io.InStream(anyerror), - is_64: bool, - endian: builtin.Endian, - file_type: ET, - arch: EM, - entry_addr: u64, - program_header_offset: u64, - section_header_offset: u64, - string_section_index: usize, - string_section: *SectionHeader, - section_headers: []SectionHeader, - program_headers: []ProgramHeader, +/// All integers are native endian. +pub const AllHeaders = struct { + header: Header, + section_headers: []Elf64_Shdr, + program_headers: []Elf64_Phdr, allocator: *mem.Allocator, - - pub fn openStream( - allocator: *mem.Allocator, - seekable_stream: *io.SeekableStream(anyerror, anyerror), - in: *io.InStream(anyerror), - ) !Elf { - var elf: Elf = undefined; - elf.allocator = allocator; - elf.seekable_stream = seekable_stream; - elf.in_stream = in; - - var magic: [4]u8 = undefined; - try in.readNoEof(magic[0..]); - if (!mem.eql(u8, &magic, "\x7fELF")) return error.InvalidFormat; - - elf.is_64 = switch (try in.readByte()) { - 1 => false, - 2 => true, - else => return error.InvalidFormat, - }; - - elf.endian = switch (try in.readByte()) { - 1 => .Little, - 2 => .Big, - else => return error.InvalidFormat, - }; - - const version_byte = try in.readByte(); - if (version_byte != 1) return error.InvalidFormat; - - // skip over padding - try seekable_stream.seekBy(9); - - elf.file_type = try in.readEnum(ET, elf.endian); - elf.arch = try in.readEnum(EM, elf.endian); - - const elf_version = try in.readInt(u32, elf.endian); - if (elf_version != 1) return error.InvalidFormat; - - if (elf.is_64) { - elf.entry_addr = try in.readInt(u64, elf.endian); - elf.program_header_offset = try in.readInt(u64, elf.endian); - elf.section_header_offset = try in.readInt(u64, elf.endian); - } else { - elf.entry_addr = @as(u64, try in.readInt(u32, elf.endian)); - elf.program_header_offset = @as(u64, try in.readInt(u32, elf.endian)); - elf.section_header_offset = @as(u64, try in.readInt(u32, elf.endian)); - } - - // skip over flags - try seekable_stream.seekBy(4); - - const header_size = try in.readInt(u16, elf.endian); - if ((elf.is_64 and header_size != @sizeOf(Elf64_Ehdr)) or (!elf.is_64 and header_size != @sizeOf(Elf32_Ehdr))) { - return error.InvalidFormat; - } - - const ph_entry_size = try in.readInt(u16, elf.endian); - const ph_entry_count = try in.readInt(u16, elf.endian); - - if ((elf.is_64 and ph_entry_size != @sizeOf(Elf64_Phdr)) or (!elf.is_64 and ph_entry_size != @sizeOf(Elf32_Phdr))) { - return error.InvalidFormat; - } - - const sh_entry_size = try in.readInt(u16, elf.endian); - const sh_entry_count = try in.readInt(u16, elf.endian); - - if ((elf.is_64 and sh_entry_size != @sizeOf(Elf64_Shdr)) or (!elf.is_64 and sh_entry_size != @sizeOf(Elf32_Shdr))) { - return error.InvalidFormat; - } - - elf.string_section_index = @as(usize, try in.readInt(u16, elf.endian)); - - if (elf.string_section_index >= sh_entry_count) return error.InvalidFormat; - - const sh_byte_count = @as(u64, sh_entry_size) * @as(u64, sh_entry_count); - const end_sh = try math.add(u64, elf.section_header_offset, sh_byte_count); - const ph_byte_count = @as(u64, ph_entry_size) * @as(u64, ph_entry_count); - const end_ph = try math.add(u64, elf.program_header_offset, ph_byte_count); - - const stream_end = try seekable_stream.getEndPos(); - if (stream_end < end_sh or stream_end < end_ph) { - return error.InvalidFormat; - } - - try seekable_stream.seekTo(elf.program_header_offset); - - elf.program_headers = try elf.allocator.alloc(ProgramHeader, ph_entry_count); - errdefer elf.allocator.free(elf.program_headers); - - if (elf.is_64) { - for (elf.program_headers) |*elf_program| { - elf_program.p_type = try in.readInt(Elf64_Word, elf.endian); - elf_program.p_flags = try in.readInt(Elf64_Word, elf.endian); - elf_program.p_offset = try in.readInt(Elf64_Off, elf.endian); - elf_program.p_vaddr = try in.readInt(Elf64_Addr, elf.endian); - elf_program.p_paddr = try in.readInt(Elf64_Addr, elf.endian); - elf_program.p_filesz = try in.readInt(Elf64_Xword, elf.endian); - elf_program.p_memsz = try in.readInt(Elf64_Xword, elf.endian); - elf_program.p_align = try in.readInt(Elf64_Xword, elf.endian); - } - } else { - for (elf.program_headers) |*elf_program| { - elf_program.p_type = @as(Elf64_Word, try in.readInt(Elf32_Word, elf.endian)); - elf_program.p_offset = @as(Elf64_Off, try in.readInt(Elf32_Off, elf.endian)); - elf_program.p_vaddr = @as(Elf64_Addr, try in.readInt(Elf32_Addr, elf.endian)); - elf_program.p_paddr = @as(Elf64_Addr, try in.readInt(Elf32_Addr, elf.endian)); - elf_program.p_filesz = @as(Elf64_Word, try in.readInt(Elf32_Word, elf.endian)); - elf_program.p_memsz = @as(Elf64_Word, try in.readInt(Elf32_Word, elf.endian)); - elf_program.p_flags = @as(Elf64_Word, try in.readInt(Elf32_Word, elf.endian)); - elf_program.p_align = @as(Elf64_Word, try in.readInt(Elf32_Word, elf.endian)); - } - } - - try seekable_stream.seekTo(elf.section_header_offset); - - elf.section_headers = try elf.allocator.alloc(SectionHeader, sh_entry_count); - errdefer elf.allocator.free(elf.section_headers); - - if (elf.is_64) { - for (elf.section_headers) |*elf_section| { - elf_section.sh_name = try in.readInt(u32, elf.endian); - elf_section.sh_type = try in.readInt(u32, elf.endian); - elf_section.sh_flags = try in.readInt(u64, elf.endian); - elf_section.sh_addr = try in.readInt(u64, elf.endian); - elf_section.sh_offset = try in.readInt(u64, elf.endian); - elf_section.sh_size = try in.readInt(u64, elf.endian); - elf_section.sh_link = try in.readInt(u32, elf.endian); - elf_section.sh_info = try in.readInt(u32, elf.endian); - elf_section.sh_addralign = try in.readInt(u64, elf.endian); - elf_section.sh_entsize = try in.readInt(u64, elf.endian); - } - } else { - for (elf.section_headers) |*elf_section| { - // TODO (multiple occurrences) allow implicit cast from %u32 -> %u64 ? - elf_section.sh_name = try in.readInt(u32, elf.endian); - elf_section.sh_type = try in.readInt(u32, elf.endian); - elf_section.sh_flags = @as(u64, try in.readInt(u32, elf.endian)); - elf_section.sh_addr = @as(u64, try in.readInt(u32, elf.endian)); - elf_section.sh_offset = @as(u64, try in.readInt(u32, elf.endian)); - elf_section.sh_size = @as(u64, try in.readInt(u32, elf.endian)); - elf_section.sh_link = try in.readInt(u32, elf.endian); - elf_section.sh_info = try in.readInt(u32, elf.endian); - elf_section.sh_addralign = @as(u64, try in.readInt(u32, elf.endian)); - elf_section.sh_entsize = @as(u64, try in.readInt(u32, elf.endian)); - } - } - - for (elf.section_headers) |*elf_section| { - if (elf_section.sh_type != SHT_NOBITS) { - const file_end_offset = try math.add(u64, elf_section.sh_offset, elf_section.sh_size); - if (stream_end < file_end_offset) return error.InvalidFormat; - } - } - - elf.string_section = &elf.section_headers[elf.string_section_index]; - if (elf.string_section.sh_type != SHT_STRTAB) { - // not a string table - return error.InvalidFormat; - } - - return elf; - } - - pub fn close(elf: *Elf) void { - elf.allocator.free(elf.section_headers); - elf.allocator.free(elf.program_headers); - } - - pub fn findSection(elf: *Elf, name: []const u8) !?*SectionHeader { - section_loop: for (elf.section_headers) |*elf_section| { - if (elf_section.sh_type == SHT_NULL) continue; - - const name_offset = elf.string_section.sh_offset + elf_section.sh_name; - try elf.seekable_stream.seekTo(name_offset); - - for (name) |expected_c| { - const target_c = try elf.in_stream.readByte(); - if (target_c == 0 or expected_c != target_c) continue :section_loop; - } - - { - const null_byte = try elf.in_stream.readByte(); - if (null_byte == 0) return elf_section; - } - } - - return null; - } - - pub fn seekToSection(elf: *Elf, elf_section: *SectionHeader) !void { - try elf.seekable_stream.seekTo(elf_section.sh_offset); - } }; +pub fn readAllHeaders(allocator: *mem.Allocator, file: File) !AllHeaders { + var hdrs: AllHeaders = .{ + .allocator = allocator, + .header = try readHeader(file), + .section_headers = undefined, + .program_headers = undefined, + }; + const is_64 = hdrs.header.is_64; + const need_bswap = hdrs.header.endian != std.builtin.endian; + + hdrs.section_headers = try allocator.alloc(Elf64_Shdr, hdrs.header.shnum); + errdefer hdrs.allocator.free(hdrs.section_headers); + + hdrs.program_headers = try allocator.alloc(Elf64_Phdr, hdrs.header.phnum); + errdefer hdrs.allocator.free(hdrs.program_headers); + + // Treat section headers and program headers as byte buffers. For 32-bit ELF and + // non-matching endian files, we post-process to correct integer endianness and offsets. + + const shdr_buf = std.mem.sliceToBytes(hdrs.section_headers)[0 .. hdrs.header.shentsize * hdrs.header.shnum]; + const phdr_buf = std.mem.sliceToBytes(hdrs.section_headers)[0 .. hdrs.header.phentsize * hdrs.header.phnum]; + + try file.preadAll(phdr_buf, hdrs.header.phoff); + try file.preadAll(shdr_buf, hdrs.header.shoff); + + const shdrs32 = @ptrCast([*]Elf32_Shdr, @alignCast(@alignOf(Elf32_Shdr), shdr_buf.ptr))[0..hdrs.header.shnum]; + const phdrs32 = @ptrCast([*]Elf32_Phdr, @alignCast(@alignOf(Elf32_Phdr), phdr_buf.ptr))[0..hdrs.header.phnum]; + for (hdrs.section_headers) |*shdr, i| { + shdr.* = .{ + .sh_name = int(is_64, need_bswap, shdrs32[i].sh_name, shdr.sh_name), + .sh_type = int(is_64, need_bswap, shdrs32[i].sh_type, shdr.sh_type), + .sh_flags = int(is_64, need_bswap, shdrs32[i].sh_flags, shdr.sh_flags), + .sh_addr = int(is_64, need_bswap, shdrs32[i].sh_addr, shdr.sh_addr), + .sh_offset = int(is_64, need_bswap, shdrs32[i].sh_offset, shdr.sh_offset), + .sh_size = int(is_64, need_bswap, shdrs32[i].sh_size, shdr.sh_size), + .sh_link = int(is_64, need_bswap, shdrs32[i].sh_link, shdr.sh_link), + .sh_info = int(is_64, need_bswap, shdrs32[i].sh_info, shdr.sh_info), + .sh_addralign = int(is_64, need_bswap, shdrs32[i].sh_addralign, shdr.sh_addralign), + .sh_entsize = int(is_64, need_bswap, shdrs32[i].sh_entsize, shdr.sh_entsize), + }; + } + for (hdrs.program_headers) |*phdr, i| { + phdr.* = .{ + .p_type = int(is_64, need_bswap, phdrs32[i].p_type, shdr.p_type), + .p_offset = int(is_64, need_bswap, phdrs32[i].p_offset, shdr.p_offset), + .p_vaddr = int(is_64, need_bswap, phdrs32[i].p_vaddr, shdr.p_vaddr), + .p_paddr = int(is_64, need_bswap, phdrs32[i].p_paddr, shdr.p_paddr), + .p_filesz = int(is_64, need_bswap, phdrs32[i].p_filesz, shdr.p_filesz), + .p_memsz = int(is_64, need_bswap, phdrs32[i].p_memsz, shdr.p_memsz), + .p_flags = int(is_64, need_bswap, phdrs32[i].p_flags, shdr.p_flags), + .p_align = int(is_64, need_bswap, phdrs32[i].p_align, shdr.p_align), + }; + } + return hdrs; +} + +pub fn int(is_64: bool, need_bswap: bool, int_32: var, int_64: var) @TypeOf(int_64) { + if (is_64) { + if (need_bswap) { + return @byteSwap(@TypeOf(int_64), int_64); + } else { + return int_64; + } + } else { + if (need_bswap) { + return @byteSwap(@TypeOf(int_32), int_32); + } else { + return int_32; + } + } +} + pub const EI_NIDENT = 16; pub const EI_CLASS = 4; diff --git a/lib/std/zig/system.zig b/lib/std/zig/system.zig index 336dd0d314..e7953a1a91 100644 --- a/lib/std/zig/system.zig +++ b/lib/std/zig/system.zig @@ -570,7 +570,7 @@ pub const NativeTargetInfo = struct { cross_target: CrossTarget, ) AbiAndDynamicLinkerFromFileError!NativeTargetInfo { var hdr_buf: [@sizeOf(elf.Elf64_Ehdr)]u8 align(@alignOf(elf.Elf64_Ehdr)) = undefined; - _ = try preadFull(file, &hdr_buf, 0, hdr_buf.len); + _ = try preadMin(file, &hdr_buf, 0, hdr_buf.len); const hdr32 = @ptrCast(*elf.Elf32_Ehdr, &hdr_buf); const hdr64 = @ptrCast(*elf.Elf64_Ehdr, &hdr_buf); if (!mem.eql(u8, hdr32.e_ident[0..4], "\x7fELF")) return error.InvalidElfMagic; @@ -587,6 +587,7 @@ pub const NativeTargetInfo = struct { elf.ELFCLASS64 => true, else => return error.InvalidElfClass, }; + const elfInt = elf.int; var phoff = elfInt(is_64, need_bswap, hdr32.e_phoff, hdr64.e_phoff); const phentsize = elfInt(is_64, need_bswap, hdr32.e_phentsize, hdr64.e_phentsize); const phnum = elfInt(is_64, need_bswap, hdr32.e_phnum, hdr64.e_phnum); @@ -610,7 +611,7 @@ pub const NativeTargetInfo = struct { // Reserve some bytes so that we can deref the 64-bit struct fields // even when the ELF file is 32-bits. const ph_reserve: usize = @sizeOf(elf.Elf64_Phdr) - @sizeOf(elf.Elf32_Phdr); - const ph_read_byte_len = try preadFull(file, ph_buf[0 .. ph_buf.len - ph_reserve], phoff, phentsize); + const ph_read_byte_len = try preadMin(file, ph_buf[0 .. ph_buf.len - ph_reserve], phoff, phentsize); var ph_buf_i: usize = 0; while (ph_buf_i < ph_read_byte_len and ph_i < phnum) : ({ ph_i += 1; @@ -625,7 +626,7 @@ pub const NativeTargetInfo = struct { const p_offset = elfInt(is_64, need_bswap, ph32.p_offset, ph64.p_offset); const p_filesz = elfInt(is_64, need_bswap, ph32.p_filesz, ph64.p_filesz); if (p_filesz > result.dynamic_linker.buffer.len) return error.NameTooLong; - _ = try preadFull(file, result.dynamic_linker.buffer[0..p_filesz], p_offset, p_filesz); + _ = try preadMin(file, result.dynamic_linker.buffer[0..p_filesz], p_offset, p_filesz); // PT_INTERP includes a null byte in p_filesz. const len = p_filesz - 1; // dynamic_linker.max_byte is "max", not "len". @@ -656,7 +657,7 @@ pub const NativeTargetInfo = struct { // Reserve some bytes so that we can deref the 64-bit struct fields // even when the ELF file is 32-bits. const dyn_reserve: usize = @sizeOf(elf.Elf64_Dyn) - @sizeOf(elf.Elf32_Dyn); - const dyn_read_byte_len = try preadFull( + const dyn_read_byte_len = try preadMin( file, dyn_buf[0 .. dyn_buf.len - dyn_reserve], dyn_off, @@ -701,14 +702,14 @@ pub const NativeTargetInfo = struct { var sh_buf: [16 * @sizeOf(elf.Elf64_Shdr)]u8 align(@alignOf(elf.Elf64_Shdr)) = undefined; if (sh_buf.len < shentsize) return error.InvalidElfFile; - _ = try preadFull(file, &sh_buf, str_section_off, shentsize); + _ = try preadMin(file, &sh_buf, str_section_off, shentsize); const shstr32 = @ptrCast(*elf.Elf32_Shdr, @alignCast(@alignOf(elf.Elf32_Shdr), &sh_buf)); const shstr64 = @ptrCast(*elf.Elf64_Shdr, @alignCast(@alignOf(elf.Elf64_Shdr), &sh_buf)); const shstrtab_off = elfInt(is_64, need_bswap, shstr32.sh_offset, shstr64.sh_offset); const shstrtab_size = elfInt(is_64, need_bswap, shstr32.sh_size, shstr64.sh_size); var strtab_buf: [4096:0]u8 = undefined; const shstrtab_len = std.math.min(shstrtab_size, strtab_buf.len); - const shstrtab_read_len = try preadFull(file, &strtab_buf, shstrtab_off, shstrtab_len); + const shstrtab_read_len = try preadMin(file, &strtab_buf, shstrtab_off, shstrtab_len); const shstrtab = strtab_buf[0..shstrtab_read_len]; const shnum = elfInt(is_64, need_bswap, hdr32.e_shnum, hdr64.e_shnum); @@ -717,7 +718,7 @@ pub const NativeTargetInfo = struct { // Reserve some bytes so that we can deref the 64-bit struct fields // even when the ELF file is 32-bits. const sh_reserve: usize = @sizeOf(elf.Elf64_Shdr) - @sizeOf(elf.Elf32_Shdr); - const sh_read_byte_len = try preadFull( + const sh_read_byte_len = try preadMin( file, sh_buf[0 .. sh_buf.len - sh_reserve], shoff, @@ -751,7 +752,7 @@ pub const NativeTargetInfo = struct { if (dynstr) |ds| { const strtab_len = std.math.min(ds.size, strtab_buf.len); - const strtab_read_len = try preadFull(file, &strtab_buf, ds.offset, shstrtab_len); + const strtab_read_len = try preadMin(file, &strtab_buf, ds.offset, shstrtab_len); const strtab = strtab_buf[0..strtab_read_len]; // TODO this pointer cast should not be necessary const rpath_list = mem.toSliceConst(u8, @ptrCast([*:0]u8, strtab[rpoff..].ptr)); @@ -813,7 +814,7 @@ pub const NativeTargetInfo = struct { return result; } - fn preadFull(file: fs.File, buf: []u8, offset: u64, min_read_len: usize) !usize { + fn preadMin(file: fs.File, buf: []u8, offset: u64, min_read_len: usize) !usize { var i: u64 = 0; while (i < min_read_len) { const len = file.pread(buf[i .. buf.len - i], offset + i) catch |err| switch (err) { @@ -853,22 +854,6 @@ pub const NativeTargetInfo = struct { abi: Target.Abi, }; - fn elfInt(is_64: bool, need_bswap: bool, int_32: var, int_64: var) @TypeOf(int_64) { - if (is_64) { - if (need_bswap) { - return @byteSwap(@TypeOf(int_64), int_64); - } else { - return int_64; - } - } else { - if (need_bswap) { - return @byteSwap(@TypeOf(int_32), int_32); - } else { - return int_32; - } - } - } - fn detectNativeCpuAndFeatures(cpu_arch: Target.Cpu.Arch, os: Target.Os, cross_target: CrossTarget) ?Target.Cpu { // Here we switch on a comptime value rather than `cpu_arch`. This is valid because `cpu_arch`, // although it is a runtime value, is guaranteed to be one of the architectures in the set From d96b6c0d9f7a4fc6e7e28dd92d18e548f00885de Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 11 Mar 2020 13:06:30 -0400 Subject: [PATCH 10/15] fix footguns in File readAll functions --- lib/std/fs.zig | 33 ++++++++++++------------------- lib/std/fs/file.zig | 42 +++++++++++++++++++++++++++++++--------- lib/std/io/in_stream.zig | 5 +---- lib/std/os/test.zig | 32 +++++++++++++++++++++++++++--- 4 files changed, 75 insertions(+), 37 deletions(-) diff --git a/lib/std/fs.zig b/lib/std/fs.zig index 892b977291..de6be91f71 100644 --- a/lib/std/fs.zig +++ b/lib/std/fs.zig @@ -96,6 +96,7 @@ pub fn updateFile(source_path: []const u8, dest_path: []const u8) !PrevStatus { /// atime, and mode of the source file so that the next call to `updateFile` will not need a copy. /// Returns the previous status of the file before updating. /// If any of the directories do not exist for dest_path, they are created. +/// TODO rework this to integrate with Dir pub fn updateFileMode(source_path: []const u8, dest_path: []const u8, mode: ?File.Mode) !PrevStatus { const my_cwd = cwd(); @@ -141,29 +142,25 @@ pub fn updateFileMode(source_path: []const u8, dest_path: []const u8, mode: ?Fil /// there is a possibility of power loss or application termination leaving temporary files present /// in the same directory as dest_path. /// Destination file will have the same mode as the source file. +/// TODO rework this to integrate with Dir pub fn copyFile(source_path: []const u8, dest_path: []const u8) !void { var in_file = try cwd().openFile(source_path, .{}); defer in_file.close(); - const mode = try in_file.mode(); - const in_stream = &in_file.inStream().stream; + const stat = try in_file.stat(); - var atomic_file = try AtomicFile.init(dest_path, mode); + var atomic_file = try AtomicFile.init(dest_path, stat.mode); defer atomic_file.deinit(); - var buf: [mem.page_size]u8 = undefined; - while (true) { - const amt = try in_stream.readFull(buf[0..]); - try atomic_file.file.write(buf[0..amt]); - if (amt != buf.len) { - return atomic_file.finish(); - } - } + try atomic_file.file.writeFileAll(in_file, .{ .in_len = stat.size }); + return atomic_file.finish(); } -/// Guaranteed to be atomic. However until https://patchwork.kernel.org/patch/9636735/ is -/// merged and readily available, +/// Guaranteed to be atomic. +/// On Linux, until https://patchwork.kernel.org/patch/9636735/ is merged and readily available, /// there is a possibility of power loss or application termination leaving temporary files present +/// in the same directory as dest_path. +/// TODO rework this to integrate with Dir pub fn copyFileMode(source_path: []const u8, dest_path: []const u8, mode: File.Mode) !void { var in_file = try cwd().openFile(source_path, .{}); defer in_file.close(); @@ -171,14 +168,8 @@ pub fn copyFileMode(source_path: []const u8, dest_path: []const u8, mode: File.M var atomic_file = try AtomicFile.init(dest_path, mode); defer atomic_file.deinit(); - var buf: [mem.page_size * 6]u8 = undefined; - while (true) { - const amt = try in_file.read(buf[0..]); - try atomic_file.file.write(buf[0..amt]); - if (amt != buf.len) { - return atomic_file.finish(); - } - } + try atomic_file.file.writeFileAll(in_file, .{}); + return atomic_file.finish(); } /// TODO update this API to avoid a getrandom syscall for every operation. It diff --git a/lib/std/fs/file.zig b/lib/std/fs/file.zig index 845e4bc75e..8362f94ca5 100644 --- a/lib/std/fs/file.zig +++ b/lib/std/fs/file.zig @@ -250,11 +250,16 @@ pub const File = struct { } } - pub fn readAll(self: File, buffer: []u8) ReadError!void { + /// Returns the number of bytes read. If the number read is smaller than `buffer.len`, it + /// means the file reached the end. Reaching the end of a file is not an error condition. + pub fn readAll(self: File, buffer: []u8) ReadError!usize { var index: usize = 0; - while (index < buffer.len) { - index += try self.read(buffer[index..]); + while (index != buffer.len) { + const amt = try self.read(buffer[index..]); + if (amt == 0) break; + index += amt; } + return index; } pub fn pread(self: File, buffer: []u8, offset: u64) PReadError!usize { @@ -265,11 +270,16 @@ pub const File = struct { } } - pub fn preadAll(self: File, buffer: []u8, offset: u64) PReadError!void { + /// Returns the number of bytes read. If the number read is smaller than `buffer.len`, it + /// means the file reached the end. Reaching the end of a file is not an error condition. + pub fn preadAll(self: File, buffer: []u8, offset: u64) PReadError!usize { var index: usize = 0; - while (index < buffer.len) { - index += try self.pread(buffer[index..], offset + index); + while (index != buffer.len) { + const amt = try self.pread(buffer[index..], offset + index); + if (amt == 0) break; + index += amt; } + return index; } pub fn readv(self: File, iovecs: []const os.iovec) ReadError!usize { @@ -280,19 +290,27 @@ pub const File = struct { } } + /// Returns the number of bytes read. If the number read is smaller than the total bytes + /// from all the buffers, it means the file reached the end. Reaching the end of a file + /// is not an error condition. /// The `iovecs` parameter is mutable because this function needs to mutate the fields in /// order to handle partial reads from the underlying OS layer. - pub fn readvAll(self: File, iovecs: []os.iovec) ReadError!void { + pub fn readvAll(self: File, iovecs: []os.iovec) ReadError!usize { if (iovecs.len == 0) return; var i: usize = 0; + var off: usize = 0; while (true) { var amt = try self.readv(iovecs[i..]); + var eof = amt == 0; + off += amt; while (amt >= iovecs[i].iov_len) { amt -= iovecs[i].iov_len; i += 1; - if (i >= iovecs.len) return; + if (i >= iovecs.len) return off; + eof = false; } + if (eof) return off; iovecs[i].iov_base += amt; iovecs[i].iov_len -= amt; } @@ -306,6 +324,9 @@ pub const File = struct { } } + /// Returns the number of bytes read. If the number read is smaller than the total bytes + /// from all the buffers, it means the file reached the end. Reaching the end of a file + /// is not an error condition. /// The `iovecs` parameter is mutable because this function needs to mutate the fields in /// order to handle partial reads from the underlying OS layer. pub fn preadvAll(self: File, iovecs: []const os.iovec, offset: u64) PReadError!void { @@ -315,12 +336,15 @@ pub const File = struct { var off: usize = 0; while (true) { var amt = try self.preadv(iovecs[i..], offset + off); + var eof = amt == 0; off += amt; while (amt >= iovecs[i].iov_len) { amt -= iovecs[i].iov_len; i += 1; - if (i >= iovecs.len) return; + if (i >= iovecs.len) return off; + eof = false; } + if (eof) return off; iovecs[i].iov_base += amt; iovecs[i].iov_len -= amt; } diff --git a/lib/std/io/in_stream.zig b/lib/std/io/in_stream.zig index 2830582482..d1d24ab427 100644 --- a/lib/std/io/in_stream.zig +++ b/lib/std/io/in_stream.zig @@ -28,10 +28,7 @@ pub fn InStream( return readFn(self.context, buffer); } - /// Deprecated: use `readAll`. - pub const readFull = readAll; - - /// Returns the number of bytes read. If the number read is smaller than buf.len, it + /// Returns the number of bytes read. If the number read is smaller than `buffer.len`, it /// means the stream reached the end. Reaching the end of a stream is not an error /// condition. pub fn readAll(self: Self, buffer: []u8) Error!usize { diff --git a/lib/std/os/test.zig b/lib/std/os/test.zig index 75a6d5db91..1ef6798b50 100644 --- a/lib/std/os/test.zig +++ b/lib/std/os/test.zig @@ -95,15 +95,41 @@ test "sendfile" { }, }; - var written_buf: [header1.len + header2.len + 10 + trailer1.len + trailer2.len]u8 = undefined; + var written_buf: [100]u8 = undefined; try dest_file.writeFileAll(src_file, .{ .in_offset = 1, .in_len = 10, .headers_and_trailers = &hdtr, .header_count = 2, }); - try dest_file.preadAll(&written_buf, 0); - expect(mem.eql(u8, &written_buf, "header1\nsecond header\nine1\nsecontrailer1\nsecond trailer\n")); + const amt = try dest_file.preadAll(&written_buf, 0); + expect(mem.eql(u8, written_buf[0..amt], "header1\nsecond header\nine1\nsecontrailer1\nsecond trailer\n")); +} + +test "fs.copyFile" { + const data = "u6wj+JmdF3qHsFPE BUlH2g4gJCmEz0PP"; + const src_file = "tmp_test_copy_file.txt"; + const dest_file = "tmp_test_copy_file2.txt"; + const dest_file2 = "tmp_test_copy_file3.txt"; + + try fs.cwd().writeFile(src_file, data); + defer fs.cwd().deleteFile(src_file) catch {}; + + try fs.copyFile(src_file, dest_file); + defer fs.cwd().deleteFile(dest_file) catch {}; + + try fs.copyFileMode(src_file, dest_file2, File.default_mode); + defer fs.cwd().deleteFile(dest_file2) catch {}; + + try expectFileContents(dest_file, data); + try expectFileContents(dest_file2, data); +} + +fn expectFileContents(file_path: []const u8, data: []const u8) !void { + const contents = try fs.cwd().readFileAlloc(testing.allocator, file_path, 1000); + defer testing.allocator.free(contents); + + testing.expectEqualSlices(u8, data, contents); } test "std.Thread.getCurrentId" { From c71991c869fc8c997c2714fe1b53caef23449c9c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 11 Mar 2020 14:34:13 -0400 Subject: [PATCH 11/15] fix compilation errors for emitRaw --- lib/std/build/emit_raw.zig | 2 +- lib/std/elf.zig | 33 ++++++++++++++++++++++++++------- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/lib/std/build/emit_raw.zig b/lib/std/build/emit_raw.zig index 63c2329947..b56305403c 100644 --- a/lib/std/build/emit_raw.zig +++ b/lib/std/build/emit_raw.zig @@ -164,7 +164,7 @@ fn emitRaw(allocator: *Allocator, elf_path: []const u8, raw_path: []const u8) !v var out_file = try fs.cwd().createFile(raw_path, .{}); defer out_file.close(); - const binary_elf_output = BinaryElfOutput.parse(allocator, elf_file); + const binary_elf_output = try BinaryElfOutput.parse(allocator, elf_file); defer binary_elf_output.deinit(); for (binary_elf_output.sections.toSlice()) |section| { diff --git a/lib/std/elf.zig b/lib/std/elf.zig index 3aefd87f09..5cb3c566af 100644 --- a/lib/std/elf.zig +++ b/lib/std/elf.zig @@ -346,13 +346,13 @@ const Header = struct { pub fn readHeader(file: File) !Header { var hdr_buf: [@sizeOf(Elf64_Ehdr)]u8 align(@alignOf(Elf64_Ehdr)) = undefined; - try in_stream.preadAll(&hdr_buf, 0); - const hdr32 = @ptrCast(*elf.Elf32_Ehdr, &hdr_buf); - const hdr64 = @ptrCast(*elf.Elf64_Ehdr, &hdr_buf); + try preadNoEof(file, &hdr_buf, 0); + const hdr32 = @ptrCast(*Elf32_Ehdr, &hdr_buf); + const hdr64 = @ptrCast(*Elf64_Ehdr, &hdr_buf); if (!mem.eql(u8, hdr32.e_ident[0..4], "\x7fELF")) return error.InvalidElfMagic; if (hdr32.e_ident[EI_VERSION] != 1) return error.InvalidElfVersion; - const endian = switch (hdr32.e_ident[elf.EI_DATA]) { + const endian = switch (hdr32.e_ident[EI_DATA]) { ELFDATA2LSB => .Little, ELFDATA2MSB => .Big, else => return error.InvalidElfEndian, @@ -407,10 +407,10 @@ pub fn readAllHeaders(allocator: *mem.Allocator, file: File) !AllHeaders { // non-matching endian files, we post-process to correct integer endianness and offsets. const shdr_buf = std.mem.sliceToBytes(hdrs.section_headers)[0 .. hdrs.header.shentsize * hdrs.header.shnum]; - const phdr_buf = std.mem.sliceToBytes(hdrs.section_headers)[0 .. hdrs.header.phentsize * hdrs.header.phnum]; + const phdr_buf = std.mem.sliceToBytes(hdrs.program_headers)[0 .. hdrs.header.phentsize * hdrs.header.phnum]; - try file.preadAll(phdr_buf, hdrs.header.phoff); - try file.preadAll(shdr_buf, hdrs.header.shoff); + try preadNoEof(file, shdr_buf, hdrs.header.shoff); + try preadNoEof(file, phdr_buf, hdrs.header.phoff); const shdrs32 = @ptrCast([*]Elf32_Shdr, @alignCast(@alignOf(Elf32_Shdr), shdr_buf.ptr))[0..hdrs.header.shnum]; const phdrs32 = @ptrCast([*]Elf32_Phdr, @alignCast(@alignOf(Elf32_Phdr), phdr_buf.ptr))[0..hdrs.header.phnum]; @@ -459,6 +459,25 @@ pub fn int(is_64: bool, need_bswap: bool, int_32: var, int_64: var) @TypeOf(int_ } } +fn preadNoEof(file: std.fs.File, buf: []u8, offset: u64) !void { + var i: u64 = 0; + while (i < buf.len) { + const len = file.pread(buf[i .. buf.len - i], offset + i) catch |err| switch (err) { + error.SystemResources => return error.SystemResources, + error.IsDir => return error.UnableToReadElfFile, + error.OperationAborted => return error.UnableToReadElfFile, + error.BrokenPipe => return error.UnableToReadElfFile, + error.Unseekable => return error.UnableToReadElfFile, + error.ConnectionResetByPeer => return error.UnableToReadElfFile, + error.InputOutput => return error.FileSystem, + error.Unexpected => return error.Unexpected, + error.WouldBlock => return error.Unexpected, + }; + if (len == 0) return error.UnexpectedEndOfFile; + i += len; + } +} + pub const EI_NIDENT = 16; pub const EI_CLASS = 4; From 431d76c0233271de6a30aa626ccb1553dc241722 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 11 Mar 2020 15:40:34 -0400 Subject: [PATCH 12/15] add std.io.StreamSource and fixes to emitRaw --- lib/std/build/emit_raw.zig | 4 +- lib/std/elf.zig | 24 ++++---- lib/std/io.zig | 2 + lib/std/io/fixed_buffer_stream.zig | 41 +++++++------- lib/std/io/stream_source.zig | 90 ++++++++++++++++++++++++++++++ 5 files changed, 127 insertions(+), 34 deletions(-) create mode 100644 lib/std/io/stream_source.zig diff --git a/lib/std/build/emit_raw.zig b/lib/std/build/emit_raw.zig index b56305403c..8b4747e8a7 100644 --- a/lib/std/build/emit_raw.zig +++ b/lib/std/build/emit_raw.zig @@ -48,8 +48,6 @@ const BinaryElfOutput = struct { }; const elf_hdrs = try std.elf.readAllHeaders(allocator, elf_file); - var binaryElfOutput = BinaryElfOutput.init(arena_allocator); - for (elf_hdrs.section_headers) |section, i| { if (sectionValidForOutput(section)) { const newSection = try allocator.create(BinaryElfSection); @@ -164,7 +162,7 @@ fn emitRaw(allocator: *Allocator, elf_path: []const u8, raw_path: []const u8) !v var out_file = try fs.cwd().createFile(raw_path, .{}); defer out_file.close(); - const binary_elf_output = try BinaryElfOutput.parse(allocator, elf_file); + var binary_elf_output = try BinaryElfOutput.parse(allocator, elf_file); defer binary_elf_output.deinit(); for (binary_elf_output.sections.toSlice()) |section| { diff --git a/lib/std/elf.zig b/lib/std/elf.zig index 5cb3c566af..127dbe8a31 100644 --- a/lib/std/elf.zig +++ b/lib/std/elf.zig @@ -1,5 +1,5 @@ -const builtin = @import("builtin"); const std = @import("std.zig"); +const builtin = std.builtin; const io = std.io; const os = std.os; const math = std.math; @@ -352,7 +352,7 @@ pub fn readHeader(file: File) !Header { if (!mem.eql(u8, hdr32.e_ident[0..4], "\x7fELF")) return error.InvalidElfMagic; if (hdr32.e_ident[EI_VERSION] != 1) return error.InvalidElfVersion; - const endian = switch (hdr32.e_ident[EI_DATA]) { + const endian: std.builtin.Endian = switch (hdr32.e_ident[EI_DATA]) { ELFDATA2LSB => .Little, ELFDATA2MSB => .Big, else => return error.InvalidElfEndian, @@ -406,8 +406,8 @@ pub fn readAllHeaders(allocator: *mem.Allocator, file: File) !AllHeaders { // Treat section headers and program headers as byte buffers. For 32-bit ELF and // non-matching endian files, we post-process to correct integer endianness and offsets. - const shdr_buf = std.mem.sliceToBytes(hdrs.section_headers)[0 .. hdrs.header.shentsize * hdrs.header.shnum]; - const phdr_buf = std.mem.sliceToBytes(hdrs.program_headers)[0 .. hdrs.header.phentsize * hdrs.header.phnum]; + const shdr_buf = std.mem.sliceAsBytes(hdrs.section_headers)[0 .. hdrs.header.shentsize * hdrs.header.shnum]; + const phdr_buf = std.mem.sliceAsBytes(hdrs.program_headers)[0 .. hdrs.header.phentsize * hdrs.header.phnum]; try preadNoEof(file, shdr_buf, hdrs.header.shoff); try preadNoEof(file, phdr_buf, hdrs.header.phoff); @@ -430,14 +430,14 @@ pub fn readAllHeaders(allocator: *mem.Allocator, file: File) !AllHeaders { } for (hdrs.program_headers) |*phdr, i| { phdr.* = .{ - .p_type = int(is_64, need_bswap, phdrs32[i].p_type, shdr.p_type), - .p_offset = int(is_64, need_bswap, phdrs32[i].p_offset, shdr.p_offset), - .p_vaddr = int(is_64, need_bswap, phdrs32[i].p_vaddr, shdr.p_vaddr), - .p_paddr = int(is_64, need_bswap, phdrs32[i].p_paddr, shdr.p_paddr), - .p_filesz = int(is_64, need_bswap, phdrs32[i].p_filesz, shdr.p_filesz), - .p_memsz = int(is_64, need_bswap, phdrs32[i].p_memsz, shdr.p_memsz), - .p_flags = int(is_64, need_bswap, phdrs32[i].p_flags, shdr.p_flags), - .p_align = int(is_64, need_bswap, phdrs32[i].p_align, shdr.p_align), + .p_type = int(is_64, need_bswap, phdrs32[i].p_type, phdr.p_type), + .p_offset = int(is_64, need_bswap, phdrs32[i].p_offset, phdr.p_offset), + .p_vaddr = int(is_64, need_bswap, phdrs32[i].p_vaddr, phdr.p_vaddr), + .p_paddr = int(is_64, need_bswap, phdrs32[i].p_paddr, phdr.p_paddr), + .p_filesz = int(is_64, need_bswap, phdrs32[i].p_filesz, phdr.p_filesz), + .p_memsz = int(is_64, need_bswap, phdrs32[i].p_memsz, phdr.p_memsz), + .p_flags = int(is_64, need_bswap, phdrs32[i].p_flags, phdr.p_flags), + .p_align = int(is_64, need_bswap, phdrs32[i].p_align, phdr.p_align), }; } return hdrs; diff --git a/lib/std/io.zig b/lib/std/io.zig index 22af5c06b1..243dd5ca95 100644 --- a/lib/std/io.zig +++ b/lib/std/io.zig @@ -126,6 +126,8 @@ pub const deserializer = @import("io/serialization.zig").deserializer; pub const BufferedAtomicFile = @import("io/buffered_atomic_file.zig").BufferedAtomicFile; +pub const StreamSource = @import("io/stream_source.zig").StreamSource; + /// Deprecated; use `std.fs.Dir.writeFile`. pub fn writeFile(path: []const u8, data: []const u8) !void { return fs.cwd().writeFile(path, data); diff --git a/lib/std/io/fixed_buffer_stream.zig b/lib/std/io/fixed_buffer_stream.zig index 0f2b542201..cc4b9c0193 100644 --- a/lib/std/io/fixed_buffer_stream.zig +++ b/lib/std/io/fixed_buffer_stream.zig @@ -12,9 +12,9 @@ pub fn FixedBufferStream(comptime Buffer: type) type { buffer: Buffer, pos: usize, - pub const ReadError = error{EndOfStream}; - pub const WriteError = error{OutOfMemory}; - pub const SeekError = error{EndOfStream}; + pub const ReadError = error{}; + pub const WriteError = error{NoSpaceLeft}; + pub const SeekError = error{}; pub const GetSeekPosError = error{}; pub const InStream = io.InStream(*Self, ReadError, read); @@ -51,16 +51,16 @@ pub fn FixedBufferStream(comptime Buffer: type) type { mem.copy(u8, dest[0..size], self.buffer[self.pos..end]); self.pos = end; - if (size == 0) return error.EndOfStream; return size; } /// If the returned number of bytes written is less than requested, the - /// buffer is full. Returns `error.OutOfMemory` when no bytes would be written. + /// buffer is full. Returns `error.NoSpaceLeft` when no bytes would be written. + /// Note: `error.NoSpaceLeft` matches the corresponding error from + /// `std.fs.File.WriteError`. pub fn write(self: *Self, bytes: []const u8) WriteError!usize { if (bytes.len == 0) return 0; - - assert(self.pos <= self.buffer.len); + if (self.pos >= self.buffer.len) return error.NoSpaceLeft; const n = if (self.pos + bytes.len <= self.buffer.len) bytes.len @@ -70,26 +70,27 @@ pub fn FixedBufferStream(comptime Buffer: type) type { mem.copy(u8, self.buffer[self.pos .. self.pos + n], bytes[0..n]); self.pos += n; - if (n == 0) return error.OutOfMemory; + if (n == 0) return error.NoSpaceLeft; return n; } pub fn seekTo(self: *Self, pos: u64) SeekError!void { - const usize_pos = std.math.cast(usize, pos) catch return error.EndOfStream; - if (usize_pos > self.buffer.len) return error.EndOfStream; + const usize_pos = std.math.cast(usize, pos) catch std.math.maxInt(usize); self.pos = usize_pos; } pub fn seekBy(self: *Self, amt: i64) SeekError!void { if (amt < 0) { - const abs_amt = std.math.cast(usize, -amt) catch return error.EndOfStream; - if (abs_amt > self.pos) return error.EndOfStream; - self.pos -= abs_amt; + const abs_amt = std.math.cast(usize, -amt) catch std.math.maxInt(usize); + if (abs_amt > self.pos) { + self.pos = 0; + } else { + self.pos -= abs_amt; + } } else { - const usize_amt = std.math.cast(usize, amt) catch return error.EndOfStream; - if (self.pos + usize_amt > self.buffer.len) return error.EndOfStream; - self.pos += usize_amt; + const usize_amt = std.math.cast(usize, amt) catch std.math.maxInt(usize); + self.pos = std.math.add(usize, self.pos, usize_amt) catch std.math.maxInt(usize); } } @@ -101,6 +102,7 @@ pub fn FixedBufferStream(comptime Buffer: type) type { return self.pos; } + /// Asserts that the seek pos is within the buffer range. pub fn getWritten(self: Self) []const u8 { return self.buffer[0..self.pos]; } @@ -140,13 +142,13 @@ test "FixedBufferStream output 2" { try fbs.outStream().writeAll("world"); testing.expect(mem.eql(u8, fbs.getWritten(), "Helloworld")); - testing.expectError(error.OutOfMemory, fbs.outStream().writeAll("!")); + testing.expectError(error.NoSpaceLeft, fbs.outStream().writeAll("!")); testing.expect(mem.eql(u8, fbs.getWritten(), "Helloworld")); fbs.reset(); testing.expect(fbs.getWritten().len == 0); - testing.expectError(error.OutOfMemory, fbs.outStream().writeAll("Hello world!")); + testing.expectError(error.NoSpaceLeft, fbs.outStream().writeAll("Hello world!")); testing.expect(mem.eql(u8, fbs.getWritten(), "Hello worl")); } @@ -164,5 +166,6 @@ test "FixedBufferStream input" { testing.expect(read == 3); testing.expect(mem.eql(u8, dest[0..3], bytes[4..7])); - testing.expectError(error.EndOfStream, fbs.inStream().read(dest[0..4])); + read = try fbs.inStream().read(dest[0..4]); + testing.expect(read == 0); } diff --git a/lib/std/io/stream_source.zig b/lib/std/io/stream_source.zig new file mode 100644 index 0000000000..7e39554337 --- /dev/null +++ b/lib/std/io/stream_source.zig @@ -0,0 +1,90 @@ +const std = @import("../std.zig"); +const io = std.io; +const testing = std.testing; + +/// Provides `io.InStream`, `io.OutStream`, and `io.SeekableStream` for in-memory buffers as +/// well as files. +/// For memory sources, if the supplied byte buffer is const, then `io.OutStream` is not available. +/// The error set of the stream functions is the error set of the corresponding file functions. +pub const StreamSource = union(enum) { + buffer: io.FixedBufferStream([]u8), + const_buffer: io.FixedBufferStream([]const u8), + file: std.fs.File, + + pub const ReadError = std.fs.File.ReadError; + pub const WriteError = std.fs.File.WriteError; + pub const SeekError = std.fs.File.SeekError; + pub const GetSeekPosError = std.fs.File.GetPosError; + + pub const InStream = io.InStream(*StreamSource, ReadError, read); + pub const OutStream = io.OutStream(*StreamSource, WriteError, write); + pub const SeekableStream = io.SeekableStream( + *StreamSource, + SeekError, + GetSeekPosError, + seekTo, + seekBy, + getPos, + getEndPos, + ); + + pub fn read(self: *StreamSource, dest: []u8) ReadError!usize { + switch (self.*) { + .buffer => |*x| return x.read(dest), + .const_buffer => |*x| return x.read(dest), + .file => |x| return x.read(dest), + } + } + + pub fn write(self: *StreamSource, bytes: []const u8) WriteError!usize { + switch (self.*) { + .buffer => |*x| return x.write(bytes), + .const_buffer => |*x| return x.write(bytes), + .file => |x| return x.write(bytes), + } + } + + pub fn seekTo(self: *StreamSource, pos: u64) SeekError!void { + switch (self.*) { + .buffer => |*x| return x.seekTo(pos), + .const_buffer => |*x| return x.seekTo(pos), + .file => |x| return x.seekTo(pos), + } + } + + pub fn seekBy(self: *StreamSource, amt: i64) SeekError!void { + switch (self.*) { + .buffer => |*x| return x.seekBy(amt), + .const_buffer => |*x| return x.seekBy(amt), + .file => |x| return x.seekBy(amt), + } + } + + pub fn getEndPos(self: *StreamSource) GetSeekPosError!u64 { + switch (self.*) { + .buffer => |*x| return x.getEndPos(), + .const_buffer => |*x| return x.getEndPos(), + .file => |x| return x.getEndPos(), + } + } + + pub fn getPos(self: *StreamSource) GetSeekPosError!u64 { + switch (self.*) { + .buffer => |*x| return x.getPos(), + .const_buffer => |*x| return x.getPos(), + .file => |x| return x.getPos(), + } + } + + pub fn inStream(self: *StreamSource) InStream { + return .{ .context = self }; + } + + pub fn outStream(self: *StreamSource) OutStream { + return .{ .context = self }; + } + + pub fn seekableStream(self: *StreamSource) SeekableStream { + return .{ .context = self }; + } +}; From 6892865ba72ea220a12a8238ce78aedab1237cce Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 11 Mar 2020 16:14:12 -0400 Subject: [PATCH 13/15] FixedBufferStream: match file semantics more by clamping pos --- lib/std/io/fixed_buffer_stream.zig | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/std/io/fixed_buffer_stream.zig b/lib/std/io/fixed_buffer_stream.zig index cc4b9c0193..0d091eea72 100644 --- a/lib/std/io/fixed_buffer_stream.zig +++ b/lib/std/io/fixed_buffer_stream.zig @@ -76,21 +76,22 @@ pub fn FixedBufferStream(comptime Buffer: type) type { } pub fn seekTo(self: *Self, pos: u64) SeekError!void { - const usize_pos = std.math.cast(usize, pos) catch std.math.maxInt(usize); - self.pos = usize_pos; + self.pos = if (std.math.cast(usize, pos)) |x| x else |_| self.buffer.len; } pub fn seekBy(self: *Self, amt: i64) SeekError!void { if (amt < 0) { - const abs_amt = std.math.cast(usize, -amt) catch std.math.maxInt(usize); - if (abs_amt > self.pos) { + const abs_amt = std.math.absCast(amt); + const abs_amt_usize = std.math.cast(usize, abs_amt) catch std.math.maxInt(usize); + if (abs_amt_usize > self.pos) { self.pos = 0; } else { - self.pos -= abs_amt; + self.pos -= abs_amt_usize; } } else { - const usize_amt = std.math.cast(usize, amt) catch std.math.maxInt(usize); - self.pos = std.math.add(usize, self.pos, usize_amt) catch std.math.maxInt(usize); + const amt_usize = std.math.cast(usize, amt) catch std.math.maxInt(usize); + const new_pos = std.math.add(usize, self.pos, amt_usize) catch std.math.maxInt(usize); + self.pos = std.math.min(self.buffer.len, new_pos); } } @@ -102,7 +103,6 @@ pub fn FixedBufferStream(comptime Buffer: type) type { return self.pos; } - /// Asserts that the seek pos is within the buffer range. pub fn getWritten(self: Self) []const u8 { return self.buffer[0..self.pos]; } From 3657a54618dafa110fd1d16e02c48e04871c9370 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 11 Mar 2020 17:34:12 -0400 Subject: [PATCH 14/15] fix regressions in elf parsing code --- lib/std/elf.zig | 155 +++++++++++++++++++++++++++++++---------- lib/std/zig/system.zig | 17 ++++- 2 files changed, 133 insertions(+), 39 deletions(-) diff --git a/lib/std/elf.zig b/lib/std/elf.zig index 127dbe8a31..7b415f675a 100644 --- a/lib/std/elf.zig +++ b/lib/std/elf.zig @@ -398,48 +398,123 @@ pub fn readAllHeaders(allocator: *mem.Allocator, file: File) !AllHeaders { const need_bswap = hdrs.header.endian != std.builtin.endian; hdrs.section_headers = try allocator.alloc(Elf64_Shdr, hdrs.header.shnum); - errdefer hdrs.allocator.free(hdrs.section_headers); + errdefer allocator.free(hdrs.section_headers); hdrs.program_headers = try allocator.alloc(Elf64_Phdr, hdrs.header.phnum); - errdefer hdrs.allocator.free(hdrs.program_headers); + errdefer allocator.free(hdrs.program_headers); - // Treat section headers and program headers as byte buffers. For 32-bit ELF and - // non-matching endian files, we post-process to correct integer endianness and offsets. + // If the ELF file is 64-bit and same-endianness, then all we have to do is + // yeet the bytes into memory. + // If only the endianness is different, they can be simply byte swapped. + if (is_64) { + const shdr_buf = std.mem.sliceAsBytes(hdrs.section_headers); + const phdr_buf = std.mem.sliceAsBytes(hdrs.program_headers); + try preadNoEof(file, shdr_buf, hdrs.header.shoff); + try preadNoEof(file, phdr_buf, hdrs.header.phoff); - const shdr_buf = std.mem.sliceAsBytes(hdrs.section_headers)[0 .. hdrs.header.shentsize * hdrs.header.shnum]; - const phdr_buf = std.mem.sliceAsBytes(hdrs.program_headers)[0 .. hdrs.header.phentsize * hdrs.header.phnum]; + if (need_bswap) { + for (hdrs.section_headers) |*shdr| { + shdr.* = .{ + .sh_name = @byteSwap(@TypeOf(shdr.sh_name), shdr.sh_name), + .sh_type = @byteSwap(@TypeOf(shdr.sh_type), shdr.sh_type), + .sh_flags = @byteSwap(@TypeOf(shdr.sh_flags), shdr.sh_flags), + .sh_addr = @byteSwap(@TypeOf(shdr.sh_addr), shdr.sh_addr), + .sh_offset = @byteSwap(@TypeOf(shdr.sh_offset), shdr.sh_offset), + .sh_size = @byteSwap(@TypeOf(shdr.sh_size), shdr.sh_size), + .sh_link = @byteSwap(@TypeOf(shdr.sh_link), shdr.sh_link), + .sh_info = @byteSwap(@TypeOf(shdr.sh_info), shdr.sh_info), + .sh_addralign = @byteSwap(@TypeOf(shdr.sh_addralign), shdr.sh_addralign), + .sh_entsize = @byteSwap(@TypeOf(shdr.sh_entsize), shdr.sh_entsize), + }; + } + for (hdrs.program_headers) |*phdr| { + phdr.* = .{ + .p_type = @byteSwap(@TypeOf(phdr.p_type), phdr.p_type), + .p_offset = @byteSwap(@TypeOf(phdr.p_offset), phdr.p_offset), + .p_vaddr = @byteSwap(@TypeOf(phdr.p_vaddr), phdr.p_vaddr), + .p_paddr = @byteSwap(@TypeOf(phdr.p_paddr), phdr.p_paddr), + .p_filesz = @byteSwap(@TypeOf(phdr.p_filesz), phdr.p_filesz), + .p_memsz = @byteSwap(@TypeOf(phdr.p_memsz), phdr.p_memsz), + .p_flags = @byteSwap(@TypeOf(phdr.p_flags), phdr.p_flags), + .p_align = @byteSwap(@TypeOf(phdr.p_align), phdr.p_align), + }; + } + } + return hdrs; + } + + const shdrs_32 = try allocator.alloc(Elf32_Shdr, hdrs.header.shnum); + defer allocator.free(shdrs_32); + + const phdrs_32 = try allocator.alloc(Elf32_Phdr, hdrs.header.phnum); + defer allocator.free(phdrs_32); + + const shdr_buf = std.mem.sliceAsBytes(shdrs_32); + const phdr_buf = std.mem.sliceAsBytes(phdrs_32); try preadNoEof(file, shdr_buf, hdrs.header.shoff); try preadNoEof(file, phdr_buf, hdrs.header.phoff); - const shdrs32 = @ptrCast([*]Elf32_Shdr, @alignCast(@alignOf(Elf32_Shdr), shdr_buf.ptr))[0..hdrs.header.shnum]; - const phdrs32 = @ptrCast([*]Elf32_Phdr, @alignCast(@alignOf(Elf32_Phdr), phdr_buf.ptr))[0..hdrs.header.phnum]; - for (hdrs.section_headers) |*shdr, i| { - shdr.* = .{ - .sh_name = int(is_64, need_bswap, shdrs32[i].sh_name, shdr.sh_name), - .sh_type = int(is_64, need_bswap, shdrs32[i].sh_type, shdr.sh_type), - .sh_flags = int(is_64, need_bswap, shdrs32[i].sh_flags, shdr.sh_flags), - .sh_addr = int(is_64, need_bswap, shdrs32[i].sh_addr, shdr.sh_addr), - .sh_offset = int(is_64, need_bswap, shdrs32[i].sh_offset, shdr.sh_offset), - .sh_size = int(is_64, need_bswap, shdrs32[i].sh_size, shdr.sh_size), - .sh_link = int(is_64, need_bswap, shdrs32[i].sh_link, shdr.sh_link), - .sh_info = int(is_64, need_bswap, shdrs32[i].sh_info, shdr.sh_info), - .sh_addralign = int(is_64, need_bswap, shdrs32[i].sh_addralign, shdr.sh_addralign), - .sh_entsize = int(is_64, need_bswap, shdrs32[i].sh_entsize, shdr.sh_entsize), - }; - } - for (hdrs.program_headers) |*phdr, i| { - phdr.* = .{ - .p_type = int(is_64, need_bswap, phdrs32[i].p_type, phdr.p_type), - .p_offset = int(is_64, need_bswap, phdrs32[i].p_offset, phdr.p_offset), - .p_vaddr = int(is_64, need_bswap, phdrs32[i].p_vaddr, phdr.p_vaddr), - .p_paddr = int(is_64, need_bswap, phdrs32[i].p_paddr, phdr.p_paddr), - .p_filesz = int(is_64, need_bswap, phdrs32[i].p_filesz, phdr.p_filesz), - .p_memsz = int(is_64, need_bswap, phdrs32[i].p_memsz, phdr.p_memsz), - .p_flags = int(is_64, need_bswap, phdrs32[i].p_flags, phdr.p_flags), - .p_align = int(is_64, need_bswap, phdrs32[i].p_align, phdr.p_align), - }; + if (need_bswap) { + for (hdrs.section_headers) |*shdr, i| { + const o = shdrs_32[i]; + shdr.* = .{ + .sh_name = @byteSwap(@TypeOf(o.sh_name), o.sh_name), + .sh_type = @byteSwap(@TypeOf(o.sh_type), o.sh_type), + .sh_flags = @byteSwap(@TypeOf(o.sh_flags), o.sh_flags), + .sh_addr = @byteSwap(@TypeOf(o.sh_addr), o.sh_addr), + .sh_offset = @byteSwap(@TypeOf(o.sh_offset), o.sh_offset), + .sh_size = @byteSwap(@TypeOf(o.sh_size), o.sh_size), + .sh_link = @byteSwap(@TypeOf(o.sh_link), o.sh_link), + .sh_info = @byteSwap(@TypeOf(o.sh_info), o.sh_info), + .sh_addralign = @byteSwap(@TypeOf(o.sh_addralign), o.sh_addralign), + .sh_entsize = @byteSwap(@TypeOf(o.sh_entsize), o.sh_entsize), + }; + } + for (hdrs.program_headers) |*phdr, i| { + const o = phdrs_32[i]; + phdr.* = .{ + .p_type = @byteSwap(@TypeOf(o.p_type), o.p_type), + .p_offset = @byteSwap(@TypeOf(o.p_offset), o.p_offset), + .p_vaddr = @byteSwap(@TypeOf(o.p_vaddr), o.p_vaddr), + .p_paddr = @byteSwap(@TypeOf(o.p_paddr), o.p_paddr), + .p_filesz = @byteSwap(@TypeOf(o.p_filesz), o.p_filesz), + .p_memsz = @byteSwap(@TypeOf(o.p_memsz), o.p_memsz), + .p_flags = @byteSwap(@TypeOf(o.p_flags), o.p_flags), + .p_align = @byteSwap(@TypeOf(o.p_align), o.p_align), + }; + } + } else { + for (hdrs.section_headers) |*shdr, i| { + const o = shdrs_32[i]; + shdr.* = .{ + .sh_name = o.sh_name, + .sh_type = o.sh_type, + .sh_flags = o.sh_flags, + .sh_addr = o.sh_addr, + .sh_offset = o.sh_offset, + .sh_size = o.sh_size, + .sh_link = o.sh_link, + .sh_info = o.sh_info, + .sh_addralign = o.sh_addralign, + .sh_entsize = o.sh_entsize, + }; + } + for (hdrs.program_headers) |*phdr, i| { + const o = phdrs_32[i]; + phdr.* = .{ + .p_type = o.p_type, + .p_offset = o.p_offset, + .p_vaddr = o.p_vaddr, + .p_paddr = o.p_paddr, + .p_filesz = o.p_filesz, + .p_memsz = o.p_memsz, + .p_flags = o.p_flags, + .p_align = o.p_align, + }; + } } + return hdrs; } @@ -451,11 +526,15 @@ pub fn int(is_64: bool, need_bswap: bool, int_32: var, int_64: var) @TypeOf(int_ return int_64; } } else { - if (need_bswap) { - return @byteSwap(@TypeOf(int_32), int_32); - } else { - return int_32; - } + return int32(need_bswap, int_32, @TypeOf(int_64)); + } +} + +pub fn int32(need_bswap: bool, int_32: var, comptime Int64: var) Int64 { + if (need_bswap) { + return @byteSwap(@TypeOf(int_32), int_32); + } else { + return int_32; } } diff --git a/lib/std/zig/system.zig b/lib/std/zig/system.zig index e7953a1a91..558b50b5b3 100644 --- a/lib/std/zig/system.zig +++ b/lib/std/zig/system.zig @@ -587,7 +587,6 @@ pub const NativeTargetInfo = struct { elf.ELFCLASS64 => true, else => return error.InvalidElfClass, }; - const elfInt = elf.int; var phoff = elfInt(is_64, need_bswap, hdr32.e_phoff, hdr64.e_phoff); const phentsize = elfInt(is_64, need_bswap, hdr32.e_phentsize, hdr64.e_phentsize); const phnum = elfInt(is_64, need_bswap, hdr32.e_phnum, hdr64.e_phnum); @@ -854,6 +853,22 @@ pub const NativeTargetInfo = struct { abi: Target.Abi, }; + pub fn elfInt(is_64: bool, need_bswap: bool, int_32: var, int_64: var) @TypeOf(int_64) { + if (is_64) { + if (need_bswap) { + return @byteSwap(@TypeOf(int_64), int_64); + } else { + return int_64; + } + } else { + if (need_bswap) { + return @byteSwap(@TypeOf(int_32), int_32); + } else { + return int_32; + } + } + } + fn detectNativeCpuAndFeatures(cpu_arch: Target.Cpu.Arch, os: Target.Os, cross_target: CrossTarget) ?Target.Cpu { // Here we switch on a comptime value rather than `cpu_arch`. This is valid because `cpu_arch`, // although it is a runtime value, is guaranteed to be one of the architectures in the set From 06d2f53ece7328e6beedd5c846a5b25798ba74e3 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 11 Mar 2020 17:39:53 -0400 Subject: [PATCH 15/15] windows: detect HANDLE_EOF in ReadFile --- lib/std/os/windows.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig index d1201108dc..6c9a1f24b7 100644 --- a/lib/std/os/windows.zig +++ b/lib/std/os/windows.zig @@ -407,6 +407,7 @@ pub fn ReadFile(in_hFile: HANDLE, buffer: []u8, offset: ?u64) ReadFileError!usiz switch (kernel32.GetLastError()) { .OPERATION_ABORTED => continue, .BROKEN_PIPE => return index, + .HANDLE_EOF => return index, else => |err| return unexpectedError(err), } }