From bdff2f43bdc112cb02e36678d079e82fd5a96605 Mon Sep 17 00:00:00 2001 From: daurnimator Date: Mon, 11 Nov 2019 02:49:35 +1100 Subject: [PATCH 1/5] std: use LinearFifo to implement io.BufferedInStreamCustom --- lib/std/io.zig | 67 +++++++++++--------------------------------------- 1 file changed, 14 insertions(+), 53 deletions(-) diff --git a/lib/std/io.zig b/lib/std/io.zig index 16bfffdcad..7baa383001 100644 --- a/lib/std/io.zig +++ b/lib/std/io.zig @@ -121,76 +121,37 @@ pub fn BufferedInStreamCustom(comptime buffer_size: usize, comptime Error: type) unbuffered_in_stream: *Stream, - buffer: [buffer_size]u8, - start_index: usize, - end_index: usize, + const FifoType = std.fifo.LinearFifo(u8, std.fifo.LinearFifoBufferType{ .Static = buffer_size }); + fifo: FifoType, pub fn init(unbuffered_in_stream: *Stream) Self { return Self{ .unbuffered_in_stream = unbuffered_in_stream, - .buffer = undefined, - - // Initialize these two fields to buffer_size so that - // in `readFn` we treat the state as being able to read - // more from the unbuffered stream. If we set them to 0 - // and 0, the code would think we already hit EOF. - .start_index = buffer_size, - .end_index = buffer_size, - + .fifo = FifoType.init(), .stream = Stream{ .readFn = readFn }, }; } fn readFn(in_stream: *Stream, dest: []u8) !usize { const self = @fieldParentPtr(Self, "stream", in_stream); - - // Hot path for one byte reads - if (dest.len == 1 and self.end_index > self.start_index) { - dest[0] = self.buffer[self.start_index]; - self.start_index += 1; - return 1; - } - var dest_index: usize = 0; - while (true) { - const dest_space = dest.len - dest_index; - if (dest_space == 0) { - return dest_index; - } - const amt_buffered = self.end_index - self.start_index; - if (amt_buffered == 0) { - assert(self.end_index <= buffer_size); - // Make sure the last read actually gave us some data - if (self.end_index == 0) { + while (dest_index < dest.len) { + const written = self.fifo.read(dest[dest_index..]); + if (written == 0) { + // fifo empty, fill it + const writable = self.fifo.writableSlice(0); + assert(writable.len > 0); + const n = try self.unbuffered_in_stream.read(writable); + if (n == 0) { // reading from the unbuffered stream returned nothing // so we have nothing left to read. return dest_index; } - // we can read more data from the unbuffered stream - if (dest_space < buffer_size) { - self.start_index = 0; - self.end_index = try self.unbuffered_in_stream.read(self.buffer[0..]); - - // Shortcut - if (self.end_index >= dest_space) { - mem.copy(u8, dest[dest_index..], self.buffer[0..dest_space]); - self.start_index = dest_space; - return dest.len; - } - } else { - // asking for so much data that buffering is actually less efficient. - // forward the request directly to the unbuffered stream - const amt_read = try self.unbuffered_in_stream.read(dest[dest_index..]); - return dest_index + amt_read; - } + self.fifo.update(n); } - - const copy_amount = math.min(dest_space, amt_buffered); - const copy_end_index = self.start_index + copy_amount; - mem.copy(u8, dest[dest_index..], self.buffer[self.start_index..copy_end_index]); - self.start_index = copy_end_index; - dest_index += copy_amount; + dest_index += written; } + return dest.len; } }; } From 38ad7daebb42d562946c092a59cfbbf94e75870a Mon Sep 17 00:00:00 2001 From: daurnimator Date: Mon, 11 Nov 2019 03:07:57 +1100 Subject: [PATCH 2/5] std: use LinearFifo to implement io.PeekStream --- lib/std/io.zig | 46 +++++++++++++-------------------------------- lib/std/io/test.zig | 8 ++++---- 2 files changed, 17 insertions(+), 37 deletions(-) diff --git a/lib/std/io.zig b/lib/std/io.zig index 7baa383001..018c863bbd 100644 --- a/lib/std/io.zig +++ b/lib/std/io.zig @@ -196,7 +196,7 @@ test "io.BufferedInStream" { /// Creates a stream which supports 'un-reading' data, so that it can be read again. /// This makes look-ahead style parsing much easier. -pub fn PeekStream(comptime buffer_size: usize, comptime InStreamError: type) type { +pub fn PeekStream(comptime buffer_type: usize, comptime InStreamError: type) type { return struct { const Self = @This(); pub const Error = InStreamError; @@ -207,55 +207,35 @@ pub fn PeekStream(comptime buffer_size: usize, comptime InStreamError: type) typ // Right now the look-ahead space is statically allocated, but a version with dynamic allocation // is not too difficult to derive from this. - buffer: [buffer_size]u8, - index: usize, - at_end: bool, + const FifoType = std.fifo.LinearFifo(u8, .{ .Static = buffer_size }); + fifo: FifoType, pub fn init(base: *Stream) Self { - return Self{ + return .{ .base = base, - .buffer = undefined, - .index = 0, - .at_end = false, + .fifo = FifoType.init(), .stream = Stream{ .readFn = readFn }, }; } - pub fn putBackByte(self: *Self, byte: u8) void { - self.buffer[self.index] = byte; - self.index += 1; + pub fn putBackByte(self: *Self, byte: u8) !void { + try self.putBack(@ptrCast([*]const u8, &byte)[0..1]); } - pub fn putBack(self: *Self, bytes: []const u8) void { - var pos = bytes.len; - while (pos != 0) { - pos -= 1; - self.putBackByte(bytes[pos]); - } + pub fn putBack(self: *Self, bytes: []const u8) !void { + try self.fifo.unget(bytes); } fn readFn(in_stream: *Stream, dest: []u8) Error!usize { const self = @fieldParentPtr(Self, "stream", in_stream); // copy over anything putBack()'d - var pos: usize = 0; - while (pos < dest.len and self.index != 0) { - dest[pos] = self.buffer[self.index - 1]; - self.index -= 1; - pos += 1; - } - - if (pos == dest.len or self.at_end) { - return pos; - } + var dest_index = self.fifo.read(dest); + if (dest_index == dest.len) return dest_index; // ask the backing stream for more - const left = dest.len - pos; - const read = try self.base.read(dest[pos..]); - assert(read <= left); - - self.at_end = (read < left); - return pos + read; + dest_index += try self.base.read(dest[dest_index..]); + return dest_index; } }; } diff --git a/lib/std/io/test.zig b/lib/std/io/test.zig index c4e9a1fc06..b2f8307310 100644 --- a/lib/std/io/test.zig +++ b/lib/std/io/test.zig @@ -97,8 +97,8 @@ test "PeekStream" { var dest: [4]u8 = undefined; - ps.putBackByte(9); - ps.putBackByte(10); + try ps.putBackByte(9); + try ps.putBackByte(10); var read = try ps.stream.read(dest[0..4]); expect(read == 4); @@ -114,8 +114,8 @@ test "PeekStream" { expect(read == 2); expect(mem.eql(u8, dest[0..2], bytes[6..8])); - ps.putBackByte(11); - ps.putBackByte(12); + try ps.putBackByte(11); + try ps.putBackByte(12); read = try ps.stream.read(dest[0..4]); expect(read == 2); From dd75cc214d087e0d3a68a76ced3a707aa456d887 Mon Sep 17 00:00:00 2001 From: daurnimator Date: Sat, 16 Nov 2019 22:34:27 +1100 Subject: [PATCH 3/5] std: let PeekStream have static/dynamic variants This completes a TODO in the existing implementation --- lib/std/io.zig | 42 +++++++++++++++++++++++++++++++----------- lib/std/io/test.zig | 2 +- 2 files changed, 32 insertions(+), 12 deletions(-) diff --git a/lib/std/io.zig b/lib/std/io.zig index 018c863bbd..2ebf8c3b87 100644 --- a/lib/std/io.zig +++ b/lib/std/io.zig @@ -196,7 +196,7 @@ test "io.BufferedInStream" { /// Creates a stream which supports 'un-reading' data, so that it can be read again. /// This makes look-ahead style parsing much easier. -pub fn PeekStream(comptime buffer_type: usize, comptime InStreamError: type) type { +pub fn PeekStream(comptime buffer_type: std.fifo.LinearFifoBufferType, comptime InStreamError: type) type { return struct { const Self = @This(); pub const Error = InStreamError; @@ -205,18 +205,38 @@ pub fn PeekStream(comptime buffer_type: usize, comptime InStreamError: type) typ stream: Stream, base: *Stream, - // Right now the look-ahead space is statically allocated, but a version with dynamic allocation - // is not too difficult to derive from this. - const FifoType = std.fifo.LinearFifo(u8, .{ .Static = buffer_size }); + const FifoType = std.fifo.LinearFifo(u8, buffer_type); fifo: FifoType, - pub fn init(base: *Stream) Self { - return .{ - .base = base, - .fifo = FifoType.init(), - .stream = Stream{ .readFn = readFn }, - }; - } + pub usingnamespace switch (buffer_type) { + .Static => struct { + pub fn init(base: *Stream) Self { + return .{ + .base = base, + .fifo = FifoType.init(), + .stream = Stream{ .readFn = readFn }, + }; + } + }, + .Slice => struct { + pub fn init(base: *Stream, buf: []u8) Self { + return .{ + .base = base, + .fifo = FifoType.init(buf), + .stream = Stream{ .readFn = readFn }, + }; + } + }, + .Dynamic => struct { + pub fn init(base: *Stream, allocator: *mem.Allocator) Self { + return .{ + .base = base, + .fifo = FifoType.init(allocator), + .stream = Stream{ .readFn = readFn }, + }; + } + }, + }; pub fn putBackByte(self: *Self, byte: u8) !void { try self.putBack(@ptrCast([*]const u8, &byte)[0..1]); diff --git a/lib/std/io/test.zig b/lib/std/io/test.zig index b2f8307310..6c194b0f3d 100644 --- a/lib/std/io/test.zig +++ b/lib/std/io/test.zig @@ -93,7 +93,7 @@ test "SliceInStream" { test "PeekStream" { const bytes = [_]u8{ 1, 2, 3, 4, 5, 6, 7, 8 }; var ss = io.SliceInStream.init(&bytes); - var ps = io.PeekStream(2, io.SliceInStream.Error).init(&ss.stream); + var ps = io.PeekStream(.{ .Static = 2 }, io.SliceInStream.Error).init(&ss.stream); var dest: [4]u8 = undefined; From 3632f31ec2991005f2269eac955cd5ef589fe5ed Mon Sep 17 00:00:00 2001 From: daurnimator Date: Mon, 11 Nov 2019 03:22:42 +1100 Subject: [PATCH 4/5] std: use LinearFifo to implement io.BufferedOutStreamCustom --- lib/std/io.zig | 41 +++++++++++------------------------------ lib/std/io/test.zig | 5 +++-- 2 files changed, 14 insertions(+), 32 deletions(-) diff --git a/lib/std/io.zig b/lib/std/io.zig index 2ebf8c3b87..4618db95e4 100644 --- a/lib/std/io.zig +++ b/lib/std/io.zig @@ -568,52 +568,33 @@ pub fn BufferedOutStreamCustom(comptime buffer_size: usize, comptime OutStreamEr unbuffered_out_stream: *Stream, - buffer: [buffer_size]u8, - index: usize, + const FifoType = std.fifo.LinearFifo(u8, std.fifo.LinearFifoBufferType{ .Static = buffer_size }); + fifo: FifoType, pub fn init(unbuffered_out_stream: *Stream) Self { return Self{ .unbuffered_out_stream = unbuffered_out_stream, - .buffer = undefined, - .index = 0, + .fifo = FifoType.init(), .stream = Stream{ .writeFn = writeFn }, }; } pub fn flush(self: *Self) !void { - try self.unbuffered_out_stream.write(self.buffer[0..self.index]); - self.index = 0; + while (true) { + const slice = self.fifo.readableSlice(0); + if (slice.len == 0) break; + try self.unbuffered_out_stream.write(slice); + self.fifo.discard(slice.len); + } } fn writeFn(out_stream: *Stream, bytes: []const u8) Error!void { const self = @fieldParentPtr(Self, "stream", out_stream); - - if (bytes.len == 1) { - // This is not required logic but a shorter path - // for single byte writes - self.buffer[self.index] = bytes[0]; - self.index += 1; - if (self.index == buffer_size) { - try self.flush(); - } - return; - } else if (bytes.len >= self.buffer.len) { + if (bytes.len >= self.fifo.writableLength()) { try self.flush(); return self.unbuffered_out_stream.write(bytes); } - var src_index: usize = 0; - - while (src_index < bytes.len) { - const dest_space_left = self.buffer.len - self.index; - const copy_amt = math.min(dest_space_left, bytes.len - src_index); - mem.copy(u8, self.buffer[self.index..], bytes[src_index .. src_index + copy_amt]); - self.index += copy_amt; - assert(self.index <= self.buffer.len); - if (self.index == self.buffer.len) { - try self.flush(); - } - src_index += copy_amt; - } + self.fifo.writeAssumeCapacity(bytes); } }; } diff --git a/lib/std/io/test.zig b/lib/std/io/test.zig index 6c194b0f3d..7716e603a0 100644 --- a/lib/std/io/test.zig +++ b/lib/std/io/test.zig @@ -5,6 +5,7 @@ const meta = std.meta; const trait = std.trait; const DefaultPrng = std.rand.DefaultPrng; const expect = std.testing.expect; +const expectEqual = std.testing.expectEqual; const expectError = std.testing.expectError; const mem = std.mem; const fs = std.fs; @@ -44,8 +45,8 @@ test "write a file, read it, then delete it" { defer file.close(); const file_size = try file.getEndPos(); - const expected_file_size = "begin".len + data.len + "end".len; - expect(file_size == expected_file_size); + const expected_file_size: u64 = "begin".len + data.len + "end".len; + expectEqual(expected_file_size, file_size); var file_in_stream = file.inStream(); var buf_stream = io.BufferedInStream(File.ReadError).init(&file_in_stream.stream); From a8d7652001525881fe17ddf42f5f86671d706b4b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 18 Feb 2020 16:48:26 -0500 Subject: [PATCH 5/5] avoid a `@ptrCast` with an array literal --- lib/std/io.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/io.zig b/lib/std/io.zig index 4618db95e4..22ba0b9bf4 100644 --- a/lib/std/io.zig +++ b/lib/std/io.zig @@ -239,7 +239,7 @@ pub fn PeekStream(comptime buffer_type: std.fifo.LinearFifoBufferType, comptime }; pub fn putBackByte(self: *Self, byte: u8) !void { - try self.putBack(@ptrCast([*]const u8, &byte)[0..1]); + try self.putBack(&[_]u8{byte}); } pub fn putBack(self: *Self, bytes: []const u8) !void {