From 1ab66f3b55bbeac3c6f4f02c86a688cdc1bb2476 Mon Sep 17 00:00:00 2001 From: tgschultz Date: Fri, 23 Nov 2018 10:02:14 -0600 Subject: [PATCH 1/6] Added serialization, bitstreams, traits for integer sign, TagPayloadType --- std/io.zig | 639 ++++++++++++++++++++++++++++++++++++++++++++- std/io_test.zig | 329 +++++++++++++++++++++++ std/meta/index.zig | 38 ++- std/meta/trait.zig | 31 +++ 4 files changed, 1029 insertions(+), 8 deletions(-) diff --git a/std/io.zig b/std/io.zig index f4122a2f8b..219db124a0 100644 --- a/std/io.zig +++ b/std/io.zig @@ -8,6 +8,8 @@ const debug = std.debug; const assert = debug.assert; const os = std.os; const mem = std.mem; +const meta = std.meta; +const trait = meta.trait; const Buffer = std.Buffer; const fmt = std.fmt; const File = std.os.File; @@ -444,6 +446,151 @@ pub const SliceInStream = struct { } }; +/// Creates a stream which allows for reading bit fields from another stream +pub fn BitInStream(endian: builtin.Endian, comptime Error: type) type { + return struct { + const Self = @This(); + + in_stream: *Stream, + bit_buffer: u7, + bit_count: u3, + stream: Stream, + + pub const Stream = InStream(Error); + const u8_bit_count = comptime meta.bitCount(u8); + const u7_bit_count = comptime meta.bitCount(u7); + const u4_bit_count = comptime meta.bitCount(u4); + + pub fn init(in_stream: *Stream) Self { + return Self{ + .in_stream = in_stream, + .bit_buffer = 0, + .bit_count = 0, + .stream = Stream{ .readFn = read }, + }; + } + + /// Reads `bits` bits from the stream and returns a specified unsigned int type + /// containing them in the least significant end, returning an error if the + /// specified number of bits could not be read. + pub fn readBitsNoEof(self: *Self, comptime U: type, bits: usize) !U { + var n: usize = undefined; + const result = try self.readBits(U, bits, &n); + if (n < bits) return error.EndOfStream; + return result; + } + + /// Reads `bits` bits from the stream and returns a specified unsigned int type + /// containing them in the least significant end. The number of bits successfully + /// read is placed in `out_bits`, as reaching the end of the stream is not an error. + pub fn readBits(self: *Self, comptime U: type, bits: usize, out_bits: *usize) Error!U { + debug.assert(trait.isUnsignedInt(U)); + + //by extending the buffer to a minimum of u8 we can cover a number of edge cases + // related to shifting and casting. + const u_bit_count = comptime meta.bitCount(U); + const buf_bit_count = bc: { + debug.assert(u_bit_count >= bits); + break :bc if (u_bit_count <= u8_bit_count) u8_bit_count else u_bit_count; + }; + const Buf = @IntType(false, buf_bit_count); + const BufShift = math.Log2Int(Buf); + + out_bits.* = usize(0); + if (U == u0 or bits == 0) return 0; + var out_buffer = Buf(0); + + if (self.bit_count > 0) { + const n = if (self.bit_count >= bits) @intCast(u3, bits) else self.bit_count; + const shift = u7_bit_count - n; + switch (endian) { + builtin.Endian.Big => { + out_buffer = Buf(self.bit_buffer >> shift); + self.bit_buffer <<= n; + }, + builtin.Endian.Little => { + const value = (self.bit_buffer << shift) >> shift; + out_buffer = Buf(value); + self.bit_buffer >>= n; + }, + } + self.bit_count -= n; + out_bits.* = n; + } + //at this point we know bit_buffer is empty + + //copy bytes until we have enough bits, then leave the rest in bit_buffer + while (out_bits.* < bits) { + const n = bits - out_bits.*; + const next_byte = self.in_stream.readByte() catch |err| { + if (err == error.EndOfStream) { + return @intCast(U, out_buffer); + } + return err; + }; + + switch (endian) { + builtin.Endian.Big => { + if (n >= u8_bit_count) { + out_buffer <<= @intCast(u3, u8_bit_count - 1); + out_buffer <<= 1; + out_buffer |= Buf(next_byte); + out_bits.* += u8_bit_count; + continue; + } + + const shift = @intCast(u3, u8_bit_count - n); + out_buffer <<= @intCast(BufShift, n); + out_buffer |= Buf(next_byte >> shift); + out_bits.* += n; + self.bit_buffer = @truncate(u7, next_byte << @intCast(u3, n - 1)); + self.bit_count = shift; + }, + builtin.Endian.Little => { + if (n >= u8_bit_count) { + out_buffer |= Buf(next_byte) << @intCast(BufShift, out_bits.*); + out_bits.* += u8_bit_count; + continue; + } + + const shift = @intCast(u3, u8_bit_count - n); + const value = (next_byte << shift) >> shift; + out_buffer |= Buf(value) << @intCast(BufShift, out_bits.*); + out_bits.* += n; + self.bit_buffer = @truncate(u7, next_byte >> @intCast(u3, n)); + self.bit_count = shift; + }, + } + } + + return @intCast(U, out_buffer); + } + + pub fn alignToByte(self: *Self) void { + self.bit_buffer = 0; + self.bit_count = 0; + } + + pub fn read(self_stream: *Stream, buffer: []u8) Error!usize { + var self = @fieldParentPtr(Self, "stream", self_stream); + + var out_bits: usize = undefined; + var out_bits_total = usize(0); + //@NOTE: I'm not sure this is a good idea, maybe alignToByte should be forced + if (self.bit_count > 0) { + for (buffer) |*b, i| { + b.* = try self.readBits(u8, u8_bit_count, &out_bits); + out_bits_total += out_bits; + } + const incomplete_byte = @boolToInt(out_bits_total % u8_bit_count > 0); + return (out_bits_total / u8_bit_count) + incomplete_byte; + } + + return self.in_stream.read(buffer); + } + }; +} + /// This is a simple OutStream that writes to a slice, and returns an error /// when it runs out of space. pub const SliceOutStream = struct { @@ -637,6 +784,137 @@ pub const BufferOutStream = struct { } }; +/// Creates a stream which allows for writing bit fields to another stream +pub fn BitOutStream(endian: builtin.Endian, comptime Error: type) type { + return struct { + const Self = @This(); + + out_stream: *Stream, + bit_buffer: u8, + bit_count: u4, + stream: Stream, + + pub const Stream = OutStream(Error); + const u8_bit_count = comptime meta.bitCount(u8); + const u4_bit_count = comptime meta.bitCount(u4); + + pub fn init(out_stream: *Stream) Self { + return Self{ + .out_stream = out_stream, + .bit_buffer = 0, + .bit_count = 0, + .stream = Stream{ .writeFn = write }, + }; + } + + /// Write the specified number of bits to the stream from the least significant bits of + /// the specified unsigned int value. Bits will only be written to the stream when there + /// are enough to fill a byte. + pub fn writeBits(self: *Self, value: var, bits: usize) Error!void { + if (bits == 0) return; + + const U = @typeOf(value); + debug.assert(trait.isUnsignedInt(U)); + + //by extending the buffer to a minimum of u8 we can cover a number of edge cases + // related to shifting and casting. + const u_bit_count = comptime meta.bitCount(U); + const buf_bit_count = bc: { + debug.assert(u_bit_count >= bits); + break :bc if (u_bit_count <= u8_bit_count) u8_bit_count else u_bit_count; + }; + const Buf = @IntType(false, buf_bit_count); + const BufShift = math.Log2Int(Buf); + + const buf_value = @intCast(Buf, value); + + const high_byte_shift = @intCast(BufShift, buf_bit_count - u8_bit_count); + var in_buffer = switch (endian) { + builtin.Endian.Big => buf_value << @intCast(BufShift, buf_bit_count - bits), + builtin.Endian.Little => buf_value, + }; + var in_bits = bits; + + if (self.bit_count > 0) { + const bits_remaining = u8_bit_count - self.bit_count; + const n = @intCast(u3, if (bits_remaining > bits) bits else bits_remaining); + switch (endian) { + builtin.Endian.Big => { + const shift = @intCast(BufShift, high_byte_shift + self.bit_count); + const v = @intCast(u8, in_buffer >> shift); + self.bit_buffer |= v; + in_buffer <<= n; + }, + builtin.Endian.Little => { + const v = @truncate(u8, in_buffer) << @intCast(u3, self.bit_count); + self.bit_buffer |= v; + in_buffer >>= n; + }, + } + self.bit_count += n; + in_bits -= n; + + //if we didn't fill the buffer, it's because bits < bits_remaining; + if (self.bit_count != u8_bit_count) return; + try self.out_stream.writeByte(self.bit_buffer); + self.bit_buffer = 0; + self.bit_count = 0; + } + //at this point we know bit_buffer is empty + + //copy bytes until we can't fill one anymore, then leave the rest in bit_buffer + while (in_bits >= u8_bit_count) { + switch (endian) { + builtin.Endian.Big => { + const v = @intCast(u8, in_buffer >> high_byte_shift); + try self.out_stream.writeByte(v); + in_buffer <<= @intCast(u3, u8_bit_count - 1); + in_buffer <<= 1; + }, + builtin.Endian.Little => { + const v = @truncate(u8, in_buffer); + try self.out_stream.writeByte(v); + in_buffer >>= @intCast(u3, u8_bit_count - 1); + in_buffer >>= 1; + }, + } + in_bits -= u8_bit_count; + } + + if (in_bits > 0) { + self.bit_count = @intCast(u4, in_bits); + self.bit_buffer = switch (endian) { + builtin.Endian.Big => @truncate(u8, in_buffer >> high_byte_shift), + builtin.Endian.Little => @truncate(u8, in_buffer), + }; + } + } + + /// Flush any remaining bits to the stream. + pub fn flushBits(self: *Self) !void { + if (self.bit_count == 0) return; + try self.out_stream.writeByte(self.bit_buffer); + self.bit_buffer = 0; + self.bit_count = 0; + } + + pub fn write(self_stream: *Stream, buffer: []const u8) Error!void { + var self = @fieldParentPtr(Self, "stream", self_stream); + + //@NOTE: I'm not sure this is a good idea, maybe flushBits should be forced + if (self.bit_count > 0) { + for (buffer) |b, i| + try self.writeBits(b, u8_bit_count); + return; + } + + return self.out_stream.write(buffer); + } + }; +} + + + pub const BufferedAtomicFile = struct { atomic_file: os.AtomicFile, file_stream: os.File.OutStream, @@ -677,11 +955,6 @@ pub const BufferedAtomicFile = struct { } }; -test "import io tests" { - comptime { - _ = @import("io_test.zig"); - } -} pub fn readLine(buf: *std.Buffer) ![]u8 { var stdin = try getStdIn(); @@ -753,3 +1026,359 @@ test "io.readLineSliceFrom" { debug.assert(mem.eql(u8, "Line 1", try readLineSliceFrom(stream, buf[0..]))); debug.assertError(readLineSliceFrom(stream, buf[0..]), error.OutOfMemory); } + +/// Creates a deserializer that deserializes types from any stream. +/// If `is_packed` is true, the data stream is treated as bit-packed, +/// otherwise data is expected to be packed to the smallest byte. +/// Types may implement a custom deserialization routine with a +/// function named `deserialize` in the form of: +/// pub fn deserialize(self: *Self, deserializer: var) !void +/// which will be called when the deserializer is used to deserialize +/// that type. It will pass a pointer to the type instance to deserialize +/// into and a pointer to the deserializer struct. +pub fn Deserializer(endian: builtin.Endian, is_packed: bool, comptime Error: type) type { + return struct { + const Self = @This(); + + in_stream: if (is_packed) BitInStream(endian, Stream.Error) else *Stream, + + pub const Stream = InStream(Error); + + pub fn init(in_stream: *Stream) Self { + return Self{ .in_stream = switch (is_packed) { + true => BitInStream(endian, Stream.Error).init(in_stream), + else => in_stream, + } }; + } + + //@BUG: inferred error issue + fn deserializeInt(self: *Self, comptime T: type) (Stream.Error || error{EndOfStream})!T { + debug.assert(trait.is(builtin.TypeId.Int)(T) or trait.is(builtin.TypeId.Float)(T)); + + const u8_bit_count = comptime meta.bitCount(u8); + const t_bit_count = comptime meta.bitCount(T); + + const U = @IntType(false, t_bit_count); + const Log2U = math.Log2Int(U); + const int_size = @sizeOf(U); + + if (is_packed) { + const result = try self.in_stream.readBitsNoEof(U, t_bit_count); + return @bitCast(T, result); + } + + var buffer: [int_size]u8 = undefined; + const read_size = try self.in_stream.read(buffer[0..]); + if (read_size < int_size) return error.EndOfStream; + + if (int_size == 1) return @bitCast(T, buffer[0]); + + var result = U(0); + for (buffer) |byte, i| { + switch (endian) { + builtin.Endian.Big => { + result = (result << @intCast(u4, u8_bit_count)) | byte; + }, + builtin.Endian.Little => { + result |= U(byte) << @intCast(Log2U, u8_bit_count * i); + }, + } + } + + return @bitCast(T, result); + } + + //@TODO: Replace this with @unionInit or whatever when it is added + // see: #1315 + fn setTag(ptr: var, tag: var) void { + const T = @typeOf(ptr); + comptime debug.assert(trait.isPtrTo(builtin.TypeId.Union)(T)); + const U = meta.Child(T); + + const info = @typeInfo(U).Union; + if (info.tag_type) |TagType| { + debug.assert(TagType == @typeOf(tag)); + + var ptr_tag = ptr: { + if (@alignOf(TagType) >= @alignOf(U)) break :ptr @ptrCast(*TagType, ptr); + const offset = comptime max: { + var max_field_size: comptime_int = 0; + for (info.fields) |field_info| { + const field_size = @sizeOf(field_info.field_type); + max_field_size = math.max(max_field_size, field_size); + } + break :max math.max(max_field_size, @alignOf(U)); + }; + break :ptr @intToPtr(*TagType, @ptrToInt(ptr) + offset); + }; + ptr_tag.* = tag; + } + } + + /// Deserializes and returns data of the specified type from the stream + pub fn deserialize(self: *Self, comptime T: type) !T { + var value: T = undefined; + try self.deserializeInto(&value); + return value; + } + + /// Deserializes data into the type pointed to by `ptr` + pub fn deserializeInto(self: *Self, ptr: var) !void { + const T = @typeOf(ptr); + debug.assert(trait.is(builtin.TypeId.Pointer)(T)); + + if (comptime trait.isSlice(T) or comptime trait.isPtrTo(builtin.TypeId.Array)(T)) { + for (ptr) |*v| + try self.deserializeInto(v); + return; + } + + comptime debug.assert(trait.isSingleItemPtr(T)); + + const C = comptime meta.Child(T); + const child_type_id = @typeId(C); + + //custom deserializer: fn(self: *Self, deserializer: var) !void + if (comptime trait.hasFn("deserialize")(C)) return ptr.deserialize(self); + + if (comptime trait.isPacked(C) and !is_packed) { + var packed_deserializer = Deserializer(endian, true, Error).init(self.in_stream); + return packed_deserializer.deserializeInto(ptr); + } + + switch (child_type_id) { + builtin.TypeId.Void => return, + builtin.TypeId.Bool => ptr.* = (try self.deserializeInt(u1)) > 0, + builtin.TypeId.Float, builtin.TypeId.Int => ptr.* = try self.deserializeInt(C), + builtin.TypeId.Struct => { + const info = @typeInfo(C).Struct; + + inline for (info.fields) |*field_info| { + const name = field_info.name; + const FieldType = field_info.field_type; + + if (FieldType == void or FieldType == u0) continue; + + //it doesn't make any sense to read pointers + if (comptime trait.is(builtin.TypeId.Pointer)(FieldType)) { + @compileError("Will not " ++ "read field " ++ name ++ " of struct " ++ + @typeName(C) ++ " because it " ++ "is of pointer-type " ++ + @typeName(FieldType) ++ "."); + } + + try self.deserializeInto(&@field(ptr, name)); + } + }, + builtin.TypeId.Union => { + const info = @typeInfo(C).Union; + if (info.tag_type) |TagType| { + //we avoid duplicate iteration over the enum tags + // by getting the int directly and casting it without + // safety. If it is bad, it will be caught anyway. + const TagInt = @TagType(TagType); + const tag = try self.deserializeInt(TagInt); + + { + @setRuntimeSafety(false); + //See: #1315 + setTag(ptr, @intToEnum(TagType, tag)); + } + + inline for (info.fields) |field_info| { + if (field_info.enum_field.?.value == tag) { + const name = field_info.name; + const FieldType = field_info.field_type; + @field(ptr, name) = FieldType(undefined); + try self.deserializeInto(&@field(ptr, name)); + return; + } + } + //This is reachable if the enum data is bad + return error.InvalidEnumTag; + } + @compileError("Cannot meaningfully deserialize " ++ @typeName(C) ++ + " because it is an untagged union Use a custom deserialize()."); + }, + builtin.TypeId.Optional => { + const OC = comptime meta.Child(C); + const exists = (try self.deserializeInt(u1)) > 0; + if (!exists) { + ptr.* = null; + return; + } + + //The way non-pointer optionals are implemented ensures a pointer to them + // will point to the value. The flag is stored at the end of that data. + var val_ptr = @ptrCast(*OC, ptr); + try self.deserializeInto(val_ptr); + //This bit ensures the null flag isn't set. Any actual copying should be + // optimized out... I hope. + ptr.* = val_ptr.*; + }, + builtin.TypeId.Enum => { + var value = try self.deserializeInt(@TagType(C)); + ptr.* = try meta.intToEnum(C, value); + }, + else => { + @compileError("Cannot deserialize " ++ @tagName(child_type_id) ++ " types (unimplemented)."); + }, + } + } + }; +} + +/// Creates a serializer that serializes types to any stream. +/// If `is_packed` is true, the data will be bit-packed into the stream. +/// Note that the you must call `serializer.flush()` when you are done +/// writing bit-packed data in order ensure any unwritten bits are committed. +/// If `is_packed` is false, data is packed to the smallest byte. In the case +/// of packed structs, the struct will written bit-packed and with the specified +/// endianess, after which data will resume being written at the next byte boundary. +/// Types may implement a custom serialization routine with a +/// function named `serialize` in the form of: +/// pub fn serialize(self: *const Self, serializer: var) !void +/// which will be called when the serializer is used to serialize that type. It will +/// pass a const pointer to the type instance to be serialized and a pointer +/// to the serializer struct. +pub fn Serializer(endian: builtin.Endian, is_packed: bool, comptime Error: type) type { + return struct { + const Self = @This(); + + out_stream: if (is_packed) BitOutStream(endian, Stream.Error) else *Stream, + + pub const Stream = OutStream(Error); + + pub fn init(out_stream: *Stream) Self { + return Self{ .out_stream = switch (is_packed) { + true => BitOutStream(endian, Stream.Error).init(out_stream), + else => out_stream, + } }; + } + + /// Flushes any unwritten bits to the stream + pub fn flush(self: *Self) Stream.Error!void { + if (is_packed) return self.out_stream.flushBits(); + } + + fn serializeInt(self: *Self, value: var) !void { + const T = @typeOf(value); + debug.assert(trait.is(builtin.TypeId.Int)(T) or trait.is(builtin.TypeId.Float)(T)); + + const t_bit_count = comptime meta.bitCount(T); + const u8_bit_count = comptime meta.bitCount(u8); + + const U = @IntType(false, t_bit_count); + const Log2U = math.Log2Int(U); + const int_size = @sizeOf(U); + + const u_value = @bitCast(U, value); + + if (is_packed) return self.out_stream.writeBits(u_value, t_bit_count); + + var buffer: [int_size]u8 = undefined; + if (int_size == 1) buffer[0] = u_value; + + for (buffer) |*byte, i| { + const idx = switch (endian) { + builtin.Endian.Big => int_size - i - 1, + builtin.Endian.Little => i, + }; + const shift = @intCast(Log2U, idx * u8_bit_count); + const v = u_value >> shift; + byte.* = if (t_bit_count < u8_bit_count) v else @truncate(u8, v); + } + + try self.out_stream.write(buffer); + } + + /// Serializes the passed value into the stream + pub fn serialize(self: *Self, value: var) !void { + const T = comptime @typeOf(value); + + if (comptime trait.isIndexable(T)) { + for (value) |v| + try self.serialize(v); + return; + } + + //custom serializer: fn(self: *const Self, serializer: var) !void + if (comptime trait.hasFn("serialize")(T)) return value.serialize(self); + + if (comptime trait.isPacked(T) and !is_packed) { + var packed_serializer = Serializer(endian, true, Error).init(self.out_stream); + try packed_serializer.serialize(value); + try packed_serializer.flush(); + return; + } + + switch (@typeId(T)) { + builtin.TypeId.Void => return, + builtin.TypeId.Bool => try self.serializeInt(u1(@boolToInt(value))), + builtin.TypeId.Float, builtin.TypeId.Int => try self.serializeInt(value), + builtin.TypeId.Struct => { + const info = @typeInfo(T); + + inline for (info.Struct.fields) |*field_info| { + const name = field_info.name; + const FieldType = field_info.field_type; + + if (FieldType == void or FieldType == u0) continue; + + //It doesn't make sense to write pointers + if (comptime trait.is(builtin.TypeId.Pointer)(FieldType)) { + @compileError("Will not " ++ "serialize field " ++ name ++ + " of struct " ++ @typeName(T) ++ " because it " ++ + "is of pointer-type " ++ @typeName(FieldType) ++ "."); + } + try self.serialize(@field(value, name)); + } + }, + builtin.TypeId.Union => { + const info = @typeInfo(T).Union; + if (info.tag_type) |TagType| { + const active_tag = meta.activeTag(value); + try self.serialize(active_tag); + //This inline loop is necessary because active_tag is a runtime + // value, but @field requires a comptime value. Our alternative + // is to check each field for a match + inline for (info.fields) |field_info| { + if (field_info.enum_field.?.value == @enumToInt(active_tag)) { + const name = field_info.name; + const FieldType = field_info.field_type; + try self.serialize(@field(value, name)); + return; + } + } + unreachable; + } + @compileError("Cannot meaningfully serialize " ++ @typeName(T) ++ + " because it is an untagged union Use a custom serialize()."); + }, + builtin.TypeId.Optional => { + if (value == null) { + try self.serializeInt(u1(@boolToInt(false))); + return; + } + try self.serializeInt(u1(@boolToInt(true))); + + const OC = comptime meta.Child(T); + + //The way non-pointer optionals are implemented ensures a pointer to them + // will point to the value. The flag is stored at the end of that data. + var val_ptr = @ptrCast(*const OC, &value); + try self.serialize(val_ptr.*); + }, + builtin.TypeId.Enum => { + try self.serializeInt(@enumToInt(value)); + }, + else => @compileError("Cannot serialize " ++ @tagName(@typeId(T)) ++ " types (unimplemented)."), + } + } + }; +} + +test "import io tests" { + comptime { + _ = @import("io_test.zig"); + } +} diff --git a/std/io_test.zig b/std/io_test.zig index 5224199527..fe06a3e2c9 100644 --- a/std/io_test.zig +++ b/std/io_test.zig @@ -1,5 +1,7 @@ const std = @import("index.zig"); const io = std.io; +const meta = std.meta; +const trait = std.trait; const DefaultPrng = std.rand.DefaultPrng; const assert = std.debug.assert; const assertError = std.debug.assertError; @@ -132,3 +134,330 @@ test "SliceOutStream" { assertError(ss.stream.write("Hello world!"), error.OutOfSpace); assert(mem.eql(u8, ss.getWritten(), "Hello worl")); } + +test "BitInStream" { + const mem_be = []u8{ 0b11001101, 0b00001011 }; + const mem_le = []u8{ 0b00011101, 0b10010101 }; + + var mem_in_be = io.SliceInStream.init(mem_be[0..]); + const InError = io.SliceInStream.Error; + var bit_stream_be = io.BitInStream(builtin.Endian.Big, InError).init(&mem_in_be.stream); + + var out_bits: usize = undefined; + + assert(1 == try bit_stream_be.readBits(u2, 1, &out_bits)); + assert(out_bits == 1); + assert(2 == try bit_stream_be.readBits(u5, 2, &out_bits)); + assert(out_bits == 2); + assert(3 == try bit_stream_be.readBits(u128, 3, &out_bits)); + assert(out_bits == 3); + assert(4 == try bit_stream_be.readBits(u8, 4, &out_bits)); + assert(out_bits == 4); + assert(5 == try bit_stream_be.readBits(u9, 5, &out_bits)); + assert(out_bits == 5); + assert(1 == try bit_stream_be.readBits(u1, 1, &out_bits)); + assert(out_bits == 1); + + mem_in_be.pos = 0; + bit_stream_be.bit_count = 0; + assert(0b110011010000101 == try bit_stream_be.readBits(u15, 15, &out_bits)); + assert(out_bits == 15); + + mem_in_be.pos = 0; + bit_stream_be.bit_count = 0; + assert(0b1100110100001011 == try bit_stream_be.readBits(u16, 16, &out_bits)); + assert(out_bits == 16); + + _ = try bit_stream_be.readBits(u0, 0, &out_bits); + + var mem_in_le = io.SliceInStream.init(mem_le[0..]); + var bit_stream_le = io.BitInStream(builtin.Endian.Little, InError).init(&mem_in_le.stream); + + assert(1 == try bit_stream_le.readBits(u2, 1, &out_bits)); + assert(out_bits == 1); + assert(2 == try bit_stream_le.readBits(u5, 2, &out_bits)); + assert(out_bits == 2); + assert(3 == try bit_stream_le.readBits(u128, 3, &out_bits)); + assert(out_bits == 3); + assert(4 == try bit_stream_le.readBits(u8, 4, &out_bits)); + assert(out_bits == 4); + assert(5 == try bit_stream_le.readBits(u9, 5, &out_bits)); + assert(out_bits == 5); + assert(1 == try bit_stream_le.readBits(u1, 1, &out_bits)); + assert(out_bits == 1); + + mem_in_le.pos = 0; + bit_stream_le.bit_count = 0; + assert(0b001010100011101 == try bit_stream_le.readBits(u15, 15, &out_bits)); + assert(out_bits == 15); + + mem_in_le.pos = 0; + bit_stream_le.bit_count = 0; + assert(0b1001010100011101 == try bit_stream_le.readBits(u16, 16, &out_bits)); + assert(out_bits == 16); + + _ = try bit_stream_le.readBits(u0, 0, &out_bits); +} + +test "BitOutStream" { + var mem_be = []u8{0} ** 2; + var mem_le = []u8{0} ** 2; + + var mem_out_be = io.SliceOutStream.init(mem_be[0..]); + const OutError = io.SliceOutStream.Error; + var bit_stream_be = io.BitOutStream(builtin.Endian.Big, OutError).init(&mem_out_be.stream); + + try bit_stream_be.writeBits(u2(1), 1); + try bit_stream_be.writeBits(u5(2), 2); + try bit_stream_be.writeBits(u128(3), 3); + try bit_stream_be.writeBits(u8(4), 4); + try bit_stream_be.writeBits(u9(5), 5); + try bit_stream_be.writeBits(u1(1), 1); + + assert(mem_be[0] == 0b11001101 and mem_be[1] == 0b00001011); + + mem_out_be.pos = 0; + + try bit_stream_be.writeBits(u15(0b110011010000101), 15); + try bit_stream_be.flushBits(); + assert(mem_be[0] == 0b11001101 and mem_be[1] == 0b00001010); + + mem_out_be.pos = 0; + try bit_stream_be.writeBits(u32(0b110011010000101), 16); + assert(mem_be[0] == 0b01100110 and mem_be[1] == 0b10000101); + + try bit_stream_be.writeBits(u0(0), 0); + + var mem_out_le = io.SliceOutStream.init(mem_le[0..]); + var bit_stream_le = io.BitOutStream(builtin.Endian.Little, OutError).init(&mem_out_le.stream); + + try bit_stream_le.writeBits(u2(1), 1); + try bit_stream_le.writeBits(u5(2), 2); + try bit_stream_le.writeBits(u128(3), 3); + try bit_stream_le.writeBits(u8(4), 4); + try bit_stream_le.writeBits(u9(5), 5); + try bit_stream_le.writeBits(u1(1), 1); + + assert(mem_le[0] == 0b00011101 and mem_le[1] == 0b10010101); + + mem_out_le.pos = 0; + try bit_stream_le.writeBits(u15(0b110011010000101), 15); + try bit_stream_le.flushBits(); + assert(mem_le[0] == 0b10000101 and mem_le[1] == 0b01100110); + + mem_out_le.pos = 0; + try bit_stream_le.writeBits(u32(0b1100110100001011), 16); + assert(mem_le[0] == 0b00001011 and mem_le[1] == 0b11001101); + + try bit_stream_le.writeBits(u0(0), 0); +} + +fn testIntSerializerDeserializer(comptime endian: builtin.Endian, comptime is_packed: bool) !void { + const max_test_bitsize = 17; + + const total_bytes = comptime blk: { + var bytes = 0; + comptime var i = 0; + while (i <= max_test_bitsize) : (i += 1) bytes += (i / 8) + @boolToInt(i % 8 > 0); + break :blk bytes * 2; + }; + + var data_mem: [total_bytes]u8 = undefined; + var out = io.SliceOutStream.init(data_mem[0..]); + const OutError = io.SliceOutStream.Error; + var out_stream = &out.stream; + var serializer = io.Serializer(endian, is_packed, OutError).init(out_stream); + + var in = io.SliceInStream.init(data_mem[0..]); + const InError = io.SliceInStream.Error; + var in_stream = &in.stream; + var deserializer = io.Deserializer(endian, is_packed, InError).init(in_stream); + + comptime var i = 0; + inline while (i <= max_test_bitsize) : (i += 1) { + const U = @IntType(false, i); + const S = @IntType(true, i); + try serializer.serializeInt(U(i)); + if (i != 0) try serializer.serializeInt(S(-1)); + } + try serializer.flush(); + + i = 0; + inline while (i <= max_test_bitsize) : (i += 1) { + const U = @IntType(false, i); + const S = @IntType(true, i); + const x = try deserializer.deserializeInt(U); + const y = if (i != 0) try deserializer.deserializeInt(S); + assert(x == U(i)); + if (i != 0) assert(y == S(-1)); + } + + const u8_bit_count = comptime meta.bitCount(u8); + //0 + 1 + 2 + ... n = (n * (n + 1)) / 2 + //and we have each for unsigned and signed, so * 2 + const total_bits = (max_test_bitsize * (max_test_bitsize + 1)); + const extra_packed_byte = @boolToInt(total_bits % u8_bit_count > 0); + const total_packed_bytes = (total_bits / u8_bit_count) + extra_packed_byte; + + + + assert(in.pos == if (is_packed) total_packed_bytes else total_bytes); +} + +test "Serializer/Deserializer Int" { + try testIntSerializerDeserializer(builtin.Endian.Big, false); + try testIntSerializerDeserializer(builtin.Endian.Little, false); + try testIntSerializerDeserializer(builtin.Endian.Big, true); + try testIntSerializerDeserializer(builtin.Endian.Little, true); +} + +fn testSerializerDeserializer(comptime endian: builtin.Endian, comptime is_packed: bool) !void { + const ColorType = enum(u4) { + RGB8 = 1, + RA16 = 2, + R32 = 3, + }; + + const TagAlign = union(enum(u32)) { + A: u8, + B: u8, + C: u8, + }; + + const Color = union(ColorType) { + RGB8: struct { + r: u8, + g: u8, + b: u8, + a: u8, + }, + RA16: struct { + r: u16, + a: u16, + }, + R32: u32, + }; + + const PackedStruct = packed struct { + f_i3: i3, + f_u2: u2, + }; + + //to test custom serialization + const Custom = struct { + f_f16: f16, + f_unused_u32: u32, + + pub fn deserialize(self: *@This(), deserializer: var) !void { + try deserializer.deserializeInto(&self.f_f16); + self.f_unused_u32 = 47; + } + + pub fn serialize(self: *const @This(), serializer: var) !void { + try serializer.serialize(self.f_f16); + } + }; + + const MyStruct = struct { + f_i3: i3, + f_u8: u8, + f_tag_align: TagAlign, + f_u24: u24, + f_i19: i19, + f_void: void, + f_f32: f32, + f_f128: f128, + f_packed_0: PackedStruct, + f_i7arr: [10]i7, + f_of64n: ?f64, + f_of64v: ?f64, + f_color_type: ColorType, + f_packed_1: PackedStruct, + f_custom: Custom, + f_color: Color, + }; + + const my_inst = MyStruct{ + .f_i3 = -1, + .f_u8 = 8, + .f_tag_align = TagAlign{ .B = 148 }, + .f_u24 = 24, + .f_i19 = 19, + .f_void = {}, + .f_f32 = 32.32, + .f_f128 = 128.128, + .f_packed_0 = PackedStruct{ .f_i3 = -1, .f_u2 = 2 }, + .f_i7arr = [10]i7{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, + .f_of64n = null, + .f_of64v = 64.64, + .f_color_type = ColorType.R32, + .f_packed_1 = PackedStruct{ .f_i3 = 1, .f_u2 = 1 }, + .f_custom = Custom{ .f_f16 = 38.63, .f_unused_u32 = 47 }, + .f_color = Color{ .R32 = 123822 }, + }; + + var data_mem: [@sizeOf(MyStruct)]u8 = undefined; + var out = io.SliceOutStream.init(data_mem[0..]); + const OutError = io.SliceOutStream.Error; + var out_stream = &out.stream; + var serializer = io.Serializer(endian, is_packed, OutError).init(out_stream); + + var in = io.SliceInStream.init(data_mem[0..]); + const InError = io.SliceInStream.Error; + var in_stream = &in.stream; + var deserializer = io.Deserializer(endian, is_packed, InError).init(in_stream); + + try serializer.serialize(my_inst); + + const my_copy = try deserializer.deserialize(MyStruct); + + assert(meta.eql(my_copy, my_inst)); +} + +test "Serializer/Deserializer generic" { + try testSerializerDeserializer(builtin.Endian.Big, false); + try testSerializerDeserializer(builtin.Endian.Little, false); + try testSerializerDeserializer(builtin.Endian.Big, true); + try testSerializerDeserializer(builtin.Endian.Little, true); +} + +fn testBadData(comptime endian: builtin.Endian, comptime is_packed: bool) !void { + const E = enum(u14) { + One = 1, + Two = 2, + }; + + const A = struct { + e: E, + }; + + const C = union(E) { + One: u14, + Two: f16, + }; + + var data_mem: [4]u8 = undefined; + var out = io.SliceOutStream.init(data_mem[0..]); + const OutError = io.SliceOutStream.Error; + var out_stream = &out.stream; + var serializer = io.Serializer(endian, is_packed, OutError).init(out_stream); + + var in = io.SliceInStream.init(data_mem[0..]); + const InError = io.SliceInStream.Error; + var in_stream = &in.stream; + var deserializer = io.Deserializer(endian, is_packed, InError).init(in_stream); + + try serializer.serialize(u14(3)); + assertError(deserializer.deserialize(A), error.InvalidEnumTag); + out.pos = 0; + try serializer.serialize(u14(3)); + try serializer.serialize(u14(88)); + assertError(deserializer.deserialize(C), error.InvalidEnumTag); +} + +test "Deserializer bad data" { + try testBadData(builtin.Endian.Big, false); + try testBadData(builtin.Endian.Little, false); + try testBadData(builtin.Endian.Big, true); + try testBadData(builtin.Endian.Little, true); +} \ No newline at end of file diff --git a/std/meta/index.zig b/std/meta/index.zig index 69a3097288..4d195a6a12 100644 --- a/std/meta/index.zig +++ b/std/meta/index.zig @@ -95,7 +95,7 @@ test "std.meta.stringToEnum" { debug.assert(null == stringToEnum(E1, "C")); } -pub fn bitCount(comptime T: type) u32 { +pub fn bitCount(comptime T: type) comptime_int { return switch (@typeInfo(T)) { TypeId.Int => |info| info.bits, TypeId.Float => |info| info.bits, @@ -108,7 +108,7 @@ test "std.meta.bitCount" { debug.assert(bitCount(f32) == 32); } -pub fn alignment(comptime T: type) u29 { +pub fn alignment(comptime T: type) comptime_int { //@alignOf works on non-pointer types const P = if (comptime trait.is(TypeId.Pointer)(T)) T else *T; return @typeInfo(P).Pointer.alignment; @@ -386,6 +386,33 @@ test "std.meta.activeTag" { debug.assert(activeTag(u) == UE.Float); } +///Given a tagged union type, and an enum, return the type of the union +/// field corresponding to the enum tag. +pub fn TagPayloadType(comptime U: type, tag: var) type { + const Tag = @typeOf(tag); + debug.assert(trait.is(builtin.TypeId.Union)(U)); + debug.assert(trait.is(builtin.TypeId.Enum)(Tag)); + + const info = @typeInfo(U).Union; + + inline for (info.fields) |field_info| { + if (field_info.enum_field.?.value == @enumToInt(tag)) return field_info.field_type; + } + unreachable; +} + +test "std.meta.TagPayloadType" { + const Event = union(enum) { + Moved: struct { + from: i32, + to: i32, + }, + }; + const MovedEvent = TagPayloadType(Event, Event.Moved); + var e: Event = undefined; + debug.assert(MovedEvent == @typeOf(e.Moved)); +} + ///Compares two of any type for equality. Containers are compared on a field-by-field basis, /// where possible. Pointers are not followed. pub fn eql(a: var, b: @typeOf(a)) bool { @@ -439,6 +466,11 @@ pub fn eql(a: var, b: @typeOf(a)) bool { builtin.TypeInfo.Pointer.Size.Slice => return a.ptr == b.ptr and a.len == b.len, } }, + builtin.TypeId.Optional => { + if(a == null and b == null) return true; + if(a == null or b == null) return false; + return eql(a.?, b.?); + }, else => return a == b, } } @@ -452,7 +484,7 @@ test "std.meta.eql" { const U = union(enum) { s: S, - f: f32, + f: ?f32, }; const s_1 = S{ diff --git a/std/meta/trait.zig b/std/meta/trait.zig index caf7f1be04..c9d9e971c9 100644 --- a/std/meta/trait.zig +++ b/std/meta/trait.zig @@ -231,6 +231,37 @@ test "std.meta.trait.isPacked" { debug.assert(!isPacked(u8)); } +/// +pub fn isUnsignedInt(comptime T: type) bool { + return switch (@typeId(T)) { + builtin.TypeId.Int => !@typeInfo(T).Int.is_signed, + else => false, + }; +} + +test "isUnsignedInt" { + debug.assert(isUnsignedInt(u32) == true); + debug.assert(isUnsignedInt(comptime_int) == false); + debug.assert(isUnsignedInt(i64) == false); + debug.assert(isUnsignedInt(f64) == false); +} + +/// +pub fn isSignedInt(comptime T: type) bool { + return switch (@typeId(T)) { + builtin.TypeId.ComptimeInt => true, + builtin.TypeId.Int => @typeInfo(T).Int.is_signed, + else => false, + }; +} + +test "isSignedInt" { + debug.assert(isSignedInt(u32) == false); + debug.assert(isSignedInt(comptime_int) == true); + debug.assert(isSignedInt(i64) == true); + debug.assert(isSignedInt(f64) == false); +} + /// pub fn isSingleItemPtr(comptime T: type) bool { if (comptime is(builtin.TypeId.Pointer)(T)) { From b6489ff90afffcf0c69490efcb5941f3bb42fc3c Mon Sep 17 00:00:00 2001 From: tgschultz Date: Fri, 23 Nov 2018 22:29:40 -0600 Subject: [PATCH 2/6] Increased range of bitwidths tested by "serialize/deserialize Int" tests. Added tests for float inf and NaN. --- std/io.zig | 7 +++++- std/io_test.zig | 62 +++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 61 insertions(+), 8 deletions(-) diff --git a/std/io.zig b/std/io.zig index 219db124a0..499422fdcc 100644 --- a/std/io.zig +++ b/std/io.zig @@ -1050,8 +1050,13 @@ pub fn Deserializer(endian: builtin.Endian, is_packed: bool, comptime Error: typ else => in_stream, } }; } + + pub fn alignToByte(self: *Self) void { + if(!is_packed) return; + self.in_stream.alignToByte(); + } - //@BUG: inferred error issue + //@BUG: inferred error issue. See: #1386 fn deserializeInt(self: *Self, comptime T: type) (Stream.Error || error{EndOfStream})!T { debug.assert(trait.is(builtin.TypeId.Int)(T) or trait.is(builtin.TypeId.Float)(T)); diff --git a/std/io_test.zig b/std/io_test.zig index fe06a3e2c9..430c8f135a 100644 --- a/std/io_test.zig +++ b/std/io_test.zig @@ -253,7 +253,8 @@ test "BitOutStream" { } fn testIntSerializerDeserializer(comptime endian: builtin.Endian, comptime is_packed: bool) !void { - const max_test_bitsize = 17; + //@NOTE: if this test is taking too long, reduce the maximum tested bitsize + const max_test_bitsize = 128; const total_bytes = comptime blk: { var bytes = 0; @@ -278,7 +279,7 @@ fn testIntSerializerDeserializer(comptime endian: builtin.Endian, comptime is_pa const U = @IntType(false, i); const S = @IntType(true, i); try serializer.serializeInt(U(i)); - if (i != 0) try serializer.serializeInt(S(-1)); + if (i != 0) try serializer.serializeInt(S(-1)) else try serializer.serialize(S(0)); } try serializer.flush(); @@ -287,9 +288,9 @@ fn testIntSerializerDeserializer(comptime endian: builtin.Endian, comptime is_pa const U = @IntType(false, i); const S = @IntType(true, i); const x = try deserializer.deserializeInt(U); - const y = if (i != 0) try deserializer.deserializeInt(S); + const y = try deserializer.deserializeInt(S); assert(x == U(i)); - if (i != 0) assert(y == S(-1)); + if (i != 0) assert(y == S(-1)) else assert(y == 0); } const u8_bit_count = comptime meta.bitCount(u8); @@ -299,8 +300,6 @@ fn testIntSerializerDeserializer(comptime endian: builtin.Endian, comptime is_pa const extra_packed_byte = @boolToInt(total_bits % u8_bit_count > 0); const total_packed_bytes = (total_bits / u8_bit_count) + extra_packed_byte; - - assert(in.pos == if (is_packed) total_packed_bytes else total_bytes); } @@ -311,6 +310,56 @@ test "Serializer/Deserializer Int" { try testIntSerializerDeserializer(builtin.Endian.Little, true); } +fn testIntSerializerDeserializerInfNaN(comptime endian: builtin.Endian, + comptime is_packed: bool) !void +{ + const mem_size = (16*2 + 32*2 + 64*2 + 128*2) / comptime meta.bitCount(u8); + var data_mem: [mem_size]u8 = undefined; + + var out = io.SliceOutStream.init(data_mem[0..]); + const OutError = io.SliceOutStream.Error; + var out_stream = &out.stream; + var serializer = io.Serializer(endian, is_packed, OutError).init(out_stream); + + var in = io.SliceInStream.init(data_mem[0..]); + const InError = io.SliceInStream.Error; + var in_stream = &in.stream; + var deserializer = io.Deserializer(endian, is_packed, InError).init(in_stream); + + //@TODO: isInf/isNan not currently implemented for f128. + try serializer.serialize(std.math.nan(f16)); + try serializer.serialize(std.math.inf(f16)); + try serializer.serialize(std.math.nan(f32)); + try serializer.serialize(std.math.inf(f32)); + try serializer.serialize(std.math.nan(f64)); + try serializer.serialize(std.math.inf(f64)); + //try serializer.serialize(std.math.nan(f128)); + //try serializer.serialize(std.math.inf(f128)); + const nan_check_f16 = try deserializer.deserialize(f16); + const inf_check_f16 = try deserializer.deserialize(f16); + const nan_check_f32 = try deserializer.deserialize(f32); + const inf_check_f32 = try deserializer.deserialize(f32); + const nan_check_f64 = try deserializer.deserialize(f64); + const inf_check_f64 = try deserializer.deserialize(f64); + //const nan_check_f128 = try deserializer.deserialize(f128); + //const inf_check_f128 = try deserializer.deserialize(f128); + assert(std.math.isNan(nan_check_f16)); + assert(std.math.isInf(inf_check_f16)); + assert(std.math.isNan(nan_check_f32)); + assert(std.math.isInf(inf_check_f32)); + assert(std.math.isNan(nan_check_f64)); + assert(std.math.isInf(inf_check_f64)); + //assert(std.math.isNan(nan_check_f128)); + //assert(std.math.isInf(inf_check_f128)); +} + +test "Serializer/Deserializer Int: Inf/NaN" { + try testIntSerializerDeserializerInfNaN(builtin.Endian.Big, false); + try testIntSerializerDeserializerInfNaN(builtin.Endian.Little, false); + try testIntSerializerDeserializerInfNaN(builtin.Endian.Big, true); + try testIntSerializerDeserializerInfNaN(builtin.Endian.Little, true); +} + fn testSerializerDeserializer(comptime endian: builtin.Endian, comptime is_packed: bool) !void { const ColorType = enum(u4) { RGB8 = 1, @@ -410,7 +459,6 @@ fn testSerializerDeserializer(comptime endian: builtin.Endian, comptime is_packe try serializer.serialize(my_inst); const my_copy = try deserializer.deserialize(MyStruct); - assert(meta.eql(my_copy, my_inst)); } From 5936bdf8a4163d0c75444e5c8554316de63e3864 Mon Sep 17 00:00:00 2001 From: tgschultz Date: Fri, 30 Nov 2018 14:31:08 -0600 Subject: [PATCH 3/6] Fixed readBits to cast errors to the correct errorset. See #1810 for why this wasn't caught earlier. --- std/io.zig | 4 +++- std/io_test.zig | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/std/io.zig b/std/io.zig index 499422fdcc..5f710a5033 100644 --- a/std/io.zig +++ b/std/io.zig @@ -526,7 +526,9 @@ pub fn BitInStream(endian: builtin.Endian, comptime Error: type) type { if (err == error.EndOfStream) { return @intCast(U, out_buffer); } - return err; + //@BUG: See #1810. Not sure if the bug is that I have to do this for some + // streams, or that I don't for streams with emtpy errorsets. + return @errSetCast(Error, err); }; switch (endian) { diff --git a/std/io_test.zig b/std/io_test.zig index 430c8f135a..ed8e0fcadb 100644 --- a/std/io_test.zig +++ b/std/io_test.zig @@ -169,6 +169,10 @@ test "BitInStream" { assert(out_bits == 16); _ = try bit_stream_be.readBits(u0, 0, &out_bits); + + assert(0 == try bit_stream_be.readBits(u1, 1, &out_bits)); + assert(out_bits == 0); + assertError(bit_stream_be.readBitsNoEof(u1, 1), error.EndOfStream); var mem_in_le = io.SliceInStream.init(mem_le[0..]); var bit_stream_le = io.BitInStream(builtin.Endian.Little, InError).init(&mem_in_le.stream); @@ -197,6 +201,10 @@ test "BitInStream" { assert(out_bits == 16); _ = try bit_stream_le.readBits(u0, 0, &out_bits); + + assert(0 == try bit_stream_le.readBits(u1, 1, &out_bits)); + assert(out_bits == 0); + assertError(bit_stream_le.readBitsNoEof(u1, 1), error.EndOfStream); } test "BitOutStream" { From 8423bd423b7a8cf32246f9d27a400a2cf14ce770 Mon Sep 17 00:00:00 2001 From: tgschultz Date: Fri, 30 Nov 2018 15:02:10 -0600 Subject: [PATCH 4/6] Added explicit test for #1810 issue to io_test.zig. --- std/io_test.zig | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/std/io_test.zig b/std/io_test.zig index ed8e0fcadb..71d46b0874 100644 --- a/std/io_test.zig +++ b/std/io_test.zig @@ -260,6 +260,54 @@ test "BitOutStream" { try bit_stream_le.writeBits(u0(0), 0); } +test "BitStreams with File Stream" { + const tmp_file_name = "temp_test_file.txt"; + { + var file = try os.File.openWrite(tmp_file_name); + defer file.close(); + + var file_out = file.outStream(); + var file_out_stream = &file_out.stream; + const OutError = os.File.WriteError; + var bit_stream = io.BitOutStream(builtin.endian, OutError).init(file_out_stream); + + try bit_stream.writeBits(u2(1), 1); + try bit_stream.writeBits(u5(2), 2); + try bit_stream.writeBits(u128(3), 3); + try bit_stream.writeBits(u8(4), 4); + try bit_stream.writeBits(u9(5), 5); + try bit_stream.writeBits(u1(1), 1); + try bit_stream.flushBits(); + } + { + var file = try os.File.openRead(tmp_file_name); + defer file.close(); + + var file_in = file.inStream(); + var file_in_stream = &file_in.stream; + const InError = os.File.ReadError; + var bit_stream = io.BitInStream(builtin.endian, InError).init(file_in_stream); + + var out_bits: usize = undefined; + + assert(1 == try bit_stream.readBits(u2, 1, &out_bits)); + assert(out_bits == 1); + assert(2 == try bit_stream.readBits(u5, 2, &out_bits)); + assert(out_bits == 2); + assert(3 == try bit_stream.readBits(u128, 3, &out_bits)); + assert(out_bits == 3); + assert(4 == try bit_stream.readBits(u8, 4, &out_bits)); + assert(out_bits == 4); + assert(5 == try bit_stream.readBits(u9, 5, &out_bits)); + assert(out_bits == 5); + assert(1 == try bit_stream.readBits(u1, 1, &out_bits)); + assert(out_bits == 1); + + assertError(bit_stream.readBitsNoEof(u1, 1), error.EndOfStream); + } + try os.deleteFile(tmp_file_name); +} + fn testIntSerializerDeserializer(comptime endian: builtin.Endian, comptime is_packed: bool) !void { //@NOTE: if this test is taking too long, reduce the maximum tested bitsize const max_test_bitsize = 128; From 1188da926ff4cddd5818d85d45b087f6207dfef9 Mon Sep 17 00:00:00 2001 From: tgschultz Date: Sun, 9 Dec 2018 20:52:16 -0600 Subject: [PATCH 5/6] Minor change to custom (de)serializer to allow them to be defined outside of the struct and aliased inside it. This will enable alternate generic serializers (i.e. one that follows pointers) on a struct-by-struct basis. --- std/io.zig | 6 +++--- std/io_test.zig | 10 +++++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/std/io.zig b/std/io.zig index 5f710a5033..c397ff6ede 100644 --- a/std/io.zig +++ b/std/io.zig @@ -1146,7 +1146,7 @@ pub fn Deserializer(endian: builtin.Endian, is_packed: bool, comptime Error: typ const child_type_id = @typeId(C); //custom deserializer: fn(self: *Self, deserializer: var) !void - if (comptime trait.hasFn("deserialize")(C)) return ptr.deserialize(self); + if (comptime trait.hasFn("deserialize")(C)) return C.deserialize(ptr, self); if (comptime trait.isPacked(C) and !is_packed) { var packed_deserializer = Deserializer(endian, true, Error).init(self.in_stream); @@ -1308,8 +1308,8 @@ pub fn Serializer(endian: builtin.Endian, is_packed: bool, comptime Error: type) return; } - //custom serializer: fn(self: *const Self, serializer: var) !void - if (comptime trait.hasFn("serialize")(T)) return value.serialize(self); + //custom serializer: fn(self: Self, serializer: var) !void + if (comptime trait.hasFn("serialize")(T)) return T.serialize(value, self); if (comptime trait.isPacked(T) and !is_packed) { var packed_serializer = Serializer(endian, true, Error).init(self.out_stream); diff --git a/std/io_test.zig b/std/io_test.zig index 71d46b0874..0bee0ddaf0 100644 --- a/std/io_test.zig +++ b/std/io_test.zig @@ -416,6 +416,10 @@ test "Serializer/Deserializer Int: Inf/NaN" { try testIntSerializerDeserializerInfNaN(builtin.Endian.Little, true); } +fn testAlternateSerializer(self: var, serializer: var) !void { + try serializer.serialize(self.f_f16); +} + fn testSerializerDeserializer(comptime endian: builtin.Endian, comptime is_packed: bool) !void { const ColorType = enum(u4) { RGB8 = 1, @@ -448,6 +452,8 @@ fn testSerializerDeserializer(comptime endian: builtin.Endian, comptime is_packe f_u2: u2, }; + + //to test custom serialization const Custom = struct { f_f16: f16, @@ -458,9 +464,7 @@ fn testSerializerDeserializer(comptime endian: builtin.Endian, comptime is_packe self.f_unused_u32 = 47; } - pub fn serialize(self: *const @This(), serializer: var) !void { - try serializer.serialize(self.f_f16); - } + pub const serialize = testAlternateSerializer; }; const MyStruct = struct { From 1a8570403f070933842db7739e7139779b7e04a5 Mon Sep 17 00:00:00 2001 From: tgschultz Date: Sun, 9 Dec 2018 20:59:51 -0600 Subject: [PATCH 6/6] Minor doc-comment fix. --- std/io.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/std/io.zig b/std/io.zig index c397ff6ede..7184787b29 100644 --- a/std/io.zig +++ b/std/io.zig @@ -1243,7 +1243,7 @@ pub fn Deserializer(endian: builtin.Endian, is_packed: bool, comptime Error: typ /// endianess, after which data will resume being written at the next byte boundary. /// Types may implement a custom serialization routine with a /// function named `serialize` in the form of: -/// pub fn serialize(self: *const Self, serializer: var) !void +/// pub fn serialize(self: Self, serializer: var) !void /// which will be called when the serializer is used to serialize that type. It will /// pass a const pointer to the type instance to be serialized and a pointer /// to the serializer struct.