diff --git a/std/io.zig b/std/io.zig index 428d95725d..57f2ef3df5 100644 --- a/std/io.zig +++ b/std/io.zig @@ -8,6 +8,8 @@ const debug = std.debug; const assert = debug.assert; const os = std.os; const mem = std.mem; +const meta = std.meta; +const trait = meta.trait; const Buffer = std.Buffer; const fmt = std.fmt; const File = std.os.File; @@ -463,6 +465,153 @@ pub const SliceInStream = struct { } }; +/// Creates a stream which allows for reading bit fields from another stream +pub fn BitInStream(endian: builtin.Endian, comptime Error: type) type { + return struct { + const Self = @This(); + + in_stream: *Stream, + bit_buffer: u7, + bit_count: u3, + stream: Stream, + + pub const Stream = InStream(Error); + const u8_bit_count = comptime meta.bitCount(u8); + const u7_bit_count = comptime meta.bitCount(u7); + const u4_bit_count = comptime meta.bitCount(u4); + + pub fn init(in_stream: *Stream) Self { + return Self{ + .in_stream = in_stream, + .bit_buffer = 0, + .bit_count = 0, + .stream = Stream{ .readFn = read }, + }; + } + + /// Reads `bits` bits from the stream and returns a specified unsigned int type + /// containing them in the least significant end, returning an error if the + /// specified number of bits could not be read. + pub fn readBitsNoEof(self: *Self, comptime U: type, bits: usize) !U { + var n: usize = undefined; + const result = try self.readBits(U, bits, &n); + if (n < bits) return error.EndOfStream; + return result; + } + + /// Reads `bits` bits from the stream and returns a specified unsigned int type + /// containing them in the least significant end. The number of bits successfully + /// read is placed in `out_bits`, as reaching the end of the stream is not an error. + pub fn readBits(self: *Self, comptime U: type, bits: usize, out_bits: *usize) Error!U { + debug.assert(trait.isUnsignedInt(U)); + + //by extending the buffer to a minimum of u8 we can cover a number of edge cases + // related to shifting and casting. + const u_bit_count = comptime meta.bitCount(U); + const buf_bit_count = bc: { + debug.assert(u_bit_count >= bits); + break :bc if (u_bit_count <= u8_bit_count) u8_bit_count else u_bit_count; + }; + const Buf = @IntType(false, buf_bit_count); + const BufShift = math.Log2Int(Buf); + + out_bits.* = usize(0); + if (U == u0 or bits == 0) return 0; + var out_buffer = Buf(0); + + if (self.bit_count > 0) { + const n = if (self.bit_count >= bits) @intCast(u3, bits) else self.bit_count; + const shift = u7_bit_count - n; + switch (endian) { + builtin.Endian.Big => { + out_buffer = Buf(self.bit_buffer >> shift); + self.bit_buffer <<= n; + }, + builtin.Endian.Little => { + const value = (self.bit_buffer << shift) >> shift; + out_buffer = Buf(value); + self.bit_buffer >>= n; + }, + } + self.bit_count -= n; + out_bits.* = n; + } + //at this point we know bit_buffer is empty + + //copy bytes until we have enough bits, then leave the rest in bit_buffer + while (out_bits.* < bits) { + const n = bits - out_bits.*; + const next_byte = self.in_stream.readByte() catch |err| { + if (err == error.EndOfStream) { + return @intCast(U, out_buffer); + } + //@BUG: See #1810. Not sure if the bug is that I have to do this for some + // streams, or that I don't for streams with emtpy errorsets. + return @errSetCast(Error, err); + }; + + switch (endian) { + builtin.Endian.Big => { + if (n >= u8_bit_count) { + out_buffer <<= @intCast(u3, u8_bit_count - 1); + out_buffer <<= 1; + out_buffer |= Buf(next_byte); + out_bits.* += u8_bit_count; + continue; + } + + const shift = @intCast(u3, u8_bit_count - n); + out_buffer <<= @intCast(BufShift, n); + out_buffer |= Buf(next_byte >> shift); + out_bits.* += n; + self.bit_buffer = @truncate(u7, next_byte << @intCast(u3, n - 1)); + self.bit_count = shift; + }, + builtin.Endian.Little => { + if (n >= u8_bit_count) { + out_buffer |= Buf(next_byte) << @intCast(BufShift, out_bits.*); + out_bits.* += u8_bit_count; + continue; + } + + const shift = @intCast(u3, u8_bit_count - n); + const value = (next_byte << shift) >> shift; + out_buffer |= Buf(value) << @intCast(BufShift, out_bits.*); + out_bits.* += n; + self.bit_buffer = @truncate(u7, next_byte >> @intCast(u3, n)); + self.bit_count = shift; + }, + } + } + + return @intCast(U, out_buffer); + } + + pub fn alignToByte(self: *Self) void { + self.bit_buffer = 0; + self.bit_count = 0; + } + + pub fn read(self_stream: *Stream, buffer: []u8) Error!usize { + var self = @fieldParentPtr(Self, "stream", self_stream); + + var out_bits: usize = undefined; + var out_bits_total = usize(0); + //@NOTE: I'm not sure this is a good idea, maybe alignToByte should be forced + if (self.bit_count > 0) { + for (buffer) |*b, i| { + b.* = try self.readBits(u8, u8_bit_count, &out_bits); + out_bits_total += out_bits; + } + const incomplete_byte = @boolToInt(out_bits_total % u8_bit_count > 0); + return (out_bits_total / u8_bit_count) + incomplete_byte; + } + + return self.in_stream.read(buffer); + } + }; +} + /// This is a simple OutStream that writes to a slice, and returns an error /// when it runs out of space. pub const SliceOutStream = struct { @@ -656,6 +805,137 @@ pub const BufferOutStream = struct { } }; +/// Creates a stream which allows for writing bit fields to another stream +pub fn BitOutStream(endian: builtin.Endian, comptime Error: type) type { + return struct { + const Self = @This(); + + out_stream: *Stream, + bit_buffer: u8, + bit_count: u4, + stream: Stream, + + pub const Stream = OutStream(Error); + const u8_bit_count = comptime meta.bitCount(u8); + const u4_bit_count = comptime meta.bitCount(u4); + + pub fn init(out_stream: *Stream) Self { + return Self{ + .out_stream = out_stream, + .bit_buffer = 0, + .bit_count = 0, + .stream = Stream{ .writeFn = write }, + }; + } + + /// Write the specified number of bits to the stream from the least significant bits of + /// the specified unsigned int value. Bits will only be written to the stream when there + /// are enough to fill a byte. + pub fn writeBits(self: *Self, value: var, bits: usize) Error!void { + if (bits == 0) return; + + const U = @typeOf(value); + debug.assert(trait.isUnsignedInt(U)); + + //by extending the buffer to a minimum of u8 we can cover a number of edge cases + // related to shifting and casting. + const u_bit_count = comptime meta.bitCount(U); + const buf_bit_count = bc: { + debug.assert(u_bit_count >= bits); + break :bc if (u_bit_count <= u8_bit_count) u8_bit_count else u_bit_count; + }; + const Buf = @IntType(false, buf_bit_count); + const BufShift = math.Log2Int(Buf); + + const buf_value = @intCast(Buf, value); + + const high_byte_shift = @intCast(BufShift, buf_bit_count - u8_bit_count); + var in_buffer = switch (endian) { + builtin.Endian.Big => buf_value << @intCast(BufShift, buf_bit_count - bits), + builtin.Endian.Little => buf_value, + }; + var in_bits = bits; + + if (self.bit_count > 0) { + const bits_remaining = u8_bit_count - self.bit_count; + const n = @intCast(u3, if (bits_remaining > bits) bits else bits_remaining); + switch (endian) { + builtin.Endian.Big => { + const shift = @intCast(BufShift, high_byte_shift + self.bit_count); + const v = @intCast(u8, in_buffer >> shift); + self.bit_buffer |= v; + in_buffer <<= n; + }, + builtin.Endian.Little => { + const v = @truncate(u8, in_buffer) << @intCast(u3, self.bit_count); + self.bit_buffer |= v; + in_buffer >>= n; + }, + } + self.bit_count += n; + in_bits -= n; + + //if we didn't fill the buffer, it's because bits < bits_remaining; + if (self.bit_count != u8_bit_count) return; + try self.out_stream.writeByte(self.bit_buffer); + self.bit_buffer = 0; + self.bit_count = 0; + } + //at this point we know bit_buffer is empty + + //copy bytes until we can't fill one anymore, then leave the rest in bit_buffer + while (in_bits >= u8_bit_count) { + switch (endian) { + builtin.Endian.Big => { + const v = @intCast(u8, in_buffer >> high_byte_shift); + try self.out_stream.writeByte(v); + in_buffer <<= @intCast(u3, u8_bit_count - 1); + in_buffer <<= 1; + }, + builtin.Endian.Little => { + const v = @truncate(u8, in_buffer); + try self.out_stream.writeByte(v); + in_buffer >>= @intCast(u3, u8_bit_count - 1); + in_buffer >>= 1; + }, + } + in_bits -= u8_bit_count; + } + + if (in_bits > 0) { + self.bit_count = @intCast(u4, in_bits); + self.bit_buffer = switch (endian) { + builtin.Endian.Big => @truncate(u8, in_buffer >> high_byte_shift), + builtin.Endian.Little => @truncate(u8, in_buffer), + }; + } + } + + /// Flush any remaining bits to the stream. + pub fn flushBits(self: *Self) !void { + if (self.bit_count == 0) return; + try self.out_stream.writeByte(self.bit_buffer); + self.bit_buffer = 0; + self.bit_count = 0; + } + + pub fn write(self_stream: *Stream, buffer: []const u8) Error!void { + var self = @fieldParentPtr(Self, "stream", self_stream); + + //@NOTE: I'm not sure this is a good idea, maybe flushBits should be forced + if (self.bit_count > 0) { + for (buffer) |b, i| + try self.writeBits(b, u8_bit_count); + return; + } + + return self.out_stream.write(buffer); + } + }; +} + + + pub const BufferedAtomicFile = struct { atomic_file: os.AtomicFile, file_stream: os.File.OutStream, @@ -696,11 +976,6 @@ pub const BufferedAtomicFile = struct { } }; -test "import io tests" { - comptime { - _ = @import("io_test.zig"); - } -} pub fn readLine(buf: *std.Buffer) ![]u8 { var stdin = try getStdIn(); @@ -772,3 +1047,364 @@ test "io.readLineSliceFrom" { debug.assert(mem.eql(u8, "Line 1", try readLineSliceFrom(stream, buf[0..]))); debug.assertError(readLineSliceFrom(stream, buf[0..]), error.OutOfMemory); } + +/// Creates a deserializer that deserializes types from any stream. +/// If `is_packed` is true, the data stream is treated as bit-packed, +/// otherwise data is expected to be packed to the smallest byte. +/// Types may implement a custom deserialization routine with a +/// function named `deserialize` in the form of: +/// pub fn deserialize(self: *Self, deserializer: var) !void +/// which will be called when the deserializer is used to deserialize +/// that type. It will pass a pointer to the type instance to deserialize +/// into and a pointer to the deserializer struct. +pub fn Deserializer(endian: builtin.Endian, is_packed: bool, comptime Error: type) type { + return struct { + const Self = @This(); + + in_stream: if (is_packed) BitInStream(endian, Stream.Error) else *Stream, + + pub const Stream = InStream(Error); + + pub fn init(in_stream: *Stream) Self { + return Self{ .in_stream = switch (is_packed) { + true => BitInStream(endian, Stream.Error).init(in_stream), + else => in_stream, + } }; + } + + pub fn alignToByte(self: *Self) void { + if(!is_packed) return; + self.in_stream.alignToByte(); + } + + //@BUG: inferred error issue. See: #1386 + fn deserializeInt(self: *Self, comptime T: type) (Stream.Error || error{EndOfStream})!T { + debug.assert(trait.is(builtin.TypeId.Int)(T) or trait.is(builtin.TypeId.Float)(T)); + + const u8_bit_count = comptime meta.bitCount(u8); + const t_bit_count = comptime meta.bitCount(T); + + const U = @IntType(false, t_bit_count); + const Log2U = math.Log2Int(U); + const int_size = @sizeOf(U); + + if (is_packed) { + const result = try self.in_stream.readBitsNoEof(U, t_bit_count); + return @bitCast(T, result); + } + + var buffer: [int_size]u8 = undefined; + const read_size = try self.in_stream.read(buffer[0..]); + if (read_size < int_size) return error.EndOfStream; + + if (int_size == 1) return @bitCast(T, buffer[0]); + + var result = U(0); + for (buffer) |byte, i| { + switch (endian) { + builtin.Endian.Big => { + result = (result << @intCast(u4, u8_bit_count)) | byte; + }, + builtin.Endian.Little => { + result |= U(byte) << @intCast(Log2U, u8_bit_count * i); + }, + } + } + + return @bitCast(T, result); + } + + //@TODO: Replace this with @unionInit or whatever when it is added + // see: #1315 + fn setTag(ptr: var, tag: var) void { + const T = @typeOf(ptr); + comptime debug.assert(trait.isPtrTo(builtin.TypeId.Union)(T)); + const U = meta.Child(T); + + const info = @typeInfo(U).Union; + if (info.tag_type) |TagType| { + debug.assert(TagType == @typeOf(tag)); + + var ptr_tag = ptr: { + if (@alignOf(TagType) >= @alignOf(U)) break :ptr @ptrCast(*TagType, ptr); + const offset = comptime max: { + var max_field_size: comptime_int = 0; + for (info.fields) |field_info| { + const field_size = @sizeOf(field_info.field_type); + max_field_size = math.max(max_field_size, field_size); + } + break :max math.max(max_field_size, @alignOf(U)); + }; + break :ptr @intToPtr(*TagType, @ptrToInt(ptr) + offset); + }; + ptr_tag.* = tag; + } + } + + /// Deserializes and returns data of the specified type from the stream + pub fn deserialize(self: *Self, comptime T: type) !T { + var value: T = undefined; + try self.deserializeInto(&value); + return value; + } + + /// Deserializes data into the type pointed to by `ptr` + pub fn deserializeInto(self: *Self, ptr: var) !void { + const T = @typeOf(ptr); + debug.assert(trait.is(builtin.TypeId.Pointer)(T)); + + if (comptime trait.isSlice(T) or comptime trait.isPtrTo(builtin.TypeId.Array)(T)) { + for (ptr) |*v| + try self.deserializeInto(v); + return; + } + + comptime debug.assert(trait.isSingleItemPtr(T)); + + const C = comptime meta.Child(T); + const child_type_id = @typeId(C); + + //custom deserializer: fn(self: *Self, deserializer: var) !void + if (comptime trait.hasFn("deserialize")(C)) return C.deserialize(ptr, self); + + if (comptime trait.isPacked(C) and !is_packed) { + var packed_deserializer = Deserializer(endian, true, Error).init(self.in_stream); + return packed_deserializer.deserializeInto(ptr); + } + + switch (child_type_id) { + builtin.TypeId.Void => return, + builtin.TypeId.Bool => ptr.* = (try self.deserializeInt(u1)) > 0, + builtin.TypeId.Float, builtin.TypeId.Int => ptr.* = try self.deserializeInt(C), + builtin.TypeId.Struct => { + const info = @typeInfo(C).Struct; + + inline for (info.fields) |*field_info| { + const name = field_info.name; + const FieldType = field_info.field_type; + + if (FieldType == void or FieldType == u0) continue; + + //it doesn't make any sense to read pointers + if (comptime trait.is(builtin.TypeId.Pointer)(FieldType)) { + @compileError("Will not " ++ "read field " ++ name ++ " of struct " ++ + @typeName(C) ++ " because it " ++ "is of pointer-type " ++ + @typeName(FieldType) ++ "."); + } + + try self.deserializeInto(&@field(ptr, name)); + } + }, + builtin.TypeId.Union => { + const info = @typeInfo(C).Union; + if (info.tag_type) |TagType| { + //we avoid duplicate iteration over the enum tags + // by getting the int directly and casting it without + // safety. If it is bad, it will be caught anyway. + const TagInt = @TagType(TagType); + const tag = try self.deserializeInt(TagInt); + + { + @setRuntimeSafety(false); + //See: #1315 + setTag(ptr, @intToEnum(TagType, tag)); + } + + inline for (info.fields) |field_info| { + if (field_info.enum_field.?.value == tag) { + const name = field_info.name; + const FieldType = field_info.field_type; + @field(ptr, name) = FieldType(undefined); + try self.deserializeInto(&@field(ptr, name)); + return; + } + } + //This is reachable if the enum data is bad + return error.InvalidEnumTag; + } + @compileError("Cannot meaningfully deserialize " ++ @typeName(C) ++ + " because it is an untagged union Use a custom deserialize()."); + }, + builtin.TypeId.Optional => { + const OC = comptime meta.Child(C); + const exists = (try self.deserializeInt(u1)) > 0; + if (!exists) { + ptr.* = null; + return; + } + + //The way non-pointer optionals are implemented ensures a pointer to them + // will point to the value. The flag is stored at the end of that data. + var val_ptr = @ptrCast(*OC, ptr); + try self.deserializeInto(val_ptr); + //This bit ensures the null flag isn't set. Any actual copying should be + // optimized out... I hope. + ptr.* = val_ptr.*; + }, + builtin.TypeId.Enum => { + var value = try self.deserializeInt(@TagType(C)); + ptr.* = try meta.intToEnum(C, value); + }, + else => { + @compileError("Cannot deserialize " ++ @tagName(child_type_id) ++ " types (unimplemented)."); + }, + } + } + }; +} + +/// Creates a serializer that serializes types to any stream. +/// If `is_packed` is true, the data will be bit-packed into the stream. +/// Note that the you must call `serializer.flush()` when you are done +/// writing bit-packed data in order ensure any unwritten bits are committed. +/// If `is_packed` is false, data is packed to the smallest byte. In the case +/// of packed structs, the struct will written bit-packed and with the specified +/// endianess, after which data will resume being written at the next byte boundary. +/// Types may implement a custom serialization routine with a +/// function named `serialize` in the form of: +/// pub fn serialize(self: Self, serializer: var) !void +/// which will be called when the serializer is used to serialize that type. It will +/// pass a const pointer to the type instance to be serialized and a pointer +/// to the serializer struct. +pub fn Serializer(endian: builtin.Endian, is_packed: bool, comptime Error: type) type { + return struct { + const Self = @This(); + + out_stream: if (is_packed) BitOutStream(endian, Stream.Error) else *Stream, + + pub const Stream = OutStream(Error); + + pub fn init(out_stream: *Stream) Self { + return Self{ .out_stream = switch (is_packed) { + true => BitOutStream(endian, Stream.Error).init(out_stream), + else => out_stream, + } }; + } + + /// Flushes any unwritten bits to the stream + pub fn flush(self: *Self) Stream.Error!void { + if (is_packed) return self.out_stream.flushBits(); + } + + fn serializeInt(self: *Self, value: var) !void { + const T = @typeOf(value); + debug.assert(trait.is(builtin.TypeId.Int)(T) or trait.is(builtin.TypeId.Float)(T)); + + const t_bit_count = comptime meta.bitCount(T); + const u8_bit_count = comptime meta.bitCount(u8); + + const U = @IntType(false, t_bit_count); + const Log2U = math.Log2Int(U); + const int_size = @sizeOf(U); + + const u_value = @bitCast(U, value); + + if (is_packed) return self.out_stream.writeBits(u_value, t_bit_count); + + var buffer: [int_size]u8 = undefined; + if (int_size == 1) buffer[0] = u_value; + + for (buffer) |*byte, i| { + const idx = switch (endian) { + builtin.Endian.Big => int_size - i - 1, + builtin.Endian.Little => i, + }; + const shift = @intCast(Log2U, idx * u8_bit_count); + const v = u_value >> shift; + byte.* = if (t_bit_count < u8_bit_count) v else @truncate(u8, v); + } + + try self.out_stream.write(buffer); + } + + /// Serializes the passed value into the stream + pub fn serialize(self: *Self, value: var) !void { + const T = comptime @typeOf(value); + + if (comptime trait.isIndexable(T)) { + for (value) |v| + try self.serialize(v); + return; + } + + //custom serializer: fn(self: Self, serializer: var) !void + if (comptime trait.hasFn("serialize")(T)) return T.serialize(value, self); + + if (comptime trait.isPacked(T) and !is_packed) { + var packed_serializer = Serializer(endian, true, Error).init(self.out_stream); + try packed_serializer.serialize(value); + try packed_serializer.flush(); + return; + } + + switch (@typeId(T)) { + builtin.TypeId.Void => return, + builtin.TypeId.Bool => try self.serializeInt(u1(@boolToInt(value))), + builtin.TypeId.Float, builtin.TypeId.Int => try self.serializeInt(value), + builtin.TypeId.Struct => { + const info = @typeInfo(T); + + inline for (info.Struct.fields) |*field_info| { + const name = field_info.name; + const FieldType = field_info.field_type; + + if (FieldType == void or FieldType == u0) continue; + + //It doesn't make sense to write pointers + if (comptime trait.is(builtin.TypeId.Pointer)(FieldType)) { + @compileError("Will not " ++ "serialize field " ++ name ++ + " of struct " ++ @typeName(T) ++ " because it " ++ + "is of pointer-type " ++ @typeName(FieldType) ++ "."); + } + try self.serialize(@field(value, name)); + } + }, + builtin.TypeId.Union => { + const info = @typeInfo(T).Union; + if (info.tag_type) |TagType| { + const active_tag = meta.activeTag(value); + try self.serialize(active_tag); + //This inline loop is necessary because active_tag is a runtime + // value, but @field requires a comptime value. Our alternative + // is to check each field for a match + inline for (info.fields) |field_info| { + if (field_info.enum_field.?.value == @enumToInt(active_tag)) { + const name = field_info.name; + const FieldType = field_info.field_type; + try self.serialize(@field(value, name)); + return; + } + } + unreachable; + } + @compileError("Cannot meaningfully serialize " ++ @typeName(T) ++ + " because it is an untagged union Use a custom serialize()."); + }, + builtin.TypeId.Optional => { + if (value == null) { + try self.serializeInt(u1(@boolToInt(false))); + return; + } + try self.serializeInt(u1(@boolToInt(true))); + + const OC = comptime meta.Child(T); + + //The way non-pointer optionals are implemented ensures a pointer to them + // will point to the value. The flag is stored at the end of that data. + var val_ptr = @ptrCast(*const OC, &value); + try self.serialize(val_ptr.*); + }, + builtin.TypeId.Enum => { + try self.serializeInt(@enumToInt(value)); + }, + else => @compileError("Cannot serialize " ++ @tagName(@typeId(T)) ++ " types (unimplemented)."), + } + } + }; +} + +test "import io tests" { + comptime { + _ = @import("io_test.zig"); + } +} diff --git a/std/io_test.zig b/std/io_test.zig index 5224199527..0bee0ddaf0 100644 --- a/std/io_test.zig +++ b/std/io_test.zig @@ -1,5 +1,7 @@ const std = @import("index.zig"); const io = std.io; +const meta = std.meta; +const trait = std.trait; const DefaultPrng = std.rand.DefaultPrng; const assert = std.debug.assert; const assertError = std.debug.assertError; @@ -132,3 +134,438 @@ test "SliceOutStream" { assertError(ss.stream.write("Hello world!"), error.OutOfSpace); assert(mem.eql(u8, ss.getWritten(), "Hello worl")); } + +test "BitInStream" { + const mem_be = []u8{ 0b11001101, 0b00001011 }; + const mem_le = []u8{ 0b00011101, 0b10010101 }; + + var mem_in_be = io.SliceInStream.init(mem_be[0..]); + const InError = io.SliceInStream.Error; + var bit_stream_be = io.BitInStream(builtin.Endian.Big, InError).init(&mem_in_be.stream); + + var out_bits: usize = undefined; + + assert(1 == try bit_stream_be.readBits(u2, 1, &out_bits)); + assert(out_bits == 1); + assert(2 == try bit_stream_be.readBits(u5, 2, &out_bits)); + assert(out_bits == 2); + assert(3 == try bit_stream_be.readBits(u128, 3, &out_bits)); + assert(out_bits == 3); + assert(4 == try bit_stream_be.readBits(u8, 4, &out_bits)); + assert(out_bits == 4); + assert(5 == try bit_stream_be.readBits(u9, 5, &out_bits)); + assert(out_bits == 5); + assert(1 == try bit_stream_be.readBits(u1, 1, &out_bits)); + assert(out_bits == 1); + + mem_in_be.pos = 0; + bit_stream_be.bit_count = 0; + assert(0b110011010000101 == try bit_stream_be.readBits(u15, 15, &out_bits)); + assert(out_bits == 15); + + mem_in_be.pos = 0; + bit_stream_be.bit_count = 0; + assert(0b1100110100001011 == try bit_stream_be.readBits(u16, 16, &out_bits)); + assert(out_bits == 16); + + _ = try bit_stream_be.readBits(u0, 0, &out_bits); + + assert(0 == try bit_stream_be.readBits(u1, 1, &out_bits)); + assert(out_bits == 0); + assertError(bit_stream_be.readBitsNoEof(u1, 1), error.EndOfStream); + + var mem_in_le = io.SliceInStream.init(mem_le[0..]); + var bit_stream_le = io.BitInStream(builtin.Endian.Little, InError).init(&mem_in_le.stream); + + assert(1 == try bit_stream_le.readBits(u2, 1, &out_bits)); + assert(out_bits == 1); + assert(2 == try bit_stream_le.readBits(u5, 2, &out_bits)); + assert(out_bits == 2); + assert(3 == try bit_stream_le.readBits(u128, 3, &out_bits)); + assert(out_bits == 3); + assert(4 == try bit_stream_le.readBits(u8, 4, &out_bits)); + assert(out_bits == 4); + assert(5 == try bit_stream_le.readBits(u9, 5, &out_bits)); + assert(out_bits == 5); + assert(1 == try bit_stream_le.readBits(u1, 1, &out_bits)); + assert(out_bits == 1); + + mem_in_le.pos = 0; + bit_stream_le.bit_count = 0; + assert(0b001010100011101 == try bit_stream_le.readBits(u15, 15, &out_bits)); + assert(out_bits == 15); + + mem_in_le.pos = 0; + bit_stream_le.bit_count = 0; + assert(0b1001010100011101 == try bit_stream_le.readBits(u16, 16, &out_bits)); + assert(out_bits == 16); + + _ = try bit_stream_le.readBits(u0, 0, &out_bits); + + assert(0 == try bit_stream_le.readBits(u1, 1, &out_bits)); + assert(out_bits == 0); + assertError(bit_stream_le.readBitsNoEof(u1, 1), error.EndOfStream); +} + +test "BitOutStream" { + var mem_be = []u8{0} ** 2; + var mem_le = []u8{0} ** 2; + + var mem_out_be = io.SliceOutStream.init(mem_be[0..]); + const OutError = io.SliceOutStream.Error; + var bit_stream_be = io.BitOutStream(builtin.Endian.Big, OutError).init(&mem_out_be.stream); + + try bit_stream_be.writeBits(u2(1), 1); + try bit_stream_be.writeBits(u5(2), 2); + try bit_stream_be.writeBits(u128(3), 3); + try bit_stream_be.writeBits(u8(4), 4); + try bit_stream_be.writeBits(u9(5), 5); + try bit_stream_be.writeBits(u1(1), 1); + + assert(mem_be[0] == 0b11001101 and mem_be[1] == 0b00001011); + + mem_out_be.pos = 0; + + try bit_stream_be.writeBits(u15(0b110011010000101), 15); + try bit_stream_be.flushBits(); + assert(mem_be[0] == 0b11001101 and mem_be[1] == 0b00001010); + + mem_out_be.pos = 0; + try bit_stream_be.writeBits(u32(0b110011010000101), 16); + assert(mem_be[0] == 0b01100110 and mem_be[1] == 0b10000101); + + try bit_stream_be.writeBits(u0(0), 0); + + var mem_out_le = io.SliceOutStream.init(mem_le[0..]); + var bit_stream_le = io.BitOutStream(builtin.Endian.Little, OutError).init(&mem_out_le.stream); + + try bit_stream_le.writeBits(u2(1), 1); + try bit_stream_le.writeBits(u5(2), 2); + try bit_stream_le.writeBits(u128(3), 3); + try bit_stream_le.writeBits(u8(4), 4); + try bit_stream_le.writeBits(u9(5), 5); + try bit_stream_le.writeBits(u1(1), 1); + + assert(mem_le[0] == 0b00011101 and mem_le[1] == 0b10010101); + + mem_out_le.pos = 0; + try bit_stream_le.writeBits(u15(0b110011010000101), 15); + try bit_stream_le.flushBits(); + assert(mem_le[0] == 0b10000101 and mem_le[1] == 0b01100110); + + mem_out_le.pos = 0; + try bit_stream_le.writeBits(u32(0b1100110100001011), 16); + assert(mem_le[0] == 0b00001011 and mem_le[1] == 0b11001101); + + try bit_stream_le.writeBits(u0(0), 0); +} + +test "BitStreams with File Stream" { + const tmp_file_name = "temp_test_file.txt"; + { + var file = try os.File.openWrite(tmp_file_name); + defer file.close(); + + var file_out = file.outStream(); + var file_out_stream = &file_out.stream; + const OutError = os.File.WriteError; + var bit_stream = io.BitOutStream(builtin.endian, OutError).init(file_out_stream); + + try bit_stream.writeBits(u2(1), 1); + try bit_stream.writeBits(u5(2), 2); + try bit_stream.writeBits(u128(3), 3); + try bit_stream.writeBits(u8(4), 4); + try bit_stream.writeBits(u9(5), 5); + try bit_stream.writeBits(u1(1), 1); + try bit_stream.flushBits(); + } + { + var file = try os.File.openRead(tmp_file_name); + defer file.close(); + + var file_in = file.inStream(); + var file_in_stream = &file_in.stream; + const InError = os.File.ReadError; + var bit_stream = io.BitInStream(builtin.endian, InError).init(file_in_stream); + + var out_bits: usize = undefined; + + assert(1 == try bit_stream.readBits(u2, 1, &out_bits)); + assert(out_bits == 1); + assert(2 == try bit_stream.readBits(u5, 2, &out_bits)); + assert(out_bits == 2); + assert(3 == try bit_stream.readBits(u128, 3, &out_bits)); + assert(out_bits == 3); + assert(4 == try bit_stream.readBits(u8, 4, &out_bits)); + assert(out_bits == 4); + assert(5 == try bit_stream.readBits(u9, 5, &out_bits)); + assert(out_bits == 5); + assert(1 == try bit_stream.readBits(u1, 1, &out_bits)); + assert(out_bits == 1); + + assertError(bit_stream.readBitsNoEof(u1, 1), error.EndOfStream); + } + try os.deleteFile(tmp_file_name); +} + +fn testIntSerializerDeserializer(comptime endian: builtin.Endian, comptime is_packed: bool) !void { + //@NOTE: if this test is taking too long, reduce the maximum tested bitsize + const max_test_bitsize = 128; + + const total_bytes = comptime blk: { + var bytes = 0; + comptime var i = 0; + while (i <= max_test_bitsize) : (i += 1) bytes += (i / 8) + @boolToInt(i % 8 > 0); + break :blk bytes * 2; + }; + + var data_mem: [total_bytes]u8 = undefined; + var out = io.SliceOutStream.init(data_mem[0..]); + const OutError = io.SliceOutStream.Error; + var out_stream = &out.stream; + var serializer = io.Serializer(endian, is_packed, OutError).init(out_stream); + + var in = io.SliceInStream.init(data_mem[0..]); + const InError = io.SliceInStream.Error; + var in_stream = &in.stream; + var deserializer = io.Deserializer(endian, is_packed, InError).init(in_stream); + + comptime var i = 0; + inline while (i <= max_test_bitsize) : (i += 1) { + const U = @IntType(false, i); + const S = @IntType(true, i); + try serializer.serializeInt(U(i)); + if (i != 0) try serializer.serializeInt(S(-1)) else try serializer.serialize(S(0)); + } + try serializer.flush(); + + i = 0; + inline while (i <= max_test_bitsize) : (i += 1) { + const U = @IntType(false, i); + const S = @IntType(true, i); + const x = try deserializer.deserializeInt(U); + const y = try deserializer.deserializeInt(S); + assert(x == U(i)); + if (i != 0) assert(y == S(-1)) else assert(y == 0); + } + + const u8_bit_count = comptime meta.bitCount(u8); + //0 + 1 + 2 + ... n = (n * (n + 1)) / 2 + //and we have each for unsigned and signed, so * 2 + const total_bits = (max_test_bitsize * (max_test_bitsize + 1)); + const extra_packed_byte = @boolToInt(total_bits % u8_bit_count > 0); + const total_packed_bytes = (total_bits / u8_bit_count) + extra_packed_byte; + + assert(in.pos == if (is_packed) total_packed_bytes else total_bytes); +} + +test "Serializer/Deserializer Int" { + try testIntSerializerDeserializer(builtin.Endian.Big, false); + try testIntSerializerDeserializer(builtin.Endian.Little, false); + try testIntSerializerDeserializer(builtin.Endian.Big, true); + try testIntSerializerDeserializer(builtin.Endian.Little, true); +} + +fn testIntSerializerDeserializerInfNaN(comptime endian: builtin.Endian, + comptime is_packed: bool) !void +{ + const mem_size = (16*2 + 32*2 + 64*2 + 128*2) / comptime meta.bitCount(u8); + var data_mem: [mem_size]u8 = undefined; + + var out = io.SliceOutStream.init(data_mem[0..]); + const OutError = io.SliceOutStream.Error; + var out_stream = &out.stream; + var serializer = io.Serializer(endian, is_packed, OutError).init(out_stream); + + var in = io.SliceInStream.init(data_mem[0..]); + const InError = io.SliceInStream.Error; + var in_stream = &in.stream; + var deserializer = io.Deserializer(endian, is_packed, InError).init(in_stream); + + //@TODO: isInf/isNan not currently implemented for f128. + try serializer.serialize(std.math.nan(f16)); + try serializer.serialize(std.math.inf(f16)); + try serializer.serialize(std.math.nan(f32)); + try serializer.serialize(std.math.inf(f32)); + try serializer.serialize(std.math.nan(f64)); + try serializer.serialize(std.math.inf(f64)); + //try serializer.serialize(std.math.nan(f128)); + //try serializer.serialize(std.math.inf(f128)); + const nan_check_f16 = try deserializer.deserialize(f16); + const inf_check_f16 = try deserializer.deserialize(f16); + const nan_check_f32 = try deserializer.deserialize(f32); + const inf_check_f32 = try deserializer.deserialize(f32); + const nan_check_f64 = try deserializer.deserialize(f64); + const inf_check_f64 = try deserializer.deserialize(f64); + //const nan_check_f128 = try deserializer.deserialize(f128); + //const inf_check_f128 = try deserializer.deserialize(f128); + assert(std.math.isNan(nan_check_f16)); + assert(std.math.isInf(inf_check_f16)); + assert(std.math.isNan(nan_check_f32)); + assert(std.math.isInf(inf_check_f32)); + assert(std.math.isNan(nan_check_f64)); + assert(std.math.isInf(inf_check_f64)); + //assert(std.math.isNan(nan_check_f128)); + //assert(std.math.isInf(inf_check_f128)); +} + +test "Serializer/Deserializer Int: Inf/NaN" { + try testIntSerializerDeserializerInfNaN(builtin.Endian.Big, false); + try testIntSerializerDeserializerInfNaN(builtin.Endian.Little, false); + try testIntSerializerDeserializerInfNaN(builtin.Endian.Big, true); + try testIntSerializerDeserializerInfNaN(builtin.Endian.Little, true); +} + +fn testAlternateSerializer(self: var, serializer: var) !void { + try serializer.serialize(self.f_f16); +} + +fn testSerializerDeserializer(comptime endian: builtin.Endian, comptime is_packed: bool) !void { + const ColorType = enum(u4) { + RGB8 = 1, + RA16 = 2, + R32 = 3, + }; + + const TagAlign = union(enum(u32)) { + A: u8, + B: u8, + C: u8, + }; + + const Color = union(ColorType) { + RGB8: struct { + r: u8, + g: u8, + b: u8, + a: u8, + }, + RA16: struct { + r: u16, + a: u16, + }, + R32: u32, + }; + + const PackedStruct = packed struct { + f_i3: i3, + f_u2: u2, + }; + + + + //to test custom serialization + const Custom = struct { + f_f16: f16, + f_unused_u32: u32, + + pub fn deserialize(self: *@This(), deserializer: var) !void { + try deserializer.deserializeInto(&self.f_f16); + self.f_unused_u32 = 47; + } + + pub const serialize = testAlternateSerializer; + }; + + const MyStruct = struct { + f_i3: i3, + f_u8: u8, + f_tag_align: TagAlign, + f_u24: u24, + f_i19: i19, + f_void: void, + f_f32: f32, + f_f128: f128, + f_packed_0: PackedStruct, + f_i7arr: [10]i7, + f_of64n: ?f64, + f_of64v: ?f64, + f_color_type: ColorType, + f_packed_1: PackedStruct, + f_custom: Custom, + f_color: Color, + }; + + const my_inst = MyStruct{ + .f_i3 = -1, + .f_u8 = 8, + .f_tag_align = TagAlign{ .B = 148 }, + .f_u24 = 24, + .f_i19 = 19, + .f_void = {}, + .f_f32 = 32.32, + .f_f128 = 128.128, + .f_packed_0 = PackedStruct{ .f_i3 = -1, .f_u2 = 2 }, + .f_i7arr = [10]i7{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, + .f_of64n = null, + .f_of64v = 64.64, + .f_color_type = ColorType.R32, + .f_packed_1 = PackedStruct{ .f_i3 = 1, .f_u2 = 1 }, + .f_custom = Custom{ .f_f16 = 38.63, .f_unused_u32 = 47 }, + .f_color = Color{ .R32 = 123822 }, + }; + + var data_mem: [@sizeOf(MyStruct)]u8 = undefined; + var out = io.SliceOutStream.init(data_mem[0..]); + const OutError = io.SliceOutStream.Error; + var out_stream = &out.stream; + var serializer = io.Serializer(endian, is_packed, OutError).init(out_stream); + + var in = io.SliceInStream.init(data_mem[0..]); + const InError = io.SliceInStream.Error; + var in_stream = &in.stream; + var deserializer = io.Deserializer(endian, is_packed, InError).init(in_stream); + + try serializer.serialize(my_inst); + + const my_copy = try deserializer.deserialize(MyStruct); + assert(meta.eql(my_copy, my_inst)); +} + +test "Serializer/Deserializer generic" { + try testSerializerDeserializer(builtin.Endian.Big, false); + try testSerializerDeserializer(builtin.Endian.Little, false); + try testSerializerDeserializer(builtin.Endian.Big, true); + try testSerializerDeserializer(builtin.Endian.Little, true); +} + +fn testBadData(comptime endian: builtin.Endian, comptime is_packed: bool) !void { + const E = enum(u14) { + One = 1, + Two = 2, + }; + + const A = struct { + e: E, + }; + + const C = union(E) { + One: u14, + Two: f16, + }; + + var data_mem: [4]u8 = undefined; + var out = io.SliceOutStream.init(data_mem[0..]); + const OutError = io.SliceOutStream.Error; + var out_stream = &out.stream; + var serializer = io.Serializer(endian, is_packed, OutError).init(out_stream); + + var in = io.SliceInStream.init(data_mem[0..]); + const InError = io.SliceInStream.Error; + var in_stream = &in.stream; + var deserializer = io.Deserializer(endian, is_packed, InError).init(in_stream); + + try serializer.serialize(u14(3)); + assertError(deserializer.deserialize(A), error.InvalidEnumTag); + out.pos = 0; + try serializer.serialize(u14(3)); + try serializer.serialize(u14(88)); + assertError(deserializer.deserialize(C), error.InvalidEnumTag); +} + +test "Deserializer bad data" { + try testBadData(builtin.Endian.Big, false); + try testBadData(builtin.Endian.Little, false); + try testBadData(builtin.Endian.Big, true); + try testBadData(builtin.Endian.Little, true); +} \ No newline at end of file diff --git a/std/meta/index.zig b/std/meta/index.zig index 69a3097288..4d195a6a12 100644 --- a/std/meta/index.zig +++ b/std/meta/index.zig @@ -95,7 +95,7 @@ test "std.meta.stringToEnum" { debug.assert(null == stringToEnum(E1, "C")); } -pub fn bitCount(comptime T: type) u32 { +pub fn bitCount(comptime T: type) comptime_int { return switch (@typeInfo(T)) { TypeId.Int => |info| info.bits, TypeId.Float => |info| info.bits, @@ -108,7 +108,7 @@ test "std.meta.bitCount" { debug.assert(bitCount(f32) == 32); } -pub fn alignment(comptime T: type) u29 { +pub fn alignment(comptime T: type) comptime_int { //@alignOf works on non-pointer types const P = if (comptime trait.is(TypeId.Pointer)(T)) T else *T; return @typeInfo(P).Pointer.alignment; @@ -386,6 +386,33 @@ test "std.meta.activeTag" { debug.assert(activeTag(u) == UE.Float); } +///Given a tagged union type, and an enum, return the type of the union +/// field corresponding to the enum tag. +pub fn TagPayloadType(comptime U: type, tag: var) type { + const Tag = @typeOf(tag); + debug.assert(trait.is(builtin.TypeId.Union)(U)); + debug.assert(trait.is(builtin.TypeId.Enum)(Tag)); + + const info = @typeInfo(U).Union; + + inline for (info.fields) |field_info| { + if (field_info.enum_field.?.value == @enumToInt(tag)) return field_info.field_type; + } + unreachable; +} + +test "std.meta.TagPayloadType" { + const Event = union(enum) { + Moved: struct { + from: i32, + to: i32, + }, + }; + const MovedEvent = TagPayloadType(Event, Event.Moved); + var e: Event = undefined; + debug.assert(MovedEvent == @typeOf(e.Moved)); +} + ///Compares two of any type for equality. Containers are compared on a field-by-field basis, /// where possible. Pointers are not followed. pub fn eql(a: var, b: @typeOf(a)) bool { @@ -439,6 +466,11 @@ pub fn eql(a: var, b: @typeOf(a)) bool { builtin.TypeInfo.Pointer.Size.Slice => return a.ptr == b.ptr and a.len == b.len, } }, + builtin.TypeId.Optional => { + if(a == null and b == null) return true; + if(a == null or b == null) return false; + return eql(a.?, b.?); + }, else => return a == b, } } @@ -452,7 +484,7 @@ test "std.meta.eql" { const U = union(enum) { s: S, - f: f32, + f: ?f32, }; const s_1 = S{ diff --git a/std/meta/trait.zig b/std/meta/trait.zig index caf7f1be04..c9d9e971c9 100644 --- a/std/meta/trait.zig +++ b/std/meta/trait.zig @@ -231,6 +231,37 @@ test "std.meta.trait.isPacked" { debug.assert(!isPacked(u8)); } +/// +pub fn isUnsignedInt(comptime T: type) bool { + return switch (@typeId(T)) { + builtin.TypeId.Int => !@typeInfo(T).Int.is_signed, + else => false, + }; +} + +test "isUnsignedInt" { + debug.assert(isUnsignedInt(u32) == true); + debug.assert(isUnsignedInt(comptime_int) == false); + debug.assert(isUnsignedInt(i64) == false); + debug.assert(isUnsignedInt(f64) == false); +} + +/// +pub fn isSignedInt(comptime T: type) bool { + return switch (@typeId(T)) { + builtin.TypeId.ComptimeInt => true, + builtin.TypeId.Int => @typeInfo(T).Int.is_signed, + else => false, + }; +} + +test "isSignedInt" { + debug.assert(isSignedInt(u32) == false); + debug.assert(isSignedInt(comptime_int) == true); + debug.assert(isSignedInt(i64) == true); + debug.assert(isSignedInt(f64) == false); +} + /// pub fn isSingleItemPtr(comptime T: type) bool { if (comptime is(builtin.TypeId.Pointer)(T)) {