diff --git a/lib/std/io.zig b/lib/std/io.zig index 3f02128a6c..feeab08804 100644 --- a/lib/std/io.zig +++ b/lib/std/io.zig @@ -178,14 +178,6 @@ pub const changeDetectionStream = @import("io/change_detection_stream.zig").chan pub const FindByteOutStream = @import("io/find_byte_out_stream.zig").FindByteOutStream; pub const findByteOutStream = @import("io/find_byte_out_stream.zig").findByteOutStream; -pub const Packing = @import("io/serialization.zig").Packing; - -pub const Serializer = @import("io/serialization.zig").Serializer; -pub const serializer = @import("io/serialization.zig").serializer; - -pub const Deserializer = @import("io/serialization.zig").Deserializer; -pub const deserializer = @import("io/serialization.zig").deserializer; - pub const BufferedAtomicFile = @import("io/buffered_atomic_file.zig").BufferedAtomicFile; pub const StreamSource = @import("io/stream_source.zig").StreamSource; @@ -220,7 +212,6 @@ test "" { _ = @import("io/writer.zig"); _ = @import("io/peek_stream.zig"); _ = @import("io/seekable_stream.zig"); - _ = @import("io/serialization.zig"); _ = @import("io/stream_source.zig"); _ = @import("io/test.zig"); } diff --git a/lib/std/io/serialization.zig b/lib/std/io/serialization.zig deleted file mode 100644 index 3fbc203242..0000000000 --- a/lib/std/io/serialization.zig +++ /dev/null @@ -1,616 +0,0 @@ -// SPDX-License-Identifier: MIT -// Copyright (c) 2015-2020 Zig Contributors -// This file is part of [zig](https://ziglang.org/), which is MIT licensed. -// The MIT license requires this copyright notice to be included in all copies -// and substantial portions of the software. -const std = @import("../std.zig"); -const builtin = std.builtin; -const io = std.io; -const assert = std.debug.assert; -const math = std.math; -const meta = std.meta; -const trait = meta.trait; -const testing = std.testing; - -pub const Packing = enum { - /// Pack data to byte alignment - Byte, - - /// Pack data to bit alignment - Bit, -}; - -/// Creates a deserializer that deserializes types from any stream. -/// If `is_packed` is true, the data stream is treated as bit-packed, -/// otherwise data is expected to be packed to the smallest byte. -/// Types may implement a custom deserialization routine with a -/// function named `deserialize` in the form of: -/// ``` -/// pub fn deserialize(self: *Self, deserializer: anytype) !void -/// ``` -/// which will be called when the deserializer is used to deserialize -/// that type. It will pass a pointer to the type instance to deserialize -/// into and a pointer to the deserializer struct. -pub fn Deserializer(comptime endian: builtin.Endian, comptime packing: Packing, comptime ReaderType: type) type { - return struct { - in_stream: if (packing == .Bit) io.BitReader(endian, ReaderType) else ReaderType, - - const Self = @This(); - - pub fn init(in_stream: ReaderType) Self { - return Self{ - .in_stream = switch (packing) { - .Bit => io.bitReader(endian, in_stream), - .Byte => in_stream, - }, - }; - } - - pub fn alignToByte(self: *Self) void { - if (packing == .Byte) return; - self.in_stream.alignToByte(); - } - - //@BUG: inferred error issue. See: #1386 - fn deserializeInt(self: *Self, comptime T: type) (ReaderType.Error || error{EndOfStream})!T { - comptime assert(trait.is(.Int)(T) or trait.is(.Float)(T)); - - const u8_bit_count = 8; - const t_bit_count = comptime meta.bitCount(T); - - const U = std.meta.Int(.unsigned, t_bit_count); - const Log2U = math.Log2Int(U); - const int_size = (t_bit_count + 7) / 8; - - if (packing == .Bit) { - const result = try self.in_stream.readBitsNoEof(U, t_bit_count); - return @bitCast(T, result); - } - - var buffer: [int_size]u8 = undefined; - const read_size = try self.in_stream.read(buffer[0..]); - if (read_size < int_size) return error.EndOfStream; - - if (int_size == 1) { - if (t_bit_count == 8) return @bitCast(T, buffer[0]); - const PossiblySignedByte = std.meta.Int(@typeInfo(T).Int.signedness, 8); - return @truncate(T, @bitCast(PossiblySignedByte, buffer[0])); - } - - var result = @as(U, 0); - for (buffer) |byte, i| { - switch (endian) { - .Big => { - result = (result << u8_bit_count) | byte; - }, - .Little => { - result |= @as(U, byte) << @intCast(Log2U, u8_bit_count * i); - }, - } - } - - return @bitCast(T, result); - } - - /// Deserializes and returns data of the specified type from the stream - pub fn deserialize(self: *Self, comptime T: type) !T { - var value: T = undefined; - try self.deserializeInto(&value); - return value; - } - - /// Deserializes data into the type pointed to by `ptr` - pub fn deserializeInto(self: *Self, ptr: anytype) !void { - const T = @TypeOf(ptr); - comptime assert(trait.is(.Pointer)(T)); - - if (comptime trait.isSlice(T) or comptime trait.isPtrTo(.Array)(T)) { - for (ptr) |*v| - try self.deserializeInto(v); - return; - } - - comptime assert(trait.isSingleItemPtr(T)); - - const C = comptime meta.Child(T); - const child_type_id = @typeInfo(C); - - //custom deserializer: fn(self: *Self, deserializer: anytype) !void - if (comptime trait.hasFn("deserialize")(C)) return C.deserialize(ptr, self); - - if (comptime trait.isPacked(C) and packing != .Bit) { - var packed_deserializer = deserializer(endian, .Bit, self.in_stream); - return packed_deserializer.deserializeInto(ptr); - } - - switch (child_type_id) { - .Void => return, - .Bool => ptr.* = (try self.deserializeInt(u1)) > 0, - .Float, .Int => ptr.* = try self.deserializeInt(C), - .Struct => { - const info = @typeInfo(C).Struct; - - inline for (info.fields) |*field_info| { - const name = field_info.name; - const FieldType = field_info.field_type; - - if (FieldType == void or FieldType == u0) continue; - - //it doesn't make any sense to read pointers - if (comptime trait.is(.Pointer)(FieldType)) { - @compileError("Will not " ++ "read field " ++ name ++ " of struct " ++ - @typeName(C) ++ " because it " ++ "is of pointer-type " ++ - @typeName(FieldType) ++ "."); - } - - try self.deserializeInto(&@field(ptr, name)); - } - }, - .Union => { - const info = @typeInfo(C).Union; - if (info.tag_type) |TagType| { - //we avoid duplicate iteration over the enum tags - // by getting the int directly and casting it without - // safety. If it is bad, it will be caught anyway. - const TagInt = @TagType(TagType); - const tag = try self.deserializeInt(TagInt); - - inline for (info.fields) |field_info| { - if (@enumToInt(@field(TagType, field_info.name)) == tag) { - const name = field_info.name; - const FieldType = field_info.field_type; - ptr.* = @unionInit(C, name, undefined); - try self.deserializeInto(&@field(ptr, name)); - return; - } - } - //This is reachable if the enum data is bad - return error.InvalidEnumTag; - } - @compileError("Cannot meaningfully deserialize " ++ @typeName(C) ++ - " because it is an untagged union. Use a custom deserialize()."); - }, - .Optional => { - const OC = comptime meta.Child(C); - const exists = (try self.deserializeInt(u1)) > 0; - if (!exists) { - ptr.* = null; - return; - } - - ptr.* = @as(OC, undefined); //make it non-null so the following .? is guaranteed safe - const val_ptr = &ptr.*.?; - try self.deserializeInto(val_ptr); - }, - .Enum => { - var value = try self.deserializeInt(@TagType(C)); - ptr.* = try meta.intToEnum(C, value); - }, - else => { - @compileError("Cannot deserialize " ++ @tagName(child_type_id) ++ " types (unimplemented)."); - }, - } - } - }; -} - -pub fn deserializer( - comptime endian: builtin.Endian, - comptime packing: Packing, - in_stream: anytype, -) Deserializer(endian, packing, @TypeOf(in_stream)) { - return Deserializer(endian, packing, @TypeOf(in_stream)).init(in_stream); -} - -/// Creates a serializer that serializes types to any stream. -/// If `is_packed` is true, the data will be bit-packed into the stream. -/// Note that the you must call `serializer.flush()` when you are done -/// writing bit-packed data in order ensure any unwritten bits are committed. -/// If `is_packed` is false, data is packed to the smallest byte. In the case -/// of packed structs, the struct will written bit-packed and with the specified -/// endianess, after which data will resume being written at the next byte boundary. -/// Types may implement a custom serialization routine with a -/// function named `serialize` in the form of: -/// ``` -/// pub fn serialize(self: Self, serializer: anytype) !void -/// ``` -/// which will be called when the serializer is used to serialize that type. It will -/// pass a const pointer to the type instance to be serialized and a pointer -/// to the serializer struct. -pub fn Serializer(comptime endian: builtin.Endian, comptime packing: Packing, comptime OutStreamType: type) type { - return struct { - out_stream: if (packing == .Bit) io.BitOutStream(endian, OutStreamType) else OutStreamType, - - const Self = @This(); - pub const Error = OutStreamType.Error; - - pub fn init(out_stream: OutStreamType) Self { - return Self{ - .out_stream = switch (packing) { - .Bit => io.bitOutStream(endian, out_stream), - .Byte => out_stream, - }, - }; - } - - /// Flushes any unwritten bits to the stream - pub fn flush(self: *Self) Error!void { - if (packing == .Bit) return self.out_stream.flushBits(); - } - - fn serializeInt(self: *Self, value: anytype) Error!void { - const T = @TypeOf(value); - comptime assert(trait.is(.Int)(T) or trait.is(.Float)(T)); - - const t_bit_count = comptime meta.bitCount(T); - const u8_bit_count = comptime meta.bitCount(u8); - - const U = std.meta.Int(.unsigned, t_bit_count); - const Log2U = math.Log2Int(U); - const int_size = (t_bit_count + 7) / 8; - - const u_value = @bitCast(U, value); - - if (packing == .Bit) return self.out_stream.writeBits(u_value, t_bit_count); - - var buffer: [int_size]u8 = undefined; - if (int_size == 1) buffer[0] = u_value; - - for (buffer) |*byte, i| { - const idx = switch (endian) { - .Big => int_size - i - 1, - .Little => i, - }; - const shift = @intCast(Log2U, idx * u8_bit_count); - const v = u_value >> shift; - byte.* = if (t_bit_count < u8_bit_count) v else @truncate(u8, v); - } - - try self.out_stream.writeAll(&buffer); - } - - /// Serializes the passed value into the stream - pub fn serialize(self: *Self, value: anytype) Error!void { - const T = comptime @TypeOf(value); - - if (comptime trait.isIndexable(T)) { - for (value) |v| - try self.serialize(v); - return; - } - - //custom serializer: fn(self: Self, serializer: anytype) !void - if (comptime trait.hasFn("serialize")(T)) return T.serialize(value, self); - - if (comptime trait.isPacked(T) and packing != .Bit) { - var packed_serializer = Serializer(endian, .Bit, OutStreamType).init(self.out_stream); - try packed_serializer.serialize(value); - try packed_serializer.flush(); - return; - } - - switch (@typeInfo(T)) { - .Void => return, - .Bool => try self.serializeInt(@as(u1, @boolToInt(value))), - .Float, .Int => try self.serializeInt(value), - .Struct => { - const info = @typeInfo(T); - - inline for (info.Struct.fields) |*field_info| { - const name = field_info.name; - const FieldType = field_info.field_type; - - if (FieldType == void or FieldType == u0) continue; - - //It doesn't make sense to write pointers - if (comptime trait.is(.Pointer)(FieldType)) { - @compileError("Will not " ++ "serialize field " ++ name ++ - " of struct " ++ @typeName(T) ++ " because it " ++ - "is of pointer-type " ++ @typeName(FieldType) ++ "."); - } - try self.serialize(@field(value, name)); - } - }, - .Union => { - const info = @typeInfo(T).Union; - if (info.tag_type) |TagType| { - const active_tag = meta.activeTag(value); - try self.serialize(active_tag); - //This inline loop is necessary because active_tag is a runtime - // value, but @field requires a comptime value. Our alternative - // is to check each field for a match - inline for (info.fields) |field_info| { - if (@field(TagType, field_info.name) == active_tag) { - const name = field_info.name; - const FieldType = field_info.field_type; - try self.serialize(@field(value, name)); - return; - } - } - unreachable; - } - @compileError("Cannot meaningfully serialize " ++ @typeName(T) ++ - " because it is an untagged union. Use a custom serialize()."); - }, - .Optional => { - if (value == null) { - try self.serializeInt(@as(u1, @boolToInt(false))); - return; - } - try self.serializeInt(@as(u1, @boolToInt(true))); - - const OC = comptime meta.Child(T); - const val_ptr = &value.?; - try self.serialize(val_ptr.*); - }, - .Enum => { - try self.serializeInt(@enumToInt(value)); - }, - else => @compileError("Cannot serialize " ++ @tagName(@typeInfo(T)) ++ " types (unimplemented)."), - } - } - }; -} - -pub fn serializer( - comptime endian: builtin.Endian, - comptime packing: Packing, - out_stream: anytype, -) Serializer(endian, packing, @TypeOf(out_stream)) { - return Serializer(endian, packing, @TypeOf(out_stream)).init(out_stream); -} - -fn testIntSerializerDeserializer(comptime endian: builtin.Endian, comptime packing: io.Packing) !void { - @setEvalBranchQuota(1500); - //@NOTE: if this test is taking too long, reduce the maximum tested bitsize - const max_test_bitsize = 128; - - const total_bytes = comptime blk: { - var bytes = 0; - comptime var i = 0; - while (i <= max_test_bitsize) : (i += 1) bytes += (i / 8) + @boolToInt(i % 8 > 0); - break :blk bytes * 2; - }; - - var data_mem: [total_bytes]u8 = undefined; - var out = io.fixedBufferStream(&data_mem); - var _serializer = serializer(endian, packing, out.outStream()); - - var in = io.fixedBufferStream(&data_mem); - var _deserializer = deserializer(endian, packing, in.reader()); - - comptime var i = 0; - inline while (i <= max_test_bitsize) : (i += 1) { - const U = std.meta.Int(.unsigned, i); - const S = std.meta.Int(.signed, i); - try _serializer.serializeInt(@as(U, i)); - if (i != 0) try _serializer.serializeInt(@as(S, -1)) else try _serializer.serialize(@as(S, 0)); - } - try _serializer.flush(); - - i = 0; - inline while (i <= max_test_bitsize) : (i += 1) { - const U = std.meta.Int(.unsigned, i); - const S = std.meta.Int(.signed, i); - const x = try _deserializer.deserializeInt(U); - const y = try _deserializer.deserializeInt(S); - testing.expect(x == @as(U, i)); - if (i != 0) testing.expect(y == @as(S, -1)) else testing.expect(y == 0); - } - - const u8_bit_count = comptime meta.bitCount(u8); - //0 + 1 + 2 + ... n = (n * (n + 1)) / 2 - //and we have each for unsigned and signed, so * 2 - const total_bits = (max_test_bitsize * (max_test_bitsize + 1)); - const extra_packed_byte = @boolToInt(total_bits % u8_bit_count > 0); - const total_packed_bytes = (total_bits / u8_bit_count) + extra_packed_byte; - - testing.expect(in.pos == if (packing == .Bit) total_packed_bytes else total_bytes); - - //Verify that empty error set works with serializer. - //deserializer is covered by FixedBufferStream - var null_serializer = io.serializer(endian, packing, std.io.null_out_stream); - try null_serializer.serialize(data_mem[0..]); - try null_serializer.flush(); -} - -test "Serializer/Deserializer Int" { - try testIntSerializerDeserializer(.Big, .Byte); - try testIntSerializerDeserializer(.Little, .Byte); - // TODO these tests are disabled due to tripping an LLVM assertion - // https://github.com/ziglang/zig/issues/2019 - //try testIntSerializerDeserializer(builtin.Endian.Big, true); - //try testIntSerializerDeserializer(builtin.Endian.Little, true); -} - -fn testIntSerializerDeserializerInfNaN( - comptime endian: builtin.Endian, - comptime packing: io.Packing, -) !void { - const mem_size = (16 * 2 + 32 * 2 + 64 * 2 + 128 * 2) / comptime meta.bitCount(u8); - var data_mem: [mem_size]u8 = undefined; - - var out = io.fixedBufferStream(&data_mem); - var _serializer = serializer(endian, packing, out.outStream()); - - var in = io.fixedBufferStream(&data_mem); - var _deserializer = deserializer(endian, packing, in.reader()); - - //@TODO: isInf/isNan not currently implemented for f128. - try _serializer.serialize(std.math.nan(f16)); - try _serializer.serialize(std.math.inf(f16)); - try _serializer.serialize(std.math.nan(f32)); - try _serializer.serialize(std.math.inf(f32)); - try _serializer.serialize(std.math.nan(f64)); - try _serializer.serialize(std.math.inf(f64)); - //try serializer.serialize(std.math.nan(f128)); - //try serializer.serialize(std.math.inf(f128)); - const nan_check_f16 = try _deserializer.deserialize(f16); - const inf_check_f16 = try _deserializer.deserialize(f16); - const nan_check_f32 = try _deserializer.deserialize(f32); - _deserializer.alignToByte(); - const inf_check_f32 = try _deserializer.deserialize(f32); - const nan_check_f64 = try _deserializer.deserialize(f64); - const inf_check_f64 = try _deserializer.deserialize(f64); - //const nan_check_f128 = try deserializer.deserialize(f128); - //const inf_check_f128 = try deserializer.deserialize(f128); - testing.expect(std.math.isNan(nan_check_f16)); - testing.expect(std.math.isInf(inf_check_f16)); - testing.expect(std.math.isNan(nan_check_f32)); - testing.expect(std.math.isInf(inf_check_f32)); - testing.expect(std.math.isNan(nan_check_f64)); - testing.expect(std.math.isInf(inf_check_f64)); - //expect(std.math.isNan(nan_check_f128)); - //expect(std.math.isInf(inf_check_f128)); -} - -test "Serializer/Deserializer Int: Inf/NaN" { - try testIntSerializerDeserializerInfNaN(.Big, .Byte); - try testIntSerializerDeserializerInfNaN(.Little, .Byte); - try testIntSerializerDeserializerInfNaN(.Big, .Bit); - try testIntSerializerDeserializerInfNaN(.Little, .Bit); -} - -fn testAlternateSerializer(self: anytype, _serializer: anytype) !void { - try _serializer.serialize(self.f_f16); -} - -fn testSerializerDeserializer(comptime endian: builtin.Endian, comptime packing: io.Packing) !void { - const ColorType = enum(u4) { - RGB8 = 1, - RA16 = 2, - R32 = 3, - }; - - const TagAlign = union(enum(u32)) { - A: u8, - B: u8, - C: u8, - }; - - const Color = union(ColorType) { - RGB8: struct { - r: u8, - g: u8, - b: u8, - a: u8, - }, - RA16: struct { - r: u16, - a: u16, - }, - R32: u32, - }; - - const PackedStruct = packed struct { - f_i3: i3, - f_u2: u2, - }; - - //to test custom serialization - const Custom = struct { - f_f16: f16, - f_unused_u32: u32, - - pub fn deserialize(self: *@This(), _deserializer: anytype) !void { - try _deserializer.deserializeInto(&self.f_f16); - self.f_unused_u32 = 47; - } - - pub const serialize = testAlternateSerializer; - }; - - const MyStruct = struct { - f_i3: i3, - f_u8: u8, - f_tag_align: TagAlign, - f_u24: u24, - f_i19: i19, - f_void: void, - f_f32: f32, - f_f128: f128, - f_packed_0: PackedStruct, - f_i7arr: [10]i7, - f_of64n: ?f64, - f_of64v: ?f64, - f_color_type: ColorType, - f_packed_1: PackedStruct, - f_custom: Custom, - f_color: Color, - }; - - const my_inst = MyStruct{ - .f_i3 = -1, - .f_u8 = 8, - .f_tag_align = TagAlign{ .B = 148 }, - .f_u24 = 24, - .f_i19 = 19, - .f_void = {}, - .f_f32 = 32.32, - .f_f128 = 128.128, - .f_packed_0 = PackedStruct{ .f_i3 = -1, .f_u2 = 2 }, - .f_i7arr = [10]i7{ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 }, - .f_of64n = null, - .f_of64v = 64.64, - .f_color_type = ColorType.R32, - .f_packed_1 = PackedStruct{ .f_i3 = 1, .f_u2 = 1 }, - .f_custom = Custom{ .f_f16 = 38.63, .f_unused_u32 = 47 }, - .f_color = Color{ .R32 = 123822 }, - }; - - var data_mem: [@sizeOf(MyStruct)]u8 = undefined; - var out = io.fixedBufferStream(&data_mem); - var _serializer = serializer(endian, packing, out.outStream()); - - var in = io.fixedBufferStream(&data_mem); - var _deserializer = deserializer(endian, packing, in.reader()); - - try _serializer.serialize(my_inst); - - const my_copy = try _deserializer.deserialize(MyStruct); - testing.expect(meta.eql(my_copy, my_inst)); -} - -test "Serializer/Deserializer generic" { - try testSerializerDeserializer(builtin.Endian.Big, .Byte); - try testSerializerDeserializer(builtin.Endian.Little, .Byte); - try testSerializerDeserializer(builtin.Endian.Big, .Bit); - try testSerializerDeserializer(builtin.Endian.Little, .Bit); -} - -fn testBadData(comptime endian: builtin.Endian, comptime packing: io.Packing) !void { - const E = enum(u14) { - One = 1, - Two = 2, - }; - - const A = struct { - e: E, - }; - - const C = union(E) { - One: u14, - Two: f16, - }; - - var data_mem: [4]u8 = undefined; - var out = io.fixedBufferStream(&data_mem); - var _serializer = serializer(endian, packing, out.outStream()); - - var in = io.fixedBufferStream(&data_mem); - var _deserializer = deserializer(endian, packing, in.reader()); - - try _serializer.serialize(@as(u14, 3)); - testing.expectError(error.InvalidEnumTag, _deserializer.deserialize(A)); - out.pos = 0; - try _serializer.serialize(@as(u14, 3)); - try _serializer.serialize(@as(u14, 88)); - testing.expectError(error.InvalidEnumTag, _deserializer.deserialize(C)); -} - -test "Deserializer bad data" { - try testBadData(.Big, .Byte); - try testBadData(.Little, .Byte); - try testBadData(.Big, .Bit); - try testBadData(.Little, .Bit); -}