From 67e6df4313a982fa08ab6ec6de1da427aaf83409 Mon Sep 17 00:00:00 2001 From: mlugg Date: Fri, 18 Jul 2025 09:53:57 +0100 Subject: [PATCH 01/26] tests: remove more old async tests The rejection of #6025 indicates that if stackless coroutines return to Zig, they will look quite different; see #23446 for the working draft proposal for their return (though it will definitely be tweaked before being accepted). Some of this test coverage was deleted in 40d11cc, but because stackless coroutines will take on a new form if re-introduced, I anticipate that essentially *none* of this coverage will be relevant. Of course, if it for some reason is, we can always grab it from the Git history. --- test/behavior/type_info.zig | 14 ----------- .../async/Frame_of_generic_function.zig | 14 ----------- .../async/bad_alignment_in_asynccall.zig | 13 ---------- .../async/exported_async_function.zig | 7 ------ ..._called_outside_of_function_definition.zig | 11 --------- .../frame_causes_function_to_be_async.zig | 13 ---------- ...eventually_is_inferred_to_become_async.zig | 15 ------------ ...c_function_pointer_passed_to_asyncCall.zig | 13 ---------- ...bad_implicit_casting_of_anyframe_types.zig | 24 ------------------- ...g_type_for_argument_tuple_to_asyncCall.zig | 14 ----------- ...suspend function call, callee suspends.zig | 20 ---------------- 11 files changed, 158 deletions(-) delete mode 100644 test/cases/compile_errors/async/Frame_of_generic_function.zig delete mode 100644 test/cases/compile_errors/async/bad_alignment_in_asynccall.zig delete mode 100644 test/cases/compile_errors/async/exported_async_function.zig delete mode 100644 test/cases/compile_errors/async/frame_called_outside_of_function_definition.zig delete mode 100644 test/cases/compile_errors/async/frame_causes_function_to_be_async.zig delete mode 100644 test/cases/compile_errors/async/non-async_function_pointer_eventually_is_inferred_to_become_async.zig delete mode 100644 test/cases/compile_errors/async/non_async_function_pointer_passed_to_asyncCall.zig delete mode 100644 test/cases/compile_errors/async/prevent_bad_implicit_casting_of_anyframe_types.zig delete mode 100644 test/cases/compile_errors/async/wrong_type_for_argument_tuple_to_asyncCall.zig delete mode 100644 test/cases/safety/nosuspend function call, callee suspends.zig diff --git a/test/behavior/type_info.zig b/test/behavior/type_info.zig index f19d90696b..aea8cbb6aa 100644 --- a/test/behavior/type_info.zig +++ b/test/behavior/type_info.zig @@ -539,20 +539,6 @@ fn add(a: i32, b: i32) i32 { return a + b; } -test "type info for async frames" { - if (true) { - // https://github.com/ziglang/zig/issues/6025 - return error.SkipZigTest; - } - - switch (@typeInfo(@Frame(add))) { - .frame => |frame| { - try expect(@as(@TypeOf(add), @ptrCast(frame.function)) == add); - }, - else => unreachable, - } -} - test "Declarations are returned in declaration order" { if (builtin.zig_backend == .stage2_sparc64) return error.SkipZigTest; // TODO if (builtin.zig_backend == .stage2_spirv) return error.SkipZigTest; diff --git a/test/cases/compile_errors/async/Frame_of_generic_function.zig b/test/cases/compile_errors/async/Frame_of_generic_function.zig deleted file mode 100644 index af0fb5c72e..0000000000 --- a/test/cases/compile_errors/async/Frame_of_generic_function.zig +++ /dev/null @@ -1,14 +0,0 @@ -export fn entry() void { - var frame: @Frame(func) = undefined; - _ = &frame; -} -fn func(comptime T: type) void { - var x: T = undefined; - _ = &x; -} - -// error -// backend=stage1 -// target=native -// -// tmp.zig:2:16: error: @Frame() of generic function diff --git a/test/cases/compile_errors/async/bad_alignment_in_asynccall.zig b/test/cases/compile_errors/async/bad_alignment_in_asynccall.zig deleted file mode 100644 index c30815e7e0..0000000000 --- a/test/cases/compile_errors/async/bad_alignment_in_asynccall.zig +++ /dev/null @@ -1,13 +0,0 @@ -export fn entry() void { - var ptr: fn () callconv(.@"async") void = func; - var bytes: [64]u8 = undefined; - _ = @asyncCall(&bytes, {}, ptr, .{}); - _ = &ptr; -} -fn func() callconv(.@"async") void {} - -// error -// backend=stage1 -// target=aarch64-linux-none -// -// tmp.zig:4:21: error: expected type '[]align(8) u8', found '*[64]u8' diff --git a/test/cases/compile_errors/async/exported_async_function.zig b/test/cases/compile_errors/async/exported_async_function.zig deleted file mode 100644 index c3be7d4b80..0000000000 --- a/test/cases/compile_errors/async/exported_async_function.zig +++ /dev/null @@ -1,7 +0,0 @@ -export fn foo() callconv(.@"async") void {} - -// error -// backend=stage1 -// target=native -// -// tmp.zig:1:1: error: exported function cannot be async diff --git a/test/cases/compile_errors/async/frame_called_outside_of_function_definition.zig b/test/cases/compile_errors/async/frame_called_outside_of_function_definition.zig deleted file mode 100644 index d140998152..0000000000 --- a/test/cases/compile_errors/async/frame_called_outside_of_function_definition.zig +++ /dev/null @@ -1,11 +0,0 @@ -var handle_undef: anyframe = undefined; -var handle_dummy: anyframe = @frame(); -export fn entry() bool { - return handle_undef == handle_dummy; -} - -// error -// backend=stage1 -// target=native -// -// tmp.zig:2:30: error: @frame() called outside of function definition diff --git a/test/cases/compile_errors/async/frame_causes_function_to_be_async.zig b/test/cases/compile_errors/async/frame_causes_function_to_be_async.zig deleted file mode 100644 index f8493b08b2..0000000000 --- a/test/cases/compile_errors/async/frame_causes_function_to_be_async.zig +++ /dev/null @@ -1,13 +0,0 @@ -export fn entry() void { - func(); -} -fn func() void { - _ = @frame(); -} - -// error -// backend=stage1 -// target=native -// -// tmp.zig:1:1: error: function with calling convention 'C' cannot be async -// tmp.zig:5:9: note: @frame() causes function to be async diff --git a/test/cases/compile_errors/async/non-async_function_pointer_eventually_is_inferred_to_become_async.zig b/test/cases/compile_errors/async/non-async_function_pointer_eventually_is_inferred_to_become_async.zig deleted file mode 100644 index e18b420028..0000000000 --- a/test/cases/compile_errors/async/non-async_function_pointer_eventually_is_inferred_to_become_async.zig +++ /dev/null @@ -1,15 +0,0 @@ -export fn a() void { - var non_async_fn: fn () void = undefined; - non_async_fn = func; -} -fn func() void { - suspend {} -} - -// error -// backend=stage1 -// target=native -// -// tmp.zig:5:1: error: 'func' cannot be async -// tmp.zig:3:20: note: required to be non-async here -// tmp.zig:6:5: note: suspends here diff --git a/test/cases/compile_errors/async/non_async_function_pointer_passed_to_asyncCall.zig b/test/cases/compile_errors/async/non_async_function_pointer_passed_to_asyncCall.zig deleted file mode 100644 index b62524f6de..0000000000 --- a/test/cases/compile_errors/async/non_async_function_pointer_passed_to_asyncCall.zig +++ /dev/null @@ -1,13 +0,0 @@ -export fn entry() void { - var ptr = afunc; - var bytes: [100]u8 align(16) = undefined; - _ = @asyncCall(&bytes, {}, ptr, .{}); - _ = &ptr; -} -fn afunc() void {} - -// error -// backend=stage1 -// target=native -// -// tmp.zig:4:32: error: expected async function, found 'fn () void' diff --git a/test/cases/compile_errors/async/prevent_bad_implicit_casting_of_anyframe_types.zig b/test/cases/compile_errors/async/prevent_bad_implicit_casting_of_anyframe_types.zig deleted file mode 100644 index 6ab99bf00d..0000000000 --- a/test/cases/compile_errors/async/prevent_bad_implicit_casting_of_anyframe_types.zig +++ /dev/null @@ -1,24 +0,0 @@ -export fn a() void { - var x: anyframe = undefined; - var y: anyframe->i32 = x; - _ = .{ &x, &y }; -} -export fn b() void { - var x: i32 = undefined; - var y: anyframe->i32 = x; - _ = .{ &x, &y }; -} -export fn c() void { - var x: @Frame(func) = undefined; - var y: anyframe->i32 = &x; - _ = .{ &x, &y }; -} -fn func() void {} - -// error -// backend=stage1 -// target=native -// -// :3:28: error: expected type 'anyframe->i32', found 'anyframe' -// :8:28: error: expected type 'anyframe->i32', found 'i32' -// tmp.zig:13:29: error: expected type 'anyframe->i32', found '*@Frame(func)' diff --git a/test/cases/compile_errors/async/wrong_type_for_argument_tuple_to_asyncCall.zig b/test/cases/compile_errors/async/wrong_type_for_argument_tuple_to_asyncCall.zig deleted file mode 100644 index 7a9be0a8cc..0000000000 --- a/test/cases/compile_errors/async/wrong_type_for_argument_tuple_to_asyncCall.zig +++ /dev/null @@ -1,14 +0,0 @@ -export fn entry1() void { - var frame: @Frame(foo) = undefined; - @asyncCall(&frame, {}, foo, {}); -} - -fn foo() i32 { - return 0; -} - -// error -// backend=stage1 -// target=native -// -// tmp.zig:3:33: error: expected tuple or struct, found 'void' diff --git a/test/cases/safety/nosuspend function call, callee suspends.zig b/test/cases/safety/nosuspend function call, callee suspends.zig deleted file mode 100644 index 50f457f314..0000000000 --- a/test/cases/safety/nosuspend function call, callee suspends.zig +++ /dev/null @@ -1,20 +0,0 @@ -const std = @import("std"); - -pub fn panic(message: []const u8, stack_trace: ?*std.builtin.StackTrace, _: ?usize) noreturn { - _ = message; - _ = stack_trace; - std.process.exit(0); -} -pub fn main() !void { - _ = nosuspend add(101, 100); - return error.TestFailed; -} -fn add(a: i32, b: i32) i32 { - if (a > 100) { - suspend {} - } - return a + b; -} -// run -// backend=stage1 -// target=native From 4e6a04929d22015c85780f08fe46d6378ae82b66 Mon Sep 17 00:00:00 2001 From: Alexandre Date: Fri, 18 Jul 2025 17:45:45 -0400 Subject: [PATCH 02/26] Changed u64 to usize to fix #24208 --- lib/std/hash/RapidHash.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/hash/RapidHash.zig b/lib/std/hash/RapidHash.zig index 030c570df9..bb1b45bcc4 100644 --- a/lib/std/hash/RapidHash.zig +++ b/lib/std/hash/RapidHash.zig @@ -20,7 +20,7 @@ pub fn hash(seed: u64, input: []const u8) u64 { if (len <= 16) { if (len >= 4) { - const d: u64 = ((len & 24) >> @intCast(len >> 3)); + const d: usize = ((len & 24) >> @intCast(len >> 3)); const e = len - 4; a = (r32(k) << 32) | r32(k[e..]); b = ((r32(k[d..]) << 32) | r32(k[(e - d)..])); From a0d16829215f56fe42548f1014cba0f2df5a1609 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 19 Jul 2025 11:49:33 -0700 Subject: [PATCH 03/26] std.hash.RapidHash: remove Its design keeps evolving. See https://github.com/Nicoshev/rapidhash/releases It's great to see the design improving, but over time, this will lead to code rot; versions that aren't widely used but would still have to live in the standard library forever and be maintained. Better to be maintained as an external dependency that applications can opt into. Then, in a few years, if a version proves to be stable and widely adopted, it could be considered for inclusion in the standard library. --- lib/std/hash.zig | 2 - lib/std/hash/RapidHash.zig | 125 ------------------------------------- lib/std/hash/benchmark.zig | 6 -- 3 files changed, 133 deletions(-) delete mode 100644 lib/std/hash/RapidHash.zig diff --git a/lib/std/hash.zig b/lib/std/hash.zig index e0eca9e391..5f1697a236 100644 --- a/lib/std/hash.zig +++ b/lib/std/hash.zig @@ -32,8 +32,6 @@ pub const CityHash64 = cityhash.CityHash64; const wyhash = @import("hash/wyhash.zig"); pub const Wyhash = wyhash.Wyhash; -pub const RapidHash = @import("hash/RapidHash.zig"); - const xxhash = @import("hash/xxhash.zig"); pub const XxHash3 = xxhash.XxHash3; pub const XxHash64 = xxhash.XxHash64; diff --git a/lib/std/hash/RapidHash.zig b/lib/std/hash/RapidHash.zig deleted file mode 100644 index bb1b45bcc4..0000000000 --- a/lib/std/hash/RapidHash.zig +++ /dev/null @@ -1,125 +0,0 @@ -const std = @import("std"); - -const readInt = std.mem.readInt; -const assert = std.debug.assert; -const expect = std.testing.expect; -const expectEqual = std.testing.expectEqual; - -const RAPID_SEED: u64 = 0xbdd89aa982704029; -const RAPID_SECRET: [3]u64 = .{ 0x2d358dccaa6c78a5, 0x8bb84b93962eacc9, 0x4b33a62ed433d4a3 }; - -pub fn hash(seed: u64, input: []const u8) u64 { - const sc = RAPID_SECRET; - const len = input.len; - var a: u64 = 0; - var b: u64 = 0; - var k = input; - var is: [3]u64 = .{ seed, 0, 0 }; - - is[0] ^= mix(seed ^ sc[0], sc[1]) ^ len; - - if (len <= 16) { - if (len >= 4) { - const d: usize = ((len & 24) >> @intCast(len >> 3)); - const e = len - 4; - a = (r32(k) << 32) | r32(k[e..]); - b = ((r32(k[d..]) << 32) | r32(k[(e - d)..])); - } else if (len > 0) - a = (@as(u64, k[0]) << 56) | (@as(u64, k[len >> 1]) << 32) | @as(u64, k[len - 1]); - } else { - var remain = len; - if (len > 48) { - is[1] = is[0]; - is[2] = is[0]; - while (remain >= 96) { - inline for (0..6) |i| { - const m1 = r64(k[8 * i * 2 ..]); - const m2 = r64(k[8 * (i * 2 + 1) ..]); - is[i % 3] = mix(m1 ^ sc[i % 3], m2 ^ is[i % 3]); - } - k = k[96..]; - remain -= 96; - } - if (remain >= 48) { - inline for (0..3) |i| { - const m1 = r64(k[8 * i * 2 ..]); - const m2 = r64(k[8 * (i * 2 + 1) ..]); - is[i] = mix(m1 ^ sc[i], m2 ^ is[i]); - } - k = k[48..]; - remain -= 48; - } - - is[0] ^= is[1] ^ is[2]; - } - - if (remain > 16) { - is[0] = mix(r64(k) ^ sc[2], r64(k[8..]) ^ is[0] ^ sc[1]); - if (remain > 32) { - is[0] = mix(r64(k[16..]) ^ sc[2], r64(k[24..]) ^ is[0]); - } - } - - a = r64(input[len - 16 ..]); - b = r64(input[len - 8 ..]); - } - - a ^= sc[1]; - b ^= is[0]; - mum(&a, &b); - return mix(a ^ sc[0] ^ len, b ^ sc[1]); -} - -test "RapidHash.hash" { - const bytes: []const u8 = "abcdefgh" ** 128; - - const sizes: [13]u64 = .{ 0, 1, 2, 3, 4, 8, 16, 32, 64, 128, 256, 512, 1024 }; - - const outcomes: [13]u64 = .{ - 0x5a6ef77074ebc84b, - 0xc11328477bc0f5d1, - 0x5644ac035e40d569, - 0x347080fbf5fcd81, - 0x56b66b8dc802bcc, - 0xb6bf9055973aac7c, - 0xed56d62eead1e402, - 0xc19072d767da8ffb, - 0x89bb40a9928a4f0d, - 0xe0af7c5e7b6e29fd, - 0x9a3ed35fbedfa11a, - 0x4c684b2119ca19fb, - 0x4b575f5bf25600d6, - }; - - var success: bool = true; - for (sizes, outcomes) |s, e| { - const r = hash(RAPID_SEED, bytes[0..s]); - - expectEqual(e, r) catch |err| { - std.debug.print("Failed on {d}: {!}\n", .{ s, err }); - success = false; - }; - } - try expect(success); -} - -inline fn mum(a: *u64, b: *u64) void { - const r = @as(u128, a.*) * b.*; - a.* = @truncate(r); - b.* = @truncate(r >> 64); -} - -inline fn mix(a: u64, b: u64) u64 { - var copy_a = a; - var copy_b = b; - mum(©_a, ©_b); - return copy_a ^ copy_b; -} - -inline fn r64(p: []const u8) u64 { - return readInt(u64, p[0..8], .little); -} - -inline fn r32(p: []const u8) u64 { - return readInt(u32, p[0..4], .little); -} diff --git a/lib/std/hash/benchmark.zig b/lib/std/hash/benchmark.zig index 980b41b8ae..84d14a35db 100644 --- a/lib/std/hash/benchmark.zig +++ b/lib/std/hash/benchmark.zig @@ -59,12 +59,6 @@ const hashes = [_]Hash{ .ty = hash.crc.Crc32, .name = "crc32", }, - Hash{ - .ty = hash.RapidHash, - .name = "rapidhash", - .has_iterative_api = false, - .init_u64 = 0, - }, Hash{ .ty = hash.CityHash32, .name = "cityhash-32", From a288266f3310e9ba98456c5e968f8ce434be6cc7 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 19 Jul 2025 16:02:06 -0700 Subject: [PATCH 04/26] std.Io.Reader: remove aggressive assert from `fill` with `.fixed("")` you should still be able to do `fill(1)` and have it return error.EndOfStream. --- lib/std/Io/Reader.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/std/Io/Reader.zig b/lib/std/Io/Reader.zig index f2a1ec7287..87e83ffd67 100644 --- a/lib/std/Io/Reader.zig +++ b/lib/std/Io/Reader.zig @@ -990,9 +990,9 @@ pub fn discardDelimiterLimit(r: *Reader, delimiter: u8, limit: Limit) DiscardDel /// Returns `error.EndOfStream` if and only if there are fewer than `n` bytes /// remaining. /// -/// Asserts buffer capacity is at least `n`. +/// If the end of stream is not encountered, asserts buffer capacity is at +/// least `n`. pub fn fill(r: *Reader, n: usize) Error!void { - assert(n <= r.buffer.len); if (r.seek + n <= r.end) { @branchHint(.likely); return; From c30df072bde3794ea5f3794d93f69e13640c6b7a Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 18 Jul 2025 19:32:42 -0700 Subject: [PATCH 05/26] std.json: update to new I/O API also do a little bit of namespace cleanup --- lib/std/json.zig | 101 +- lib/std/json/Scanner.zig | 1767 ++++++++++++++++++++++++++++++ lib/std/json/Stringify.zig | 999 +++++++++++++++++ lib/std/json/dynamic.zig | 18 +- lib/std/json/dynamic_test.zig | 40 +- lib/std/json/fmt.zig | 40 - lib/std/json/hashmap_test.zig | 18 +- lib/std/json/scanner.zig | 1776 ------------------------------- lib/std/json/scanner_test.zig | 78 +- lib/std/json/static.zig | 10 +- lib/std/json/static_test.zig | 30 +- lib/std/json/stringify.zig | 772 -------------- lib/std/json/stringify_test.zig | 504 --------- lib/std/json/test.zig | 11 +- 14 files changed, 2920 insertions(+), 3244 deletions(-) create mode 100644 lib/std/json/Scanner.zig create mode 100644 lib/std/json/Stringify.zig delete mode 100644 lib/std/json/fmt.zig delete mode 100644 lib/std/json/scanner.zig delete mode 100644 lib/std/json/stringify.zig delete mode 100644 lib/std/json/stringify_test.zig diff --git a/lib/std/json.zig b/lib/std/json.zig index c0fb064c6a..f81ac1cd65 100644 --- a/lib/std/json.zig +++ b/lib/std/json.zig @@ -10,8 +10,8 @@ //! The high-level `stringify` serializes a Zig or `Value` type into JSON. const builtin = @import("builtin"); -const testing = @import("std").testing; -const ArrayList = @import("std").ArrayList; +const std = @import("std"); +const testing = std.testing; test Scanner { var scanner = Scanner.initCompleteInput(testing.allocator, "{\"foo\": 123}\n"); @@ -41,11 +41,13 @@ test Value { try testing.expectEqualSlices(u8, "goes", parsed.value.object.get("anything").?.string); } -test writeStream { - var out = ArrayList(u8).init(testing.allocator); +test Stringify { + var out: std.io.Writer.Allocating = .init(testing.allocator); + var write_stream: Stringify = .{ + .writer = &out.writer, + .options = .{ .whitespace = .indent_2 }, + }; defer out.deinit(); - var write_stream = writeStream(out.writer(), .{ .whitespace = .indent_2 }); - defer write_stream.deinit(); try write_stream.beginObject(); try write_stream.objectField("foo"); try write_stream.write(123); @@ -55,16 +57,7 @@ test writeStream { \\ "foo": 123 \\} ; - try testing.expectEqualSlices(u8, expected, out.items); -} - -test stringify { - var out = ArrayList(u8).init(testing.allocator); - defer out.deinit(); - - const T = struct { a: i32, b: []const u8 }; - try stringify(T{ .a = 123, .b = "xy" }, .{}, out.writer()); - try testing.expectEqualSlices(u8, "{\"a\":123,\"b\":\"xy\"}", out.items); + try testing.expectEqualSlices(u8, expected, out.getWritten()); } pub const ObjectMap = @import("json/dynamic.zig").ObjectMap; @@ -73,18 +66,18 @@ pub const Value = @import("json/dynamic.zig").Value; pub const ArrayHashMap = @import("json/hashmap.zig").ArrayHashMap; -pub const validate = @import("json/scanner.zig").validate; -pub const Error = @import("json/scanner.zig").Error; -pub const reader = @import("json/scanner.zig").reader; -pub const default_buffer_size = @import("json/scanner.zig").default_buffer_size; -pub const Token = @import("json/scanner.zig").Token; -pub const TokenType = @import("json/scanner.zig").TokenType; -pub const Diagnostics = @import("json/scanner.zig").Diagnostics; -pub const AllocWhen = @import("json/scanner.zig").AllocWhen; -pub const default_max_value_len = @import("json/scanner.zig").default_max_value_len; -pub const Reader = @import("json/scanner.zig").Reader; -pub const Scanner = @import("json/scanner.zig").Scanner; -pub const isNumberFormattedLikeAnInteger = @import("json/scanner.zig").isNumberFormattedLikeAnInteger; +pub const Scanner = @import("json/Scanner.zig"); +pub const validate = Scanner.validate; +pub const Error = Scanner.Error; +pub const reader = Scanner.reader; +pub const default_buffer_size = Scanner.default_buffer_size; +pub const Token = Scanner.Token; +pub const TokenType = Scanner.TokenType; +pub const Diagnostics = Scanner.Diagnostics; +pub const AllocWhen = Scanner.AllocWhen; +pub const default_max_value_len = Scanner.default_max_value_len; +pub const Reader = Scanner.Reader; +pub const isNumberFormattedLikeAnInteger = Scanner.isNumberFormattedLikeAnInteger; pub const ParseOptions = @import("json/static.zig").ParseOptions; pub const Parsed = @import("json/static.zig").Parsed; @@ -99,27 +92,49 @@ pub const innerParseFromValue = @import("json/static.zig").innerParseFromValue; pub const ParseError = @import("json/static.zig").ParseError; pub const ParseFromValueError = @import("json/static.zig").ParseFromValueError; -pub const StringifyOptions = @import("json/stringify.zig").StringifyOptions; -pub const stringify = @import("json/stringify.zig").stringify; -pub const stringifyMaxDepth = @import("json/stringify.zig").stringifyMaxDepth; -pub const stringifyArbitraryDepth = @import("json/stringify.zig").stringifyArbitraryDepth; -pub const stringifyAlloc = @import("json/stringify.zig").stringifyAlloc; -pub const writeStream = @import("json/stringify.zig").writeStream; -pub const writeStreamMaxDepth = @import("json/stringify.zig").writeStreamMaxDepth; -pub const writeStreamArbitraryDepth = @import("json/stringify.zig").writeStreamArbitraryDepth; -pub const WriteStream = @import("json/stringify.zig").WriteStream; -pub const encodeJsonString = @import("json/stringify.zig").encodeJsonString; -pub const encodeJsonStringChars = @import("json/stringify.zig").encodeJsonStringChars; +pub const Stringify = @import("json/Stringify.zig"); -pub const Formatter = @import("json/fmt.zig").Formatter; -pub const fmt = @import("json/fmt.zig").fmt; +/// Returns a formatter that formats the given value using stringify. +pub fn fmt(value: anytype, options: Stringify.Options) Formatter(@TypeOf(value)) { + return Formatter(@TypeOf(value)){ .value = value, .options = options }; +} + +test fmt { + const expectFmt = std.testing.expectFmt; + try expectFmt("123", "{f}", .{fmt(@as(u32, 123), .{})}); + try expectFmt( + \\{"num":927,"msg":"hello","sub":{"mybool":true}} + , "{f}", .{fmt(struct { + num: u32, + msg: []const u8, + sub: struct { + mybool: bool, + }, + }{ + .num = 927, + .msg = "hello", + .sub = .{ .mybool = true }, + }, .{})}); +} + +/// Formats the given value using stringify. +pub fn Formatter(comptime T: type) type { + return struct { + value: T, + options: Stringify.Options, + + pub fn format(self: @This(), writer: *std.Io.Writer) std.Io.Writer.Error!void { + try Stringify.value(self.value, self.options, writer); + } + }; +} test { _ = @import("json/test.zig"); - _ = @import("json/scanner.zig"); + _ = Scanner; _ = @import("json/dynamic.zig"); _ = @import("json/hashmap.zig"); _ = @import("json/static.zig"); - _ = @import("json/stringify.zig"); + _ = Stringify; _ = @import("json/JSONTestSuite_test.zig"); } diff --git a/lib/std/json/Scanner.zig b/lib/std/json/Scanner.zig new file mode 100644 index 0000000000..b9c3c506a5 --- /dev/null +++ b/lib/std/json/Scanner.zig @@ -0,0 +1,1767 @@ +//! The lowest level parsing API in this package; +//! supports streaming input with a low memory footprint. +//! The memory requirement is `O(d)` where d is the nesting depth of `[]` or `{}` containers in the input. +//! Specifically `d/8` bytes are required for this purpose, +//! with some extra buffer according to the implementation of `std.ArrayList`. +//! +//! This scanner can emit partial tokens; see `std.json.Token`. +//! The input to this class is a sequence of input buffers that you must supply one at a time. +//! Call `feedInput()` with the first buffer, then call `next()` repeatedly until `error.BufferUnderrun` is returned. +//! Then call `feedInput()` again and so forth. +//! Call `endInput()` when the last input buffer has been given to `feedInput()`, either immediately after calling `feedInput()`, +//! or when `error.BufferUnderrun` requests more data and there is no more. +//! Be sure to call `next()` after calling `endInput()` until `Token.end_of_document` has been returned. +//! +//! Notes on standards compliance: https://datatracker.ietf.org/doc/html/rfc8259 +//! * RFC 8259 requires JSON documents be valid UTF-8, +//! but makes an allowance for systems that are "part of a closed ecosystem". +//! I have no idea what that's supposed to mean in the context of a standard specification. +//! This implementation requires inputs to be valid UTF-8. +//! * RFC 8259 contradicts itself regarding whether lowercase is allowed in \u hex digits, +//! but this is probably a bug in the spec, and it's clear that lowercase is meant to be allowed. +//! (RFC 5234 defines HEXDIG to only allow uppercase.) +//! * When RFC 8259 refers to a "character", I assume they really mean a "Unicode scalar value". +//! See http://www.unicode.org/glossary/#unicode_scalar_value . +//! * RFC 8259 doesn't explicitly disallow unpaired surrogate halves in \u escape sequences, +//! but vaguely implies that \u escapes are for encoding Unicode "characters" (i.e. Unicode scalar values?), +//! which would mean that unpaired surrogate halves are forbidden. +//! By contrast ECMA-404 (a competing(/compatible?) JSON standard, which JavaScript's JSON.parse() conforms to) +//! explicitly allows unpaired surrogate halves. +//! This implementation forbids unpaired surrogate halves in \u sequences. +//! If a high surrogate half appears in a \u sequence, +//! then a low surrogate half must immediately follow in \u notation. +//! * RFC 8259 allows implementations to "accept non-JSON forms or extensions". +//! This implementation does not accept any of that. +//! * RFC 8259 allows implementations to put limits on "the size of texts", +//! "the maximum depth of nesting", "the range and precision of numbers", +//! and "the length and character contents of strings". +//! This low-level implementation does not limit these, +//! except where noted above, and except that nesting depth requires memory allocation. +//! Note that this low-level API does not interpret numbers numerically, +//! but simply emits their source form for some higher level code to make sense of. +//! * This low-level implementation allows duplicate object keys, +//! and key/value pairs are emitted in the order they appear in the input. + +const Scanner = @This(); +const std = @import("std"); + +const Allocator = std.mem.Allocator; +const ArrayList = std.ArrayList; +const assert = std.debug.assert; +const BitStack = std.BitStack; + +state: State = .value, +string_is_object_key: bool = false, +stack: BitStack, +value_start: usize = undefined, +utf16_code_units: [2]u16 = undefined, + +input: []const u8 = "", +cursor: usize = 0, +is_end_of_input: bool = false, +diagnostics: ?*Diagnostics = null, + +/// The allocator is only used to track `[]` and `{}` nesting levels. +pub fn initStreaming(allocator: Allocator) @This() { + return .{ + .stack = BitStack.init(allocator), + }; +} +/// Use this if your input is a single slice. +/// This is effectively equivalent to: +/// ``` +/// initStreaming(allocator); +/// feedInput(complete_input); +/// endInput(); +/// ``` +pub fn initCompleteInput(allocator: Allocator, complete_input: []const u8) @This() { + return .{ + .stack = BitStack.init(allocator), + .input = complete_input, + .is_end_of_input = true, + }; +} +pub fn deinit(self: *@This()) void { + self.stack.deinit(); + self.* = undefined; +} + +pub fn enableDiagnostics(self: *@This(), diagnostics: *Diagnostics) void { + diagnostics.cursor_pointer = &self.cursor; + self.diagnostics = diagnostics; +} + +/// Call this whenever you get `error.BufferUnderrun` from `next()`. +/// When there is no more input to provide, call `endInput()`. +pub fn feedInput(self: *@This(), input: []const u8) void { + assert(self.cursor == self.input.len); // Not done with the last input slice. + if (self.diagnostics) |diag| { + diag.total_bytes_before_current_input += self.input.len; + // This usually goes "negative" to measure how far before the beginning + // of the new buffer the current line started. + diag.line_start_cursor -%= self.cursor; + } + self.input = input; + self.cursor = 0; + self.value_start = 0; +} +/// Call this when you will no longer call `feedInput()` anymore. +/// This can be called either immediately after the last `feedInput()`, +/// or at any time afterward, such as when getting `error.BufferUnderrun` from `next()`. +/// Don't forget to call `next*()` after `endInput()` until you get `.end_of_document`. +pub fn endInput(self: *@This()) void { + self.is_end_of_input = true; +} + +pub const NextError = Error || Allocator.Error || error{BufferUnderrun}; +pub const AllocError = Error || Allocator.Error || error{ValueTooLong}; +pub const PeekError = Error || error{BufferUnderrun}; +pub const SkipError = Error || Allocator.Error; +pub const AllocIntoArrayListError = AllocError || error{BufferUnderrun}; + +/// Equivalent to `nextAllocMax(allocator, when, default_max_value_len);` +/// This function is only available after `endInput()` (or `initCompleteInput()`) has been called. +/// See also `std.json.Token` for documentation of `nextAlloc*()` function behavior. +pub fn nextAlloc(self: *@This(), allocator: Allocator, when: AllocWhen) AllocError!Token { + return self.nextAllocMax(allocator, when, default_max_value_len); +} + +/// This function is only available after `endInput()` (or `initCompleteInput()`) has been called. +/// See also `std.json.Token` for documentation of `nextAlloc*()` function behavior. +pub fn nextAllocMax(self: *@This(), allocator: Allocator, when: AllocWhen, max_value_len: usize) AllocError!Token { + assert(self.is_end_of_input); // This function is not available in streaming mode. + const token_type = self.peekNextTokenType() catch |e| switch (e) { + error.BufferUnderrun => unreachable, + else => |err| return err, + }; + switch (token_type) { + .number, .string => { + var value_list = ArrayList(u8).init(allocator); + errdefer { + value_list.deinit(); + } + if (self.allocNextIntoArrayListMax(&value_list, when, max_value_len) catch |e| switch (e) { + error.BufferUnderrun => unreachable, + else => |err| return err, + }) |slice| { + return if (token_type == .number) + Token{ .number = slice } + else + Token{ .string = slice }; + } else { + return if (token_type == .number) + Token{ .allocated_number = try value_list.toOwnedSlice() } + else + Token{ .allocated_string = try value_list.toOwnedSlice() }; + } + }, + + // Simple tokens never alloc. + .object_begin, + .object_end, + .array_begin, + .array_end, + .true, + .false, + .null, + .end_of_document, + => return self.next() catch |e| switch (e) { + error.BufferUnderrun => unreachable, + else => |err| return err, + }, + } +} + +/// Equivalent to `allocNextIntoArrayListMax(value_list, when, default_max_value_len);` +pub fn allocNextIntoArrayList(self: *@This(), value_list: *ArrayList(u8), when: AllocWhen) AllocIntoArrayListError!?[]const u8 { + return self.allocNextIntoArrayListMax(value_list, when, default_max_value_len); +} +/// The next token type must be either `.number` or `.string`. See `peekNextTokenType()`. +/// When allocation is not necessary with `.alloc_if_needed`, +/// this method returns the content slice from the input buffer, and `value_list` is not touched. +/// When allocation is necessary or with `.alloc_always`, this method concatenates partial tokens into the given `value_list`, +/// and returns `null` once the final `.number` or `.string` token has been written into it. +/// In case of an `error.BufferUnderrun`, partial values will be left in the given value_list. +/// The given `value_list` is never reset by this method, so an `error.BufferUnderrun` situation +/// can be resumed by passing the same array list in again. +/// This method does not indicate whether the token content being returned is for a `.number` or `.string` token type; +/// the caller of this method is expected to know which type of token is being processed. +pub fn allocNextIntoArrayListMax(self: *@This(), value_list: *ArrayList(u8), when: AllocWhen, max_value_len: usize) AllocIntoArrayListError!?[]const u8 { + while (true) { + const token = try self.next(); + switch (token) { + // Accumulate partial values. + .partial_number, .partial_string => |slice| { + try appendSlice(value_list, slice, max_value_len); + }, + .partial_string_escaped_1 => |buf| { + try appendSlice(value_list, buf[0..], max_value_len); + }, + .partial_string_escaped_2 => |buf| { + try appendSlice(value_list, buf[0..], max_value_len); + }, + .partial_string_escaped_3 => |buf| { + try appendSlice(value_list, buf[0..], max_value_len); + }, + .partial_string_escaped_4 => |buf| { + try appendSlice(value_list, buf[0..], max_value_len); + }, + + // Return complete values. + .number => |slice| { + if (when == .alloc_if_needed and value_list.items.len == 0) { + // No alloc necessary. + return slice; + } + try appendSlice(value_list, slice, max_value_len); + // The token is complete. + return null; + }, + .string => |slice| { + if (when == .alloc_if_needed and value_list.items.len == 0) { + // No alloc necessary. + return slice; + } + try appendSlice(value_list, slice, max_value_len); + // The token is complete. + return null; + }, + + .object_begin, + .object_end, + .array_begin, + .array_end, + .true, + .false, + .null, + .end_of_document, + => unreachable, // Only .number and .string token types are allowed here. Check peekNextTokenType() before calling this. + + .allocated_number, .allocated_string => unreachable, + } + } +} + +/// This function is only available after `endInput()` (or `initCompleteInput()`) has been called. +/// If the next token type is `.object_begin` or `.array_begin`, +/// this function calls `next()` repeatedly until the corresponding `.object_end` or `.array_end` is found. +/// If the next token type is `.number` or `.string`, +/// this function calls `next()` repeatedly until the (non `.partial_*`) `.number` or `.string` token is found. +/// If the next token type is `.true`, `.false`, or `.null`, this function calls `next()` once. +/// The next token type must not be `.object_end`, `.array_end`, or `.end_of_document`; +/// see `peekNextTokenType()`. +pub fn skipValue(self: *@This()) SkipError!void { + assert(self.is_end_of_input); // This function is not available in streaming mode. + switch (self.peekNextTokenType() catch |e| switch (e) { + error.BufferUnderrun => unreachable, + else => |err| return err, + }) { + .object_begin, .array_begin => { + self.skipUntilStackHeight(self.stackHeight()) catch |e| switch (e) { + error.BufferUnderrun => unreachable, + else => |err| return err, + }; + }, + .number, .string => { + while (true) { + switch (self.next() catch |e| switch (e) { + error.BufferUnderrun => unreachable, + else => |err| return err, + }) { + .partial_number, + .partial_string, + .partial_string_escaped_1, + .partial_string_escaped_2, + .partial_string_escaped_3, + .partial_string_escaped_4, + => continue, + + .number, .string => break, + + else => unreachable, + } + } + }, + .true, .false, .null => { + _ = self.next() catch |e| switch (e) { + error.BufferUnderrun => unreachable, + else => |err| return err, + }; + }, + + .object_end, .array_end, .end_of_document => unreachable, // Attempt to skip a non-value token. + } +} + +/// Skip tokens until an `.object_end` or `.array_end` token results in a `stackHeight()` equal the given stack height. +/// Unlike `skipValue()`, this function is available in streaming mode. +pub fn skipUntilStackHeight(self: *@This(), terminal_stack_height: usize) NextError!void { + while (true) { + switch (try self.next()) { + .object_end, .array_end => { + if (self.stackHeight() == terminal_stack_height) break; + }, + .end_of_document => unreachable, + else => continue, + } + } +} + +/// The depth of `{}` or `[]` nesting levels at the current position. +pub fn stackHeight(self: *const @This()) usize { + return self.stack.bit_len; +} + +/// Pre allocate memory to hold the given number of nesting levels. +/// `stackHeight()` up to the given number will not cause allocations. +pub fn ensureTotalStackCapacity(self: *@This(), height: usize) Allocator.Error!void { + try self.stack.ensureTotalCapacity(height); +} + +/// See `std.json.Token` for documentation of this function. +pub fn next(self: *@This()) NextError!Token { + state_loop: while (true) { + switch (self.state) { + .value => { + switch (try self.skipWhitespaceExpectByte()) { + // Object, Array + '{' => { + try self.stack.push(OBJECT_MODE); + self.cursor += 1; + self.state = .object_start; + return .object_begin; + }, + '[' => { + try self.stack.push(ARRAY_MODE); + self.cursor += 1; + self.state = .array_start; + return .array_begin; + }, + + // String + '"' => { + self.cursor += 1; + self.value_start = self.cursor; + self.state = .string; + continue :state_loop; + }, + + // Number + '1'...'9' => { + self.value_start = self.cursor; + self.cursor += 1; + self.state = .number_int; + continue :state_loop; + }, + '0' => { + self.value_start = self.cursor; + self.cursor += 1; + self.state = .number_leading_zero; + continue :state_loop; + }, + '-' => { + self.value_start = self.cursor; + self.cursor += 1; + self.state = .number_minus; + continue :state_loop; + }, + + // literal values + 't' => { + self.cursor += 1; + self.state = .literal_t; + continue :state_loop; + }, + 'f' => { + self.cursor += 1; + self.state = .literal_f; + continue :state_loop; + }, + 'n' => { + self.cursor += 1; + self.state = .literal_n; + continue :state_loop; + }, + + else => return error.SyntaxError, + } + }, + + .post_value => { + if (try self.skipWhitespaceCheckEnd()) return .end_of_document; + + const c = self.input[self.cursor]; + if (self.string_is_object_key) { + self.string_is_object_key = false; + switch (c) { + ':' => { + self.cursor += 1; + self.state = .value; + continue :state_loop; + }, + else => return error.SyntaxError, + } + } + + switch (c) { + '}' => { + if (self.stack.pop() != OBJECT_MODE) return error.SyntaxError; + self.cursor += 1; + // stay in .post_value state. + return .object_end; + }, + ']' => { + if (self.stack.pop() != ARRAY_MODE) return error.SyntaxError; + self.cursor += 1; + // stay in .post_value state. + return .array_end; + }, + ',' => { + switch (self.stack.peek()) { + OBJECT_MODE => { + self.state = .object_post_comma; + }, + ARRAY_MODE => { + self.state = .value; + }, + } + self.cursor += 1; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + + .object_start => { + switch (try self.skipWhitespaceExpectByte()) { + '"' => { + self.cursor += 1; + self.value_start = self.cursor; + self.state = .string; + self.string_is_object_key = true; + continue :state_loop; + }, + '}' => { + self.cursor += 1; + _ = self.stack.pop(); + self.state = .post_value; + return .object_end; + }, + else => return error.SyntaxError, + } + }, + .object_post_comma => { + switch (try self.skipWhitespaceExpectByte()) { + '"' => { + self.cursor += 1; + self.value_start = self.cursor; + self.state = .string; + self.string_is_object_key = true; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + + .array_start => { + switch (try self.skipWhitespaceExpectByte()) { + ']' => { + self.cursor += 1; + _ = self.stack.pop(); + self.state = .post_value; + return .array_end; + }, + else => { + self.state = .value; + continue :state_loop; + }, + } + }, + + .number_minus => { + if (self.cursor >= self.input.len) return self.endOfBufferInNumber(false); + switch (self.input[self.cursor]) { + '0' => { + self.cursor += 1; + self.state = .number_leading_zero; + continue :state_loop; + }, + '1'...'9' => { + self.cursor += 1; + self.state = .number_int; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + .number_leading_zero => { + if (self.cursor >= self.input.len) return self.endOfBufferInNumber(true); + switch (self.input[self.cursor]) { + '.' => { + self.cursor += 1; + self.state = .number_post_dot; + continue :state_loop; + }, + 'e', 'E' => { + self.cursor += 1; + self.state = .number_post_e; + continue :state_loop; + }, + else => { + self.state = .post_value; + return Token{ .number = self.takeValueSlice() }; + }, + } + }, + .number_int => { + while (self.cursor < self.input.len) : (self.cursor += 1) { + switch (self.input[self.cursor]) { + '0'...'9' => continue, + '.' => { + self.cursor += 1; + self.state = .number_post_dot; + continue :state_loop; + }, + 'e', 'E' => { + self.cursor += 1; + self.state = .number_post_e; + continue :state_loop; + }, + else => { + self.state = .post_value; + return Token{ .number = self.takeValueSlice() }; + }, + } + } + return self.endOfBufferInNumber(true); + }, + .number_post_dot => { + if (self.cursor >= self.input.len) return self.endOfBufferInNumber(false); + switch (self.input[self.cursor]) { + '0'...'9' => { + self.cursor += 1; + self.state = .number_frac; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + .number_frac => { + while (self.cursor < self.input.len) : (self.cursor += 1) { + switch (self.input[self.cursor]) { + '0'...'9' => continue, + 'e', 'E' => { + self.cursor += 1; + self.state = .number_post_e; + continue :state_loop; + }, + else => { + self.state = .post_value; + return Token{ .number = self.takeValueSlice() }; + }, + } + } + return self.endOfBufferInNumber(true); + }, + .number_post_e => { + if (self.cursor >= self.input.len) return self.endOfBufferInNumber(false); + switch (self.input[self.cursor]) { + '0'...'9' => { + self.cursor += 1; + self.state = .number_exp; + continue :state_loop; + }, + '+', '-' => { + self.cursor += 1; + self.state = .number_post_e_sign; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + .number_post_e_sign => { + if (self.cursor >= self.input.len) return self.endOfBufferInNumber(false); + switch (self.input[self.cursor]) { + '0'...'9' => { + self.cursor += 1; + self.state = .number_exp; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + .number_exp => { + while (self.cursor < self.input.len) : (self.cursor += 1) { + switch (self.input[self.cursor]) { + '0'...'9' => continue, + else => { + self.state = .post_value; + return Token{ .number = self.takeValueSlice() }; + }, + } + } + return self.endOfBufferInNumber(true); + }, + + .string => { + while (self.cursor < self.input.len) : (self.cursor += 1) { + switch (self.input[self.cursor]) { + 0...0x1f => return error.SyntaxError, // Bare ASCII control code in string. + + // ASCII plain text. + 0x20...('"' - 1), ('"' + 1)...('\\' - 1), ('\\' + 1)...0x7F => continue, + + // Special characters. + '"' => { + const result = Token{ .string = self.takeValueSlice() }; + self.cursor += 1; + self.state = .post_value; + return result; + }, + '\\' => { + const slice = self.takeValueSlice(); + self.cursor += 1; + self.state = .string_backslash; + if (slice.len > 0) return Token{ .partial_string = slice }; + continue :state_loop; + }, + + // UTF-8 validation. + // See http://unicode.org/mail-arch/unicode-ml/y2003-m02/att-0467/01-The_Algorithm_to_Valide_an_UTF-8_String + 0xC2...0xDF => { + self.cursor += 1; + self.state = .string_utf8_last_byte; + continue :state_loop; + }, + 0xE0 => { + self.cursor += 1; + self.state = .string_utf8_second_to_last_byte_guard_against_overlong; + continue :state_loop; + }, + 0xE1...0xEC, 0xEE...0xEF => { + self.cursor += 1; + self.state = .string_utf8_second_to_last_byte; + continue :state_loop; + }, + 0xED => { + self.cursor += 1; + self.state = .string_utf8_second_to_last_byte_guard_against_surrogate_half; + continue :state_loop; + }, + 0xF0 => { + self.cursor += 1; + self.state = .string_utf8_third_to_last_byte_guard_against_overlong; + continue :state_loop; + }, + 0xF1...0xF3 => { + self.cursor += 1; + self.state = .string_utf8_third_to_last_byte; + continue :state_loop; + }, + 0xF4 => { + self.cursor += 1; + self.state = .string_utf8_third_to_last_byte_guard_against_too_large; + continue :state_loop; + }, + 0x80...0xC1, 0xF5...0xFF => return error.SyntaxError, // Invalid UTF-8. + } + } + if (self.is_end_of_input) return error.UnexpectedEndOfInput; + const slice = self.takeValueSlice(); + if (slice.len > 0) return Token{ .partial_string = slice }; + return error.BufferUnderrun; + }, + .string_backslash => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + switch (self.input[self.cursor]) { + '"', '\\', '/' => { + // Since these characters now represent themselves literally, + // we can simply begin the next plaintext slice here. + self.value_start = self.cursor; + self.cursor += 1; + self.state = .string; + continue :state_loop; + }, + 'b' => { + self.cursor += 1; + self.value_start = self.cursor; + self.state = .string; + return Token{ .partial_string_escaped_1 = [_]u8{0x08} }; + }, + 'f' => { + self.cursor += 1; + self.value_start = self.cursor; + self.state = .string; + return Token{ .partial_string_escaped_1 = [_]u8{0x0c} }; + }, + 'n' => { + self.cursor += 1; + self.value_start = self.cursor; + self.state = .string; + return Token{ .partial_string_escaped_1 = [_]u8{'\n'} }; + }, + 'r' => { + self.cursor += 1; + self.value_start = self.cursor; + self.state = .string; + return Token{ .partial_string_escaped_1 = [_]u8{'\r'} }; + }, + 't' => { + self.cursor += 1; + self.value_start = self.cursor; + self.state = .string; + return Token{ .partial_string_escaped_1 = [_]u8{'\t'} }; + }, + 'u' => { + self.cursor += 1; + self.state = .string_backslash_u; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + .string_backslash_u => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + const c = self.input[self.cursor]; + switch (c) { + '0'...'9' => { + self.utf16_code_units[0] = @as(u16, c - '0') << 12; + }, + 'A'...'F' => { + self.utf16_code_units[0] = @as(u16, c - 'A' + 10) << 12; + }, + 'a'...'f' => { + self.utf16_code_units[0] = @as(u16, c - 'a' + 10) << 12; + }, + else => return error.SyntaxError, + } + self.cursor += 1; + self.state = .string_backslash_u_1; + continue :state_loop; + }, + .string_backslash_u_1 => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + const c = self.input[self.cursor]; + switch (c) { + '0'...'9' => { + self.utf16_code_units[0] |= @as(u16, c - '0') << 8; + }, + 'A'...'F' => { + self.utf16_code_units[0] |= @as(u16, c - 'A' + 10) << 8; + }, + 'a'...'f' => { + self.utf16_code_units[0] |= @as(u16, c - 'a' + 10) << 8; + }, + else => return error.SyntaxError, + } + self.cursor += 1; + self.state = .string_backslash_u_2; + continue :state_loop; + }, + .string_backslash_u_2 => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + const c = self.input[self.cursor]; + switch (c) { + '0'...'9' => { + self.utf16_code_units[0] |= @as(u16, c - '0') << 4; + }, + 'A'...'F' => { + self.utf16_code_units[0] |= @as(u16, c - 'A' + 10) << 4; + }, + 'a'...'f' => { + self.utf16_code_units[0] |= @as(u16, c - 'a' + 10) << 4; + }, + else => return error.SyntaxError, + } + self.cursor += 1; + self.state = .string_backslash_u_3; + continue :state_loop; + }, + .string_backslash_u_3 => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + const c = self.input[self.cursor]; + switch (c) { + '0'...'9' => { + self.utf16_code_units[0] |= c - '0'; + }, + 'A'...'F' => { + self.utf16_code_units[0] |= c - 'A' + 10; + }, + 'a'...'f' => { + self.utf16_code_units[0] |= c - 'a' + 10; + }, + else => return error.SyntaxError, + } + self.cursor += 1; + if (std.unicode.utf16IsHighSurrogate(self.utf16_code_units[0])) { + self.state = .string_surrogate_half; + continue :state_loop; + } else if (std.unicode.utf16IsLowSurrogate(self.utf16_code_units[0])) { + return error.SyntaxError; // Unexpected low surrogate half. + } else { + self.value_start = self.cursor; + self.state = .string; + return partialStringCodepoint(self.utf16_code_units[0]); + } + }, + .string_surrogate_half => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + switch (self.input[self.cursor]) { + '\\' => { + self.cursor += 1; + self.state = .string_surrogate_half_backslash; + continue :state_loop; + }, + else => return error.SyntaxError, // Expected low surrogate half. + } + }, + .string_surrogate_half_backslash => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + switch (self.input[self.cursor]) { + 'u' => { + self.cursor += 1; + self.state = .string_surrogate_half_backslash_u; + continue :state_loop; + }, + else => return error.SyntaxError, // Expected low surrogate half. + } + }, + .string_surrogate_half_backslash_u => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + switch (self.input[self.cursor]) { + 'D', 'd' => { + self.cursor += 1; + self.utf16_code_units[1] = 0xD << 12; + self.state = .string_surrogate_half_backslash_u_1; + continue :state_loop; + }, + else => return error.SyntaxError, // Expected low surrogate half. + } + }, + .string_surrogate_half_backslash_u_1 => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + const c = self.input[self.cursor]; + switch (c) { + 'C'...'F' => { + self.cursor += 1; + self.utf16_code_units[1] |= @as(u16, c - 'A' + 10) << 8; + self.state = .string_surrogate_half_backslash_u_2; + continue :state_loop; + }, + 'c'...'f' => { + self.cursor += 1; + self.utf16_code_units[1] |= @as(u16, c - 'a' + 10) << 8; + self.state = .string_surrogate_half_backslash_u_2; + continue :state_loop; + }, + else => return error.SyntaxError, // Expected low surrogate half. + } + }, + .string_surrogate_half_backslash_u_2 => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + const c = self.input[self.cursor]; + switch (c) { + '0'...'9' => { + self.cursor += 1; + self.utf16_code_units[1] |= @as(u16, c - '0') << 4; + self.state = .string_surrogate_half_backslash_u_3; + continue :state_loop; + }, + 'A'...'F' => { + self.cursor += 1; + self.utf16_code_units[1] |= @as(u16, c - 'A' + 10) << 4; + self.state = .string_surrogate_half_backslash_u_3; + continue :state_loop; + }, + 'a'...'f' => { + self.cursor += 1; + self.utf16_code_units[1] |= @as(u16, c - 'a' + 10) << 4; + self.state = .string_surrogate_half_backslash_u_3; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + .string_surrogate_half_backslash_u_3 => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + const c = self.input[self.cursor]; + switch (c) { + '0'...'9' => { + self.utf16_code_units[1] |= c - '0'; + }, + 'A'...'F' => { + self.utf16_code_units[1] |= c - 'A' + 10; + }, + 'a'...'f' => { + self.utf16_code_units[1] |= c - 'a' + 10; + }, + else => return error.SyntaxError, + } + self.cursor += 1; + self.value_start = self.cursor; + self.state = .string; + const code_point = std.unicode.utf16DecodeSurrogatePair(&self.utf16_code_units) catch unreachable; + return partialStringCodepoint(code_point); + }, + + .string_utf8_last_byte => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + switch (self.input[self.cursor]) { + 0x80...0xBF => { + self.cursor += 1; + self.state = .string; + continue :state_loop; + }, + else => return error.SyntaxError, // Invalid UTF-8. + } + }, + .string_utf8_second_to_last_byte => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + switch (self.input[self.cursor]) { + 0x80...0xBF => { + self.cursor += 1; + self.state = .string_utf8_last_byte; + continue :state_loop; + }, + else => return error.SyntaxError, // Invalid UTF-8. + } + }, + .string_utf8_second_to_last_byte_guard_against_overlong => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + switch (self.input[self.cursor]) { + 0xA0...0xBF => { + self.cursor += 1; + self.state = .string_utf8_last_byte; + continue :state_loop; + }, + else => return error.SyntaxError, // Invalid UTF-8. + } + }, + .string_utf8_second_to_last_byte_guard_against_surrogate_half => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + switch (self.input[self.cursor]) { + 0x80...0x9F => { + self.cursor += 1; + self.state = .string_utf8_last_byte; + continue :state_loop; + }, + else => return error.SyntaxError, // Invalid UTF-8. + } + }, + .string_utf8_third_to_last_byte => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + switch (self.input[self.cursor]) { + 0x80...0xBF => { + self.cursor += 1; + self.state = .string_utf8_second_to_last_byte; + continue :state_loop; + }, + else => return error.SyntaxError, // Invalid UTF-8. + } + }, + .string_utf8_third_to_last_byte_guard_against_overlong => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + switch (self.input[self.cursor]) { + 0x90...0xBF => { + self.cursor += 1; + self.state = .string_utf8_second_to_last_byte; + continue :state_loop; + }, + else => return error.SyntaxError, // Invalid UTF-8. + } + }, + .string_utf8_third_to_last_byte_guard_against_too_large => { + if (self.cursor >= self.input.len) return self.endOfBufferInString(); + switch (self.input[self.cursor]) { + 0x80...0x8F => { + self.cursor += 1; + self.state = .string_utf8_second_to_last_byte; + continue :state_loop; + }, + else => return error.SyntaxError, // Invalid UTF-8. + } + }, + + .literal_t => { + switch (try self.expectByte()) { + 'r' => { + self.cursor += 1; + self.state = .literal_tr; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + .literal_tr => { + switch (try self.expectByte()) { + 'u' => { + self.cursor += 1; + self.state = .literal_tru; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + .literal_tru => { + switch (try self.expectByte()) { + 'e' => { + self.cursor += 1; + self.state = .post_value; + return .true; + }, + else => return error.SyntaxError, + } + }, + .literal_f => { + switch (try self.expectByte()) { + 'a' => { + self.cursor += 1; + self.state = .literal_fa; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + .literal_fa => { + switch (try self.expectByte()) { + 'l' => { + self.cursor += 1; + self.state = .literal_fal; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + .literal_fal => { + switch (try self.expectByte()) { + 's' => { + self.cursor += 1; + self.state = .literal_fals; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + .literal_fals => { + switch (try self.expectByte()) { + 'e' => { + self.cursor += 1; + self.state = .post_value; + return .false; + }, + else => return error.SyntaxError, + } + }, + .literal_n => { + switch (try self.expectByte()) { + 'u' => { + self.cursor += 1; + self.state = .literal_nu; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + .literal_nu => { + switch (try self.expectByte()) { + 'l' => { + self.cursor += 1; + self.state = .literal_nul; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + .literal_nul => { + switch (try self.expectByte()) { + 'l' => { + self.cursor += 1; + self.state = .post_value; + return .null; + }, + else => return error.SyntaxError, + } + }, + } + unreachable; + } +} + +/// Seeks ahead in the input until the first byte of the next token (or the end of the input) +/// determines which type of token will be returned from the next `next*()` call. +/// This function is idempotent, only advancing past commas, colons, and inter-token whitespace. +pub fn peekNextTokenType(self: *@This()) PeekError!TokenType { + state_loop: while (true) { + switch (self.state) { + .value => { + switch (try self.skipWhitespaceExpectByte()) { + '{' => return .object_begin, + '[' => return .array_begin, + '"' => return .string, + '-', '0'...'9' => return .number, + 't' => return .true, + 'f' => return .false, + 'n' => return .null, + else => return error.SyntaxError, + } + }, + + .post_value => { + if (try self.skipWhitespaceCheckEnd()) return .end_of_document; + + const c = self.input[self.cursor]; + if (self.string_is_object_key) { + self.string_is_object_key = false; + switch (c) { + ':' => { + self.cursor += 1; + self.state = .value; + continue :state_loop; + }, + else => return error.SyntaxError, + } + } + + switch (c) { + '}' => return .object_end, + ']' => return .array_end, + ',' => { + switch (self.stack.peek()) { + OBJECT_MODE => { + self.state = .object_post_comma; + }, + ARRAY_MODE => { + self.state = .value; + }, + } + self.cursor += 1; + continue :state_loop; + }, + else => return error.SyntaxError, + } + }, + + .object_start => { + switch (try self.skipWhitespaceExpectByte()) { + '"' => return .string, + '}' => return .object_end, + else => return error.SyntaxError, + } + }, + .object_post_comma => { + switch (try self.skipWhitespaceExpectByte()) { + '"' => return .string, + else => return error.SyntaxError, + } + }, + + .array_start => { + switch (try self.skipWhitespaceExpectByte()) { + ']' => return .array_end, + else => { + self.state = .value; + continue :state_loop; + }, + } + }, + + .number_minus, + .number_leading_zero, + .number_int, + .number_post_dot, + .number_frac, + .number_post_e, + .number_post_e_sign, + .number_exp, + => return .number, + + .string, + .string_backslash, + .string_backslash_u, + .string_backslash_u_1, + .string_backslash_u_2, + .string_backslash_u_3, + .string_surrogate_half, + .string_surrogate_half_backslash, + .string_surrogate_half_backslash_u, + .string_surrogate_half_backslash_u_1, + .string_surrogate_half_backslash_u_2, + .string_surrogate_half_backslash_u_3, + => return .string, + + .string_utf8_last_byte, + .string_utf8_second_to_last_byte, + .string_utf8_second_to_last_byte_guard_against_overlong, + .string_utf8_second_to_last_byte_guard_against_surrogate_half, + .string_utf8_third_to_last_byte, + .string_utf8_third_to_last_byte_guard_against_overlong, + .string_utf8_third_to_last_byte_guard_against_too_large, + => return .string, + + .literal_t, + .literal_tr, + .literal_tru, + => return .true, + .literal_f, + .literal_fa, + .literal_fal, + .literal_fals, + => return .false, + .literal_n, + .literal_nu, + .literal_nul, + => return .null, + } + unreachable; + } +} + +const State = enum { + value, + post_value, + + object_start, + object_post_comma, + + array_start, + + number_minus, + number_leading_zero, + number_int, + number_post_dot, + number_frac, + number_post_e, + number_post_e_sign, + number_exp, + + string, + string_backslash, + string_backslash_u, + string_backslash_u_1, + string_backslash_u_2, + string_backslash_u_3, + string_surrogate_half, + string_surrogate_half_backslash, + string_surrogate_half_backslash_u, + string_surrogate_half_backslash_u_1, + string_surrogate_half_backslash_u_2, + string_surrogate_half_backslash_u_3, + + // From http://unicode.org/mail-arch/unicode-ml/y2003-m02/att-0467/01-The_Algorithm_to_Valide_an_UTF-8_String + string_utf8_last_byte, // State A + string_utf8_second_to_last_byte, // State B + string_utf8_second_to_last_byte_guard_against_overlong, // State C + string_utf8_second_to_last_byte_guard_against_surrogate_half, // State D + string_utf8_third_to_last_byte, // State E + string_utf8_third_to_last_byte_guard_against_overlong, // State F + string_utf8_third_to_last_byte_guard_against_too_large, // State G + + literal_t, + literal_tr, + literal_tru, + literal_f, + literal_fa, + literal_fal, + literal_fals, + literal_n, + literal_nu, + literal_nul, +}; + +fn expectByte(self: *const @This()) !u8 { + if (self.cursor < self.input.len) { + return self.input[self.cursor]; + } + // No byte. + if (self.is_end_of_input) return error.UnexpectedEndOfInput; + return error.BufferUnderrun; +} + +fn skipWhitespace(self: *@This()) void { + while (self.cursor < self.input.len) : (self.cursor += 1) { + switch (self.input[self.cursor]) { + // Whitespace + ' ', '\t', '\r' => continue, + '\n' => { + if (self.diagnostics) |diag| { + diag.line_number += 1; + // This will count the newline itself, + // which means a straight-forward subtraction will give a 1-based column number. + diag.line_start_cursor = self.cursor; + } + continue; + }, + else => return, + } + } +} + +fn skipWhitespaceExpectByte(self: *@This()) !u8 { + self.skipWhitespace(); + return self.expectByte(); +} + +fn skipWhitespaceCheckEnd(self: *@This()) !bool { + self.skipWhitespace(); + if (self.cursor >= self.input.len) { + // End of buffer. + if (self.is_end_of_input) { + // End of everything. + if (self.stackHeight() == 0) { + // We did it! + return true; + } + return error.UnexpectedEndOfInput; + } + return error.BufferUnderrun; + } + if (self.stackHeight() == 0) return error.SyntaxError; + return false; +} + +fn takeValueSlice(self: *@This()) []const u8 { + const slice = self.input[self.value_start..self.cursor]; + self.value_start = self.cursor; + return slice; +} +fn takeValueSliceMinusTrailingOffset(self: *@This(), trailing_negative_offset: usize) []const u8 { + // Check if the escape sequence started before the current input buffer. + // (The algebra here is awkward to avoid unsigned underflow, + // but it's just making sure the slice on the next line isn't UB.) + if (self.cursor <= self.value_start + trailing_negative_offset) return ""; + const slice = self.input[self.value_start .. self.cursor - trailing_negative_offset]; + // When trailing_negative_offset is non-zero, setting self.value_start doesn't matter, + // because we always set it again while emitting the .partial_string_escaped_*. + self.value_start = self.cursor; + return slice; +} + +fn endOfBufferInNumber(self: *@This(), allow_end: bool) !Token { + const slice = self.takeValueSlice(); + if (self.is_end_of_input) { + if (!allow_end) return error.UnexpectedEndOfInput; + self.state = .post_value; + return Token{ .number = slice }; + } + if (slice.len == 0) return error.BufferUnderrun; + return Token{ .partial_number = slice }; +} + +fn endOfBufferInString(self: *@This()) !Token { + if (self.is_end_of_input) return error.UnexpectedEndOfInput; + const slice = self.takeValueSliceMinusTrailingOffset(switch (self.state) { + // Don't include the escape sequence in the partial string. + .string_backslash => 1, + .string_backslash_u => 2, + .string_backslash_u_1 => 3, + .string_backslash_u_2 => 4, + .string_backslash_u_3 => 5, + .string_surrogate_half => 6, + .string_surrogate_half_backslash => 7, + .string_surrogate_half_backslash_u => 8, + .string_surrogate_half_backslash_u_1 => 9, + .string_surrogate_half_backslash_u_2 => 10, + .string_surrogate_half_backslash_u_3 => 11, + + // Include everything up to the cursor otherwise. + .string, + .string_utf8_last_byte, + .string_utf8_second_to_last_byte, + .string_utf8_second_to_last_byte_guard_against_overlong, + .string_utf8_second_to_last_byte_guard_against_surrogate_half, + .string_utf8_third_to_last_byte, + .string_utf8_third_to_last_byte_guard_against_overlong, + .string_utf8_third_to_last_byte_guard_against_too_large, + => 0, + + else => unreachable, + }); + if (slice.len == 0) return error.BufferUnderrun; + return Token{ .partial_string = slice }; +} + +fn partialStringCodepoint(code_point: u21) Token { + var buf: [4]u8 = undefined; + switch (std.unicode.utf8Encode(code_point, &buf) catch unreachable) { + 1 => return Token{ .partial_string_escaped_1 = buf[0..1].* }, + 2 => return Token{ .partial_string_escaped_2 = buf[0..2].* }, + 3 => return Token{ .partial_string_escaped_3 = buf[0..3].* }, + 4 => return Token{ .partial_string_escaped_4 = buf[0..4].* }, + else => unreachable, + } +} + +/// Scan the input and check for malformed JSON. +/// On `SyntaxError` or `UnexpectedEndOfInput`, returns `false`. +/// Returns any errors from the allocator as-is, which is unlikely, +/// but can be caused by extreme nesting depth in the input. +pub fn validate(allocator: Allocator, s: []const u8) Allocator.Error!bool { + var scanner = Scanner.initCompleteInput(allocator, s); + defer scanner.deinit(); + + while (true) { + const token = scanner.next() catch |err| switch (err) { + error.SyntaxError, error.UnexpectedEndOfInput => return false, + error.OutOfMemory => return error.OutOfMemory, + error.BufferUnderrun => unreachable, + }; + if (token == .end_of_document) break; + } + + return true; +} + +/// The parsing errors are divided into two categories: +/// * `SyntaxError` is for clearly malformed JSON documents, +/// such as giving an input document that isn't JSON at all. +/// * `UnexpectedEndOfInput` is for signaling that everything's been +/// valid so far, but the input appears to be truncated for some reason. +/// Note that a completely empty (or whitespace-only) input will give `UnexpectedEndOfInput`. +pub const Error = error{ SyntaxError, UnexpectedEndOfInput }; + +/// Used by `json.reader`. +pub const default_buffer_size = 0x1000; + +/// The tokens emitted by `std.json.Scanner` and `std.json.Reader` `.next*()` functions follow this grammar: +/// ``` +/// = .end_of_document +/// = +/// | +/// | +/// | +/// | +/// | .true +/// | .false +/// | .null +/// = .object_begin ( )* .object_end +/// = .array_begin ( )* .array_end +/// = +/// = +/// ``` +/// +/// What you get for `` and `` values depends on which `next*()` method you call: +/// +/// ``` +/// next(): +/// = ( .partial_number )* .number +/// = ( )* .string +/// = +/// | .partial_string +/// | .partial_string_escaped_1 +/// | .partial_string_escaped_2 +/// | .partial_string_escaped_3 +/// | .partial_string_escaped_4 +/// +/// nextAlloc*(..., .alloc_always): +/// = .allocated_number +/// = .allocated_string +/// +/// nextAlloc*(..., .alloc_if_needed): +/// = +/// | .number +/// | .allocated_number +/// = +/// | .string +/// | .allocated_string +/// ``` +/// +/// For all tokens with a `[]const u8`, `[]u8`, or `[n]u8` payload, the payload represents the content of the value. +/// For number values, this is the representation of the number exactly as it appears in the input. +/// For strings, this is the content of the string after resolving escape sequences. +/// +/// For `.allocated_number` and `.allocated_string`, the `[]u8` payloads are allocations made with the given allocator. +/// You are responsible for managing that memory. `json.Reader.deinit()` does *not* free those allocations. +/// +/// The `.partial_*` tokens indicate that a value spans multiple input buffers or that a string contains escape sequences. +/// To get a complete value in memory, you need to concatenate the values yourself. +/// Calling `nextAlloc*()` does this for you, and returns an `.allocated_*` token with the result. +/// +/// For tokens with a `[]const u8` payload, the payload is a slice into the current input buffer. +/// The memory may become undefined during the next call to `json.Scanner.feedInput()` +/// or any `json.Reader` method whose return error set includes `json.Error`. +/// To keep the value persistently, it recommended to make a copy or to use `.alloc_always`, +/// which makes a copy for you. +/// +/// Note that `.number` and `.string` tokens that follow `.partial_*` tokens may have `0` length to indicate that +/// the previously partial value is completed with no additional bytes. +/// (This can happen when the break between input buffers happens to land on the exact end of a value. E.g. `"[1234"`, `"]"`.) +/// `.partial_*` tokens never have `0` length. +/// +/// The recommended strategy for using the different `next*()` methods is something like this: +/// +/// When you're expecting an object key, use `.alloc_if_needed`. +/// You often don't need a copy of the key string to persist; you might just check which field it is. +/// In the case that the key happens to require an allocation, free it immediately after checking it. +/// +/// When you're expecting a meaningful string value (such as on the right of a `:`), +/// use `.alloc_always` in order to keep the value valid throughout parsing the rest of the document. +/// +/// When you're expecting a number value, use `.alloc_if_needed`. +/// You're probably going to be parsing the string representation of the number into a numeric representation, +/// so you need the complete string representation only temporarily. +/// +/// When you're skipping an unrecognized value, use `skipValue()`. +pub const Token = union(enum) { + object_begin, + object_end, + array_begin, + array_end, + + true, + false, + null, + + number: []const u8, + partial_number: []const u8, + allocated_number: []u8, + + string: []const u8, + partial_string: []const u8, + partial_string_escaped_1: [1]u8, + partial_string_escaped_2: [2]u8, + partial_string_escaped_3: [3]u8, + partial_string_escaped_4: [4]u8, + allocated_string: []u8, + + end_of_document, +}; + +/// This is only used in `peekNextTokenType()` and gives a categorization based on the first byte of the next token that will be emitted from a `next*()` call. +pub const TokenType = enum { + object_begin, + object_end, + array_begin, + array_end, + true, + false, + null, + number, + string, + end_of_document, +}; + +/// To enable diagnostics, declare `var diagnostics = Diagnostics{};` then call `source.enableDiagnostics(&diagnostics);` +/// where `source` is either a `std.json.Reader` or a `std.json.Scanner` that has just been initialized. +/// At any time, notably just after an error, call `getLine()`, `getColumn()`, and/or `getByteOffset()` +/// to get meaningful information from this. +pub const Diagnostics = struct { + line_number: u64 = 1, + line_start_cursor: usize = @as(usize, @bitCast(@as(isize, -1))), // Start just "before" the input buffer to get a 1-based column for line 1. + total_bytes_before_current_input: u64 = 0, + cursor_pointer: *const usize = undefined, + + /// Starts at 1. + pub fn getLine(self: *const @This()) u64 { + return self.line_number; + } + /// Starts at 1. + pub fn getColumn(self: *const @This()) u64 { + return self.cursor_pointer.* -% self.line_start_cursor; + } + /// Starts at 0. Measures the byte offset since the start of the input. + pub fn getByteOffset(self: *const @This()) u64 { + return self.total_bytes_before_current_input + self.cursor_pointer.*; + } +}; + +/// See the documentation for `std.json.Token`. +pub const AllocWhen = enum { alloc_if_needed, alloc_always }; + +/// For security, the maximum size allocated to store a single string or number value is limited to 4MiB by default. +/// This limit can be specified by calling `nextAllocMax()` instead of `nextAlloc()`. +pub const default_max_value_len = 4 * 1024 * 1024; + +/// All `next*()` methods here handle `error.BufferUnderrun` from `std.json.Scanner`, and then read from the reader. +pub const Reader = struct { + scanner: Scanner, + reader: *std.Io.Reader, + + /// The allocator is only used to track `[]` and `{}` nesting levels. + pub fn init(allocator: Allocator, io_reader: *std.Io.Reader) @This() { + return .{ + .scanner = Scanner.initStreaming(allocator), + .reader = io_reader, + }; + } + pub fn deinit(self: *@This()) void { + self.scanner.deinit(); + self.* = undefined; + } + + /// Calls `std.json.Scanner.enableDiagnostics`. + pub fn enableDiagnostics(self: *@This(), diagnostics: *Diagnostics) void { + self.scanner.enableDiagnostics(diagnostics); + } + + pub const NextError = std.Io.Reader.Error || Error || Allocator.Error; + pub const SkipError = Reader.NextError; + pub const AllocError = Reader.NextError || error{ValueTooLong}; + pub const PeekError = std.Io.Reader.Error || Error; + + /// Equivalent to `nextAllocMax(allocator, when, default_max_value_len);` + /// See also `std.json.Token` for documentation of `nextAlloc*()` function behavior. + pub fn nextAlloc(self: *@This(), allocator: Allocator, when: AllocWhen) Reader.AllocError!Token { + return self.nextAllocMax(allocator, when, default_max_value_len); + } + /// See also `std.json.Token` for documentation of `nextAlloc*()` function behavior. + pub fn nextAllocMax(self: *@This(), allocator: Allocator, when: AllocWhen, max_value_len: usize) Reader.AllocError!Token { + const token_type = try self.peekNextTokenType(); + switch (token_type) { + .number, .string => { + var value_list = ArrayList(u8).init(allocator); + errdefer { + value_list.deinit(); + } + if (try self.allocNextIntoArrayListMax(&value_list, when, max_value_len)) |slice| { + return if (token_type == .number) + Token{ .number = slice } + else + Token{ .string = slice }; + } else { + return if (token_type == .number) + Token{ .allocated_number = try value_list.toOwnedSlice() } + else + Token{ .allocated_string = try value_list.toOwnedSlice() }; + } + }, + + // Simple tokens never alloc. + .object_begin, + .object_end, + .array_begin, + .array_end, + .true, + .false, + .null, + .end_of_document, + => return try self.next(), + } + } + + /// Equivalent to `allocNextIntoArrayListMax(value_list, when, default_max_value_len);` + pub fn allocNextIntoArrayList(self: *@This(), value_list: *ArrayList(u8), when: AllocWhen) Reader.AllocError!?[]const u8 { + return self.allocNextIntoArrayListMax(value_list, when, default_max_value_len); + } + /// Calls `std.json.Scanner.allocNextIntoArrayListMax` and handles `error.BufferUnderrun`. + pub fn allocNextIntoArrayListMax(self: *@This(), value_list: *ArrayList(u8), when: AllocWhen, max_value_len: usize) Reader.AllocError!?[]const u8 { + while (true) { + return self.scanner.allocNextIntoArrayListMax(value_list, when, max_value_len) catch |err| switch (err) { + error.BufferUnderrun => { + try self.refillBuffer(); + continue; + }, + else => |other_err| return other_err, + }; + } + } + + /// Like `std.json.Scanner.skipValue`, but handles `error.BufferUnderrun`. + pub fn skipValue(self: *@This()) Reader.SkipError!void { + switch (try self.peekNextTokenType()) { + .object_begin, .array_begin => { + try self.skipUntilStackHeight(self.stackHeight()); + }, + .number, .string => { + while (true) { + switch (try self.next()) { + .partial_number, + .partial_string, + .partial_string_escaped_1, + .partial_string_escaped_2, + .partial_string_escaped_3, + .partial_string_escaped_4, + => continue, + + .number, .string => break, + + else => unreachable, + } + } + }, + .true, .false, .null => { + _ = try self.next(); + }, + + .object_end, .array_end, .end_of_document => unreachable, // Attempt to skip a non-value token. + } + } + /// Like `std.json.Scanner.skipUntilStackHeight()` but handles `error.BufferUnderrun`. + pub fn skipUntilStackHeight(self: *@This(), terminal_stack_height: usize) Reader.NextError!void { + while (true) { + return self.scanner.skipUntilStackHeight(terminal_stack_height) catch |err| switch (err) { + error.BufferUnderrun => { + try self.refillBuffer(); + continue; + }, + else => |other_err| return other_err, + }; + } + } + + /// Calls `std.json.Scanner.stackHeight`. + pub fn stackHeight(self: *const @This()) usize { + return self.scanner.stackHeight(); + } + /// Calls `std.json.Scanner.ensureTotalStackCapacity`. + pub fn ensureTotalStackCapacity(self: *@This(), height: usize) Allocator.Error!void { + try self.scanner.ensureTotalStackCapacity(height); + } + + /// See `std.json.Token` for documentation of this function. + pub fn next(self: *@This()) Reader.NextError!Token { + while (true) { + return self.scanner.next() catch |err| switch (err) { + error.BufferUnderrun => { + try self.refillBuffer(); + continue; + }, + else => |other_err| return other_err, + }; + } + } + + /// See `std.json.Scanner.peekNextTokenType()`. + pub fn peekNextTokenType(self: *@This()) Reader.PeekError!TokenType { + while (true) { + return self.scanner.peekNextTokenType() catch |err| switch (err) { + error.BufferUnderrun => { + try self.refillBuffer(); + continue; + }, + else => |other_err| return other_err, + }; + } + } + + fn refillBuffer(self: *@This()) std.Io.Reader.Error!void { + const input = self.reader.peekGreedy(1) catch |err| switch (err) { + error.ReadFailed => return error.ReadFailed, + error.EndOfStream => return self.scanner.endInput(), + }; + self.reader.toss(input.len); + self.scanner.feedInput(input); + } +}; + +const OBJECT_MODE = 0; +const ARRAY_MODE = 1; + +fn appendSlice(list: *std.ArrayList(u8), buf: []const u8, max_value_len: usize) !void { + const new_len = std.math.add(usize, list.items.len, buf.len) catch return error.ValueTooLong; + if (new_len > max_value_len) return error.ValueTooLong; + try list.appendSlice(buf); +} + +/// For the slice you get from a `Token.number` or `Token.allocated_number`, +/// this function returns true if the number doesn't contain any fraction or exponent components, and is not `-0`. +/// Note, the numeric value encoded by the value may still be an integer, such as `1.0`. +/// This function is meant to give a hint about whether integer parsing or float parsing should be used on the value. +/// This function will not give meaningful results on non-numeric input. +pub fn isNumberFormattedLikeAnInteger(value: []const u8) bool { + if (std.mem.eql(u8, value, "-0")) return false; + return std.mem.indexOfAny(u8, value, ".eE") == null; +} + +test { + _ = @import("./scanner_test.zig"); +} diff --git a/lib/std/json/Stringify.zig b/lib/std/json/Stringify.zig new file mode 100644 index 0000000000..4d79217a87 --- /dev/null +++ b/lib/std/json/Stringify.zig @@ -0,0 +1,999 @@ +//! Writes JSON ([RFC8259](https://tools.ietf.org/html/rfc8259)) formatted data +//! to a stream. +//! +//! The sequence of method calls to write JSON content must follow this grammar: +//! ``` +//! = +//! = +//! | +//! | +//! | write +//! | print +//! | +//! = beginObject ( )* endObject +//! = objectField | objectFieldRaw | +//! = beginArray ( )* endArray +//! = beginWriteRaw ( stream.writeAll )* endWriteRaw +//! = beginObjectFieldRaw ( stream.writeAll )* endObjectFieldRaw +//! ``` + +const std = @import("../std.zig"); +const assert = std.debug.assert; +const Allocator = std.mem.Allocator; +const ArrayList = std.ArrayList; +const BitStack = std.BitStack; +const Stringify = @This(); +const Writer = std.io.Writer; + +const IndentationMode = enum(u1) { + object = 0, + array = 1, +}; + +writer: *Writer, +options: Options = .{}, +indent_level: usize = 0, +next_punctuation: enum { + the_beginning, + none, + comma, + colon, +} = .the_beginning, + +nesting_stack: switch (safety_checks) { + .checked_to_fixed_depth => |fixed_buffer_size| [(fixed_buffer_size + 7) >> 3]u8, + .assumed_correct => void, +} = switch (safety_checks) { + .checked_to_fixed_depth => @splat(0), + .assumed_correct => {}, +}, + +raw_streaming_mode: if (build_mode_has_safety) + enum { none, value, objectField } +else + void = if (build_mode_has_safety) .none else {}, + +const build_mode_has_safety = switch (@import("builtin").mode) { + .Debug, .ReleaseSafe => true, + .ReleaseFast, .ReleaseSmall => false, +}; + +/// The `safety_checks_hint` parameter determines how much memory is used to enable assertions that the above grammar is being followed, +/// e.g. tripping an assertion rather than allowing `endObject` to emit the final `}` in `[[[]]}`. +/// "Depth" in this context means the depth of nested `[]` or `{}` expressions +/// (or equivalently the amount of recursion on the `` grammar expression above). +/// For example, emitting the JSON `[[[]]]` requires a depth of 3. +/// If `.checked_to_fixed_depth` is used, there is additionally an assertion that the nesting depth never exceeds the given limit. +/// `.checked_to_fixed_depth` embeds the storage required in the `Stringify` struct. +/// `.assumed_correct` requires no space and performs none of these assertions. +/// In `ReleaseFast` and `ReleaseSmall` mode, the given `safety_checks_hint` is ignored and is always treated as `.assumed_correct`. +const safety_checks_hint: union(enum) { + /// Rounded up to the nearest multiple of 8. + checked_to_fixed_depth: usize, + assumed_correct, +} = .{ .checked_to_fixed_depth = 256 }; + +const safety_checks: @TypeOf(safety_checks_hint) = if (build_mode_has_safety) + safety_checks_hint +else + .assumed_correct; + +pub const Error = Writer.Error; + +pub fn beginArray(self: *Stringify) Error!void { + if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); + try self.valueStart(); + try self.writer.writeByte('['); + try self.pushIndentation(.array); + self.next_punctuation = .none; +} + +pub fn beginObject(self: *Stringify) Error!void { + if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); + try self.valueStart(); + try self.writer.writeByte('{'); + try self.pushIndentation(.object); + self.next_punctuation = .none; +} + +pub fn endArray(self: *Stringify) Error!void { + if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); + self.popIndentation(.array); + switch (self.next_punctuation) { + .none => {}, + .comma => { + try self.indent(); + }, + .the_beginning, .colon => unreachable, + } + try self.writer.writeByte(']'); + self.valueDone(); +} + +pub fn endObject(self: *Stringify) Error!void { + if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); + self.popIndentation(.object); + switch (self.next_punctuation) { + .none => {}, + .comma => { + try self.indent(); + }, + .the_beginning, .colon => unreachable, + } + try self.writer.writeByte('}'); + self.valueDone(); +} + +fn pushIndentation(self: *Stringify, mode: IndentationMode) !void { + switch (safety_checks) { + .checked_to_fixed_depth => { + BitStack.pushWithStateAssumeCapacity(&self.nesting_stack, &self.indent_level, @intFromEnum(mode)); + }, + .assumed_correct => { + self.indent_level += 1; + }, + } +} +fn popIndentation(self: *Stringify, expected_mode: IndentationMode) void { + switch (safety_checks) { + .checked_to_fixed_depth => { + assert(BitStack.popWithState(&self.nesting_stack, &self.indent_level) == @intFromEnum(expected_mode)); + }, + .assumed_correct => { + self.indent_level -= 1; + }, + } +} + +fn indent(self: *Stringify) !void { + var char: u8 = ' '; + const n_chars = switch (self.options.whitespace) { + .minified => return, + .indent_1 => 1 * self.indent_level, + .indent_2 => 2 * self.indent_level, + .indent_3 => 3 * self.indent_level, + .indent_4 => 4 * self.indent_level, + .indent_8 => 8 * self.indent_level, + .indent_tab => blk: { + char = '\t'; + break :blk self.indent_level; + }, + }; + try self.writer.writeByte('\n'); + try self.writer.splatByteAll(char, n_chars); +} + +fn valueStart(self: *Stringify) !void { + if (self.isObjectKeyExpected()) |is_it| assert(!is_it); // Call objectField*(), not write(), for object keys. + return self.valueStartAssumeTypeOk(); +} +fn objectFieldStart(self: *Stringify) !void { + if (self.isObjectKeyExpected()) |is_it| assert(is_it); // Expected write(), not objectField*(). + return self.valueStartAssumeTypeOk(); +} +fn valueStartAssumeTypeOk(self: *Stringify) !void { + assert(!self.isComplete()); // JSON document already complete. + switch (self.next_punctuation) { + .the_beginning => { + // No indentation for the very beginning. + }, + .none => { + // First item in a container. + try self.indent(); + }, + .comma => { + // Subsequent item in a container. + try self.writer.writeByte(','); + try self.indent(); + }, + .colon => { + try self.writer.writeByte(':'); + if (self.options.whitespace != .minified) { + try self.writer.writeByte(' '); + } + }, + } +} +fn valueDone(self: *Stringify) void { + self.next_punctuation = .comma; +} + +// Only when safety is enabled: +fn isObjectKeyExpected(self: *const Stringify) ?bool { + switch (safety_checks) { + .checked_to_fixed_depth => return self.indent_level > 0 and + BitStack.peekWithState(&self.nesting_stack, self.indent_level) == @intFromEnum(IndentationMode.object) and + self.next_punctuation != .colon, + .assumed_correct => return null, + } +} +fn isComplete(self: *const Stringify) bool { + return self.indent_level == 0 and self.next_punctuation == .comma; +} + +/// An alternative to calling `write` that formats a value with `std.fmt`. +/// This function does the usual punctuation and indentation formatting +/// assuming the resulting formatted string represents a single complete value; +/// e.g. `"1"`, `"[]"`, `"[1,2]"`, not `"1,2"`. +/// This function may be useful for doing your own number formatting. +pub fn print(self: *Stringify, comptime fmt: []const u8, args: anytype) Error!void { + if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); + try self.valueStart(); + try self.writer.print(fmt, args); + self.valueDone(); +} + +test print { + var out_buf: [1024]u8 = undefined; + var out: Writer = .fixed(&out_buf); + + var w: Stringify = .{ .writer = &out, .options = .{ .whitespace = .indent_2 } }; + + try w.beginObject(); + try w.objectField("a"); + try w.print("[ ]", .{}); + try w.objectField("b"); + try w.beginArray(); + try w.print("[{s}] ", .{"[]"}); + try w.print(" {}", .{12345}); + try w.endArray(); + try w.endObject(); + + const expected = + \\{ + \\ "a": [ ], + \\ "b": [ + \\ [[]] , + \\ 12345 + \\ ] + \\} + ; + try std.testing.expectEqualStrings(expected, out.buffered()); +} + +/// An alternative to calling `write` that allows you to write directly to the `.writer` field, e.g. with `.writer.writeAll()`. +/// Call `beginWriteRaw()`, then write a complete value (including any quotes if necessary) directly to the `.writer` field, +/// then call `endWriteRaw()`. +/// This can be useful for streaming very long strings into the output without needing it all buffered in memory. +pub fn beginWriteRaw(self: *Stringify) !void { + if (build_mode_has_safety) { + assert(self.raw_streaming_mode == .none); + self.raw_streaming_mode = .value; + } + try self.valueStart(); +} + +/// See `beginWriteRaw`. +pub fn endWriteRaw(self: *Stringify) void { + if (build_mode_has_safety) { + assert(self.raw_streaming_mode == .value); + self.raw_streaming_mode = .none; + } + self.valueDone(); +} + +/// See `Stringify` for when to call this method. +/// `key` is the string content of the property name. +/// Surrounding quotes will be added and any special characters will be escaped. +/// See also `objectFieldRaw`. +pub fn objectField(self: *Stringify, key: []const u8) Error!void { + if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); + try self.objectFieldStart(); + try encodeJsonString(key, self.options, self.writer); + self.next_punctuation = .colon; +} +/// See `Stringify` for when to call this method. +/// `quoted_key` is the complete bytes of the key including quotes and any necessary escape sequences. +/// A few assertions are performed on the given value to ensure that the caller of this function understands the API contract. +/// See also `objectField`. +pub fn objectFieldRaw(self: *Stringify, quoted_key: []const u8) Error!void { + if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); + assert(quoted_key.len >= 2 and quoted_key[0] == '"' and quoted_key[quoted_key.len - 1] == '"'); // quoted_key should be "quoted". + try self.objectFieldStart(); + try self.writer.writeAll(quoted_key); + self.next_punctuation = .colon; +} + +/// In the rare case that you need to write very long object field names, +/// this is an alternative to `objectField` and `objectFieldRaw` that allows you to write directly to the `.writer` field +/// similar to `beginWriteRaw`. +/// Call `endObjectFieldRaw()` when you're done. +pub fn beginObjectFieldRaw(self: *Stringify) !void { + if (build_mode_has_safety) { + assert(self.raw_streaming_mode == .none); + self.raw_streaming_mode = .objectField; + } + try self.objectFieldStart(); +} + +/// See `beginObjectFieldRaw`. +pub fn endObjectFieldRaw(self: *Stringify) void { + if (build_mode_has_safety) { + assert(self.raw_streaming_mode == .objectField); + self.raw_streaming_mode = .none; + } + self.next_punctuation = .colon; +} + +/// Renders the given Zig value as JSON. +/// +/// Supported types: +/// * Zig `bool` -> JSON `true` or `false`. +/// * Zig `?T` -> `null` or the rendering of `T`. +/// * Zig `i32`, `u64`, etc. -> JSON number or string. +/// * When option `emit_nonportable_numbers_as_strings` is true, if the value is outside the range `+-1<<53` (the precise integer range of f64), it is rendered as a JSON string in base 10. Otherwise, it is rendered as JSON number. +/// * Zig floats -> JSON number or string. +/// * If the value cannot be precisely represented by an f64, it is rendered as a JSON string. Otherwise, it is rendered as JSON number. +/// * TODO: Float rendering will likely change in the future, e.g. to remove the unnecessary "e+00". +/// * Zig `[]const u8`, `[]u8`, `*[N]u8`, `@Vector(N, u8)`, and similar -> JSON string. +/// * See `Options.emit_strings_as_arrays`. +/// * If the content is not valid UTF-8, rendered as an array of numbers instead. +/// * Zig `[]T`, `[N]T`, `*[N]T`, `@Vector(N, T)`, and similar -> JSON array of the rendering of each item. +/// * Zig tuple -> JSON array of the rendering of each item. +/// * Zig `struct` -> JSON object with each field in declaration order. +/// * If the struct declares a method `pub fn jsonStringify(self: *@This(), jw: anytype) !void`, it is called to do the serialization instead of the default behavior. The given `jw` is a pointer to this `Stringify`. See `std.json.Value` for an example. +/// * See `Options.emit_null_optional_fields`. +/// * Zig `union(enum)` -> JSON object with one field named for the active tag and a value representing the payload. +/// * If the payload is `void`, then the emitted value is `{}`. +/// * If the union declares a method `pub fn jsonStringify(self: *@This(), jw: anytype) !void`, it is called to do the serialization instead of the default behavior. The given `jw` is a pointer to this `Stringify`. +/// * Zig `enum` -> JSON string naming the active tag. +/// * If the enum declares a method `pub fn jsonStringify(self: *@This(), jw: anytype) !void`, it is called to do the serialization instead of the default behavior. The given `jw` is a pointer to this `Stringify`. +/// * If the enum is non-exhaustive, unnamed values are rendered as integers. +/// * Zig untyped enum literal -> JSON string naming the active tag. +/// * Zig error -> JSON string naming the error. +/// * Zig `*T` -> the rendering of `T`. Note there is no guard against circular-reference infinite recursion. +/// +/// See also alternative functions `print` and `beginWriteRaw`. +/// For writing object field names, use `objectField` instead. +pub fn write(self: *Stringify, v: anytype) Error!void { + if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); + const T = @TypeOf(v); + switch (@typeInfo(T)) { + .int => { + try self.valueStart(); + if (self.options.emit_nonportable_numbers_as_strings and + (v <= -(1 << 53) or v >= (1 << 53))) + { + try self.writer.print("\"{}\"", .{v}); + } else { + try self.writer.print("{}", .{v}); + } + self.valueDone(); + return; + }, + .comptime_int => { + return self.write(@as(std.math.IntFittingRange(v, v), v)); + }, + .float, .comptime_float => { + if (@as(f64, @floatCast(v)) == v) { + try self.valueStart(); + try self.writer.print("{}", .{@as(f64, @floatCast(v))}); + self.valueDone(); + return; + } + try self.valueStart(); + try self.writer.print("\"{}\"", .{v}); + self.valueDone(); + return; + }, + + .bool => { + try self.valueStart(); + try self.writer.writeAll(if (v) "true" else "false"); + self.valueDone(); + return; + }, + .null => { + try self.valueStart(); + try self.writer.writeAll("null"); + self.valueDone(); + return; + }, + .optional => { + if (v) |payload| { + return try self.write(payload); + } else { + return try self.write(null); + } + }, + .@"enum" => |enum_info| { + if (std.meta.hasFn(T, "jsonStringify")) { + return v.jsonStringify(self); + } + + if (!enum_info.is_exhaustive) { + inline for (enum_info.fields) |field| { + if (v == @field(T, field.name)) { + break; + } + } else { + return self.write(@intFromEnum(v)); + } + } + + return self.stringValue(@tagName(v)); + }, + .enum_literal => { + return self.stringValue(@tagName(v)); + }, + .@"union" => { + if (std.meta.hasFn(T, "jsonStringify")) { + return v.jsonStringify(self); + } + + const info = @typeInfo(T).@"union"; + if (info.tag_type) |UnionTagType| { + try self.beginObject(); + inline for (info.fields) |u_field| { + if (v == @field(UnionTagType, u_field.name)) { + try self.objectField(u_field.name); + if (u_field.type == void) { + // void v is {} + try self.beginObject(); + try self.endObject(); + } else { + try self.write(@field(v, u_field.name)); + } + break; + } + } else { + unreachable; // No active tag? + } + try self.endObject(); + return; + } else { + @compileError("Unable to stringify untagged union '" ++ @typeName(T) ++ "'"); + } + }, + .@"struct" => |S| { + if (std.meta.hasFn(T, "jsonStringify")) { + return v.jsonStringify(self); + } + + if (S.is_tuple) { + try self.beginArray(); + } else { + try self.beginObject(); + } + inline for (S.fields) |Field| { + // don't include void fields + if (Field.type == void) continue; + + var emit_field = true; + + // don't include optional fields that are null when emit_null_optional_fields is set to false + if (@typeInfo(Field.type) == .optional) { + if (self.options.emit_null_optional_fields == false) { + if (@field(v, Field.name) == null) { + emit_field = false; + } + } + } + + if (emit_field) { + if (!S.is_tuple) { + try self.objectField(Field.name); + } + try self.write(@field(v, Field.name)); + } + } + if (S.is_tuple) { + try self.endArray(); + } else { + try self.endObject(); + } + return; + }, + .error_set => return self.stringValue(@errorName(v)), + .pointer => |ptr_info| switch (ptr_info.size) { + .one => switch (@typeInfo(ptr_info.child)) { + .array => { + // Coerce `*[N]T` to `[]const T`. + const Slice = []const std.meta.Elem(ptr_info.child); + return self.write(@as(Slice, v)); + }, + else => { + return self.write(v.*); + }, + }, + .many, .slice => { + if (ptr_info.size == .many and ptr_info.sentinel() == null) + @compileError("unable to stringify type '" ++ @typeName(T) ++ "' without sentinel"); + const slice = if (ptr_info.size == .many) std.mem.span(v) else v; + + if (ptr_info.child == u8) { + // This is a []const u8, or some similar Zig string. + if (!self.options.emit_strings_as_arrays and std.unicode.utf8ValidateSlice(slice)) { + return self.stringValue(slice); + } + } + + try self.beginArray(); + for (slice) |x| { + try self.write(x); + } + try self.endArray(); + return; + }, + else => @compileError("Unable to stringify type '" ++ @typeName(T) ++ "'"), + }, + .array => { + // Coerce `[N]T` to `*const [N]T` (and then to `[]const T`). + return self.write(&v); + }, + .vector => |info| { + const array: [info.len]info.child = v; + return self.write(&array); + }, + else => @compileError("Unable to stringify type '" ++ @typeName(T) ++ "'"), + } + unreachable; +} + +fn stringValue(self: *Stringify, s: []const u8) !void { + try self.valueStart(); + try encodeJsonString(s, self.options, self.writer); + self.valueDone(); +} + +pub const Options = struct { + /// Controls the whitespace emitted. + /// The default `.minified` is a compact encoding with no whitespace between tokens. + /// Any setting other than `.minified` will use newlines, indentation, and a space after each ':'. + /// `.indent_1` means 1 space for each indentation level, `.indent_2` means 2 spaces, etc. + /// `.indent_tab` uses a tab for each indentation level. + whitespace: enum { + minified, + indent_1, + indent_2, + indent_3, + indent_4, + indent_8, + indent_tab, + } = .minified, + + /// Should optional fields with null value be written? + emit_null_optional_fields: bool = true, + + /// Arrays/slices of u8 are typically encoded as JSON strings. + /// This option emits them as arrays of numbers instead. + /// Does not affect calls to `objectField*()`. + emit_strings_as_arrays: bool = false, + + /// Should unicode characters be escaped in strings? + escape_unicode: bool = false, + + /// When true, renders numbers outside the range `+-1<<53` (the precise integer range of f64) as JSON strings in base 10. + emit_nonportable_numbers_as_strings: bool = false, +}; + +/// Writes the given value to the `Writer` writer. +/// See `Stringify` for how the given value is serialized into JSON. +/// The maximum nesting depth of the output JSON document is 256. +pub fn value(v: anytype, options: Options, writer: *Writer) Error!void { + var s: Stringify = .{ .writer = writer, .options = options }; + try s.write(v); +} + +test value { + var out: std.io.Writer.Allocating = .init(std.testing.allocator); + const writer = &out.writer; + defer out.deinit(); + + const T = struct { a: i32, b: []const u8 }; + try value(T{ .a = 123, .b = "xy" }, .{}, writer); + try std.testing.expectEqualSlices(u8, "{\"a\":123,\"b\":\"xy\"}", out.getWritten()); + + try testStringify("9999999999999999", 9999999999999999, .{}); + try testStringify("\"9999999999999999\"", 9999999999999999, .{ .emit_nonportable_numbers_as_strings = true }); + + try testStringify("[1,1]", @as(@Vector(2, u32), @splat(1)), .{}); + try testStringify("\"AA\"", @as(@Vector(2, u8), @splat('A')), .{}); + try testStringify("[65,65]", @as(@Vector(2, u8), @splat('A')), .{ .emit_strings_as_arrays = true }); + + // void field + try testStringify("{\"foo\":42}", struct { + foo: u32, + bar: void = {}, + }{ .foo = 42 }, .{}); + + const Tuple = struct { []const u8, usize }; + try testStringify("[\"foo\",42]", Tuple{ "foo", 42 }, .{}); + + comptime { + testStringify("false", false, .{}) catch unreachable; + const MyStruct = struct { foo: u32 }; + testStringify("[{\"foo\":42},{\"foo\":100},{\"foo\":1000}]", [_]MyStruct{ + MyStruct{ .foo = 42 }, + MyStruct{ .foo = 100 }, + MyStruct{ .foo = 1000 }, + }, .{}) catch unreachable; + } +} + +/// Calls `value` and stores the result in dynamically allocated memory instead +/// of taking a writer. +/// +/// Caller owns returned memory. +pub fn valueAlloc(gpa: Allocator, v: anytype, options: Options) error{OutOfMemory}![]u8 { + var aw: std.io.Writer.Allocating = .init(gpa); + defer aw.deinit(); + value(v, options, &aw.writer) catch return error.OutOfMemory; + return aw.toOwnedSlice(); +} + +test valueAlloc { + const allocator = std.testing.allocator; + const expected = + \\{"foo":"bar","answer":42,"my_friend":"sammy"} + ; + const actual = try valueAlloc(allocator, .{ .foo = "bar", .answer = 42, .my_friend = "sammy" }, .{}); + defer allocator.free(actual); + + try std.testing.expectEqualStrings(expected, actual); +} + +fn outputUnicodeEscape(codepoint: u21, w: *Writer) Error!void { + if (codepoint <= 0xFFFF) { + // If the character is in the Basic Multilingual Plane (U+0000 through U+FFFF), + // then it may be represented as a six-character sequence: a reverse solidus, followed + // by the lowercase letter u, followed by four hexadecimal digits that encode the character's code point. + try w.writeAll("\\u"); + try w.printInt(codepoint, 16, .lower, .{ .width = 4, .fill = '0' }); + } else { + assert(codepoint <= 0x10FFFF); + // To escape an extended character that is not in the Basic Multilingual Plane, + // the character is represented as a 12-character sequence, encoding the UTF-16 surrogate pair. + const high = @as(u16, @intCast((codepoint - 0x10000) >> 10)) + 0xD800; + const low = @as(u16, @intCast(codepoint & 0x3FF)) + 0xDC00; + try w.writeAll("\\u"); + try w.printInt(high, 16, .lower, .{ .width = 4, .fill = '0' }); + try w.writeAll("\\u"); + try w.printInt(low, 16, .lower, .{ .width = 4, .fill = '0' }); + } +} + +fn outputSpecialEscape(c: u8, writer: *Writer) Error!void { + switch (c) { + '\\' => try writer.writeAll("\\\\"), + '\"' => try writer.writeAll("\\\""), + 0x08 => try writer.writeAll("\\b"), + 0x0C => try writer.writeAll("\\f"), + '\n' => try writer.writeAll("\\n"), + '\r' => try writer.writeAll("\\r"), + '\t' => try writer.writeAll("\\t"), + else => try outputUnicodeEscape(c, writer), + } +} + +/// Write `string` to `writer` as a JSON encoded string. +pub fn encodeJsonString(string: []const u8, options: Options, writer: *Writer) Error!void { + try writer.writeByte('\"'); + try encodeJsonStringChars(string, options, writer); + try writer.writeByte('\"'); +} + +/// Write `chars` to `writer` as JSON encoded string characters. +pub fn encodeJsonStringChars(chars: []const u8, options: Options, writer: *Writer) Error!void { + var write_cursor: usize = 0; + var i: usize = 0; + if (options.escape_unicode) { + while (i < chars.len) : (i += 1) { + switch (chars[i]) { + // normal ascii character + 0x20...0x21, 0x23...0x5B, 0x5D...0x7E => {}, + 0x00...0x1F, '\\', '\"' => { + // Always must escape these. + try writer.writeAll(chars[write_cursor..i]); + try outputSpecialEscape(chars[i], writer); + write_cursor = i + 1; + }, + 0x7F...0xFF => { + try writer.writeAll(chars[write_cursor..i]); + const ulen = std.unicode.utf8ByteSequenceLength(chars[i]) catch unreachable; + const codepoint = std.unicode.utf8Decode(chars[i..][0..ulen]) catch unreachable; + try outputUnicodeEscape(codepoint, writer); + i += ulen - 1; + write_cursor = i + 1; + }, + } + } + } else { + while (i < chars.len) : (i += 1) { + switch (chars[i]) { + // normal bytes + 0x20...0x21, 0x23...0x5B, 0x5D...0xFF => {}, + 0x00...0x1F, '\\', '\"' => { + // Always must escape these. + try writer.writeAll(chars[write_cursor..i]); + try outputSpecialEscape(chars[i], writer); + write_cursor = i + 1; + }, + } + } + } + try writer.writeAll(chars[write_cursor..chars.len]); +} + +test "json write stream" { + var out_buf: [1024]u8 = undefined; + var out: Writer = .fixed(&out_buf); + var w: Stringify = .{ .writer = &out, .options = .{ .whitespace = .indent_2 } }; + try testBasicWriteStream(&w); +} + +fn testBasicWriteStream(w: *Stringify) !void { + w.writer.end = 0; + + try w.beginObject(); + + try w.objectField("object"); + var arena_allocator = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena_allocator.deinit(); + try w.write(try getJsonObject(arena_allocator.allocator())); + + try w.objectFieldRaw("\"string\""); + try w.write("This is a string"); + + try w.objectField("array"); + try w.beginArray(); + try w.write("Another string"); + try w.write(@as(i32, 1)); + try w.write(@as(f32, 3.5)); + try w.endArray(); + + try w.objectField("int"); + try w.write(@as(i32, 10)); + + try w.objectField("float"); + try w.write(@as(f32, 3.5)); + + try w.endObject(); + + const expected = + \\{ + \\ "object": { + \\ "one": 1, + \\ "two": 2 + \\ }, + \\ "string": "This is a string", + \\ "array": [ + \\ "Another string", + \\ 1, + \\ 3.5 + \\ ], + \\ "int": 10, + \\ "float": 3.5 + \\} + ; + try std.testing.expectEqualStrings(expected, w.writer.buffered()); +} + +fn getJsonObject(allocator: std.mem.Allocator) !std.json.Value { + var v: std.json.Value = .{ .object = std.json.ObjectMap.init(allocator) }; + try v.object.put("one", std.json.Value{ .integer = @as(i64, @intCast(1)) }); + try v.object.put("two", std.json.Value{ .float = 2.0 }); + return v; +} + +test "stringify null optional fields" { + const MyStruct = struct { + optional: ?[]const u8 = null, + required: []const u8 = "something", + another_optional: ?[]const u8 = null, + another_required: []const u8 = "something else", + }; + try testStringify( + \\{"optional":null,"required":"something","another_optional":null,"another_required":"something else"} + , + MyStruct{}, + .{}, + ); + try testStringify( + \\{"required":"something","another_required":"something else"} + , + MyStruct{}, + .{ .emit_null_optional_fields = false }, + ); +} + +test "stringify basic types" { + try testStringify("false", false, .{}); + try testStringify("true", true, .{}); + try testStringify("null", @as(?u8, null), .{}); + try testStringify("null", @as(?*u32, null), .{}); + try testStringify("42", 42, .{}); + try testStringify("42", 42.0, .{}); + try testStringify("42", @as(u8, 42), .{}); + try testStringify("42", @as(u128, 42), .{}); + try testStringify("9999999999999999", 9999999999999999, .{}); + try testStringify("42", @as(f32, 42), .{}); + try testStringify("42", @as(f64, 42), .{}); + try testStringify("\"ItBroke\"", @as(anyerror, error.ItBroke), .{}); + try testStringify("\"ItBroke\"", error.ItBroke, .{}); +} + +test "stringify string" { + try testStringify("\"hello\"", "hello", .{}); + try testStringify("\"with\\nescapes\\r\"", "with\nescapes\r", .{}); + try testStringify("\"with\\nescapes\\r\"", "with\nescapes\r", .{ .escape_unicode = true }); + try testStringify("\"with unicode\\u0001\"", "with unicode\u{1}", .{}); + try testStringify("\"with unicode\\u0001\"", "with unicode\u{1}", .{ .escape_unicode = true }); + try testStringify("\"with unicode\u{80}\"", "with unicode\u{80}", .{}); + try testStringify("\"with unicode\\u0080\"", "with unicode\u{80}", .{ .escape_unicode = true }); + try testStringify("\"with unicode\u{FF}\"", "with unicode\u{FF}", .{}); + try testStringify("\"with unicode\\u00ff\"", "with unicode\u{FF}", .{ .escape_unicode = true }); + try testStringify("\"with unicode\u{100}\"", "with unicode\u{100}", .{}); + try testStringify("\"with unicode\\u0100\"", "with unicode\u{100}", .{ .escape_unicode = true }); + try testStringify("\"with unicode\u{800}\"", "with unicode\u{800}", .{}); + try testStringify("\"with unicode\\u0800\"", "with unicode\u{800}", .{ .escape_unicode = true }); + try testStringify("\"with unicode\u{8000}\"", "with unicode\u{8000}", .{}); + try testStringify("\"with unicode\\u8000\"", "with unicode\u{8000}", .{ .escape_unicode = true }); + try testStringify("\"with unicode\u{D799}\"", "with unicode\u{D799}", .{}); + try testStringify("\"with unicode\\ud799\"", "with unicode\u{D799}", .{ .escape_unicode = true }); + try testStringify("\"with unicode\u{10000}\"", "with unicode\u{10000}", .{}); + try testStringify("\"with unicode\\ud800\\udc00\"", "with unicode\u{10000}", .{ .escape_unicode = true }); + try testStringify("\"with unicode\u{10FFFF}\"", "with unicode\u{10FFFF}", .{}); + try testStringify("\"with unicode\\udbff\\udfff\"", "with unicode\u{10FFFF}", .{ .escape_unicode = true }); +} + +test "stringify many-item sentinel-terminated string" { + try testStringify("\"hello\"", @as([*:0]const u8, "hello"), .{}); + try testStringify("\"with\\nescapes\\r\"", @as([*:0]const u8, "with\nescapes\r"), .{ .escape_unicode = true }); + try testStringify("\"with unicode\\u0001\"", @as([*:0]const u8, "with unicode\u{1}"), .{ .escape_unicode = true }); +} + +test "stringify enums" { + const E = enum { + foo, + bar, + }; + try testStringify("\"foo\"", E.foo, .{}); + try testStringify("\"bar\"", E.bar, .{}); +} + +test "stringify non-exhaustive enum" { + const E = enum(u8) { + foo = 0, + _, + }; + try testStringify("\"foo\"", E.foo, .{}); + try testStringify("1", @as(E, @enumFromInt(1)), .{}); +} + +test "stringify enum literals" { + try testStringify("\"foo\"", .foo, .{}); + try testStringify("\"bar\"", .bar, .{}); +} + +test "stringify tagged unions" { + const T = union(enum) { + nothing, + foo: u32, + bar: bool, + }; + try testStringify("{\"nothing\":{}}", T{ .nothing = {} }, .{}); + try testStringify("{\"foo\":42}", T{ .foo = 42 }, .{}); + try testStringify("{\"bar\":true}", T{ .bar = true }, .{}); +} + +test "stringify struct" { + try testStringify("{\"foo\":42}", struct { + foo: u32, + }{ .foo = 42 }, .{}); +} + +test "emit_strings_as_arrays" { + // Should only affect string values, not object keys. + try testStringify("{\"foo\":\"bar\"}", .{ .foo = "bar" }, .{}); + try testStringify("{\"foo\":[98,97,114]}", .{ .foo = "bar" }, .{ .emit_strings_as_arrays = true }); + // Should *not* affect these types: + try testStringify("\"foo\"", @as(enum { foo, bar }, .foo), .{ .emit_strings_as_arrays = true }); + try testStringify("\"ItBroke\"", error.ItBroke, .{ .emit_strings_as_arrays = true }); + // Should work on these: + try testStringify("\"bar\"", @Vector(3, u8){ 'b', 'a', 'r' }, .{}); + try testStringify("[98,97,114]", @Vector(3, u8){ 'b', 'a', 'r' }, .{ .emit_strings_as_arrays = true }); + try testStringify("\"bar\"", [3]u8{ 'b', 'a', 'r' }, .{}); + try testStringify("[98,97,114]", [3]u8{ 'b', 'a', 'r' }, .{ .emit_strings_as_arrays = true }); +} + +test "stringify struct with indentation" { + try testStringify( + \\{ + \\ "foo": 42, + \\ "bar": [ + \\ 1, + \\ 2, + \\ 3 + \\ ] + \\} + , + struct { + foo: u32, + bar: [3]u32, + }{ + .foo = 42, + .bar = .{ 1, 2, 3 }, + }, + .{ .whitespace = .indent_4 }, + ); + try testStringify( + "{\n\t\"foo\": 42,\n\t\"bar\": [\n\t\t1,\n\t\t2,\n\t\t3\n\t]\n}", + struct { + foo: u32, + bar: [3]u32, + }{ + .foo = 42, + .bar = .{ 1, 2, 3 }, + }, + .{ .whitespace = .indent_tab }, + ); + try testStringify( + \\{"foo":42,"bar":[1,2,3]} + , + struct { + foo: u32, + bar: [3]u32, + }{ + .foo = 42, + .bar = .{ 1, 2, 3 }, + }, + .{ .whitespace = .minified }, + ); +} + +test "stringify array of structs" { + const MyStruct = struct { + foo: u32, + }; + try testStringify("[{\"foo\":42},{\"foo\":100},{\"foo\":1000}]", [_]MyStruct{ + MyStruct{ .foo = 42 }, + MyStruct{ .foo = 100 }, + MyStruct{ .foo = 1000 }, + }, .{}); +} + +test "stringify struct with custom stringifier" { + try testStringify("[\"something special\",42]", struct { + foo: u32, + const Self = @This(); + pub fn jsonStringify(v: @This(), jws: anytype) !void { + _ = v; + try jws.beginArray(); + try jws.write("something special"); + try jws.write(42); + try jws.endArray(); + } + }{ .foo = 42 }, .{}); +} + +fn testStringify(expected: []const u8, v: anytype, options: Options) !void { + var buffer: [4096]u8 = undefined; + var w: Writer = .fixed(&buffer); + try value(v, options, &w); + try std.testing.expectEqualStrings(expected, w.buffered()); +} + +test "raw streaming" { + var out_buf: [1024]u8 = undefined; + var out: Writer = .fixed(&out_buf); + + var w: Stringify = .{ .writer = &out, .options = .{ .whitespace = .indent_2 } }; + try w.beginObject(); + try w.beginObjectFieldRaw(); + try w.writer.writeAll("\"long"); + try w.writer.writeAll(" key\""); + w.endObjectFieldRaw(); + try w.beginWriteRaw(); + try w.writer.writeAll("\"long"); + try w.writer.writeAll(" value\""); + w.endWriteRaw(); + try w.endObject(); + + const expected = + \\{ + \\ "long key": "long value" + \\} + ; + try std.testing.expectEqualStrings(expected, w.writer.buffered()); +} diff --git a/lib/std/json/dynamic.zig b/lib/std/json/dynamic.zig index 4d24444390..b47e7e1067 100644 --- a/lib/std/json/dynamic.zig +++ b/lib/std/json/dynamic.zig @@ -4,17 +4,12 @@ const ArenaAllocator = std.heap.ArenaAllocator; const ArrayList = std.ArrayList; const StringArrayHashMap = std.StringArrayHashMap; const Allocator = std.mem.Allocator; - -const StringifyOptions = @import("./stringify.zig").StringifyOptions; -const stringify = @import("./stringify.zig").stringify; +const json = std.json; const ParseOptions = @import("./static.zig").ParseOptions; const ParseError = @import("./static.zig").ParseError; -const JsonScanner = @import("./scanner.zig").Scanner; -const AllocWhen = @import("./scanner.zig").AllocWhen; -const Token = @import("./scanner.zig").Token; -const isNumberFormattedLikeAnInteger = @import("./scanner.zig").isNumberFormattedLikeAnInteger; +const isNumberFormattedLikeAnInteger = @import("Scanner.zig").isNumberFormattedLikeAnInteger; pub const ObjectMap = StringArrayHashMap(Value); pub const Array = ArrayList(Value); @@ -52,12 +47,11 @@ pub const Value = union(enum) { } } - pub fn dump(self: Value) void { - std.debug.lockStdErr(); - defer std.debug.unlockStdErr(); + pub fn dump(v: Value) void { + const w = std.debug.lockStderrWriter(&.{}); + defer std.debug.unlockStderrWriter(); - const stderr = std.fs.File.stderr().deprecatedWriter(); - stringify(self, .{}, stderr) catch return; + json.Stringify.value(v, .{}, w) catch return; } pub fn jsonStringify(value: @This(), jws: anytype) !void { diff --git a/lib/std/json/dynamic_test.zig b/lib/std/json/dynamic_test.zig index 1362e3cfad..9d991f2c50 100644 --- a/lib/std/json/dynamic_test.zig +++ b/lib/std/json/dynamic_test.zig @@ -1,8 +1,10 @@ const std = @import("std"); +const json = std.json; const mem = std.mem; const testing = std.testing; const ArenaAllocator = std.heap.ArenaAllocator; const Allocator = std.mem.Allocator; +const Writer = std.io.Writer; const ObjectMap = @import("dynamic.zig").ObjectMap; const Array = @import("dynamic.zig").Array; @@ -14,8 +16,7 @@ const parseFromTokenSource = @import("static.zig").parseFromTokenSource; const parseFromValueLeaky = @import("static.zig").parseFromValueLeaky; const ParseOptions = @import("static.zig").ParseOptions; -const jsonReader = @import("scanner.zig").reader; -const JsonReader = @import("scanner.zig").Reader; +const Scanner = @import("Scanner.zig"); test "json.parser.dynamic" { const s = @@ -70,14 +71,10 @@ test "json.parser.dynamic" { try testing.expect(mem.eql(u8, large_int.number_string, "18446744073709551615")); } -const writeStream = @import("./stringify.zig").writeStream; test "write json then parse it" { var out_buffer: [1000]u8 = undefined; - - var fixed_buffer_stream = std.io.fixedBufferStream(&out_buffer); - const out_stream = fixed_buffer_stream.writer(); - var jw = writeStream(out_stream, .{}); - defer jw.deinit(); + var fixed_writer: Writer = .fixed(&out_buffer); + var jw: json.Stringify = .{ .writer = &fixed_writer, .options = .{} }; try jw.beginObject(); @@ -101,8 +98,8 @@ test "write json then parse it" { try jw.endObject(); - fixed_buffer_stream = std.io.fixedBufferStream(fixed_buffer_stream.getWritten()); - var json_reader = jsonReader(testing.allocator, fixed_buffer_stream.reader()); + var fbs: std.Io.Reader = .fixed(fixed_writer.buffered()); + var json_reader: Scanner.Reader = .init(testing.allocator, &fbs); defer json_reader.deinit(); var parsed = try parseFromTokenSource(Value, testing.allocator, &json_reader, .{}); defer parsed.deinit(); @@ -242,10 +239,9 @@ test "Value.jsonStringify" { .{ .object = obj }, }; var buffer: [0x1000]u8 = undefined; - var fbs = std.io.fixedBufferStream(&buffer); + var fixed_writer: Writer = .fixed(&buffer); - var jw = writeStream(fbs.writer(), .{ .whitespace = .indent_1 }); - defer jw.deinit(); + var jw: json.Stringify = .{ .writer = &fixed_writer, .options = .{ .whitespace = .indent_1 } }; try jw.write(array); const expected = @@ -266,7 +262,7 @@ test "Value.jsonStringify" { \\ } \\] ; - try testing.expectEqualStrings(expected, fbs.getWritten()); + try testing.expectEqualStrings(expected, fixed_writer.buffered()); } test "parseFromValue(std.json.Value,...)" { @@ -334,8 +330,8 @@ test "polymorphic parsing" { test "long object value" { const value = "01234567890123456789"; const doc = "{\"key\":\"" ++ value ++ "\"}"; - var fbs = std.io.fixedBufferStream(doc); - var reader = smallBufferJsonReader(testing.allocator, fbs.reader()); + var fbs: std.Io.Reader = .fixed(doc); + var reader = smallBufferJsonReader(testing.allocator, &fbs); defer reader.deinit(); var parsed = try parseFromTokenSource(Value, testing.allocator, &reader, .{}); defer parsed.deinit(); @@ -367,8 +363,8 @@ test "many object keys" { \\ "k5": "v5" \\} ; - var fbs = std.io.fixedBufferStream(doc); - var reader = smallBufferJsonReader(testing.allocator, fbs.reader()); + var fbs: std.Io.Reader = .fixed(doc); + var reader = smallBufferJsonReader(testing.allocator, &fbs); defer reader.deinit(); var parsed = try parseFromTokenSource(Value, testing.allocator, &reader, .{}); defer parsed.deinit(); @@ -382,8 +378,8 @@ test "many object keys" { test "negative zero" { const doc = "-0"; - var fbs = std.io.fixedBufferStream(doc); - var reader = smallBufferJsonReader(testing.allocator, fbs.reader()); + var fbs: std.Io.Reader = .fixed(doc); + var reader = smallBufferJsonReader(testing.allocator, &fbs); defer reader.deinit(); var parsed = try parseFromTokenSource(Value, testing.allocator, &reader, .{}); defer parsed.deinit(); @@ -391,6 +387,6 @@ test "negative zero" { try testing.expect(std.math.isNegativeZero(parsed.value.float)); } -fn smallBufferJsonReader(allocator: Allocator, io_reader: anytype) JsonReader(16, @TypeOf(io_reader)) { - return JsonReader(16, @TypeOf(io_reader)).init(allocator, io_reader); +fn smallBufferJsonReader(allocator: Allocator, io_reader: anytype) Scanner.Reader { + return .init(allocator, io_reader); } diff --git a/lib/std/json/fmt.zig b/lib/std/json/fmt.zig deleted file mode 100644 index c526dce1da..0000000000 --- a/lib/std/json/fmt.zig +++ /dev/null @@ -1,40 +0,0 @@ -const std = @import("../std.zig"); -const assert = std.debug.assert; - -const stringify = @import("stringify.zig").stringify; -const StringifyOptions = @import("stringify.zig").StringifyOptions; - -/// Returns a formatter that formats the given value using stringify. -pub fn fmt(value: anytype, options: StringifyOptions) Formatter(@TypeOf(value)) { - return Formatter(@TypeOf(value)){ .value = value, .options = options }; -} - -/// Formats the given value using stringify. -pub fn Formatter(comptime T: type) type { - return struct { - value: T, - options: StringifyOptions, - - pub fn format(self: @This(), writer: *std.io.Writer) std.io.Writer.Error!void { - try stringify(self.value, self.options, writer); - } - }; -} - -test fmt { - const expectFmt = std.testing.expectFmt; - try expectFmt("123", "{}", .{fmt(@as(u32, 123), .{})}); - try expectFmt( - \\{"num":927,"msg":"hello","sub":{"mybool":true}} - , "{}", .{fmt(struct { - num: u32, - msg: []const u8, - sub: struct { - mybool: bool, - }, - }{ - .num = 927, - .msg = "hello", - .sub = .{ .mybool = true }, - }, .{})}); -} diff --git a/lib/std/json/hashmap_test.zig b/lib/std/json/hashmap_test.zig index 49d8caffae..0544eaa68b 100644 --- a/lib/std/json/hashmap_test.zig +++ b/lib/std/json/hashmap_test.zig @@ -1,4 +1,5 @@ const std = @import("std"); +const json = std.json; const testing = std.testing; const ArrayHashMap = @import("hashmap.zig").ArrayHashMap; @@ -7,10 +8,9 @@ const parseFromSlice = @import("static.zig").parseFromSlice; const parseFromSliceLeaky = @import("static.zig").parseFromSliceLeaky; const parseFromTokenSource = @import("static.zig").parseFromTokenSource; const parseFromValue = @import("static.zig").parseFromValue; -const stringifyAlloc = @import("stringify.zig").stringifyAlloc; const Value = @import("dynamic.zig").Value; -const jsonReader = @import("./scanner.zig").reader; +const Scanner = @import("Scanner.zig"); const T = struct { i: i32, @@ -39,8 +39,8 @@ test "parse json hashmap while streaming" { \\ "xyz": {"i": 1, "s": "w"} \\} ; - var stream = std.io.fixedBufferStream(doc); - var json_reader = jsonReader(testing.allocator, stream.reader()); + var stream: std.Io.Reader = .fixed(doc); + var json_reader: Scanner.Reader = .init(testing.allocator, &stream); var parsed = try parseFromTokenSource( ArrayHashMap(T), @@ -89,7 +89,7 @@ test "stringify json hashmap" { var value = ArrayHashMap(T){}; defer value.deinit(testing.allocator); { - const doc = try stringifyAlloc(testing.allocator, value, .{}); + const doc = try json.Stringify.valueAlloc(testing.allocator, value, .{}); defer testing.allocator.free(doc); try testing.expectEqualStrings("{}", doc); } @@ -98,7 +98,7 @@ test "stringify json hashmap" { try value.map.put(testing.allocator, "xyz", .{ .i = 1, .s = "w" }); { - const doc = try stringifyAlloc(testing.allocator, value, .{}); + const doc = try json.Stringify.valueAlloc(testing.allocator, value, .{}); defer testing.allocator.free(doc); try testing.expectEqualStrings( \\{"abc":{"i":0,"s":"d"},"xyz":{"i":1,"s":"w"}} @@ -107,7 +107,7 @@ test "stringify json hashmap" { try testing.expect(value.map.swapRemove("abc")); { - const doc = try stringifyAlloc(testing.allocator, value, .{}); + const doc = try json.Stringify.valueAlloc(testing.allocator, value, .{}); defer testing.allocator.free(doc); try testing.expectEqualStrings( \\{"xyz":{"i":1,"s":"w"}} @@ -116,7 +116,7 @@ test "stringify json hashmap" { try testing.expect(value.map.swapRemove("xyz")); { - const doc = try stringifyAlloc(testing.allocator, value, .{}); + const doc = try json.Stringify.valueAlloc(testing.allocator, value, .{}); defer testing.allocator.free(doc); try testing.expectEqualStrings("{}", doc); } @@ -129,7 +129,7 @@ test "stringify json hashmap whitespace" { try value.map.put(testing.allocator, "xyz", .{ .i = 1, .s = "w" }); { - const doc = try stringifyAlloc(testing.allocator, value, .{ .whitespace = .indent_2 }); + const doc = try json.Stringify.valueAlloc(testing.allocator, value, .{ .whitespace = .indent_2 }); defer testing.allocator.free(doc); try testing.expectEqualStrings( \\{ diff --git a/lib/std/json/scanner.zig b/lib/std/json/scanner.zig deleted file mode 100644 index 1836d6775b..0000000000 --- a/lib/std/json/scanner.zig +++ /dev/null @@ -1,1776 +0,0 @@ -// Notes on standards compliance: https://datatracker.ietf.org/doc/html/rfc8259 -// * RFC 8259 requires JSON documents be valid UTF-8, -// but makes an allowance for systems that are "part of a closed ecosystem". -// I have no idea what that's supposed to mean in the context of a standard specification. -// This implementation requires inputs to be valid UTF-8. -// * RFC 8259 contradicts itself regarding whether lowercase is allowed in \u hex digits, -// but this is probably a bug in the spec, and it's clear that lowercase is meant to be allowed. -// (RFC 5234 defines HEXDIG to only allow uppercase.) -// * When RFC 8259 refers to a "character", I assume they really mean a "Unicode scalar value". -// See http://www.unicode.org/glossary/#unicode_scalar_value . -// * RFC 8259 doesn't explicitly disallow unpaired surrogate halves in \u escape sequences, -// but vaguely implies that \u escapes are for encoding Unicode "characters" (i.e. Unicode scalar values?), -// which would mean that unpaired surrogate halves are forbidden. -// By contrast ECMA-404 (a competing(/compatible?) JSON standard, which JavaScript's JSON.parse() conforms to) -// explicitly allows unpaired surrogate halves. -// This implementation forbids unpaired surrogate halves in \u sequences. -// If a high surrogate half appears in a \u sequence, -// then a low surrogate half must immediately follow in \u notation. -// * RFC 8259 allows implementations to "accept non-JSON forms or extensions". -// This implementation does not accept any of that. -// * RFC 8259 allows implementations to put limits on "the size of texts", -// "the maximum depth of nesting", "the range and precision of numbers", -// and "the length and character contents of strings". -// This low-level implementation does not limit these, -// except where noted above, and except that nesting depth requires memory allocation. -// Note that this low-level API does not interpret numbers numerically, -// but simply emits their source form for some higher level code to make sense of. -// * This low-level implementation allows duplicate object keys, -// and key/value pairs are emitted in the order they appear in the input. - -const std = @import("std"); - -const Allocator = std.mem.Allocator; -const ArrayList = std.ArrayList; -const assert = std.debug.assert; -const BitStack = std.BitStack; - -/// Scan the input and check for malformed JSON. -/// On `SyntaxError` or `UnexpectedEndOfInput`, returns `false`. -/// Returns any errors from the allocator as-is, which is unlikely, -/// but can be caused by extreme nesting depth in the input. -pub fn validate(allocator: Allocator, s: []const u8) Allocator.Error!bool { - var scanner = Scanner.initCompleteInput(allocator, s); - defer scanner.deinit(); - - while (true) { - const token = scanner.next() catch |err| switch (err) { - error.SyntaxError, error.UnexpectedEndOfInput => return false, - error.OutOfMemory => return error.OutOfMemory, - error.BufferUnderrun => unreachable, - }; - if (token == .end_of_document) break; - } - - return true; -} - -/// The parsing errors are divided into two categories: -/// * `SyntaxError` is for clearly malformed JSON documents, -/// such as giving an input document that isn't JSON at all. -/// * `UnexpectedEndOfInput` is for signaling that everything's been -/// valid so far, but the input appears to be truncated for some reason. -/// Note that a completely empty (or whitespace-only) input will give `UnexpectedEndOfInput`. -pub const Error = error{ SyntaxError, UnexpectedEndOfInput }; - -/// Calls `std.json.Reader` with `std.json.default_buffer_size`. -pub fn reader(allocator: Allocator, io_reader: anytype) Reader(default_buffer_size, @TypeOf(io_reader)) { - return Reader(default_buffer_size, @TypeOf(io_reader)).init(allocator, io_reader); -} -/// Used by `json.reader`. -pub const default_buffer_size = 0x1000; - -/// The tokens emitted by `std.json.Scanner` and `std.json.Reader` `.next*()` functions follow this grammar: -/// ``` -/// = .end_of_document -/// = -/// | -/// | -/// | -/// | -/// | .true -/// | .false -/// | .null -/// = .object_begin ( )* .object_end -/// = .array_begin ( )* .array_end -/// = -/// = -/// ``` -/// -/// What you get for `` and `` values depends on which `next*()` method you call: -/// -/// ``` -/// next(): -/// = ( .partial_number )* .number -/// = ( )* .string -/// = -/// | .partial_string -/// | .partial_string_escaped_1 -/// | .partial_string_escaped_2 -/// | .partial_string_escaped_3 -/// | .partial_string_escaped_4 -/// -/// nextAlloc*(..., .alloc_always): -/// = .allocated_number -/// = .allocated_string -/// -/// nextAlloc*(..., .alloc_if_needed): -/// = -/// | .number -/// | .allocated_number -/// = -/// | .string -/// | .allocated_string -/// ``` -/// -/// For all tokens with a `[]const u8`, `[]u8`, or `[n]u8` payload, the payload represents the content of the value. -/// For number values, this is the representation of the number exactly as it appears in the input. -/// For strings, this is the content of the string after resolving escape sequences. -/// -/// For `.allocated_number` and `.allocated_string`, the `[]u8` payloads are allocations made with the given allocator. -/// You are responsible for managing that memory. `json.Reader.deinit()` does *not* free those allocations. -/// -/// The `.partial_*` tokens indicate that a value spans multiple input buffers or that a string contains escape sequences. -/// To get a complete value in memory, you need to concatenate the values yourself. -/// Calling `nextAlloc*()` does this for you, and returns an `.allocated_*` token with the result. -/// -/// For tokens with a `[]const u8` payload, the payload is a slice into the current input buffer. -/// The memory may become undefined during the next call to `json.Scanner.feedInput()` -/// or any `json.Reader` method whose return error set includes `json.Error`. -/// To keep the value persistently, it recommended to make a copy or to use `.alloc_always`, -/// which makes a copy for you. -/// -/// Note that `.number` and `.string` tokens that follow `.partial_*` tokens may have `0` length to indicate that -/// the previously partial value is completed with no additional bytes. -/// (This can happen when the break between input buffers happens to land on the exact end of a value. E.g. `"[1234"`, `"]"`.) -/// `.partial_*` tokens never have `0` length. -/// -/// The recommended strategy for using the different `next*()` methods is something like this: -/// -/// When you're expecting an object key, use `.alloc_if_needed`. -/// You often don't need a copy of the key string to persist; you might just check which field it is. -/// In the case that the key happens to require an allocation, free it immediately after checking it. -/// -/// When you're expecting a meaningful string value (such as on the right of a `:`), -/// use `.alloc_always` in order to keep the value valid throughout parsing the rest of the document. -/// -/// When you're expecting a number value, use `.alloc_if_needed`. -/// You're probably going to be parsing the string representation of the number into a numeric representation, -/// so you need the complete string representation only temporarily. -/// -/// When you're skipping an unrecognized value, use `skipValue()`. -pub const Token = union(enum) { - object_begin, - object_end, - array_begin, - array_end, - - true, - false, - null, - - number: []const u8, - partial_number: []const u8, - allocated_number: []u8, - - string: []const u8, - partial_string: []const u8, - partial_string_escaped_1: [1]u8, - partial_string_escaped_2: [2]u8, - partial_string_escaped_3: [3]u8, - partial_string_escaped_4: [4]u8, - allocated_string: []u8, - - end_of_document, -}; - -/// This is only used in `peekNextTokenType()` and gives a categorization based on the first byte of the next token that will be emitted from a `next*()` call. -pub const TokenType = enum { - object_begin, - object_end, - array_begin, - array_end, - true, - false, - null, - number, - string, - end_of_document, -}; - -/// To enable diagnostics, declare `var diagnostics = Diagnostics{};` then call `source.enableDiagnostics(&diagnostics);` -/// where `source` is either a `std.json.Reader` or a `std.json.Scanner` that has just been initialized. -/// At any time, notably just after an error, call `getLine()`, `getColumn()`, and/or `getByteOffset()` -/// to get meaningful information from this. -pub const Diagnostics = struct { - line_number: u64 = 1, - line_start_cursor: usize = @as(usize, @bitCast(@as(isize, -1))), // Start just "before" the input buffer to get a 1-based column for line 1. - total_bytes_before_current_input: u64 = 0, - cursor_pointer: *const usize = undefined, - - /// Starts at 1. - pub fn getLine(self: *const @This()) u64 { - return self.line_number; - } - /// Starts at 1. - pub fn getColumn(self: *const @This()) u64 { - return self.cursor_pointer.* -% self.line_start_cursor; - } - /// Starts at 0. Measures the byte offset since the start of the input. - pub fn getByteOffset(self: *const @This()) u64 { - return self.total_bytes_before_current_input + self.cursor_pointer.*; - } -}; - -/// See the documentation for `std.json.Token`. -pub const AllocWhen = enum { alloc_if_needed, alloc_always }; - -/// For security, the maximum size allocated to store a single string or number value is limited to 4MiB by default. -/// This limit can be specified by calling `nextAllocMax()` instead of `nextAlloc()`. -pub const default_max_value_len = 4 * 1024 * 1024; - -/// Connects a `std.io.GenericReader` to a `std.json.Scanner`. -/// All `next*()` methods here handle `error.BufferUnderrun` from `std.json.Scanner`, and then read from the reader. -pub fn Reader(comptime buffer_size: usize, comptime ReaderType: type) type { - return struct { - scanner: Scanner, - reader: ReaderType, - - buffer: [buffer_size]u8 = undefined, - - /// The allocator is only used to track `[]` and `{}` nesting levels. - pub fn init(allocator: Allocator, io_reader: ReaderType) @This() { - return .{ - .scanner = Scanner.initStreaming(allocator), - .reader = io_reader, - }; - } - pub fn deinit(self: *@This()) void { - self.scanner.deinit(); - self.* = undefined; - } - - /// Calls `std.json.Scanner.enableDiagnostics`. - pub fn enableDiagnostics(self: *@This(), diagnostics: *Diagnostics) void { - self.scanner.enableDiagnostics(diagnostics); - } - - pub const NextError = ReaderType.Error || Error || Allocator.Error; - pub const SkipError = NextError; - pub const AllocError = NextError || error{ValueTooLong}; - pub const PeekError = ReaderType.Error || Error; - - /// Equivalent to `nextAllocMax(allocator, when, default_max_value_len);` - /// See also `std.json.Token` for documentation of `nextAlloc*()` function behavior. - pub fn nextAlloc(self: *@This(), allocator: Allocator, when: AllocWhen) AllocError!Token { - return self.nextAllocMax(allocator, when, default_max_value_len); - } - /// See also `std.json.Token` for documentation of `nextAlloc*()` function behavior. - pub fn nextAllocMax(self: *@This(), allocator: Allocator, when: AllocWhen, max_value_len: usize) AllocError!Token { - const token_type = try self.peekNextTokenType(); - switch (token_type) { - .number, .string => { - var value_list = ArrayList(u8).init(allocator); - errdefer { - value_list.deinit(); - } - if (try self.allocNextIntoArrayListMax(&value_list, when, max_value_len)) |slice| { - return if (token_type == .number) - Token{ .number = slice } - else - Token{ .string = slice }; - } else { - return if (token_type == .number) - Token{ .allocated_number = try value_list.toOwnedSlice() } - else - Token{ .allocated_string = try value_list.toOwnedSlice() }; - } - }, - - // Simple tokens never alloc. - .object_begin, - .object_end, - .array_begin, - .array_end, - .true, - .false, - .null, - .end_of_document, - => return try self.next(), - } - } - - /// Equivalent to `allocNextIntoArrayListMax(value_list, when, default_max_value_len);` - pub fn allocNextIntoArrayList(self: *@This(), value_list: *ArrayList(u8), when: AllocWhen) AllocError!?[]const u8 { - return self.allocNextIntoArrayListMax(value_list, when, default_max_value_len); - } - /// Calls `std.json.Scanner.allocNextIntoArrayListMax` and handles `error.BufferUnderrun`. - pub fn allocNextIntoArrayListMax(self: *@This(), value_list: *ArrayList(u8), when: AllocWhen, max_value_len: usize) AllocError!?[]const u8 { - while (true) { - return self.scanner.allocNextIntoArrayListMax(value_list, when, max_value_len) catch |err| switch (err) { - error.BufferUnderrun => { - try self.refillBuffer(); - continue; - }, - else => |other_err| return other_err, - }; - } - } - - /// Like `std.json.Scanner.skipValue`, but handles `error.BufferUnderrun`. - pub fn skipValue(self: *@This()) SkipError!void { - switch (try self.peekNextTokenType()) { - .object_begin, .array_begin => { - try self.skipUntilStackHeight(self.stackHeight()); - }, - .number, .string => { - while (true) { - switch (try self.next()) { - .partial_number, - .partial_string, - .partial_string_escaped_1, - .partial_string_escaped_2, - .partial_string_escaped_3, - .partial_string_escaped_4, - => continue, - - .number, .string => break, - - else => unreachable, - } - } - }, - .true, .false, .null => { - _ = try self.next(); - }, - - .object_end, .array_end, .end_of_document => unreachable, // Attempt to skip a non-value token. - } - } - /// Like `std.json.Scanner.skipUntilStackHeight()` but handles `error.BufferUnderrun`. - pub fn skipUntilStackHeight(self: *@This(), terminal_stack_height: usize) NextError!void { - while (true) { - return self.scanner.skipUntilStackHeight(terminal_stack_height) catch |err| switch (err) { - error.BufferUnderrun => { - try self.refillBuffer(); - continue; - }, - else => |other_err| return other_err, - }; - } - } - - /// Calls `std.json.Scanner.stackHeight`. - pub fn stackHeight(self: *const @This()) usize { - return self.scanner.stackHeight(); - } - /// Calls `std.json.Scanner.ensureTotalStackCapacity`. - pub fn ensureTotalStackCapacity(self: *@This(), height: usize) Allocator.Error!void { - try self.scanner.ensureTotalStackCapacity(height); - } - - /// See `std.json.Token` for documentation of this function. - pub fn next(self: *@This()) NextError!Token { - while (true) { - return self.scanner.next() catch |err| switch (err) { - error.BufferUnderrun => { - try self.refillBuffer(); - continue; - }, - else => |other_err| return other_err, - }; - } - } - - /// See `std.json.Scanner.peekNextTokenType()`. - pub fn peekNextTokenType(self: *@This()) PeekError!TokenType { - while (true) { - return self.scanner.peekNextTokenType() catch |err| switch (err) { - error.BufferUnderrun => { - try self.refillBuffer(); - continue; - }, - else => |other_err| return other_err, - }; - } - } - - fn refillBuffer(self: *@This()) ReaderType.Error!void { - const input = self.buffer[0..try self.reader.read(self.buffer[0..])]; - if (input.len > 0) { - self.scanner.feedInput(input); - } else { - self.scanner.endInput(); - } - } - }; -} - -/// The lowest level parsing API in this package; -/// supports streaming input with a low memory footprint. -/// The memory requirement is `O(d)` where d is the nesting depth of `[]` or `{}` containers in the input. -/// Specifically `d/8` bytes are required for this purpose, -/// with some extra buffer according to the implementation of `std.ArrayList`. -/// -/// This scanner can emit partial tokens; see `std.json.Token`. -/// The input to this class is a sequence of input buffers that you must supply one at a time. -/// Call `feedInput()` with the first buffer, then call `next()` repeatedly until `error.BufferUnderrun` is returned. -/// Then call `feedInput()` again and so forth. -/// Call `endInput()` when the last input buffer has been given to `feedInput()`, either immediately after calling `feedInput()`, -/// or when `error.BufferUnderrun` requests more data and there is no more. -/// Be sure to call `next()` after calling `endInput()` until `Token.end_of_document` has been returned. -pub const Scanner = struct { - state: State = .value, - string_is_object_key: bool = false, - stack: BitStack, - value_start: usize = undefined, - utf16_code_units: [2]u16 = undefined, - - input: []const u8 = "", - cursor: usize = 0, - is_end_of_input: bool = false, - diagnostics: ?*Diagnostics = null, - - /// The allocator is only used to track `[]` and `{}` nesting levels. - pub fn initStreaming(allocator: Allocator) @This() { - return .{ - .stack = BitStack.init(allocator), - }; - } - /// Use this if your input is a single slice. - /// This is effectively equivalent to: - /// ``` - /// initStreaming(allocator); - /// feedInput(complete_input); - /// endInput(); - /// ``` - pub fn initCompleteInput(allocator: Allocator, complete_input: []const u8) @This() { - return .{ - .stack = BitStack.init(allocator), - .input = complete_input, - .is_end_of_input = true, - }; - } - pub fn deinit(self: *@This()) void { - self.stack.deinit(); - self.* = undefined; - } - - pub fn enableDiagnostics(self: *@This(), diagnostics: *Diagnostics) void { - diagnostics.cursor_pointer = &self.cursor; - self.diagnostics = diagnostics; - } - - /// Call this whenever you get `error.BufferUnderrun` from `next()`. - /// When there is no more input to provide, call `endInput()`. - pub fn feedInput(self: *@This(), input: []const u8) void { - assert(self.cursor == self.input.len); // Not done with the last input slice. - if (self.diagnostics) |diag| { - diag.total_bytes_before_current_input += self.input.len; - // This usually goes "negative" to measure how far before the beginning - // of the new buffer the current line started. - diag.line_start_cursor -%= self.cursor; - } - self.input = input; - self.cursor = 0; - self.value_start = 0; - } - /// Call this when you will no longer call `feedInput()` anymore. - /// This can be called either immediately after the last `feedInput()`, - /// or at any time afterward, such as when getting `error.BufferUnderrun` from `next()`. - /// Don't forget to call `next*()` after `endInput()` until you get `.end_of_document`. - pub fn endInput(self: *@This()) void { - self.is_end_of_input = true; - } - - pub const NextError = Error || Allocator.Error || error{BufferUnderrun}; - pub const AllocError = Error || Allocator.Error || error{ValueTooLong}; - pub const PeekError = Error || error{BufferUnderrun}; - pub const SkipError = Error || Allocator.Error; - pub const AllocIntoArrayListError = AllocError || error{BufferUnderrun}; - - /// Equivalent to `nextAllocMax(allocator, when, default_max_value_len);` - /// This function is only available after `endInput()` (or `initCompleteInput()`) has been called. - /// See also `std.json.Token` for documentation of `nextAlloc*()` function behavior. - pub fn nextAlloc(self: *@This(), allocator: Allocator, when: AllocWhen) AllocError!Token { - return self.nextAllocMax(allocator, when, default_max_value_len); - } - - /// This function is only available after `endInput()` (or `initCompleteInput()`) has been called. - /// See also `std.json.Token` for documentation of `nextAlloc*()` function behavior. - pub fn nextAllocMax(self: *@This(), allocator: Allocator, when: AllocWhen, max_value_len: usize) AllocError!Token { - assert(self.is_end_of_input); // This function is not available in streaming mode. - const token_type = self.peekNextTokenType() catch |e| switch (e) { - error.BufferUnderrun => unreachable, - else => |err| return err, - }; - switch (token_type) { - .number, .string => { - var value_list = ArrayList(u8).init(allocator); - errdefer { - value_list.deinit(); - } - if (self.allocNextIntoArrayListMax(&value_list, when, max_value_len) catch |e| switch (e) { - error.BufferUnderrun => unreachable, - else => |err| return err, - }) |slice| { - return if (token_type == .number) - Token{ .number = slice } - else - Token{ .string = slice }; - } else { - return if (token_type == .number) - Token{ .allocated_number = try value_list.toOwnedSlice() } - else - Token{ .allocated_string = try value_list.toOwnedSlice() }; - } - }, - - // Simple tokens never alloc. - .object_begin, - .object_end, - .array_begin, - .array_end, - .true, - .false, - .null, - .end_of_document, - => return self.next() catch |e| switch (e) { - error.BufferUnderrun => unreachable, - else => |err| return err, - }, - } - } - - /// Equivalent to `allocNextIntoArrayListMax(value_list, when, default_max_value_len);` - pub fn allocNextIntoArrayList(self: *@This(), value_list: *ArrayList(u8), when: AllocWhen) AllocIntoArrayListError!?[]const u8 { - return self.allocNextIntoArrayListMax(value_list, when, default_max_value_len); - } - /// The next token type must be either `.number` or `.string`. See `peekNextTokenType()`. - /// When allocation is not necessary with `.alloc_if_needed`, - /// this method returns the content slice from the input buffer, and `value_list` is not touched. - /// When allocation is necessary or with `.alloc_always`, this method concatenates partial tokens into the given `value_list`, - /// and returns `null` once the final `.number` or `.string` token has been written into it. - /// In case of an `error.BufferUnderrun`, partial values will be left in the given value_list. - /// The given `value_list` is never reset by this method, so an `error.BufferUnderrun` situation - /// can be resumed by passing the same array list in again. - /// This method does not indicate whether the token content being returned is for a `.number` or `.string` token type; - /// the caller of this method is expected to know which type of token is being processed. - pub fn allocNextIntoArrayListMax(self: *@This(), value_list: *ArrayList(u8), when: AllocWhen, max_value_len: usize) AllocIntoArrayListError!?[]const u8 { - while (true) { - const token = try self.next(); - switch (token) { - // Accumulate partial values. - .partial_number, .partial_string => |slice| { - try appendSlice(value_list, slice, max_value_len); - }, - .partial_string_escaped_1 => |buf| { - try appendSlice(value_list, buf[0..], max_value_len); - }, - .partial_string_escaped_2 => |buf| { - try appendSlice(value_list, buf[0..], max_value_len); - }, - .partial_string_escaped_3 => |buf| { - try appendSlice(value_list, buf[0..], max_value_len); - }, - .partial_string_escaped_4 => |buf| { - try appendSlice(value_list, buf[0..], max_value_len); - }, - - // Return complete values. - .number => |slice| { - if (when == .alloc_if_needed and value_list.items.len == 0) { - // No alloc necessary. - return slice; - } - try appendSlice(value_list, slice, max_value_len); - // The token is complete. - return null; - }, - .string => |slice| { - if (when == .alloc_if_needed and value_list.items.len == 0) { - // No alloc necessary. - return slice; - } - try appendSlice(value_list, slice, max_value_len); - // The token is complete. - return null; - }, - - .object_begin, - .object_end, - .array_begin, - .array_end, - .true, - .false, - .null, - .end_of_document, - => unreachable, // Only .number and .string token types are allowed here. Check peekNextTokenType() before calling this. - - .allocated_number, .allocated_string => unreachable, - } - } - } - - /// This function is only available after `endInput()` (or `initCompleteInput()`) has been called. - /// If the next token type is `.object_begin` or `.array_begin`, - /// this function calls `next()` repeatedly until the corresponding `.object_end` or `.array_end` is found. - /// If the next token type is `.number` or `.string`, - /// this function calls `next()` repeatedly until the (non `.partial_*`) `.number` or `.string` token is found. - /// If the next token type is `.true`, `.false`, or `.null`, this function calls `next()` once. - /// The next token type must not be `.object_end`, `.array_end`, or `.end_of_document`; - /// see `peekNextTokenType()`. - pub fn skipValue(self: *@This()) SkipError!void { - assert(self.is_end_of_input); // This function is not available in streaming mode. - switch (self.peekNextTokenType() catch |e| switch (e) { - error.BufferUnderrun => unreachable, - else => |err| return err, - }) { - .object_begin, .array_begin => { - self.skipUntilStackHeight(self.stackHeight()) catch |e| switch (e) { - error.BufferUnderrun => unreachable, - else => |err| return err, - }; - }, - .number, .string => { - while (true) { - switch (self.next() catch |e| switch (e) { - error.BufferUnderrun => unreachable, - else => |err| return err, - }) { - .partial_number, - .partial_string, - .partial_string_escaped_1, - .partial_string_escaped_2, - .partial_string_escaped_3, - .partial_string_escaped_4, - => continue, - - .number, .string => break, - - else => unreachable, - } - } - }, - .true, .false, .null => { - _ = self.next() catch |e| switch (e) { - error.BufferUnderrun => unreachable, - else => |err| return err, - }; - }, - - .object_end, .array_end, .end_of_document => unreachable, // Attempt to skip a non-value token. - } - } - - /// Skip tokens until an `.object_end` or `.array_end` token results in a `stackHeight()` equal the given stack height. - /// Unlike `skipValue()`, this function is available in streaming mode. - pub fn skipUntilStackHeight(self: *@This(), terminal_stack_height: usize) NextError!void { - while (true) { - switch (try self.next()) { - .object_end, .array_end => { - if (self.stackHeight() == terminal_stack_height) break; - }, - .end_of_document => unreachable, - else => continue, - } - } - } - - /// The depth of `{}` or `[]` nesting levels at the current position. - pub fn stackHeight(self: *const @This()) usize { - return self.stack.bit_len; - } - - /// Pre allocate memory to hold the given number of nesting levels. - /// `stackHeight()` up to the given number will not cause allocations. - pub fn ensureTotalStackCapacity(self: *@This(), height: usize) Allocator.Error!void { - try self.stack.ensureTotalCapacity(height); - } - - /// See `std.json.Token` for documentation of this function. - pub fn next(self: *@This()) NextError!Token { - state_loop: while (true) { - switch (self.state) { - .value => { - switch (try self.skipWhitespaceExpectByte()) { - // Object, Array - '{' => { - try self.stack.push(OBJECT_MODE); - self.cursor += 1; - self.state = .object_start; - return .object_begin; - }, - '[' => { - try self.stack.push(ARRAY_MODE); - self.cursor += 1; - self.state = .array_start; - return .array_begin; - }, - - // String - '"' => { - self.cursor += 1; - self.value_start = self.cursor; - self.state = .string; - continue :state_loop; - }, - - // Number - '1'...'9' => { - self.value_start = self.cursor; - self.cursor += 1; - self.state = .number_int; - continue :state_loop; - }, - '0' => { - self.value_start = self.cursor; - self.cursor += 1; - self.state = .number_leading_zero; - continue :state_loop; - }, - '-' => { - self.value_start = self.cursor; - self.cursor += 1; - self.state = .number_minus; - continue :state_loop; - }, - - // literal values - 't' => { - self.cursor += 1; - self.state = .literal_t; - continue :state_loop; - }, - 'f' => { - self.cursor += 1; - self.state = .literal_f; - continue :state_loop; - }, - 'n' => { - self.cursor += 1; - self.state = .literal_n; - continue :state_loop; - }, - - else => return error.SyntaxError, - } - }, - - .post_value => { - if (try self.skipWhitespaceCheckEnd()) return .end_of_document; - - const c = self.input[self.cursor]; - if (self.string_is_object_key) { - self.string_is_object_key = false; - switch (c) { - ':' => { - self.cursor += 1; - self.state = .value; - continue :state_loop; - }, - else => return error.SyntaxError, - } - } - - switch (c) { - '}' => { - if (self.stack.pop() != OBJECT_MODE) return error.SyntaxError; - self.cursor += 1; - // stay in .post_value state. - return .object_end; - }, - ']' => { - if (self.stack.pop() != ARRAY_MODE) return error.SyntaxError; - self.cursor += 1; - // stay in .post_value state. - return .array_end; - }, - ',' => { - switch (self.stack.peek()) { - OBJECT_MODE => { - self.state = .object_post_comma; - }, - ARRAY_MODE => { - self.state = .value; - }, - } - self.cursor += 1; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - - .object_start => { - switch (try self.skipWhitespaceExpectByte()) { - '"' => { - self.cursor += 1; - self.value_start = self.cursor; - self.state = .string; - self.string_is_object_key = true; - continue :state_loop; - }, - '}' => { - self.cursor += 1; - _ = self.stack.pop(); - self.state = .post_value; - return .object_end; - }, - else => return error.SyntaxError, - } - }, - .object_post_comma => { - switch (try self.skipWhitespaceExpectByte()) { - '"' => { - self.cursor += 1; - self.value_start = self.cursor; - self.state = .string; - self.string_is_object_key = true; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - - .array_start => { - switch (try self.skipWhitespaceExpectByte()) { - ']' => { - self.cursor += 1; - _ = self.stack.pop(); - self.state = .post_value; - return .array_end; - }, - else => { - self.state = .value; - continue :state_loop; - }, - } - }, - - .number_minus => { - if (self.cursor >= self.input.len) return self.endOfBufferInNumber(false); - switch (self.input[self.cursor]) { - '0' => { - self.cursor += 1; - self.state = .number_leading_zero; - continue :state_loop; - }, - '1'...'9' => { - self.cursor += 1; - self.state = .number_int; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - .number_leading_zero => { - if (self.cursor >= self.input.len) return self.endOfBufferInNumber(true); - switch (self.input[self.cursor]) { - '.' => { - self.cursor += 1; - self.state = .number_post_dot; - continue :state_loop; - }, - 'e', 'E' => { - self.cursor += 1; - self.state = .number_post_e; - continue :state_loop; - }, - else => { - self.state = .post_value; - return Token{ .number = self.takeValueSlice() }; - }, - } - }, - .number_int => { - while (self.cursor < self.input.len) : (self.cursor += 1) { - switch (self.input[self.cursor]) { - '0'...'9' => continue, - '.' => { - self.cursor += 1; - self.state = .number_post_dot; - continue :state_loop; - }, - 'e', 'E' => { - self.cursor += 1; - self.state = .number_post_e; - continue :state_loop; - }, - else => { - self.state = .post_value; - return Token{ .number = self.takeValueSlice() }; - }, - } - } - return self.endOfBufferInNumber(true); - }, - .number_post_dot => { - if (self.cursor >= self.input.len) return self.endOfBufferInNumber(false); - switch (self.input[self.cursor]) { - '0'...'9' => { - self.cursor += 1; - self.state = .number_frac; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - .number_frac => { - while (self.cursor < self.input.len) : (self.cursor += 1) { - switch (self.input[self.cursor]) { - '0'...'9' => continue, - 'e', 'E' => { - self.cursor += 1; - self.state = .number_post_e; - continue :state_loop; - }, - else => { - self.state = .post_value; - return Token{ .number = self.takeValueSlice() }; - }, - } - } - return self.endOfBufferInNumber(true); - }, - .number_post_e => { - if (self.cursor >= self.input.len) return self.endOfBufferInNumber(false); - switch (self.input[self.cursor]) { - '0'...'9' => { - self.cursor += 1; - self.state = .number_exp; - continue :state_loop; - }, - '+', '-' => { - self.cursor += 1; - self.state = .number_post_e_sign; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - .number_post_e_sign => { - if (self.cursor >= self.input.len) return self.endOfBufferInNumber(false); - switch (self.input[self.cursor]) { - '0'...'9' => { - self.cursor += 1; - self.state = .number_exp; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - .number_exp => { - while (self.cursor < self.input.len) : (self.cursor += 1) { - switch (self.input[self.cursor]) { - '0'...'9' => continue, - else => { - self.state = .post_value; - return Token{ .number = self.takeValueSlice() }; - }, - } - } - return self.endOfBufferInNumber(true); - }, - - .string => { - while (self.cursor < self.input.len) : (self.cursor += 1) { - switch (self.input[self.cursor]) { - 0...0x1f => return error.SyntaxError, // Bare ASCII control code in string. - - // ASCII plain text. - 0x20...('"' - 1), ('"' + 1)...('\\' - 1), ('\\' + 1)...0x7F => continue, - - // Special characters. - '"' => { - const result = Token{ .string = self.takeValueSlice() }; - self.cursor += 1; - self.state = .post_value; - return result; - }, - '\\' => { - const slice = self.takeValueSlice(); - self.cursor += 1; - self.state = .string_backslash; - if (slice.len > 0) return Token{ .partial_string = slice }; - continue :state_loop; - }, - - // UTF-8 validation. - // See http://unicode.org/mail-arch/unicode-ml/y2003-m02/att-0467/01-The_Algorithm_to_Valide_an_UTF-8_String - 0xC2...0xDF => { - self.cursor += 1; - self.state = .string_utf8_last_byte; - continue :state_loop; - }, - 0xE0 => { - self.cursor += 1; - self.state = .string_utf8_second_to_last_byte_guard_against_overlong; - continue :state_loop; - }, - 0xE1...0xEC, 0xEE...0xEF => { - self.cursor += 1; - self.state = .string_utf8_second_to_last_byte; - continue :state_loop; - }, - 0xED => { - self.cursor += 1; - self.state = .string_utf8_second_to_last_byte_guard_against_surrogate_half; - continue :state_loop; - }, - 0xF0 => { - self.cursor += 1; - self.state = .string_utf8_third_to_last_byte_guard_against_overlong; - continue :state_loop; - }, - 0xF1...0xF3 => { - self.cursor += 1; - self.state = .string_utf8_third_to_last_byte; - continue :state_loop; - }, - 0xF4 => { - self.cursor += 1; - self.state = .string_utf8_third_to_last_byte_guard_against_too_large; - continue :state_loop; - }, - 0x80...0xC1, 0xF5...0xFF => return error.SyntaxError, // Invalid UTF-8. - } - } - if (self.is_end_of_input) return error.UnexpectedEndOfInput; - const slice = self.takeValueSlice(); - if (slice.len > 0) return Token{ .partial_string = slice }; - return error.BufferUnderrun; - }, - .string_backslash => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - switch (self.input[self.cursor]) { - '"', '\\', '/' => { - // Since these characters now represent themselves literally, - // we can simply begin the next plaintext slice here. - self.value_start = self.cursor; - self.cursor += 1; - self.state = .string; - continue :state_loop; - }, - 'b' => { - self.cursor += 1; - self.value_start = self.cursor; - self.state = .string; - return Token{ .partial_string_escaped_1 = [_]u8{0x08} }; - }, - 'f' => { - self.cursor += 1; - self.value_start = self.cursor; - self.state = .string; - return Token{ .partial_string_escaped_1 = [_]u8{0x0c} }; - }, - 'n' => { - self.cursor += 1; - self.value_start = self.cursor; - self.state = .string; - return Token{ .partial_string_escaped_1 = [_]u8{'\n'} }; - }, - 'r' => { - self.cursor += 1; - self.value_start = self.cursor; - self.state = .string; - return Token{ .partial_string_escaped_1 = [_]u8{'\r'} }; - }, - 't' => { - self.cursor += 1; - self.value_start = self.cursor; - self.state = .string; - return Token{ .partial_string_escaped_1 = [_]u8{'\t'} }; - }, - 'u' => { - self.cursor += 1; - self.state = .string_backslash_u; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - .string_backslash_u => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - const c = self.input[self.cursor]; - switch (c) { - '0'...'9' => { - self.utf16_code_units[0] = @as(u16, c - '0') << 12; - }, - 'A'...'F' => { - self.utf16_code_units[0] = @as(u16, c - 'A' + 10) << 12; - }, - 'a'...'f' => { - self.utf16_code_units[0] = @as(u16, c - 'a' + 10) << 12; - }, - else => return error.SyntaxError, - } - self.cursor += 1; - self.state = .string_backslash_u_1; - continue :state_loop; - }, - .string_backslash_u_1 => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - const c = self.input[self.cursor]; - switch (c) { - '0'...'9' => { - self.utf16_code_units[0] |= @as(u16, c - '0') << 8; - }, - 'A'...'F' => { - self.utf16_code_units[0] |= @as(u16, c - 'A' + 10) << 8; - }, - 'a'...'f' => { - self.utf16_code_units[0] |= @as(u16, c - 'a' + 10) << 8; - }, - else => return error.SyntaxError, - } - self.cursor += 1; - self.state = .string_backslash_u_2; - continue :state_loop; - }, - .string_backslash_u_2 => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - const c = self.input[self.cursor]; - switch (c) { - '0'...'9' => { - self.utf16_code_units[0] |= @as(u16, c - '0') << 4; - }, - 'A'...'F' => { - self.utf16_code_units[0] |= @as(u16, c - 'A' + 10) << 4; - }, - 'a'...'f' => { - self.utf16_code_units[0] |= @as(u16, c - 'a' + 10) << 4; - }, - else => return error.SyntaxError, - } - self.cursor += 1; - self.state = .string_backslash_u_3; - continue :state_loop; - }, - .string_backslash_u_3 => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - const c = self.input[self.cursor]; - switch (c) { - '0'...'9' => { - self.utf16_code_units[0] |= c - '0'; - }, - 'A'...'F' => { - self.utf16_code_units[0] |= c - 'A' + 10; - }, - 'a'...'f' => { - self.utf16_code_units[0] |= c - 'a' + 10; - }, - else => return error.SyntaxError, - } - self.cursor += 1; - if (std.unicode.utf16IsHighSurrogate(self.utf16_code_units[0])) { - self.state = .string_surrogate_half; - continue :state_loop; - } else if (std.unicode.utf16IsLowSurrogate(self.utf16_code_units[0])) { - return error.SyntaxError; // Unexpected low surrogate half. - } else { - self.value_start = self.cursor; - self.state = .string; - return partialStringCodepoint(self.utf16_code_units[0]); - } - }, - .string_surrogate_half => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - switch (self.input[self.cursor]) { - '\\' => { - self.cursor += 1; - self.state = .string_surrogate_half_backslash; - continue :state_loop; - }, - else => return error.SyntaxError, // Expected low surrogate half. - } - }, - .string_surrogate_half_backslash => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - switch (self.input[self.cursor]) { - 'u' => { - self.cursor += 1; - self.state = .string_surrogate_half_backslash_u; - continue :state_loop; - }, - else => return error.SyntaxError, // Expected low surrogate half. - } - }, - .string_surrogate_half_backslash_u => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - switch (self.input[self.cursor]) { - 'D', 'd' => { - self.cursor += 1; - self.utf16_code_units[1] = 0xD << 12; - self.state = .string_surrogate_half_backslash_u_1; - continue :state_loop; - }, - else => return error.SyntaxError, // Expected low surrogate half. - } - }, - .string_surrogate_half_backslash_u_1 => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - const c = self.input[self.cursor]; - switch (c) { - 'C'...'F' => { - self.cursor += 1; - self.utf16_code_units[1] |= @as(u16, c - 'A' + 10) << 8; - self.state = .string_surrogate_half_backslash_u_2; - continue :state_loop; - }, - 'c'...'f' => { - self.cursor += 1; - self.utf16_code_units[1] |= @as(u16, c - 'a' + 10) << 8; - self.state = .string_surrogate_half_backslash_u_2; - continue :state_loop; - }, - else => return error.SyntaxError, // Expected low surrogate half. - } - }, - .string_surrogate_half_backslash_u_2 => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - const c = self.input[self.cursor]; - switch (c) { - '0'...'9' => { - self.cursor += 1; - self.utf16_code_units[1] |= @as(u16, c - '0') << 4; - self.state = .string_surrogate_half_backslash_u_3; - continue :state_loop; - }, - 'A'...'F' => { - self.cursor += 1; - self.utf16_code_units[1] |= @as(u16, c - 'A' + 10) << 4; - self.state = .string_surrogate_half_backslash_u_3; - continue :state_loop; - }, - 'a'...'f' => { - self.cursor += 1; - self.utf16_code_units[1] |= @as(u16, c - 'a' + 10) << 4; - self.state = .string_surrogate_half_backslash_u_3; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - .string_surrogate_half_backslash_u_3 => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - const c = self.input[self.cursor]; - switch (c) { - '0'...'9' => { - self.utf16_code_units[1] |= c - '0'; - }, - 'A'...'F' => { - self.utf16_code_units[1] |= c - 'A' + 10; - }, - 'a'...'f' => { - self.utf16_code_units[1] |= c - 'a' + 10; - }, - else => return error.SyntaxError, - } - self.cursor += 1; - self.value_start = self.cursor; - self.state = .string; - const code_point = std.unicode.utf16DecodeSurrogatePair(&self.utf16_code_units) catch unreachable; - return partialStringCodepoint(code_point); - }, - - .string_utf8_last_byte => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - switch (self.input[self.cursor]) { - 0x80...0xBF => { - self.cursor += 1; - self.state = .string; - continue :state_loop; - }, - else => return error.SyntaxError, // Invalid UTF-8. - } - }, - .string_utf8_second_to_last_byte => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - switch (self.input[self.cursor]) { - 0x80...0xBF => { - self.cursor += 1; - self.state = .string_utf8_last_byte; - continue :state_loop; - }, - else => return error.SyntaxError, // Invalid UTF-8. - } - }, - .string_utf8_second_to_last_byte_guard_against_overlong => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - switch (self.input[self.cursor]) { - 0xA0...0xBF => { - self.cursor += 1; - self.state = .string_utf8_last_byte; - continue :state_loop; - }, - else => return error.SyntaxError, // Invalid UTF-8. - } - }, - .string_utf8_second_to_last_byte_guard_against_surrogate_half => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - switch (self.input[self.cursor]) { - 0x80...0x9F => { - self.cursor += 1; - self.state = .string_utf8_last_byte; - continue :state_loop; - }, - else => return error.SyntaxError, // Invalid UTF-8. - } - }, - .string_utf8_third_to_last_byte => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - switch (self.input[self.cursor]) { - 0x80...0xBF => { - self.cursor += 1; - self.state = .string_utf8_second_to_last_byte; - continue :state_loop; - }, - else => return error.SyntaxError, // Invalid UTF-8. - } - }, - .string_utf8_third_to_last_byte_guard_against_overlong => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - switch (self.input[self.cursor]) { - 0x90...0xBF => { - self.cursor += 1; - self.state = .string_utf8_second_to_last_byte; - continue :state_loop; - }, - else => return error.SyntaxError, // Invalid UTF-8. - } - }, - .string_utf8_third_to_last_byte_guard_against_too_large => { - if (self.cursor >= self.input.len) return self.endOfBufferInString(); - switch (self.input[self.cursor]) { - 0x80...0x8F => { - self.cursor += 1; - self.state = .string_utf8_second_to_last_byte; - continue :state_loop; - }, - else => return error.SyntaxError, // Invalid UTF-8. - } - }, - - .literal_t => { - switch (try self.expectByte()) { - 'r' => { - self.cursor += 1; - self.state = .literal_tr; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - .literal_tr => { - switch (try self.expectByte()) { - 'u' => { - self.cursor += 1; - self.state = .literal_tru; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - .literal_tru => { - switch (try self.expectByte()) { - 'e' => { - self.cursor += 1; - self.state = .post_value; - return .true; - }, - else => return error.SyntaxError, - } - }, - .literal_f => { - switch (try self.expectByte()) { - 'a' => { - self.cursor += 1; - self.state = .literal_fa; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - .literal_fa => { - switch (try self.expectByte()) { - 'l' => { - self.cursor += 1; - self.state = .literal_fal; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - .literal_fal => { - switch (try self.expectByte()) { - 's' => { - self.cursor += 1; - self.state = .literal_fals; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - .literal_fals => { - switch (try self.expectByte()) { - 'e' => { - self.cursor += 1; - self.state = .post_value; - return .false; - }, - else => return error.SyntaxError, - } - }, - .literal_n => { - switch (try self.expectByte()) { - 'u' => { - self.cursor += 1; - self.state = .literal_nu; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - .literal_nu => { - switch (try self.expectByte()) { - 'l' => { - self.cursor += 1; - self.state = .literal_nul; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - .literal_nul => { - switch (try self.expectByte()) { - 'l' => { - self.cursor += 1; - self.state = .post_value; - return .null; - }, - else => return error.SyntaxError, - } - }, - } - unreachable; - } - } - - /// Seeks ahead in the input until the first byte of the next token (or the end of the input) - /// determines which type of token will be returned from the next `next*()` call. - /// This function is idempotent, only advancing past commas, colons, and inter-token whitespace. - pub fn peekNextTokenType(self: *@This()) PeekError!TokenType { - state_loop: while (true) { - switch (self.state) { - .value => { - switch (try self.skipWhitespaceExpectByte()) { - '{' => return .object_begin, - '[' => return .array_begin, - '"' => return .string, - '-', '0'...'9' => return .number, - 't' => return .true, - 'f' => return .false, - 'n' => return .null, - else => return error.SyntaxError, - } - }, - - .post_value => { - if (try self.skipWhitespaceCheckEnd()) return .end_of_document; - - const c = self.input[self.cursor]; - if (self.string_is_object_key) { - self.string_is_object_key = false; - switch (c) { - ':' => { - self.cursor += 1; - self.state = .value; - continue :state_loop; - }, - else => return error.SyntaxError, - } - } - - switch (c) { - '}' => return .object_end, - ']' => return .array_end, - ',' => { - switch (self.stack.peek()) { - OBJECT_MODE => { - self.state = .object_post_comma; - }, - ARRAY_MODE => { - self.state = .value; - }, - } - self.cursor += 1; - continue :state_loop; - }, - else => return error.SyntaxError, - } - }, - - .object_start => { - switch (try self.skipWhitespaceExpectByte()) { - '"' => return .string, - '}' => return .object_end, - else => return error.SyntaxError, - } - }, - .object_post_comma => { - switch (try self.skipWhitespaceExpectByte()) { - '"' => return .string, - else => return error.SyntaxError, - } - }, - - .array_start => { - switch (try self.skipWhitespaceExpectByte()) { - ']' => return .array_end, - else => { - self.state = .value; - continue :state_loop; - }, - } - }, - - .number_minus, - .number_leading_zero, - .number_int, - .number_post_dot, - .number_frac, - .number_post_e, - .number_post_e_sign, - .number_exp, - => return .number, - - .string, - .string_backslash, - .string_backslash_u, - .string_backslash_u_1, - .string_backslash_u_2, - .string_backslash_u_3, - .string_surrogate_half, - .string_surrogate_half_backslash, - .string_surrogate_half_backslash_u, - .string_surrogate_half_backslash_u_1, - .string_surrogate_half_backslash_u_2, - .string_surrogate_half_backslash_u_3, - => return .string, - - .string_utf8_last_byte, - .string_utf8_second_to_last_byte, - .string_utf8_second_to_last_byte_guard_against_overlong, - .string_utf8_second_to_last_byte_guard_against_surrogate_half, - .string_utf8_third_to_last_byte, - .string_utf8_third_to_last_byte_guard_against_overlong, - .string_utf8_third_to_last_byte_guard_against_too_large, - => return .string, - - .literal_t, - .literal_tr, - .literal_tru, - => return .true, - .literal_f, - .literal_fa, - .literal_fal, - .literal_fals, - => return .false, - .literal_n, - .literal_nu, - .literal_nul, - => return .null, - } - unreachable; - } - } - - const State = enum { - value, - post_value, - - object_start, - object_post_comma, - - array_start, - - number_minus, - number_leading_zero, - number_int, - number_post_dot, - number_frac, - number_post_e, - number_post_e_sign, - number_exp, - - string, - string_backslash, - string_backslash_u, - string_backslash_u_1, - string_backslash_u_2, - string_backslash_u_3, - string_surrogate_half, - string_surrogate_half_backslash, - string_surrogate_half_backslash_u, - string_surrogate_half_backslash_u_1, - string_surrogate_half_backslash_u_2, - string_surrogate_half_backslash_u_3, - - // From http://unicode.org/mail-arch/unicode-ml/y2003-m02/att-0467/01-The_Algorithm_to_Valide_an_UTF-8_String - string_utf8_last_byte, // State A - string_utf8_second_to_last_byte, // State B - string_utf8_second_to_last_byte_guard_against_overlong, // State C - string_utf8_second_to_last_byte_guard_against_surrogate_half, // State D - string_utf8_third_to_last_byte, // State E - string_utf8_third_to_last_byte_guard_against_overlong, // State F - string_utf8_third_to_last_byte_guard_against_too_large, // State G - - literal_t, - literal_tr, - literal_tru, - literal_f, - literal_fa, - literal_fal, - literal_fals, - literal_n, - literal_nu, - literal_nul, - }; - - fn expectByte(self: *const @This()) !u8 { - if (self.cursor < self.input.len) { - return self.input[self.cursor]; - } - // No byte. - if (self.is_end_of_input) return error.UnexpectedEndOfInput; - return error.BufferUnderrun; - } - - fn skipWhitespace(self: *@This()) void { - while (self.cursor < self.input.len) : (self.cursor += 1) { - switch (self.input[self.cursor]) { - // Whitespace - ' ', '\t', '\r' => continue, - '\n' => { - if (self.diagnostics) |diag| { - diag.line_number += 1; - // This will count the newline itself, - // which means a straight-forward subtraction will give a 1-based column number. - diag.line_start_cursor = self.cursor; - } - continue; - }, - else => return, - } - } - } - - fn skipWhitespaceExpectByte(self: *@This()) !u8 { - self.skipWhitespace(); - return self.expectByte(); - } - - fn skipWhitespaceCheckEnd(self: *@This()) !bool { - self.skipWhitespace(); - if (self.cursor >= self.input.len) { - // End of buffer. - if (self.is_end_of_input) { - // End of everything. - if (self.stackHeight() == 0) { - // We did it! - return true; - } - return error.UnexpectedEndOfInput; - } - return error.BufferUnderrun; - } - if (self.stackHeight() == 0) return error.SyntaxError; - return false; - } - - fn takeValueSlice(self: *@This()) []const u8 { - const slice = self.input[self.value_start..self.cursor]; - self.value_start = self.cursor; - return slice; - } - fn takeValueSliceMinusTrailingOffset(self: *@This(), trailing_negative_offset: usize) []const u8 { - // Check if the escape sequence started before the current input buffer. - // (The algebra here is awkward to avoid unsigned underflow, - // but it's just making sure the slice on the next line isn't UB.) - if (self.cursor <= self.value_start + trailing_negative_offset) return ""; - const slice = self.input[self.value_start .. self.cursor - trailing_negative_offset]; - // When trailing_negative_offset is non-zero, setting self.value_start doesn't matter, - // because we always set it again while emitting the .partial_string_escaped_*. - self.value_start = self.cursor; - return slice; - } - - fn endOfBufferInNumber(self: *@This(), allow_end: bool) !Token { - const slice = self.takeValueSlice(); - if (self.is_end_of_input) { - if (!allow_end) return error.UnexpectedEndOfInput; - self.state = .post_value; - return Token{ .number = slice }; - } - if (slice.len == 0) return error.BufferUnderrun; - return Token{ .partial_number = slice }; - } - - fn endOfBufferInString(self: *@This()) !Token { - if (self.is_end_of_input) return error.UnexpectedEndOfInput; - const slice = self.takeValueSliceMinusTrailingOffset(switch (self.state) { - // Don't include the escape sequence in the partial string. - .string_backslash => 1, - .string_backslash_u => 2, - .string_backslash_u_1 => 3, - .string_backslash_u_2 => 4, - .string_backslash_u_3 => 5, - .string_surrogate_half => 6, - .string_surrogate_half_backslash => 7, - .string_surrogate_half_backslash_u => 8, - .string_surrogate_half_backslash_u_1 => 9, - .string_surrogate_half_backslash_u_2 => 10, - .string_surrogate_half_backslash_u_3 => 11, - - // Include everything up to the cursor otherwise. - .string, - .string_utf8_last_byte, - .string_utf8_second_to_last_byte, - .string_utf8_second_to_last_byte_guard_against_overlong, - .string_utf8_second_to_last_byte_guard_against_surrogate_half, - .string_utf8_third_to_last_byte, - .string_utf8_third_to_last_byte_guard_against_overlong, - .string_utf8_third_to_last_byte_guard_against_too_large, - => 0, - - else => unreachable, - }); - if (slice.len == 0) return error.BufferUnderrun; - return Token{ .partial_string = slice }; - } - - fn partialStringCodepoint(code_point: u21) Token { - var buf: [4]u8 = undefined; - switch (std.unicode.utf8Encode(code_point, &buf) catch unreachable) { - 1 => return Token{ .partial_string_escaped_1 = buf[0..1].* }, - 2 => return Token{ .partial_string_escaped_2 = buf[0..2].* }, - 3 => return Token{ .partial_string_escaped_3 = buf[0..3].* }, - 4 => return Token{ .partial_string_escaped_4 = buf[0..4].* }, - else => unreachable, - } - } -}; - -const OBJECT_MODE = 0; -const ARRAY_MODE = 1; - -fn appendSlice(list: *std.ArrayList(u8), buf: []const u8, max_value_len: usize) !void { - const new_len = std.math.add(usize, list.items.len, buf.len) catch return error.ValueTooLong; - if (new_len > max_value_len) return error.ValueTooLong; - try list.appendSlice(buf); -} - -/// For the slice you get from a `Token.number` or `Token.allocated_number`, -/// this function returns true if the number doesn't contain any fraction or exponent components, and is not `-0`. -/// Note, the numeric value encoded by the value may still be an integer, such as `1.0`. -/// This function is meant to give a hint about whether integer parsing or float parsing should be used on the value. -/// This function will not give meaningful results on non-numeric input. -pub fn isNumberFormattedLikeAnInteger(value: []const u8) bool { - if (std.mem.eql(u8, value, "-0")) return false; - return std.mem.indexOfAny(u8, value, ".eE") == null; -} - -test { - _ = @import("./scanner_test.zig"); -} diff --git a/lib/std/json/scanner_test.zig b/lib/std/json/scanner_test.zig index d085cb661a..eb5d5cb75e 100644 --- a/lib/std/json/scanner_test.zig +++ b/lib/std/json/scanner_test.zig @@ -1,13 +1,11 @@ const std = @import("std"); -const JsonScanner = @import("./scanner.zig").Scanner; -const jsonReader = @import("./scanner.zig").reader; -const JsonReader = @import("./scanner.zig").Reader; -const Token = @import("./scanner.zig").Token; -const TokenType = @import("./scanner.zig").TokenType; -const Diagnostics = @import("./scanner.zig").Diagnostics; -const Error = @import("./scanner.zig").Error; -const validate = @import("./scanner.zig").validate; -const isNumberFormattedLikeAnInteger = @import("./scanner.zig").isNumberFormattedLikeAnInteger; +const Scanner = @import("Scanner.zig"); +const Token = Scanner.Token; +const TokenType = Scanner.TokenType; +const Diagnostics = Scanner.Diagnostics; +const Error = Scanner.Error; +const validate = Scanner.validate; +const isNumberFormattedLikeAnInteger = Scanner.isNumberFormattedLikeAnInteger; const example_document_str = \\{ @@ -36,7 +34,7 @@ fn expectPeekNext(scanner_or_reader: anytype, expected_token_type: TokenType, ex } test "token" { - var scanner = JsonScanner.initCompleteInput(std.testing.allocator, example_document_str); + var scanner = Scanner.initCompleteInput(std.testing.allocator, example_document_str); defer scanner.deinit(); try expectNext(&scanner, .object_begin); @@ -138,23 +136,25 @@ fn testAllTypes(source: anytype, large_buffer: bool) !void { } test "peek all types" { - var scanner = JsonScanner.initCompleteInput(std.testing.allocator, all_types_test_case); + var scanner = Scanner.initCompleteInput(std.testing.allocator, all_types_test_case); defer scanner.deinit(); try testAllTypes(&scanner, true); - var stream = std.io.fixedBufferStream(all_types_test_case); - var json_reader = jsonReader(std.testing.allocator, stream.reader()); + var stream: std.Io.Reader = .fixed(all_types_test_case); + var json_reader: Scanner.Reader = .init(std.testing.allocator, &stream); defer json_reader.deinit(); try testAllTypes(&json_reader, true); - var tiny_stream = std.io.fixedBufferStream(all_types_test_case); - var tiny_json_reader = JsonReader(1, @TypeOf(tiny_stream.reader())).init(std.testing.allocator, tiny_stream.reader()); + var tiny_buffer: [1]u8 = undefined; + var tiny_stream: std.testing.Reader = .init(&tiny_buffer, &.{.{ .buffer = all_types_test_case }}); + tiny_stream.artificial_limit = .limited(1); + var tiny_json_reader: Scanner.Reader = .init(std.testing.allocator, &tiny_stream.interface); defer tiny_json_reader.deinit(); try testAllTypes(&tiny_json_reader, false); } test "token mismatched close" { - var scanner = JsonScanner.initCompleteInput(std.testing.allocator, "[102, 111, 111 }"); + var scanner = Scanner.initCompleteInput(std.testing.allocator, "[102, 111, 111 }"); defer scanner.deinit(); try expectNext(&scanner, .array_begin); try expectNext(&scanner, Token{ .number = "102" }); @@ -164,15 +164,15 @@ test "token mismatched close" { } test "token premature object close" { - var scanner = JsonScanner.initCompleteInput(std.testing.allocator, "{ \"key\": }"); + var scanner = Scanner.initCompleteInput(std.testing.allocator, "{ \"key\": }"); defer scanner.deinit(); try expectNext(&scanner, .object_begin); try expectNext(&scanner, Token{ .string = "key" }); try std.testing.expectError(error.SyntaxError, scanner.next()); } -test "JsonScanner basic" { - var scanner = JsonScanner.initCompleteInput(std.testing.allocator, example_document_str); +test "Scanner basic" { + var scanner = Scanner.initCompleteInput(std.testing.allocator, example_document_str); defer scanner.deinit(); while (true) { @@ -181,10 +181,10 @@ test "JsonScanner basic" { } } -test "JsonReader basic" { - var stream = std.io.fixedBufferStream(example_document_str); +test "Scanner.Reader basic" { + var stream: std.Io.Reader = .fixed(example_document_str); - var json_reader = jsonReader(std.testing.allocator, stream.reader()); + var json_reader: Scanner.Reader = .init(std.testing.allocator, &stream); defer json_reader.deinit(); while (true) { @@ -215,7 +215,7 @@ const number_test_items = blk: { test "numbers" { for (number_test_items) |number_str| { - var scanner = JsonScanner.initCompleteInput(std.testing.allocator, number_str); + var scanner = Scanner.initCompleteInput(std.testing.allocator, number_str); defer scanner.deinit(); const token = try scanner.next(); @@ -243,10 +243,10 @@ const string_test_cases = .{ test "strings" { inline for (string_test_cases) |tuple| { - var stream = std.io.fixedBufferStream("\"" ++ tuple[0] ++ "\""); + var stream: std.Io.Reader = .fixed("\"" ++ tuple[0] ++ "\""); var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit(); - var json_reader = jsonReader(std.testing.allocator, stream.reader()); + var json_reader: Scanner.Reader = .init(std.testing.allocator, &stream); defer json_reader.deinit(); const token = try json_reader.nextAlloc(arena.allocator(), .alloc_if_needed); @@ -289,7 +289,7 @@ test "nesting" { } fn expectMaybeError(document_str: []const u8, maybe_error: ?Error) !void { - var scanner = JsonScanner.initCompleteInput(std.testing.allocator, document_str); + var scanner = Scanner.initCompleteInput(std.testing.allocator, document_str); defer scanner.deinit(); while (true) { @@ -352,12 +352,12 @@ fn expectEqualTokens(expected_token: Token, actual_token: Token) !void { } fn testTinyBufferSize(document_str: []const u8) !void { - var tiny_stream = std.io.fixedBufferStream(document_str); - var normal_stream = std.io.fixedBufferStream(document_str); + var tiny_stream: std.Io.Reader = .fixed(document_str); + var normal_stream: std.Io.Reader = .fixed(document_str); - var tiny_json_reader = JsonReader(1, @TypeOf(tiny_stream.reader())).init(std.testing.allocator, tiny_stream.reader()); + var tiny_json_reader: Scanner.Reader = .init(std.testing.allocator, &tiny_stream); defer tiny_json_reader.deinit(); - var normal_json_reader = JsonReader(0x1000, @TypeOf(normal_stream.reader())).init(std.testing.allocator, normal_stream.reader()); + var normal_json_reader: Scanner.Reader = .init(std.testing.allocator, &normal_stream); defer normal_json_reader.deinit(); expectEqualStreamOfTokens(&normal_json_reader, &tiny_json_reader) catch |err| { @@ -397,13 +397,13 @@ test "validate" { } fn testSkipValue(s: []const u8) !void { - var scanner = JsonScanner.initCompleteInput(std.testing.allocator, s); + var scanner = Scanner.initCompleteInput(std.testing.allocator, s); defer scanner.deinit(); try scanner.skipValue(); try expectEqualTokens(.end_of_document, try scanner.next()); - var stream = std.io.fixedBufferStream(s); - var json_reader = jsonReader(std.testing.allocator, stream.reader()); + var stream: std.Io.Reader = .fixed(s); + var json_reader: Scanner.Reader = .init(std.testing.allocator, &stream); defer json_reader.deinit(); try json_reader.skipValue(); try expectEqualTokens(.end_of_document, try json_reader.next()); @@ -441,7 +441,7 @@ fn testEnsureStackCapacity(do_ensure: bool) !void { try input_string.appendNTimes(std.testing.allocator, ']', nestings); defer input_string.deinit(std.testing.allocator); - var scanner = JsonScanner.initCompleteInput(failing_allocator, input_string.items); + var scanner = Scanner.initCompleteInput(failing_allocator, input_string.items); defer scanner.deinit(); if (do_ensure) { @@ -473,17 +473,17 @@ fn testDiagnosticsFromSource(expected_error: ?anyerror, line: u64, col: u64, byt try std.testing.expectEqual(byte_offset, diagnostics.getByteOffset()); } fn testDiagnostics(expected_error: ?anyerror, line: u64, col: u64, byte_offset: u64, s: []const u8) !void { - var scanner = JsonScanner.initCompleteInput(std.testing.allocator, s); + var scanner = Scanner.initCompleteInput(std.testing.allocator, s); defer scanner.deinit(); try testDiagnosticsFromSource(expected_error, line, col, byte_offset, &scanner); - var tiny_stream = std.io.fixedBufferStream(s); - var tiny_json_reader = JsonReader(1, @TypeOf(tiny_stream.reader())).init(std.testing.allocator, tiny_stream.reader()); + var tiny_stream: std.Io.Reader = .fixed(s); + var tiny_json_reader: Scanner.Reader = .init(std.testing.allocator, &tiny_stream); defer tiny_json_reader.deinit(); try testDiagnosticsFromSource(expected_error, line, col, byte_offset, &tiny_json_reader); - var medium_stream = std.io.fixedBufferStream(s); - var medium_json_reader = JsonReader(5, @TypeOf(medium_stream.reader())).init(std.testing.allocator, medium_stream.reader()); + var medium_stream: std.Io.Reader = .fixed(s); + var medium_json_reader: Scanner.Reader = .init(std.testing.allocator, &medium_stream); defer medium_json_reader.deinit(); try testDiagnosticsFromSource(expected_error, line, col, byte_offset, &medium_json_reader); } diff --git a/lib/std/json/static.zig b/lib/std/json/static.zig index 2504d59100..44469adf4c 100644 --- a/lib/std/json/static.zig +++ b/lib/std/json/static.zig @@ -4,11 +4,11 @@ const Allocator = std.mem.Allocator; const ArenaAllocator = std.heap.ArenaAllocator; const ArrayList = std.ArrayList; -const Scanner = @import("./scanner.zig").Scanner; -const Token = @import("./scanner.zig").Token; -const AllocWhen = @import("./scanner.zig").AllocWhen; -const default_max_value_len = @import("./scanner.zig").default_max_value_len; -const isNumberFormattedLikeAnInteger = @import("./scanner.zig").isNumberFormattedLikeAnInteger; +const Scanner = @import("Scanner.zig"); +const Token = Scanner.Token; +const AllocWhen = Scanner.AllocWhen; +const default_max_value_len = Scanner.default_max_value_len; +const isNumberFormattedLikeAnInteger = Scanner.isNumberFormattedLikeAnInteger; const Value = @import("./dynamic.zig").Value; const Array = @import("./dynamic.zig").Array; diff --git a/lib/std/json/static_test.zig b/lib/std/json/static_test.zig index 3a1919e40c..735ccd82e1 100644 --- a/lib/std/json/static_test.zig +++ b/lib/std/json/static_test.zig @@ -12,9 +12,7 @@ const parseFromValue = @import("./static.zig").parseFromValue; const parseFromValueLeaky = @import("./static.zig").parseFromValueLeaky; const ParseOptions = @import("./static.zig").ParseOptions; -const JsonScanner = @import("./scanner.zig").Scanner; -const jsonReader = @import("./scanner.zig").reader; -const Diagnostics = @import("./scanner.zig").Diagnostics; +const Scanner = @import("Scanner.zig"); const Value = @import("./dynamic.zig").Value; @@ -300,9 +298,9 @@ const subnamespaces_0_doc = fn testAllParseFunctions(comptime T: type, expected: T, doc: []const u8) !void { // First do the one with the debug info in case we get a SyntaxError or something. { - var scanner = JsonScanner.initCompleteInput(testing.allocator, doc); + var scanner = Scanner.initCompleteInput(testing.allocator, doc); defer scanner.deinit(); - var diagnostics = Diagnostics{}; + var diagnostics = Scanner.Diagnostics{}; scanner.enableDiagnostics(&diagnostics); var parsed = parseFromTokenSource(T, testing.allocator, &scanner, .{}) catch |e| { std.debug.print("at line,col: {}:{}\n", .{ diagnostics.getLine(), diagnostics.getColumn() }); @@ -317,8 +315,8 @@ fn testAllParseFunctions(comptime T: type, expected: T, doc: []const u8) !void { try testing.expectEqualDeep(expected, parsed.value); } { - var stream = std.io.fixedBufferStream(doc); - var json_reader = jsonReader(std.testing.allocator, stream.reader()); + var stream: std.Io.Reader = .fixed(doc); + var json_reader: Scanner.Reader = .init(std.testing.allocator, &stream); defer json_reader.deinit(); var parsed = try parseFromTokenSource(T, testing.allocator, &json_reader, .{}); defer parsed.deinit(); @@ -331,13 +329,13 @@ fn testAllParseFunctions(comptime T: type, expected: T, doc: []const u8) !void { try testing.expectEqualDeep(expected, try parseFromSliceLeaky(T, arena.allocator(), doc, .{})); } { - var scanner = JsonScanner.initCompleteInput(testing.allocator, doc); + var scanner = Scanner.initCompleteInput(testing.allocator, doc); defer scanner.deinit(); try testing.expectEqualDeep(expected, try parseFromTokenSourceLeaky(T, arena.allocator(), &scanner, .{})); } { - var stream = std.io.fixedBufferStream(doc); - var json_reader = jsonReader(std.testing.allocator, stream.reader()); + var stream: std.Io.Reader = .fixed(doc); + var json_reader: Scanner.Reader = .init(std.testing.allocator, &stream); defer json_reader.deinit(); try testing.expectEqualDeep(expected, try parseFromTokenSourceLeaky(T, arena.allocator(), &json_reader, .{})); } @@ -763,7 +761,7 @@ test "parse exponential into int" { test "parseFromTokenSource" { { - var scanner = JsonScanner.initCompleteInput(testing.allocator, "123"); + var scanner = Scanner.initCompleteInput(testing.allocator, "123"); defer scanner.deinit(); var parsed = try parseFromTokenSource(u32, testing.allocator, &scanner, .{}); defer parsed.deinit(); @@ -771,8 +769,8 @@ test "parseFromTokenSource" { } { - var stream = std.io.fixedBufferStream("123"); - var json_reader = jsonReader(std.testing.allocator, stream.reader()); + var stream: std.Io.Reader = .fixed("123"); + var json_reader: Scanner.Reader = .init(std.testing.allocator, &stream); defer json_reader.deinit(); var parsed = try parseFromTokenSource(u32, testing.allocator, &json_reader, .{}); defer parsed.deinit(); @@ -836,7 +834,7 @@ test "json parse partial" { \\} ; const allocator = testing.allocator; - var scanner = JsonScanner.initCompleteInput(allocator, str); + var scanner = Scanner.initCompleteInput(allocator, str); defer scanner.deinit(); var arena = ArenaAllocator.init(allocator); @@ -886,8 +884,8 @@ test "json parse allocate when streaming" { var arena = ArenaAllocator.init(allocator); defer arena.deinit(); - var stream = std.io.fixedBufferStream(str); - var json_reader = jsonReader(std.testing.allocator, stream.reader()); + var stream: std.Io.Reader = .fixed(str); + var json_reader: Scanner.Reader = .init(std.testing.allocator, &stream); const parsed = parseFromTokenSourceLeaky(T, arena.allocator(), &json_reader, .{}) catch |err| { json_reader.deinit(); diff --git a/lib/std/json/stringify.zig b/lib/std/json/stringify.zig deleted file mode 100644 index aa49573695..0000000000 --- a/lib/std/json/stringify.zig +++ /dev/null @@ -1,772 +0,0 @@ -const std = @import("std"); -const assert = std.debug.assert; -const Allocator = std.mem.Allocator; -const ArrayList = std.ArrayList; -const BitStack = std.BitStack; - -const OBJECT_MODE = 0; -const ARRAY_MODE = 1; - -pub const StringifyOptions = struct { - /// Controls the whitespace emitted. - /// The default `.minified` is a compact encoding with no whitespace between tokens. - /// Any setting other than `.minified` will use newlines, indentation, and a space after each ':'. - /// `.indent_1` means 1 space for each indentation level, `.indent_2` means 2 spaces, etc. - /// `.indent_tab` uses a tab for each indentation level. - whitespace: enum { - minified, - indent_1, - indent_2, - indent_3, - indent_4, - indent_8, - indent_tab, - } = .minified, - - /// Should optional fields with null value be written? - emit_null_optional_fields: bool = true, - - /// Arrays/slices of u8 are typically encoded as JSON strings. - /// This option emits them as arrays of numbers instead. - /// Does not affect calls to `objectField*()`. - emit_strings_as_arrays: bool = false, - - /// Should unicode characters be escaped in strings? - escape_unicode: bool = false, - - /// When true, renders numbers outside the range `+-1<<53` (the precise integer range of f64) as JSON strings in base 10. - emit_nonportable_numbers_as_strings: bool = false, -}; - -/// Writes the given value to the `std.io.GenericWriter` stream. -/// See `WriteStream` for how the given value is serialized into JSON. -/// The maximum nesting depth of the output JSON document is 256. -/// See also `stringifyMaxDepth` and `stringifyArbitraryDepth`. -pub fn stringify( - value: anytype, - options: StringifyOptions, - out_stream: anytype, -) @TypeOf(out_stream).Error!void { - var jw = writeStream(out_stream, options); - defer jw.deinit(); - try jw.write(value); -} - -/// Like `stringify` with configurable nesting depth. -/// `max_depth` is rounded up to the nearest multiple of 8. -/// Give `null` for `max_depth` to disable some safety checks and allow arbitrary nesting depth. -/// See `writeStreamMaxDepth` for more info. -pub fn stringifyMaxDepth( - value: anytype, - options: StringifyOptions, - out_stream: anytype, - comptime max_depth: ?usize, -) @TypeOf(out_stream).Error!void { - var jw = writeStreamMaxDepth(out_stream, options, max_depth); - try jw.write(value); -} - -/// Like `stringify` but takes an allocator to facilitate safety checks while allowing arbitrary nesting depth. -/// These safety checks can be helpful when debugging custom `jsonStringify` implementations; -/// See `WriteStream`. -pub fn stringifyArbitraryDepth( - allocator: Allocator, - value: anytype, - options: StringifyOptions, - out_stream: anytype, -) WriteStream(@TypeOf(out_stream), .checked_to_arbitrary_depth).Error!void { - var jw = writeStreamArbitraryDepth(allocator, out_stream, options); - defer jw.deinit(); - try jw.write(value); -} - -/// Calls `stringifyArbitraryDepth` and stores the result in dynamically allocated memory -/// instead of taking a `std.io.GenericWriter`. -/// -/// Caller owns returned memory. -pub fn stringifyAlloc( - allocator: Allocator, - value: anytype, - options: StringifyOptions, -) error{OutOfMemory}![]u8 { - var list = std.ArrayList(u8).init(allocator); - errdefer list.deinit(); - try stringifyArbitraryDepth(allocator, value, options, list.writer()); - return list.toOwnedSlice(); -} - -/// See `WriteStream` for documentation. -/// Equivalent to calling `writeStreamMaxDepth` with a depth of `256`. -/// -/// The caller does *not* need to call `deinit()` on the returned object. -pub fn writeStream( - out_stream: anytype, - options: StringifyOptions, -) WriteStream(@TypeOf(out_stream), .{ .checked_to_fixed_depth = 256 }) { - return writeStreamMaxDepth(out_stream, options, 256); -} - -/// See `WriteStream` for documentation. -/// The returned object includes 1 bit of size per `max_depth` to enable safety checks on the order of method calls; -/// see the grammar in the `WriteStream` documentation. -/// `max_depth` is rounded up to the nearest multiple of 8. -/// If the nesting depth exceeds `max_depth`, it is detectable illegal behavior. -/// Give `null` for `max_depth` to disable safety checks for the grammar and allow arbitrary nesting depth. -/// In `ReleaseFast` and `ReleaseSmall`, `max_depth` is ignored, effectively equivalent to passing `null`. -/// Alternatively, see `writeStreamArbitraryDepth` to do safety checks to arbitrary depth. -/// -/// The caller does *not* need to call `deinit()` on the returned object. -pub fn writeStreamMaxDepth( - out_stream: anytype, - options: StringifyOptions, - comptime max_depth: ?usize, -) WriteStream( - @TypeOf(out_stream), - if (max_depth) |d| .{ .checked_to_fixed_depth = d } else .assumed_correct, -) { - return WriteStream( - @TypeOf(out_stream), - if (max_depth) |d| .{ .checked_to_fixed_depth = d } else .assumed_correct, - ).init(undefined, out_stream, options); -} - -/// See `WriteStream` for documentation. -/// This version of the write stream enables safety checks to arbitrarily deep nesting levels -/// by using the given allocator. -/// The caller should call `deinit()` on the returned object to free allocated memory. -/// -/// In `ReleaseFast` and `ReleaseSmall` mode, this function is effectively equivalent to calling `writeStreamMaxDepth(..., null)`; -/// in those build modes, the allocator is *not used*. -pub fn writeStreamArbitraryDepth( - allocator: Allocator, - out_stream: anytype, - options: StringifyOptions, -) WriteStream(@TypeOf(out_stream), .checked_to_arbitrary_depth) { - return WriteStream(@TypeOf(out_stream), .checked_to_arbitrary_depth).init(allocator, out_stream, options); -} - -/// Writes JSON ([RFC8259](https://tools.ietf.org/html/rfc8259)) formatted data -/// to a stream. -/// -/// The sequence of method calls to write JSON content must follow this grammar: -/// ``` -/// = -/// = -/// | -/// | -/// | write -/// | print -/// | -/// = beginObject ( )* endObject -/// = objectField | objectFieldRaw | -/// = beginArray ( )* endArray -/// = beginWriteRaw ( stream.writeAll )* endWriteRaw -/// = beginObjectFieldRaw ( stream.writeAll )* endObjectFieldRaw -/// ``` -/// -/// The `safety_checks_hint` parameter determines how much memory is used to enable assertions that the above grammar is being followed, -/// e.g. tripping an assertion rather than allowing `endObject` to emit the final `}` in `[[[]]}`. -/// "Depth" in this context means the depth of nested `[]` or `{}` expressions -/// (or equivalently the amount of recursion on the `` grammar expression above). -/// For example, emitting the JSON `[[[]]]` requires a depth of 3. -/// If `.checked_to_fixed_depth` is used, there is additionally an assertion that the nesting depth never exceeds the given limit. -/// `.checked_to_arbitrary_depth` requires a runtime allocator for the memory. -/// `.checked_to_fixed_depth` embeds the storage required in the `WriteStream` struct. -/// `.assumed_correct` requires no space and performs none of these assertions. -/// In `ReleaseFast` and `ReleaseSmall` mode, the given `safety_checks_hint` is ignored and is always treated as `.assumed_correct`. -pub fn WriteStream( - comptime OutStream: type, - comptime safety_checks_hint: union(enum) { - checked_to_arbitrary_depth, - checked_to_fixed_depth: usize, // Rounded up to the nearest multiple of 8. - assumed_correct, - }, -) type { - return struct { - const Self = @This(); - const build_mode_has_safety = switch (@import("builtin").mode) { - .Debug, .ReleaseSafe => true, - .ReleaseFast, .ReleaseSmall => false, - }; - const safety_checks: @TypeOf(safety_checks_hint) = if (build_mode_has_safety) - safety_checks_hint - else - .assumed_correct; - - pub const Stream = OutStream; - pub const Error = switch (safety_checks) { - .checked_to_arbitrary_depth => Stream.Error || error{OutOfMemory}, - .checked_to_fixed_depth, .assumed_correct => Stream.Error, - }; - - options: StringifyOptions, - - stream: OutStream, - indent_level: usize = 0, - next_punctuation: enum { - the_beginning, - none, - comma, - colon, - } = .the_beginning, - - nesting_stack: switch (safety_checks) { - .checked_to_arbitrary_depth => BitStack, - .checked_to_fixed_depth => |fixed_buffer_size| [(fixed_buffer_size + 7) >> 3]u8, - .assumed_correct => void, - }, - - raw_streaming_mode: if (build_mode_has_safety) - enum { none, value, objectField } - else - void = if (build_mode_has_safety) .none else {}, - - pub fn init(safety_allocator: Allocator, stream: OutStream, options: StringifyOptions) Self { - return .{ - .options = options, - .stream = stream, - .nesting_stack = switch (safety_checks) { - .checked_to_arbitrary_depth => BitStack.init(safety_allocator), - .checked_to_fixed_depth => |fixed_buffer_size| [_]u8{0} ** ((fixed_buffer_size + 7) >> 3), - .assumed_correct => {}, - }, - }; - } - - /// Only necessary with .checked_to_arbitrary_depth. - pub fn deinit(self: *Self) void { - switch (safety_checks) { - .checked_to_arbitrary_depth => self.nesting_stack.deinit(), - .checked_to_fixed_depth, .assumed_correct => {}, - } - self.* = undefined; - } - - pub fn beginArray(self: *Self) Error!void { - if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); - try self.valueStart(); - try self.stream.writeByte('['); - try self.pushIndentation(ARRAY_MODE); - self.next_punctuation = .none; - } - - pub fn beginObject(self: *Self) Error!void { - if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); - try self.valueStart(); - try self.stream.writeByte('{'); - try self.pushIndentation(OBJECT_MODE); - self.next_punctuation = .none; - } - - pub fn endArray(self: *Self) Error!void { - if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); - self.popIndentation(ARRAY_MODE); - switch (self.next_punctuation) { - .none => {}, - .comma => { - try self.indent(); - }, - .the_beginning, .colon => unreachable, - } - try self.stream.writeByte(']'); - self.valueDone(); - } - - pub fn endObject(self: *Self) Error!void { - if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); - self.popIndentation(OBJECT_MODE); - switch (self.next_punctuation) { - .none => {}, - .comma => { - try self.indent(); - }, - .the_beginning, .colon => unreachable, - } - try self.stream.writeByte('}'); - self.valueDone(); - } - - fn pushIndentation(self: *Self, mode: u1) !void { - switch (safety_checks) { - .checked_to_arbitrary_depth => { - try self.nesting_stack.push(mode); - self.indent_level += 1; - }, - .checked_to_fixed_depth => { - BitStack.pushWithStateAssumeCapacity(&self.nesting_stack, &self.indent_level, mode); - }, - .assumed_correct => { - self.indent_level += 1; - }, - } - } - fn popIndentation(self: *Self, assert_its_this_one: u1) void { - switch (safety_checks) { - .checked_to_arbitrary_depth => { - assert(self.nesting_stack.pop() == assert_its_this_one); - self.indent_level -= 1; - }, - .checked_to_fixed_depth => { - assert(BitStack.popWithState(&self.nesting_stack, &self.indent_level) == assert_its_this_one); - }, - .assumed_correct => { - self.indent_level -= 1; - }, - } - } - - fn indent(self: *Self) !void { - var char: u8 = ' '; - const n_chars = switch (self.options.whitespace) { - .minified => return, - .indent_1 => 1 * self.indent_level, - .indent_2 => 2 * self.indent_level, - .indent_3 => 3 * self.indent_level, - .indent_4 => 4 * self.indent_level, - .indent_8 => 8 * self.indent_level, - .indent_tab => blk: { - char = '\t'; - break :blk self.indent_level; - }, - }; - try self.stream.writeByte('\n'); - try self.stream.writeByteNTimes(char, n_chars); - } - - fn valueStart(self: *Self) !void { - if (self.isObjectKeyExpected()) |is_it| assert(!is_it); // Call objectField*(), not write(), for object keys. - return self.valueStartAssumeTypeOk(); - } - fn objectFieldStart(self: *Self) !void { - if (self.isObjectKeyExpected()) |is_it| assert(is_it); // Expected write(), not objectField*(). - return self.valueStartAssumeTypeOk(); - } - fn valueStartAssumeTypeOk(self: *Self) !void { - assert(!self.isComplete()); // JSON document already complete. - switch (self.next_punctuation) { - .the_beginning => { - // No indentation for the very beginning. - }, - .none => { - // First item in a container. - try self.indent(); - }, - .comma => { - // Subsequent item in a container. - try self.stream.writeByte(','); - try self.indent(); - }, - .colon => { - try self.stream.writeByte(':'); - if (self.options.whitespace != .minified) { - try self.stream.writeByte(' '); - } - }, - } - } - fn valueDone(self: *Self) void { - self.next_punctuation = .comma; - } - - // Only when safety is enabled: - fn isObjectKeyExpected(self: *const Self) ?bool { - switch (safety_checks) { - .checked_to_arbitrary_depth => return self.indent_level > 0 and - self.nesting_stack.peek() == OBJECT_MODE and - self.next_punctuation != .colon, - .checked_to_fixed_depth => return self.indent_level > 0 and - BitStack.peekWithState(&self.nesting_stack, self.indent_level) == OBJECT_MODE and - self.next_punctuation != .colon, - .assumed_correct => return null, - } - } - fn isComplete(self: *const Self) bool { - return self.indent_level == 0 and self.next_punctuation == .comma; - } - - /// An alternative to calling `write` that formats a value with `std.fmt`. - /// This function does the usual punctuation and indentation formatting - /// assuming the resulting formatted string represents a single complete value; - /// e.g. `"1"`, `"[]"`, `"[1,2]"`, not `"1,2"`. - /// This function may be useful for doing your own number formatting. - pub fn print(self: *Self, comptime fmt: []const u8, args: anytype) Error!void { - if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); - try self.valueStart(); - try self.stream.print(fmt, args); - self.valueDone(); - } - - /// An alternative to calling `write` that allows you to write directly to the `.stream` field, e.g. with `.stream.writeAll()`. - /// Call `beginWriteRaw()`, then write a complete value (including any quotes if necessary) directly to the `.stream` field, - /// then call `endWriteRaw()`. - /// This can be useful for streaming very long strings into the output without needing it all buffered in memory. - pub fn beginWriteRaw(self: *Self) !void { - if (build_mode_has_safety) { - assert(self.raw_streaming_mode == .none); - self.raw_streaming_mode = .value; - } - try self.valueStart(); - } - - /// See `beginWriteRaw`. - pub fn endWriteRaw(self: *Self) void { - if (build_mode_has_safety) { - assert(self.raw_streaming_mode == .value); - self.raw_streaming_mode = .none; - } - self.valueDone(); - } - - /// See `WriteStream` for when to call this method. - /// `key` is the string content of the property name. - /// Surrounding quotes will be added and any special characters will be escaped. - /// See also `objectFieldRaw`. - pub fn objectField(self: *Self, key: []const u8) Error!void { - if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); - try self.objectFieldStart(); - try encodeJsonString(key, self.options, self.stream); - self.next_punctuation = .colon; - } - /// See `WriteStream` for when to call this method. - /// `quoted_key` is the complete bytes of the key including quotes and any necessary escape sequences. - /// A few assertions are performed on the given value to ensure that the caller of this function understands the API contract. - /// See also `objectField`. - pub fn objectFieldRaw(self: *Self, quoted_key: []const u8) Error!void { - if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); - assert(quoted_key.len >= 2 and quoted_key[0] == '"' and quoted_key[quoted_key.len - 1] == '"'); // quoted_key should be "quoted". - try self.objectFieldStart(); - try self.stream.writeAll(quoted_key); - self.next_punctuation = .colon; - } - - /// In the rare case that you need to write very long object field names, - /// this is an alternative to `objectField` and `objectFieldRaw` that allows you to write directly to the `.stream` field - /// similar to `beginWriteRaw`. - /// Call `endObjectFieldRaw()` when you're done. - pub fn beginObjectFieldRaw(self: *Self) !void { - if (build_mode_has_safety) { - assert(self.raw_streaming_mode == .none); - self.raw_streaming_mode = .objectField; - } - try self.objectFieldStart(); - } - - /// See `beginObjectFieldRaw`. - pub fn endObjectFieldRaw(self: *Self) void { - if (build_mode_has_safety) { - assert(self.raw_streaming_mode == .objectField); - self.raw_streaming_mode = .none; - } - self.next_punctuation = .colon; - } - - /// Renders the given Zig value as JSON. - /// - /// Supported types: - /// * Zig `bool` -> JSON `true` or `false`. - /// * Zig `?T` -> `null` or the rendering of `T`. - /// * Zig `i32`, `u64`, etc. -> JSON number or string. - /// * When option `emit_nonportable_numbers_as_strings` is true, if the value is outside the range `+-1<<53` (the precise integer range of f64), it is rendered as a JSON string in base 10. Otherwise, it is rendered as JSON number. - /// * Zig floats -> JSON number or string. - /// * If the value cannot be precisely represented by an f64, it is rendered as a JSON string. Otherwise, it is rendered as JSON number. - /// * Zig `[]const u8`, `[]u8`, `*[N]u8`, `@Vector(N, u8)`, and similar -> JSON string. - /// * See `StringifyOptions.emit_strings_as_arrays`. - /// * If the content is not valid UTF-8, rendered as an array of numbers instead. - /// * Zig `[]T`, `[N]T`, `*[N]T`, `@Vector(N, T)`, and similar -> JSON array of the rendering of each item. - /// * Zig tuple -> JSON array of the rendering of each item. - /// * Zig `struct` -> JSON object with each field in declaration order. - /// * If the struct declares a method `pub fn jsonStringify(self: *@This(), jw: anytype) !void`, it is called to do the serialization instead of the default behavior. The given `jw` is a pointer to this `WriteStream`. See `std.json.Value` for an example. - /// * See `StringifyOptions.emit_null_optional_fields`. - /// * Zig `union(enum)` -> JSON object with one field named for the active tag and a value representing the payload. - /// * If the payload is `void`, then the emitted value is `{}`. - /// * If the union declares a method `pub fn jsonStringify(self: *@This(), jw: anytype) !void`, it is called to do the serialization instead of the default behavior. The given `jw` is a pointer to this `WriteStream`. - /// * Zig `enum` -> JSON string naming the active tag. - /// * If the enum declares a method `pub fn jsonStringify(self: *@This(), jw: anytype) !void`, it is called to do the serialization instead of the default behavior. The given `jw` is a pointer to this `WriteStream`. - /// * If the enum is non-exhaustive, unnamed values are rendered as integers. - /// * Zig untyped enum literal -> JSON string naming the active tag. - /// * Zig error -> JSON string naming the error. - /// * Zig `*T` -> the rendering of `T`. Note there is no guard against circular-reference infinite recursion. - /// - /// See also alternative functions `print` and `beginWriteRaw`. - /// For writing object field names, use `objectField` instead. - pub fn write(self: *Self, value: anytype) Error!void { - if (build_mode_has_safety) assert(self.raw_streaming_mode == .none); - const T = @TypeOf(value); - switch (@typeInfo(T)) { - .int => { - try self.valueStart(); - if (self.options.emit_nonportable_numbers_as_strings and - (value <= -(1 << 53) or value >= (1 << 53))) - { - try self.stream.print("\"{}\"", .{value}); - } else { - try self.stream.print("{}", .{value}); - } - self.valueDone(); - return; - }, - .comptime_int => { - return self.write(@as(std.math.IntFittingRange(value, value), value)); - }, - .float, .comptime_float => { - if (@as(f64, @floatCast(value)) == value) { - try self.valueStart(); - try self.stream.print("{}", .{@as(f64, @floatCast(value))}); - self.valueDone(); - return; - } - try self.valueStart(); - try self.stream.print("\"{}\"", .{value}); - self.valueDone(); - return; - }, - - .bool => { - try self.valueStart(); - try self.stream.writeAll(if (value) "true" else "false"); - self.valueDone(); - return; - }, - .null => { - try self.valueStart(); - try self.stream.writeAll("null"); - self.valueDone(); - return; - }, - .optional => { - if (value) |payload| { - return try self.write(payload); - } else { - return try self.write(null); - } - }, - .@"enum" => |enum_info| { - if (std.meta.hasFn(T, "jsonStringify")) { - return value.jsonStringify(self); - } - - if (!enum_info.is_exhaustive) { - inline for (enum_info.fields) |field| { - if (value == @field(T, field.name)) { - break; - } - } else { - return self.write(@intFromEnum(value)); - } - } - - return self.stringValue(@tagName(value)); - }, - .enum_literal => { - return self.stringValue(@tagName(value)); - }, - .@"union" => { - if (std.meta.hasFn(T, "jsonStringify")) { - return value.jsonStringify(self); - } - - const info = @typeInfo(T).@"union"; - if (info.tag_type) |UnionTagType| { - try self.beginObject(); - inline for (info.fields) |u_field| { - if (value == @field(UnionTagType, u_field.name)) { - try self.objectField(u_field.name); - if (u_field.type == void) { - // void value is {} - try self.beginObject(); - try self.endObject(); - } else { - try self.write(@field(value, u_field.name)); - } - break; - } - } else { - unreachable; // No active tag? - } - try self.endObject(); - return; - } else { - @compileError("Unable to stringify untagged union '" ++ @typeName(T) ++ "'"); - } - }, - .@"struct" => |S| { - if (std.meta.hasFn(T, "jsonStringify")) { - return value.jsonStringify(self); - } - - if (S.is_tuple) { - try self.beginArray(); - } else { - try self.beginObject(); - } - inline for (S.fields) |Field| { - // don't include void fields - if (Field.type == void) continue; - - var emit_field = true; - - // don't include optional fields that are null when emit_null_optional_fields is set to false - if (@typeInfo(Field.type) == .optional) { - if (self.options.emit_null_optional_fields == false) { - if (@field(value, Field.name) == null) { - emit_field = false; - } - } - } - - if (emit_field) { - if (!S.is_tuple) { - try self.objectField(Field.name); - } - try self.write(@field(value, Field.name)); - } - } - if (S.is_tuple) { - try self.endArray(); - } else { - try self.endObject(); - } - return; - }, - .error_set => return self.stringValue(@errorName(value)), - .pointer => |ptr_info| switch (ptr_info.size) { - .one => switch (@typeInfo(ptr_info.child)) { - .array => { - // Coerce `*[N]T` to `[]const T`. - const Slice = []const std.meta.Elem(ptr_info.child); - return self.write(@as(Slice, value)); - }, - else => { - return self.write(value.*); - }, - }, - .many, .slice => { - if (ptr_info.size == .many and ptr_info.sentinel() == null) - @compileError("unable to stringify type '" ++ @typeName(T) ++ "' without sentinel"); - const slice = if (ptr_info.size == .many) std.mem.span(value) else value; - - if (ptr_info.child == u8) { - // This is a []const u8, or some similar Zig string. - if (!self.options.emit_strings_as_arrays and std.unicode.utf8ValidateSlice(slice)) { - return self.stringValue(slice); - } - } - - try self.beginArray(); - for (slice) |x| { - try self.write(x); - } - try self.endArray(); - return; - }, - else => @compileError("Unable to stringify type '" ++ @typeName(T) ++ "'"), - }, - .array => { - // Coerce `[N]T` to `*const [N]T` (and then to `[]const T`). - return self.write(&value); - }, - .vector => |info| { - const array: [info.len]info.child = value; - return self.write(&array); - }, - else => @compileError("Unable to stringify type '" ++ @typeName(T) ++ "'"), - } - unreachable; - } - - fn stringValue(self: *Self, s: []const u8) !void { - try self.valueStart(); - try encodeJsonString(s, self.options, self.stream); - self.valueDone(); - } - }; -} - -fn outputUnicodeEscape(codepoint: u21, out_stream: anytype) !void { - if (codepoint <= 0xFFFF) { - // If the character is in the Basic Multilingual Plane (U+0000 through U+FFFF), - // then it may be represented as a six-character sequence: a reverse solidus, followed - // by the lowercase letter u, followed by four hexadecimal digits that encode the character's code point. - try out_stream.writeAll("\\u"); - //try w.printInt("x", .{ .width = 4, .fill = '0' }, codepoint); - try std.fmt.format(out_stream, "{x:0>4}", .{codepoint}); - } else { - assert(codepoint <= 0x10FFFF); - // To escape an extended character that is not in the Basic Multilingual Plane, - // the character is represented as a 12-character sequence, encoding the UTF-16 surrogate pair. - const high = @as(u16, @intCast((codepoint - 0x10000) >> 10)) + 0xD800; - const low = @as(u16, @intCast(codepoint & 0x3FF)) + 0xDC00; - try out_stream.writeAll("\\u"); - //try w.printInt("x", .{ .width = 4, .fill = '0' }, high); - try std.fmt.format(out_stream, "{x:0>4}", .{high}); - try out_stream.writeAll("\\u"); - //try w.printInt("x", .{ .width = 4, .fill = '0' }, low); - try std.fmt.format(out_stream, "{x:0>4}", .{low}); - } -} - -fn outputSpecialEscape(c: u8, writer: anytype) !void { - switch (c) { - '\\' => try writer.writeAll("\\\\"), - '\"' => try writer.writeAll("\\\""), - 0x08 => try writer.writeAll("\\b"), - 0x0C => try writer.writeAll("\\f"), - '\n' => try writer.writeAll("\\n"), - '\r' => try writer.writeAll("\\r"), - '\t' => try writer.writeAll("\\t"), - else => try outputUnicodeEscape(c, writer), - } -} - -/// Write `string` to `writer` as a JSON encoded string. -pub fn encodeJsonString(string: []const u8, options: StringifyOptions, writer: anytype) !void { - try writer.writeByte('\"'); - try encodeJsonStringChars(string, options, writer); - try writer.writeByte('\"'); -} - -/// Write `chars` to `writer` as JSON encoded string characters. -pub fn encodeJsonStringChars(chars: []const u8, options: StringifyOptions, writer: anytype) !void { - var write_cursor: usize = 0; - var i: usize = 0; - if (options.escape_unicode) { - while (i < chars.len) : (i += 1) { - switch (chars[i]) { - // normal ascii character - 0x20...0x21, 0x23...0x5B, 0x5D...0x7E => {}, - 0x00...0x1F, '\\', '\"' => { - // Always must escape these. - try writer.writeAll(chars[write_cursor..i]); - try outputSpecialEscape(chars[i], writer); - write_cursor = i + 1; - }, - 0x7F...0xFF => { - try writer.writeAll(chars[write_cursor..i]); - const ulen = std.unicode.utf8ByteSequenceLength(chars[i]) catch unreachable; - const codepoint = std.unicode.utf8Decode(chars[i..][0..ulen]) catch unreachable; - try outputUnicodeEscape(codepoint, writer); - i += ulen - 1; - write_cursor = i + 1; - }, - } - } - } else { - while (i < chars.len) : (i += 1) { - switch (chars[i]) { - // normal bytes - 0x20...0x21, 0x23...0x5B, 0x5D...0xFF => {}, - 0x00...0x1F, '\\', '\"' => { - // Always must escape these. - try writer.writeAll(chars[write_cursor..i]); - try outputSpecialEscape(chars[i], writer); - write_cursor = i + 1; - }, - } - } - } - try writer.writeAll(chars[write_cursor..chars.len]); -} - -test { - _ = @import("./stringify_test.zig"); -} diff --git a/lib/std/json/stringify_test.zig b/lib/std/json/stringify_test.zig deleted file mode 100644 index 22dd504285..0000000000 --- a/lib/std/json/stringify_test.zig +++ /dev/null @@ -1,504 +0,0 @@ -const std = @import("std"); -const mem = std.mem; -const testing = std.testing; - -const ObjectMap = @import("dynamic.zig").ObjectMap; -const Value = @import("dynamic.zig").Value; - -const StringifyOptions = @import("stringify.zig").StringifyOptions; -const stringify = @import("stringify.zig").stringify; -const stringifyMaxDepth = @import("stringify.zig").stringifyMaxDepth; -const stringifyArbitraryDepth = @import("stringify.zig").stringifyArbitraryDepth; -const stringifyAlloc = @import("stringify.zig").stringifyAlloc; -const writeStream = @import("stringify.zig").writeStream; -const writeStreamMaxDepth = @import("stringify.zig").writeStreamMaxDepth; -const writeStreamArbitraryDepth = @import("stringify.zig").writeStreamArbitraryDepth; - -test "json write stream" { - var out_buf: [1024]u8 = undefined; - var slice_stream = std.io.fixedBufferStream(&out_buf); - const out = slice_stream.writer(); - - { - var w = writeStream(out, .{ .whitespace = .indent_2 }); - try testBasicWriteStream(&w, &slice_stream); - } - - { - var w = writeStreamMaxDepth(out, .{ .whitespace = .indent_2 }, 8); - try testBasicWriteStream(&w, &slice_stream); - } - - { - var w = writeStreamMaxDepth(out, .{ .whitespace = .indent_2 }, null); - try testBasicWriteStream(&w, &slice_stream); - } - - { - var w = writeStreamArbitraryDepth(testing.allocator, out, .{ .whitespace = .indent_2 }); - defer w.deinit(); - try testBasicWriteStream(&w, &slice_stream); - } -} - -fn testBasicWriteStream(w: anytype, slice_stream: anytype) !void { - slice_stream.reset(); - - try w.beginObject(); - - try w.objectField("object"); - var arena_allocator = std.heap.ArenaAllocator.init(testing.allocator); - defer arena_allocator.deinit(); - try w.write(try getJsonObject(arena_allocator.allocator())); - - try w.objectFieldRaw("\"string\""); - try w.write("This is a string"); - - try w.objectField("array"); - try w.beginArray(); - try w.write("Another string"); - try w.write(@as(i32, 1)); - try w.write(@as(f32, 3.5)); - try w.endArray(); - - try w.objectField("int"); - try w.write(@as(i32, 10)); - - try w.objectField("float"); - try w.write(@as(f32, 3.5)); - - try w.endObject(); - - const result = slice_stream.getWritten(); - const expected = - \\{ - \\ "object": { - \\ "one": 1, - \\ "two": 2 - \\ }, - \\ "string": "This is a string", - \\ "array": [ - \\ "Another string", - \\ 1, - \\ 3.5 - \\ ], - \\ "int": 10, - \\ "float": 3.5 - \\} - ; - try std.testing.expectEqualStrings(expected, result); -} - -fn getJsonObject(allocator: std.mem.Allocator) !Value { - var value = Value{ .object = ObjectMap.init(allocator) }; - try value.object.put("one", Value{ .integer = @as(i64, @intCast(1)) }); - try value.object.put("two", Value{ .float = 2.0 }); - return value; -} - -test "stringify null optional fields" { - const MyStruct = struct { - optional: ?[]const u8 = null, - required: []const u8 = "something", - another_optional: ?[]const u8 = null, - another_required: []const u8 = "something else", - }; - try testStringify( - \\{"optional":null,"required":"something","another_optional":null,"another_required":"something else"} - , - MyStruct{}, - .{}, - ); - try testStringify( - \\{"required":"something","another_required":"something else"} - , - MyStruct{}, - .{ .emit_null_optional_fields = false }, - ); -} - -test "stringify basic types" { - try testStringify("false", false, .{}); - try testStringify("true", true, .{}); - try testStringify("null", @as(?u8, null), .{}); - try testStringify("null", @as(?*u32, null), .{}); - try testStringify("42", 42, .{}); - try testStringify("42", 42.0, .{}); - try testStringify("42", @as(u8, 42), .{}); - try testStringify("42", @as(u128, 42), .{}); - try testStringify("9999999999999999", 9999999999999999, .{}); - try testStringify("42", @as(f32, 42), .{}); - try testStringify("42", @as(f64, 42), .{}); - try testStringify("\"ItBroke\"", @as(anyerror, error.ItBroke), .{}); - try testStringify("\"ItBroke\"", error.ItBroke, .{}); -} - -test "stringify string" { - try testStringify("\"hello\"", "hello", .{}); - try testStringify("\"with\\nescapes\\r\"", "with\nescapes\r", .{}); - try testStringify("\"with\\nescapes\\r\"", "with\nescapes\r", .{ .escape_unicode = true }); - try testStringify("\"with unicode\\u0001\"", "with unicode\u{1}", .{}); - try testStringify("\"with unicode\\u0001\"", "with unicode\u{1}", .{ .escape_unicode = true }); - try testStringify("\"with unicode\u{80}\"", "with unicode\u{80}", .{}); - try testStringify("\"with unicode\\u0080\"", "with unicode\u{80}", .{ .escape_unicode = true }); - try testStringify("\"with unicode\u{FF}\"", "with unicode\u{FF}", .{}); - try testStringify("\"with unicode\\u00ff\"", "with unicode\u{FF}", .{ .escape_unicode = true }); - try testStringify("\"with unicode\u{100}\"", "with unicode\u{100}", .{}); - try testStringify("\"with unicode\\u0100\"", "with unicode\u{100}", .{ .escape_unicode = true }); - try testStringify("\"with unicode\u{800}\"", "with unicode\u{800}", .{}); - try testStringify("\"with unicode\\u0800\"", "with unicode\u{800}", .{ .escape_unicode = true }); - try testStringify("\"with unicode\u{8000}\"", "with unicode\u{8000}", .{}); - try testStringify("\"with unicode\\u8000\"", "with unicode\u{8000}", .{ .escape_unicode = true }); - try testStringify("\"with unicode\u{D799}\"", "with unicode\u{D799}", .{}); - try testStringify("\"with unicode\\ud799\"", "with unicode\u{D799}", .{ .escape_unicode = true }); - try testStringify("\"with unicode\u{10000}\"", "with unicode\u{10000}", .{}); - try testStringify("\"with unicode\\ud800\\udc00\"", "with unicode\u{10000}", .{ .escape_unicode = true }); - try testStringify("\"with unicode\u{10FFFF}\"", "with unicode\u{10FFFF}", .{}); - try testStringify("\"with unicode\\udbff\\udfff\"", "with unicode\u{10FFFF}", .{ .escape_unicode = true }); -} - -test "stringify many-item sentinel-terminated string" { - try testStringify("\"hello\"", @as([*:0]const u8, "hello"), .{}); - try testStringify("\"with\\nescapes\\r\"", @as([*:0]const u8, "with\nescapes\r"), .{ .escape_unicode = true }); - try testStringify("\"with unicode\\u0001\"", @as([*:0]const u8, "with unicode\u{1}"), .{ .escape_unicode = true }); -} - -test "stringify enums" { - const E = enum { - foo, - bar, - }; - try testStringify("\"foo\"", E.foo, .{}); - try testStringify("\"bar\"", E.bar, .{}); -} - -test "stringify non-exhaustive enum" { - const E = enum(u8) { - foo = 0, - _, - }; - try testStringify("\"foo\"", E.foo, .{}); - try testStringify("1", @as(E, @enumFromInt(1)), .{}); -} - -test "stringify enum literals" { - try testStringify("\"foo\"", .foo, .{}); - try testStringify("\"bar\"", .bar, .{}); -} - -test "stringify tagged unions" { - const T = union(enum) { - nothing, - foo: u32, - bar: bool, - }; - try testStringify("{\"nothing\":{}}", T{ .nothing = {} }, .{}); - try testStringify("{\"foo\":42}", T{ .foo = 42 }, .{}); - try testStringify("{\"bar\":true}", T{ .bar = true }, .{}); -} - -test "stringify struct" { - try testStringify("{\"foo\":42}", struct { - foo: u32, - }{ .foo = 42 }, .{}); -} - -test "emit_strings_as_arrays" { - // Should only affect string values, not object keys. - try testStringify("{\"foo\":\"bar\"}", .{ .foo = "bar" }, .{}); - try testStringify("{\"foo\":[98,97,114]}", .{ .foo = "bar" }, .{ .emit_strings_as_arrays = true }); - // Should *not* affect these types: - try testStringify("\"foo\"", @as(enum { foo, bar }, .foo), .{ .emit_strings_as_arrays = true }); - try testStringify("\"ItBroke\"", error.ItBroke, .{ .emit_strings_as_arrays = true }); - // Should work on these: - try testStringify("\"bar\"", @Vector(3, u8){ 'b', 'a', 'r' }, .{}); - try testStringify("[98,97,114]", @Vector(3, u8){ 'b', 'a', 'r' }, .{ .emit_strings_as_arrays = true }); - try testStringify("\"bar\"", [3]u8{ 'b', 'a', 'r' }, .{}); - try testStringify("[98,97,114]", [3]u8{ 'b', 'a', 'r' }, .{ .emit_strings_as_arrays = true }); -} - -test "stringify struct with indentation" { - try testStringify( - \\{ - \\ "foo": 42, - \\ "bar": [ - \\ 1, - \\ 2, - \\ 3 - \\ ] - \\} - , - struct { - foo: u32, - bar: [3]u32, - }{ - .foo = 42, - .bar = .{ 1, 2, 3 }, - }, - .{ .whitespace = .indent_4 }, - ); - try testStringify( - "{\n\t\"foo\": 42,\n\t\"bar\": [\n\t\t1,\n\t\t2,\n\t\t3\n\t]\n}", - struct { - foo: u32, - bar: [3]u32, - }{ - .foo = 42, - .bar = .{ 1, 2, 3 }, - }, - .{ .whitespace = .indent_tab }, - ); - try testStringify( - \\{"foo":42,"bar":[1,2,3]} - , - struct { - foo: u32, - bar: [3]u32, - }{ - .foo = 42, - .bar = .{ 1, 2, 3 }, - }, - .{ .whitespace = .minified }, - ); -} - -test "stringify struct with void field" { - try testStringify("{\"foo\":42}", struct { - foo: u32, - bar: void = {}, - }{ .foo = 42 }, .{}); -} - -test "stringify array of structs" { - const MyStruct = struct { - foo: u32, - }; - try testStringify("[{\"foo\":42},{\"foo\":100},{\"foo\":1000}]", [_]MyStruct{ - MyStruct{ .foo = 42 }, - MyStruct{ .foo = 100 }, - MyStruct{ .foo = 1000 }, - }, .{}); -} - -test "stringify struct with custom stringifier" { - try testStringify("[\"something special\",42]", struct { - foo: u32, - const Self = @This(); - pub fn jsonStringify(value: @This(), jws: anytype) !void { - _ = value; - try jws.beginArray(); - try jws.write("something special"); - try jws.write(42); - try jws.endArray(); - } - }{ .foo = 42 }, .{}); -} - -test "stringify vector" { - try testStringify("[1,1]", @as(@Vector(2, u32), @splat(1)), .{}); - try testStringify("\"AA\"", @as(@Vector(2, u8), @splat('A')), .{}); - try testStringify("[65,65]", @as(@Vector(2, u8), @splat('A')), .{ .emit_strings_as_arrays = true }); -} - -test "stringify tuple" { - try testStringify("[\"foo\",42]", std.meta.Tuple(&.{ []const u8, usize }){ "foo", 42 }, .{}); -} - -fn testStringify(expected: []const u8, value: anytype, options: StringifyOptions) !void { - const ValidationWriter = struct { - const Self = @This(); - pub const Writer = std.io.GenericWriter(*Self, Error, write); - pub const Error = error{ - TooMuchData, - DifferentData, - }; - - expected_remaining: []const u8, - - fn init(exp: []const u8) Self { - return .{ .expected_remaining = exp }; - } - - pub fn writer(self: *Self) Writer { - return .{ .context = self }; - } - - fn write(self: *Self, bytes: []const u8) Error!usize { - if (self.expected_remaining.len < bytes.len) { - std.debug.print( - \\====== expected this output: ========= - \\{s} - \\======== instead found this: ========= - \\{s} - \\====================================== - , .{ - self.expected_remaining, - bytes, - }); - return error.TooMuchData; - } - if (!mem.eql(u8, self.expected_remaining[0..bytes.len], bytes)) { - std.debug.print( - \\====== expected this output: ========= - \\{s} - \\======== instead found this: ========= - \\{s} - \\====================================== - , .{ - self.expected_remaining[0..bytes.len], - bytes, - }); - return error.DifferentData; - } - self.expected_remaining = self.expected_remaining[bytes.len..]; - return bytes.len; - } - }; - - var vos = ValidationWriter.init(expected); - try stringifyArbitraryDepth(testing.allocator, value, options, vos.writer()); - if (vos.expected_remaining.len > 0) return error.NotEnoughData; - - // Also test with safety disabled. - try testStringifyMaxDepth(expected, value, options, null); - try testStringifyArbitraryDepth(expected, value, options); -} - -fn testStringifyMaxDepth(expected: []const u8, value: anytype, options: StringifyOptions, comptime max_depth: ?usize) !void { - var out_buf: [1024]u8 = undefined; - var slice_stream = std.io.fixedBufferStream(&out_buf); - const out = slice_stream.writer(); - - try stringifyMaxDepth(value, options, out, max_depth); - const got = slice_stream.getWritten(); - - try testing.expectEqualStrings(expected, got); -} - -fn testStringifyArbitraryDepth(expected: []const u8, value: anytype, options: StringifyOptions) !void { - var out_buf: [1024]u8 = undefined; - var slice_stream = std.io.fixedBufferStream(&out_buf); - const out = slice_stream.writer(); - - try stringifyArbitraryDepth(testing.allocator, value, options, out); - const got = slice_stream.getWritten(); - - try testing.expectEqualStrings(expected, got); -} - -test "stringify alloc" { - const allocator = std.testing.allocator; - const expected = - \\{"foo":"bar","answer":42,"my_friend":"sammy"} - ; - const actual = try stringifyAlloc(allocator, .{ .foo = "bar", .answer = 42, .my_friend = "sammy" }, .{}); - defer allocator.free(actual); - - try std.testing.expectEqualStrings(expected, actual); -} - -test "comptime stringify" { - comptime testStringifyMaxDepth("false", false, .{}, null) catch unreachable; - comptime testStringifyMaxDepth("false", false, .{}, 0) catch unreachable; - comptime testStringifyArbitraryDepth("false", false, .{}) catch unreachable; - - const MyStruct = struct { - foo: u32, - }; - comptime testStringifyMaxDepth("[{\"foo\":42},{\"foo\":100},{\"foo\":1000}]", [_]MyStruct{ - MyStruct{ .foo = 42 }, - MyStruct{ .foo = 100 }, - MyStruct{ .foo = 1000 }, - }, .{}, null) catch unreachable; - comptime testStringifyMaxDepth("[{\"foo\":42},{\"foo\":100},{\"foo\":1000}]", [_]MyStruct{ - MyStruct{ .foo = 42 }, - MyStruct{ .foo = 100 }, - MyStruct{ .foo = 1000 }, - }, .{}, 8) catch unreachable; -} - -test "print" { - var out_buf: [1024]u8 = undefined; - var slice_stream = std.io.fixedBufferStream(&out_buf); - const out = slice_stream.writer(); - - var w = writeStream(out, .{ .whitespace = .indent_2 }); - defer w.deinit(); - - try w.beginObject(); - try w.objectField("a"); - try w.print("[ ]", .{}); - try w.objectField("b"); - try w.beginArray(); - try w.print("[{s}] ", .{"[]"}); - try w.print(" {}", .{12345}); - try w.endArray(); - try w.endObject(); - - const result = slice_stream.getWritten(); - const expected = - \\{ - \\ "a": [ ], - \\ "b": [ - \\ [[]] , - \\ 12345 - \\ ] - \\} - ; - try std.testing.expectEqualStrings(expected, result); -} - -test "nonportable numbers" { - try testStringify("9999999999999999", 9999999999999999, .{}); - try testStringify("\"9999999999999999\"", 9999999999999999, .{ .emit_nonportable_numbers_as_strings = true }); -} - -test "stringify raw streaming" { - var out_buf: [1024]u8 = undefined; - var slice_stream = std.io.fixedBufferStream(&out_buf); - const out = slice_stream.writer(); - - { - var w = writeStream(out, .{ .whitespace = .indent_2 }); - try testRawStreaming(&w, &slice_stream); - } - - { - var w = writeStreamMaxDepth(out, .{ .whitespace = .indent_2 }, 8); - try testRawStreaming(&w, &slice_stream); - } - - { - var w = writeStreamMaxDepth(out, .{ .whitespace = .indent_2 }, null); - try testRawStreaming(&w, &slice_stream); - } - - { - var w = writeStreamArbitraryDepth(testing.allocator, out, .{ .whitespace = .indent_2 }); - defer w.deinit(); - try testRawStreaming(&w, &slice_stream); - } -} - -fn testRawStreaming(w: anytype, slice_stream: anytype) !void { - slice_stream.reset(); - - try w.beginObject(); - try w.beginObjectFieldRaw(); - try w.stream.writeAll("\"long"); - try w.stream.writeAll(" key\""); - w.endObjectFieldRaw(); - try w.beginWriteRaw(); - try w.stream.writeAll("\"long"); - try w.stream.writeAll(" value\""); - w.endWriteRaw(); - try w.endObject(); - - const result = slice_stream.getWritten(); - const expected = - \\{ - \\ "long key": "long value" - \\} - ; - try std.testing.expectEqualStrings(expected, result); -} diff --git a/lib/std/json/test.zig b/lib/std/json/test.zig index 136e8e34d1..d3d803e939 100644 --- a/lib/std/json/test.zig +++ b/lib/std/json/test.zig @@ -1,10 +1,9 @@ const std = @import("std"); +const json = std.json; const testing = std.testing; const parseFromSlice = @import("./static.zig").parseFromSlice; -const validate = @import("./scanner.zig").validate; -const JsonScanner = @import("./scanner.zig").Scanner; +const Scanner = @import("./Scanner.zig"); const Value = @import("./dynamic.zig").Value; -const stringifyAlloc = @import("./stringify.zig").stringifyAlloc; // Support for JSONTestSuite.zig pub fn ok(s: []const u8) !void { @@ -20,7 +19,7 @@ pub fn any(s: []const u8) !void { testHighLevelDynamicParser(s) catch {}; } fn testLowLevelScanner(s: []const u8) !void { - var scanner = JsonScanner.initCompleteInput(testing.allocator, s); + var scanner = Scanner.initCompleteInput(testing.allocator, s); defer scanner.deinit(); while (true) { const token = try scanner.next(); @@ -47,12 +46,12 @@ test "n_object_closed_missing_value" { } fn roundTrip(s: []const u8) !void { - try testing.expect(try validate(testing.allocator, s)); + try testing.expect(try Scanner.validate(testing.allocator, s)); var parsed = try parseFromSlice(Value, testing.allocator, s, .{}); defer parsed.deinit(); - const rendered = try stringifyAlloc(testing.allocator, parsed.value, .{}); + const rendered = try json.Stringify.valueAlloc(testing.allocator, parsed.value, .{}); defer testing.allocator.free(rendered); try testing.expectEqualStrings(s, rendered); From b956b021878cd7b67f01c30a30fcb085887f24dc Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 19 Jul 2025 16:24:37 -0700 Subject: [PATCH 06/26] `zig env`: update std.json API --- src/print_env.zig | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/print_env.zig b/src/print_env.zig index af433ddfc1..d1546abc6c 100644 --- a/src/print_env.zig +++ b/src/print_env.zig @@ -21,10 +21,10 @@ pub fn cmdEnv(arena: Allocator, args: []const []const u8) !void { const host = try std.zig.system.resolveTargetQuery(.{}); const triple = try host.zigTriple(arena); - var bw = std.io.bufferedWriter(std.fs.File.stdout().deprecatedWriter()); - const w = bw.writer(); + var buffer: [1024]u8 = undefined; + var stdout_writer = std.fs.File.stdout().writer(&buffer); - var jws = std.json.writeStream(w, .{ .whitespace = .indent_1 }); + var jws: std.json.Stringify = .{ .writer = &stdout_writer.interface, .options = .{ .whitespace = .indent_1 } }; try jws.beginObject(); @@ -55,7 +55,7 @@ pub fn cmdEnv(arena: Allocator, args: []const []const u8) !void { try jws.endObject(); try jws.endObject(); - try w.writeByte('\n'); - try bw.flush(); + try stdout_writer.interface.writeByte('\n'); + try stdout_writer.interface.flush(); } From c3da98cf5a8b8a3164db78998bda848100871918 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 19 Jul 2025 16:59:16 -0700 Subject: [PATCH 07/26] std.zon: update to new I/O API --- lib/std/Build/Cache/Path.zig | 8 +- lib/std/zig.zig | 29 +- lib/std/zig/Ast.zig | 2 +- lib/std/zon/parse.zig | 26 +- lib/std/zon/stringify.zig | 1907 +++++++++++++++++----------------- 5 files changed, 979 insertions(+), 993 deletions(-) diff --git a/lib/std/Build/Cache/Path.zig b/lib/std/Build/Cache/Path.zig index a0a58067fc..efd0f86105 100644 --- a/lib/std/Build/Cache/Path.zig +++ b/lib/std/Build/Cache/Path.zig @@ -161,17 +161,19 @@ pub fn formatEscapeString(path: Path, writer: *std.io.Writer) std.io.Writer.Erro } } +/// Deprecated, use double quoted escape to print paths. pub fn fmtEscapeChar(path: Path) std.fmt.Formatter(Path, formatEscapeChar) { return .{ .data = path }; } +/// Deprecated, use double quoted escape to print paths. pub fn formatEscapeChar(path: Path, writer: *std.io.Writer) std.io.Writer.Error!void { if (path.root_dir.path) |p| { - try std.zig.charEscape(p, writer); - if (path.sub_path.len > 0) try std.zig.charEscape(fs.path.sep_str, writer); + for (p) |byte| try std.zig.charEscape(byte, writer); + if (path.sub_path.len > 0) try writer.writeByte(fs.path.sep); } if (path.sub_path.len > 0) { - try std.zig.charEscape(path.sub_path, writer); + for (path.sub_path) |byte| try std.zig.charEscape(byte, writer); } } diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 51eac1b0a6..4d75bfa2db 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -446,8 +446,8 @@ pub fn fmtString(bytes: []const u8) std.fmt.Formatter([]const u8, stringEscape) } /// Return a formatter for escaping a single quoted Zig string. -pub fn fmtChar(bytes: []const u8) std.fmt.Formatter([]const u8, charEscape) { - return .{ .data = bytes }; +pub fn fmtChar(c: u21) std.fmt.Formatter(u21, charEscape) { + return .{ .data = c }; } test fmtString { @@ -458,9 +458,7 @@ test fmtString { } test fmtChar { - try std.testing.expectFmt( - \\" \\ hi \x07 \x11 " derp \'" - , "\"{f}\"", .{fmtChar(" \\ hi \x07 \x11 \" derp '")}); + try std.testing.expectFmt("c \\u{26a1}", "{f} {f}", .{ fmtChar('c'), fmtChar('âš¡') }); } /// Print the string as escaped contents of a double quoted string. @@ -480,21 +478,26 @@ pub fn stringEscape(bytes: []const u8, w: *Writer) Writer.Error!void { }; } -/// Print the string as escaped contents of a single-quoted string. -pub fn charEscape(bytes: []const u8, w: *Writer) Writer.Error!void { - for (bytes) |byte| switch (byte) { +/// Print as escaped contents of a single-quoted string. +pub fn charEscape(codepoint: u21, w: *Writer) Writer.Error!void { + switch (codepoint) { '\n' => try w.writeAll("\\n"), '\r' => try w.writeAll("\\r"), '\t' => try w.writeAll("\\t"), '\\' => try w.writeAll("\\\\"), - '"' => try w.writeByte('"'), '\'' => try w.writeAll("\\'"), - ' ', '!', '#'...'&', '('...'[', ']'...'~' => try w.writeByte(byte), + '"', ' ', '!', '#'...'&', '('...'[', ']'...'~' => try w.writeByte(@intCast(codepoint)), else => { - try w.writeAll("\\x"); - try w.printInt(byte, 16, .lower, .{ .width = 2, .fill = '0' }); + if (std.math.cast(u8, codepoint)) |byte| { + try w.writeAll("\\x"); + try w.printInt(byte, 16, .lower, .{ .width = 2, .fill = '0' }); + } else { + try w.writeAll("\\u{"); + try w.printInt(codepoint, 16, .lower, .{}); + try w.writeByte('}'); + } }, - }; + } } pub fn isValidId(bytes: []const u8) bool { diff --git a/lib/std/zig/Ast.zig b/lib/std/zig/Ast.zig index 1f36c0fdbf..c15693fd62 100644 --- a/lib/std/zig/Ast.zig +++ b/lib/std/zig/Ast.zig @@ -574,7 +574,7 @@ pub fn renderError(tree: Ast, parse_error: Error, w: *Writer) Writer.Error!void '/' => "comment", else => unreachable, }, - std.zig.fmtChar(tok_slice[parse_error.extra.offset..][0..1]), + std.zig.fmtChar(tok_slice[parse_error.extra.offset]), }); }, diff --git a/lib/std/zon/parse.zig b/lib/std/zon/parse.zig index d5df5693ec..96a7fa6595 100644 --- a/lib/std/zon/parse.zig +++ b/lib/std/zon/parse.zig @@ -64,14 +64,14 @@ pub const Error = union(enum) { } }; - fn formatMessage(self: []const u8, w: *std.io.Writer) std.io.Writer.Error!void { + fn formatMessage(self: []const u8, w: *std.Io.Writer) std.Io.Writer.Error!void { // Just writes the string for now, but we're keeping this behind a formatter so we have // the option to extend it in the future to print more advanced messages (like `Error` // does) without breaking the API. try w.writeAll(self); } - pub fn fmtMessage(self: Note, diag: *const Diagnostics) std.fmt.Formatter([]const u8, Note.formatMessage) { + pub fn fmtMessage(self: Note, diag: *const Diagnostics) std.fmt.Alt([]const u8, Note.formatMessage) { return .{ .data = switch (self) { .zoir => |note| note.msg.get(diag.zoir), .type_check => |note| note.msg, @@ -147,14 +147,14 @@ pub const Error = union(enum) { diag: *const Diagnostics, }; - fn formatMessage(self: FormatMessage, w: *std.io.Writer) std.io.Writer.Error!void { + fn formatMessage(self: FormatMessage, w: *std.Io.Writer) std.Io.Writer.Error!void { switch (self.err) { .zoir => |err| try w.writeAll(err.msg.get(self.diag.zoir)), .type_check => |tc| try w.writeAll(tc.message), } } - pub fn fmtMessage(self: @This(), diag: *const Diagnostics) std.fmt.Formatter(FormatMessage, formatMessage) { + pub fn fmtMessage(self: @This(), diag: *const Diagnostics) std.fmt.Alt(FormatMessage, formatMessage) { return .{ .data = .{ .err = self, .diag = diag, @@ -226,7 +226,7 @@ pub const Diagnostics = struct { return .{ .diag = self }; } - pub fn format(self: *const @This(), w: *std.io.Writer) std.io.Writer.Error!void { + pub fn format(self: *const @This(), w: *std.Io.Writer) std.Io.Writer.Error!void { var errors = self.iterateErrors(); while (errors.next()) |err| { const loc = err.getLocation(self); @@ -606,7 +606,7 @@ const Parser = struct { } } - fn parseSlicePointer(self: *@This(), T: type, node: Zoir.Node.Index) !T { + fn parseSlicePointer(self: *@This(), T: type, node: Zoir.Node.Index) ParseExprInnerError!T { switch (node.get(self.zoir)) { .string_literal => return self.parseString(T, node), .array_literal => |nodes| return self.parseSlice(T, nodes), @@ -1048,6 +1048,7 @@ const Parser = struct { name: []const u8, ) error{ OutOfMemory, ParseZon } { @branchHint(.cold); + const gpa = self.gpa; const token = if (field) |f| b: { var buf: [2]Ast.Node.Index = undefined; const struct_init = self.ast.fullStructInit(&buf, node.getAstNode(self.zoir)).?; @@ -1065,13 +1066,12 @@ const Parser = struct { }; } else b: { const msg = "supported: "; - var buf: std.ArrayListUnmanaged(u8) = try .initCapacity(self.gpa, 64); - defer buf.deinit(self.gpa); - const writer = buf.writer(self.gpa); - try writer.writeAll(msg); + var buf: std.ArrayListUnmanaged(u8) = try .initCapacity(gpa, 64); + defer buf.deinit(gpa); + try buf.appendSlice(gpa, msg); inline for (info.fields, 0..) |field_info, i| { - if (i != 0) try writer.writeAll(", "); - try writer.print("'{f}'", .{std.zig.fmtIdFlags(field_info.name, .{ + if (i != 0) try buf.appendSlice(gpa, ", "); + try buf.print(gpa, "'{f}'", .{std.zig.fmtIdFlags(field_info.name, .{ .allow_primitive = true, .allow_underscore = true, })}); @@ -1079,7 +1079,7 @@ const Parser = struct { break :b .{ .token = token, .offset = 0, - .msg = try buf.toOwnedSlice(self.gpa), + .msg = try buf.toOwnedSlice(gpa), .owned = true, }; }; diff --git a/lib/std/zon/stringify.zig b/lib/std/zon/stringify.zig index 1a38dc7579..4cb2640063 100644 --- a/lib/std/zon/stringify.zig +++ b/lib/std/zon/stringify.zig @@ -22,6 +22,7 @@ const std = @import("std"); const assert = std.debug.assert; +const Writer = std.Io.Writer; /// Options for `serialize`. pub const SerializeOptions = struct { @@ -40,15 +41,12 @@ pub const SerializeOptions = struct { /// Serialize the given value as ZON. /// /// It is asserted at comptime that `@TypeOf(val)` is not a recursive type. -pub fn serialize( - val: anytype, - options: SerializeOptions, - writer: anytype, -) @TypeOf(writer).Error!void { - var sz = serializer(writer, .{ - .whitespace = options.whitespace, - }); - try sz.value(val, .{ +pub fn serialize(val: anytype, options: SerializeOptions, writer: *Writer) Writer.Error!void { + var s: Serializer = .{ + .writer = writer, + .options = .{ .whitespace = options.whitespace }, + }; + try s.value(val, .{ .emit_codepoint_literals = options.emit_codepoint_literals, .emit_strings_as_containers = options.emit_strings_as_containers, .emit_default_optional_fields = options.emit_default_optional_fields, @@ -62,13 +60,14 @@ pub fn serialize( pub fn serializeMaxDepth( val: anytype, options: SerializeOptions, - writer: anytype, + writer: *Writer, depth: usize, -) (@TypeOf(writer).Error || error{ExceededMaxDepth})!void { - var sz = serializer(writer, .{ - .whitespace = options.whitespace, - }); - try sz.valueMaxDepth(val, .{ +) Serializer.DepthError!void { + var s: Serializer = .{ + .writer = writer, + .options = .{ .whitespace = options.whitespace }, + }; + try s.valueMaxDepth(val, .{ .emit_codepoint_literals = options.emit_codepoint_literals, .emit_strings_as_containers = options.emit_strings_as_containers, .emit_default_optional_fields = options.emit_default_optional_fields, @@ -81,44 +80,45 @@ pub fn serializeMaxDepth( pub fn serializeArbitraryDepth( val: anytype, options: SerializeOptions, - writer: anytype, -) @TypeOf(writer).Error!void { - var sz = serializer(writer, .{ - .whitespace = options.whitespace, - }); - try sz.valueArbitraryDepth(val, .{ + writer: *Writer, +) Serializer.Error!void { + var s: Serializer = .{ + .writer = writer, + .options = .{ .whitespace = options.whitespace }, + }; + try s.valueArbitraryDepth(val, .{ .emit_codepoint_literals = options.emit_codepoint_literals, .emit_strings_as_containers = options.emit_strings_as_containers, .emit_default_optional_fields = options.emit_default_optional_fields, }); } -fn typeIsRecursive(comptime T: type) bool { - return comptime typeIsRecursiveImpl(T, &.{}); +inline fn typeIsRecursive(comptime T: type) bool { + return comptime typeIsRecursiveInner(T, &.{}); } -fn typeIsRecursiveImpl(comptime T: type, comptime prev_visited: []const type) bool { +fn typeIsRecursiveInner(comptime T: type, comptime prev_visited: []const type) bool { for (prev_visited) |V| { if (V == T) return true; } const visited = prev_visited ++ .{T}; return switch (@typeInfo(T)) { - .pointer => |pointer| typeIsRecursiveImpl(pointer.child, visited), - .optional => |optional| typeIsRecursiveImpl(optional.child, visited), - .array => |array| typeIsRecursiveImpl(array.child, visited), - .vector => |vector| typeIsRecursiveImpl(vector.child, visited), + .pointer => |pointer| typeIsRecursiveInner(pointer.child, visited), + .optional => |optional| typeIsRecursiveInner(optional.child, visited), + .array => |array| typeIsRecursiveInner(array.child, visited), + .vector => |vector| typeIsRecursiveInner(vector.child, visited), .@"struct" => |@"struct"| for (@"struct".fields) |field| { - if (typeIsRecursiveImpl(field.type, visited)) break true; + if (typeIsRecursiveInner(field.type, visited)) break true; } else false, .@"union" => |@"union"| inline for (@"union".fields) |field| { - if (typeIsRecursiveImpl(field.type, visited)) break true; + if (typeIsRecursiveInner(field.type, visited)) break true; } else false, else => false, }; } -fn canSerializeType(T: type) bool { +inline fn canSerializeType(T: type) bool { comptime return canSerializeTypeInner(T, &.{}, false); } @@ -343,12 +343,6 @@ test "std.zon checkValueDepth" { try expectValueDepthEquals(3, @as([]const []const u8, &.{&.{ 1, 2, 3 }})); } -/// Options for `Serializer`. -pub const SerializerOptions = struct { - /// If false, only syntactically necessary whitespace is emitted. - whitespace: bool = true, -}; - /// Determines when to emit Unicode code point literals as opposed to integer literals. pub const EmitCodepointLiterals = enum { /// Never emit Unicode code point literals. @@ -440,634 +434,616 @@ pub const SerializeContainerOptions = struct { /// For manual serialization of containers, see: /// * `beginStruct` /// * `beginTuple` -/// -/// # Example -/// ```zig -/// var sz = serializer(writer, .{}); -/// var vec2 = try sz.beginStruct(.{}); -/// try vec2.field("x", 1.5, .{}); -/// try vec2.fieldPrefix(); -/// try sz.value(2.5); -/// try vec2.end(); -/// ``` -pub fn Serializer(Writer: type) type { - return struct { - const Self = @This(); +pub const Serializer = struct { + options: Options = .{}, + indent_level: u8 = 0, + writer: *Writer, - options: SerializerOptions, - indent_level: u8, - writer: Writer, + pub const Error = Writer.Error; + pub const DepthError = Error || error{ExceededMaxDepth}; - /// Initialize a serializer. - fn init(writer: Writer, options: SerializerOptions) Self { + pub const Options = struct { + /// If false, only syntactically necessary whitespace is emitted. + whitespace: bool = true, + }; + + /// Serialize a value, similar to `serialize`. + pub fn value(self: *Serializer, val: anytype, options: ValueOptions) Error!void { + comptime assert(!typeIsRecursive(@TypeOf(val))); + return self.valueArbitraryDepth(val, options); + } + + /// Serialize a value, similar to `serializeMaxDepth`. + /// Can return `error.ExceededMaxDepth`. + pub fn valueMaxDepth(self: *Serializer, val: anytype, options: ValueOptions, depth: usize) DepthError!void { + try checkValueDepth(val, depth); + return self.valueArbitraryDepth(val, options); + } + + /// Serialize a value, similar to `serializeArbitraryDepth`. + pub fn valueArbitraryDepth(self: *Serializer, val: anytype, options: ValueOptions) Error!void { + comptime assert(canSerializeType(@TypeOf(val))); + switch (@typeInfo(@TypeOf(val))) { + .int, .comptime_int => if (options.emit_codepoint_literals.emitAsCodepoint(val)) |c| { + self.codePoint(c) catch |err| switch (err) { + error.InvalidCodepoint => unreachable, // Already validated + else => |e| return e, + }; + } else { + try self.int(val); + }, + .float, .comptime_float => try self.float(val), + .bool, .null => try self.writer.print("{}", .{val}), + .enum_literal => try self.ident(@tagName(val)), + .@"enum" => try self.ident(@tagName(val)), + .pointer => |pointer| { + // Try to serialize as a string + const item: ?type = switch (@typeInfo(pointer.child)) { + .array => |array| array.child, + else => if (pointer.size == .slice) pointer.child else null, + }; + if (item == u8 and + (pointer.sentinel() == null or pointer.sentinel() == 0) and + !options.emit_strings_as_containers) + { + return try self.string(val); + } + + // Serialize as either a tuple or as the child type + switch (pointer.size) { + .slice => try self.tupleImpl(val, options), + .one => try self.valueArbitraryDepth(val.*, options), + else => comptime unreachable, + } + }, + .array => { + var container = try self.beginTuple( + .{ .whitespace_style = .{ .fields = val.len } }, + ); + for (val) |item_val| { + try container.fieldArbitraryDepth(item_val, options); + } + try container.end(); + }, + .@"struct" => |@"struct"| if (@"struct".is_tuple) { + var container = try self.beginTuple( + .{ .whitespace_style = .{ .fields = @"struct".fields.len } }, + ); + inline for (val) |field_value| { + try container.fieldArbitraryDepth(field_value, options); + } + try container.end(); + } else { + // Decide which fields to emit + const fields, const skipped: [@"struct".fields.len]bool = if (options.emit_default_optional_fields) b: { + break :b .{ @"struct".fields.len, @splat(false) }; + } else b: { + var fields = @"struct".fields.len; + var skipped: [@"struct".fields.len]bool = @splat(false); + inline for (@"struct".fields, &skipped) |field_info, *skip| { + if (field_info.default_value_ptr) |ptr| { + const default: *const field_info.type = @ptrCast(@alignCast(ptr)); + const field_value = @field(val, field_info.name); + if (std.meta.eql(field_value, default.*)) { + skip.* = true; + fields -= 1; + } + } + } + break :b .{ fields, skipped }; + }; + + // Emit those fields + var container = try self.beginStruct( + .{ .whitespace_style = .{ .fields = fields } }, + ); + inline for (@"struct".fields, skipped) |field_info, skip| { + if (!skip) { + try container.fieldArbitraryDepth( + field_info.name, + @field(val, field_info.name), + options, + ); + } + } + try container.end(); + }, + .@"union" => |@"union"| { + comptime assert(@"union".tag_type != null); + switch (val) { + inline else => |pl, tag| if (@TypeOf(pl) == void) + try self.writer.print(".{s}", .{@tagName(tag)}) + else { + var container = try self.beginStruct(.{ .whitespace_style = .{ .fields = 1 } }); + + try container.fieldArbitraryDepth( + @tagName(tag), + pl, + options, + ); + + try container.end(); + }, + } + }, + .optional => if (val) |inner| { + try self.valueArbitraryDepth(inner, options); + } else { + try self.writer.writeAll("null"); + }, + .vector => |vector| { + var container = try self.beginTuple( + .{ .whitespace_style = .{ .fields = vector.len } }, + ); + for (0..vector.len) |i| { + try container.fieldArbitraryDepth(val[i], options); + } + try container.end(); + }, + + else => comptime unreachable, + } + } + + /// Serialize an integer. + pub fn int(self: *Serializer, val: anytype) Error!void { + try self.writer.printInt(val, 10, .lower, .{}); + } + + /// Serialize a float. + pub fn float(self: *Serializer, val: anytype) Error!void { + switch (@typeInfo(@TypeOf(val))) { + .float => if (std.math.isNan(val)) { + return self.writer.writeAll("nan"); + } else if (std.math.isPositiveInf(val)) { + return self.writer.writeAll("inf"); + } else if (std.math.isNegativeInf(val)) { + return self.writer.writeAll("-inf"); + } else if (std.math.isNegativeZero(val)) { + return self.writer.writeAll("-0.0"); + } else { + try self.writer.print("{d}", .{val}); + }, + .comptime_float => if (val == 0) { + return self.writer.writeAll("0"); + } else { + try self.writer.print("{d}", .{val}); + }, + else => comptime unreachable, + } + } + + /// Serialize `name` as an identifier prefixed with `.`. + /// + /// Escapes the identifier if necessary. + pub fn ident(self: *Serializer, name: []const u8) Error!void { + try self.writer.print(".{f}", .{std.zig.fmtIdPU(name)}); + } + + pub const CodePointError = Error || error{InvalidCodepoint}; + + /// Serialize `val` as a Unicode codepoint. + /// + /// Returns `error.InvalidCodepoint` if `val` is not a valid Unicode codepoint. + pub fn codePoint(self: *Serializer, val: u21) CodePointError!void { + try self.writer.print("'{f}'", .{std.zig.fmtChar(val)}); + } + + /// Like `value`, but always serializes `val` as a tuple. + /// + /// Will fail at comptime if `val` is not a tuple, array, pointer to an array, or slice. + pub fn tuple(self: *Serializer, val: anytype, options: ValueOptions) Error!void { + comptime assert(!typeIsRecursive(@TypeOf(val))); + try self.tupleArbitraryDepth(val, options); + } + + /// Like `tuple`, but recursive types are allowed. + /// + /// Returns `error.ExceededMaxDepth` if `depth` is exceeded. + pub fn tupleMaxDepth( + self: *Serializer, + val: anytype, + options: ValueOptions, + depth: usize, + ) DepthError!void { + try checkValueDepth(val, depth); + try self.tupleArbitraryDepth(val, options); + } + + /// Like `tuple`, but recursive types are allowed. + /// + /// It is the caller's responsibility to ensure that `val` does not contain cycles. + pub fn tupleArbitraryDepth( + self: *Serializer, + val: anytype, + options: ValueOptions, + ) Error!void { + try self.tupleImpl(val, options); + } + + fn tupleImpl(self: *Serializer, val: anytype, options: ValueOptions) Error!void { + comptime assert(canSerializeType(@TypeOf(val))); + switch (@typeInfo(@TypeOf(val))) { + .@"struct" => { + var container = try self.beginTuple(.{ .whitespace_style = .{ .fields = val.len } }); + inline for (val) |item_val| { + try container.fieldArbitraryDepth(item_val, options); + } + try container.end(); + }, + .pointer, .array => { + var container = try self.beginTuple(.{ .whitespace_style = .{ .fields = val.len } }); + for (val) |item_val| { + try container.fieldArbitraryDepth(item_val, options); + } + try container.end(); + }, + else => comptime unreachable, + } + } + + /// Like `value`, but always serializes `val` as a string. + pub fn string(self: *Serializer, val: []const u8) Error!void { + try self.writer.print("\"{f}\"", .{std.zig.fmtString(val)}); + } + + /// Options for formatting multiline strings. + pub const MultilineStringOptions = struct { + /// If top level is true, whitespace before and after the multiline string is elided. + /// If it is true, a newline is printed, then the value, followed by a newline, and if + /// whitespace is true any necessary indentation follows. + top_level: bool = false, + }; + + pub const MultilineStringError = Error || error{InnerCarriageReturn}; + + /// Like `value`, but always serializes to a multiline string literal. + /// + /// Returns `error.InnerCarriageReturn` if `val` contains a CR not followed by a newline, + /// since multiline strings cannot represent CR without a following newline. + pub fn multilineString( + self: *Serializer, + val: []const u8, + options: MultilineStringOptions, + ) MultilineStringError!void { + // Make sure the string does not contain any carriage returns not followed by a newline + var i: usize = 0; + while (i < val.len) : (i += 1) { + if (val[i] == '\r') { + if (i + 1 < val.len) { + if (val[i + 1] == '\n') { + i += 1; + continue; + } + } + return error.InnerCarriageReturn; + } + } + + if (!options.top_level) { + try self.newline(); + try self.indent(); + } + + try self.writer.writeAll("\\\\"); + for (val) |c| { + if (c != '\r') { + try self.writer.writeByte(c); // We write newlines here even if whitespace off + if (c == '\n') { + try self.indent(); + try self.writer.writeAll("\\\\"); + } + } + } + + if (!options.top_level) { + try self.writer.writeByte('\n'); // Even if whitespace off + try self.indent(); + } + } + + /// Create a `Struct` for writing ZON structs field by field. + pub fn beginStruct( + self: *Serializer, + options: SerializeContainerOptions, + ) Error!Struct { + return Struct.begin(self, options); + } + + /// Creates a `Tuple` for writing ZON tuples field by field. + pub fn beginTuple( + self: *Serializer, + options: SerializeContainerOptions, + ) Error!Tuple { + return Tuple.begin(self, options); + } + + fn indent(self: *Serializer) Error!void { + if (self.options.whitespace) { + try self.writer.splatByteAll(' ', 4 * self.indent_level); + } + } + + fn newline(self: *Serializer) Error!void { + if (self.options.whitespace) { + try self.writer.writeByte('\n'); + } + } + + fn newlineOrSpace(self: *Serializer, len: usize) Error!void { + if (self.containerShouldWrap(len)) { + try self.newline(); + } else { + try self.space(); + } + } + + fn space(self: *Serializer) Error!void { + if (self.options.whitespace) { + try self.writer.writeByte(' '); + } + } + + /// Writes ZON tuples field by field. + pub const Tuple = struct { + container: Container, + + fn begin(parent: *Serializer, options: SerializeContainerOptions) Error!Tuple { return .{ - .options = options, - .writer = writer, - .indent_level = 0, + .container = try Container.begin(parent, .anon, options), }; } - /// Serialize a value, similar to `serialize`. - pub fn value(self: *Self, val: anytype, options: ValueOptions) Writer.Error!void { - comptime assert(!typeIsRecursive(@TypeOf(val))); - return self.valueArbitraryDepth(val, options); + /// Finishes serializing the tuple. + /// + /// Prints a trailing comma as configured when appropriate, and the closing bracket. + pub fn end(self: *Tuple) Error!void { + try self.container.end(); + self.* = undefined; } - /// Serialize a value, similar to `serializeMaxDepth`. - pub fn valueMaxDepth( - self: *Self, + /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `value`. + pub fn field( + self: *Tuple, val: anytype, options: ValueOptions, - depth: usize, - ) (Writer.Error || error{ExceededMaxDepth})!void { - try checkValueDepth(val, depth); - return self.valueArbitraryDepth(val, options); + ) Error!void { + try self.container.field(null, val, options); } - /// Serialize a value, similar to `serializeArbitraryDepth`. - pub fn valueArbitraryDepth( - self: *Self, - val: anytype, - options: ValueOptions, - ) Writer.Error!void { - comptime assert(canSerializeType(@TypeOf(val))); - switch (@typeInfo(@TypeOf(val))) { - .int, .comptime_int => if (options.emit_codepoint_literals.emitAsCodepoint(val)) |c| { - self.codePoint(c) catch |err| switch (err) { - error.InvalidCodepoint => unreachable, // Already validated - else => |e| return e, - }; - } else { - try self.int(val); - }, - .float, .comptime_float => try self.float(val), - .bool, .null => try std.fmt.format(self.writer, "{}", .{val}), - .enum_literal => try self.ident(@tagName(val)), - .@"enum" => try self.ident(@tagName(val)), - .pointer => |pointer| { - // Try to serialize as a string - const item: ?type = switch (@typeInfo(pointer.child)) { - .array => |array| array.child, - else => if (pointer.size == .slice) pointer.child else null, - }; - if (item == u8 and - (pointer.sentinel() == null or pointer.sentinel() == 0) and - !options.emit_strings_as_containers) - { - return try self.string(val); - } - - // Serialize as either a tuple or as the child type - switch (pointer.size) { - .slice => try self.tupleImpl(val, options), - .one => try self.valueArbitraryDepth(val.*, options), - else => comptime unreachable, - } - }, - .array => { - var container = try self.beginTuple( - .{ .whitespace_style = .{ .fields = val.len } }, - ); - for (val) |item_val| { - try container.fieldArbitraryDepth(item_val, options); - } - try container.end(); - }, - .@"struct" => |@"struct"| if (@"struct".is_tuple) { - var container = try self.beginTuple( - .{ .whitespace_style = .{ .fields = @"struct".fields.len } }, - ); - inline for (val) |field_value| { - try container.fieldArbitraryDepth(field_value, options); - } - try container.end(); - } else { - // Decide which fields to emit - const fields, const skipped: [@"struct".fields.len]bool = if (options.emit_default_optional_fields) b: { - break :b .{ @"struct".fields.len, @splat(false) }; - } else b: { - var fields = @"struct".fields.len; - var skipped: [@"struct".fields.len]bool = @splat(false); - inline for (@"struct".fields, &skipped) |field_info, *skip| { - if (field_info.default_value_ptr) |ptr| { - const default: *const field_info.type = @ptrCast(@alignCast(ptr)); - const field_value = @field(val, field_info.name); - if (std.meta.eql(field_value, default.*)) { - skip.* = true; - fields -= 1; - } - } - } - break :b .{ fields, skipped }; - }; - - // Emit those fields - var container = try self.beginStruct( - .{ .whitespace_style = .{ .fields = fields } }, - ); - inline for (@"struct".fields, skipped) |field_info, skip| { - if (!skip) { - try container.fieldArbitraryDepth( - field_info.name, - @field(val, field_info.name), - options, - ); - } - } - try container.end(); - }, - .@"union" => |@"union"| { - comptime assert(@"union".tag_type != null); - switch (val) { - inline else => |pl, tag| if (@TypeOf(pl) == void) - try self.writer.print(".{s}", .{@tagName(tag)}) - else { - var container = try self.beginStruct(.{ .whitespace_style = .{ .fields = 1 } }); - - try container.fieldArbitraryDepth( - @tagName(tag), - pl, - options, - ); - - try container.end(); - }, - } - }, - .optional => if (val) |inner| { - try self.valueArbitraryDepth(inner, options); - } else { - try self.writer.writeAll("null"); - }, - .vector => |vector| { - var container = try self.beginTuple( - .{ .whitespace_style = .{ .fields = vector.len } }, - ); - for (0..vector.len) |i| { - try container.fieldArbitraryDepth(val[i], options); - } - try container.end(); - }, - - else => comptime unreachable, - } - } - - /// Serialize an integer. - pub fn int(self: *Self, val: anytype) Writer.Error!void { - //try self.writer.printInt(val, 10, .lower, .{}); - try std.fmt.format(self.writer, "{d}", .{val}); - } - - /// Serialize a float. - pub fn float(self: *Self, val: anytype) Writer.Error!void { - switch (@typeInfo(@TypeOf(val))) { - .float => if (std.math.isNan(val)) { - return self.writer.writeAll("nan"); - } else if (std.math.isPositiveInf(val)) { - return self.writer.writeAll("inf"); - } else if (std.math.isNegativeInf(val)) { - return self.writer.writeAll("-inf"); - } else if (std.math.isNegativeZero(val)) { - return self.writer.writeAll("-0.0"); - } else { - try std.fmt.format(self.writer, "{d}", .{val}); - }, - .comptime_float => if (val == 0) { - return self.writer.writeAll("0"); - } else { - try std.fmt.format(self.writer, "{d}", .{val}); - }, - else => comptime unreachable, - } - } - - /// Serialize `name` as an identifier prefixed with `.`. - /// - /// Escapes the identifier if necessary. - pub fn ident(self: *Self, name: []const u8) Writer.Error!void { - try self.writer.print(".{f}", .{std.zig.fmtIdPU(name)}); - } - - /// Serialize `val` as a Unicode codepoint. - /// - /// Returns `error.InvalidCodepoint` if `val` is not a valid Unicode codepoint. - pub fn codePoint( - self: *Self, - val: u21, - ) (Writer.Error || error{InvalidCodepoint})!void { - var buf: [8]u8 = undefined; - const len = std.unicode.utf8Encode(val, &buf) catch return error.InvalidCodepoint; - const str = buf[0..len]; - try std.fmt.format(self.writer, "'{f}'", .{std.zig.fmtChar(str)}); - } - - /// Like `value`, but always serializes `val` as a tuple. - /// - /// Will fail at comptime if `val` is not a tuple, array, pointer to an array, or slice. - pub fn tuple(self: *Self, val: anytype, options: ValueOptions) Writer.Error!void { - comptime assert(!typeIsRecursive(@TypeOf(val))); - try self.tupleArbitraryDepth(val, options); - } - - /// Like `tuple`, but recursive types are allowed. - /// + /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `valueMaxDepth`. /// Returns `error.ExceededMaxDepth` if `depth` is exceeded. - pub fn tupleMaxDepth( - self: *Self, + pub fn fieldMaxDepth( + self: *Tuple, val: anytype, options: ValueOptions, depth: usize, - ) (Writer.Error || error{ExceededMaxDepth})!void { - try checkValueDepth(val, depth); - try self.tupleArbitraryDepth(val, options); + ) DepthError!void { + try self.container.fieldMaxDepth(null, val, options, depth); } - /// Like `tuple`, but recursive types are allowed. - /// - /// It is the caller's responsibility to ensure that `val` does not contain cycles. - pub fn tupleArbitraryDepth( - self: *Self, + /// Serialize a field. Equivalent to calling `fieldPrefix` followed by + /// `valueArbitraryDepth`. + pub fn fieldArbitraryDepth( + self: *Tuple, val: anytype, options: ValueOptions, - ) Writer.Error!void { - try self.tupleImpl(val, options); + ) Error!void { + try self.container.fieldArbitraryDepth(null, val, options); } - fn tupleImpl(self: *Self, val: anytype, options: ValueOptions) Writer.Error!void { - comptime assert(canSerializeType(@TypeOf(val))); - switch (@typeInfo(@TypeOf(val))) { - .@"struct" => { - var container = try self.beginTuple(.{ .whitespace_style = .{ .fields = val.len } }); - inline for (val) |item_val| { - try container.fieldArbitraryDepth(item_val, options); - } - try container.end(); - }, - .pointer, .array => { - var container = try self.beginTuple(.{ .whitespace_style = .{ .fields = val.len } }); - for (val) |item_val| { - try container.fieldArbitraryDepth(item_val, options); - } - try container.end(); - }, - else => comptime unreachable, - } + /// Starts a field with a struct as a value. Returns the struct. + pub fn beginStructField( + self: *Tuple, + options: SerializeContainerOptions, + ) Error!Struct { + try self.fieldPrefix(); + return self.container.serializer.beginStruct(options); } - /// Like `value`, but always serializes `val` as a string. - pub fn string(self: *Self, val: []const u8) Writer.Error!void { - try std.fmt.format(self.writer, "\"{f}\"", .{std.zig.fmtString(val)}); + /// Starts a field with a tuple as a value. Returns the tuple. + pub fn beginTupleField( + self: *Tuple, + options: SerializeContainerOptions, + ) Error!Tuple { + try self.fieldPrefix(); + return self.container.serializer.beginTuple(options); } - /// Options for formatting multiline strings. - pub const MultilineStringOptions = struct { - /// If top level is true, whitespace before and after the multiline string is elided. - /// If it is true, a newline is printed, then the value, followed by a newline, and if - /// whitespace is true any necessary indentation follows. - top_level: bool = false, - }; + /// Print a field prefix. This prints any necessary commas, and whitespace as + /// configured. Useful if you want to serialize the field value yourself. + pub fn fieldPrefix(self: *Tuple) Error!void { + try self.container.fieldPrefix(null); + } + }; - /// Like `value`, but always serializes to a multiline string literal. + /// Writes ZON structs field by field. + pub const Struct = struct { + container: Container, + + fn begin(parent: *Serializer, options: SerializeContainerOptions) Error!Struct { + return .{ + .container = try Container.begin(parent, .named, options), + }; + } + + /// Finishes serializing the struct. /// - /// Returns `error.InnerCarriageReturn` if `val` contains a CR not followed by a newline, - /// since multiline strings cannot represent CR without a following newline. - pub fn multilineString( - self: *Self, - val: []const u8, - options: MultilineStringOptions, - ) (Writer.Error || error{InnerCarriageReturn})!void { - // Make sure the string does not contain any carriage returns not followed by a newline - var i: usize = 0; - while (i < val.len) : (i += 1) { - if (val[i] == '\r') { - if (i + 1 < val.len) { - if (val[i + 1] == '\n') { - i += 1; - continue; - } - } - return error.InnerCarriageReturn; - } - } - - if (!options.top_level) { - try self.newline(); - try self.indent(); - } - - try self.writer.writeAll("\\\\"); - for (val) |c| { - if (c != '\r') { - try self.writer.writeByte(c); // We write newlines here even if whitespace off - if (c == '\n') { - try self.indent(); - try self.writer.writeAll("\\\\"); - } - } - } - - if (!options.top_level) { - try self.writer.writeByte('\n'); // Even if whitespace off - try self.indent(); - } + /// Prints a trailing comma as configured when appropriate, and the closing bracket. + pub fn end(self: *Struct) Error!void { + try self.container.end(); + self.* = undefined; } - /// Create a `Struct` for writing ZON structs field by field. - pub fn beginStruct( - self: *Self, + /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `value`. + pub fn field( + self: *Struct, + name: []const u8, + val: anytype, + options: ValueOptions, + ) Error!void { + try self.container.field(name, val, options); + } + + /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `valueMaxDepth`. + /// Returns `error.ExceededMaxDepth` if `depth` is exceeded. + pub fn fieldMaxDepth( + self: *Struct, + name: []const u8, + val: anytype, + options: ValueOptions, + depth: usize, + ) DepthError!void { + try self.container.fieldMaxDepth(name, val, options, depth); + } + + /// Serialize a field. Equivalent to calling `fieldPrefix` followed by + /// `valueArbitraryDepth`. + pub fn fieldArbitraryDepth( + self: *Struct, + name: []const u8, + val: anytype, + options: ValueOptions, + ) Error!void { + try self.container.fieldArbitraryDepth(name, val, options); + } + + /// Starts a field with a struct as a value. Returns the struct. + pub fn beginStructField( + self: *Struct, + name: []const u8, options: SerializeContainerOptions, - ) Writer.Error!Struct { - return Struct.begin(self, options); + ) Error!Struct { + try self.fieldPrefix(name); + return self.container.serializer.beginStruct(options); } - /// Creates a `Tuple` for writing ZON tuples field by field. - pub fn beginTuple( - self: *Self, + /// Starts a field with a tuple as a value. Returns the tuple. + pub fn beginTupleField( + self: *Struct, + name: []const u8, options: SerializeContainerOptions, - ) Writer.Error!Tuple { - return Tuple.begin(self, options); + ) Error!Tuple { + try self.fieldPrefix(name); + return self.container.serializer.beginTuple(options); } - fn indent(self: *Self) Writer.Error!void { - if (self.options.whitespace) { - try self.writer.writeByteNTimes(' ', 4 * self.indent_level); - } + /// Print a field prefix. This prints any necessary commas, the field name (escaped if + /// necessary) and whitespace as configured. Useful if you want to serialize the field + /// value yourself. + pub fn fieldPrefix(self: *Struct, name: []const u8) Error!void { + try self.container.fieldPrefix(name); } + }; - fn newline(self: *Self) Writer.Error!void { - if (self.options.whitespace) { - try self.writer.writeByte('\n'); - } - } + const Container = struct { + const FieldStyle = enum { named, anon }; - fn newlineOrSpace(self: *Self, len: usize) Writer.Error!void { - if (self.containerShouldWrap(len)) { - try self.newline(); - } else { - try self.space(); - } - } + serializer: *Serializer, + field_style: FieldStyle, + options: SerializeContainerOptions, + empty: bool, - fn space(self: *Self) Writer.Error!void { - if (self.options.whitespace) { - try self.writer.writeByte(' '); - } - } - - /// Writes ZON tuples field by field. - pub const Tuple = struct { - container: Container, - - fn begin(parent: *Self, options: SerializeContainerOptions) Writer.Error!Tuple { - return .{ - .container = try Container.begin(parent, .anon, options), - }; - } - - /// Finishes serializing the tuple. - /// - /// Prints a trailing comma as configured when appropriate, and the closing bracket. - pub fn end(self: *Tuple) Writer.Error!void { - try self.container.end(); - self.* = undefined; - } - - /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `value`. - pub fn field( - self: *Tuple, - val: anytype, - options: ValueOptions, - ) Writer.Error!void { - try self.container.field(null, val, options); - } - - /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `valueMaxDepth`. - pub fn fieldMaxDepth( - self: *Tuple, - val: anytype, - options: ValueOptions, - depth: usize, - ) (Writer.Error || error{ExceededMaxDepth})!void { - try self.container.fieldMaxDepth(null, val, options, depth); - } - - /// Serialize a field. Equivalent to calling `fieldPrefix` followed by - /// `valueArbitraryDepth`. - pub fn fieldArbitraryDepth( - self: *Tuple, - val: anytype, - options: ValueOptions, - ) Writer.Error!void { - try self.container.fieldArbitraryDepth(null, val, options); - } - - /// Starts a field with a struct as a value. Returns the struct. - pub fn beginStructField( - self: *Tuple, - options: SerializeContainerOptions, - ) Writer.Error!Struct { - try self.fieldPrefix(); - return self.container.serializer.beginStruct(options); - } - - /// Starts a field with a tuple as a value. Returns the tuple. - pub fn beginTupleField( - self: *Tuple, - options: SerializeContainerOptions, - ) Writer.Error!Tuple { - try self.fieldPrefix(); - return self.container.serializer.beginTuple(options); - } - - /// Print a field prefix. This prints any necessary commas, and whitespace as - /// configured. Useful if you want to serialize the field value yourself. - pub fn fieldPrefix(self: *Tuple) Writer.Error!void { - try self.container.fieldPrefix(null); - } - }; - - /// Writes ZON structs field by field. - pub const Struct = struct { - container: Container, - - fn begin(parent: *Self, options: SerializeContainerOptions) Writer.Error!Struct { - return .{ - .container = try Container.begin(parent, .named, options), - }; - } - - /// Finishes serializing the struct. - /// - /// Prints a trailing comma as configured when appropriate, and the closing bracket. - pub fn end(self: *Struct) Writer.Error!void { - try self.container.end(); - self.* = undefined; - } - - /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `value`. - pub fn field( - self: *Struct, - name: []const u8, - val: anytype, - options: ValueOptions, - ) Writer.Error!void { - try self.container.field(name, val, options); - } - - /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `valueMaxDepth`. - pub fn fieldMaxDepth( - self: *Struct, - name: []const u8, - val: anytype, - options: ValueOptions, - depth: usize, - ) (Writer.Error || error{ExceededMaxDepth})!void { - try self.container.fieldMaxDepth(name, val, options, depth); - } - - /// Serialize a field. Equivalent to calling `fieldPrefix` followed by - /// `valueArbitraryDepth`. - pub fn fieldArbitraryDepth( - self: *Struct, - name: []const u8, - val: anytype, - options: ValueOptions, - ) Writer.Error!void { - try self.container.fieldArbitraryDepth(name, val, options); - } - - /// Starts a field with a struct as a value. Returns the struct. - pub fn beginStructField( - self: *Struct, - name: []const u8, - options: SerializeContainerOptions, - ) Writer.Error!Struct { - try self.fieldPrefix(name); - return self.container.serializer.beginStruct(options); - } - - /// Starts a field with a tuple as a value. Returns the tuple. - pub fn beginTupleField( - self: *Struct, - name: []const u8, - options: SerializeContainerOptions, - ) Writer.Error!Tuple { - try self.fieldPrefix(name); - return self.container.serializer.beginTuple(options); - } - - /// Print a field prefix. This prints any necessary commas, the field name (escaped if - /// necessary) and whitespace as configured. Useful if you want to serialize the field - /// value yourself. - pub fn fieldPrefix(self: *Struct, name: []const u8) Writer.Error!void { - try self.container.fieldPrefix(name); - } - }; - - const Container = struct { - const FieldStyle = enum { named, anon }; - - serializer: *Self, + fn begin( + sz: *Serializer, field_style: FieldStyle, options: SerializeContainerOptions, - empty: bool, + ) Error!Container { + if (options.shouldWrap()) sz.indent_level +|= 1; + try sz.writer.writeAll(".{"); + return .{ + .serializer = sz, + .field_style = field_style, + .options = options, + .empty = true, + }; + } - fn begin( - sz: *Self, - field_style: FieldStyle, - options: SerializeContainerOptions, - ) Writer.Error!Container { - if (options.shouldWrap()) sz.indent_level +|= 1; - try sz.writer.writeAll(".{"); - return .{ - .serializer = sz, - .field_style = field_style, - .options = options, - .empty = true, - }; - } - - fn end(self: *Container) Writer.Error!void { - if (self.options.shouldWrap()) self.serializer.indent_level -|= 1; - if (!self.empty) { - if (self.options.shouldWrap()) { - if (self.serializer.options.whitespace) { - try self.serializer.writer.writeByte(','); - } - try self.serializer.newline(); - try self.serializer.indent(); - } else if (!self.shouldElideSpaces()) { - try self.serializer.space(); - } - } - try self.serializer.writer.writeByte('}'); - self.* = undefined; - } - - fn fieldPrefix(self: *Container, name: ?[]const u8) Writer.Error!void { - if (!self.empty) { - try self.serializer.writer.writeByte(','); - } - self.empty = false; + fn end(self: *Container) Error!void { + if (self.options.shouldWrap()) self.serializer.indent_level -|= 1; + if (!self.empty) { if (self.options.shouldWrap()) { + if (self.serializer.options.whitespace) { + try self.serializer.writer.writeByte(','); + } try self.serializer.newline(); + try self.serializer.indent(); } else if (!self.shouldElideSpaces()) { try self.serializer.space(); } - if (self.options.shouldWrap()) try self.serializer.indent(); - if (name) |n| { - try self.serializer.ident(n); - try self.serializer.space(); - try self.serializer.writer.writeByte('='); - try self.serializer.space(); - } } + try self.serializer.writer.writeByte('}'); + self.* = undefined; + } - fn field( - self: *Container, - name: ?[]const u8, - val: anytype, - options: ValueOptions, - ) Writer.Error!void { - comptime assert(!typeIsRecursive(@TypeOf(val))); - try self.fieldArbitraryDepth(name, val, options); + fn fieldPrefix(self: *Container, name: ?[]const u8) Error!void { + if (!self.empty) { + try self.serializer.writer.writeByte(','); } + self.empty = false; + if (self.options.shouldWrap()) { + try self.serializer.newline(); + } else if (!self.shouldElideSpaces()) { + try self.serializer.space(); + } + if (self.options.shouldWrap()) try self.serializer.indent(); + if (name) |n| { + try self.serializer.ident(n); + try self.serializer.space(); + try self.serializer.writer.writeByte('='); + try self.serializer.space(); + } + } - fn fieldMaxDepth( - self: *Container, - name: ?[]const u8, - val: anytype, - options: ValueOptions, - depth: usize, - ) (Writer.Error || error{ExceededMaxDepth})!void { - try checkValueDepth(val, depth); - try self.fieldArbitraryDepth(name, val, options); - } + fn field( + self: *Container, + name: ?[]const u8, + val: anytype, + options: ValueOptions, + ) Error!void { + comptime assert(!typeIsRecursive(@TypeOf(val))); + try self.fieldArbitraryDepth(name, val, options); + } - fn fieldArbitraryDepth( - self: *Container, - name: ?[]const u8, - val: anytype, - options: ValueOptions, - ) Writer.Error!void { - try self.fieldPrefix(name); - try self.serializer.valueArbitraryDepth(val, options); - } + /// Returns `error.ExceededMaxDepth` if `depth` is exceeded. + fn fieldMaxDepth( + self: *Container, + name: ?[]const u8, + val: anytype, + options: ValueOptions, + depth: usize, + ) DepthError!void { + try checkValueDepth(val, depth); + try self.fieldArbitraryDepth(name, val, options); + } - fn shouldElideSpaces(self: *const Container) bool { - return switch (self.options.whitespace_style) { - .fields => |fields| self.field_style != .named and fields == 1, - else => false, - }; - } - }; + fn fieldArbitraryDepth( + self: *Container, + name: ?[]const u8, + val: anytype, + options: ValueOptions, + ) Error!void { + try self.fieldPrefix(name); + try self.serializer.valueArbitraryDepth(val, options); + } + + fn shouldElideSpaces(self: *const Container) bool { + return switch (self.options.whitespace_style) { + .fields => |fields| self.field_style != .named and fields == 1, + else => false, + }; + } }; -} +}; -/// Creates a new `Serializer` with the given writer and options. -pub fn serializer(writer: anytype, options: SerializerOptions) Serializer(@TypeOf(writer)) { - return .init(writer, options); +test Serializer { + var discarding: Writer.Discarding = .init(&.{}); + var s: Serializer = .{ .writer = &discarding.writer }; + var vec2 = try s.beginStruct(.{}); + try vec2.field("x", 1.5, .{}); + try vec2.fieldPrefix("prefix"); + try s.value(2.5, .{}); + try vec2.end(); } fn expectSerializeEqual( @@ -1075,10 +1051,12 @@ fn expectSerializeEqual( value: anytype, options: SerializeOptions, ) !void { - var buf = std.ArrayList(u8).init(std.testing.allocator); - defer buf.deinit(); - try serialize(value, options, buf.writer()); - try std.testing.expectEqualStrings(expected, buf.items); + var aw: Writer.Allocating = .init(std.testing.allocator); + const bw = &aw.writer; + defer aw.deinit(); + + try serialize(value, options, bw); + try std.testing.expectEqualStrings(expected, aw.getWritten()); } test "std.zon stringify whitespace, high level API" { @@ -1175,59 +1153,59 @@ test "std.zon stringify whitespace, high level API" { } test "std.zon stringify whitespace, low level API" { - var buf = std.ArrayList(u8).init(std.testing.allocator); - defer buf.deinit(); - var sz = serializer(buf.writer(), .{}); + var aw: Writer.Allocating = .init(std.testing.allocator); + var s: Serializer = .{ .writer = &aw.writer }; + defer aw.deinit(); - inline for (.{ true, false }) |whitespace| { - sz.options = .{ .whitespace = whitespace }; + for ([2]bool{ true, false }) |whitespace| { + s.options = .{ .whitespace = whitespace }; // Empty containers { - var container = try sz.beginStruct(.{}); + var container = try s.beginStruct(.{}); try container.end(); - try std.testing.expectEqualStrings(".{}", buf.items); - buf.clearRetainingCapacity(); + try std.testing.expectEqualStrings(".{}", aw.getWritten()); + aw.clearRetainingCapacity(); } { - var container = try sz.beginTuple(.{}); + var container = try s.beginTuple(.{}); try container.end(); - try std.testing.expectEqualStrings(".{}", buf.items); - buf.clearRetainingCapacity(); + try std.testing.expectEqualStrings(".{}", aw.getWritten()); + aw.clearRetainingCapacity(); } { - var container = try sz.beginStruct(.{ .whitespace_style = .{ .wrap = false } }); + var container = try s.beginStruct(.{ .whitespace_style = .{ .wrap = false } }); try container.end(); - try std.testing.expectEqualStrings(".{}", buf.items); - buf.clearRetainingCapacity(); + try std.testing.expectEqualStrings(".{}", aw.getWritten()); + aw.clearRetainingCapacity(); } { - var container = try sz.beginTuple(.{ .whitespace_style = .{ .wrap = false } }); + var container = try s.beginTuple(.{ .whitespace_style = .{ .wrap = false } }); try container.end(); - try std.testing.expectEqualStrings(".{}", buf.items); - buf.clearRetainingCapacity(); + try std.testing.expectEqualStrings(".{}", aw.getWritten()); + aw.clearRetainingCapacity(); } { - var container = try sz.beginStruct(.{ .whitespace_style = .{ .fields = 0 } }); + var container = try s.beginStruct(.{ .whitespace_style = .{ .fields = 0 } }); try container.end(); - try std.testing.expectEqualStrings(".{}", buf.items); - buf.clearRetainingCapacity(); + try std.testing.expectEqualStrings(".{}", aw.getWritten()); + aw.clearRetainingCapacity(); } { - var container = try sz.beginTuple(.{ .whitespace_style = .{ .fields = 0 } }); + var container = try s.beginTuple(.{ .whitespace_style = .{ .fields = 0 } }); try container.end(); - try std.testing.expectEqualStrings(".{}", buf.items); - buf.clearRetainingCapacity(); + try std.testing.expectEqualStrings(".{}", aw.getWritten()); + aw.clearRetainingCapacity(); } // Size 1 { - var container = try sz.beginStruct(.{}); + var container = try s.beginStruct(.{}); try container.field("a", 1, .{}); try container.end(); if (whitespace) { @@ -1235,15 +1213,15 @@ test "std.zon stringify whitespace, low level API" { \\.{ \\ .a = 1, \\} - , buf.items); + , aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{.a=1}", buf.items); + try std.testing.expectEqualStrings(".{.a=1}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { - var container = try sz.beginTuple(.{}); + var container = try s.beginTuple(.{}); try container.field(1, .{}); try container.end(); if (whitespace) { @@ -1251,62 +1229,62 @@ test "std.zon stringify whitespace, low level API" { \\.{ \\ 1, \\} - , buf.items); + , aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{1}", buf.items); + try std.testing.expectEqualStrings(".{1}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { - var container = try sz.beginStruct(.{ .whitespace_style = .{ .wrap = false } }); + var container = try s.beginStruct(.{ .whitespace_style = .{ .wrap = false } }); try container.field("a", 1, .{}); try container.end(); if (whitespace) { - try std.testing.expectEqualStrings(".{ .a = 1 }", buf.items); + try std.testing.expectEqualStrings(".{ .a = 1 }", aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{.a=1}", buf.items); + try std.testing.expectEqualStrings(".{.a=1}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { // We get extra spaces here, since we didn't know up front that there would only be one // field. - var container = try sz.beginTuple(.{ .whitespace_style = .{ .wrap = false } }); + var container = try s.beginTuple(.{ .whitespace_style = .{ .wrap = false } }); try container.field(1, .{}); try container.end(); if (whitespace) { - try std.testing.expectEqualStrings(".{ 1 }", buf.items); + try std.testing.expectEqualStrings(".{ 1 }", aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{1}", buf.items); + try std.testing.expectEqualStrings(".{1}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { - var container = try sz.beginStruct(.{ .whitespace_style = .{ .fields = 1 } }); + var container = try s.beginStruct(.{ .whitespace_style = .{ .fields = 1 } }); try container.field("a", 1, .{}); try container.end(); if (whitespace) { - try std.testing.expectEqualStrings(".{ .a = 1 }", buf.items); + try std.testing.expectEqualStrings(".{ .a = 1 }", aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{.a=1}", buf.items); + try std.testing.expectEqualStrings(".{.a=1}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { - var container = try sz.beginTuple(.{ .whitespace_style = .{ .fields = 1 } }); + var container = try s.beginTuple(.{ .whitespace_style = .{ .fields = 1 } }); try container.field(1, .{}); try container.end(); - try std.testing.expectEqualStrings(".{1}", buf.items); - buf.clearRetainingCapacity(); + try std.testing.expectEqualStrings(".{1}", aw.getWritten()); + aw.clearRetainingCapacity(); } // Size 2 { - var container = try sz.beginStruct(.{}); + var container = try s.beginStruct(.{}); try container.field("a", 1, .{}); try container.field("b", 2, .{}); try container.end(); @@ -1316,15 +1294,15 @@ test "std.zon stringify whitespace, low level API" { \\ .a = 1, \\ .b = 2, \\} - , buf.items); + , aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{.a=1,.b=2}", buf.items); + try std.testing.expectEqualStrings(".{.a=1,.b=2}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { - var container = try sz.beginTuple(.{}); + var container = try s.beginTuple(.{}); try container.field(1, .{}); try container.field(2, .{}); try container.end(); @@ -1334,68 +1312,68 @@ test "std.zon stringify whitespace, low level API" { \\ 1, \\ 2, \\} - , buf.items); + , aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{1,2}", buf.items); + try std.testing.expectEqualStrings(".{1,2}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { - var container = try sz.beginStruct(.{ .whitespace_style = .{ .wrap = false } }); + var container = try s.beginStruct(.{ .whitespace_style = .{ .wrap = false } }); try container.field("a", 1, .{}); try container.field("b", 2, .{}); try container.end(); if (whitespace) { - try std.testing.expectEqualStrings(".{ .a = 1, .b = 2 }", buf.items); + try std.testing.expectEqualStrings(".{ .a = 1, .b = 2 }", aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{.a=1,.b=2}", buf.items); + try std.testing.expectEqualStrings(".{.a=1,.b=2}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { - var container = try sz.beginTuple(.{ .whitespace_style = .{ .wrap = false } }); + var container = try s.beginTuple(.{ .whitespace_style = .{ .wrap = false } }); try container.field(1, .{}); try container.field(2, .{}); try container.end(); if (whitespace) { - try std.testing.expectEqualStrings(".{ 1, 2 }", buf.items); + try std.testing.expectEqualStrings(".{ 1, 2 }", aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{1,2}", buf.items); + try std.testing.expectEqualStrings(".{1,2}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { - var container = try sz.beginStruct(.{ .whitespace_style = .{ .fields = 2 } }); + var container = try s.beginStruct(.{ .whitespace_style = .{ .fields = 2 } }); try container.field("a", 1, .{}); try container.field("b", 2, .{}); try container.end(); if (whitespace) { - try std.testing.expectEqualStrings(".{ .a = 1, .b = 2 }", buf.items); + try std.testing.expectEqualStrings(".{ .a = 1, .b = 2 }", aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{.a=1,.b=2}", buf.items); + try std.testing.expectEqualStrings(".{.a=1,.b=2}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { - var container = try sz.beginTuple(.{ .whitespace_style = .{ .fields = 2 } }); + var container = try s.beginTuple(.{ .whitespace_style = .{ .fields = 2 } }); try container.field(1, .{}); try container.field(2, .{}); try container.end(); if (whitespace) { - try std.testing.expectEqualStrings(".{ 1, 2 }", buf.items); + try std.testing.expectEqualStrings(".{ 1, 2 }", aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{1,2}", buf.items); + try std.testing.expectEqualStrings(".{1,2}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } // Size 3 { - var container = try sz.beginStruct(.{}); + var container = try s.beginStruct(.{}); try container.field("a", 1, .{}); try container.field("b", 2, .{}); try container.field("c", 3, .{}); @@ -1407,15 +1385,15 @@ test "std.zon stringify whitespace, low level API" { \\ .b = 2, \\ .c = 3, \\} - , buf.items); + , aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{.a=1,.b=2,.c=3}", buf.items); + try std.testing.expectEqualStrings(".{.a=1,.b=2,.c=3}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { - var container = try sz.beginTuple(.{}); + var container = try s.beginTuple(.{}); try container.field(1, .{}); try container.field(2, .{}); try container.field(3, .{}); @@ -1427,43 +1405,43 @@ test "std.zon stringify whitespace, low level API" { \\ 2, \\ 3, \\} - , buf.items); + , aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{1,2,3}", buf.items); + try std.testing.expectEqualStrings(".{1,2,3}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { - var container = try sz.beginStruct(.{ .whitespace_style = .{ .wrap = false } }); + var container = try s.beginStruct(.{ .whitespace_style = .{ .wrap = false } }); try container.field("a", 1, .{}); try container.field("b", 2, .{}); try container.field("c", 3, .{}); try container.end(); if (whitespace) { - try std.testing.expectEqualStrings(".{ .a = 1, .b = 2, .c = 3 }", buf.items); + try std.testing.expectEqualStrings(".{ .a = 1, .b = 2, .c = 3 }", aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{.a=1,.b=2,.c=3}", buf.items); + try std.testing.expectEqualStrings(".{.a=1,.b=2,.c=3}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { - var container = try sz.beginTuple(.{ .whitespace_style = .{ .wrap = false } }); + var container = try s.beginTuple(.{ .whitespace_style = .{ .wrap = false } }); try container.field(1, .{}); try container.field(2, .{}); try container.field(3, .{}); try container.end(); if (whitespace) { - try std.testing.expectEqualStrings(".{ 1, 2, 3 }", buf.items); + try std.testing.expectEqualStrings(".{ 1, 2, 3 }", aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{1,2,3}", buf.items); + try std.testing.expectEqualStrings(".{1,2,3}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { - var container = try sz.beginStruct(.{ .whitespace_style = .{ .fields = 3 } }); + var container = try s.beginStruct(.{ .whitespace_style = .{ .fields = 3 } }); try container.field("a", 1, .{}); try container.field("b", 2, .{}); try container.field("c", 3, .{}); @@ -1475,15 +1453,15 @@ test "std.zon stringify whitespace, low level API" { \\ .b = 2, \\ .c = 3, \\} - , buf.items); + , aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{.a=1,.b=2,.c=3}", buf.items); + try std.testing.expectEqualStrings(".{.a=1,.b=2,.c=3}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { - var container = try sz.beginTuple(.{ .whitespace_style = .{ .fields = 3 } }); + var container = try s.beginTuple(.{ .whitespace_style = .{ .fields = 3 } }); try container.field(1, .{}); try container.field(2, .{}); try container.field(3, .{}); @@ -1495,16 +1473,16 @@ test "std.zon stringify whitespace, low level API" { \\ 2, \\ 3, \\} - , buf.items); + , aw.getWritten()); } else { - try std.testing.expectEqualStrings(".{1,2,3}", buf.items); + try std.testing.expectEqualStrings(".{1,2,3}", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } // Nested objects where the outer container doesn't wrap but the inner containers do { - var container = try sz.beginStruct(.{ .whitespace_style = .{ .wrap = false } }); + var container = try s.beginStruct(.{ .whitespace_style = .{ .wrap = false } }); try container.field("first", .{ 1, 2, 3 }, .{}); try container.field("second", .{ 4, 5, 6 }, .{}); try container.end(); @@ -1519,139 +1497,141 @@ test "std.zon stringify whitespace, low level API" { \\ 5, \\ 6, \\} } - , buf.items); + , aw.getWritten()); } else { try std.testing.expectEqualStrings( ".{.first=.{1,2,3},.second=.{4,5,6}}", - buf.items, + aw.getWritten(), ); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } } } test "std.zon stringify utf8 codepoints" { - var buf = std.ArrayList(u8).init(std.testing.allocator); - defer buf.deinit(); - var sz = serializer(buf.writer(), .{}); + var aw: Writer.Allocating = .init(std.testing.allocator); + var s: Serializer = .{ .writer = &aw.writer }; + defer aw.deinit(); // Printable ASCII - try sz.int('a'); - try std.testing.expectEqualStrings("97", buf.items); - buf.clearRetainingCapacity(); + try s.int('a'); + try std.testing.expectEqualStrings("97", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.codePoint('a'); - try std.testing.expectEqualStrings("'a'", buf.items); - buf.clearRetainingCapacity(); + try s.codePoint('a'); + try std.testing.expectEqualStrings("'a'", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value('a', .{ .emit_codepoint_literals = .always }); - try std.testing.expectEqualStrings("'a'", buf.items); - buf.clearRetainingCapacity(); + try s.value('a', .{ .emit_codepoint_literals = .always }); + try std.testing.expectEqualStrings("'a'", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value('a', .{ .emit_codepoint_literals = .printable_ascii }); - try std.testing.expectEqualStrings("'a'", buf.items); - buf.clearRetainingCapacity(); + try s.value('a', .{ .emit_codepoint_literals = .printable_ascii }); + try std.testing.expectEqualStrings("'a'", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value('a', .{ .emit_codepoint_literals = .never }); - try std.testing.expectEqualStrings("97", buf.items); - buf.clearRetainingCapacity(); + try s.value('a', .{ .emit_codepoint_literals = .never }); + try std.testing.expectEqualStrings("97", aw.getWritten()); + aw.clearRetainingCapacity(); // Short escaped codepoint - try sz.int('\n'); - try std.testing.expectEqualStrings("10", buf.items); - buf.clearRetainingCapacity(); + try s.int('\n'); + try std.testing.expectEqualStrings("10", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.codePoint('\n'); - try std.testing.expectEqualStrings("'\\n'", buf.items); - buf.clearRetainingCapacity(); + try s.codePoint('\n'); + try std.testing.expectEqualStrings("'\\n'", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value('\n', .{ .emit_codepoint_literals = .always }); - try std.testing.expectEqualStrings("'\\n'", buf.items); - buf.clearRetainingCapacity(); + try s.value('\n', .{ .emit_codepoint_literals = .always }); + try std.testing.expectEqualStrings("'\\n'", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value('\n', .{ .emit_codepoint_literals = .printable_ascii }); - try std.testing.expectEqualStrings("10", buf.items); - buf.clearRetainingCapacity(); + try s.value('\n', .{ .emit_codepoint_literals = .printable_ascii }); + try std.testing.expectEqualStrings("10", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value('\n', .{ .emit_codepoint_literals = .never }); - try std.testing.expectEqualStrings("10", buf.items); - buf.clearRetainingCapacity(); + try s.value('\n', .{ .emit_codepoint_literals = .never }); + try std.testing.expectEqualStrings("10", aw.getWritten()); + aw.clearRetainingCapacity(); // Large codepoint - try sz.int('âš¡'); - try std.testing.expectEqualStrings("9889", buf.items); - buf.clearRetainingCapacity(); + try s.int('âš¡'); + try std.testing.expectEqualStrings("9889", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.codePoint('âš¡'); - try std.testing.expectEqualStrings("'\\xe2\\x9a\\xa1'", buf.items); - buf.clearRetainingCapacity(); + try s.codePoint('âš¡'); + try std.testing.expectEqualStrings("'\\u{26a1}'", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value('âš¡', .{ .emit_codepoint_literals = .always }); - try std.testing.expectEqualStrings("'\\xe2\\x9a\\xa1'", buf.items); - buf.clearRetainingCapacity(); + try s.value('âš¡', .{ .emit_codepoint_literals = .always }); + try std.testing.expectEqualStrings("'\\u{26a1}'", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value('âš¡', .{ .emit_codepoint_literals = .printable_ascii }); - try std.testing.expectEqualStrings("9889", buf.items); - buf.clearRetainingCapacity(); + try s.value('âš¡', .{ .emit_codepoint_literals = .printable_ascii }); + try std.testing.expectEqualStrings("9889", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value('âš¡', .{ .emit_codepoint_literals = .never }); - try std.testing.expectEqualStrings("9889", buf.items); - buf.clearRetainingCapacity(); + try s.value('âš¡', .{ .emit_codepoint_literals = .never }); + try std.testing.expectEqualStrings("9889", aw.getWritten()); + aw.clearRetainingCapacity(); // Invalid codepoint - try std.testing.expectError(error.InvalidCodepoint, sz.codePoint(0x110000 + 1)); + try s.codePoint(0x110000 + 1); + try std.testing.expectEqualStrings("'\\u{110001}'", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.int(0x110000 + 1); - try std.testing.expectEqualStrings("1114113", buf.items); - buf.clearRetainingCapacity(); + try s.int(0x110000 + 1); + try std.testing.expectEqualStrings("1114113", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value(0x110000 + 1, .{ .emit_codepoint_literals = .always }); - try std.testing.expectEqualStrings("1114113", buf.items); - buf.clearRetainingCapacity(); + try s.value(0x110000 + 1, .{ .emit_codepoint_literals = .always }); + try std.testing.expectEqualStrings("1114113", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value(0x110000 + 1, .{ .emit_codepoint_literals = .printable_ascii }); - try std.testing.expectEqualStrings("1114113", buf.items); - buf.clearRetainingCapacity(); + try s.value(0x110000 + 1, .{ .emit_codepoint_literals = .printable_ascii }); + try std.testing.expectEqualStrings("1114113", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value(0x110000 + 1, .{ .emit_codepoint_literals = .never }); - try std.testing.expectEqualStrings("1114113", buf.items); - buf.clearRetainingCapacity(); + try s.value(0x110000 + 1, .{ .emit_codepoint_literals = .never }); + try std.testing.expectEqualStrings("1114113", aw.getWritten()); + aw.clearRetainingCapacity(); // Valid codepoint, not a codepoint type - try sz.value(@as(u22, 'a'), .{ .emit_codepoint_literals = .always }); - try std.testing.expectEqualStrings("97", buf.items); - buf.clearRetainingCapacity(); + try s.value(@as(u22, 'a'), .{ .emit_codepoint_literals = .always }); + try std.testing.expectEqualStrings("97", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value(@as(u22, 'a'), .{ .emit_codepoint_literals = .printable_ascii }); - try std.testing.expectEqualStrings("97", buf.items); - buf.clearRetainingCapacity(); + try s.value(@as(u22, 'a'), .{ .emit_codepoint_literals = .printable_ascii }); + try std.testing.expectEqualStrings("97", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value(@as(i32, 'a'), .{ .emit_codepoint_literals = .never }); - try std.testing.expectEqualStrings("97", buf.items); - buf.clearRetainingCapacity(); + try s.value(@as(i32, 'a'), .{ .emit_codepoint_literals = .never }); + try std.testing.expectEqualStrings("97", aw.getWritten()); + aw.clearRetainingCapacity(); // Make sure value options are passed to children - try sz.value(.{ .c = 'âš¡' }, .{ .emit_codepoint_literals = .always }); - try std.testing.expectEqualStrings(".{ .c = '\\xe2\\x9a\\xa1' }", buf.items); - buf.clearRetainingCapacity(); + try s.value(.{ .c = 'âš¡' }, .{ .emit_codepoint_literals = .always }); + try std.testing.expectEqualStrings(".{ .c = '\\u{26a1}' }", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value(.{ .c = 'âš¡' }, .{ .emit_codepoint_literals = .never }); - try std.testing.expectEqualStrings(".{ .c = 9889 }", buf.items); - buf.clearRetainingCapacity(); + try s.value(.{ .c = 'âš¡' }, .{ .emit_codepoint_literals = .never }); + try std.testing.expectEqualStrings(".{ .c = 9889 }", aw.getWritten()); + aw.clearRetainingCapacity(); } test "std.zon stringify strings" { - var buf = std.ArrayList(u8).init(std.testing.allocator); - defer buf.deinit(); - var sz = serializer(buf.writer(), .{}); + var aw: Writer.Allocating = .init(std.testing.allocator); + var s: Serializer = .{ .writer = &aw.writer }; + defer aw.deinit(); // Minimal case - try sz.string("abcâš¡\n"); - try std.testing.expectEqualStrings("\"abc\\xe2\\x9a\\xa1\\n\"", buf.items); - buf.clearRetainingCapacity(); + try s.string("abcâš¡\n"); + try std.testing.expectEqualStrings("\"abc\\xe2\\x9a\\xa1\\n\"", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.tuple("abcâš¡\n", .{}); + try s.tuple("abcâš¡\n", .{}); try std.testing.expectEqualStrings( \\.{ \\ 97, @@ -1662,14 +1642,14 @@ test "std.zon stringify strings" { \\ 161, \\ 10, \\} - , buf.items); - buf.clearRetainingCapacity(); + , aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value("abcâš¡\n", .{}); - try std.testing.expectEqualStrings("\"abc\\xe2\\x9a\\xa1\\n\"", buf.items); - buf.clearRetainingCapacity(); + try s.value("abcâš¡\n", .{}); + try std.testing.expectEqualStrings("\"abc\\xe2\\x9a\\xa1\\n\"", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value("abcâš¡\n", .{ .emit_strings_as_containers = true }); + try s.value("abcâš¡\n", .{ .emit_strings_as_containers = true }); try std.testing.expectEqualStrings( \\.{ \\ 97, @@ -1680,113 +1660,113 @@ test "std.zon stringify strings" { \\ 161, \\ 10, \\} - , buf.items); - buf.clearRetainingCapacity(); + , aw.getWritten()); + aw.clearRetainingCapacity(); // Value options are inherited by children - try sz.value(.{ .str = "abc" }, .{}); - try std.testing.expectEqualStrings(".{ .str = \"abc\" }", buf.items); - buf.clearRetainingCapacity(); + try s.value(.{ .str = "abc" }, .{}); + try std.testing.expectEqualStrings(".{ .str = \"abc\" }", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.value(.{ .str = "abc" }, .{ .emit_strings_as_containers = true }); + try s.value(.{ .str = "abc" }, .{ .emit_strings_as_containers = true }); try std.testing.expectEqualStrings( \\.{ .str = .{ \\ 97, \\ 98, \\ 99, \\} } - , buf.items); - buf.clearRetainingCapacity(); + , aw.getWritten()); + aw.clearRetainingCapacity(); // Arrays (rather than pointers to arrays) of u8s are not considered strings, so that data can // round trip correctly. - try sz.value("abc".*, .{}); + try s.value("abc".*, .{}); try std.testing.expectEqualStrings( \\.{ \\ 97, \\ 98, \\ 99, \\} - , buf.items); - buf.clearRetainingCapacity(); + , aw.getWritten()); + aw.clearRetainingCapacity(); } test "std.zon stringify multiline strings" { - var buf = std.ArrayList(u8).init(std.testing.allocator); - defer buf.deinit(); - var sz = serializer(buf.writer(), .{}); + var aw: Writer.Allocating = .init(std.testing.allocator); + var s: Serializer = .{ .writer = &aw.writer }; + defer aw.deinit(); inline for (.{ true, false }) |whitespace| { - sz.options.whitespace = whitespace; + s.options.whitespace = whitespace; { - try sz.multilineString("", .{ .top_level = true }); - try std.testing.expectEqualStrings("\\\\", buf.items); - buf.clearRetainingCapacity(); + try s.multilineString("", .{ .top_level = true }); + try std.testing.expectEqualStrings("\\\\", aw.getWritten()); + aw.clearRetainingCapacity(); } { - try sz.multilineString("abcâš¡", .{ .top_level = true }); - try std.testing.expectEqualStrings("\\\\abcâš¡", buf.items); - buf.clearRetainingCapacity(); + try s.multilineString("abcâš¡", .{ .top_level = true }); + try std.testing.expectEqualStrings("\\\\abcâš¡", aw.getWritten()); + aw.clearRetainingCapacity(); } { - try sz.multilineString("abcâš¡\ndef", .{ .top_level = true }); - try std.testing.expectEqualStrings("\\\\abcâš¡\n\\\\def", buf.items); - buf.clearRetainingCapacity(); + try s.multilineString("abcâš¡\ndef", .{ .top_level = true }); + try std.testing.expectEqualStrings("\\\\abcâš¡\n\\\\def", aw.getWritten()); + aw.clearRetainingCapacity(); } { - try sz.multilineString("abcâš¡\r\ndef", .{ .top_level = true }); - try std.testing.expectEqualStrings("\\\\abcâš¡\n\\\\def", buf.items); - buf.clearRetainingCapacity(); + try s.multilineString("abcâš¡\r\ndef", .{ .top_level = true }); + try std.testing.expectEqualStrings("\\\\abcâš¡\n\\\\def", aw.getWritten()); + aw.clearRetainingCapacity(); } { - try sz.multilineString("\nabcâš¡", .{ .top_level = true }); - try std.testing.expectEqualStrings("\\\\\n\\\\abcâš¡", buf.items); - buf.clearRetainingCapacity(); + try s.multilineString("\nabcâš¡", .{ .top_level = true }); + try std.testing.expectEqualStrings("\\\\\n\\\\abcâš¡", aw.getWritten()); + aw.clearRetainingCapacity(); } { - try sz.multilineString("\r\nabcâš¡", .{ .top_level = true }); - try std.testing.expectEqualStrings("\\\\\n\\\\abcâš¡", buf.items); - buf.clearRetainingCapacity(); + try s.multilineString("\r\nabcâš¡", .{ .top_level = true }); + try std.testing.expectEqualStrings("\\\\\n\\\\abcâš¡", aw.getWritten()); + aw.clearRetainingCapacity(); } { - try sz.multilineString("abc\ndef", .{}); + try s.multilineString("abc\ndef", .{}); if (whitespace) { - try std.testing.expectEqualStrings("\n\\\\abc\n\\\\def\n", buf.items); + try std.testing.expectEqualStrings("\n\\\\abc\n\\\\def\n", aw.getWritten()); } else { - try std.testing.expectEqualStrings("\\\\abc\n\\\\def\n", buf.items); + try std.testing.expectEqualStrings("\\\\abc\n\\\\def\n", aw.getWritten()); } - buf.clearRetainingCapacity(); + aw.clearRetainingCapacity(); } { const str: []const u8 = &.{ 'a', '\r', 'c' }; - try sz.string(str); - try std.testing.expectEqualStrings("\"a\\rc\"", buf.items); - buf.clearRetainingCapacity(); + try s.string(str); + try std.testing.expectEqualStrings("\"a\\rc\"", aw.getWritten()); + aw.clearRetainingCapacity(); } { try std.testing.expectError( error.InnerCarriageReturn, - sz.multilineString(@as([]const u8, &.{ 'a', '\r', 'c' }), .{}), + s.multilineString(@as([]const u8, &.{ 'a', '\r', 'c' }), .{}), ); try std.testing.expectError( error.InnerCarriageReturn, - sz.multilineString(@as([]const u8, &.{ 'a', '\r', 'c', '\n' }), .{}), + s.multilineString(@as([]const u8, &.{ 'a', '\r', 'c', '\n' }), .{}), ); try std.testing.expectError( error.InnerCarriageReturn, - sz.multilineString(@as([]const u8, &.{ 'a', '\r', 'c', '\r', '\n' }), .{}), + s.multilineString(@as([]const u8, &.{ 'a', '\r', 'c', '\r', '\n' }), .{}), ); - try std.testing.expectEqualStrings("", buf.items); - buf.clearRetainingCapacity(); + try std.testing.expectEqualStrings("", aw.getWritten()); + aw.clearRetainingCapacity(); } } } @@ -1932,42 +1912,43 @@ test "std.zon stringify skip default fields" { } test "std.zon depth limits" { - var buf = std.ArrayList(u8).init(std.testing.allocator); - defer buf.deinit(); + var aw: Writer.Allocating = .init(std.testing.allocator); + const bw = &aw.writer; + defer aw.deinit(); const Recurse = struct { r: []const @This() }; // Normal operation - try serializeMaxDepth(.{ 1, .{ 2, 3 } }, .{}, buf.writer(), 16); - try std.testing.expectEqualStrings(".{ 1, .{ 2, 3 } }", buf.items); - buf.clearRetainingCapacity(); + try serializeMaxDepth(.{ 1, .{ 2, 3 } }, .{}, bw, 16); + try std.testing.expectEqualStrings(".{ 1, .{ 2, 3 } }", aw.getWritten()); + aw.clearRetainingCapacity(); - try serializeArbitraryDepth(.{ 1, .{ 2, 3 } }, .{}, buf.writer()); - try std.testing.expectEqualStrings(".{ 1, .{ 2, 3 } }", buf.items); - buf.clearRetainingCapacity(); + try serializeArbitraryDepth(.{ 1, .{ 2, 3 } }, .{}, bw); + try std.testing.expectEqualStrings(".{ 1, .{ 2, 3 } }", aw.getWritten()); + aw.clearRetainingCapacity(); // Max depth failing on non recursive type try std.testing.expectError( error.ExceededMaxDepth, - serializeMaxDepth(.{ 1, .{ 2, .{ 3, 4 } } }, .{}, buf.writer(), 3), + serializeMaxDepth(.{ 1, .{ 2, .{ 3, 4 } } }, .{}, bw, 3), ); - try std.testing.expectEqualStrings("", buf.items); - buf.clearRetainingCapacity(); + try std.testing.expectEqualStrings("", aw.getWritten()); + aw.clearRetainingCapacity(); // Max depth passing on recursive type { const maybe_recurse = Recurse{ .r = &.{} }; - try serializeMaxDepth(maybe_recurse, .{}, buf.writer(), 2); - try std.testing.expectEqualStrings(".{ .r = .{} }", buf.items); - buf.clearRetainingCapacity(); + try serializeMaxDepth(maybe_recurse, .{}, bw, 2); + try std.testing.expectEqualStrings(".{ .r = .{} }", aw.getWritten()); + aw.clearRetainingCapacity(); } // Unchecked passing on recursive type { const maybe_recurse = Recurse{ .r = &.{} }; - try serializeArbitraryDepth(maybe_recurse, .{}, buf.writer()); - try std.testing.expectEqualStrings(".{ .r = .{} }", buf.items); - buf.clearRetainingCapacity(); + try serializeArbitraryDepth(maybe_recurse, .{}, bw); + try std.testing.expectEqualStrings(".{ .r = .{} }", aw.getWritten()); + aw.clearRetainingCapacity(); } // Max depth failing on recursive type due to depth @@ -1976,10 +1957,10 @@ test "std.zon depth limits" { maybe_recurse.r = &.{.{ .r = &.{} }}; try std.testing.expectError( error.ExceededMaxDepth, - serializeMaxDepth(maybe_recurse, .{}, buf.writer(), 2), + serializeMaxDepth(maybe_recurse, .{}, bw, 2), ); - try std.testing.expectEqualStrings("", buf.items); - buf.clearRetainingCapacity(); + try std.testing.expectEqualStrings("", aw.getWritten()); + aw.clearRetainingCapacity(); } // Same but for a slice @@ -1989,23 +1970,23 @@ test "std.zon depth limits" { try std.testing.expectError( error.ExceededMaxDepth, - serializeMaxDepth(maybe_recurse, .{}, buf.writer(), 2), + serializeMaxDepth(maybe_recurse, .{}, bw, 2), ); - try std.testing.expectEqualStrings("", buf.items); - buf.clearRetainingCapacity(); + try std.testing.expectEqualStrings("", aw.getWritten()); + aw.clearRetainingCapacity(); - var sz = serializer(buf.writer(), .{}); + var s: Serializer = .{ .writer = bw }; try std.testing.expectError( error.ExceededMaxDepth, - sz.tupleMaxDepth(maybe_recurse, .{}, 2), + s.tupleMaxDepth(maybe_recurse, .{}, 2), ); - try std.testing.expectEqualStrings("", buf.items); - buf.clearRetainingCapacity(); + try std.testing.expectEqualStrings("", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.tupleArbitraryDepth(maybe_recurse, .{}); - try std.testing.expectEqualStrings(".{.{ .r = .{} }}", buf.items); - buf.clearRetainingCapacity(); + try s.tupleArbitraryDepth(maybe_recurse, .{}); + try std.testing.expectEqualStrings(".{.{ .r = .{} }}", aw.getWritten()); + aw.clearRetainingCapacity(); } // A slice succeeding @@ -2013,19 +1994,19 @@ test "std.zon depth limits" { var temp: [1]Recurse = .{.{ .r = &.{} }}; const maybe_recurse: []const Recurse = &temp; - try serializeMaxDepth(maybe_recurse, .{}, buf.writer(), 3); - try std.testing.expectEqualStrings(".{.{ .r = .{} }}", buf.items); - buf.clearRetainingCapacity(); + try serializeMaxDepth(maybe_recurse, .{}, bw, 3); + try std.testing.expectEqualStrings(".{.{ .r = .{} }}", aw.getWritten()); + aw.clearRetainingCapacity(); - var sz = serializer(buf.writer(), .{}); + var s: Serializer = .{ .writer = bw }; - try sz.tupleMaxDepth(maybe_recurse, .{}, 3); - try std.testing.expectEqualStrings(".{.{ .r = .{} }}", buf.items); - buf.clearRetainingCapacity(); + try s.tupleMaxDepth(maybe_recurse, .{}, 3); + try std.testing.expectEqualStrings(".{.{ .r = .{} }}", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.tupleArbitraryDepth(maybe_recurse, .{}); - try std.testing.expectEqualStrings(".{.{ .r = .{} }}", buf.items); - buf.clearRetainingCapacity(); + try s.tupleArbitraryDepth(maybe_recurse, .{}); + try std.testing.expectEqualStrings(".{.{ .r = .{} }}", aw.getWritten()); + aw.clearRetainingCapacity(); } // Max depth failing on recursive type due to recursion @@ -2036,46 +2017,46 @@ test "std.zon depth limits" { try std.testing.expectError( error.ExceededMaxDepth, - serializeMaxDepth(maybe_recurse, .{}, buf.writer(), 128), + serializeMaxDepth(maybe_recurse, .{}, bw, 128), ); - try std.testing.expectEqualStrings("", buf.items); - buf.clearRetainingCapacity(); + try std.testing.expectEqualStrings("", aw.getWritten()); + aw.clearRetainingCapacity(); - var sz = serializer(buf.writer(), .{}); + var s: Serializer = .{ .writer = bw }; try std.testing.expectError( error.ExceededMaxDepth, - sz.tupleMaxDepth(maybe_recurse, .{}, 128), + s.tupleMaxDepth(maybe_recurse, .{}, 128), ); - try std.testing.expectEqualStrings("", buf.items); - buf.clearRetainingCapacity(); + try std.testing.expectEqualStrings("", aw.getWritten()); + aw.clearRetainingCapacity(); } // Max depth on other parts of the lower level API { - var sz = serializer(buf.writer(), .{}); + var s: Serializer = .{ .writer = bw }; const maybe_recurse: []const Recurse = &.{}; - try std.testing.expectError(error.ExceededMaxDepth, sz.valueMaxDepth(1, .{}, 0)); - try sz.valueMaxDepth(2, .{}, 1); - try sz.value(3, .{}); - try sz.valueArbitraryDepth(maybe_recurse, .{}); + try std.testing.expectError(error.ExceededMaxDepth, s.valueMaxDepth(1, .{}, 0)); + try s.valueMaxDepth(2, .{}, 1); + try s.value(3, .{}); + try s.valueArbitraryDepth(maybe_recurse, .{}); - var s = try sz.beginStruct(.{}); - try std.testing.expectError(error.ExceededMaxDepth, s.fieldMaxDepth("a", 1, .{}, 0)); - try s.fieldMaxDepth("b", 4, .{}, 1); - try s.field("c", 5, .{}); - try s.fieldArbitraryDepth("d", maybe_recurse, .{}); - try s.end(); + var wip_struct = try s.beginStruct(.{}); + try std.testing.expectError(error.ExceededMaxDepth, wip_struct.fieldMaxDepth("a", 1, .{}, 0)); + try wip_struct.fieldMaxDepth("b", 4, .{}, 1); + try wip_struct.field("c", 5, .{}); + try wip_struct.fieldArbitraryDepth("d", maybe_recurse, .{}); + try wip_struct.end(); - var t = try sz.beginTuple(.{}); + var t = try s.beginTuple(.{}); try std.testing.expectError(error.ExceededMaxDepth, t.fieldMaxDepth(1, .{}, 0)); try t.fieldMaxDepth(6, .{}, 1); try t.field(7, .{}); try t.fieldArbitraryDepth(maybe_recurse, .{}); try t.end(); - var a = try sz.beginTuple(.{}); + var a = try s.beginTuple(.{}); try std.testing.expectError(error.ExceededMaxDepth, a.fieldMaxDepth(1, .{}, 0)); try a.fieldMaxDepth(8, .{}, 1); try a.field(9, .{}); @@ -2096,7 +2077,7 @@ test "std.zon depth limits" { \\ 9, \\ .{}, \\} - , buf.items); + , aw.getWritten()); } } @@ -2192,42 +2173,42 @@ test "std.zon stringify primitives" { } test "std.zon stringify ident" { - var buf = std.ArrayList(u8).init(std.testing.allocator); - defer buf.deinit(); - var sz = serializer(buf.writer(), .{}); + var aw: Writer.Allocating = .init(std.testing.allocator); + var s: Serializer = .{ .writer = &aw.writer }; + defer aw.deinit(); try expectSerializeEqual(".{ .a = 0 }", .{ .a = 0 }, .{}); - try sz.ident("a"); - try std.testing.expectEqualStrings(".a", buf.items); - buf.clearRetainingCapacity(); + try s.ident("a"); + try std.testing.expectEqualStrings(".a", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.ident("foo_1"); - try std.testing.expectEqualStrings(".foo_1", buf.items); - buf.clearRetainingCapacity(); + try s.ident("foo_1"); + try std.testing.expectEqualStrings(".foo_1", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.ident("_foo_1"); - try std.testing.expectEqualStrings("._foo_1", buf.items); - buf.clearRetainingCapacity(); + try s.ident("_foo_1"); + try std.testing.expectEqualStrings("._foo_1", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.ident("foo bar"); - try std.testing.expectEqualStrings(".@\"foo bar\"", buf.items); - buf.clearRetainingCapacity(); + try s.ident("foo bar"); + try std.testing.expectEqualStrings(".@\"foo bar\"", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.ident("1foo"); - try std.testing.expectEqualStrings(".@\"1foo\"", buf.items); - buf.clearRetainingCapacity(); + try s.ident("1foo"); + try std.testing.expectEqualStrings(".@\"1foo\"", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.ident("var"); - try std.testing.expectEqualStrings(".@\"var\"", buf.items); - buf.clearRetainingCapacity(); + try s.ident("var"); + try std.testing.expectEqualStrings(".@\"var\"", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.ident("true"); - try std.testing.expectEqualStrings(".true", buf.items); - buf.clearRetainingCapacity(); + try s.ident("true"); + try std.testing.expectEqualStrings(".true", aw.getWritten()); + aw.clearRetainingCapacity(); - try sz.ident("_"); - try std.testing.expectEqualStrings("._", buf.items); - buf.clearRetainingCapacity(); + try s.ident("_"); + try std.testing.expectEqualStrings("._", aw.getWritten()); + aw.clearRetainingCapacity(); const Enum = enum { @"foo bar", @@ -2239,40 +2220,40 @@ test "std.zon stringify ident" { } test "std.zon stringify as tuple" { - var buf = std.ArrayList(u8).init(std.testing.allocator); - defer buf.deinit(); - var sz = serializer(buf.writer(), .{}); + var aw: Writer.Allocating = .init(std.testing.allocator); + var s: Serializer = .{ .writer = &aw.writer }; + defer aw.deinit(); // Tuples - try sz.tuple(.{ 1, 2 }, .{}); - try std.testing.expectEqualStrings(".{ 1, 2 }", buf.items); - buf.clearRetainingCapacity(); + try s.tuple(.{ 1, 2 }, .{}); + try std.testing.expectEqualStrings(".{ 1, 2 }", aw.getWritten()); + aw.clearRetainingCapacity(); // Slice - try sz.tuple(@as([]const u8, &.{ 1, 2 }), .{}); - try std.testing.expectEqualStrings(".{ 1, 2 }", buf.items); - buf.clearRetainingCapacity(); + try s.tuple(@as([]const u8, &.{ 1, 2 }), .{}); + try std.testing.expectEqualStrings(".{ 1, 2 }", aw.getWritten()); + aw.clearRetainingCapacity(); // Array - try sz.tuple([2]u8{ 1, 2 }, .{}); - try std.testing.expectEqualStrings(".{ 1, 2 }", buf.items); - buf.clearRetainingCapacity(); + try s.tuple([2]u8{ 1, 2 }, .{}); + try std.testing.expectEqualStrings(".{ 1, 2 }", aw.getWritten()); + aw.clearRetainingCapacity(); } test "std.zon stringify as float" { - var buf = std.ArrayList(u8).init(std.testing.allocator); - defer buf.deinit(); - var sz = serializer(buf.writer(), .{}); + var aw: Writer.Allocating = .init(std.testing.allocator); + var s: Serializer = .{ .writer = &aw.writer }; + defer aw.deinit(); // Comptime float - try sz.float(2.5); - try std.testing.expectEqualStrings("2.5", buf.items); - buf.clearRetainingCapacity(); + try s.float(2.5); + try std.testing.expectEqualStrings("2.5", aw.getWritten()); + aw.clearRetainingCapacity(); // Sized float - try sz.float(@as(f32, 2.5)); - try std.testing.expectEqualStrings("2.5", buf.items); - buf.clearRetainingCapacity(); + try s.float(@as(f32, 2.5)); + try std.testing.expectEqualStrings("2.5", aw.getWritten()); + aw.clearRetainingCapacity(); } test "std.zon stringify vector" { @@ -2364,13 +2345,13 @@ test "std.zon pointers" { } test "std.zon tuple/struct field" { - var buf = std.ArrayList(u8).init(std.testing.allocator); - defer buf.deinit(); - var sz = serializer(buf.writer(), .{}); + var aw: Writer.Allocating = .init(std.testing.allocator); + var s: Serializer = .{ .writer = &aw.writer }; + defer aw.deinit(); // Test on structs { - var root = try sz.beginStruct(.{}); + var root = try s.beginStruct(.{}); { var tuple = try root.beginTupleField("foo", .{}); try tuple.field(0, .{}); @@ -2396,13 +2377,13 @@ test "std.zon tuple/struct field" { \\ .b = 1, \\ }, \\} - , buf.items); - buf.clearRetainingCapacity(); + , aw.getWritten()); + aw.clearRetainingCapacity(); } // Test on tuples { - var root = try sz.beginTuple(.{}); + var root = try s.beginTuple(.{}); { var tuple = try root.beginTupleField(.{}); try tuple.field(0, .{}); @@ -2428,7 +2409,7 @@ test "std.zon tuple/struct field" { \\ .b = 1, \\ }, \\} - , buf.items); - buf.clearRetainingCapacity(); + , aw.getWritten()); + aw.clearRetainingCapacity(); } } From 737b13357ed04a33bfea17f58674deba8f17f51e Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 19 Jul 2025 17:11:47 -0700 Subject: [PATCH 08/26] resinator: fix std.json API usage --- lib/compiler/resinator/main.zig | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/compiler/resinator/main.zig b/lib/compiler/resinator/main.zig index 903e0a2f71..29cf716600 100644 --- a/lib/compiler/resinator/main.zig +++ b/lib/compiler/resinator/main.zig @@ -290,12 +290,14 @@ pub fn main() !void { }; defer depfile.close(); - const depfile_writer = depfile.deprecatedWriter(); - var depfile_buffered_writer = std.io.bufferedWriter(depfile_writer); + var depfile_buffer: [1024]u8 = undefined; + var depfile_writer = depfile.writer(&depfile_buffer); switch (options.depfile_fmt) { .json => { - var write_stream = std.json.writeStream(depfile_buffered_writer.writer(), .{ .whitespace = .indent_2 }); - defer write_stream.deinit(); + var write_stream: std.json.Stringify = .{ + .writer = &depfile_writer.interface, + .options = .{ .whitespace = .indent_2 }, + }; try write_stream.beginArray(); for (dependencies_list.items) |dep_path| { @@ -304,7 +306,7 @@ pub fn main() !void { try write_stream.endArray(); }, } - try depfile_buffered_writer.flush(); + try depfile_writer.interface.flush(); } } From 0fb7a0a94bf6f9e329008d8b5b819a8d1d7124b0 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 19 Jul 2025 17:33:44 -0700 Subject: [PATCH 09/26] std.zon: better namespace for Serializer --- lib/std/zon.zig | 1 + lib/std/zon/Serializer.zig | 929 ++++++++++++++++++++++++++++++++++++ lib/std/zon/stringify.zig | 936 +------------------------------------ src/main.zig | 9 +- src/print_env.zig | 49 +- src/print_targets.zig | 12 +- src/translate_c.zig | 2 +- 7 files changed, 957 insertions(+), 981 deletions(-) create mode 100644 lib/std/zon/Serializer.zig diff --git a/lib/std/zon.zig b/lib/std/zon.zig index 252331057a..9ac02cf741 100644 --- a/lib/std/zon.zig +++ b/lib/std/zon.zig @@ -38,6 +38,7 @@ pub const parse = @import("zon/parse.zig"); pub const stringify = @import("zon/stringify.zig"); +pub const Serializer = @import("zon/Serializer.zig"); test { _ = parse; diff --git a/lib/std/zon/Serializer.zig b/lib/std/zon/Serializer.zig new file mode 100644 index 0000000000..b65b13bf97 --- /dev/null +++ b/lib/std/zon/Serializer.zig @@ -0,0 +1,929 @@ +//! Lower level control over serialization, you can create a new instance with `serializer`. +//! +//! Useful when you want control over which fields are serialized, how they're represented, +//! or want to write a ZON object that does not exist in memory. +//! +//! You can serialize values with `value`. To serialize recursive types, the following are provided: +//! * `valueMaxDepth` +//! * `valueArbitraryDepth` +//! +//! You can also serialize values using specific notations: +//! * `int` +//! * `float` +//! * `codePoint` +//! * `tuple` +//! * `tupleMaxDepth` +//! * `tupleArbitraryDepth` +//! * `string` +//! * `multilineString` +//! +//! For manual serialization of containers, see: +//! * `beginStruct` +//! * `beginTuple` + +options: Options = .{}, +indent_level: u8 = 0, +writer: *Writer, + +const Serializer = @This(); +const std = @import("std"); +const assert = std.debug.assert; +const Writer = std.Io.Writer; + +pub const Error = Writer.Error; +pub const DepthError = Error || error{ExceededMaxDepth}; + +pub const Options = struct { + /// If false, only syntactically necessary whitespace is emitted. + whitespace: bool = true, +}; + +/// Options for manual serialization of container types. +pub const ContainerOptions = struct { + /// The whitespace style that should be used for this container. Ignored if whitespace is off. + whitespace_style: union(enum) { + /// If true, wrap every field. If false do not. + wrap: bool, + /// Automatically decide whether to wrap or not based on the number of fields. Following + /// the standard rule of thumb, containers with more than two fields are wrapped. + fields: usize, + } = .{ .wrap = true }, + + fn shouldWrap(self: ContainerOptions) bool { + return switch (self.whitespace_style) { + .wrap => |wrap| wrap, + .fields => |fields| fields > 2, + }; + } +}; + +/// Options for serialization of an individual value. +/// +/// See `SerializeOptions` for more information on these options. +pub const ValueOptions = struct { + emit_codepoint_literals: EmitCodepointLiterals = .never, + emit_strings_as_containers: bool = false, + emit_default_optional_fields: bool = true, +}; + +/// Determines when to emit Unicode code point literals as opposed to integer literals. +pub const EmitCodepointLiterals = enum { + /// Never emit Unicode code point literals. + never, + /// Emit Unicode code point literals for any `u8` in the printable ASCII range. + printable_ascii, + /// Emit Unicode code point literals for any unsigned integer with 21 bits or fewer + /// whose value is a valid non-surrogate code point. + always, + + /// If the value should be emitted as a Unicode codepoint, return it as a u21. + fn emitAsCodepoint(self: @This(), val: anytype) ?u21 { + // Rule out incompatible integer types + switch (@typeInfo(@TypeOf(val))) { + .int => |int_info| if (int_info.signedness == .signed or int_info.bits > 21) { + return null; + }, + .comptime_int => {}, + else => comptime unreachable, + } + + // Return null if the value shouldn't be printed as a Unicode codepoint, or the value casted + // to a u21 if it should. + switch (self) { + .always => { + const c = std.math.cast(u21, val) orelse return null; + if (!std.unicode.utf8ValidCodepoint(c)) return null; + return c; + }, + .printable_ascii => { + const c = std.math.cast(u8, val) orelse return null; + if (!std.ascii.isPrint(c)) return null; + return c; + }, + .never => { + return null; + }, + } + } +}; + +/// Serialize a value, similar to `serialize`. +pub fn value(self: *Serializer, val: anytype, options: ValueOptions) Error!void { + comptime assert(!typeIsRecursive(@TypeOf(val))); + return self.valueArbitraryDepth(val, options); +} + +/// Serialize a value, similar to `serializeMaxDepth`. +/// Can return `error.ExceededMaxDepth`. +pub fn valueMaxDepth(self: *Serializer, val: anytype, options: ValueOptions, depth: usize) DepthError!void { + try checkValueDepth(val, depth); + return self.valueArbitraryDepth(val, options); +} + +/// Serialize a value, similar to `serializeArbitraryDepth`. +pub fn valueArbitraryDepth(self: *Serializer, val: anytype, options: ValueOptions) Error!void { + comptime assert(canSerializeType(@TypeOf(val))); + switch (@typeInfo(@TypeOf(val))) { + .int, .comptime_int => if (options.emit_codepoint_literals.emitAsCodepoint(val)) |c| { + self.codePoint(c) catch |err| switch (err) { + error.InvalidCodepoint => unreachable, // Already validated + else => |e| return e, + }; + } else { + try self.int(val); + }, + .float, .comptime_float => try self.float(val), + .bool, .null => try self.writer.print("{}", .{val}), + .enum_literal => try self.ident(@tagName(val)), + .@"enum" => try self.ident(@tagName(val)), + .pointer => |pointer| { + // Try to serialize as a string + const item: ?type = switch (@typeInfo(pointer.child)) { + .array => |array| array.child, + else => if (pointer.size == .slice) pointer.child else null, + }; + if (item == u8 and + (pointer.sentinel() == null or pointer.sentinel() == 0) and + !options.emit_strings_as_containers) + { + return try self.string(val); + } + + // Serialize as either a tuple or as the child type + switch (pointer.size) { + .slice => try self.tupleImpl(val, options), + .one => try self.valueArbitraryDepth(val.*, options), + else => comptime unreachable, + } + }, + .array => { + var container = try self.beginTuple( + .{ .whitespace_style = .{ .fields = val.len } }, + ); + for (val) |item_val| { + try container.fieldArbitraryDepth(item_val, options); + } + try container.end(); + }, + .@"struct" => |@"struct"| if (@"struct".is_tuple) { + var container = try self.beginTuple( + .{ .whitespace_style = .{ .fields = @"struct".fields.len } }, + ); + inline for (val) |field_value| { + try container.fieldArbitraryDepth(field_value, options); + } + try container.end(); + } else { + // Decide which fields to emit + const fields, const skipped: [@"struct".fields.len]bool = if (options.emit_default_optional_fields) b: { + break :b .{ @"struct".fields.len, @splat(false) }; + } else b: { + var fields = @"struct".fields.len; + var skipped: [@"struct".fields.len]bool = @splat(false); + inline for (@"struct".fields, &skipped) |field_info, *skip| { + if (field_info.default_value_ptr) |ptr| { + const default: *const field_info.type = @ptrCast(@alignCast(ptr)); + const field_value = @field(val, field_info.name); + if (std.meta.eql(field_value, default.*)) { + skip.* = true; + fields -= 1; + } + } + } + break :b .{ fields, skipped }; + }; + + // Emit those fields + var container = try self.beginStruct( + .{ .whitespace_style = .{ .fields = fields } }, + ); + inline for (@"struct".fields, skipped) |field_info, skip| { + if (!skip) { + try container.fieldArbitraryDepth( + field_info.name, + @field(val, field_info.name), + options, + ); + } + } + try container.end(); + }, + .@"union" => |@"union"| { + comptime assert(@"union".tag_type != null); + switch (val) { + inline else => |pl, tag| if (@TypeOf(pl) == void) + try self.writer.print(".{s}", .{@tagName(tag)}) + else { + var container = try self.beginStruct(.{ .whitespace_style = .{ .fields = 1 } }); + + try container.fieldArbitraryDepth( + @tagName(tag), + pl, + options, + ); + + try container.end(); + }, + } + }, + .optional => if (val) |inner| { + try self.valueArbitraryDepth(inner, options); + } else { + try self.writer.writeAll("null"); + }, + .vector => |vector| { + var container = try self.beginTuple( + .{ .whitespace_style = .{ .fields = vector.len } }, + ); + for (0..vector.len) |i| { + try container.fieldArbitraryDepth(val[i], options); + } + try container.end(); + }, + + else => comptime unreachable, + } +} + +/// Serialize an integer. +pub fn int(self: *Serializer, val: anytype) Error!void { + try self.writer.printInt(val, 10, .lower, .{}); +} + +/// Serialize a float. +pub fn float(self: *Serializer, val: anytype) Error!void { + switch (@typeInfo(@TypeOf(val))) { + .float => if (std.math.isNan(val)) { + return self.writer.writeAll("nan"); + } else if (std.math.isPositiveInf(val)) { + return self.writer.writeAll("inf"); + } else if (std.math.isNegativeInf(val)) { + return self.writer.writeAll("-inf"); + } else if (std.math.isNegativeZero(val)) { + return self.writer.writeAll("-0.0"); + } else { + try self.writer.print("{d}", .{val}); + }, + .comptime_float => if (val == 0) { + return self.writer.writeAll("0"); + } else { + try self.writer.print("{d}", .{val}); + }, + else => comptime unreachable, + } +} + +/// Serialize `name` as an identifier prefixed with `.`. +/// +/// Escapes the identifier if necessary. +pub fn ident(self: *Serializer, name: []const u8) Error!void { + try self.writer.print(".{f}", .{std.zig.fmtIdPU(name)}); +} + +pub const CodePointError = Error || error{InvalidCodepoint}; + +/// Serialize `val` as a Unicode codepoint. +/// +/// Returns `error.InvalidCodepoint` if `val` is not a valid Unicode codepoint. +pub fn codePoint(self: *Serializer, val: u21) CodePointError!void { + try self.writer.print("'{f}'", .{std.zig.fmtChar(val)}); +} + +/// Like `value`, but always serializes `val` as a tuple. +/// +/// Will fail at comptime if `val` is not a tuple, array, pointer to an array, or slice. +pub fn tuple(self: *Serializer, val: anytype, options: ValueOptions) Error!void { + comptime assert(!typeIsRecursive(@TypeOf(val))); + try self.tupleArbitraryDepth(val, options); +} + +/// Like `tuple`, but recursive types are allowed. +/// +/// Returns `error.ExceededMaxDepth` if `depth` is exceeded. +pub fn tupleMaxDepth( + self: *Serializer, + val: anytype, + options: ValueOptions, + depth: usize, +) DepthError!void { + try checkValueDepth(val, depth); + try self.tupleArbitraryDepth(val, options); +} + +/// Like `tuple`, but recursive types are allowed. +/// +/// It is the caller's responsibility to ensure that `val` does not contain cycles. +pub fn tupleArbitraryDepth( + self: *Serializer, + val: anytype, + options: ValueOptions, +) Error!void { + try self.tupleImpl(val, options); +} + +fn tupleImpl(self: *Serializer, val: anytype, options: ValueOptions) Error!void { + comptime assert(canSerializeType(@TypeOf(val))); + switch (@typeInfo(@TypeOf(val))) { + .@"struct" => { + var container = try self.beginTuple(.{ .whitespace_style = .{ .fields = val.len } }); + inline for (val) |item_val| { + try container.fieldArbitraryDepth(item_val, options); + } + try container.end(); + }, + .pointer, .array => { + var container = try self.beginTuple(.{ .whitespace_style = .{ .fields = val.len } }); + for (val) |item_val| { + try container.fieldArbitraryDepth(item_val, options); + } + try container.end(); + }, + else => comptime unreachable, + } +} + +/// Like `value`, but always serializes `val` as a string. +pub fn string(self: *Serializer, val: []const u8) Error!void { + try self.writer.print("\"{f}\"", .{std.zig.fmtString(val)}); +} + +/// Options for formatting multiline strings. +pub const MultilineStringOptions = struct { + /// If top level is true, whitespace before and after the multiline string is elided. + /// If it is true, a newline is printed, then the value, followed by a newline, and if + /// whitespace is true any necessary indentation follows. + top_level: bool = false, +}; + +pub const MultilineStringError = Error || error{InnerCarriageReturn}; + +/// Like `value`, but always serializes to a multiline string literal. +/// +/// Returns `error.InnerCarriageReturn` if `val` contains a CR not followed by a newline, +/// since multiline strings cannot represent CR without a following newline. +pub fn multilineString( + self: *Serializer, + val: []const u8, + options: MultilineStringOptions, +) MultilineStringError!void { + // Make sure the string does not contain any carriage returns not followed by a newline + var i: usize = 0; + while (i < val.len) : (i += 1) { + if (val[i] == '\r') { + if (i + 1 < val.len) { + if (val[i + 1] == '\n') { + i += 1; + continue; + } + } + return error.InnerCarriageReturn; + } + } + + if (!options.top_level) { + try self.newline(); + try self.indent(); + } + + try self.writer.writeAll("\\\\"); + for (val) |c| { + if (c != '\r') { + try self.writer.writeByte(c); // We write newlines here even if whitespace off + if (c == '\n') { + try self.indent(); + try self.writer.writeAll("\\\\"); + } + } + } + + if (!options.top_level) { + try self.writer.writeByte('\n'); // Even if whitespace off + try self.indent(); + } +} + +/// Create a `Struct` for writing ZON structs field by field. +pub fn beginStruct(self: *Serializer, options: ContainerOptions) Error!Struct { + return Struct.begin(self, options); +} + +/// Creates a `Tuple` for writing ZON tuples field by field. +pub fn beginTuple(self: *Serializer, options: ContainerOptions) Error!Tuple { + return Tuple.begin(self, options); +} + +fn indent(self: *Serializer) Error!void { + if (self.options.whitespace) { + try self.writer.splatByteAll(' ', 4 * self.indent_level); + } +} + +fn newline(self: *Serializer) Error!void { + if (self.options.whitespace) { + try self.writer.writeByte('\n'); + } +} + +fn newlineOrSpace(self: *Serializer, len: usize) Error!void { + if (self.containerShouldWrap(len)) { + try self.newline(); + } else { + try self.space(); + } +} + +fn space(self: *Serializer) Error!void { + if (self.options.whitespace) { + try self.writer.writeByte(' '); + } +} + +/// Writes ZON tuples field by field. +pub const Tuple = struct { + container: Container, + + fn begin(parent: *Serializer, options: ContainerOptions) Error!Tuple { + return .{ + .container = try Container.begin(parent, .anon, options), + }; + } + + /// Finishes serializing the tuple. + /// + /// Prints a trailing comma as configured when appropriate, and the closing bracket. + pub fn end(self: *Tuple) Error!void { + try self.container.end(); + self.* = undefined; + } + + /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `value`. + pub fn field( + self: *Tuple, + val: anytype, + options: ValueOptions, + ) Error!void { + try self.container.field(null, val, options); + } + + /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `valueMaxDepth`. + /// Returns `error.ExceededMaxDepth` if `depth` is exceeded. + pub fn fieldMaxDepth( + self: *Tuple, + val: anytype, + options: ValueOptions, + depth: usize, + ) DepthError!void { + try self.container.fieldMaxDepth(null, val, options, depth); + } + + /// Serialize a field. Equivalent to calling `fieldPrefix` followed by + /// `valueArbitraryDepth`. + pub fn fieldArbitraryDepth( + self: *Tuple, + val: anytype, + options: ValueOptions, + ) Error!void { + try self.container.fieldArbitraryDepth(null, val, options); + } + + /// Starts a field with a struct as a value. Returns the struct. + pub fn beginStructField( + self: *Tuple, + options: ContainerOptions, + ) Error!Struct { + try self.fieldPrefix(); + return self.container.serializer.beginStruct(options); + } + + /// Starts a field with a tuple as a value. Returns the tuple. + pub fn beginTupleField( + self: *Tuple, + options: ContainerOptions, + ) Error!Tuple { + try self.fieldPrefix(); + return self.container.serializer.beginTuple(options); + } + + /// Print a field prefix. This prints any necessary commas, and whitespace as + /// configured. Useful if you want to serialize the field value yourself. + pub fn fieldPrefix(self: *Tuple) Error!void { + try self.container.fieldPrefix(null); + } +}; + +/// Writes ZON structs field by field. +pub const Struct = struct { + container: Container, + + fn begin(parent: *Serializer, options: ContainerOptions) Error!Struct { + return .{ + .container = try Container.begin(parent, .named, options), + }; + } + + /// Finishes serializing the struct. + /// + /// Prints a trailing comma as configured when appropriate, and the closing bracket. + pub fn end(self: *Struct) Error!void { + try self.container.end(); + self.* = undefined; + } + + /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `value`. + pub fn field( + self: *Struct, + name: []const u8, + val: anytype, + options: ValueOptions, + ) Error!void { + try self.container.field(name, val, options); + } + + /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `valueMaxDepth`. + /// Returns `error.ExceededMaxDepth` if `depth` is exceeded. + pub fn fieldMaxDepth( + self: *Struct, + name: []const u8, + val: anytype, + options: ValueOptions, + depth: usize, + ) DepthError!void { + try self.container.fieldMaxDepth(name, val, options, depth); + } + + /// Serialize a field. Equivalent to calling `fieldPrefix` followed by + /// `valueArbitraryDepth`. + pub fn fieldArbitraryDepth( + self: *Struct, + name: []const u8, + val: anytype, + options: ValueOptions, + ) Error!void { + try self.container.fieldArbitraryDepth(name, val, options); + } + + /// Starts a field with a struct as a value. Returns the struct. + pub fn beginStructField( + self: *Struct, + name: []const u8, + options: ContainerOptions, + ) Error!Struct { + try self.fieldPrefix(name); + return self.container.serializer.beginStruct(options); + } + + /// Starts a field with a tuple as a value. Returns the tuple. + pub fn beginTupleField( + self: *Struct, + name: []const u8, + options: ContainerOptions, + ) Error!Tuple { + try self.fieldPrefix(name); + return self.container.serializer.beginTuple(options); + } + + /// Print a field prefix. This prints any necessary commas, the field name (escaped if + /// necessary) and whitespace as configured. Useful if you want to serialize the field + /// value yourself. + pub fn fieldPrefix(self: *Struct, name: []const u8) Error!void { + try self.container.fieldPrefix(name); + } +}; + +const Container = struct { + const FieldStyle = enum { named, anon }; + + serializer: *Serializer, + field_style: FieldStyle, + options: ContainerOptions, + empty: bool, + + fn begin( + sz: *Serializer, + field_style: FieldStyle, + options: ContainerOptions, + ) Error!Container { + if (options.shouldWrap()) sz.indent_level +|= 1; + try sz.writer.writeAll(".{"); + return .{ + .serializer = sz, + .field_style = field_style, + .options = options, + .empty = true, + }; + } + + fn end(self: *Container) Error!void { + if (self.options.shouldWrap()) self.serializer.indent_level -|= 1; + if (!self.empty) { + if (self.options.shouldWrap()) { + if (self.serializer.options.whitespace) { + try self.serializer.writer.writeByte(','); + } + try self.serializer.newline(); + try self.serializer.indent(); + } else if (!self.shouldElideSpaces()) { + try self.serializer.space(); + } + } + try self.serializer.writer.writeByte('}'); + self.* = undefined; + } + + fn fieldPrefix(self: *Container, name: ?[]const u8) Error!void { + if (!self.empty) { + try self.serializer.writer.writeByte(','); + } + self.empty = false; + if (self.options.shouldWrap()) { + try self.serializer.newline(); + } else if (!self.shouldElideSpaces()) { + try self.serializer.space(); + } + if (self.options.shouldWrap()) try self.serializer.indent(); + if (name) |n| { + try self.serializer.ident(n); + try self.serializer.space(); + try self.serializer.writer.writeByte('='); + try self.serializer.space(); + } + } + + fn field( + self: *Container, + name: ?[]const u8, + val: anytype, + options: ValueOptions, + ) Error!void { + comptime assert(!typeIsRecursive(@TypeOf(val))); + try self.fieldArbitraryDepth(name, val, options); + } + + /// Returns `error.ExceededMaxDepth` if `depth` is exceeded. + fn fieldMaxDepth( + self: *Container, + name: ?[]const u8, + val: anytype, + options: ValueOptions, + depth: usize, + ) DepthError!void { + try checkValueDepth(val, depth); + try self.fieldArbitraryDepth(name, val, options); + } + + fn fieldArbitraryDepth( + self: *Container, + name: ?[]const u8, + val: anytype, + options: ValueOptions, + ) Error!void { + try self.fieldPrefix(name); + try self.serializer.valueArbitraryDepth(val, options); + } + + fn shouldElideSpaces(self: *const Container) bool { + return switch (self.options.whitespace_style) { + .fields => |fields| self.field_style != .named and fields == 1, + else => false, + }; + } +}; + +test Serializer { + var discarding: Writer.Discarding = .init(&.{}); + var s: Serializer = .{ .writer = &discarding.writer }; + var vec2 = try s.beginStruct(.{}); + try vec2.field("x", 1.5, .{}); + try vec2.fieldPrefix("prefix"); + try s.value(2.5, .{}); + try vec2.end(); +} + +inline fn typeIsRecursive(comptime T: type) bool { + return comptime typeIsRecursiveInner(T, &.{}); +} + +fn typeIsRecursiveInner(comptime T: type, comptime prev_visited: []const type) bool { + for (prev_visited) |V| { + if (V == T) return true; + } + const visited = prev_visited ++ .{T}; + + return switch (@typeInfo(T)) { + .pointer => |pointer| typeIsRecursiveInner(pointer.child, visited), + .optional => |optional| typeIsRecursiveInner(optional.child, visited), + .array => |array| typeIsRecursiveInner(array.child, visited), + .vector => |vector| typeIsRecursiveInner(vector.child, visited), + .@"struct" => |@"struct"| for (@"struct".fields) |field| { + if (typeIsRecursiveInner(field.type, visited)) break true; + } else false, + .@"union" => |@"union"| inline for (@"union".fields) |field| { + if (typeIsRecursiveInner(field.type, visited)) break true; + } else false, + else => false, + }; +} + +test typeIsRecursive { + try std.testing.expect(!typeIsRecursive(bool)); + try std.testing.expect(!typeIsRecursive(struct { x: i32, y: i32 })); + try std.testing.expect(!typeIsRecursive(struct { i32, i32 })); + try std.testing.expect(typeIsRecursive(struct { x: i32, y: i32, z: *@This() })); + try std.testing.expect(typeIsRecursive(struct { + a: struct { + const A = @This(); + b: struct { + c: *struct { + a: ?A, + }, + }, + }, + })); + try std.testing.expect(typeIsRecursive(struct { + a: [3]*@This(), + })); + try std.testing.expect(typeIsRecursive(struct { + a: union { a: i32, b: *@This() }, + })); +} + +fn checkValueDepth(val: anytype, depth: usize) error{ExceededMaxDepth}!void { + if (depth == 0) return error.ExceededMaxDepth; + const child_depth = depth - 1; + + switch (@typeInfo(@TypeOf(val))) { + .pointer => |pointer| switch (pointer.size) { + .one => try checkValueDepth(val.*, child_depth), + .slice => for (val) |item| { + try checkValueDepth(item, child_depth); + }, + .c, .many => {}, + }, + .array => for (val) |item| { + try checkValueDepth(item, child_depth); + }, + .@"struct" => |@"struct"| inline for (@"struct".fields) |field_info| { + try checkValueDepth(@field(val, field_info.name), child_depth); + }, + .@"union" => |@"union"| if (@"union".tag_type == null) { + return; + } else switch (val) { + inline else => |payload| { + return checkValueDepth(payload, child_depth); + }, + }, + .optional => if (val) |inner| try checkValueDepth(inner, child_depth), + else => {}, + } +} + +fn expectValueDepthEquals(expected: usize, v: anytype) !void { + try checkValueDepth(v, expected); + try std.testing.expectError(error.ExceededMaxDepth, checkValueDepth(v, expected - 1)); +} + +test checkValueDepth { + try expectValueDepthEquals(1, 10); + try expectValueDepthEquals(2, .{ .x = 1, .y = 2 }); + try expectValueDepthEquals(2, .{ 1, 2 }); + try expectValueDepthEquals(3, .{ 1, .{ 2, 3 } }); + try expectValueDepthEquals(3, .{ .{ 1, 2 }, 3 }); + try expectValueDepthEquals(3, .{ .x = 0, .y = 1, .z = .{ .x = 3 } }); + try expectValueDepthEquals(3, .{ .x = 0, .y = .{ .x = 1 }, .z = 2 }); + try expectValueDepthEquals(3, .{ .x = .{ .x = 0 }, .y = 1, .z = 2 }); + try expectValueDepthEquals(2, @as(?u32, 1)); + try expectValueDepthEquals(1, @as(?u32, null)); + try expectValueDepthEquals(1, null); + try expectValueDepthEquals(2, &1); + try expectValueDepthEquals(3, &@as(?u32, 1)); + + const Union = union(enum) { + x: u32, + y: struct { x: u32 }, + }; + try expectValueDepthEquals(2, Union{ .x = 1 }); + try expectValueDepthEquals(3, Union{ .y = .{ .x = 1 } }); + + const Recurse = struct { r: ?*const @This() }; + try expectValueDepthEquals(2, Recurse{ .r = null }); + try expectValueDepthEquals(5, Recurse{ .r = &Recurse{ .r = null } }); + try expectValueDepthEquals(8, Recurse{ .r = &Recurse{ .r = &Recurse{ .r = null } } }); + + try expectValueDepthEquals(2, @as([]const u8, &.{ 1, 2, 3 })); + try expectValueDepthEquals(3, @as([]const []const u8, &.{&.{ 1, 2, 3 }})); +} + +inline fn canSerializeType(T: type) bool { + comptime return canSerializeTypeInner(T, &.{}, false); +} + +fn canSerializeTypeInner( + T: type, + /// Visited structs and unions, to avoid infinite recursion. + /// Tracking more types is unnecessary, and a little complex due to optional nesting. + visited: []const type, + parent_is_optional: bool, +) bool { + return switch (@typeInfo(T)) { + .bool, + .int, + .float, + .comptime_float, + .comptime_int, + .null, + .enum_literal, + => true, + + .noreturn, + .void, + .type, + .undefined, + .error_union, + .error_set, + .@"fn", + .frame, + .@"anyframe", + .@"opaque", + => false, + + .@"enum" => |@"enum"| @"enum".is_exhaustive, + + .pointer => |pointer| switch (pointer.size) { + .one => canSerializeTypeInner(pointer.child, visited, parent_is_optional), + .slice => canSerializeTypeInner(pointer.child, visited, false), + .many, .c => false, + }, + + .optional => |optional| if (parent_is_optional) + false + else + canSerializeTypeInner(optional.child, visited, true), + + .array => |array| canSerializeTypeInner(array.child, visited, false), + .vector => |vector| canSerializeTypeInner(vector.child, visited, false), + + .@"struct" => |@"struct"| { + for (visited) |V| if (T == V) return true; + const new_visited = visited ++ .{T}; + for (@"struct".fields) |field| { + if (!canSerializeTypeInner(field.type, new_visited, false)) return false; + } + return true; + }, + .@"union" => |@"union"| { + for (visited) |V| if (T == V) return true; + const new_visited = visited ++ .{T}; + if (@"union".tag_type == null) return false; + for (@"union".fields) |field| { + if (field.type != void and !canSerializeTypeInner(field.type, new_visited, false)) { + return false; + } + } + return true; + }, + }; +} + +test canSerializeType { + try std.testing.expect(!comptime canSerializeType(void)); + try std.testing.expect(!comptime canSerializeType(struct { f: [*]u8 })); + try std.testing.expect(!comptime canSerializeType(struct { error{foo} })); + try std.testing.expect(!comptime canSerializeType(union(enum) { a: void, f: [*c]u8 })); + try std.testing.expect(!comptime canSerializeType(@Vector(0, [*c]u8))); + try std.testing.expect(!comptime canSerializeType(*?[*c]u8)); + try std.testing.expect(!comptime canSerializeType(enum(u8) { _ })); + try std.testing.expect(!comptime canSerializeType(union { foo: void })); + try std.testing.expect(comptime canSerializeType(union(enum) { foo: void })); + try std.testing.expect(comptime canSerializeType(comptime_float)); + try std.testing.expect(comptime canSerializeType(comptime_int)); + try std.testing.expect(!comptime canSerializeType(struct { comptime foo: ??u8 = null })); + try std.testing.expect(comptime canSerializeType(@TypeOf(.foo))); + try std.testing.expect(comptime canSerializeType(?u8)); + try std.testing.expect(comptime canSerializeType(*?*u8)); + try std.testing.expect(comptime canSerializeType(?struct { + foo: ?struct { + ?union(enum) { + a: ?@Vector(0, ?*u8), + }, + ?struct { + f: ?[]?u8, + }, + }, + })); + try std.testing.expect(!comptime canSerializeType(??u8)); + try std.testing.expect(!comptime canSerializeType(?*?u8)); + try std.testing.expect(!comptime canSerializeType(*?*?*u8)); + try std.testing.expect(comptime canSerializeType(struct { x: comptime_int = 2 })); + try std.testing.expect(comptime canSerializeType(struct { x: comptime_float = 2 })); + try std.testing.expect(comptime canSerializeType(struct { comptime_int })); + try std.testing.expect(comptime canSerializeType(struct { comptime x: @TypeOf(.foo) = .foo })); + const Recursive = struct { foo: ?*@This() }; + try std.testing.expect(comptime canSerializeType(Recursive)); + + // Make sure we validate nested optional before we early out due to already having seen + // a type recursion! + try std.testing.expect(!comptime canSerializeType(struct { + add_to_visited: ?u8, + retrieve_from_visited: ??u8, + })); +} diff --git a/lib/std/zon/stringify.zig b/lib/std/zon/stringify.zig index 4cb2640063..785a303f22 100644 --- a/lib/std/zon/stringify.zig +++ b/lib/std/zon/stringify.zig @@ -23,13 +23,13 @@ const std = @import("std"); const assert = std.debug.assert; const Writer = std.Io.Writer; +const Serializer = std.zon.Serializer; -/// Options for `serialize`. pub const SerializeOptions = struct { /// If false, whitespace is omitted. Otherwise whitespace is emitted in standard Zig style. whitespace: bool = true, /// Determines when to emit Unicode code point literals as opposed to integer literals. - emit_codepoint_literals: EmitCodepointLiterals = .never, + emit_codepoint_literals: Serializer.EmitCodepointLiterals = .never, /// If true, slices of `u8`s, and pointers to arrays of `u8` are serialized as containers. /// Otherwise they are serialized as string literals. emit_strings_as_containers: bool = false, @@ -93,102 +93,6 @@ pub fn serializeArbitraryDepth( }); } -inline fn typeIsRecursive(comptime T: type) bool { - return comptime typeIsRecursiveInner(T, &.{}); -} - -fn typeIsRecursiveInner(comptime T: type, comptime prev_visited: []const type) bool { - for (prev_visited) |V| { - if (V == T) return true; - } - const visited = prev_visited ++ .{T}; - - return switch (@typeInfo(T)) { - .pointer => |pointer| typeIsRecursiveInner(pointer.child, visited), - .optional => |optional| typeIsRecursiveInner(optional.child, visited), - .array => |array| typeIsRecursiveInner(array.child, visited), - .vector => |vector| typeIsRecursiveInner(vector.child, visited), - .@"struct" => |@"struct"| for (@"struct".fields) |field| { - if (typeIsRecursiveInner(field.type, visited)) break true; - } else false, - .@"union" => |@"union"| inline for (@"union".fields) |field| { - if (typeIsRecursiveInner(field.type, visited)) break true; - } else false, - else => false, - }; -} - -inline fn canSerializeType(T: type) bool { - comptime return canSerializeTypeInner(T, &.{}, false); -} - -fn canSerializeTypeInner( - T: type, - /// Visited structs and unions, to avoid infinite recursion. - /// Tracking more types is unnecessary, and a little complex due to optional nesting. - visited: []const type, - parent_is_optional: bool, -) bool { - return switch (@typeInfo(T)) { - .bool, - .int, - .float, - .comptime_float, - .comptime_int, - .null, - .enum_literal, - => true, - - .noreturn, - .void, - .type, - .undefined, - .error_union, - .error_set, - .@"fn", - .frame, - .@"anyframe", - .@"opaque", - => false, - - .@"enum" => |@"enum"| @"enum".is_exhaustive, - - .pointer => |pointer| switch (pointer.size) { - .one => canSerializeTypeInner(pointer.child, visited, parent_is_optional), - .slice => canSerializeTypeInner(pointer.child, visited, false), - .many, .c => false, - }, - - .optional => |optional| if (parent_is_optional) - false - else - canSerializeTypeInner(optional.child, visited, true), - - .array => |array| canSerializeTypeInner(array.child, visited, false), - .vector => |vector| canSerializeTypeInner(vector.child, visited, false), - - .@"struct" => |@"struct"| { - for (visited) |V| if (T == V) return true; - const new_visited = visited ++ .{T}; - for (@"struct".fields) |field| { - if (!canSerializeTypeInner(field.type, new_visited, false)) return false; - } - return true; - }, - .@"union" => |@"union"| { - for (visited) |V| if (T == V) return true; - const new_visited = visited ++ .{T}; - if (@"union".tag_type == null) return false; - for (@"union".fields) |field| { - if (field.type != void and !canSerializeTypeInner(field.type, new_visited, false)) { - return false; - } - } - return true; - }, - }; -} - fn isNestedOptional(T: type) bool { comptime switch (@typeInfo(T)) { .optional => |optional| return isNestedOptionalInner(optional.child), @@ -210,842 +114,6 @@ fn isNestedOptionalInner(T: type) bool { } } -test "std.zon stringify canSerializeType" { - try std.testing.expect(!comptime canSerializeType(void)); - try std.testing.expect(!comptime canSerializeType(struct { f: [*]u8 })); - try std.testing.expect(!comptime canSerializeType(struct { error{foo} })); - try std.testing.expect(!comptime canSerializeType(union(enum) { a: void, f: [*c]u8 })); - try std.testing.expect(!comptime canSerializeType(@Vector(0, [*c]u8))); - try std.testing.expect(!comptime canSerializeType(*?[*c]u8)); - try std.testing.expect(!comptime canSerializeType(enum(u8) { _ })); - try std.testing.expect(!comptime canSerializeType(union { foo: void })); - try std.testing.expect(comptime canSerializeType(union(enum) { foo: void })); - try std.testing.expect(comptime canSerializeType(comptime_float)); - try std.testing.expect(comptime canSerializeType(comptime_int)); - try std.testing.expect(!comptime canSerializeType(struct { comptime foo: ??u8 = null })); - try std.testing.expect(comptime canSerializeType(@TypeOf(.foo))); - try std.testing.expect(comptime canSerializeType(?u8)); - try std.testing.expect(comptime canSerializeType(*?*u8)); - try std.testing.expect(comptime canSerializeType(?struct { - foo: ?struct { - ?union(enum) { - a: ?@Vector(0, ?*u8), - }, - ?struct { - f: ?[]?u8, - }, - }, - })); - try std.testing.expect(!comptime canSerializeType(??u8)); - try std.testing.expect(!comptime canSerializeType(?*?u8)); - try std.testing.expect(!comptime canSerializeType(*?*?*u8)); - try std.testing.expect(comptime canSerializeType(struct { x: comptime_int = 2 })); - try std.testing.expect(comptime canSerializeType(struct { x: comptime_float = 2 })); - try std.testing.expect(comptime canSerializeType(struct { comptime_int })); - try std.testing.expect(comptime canSerializeType(struct { comptime x: @TypeOf(.foo) = .foo })); - const Recursive = struct { foo: ?*@This() }; - try std.testing.expect(comptime canSerializeType(Recursive)); - - // Make sure we validate nested optional before we early out due to already having seen - // a type recursion! - try std.testing.expect(!comptime canSerializeType(struct { - add_to_visited: ?u8, - retrieve_from_visited: ??u8, - })); -} - -test "std.zon typeIsRecursive" { - try std.testing.expect(!typeIsRecursive(bool)); - try std.testing.expect(!typeIsRecursive(struct { x: i32, y: i32 })); - try std.testing.expect(!typeIsRecursive(struct { i32, i32 })); - try std.testing.expect(typeIsRecursive(struct { x: i32, y: i32, z: *@This() })); - try std.testing.expect(typeIsRecursive(struct { - a: struct { - const A = @This(); - b: struct { - c: *struct { - a: ?A, - }, - }, - }, - })); - try std.testing.expect(typeIsRecursive(struct { - a: [3]*@This(), - })); - try std.testing.expect(typeIsRecursive(struct { - a: union { a: i32, b: *@This() }, - })); -} - -fn checkValueDepth(val: anytype, depth: usize) error{ExceededMaxDepth}!void { - if (depth == 0) return error.ExceededMaxDepth; - const child_depth = depth - 1; - - switch (@typeInfo(@TypeOf(val))) { - .pointer => |pointer| switch (pointer.size) { - .one => try checkValueDepth(val.*, child_depth), - .slice => for (val) |item| { - try checkValueDepth(item, child_depth); - }, - .c, .many => {}, - }, - .array => for (val) |item| { - try checkValueDepth(item, child_depth); - }, - .@"struct" => |@"struct"| inline for (@"struct".fields) |field_info| { - try checkValueDepth(@field(val, field_info.name), child_depth); - }, - .@"union" => |@"union"| if (@"union".tag_type == null) { - return; - } else switch (val) { - inline else => |payload| { - return checkValueDepth(payload, child_depth); - }, - }, - .optional => if (val) |inner| try checkValueDepth(inner, child_depth), - else => {}, - } -} - -fn expectValueDepthEquals(expected: usize, value: anytype) !void { - try checkValueDepth(value, expected); - try std.testing.expectError(error.ExceededMaxDepth, checkValueDepth(value, expected - 1)); -} - -test "std.zon checkValueDepth" { - try expectValueDepthEquals(1, 10); - try expectValueDepthEquals(2, .{ .x = 1, .y = 2 }); - try expectValueDepthEquals(2, .{ 1, 2 }); - try expectValueDepthEquals(3, .{ 1, .{ 2, 3 } }); - try expectValueDepthEquals(3, .{ .{ 1, 2 }, 3 }); - try expectValueDepthEquals(3, .{ .x = 0, .y = 1, .z = .{ .x = 3 } }); - try expectValueDepthEquals(3, .{ .x = 0, .y = .{ .x = 1 }, .z = 2 }); - try expectValueDepthEquals(3, .{ .x = .{ .x = 0 }, .y = 1, .z = 2 }); - try expectValueDepthEquals(2, @as(?u32, 1)); - try expectValueDepthEquals(1, @as(?u32, null)); - try expectValueDepthEquals(1, null); - try expectValueDepthEquals(2, &1); - try expectValueDepthEquals(3, &@as(?u32, 1)); - - const Union = union(enum) { - x: u32, - y: struct { x: u32 }, - }; - try expectValueDepthEquals(2, Union{ .x = 1 }); - try expectValueDepthEquals(3, Union{ .y = .{ .x = 1 } }); - - const Recurse = struct { r: ?*const @This() }; - try expectValueDepthEquals(2, Recurse{ .r = null }); - try expectValueDepthEquals(5, Recurse{ .r = &Recurse{ .r = null } }); - try expectValueDepthEquals(8, Recurse{ .r = &Recurse{ .r = &Recurse{ .r = null } } }); - - try expectValueDepthEquals(2, @as([]const u8, &.{ 1, 2, 3 })); - try expectValueDepthEquals(3, @as([]const []const u8, &.{&.{ 1, 2, 3 }})); -} - -/// Determines when to emit Unicode code point literals as opposed to integer literals. -pub const EmitCodepointLiterals = enum { - /// Never emit Unicode code point literals. - never, - /// Emit Unicode code point literals for any `u8` in the printable ASCII range. - printable_ascii, - /// Emit Unicode code point literals for any unsigned integer with 21 bits or fewer - /// whose value is a valid non-surrogate code point. - always, - - /// If the value should be emitted as a Unicode codepoint, return it as a u21. - fn emitAsCodepoint(self: @This(), val: anytype) ?u21 { - // Rule out incompatible integer types - switch (@typeInfo(@TypeOf(val))) { - .int => |int_info| if (int_info.signedness == .signed or int_info.bits > 21) { - return null; - }, - .comptime_int => {}, - else => comptime unreachable, - } - - // Return null if the value shouldn't be printed as a Unicode codepoint, or the value casted - // to a u21 if it should. - switch (self) { - .always => { - const c = std.math.cast(u21, val) orelse return null; - if (!std.unicode.utf8ValidCodepoint(c)) return null; - return c; - }, - .printable_ascii => { - const c = std.math.cast(u8, val) orelse return null; - if (!std.ascii.isPrint(c)) return null; - return c; - }, - .never => { - return null; - }, - } - } -}; - -/// Options for serialization of an individual value. -/// -/// See `SerializeOptions` for more information on these options. -pub const ValueOptions = struct { - emit_codepoint_literals: EmitCodepointLiterals = .never, - emit_strings_as_containers: bool = false, - emit_default_optional_fields: bool = true, -}; - -/// Options for manual serialization of container types. -pub const SerializeContainerOptions = struct { - /// The whitespace style that should be used for this container. Ignored if whitespace is off. - whitespace_style: union(enum) { - /// If true, wrap every field. If false do not. - wrap: bool, - /// Automatically decide whether to wrap or not based on the number of fields. Following - /// the standard rule of thumb, containers with more than two fields are wrapped. - fields: usize, - } = .{ .wrap = true }, - - fn shouldWrap(self: SerializeContainerOptions) bool { - return switch (self.whitespace_style) { - .wrap => |wrap| wrap, - .fields => |fields| fields > 2, - }; - } -}; - -/// Lower level control over serialization, you can create a new instance with `serializer`. -/// -/// Useful when you want control over which fields are serialized, how they're represented, -/// or want to write a ZON object that does not exist in memory. -/// -/// You can serialize values with `value`. To serialize recursive types, the following are provided: -/// * `valueMaxDepth` -/// * `valueArbitraryDepth` -/// -/// You can also serialize values using specific notations: -/// * `int` -/// * `float` -/// * `codePoint` -/// * `tuple` -/// * `tupleMaxDepth` -/// * `tupleArbitraryDepth` -/// * `string` -/// * `multilineString` -/// -/// For manual serialization of containers, see: -/// * `beginStruct` -/// * `beginTuple` -pub const Serializer = struct { - options: Options = .{}, - indent_level: u8 = 0, - writer: *Writer, - - pub const Error = Writer.Error; - pub const DepthError = Error || error{ExceededMaxDepth}; - - pub const Options = struct { - /// If false, only syntactically necessary whitespace is emitted. - whitespace: bool = true, - }; - - /// Serialize a value, similar to `serialize`. - pub fn value(self: *Serializer, val: anytype, options: ValueOptions) Error!void { - comptime assert(!typeIsRecursive(@TypeOf(val))); - return self.valueArbitraryDepth(val, options); - } - - /// Serialize a value, similar to `serializeMaxDepth`. - /// Can return `error.ExceededMaxDepth`. - pub fn valueMaxDepth(self: *Serializer, val: anytype, options: ValueOptions, depth: usize) DepthError!void { - try checkValueDepth(val, depth); - return self.valueArbitraryDepth(val, options); - } - - /// Serialize a value, similar to `serializeArbitraryDepth`. - pub fn valueArbitraryDepth(self: *Serializer, val: anytype, options: ValueOptions) Error!void { - comptime assert(canSerializeType(@TypeOf(val))); - switch (@typeInfo(@TypeOf(val))) { - .int, .comptime_int => if (options.emit_codepoint_literals.emitAsCodepoint(val)) |c| { - self.codePoint(c) catch |err| switch (err) { - error.InvalidCodepoint => unreachable, // Already validated - else => |e| return e, - }; - } else { - try self.int(val); - }, - .float, .comptime_float => try self.float(val), - .bool, .null => try self.writer.print("{}", .{val}), - .enum_literal => try self.ident(@tagName(val)), - .@"enum" => try self.ident(@tagName(val)), - .pointer => |pointer| { - // Try to serialize as a string - const item: ?type = switch (@typeInfo(pointer.child)) { - .array => |array| array.child, - else => if (pointer.size == .slice) pointer.child else null, - }; - if (item == u8 and - (pointer.sentinel() == null or pointer.sentinel() == 0) and - !options.emit_strings_as_containers) - { - return try self.string(val); - } - - // Serialize as either a tuple or as the child type - switch (pointer.size) { - .slice => try self.tupleImpl(val, options), - .one => try self.valueArbitraryDepth(val.*, options), - else => comptime unreachable, - } - }, - .array => { - var container = try self.beginTuple( - .{ .whitespace_style = .{ .fields = val.len } }, - ); - for (val) |item_val| { - try container.fieldArbitraryDepth(item_val, options); - } - try container.end(); - }, - .@"struct" => |@"struct"| if (@"struct".is_tuple) { - var container = try self.beginTuple( - .{ .whitespace_style = .{ .fields = @"struct".fields.len } }, - ); - inline for (val) |field_value| { - try container.fieldArbitraryDepth(field_value, options); - } - try container.end(); - } else { - // Decide which fields to emit - const fields, const skipped: [@"struct".fields.len]bool = if (options.emit_default_optional_fields) b: { - break :b .{ @"struct".fields.len, @splat(false) }; - } else b: { - var fields = @"struct".fields.len; - var skipped: [@"struct".fields.len]bool = @splat(false); - inline for (@"struct".fields, &skipped) |field_info, *skip| { - if (field_info.default_value_ptr) |ptr| { - const default: *const field_info.type = @ptrCast(@alignCast(ptr)); - const field_value = @field(val, field_info.name); - if (std.meta.eql(field_value, default.*)) { - skip.* = true; - fields -= 1; - } - } - } - break :b .{ fields, skipped }; - }; - - // Emit those fields - var container = try self.beginStruct( - .{ .whitespace_style = .{ .fields = fields } }, - ); - inline for (@"struct".fields, skipped) |field_info, skip| { - if (!skip) { - try container.fieldArbitraryDepth( - field_info.name, - @field(val, field_info.name), - options, - ); - } - } - try container.end(); - }, - .@"union" => |@"union"| { - comptime assert(@"union".tag_type != null); - switch (val) { - inline else => |pl, tag| if (@TypeOf(pl) == void) - try self.writer.print(".{s}", .{@tagName(tag)}) - else { - var container = try self.beginStruct(.{ .whitespace_style = .{ .fields = 1 } }); - - try container.fieldArbitraryDepth( - @tagName(tag), - pl, - options, - ); - - try container.end(); - }, - } - }, - .optional => if (val) |inner| { - try self.valueArbitraryDepth(inner, options); - } else { - try self.writer.writeAll("null"); - }, - .vector => |vector| { - var container = try self.beginTuple( - .{ .whitespace_style = .{ .fields = vector.len } }, - ); - for (0..vector.len) |i| { - try container.fieldArbitraryDepth(val[i], options); - } - try container.end(); - }, - - else => comptime unreachable, - } - } - - /// Serialize an integer. - pub fn int(self: *Serializer, val: anytype) Error!void { - try self.writer.printInt(val, 10, .lower, .{}); - } - - /// Serialize a float. - pub fn float(self: *Serializer, val: anytype) Error!void { - switch (@typeInfo(@TypeOf(val))) { - .float => if (std.math.isNan(val)) { - return self.writer.writeAll("nan"); - } else if (std.math.isPositiveInf(val)) { - return self.writer.writeAll("inf"); - } else if (std.math.isNegativeInf(val)) { - return self.writer.writeAll("-inf"); - } else if (std.math.isNegativeZero(val)) { - return self.writer.writeAll("-0.0"); - } else { - try self.writer.print("{d}", .{val}); - }, - .comptime_float => if (val == 0) { - return self.writer.writeAll("0"); - } else { - try self.writer.print("{d}", .{val}); - }, - else => comptime unreachable, - } - } - - /// Serialize `name` as an identifier prefixed with `.`. - /// - /// Escapes the identifier if necessary. - pub fn ident(self: *Serializer, name: []const u8) Error!void { - try self.writer.print(".{f}", .{std.zig.fmtIdPU(name)}); - } - - pub const CodePointError = Error || error{InvalidCodepoint}; - - /// Serialize `val` as a Unicode codepoint. - /// - /// Returns `error.InvalidCodepoint` if `val` is not a valid Unicode codepoint. - pub fn codePoint(self: *Serializer, val: u21) CodePointError!void { - try self.writer.print("'{f}'", .{std.zig.fmtChar(val)}); - } - - /// Like `value`, but always serializes `val` as a tuple. - /// - /// Will fail at comptime if `val` is not a tuple, array, pointer to an array, or slice. - pub fn tuple(self: *Serializer, val: anytype, options: ValueOptions) Error!void { - comptime assert(!typeIsRecursive(@TypeOf(val))); - try self.tupleArbitraryDepth(val, options); - } - - /// Like `tuple`, but recursive types are allowed. - /// - /// Returns `error.ExceededMaxDepth` if `depth` is exceeded. - pub fn tupleMaxDepth( - self: *Serializer, - val: anytype, - options: ValueOptions, - depth: usize, - ) DepthError!void { - try checkValueDepth(val, depth); - try self.tupleArbitraryDepth(val, options); - } - - /// Like `tuple`, but recursive types are allowed. - /// - /// It is the caller's responsibility to ensure that `val` does not contain cycles. - pub fn tupleArbitraryDepth( - self: *Serializer, - val: anytype, - options: ValueOptions, - ) Error!void { - try self.tupleImpl(val, options); - } - - fn tupleImpl(self: *Serializer, val: anytype, options: ValueOptions) Error!void { - comptime assert(canSerializeType(@TypeOf(val))); - switch (@typeInfo(@TypeOf(val))) { - .@"struct" => { - var container = try self.beginTuple(.{ .whitespace_style = .{ .fields = val.len } }); - inline for (val) |item_val| { - try container.fieldArbitraryDepth(item_val, options); - } - try container.end(); - }, - .pointer, .array => { - var container = try self.beginTuple(.{ .whitespace_style = .{ .fields = val.len } }); - for (val) |item_val| { - try container.fieldArbitraryDepth(item_val, options); - } - try container.end(); - }, - else => comptime unreachable, - } - } - - /// Like `value`, but always serializes `val` as a string. - pub fn string(self: *Serializer, val: []const u8) Error!void { - try self.writer.print("\"{f}\"", .{std.zig.fmtString(val)}); - } - - /// Options for formatting multiline strings. - pub const MultilineStringOptions = struct { - /// If top level is true, whitespace before and after the multiline string is elided. - /// If it is true, a newline is printed, then the value, followed by a newline, and if - /// whitespace is true any necessary indentation follows. - top_level: bool = false, - }; - - pub const MultilineStringError = Error || error{InnerCarriageReturn}; - - /// Like `value`, but always serializes to a multiline string literal. - /// - /// Returns `error.InnerCarriageReturn` if `val` contains a CR not followed by a newline, - /// since multiline strings cannot represent CR without a following newline. - pub fn multilineString( - self: *Serializer, - val: []const u8, - options: MultilineStringOptions, - ) MultilineStringError!void { - // Make sure the string does not contain any carriage returns not followed by a newline - var i: usize = 0; - while (i < val.len) : (i += 1) { - if (val[i] == '\r') { - if (i + 1 < val.len) { - if (val[i + 1] == '\n') { - i += 1; - continue; - } - } - return error.InnerCarriageReturn; - } - } - - if (!options.top_level) { - try self.newline(); - try self.indent(); - } - - try self.writer.writeAll("\\\\"); - for (val) |c| { - if (c != '\r') { - try self.writer.writeByte(c); // We write newlines here even if whitespace off - if (c == '\n') { - try self.indent(); - try self.writer.writeAll("\\\\"); - } - } - } - - if (!options.top_level) { - try self.writer.writeByte('\n'); // Even if whitespace off - try self.indent(); - } - } - - /// Create a `Struct` for writing ZON structs field by field. - pub fn beginStruct( - self: *Serializer, - options: SerializeContainerOptions, - ) Error!Struct { - return Struct.begin(self, options); - } - - /// Creates a `Tuple` for writing ZON tuples field by field. - pub fn beginTuple( - self: *Serializer, - options: SerializeContainerOptions, - ) Error!Tuple { - return Tuple.begin(self, options); - } - - fn indent(self: *Serializer) Error!void { - if (self.options.whitespace) { - try self.writer.splatByteAll(' ', 4 * self.indent_level); - } - } - - fn newline(self: *Serializer) Error!void { - if (self.options.whitespace) { - try self.writer.writeByte('\n'); - } - } - - fn newlineOrSpace(self: *Serializer, len: usize) Error!void { - if (self.containerShouldWrap(len)) { - try self.newline(); - } else { - try self.space(); - } - } - - fn space(self: *Serializer) Error!void { - if (self.options.whitespace) { - try self.writer.writeByte(' '); - } - } - - /// Writes ZON tuples field by field. - pub const Tuple = struct { - container: Container, - - fn begin(parent: *Serializer, options: SerializeContainerOptions) Error!Tuple { - return .{ - .container = try Container.begin(parent, .anon, options), - }; - } - - /// Finishes serializing the tuple. - /// - /// Prints a trailing comma as configured when appropriate, and the closing bracket. - pub fn end(self: *Tuple) Error!void { - try self.container.end(); - self.* = undefined; - } - - /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `value`. - pub fn field( - self: *Tuple, - val: anytype, - options: ValueOptions, - ) Error!void { - try self.container.field(null, val, options); - } - - /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `valueMaxDepth`. - /// Returns `error.ExceededMaxDepth` if `depth` is exceeded. - pub fn fieldMaxDepth( - self: *Tuple, - val: anytype, - options: ValueOptions, - depth: usize, - ) DepthError!void { - try self.container.fieldMaxDepth(null, val, options, depth); - } - - /// Serialize a field. Equivalent to calling `fieldPrefix` followed by - /// `valueArbitraryDepth`. - pub fn fieldArbitraryDepth( - self: *Tuple, - val: anytype, - options: ValueOptions, - ) Error!void { - try self.container.fieldArbitraryDepth(null, val, options); - } - - /// Starts a field with a struct as a value. Returns the struct. - pub fn beginStructField( - self: *Tuple, - options: SerializeContainerOptions, - ) Error!Struct { - try self.fieldPrefix(); - return self.container.serializer.beginStruct(options); - } - - /// Starts a field with a tuple as a value. Returns the tuple. - pub fn beginTupleField( - self: *Tuple, - options: SerializeContainerOptions, - ) Error!Tuple { - try self.fieldPrefix(); - return self.container.serializer.beginTuple(options); - } - - /// Print a field prefix. This prints any necessary commas, and whitespace as - /// configured. Useful if you want to serialize the field value yourself. - pub fn fieldPrefix(self: *Tuple) Error!void { - try self.container.fieldPrefix(null); - } - }; - - /// Writes ZON structs field by field. - pub const Struct = struct { - container: Container, - - fn begin(parent: *Serializer, options: SerializeContainerOptions) Error!Struct { - return .{ - .container = try Container.begin(parent, .named, options), - }; - } - - /// Finishes serializing the struct. - /// - /// Prints a trailing comma as configured when appropriate, and the closing bracket. - pub fn end(self: *Struct) Error!void { - try self.container.end(); - self.* = undefined; - } - - /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `value`. - pub fn field( - self: *Struct, - name: []const u8, - val: anytype, - options: ValueOptions, - ) Error!void { - try self.container.field(name, val, options); - } - - /// Serialize a field. Equivalent to calling `fieldPrefix` followed by `valueMaxDepth`. - /// Returns `error.ExceededMaxDepth` if `depth` is exceeded. - pub fn fieldMaxDepth( - self: *Struct, - name: []const u8, - val: anytype, - options: ValueOptions, - depth: usize, - ) DepthError!void { - try self.container.fieldMaxDepth(name, val, options, depth); - } - - /// Serialize a field. Equivalent to calling `fieldPrefix` followed by - /// `valueArbitraryDepth`. - pub fn fieldArbitraryDepth( - self: *Struct, - name: []const u8, - val: anytype, - options: ValueOptions, - ) Error!void { - try self.container.fieldArbitraryDepth(name, val, options); - } - - /// Starts a field with a struct as a value. Returns the struct. - pub fn beginStructField( - self: *Struct, - name: []const u8, - options: SerializeContainerOptions, - ) Error!Struct { - try self.fieldPrefix(name); - return self.container.serializer.beginStruct(options); - } - - /// Starts a field with a tuple as a value. Returns the tuple. - pub fn beginTupleField( - self: *Struct, - name: []const u8, - options: SerializeContainerOptions, - ) Error!Tuple { - try self.fieldPrefix(name); - return self.container.serializer.beginTuple(options); - } - - /// Print a field prefix. This prints any necessary commas, the field name (escaped if - /// necessary) and whitespace as configured. Useful if you want to serialize the field - /// value yourself. - pub fn fieldPrefix(self: *Struct, name: []const u8) Error!void { - try self.container.fieldPrefix(name); - } - }; - - const Container = struct { - const FieldStyle = enum { named, anon }; - - serializer: *Serializer, - field_style: FieldStyle, - options: SerializeContainerOptions, - empty: bool, - - fn begin( - sz: *Serializer, - field_style: FieldStyle, - options: SerializeContainerOptions, - ) Error!Container { - if (options.shouldWrap()) sz.indent_level +|= 1; - try sz.writer.writeAll(".{"); - return .{ - .serializer = sz, - .field_style = field_style, - .options = options, - .empty = true, - }; - } - - fn end(self: *Container) Error!void { - if (self.options.shouldWrap()) self.serializer.indent_level -|= 1; - if (!self.empty) { - if (self.options.shouldWrap()) { - if (self.serializer.options.whitespace) { - try self.serializer.writer.writeByte(','); - } - try self.serializer.newline(); - try self.serializer.indent(); - } else if (!self.shouldElideSpaces()) { - try self.serializer.space(); - } - } - try self.serializer.writer.writeByte('}'); - self.* = undefined; - } - - fn fieldPrefix(self: *Container, name: ?[]const u8) Error!void { - if (!self.empty) { - try self.serializer.writer.writeByte(','); - } - self.empty = false; - if (self.options.shouldWrap()) { - try self.serializer.newline(); - } else if (!self.shouldElideSpaces()) { - try self.serializer.space(); - } - if (self.options.shouldWrap()) try self.serializer.indent(); - if (name) |n| { - try self.serializer.ident(n); - try self.serializer.space(); - try self.serializer.writer.writeByte('='); - try self.serializer.space(); - } - } - - fn field( - self: *Container, - name: ?[]const u8, - val: anytype, - options: ValueOptions, - ) Error!void { - comptime assert(!typeIsRecursive(@TypeOf(val))); - try self.fieldArbitraryDepth(name, val, options); - } - - /// Returns `error.ExceededMaxDepth` if `depth` is exceeded. - fn fieldMaxDepth( - self: *Container, - name: ?[]const u8, - val: anytype, - options: ValueOptions, - depth: usize, - ) DepthError!void { - try checkValueDepth(val, depth); - try self.fieldArbitraryDepth(name, val, options); - } - - fn fieldArbitraryDepth( - self: *Container, - name: ?[]const u8, - val: anytype, - options: ValueOptions, - ) Error!void { - try self.fieldPrefix(name); - try self.serializer.valueArbitraryDepth(val, options); - } - - fn shouldElideSpaces(self: *const Container) bool { - return switch (self.options.whitespace_style) { - .fields => |fields| self.field_style != .named and fields == 1, - else => false, - }; - } - }; -}; - -test Serializer { - var discarding: Writer.Discarding = .init(&.{}); - var s: Serializer = .{ .writer = &discarding.writer }; - var vec2 = try s.beginStruct(.{}); - try vec2.field("x", 1.5, .{}); - try vec2.fieldPrefix("prefix"); - try s.value(2.5, .{}); - try vec2.end(); -} - fn expectSerializeEqual( expected: []const u8, value: anytype, diff --git a/src/main.zig b/src/main.zig index 019ae8bc7e..2eb5762a0c 100644 --- a/src/main.zig +++ b/src/main.zig @@ -344,8 +344,9 @@ fn mainArgs(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { } else if (mem.eql(u8, cmd, "targets")) { dev.check(.targets_command); const host = std.zig.resolveTargetQueryOrFatal(.{}); - const stdout = fs.File.stdout().deprecatedWriter(); - return @import("print_targets.zig").cmdTargets(arena, cmd_args, stdout, &host); + var stdout_writer = fs.File.stdout().writer(&stdio_buffer); + try @import("print_targets.zig").cmdTargets(arena, cmd_args, &stdout_writer.interface, &host); + return stdout_writer.interface.flush(); } else if (mem.eql(u8, cmd, "version")) { dev.check(.version_command); try fs.File.stdout().writeAll(build_options.version ++ "\n"); @@ -356,7 +357,9 @@ fn mainArgs(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { } else if (mem.eql(u8, cmd, "env")) { dev.check(.env_command); verifyLibcxxCorrectlyLinked(); - return @import("print_env.zig").cmdEnv(arena, cmd_args); + var stdout_writer = fs.File.stdout().writer(&stdio_buffer); + try @import("print_env.zig").cmdEnv(arena, &stdout_writer.interface); + return stdout_writer.interface.flush(); } else if (mem.eql(u8, cmd, "reduce")) { return jitCmd(gpa, arena, cmd_args, .{ .cmd_name = "reduce", diff --git a/src/print_env.zig b/src/print_env.zig index d1546abc6c..d1251c0d62 100644 --- a/src/print_env.zig +++ b/src/print_env.zig @@ -4,8 +4,7 @@ const introspect = @import("introspect.zig"); const Allocator = std.mem.Allocator; const fatal = std.process.fatal; -pub fn cmdEnv(arena: Allocator, args: []const []const u8) !void { - _ = args; +pub fn cmdEnv(arena: Allocator, out: *std.Io.Writer) !void { const cwd_path = try introspect.getResolvedCwd(arena); const self_exe_path = try std.fs.selfExePathAlloc(arena); @@ -21,41 +20,21 @@ pub fn cmdEnv(arena: Allocator, args: []const []const u8) !void { const host = try std.zig.system.resolveTargetQuery(.{}); const triple = try host.zigTriple(arena); - var buffer: [1024]u8 = undefined; - var stdout_writer = std.fs.File.stdout().writer(&buffer); + var serializer: std.zon.Serializer = .{ .writer = out }; + var root = try serializer.beginStruct(.{}); - var jws: std.json.Stringify = .{ .writer = &stdout_writer.interface, .options = .{ .whitespace = .indent_1 } }; - - try jws.beginObject(); - - try jws.objectField("zig_exe"); - try jws.write(self_exe_path); - - try jws.objectField("lib_dir"); - try jws.write(zig_lib_directory.path.?); - - try jws.objectField("std_dir"); - try jws.write(zig_std_dir); - - try jws.objectField("global_cache_dir"); - try jws.write(global_cache_dir); - - try jws.objectField("version"); - try jws.write(build_options.version); - - try jws.objectField("target"); - try jws.write(triple); - - try jws.objectField("env"); - try jws.beginObject(); + try root.field("zig_exe", self_exe_path, .{}); + try root.field("lib_dir", zig_lib_directory.path.?, .{}); + try root.field("std_dir", zig_std_dir, .{}); + try root.field("global_cache_dir", global_cache_dir, .{}); + try root.field("version", build_options.version, .{}); + try root.field("target", triple, .{}); + var env = try root.beginStructField("env", .{}); inline for (@typeInfo(std.zig.EnvVar).@"enum".fields) |field| { - try jws.objectField(field.name); - try jws.write(try @field(std.zig.EnvVar, field.name).get(arena)); + try env.field(field.name, try @field(std.zig.EnvVar, field.name).get(arena), .{}); } - try jws.endObject(); + try env.end(); + try root.end(); - try jws.endObject(); - - try stdout_writer.interface.writeByte('\n'); - try stdout_writer.interface.flush(); + try out.writeByte('\n'); } diff --git a/src/print_targets.zig b/src/print_targets.zig index e234aeda26..e04842bb7e 100644 --- a/src/print_targets.zig +++ b/src/print_targets.zig @@ -14,8 +14,7 @@ const introspect = @import("introspect.zig"); pub fn cmdTargets( allocator: Allocator, args: []const []const u8, - /// Output stream - stdout: anytype, + out: *std.Io.Writer, native_target: *const Target, ) !void { _ = args; @@ -38,12 +37,10 @@ pub fn cmdTargets( const glibc_abi = try glibc.loadMetaData(allocator, abilists_contents); defer glibc_abi.destroy(allocator); - var bw = io.bufferedWriter(stdout); - const w = bw.writer(); - var sz = std.zon.stringify.serializer(w, .{}); + var serializer: std.zon.Serializer = .{ .writer = out }; { - var root_obj = try sz.beginStruct(.{}); + var root_obj = try serializer.beginStruct(.{}); try root_obj.field("arch", meta.fieldNames(Target.Cpu.Arch), .{}); try root_obj.field("os", meta.fieldNames(Target.Os.Tag), .{}); @@ -136,6 +133,5 @@ pub fn cmdTargets( try root_obj.end(); } - try w.writeByte('\n'); - return bw.flush(); + try out.writeByte('\n'); } diff --git a/src/translate_c.zig b/src/translate_c.zig index 301e0a219d..f1f3ad8659 100644 --- a/src/translate_c.zig +++ b/src/translate_c.zig @@ -3338,7 +3338,7 @@ fn transPredefinedExpr(c: *Context, scope: *Scope, expr: *const clang.Predefined fn transCreateCharLitNode(c: *Context, narrow: bool, val: u32) TransError!Node { return Tag.char_literal.create(c.arena, if (narrow) - try std.fmt.allocPrint(c.arena, "'{f}'", .{std.zig.fmtChar(&.{@as(u8, @intCast(val))})}) + try std.fmt.allocPrint(c.arena, "'{f}'", .{std.zig.fmtChar(@intCast(val))}) else try std.fmt.allocPrint(c.arena, "'\\u{{{x}}}'", .{val})); } From 592f1043dc704faf79bde7149917d4155b3747f8 Mon Sep 17 00:00:00 2001 From: Jacob Young Date: Sat, 19 Jul 2025 13:54:31 -0400 Subject: [PATCH 10/26] cbe: fix comptime-known packed unions --- src/codegen/c.zig | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/codegen/c.zig b/src/codegen/c.zig index 955a83bc33..c726c05e1b 100644 --- a/src/codegen/c.zig +++ b/src/codegen/c.zig @@ -2438,7 +2438,10 @@ pub const DeclGen = struct { const ty = val.typeOf(zcu); return .{ .data = .{ .dg = dg, - .int_info = ty.intInfo(zcu), + .int_info = if (ty.zigTypeTag(zcu) == .@"union" and ty.containerLayout(zcu) == .@"packed") + .{ .signedness = .unsigned, .bits = @intCast(ty.bitSize(zcu)) } + else + ty.intInfo(zcu), .kind = kind, .ctype = try dg.ctypeFromType(ty, kind), .val = val, From 93378e2e7be964a7362a9875e3292e9d6a697177 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Wed, 16 Jul 2025 19:47:58 -0700 Subject: [PATCH 11/26] std.zig: finish updating to new I/O API --- lib/compiler/objcopy.zig | 18 +- lib/compiler/resinator/main.zig | 6 +- lib/compiler/test_runner.zig | 11 +- lib/std/zig.zig | 1 + lib/std/zig/LibCInstallation.zig | 7 +- lib/std/zig/Server.zig | 214 +++++++---------------- lib/std/zig/WindowsSdk.zig | 14 +- lib/std/zig/llvm.zig | 6 + lib/std/zig/llvm/BitcodeReader.zig | 28 +-- lib/std/zig/perf_test.zig | 14 +- lib/std/zig/system/linux.zig | 35 ++-- src/Compilation.zig | 85 +++++----- src/deprecated.zig | 262 ----------------------------- src/main.zig | 46 ++--- 14 files changed, 201 insertions(+), 546 deletions(-) diff --git a/lib/compiler/objcopy.zig b/lib/compiler/objcopy.zig index 98bf5d2bcf..52ffe208f6 100644 --- a/lib/compiler/objcopy.zig +++ b/lib/compiler/objcopy.zig @@ -10,6 +10,9 @@ const assert = std.debug.assert; const fatal = std.process.fatal; const Server = std.zig.Server; +var stdin_buffer: [1024]u8 = undefined; +var stdout_buffer: [1024]u8 = undefined; + pub fn main() !void { var arena_instance = std.heap.ArenaAllocator.init(std.heap.page_allocator); defer arena_instance.deinit(); @@ -22,11 +25,8 @@ pub fn main() !void { return cmdObjCopy(gpa, arena, args[1..]); } -fn cmdObjCopy( - gpa: Allocator, - arena: Allocator, - args: []const []const u8, -) !void { +fn cmdObjCopy(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { + _ = gpa; var i: usize = 0; var opt_out_fmt: ?std.Target.ObjectFormat = null; var opt_input: ?[]const u8 = null; @@ -225,13 +225,13 @@ fn cmdObjCopy( } if (listen) { + var stdin_reader = fs.File.stdin().reader(&stdin_buffer); + var stdout_writer = fs.File.stdout().writer(&stdout_buffer); var server = try Server.init(.{ - .gpa = gpa, - .in = .stdin(), - .out = .stdout(), + .in = &stdin_reader.interface, + .out = &stdout_writer.interface, .zig_version = builtin.zig_version_string, }); - defer server.deinit(); var seen_update = false; while (true) { diff --git a/lib/compiler/resinator/main.zig b/lib/compiler/resinator/main.zig index 903e0a2f71..4c952c03c4 100644 --- a/lib/compiler/resinator/main.zig +++ b/lib/compiler/resinator/main.zig @@ -13,6 +13,8 @@ const hasDisjointCodePage = @import("disjoint_code_page.zig").hasDisjointCodePag const fmtResourceType = @import("res.zig").NameOrOrdinal.fmtResourceType; const aro = @import("aro"); +var stdout_buffer: [1024]u8 = undefined; + pub fn main() !void { var gpa: std.heap.GeneralPurposeAllocator(.{}) = .init; defer std.debug.assert(gpa.deinit() == .ok); @@ -41,12 +43,12 @@ pub fn main() !void { cli_args = args[3..]; } + var stdout_writer2 = std.fs.File.stdout().writer(&stdout_buffer); var error_handler: ErrorHandler = switch (zig_integration) { true => .{ .server = .{ - .out = std.fs.File.stdout(), + .out = &stdout_writer2.interface, .in = undefined, // won't be receiving messages - .receive_fifo = undefined, // won't be receiving messages }, }, false => .{ diff --git a/lib/compiler/test_runner.zig b/lib/compiler/test_runner.zig index 929bd1c417..a69066f09c 100644 --- a/lib/compiler/test_runner.zig +++ b/lib/compiler/test_runner.zig @@ -2,7 +2,6 @@ const builtin = @import("builtin"); const std = @import("std"); -const io = std.io; const testing = std.testing; const assert = std.debug.assert; @@ -13,6 +12,8 @@ pub const std_options: std.Options = .{ var log_err_count: usize = 0; var fba_buffer: [8192]u8 = undefined; var fba = std.heap.FixedBufferAllocator.init(&fba_buffer); +var stdin_buffer: [std.heap.page_size_min]u8 align(std.heap.page_size_min) = undefined; +var stdout_buffer: [std.heap.page_size_min]u8 align(std.heap.page_size_min) = undefined; const crippled = switch (builtin.zig_backend) { .stage2_powerpc, @@ -67,13 +68,13 @@ pub fn main() void { fn mainServer() !void { @disableInstrumentation(); + var stdin_reader = std.fs.File.stdin().reader(&stdin_buffer); + var stdout_writer = std.fs.File.stdout().writer(&stdout_buffer); var server = try std.zig.Server.init(.{ - .gpa = fba.allocator(), - .in = .stdin(), - .out = .stdout(), + .in = &stdin_reader.interface, + .out = &stdout_writer.interface, .zig_version = builtin.zig_version_string, }); - defer server.deinit(); if (builtin.fuzz) { const coverage_id = fuzzer_coverage_id(); diff --git a/lib/std/zig.zig b/lib/std/zig.zig index 51eac1b0a6..ad264a9b33 100644 --- a/lib/std/zig.zig +++ b/lib/std/zig.zig @@ -905,4 +905,5 @@ test { _ = system; _ = target; _ = c_translation; + _ = llvm; } diff --git a/lib/std/zig/LibCInstallation.zig b/lib/std/zig/LibCInstallation.zig index e2b7cd233d..3e03d1ff2e 100644 --- a/lib/std/zig/LibCInstallation.zig +++ b/lib/std/zig/LibCInstallation.zig @@ -370,7 +370,7 @@ fn findNativeIncludeDirWindows( for (installs) |install| { result_buf.shrinkAndFree(0); - try result_buf.writer().print("{s}\\Include\\{s}\\ucrt", .{ install.path, install.version }); + try result_buf.print("{s}\\Include\\{s}\\ucrt", .{ install.path, install.version }); var dir = fs.cwd().openDir(result_buf.items, .{}) catch |err| switch (err) { error.FileNotFound, @@ -417,7 +417,7 @@ fn findNativeCrtDirWindows( for (installs) |install| { result_buf.shrinkAndFree(0); - try result_buf.writer().print("{s}\\Lib\\{s}\\ucrt\\{s}", .{ install.path, install.version, arch_sub_dir }); + try result_buf.print("{s}\\Lib\\{s}\\ucrt\\{s}", .{ install.path, install.version, arch_sub_dir }); var dir = fs.cwd().openDir(result_buf.items, .{}) catch |err| switch (err) { error.FileNotFound, @@ -484,8 +484,7 @@ fn findNativeKernel32LibDir( for (installs) |install| { result_buf.shrinkAndFree(0); - const stream = result_buf.writer(); - try stream.print("{s}\\Lib\\{s}\\um\\{s}", .{ install.path, install.version, arch_sub_dir }); + try result_buf.print("{s}\\Lib\\{s}\\um\\{s}", .{ install.path, install.version, arch_sub_dir }); var dir = fs.cwd().openDir(result_buf.items, .{}) catch |err| switch (err) { error.FileNotFound, diff --git a/lib/std/zig/Server.zig b/lib/std/zig/Server.zig index 2f03c78083..38ad45e1e1 100644 --- a/lib/std/zig/Server.zig +++ b/lib/std/zig/Server.zig @@ -1,6 +1,20 @@ -in: std.fs.File, -out: std.fs.File, -receive_fifo: std.fifo.LinearFifo(u8, .Dynamic), +const Server = @This(); + +const builtin = @import("builtin"); + +const std = @import("std"); +const Allocator = std.mem.Allocator; +const assert = std.debug.assert; +const native_endian = builtin.target.cpu.arch.endian(); +const need_bswap = native_endian != .little; +const Cache = std.Build.Cache; +const OutMessage = std.zig.Server.Message; +const InMessage = std.zig.Client.Message; +const Reader = std.Io.Reader; +const Writer = std.Io.Writer; + +in: *Reader, +out: *Writer, pub const Message = struct { pub const Header = extern struct { @@ -94,9 +108,8 @@ pub const Message = struct { }; pub const Options = struct { - gpa: Allocator, - in: std.fs.File, - out: std.fs.File, + in: *Reader, + out: *Writer, zig_version: []const u8, }; @@ -104,96 +117,40 @@ pub fn init(options: Options) !Server { var s: Server = .{ .in = options.in, .out = options.out, - .receive_fifo = std.fifo.LinearFifo(u8, .Dynamic).init(options.gpa), }; try s.serveStringMessage(.zig_version, options.zig_version); return s; } -pub fn deinit(s: *Server) void { - s.receive_fifo.deinit(); - s.* = undefined; -} - pub fn receiveMessage(s: *Server) !InMessage.Header { - const Header = InMessage.Header; - const fifo = &s.receive_fifo; - var last_amt_zero = false; - - while (true) { - const buf = fifo.readableSlice(0); - assert(fifo.readableLength() == buf.len); - if (buf.len >= @sizeOf(Header)) { - const header: *align(1) const Header = @ptrCast(buf[0..@sizeOf(Header)]); - const bytes_len = bswap(header.bytes_len); - const tag = bswap(header.tag); - - if (buf.len - @sizeOf(Header) >= bytes_len) { - fifo.discard(@sizeOf(Header)); - return .{ - .tag = tag, - .bytes_len = bytes_len, - }; - } else { - const needed = bytes_len - (buf.len - @sizeOf(Header)); - const write_buffer = try fifo.writableWithSize(needed); - const amt = try s.in.read(write_buffer); - fifo.update(amt); - continue; - } - } - - const write_buffer = try fifo.writableWithSize(256); - const amt = try s.in.read(write_buffer); - fifo.update(amt); - if (amt == 0) { - if (last_amt_zero) return error.BrokenPipe; - last_amt_zero = true; - } - } + return s.in.takeStruct(InMessage.Header, .little); } pub fn receiveBody_u32(s: *Server) !u32 { - const fifo = &s.receive_fifo; - const buf = fifo.readableSlice(0); - const result = @as(*align(1) const u32, @ptrCast(buf[0..4])).*; - fifo.discard(4); - return bswap(result); + return s.in.takeInt(u32, .little); } pub fn serveStringMessage(s: *Server, tag: OutMessage.Tag, msg: []const u8) !void { - return s.serveMessage(.{ + try s.serveMessageHeader(.{ .tag = tag, - .bytes_len = @as(u32, @intCast(msg.len)), - }, &.{msg}); + .bytes_len = @intCast(msg.len), + }); + try s.out.writeAll(msg); + try s.out.flush(); } -pub fn serveMessage( - s: *const Server, - header: OutMessage.Header, - bufs: []const []const u8, -) !void { - var iovecs: [10]std.posix.iovec_const = undefined; - const header_le = bswap(header); - iovecs[0] = .{ - .base = @as([*]const u8, @ptrCast(&header_le)), - .len = @sizeOf(OutMessage.Header), - }; - for (bufs, iovecs[1 .. bufs.len + 1]) |buf, *iovec| { - iovec.* = .{ - .base = buf.ptr, - .len = buf.len, - }; - } - try s.out.writevAll(iovecs[0 .. bufs.len + 1]); +/// Don't forget to flush! +pub fn serveMessageHeader(s: *const Server, header: OutMessage.Header) !void { + try s.out.writeStruct(header, .little); } -pub fn serveU64Message(s: *Server, tag: OutMessage.Tag, int: u64) !void { - const msg_le = bswap(int); - return s.serveMessage(.{ +pub fn serveU64Message(s: *const Server, tag: OutMessage.Tag, int: u64) !void { + try serveMessageHeader(s, .{ .tag = tag, .bytes_len = @sizeOf(u64), - }, &.{std.mem.asBytes(&msg_le)}); + }); + try s.out.writeInt(u64, int, .little); + try s.out.flush(); } pub fn serveEmitDigest( @@ -201,26 +158,22 @@ pub fn serveEmitDigest( digest: *const [Cache.bin_digest_len]u8, header: OutMessage.EmitDigest, ) !void { - try s.serveMessage(.{ + try s.serveMessageHeader(.{ .tag = .emit_digest, .bytes_len = @intCast(digest.len + @sizeOf(OutMessage.EmitDigest)), - }, &.{ - std.mem.asBytes(&header), - digest, }); + try s.out.writeStruct(header, .little); + try s.out.writeAll(digest); + try s.out.flush(); } -pub fn serveTestResults( - s: *Server, - msg: OutMessage.TestResults, -) !void { - const msg_le = bswap(msg); - try s.serveMessage(.{ +pub fn serveTestResults(s: *Server, msg: OutMessage.TestResults) !void { + try s.serveMessageHeader(.{ .tag = .test_results, .bytes_len = @intCast(@sizeOf(OutMessage.TestResults)), - }, &.{ - std.mem.asBytes(&msg_le), }); + try s.out.writeStruct(msg, .little); + try s.out.flush(); } pub fn serveErrorBundle(s: *Server, error_bundle: std.zig.ErrorBundle) !void { @@ -230,91 +183,38 @@ pub fn serveErrorBundle(s: *Server, error_bundle: std.zig.ErrorBundle) !void { }; const bytes_len = @sizeOf(OutMessage.ErrorBundle) + 4 * error_bundle.extra.len + error_bundle.string_bytes.len; - try s.serveMessage(.{ + try s.serveMessageHeader(.{ .tag = .error_bundle, .bytes_len = @intCast(bytes_len), - }, &.{ - std.mem.asBytes(&eb_hdr), - // TODO: implement @ptrCast between slices changing the length - std.mem.sliceAsBytes(error_bundle.extra), - error_bundle.string_bytes, }); + try s.out.writeStruct(eb_hdr, .little); + try s.out.writeSliceEndian(u32, error_bundle.extra, .little); + try s.out.writeAll(error_bundle.string_bytes); + try s.out.flush(); } pub const TestMetadata = struct { - names: []u32, - expected_panic_msgs: []u32, + names: []const u32, + expected_panic_msgs: []const u32, string_bytes: []const u8, }; pub fn serveTestMetadata(s: *Server, test_metadata: TestMetadata) !void { const header: OutMessage.TestMetadata = .{ - .tests_len = bswap(@as(u32, @intCast(test_metadata.names.len))), - .string_bytes_len = bswap(@as(u32, @intCast(test_metadata.string_bytes.len))), + .tests_len = @as(u32, @intCast(test_metadata.names.len)), + .string_bytes_len = @as(u32, @intCast(test_metadata.string_bytes.len)), }; const trailing = 2; const bytes_len = @sizeOf(OutMessage.TestMetadata) + trailing * @sizeOf(u32) * test_metadata.names.len + test_metadata.string_bytes.len; - if (need_bswap) { - bswap_u32_array(test_metadata.names); - bswap_u32_array(test_metadata.expected_panic_msgs); - } - defer if (need_bswap) { - bswap_u32_array(test_metadata.names); - bswap_u32_array(test_metadata.expected_panic_msgs); - }; - - return s.serveMessage(.{ + try s.serveMessageHeader(.{ .tag = .test_metadata, .bytes_len = @intCast(bytes_len), - }, &.{ - std.mem.asBytes(&header), - // TODO: implement @ptrCast between slices changing the length - std.mem.sliceAsBytes(test_metadata.names), - std.mem.sliceAsBytes(test_metadata.expected_panic_msgs), - test_metadata.string_bytes, }); + try s.out.writeStruct(header, .little); + try s.out.writeSliceEndian(u32, test_metadata.names, .little); + try s.out.writeSliceEndian(u32, test_metadata.expected_panic_msgs, .little); + try s.out.writeAll(test_metadata.string_bytes); + try s.out.flush(); } - -fn bswap(x: anytype) @TypeOf(x) { - if (!need_bswap) return x; - - const T = @TypeOf(x); - switch (@typeInfo(T)) { - .@"enum" => return @as(T, @enumFromInt(@byteSwap(@intFromEnum(x)))), - .int => return @byteSwap(x), - .@"struct" => |info| switch (info.layout) { - .@"extern" => { - var result: T = undefined; - inline for (info.fields) |field| { - @field(result, field.name) = bswap(@field(x, field.name)); - } - return result; - }, - .@"packed" => { - const I = info.backing_integer.?; - return @as(T, @bitCast(@byteSwap(@as(I, @bitCast(x))))); - }, - .auto => @compileError("auto layout struct"), - }, - else => @compileError("bswap on type " ++ @typeName(T)), - } -} - -fn bswap_u32_array(slice: []u32) void { - comptime assert(need_bswap); - for (slice) |*elem| elem.* = @byteSwap(elem.*); -} - -const OutMessage = std.zig.Server.Message; -const InMessage = std.zig.Client.Message; - -const Server = @This(); -const builtin = @import("builtin"); -const std = @import("std"); -const Allocator = std.mem.Allocator; -const assert = std.debug.assert; -const native_endian = builtin.target.cpu.arch.endian(); -const need_bswap = native_endian != .little; -const Cache = std.Build.Cache; diff --git a/lib/std/zig/WindowsSdk.zig b/lib/std/zig/WindowsSdk.zig index 1bcf45fb74..61e1defb12 100644 --- a/lib/std/zig/WindowsSdk.zig +++ b/lib/std/zig/WindowsSdk.zig @@ -1,11 +1,12 @@ +const WindowsSdk = @This(); +const builtin = @import("builtin"); +const std = @import("std"); +const Writer = std.Io.Writer; + windows10sdk: ?Installation, windows81sdk: ?Installation, msvc_lib_dir: ?[]const u8, -const WindowsSdk = @This(); -const std = @import("std"); -const builtin = @import("builtin"); - const windows = std.os.windows; const RRF = windows.advapi32.RRF; @@ -759,14 +760,13 @@ const MsvcLibDir = struct { while (instances_dir_it.next() catch return error.PathNotFound) |entry| { if (entry.kind != .directory) continue; - var fbs = std.io.fixedBufferStream(&state_subpath_buf); - const writer = fbs.writer(); + var writer: Writer = .fixed(&state_subpath_buf); writer.writeAll(entry.name) catch unreachable; writer.writeByte(std.fs.path.sep) catch unreachable; writer.writeAll("state.json") catch unreachable; - const json_contents = instances_dir.readFileAlloc(allocator, fbs.getWritten(), std.math.maxInt(usize)) catch continue; + const json_contents = instances_dir.readFileAlloc(allocator, writer.buffered(), std.math.maxInt(usize)) catch continue; defer allocator.free(json_contents); var parsed = std.json.parseFromSlice(std.json.Value, allocator, json_contents, .{}) catch continue; diff --git a/lib/std/zig/llvm.zig b/lib/std/zig/llvm.zig index c2e1ed9c56..c45ffe9083 100644 --- a/lib/std/zig/llvm.zig +++ b/lib/std/zig/llvm.zig @@ -1,3 +1,9 @@ pub const BitcodeReader = @import("llvm/BitcodeReader.zig"); pub const bitcode_writer = @import("llvm/bitcode_writer.zig"); pub const Builder = @import("llvm/Builder.zig"); + +test { + _ = BitcodeReader; + _ = bitcode_writer; + _ = Builder; +} diff --git a/lib/std/zig/llvm/BitcodeReader.zig b/lib/std/zig/llvm/BitcodeReader.zig index ea18b7978e..f1d47e93c0 100644 --- a/lib/std/zig/llvm/BitcodeReader.zig +++ b/lib/std/zig/llvm/BitcodeReader.zig @@ -1,6 +1,11 @@ +const BitcodeReader = @This(); + +const std = @import("../../std.zig"); +const assert = std.debug.assert; + allocator: std.mem.Allocator, record_arena: std.heap.ArenaAllocator.State, -reader: std.io.AnyReader, +reader: *std.Io.Reader, keep_names: bool, bit_buffer: u32, bit_offset: u5, @@ -93,7 +98,7 @@ pub const Record = struct { }; pub const InitOptions = struct { - reader: std.io.AnyReader, + reader: *std.Io.Reader, keep_names: bool = false, }; pub fn init(allocator: std.mem.Allocator, options: InitOptions) BitcodeReader { @@ -172,7 +177,7 @@ pub fn next(bc: *BitcodeReader) !?Item { pub fn skipBlock(bc: *BitcodeReader, block: Block) !void { assert(bc.bit_offset == 0); - try bc.reader.skipBytes(@as(u34, block.len) * 4, .{}); + try bc.reader.discardAll(4 * @as(u34, block.len)); try bc.endBlock(); } @@ -371,19 +376,19 @@ fn align32Bits(bc: *BitcodeReader) void { fn read32Bits(bc: *BitcodeReader) !u32 { assert(bc.bit_offset == 0); - return bc.reader.readInt(u32, .little); + return bc.reader.takeInt(u32, .little); } fn readBytes(bc: *BitcodeReader, bytes: []u8) !void { assert(bc.bit_offset == 0); - try bc.reader.readNoEof(bytes); + try bc.reader.readSliceAll(bytes); const trailing_bytes = bytes.len % 4; if (trailing_bytes > 0) { - var bit_buffer = [1]u8{0} ** 4; - try bc.reader.readNoEof(bit_buffer[trailing_bytes..]); + var bit_buffer: [4]u8 = @splat(0); + try bc.reader.readSliceAll(bit_buffer[trailing_bytes..]); bc.bit_buffer = std.mem.readInt(u32, &bit_buffer, .little); - bc.bit_offset = @intCast(trailing_bytes * 8); + bc.bit_offset = @intCast(8 * trailing_bytes); } } @@ -509,7 +514,6 @@ const Abbrev = struct { }; }; -const assert = std.debug.assert; -const std = @import("../../std.zig"); - -const BitcodeReader = @This(); +test { + _ = &skipBlock; +} diff --git a/lib/std/zig/perf_test.zig b/lib/std/zig/perf_test.zig index 087b081475..1566a15d2d 100644 --- a/lib/std/zig/perf_test.zig +++ b/lib/std/zig/perf_test.zig @@ -1,7 +1,6 @@ const std = @import("std"); const mem = std.mem; const Tokenizer = std.zig.Tokenizer; -const io = std.io; const fmtIntSizeBin = std.fmt.fmtIntSizeBin; const source = @embedFile("../os.zig"); @@ -22,16 +21,15 @@ pub fn main() !void { const bytes_per_sec_float = @as(f64, @floatFromInt(source.len * iterations)) / elapsed_s; const bytes_per_sec = @as(u64, @intFromFloat(@floor(bytes_per_sec_float))); - var stdout_file: std.fs.File = .stdout(); - const stdout = stdout_file.deprecatedWriter(); - try stdout.print("parsing speed: {:.2}/s, {:.2} used \n", .{ - fmtIntSizeBin(bytes_per_sec), - fmtIntSizeBin(memory_used), - }); + var stdout_buffer: [1024]u8 = undefined; + var stdout_writer = std.fs.File.stdout().writer(&stdout_buffer); + const stdout = &stdout_writer.interface; + try stdout.print("parsing speed: {Bi:.2}/s, {Bi:.2} used \n", .{ bytes_per_sec, memory_used }); + try stdout.flush(); } fn testOnce() usize { - var fixed_buf_alloc = std.heap.FixedBufferAllocator.init(fixed_buffer_mem[0..]); + var fixed_buf_alloc = std.heap.FixedBufferAllocator.init(&fixed_buffer_mem); const allocator = fixed_buf_alloc.allocator(); _ = std.zig.Ast.parse(allocator, source, .zig) catch @panic("parse failure"); return fixed_buf_alloc.end_index; diff --git a/lib/std/zig/system/linux.zig b/lib/std/zig/system/linux.zig index 8044e1969d..df70e71f5b 100644 --- a/lib/std/zig/system/linux.zig +++ b/lib/std/zig/system/linux.zig @@ -1,7 +1,6 @@ const std = @import("std"); const builtin = @import("builtin"); const mem = std.mem; -const io = std.io; const fs = std.fs; const fmt = std.fmt; const testing = std.testing; @@ -344,8 +343,8 @@ fn testParser( expected_model: *const Target.Cpu.Model, input: []const u8, ) !void { - var fbs = io.fixedBufferStream(input); - const result = try parser.parse(arch, fbs.reader()); + var r: std.Io.Reader = .fixed(input); + const result = try parser.parse(arch, &r); try testing.expectEqual(expected_model, result.?.model); try testing.expect(expected_model.features.eql(result.?.features)); } @@ -357,20 +356,17 @@ fn testParser( // When all the lines have been analyzed the finalize method is called. fn CpuinfoParser(comptime impl: anytype) type { return struct { - fn parse(arch: Target.Cpu.Arch, reader: anytype) anyerror!?Target.Cpu { - var line_buf: [1024]u8 = undefined; + fn parse(arch: Target.Cpu.Arch, reader: *std.Io.Reader) !?Target.Cpu { var obj: impl = .{}; - - while (true) { - const line = (try reader.readUntilDelimiterOrEof(&line_buf, '\n')) orelse break; + while (reader.takeDelimiterExclusive('\n')) |line| { const colon_pos = mem.indexOfScalar(u8, line, ':') orelse continue; const key = mem.trimEnd(u8, line[0..colon_pos], " \t"); const value = mem.trimStart(u8, line[colon_pos + 1 ..], " \t"); - - if (!try obj.line_hook(key, value)) - break; + if (!try obj.line_hook(key, value)) break; + } else |err| switch (err) { + error.EndOfStream => {}, + else => |e| return e, } - return obj.finalize(arch); } }; @@ -383,15 +379,18 @@ inline fn getAArch64CpuFeature(comptime feat_reg: []const u8) u64 { } pub fn detectNativeCpuAndFeatures() ?Target.Cpu { - var f = fs.openFileAbsolute("/proc/cpuinfo", .{}) catch |err| switch (err) { + var file = fs.openFileAbsolute("/proc/cpuinfo", .{}) catch |err| switch (err) { else => return null, }; - defer f.close(); + defer file.close(); + + var buffer: [4096]u8 = undefined; // "flags" lines can get pretty long. + var file_reader = file.reader(&buffer); const current_arch = builtin.cpu.arch; switch (current_arch) { .arm, .armeb, .thumb, .thumbeb => { - return ArmCpuinfoParser.parse(current_arch, f.deprecatedReader()) catch null; + return ArmCpuinfoParser.parse(current_arch, &file_reader.interface) catch null; }, .aarch64, .aarch64_be => { const registers = [12]u64{ @@ -413,13 +412,13 @@ pub fn detectNativeCpuAndFeatures() ?Target.Cpu { return core; }, .sparc64 => { - return SparcCpuinfoParser.parse(current_arch, f.deprecatedReader()) catch null; + return SparcCpuinfoParser.parse(current_arch, &file_reader.interface) catch null; }, .powerpc, .powerpcle, .powerpc64, .powerpc64le => { - return PowerpcCpuinfoParser.parse(current_arch, f.deprecatedReader()) catch null; + return PowerpcCpuinfoParser.parse(current_arch, &file_reader.interface) catch null; }, .riscv64, .riscv32 => { - return RiscvCpuinfoParser.parse(current_arch, f.deprecatedReader()) catch null; + return RiscvCpuinfoParser.parse(current_arch, &file_reader.interface) catch null; }, else => {}, } diff --git a/src/Compilation.zig b/src/Compilation.zig index 2e20c6edbe..469a13b4a9 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -12,6 +12,7 @@ const ThreadPool = std.Thread.Pool; const WaitGroup = std.Thread.WaitGroup; const ErrorBundle = std.zig.ErrorBundle; const fatal = std.process.fatal; +const Writer = std.io.Writer; const Value = @import("Value.zig"); const Type = @import("Type.zig"); @@ -44,6 +45,8 @@ const Builtin = @import("Builtin.zig"); const LlvmObject = @import("codegen/llvm.zig").Object; const dev = @import("dev.zig"); +const DeprecatedLinearFifo = @import("deprecated.zig").LinearFifo; + pub const Config = @import("Compilation/Config.zig"); /// General-purpose allocator. Used for both temporary and long-term storage. @@ -121,15 +124,15 @@ work_queues: [ } break :len len; } -]std.fifo.LinearFifo(Job, .Dynamic), +]DeprecatedLinearFifo(Job), /// These jobs are to invoke the Clang compiler to create an object file, which /// gets linked with the Compilation. -c_object_work_queue: std.fifo.LinearFifo(*CObject, .Dynamic), +c_object_work_queue: DeprecatedLinearFifo(*CObject), /// These jobs are to invoke the RC compiler to create a compiled resource file (.res), which /// gets linked with the Compilation. -win32_resource_work_queue: if (dev.env.supports(.win32_resource)) std.fifo.LinearFifo(*Win32Resource, .Dynamic) else struct { +win32_resource_work_queue: if (dev.env.supports(.win32_resource)) DeprecatedLinearFifo(*Win32Resource) else struct { pub fn ensureUnusedCapacity(_: @This(), _: u0) error{}!void {} pub fn readItem(_: @This()) ?noreturn { return null; @@ -995,13 +998,13 @@ pub const CObject = struct { const file = fs.cwd().openFile(file_name, .{}) catch break :source_line 0; defer file.close(); - file.seekTo(diag.src_loc.offset + 1 - diag.src_loc.column) catch break :source_line 0; - - var line = std.ArrayList(u8).init(eb.gpa); - defer line.deinit(); - file.deprecatedReader().readUntilDelimiterArrayList(&line, '\n', 1 << 10) catch break :source_line 0; - - break :source_line try eb.addString(line.items); + var buffer: [1024]u8 = undefined; + var file_reader = file.reader(&buffer); + file_reader.seekTo(diag.src_loc.offset + 1 - diag.src_loc.column) catch break :source_line 0; + var aw: Writer.Allocating = .init(eb.gpa); + defer aw.deinit(); + _ = file_reader.interface.streamDelimiterEnding(&aw.writer, '\n') catch break :source_line 0; + break :source_line try eb.addString(aw.getWritten()); }; return .{ @@ -1067,11 +1070,11 @@ pub const CObject = struct { } }; + var buffer: [1024]u8 = undefined; const file = try fs.cwd().openFile(path, .{}); defer file.close(); - var br = std.io.bufferedReader(file.deprecatedReader()); - const reader = br.reader(); - var bc = std.zig.llvm.BitcodeReader.init(gpa, .{ .reader = reader.any() }); + var file_reader = file.reader(&buffer); + var bc = std.zig.llvm.BitcodeReader.init(gpa, .{ .reader = &file_reader.interface }); defer bc.deinit(); var file_names: std.AutoArrayHashMapUnmanaged(u32, []const u8) = .empty; @@ -1873,15 +1876,12 @@ pub fn create(gpa: Allocator, arena: Allocator, options: CreateOptions) !*Compil if (options.verbose_llvm_cpu_features) { if (options.root_mod.resolved_target.llvm_cpu_features) |cf| print: { - std.debug.lockStdErr(); - defer std.debug.unlockStdErr(); - const stderr = fs.File.stderr().deprecatedWriter(); - nosuspend { - stderr.print("compilation: {s}\n", .{options.root_name}) catch break :print; - stderr.print(" target: {s}\n", .{try target.zigTriple(arena)}) catch break :print; - stderr.print(" cpu: {s}\n", .{target.cpu.model.name}) catch break :print; - stderr.print(" features: {s}\n", .{cf}) catch {}; - } + const stderr_w = std.debug.lockStderrWriter(&.{}); + defer std.debug.unlockStderrWriter(); + stderr_w.print("compilation: {s}\n", .{options.root_name}) catch break :print; + stderr_w.print(" target: {s}\n", .{try target.zigTriple(arena)}) catch break :print; + stderr_w.print(" cpu: {s}\n", .{target.cpu.model.name}) catch break :print; + stderr_w.print(" features: {s}\n", .{cf}) catch {}; } } @@ -2483,7 +2483,7 @@ pub fn destroy(comp: *Compilation) void { if (comp.zcu) |zcu| zcu.deinit(); comp.cache_use.deinit(); - for (comp.work_queues) |work_queue| work_queue.deinit(); + for (&comp.work_queues) |*work_queue| work_queue.deinit(); comp.c_object_work_queue.deinit(); comp.win32_resource_work_queue.deinit(); @@ -3931,11 +3931,12 @@ pub fn getAllErrorsAlloc(comp: *Compilation) !ErrorBundle { // This AU is referenced and has a transitive compile error, meaning it referenced something with a compile error. // However, we haven't reported any such error. // This is a compiler bug. - const stderr = fs.File.stderr().deprecatedWriter(); - try stderr.writeAll("referenced transitive analysis errors, but none actually emitted\n"); - try stderr.print("{f} [transitive failure]\n", .{zcu.fmtAnalUnit(failed_unit)}); + var stderr_w = std.debug.lockStderrWriter(&.{}); + defer std.debug.unlockStderrWriter(); + try stderr_w.writeAll("referenced transitive analysis errors, but none actually emitted\n"); + try stderr_w.print("{f} [transitive failure]\n", .{zcu.fmtAnalUnit(failed_unit)}); while (ref) |r| { - try stderr.print("referenced by: {f}{s}\n", .{ + try stderr_w.print("referenced by: {f}{s}\n", .{ zcu.fmtAnalUnit(r.referencer), if (zcu.transitive_failed_analysis.contains(r.referencer)) " [transitive failure]" else "", }); @@ -5849,7 +5850,9 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Pr try child.spawn(); - const stderr = try child.stderr.?.deprecatedReader().readAllAlloc(arena, std.math.maxInt(usize)); + var small_buffer: [1]u8 = undefined; + var stderr_reader = child.stderr.?.reader(&small_buffer); + const stderr = try stderr_reader.interface.allocRemaining(arena, .unlimited); const term = child.wait() catch |err| { return comp.failCObj(c_object, "failed to spawn zig clang {s}: {s}", .{ argv.items[0], @errorName(err) }); @@ -6213,13 +6216,10 @@ fn spawnZigRc( const stdout = poller.fifo(.stdout); poll: while (true) { - while (stdout.readableLength() < @sizeOf(std.zig.Server.Message.Header)) { - if (!(try poller.poll())) break :poll; - } - const header = stdout.reader().readStruct(std.zig.Server.Message.Header) catch unreachable; - while (stdout.readableLength() < header.bytes_len) { - if (!(try poller.poll())) break :poll; - } + while (stdout.readableLength() < @sizeOf(std.zig.Server.Message.Header)) if (!try poller.poll()) break :poll; + var header: std.zig.Server.Message.Header = undefined; + assert(stdout.read(std.mem.asBytes(&header)) == @sizeOf(std.zig.Server.Message.Header)); + while (stdout.readableLength() < header.bytes_len) if (!try poller.poll()) break :poll; const body = stdout.readableSliceOfLen(header.bytes_len); switch (header.tag) { @@ -7209,13 +7209,16 @@ pub fn lockAndSetMiscFailure( } pub fn dump_argv(argv: []const []const u8) void { - std.debug.lockStdErr(); - defer std.debug.unlockStdErr(); - const stderr = fs.File.stderr().deprecatedWriter(); - for (argv[0 .. argv.len - 1]) |arg| { - nosuspend stderr.print("{s} ", .{arg}) catch return; + var buffer: [64]u8 = undefined; + const stderr = std.debug.lockStderrWriter(&buffer); + defer std.debug.unlockStderrWriter(); + nosuspend { + for (argv) |arg| { + stderr.writeAll(arg) catch return; + (stderr.writableArray(1) catch return)[0] = ' '; + } + stderr.buffer[stderr.end - 1] = '\n'; } - nosuspend stderr.print("{s}\n", .{argv[argv.len - 1]}) catch {}; } pub fn getZigBackend(comp: Compilation) std.builtin.CompilerBackend { diff --git a/src/deprecated.zig b/src/deprecated.zig index 1f7d5c8c25..68c712b3b1 100644 --- a/src/deprecated.zig +++ b/src/deprecated.zig @@ -52,15 +52,6 @@ pub fn LinearFifo(comptime T: type) type { } } - /// Reduce allocated capacity to `size`. - pub fn shrink(self: *Self, size: usize) void { - assert(size >= self.count); - self.realign(); - self.buf = self.allocator.realloc(self.buf, size) catch |e| switch (e) { - error.OutOfMemory => return, // no problem, capacity is still correct then. - }; - } - /// Ensure that the buffer can fit at least `size` items pub fn ensureTotalCapacity(self: *Self, size: usize) !void { if (self.buf.len >= size) return; @@ -76,11 +67,6 @@ pub fn LinearFifo(comptime T: type) type { return try self.ensureTotalCapacity(math.add(usize, self.count, size) catch return error.OutOfMemory); } - /// Returns number of items currently in fifo - pub fn readableLength(self: Self) usize { - return self.count; - } - /// Returns a writable slice from the 'read' end of the fifo fn readableSliceMut(self: Self, offset: usize) []T { if (offset > self.count) return &[_]T{}; @@ -95,22 +81,6 @@ pub fn LinearFifo(comptime T: type) type { } } - /// Returns a readable slice from `offset` - pub fn readableSlice(self: Self, offset: usize) []const T { - return self.readableSliceMut(offset); - } - - pub fn readableSliceOfLen(self: *Self, len: usize) []const T { - assert(len <= self.count); - const buf = self.readableSlice(0); - if (buf.len >= len) { - return buf[0..len]; - } else { - self.realign(); - return self.readableSlice(0)[0..len]; - } - } - /// Discard first `count` items in the fifo pub fn discard(self: *Self, count: usize) void { assert(count <= self.count); @@ -143,28 +113,6 @@ pub fn LinearFifo(comptime T: type) type { return c; } - /// Read data from the fifo into `dst`, returns number of items copied. - pub fn read(self: *Self, dst: []T) usize { - var dst_left = dst; - - while (dst_left.len > 0) { - const slice = self.readableSlice(0); - if (slice.len == 0) break; - const n = @min(slice.len, dst_left.len); - @memcpy(dst_left[0..n], slice[0..n]); - self.discard(n); - dst_left = dst_left[n..]; - } - - return dst.len - dst_left.len; - } - - /// Same as `read` except it returns an error union - /// The purpose of this function existing is to match `std.io.Reader` API. - fn readFn(self: *Self, dest: []u8) error{}!usize { - return self.read(dest); - } - /// Returns number of items available in fifo pub fn writableLength(self: Self) usize { return self.buf.len - self.count; @@ -183,20 +131,6 @@ pub fn LinearFifo(comptime T: type) type { } } - /// Returns a writable buffer of at least `size` items, allocating memory as needed. - /// Use `fifo.update` once you've written data to it. - pub fn writableWithSize(self: *Self, size: usize) ![]T { - try self.ensureUnusedCapacity(size); - - // try to avoid realigning buffer - var slice = self.writableSlice(0); - if (slice.len < size) { - self.realign(); - slice = self.writableSlice(0); - } - return slice; - } - /// Update the tail location of the buffer (usually follows use of writable/writableWithSize) pub fn update(self: *Self, count: usize) void { assert(self.count + count <= self.buf.len); @@ -231,201 +165,5 @@ pub fn LinearFifo(comptime T: type) type { self.buf[tail] = item; self.update(1); } - - /// Appends the data in `src` to the fifo. - /// Allocates more memory as necessary - pub fn write(self: *Self, src: []const T) !void { - try self.ensureUnusedCapacity(src.len); - - return self.writeAssumeCapacity(src); - } - - /// Same as `write` except it returns the number of bytes written, which is always the same - /// as `bytes.len`. The purpose of this function existing is to match `std.io.Writer` API. - fn appendWrite(self: *Self, bytes: []const u8) error{OutOfMemory}!usize { - try self.write(bytes); - return bytes.len; - } - - /// Make `count` items available before the current read location - fn rewind(self: *Self, count: usize) void { - assert(self.writableLength() >= count); - - var head = self.head + (self.buf.len - count); - head &= self.buf.len - 1; - self.head = head; - self.count += count; - } - - /// Place data back into the read stream - pub fn unget(self: *Self, src: []const T) !void { - try self.ensureUnusedCapacity(src.len); - - self.rewind(src.len); - - const slice = self.readableSliceMut(0); - if (src.len < slice.len) { - @memcpy(slice[0..src.len], src); - } else { - @memcpy(slice, src[0..slice.len]); - const slice2 = self.readableSliceMut(slice.len); - @memcpy(slice2[0 .. src.len - slice.len], src[slice.len..]); - } - } - - /// Returns the item at `offset`. - /// Asserts offset is within bounds. - pub fn peekItem(self: Self, offset: usize) T { - assert(offset < self.count); - - var index = self.head + offset; - index &= self.buf.len - 1; - return self.buf[index]; - } - - pub fn toOwnedSlice(self: *Self) Allocator.Error![]T { - if (self.head != 0) self.realign(); - assert(self.head == 0); - assert(self.count <= self.buf.len); - const allocator = self.allocator; - if (allocator.resize(self.buf, self.count)) { - const result = self.buf[0..self.count]; - self.* = Self.init(allocator); - return result; - } - const new_memory = try allocator.dupe(T, self.buf[0..self.count]); - allocator.free(self.buf); - self.* = Self.init(allocator); - return new_memory; - } }; } - -test "LinearFifo(u8, .Dynamic) discard(0) from empty buffer should not error on overflow" { - var fifo = LinearFifo(u8, .Dynamic).init(testing.allocator); - defer fifo.deinit(); - - // If overflow is not explicitly allowed this will crash in debug / safe mode - fifo.discard(0); -} - -test "LinearFifo(u8, .Dynamic)" { - var fifo = LinearFifo(u8, .Dynamic).init(testing.allocator); - defer fifo.deinit(); - - try fifo.write("HELLO"); - try testing.expectEqual(@as(usize, 5), fifo.readableLength()); - try testing.expectEqualSlices(u8, "HELLO", fifo.readableSlice(0)); - - { - var i: usize = 0; - while (i < 5) : (i += 1) { - try fifo.write(&[_]u8{fifo.peekItem(i)}); - } - try testing.expectEqual(@as(usize, 10), fifo.readableLength()); - try testing.expectEqualSlices(u8, "HELLOHELLO", fifo.readableSlice(0)); - } - - { - try testing.expectEqual(@as(u8, 'H'), fifo.readItem().?); - try testing.expectEqual(@as(u8, 'E'), fifo.readItem().?); - try testing.expectEqual(@as(u8, 'L'), fifo.readItem().?); - try testing.expectEqual(@as(u8, 'L'), fifo.readItem().?); - try testing.expectEqual(@as(u8, 'O'), fifo.readItem().?); - } - try testing.expectEqual(@as(usize, 5), fifo.readableLength()); - - { // Writes that wrap around - try testing.expectEqual(@as(usize, 11), fifo.writableLength()); - try testing.expectEqual(@as(usize, 6), fifo.writableSlice(0).len); - fifo.writeAssumeCapacity("6 {}, .stdio => { + var stdin_reader = fs.File.stdin().reader(&stdin_buffer); + var stdout_writer = fs.File.stdout().writer(&stdout_buffer); try serve( comp, - .stdin(), - .stdout(), + &stdin_reader.interface, + &stdout_writer.interface, test_exec_args.items, self_exe_path, arg_mode, @@ -3584,10 +3588,13 @@ fn buildOutputType( const conn = try server.accept(); defer conn.stream.close(); + var input = conn.stream.reader(&stdin_buffer); + var output = conn.stream.writer(&stdout_buffer); + try serve( comp, - .{ .handle = conn.stream.handle }, - .{ .handle = conn.stream.handle }, + input.interface(), + &output.interface, test_exec_args.items, self_exe_path, arg_mode, @@ -4053,8 +4060,8 @@ fn saveState(comp: *Compilation, incremental: bool) void { fn serve( comp: *Compilation, - in: fs.File, - out: fs.File, + in: *std.Io.Reader, + out: *std.Io.Writer, test_exec_args: []const ?[]const u8, self_exe_path: ?[]const u8, arg_mode: ArgMode, @@ -4064,12 +4071,10 @@ fn serve( const gpa = comp.gpa; var server = try Server.init(.{ - .gpa = gpa, .in = in, .out = out, .zig_version = build_options.version, }); - defer server.deinit(); var child_pid: ?std.process.Child.Id = null; @@ -5491,10 +5496,10 @@ fn jitCmd( defer comp.destroy(); if (options.server) { + var stdout_writer = fs.File.stdout().writer(&stdout_buffer); var server: std.zig.Server = .{ - .out = fs.File.stdout(), + .out = &stdout_writer.interface, .in = undefined, // won't be receiving messages - .receive_fifo = undefined, // won't be receiving messages }; try comp.update(root_prog_node); @@ -6058,7 +6063,7 @@ fn cmdAstCheck( }; } else fs.File.stdin(); defer if (zig_source_path != null) f.close(); - var file_reader: fs.File.Reader = f.reader(&stdio_buffer); + var file_reader: fs.File.Reader = f.reader(&stdin_buffer); break :s std.zig.readSourceFileToEndAlloc(arena, &file_reader) catch |err| { fatal("unable to load file '{s}' for ast-check: {s}", .{ display_path, @errorName(err) }); }; @@ -6076,7 +6081,7 @@ fn cmdAstCheck( const tree = try Ast.parse(arena, source, mode); - var stdout_writer = fs.File.stdout().writerStreaming(&stdio_buffer); + var stdout_writer = fs.File.stdout().writerStreaming(&stdout_buffer); const stdout_bw = &stdout_writer.interface; switch (mode) { .zig => { @@ -6291,7 +6296,7 @@ fn detectNativeCpuWithLLVM( } fn printCpu(cpu: std.Target.Cpu) !void { - var stdout_writer = fs.File.stdout().writerStreaming(&stdio_buffer); + var stdout_writer = fs.File.stdout().writerStreaming(&stdout_buffer); const stdout_bw = &stdout_writer.interface; if (cpu.model.llvm_name) |llvm_name| { @@ -6340,7 +6345,7 @@ fn cmdDumpLlvmInts( const dl = tm.createTargetDataLayout(); const context = llvm.Context.create(); - var stdout_writer = fs.File.stdout().writerStreaming(&stdio_buffer); + var stdout_writer = fs.File.stdout().writerStreaming(&stdout_buffer); const stdout_bw = &stdout_writer.interface; for ([_]u16{ 1, 8, 16, 32, 64, 128, 256 }) |bits| { const int_type = context.intType(bits); @@ -6369,9 +6374,8 @@ fn cmdDumpZir( defer f.close(); const zir = try Zcu.loadZirCache(arena, f); - var stdout_writer = fs.File.stdout().writerStreaming(&stdio_buffer); + var stdout_writer = fs.File.stdout().writerStreaming(&stdout_buffer); const stdout_bw = &stdout_writer.interface; - { const instruction_bytes = zir.instructions.len * // Here we don't use @sizeOf(Zir.Inst.Data) because it would include @@ -6417,7 +6421,7 @@ fn cmdChangelist( var f = fs.cwd().openFile(old_source_path, .{}) catch |err| fatal("unable to open old source file '{s}': {s}", .{ old_source_path, @errorName(err) }); defer f.close(); - var file_reader: fs.File.Reader = f.reader(&stdio_buffer); + var file_reader: fs.File.Reader = f.reader(&stdin_buffer); break :source std.zig.readSourceFileToEndAlloc(arena, &file_reader) catch |err| fatal("unable to read old source file '{s}': {s}", .{ old_source_path, @errorName(err) }); }; @@ -6425,7 +6429,7 @@ fn cmdChangelist( var f = fs.cwd().openFile(new_source_path, .{}) catch |err| fatal("unable to open new source file '{s}': {s}", .{ new_source_path, @errorName(err) }); defer f.close(); - var file_reader: fs.File.Reader = f.reader(&stdio_buffer); + var file_reader: fs.File.Reader = f.reader(&stdin_buffer); break :source std.zig.readSourceFileToEndAlloc(arena, &file_reader) catch |err| fatal("unable to read new source file '{s}': {s}", .{ new_source_path, @errorName(err) }); }; @@ -6457,7 +6461,7 @@ fn cmdChangelist( var inst_map: std.AutoHashMapUnmanaged(Zir.Inst.Index, Zir.Inst.Index) = .empty; try Zcu.mapOldZirToNew(arena, old_zir, new_zir, &inst_map); - var stdout_writer = fs.File.stdout().writerStreaming(&stdio_buffer); + var stdout_writer = fs.File.stdout().writerStreaming(&stdout_buffer); const stdout_bw = &stdout_writer.interface; { try stdout_bw.print("Instruction mappings:\n", .{}); @@ -6917,7 +6921,7 @@ fn cmdFetch( const name = switch (save) { .no => { - var stdout = fs.File.stdout().writerStreaming(&stdio_buffer); + var stdout = fs.File.stdout().writerStreaming(&stdout_buffer); try stdout.interface.print("{s}\n", .{package_hash_slice}); try stdout.interface.flush(); return cleanExit(); From bad836a69bf3c1da29f39ed1a6c8bf6c66febfb6 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 18 Jul 2025 06:45:25 -0700 Subject: [PATCH 12/26] Compilation: revert some stuff --- src/Compilation.zig | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index 469a13b4a9..916a025bc9 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -998,13 +998,13 @@ pub const CObject = struct { const file = fs.cwd().openFile(file_name, .{}) catch break :source_line 0; defer file.close(); - var buffer: [1024]u8 = undefined; - var file_reader = file.reader(&buffer); - file_reader.seekTo(diag.src_loc.offset + 1 - diag.src_loc.column) catch break :source_line 0; - var aw: Writer.Allocating = .init(eb.gpa); - defer aw.deinit(); - _ = file_reader.interface.streamDelimiterEnding(&aw.writer, '\n') catch break :source_line 0; - break :source_line try eb.addString(aw.getWritten()); + file.seekTo(diag.src_loc.offset + 1 - diag.src_loc.column) catch break :source_line 0; + + var line = std.ArrayList(u8).init(eb.gpa); + defer line.deinit(); + file.deprecatedReader().readUntilDelimiterArrayList(&line, '\n', 1 << 10) catch break :source_line 0; + + break :source_line try eb.addString(line.items); }; return .{ @@ -5850,9 +5850,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_obj_prog_node: std.Pr try child.spawn(); - var small_buffer: [1]u8 = undefined; - var stderr_reader = child.stderr.?.reader(&small_buffer); - const stderr = try stderr_reader.interface.allocRemaining(arena, .unlimited); + const stderr = try child.stderr.?.deprecatedReader().readAllAlloc(arena, std.math.maxInt(usize)); const term = child.wait() catch |err| { return comp.failCObj(c_object, "failed to spawn zig clang {s}: {s}", .{ argv.items[0], @errorName(err) }); From 83d1f88ac5f053f11b91d9c3d51f5c63355b8ca3 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 18 Jul 2025 10:42:04 -0700 Subject: [PATCH 13/26] std.debug: add assertAligned --- lib/std/debug.zig | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/std/debug.zig b/lib/std/debug.zig index a7b88e4257..655852c65e 100644 --- a/lib/std/debug.zig +++ b/lib/std/debug.zig @@ -566,6 +566,13 @@ pub fn assertReadable(slice: []const volatile u8) void { for (slice) |*byte| _ = byte.*; } +/// Invokes detectable illegal behavior when the provided array is not aligned +/// to the provided amount. +pub fn assertAligned(ptr: anytype, comptime alignment: std.mem.Alignment) void { + const aligned_ptr: *align(alignment.toByteUnits()) anyopaque = @alignCast(@ptrCast(ptr)); + _ = aligned_ptr; +} + /// Equivalent to `@panic` but with a formatted message. pub fn panic(comptime format: []const u8, args: anytype) noreturn { @branchHint(.cold); From bd64bf0e47e75481569dcc3ba519e3d2e7f7b276 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 18 Jul 2025 10:42:24 -0700 Subject: [PATCH 14/26] std.mem: add byteSwapAllElements --- lib/std/mem.zig | 36 ++++++++++++++++++++---------------- 1 file changed, 20 insertions(+), 16 deletions(-) diff --git a/lib/std/mem.zig b/lib/std/mem.zig index 33e68eedad..1a61076f32 100644 --- a/lib/std/mem.zig +++ b/lib/std/mem.zig @@ -2179,22 +2179,8 @@ pub fn byteSwapAllFields(comptime S: type, ptr: *S) void { const BackingInt = std.meta.Int(.unsigned, @bitSizeOf(S)); ptr.* = @bitCast(@byteSwap(@as(BackingInt, @bitCast(ptr.*)))); }, - .array => { - for (ptr) |*item| { - switch (@typeInfo(@TypeOf(item.*))) { - .@"struct", .@"union", .array => byteSwapAllFields(@TypeOf(item.*), item), - .@"enum" => { - item.* = @enumFromInt(@byteSwap(@intFromEnum(item.*))); - }, - .bool => {}, - .float => |float_info| { - item.* = @bitCast(@byteSwap(@as(std.meta.Int(.unsigned, float_info.bits), @bitCast(item.*)))); - }, - else => { - item.* = @byteSwap(item.*); - }, - } - } + .array => |info| { + byteSwapAllElements(info.child, ptr); }, else => { ptr.* = @byteSwap(ptr.*); @@ -2258,6 +2244,24 @@ test byteSwapAllFields { }, k); } +pub fn byteSwapAllElements(comptime Elem: type, slice: []Elem) void { + for (slice) |*elem| { + switch (@typeInfo(@TypeOf(elem.*))) { + .@"struct", .@"union", .array => byteSwapAllFields(@TypeOf(elem.*), elem), + .@"enum" => { + elem.* = @enumFromInt(@byteSwap(@intFromEnum(elem.*))); + }, + .bool => {}, + .float => |float_info| { + elem.* = @bitCast(@byteSwap(@as(std.meta.Int(.unsigned, float_info.bits), @bitCast(elem.*)))); + }, + else => { + elem.* = @byteSwap(elem.*); + }, + } + } +} + /// Returns an iterator that iterates over the slices of `buffer` that are not /// any of the items in `delimiters`. /// From 8489bab1f46ef7a5cdcf598b10f1150a9e0c2f0d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 18 Jul 2025 10:43:02 -0700 Subject: [PATCH 15/26] std.Io.Writer: add missing writeSliceSwap --- lib/std/Io/Writer.zig | 28 +++++++++++++++++++++++++++- lib/std/zig/Server.zig | 2 ++ 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/lib/std/Io/Writer.zig b/lib/std/Io/Writer.zig index 4b0e142fb0..11bc05a00d 100644 --- a/lib/std/Io/Writer.zig +++ b/lib/std/Io/Writer.zig @@ -851,6 +851,9 @@ pub inline fn writeStruct(w: *Writer, value: anytype, endian: std.builtin.Endian } } +/// If, `endian` is not native, +/// * Asserts that the buffer storage capacity is at least enough to store `@sizeOf(Elem)` +/// * Asserts that the buffer is aligned enough for `@alignOf(Elem)`. pub inline fn writeSliceEndian( w: *Writer, Elem: type, @@ -860,7 +863,22 @@ pub inline fn writeSliceEndian( if (native_endian == endian) { return writeAll(w, @ptrCast(slice)); } else { - return w.writeArraySwap(w, Elem, slice); + return writeSliceSwap(w, Elem, slice); + } +} + +/// Asserts that the buffer storage capacity is at least enough to store `@sizeOf(Elem)` +/// +/// Asserts that the buffer is aligned enough for `@alignOf(Elem)`. +pub fn writeSliceSwap(w: *Writer, Elem: type, slice: []const Elem) Error!void { + var i: usize = 0; + while (i < slice.len) { + const dest_bytes = try w.writableSliceGreedy(@sizeOf(Elem)); + const dest: []Elem = @alignCast(@ptrCast(dest_bytes[0 .. dest_bytes.len - dest_bytes.len % @sizeOf(Elem)])); + const copy_len = @min(dest.len, slice.len - i); + @memcpy(dest[0..copy_len], slice[i..][0..copy_len]); + i += copy_len; + std.mem.byteSwapAllElements(Elem, dest); } } @@ -2630,3 +2648,11 @@ test writeStruct { }, &buffer); } } + +test writeSliceEndian { + var buffer: [4]u8 align(2) = undefined; + var w: Writer = .fixed(&buffer); + const array: [2]u16 = .{ 0x1234, 0x5678 }; + try writeSliceEndian(&w, u16, &array, .big); + try testing.expectEqualSlices(u8, &.{ 0x12, 0x34, 0x56, 0x78 }, &buffer); +} diff --git a/lib/std/zig/Server.zig b/lib/std/zig/Server.zig index 38ad45e1e1..8fc016d284 100644 --- a/lib/std/zig/Server.zig +++ b/lib/std/zig/Server.zig @@ -118,6 +118,8 @@ pub fn init(options: Options) !Server { .in = options.in, .out = options.out, }; + assert(s.out.buffer.len >= 4); + std.debug.assertAligned(s.out.buffer.ptr, .@"4"); try s.serveStringMessage(.zig_version, options.zig_version); return s; } From b956ae20af86384ef5d15b2417b0bf095b5b3740 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 18 Jul 2025 11:15:12 -0700 Subject: [PATCH 16/26] frontend: align those stdio buffers --- src/main.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main.zig b/src/main.zig index 9958c461b1..dc931f119b 100644 --- a/src/main.zig +++ b/src/main.zig @@ -66,9 +66,9 @@ pub fn wasi_cwd() std.os.wasi.fd_t { const fatal = std.process.fatal; /// This can be global since stdin is a singleton. -var stdin_buffer: [4096]u8 = undefined; +var stdin_buffer: [4096]u8 align(std.heap.page_size_min) = undefined; /// This can be global since stdout is a singleton. -var stdout_buffer: [4096]u8 = undefined; +var stdout_buffer: [4096]u8 align(std.heap.page_size_min) = undefined; /// Shaming all the locations that inappropriately use an O(N) search algorithm. /// Please delete this and fix the compilation errors! From d396780925914025c42576d45a2f668a64f7336b Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 18 Jul 2025 11:16:31 -0700 Subject: [PATCH 17/26] Compilation: unrevert some stuff --- src/Compilation.zig | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index 916a025bc9..b5597017c4 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -998,13 +998,13 @@ pub const CObject = struct { const file = fs.cwd().openFile(file_name, .{}) catch break :source_line 0; defer file.close(); - file.seekTo(diag.src_loc.offset + 1 - diag.src_loc.column) catch break :source_line 0; - - var line = std.ArrayList(u8).init(eb.gpa); - defer line.deinit(); - file.deprecatedReader().readUntilDelimiterArrayList(&line, '\n', 1 << 10) catch break :source_line 0; - - break :source_line try eb.addString(line.items); + var buffer: [1024]u8 = undefined; + var file_reader = file.reader(&buffer); + file_reader.seekTo(diag.src_loc.offset + 1 - diag.src_loc.column) catch break :source_line 0; + var aw: Writer.Allocating = .init(eb.gpa); + defer aw.deinit(); + _ = file_reader.interface.streamDelimiterEnding(&aw.writer, '\n') catch break :source_line 0; + break :source_line try eb.addString(aw.getWritten()); }; return .{ From 83960e0eb068f786c46c3fe559016e1e9faea3cd Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 18 Jul 2025 17:38:13 -0700 Subject: [PATCH 18/26] disable -fno-llvm -target wasm32-wasi testing no active maintainer, and it's failing to lower some basic stuff --- test/src/Cases.zig | 2 -- test/tests.zig | 19 ++++++++++--------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/test/src/Cases.zig b/test/src/Cases.zig index 522fe6b385..a92f9fd158 100644 --- a/test/src/Cases.zig +++ b/test/src/Cases.zig @@ -800,8 +800,6 @@ const TestManifestConfigDefaults = struct { } // Windows defaults = defaults ++ "x86_64-windows" ++ ","; - // Wasm - defaults = defaults ++ "wasm32-wasi"; break :blk defaults; }; } else if (std.mem.eql(u8, key, "output_mode")) { diff --git a/test/tests.zig b/test/tests.zig index 2a6c4166fa..db4407172f 100644 --- a/test/tests.zig +++ b/test/tests.zig @@ -1335,15 +1335,16 @@ const test_targets = blk: { // WASI Targets - .{ - .target = .{ - .cpu_arch = .wasm32, - .os_tag = .wasi, - .abi = .none, - }, - .use_llvm = false, - .use_lld = false, - }, + // TODO: lowerTry for pointers + //.{ + // .target = .{ + // .cpu_arch = .wasm32, + // .os_tag = .wasi, + // .abi = .none, + // }, + // .use_llvm = false, + // .use_lld = false, + //}, .{ .target = .{ .cpu_arch = .wasm32, From 741a66e03cd8644d01b38849f2bd8f70cae6beca Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 18 Jul 2025 20:51:37 -0700 Subject: [PATCH 19/26] std.zig.llvm.BitcodeReader: fix 32-bit skipBlock --- lib/std/zig/llvm/BitcodeReader.zig | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/std/zig/llvm/BitcodeReader.zig b/lib/std/zig/llvm/BitcodeReader.zig index f1d47e93c0..f691f12a8c 100644 --- a/lib/std/zig/llvm/BitcodeReader.zig +++ b/lib/std/zig/llvm/BitcodeReader.zig @@ -177,7 +177,7 @@ pub fn next(bc: *BitcodeReader) !?Item { pub fn skipBlock(bc: *BitcodeReader, block: Block) !void { assert(bc.bit_offset == 0); - try bc.reader.discardAll(4 * @as(u34, block.len)); + try bc.reader.discardAll(4 * @as(usize, block.len)); try bc.endBlock(); } From b4fd57a9c114748afb9ba0a04bede61089a02ddf Mon Sep 17 00:00:00 2001 From: Jacob Young Date: Sat, 19 Jul 2025 18:26:56 -0400 Subject: [PATCH 20/26] llvm: workaround crashes in llvm loop optimizations Workaround for #24383 --- src/codegen/llvm.zig | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/codegen/llvm.zig b/src/codegen/llvm.zig index b3391cfb6a..a570dd5ec0 100644 --- a/src/codegen/llvm.zig +++ b/src/codegen/llvm.zig @@ -6385,6 +6385,9 @@ pub const FuncGen = struct { // * https://github.com/llvm/llvm-project/blob/56905dab7da50bccfcceaeb496b206ff476127e1/llvm/test/MC/WebAssembly/blockaddress.ll if (zcu.comp.getTarget().cpu.arch.isWasm()) break :jmp_table null; + // Workaround for https://github.com/ziglang/zig/issues/24383: + if (self.ng.ownerModule().optimize_mode == .ReleaseSafe) break :jmp_table null; + // On a 64-bit target, 1024 pointers in our jump table is about 8K of pointers. This seems just // about acceptable - it won't fill L1d cache on most CPUs. const max_table_len = 1024; From c40fb96ca358e2ef28aecc2b7ebc5ffab43ccac8 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sat, 19 Jul 2025 21:53:48 -0700 Subject: [PATCH 21/26] std.Io.Writer: fix writeSliceSwap tried to be too clever, wrote bad code --- lib/compiler/test_runner.zig | 12 ++++++------ lib/std/Build/Step/Run.zig | 2 +- lib/std/Io/Reader.zig | 34 +++++++++++++++++----------------- lib/std/Io/Writer.zig | 23 +++++++---------------- lib/std/zig/Server.zig | 6 ++---- src/Zcu.zig | 2 +- src/Zcu/PerThread.zig | 2 +- 7 files changed, 35 insertions(+), 46 deletions(-) diff --git a/lib/compiler/test_runner.zig b/lib/compiler/test_runner.zig index a69066f09c..8b60a75399 100644 --- a/lib/compiler/test_runner.zig +++ b/lib/compiler/test_runner.zig @@ -10,10 +10,10 @@ pub const std_options: std.Options = .{ }; var log_err_count: usize = 0; -var fba_buffer: [8192]u8 = undefined; var fba = std.heap.FixedBufferAllocator.init(&fba_buffer); -var stdin_buffer: [std.heap.page_size_min]u8 align(std.heap.page_size_min) = undefined; -var stdout_buffer: [std.heap.page_size_min]u8 align(std.heap.page_size_min) = undefined; +var fba_buffer: [8192]u8 = undefined; +var stdin_buffer: [4096]u8 = undefined; +var stdout_buffer: [4096]u8 = undefined; const crippled = switch (builtin.zig_backend) { .stage2_powerpc, @@ -68,8 +68,8 @@ pub fn main() void { fn mainServer() !void { @disableInstrumentation(); - var stdin_reader = std.fs.File.stdin().reader(&stdin_buffer); - var stdout_writer = std.fs.File.stdout().writer(&stdout_buffer); + var stdin_reader = std.fs.File.stdin().readerStreaming(&stdin_buffer); + var stdout_writer = std.fs.File.stdout().writerStreaming(&stdout_buffer); var server = try std.zig.Server.init(.{ .in = &stdin_reader.interface, .out = &stdout_writer.interface, @@ -104,7 +104,7 @@ fn mainServer() !void { defer testing.allocator.free(expected_panic_msgs); for (test_fns, names, expected_panic_msgs) |test_fn, *name, *expected_panic_msg| { - name.* = @as(u32, @intCast(string_bytes.items.len)); + name.* = @intCast(string_bytes.items.len); try string_bytes.ensureUnusedCapacity(testing.allocator, test_fn.name.len + 1); string_bytes.appendSliceAssumeCapacity(test_fn.name); string_bytes.appendAssumeCapacity(0); diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index b4e2e73c0a..1742da33c8 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -1742,7 +1742,7 @@ fn sendMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag) !void { .tag = tag, .bytes_len = 0, }; - try file.writeAll(std.mem.asBytes(&header)); + try file.writeAll(@ptrCast(&header)); } fn sendRunTestMessage(file: std.fs.File, tag: std.zig.Client.Message.Tag, index: u32) !void { diff --git a/lib/std/Io/Reader.zig b/lib/std/Io/Reader.zig index f2a1ec7287..497b906c97 100644 --- a/lib/std/Io/Reader.zig +++ b/lib/std/Io/Reader.zig @@ -1108,9 +1108,9 @@ pub fn takeVarInt(r: *Reader, comptime Int: type, endian: std.builtin.Endian, n: /// Asserts the buffer was initialized with a capacity at least `@sizeOf(T)`. /// /// See also: -/// * `peekStructReference` +/// * `peekStructPointer` /// * `takeStruct` -pub fn takeStructReference(r: *Reader, comptime T: type) Error!*align(1) T { +pub fn takeStructPointer(r: *Reader, comptime T: type) Error!*align(1) T { // Only extern and packed structs have defined in-memory layout. comptime assert(@typeInfo(T).@"struct".layout != .auto); return @ptrCast(try r.takeArray(@sizeOf(T))); @@ -1122,9 +1122,9 @@ pub fn takeStructReference(r: *Reader, comptime T: type) Error!*align(1) T { /// Asserts the buffer was initialized with a capacity at least `@sizeOf(T)`. /// /// See also: -/// * `takeStructReference` +/// * `takeStructPointer` /// * `peekStruct` -pub fn peekStructReference(r: *Reader, comptime T: type) Error!*align(1) T { +pub fn peekStructPointer(r: *Reader, comptime T: type) Error!*align(1) T { // Only extern and packed structs have defined in-memory layout. comptime assert(@typeInfo(T).@"struct".layout != .auto); return @ptrCast(try r.peekArray(@sizeOf(T))); @@ -1136,14 +1136,14 @@ pub fn peekStructReference(r: *Reader, comptime T: type) Error!*align(1) T { /// when `endian` is comptime-known and matches the host endianness. /// /// See also: -/// * `takeStructReference` +/// * `takeStructPointer` /// * `peekStruct` pub inline fn takeStruct(r: *Reader, comptime T: type, endian: std.builtin.Endian) Error!T { switch (@typeInfo(T)) { .@"struct" => |info| switch (info.layout) { .auto => @compileError("ill-defined memory layout"), .@"extern" => { - var res = (try r.takeStructReference(T)).*; + var res = (try r.takeStructPointer(T)).*; if (native_endian != endian) std.mem.byteSwapAllFields(T, &res); return res; }, @@ -1162,13 +1162,13 @@ pub inline fn takeStruct(r: *Reader, comptime T: type, endian: std.builtin.Endia /// /// See also: /// * `takeStruct` -/// * `peekStructReference` +/// * `peekStructPointer` pub inline fn peekStruct(r: *Reader, comptime T: type, endian: std.builtin.Endian) Error!T { switch (@typeInfo(T)) { .@"struct" => |info| switch (info.layout) { .auto => @compileError("ill-defined memory layout"), .@"extern" => { - var res = (try r.peekStructReference(T)).*; + var res = (try r.peekStructPointer(T)).*; if (native_endian != endian) std.mem.byteSwapAllFields(T, &res); return res; }, @@ -1557,27 +1557,27 @@ test takeVarInt { try testing.expectError(error.EndOfStream, r.takeVarInt(u16, .little, 1)); } -test takeStructReference { +test takeStructPointer { var r: Reader = .fixed(&.{ 0x12, 0x00, 0x34, 0x56 }); const S = extern struct { a: u8, b: u16 }; switch (native_endian) { - .little => try testing.expectEqual(@as(S, .{ .a = 0x12, .b = 0x5634 }), (try r.takeStructReference(S)).*), - .big => try testing.expectEqual(@as(S, .{ .a = 0x12, .b = 0x3456 }), (try r.takeStructReference(S)).*), + .little => try testing.expectEqual(@as(S, .{ .a = 0x12, .b = 0x5634 }), (try r.takeStructPointer(S)).*), + .big => try testing.expectEqual(@as(S, .{ .a = 0x12, .b = 0x3456 }), (try r.takeStructPointer(S)).*), } - try testing.expectError(error.EndOfStream, r.takeStructReference(S)); + try testing.expectError(error.EndOfStream, r.takeStructPointer(S)); } -test peekStructReference { +test peekStructPointer { var r: Reader = .fixed(&.{ 0x12, 0x00, 0x34, 0x56 }); const S = extern struct { a: u8, b: u16 }; switch (native_endian) { .little => { - try testing.expectEqual(@as(S, .{ .a = 0x12, .b = 0x5634 }), (try r.peekStructReference(S)).*); - try testing.expectEqual(@as(S, .{ .a = 0x12, .b = 0x5634 }), (try r.peekStructReference(S)).*); + try testing.expectEqual(@as(S, .{ .a = 0x12, .b = 0x5634 }), (try r.peekStructPointer(S)).*); + try testing.expectEqual(@as(S, .{ .a = 0x12, .b = 0x5634 }), (try r.peekStructPointer(S)).*); }, .big => { - try testing.expectEqual(@as(S, .{ .a = 0x12, .b = 0x3456 }), (try r.peekStructReference(S)).*); - try testing.expectEqual(@as(S, .{ .a = 0x12, .b = 0x3456 }), (try r.peekStructReference(S)).*); + try testing.expectEqual(@as(S, .{ .a = 0x12, .b = 0x3456 }), (try r.peekStructPointer(S)).*); + try testing.expectEqual(@as(S, .{ .a = 0x12, .b = 0x3456 }), (try r.peekStructPointer(S)).*); }, } } diff --git a/lib/std/Io/Writer.zig b/lib/std/Io/Writer.zig index 11bc05a00d..4d8f04b246 100644 --- a/lib/std/Io/Writer.zig +++ b/lib/std/Io/Writer.zig @@ -851,9 +851,6 @@ pub inline fn writeStruct(w: *Writer, value: anytype, endian: std.builtin.Endian } } -/// If, `endian` is not native, -/// * Asserts that the buffer storage capacity is at least enough to store `@sizeOf(Elem)` -/// * Asserts that the buffer is aligned enough for `@alignOf(Elem)`. pub inline fn writeSliceEndian( w: *Writer, Elem: type, @@ -867,18 +864,11 @@ pub inline fn writeSliceEndian( } } -/// Asserts that the buffer storage capacity is at least enough to store `@sizeOf(Elem)` -/// -/// Asserts that the buffer is aligned enough for `@alignOf(Elem)`. pub fn writeSliceSwap(w: *Writer, Elem: type, slice: []const Elem) Error!void { - var i: usize = 0; - while (i < slice.len) { - const dest_bytes = try w.writableSliceGreedy(@sizeOf(Elem)); - const dest: []Elem = @alignCast(@ptrCast(dest_bytes[0 .. dest_bytes.len - dest_bytes.len % @sizeOf(Elem)])); - const copy_len = @min(dest.len, slice.len - i); - @memcpy(dest[0..copy_len], slice[i..][0..copy_len]); - i += copy_len; - std.mem.byteSwapAllElements(Elem, dest); + for (slice) |elem| { + var tmp = elem; + std.mem.byteSwapAllFields(Elem, &tmp); + try w.writeAll(@ptrCast(&tmp)); } } @@ -2650,9 +2640,10 @@ test writeStruct { } test writeSliceEndian { - var buffer: [4]u8 align(2) = undefined; + var buffer: [5]u8 align(2) = undefined; var w: Writer = .fixed(&buffer); + try w.writeByte('x'); const array: [2]u16 = .{ 0x1234, 0x5678 }; try writeSliceEndian(&w, u16, &array, .big); - try testing.expectEqualSlices(u8, &.{ 0x12, 0x34, 0x56, 0x78 }, &buffer); + try testing.expectEqualSlices(u8, &.{ 'x', 0x12, 0x34, 0x56, 0x78 }, &buffer); } diff --git a/lib/std/zig/Server.zig b/lib/std/zig/Server.zig index 8fc016d284..12bd259b16 100644 --- a/lib/std/zig/Server.zig +++ b/lib/std/zig/Server.zig @@ -118,8 +118,6 @@ pub fn init(options: Options) !Server { .in = options.in, .out = options.out, }; - assert(s.out.buffer.len >= 4); - std.debug.assertAligned(s.out.buffer.ptr, .@"4"); try s.serveStringMessage(.zig_version, options.zig_version); return s; } @@ -203,8 +201,8 @@ pub const TestMetadata = struct { pub fn serveTestMetadata(s: *Server, test_metadata: TestMetadata) !void { const header: OutMessage.TestMetadata = .{ - .tests_len = @as(u32, @intCast(test_metadata.names.len)), - .string_bytes_len = @as(u32, @intCast(test_metadata.string_bytes.len)), + .tests_len = @intCast(test_metadata.names.len), + .string_bytes_len = @intCast(test_metadata.string_bytes.len), }; const trailing = 2; const bytes_len = @sizeOf(OutMessage.TestMetadata) + diff --git a/src/Zcu.zig b/src/Zcu.zig index 897bb6e89e..d337f0b943 100644 --- a/src/Zcu.zig +++ b/src/Zcu.zig @@ -2821,7 +2821,7 @@ pub fn loadZirCache(gpa: Allocator, cache_file: std.fs.File) !Zir { var buffer: [2000]u8 = undefined; var file_reader = cache_file.reader(&buffer); return result: { - const header = file_reader.interface.takeStructReference(Zir.Header) catch |err| break :result err; + const header = file_reader.interface.takeStructPointer(Zir.Header) catch |err| break :result err; break :result loadZirCacheBody(gpa, header.*, &file_reader.interface); } catch |err| switch (err) { error.ReadFailed => return file_reader.err.?, diff --git a/src/Zcu/PerThread.zig b/src/Zcu/PerThread.zig index dc2308add1..26f008e1c8 100644 --- a/src/Zcu/PerThread.zig +++ b/src/Zcu/PerThread.zig @@ -349,7 +349,7 @@ fn loadZirZoirCache( const cache_br = &cache_fr.interface; // First we read the header to determine the lengths of arrays. - const header = (cache_br.takeStructReference(Header) catch |err| switch (err) { + const header = (cache_br.takeStructPointer(Header) catch |err| switch (err) { error.ReadFailed => return cache_fr.err.?, // This can happen if Zig bails out of this function between creating // the cached file and writing it. From c58cce799932a5b9a735ac359794ec8bcde61633 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alex=20R=C3=B8nne=20Petersen?= Date: Sun, 20 Jul 2025 01:15:46 +0200 Subject: [PATCH 22/26] std.Build.Step.Run: fix up 681d324c49e7cdc773cc891ea49ed69dd03c23c7 https://github.com/ziglang/zig/pull/24151/files#r2217494741 --- lib/std/Build/Step/Run.zig | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/lib/std/Build/Step/Run.zig b/lib/std/Build/Step/Run.zig index b4e2e73c0a..b8d1a3a9cb 100644 --- a/lib/std/Build/Step/Run.zig +++ b/lib/std/Build/Step/Run.zig @@ -1118,10 +1118,12 @@ fn runCommand( // Wine's excessive stderr logging is only situationally helpful. Disable it by default, but // allow the user to override it (e.g. with `WINEDEBUG=err+all`) if desired. if (env_map.get("WINEDEBUG") == null) { - // We don't own `env_map` at this point, so turn it into a copy before modifying it. - env_map = arena.create(EnvMap) catch @panic("OOM"); - env_map.hash_map = try env_map.hash_map.cloneWithAllocator(arena); - try env_map.put("WINEDEBUG", "-all"); + // We don't own `env_map` at this point, so create a copy in order to modify it. + const new_env_map = arena.create(EnvMap) catch @panic("OOM"); + new_env_map.hash_map = try env_map.hash_map.cloneWithAllocator(arena); + try new_env_map.put("WINEDEBUG", "-all"); + + env_map = new_env_map; } } else { return failForeign(run, "-fwine", argv[0], exe); From 4780cc50cf7e42f6af3eb71ef3897f4b341215b4 Mon Sep 17 00:00:00 2001 From: Marc Tiehuis Date: Sun, 20 Jul 2025 13:57:31 +1200 Subject: [PATCH 23/26] std.Io.Writer: support alignment for `{t}` specifier --- lib/std/Io/Writer.zig | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/lib/std/Io/Writer.zig b/lib/std/Io/Writer.zig index 4b0e142fb0..98fec6196f 100644 --- a/lib/std/Io/Writer.zig +++ b/lib/std/Io/Writer.zig @@ -1123,8 +1123,8 @@ pub fn printValue( else => invalidFmtError(fmt, value), }, 't' => switch (@typeInfo(T)) { - .error_set => return w.writeAll(@errorName(value)), - .@"enum", .@"union" => return w.writeAll(@tagName(value)), + .error_set => return w.alignBufferOptions(@errorName(value), options), + .@"enum", .@"union" => return w.alignBufferOptions(@tagName(value), options), else => invalidFmtError(fmt, value), }, else => {}, @@ -2134,6 +2134,14 @@ test "bytes.hex" { try testing.expectFmt("lowercase: 000ebabe\n", "lowercase: {x}\n", .{bytes_with_zeros}); } +test "padding" { + const foo: enum { foo } = .foo; + try testing.expectFmt("tag: |foo |\n", "tag: |{t:<4}|\n", .{foo}); + + const bar: error{bar} = error.bar; + try testing.expectFmt("error: |bar |\n", "error: |{t:<4}|\n", .{bar}); +} + test fixed { { var buf: [255]u8 = undefined; From 14bb533203b87d7ef8c7d9434f8efba469ed1b0f Mon Sep 17 00:00:00 2001 From: antlilja Date: Sun, 20 Jul 2025 13:48:24 +0200 Subject: [PATCH 24/26] use stdout_buffer instead of stdio_buffer in main.zig --- src/main.zig | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main.zig b/src/main.zig index d55ec85b90..7ad40e1a68 100644 --- a/src/main.zig +++ b/src/main.zig @@ -346,7 +346,7 @@ fn mainArgs(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { } else if (mem.eql(u8, cmd, "targets")) { dev.check(.targets_command); const host = std.zig.resolveTargetQueryOrFatal(.{}); - var stdout_writer = fs.File.stdout().writer(&stdio_buffer); + var stdout_writer = fs.File.stdout().writer(&stdout_buffer); try @import("print_targets.zig").cmdTargets(arena, cmd_args, &stdout_writer.interface, &host); return stdout_writer.interface.flush(); } else if (mem.eql(u8, cmd, "version")) { @@ -359,7 +359,7 @@ fn mainArgs(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { } else if (mem.eql(u8, cmd, "env")) { dev.check(.env_command); verifyLibcxxCorrectlyLinked(); - var stdout_writer = fs.File.stdout().writer(&stdio_buffer); + var stdout_writer = fs.File.stdout().writer(&stdout_buffer); try @import("print_env.zig").cmdEnv(arena, &stdout_writer.interface); return stdout_writer.interface.flush(); } else if (mem.eql(u8, cmd, "reduce")) { From f657767b600064c520fac0e72da6016508a8b601 Mon Sep 17 00:00:00 2001 From: Kendall Condon Date: Sun, 20 Jul 2025 13:30:57 -0400 Subject: [PATCH 25/26] langref: upgrade grammar.y for asm clobber change --- doc/langref.html.in | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/langref.html.in b/doc/langref.html.in index dcf13e812d..e8189e5c42 100644 --- a/doc/langref.html.in +++ b/doc/langref.html.in @@ -7987,7 +7987,7 @@ AsmInput <- COLON AsmInputList AsmClobbers? AsmInputItem <- LBRACKET IDENTIFIER RBRACKET STRINGLITERAL LPAREN Expr RPAREN -AsmClobbers <- COLON StringList +AsmClobbers <- COLON Expr # *** Helper grammar *** BreakLabel <- COLON IDENTIFIER From e4abdf5a133ac4822644da1dabd5437ac751d78d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 20 Jul 2025 11:23:12 -0700 Subject: [PATCH 26/26] std.Io.Reader: fix takeStruct/peekStruct packed closes #24516 --- lib/std/Io/Reader.zig | 25 +++++++++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/lib/std/Io/Reader.zig b/lib/std/Io/Reader.zig index 659ebf2f97..f25e113522 100644 --- a/lib/std/Io/Reader.zig +++ b/lib/std/Io/Reader.zig @@ -1148,7 +1148,7 @@ pub inline fn takeStruct(r: *Reader, comptime T: type, endian: std.builtin.Endia return res; }, .@"packed" => { - return takeInt(r, info.backing_integer.?, endian); + return @bitCast(try takeInt(r, info.backing_integer.?, endian)); }, }, else => @compileError("not a struct"), @@ -1173,7 +1173,7 @@ pub inline fn peekStruct(r: *Reader, comptime T: type, endian: std.builtin.Endia return res; }, .@"packed" => { - return peekInt(r, info.backing_integer.?, endian); + return @bitCast(try peekInt(r, info.backing_integer.?, endian)); }, }, else => @compileError("not a struct"), @@ -1724,6 +1724,27 @@ test "takeDelimiterInclusive when it rebases" { } } +test "takeStruct and peekStruct packed" { + var r: Reader = .fixed(&.{ 0b11110000, 0b00110011 }); + const S = packed struct(u16) { a: u2, b: u6, c: u7, d: u1 }; + + try testing.expectEqual(@as(S, .{ + .a = 0b11, + .b = 0b001100, + .c = 0b1110000, + .d = 0b1, + }), try r.peekStruct(S, .big)); + + try testing.expectEqual(@as(S, .{ + .a = 0b11, + .b = 0b001100, + .c = 0b1110000, + .d = 0b1, + }), try r.takeStruct(S, .big)); + + try testing.expectError(error.EndOfStream, r.takeStruct(S, .little)); +} + /// Provides a `Reader` implementation by passing data from an underlying /// reader through `Hasher.update`. ///