diff --git a/lib/std/compress/flate.zig b/lib/std/compress/flate.zig index 79ebfd8869..4dbfe7f75a 100644 --- a/lib/std/compress/flate.zig +++ b/lib/std/compress/flate.zig @@ -1,5 +1,10 @@ const std = @import("../std.zig"); +/// When decompressing, the output buffer is used as the history window, so +/// less than this may result in failure to decompress streams that were +/// compressed with a larger window. +pub const max_window_len = 1 << 16; + /// Deflate is a lossless data compression file format that uses a combination /// of LZ77 and Huffman coding. pub const deflate = @import("flate/deflate.zig"); diff --git a/lib/std/compress/flate/inflate.zig b/lib/std/compress/flate/inflate.zig index 1d89e7f7c8..166ea94cd4 100644 --- a/lib/std/compress/flate/inflate.zig +++ b/lib/std/compress/flate/inflate.zig @@ -403,6 +403,10 @@ pub fn Inflate(comptime container: Container, comptime Lookahead: type) type { }, }; } + + pub fn readable(self: *Self, buffer: []u8) std.io.BufferedReader { + return reader(self).buffered(buffer); + } }; } diff --git a/lib/std/compress/zlib.zig b/lib/std/compress/zlib.zig index 8b0d1d6122..35fede8eeb 100644 --- a/lib/std/compress/zlib.zig +++ b/lib/std/compress/zlib.zig @@ -2,6 +2,11 @@ const std = @import("../std.zig"); const deflate = @import("flate/deflate.zig"); const inflate = @import("flate/inflate.zig"); +/// When decompressing, the output buffer is used as the history window, so +/// less than this may result in failure to decompress streams that were +/// compressed with a larger window. +pub const max_window_len = std.compress.flate.max_window_len; + /// Decompress compressed data from reader and write plain data to the writer. pub fn decompress(reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) !void { try inflate.decompress(.zlib, reader, writer); diff --git a/lib/std/crypto.zig b/lib/std/crypto.zig index 8482502828..5582e91e59 100644 --- a/lib/std/crypto.zig +++ b/lib/std/crypto.zig @@ -1,5 +1,7 @@ //! Cryptography. +const std = @import("std.zig"); +const assert = std.debug.assert; const root = @import("root"); pub const timing_safe = @import("crypto/timing_safe.zig"); @@ -119,7 +121,7 @@ pub const hash = struct { pub const blake2 = @import("crypto/blake2.zig"); pub const Blake3 = @import("crypto/blake3.zig").Blake3; pub const Md5 = @import("crypto/md5.zig").Md5; - pub const Sha1 = @import("crypto/sha1.zig").Sha1; + pub const Sha1 = @import("crypto/Sha1.zig"); pub const sha2 = @import("crypto/sha2.zig"); pub const sha3 = @import("crypto/sha3.zig"); pub const composition = @import("crypto/hash_composition.zig"); @@ -217,8 +219,6 @@ pub const random = @import("crypto/tlcsprng.zig").interface; /// Encoding and decoding pub const codecs = @import("crypto/codecs.zig"); -const std = @import("std.zig"); - pub const errors = @import("crypto/errors.zig"); pub const tls = @import("crypto/tls.zig"); diff --git a/lib/std/crypto/Sha1.zig b/lib/std/crypto/Sha1.zig new file mode 100644 index 0000000000..228a1a74c2 --- /dev/null +++ b/lib/std/crypto/Sha1.zig @@ -0,0 +1,424 @@ +//! The SHA-1 function is now considered cryptographically broken. +//! Namely, it is feasible to find multiple inputs producing the same hash. +//! For a fast-performing, cryptographically secure hash function, see SHA512/256, BLAKE2 or BLAKE3. + +const Sha1 = @This(); +const std = @import("../std.zig"); +const mem = std.mem; +const math = std.math; +const assert = std.debug.assert; + +pub const block_length = 64; +pub const digest_length = 20; +pub const Options = struct {}; + +s: [5]u32, +/// Streaming Cache +buf: [64]u8, +buf_end: u8, +total_len: u64, + +pub fn init(options: Options) Sha1 { + _ = options; + return .{ + .s = [_]u32{ + 0x67452301, + 0xEFCDAB89, + 0x98BADCFE, + 0x10325476, + 0xC3D2E1F0, + }, + .buf = undefined, + .buf_end = 0, + .total_len = 0, + }; +} + +pub fn hash(b: []const u8, out: *[digest_length]u8, options: Options) void { + var d = Sha1.init(options); + d.update(b); + d.final(out); +} + +pub fn update(d: *Sha1, b: []const u8) void { + var off: usize = 0; + + // Partial buffer exists from previous update. Copy into buffer then hash. + const unused_buf = d.buf[d.buf_end..]; + if (unused_buf.len < d.buf.len and b.len >= unused_buf.len) { + @memcpy(unused_buf, b[0..unused_buf.len]); + off += unused_buf.len; + round(&d.s, &d.buf); + d.buf_end = 0; + } + + // Full middle blocks. + while (off + 64 <= b.len) : (off += 64) { + round(&d.s, b[off..][0..64]); + } + + // Copy any remainder for next pass. + const remainder = b[off..]; + @memcpy(d.buf[d.buf_end..][0..remainder.len], remainder); + d.buf_end = @intCast(d.buf_end + remainder.len); + d.total_len += b.len; +} + +pub fn peek(d: Sha1) [digest_length]u8 { + var copy = d; + return copy.finalResult(); +} + +pub fn final(d: *Sha1, out: *[digest_length]u8) void { + // The buffer here will never be completely full. + @memset(d.buf[d.buf_end..], 0); + + // Append padding bits. + d.buf[d.buf_end] = 0x80; + d.buf_end += 1; + + // > 448 mod 512 so need to add an extra round to wrap around. + if (64 - d.buf_end < 8) { + round(&d.s, d.buf[0..]); + @memset(d.buf[0..], 0); + } + + // Append message length. + var i: usize = 1; + var len = d.total_len >> 5; + d.buf[63] = @as(u8, @intCast(d.total_len & 0x1f)) << 3; + while (i < 8) : (i += 1) { + d.buf[63 - i] = @as(u8, @intCast(len & 0xff)); + len >>= 8; + } + + round(&d.s, d.buf[0..]); + + for (d.s, 0..) |s, j| { + mem.writeInt(u32, out[4 * j ..][0..4], s, .big); + } +} + +pub fn finalResult(d: *Sha1) [digest_length]u8 { + var result: [digest_length]u8 = undefined; + d.final(&result); + return result; +} + +pub fn round(d_s: *[5]u32, b: *const [64]u8) void { + var s: [16]u32 = undefined; + + var v: [5]u32 = [_]u32{ + d_s[0], + d_s[1], + d_s[2], + d_s[3], + d_s[4], + }; + + const round0a = comptime [_]RoundParam{ + .abcdei(0, 1, 2, 3, 4, 0), + .abcdei(4, 0, 1, 2, 3, 1), + .abcdei(3, 4, 0, 1, 2, 2), + .abcdei(2, 3, 4, 0, 1, 3), + .abcdei(1, 2, 3, 4, 0, 4), + .abcdei(0, 1, 2, 3, 4, 5), + .abcdei(4, 0, 1, 2, 3, 6), + .abcdei(3, 4, 0, 1, 2, 7), + .abcdei(2, 3, 4, 0, 1, 8), + .abcdei(1, 2, 3, 4, 0, 9), + .abcdei(0, 1, 2, 3, 4, 10), + .abcdei(4, 0, 1, 2, 3, 11), + .abcdei(3, 4, 0, 1, 2, 12), + .abcdei(2, 3, 4, 0, 1, 13), + .abcdei(1, 2, 3, 4, 0, 14), + .abcdei(0, 1, 2, 3, 4, 15), + }; + inline for (round0a) |r| { + s[r.i] = mem.readInt(u32, b[r.i * 4 ..][0..4], .big); + + v[r.e] = v[r.e] +% math.rotl(u32, v[r.a], @as(u32, 5)) +% 0x5A827999 +% s[r.i & 0xf] +% ((v[r.b] & v[r.c]) | (~v[r.b] & v[r.d])); + v[r.b] = math.rotl(u32, v[r.b], @as(u32, 30)); + } + + const round0b = comptime [_]RoundParam{ + .abcdei(4, 0, 1, 2, 3, 16), + .abcdei(3, 4, 0, 1, 2, 17), + .abcdei(2, 3, 4, 0, 1, 18), + .abcdei(1, 2, 3, 4, 0, 19), + }; + inline for (round0b) |r| { + const t = s[(r.i - 3) & 0xf] ^ s[(r.i - 8) & 0xf] ^ s[(r.i - 14) & 0xf] ^ s[(r.i - 16) & 0xf]; + s[r.i & 0xf] = math.rotl(u32, t, @as(u32, 1)); + + v[r.e] = v[r.e] +% math.rotl(u32, v[r.a], @as(u32, 5)) +% 0x5A827999 +% s[r.i & 0xf] +% ((v[r.b] & v[r.c]) | (~v[r.b] & v[r.d])); + v[r.b] = math.rotl(u32, v[r.b], @as(u32, 30)); + } + + const round1 = comptime [_]RoundParam{ + .abcdei(0, 1, 2, 3, 4, 20), + .abcdei(4, 0, 1, 2, 3, 21), + .abcdei(3, 4, 0, 1, 2, 22), + .abcdei(2, 3, 4, 0, 1, 23), + .abcdei(1, 2, 3, 4, 0, 24), + .abcdei(0, 1, 2, 3, 4, 25), + .abcdei(4, 0, 1, 2, 3, 26), + .abcdei(3, 4, 0, 1, 2, 27), + .abcdei(2, 3, 4, 0, 1, 28), + .abcdei(1, 2, 3, 4, 0, 29), + .abcdei(0, 1, 2, 3, 4, 30), + .abcdei(4, 0, 1, 2, 3, 31), + .abcdei(3, 4, 0, 1, 2, 32), + .abcdei(2, 3, 4, 0, 1, 33), + .abcdei(1, 2, 3, 4, 0, 34), + .abcdei(0, 1, 2, 3, 4, 35), + .abcdei(4, 0, 1, 2, 3, 36), + .abcdei(3, 4, 0, 1, 2, 37), + .abcdei(2, 3, 4, 0, 1, 38), + .abcdei(1, 2, 3, 4, 0, 39), + }; + inline for (round1) |r| { + const t = s[(r.i - 3) & 0xf] ^ s[(r.i - 8) & 0xf] ^ s[(r.i - 14) & 0xf] ^ s[(r.i - 16) & 0xf]; + s[r.i & 0xf] = math.rotl(u32, t, @as(u32, 1)); + + v[r.e] = v[r.e] +% math.rotl(u32, v[r.a], @as(u32, 5)) +% 0x6ED9EBA1 +% s[r.i & 0xf] +% (v[r.b] ^ v[r.c] ^ v[r.d]); + v[r.b] = math.rotl(u32, v[r.b], @as(u32, 30)); + } + + const round2 = comptime [_]RoundParam{ + .abcdei(0, 1, 2, 3, 4, 40), + .abcdei(4, 0, 1, 2, 3, 41), + .abcdei(3, 4, 0, 1, 2, 42), + .abcdei(2, 3, 4, 0, 1, 43), + .abcdei(1, 2, 3, 4, 0, 44), + .abcdei(0, 1, 2, 3, 4, 45), + .abcdei(4, 0, 1, 2, 3, 46), + .abcdei(3, 4, 0, 1, 2, 47), + .abcdei(2, 3, 4, 0, 1, 48), + .abcdei(1, 2, 3, 4, 0, 49), + .abcdei(0, 1, 2, 3, 4, 50), + .abcdei(4, 0, 1, 2, 3, 51), + .abcdei(3, 4, 0, 1, 2, 52), + .abcdei(2, 3, 4, 0, 1, 53), + .abcdei(1, 2, 3, 4, 0, 54), + .abcdei(0, 1, 2, 3, 4, 55), + .abcdei(4, 0, 1, 2, 3, 56), + .abcdei(3, 4, 0, 1, 2, 57), + .abcdei(2, 3, 4, 0, 1, 58), + .abcdei(1, 2, 3, 4, 0, 59), + }; + inline for (round2) |r| { + const t = s[(r.i - 3) & 0xf] ^ s[(r.i - 8) & 0xf] ^ s[(r.i - 14) & 0xf] ^ s[(r.i - 16) & 0xf]; + s[r.i & 0xf] = math.rotl(u32, t, @as(u32, 1)); + + v[r.e] = v[r.e] +% math.rotl(u32, v[r.a], @as(u32, 5)) +% 0x8F1BBCDC +% s[r.i & 0xf] +% ((v[r.b] & v[r.c]) ^ (v[r.b] & v[r.d]) ^ (v[r.c] & v[r.d])); + v[r.b] = math.rotl(u32, v[r.b], @as(u32, 30)); + } + + const round3 = comptime [_]RoundParam{ + .abcdei(0, 1, 2, 3, 4, 60), + .abcdei(4, 0, 1, 2, 3, 61), + .abcdei(3, 4, 0, 1, 2, 62), + .abcdei(2, 3, 4, 0, 1, 63), + .abcdei(1, 2, 3, 4, 0, 64), + .abcdei(0, 1, 2, 3, 4, 65), + .abcdei(4, 0, 1, 2, 3, 66), + .abcdei(3, 4, 0, 1, 2, 67), + .abcdei(2, 3, 4, 0, 1, 68), + .abcdei(1, 2, 3, 4, 0, 69), + .abcdei(0, 1, 2, 3, 4, 70), + .abcdei(4, 0, 1, 2, 3, 71), + .abcdei(3, 4, 0, 1, 2, 72), + .abcdei(2, 3, 4, 0, 1, 73), + .abcdei(1, 2, 3, 4, 0, 74), + .abcdei(0, 1, 2, 3, 4, 75), + .abcdei(4, 0, 1, 2, 3, 76), + .abcdei(3, 4, 0, 1, 2, 77), + .abcdei(2, 3, 4, 0, 1, 78), + .abcdei(1, 2, 3, 4, 0, 79), + }; + inline for (round3) |r| { + const t = s[(r.i - 3) & 0xf] ^ s[(r.i - 8) & 0xf] ^ s[(r.i - 14) & 0xf] ^ s[(r.i - 16) & 0xf]; + s[r.i & 0xf] = math.rotl(u32, t, @as(u32, 1)); + + v[r.e] = v[r.e] +% math.rotl(u32, v[r.a], @as(u32, 5)) +% 0xCA62C1D6 +% s[r.i & 0xf] +% (v[r.b] ^ v[r.c] ^ v[r.d]); + v[r.b] = math.rotl(u32, v[r.b], @as(u32, 30)); + } + + d_s[0] +%= v[0]; + d_s[1] +%= v[1]; + d_s[2] +%= v[2]; + d_s[3] +%= v[3]; + d_s[4] +%= v[4]; +} + +pub fn writable(sha1: *Sha1, buffer: []u8) std.io.BufferedWriter { + return .{ + .unbuffered_writer = .{ + .context = sha1, + .vtable = &.{ + .writeSplat = writeSplat, + .writeFile = std.io.Writer.unimplementedWriteFile, + }, + }, + .buffer = buffer, + }; +} + +fn writeSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) std.io.Writer.Error!usize { + const sha1: *Sha1 = @ptrCast(@alignCast(context)); + const start_total = sha1.total_len; + if (sha1.buf_end == 0) { + try writeSplatAligned(sha1, data, splat); + const n: usize = @intCast(sha1.total_len - start_total); + if (n > 0) return n; + } + for (data[0 .. data.len - 1]) |slice| { + const copy_len = @min(slice.len, sha1.buf.len - sha1.buf_end); + @memcpy(sha1.buf[sha1.buf_end..][0..copy_len], slice[0..copy_len]); + sha1.total_len += copy_len; + if (sha1.buf.len - sha1.buf_end - copy_len == 0) { + round(&sha1.s, &sha1.buf); + sha1.buf_end = 0; + return @intCast(sha1.total_len - start_total); + } + sha1.buf_end = @intCast(sha1.buf_end + copy_len); + } + const slice = data[data.len - 1]; + for (0..splat) |_| { + const copy_len = @min(slice.len, sha1.buf.len - sha1.buf_end); + @memcpy(sha1.buf[sha1.buf_end..][0..copy_len], slice[0..copy_len]); + sha1.total_len += copy_len; + if (sha1.buf.len - sha1.buf_end - copy_len == 0) { + round(&sha1.s, &sha1.buf); + sha1.buf_end = 0; + return @intCast(sha1.total_len - start_total); + } + sha1.buf_end = @intCast(sha1.buf_end + copy_len); + } + return @intCast(sha1.total_len - start_total); +} + +fn writeSplatAligned(sha1: *Sha1, data: []const []const u8, splat: usize) std.io.Writer.Error!void { + assert(sha1.buf_end == 0); + for (data[0 .. data.len - 1]) |slice| { + var off: usize = 0; + while (off < slice.len) { + if (off + 64 > slice.len) { + sha1.total_len += off; + return; + } + round(&sha1.s, slice[off..][0..64]); + off += 64; + } + sha1.total_len += off; + } + const last = data[data.len - 1]; + if (last.len * splat < 64) return; + if (last.len == 1) { + @memset(&sha1.buf, last[0]); + for (0..splat / 64) |_| round(&sha1.s, &sha1.buf); + sha1.total_len += (splat / 64) * 64; + return; + } + if (last.len >= 64) { + for (0..splat) |_| { + var off: usize = 0; + while (off < last.len) { + if (off + 64 > last.len) { + sha1.total_len += off; + return; + } + round(&sha1.s, last[off..][0..64]); + off += 64; + } + } + sha1.total_len += last.len * splat; + return; + } + // Opportunity: if last.len is less than 64, we could fill up the buffer + // with the pattern repeated then do rounds. +} + +const RoundParam = struct { + a: usize, + b: usize, + c: usize, + d: usize, + e: usize, + i: u32, + + fn abcdei(a: usize, b: usize, c: usize, d: usize, e: usize, i: u32) RoundParam { + return .{ + .a = a, + .b = b, + .c = c, + .d = d, + .e = e, + .i = i, + }; + } +}; + +const htest = @import("test.zig"); + +test "sha1 single" { + try htest.assertEqualHash(Sha1, "da39a3ee5e6b4b0d3255bfef95601890afd80709", ""); + try htest.assertEqualHash(Sha1, "a9993e364706816aba3e25717850c26c9cd0d89d", "abc"); + try htest.assertEqualHash(Sha1, "a49b2446a02c645bf419f995b67091253a04a259", "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"); +} + +test "sha1 streaming" { + var h = Sha1.init(.{}); + var out: [20]u8 = undefined; + + h.final(&out); + try htest.assertEqual("da39a3ee5e6b4b0d3255bfef95601890afd80709", out[0..]); + + h = Sha1.init(.{}); + h.update("abc"); + h.final(&out); + try htest.assertEqual("a9993e364706816aba3e25717850c26c9cd0d89d", out[0..]); + + h = Sha1.init(.{}); + h.update("a"); + h.update("b"); + h.update("c"); + h.final(&out); + try htest.assertEqual("a9993e364706816aba3e25717850c26c9cd0d89d", out[0..]); +} + +test "sha1 aligned final" { + var block = [_]u8{0} ** Sha1.block_length; + var out: [Sha1.digest_length]u8 = undefined; + + var h = Sha1.init(.{}); + h.update(&block); + h.final(out[0..]); +} + +test "splat" { + var vecs = [_][]const u8{ + "hello", + "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstuyyyyyyyyyyyyyyyyyyyyyyyyy", + "x", + }; + const splat_len = 512; + const update_result = r: { + var sha1: Sha1 = .init(.{}); + sha1.update(vecs[0]); + sha1.update(vecs[1]); + var buffer: [splat_len]u8 = undefined; + @memset(&buffer, vecs[2][0]); + sha1.update(&buffer); + break :r sha1.finalResult(); + }; + const stream_result = r: { + var sha1: Sha1 = .init(.{}); + var bw = sha1.writable(&.{}); + try bw.writeSplatAll(&vecs, splat_len); + try std.testing.expectEqual(vecs[0].len + vecs[1].len + vecs[2].len * splat_len, sha1.total_len); + break :r sha1.finalResult(); + }; + try std.testing.expectEqualSlices(u8, &update_result, &stream_result); +} diff --git a/lib/std/crypto/aegis.zig b/lib/std/crypto/aegis.zig index 2f565ce490..dedd962059 100644 --- a/lib/std/crypto/aegis.zig +++ b/lib/std/crypto/aegis.zig @@ -801,18 +801,6 @@ fn AegisMac(comptime T: type) type { ctx.update(msg); ctx.final(out); } - - pub const Error = error{}; - pub const Writer = std.io.Writer(*Mac, Error, write); - - fn write(self: *Mac, bytes: []const u8) Error!usize { - self.update(bytes); - return bytes.len; - } - - pub fn writer(self: *Mac) Writer { - return .{ .context = self }; - } }; } diff --git a/lib/std/crypto/blake2.zig b/lib/std/crypto/blake2.zig index 1a285080b5..28754b4cae 100644 --- a/lib/std/crypto/blake2.zig +++ b/lib/std/crypto/blake2.zig @@ -185,18 +185,6 @@ pub fn Blake2s(comptime out_bits: usize) type { r.* ^= v[i] ^ v[i + 8]; } } - - pub const Error = error{}; - pub const Writer = std.io.Writer(*Self, Error, write); - - fn write(self: *Self, bytes: []const u8) Error!usize { - self.update(bytes); - return bytes.len; - } - - pub fn writer(self: *Self) Writer { - return .{ .context = self }; - } }; } diff --git a/lib/std/crypto/blake3.zig b/lib/std/crypto/blake3.zig index 585c338417..5fb9f6ee76 100644 --- a/lib/std/crypto/blake3.zig +++ b/lib/std/crypto/blake3.zig @@ -474,18 +474,6 @@ pub const Blake3 = struct { } output.rootOutputBytes(out_slice); } - - pub const Error = error{}; - pub const Writer = std.io.Writer(*Blake3, Error, write); - - fn write(self: *Blake3, bytes: []const u8) Error!usize { - self.update(bytes); - return bytes.len; - } - - pub fn writer(self: *Blake3) Writer { - return .{ .context = self }; - } }; // Use named type declarations to workaround crash with anonymous structs (issue #4373). diff --git a/lib/std/crypto/codecs/asn1.zig b/lib/std/crypto/codecs/asn1.zig index 7921c70118..5ae156ecbd 100644 --- a/lib/std/crypto/codecs/asn1.zig +++ b/lib/std/crypto/codecs/asn1.zig @@ -90,39 +90,32 @@ pub const Tag = struct { }; } - pub fn encode(self: Tag, writer: anytype) @TypeOf(writer).Error!void { - var tag1 = FirstTag{ + pub fn encode(self: Tag, writer: *std.io.BufferedWriter) std.io.Writer.Error!void { + var tag1: FirstTag = .{ .number = undefined, .constructed = self.constructed, .class = self.class, }; - - var buffer: [3]u8 = undefined; - var stream = std.io.fixedBufferStream(&buffer); - var writer2 = stream.writer(); - switch (@intFromEnum(self.number)) { 0...std.math.maxInt(u5) => |n| { tag1.number = @intCast(n); - writer2.writeByte(@bitCast(tag1)) catch unreachable; + try writer.writeByte(@bitCast(tag1)); }, std.math.maxInt(u5) + 1...std.math.maxInt(u7) => |n| { tag1.number = 15; const tag2 = NextTag{ .number = @intCast(n), .continues = false }; - writer2.writeByte(@bitCast(tag1)) catch unreachable; - writer2.writeByte(@bitCast(tag2)) catch unreachable; + try writer.writeByte(@bitCast(tag1)); + try writer.writeByte(@bitCast(tag2)); }, else => |n| { tag1.number = 15; const tag2 = NextTag{ .number = @intCast(n >> 7), .continues = true }; const tag3 = NextTag{ .number = @truncate(n), .continues = false }; - writer2.writeByte(@bitCast(tag1)) catch unreachable; - writer2.writeByte(@bitCast(tag2)) catch unreachable; - writer2.writeByte(@bitCast(tag3)) catch unreachable; + try writer.writeByte(@bitCast(tag1)); + try writer.writeByte(@bitCast(tag2)); + try writer.writeByte(@bitCast(tag3)); }, } - - _ = try writer.write(stream.getWritten()); } const FirstTag = packed struct(u8) { number: u5, constructed: bool, class: Tag.Class }; diff --git a/lib/std/crypto/codecs/asn1/Oid.zig b/lib/std/crypto/codecs/asn1/Oid.zig index edca552050..4c20831c8b 100644 --- a/lib/std/crypto/codecs/asn1/Oid.zig +++ b/lib/std/crypto/codecs/asn1/Oid.zig @@ -4,9 +4,12 @@ //! organizations, or policy documents. encoded: []const u8, -pub const InitError = std.fmt.ParseIntError || error{MissingPrefix} || std.io.FixedBufferStream(u8).WriteError; +pub const EncodeError = error{ + WriteFailed, + MissingPrefix, +}; -pub fn fromDot(dot_notation: []const u8, out: []u8) InitError!Oid { +pub fn encode(dot_notation: []const u8, out: *std.io.BufferedWriter) EncodeError!void { var split = std.mem.splitScalar(u8, dot_notation, '.'); const first_str = split.next() orelse return error.MissingPrefix; const second_str = split.next() orelse return error.MissingPrefix; @@ -14,10 +17,7 @@ pub fn fromDot(dot_notation: []const u8, out: []u8) InitError!Oid { const first = try std.fmt.parseInt(u8, first_str, 10); const second = try std.fmt.parseInt(u8, second_str, 10); - var stream = std.io.fixedBufferStream(out); - var writer = stream.writer(); - - try writer.writeByte(first * 40 + second); + try out.writeByte(first * 40 + second); var i: usize = 1; while (split.next()) |s| { @@ -28,16 +28,26 @@ pub fn fromDot(dot_notation: []const u8, out: []u8) InitError!Oid { const place = std.math.pow(Arc, encoding_base, n_bytes - @as(Arc, @intCast(j))); const digit: u8 = @intCast(@divFloor(parsed, place)); - try writer.writeByte(digit | 0x80); + try out.writeByte(digit | 0x80); parsed -= digit * place; i += 1; } - try writer.writeByte(@intCast(parsed)); + try out.writeByte(@intCast(parsed)); i += 1; } +} - return .{ .encoded = stream.getWritten() }; +pub const InitError = std.fmt.ParseIntError || error{ MissingPrefix, BufferTooSmall }; + +pub fn fromDot(dot_notation: []const u8, out: []u8) InitError!Oid { + var bw: std.io.BufferedWriter = undefined; + bw.initFixed(out); + encode(dot_notation, &bw) catch |err| switch (err) { + error.WriteFailed => return error.BufferTooSmall, + else => |e| return e, + }; + return .{ .encoded = bw.getWritten() }; } test fromDot { @@ -48,7 +58,7 @@ test fromDot { } } -pub fn toDot(self: Oid, writer: anytype) @TypeOf(writer).Error!void { +pub fn toDot(self: Oid, writer: *std.io.BufferedWriter) std.io.Writer.Error!void { const encoded = self.encoded; const first = @divTrunc(encoded[0], 40); const second = encoded[0] - first * 40; @@ -80,9 +90,10 @@ test toDot { var buf: [256]u8 = undefined; for (test_cases) |t| { - var stream = std.io.fixedBufferStream(&buf); - try toDot(Oid{ .encoded = t.encoded }, stream.writer()); - try std.testing.expectEqualStrings(t.dot_notation, stream.getWritten()); + var bw: std.io.BufferedWriter = undefined; + bw.initFixed(&buf); + try toDot(Oid{ .encoded = t.encoded }, &bw); + try std.testing.expectEqualStrings(t.dot_notation, bw.getWritten()); } } diff --git a/lib/std/crypto/phc_encoding.zig b/lib/std/crypto/phc_encoding.zig index 1aa7257bbb..c23702747a 100644 --- a/lib/std/crypto/phc_encoding.zig +++ b/lib/std/crypto/phc_encoding.zig @@ -196,9 +196,11 @@ pub fn serialize(params: anytype, str: []u8) Error![]const u8 { /// Compute the number of bytes required to serialize `params` pub fn calcSize(params: anytype) usize { - var buf = io.countingWriter(io.null_writer); - serializeTo(params, buf.writer()) catch unreachable; - return @as(usize, @intCast(buf.bytes_written)); + var null_writer: std.io.Writer.Null = .{}; + var trash: [128]u8 = undefined; + var bw = null_writer.writable(&trash); + serializeTo(params, &bw) catch unreachable; + return bw.count; } fn serializeTo(params: anytype, out: *std.io.BufferedWriter) !void { diff --git a/lib/std/crypto/sha1.zig b/lib/std/crypto/sha1.zig deleted file mode 100644 index 2968517b68..0000000000 --- a/lib/std/crypto/sha1.zig +++ /dev/null @@ -1,319 +0,0 @@ -const std = @import("../std.zig"); -const mem = std.mem; -const math = std.math; - -const RoundParam = struct { - a: usize, - b: usize, - c: usize, - d: usize, - e: usize, - i: u32, -}; - -fn roundParam(a: usize, b: usize, c: usize, d: usize, e: usize, i: u32) RoundParam { - return RoundParam{ - .a = a, - .b = b, - .c = c, - .d = d, - .e = e, - .i = i, - }; -} - -/// The SHA-1 function is now considered cryptographically broken. -/// Namely, it is feasible to find multiple inputs producing the same hash. -/// For a fast-performing, cryptographically secure hash function, see SHA512/256, BLAKE2 or BLAKE3. -pub const Sha1 = struct { - const Self = @This(); - pub const block_length = 64; - pub const digest_length = 20; - pub const Options = struct {}; - - s: [5]u32, - // Streaming Cache - buf: [64]u8 = undefined, - buf_len: u8 = 0, - total_len: u64 = 0, - - pub fn init(options: Options) Self { - _ = options; - return Self{ - .s = [_]u32{ - 0x67452301, - 0xEFCDAB89, - 0x98BADCFE, - 0x10325476, - 0xC3D2E1F0, - }, - }; - } - - pub fn hash(b: []const u8, out: *[digest_length]u8, options: Options) void { - var d = Sha1.init(options); - d.update(b); - d.final(out); - } - - pub fn update(d: *Self, b: []const u8) void { - var off: usize = 0; - - // Partial buffer exists from previous update. Copy into buffer then hash. - if (d.buf_len != 0 and d.buf_len + b.len >= 64) { - off += 64 - d.buf_len; - @memcpy(d.buf[d.buf_len..][0..off], b[0..off]); - - d.round(d.buf[0..]); - d.buf_len = 0; - } - - // Full middle blocks. - while (off + 64 <= b.len) : (off += 64) { - d.round(b[off..][0..64]); - } - - // Copy any remainder for next pass. - @memcpy(d.buf[d.buf_len..][0 .. b.len - off], b[off..]); - d.buf_len += @as(u8, @intCast(b[off..].len)); - - d.total_len += b.len; - } - - pub fn peek(d: Self) [digest_length]u8 { - var copy = d; - return copy.finalResult(); - } - - pub fn final(d: *Self, out: *[digest_length]u8) void { - // The buffer here will never be completely full. - @memset(d.buf[d.buf_len..], 0); - - // Append padding bits. - d.buf[d.buf_len] = 0x80; - d.buf_len += 1; - - // > 448 mod 512 so need to add an extra round to wrap around. - if (64 - d.buf_len < 8) { - d.round(d.buf[0..]); - @memset(d.buf[0..], 0); - } - - // Append message length. - var i: usize = 1; - var len = d.total_len >> 5; - d.buf[63] = @as(u8, @intCast(d.total_len & 0x1f)) << 3; - while (i < 8) : (i += 1) { - d.buf[63 - i] = @as(u8, @intCast(len & 0xff)); - len >>= 8; - } - - d.round(d.buf[0..]); - - for (d.s, 0..) |s, j| { - mem.writeInt(u32, out[4 * j ..][0..4], s, .big); - } - } - - pub fn finalResult(d: *Self) [digest_length]u8 { - var result: [digest_length]u8 = undefined; - d.final(&result); - return result; - } - - fn round(d: *Self, b: *const [64]u8) void { - var s: [16]u32 = undefined; - - var v: [5]u32 = [_]u32{ - d.s[0], - d.s[1], - d.s[2], - d.s[3], - d.s[4], - }; - - const round0a = comptime [_]RoundParam{ - roundParam(0, 1, 2, 3, 4, 0), - roundParam(4, 0, 1, 2, 3, 1), - roundParam(3, 4, 0, 1, 2, 2), - roundParam(2, 3, 4, 0, 1, 3), - roundParam(1, 2, 3, 4, 0, 4), - roundParam(0, 1, 2, 3, 4, 5), - roundParam(4, 0, 1, 2, 3, 6), - roundParam(3, 4, 0, 1, 2, 7), - roundParam(2, 3, 4, 0, 1, 8), - roundParam(1, 2, 3, 4, 0, 9), - roundParam(0, 1, 2, 3, 4, 10), - roundParam(4, 0, 1, 2, 3, 11), - roundParam(3, 4, 0, 1, 2, 12), - roundParam(2, 3, 4, 0, 1, 13), - roundParam(1, 2, 3, 4, 0, 14), - roundParam(0, 1, 2, 3, 4, 15), - }; - inline for (round0a) |r| { - s[r.i] = mem.readInt(u32, b[r.i * 4 ..][0..4], .big); - - v[r.e] = v[r.e] +% math.rotl(u32, v[r.a], @as(u32, 5)) +% 0x5A827999 +% s[r.i & 0xf] +% ((v[r.b] & v[r.c]) | (~v[r.b] & v[r.d])); - v[r.b] = math.rotl(u32, v[r.b], @as(u32, 30)); - } - - const round0b = comptime [_]RoundParam{ - roundParam(4, 0, 1, 2, 3, 16), - roundParam(3, 4, 0, 1, 2, 17), - roundParam(2, 3, 4, 0, 1, 18), - roundParam(1, 2, 3, 4, 0, 19), - }; - inline for (round0b) |r| { - const t = s[(r.i - 3) & 0xf] ^ s[(r.i - 8) & 0xf] ^ s[(r.i - 14) & 0xf] ^ s[(r.i - 16) & 0xf]; - s[r.i & 0xf] = math.rotl(u32, t, @as(u32, 1)); - - v[r.e] = v[r.e] +% math.rotl(u32, v[r.a], @as(u32, 5)) +% 0x5A827999 +% s[r.i & 0xf] +% ((v[r.b] & v[r.c]) | (~v[r.b] & v[r.d])); - v[r.b] = math.rotl(u32, v[r.b], @as(u32, 30)); - } - - const round1 = comptime [_]RoundParam{ - roundParam(0, 1, 2, 3, 4, 20), - roundParam(4, 0, 1, 2, 3, 21), - roundParam(3, 4, 0, 1, 2, 22), - roundParam(2, 3, 4, 0, 1, 23), - roundParam(1, 2, 3, 4, 0, 24), - roundParam(0, 1, 2, 3, 4, 25), - roundParam(4, 0, 1, 2, 3, 26), - roundParam(3, 4, 0, 1, 2, 27), - roundParam(2, 3, 4, 0, 1, 28), - roundParam(1, 2, 3, 4, 0, 29), - roundParam(0, 1, 2, 3, 4, 30), - roundParam(4, 0, 1, 2, 3, 31), - roundParam(3, 4, 0, 1, 2, 32), - roundParam(2, 3, 4, 0, 1, 33), - roundParam(1, 2, 3, 4, 0, 34), - roundParam(0, 1, 2, 3, 4, 35), - roundParam(4, 0, 1, 2, 3, 36), - roundParam(3, 4, 0, 1, 2, 37), - roundParam(2, 3, 4, 0, 1, 38), - roundParam(1, 2, 3, 4, 0, 39), - }; - inline for (round1) |r| { - const t = s[(r.i - 3) & 0xf] ^ s[(r.i - 8) & 0xf] ^ s[(r.i - 14) & 0xf] ^ s[(r.i - 16) & 0xf]; - s[r.i & 0xf] = math.rotl(u32, t, @as(u32, 1)); - - v[r.e] = v[r.e] +% math.rotl(u32, v[r.a], @as(u32, 5)) +% 0x6ED9EBA1 +% s[r.i & 0xf] +% (v[r.b] ^ v[r.c] ^ v[r.d]); - v[r.b] = math.rotl(u32, v[r.b], @as(u32, 30)); - } - - const round2 = comptime [_]RoundParam{ - roundParam(0, 1, 2, 3, 4, 40), - roundParam(4, 0, 1, 2, 3, 41), - roundParam(3, 4, 0, 1, 2, 42), - roundParam(2, 3, 4, 0, 1, 43), - roundParam(1, 2, 3, 4, 0, 44), - roundParam(0, 1, 2, 3, 4, 45), - roundParam(4, 0, 1, 2, 3, 46), - roundParam(3, 4, 0, 1, 2, 47), - roundParam(2, 3, 4, 0, 1, 48), - roundParam(1, 2, 3, 4, 0, 49), - roundParam(0, 1, 2, 3, 4, 50), - roundParam(4, 0, 1, 2, 3, 51), - roundParam(3, 4, 0, 1, 2, 52), - roundParam(2, 3, 4, 0, 1, 53), - roundParam(1, 2, 3, 4, 0, 54), - roundParam(0, 1, 2, 3, 4, 55), - roundParam(4, 0, 1, 2, 3, 56), - roundParam(3, 4, 0, 1, 2, 57), - roundParam(2, 3, 4, 0, 1, 58), - roundParam(1, 2, 3, 4, 0, 59), - }; - inline for (round2) |r| { - const t = s[(r.i - 3) & 0xf] ^ s[(r.i - 8) & 0xf] ^ s[(r.i - 14) & 0xf] ^ s[(r.i - 16) & 0xf]; - s[r.i & 0xf] = math.rotl(u32, t, @as(u32, 1)); - - v[r.e] = v[r.e] +% math.rotl(u32, v[r.a], @as(u32, 5)) +% 0x8F1BBCDC +% s[r.i & 0xf] +% ((v[r.b] & v[r.c]) ^ (v[r.b] & v[r.d]) ^ (v[r.c] & v[r.d])); - v[r.b] = math.rotl(u32, v[r.b], @as(u32, 30)); - } - - const round3 = comptime [_]RoundParam{ - roundParam(0, 1, 2, 3, 4, 60), - roundParam(4, 0, 1, 2, 3, 61), - roundParam(3, 4, 0, 1, 2, 62), - roundParam(2, 3, 4, 0, 1, 63), - roundParam(1, 2, 3, 4, 0, 64), - roundParam(0, 1, 2, 3, 4, 65), - roundParam(4, 0, 1, 2, 3, 66), - roundParam(3, 4, 0, 1, 2, 67), - roundParam(2, 3, 4, 0, 1, 68), - roundParam(1, 2, 3, 4, 0, 69), - roundParam(0, 1, 2, 3, 4, 70), - roundParam(4, 0, 1, 2, 3, 71), - roundParam(3, 4, 0, 1, 2, 72), - roundParam(2, 3, 4, 0, 1, 73), - roundParam(1, 2, 3, 4, 0, 74), - roundParam(0, 1, 2, 3, 4, 75), - roundParam(4, 0, 1, 2, 3, 76), - roundParam(3, 4, 0, 1, 2, 77), - roundParam(2, 3, 4, 0, 1, 78), - roundParam(1, 2, 3, 4, 0, 79), - }; - inline for (round3) |r| { - const t = s[(r.i - 3) & 0xf] ^ s[(r.i - 8) & 0xf] ^ s[(r.i - 14) & 0xf] ^ s[(r.i - 16) & 0xf]; - s[r.i & 0xf] = math.rotl(u32, t, @as(u32, 1)); - - v[r.e] = v[r.e] +% math.rotl(u32, v[r.a], @as(u32, 5)) +% 0xCA62C1D6 +% s[r.i & 0xf] +% (v[r.b] ^ v[r.c] ^ v[r.d]); - v[r.b] = math.rotl(u32, v[r.b], @as(u32, 30)); - } - - d.s[0] +%= v[0]; - d.s[1] +%= v[1]; - d.s[2] +%= v[2]; - d.s[3] +%= v[3]; - d.s[4] +%= v[4]; - } - - pub const Error = error{}; - pub const Writer = std.io.Writer(*Self, Error, write); - - fn write(self: *Self, bytes: []const u8) Error!usize { - self.update(bytes); - return bytes.len; - } - - pub fn writer(self: *Self) Writer { - return .{ .context = self }; - } -}; - -const htest = @import("test.zig"); - -test "sha1 single" { - try htest.assertEqualHash(Sha1, "da39a3ee5e6b4b0d3255bfef95601890afd80709", ""); - try htest.assertEqualHash(Sha1, "a9993e364706816aba3e25717850c26c9cd0d89d", "abc"); - try htest.assertEqualHash(Sha1, "a49b2446a02c645bf419f995b67091253a04a259", "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"); -} - -test "sha1 streaming" { - var h = Sha1.init(.{}); - var out: [20]u8 = undefined; - - h.final(&out); - try htest.assertEqual("da39a3ee5e6b4b0d3255bfef95601890afd80709", out[0..]); - - h = Sha1.init(.{}); - h.update("abc"); - h.final(&out); - try htest.assertEqual("a9993e364706816aba3e25717850c26c9cd0d89d", out[0..]); - - h = Sha1.init(.{}); - h.update("a"); - h.update("b"); - h.update("c"); - h.final(&out); - try htest.assertEqual("a9993e364706816aba3e25717850c26c9cd0d89d", out[0..]); -} - -test "sha1 aligned final" { - var block = [_]u8{0} ** Sha1.block_length; - var out: [Sha1.digest_length]u8 = undefined; - - var h = Sha1.init(.{}); - h.update(&block); - h.final(out[0..]); -} diff --git a/lib/std/crypto/sha2.zig b/lib/std/crypto/sha2.zig index edcb2ecb88..9a0cfc0849 100644 --- a/lib/std/crypto/sha2.zig +++ b/lib/std/crypto/sha2.zig @@ -95,14 +95,19 @@ fn Sha2x32(comptime iv: Iv32, digest_bits: comptime_int) type { pub const Options = struct {}; s: [8]u32 align(16), - // Streaming Cache - buf: [64]u8 = undefined, - buf_len: u8 = 0, - total_len: u64 = 0, + /// Streaming Cache + buf: [64]u8, + buf_len: u8, + total_len: u64, pub fn init(options: Options) Self { _ = options; - return Self{ .s = iv }; + return .{ + .s = iv, + .buf = undefined, + .buf_len = 0, + .total_len = 0, + }; } pub fn hash(b: []const u8, out: *[digest_length]u8, options: Options) void { @@ -377,16 +382,25 @@ fn Sha2x32(comptime iv: Iv32, digest_bits: comptime_int) type { for (&d.s, v) |*dv, vv| dv.* +%= vv; } - pub const Error = error{}; - pub const Writer = std.io.Writer(*Self, Error, write); - - fn write(self: *Self, bytes: []const u8) Error!usize { - self.update(bytes); - return bytes.len; + pub fn writable(this: *@This(), buffer: []u8) std.io.BufferedWriter { + return .{ + .unbuffered_writer = .{ + .context = this, + .vtable = &.{ + .writeSplat = writeSplat, + .writeFile = std.io.Writer.unimplementedWriteFile, + }, + }, + .buffer = buffer, + }; } - pub fn writer(self: *Self) Writer { - return .{ .context = self }; + fn writeSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) std.io.Writer.Error!usize { + const this: *@This() = @ptrCast(@alignCast(context)); + const start_total = this.total_len; + for (data[0 .. data.len - 1]) |slice| this.update(slice); + for (0..splat) |_| this.update(data[data.len - 1]); + return @intCast(this.total_len - start_total); } }; } diff --git a/lib/std/crypto/sha3.zig b/lib/std/crypto/sha3.zig index a001538c1d..84cd0c2b0e 100644 --- a/lib/std/crypto/sha3.zig +++ b/lib/std/crypto/sha3.zig @@ -80,18 +80,6 @@ pub fn Keccak(comptime f: u11, comptime output_bits: u11, comptime default_delim self.st.pad(); self.st.squeeze(out[0..]); } - - pub const Error = error{}; - pub const Writer = std.io.Writer(*Self, Error, write); - - fn write(self: *Self, bytes: []const u8) Error!usize { - self.update(bytes); - return bytes.len; - } - - pub fn writer(self: *Self) Writer { - return .{ .context = self }; - } }; } @@ -191,18 +179,6 @@ fn ShakeLike(comptime security_level: u11, comptime default_delim: u8, comptime pub fn fillBlock(self: *Self) void { self.st.fillBlock(); } - - pub const Error = error{}; - pub const Writer = std.io.Writer(*Self, Error, write); - - fn write(self: *Self, bytes: []const u8) Error!usize { - self.update(bytes); - return bytes.len; - } - - pub fn writer(self: *Self) Writer { - return .{ .context = self }; - } }; } @@ -284,18 +260,6 @@ fn CShakeLike(comptime security_level: u11, comptime default_delim: u8, comptime pub fn fillBlock(self: *Self) void { self.shaker.fillBlock(); } - - pub const Error = error{}; - pub const Writer = std.io.Writer(*Self, Error, write); - - fn write(self: *Self, bytes: []const u8) Error!usize { - self.update(bytes); - return bytes.len; - } - - pub fn writer(self: *Self) Writer { - return .{ .context = self }; - } }; } @@ -390,18 +354,6 @@ fn KMacLike(comptime security_level: u11, comptime default_delim: u8, comptime r ctx.update(msg); ctx.final(out); } - - pub const Error = error{}; - pub const Writer = std.io.Writer(*Self, Error, write); - - fn write(self: *Self, bytes: []const u8) Error!usize { - self.update(bytes); - return bytes.len; - } - - pub fn writer(self: *Self) Writer { - return .{ .context = self }; - } }; } @@ -482,18 +434,6 @@ fn TupleHashLike(comptime security_level: u11, comptime default_delim: u8, compt } self.cshaker.squeeze(out); } - - pub const Error = error{}; - pub const Writer = std.io.Writer(*Self, Error, write); - - fn write(self: *Self, bytes: []const u8) Error!usize { - self.update(bytes); - return bytes.len; - } - - pub fn writer(self: *Self) Writer { - return .{ .context = self }; - } }; } diff --git a/lib/std/crypto/siphash.zig b/lib/std/crypto/siphash.zig index a4cd904ba8..cf595327c5 100644 --- a/lib/std/crypto/siphash.zig +++ b/lib/std/crypto/siphash.zig @@ -238,48 +238,6 @@ fn SipHash(comptime T: type, comptime c_rounds: usize, comptime d_rounds: usize) pub fn toInt(msg: []const u8, key: *const [key_length]u8) T { return State.hash(msg, key); } - - pub fn writer(self: *Self) std.io.Writer { - return .{ - .context = self, - .vtable = &.{ - .writeSplat = &writeSplat, - .writeFile = &writeFile, - }, - }; - } - - fn writeSplat(ctx: ?*anyopaque, data: []const []const u8, splat: usize) std.io.Writer.Error!usize { - const self: *Self = @alignCast(@ptrCast(ctx)); - var len: usize = 0; - for (data[0 .. data.len - 1]) |slice| { - self.update(slice); - len += slice.len; - } - { - const slice = data[data.len - 1]; - for (0..splat) |_| self.update(slice); - len += slice.len * splat; - } - return len; - } - - fn writeFile( - ctx: ?*anyopaque, - file: std.fs.File, - offset: std.io.Writer.Offset, - limit: std.io.Writer.Limit, - headers_and_trailers: []const []const u8, - headers_len: usize, - ) std.io.Writer.Error!usize { - _ = ctx; - _ = file; - _ = offset; - _ = limit; - _ = headers_and_trailers; - _ = headers_len; - @panic("TODO"); - } }; } diff --git a/lib/std/debug/Dwarf.zig b/lib/std/debug/Dwarf.zig index 4bca4f1c9d..e81e997948 100644 --- a/lib/std/debug/Dwarf.zig +++ b/lib/std/debug/Dwarf.zig @@ -2176,7 +2176,7 @@ pub const ElfModule = struct { parent_mapped_mem: ?[]align(std.heap.page_size_min) const u8, elf_filename: ?[]const u8, ) LoadError!Dwarf.ElfModule { - if (expected_crc) |crc| if (crc != std.hash.crc.Crc32.hash(mapped_mem)) return error.InvalidDebugInfo; + if (expected_crc) |crc| if (crc != std.hash.Crc32.hash(mapped_mem)) return error.InvalidDebugInfo; const hdr: *const elf.Ehdr = @ptrCast(&mapped_mem[0]); if (!mem.eql(u8, hdr.e_ident[0..4], elf.MAGIC)) return error.InvalidElfMagic; diff --git a/lib/std/fs/File.zig b/lib/std/fs/File.zig index 8161887552..6a3589aae8 100644 --- a/lib/std/fs/File.zig +++ b/lib/std/fs/File.zig @@ -934,6 +934,10 @@ pub const Reader = struct { }; } + pub fn readable(r: *Reader, buffer: []u8) std.io.BufferedReader { + return interface(r).buffered(buffer); + } + pub fn getSize(r: *Reader) GetEndPosError!u64 { return r.size orelse { if (r.size_err) |err| return err; @@ -1228,6 +1232,7 @@ pub const Writer = struct { pos: u64 = 0, sendfile_err: ?SendfileError = null, read_err: ?ReadError = null, + seek_err: ?SeekError = null, pub const Mode = Reader.Mode; @@ -1250,6 +1255,20 @@ pub const Writer = struct { }; } + pub fn writable(w: *Writer, buffer: []u8) std.io.BufferedWriter { + return interface(w).buffered(buffer); + } + + pub fn moveToReader(w: *Writer) Reader { + defer w.* = undefined; + return .{ + .file = w.file, + .mode = w.mode, + .pos = w.pos, + .seek_err = w.seek_err, + }; + } + pub fn writeSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) std.io.Writer.Error!usize { const w: *Writer = @ptrCast(@alignCast(context)); const handle = w.file.handle; @@ -1347,6 +1366,21 @@ pub const Writer = struct { } return error.Unimplemented; } + + pub fn seekTo(w: *Writer, offset: u64) SeekError!void { + if (w.seek_err) |err| return err; + switch (w.mode) { + .positional, .positional_reading => { + w.pos = offset; + }, + .streaming, .streaming_reading => { + posix.lseek_SET(w.file.handle, offset) catch |err| { + w.seek_err = err; + return err; + }; + }, + } + } }; /// Defaults to positional reading; falls back to streaming. diff --git a/lib/std/hash.zig b/lib/std/hash.zig index 157dad67a7..d01cbde6c1 100644 --- a/lib/std/hash.zig +++ b/lib/std/hash.zig @@ -6,9 +6,8 @@ pub const autoHash = auto_hash.autoHash; pub const autoHashStrat = auto_hash.hash; pub const Strategy = auto_hash.HashStrategy; -// pub for polynomials + generic crc32 construction pub const crc = @import("hash/crc.zig"); -pub const Crc32 = crc.Crc32; +pub const Crc32 = crc.Crc32IsoHdlc; const fnv = @import("hash/fnv.zig"); pub const Fnv1a_32 = fnv.Fnv1a_32; diff --git a/lib/std/hash/crc.zig b/lib/std/hash/crc.zig index c0a418a0c2..e48b9eac97 100644 --- a/lib/std/hash/crc.zig +++ b/lib/std/hash/crc.zig @@ -1,19 +1,127 @@ -//! This file is auto-generated by tools/update_crc_catalog.zig. +const std = @import("../std.zig"); -const impl = @import("crc/impl.zig"); +pub fn Generic(comptime W: type, comptime algorithm: Algorithm(W)) type { + return struct { + const Self = @This(); + const I = if (@bitSizeOf(W) < 8) u8 else W; + const lookup_table = blk: { + @setEvalBranchQuota(2500); -pub const Crc = impl.Crc; -pub const Polynomial = impl.Polynomial; -pub const Crc32WithPoly = impl.Crc32WithPoly; -pub const Crc32SmallWithPoly = impl.Crc32SmallWithPoly; + const poly = if (algorithm.reflect_input) + @bitReverse(@as(I, algorithm.polynomial)) >> (@bitSizeOf(I) - @bitSizeOf(W)) + else + @as(I, algorithm.polynomial) << (@bitSizeOf(I) - @bitSizeOf(W)); -pub const Crc32 = Crc32IsoHdlc; + var table: [256]I = undefined; + for (&table, 0..) |*e, i| { + var crc: I = i; + if (algorithm.reflect_input) { + var j: usize = 0; + while (j < 8) : (j += 1) { + crc = (crc >> 1) ^ ((crc & 1) * poly); + } + } else { + crc <<= @bitSizeOf(I) - 8; + var j: usize = 0; + while (j < 8) : (j += 1) { + crc = (crc << 1) ^ (((crc >> (@bitSizeOf(I) - 1)) & 1) * poly); + } + } + e.* = crc; + } + break :blk table; + }; -test { - _ = @import("crc/test.zig"); + crc: I, + + pub fn init() Self { + const initial = if (algorithm.reflect_input) + @bitReverse(@as(I, algorithm.initial)) >> (@bitSizeOf(I) - @bitSizeOf(W)) + else + @as(I, algorithm.initial) << (@bitSizeOf(I) - @bitSizeOf(W)); + return .{ .crc = initial }; + } + + inline fn tableEntry(index: I) I { + return lookup_table[@as(u8, @intCast(index & 0xFF))]; + } + + pub fn updateByte(self: *Self, byte: u8) void { + if (@bitSizeOf(I) <= 8) { + self.crc = tableEntry(self.crc ^ byte); + } else if (algorithm.reflect_input) { + const table_index = self.crc ^ byte; + self.crc = tableEntry(table_index) ^ (self.crc >> 8); + } else { + const table_index = (self.crc >> (@bitSizeOf(I) - 8)) ^ byte; + self.crc = tableEntry(table_index) ^ (self.crc << 8); + } + } + + pub fn update(self: *Self, bytes: []const u8) void { + for (bytes) |byte| updateByte(self, byte); + } + + pub fn final(self: Self) W { + var c = self.crc; + if (algorithm.reflect_input != algorithm.reflect_output) { + c = @bitReverse(c); + } + if (!algorithm.reflect_output) { + c >>= @bitSizeOf(I) - @bitSizeOf(W); + } + return @intCast(c ^ algorithm.xor_output); + } + + pub fn hash(bytes: []const u8) W { + var c = Self.init(); + c.update(bytes); + return c.final(); + } + + pub fn writable(self: *Self, buffer: []u8) std.io.BufferedWriter { + return .{ + .unbuffered_writer = .{ + .context = self, + .vtable = &.{ + .writeSplat = writeSplat, + .writeFile = std.io.Writer.unimplementedWriteFile, + }, + }, + .buffer = buffer, + }; + } + + fn writeSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) std.io.Writer.Error!usize { + const self: *Self = @ptrCast(@alignCast(context)); + var n: usize = 0; + for (data[0 .. data.len - 1]) |slice| { + self.update(slice); + n += slice.len; + } + const last = data[data.len - 1]; + if (last.len == 1) { + for (0..splat) |_| self.updateByte(last[0]); + return n + splat; + } else { + for (0..splat) |_| self.update(last); + return n + last.len * splat; + } + } + }; } -pub const Crc3Gsm = Crc(u3, .{ +pub fn Algorithm(comptime W: type) type { + return struct { + polynomial: W, + initial: W, + reflect_input: bool, + reflect_output: bool, + xor_output: W, + }; +} + +pub const Crc3Gsm = Generic(u3, .{ .polynomial = 0x3, .initial = 0x0, .reflect_input = false, @@ -21,7 +129,7 @@ pub const Crc3Gsm = Crc(u3, .{ .xor_output = 0x7, }); -pub const Crc3Rohc = Crc(u3, .{ +pub const Crc3Rohc = Generic(u3, .{ .polynomial = 0x3, .initial = 0x7, .reflect_input = true, @@ -29,7 +137,7 @@ pub const Crc3Rohc = Crc(u3, .{ .xor_output = 0x0, }); -pub const Crc4G704 = Crc(u4, .{ +pub const Crc4G704 = Generic(u4, .{ .polynomial = 0x3, .initial = 0x0, .reflect_input = true, @@ -37,7 +145,7 @@ pub const Crc4G704 = Crc(u4, .{ .xor_output = 0x0, }); -pub const Crc4Interlaken = Crc(u4, .{ +pub const Crc4Interlaken = Generic(u4, .{ .polynomial = 0x3, .initial = 0xf, .reflect_input = false, @@ -45,7 +153,7 @@ pub const Crc4Interlaken = Crc(u4, .{ .xor_output = 0xf, }); -pub const Crc5EpcC1g2 = Crc(u5, .{ +pub const Crc5EpcC1g2 = Generic(u5, .{ .polynomial = 0x09, .initial = 0x09, .reflect_input = false, @@ -53,7 +161,7 @@ pub const Crc5EpcC1g2 = Crc(u5, .{ .xor_output = 0x00, }); -pub const Crc5G704 = Crc(u5, .{ +pub const Crc5G704 = Generic(u5, .{ .polynomial = 0x15, .initial = 0x00, .reflect_input = true, @@ -61,7 +169,7 @@ pub const Crc5G704 = Crc(u5, .{ .xor_output = 0x00, }); -pub const Crc5Usb = Crc(u5, .{ +pub const Crc5Usb = Generic(u5, .{ .polynomial = 0x05, .initial = 0x1f, .reflect_input = true, @@ -69,7 +177,7 @@ pub const Crc5Usb = Crc(u5, .{ .xor_output = 0x1f, }); -pub const Crc6Cdma2000A = Crc(u6, .{ +pub const Crc6Cdma2000A = Generic(u6, .{ .polynomial = 0x27, .initial = 0x3f, .reflect_input = false, @@ -77,7 +185,7 @@ pub const Crc6Cdma2000A = Crc(u6, .{ .xor_output = 0x00, }); -pub const Crc6Cdma2000B = Crc(u6, .{ +pub const Crc6Cdma2000B = Generic(u6, .{ .polynomial = 0x07, .initial = 0x3f, .reflect_input = false, @@ -85,7 +193,7 @@ pub const Crc6Cdma2000B = Crc(u6, .{ .xor_output = 0x00, }); -pub const Crc6Darc = Crc(u6, .{ +pub const Crc6Darc = Generic(u6, .{ .polynomial = 0x19, .initial = 0x00, .reflect_input = true, @@ -93,7 +201,7 @@ pub const Crc6Darc = Crc(u6, .{ .xor_output = 0x00, }); -pub const Crc6G704 = Crc(u6, .{ +pub const Crc6G704 = Generic(u6, .{ .polynomial = 0x03, .initial = 0x00, .reflect_input = true, @@ -101,7 +209,7 @@ pub const Crc6G704 = Crc(u6, .{ .xor_output = 0x00, }); -pub const Crc6Gsm = Crc(u6, .{ +pub const Crc6Gsm = Generic(u6, .{ .polynomial = 0x2f, .initial = 0x00, .reflect_input = false, @@ -109,7 +217,7 @@ pub const Crc6Gsm = Crc(u6, .{ .xor_output = 0x3f, }); -pub const Crc7Mmc = Crc(u7, .{ +pub const Crc7Mmc = Generic(u7, .{ .polynomial = 0x09, .initial = 0x00, .reflect_input = false, @@ -117,7 +225,7 @@ pub const Crc7Mmc = Crc(u7, .{ .xor_output = 0x00, }); -pub const Crc7Rohc = Crc(u7, .{ +pub const Crc7Rohc = Generic(u7, .{ .polynomial = 0x4f, .initial = 0x7f, .reflect_input = true, @@ -125,7 +233,7 @@ pub const Crc7Rohc = Crc(u7, .{ .xor_output = 0x00, }); -pub const Crc7Umts = Crc(u7, .{ +pub const Crc7Umts = Generic(u7, .{ .polynomial = 0x45, .initial = 0x00, .reflect_input = false, @@ -133,7 +241,7 @@ pub const Crc7Umts = Crc(u7, .{ .xor_output = 0x00, }); -pub const Crc8Autosar = Crc(u8, .{ +pub const Crc8Autosar = Generic(u8, .{ .polynomial = 0x2f, .initial = 0xff, .reflect_input = false, @@ -141,7 +249,7 @@ pub const Crc8Autosar = Crc(u8, .{ .xor_output = 0xff, }); -pub const Crc8Bluetooth = Crc(u8, .{ +pub const Crc8Bluetooth = Generic(u8, .{ .polynomial = 0xa7, .initial = 0x00, .reflect_input = true, @@ -149,7 +257,7 @@ pub const Crc8Bluetooth = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc8Cdma2000 = Crc(u8, .{ +pub const Crc8Cdma2000 = Generic(u8, .{ .polynomial = 0x9b, .initial = 0xff, .reflect_input = false, @@ -157,7 +265,7 @@ pub const Crc8Cdma2000 = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc8Darc = Crc(u8, .{ +pub const Crc8Darc = Generic(u8, .{ .polynomial = 0x39, .initial = 0x00, .reflect_input = true, @@ -165,7 +273,7 @@ pub const Crc8Darc = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc8DvbS2 = Crc(u8, .{ +pub const Crc8DvbS2 = Generic(u8, .{ .polynomial = 0xd5, .initial = 0x00, .reflect_input = false, @@ -173,7 +281,7 @@ pub const Crc8DvbS2 = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc8GsmA = Crc(u8, .{ +pub const Crc8GsmA = Generic(u8, .{ .polynomial = 0x1d, .initial = 0x00, .reflect_input = false, @@ -181,7 +289,7 @@ pub const Crc8GsmA = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc8GsmB = Crc(u8, .{ +pub const Crc8GsmB = Generic(u8, .{ .polynomial = 0x49, .initial = 0x00, .reflect_input = false, @@ -189,7 +297,7 @@ pub const Crc8GsmB = Crc(u8, .{ .xor_output = 0xff, }); -pub const Crc8Hitag = Crc(u8, .{ +pub const Crc8Hitag = Generic(u8, .{ .polynomial = 0x1d, .initial = 0xff, .reflect_input = false, @@ -197,7 +305,7 @@ pub const Crc8Hitag = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc8I4321 = Crc(u8, .{ +pub const Crc8I4321 = Generic(u8, .{ .polynomial = 0x07, .initial = 0x00, .reflect_input = false, @@ -205,7 +313,7 @@ pub const Crc8I4321 = Crc(u8, .{ .xor_output = 0x55, }); -pub const Crc8ICode = Crc(u8, .{ +pub const Crc8ICode = Generic(u8, .{ .polynomial = 0x1d, .initial = 0xfd, .reflect_input = false, @@ -213,7 +321,7 @@ pub const Crc8ICode = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc8Lte = Crc(u8, .{ +pub const Crc8Lte = Generic(u8, .{ .polynomial = 0x9b, .initial = 0x00, .reflect_input = false, @@ -221,7 +329,7 @@ pub const Crc8Lte = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc8MaximDow = Crc(u8, .{ +pub const Crc8MaximDow = Generic(u8, .{ .polynomial = 0x31, .initial = 0x00, .reflect_input = true, @@ -229,7 +337,7 @@ pub const Crc8MaximDow = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc8MifareMad = Crc(u8, .{ +pub const Crc8MifareMad = Generic(u8, .{ .polynomial = 0x1d, .initial = 0xc7, .reflect_input = false, @@ -237,7 +345,7 @@ pub const Crc8MifareMad = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc8Nrsc5 = Crc(u8, .{ +pub const Crc8Nrsc5 = Generic(u8, .{ .polynomial = 0x31, .initial = 0xff, .reflect_input = false, @@ -245,7 +353,7 @@ pub const Crc8Nrsc5 = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc8Opensafety = Crc(u8, .{ +pub const Crc8Opensafety = Generic(u8, .{ .polynomial = 0x2f, .initial = 0x00, .reflect_input = false, @@ -253,7 +361,7 @@ pub const Crc8Opensafety = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc8Rohc = Crc(u8, .{ +pub const Crc8Rohc = Generic(u8, .{ .polynomial = 0x07, .initial = 0xff, .reflect_input = true, @@ -261,7 +369,7 @@ pub const Crc8Rohc = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc8SaeJ1850 = Crc(u8, .{ +pub const Crc8SaeJ1850 = Generic(u8, .{ .polynomial = 0x1d, .initial = 0xff, .reflect_input = false, @@ -269,7 +377,7 @@ pub const Crc8SaeJ1850 = Crc(u8, .{ .xor_output = 0xff, }); -pub const Crc8Smbus = Crc(u8, .{ +pub const Crc8Smbus = Generic(u8, .{ .polynomial = 0x07, .initial = 0x00, .reflect_input = false, @@ -277,7 +385,7 @@ pub const Crc8Smbus = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc8Tech3250 = Crc(u8, .{ +pub const Crc8Tech3250 = Generic(u8, .{ .polynomial = 0x1d, .initial = 0xff, .reflect_input = true, @@ -285,7 +393,7 @@ pub const Crc8Tech3250 = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc8Wcdma = Crc(u8, .{ +pub const Crc8Wcdma = Generic(u8, .{ .polynomial = 0x9b, .initial = 0x00, .reflect_input = true, @@ -293,7 +401,7 @@ pub const Crc8Wcdma = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc10Atm = Crc(u10, .{ +pub const Crc10Atm = Generic(u10, .{ .polynomial = 0x233, .initial = 0x000, .reflect_input = false, @@ -301,7 +409,7 @@ pub const Crc10Atm = Crc(u10, .{ .xor_output = 0x000, }); -pub const Crc10Cdma2000 = Crc(u10, .{ +pub const Crc10Cdma2000 = Generic(u10, .{ .polynomial = 0x3d9, .initial = 0x3ff, .reflect_input = false, @@ -309,7 +417,7 @@ pub const Crc10Cdma2000 = Crc(u10, .{ .xor_output = 0x000, }); -pub const Crc10Gsm = Crc(u10, .{ +pub const Crc10Gsm = Generic(u10, .{ .polynomial = 0x175, .initial = 0x000, .reflect_input = false, @@ -317,7 +425,7 @@ pub const Crc10Gsm = Crc(u10, .{ .xor_output = 0x3ff, }); -pub const Crc11Flexray = Crc(u11, .{ +pub const Crc11Flexray = Generic(u11, .{ .polynomial = 0x385, .initial = 0x01a, .reflect_input = false, @@ -325,7 +433,7 @@ pub const Crc11Flexray = Crc(u11, .{ .xor_output = 0x000, }); -pub const Crc11Umts = Crc(u11, .{ +pub const Crc11Umts = Generic(u11, .{ .polynomial = 0x307, .initial = 0x000, .reflect_input = false, @@ -333,7 +441,7 @@ pub const Crc11Umts = Crc(u11, .{ .xor_output = 0x000, }); -pub const Crc12Cdma2000 = Crc(u12, .{ +pub const Crc12Cdma2000 = Generic(u12, .{ .polynomial = 0xf13, .initial = 0xfff, .reflect_input = false, @@ -341,7 +449,7 @@ pub const Crc12Cdma2000 = Crc(u12, .{ .xor_output = 0x000, }); -pub const Crc12Dect = Crc(u12, .{ +pub const Crc12Dect = Generic(u12, .{ .polynomial = 0x80f, .initial = 0x000, .reflect_input = false, @@ -349,7 +457,7 @@ pub const Crc12Dect = Crc(u12, .{ .xor_output = 0x000, }); -pub const Crc12Gsm = Crc(u12, .{ +pub const Crc12Gsm = Generic(u12, .{ .polynomial = 0xd31, .initial = 0x000, .reflect_input = false, @@ -357,7 +465,7 @@ pub const Crc12Gsm = Crc(u12, .{ .xor_output = 0xfff, }); -pub const Crc12Umts = Crc(u12, .{ +pub const Crc12Umts = Generic(u12, .{ .polynomial = 0x80f, .initial = 0x000, .reflect_input = false, @@ -365,7 +473,7 @@ pub const Crc12Umts = Crc(u12, .{ .xor_output = 0x000, }); -pub const Crc13Bbc = Crc(u13, .{ +pub const Crc13Bbc = Generic(u13, .{ .polynomial = 0x1cf5, .initial = 0x0000, .reflect_input = false, @@ -373,7 +481,7 @@ pub const Crc13Bbc = Crc(u13, .{ .xor_output = 0x0000, }); -pub const Crc14Darc = Crc(u14, .{ +pub const Crc14Darc = Generic(u14, .{ .polynomial = 0x0805, .initial = 0x0000, .reflect_input = true, @@ -381,7 +489,7 @@ pub const Crc14Darc = Crc(u14, .{ .xor_output = 0x0000, }); -pub const Crc14Gsm = Crc(u14, .{ +pub const Crc14Gsm = Generic(u14, .{ .polynomial = 0x202d, .initial = 0x0000, .reflect_input = false, @@ -389,7 +497,7 @@ pub const Crc14Gsm = Crc(u14, .{ .xor_output = 0x3fff, }); -pub const Crc15Can = Crc(u15, .{ +pub const Crc15Can = Generic(u15, .{ .polynomial = 0x4599, .initial = 0x0000, .reflect_input = false, @@ -397,7 +505,7 @@ pub const Crc15Can = Crc(u15, .{ .xor_output = 0x0000, }); -pub const Crc15Mpt1327 = Crc(u15, .{ +pub const Crc15Mpt1327 = Generic(u15, .{ .polynomial = 0x6815, .initial = 0x0000, .reflect_input = false, @@ -405,7 +513,7 @@ pub const Crc15Mpt1327 = Crc(u15, .{ .xor_output = 0x0001, }); -pub const Crc16Arc = Crc(u16, .{ +pub const Crc16Arc = Generic(u16, .{ .polynomial = 0x8005, .initial = 0x0000, .reflect_input = true, @@ -413,7 +521,7 @@ pub const Crc16Arc = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16Cdma2000 = Crc(u16, .{ +pub const Crc16Cdma2000 = Generic(u16, .{ .polynomial = 0xc867, .initial = 0xffff, .reflect_input = false, @@ -421,7 +529,7 @@ pub const Crc16Cdma2000 = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16Cms = Crc(u16, .{ +pub const Crc16Cms = Generic(u16, .{ .polynomial = 0x8005, .initial = 0xffff, .reflect_input = false, @@ -429,7 +537,7 @@ pub const Crc16Cms = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16Dds110 = Crc(u16, .{ +pub const Crc16Dds110 = Generic(u16, .{ .polynomial = 0x8005, .initial = 0x800d, .reflect_input = false, @@ -437,7 +545,7 @@ pub const Crc16Dds110 = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16DectR = Crc(u16, .{ +pub const Crc16DectR = Generic(u16, .{ .polynomial = 0x0589, .initial = 0x0000, .reflect_input = false, @@ -445,7 +553,7 @@ pub const Crc16DectR = Crc(u16, .{ .xor_output = 0x0001, }); -pub const Crc16DectX = Crc(u16, .{ +pub const Crc16DectX = Generic(u16, .{ .polynomial = 0x0589, .initial = 0x0000, .reflect_input = false, @@ -453,7 +561,7 @@ pub const Crc16DectX = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16Dnp = Crc(u16, .{ +pub const Crc16Dnp = Generic(u16, .{ .polynomial = 0x3d65, .initial = 0x0000, .reflect_input = true, @@ -461,7 +569,7 @@ pub const Crc16Dnp = Crc(u16, .{ .xor_output = 0xffff, }); -pub const Crc16En13757 = Crc(u16, .{ +pub const Crc16En13757 = Generic(u16, .{ .polynomial = 0x3d65, .initial = 0x0000, .reflect_input = false, @@ -469,7 +577,7 @@ pub const Crc16En13757 = Crc(u16, .{ .xor_output = 0xffff, }); -pub const Crc16Genibus = Crc(u16, .{ +pub const Crc16Genibus = Generic(u16, .{ .polynomial = 0x1021, .initial = 0xffff, .reflect_input = false, @@ -477,7 +585,7 @@ pub const Crc16Genibus = Crc(u16, .{ .xor_output = 0xffff, }); -pub const Crc16Gsm = Crc(u16, .{ +pub const Crc16Gsm = Generic(u16, .{ .polynomial = 0x1021, .initial = 0x0000, .reflect_input = false, @@ -485,7 +593,7 @@ pub const Crc16Gsm = Crc(u16, .{ .xor_output = 0xffff, }); -pub const Crc16Ibm3740 = Crc(u16, .{ +pub const Crc16Ibm3740 = Generic(u16, .{ .polynomial = 0x1021, .initial = 0xffff, .reflect_input = false, @@ -493,7 +601,7 @@ pub const Crc16Ibm3740 = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16IbmSdlc = Crc(u16, .{ +pub const Crc16IbmSdlc = Generic(u16, .{ .polynomial = 0x1021, .initial = 0xffff, .reflect_input = true, @@ -501,7 +609,7 @@ pub const Crc16IbmSdlc = Crc(u16, .{ .xor_output = 0xffff, }); -pub const Crc16IsoIec144433A = Crc(u16, .{ +pub const Crc16IsoIec144433A = Generic(u16, .{ .polynomial = 0x1021, .initial = 0xc6c6, .reflect_input = true, @@ -509,7 +617,7 @@ pub const Crc16IsoIec144433A = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16Kermit = Crc(u16, .{ +pub const Crc16Kermit = Generic(u16, .{ .polynomial = 0x1021, .initial = 0x0000, .reflect_input = true, @@ -517,7 +625,7 @@ pub const Crc16Kermit = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16Lj1200 = Crc(u16, .{ +pub const Crc16Lj1200 = Generic(u16, .{ .polynomial = 0x6f63, .initial = 0x0000, .reflect_input = false, @@ -525,7 +633,7 @@ pub const Crc16Lj1200 = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16M17 = Crc(u16, .{ +pub const Crc16M17 = Generic(u16, .{ .polynomial = 0x5935, .initial = 0xffff, .reflect_input = false, @@ -533,7 +641,7 @@ pub const Crc16M17 = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16MaximDow = Crc(u16, .{ +pub const Crc16MaximDow = Generic(u16, .{ .polynomial = 0x8005, .initial = 0x0000, .reflect_input = true, @@ -541,7 +649,7 @@ pub const Crc16MaximDow = Crc(u16, .{ .xor_output = 0xffff, }); -pub const Crc16Mcrf4xx = Crc(u16, .{ +pub const Crc16Mcrf4xx = Generic(u16, .{ .polynomial = 0x1021, .initial = 0xffff, .reflect_input = true, @@ -549,7 +657,7 @@ pub const Crc16Mcrf4xx = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16Modbus = Crc(u16, .{ +pub const Crc16Modbus = Generic(u16, .{ .polynomial = 0x8005, .initial = 0xffff, .reflect_input = true, @@ -557,7 +665,7 @@ pub const Crc16Modbus = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16Nrsc5 = Crc(u16, .{ +pub const Crc16Nrsc5 = Generic(u16, .{ .polynomial = 0x080b, .initial = 0xffff, .reflect_input = true, @@ -565,7 +673,7 @@ pub const Crc16Nrsc5 = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16OpensafetyA = Crc(u16, .{ +pub const Crc16OpensafetyA = Generic(u16, .{ .polynomial = 0x5935, .initial = 0x0000, .reflect_input = false, @@ -573,7 +681,7 @@ pub const Crc16OpensafetyA = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16OpensafetyB = Crc(u16, .{ +pub const Crc16OpensafetyB = Generic(u16, .{ .polynomial = 0x755b, .initial = 0x0000, .reflect_input = false, @@ -581,7 +689,7 @@ pub const Crc16OpensafetyB = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16Profibus = Crc(u16, .{ +pub const Crc16Profibus = Generic(u16, .{ .polynomial = 0x1dcf, .initial = 0xffff, .reflect_input = false, @@ -589,7 +697,7 @@ pub const Crc16Profibus = Crc(u16, .{ .xor_output = 0xffff, }); -pub const Crc16Riello = Crc(u16, .{ +pub const Crc16Riello = Generic(u16, .{ .polynomial = 0x1021, .initial = 0xb2aa, .reflect_input = true, @@ -597,7 +705,7 @@ pub const Crc16Riello = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16SpiFujitsu = Crc(u16, .{ +pub const Crc16SpiFujitsu = Generic(u16, .{ .polynomial = 0x1021, .initial = 0x1d0f, .reflect_input = false, @@ -605,7 +713,7 @@ pub const Crc16SpiFujitsu = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16T10Dif = Crc(u16, .{ +pub const Crc16T10Dif = Generic(u16, .{ .polynomial = 0x8bb7, .initial = 0x0000, .reflect_input = false, @@ -613,7 +721,7 @@ pub const Crc16T10Dif = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16Teledisk = Crc(u16, .{ +pub const Crc16Teledisk = Generic(u16, .{ .polynomial = 0xa097, .initial = 0x0000, .reflect_input = false, @@ -621,7 +729,7 @@ pub const Crc16Teledisk = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16Tms37157 = Crc(u16, .{ +pub const Crc16Tms37157 = Generic(u16, .{ .polynomial = 0x1021, .initial = 0x89ec, .reflect_input = true, @@ -629,7 +737,7 @@ pub const Crc16Tms37157 = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16Umts = Crc(u16, .{ +pub const Crc16Umts = Generic(u16, .{ .polynomial = 0x8005, .initial = 0x0000, .reflect_input = false, @@ -637,7 +745,7 @@ pub const Crc16Umts = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16Usb = Crc(u16, .{ +pub const Crc16Usb = Generic(u16, .{ .polynomial = 0x8005, .initial = 0xffff, .reflect_input = true, @@ -645,7 +753,7 @@ pub const Crc16Usb = Crc(u16, .{ .xor_output = 0xffff, }); -pub const Crc16Xmodem = Crc(u16, .{ +pub const Crc16Xmodem = Generic(u16, .{ .polynomial = 0x1021, .initial = 0x0000, .reflect_input = false, @@ -653,7 +761,7 @@ pub const Crc16Xmodem = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc17CanFd = Crc(u17, .{ +pub const Crc17CanFd = Generic(u17, .{ .polynomial = 0x1685b, .initial = 0x00000, .reflect_input = false, @@ -661,7 +769,7 @@ pub const Crc17CanFd = Crc(u17, .{ .xor_output = 0x00000, }); -pub const Crc21CanFd = Crc(u21, .{ +pub const Crc21CanFd = Generic(u21, .{ .polynomial = 0x102899, .initial = 0x000000, .reflect_input = false, @@ -669,7 +777,7 @@ pub const Crc21CanFd = Crc(u21, .{ .xor_output = 0x000000, }); -pub const Crc24Ble = Crc(u24, .{ +pub const Crc24Ble = Generic(u24, .{ .polynomial = 0x00065b, .initial = 0x555555, .reflect_input = true, @@ -677,7 +785,7 @@ pub const Crc24Ble = Crc(u24, .{ .xor_output = 0x000000, }); -pub const Crc24FlexrayA = Crc(u24, .{ +pub const Crc24FlexrayA = Generic(u24, .{ .polynomial = 0x5d6dcb, .initial = 0xfedcba, .reflect_input = false, @@ -685,7 +793,7 @@ pub const Crc24FlexrayA = Crc(u24, .{ .xor_output = 0x000000, }); -pub const Crc24FlexrayB = Crc(u24, .{ +pub const Crc24FlexrayB = Generic(u24, .{ .polynomial = 0x5d6dcb, .initial = 0xabcdef, .reflect_input = false, @@ -693,7 +801,7 @@ pub const Crc24FlexrayB = Crc(u24, .{ .xor_output = 0x000000, }); -pub const Crc24Interlaken = Crc(u24, .{ +pub const Crc24Interlaken = Generic(u24, .{ .polynomial = 0x328b63, .initial = 0xffffff, .reflect_input = false, @@ -701,7 +809,7 @@ pub const Crc24Interlaken = Crc(u24, .{ .xor_output = 0xffffff, }); -pub const Crc24LteA = Crc(u24, .{ +pub const Crc24LteA = Generic(u24, .{ .polynomial = 0x864cfb, .initial = 0x000000, .reflect_input = false, @@ -709,7 +817,7 @@ pub const Crc24LteA = Crc(u24, .{ .xor_output = 0x000000, }); -pub const Crc24LteB = Crc(u24, .{ +pub const Crc24LteB = Generic(u24, .{ .polynomial = 0x800063, .initial = 0x000000, .reflect_input = false, @@ -717,7 +825,7 @@ pub const Crc24LteB = Crc(u24, .{ .xor_output = 0x000000, }); -pub const Crc24Openpgp = Crc(u24, .{ +pub const Crc24Openpgp = Generic(u24, .{ .polynomial = 0x864cfb, .initial = 0xb704ce, .reflect_input = false, @@ -725,7 +833,7 @@ pub const Crc24Openpgp = Crc(u24, .{ .xor_output = 0x000000, }); -pub const Crc24Os9 = Crc(u24, .{ +pub const Crc24Os9 = Generic(u24, .{ .polynomial = 0x800063, .initial = 0xffffff, .reflect_input = false, @@ -733,7 +841,7 @@ pub const Crc24Os9 = Crc(u24, .{ .xor_output = 0xffffff, }); -pub const Crc30Cdma = Crc(u30, .{ +pub const Crc30Cdma = Generic(u30, .{ .polynomial = 0x2030b9c7, .initial = 0x3fffffff, .reflect_input = false, @@ -741,7 +849,7 @@ pub const Crc30Cdma = Crc(u30, .{ .xor_output = 0x3fffffff, }); -pub const Crc31Philips = Crc(u31, .{ +pub const Crc31Philips = Generic(u31, .{ .polynomial = 0x04c11db7, .initial = 0x7fffffff, .reflect_input = false, @@ -749,7 +857,7 @@ pub const Crc31Philips = Crc(u31, .{ .xor_output = 0x7fffffff, }); -pub const Crc32Aixm = Crc(u32, .{ +pub const Crc32Aixm = Generic(u32, .{ .polynomial = 0x814141ab, .initial = 0x00000000, .reflect_input = false, @@ -757,7 +865,7 @@ pub const Crc32Aixm = Crc(u32, .{ .xor_output = 0x00000000, }); -pub const Crc32Autosar = Crc(u32, .{ +pub const Crc32Autosar = Generic(u32, .{ .polynomial = 0xf4acfb13, .initial = 0xffffffff, .reflect_input = true, @@ -765,7 +873,7 @@ pub const Crc32Autosar = Crc(u32, .{ .xor_output = 0xffffffff, }); -pub const Crc32Base91D = Crc(u32, .{ +pub const Crc32Base91D = Generic(u32, .{ .polynomial = 0xa833982b, .initial = 0xffffffff, .reflect_input = true, @@ -773,7 +881,7 @@ pub const Crc32Base91D = Crc(u32, .{ .xor_output = 0xffffffff, }); -pub const Crc32Bzip2 = Crc(u32, .{ +pub const Crc32Bzip2 = Generic(u32, .{ .polynomial = 0x04c11db7, .initial = 0xffffffff, .reflect_input = false, @@ -781,7 +889,7 @@ pub const Crc32Bzip2 = Crc(u32, .{ .xor_output = 0xffffffff, }); -pub const Crc32CdRomEdc = Crc(u32, .{ +pub const Crc32CdRomEdc = Generic(u32, .{ .polynomial = 0x8001801b, .initial = 0x00000000, .reflect_input = true, @@ -789,7 +897,7 @@ pub const Crc32CdRomEdc = Crc(u32, .{ .xor_output = 0x00000000, }); -pub const Crc32Cksum = Crc(u32, .{ +pub const Crc32Cksum = Generic(u32, .{ .polynomial = 0x04c11db7, .initial = 0x00000000, .reflect_input = false, @@ -797,7 +905,7 @@ pub const Crc32Cksum = Crc(u32, .{ .xor_output = 0xffffffff, }); -pub const Crc32Iscsi = Crc(u32, .{ +pub const Crc32Iscsi = Generic(u32, .{ .polynomial = 0x1edc6f41, .initial = 0xffffffff, .reflect_input = true, @@ -805,7 +913,7 @@ pub const Crc32Iscsi = Crc(u32, .{ .xor_output = 0xffffffff, }); -pub const Crc32IsoHdlc = Crc(u32, .{ +pub const Crc32IsoHdlc = Generic(u32, .{ .polynomial = 0x04c11db7, .initial = 0xffffffff, .reflect_input = true, @@ -813,7 +921,7 @@ pub const Crc32IsoHdlc = Crc(u32, .{ .xor_output = 0xffffffff, }); -pub const Crc32Jamcrc = Crc(u32, .{ +pub const Crc32Jamcrc = Generic(u32, .{ .polynomial = 0x04c11db7, .initial = 0xffffffff, .reflect_input = true, @@ -821,7 +929,7 @@ pub const Crc32Jamcrc = Crc(u32, .{ .xor_output = 0x00000000, }); -pub const Crc32Koopman = Crc(u32, .{ +pub const Crc32Koopman = Generic(u32, .{ .polynomial = 0x741b8cd7, .initial = 0xffffffff, .reflect_input = true, @@ -829,7 +937,7 @@ pub const Crc32Koopman = Crc(u32, .{ .xor_output = 0xffffffff, }); -pub const Crc32Mef = Crc(u32, .{ +pub const Crc32Mef = Generic(u32, .{ .polynomial = 0x741b8cd7, .initial = 0xffffffff, .reflect_input = true, @@ -837,7 +945,7 @@ pub const Crc32Mef = Crc(u32, .{ .xor_output = 0x00000000, }); -pub const Crc32Mpeg2 = Crc(u32, .{ +pub const Crc32Mpeg2 = Generic(u32, .{ .polynomial = 0x04c11db7, .initial = 0xffffffff, .reflect_input = false, @@ -845,7 +953,7 @@ pub const Crc32Mpeg2 = Crc(u32, .{ .xor_output = 0x00000000, }); -pub const Crc32Xfer = Crc(u32, .{ +pub const Crc32Xfer = Generic(u32, .{ .polynomial = 0x000000af, .initial = 0x00000000, .reflect_input = false, @@ -853,7 +961,7 @@ pub const Crc32Xfer = Crc(u32, .{ .xor_output = 0x00000000, }); -pub const Crc40Gsm = Crc(u40, .{ +pub const Crc40Gsm = Generic(u40, .{ .polynomial = 0x0004820009, .initial = 0x0000000000, .reflect_input = false, @@ -861,7 +969,7 @@ pub const Crc40Gsm = Crc(u40, .{ .xor_output = 0xffffffffff, }); -pub const Crc64Ecma182 = Crc(u64, .{ +pub const Crc64Ecma182 = Generic(u64, .{ .polynomial = 0x42f0e1eba9ea3693, .initial = 0x0000000000000000, .reflect_input = false, @@ -869,7 +977,7 @@ pub const Crc64Ecma182 = Crc(u64, .{ .xor_output = 0x0000000000000000, }); -pub const Crc64GoIso = Crc(u64, .{ +pub const Crc64GoIso = Generic(u64, .{ .polynomial = 0x000000000000001b, .initial = 0xffffffffffffffff, .reflect_input = true, @@ -877,7 +985,7 @@ pub const Crc64GoIso = Crc(u64, .{ .xor_output = 0xffffffffffffffff, }); -pub const Crc64Ms = Crc(u64, .{ +pub const Crc64Ms = Generic(u64, .{ .polynomial = 0x259c84cba6426349, .initial = 0xffffffffffffffff, .reflect_input = true, @@ -885,7 +993,7 @@ pub const Crc64Ms = Crc(u64, .{ .xor_output = 0x0000000000000000, }); -pub const Crc64Redis = Crc(u64, .{ +pub const Crc64Redis = Generic(u64, .{ .polynomial = 0xad93d23594c935a9, .initial = 0x0000000000000000, .reflect_input = true, @@ -893,7 +1001,7 @@ pub const Crc64Redis = Crc(u64, .{ .xor_output = 0x0000000000000000, }); -pub const Crc64We = Crc(u64, .{ +pub const Crc64We = Generic(u64, .{ .polynomial = 0x42f0e1eba9ea3693, .initial = 0xffffffffffffffff, .reflect_input = false, @@ -901,7 +1009,7 @@ pub const Crc64We = Crc(u64, .{ .xor_output = 0xffffffffffffffff, }); -pub const Crc64Xz = Crc(u64, .{ +pub const Crc64Xz = Generic(u64, .{ .polynomial = 0x42f0e1eba9ea3693, .initial = 0xffffffffffffffff, .reflect_input = true, @@ -909,10 +1017,14 @@ pub const Crc64Xz = Crc(u64, .{ .xor_output = 0xffffffffffffffff, }); -pub const Crc82Darc = Crc(u82, .{ +pub const Crc82Darc = Generic(u82, .{ .polynomial = 0x0308c0111011401440411, .initial = 0x000000000000000000000, .reflect_input = true, .reflect_output = true, .xor_output = 0x000000000000000000000, }); + +test { + _ = @import("crc/test.zig"); +} diff --git a/lib/std/hash/crc/impl.zig b/lib/std/hash/crc/impl.zig deleted file mode 100644 index 253a7b0a62..0000000000 --- a/lib/std/hash/crc/impl.zig +++ /dev/null @@ -1,112 +0,0 @@ -// There is a generic CRC implementation "Crc()" which can be parameterized via -// the Algorithm struct for a plethora of uses. -// -// The primary interface for all of the standard CRC algorithms is the -// generated file "crc.zig", which uses the implementation code here to define -// many standard CRCs. - -const std = @import("std"); - -pub fn Algorithm(comptime W: type) type { - return struct { - polynomial: W, - initial: W, - reflect_input: bool, - reflect_output: bool, - xor_output: W, - }; -} - -pub fn Crc(comptime W: type, comptime algorithm: Algorithm(W)) type { - return struct { - const Self = @This(); - const I = if (@bitSizeOf(W) < 8) u8 else W; - const lookup_table = blk: { - @setEvalBranchQuota(2500); - - const poly = if (algorithm.reflect_input) - @bitReverse(@as(I, algorithm.polynomial)) >> (@bitSizeOf(I) - @bitSizeOf(W)) - else - @as(I, algorithm.polynomial) << (@bitSizeOf(I) - @bitSizeOf(W)); - - var table: [256]I = undefined; - for (&table, 0..) |*e, i| { - var crc: I = i; - if (algorithm.reflect_input) { - var j: usize = 0; - while (j < 8) : (j += 1) { - crc = (crc >> 1) ^ ((crc & 1) * poly); - } - } else { - crc <<= @bitSizeOf(I) - 8; - var j: usize = 0; - while (j < 8) : (j += 1) { - crc = (crc << 1) ^ (((crc >> (@bitSizeOf(I) - 1)) & 1) * poly); - } - } - e.* = crc; - } - break :blk table; - }; - - crc: I, - - pub fn init() Self { - const initial = if (algorithm.reflect_input) - @bitReverse(@as(I, algorithm.initial)) >> (@bitSizeOf(I) - @bitSizeOf(W)) - else - @as(I, algorithm.initial) << (@bitSizeOf(I) - @bitSizeOf(W)); - return Self{ .crc = initial }; - } - - inline fn tableEntry(index: I) I { - return lookup_table[@as(u8, @intCast(index & 0xFF))]; - } - - pub fn update(self: *Self, bytes: []const u8) void { - var i: usize = 0; - if (@bitSizeOf(I) <= 8) { - while (i < bytes.len) : (i += 1) { - self.crc = tableEntry(self.crc ^ bytes[i]); - } - } else if (algorithm.reflect_input) { - while (i < bytes.len) : (i += 1) { - const table_index = self.crc ^ bytes[i]; - self.crc = tableEntry(table_index) ^ (self.crc >> 8); - } - } else { - while (i < bytes.len) : (i += 1) { - const table_index = (self.crc >> (@bitSizeOf(I) - 8)) ^ bytes[i]; - self.crc = tableEntry(table_index) ^ (self.crc << 8); - } - } - } - - pub fn final(self: Self) W { - var c = self.crc; - if (algorithm.reflect_input != algorithm.reflect_output) { - c = @bitReverse(c); - } - if (!algorithm.reflect_output) { - c >>= @bitSizeOf(I) - @bitSizeOf(W); - } - return @as(W, @intCast(c ^ algorithm.xor_output)); - } - - pub fn hash(bytes: []const u8) W { - var c = Self.init(); - c.update(bytes); - return c.final(); - } - }; -} - -pub const Polynomial = enum(u32) { - IEEE = @compileError("use Crc with algorithm .Crc32IsoHdlc"), - Castagnoli = @compileError("use Crc with algorithm .Crc32Iscsi"), - Koopman = @compileError("use Crc with algorithm .Crc32Koopman"), - _, -}; - -pub const Crc32WithPoly = @compileError("use Crc instead"); -pub const Crc32SmallWithPoly = @compileError("use Crc instead"); diff --git a/lib/std/io/BufferedReader.zig b/lib/std/io/BufferedReader.zig index 52a3d68696..6d0870ddf4 100644 --- a/lib/std/io/BufferedReader.zig +++ b/lib/std/io/BufferedReader.zig @@ -35,6 +35,10 @@ pub fn bufferContents(br: *BufferedReader) []u8 { return br.buffer[br.seek..br.end]; } +pub fn bufferedLen(br: *const BufferedReader) usize { + return br.end - br.seek; +} + /// Although `BufferedReader` can easily satisfy the `Reader` interface, it's /// generally more practical to pass a `BufferedReader` instance itself around, /// since it will result in fewer calls across vtable boundaries. @@ -49,6 +53,10 @@ pub fn reader(br: *BufferedReader) Reader { }; } +pub fn hashed(br: *BufferedReader, hasher: anytype) Reader.Hashed(@TypeOf(hasher)) { + return .{ .in = br, .hasher = hasher }; +} + /// Equivalent semantics to `std.io.Reader.VTable.readVec`. pub fn readVec(br: *BufferedReader, data: []const []u8) Reader.Error!usize { return readVecLimit(br, data, .unlimited); @@ -401,6 +409,30 @@ pub fn readSliceShort(br: *BufferedReader, buffer: []u8) Reader.ShortError!usize } } +/// Fill `buffer` with the next `buffer.len` bytes from the stream, advancing +/// the seek position. +/// +/// Invalidates previously returned values from `peek`. +/// +/// If the provided buffer cannot be filled completely, `error.EndOfStream` is +/// returned instead. +/// +/// The function is inline to avoid the dead code in case `endian` is +/// comptime-known and matches host endianness. +/// +/// See also: +/// * `readSlice` +/// * `readSliceEndianAlloc` +pub inline fn readSliceEndian( + br: *BufferedReader, + comptime Elem: type, + buffer: []Elem, + endian: std.builtin.Endian, +) Reader.Error!void { + try readSlice(br, @ptrCast(buffer)); + if (native_endian != endian) for (buffer) |*elem| std.mem.byteSwapAllFields(Elem, elem); +} + pub const ReadAllocError = Reader.Error || Allocator.Error; /// The function is inline to avoid the dead code in case `endian` is @@ -408,14 +440,14 @@ pub const ReadAllocError = Reader.Error || Allocator.Error; pub inline fn readSliceEndianAlloc( br: *BufferedReader, allocator: Allocator, - Elem: type, + comptime Elem: type, len: usize, endian: std.builtin.Endian, ) ReadAllocError![]Elem { const dest = try allocator.alloc(Elem, len); errdefer allocator.free(dest); try readSlice(br, @ptrCast(dest)); - if (native_endian != endian) std.mem.byteSwapAllFields(Elem, dest); + if (native_endian != endian) for (dest) |*elem| std.mem.byteSwapAllFields(Elem, elem); return dest; } diff --git a/lib/std/io/BufferedWriter.zig b/lib/std/io/BufferedWriter.zig index 30c6ec957d..dc05296fe2 100644 --- a/lib/std/io/BufferedWriter.zig +++ b/lib/std/io/BufferedWriter.zig @@ -58,6 +58,10 @@ pub fn initFixed(bw: *BufferedWriter, buffer: []u8) void { }; } +pub fn hashed(bw: *BufferedWriter, hasher: anytype) Writer.Hashed(@TypeOf(hasher)) { + return .{ .out = bw, .hasher = hasher }; +} + /// This function is available when using `initFixed`. pub fn getWritten(bw: *const BufferedWriter) []u8 { assert(bw.unbuffered_writer.vtable == &fixed_vtable); @@ -157,7 +161,7 @@ pub fn advance(bw: *BufferedWriter, n: usize) void { } /// The `data` parameter is mutable because this function needs to mutate the -/// fields in order to handle partial writes from `Writer.VTable.writeVec`. +/// fields in order to handle partial writes from `Writer.VTable.writeSplat`. pub fn writeVecAll(bw: *BufferedWriter, data: [][]const u8) Writer.Error!void { var index: usize = 0; var truncate: usize = 0; @@ -175,6 +179,36 @@ pub fn writeVecAll(bw: *BufferedWriter, data: [][]const u8) Writer.Error!void { } } +/// The `data` parameter is mutable because this function needs to mutate the +/// fields in order to handle partial writes from `Writer.VTable.writeSplat`. +pub fn writeSplatAll(bw: *BufferedWriter, data: [][]const u8, splat: usize) Writer.Error!void { + var index: usize = 0; + var truncate: usize = 0; + var remaining_splat = splat; + while (index + 1 < data.len) { + { + const untruncated = data[index]; + data[index] = untruncated[truncate..]; + defer data[index] = untruncated; + truncate += try bw.writeSplat(data[index..], remaining_splat); + } + while (truncate >= data[index].len) { + if (index + 1 < data.len) { + truncate -= data[index].len; + index += 1; + } else { + const last = data[data.len - 1]; + remaining_splat -= @divExact(truncate, last.len); + while (remaining_splat > 0) { + const n = try bw.writeSplat(data[data.len - 1 ..][0..1], remaining_splat); + remaining_splat -= @divExact(n, last.len); + } + return; + } + } + } +} + /// If the number of bytes to write based on `data` and `splat` fits inside /// `unusedCapacitySlice`, this function is guaranteed to not fail, not call /// into the underlying writer, and return the full number of bytes. @@ -443,7 +477,7 @@ pub fn splatBytesAll(bw: *BufferedWriter, bytes: []const u8, splat: usize) Write /// Writes the same slice many times, allowing short writes. /// -/// Does maximum of one underlying `Writer.VTable.writeVec`. +/// Does maximum of one underlying `Writer.VTable.writeSplat`. pub fn splatBytes(bw: *BufferedWriter, bytes: []const u8, n: usize) Writer.Error!usize { return passthruWriteSplat(bw, &.{bytes}, n); } @@ -621,7 +655,7 @@ pub const WriteFileOptions = struct { /// size here will save one syscall. limit: Writer.Limit = .unlimited, /// Headers and trailers must be passed together so that in case `len` is - /// zero, they can be forwarded directly to `Writer.VTable.writeVec`. + /// zero, they can be forwarded directly to `Writer.VTable.writeSplat`. /// /// The parameter is mutable because this function needs to mutate the /// fields in order to handle partial writes from `Writer.VTable.writeFile`. diff --git a/lib/std/io/Reader.zig b/lib/std/io/Reader.zig index 343de016ea..b2763ddbfc 100644 --- a/lib/std/io/Reader.zig +++ b/lib/std/io/Reader.zig @@ -331,3 +331,71 @@ test "readAlloc when the backing reader provides one byte at a time" { defer std.testing.allocator.free(res); try std.testing.expectEqualStrings(str, res); } + +/// Provides a `Reader` implementation by passing data from an underlying +/// reader through `Hasher.update`. +/// +/// The underlying reader is best unbuffered. +/// +/// This implementation makes suboptimal buffering decisions due to being +/// generic. A better solution will involve creating a reader for each hash +/// function, where the discard buffer can be tailored to the hash +/// implementation details. +pub fn Hashed(comptime Hasher: type) type { + return struct { + in: *BufferedReader, + hasher: Hasher, + + pub fn readable(this: *@This(), buffer: []u8) BufferedReader { + return .{ + .unbuffered_reader = .{ + .context = this, + .vtable = &.{ + .read = @This().read, + .readVec = @This().readVec, + .discard = @This().discard, + }, + }, + .buffer = buffer, + .end = 0, + .seek = 0, + }; + } + + fn read(context: ?*anyopaque, bw: *BufferedWriter, limit: Limit) RwError!usize { + const this: *@This() = @alignCast(@ptrCast(context)); + const slice = limit.slice(try bw.writableSliceGreedy(1)); + const n = try this.in.readVec(&.{slice}); + this.hasher.update(slice[0..n]); + bw.advance(n); + return n; + } + + fn discard(context: ?*anyopaque, limit: Limit) Error!usize { + const this: *@This() = @alignCast(@ptrCast(context)); + var bw = this.hasher.writable(&.{}); + const n = this.in.read(&bw, limit) catch |err| switch (err) { + error.WriteFailed => unreachable, + else => |e| return e, + }; + return n; + } + + fn readVec(context: ?*anyopaque, data: []const []u8) Error!usize { + const this: *@This() = @alignCast(@ptrCast(context)); + const n = try this.in.readVec(data); + var remaining: usize = n; + for (data) |slice| { + if (remaining < slice.len) { + this.hasher.update(slice[0..remaining]); + return n; + } else { + remaining -= slice.len; + this.hasher.update(slice); + } + } + assert(remaining == 0); + return n; + } + }; +} diff --git a/lib/std/io/Writer.zig b/lib/std/io/Writer.zig index f00ba27daf..e89235c80f 100644 --- a/lib/std/io/Writer.zig +++ b/lib/std/io/Writer.zig @@ -136,8 +136,8 @@ pub fn failingWriteSplat(context: ?*anyopaque, data: []const []const u8, splat: pub fn failingWriteFile( context: ?*anyopaque, file: std.fs.File, - offset: std.io.Writer.Offset, - limit: std.io.Writer.Limit, + offset: Offset, + limit: Limit, headers_and_trailers: []const []const u8, headers_len: usize, ) FileError!usize { @@ -158,11 +158,13 @@ pub const failing: Writer = .{ }, }; +/// For use when the `Writer` implementation can cannot offer a more efficient +/// implementation than a basic read/write loop on the file. pub fn unimplementedWriteFile( context: ?*anyopaque, file: std.fs.File, - offset: std.io.Writer.Offset, - limit: std.io.Writer.Limit, + offset: Offset, + limit: Limit, headers_and_trailers: []const []const u8, headers_len: usize, ) FileError!usize { @@ -175,6 +177,84 @@ pub fn unimplementedWriteFile( return error.Unimplemented; } +/// Provides a `Writer` implementation based on calling `Hasher.update`, sending +/// all data also to an underlying `std.io.BufferedWriter`. +/// +/// When using this, the underlying writer is best unbuffered because all +/// writes are passed on directly to it. +/// +/// This implementation makes suboptimal buffering decisions due to being +/// generic. A better solution will involve creating a writer for each hash +/// function, where the splat buffer can be tailored to the hash implementation +/// details. +pub fn Hashed(comptime Hasher: type) type { + return struct { + out: *std.io.BufferedWriter, + hasher: Hasher, + + pub fn writable(this: *@This(), buffer: []u8) std.io.BufferedWriter { + return .{ + .unbuffered_writer = .{ + .context = this, + .vtable = &.{ + .writeSplat = @This().writeSplat, + .writeFile = Writer.unimplementedWriteFile, + }, + }, + .buffer = buffer, + }; + } + + fn writeSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) Writer.Error!usize { + const this: *@This() = @alignCast(@ptrCast(context)); + const n = try this.out.writeSplat(data, splat); + const short_data = data[0 .. data.len - @intFromBool(splat == 0)]; + var remaining: usize = n; + for (short_data) |slice| { + if (remaining < slice.len) { + this.hasher.update(slice[0..remaining]); + return n; + } else { + remaining -= slice.len; + this.hasher.update(slice); + } + } + const remaining_splat = switch (splat) { + 0, 1 => { + assert(remaining == 0); + return n; + }, + else => splat - 1, + }; + const last = data[data.len - 1]; + assert(remaining == remaining_splat * last.len); + switch (last.len) { + 0 => { + assert(remaining == 0); + return n; + }, + 1 => { + var buffer: [64]u8 = undefined; + @memset(&buffer, last[0]); + while (remaining > 0) { + const update_len = @min(remaining, buffer.len); + this.hasher.update(buffer[0..update_len]); + remaining -= update_len; + } + return n; + }, + else => {}, + } + while (remaining > 0) { + const update_len = @min(remaining, last.len); + this.hasher.update(last[0..update_len]); + remaining -= update_len; + } + return n; + } + }; +} + test { _ = Null; } diff --git a/lib/std/io/Writer/Null.zig b/lib/std/io/Writer/Null.zig index 4ae90844d3..22616bdfd7 100644 --- a/lib/std/io/Writer/Null.zig +++ b/lib/std/io/Writer/Null.zig @@ -19,6 +19,10 @@ pub fn writer(nw: *NullWriter) Writer { }; } +pub fn writable(nw: *NullWriter, buffer: []u8) std.io.BufferedWriter { + return writer(nw).buffered(buffer); +} + fn writeSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) Writer.Error!usize { _ = context; const headers = data[0 .. data.len - 1]; diff --git a/lib/std/mem.zig b/lib/std/mem.zig index 59c26cc887..2e224e5db7 100644 --- a/lib/std/mem.zig +++ b/lib/std/mem.zig @@ -2196,7 +2196,9 @@ pub fn byteSwapAllFields(comptime S: type, ptr: *S) void { } } }, - else => @compileError("byteSwapAllFields expects a struct or array as the first argument"), + else => { + ptr.* = @byteSwap(ptr.*); + }, } } diff --git a/src/Package/Fetch/git.zig b/src/Package/Fetch/git.zig index fe8ee5f667..89ef199996 100644 --- a/src/Package/Fetch/git.zig +++ b/src/Package/Fetch/git.zig @@ -11,6 +11,7 @@ const Allocator = mem.Allocator; const Sha1 = std.crypto.hash.Sha1; const Sha256 = std.crypto.hash.sha2.Sha256; const assert = std.debug.assert; +const zlib = std.compress.zlib; /// The ID of a Git object. pub const Oid = union(Format) { @@ -52,7 +53,6 @@ pub const Oid = union(Format) { }; } - // Must be public for use from HashedReader and HashedWriter. pub fn update(hasher: *Hasher, b: []const u8) void { switch (hasher.*) { inline else => |*inner| inner.update(b), @@ -64,6 +64,12 @@ pub const Oid = union(Format) { inline else => |*inner, tag| @unionInit(Oid, @tagName(tag), inner.finalResult()), }; } + + pub fn writable(hasher: *Hasher, buffer: []u8) std.io.BufferedWriter { + return switch (hasher.*) { + inline else => |*inner| inner.writable(buffer), + }; + } }; pub fn fromBytes(oid_format: Format, bytes: []const u8) Oid { @@ -73,9 +79,18 @@ pub const Oid = union(Format) { }; } - pub fn readBytes(oid_format: Format, reader: anytype) @TypeOf(reader).NoEofError!Oid { + pub fn readBytes(oid_format: Format, reader: *std.io.BufferedReader) std.io.Reader.Error!Oid { return switch (oid_format) { - inline else => |tag| @unionInit(Oid, @tagName(tag), try reader.readBytesNoEof(tag.byteLength())), + .sha1 => { + var result: Oid = .{ .sha1 = undefined }; + try reader.readSlice(&result.sha1); + return result; + }, + .sha256 => { + var result: Oid = .{ .sha256 = undefined }; + try reader.readSlice(&result.sha256); + return result; + }, }; } @@ -167,8 +182,15 @@ pub const Diagnostics = struct { pub const Repository = struct { odb: Odb, - pub fn init(allocator: Allocator, format: Oid.Format, pack_file: std.fs.File, index_file: std.fs.File) !Repository { - return .{ .odb = try Odb.init(allocator, format, pack_file, index_file) }; + pub fn init( + repo: *Repository, + allocator: Allocator, + format: Oid.Format, + pack_file: *std.fs.File.Reader, + index_file: *std.fs.File.Reader, + ) !void { + repo.* = .{ .odb = undefined }; + try repo.odb.init(allocator, format, pack_file, index_file); } pub fn deinit(repository: *Repository) void { @@ -337,24 +359,32 @@ pub const Repository = struct { /// [pack-format](https://git-scm.com/docs/pack-format). const Odb = struct { format: Oid.Format, - pack_file: std.fs.File, + pack_file: *std.fs.File.Reader, index_header: IndexHeader, - index_file: std.fs.File, + index_file: *std.fs.File.Reader, cache: ObjectCache = .{}, allocator: Allocator, /// Initializes the database from open pack and index files. - fn init(allocator: Allocator, format: Oid.Format, pack_file: std.fs.File, index_file: std.fs.File) !Odb { + fn init( + odb: *Odb, + allocator: Allocator, + format: Oid.Format, + pack_file: *std.fs.File.Reader, + index_file: *std.fs.File.Reader, + ) !void { try pack_file.seekTo(0); try index_file.seekTo(0); - const index_header = try IndexHeader.read(index_file.reader()); - return .{ + odb.* = .{ .format = format, .pack_file = pack_file, - .index_header = index_header, + .index_header = undefined, .index_file = index_file, .allocator = allocator, }; + var buffer: [1032]u8 = undefined; + var index_file_br = index_file.readable(&buffer); + try odb.index_header.read(&index_file_br); } fn deinit(odb: *Odb) void { @@ -364,27 +394,30 @@ const Odb = struct { /// Reads the object at the current position in the database. fn readObject(odb: *Odb) !Object { - var base_offset = try odb.pack_file.getPos(); + var pack_read_buffer: [64]u8 = undefined; + var base_offset = odb.pack_file.pos; + var pack_br = odb.pack_file.readable(&pack_read_buffer); var base_header: EntryHeader = undefined; var delta_offsets: std.ArrayListUnmanaged(u64) = .empty; defer delta_offsets.deinit(odb.allocator); const base_object = while (true) { if (odb.cache.get(base_offset)) |base_object| break base_object; - base_header = try EntryHeader.read(odb.format, odb.pack_file.reader()); + base_header = try EntryHeader.read(odb.format, &pack_br); switch (base_header) { .ofs_delta => |ofs_delta| { try delta_offsets.append(odb.allocator, base_offset); base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidFormat; try odb.pack_file.seekTo(base_offset); + pack_br = odb.pack_file.readable(&pack_read_buffer); }, .ref_delta => |ref_delta| { try delta_offsets.append(odb.allocator, base_offset); try odb.seekOid(ref_delta.base_object); - base_offset = try odb.pack_file.getPos(); + base_offset = odb.pack_file.pos - pack_br.bufferedLen(); }, else => { - const base_data = try readObjectRaw(odb.allocator, odb.pack_file.reader(), base_header.uncompressedLength()); + const base_data = try readObjectRaw(odb.allocator, &pack_br, base_header.uncompressedLength()); errdefer odb.allocator.free(base_data); const base_object: Object = .{ .type = base_header.objectType(), .data = base_data }; try odb.cache.put(odb.allocator, base_offset, base_object); @@ -414,7 +447,8 @@ const Odb = struct { const found_index = while (start_index < end_index) { const mid_index = start_index + (end_index - start_index) / 2; try odb.index_file.seekTo(IndexHeader.size + mid_index * oid_length); - const mid_oid = try Oid.readBytes(odb.format, odb.index_file.reader()); + var br = odb.index_file.interface().unbuffered(); + const mid_oid = try Oid.readBytes(odb.format, &br); switch (mem.order(u8, mid_oid.slice(), oid.slice())) { .lt => start_index = mid_index + 1, .gt => end_index = mid_index, @@ -424,13 +458,16 @@ const Odb = struct { const n_objects = odb.index_header.fan_out_table[255]; const offset_values_start = IndexHeader.size + n_objects * (oid_length + 4); + var buffer: [8]u8 = undefined; try odb.index_file.seekTo(offset_values_start + found_index * 4); - const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try odb.index_file.reader().readInt(u32, .big)); + var br = odb.index_file.interface().buffered(&buffer); + const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try br.takeInt(u32, .big)); const pack_offset = pack_offset: { if (l1_offset.big) { const l2_offset_values_start = offset_values_start + n_objects * 4; try odb.index_file.seekTo(l2_offset_values_start + l1_offset.value * 4); - break :pack_offset try odb.index_file.reader().readInt(u64, .big); + br = odb.index_file.interface().buffered(&buffer); + break :pack_offset try br.takeInt(u64, .big); } else { break :pack_offset l1_offset.value; } @@ -556,7 +593,7 @@ const Packet = union(enum) { const max_data_length = 65516; /// Reads a packet in pkt-line format. - fn read(reader: anytype, buf: *[max_data_length]u8) !Packet { + fn read(reader: *std.io.BufferedReader, buf: *[max_data_length]u8) !Packet { const length = std.fmt.parseUnsigned(u16, &try reader.readBytesNoEof(4), 16) catch return error.InvalidPacket; switch (length) { 0 => return .flush, @@ -571,7 +608,7 @@ const Packet = union(enum) { } /// Writes a packet in pkt-line format. - fn write(packet: Packet, writer: anytype) !void { + fn write(packet: Packet, writer: *std.io.BufferedWriter) !void { switch (packet) { .flush => try writer.writeAll("0000"), .delimiter => try writer.writeAll("0001"), @@ -1070,21 +1107,12 @@ const PackHeader = struct { const signature = "PACK"; const supported_version = 2; - fn read(reader: anytype) !PackHeader { - const actual_signature = reader.readBytesNoEof(4) catch |e| switch (e) { - error.EndOfStream => return error.InvalidHeader, - else => |other| return other, - }; - if (!mem.eql(u8, &actual_signature, signature)) return error.InvalidHeader; - const version = reader.readInt(u32, .big) catch |e| switch (e) { - error.EndOfStream => return error.InvalidHeader, - else => |other| return other, - }; + fn read(reader: *std.io.BufferedReader) !PackHeader { + const actual_signature = try reader.take(4); + if (!mem.eql(u8, actual_signature, signature)) return error.InvalidHeader; + const version = try reader.takeInt(u32, .big); if (version != supported_version) return error.UnsupportedVersion; - const total_objects = reader.readInt(u32, .big) catch |e| switch (e) { - error.EndOfStream => return error.InvalidHeader, - else => |other| return other, - }; + const total_objects = try reader.takeInt(u32, .big); return .{ .total_objects = total_objects }; } }; @@ -1133,12 +1161,9 @@ const EntryHeader = union(Type) { }; } - fn read(format: Oid.Format, reader: anytype) !EntryHeader { + fn read(format: Oid.Format, reader: *std.io.BufferedReader) !EntryHeader { const InitialByte = packed struct { len: u4, type: u3, has_next: bool }; - const initial: InitialByte = @bitCast(reader.readByte() catch |e| switch (e) { - error.EndOfStream => return error.InvalidFormat, - else => |other| return other, - }); + const initial: InitialByte = @bitCast(try reader.takeByte()); const rest_len = if (initial.has_next) try readSizeVarInt(reader) else 0; var uncompressed_length: u64 = initial.len; uncompressed_length |= std.math.shlExact(u64, rest_len, 4) catch return error.InvalidFormat; @@ -1162,25 +1187,25 @@ const EntryHeader = union(Type) { } }; -fn readSizeVarInt(r: anytype) !u64 { +fn readSizeVarInt(r: *std.io.BufferedReader) !u64 { const Byte = packed struct { value: u7, has_next: bool }; - var b: Byte = @bitCast(try r.readByte()); + var b: Byte = @bitCast(try r.takeByte()); var value: u64 = b.value; var shift: u6 = 0; while (b.has_next) { - b = @bitCast(try r.readByte()); + b = @bitCast(try r.takeByte()); shift = std.math.add(u6, shift, 7) catch return error.InvalidFormat; value |= @as(u64, b.value) << shift; } return value; } -fn readOffsetVarInt(r: anytype) !u64 { +fn readOffsetVarInt(r: *std.io.BufferedReader) !u64 { const Byte = packed struct { value: u7, has_next: bool }; - var b: Byte = @bitCast(try r.readByte()); + var b: Byte = @bitCast(try r.takeByte()); var value: u64 = b.value; while (b.has_next) { - b = @bitCast(try r.readByte()); + b = @bitCast(try r.takeByte()); value = std.math.shlExact(u64, value + 1, 7) catch return error.InvalidFormat; value |= b.value; } @@ -1194,19 +1219,12 @@ const IndexHeader = struct { const supported_version = 2; const size = 4 + 4 + @sizeOf([256]u32); - fn read(reader: anytype) !IndexHeader { - var header_bytes = try reader.readBytesNoEof(size); - if (!mem.eql(u8, header_bytes[0..4], signature)) return error.InvalidHeader; - const version = mem.readInt(u32, header_bytes[4..8], .big); + fn read(index_header: *IndexHeader, br: *std.io.BufferedReader) !void { + const sig = try br.take(4); + if (!mem.eql(u8, sig, signature)) return error.InvalidHeader; + const version = try br.takeInt(u32, .big); if (version != supported_version) return error.UnsupportedVersion; - - var fan_out_table: [256]u32 = undefined; - var fan_out_table_stream = std.io.fixedBufferStream(header_bytes[8..]); - const fan_out_table_reader = fan_out_table_stream.reader(); - for (&fan_out_table) |*entry| { - entry.* = fan_out_table_reader.readInt(u32, .big) catch unreachable; - } - return .{ .fan_out_table = fan_out_table }; + try br.readSliceEndian(u32, &index_header.fan_out_table, .big); } }; @@ -1217,7 +1235,12 @@ const IndexEntry = struct { /// Writes out a version 2 index for the given packfile, as documented in /// [pack-format](https://git-scm.com/docs/pack-format). -pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, index_writer: anytype) !void { +pub fn indexPack( + allocator: Allocator, + format: Oid.Format, + pack: *std.fs.File.Reader, + index_writer: *std.fs.File.Writer, +) !void { try pack.seekTo(0); var index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry) = .empty; @@ -1270,8 +1293,10 @@ pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, in } @memset(fan_out_table[fan_out_index..], count); - var index_hashed_writer = std.compress.hashedWriter(index_writer, Oid.Hasher.init(format)); - const writer = index_hashed_writer.writer(); + var index_writer_bw = index_writer.writable(&.{}); + var index_hashed_writer = index_writer_bw.hashed(Oid.Hasher.init(format)); + var write_buffer: [256]u8 = undefined; + var writer = index_hashed_writer.writable(&write_buffer); try writer.writeAll(IndexHeader.signature); try writer.writeInt(u32, IndexHeader.supported_version, .big); for (fan_out_table) |fan_out_entry| { @@ -1303,8 +1328,9 @@ pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, in } try writer.writeAll(pack_checksum.slice()); + try writer.flush(); const index_checksum = index_hashed_writer.hasher.finalResult(); - try index_writer.writeAll(index_checksum.slice()); + try index_writer_bw.writeAll(index_checksum.slice()); } /// Performs the first pass over the packfile data for index construction. @@ -1314,50 +1340,46 @@ pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, in fn indexPackFirstPass( allocator: Allocator, format: Oid.Format, - pack: std.fs.File, + pack: *std.fs.File.Reader, index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry), pending_deltas: *std.ArrayListUnmanaged(IndexEntry), ) !Oid { - var pack_buffered_reader = std.io.bufferedReader(pack.reader()); - var pack_counting_reader = std.io.countingReader(pack_buffered_reader.reader()); - var pack_hashed_reader = std.compress.hashedReader(pack_counting_reader.reader(), Oid.Hasher.init(format)); - const pack_reader = pack_hashed_reader.reader(); + var pack_br = pack.readable(&.{}); + var pack_hashed_reader = pack_br.hashed(Oid.Hasher.init(format)); + var pack_buffer: [2048]u8 = undefined; // Reasonably large buffer for file system. + var pack_hashed_br = pack_hashed_reader.readable(&pack_buffer); - const pack_header = try PackHeader.read(pack_reader); + const pack_header = try PackHeader.read(&pack_hashed_br); - var current_entry: u32 = 0; - while (current_entry < pack_header.total_objects) : (current_entry += 1) { - const entry_offset = pack_counting_reader.bytes_read; - var entry_crc32_reader = std.compress.hashedReader(pack_reader, std.hash.Crc32.init()); - const entry_header = try EntryHeader.read(format, entry_crc32_reader.reader()); + for (0..pack_header.total_objects) |_| { + const entry_offset = pack.pos - pack_hashed_br.bufferContents().len; + var entry_crc32_reader = pack_hashed_br.hashed(std.hash.Crc32.init()); + var entry_buffer: [64]u8 = undefined; // Buffer only needed for loading EntryHeader. + var entry_crc32_br = entry_crc32_reader.readable(&entry_buffer); + const entry_header = try EntryHeader.read(format, &entry_crc32_br); + var entry_decompress_stream: zlib.Decompressor = .init(&entry_crc32_br); + // Decompress uses large output buffer; no input buffer needed. + var entry_decompress_br = entry_decompress_stream.readable(&.{}); switch (entry_header) { .commit, .tree, .blob, .tag => |object| { - var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader()); - var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader()); - var entry_hashed_writer = std.compress.hashedWriter(std.io.null_writer, Oid.Hasher.init(format)); - const entry_writer = entry_hashed_writer.writer(); + var oid_hasher = Oid.Hasher.init(format); + var oid_hasher_buffer: [zlib.max_window_len]u8 = undefined; + var oid_hasher_bw = oid_hasher.writable(&oid_hasher_buffer); // The object header is not included in the pack data but is - // part of the object's ID - try entry_writer.print("{s} {}\x00", .{ @tagName(entry_header), object.uncompressed_length }); - var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); - try fifo.pump(entry_counting_reader.reader(), entry_writer); - if (entry_counting_reader.bytes_read != object.uncompressed_length) { - return error.InvalidObject; - } - const oid = entry_hashed_writer.hasher.finalResult(); + // part of the object's ID. + try oid_hasher_bw.print("{s} {d}\x00", .{ @tagName(entry_header), object.uncompressed_length }); + const n = try entry_decompress_br.readRemaining(&oid_hasher_bw); + if (n != object.uncompressed_length) return error.InvalidObject; + try oid_hasher_bw.flush(); + const oid = oid_hasher.finalResult(); try index_entries.put(allocator, oid, .{ .offset = entry_offset, .crc32 = entry_crc32_reader.hasher.final(), }); }, inline .ofs_delta, .ref_delta => |delta| { - var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader()); - var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader()); - var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); - try fifo.pump(entry_counting_reader.reader(), std.io.null_writer); - if (entry_counting_reader.bytes_read != delta.uncompressed_length) { - return error.InvalidObject; - } + const n = try entry_decompress_br.discardRemaining(); + if (n != delta.uncompressed_length) return error.InvalidObject; try pending_deltas.append(allocator, .{ .offset = entry_offset, .crc32 = entry_crc32_reader.hasher.final(), @@ -1367,15 +1389,11 @@ fn indexPackFirstPass( } const pack_checksum = pack_hashed_reader.hasher.finalResult(); - const recorded_checksum = try Oid.readBytes(format, pack_buffered_reader.reader()); + const recorded_checksum = try Oid.readBytes(format, &pack_br); if (!mem.eql(u8, pack_checksum.slice(), recorded_checksum.slice())) { return error.CorruptedPack; } - _ = pack_reader.readByte() catch |e| switch (e) { - error.EndOfStream => return pack_checksum, - else => |other| return other, - }; - return error.InvalidFormat; + return pack_checksum; } /// Attempts to determine the final object ID of the given deltified object. @@ -1384,7 +1402,7 @@ fn indexPackFirstPass( fn indexPackHashDelta( allocator: Allocator, format: Oid.Format, - pack: std.fs.File, + pack: *std.fs.File.Reader, delta: IndexEntry, index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry), cache: *ObjectCache, @@ -1398,7 +1416,9 @@ fn indexPackHashDelta( if (cache.get(base_offset)) |base_object| break base_object; try pack.seekTo(base_offset); - base_header = try EntryHeader.read(format, pack.reader()); + var pack_read_buffer: [64]u8 = undefined; + var pack_br = pack.readable(&pack_read_buffer); + base_header = try EntryHeader.read(format, &pack_br); switch (base_header) { .ofs_delta => |ofs_delta| { try delta_offsets.append(allocator, base_offset); @@ -1409,7 +1429,7 @@ fn indexPackHashDelta( base_offset = (index_entries.get(ref_delta.base_object) orelse return null).offset; }, else => { - const base_data = try readObjectRaw(allocator, pack.reader(), base_header.uncompressedLength()); + const base_data = try readObjectRaw(allocator, &pack_br, base_header.uncompressedLength()); errdefer allocator.free(base_data); const base_object: Object = .{ .type = base_header.objectType(), .data = base_data }; try cache.put(allocator, base_offset, base_object); @@ -1421,9 +1441,12 @@ fn indexPackHashDelta( const base_data = try resolveDeltaChain(allocator, format, pack, base_object, delta_offsets.items, cache); var entry_hasher: Oid.Hasher = .init(format); - var entry_hashed_writer = std.compress.hashedWriter(std.io.null_writer, &entry_hasher); - try entry_hashed_writer.writer().print("{s} {}\x00", .{ @tagName(base_object.type), base_data.len }); - entry_hasher.update(base_data); + var entry_hasher_buffer: [64]u8 = undefined; + var entry_hasher_bw = entry_hasher.writable(&entry_hasher_buffer); + // Writes to hashers cannot fail. + entry_hasher_bw.print("{s} {d}\x00", .{ @tagName(base_object.type), base_data.len }) catch unreachable; + entry_hasher_bw.writeAll(base_data) catch unreachable; + entry_hasher_bw.flush() catch unreachable; return entry_hasher.finalResult(); } @@ -1434,7 +1457,7 @@ fn indexPackHashDelta( fn resolveDeltaChain( allocator: Allocator, format: Oid.Format, - pack: std.fs.File, + pack: *std.fs.File.Reader, base_object: Object, delta_offsets: []const u64, cache: *ObjectCache, @@ -1446,21 +1469,22 @@ fn resolveDeltaChain( const delta_offset = delta_offsets[i]; try pack.seekTo(delta_offset); - const delta_header = try EntryHeader.read(format, pack.reader()); - const delta_data = try readObjectRaw(allocator, pack.reader(), delta_header.uncompressedLength()); - defer allocator.free(delta_data); - var delta_stream = std.io.fixedBufferStream(delta_data); - const delta_reader = delta_stream.reader(); - _ = try readSizeVarInt(delta_reader); // base object size - const expanded_size = try readSizeVarInt(delta_reader); - + var pack_read_buffer: [64]u8 = undefined; + var pack_br = pack.readable(&pack_read_buffer); + const delta_header = try EntryHeader.read(format, &pack_br); + _ = delta_header; + var delta_decompress: zlib.Decompressor = .init(&pack_br); + var delta_decompress_buffer: [zlib.max_window_len]u8 = undefined; + var delta_reader = delta_decompress.readable(&delta_decompress_buffer); + _ = try readSizeVarInt(&delta_reader); // base object size + const expanded_size = try readSizeVarInt(&delta_reader); const expanded_alloc_size = std.math.cast(usize, expanded_size) orelse return error.ObjectTooLarge; const expanded_data = try allocator.alloc(u8, expanded_alloc_size); errdefer allocator.free(expanded_data); - var expanded_delta_stream = std.io.fixedBufferStream(expanded_data); - var base_stream = std.io.fixedBufferStream(base_data); - try expandDelta(&base_stream, delta_reader, expanded_delta_stream.writer()); - if (expanded_delta_stream.pos != expanded_size) return error.InvalidObject; + var expanded_delta_stream: std.io.BufferedWriter = undefined; + expanded_delta_stream.initFixed(expanded_data); + try expandDelta(base_data, &delta_reader, &expanded_delta_stream); + if (expanded_delta_stream.end != expanded_size) return error.InvalidObject; try cache.put(allocator, delta_offset, .{ .type = base_object.type, .data = expanded_data }); base_data = expanded_data; @@ -1468,31 +1492,23 @@ fn resolveDeltaChain( return base_data; } -/// Reads the complete contents of an object from `reader`. This function may -/// read more bytes than required from `reader`, so the reader position after -/// returning is not reliable. -fn readObjectRaw(allocator: Allocator, reader: anytype, size: u64) ![]u8 { +/// Reads the complete contents of an object from `reader`. +fn readObjectRaw(gpa: Allocator, reader: *std.io.BufferedReader, size: u64) ![]u8 { const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge; - var buffered_reader = std.io.bufferedReader(reader); - var decompress_stream = std.compress.zlib.decompressor(buffered_reader.reader()); - const data = try allocator.alloc(u8, alloc_size); - errdefer allocator.free(data); - try decompress_stream.reader().readNoEof(data); - _ = decompress_stream.reader().readByte() catch |e| switch (e) { - error.EndOfStream => return data, - else => |other| return other, - }; - return error.InvalidFormat; + var decompress: zlib.Decompressor = .init(reader); + var buffer: std.ArrayListUnmanaged(u8) = .empty; + defer buffer.deinit(gpa); + try decompress.reader().readRemainingArrayList(gpa, null, &buffer, .limited(alloc_size), zlib.max_window_len); + if (buffer.items.len < size) return error.EndOfStream; + return buffer.toOwnedSlice(gpa); } -/// Expands delta data from `delta_reader` to `writer`. `base_object` must -/// support `reader` and `seekTo` (such as a `std.io.FixedBufferStream`). -/// /// The format of the delta data is documented in /// [pack-format](https://git-scm.com/docs/pack-format). -fn expandDelta(base_object: anytype, delta_reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) !void { +fn expandDelta(base_object: []const u8, delta_reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) !void { + var base_offset: u32 = 0; while (true) { - const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.readByte() catch |e| switch (e) { + const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.takeByte() catch |e| switch (e) { error.EndOfStream => return, else => |other| return other, }); @@ -1507,23 +1523,22 @@ fn expandDelta(base_object: anytype, delta_reader: *std.io.BufferedReader, write size3: bool, } = @bitCast(inst.value); const offset_parts: packed struct { offset1: u8, offset2: u8, offset3: u8, offset4: u8 } = .{ - .offset1 = if (available.offset1) try delta_reader.readByte() else 0, - .offset2 = if (available.offset2) try delta_reader.readByte() else 0, - .offset3 = if (available.offset3) try delta_reader.readByte() else 0, - .offset4 = if (available.offset4) try delta_reader.readByte() else 0, + .offset1 = if (available.offset1) try delta_reader.takeByte() else 0, + .offset2 = if (available.offset2) try delta_reader.takeByte() else 0, + .offset3 = if (available.offset3) try delta_reader.takeByte() else 0, + .offset4 = if (available.offset4) try delta_reader.takeByte() else 0, }; - const offset: u32 = @bitCast(offset_parts); + base_offset = @bitCast(offset_parts); const size_parts: packed struct { size1: u8, size2: u8, size3: u8 } = .{ - .size1 = if (available.size1) try delta_reader.readByte() else 0, - .size2 = if (available.size2) try delta_reader.readByte() else 0, - .size3 = if (available.size3) try delta_reader.readByte() else 0, + .size1 = if (available.size1) try delta_reader.takeByte() else 0, + .size2 = if (available.size2) try delta_reader.takeByte() else 0, + .size3 = if (available.size3) try delta_reader.takeByte() else 0, }; var size: u24 = @bitCast(size_parts); if (size == 0) size = 0x10000; - try base_object.seekTo(offset); - var base_object_br = base_object.reader(); - try base_object_br.readAll(writer, .limited(size)); + try writer.writeAll(base_object[base_offset..][0..size]); + base_offset += size; } else if (inst.value != 0) { try delta_reader.readAll(writer, .limited(inst.value)); } else { @@ -1557,7 +1572,8 @@ fn runRepositoryTest(comptime format: Oid.Format, head_commit: []const u8) !void var index_file = try git_dir.dir.createFile("testrepo.idx", .{ .read = true }); defer index_file.close(); - try indexPack(testing.allocator, format, pack_file, index_file.writer()); + var index_file_writer = index_file.writer(); + try indexPack(testing.allocator, format, pack_file, &index_file_writer); // Arbitrary size limit on files read while checking the repository contents // (all files in the test repo are known to be smaller than this) @@ -1571,7 +1587,8 @@ fn runRepositoryTest(comptime format: Oid.Format, head_commit: []const u8) !void const testrepo_idx = @embedFile("git/testdata/testrepo-" ++ @tagName(format) ++ ".idx"); try testing.expectEqualSlices(u8, testrepo_idx, index_file_data); - var repository = try Repository.init(testing.allocator, format, pack_file, index_file); + var index_file_reader = index_file_writer.moveToReader(); + var repository = try Repository.init(testing.allocator, format, pack_file, &index_file_reader); defer repository.deinit(); var worktree = testing.tmpDir(.{ .iterate = true }); @@ -1652,10 +1669,12 @@ test "SHA-256 packfile indexing and checkout" { /// Checks out a commit of a packfile. Intended for experimenting with and /// benchmarking possible optimizations to the indexing and checkout behavior. pub fn main() !void { - const allocator = std.heap.c_allocator; + var debug_allocator: std.heap.DebugAllocator(.{}) = .init; + defer _ = debug_allocator.deinit(); + const gpa = if (std.debug.runtime_safety) debug_allocator.allocator() else std.heap.smp_allocator; - const args = try std.process.argsAlloc(allocator); - defer std.process.argsFree(allocator, args); + const args = try std.process.argsAlloc(gpa); + defer std.process.argsFree(gpa, args); if (args.len != 5) { return error.InvalidArguments; // Arguments: format packfile commit worktree } @@ -1674,15 +1693,17 @@ pub fn main() !void { std.debug.print("Starting index...\n", .{}); var index_file = try git_dir.createFile("idx", .{ .read = true }); defer index_file.close(); - var index_buffered_writer = std.io.bufferedWriter(index_file.writer()); - try indexPack(allocator, format, pack_file, index_buffered_writer.writer()); - try index_buffered_writer.flush(); + var index_file_writer = index_file.writer(); + var pack_file_reader = pack_file.reader(); + try indexPack(gpa, format, &pack_file_reader, &index_file_writer); try index_file.sync(); std.debug.print("Starting checkout...\n", .{}); - var repository = try Repository.init(allocator, format, pack_file, index_file); + var index_file_reader = index_file_writer.moveToReader(); + var repository: Repository = undefined; + try repository.init(gpa, format, &pack_file_reader, &index_file_reader); defer repository.deinit(); - var diagnostics: Diagnostics = .{ .allocator = allocator }; + var diagnostics: Diagnostics = .{ .allocator = gpa }; defer diagnostics.deinit(); try repository.checkout(worktree, commit, &diagnostics); diff --git a/src/main.zig b/src/main.zig index 561935035f..e93351f2d0 100644 --- a/src/main.zig +++ b/src/main.zig @@ -3330,12 +3330,11 @@ fn buildOutputType( // for the hashing algorithm here and in the cache are the same. // We are providing our own cache key, because this file has nothing // to do with the cache manifest. - var hasher = Cache.Hasher.init("0123456789abcdef"); var file_writer = f.writer(); - var file_writer_bw = file_writer.interface().unbuffered(); - var hasher_writer = hasher.writer(&file_writer_bw); + var file_writer_bw = file_writer.writable(&.{}); + var hasher_writer = file_writer_bw.hashed(Cache.Hasher.init("0123456789abcdef")); var buffer: [1000]u8 = undefined; - var bw = hasher_writer.interface().buffered(&buffer); + var bw = hasher_writer.writable(&buffer); bw.writeFileAll(.stdin(), .{}) catch |err| switch (err) { error.WriteFailed => fatal("failed to write {s}: {s}", .{ dump_path, file_writer.err.? }), else => fatal("failed to pipe stdin to {s}: {s}", .{ dump_path, err }),