From 06b44a0afa849556f613fcc5939db99fec38883f Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 9 May 2025 16:59:13 -0700 Subject: [PATCH] update git fetching logic to new reader/writer API - flatten std.crypto.hash.Sha1 and give it a writable interface that optimizes splats - flatten std.hash.crc and give it a writable interface that optimizes splats - remove old writer impls from std.crypto - add fs.File.Writer.moveToReader - add fs.File.Writer.seekTo - add std.io.Reader.Hashed and std.io.Writer.Hashed which are passthrough streams. Instead of passing through to null writer, use the writable interface implemented directly on hashers which doesn't have to account for passing through the data. - add std.io.BufferedWriter.writeSplatAll --- lib/std/compress/flate.zig | 5 + lib/std/compress/flate/inflate.zig | 4 + lib/std/compress/zlib.zig | 5 + lib/std/crypto.zig | 6 +- lib/std/crypto/Sha1.zig | 424 +++++++++++++++++++++++++++++ lib/std/crypto/aegis.zig | 12 - lib/std/crypto/blake2.zig | 12 - lib/std/crypto/blake3.zig | 12 - lib/std/crypto/codecs/asn1.zig | 23 +- lib/std/crypto/codecs/asn1/Oid.zig | 37 ++- lib/std/crypto/phc_encoding.zig | 8 +- lib/std/crypto/sha1.zig | 319 ---------------------- lib/std/crypto/sha2.zig | 40 ++- lib/std/crypto/sha3.zig | 60 ---- lib/std/crypto/siphash.zig | 42 --- lib/std/debug/Dwarf.zig | 2 +- lib/std/fs/File.zig | 34 +++ lib/std/hash.zig | 3 +- lib/std/hash/crc.zig | 356 +++++++++++++++--------- lib/std/hash/crc/impl.zig | 112 -------- lib/std/io/BufferedReader.zig | 36 ++- lib/std/io/BufferedWriter.zig | 40 ++- lib/std/io/Reader.zig | 68 +++++ lib/std/io/Writer.zig | 88 +++++- lib/std/io/Writer/Null.zig | 4 + lib/std/mem.zig | 4 +- src/Package/Fetch/git.zig | 333 +++++++++++----------- src/main.zig | 7 +- 28 files changed, 1185 insertions(+), 911 deletions(-) create mode 100644 lib/std/crypto/Sha1.zig delete mode 100644 lib/std/crypto/sha1.zig delete mode 100644 lib/std/hash/crc/impl.zig diff --git a/lib/std/compress/flate.zig b/lib/std/compress/flate.zig index 79ebfd8869..4dbfe7f75a 100644 --- a/lib/std/compress/flate.zig +++ b/lib/std/compress/flate.zig @@ -1,5 +1,10 @@ const std = @import("../std.zig"); +/// When decompressing, the output buffer is used as the history window, so +/// less than this may result in failure to decompress streams that were +/// compressed with a larger window. +pub const max_window_len = 1 << 16; + /// Deflate is a lossless data compression file format that uses a combination /// of LZ77 and Huffman coding. pub const deflate = @import("flate/deflate.zig"); diff --git a/lib/std/compress/flate/inflate.zig b/lib/std/compress/flate/inflate.zig index 1d89e7f7c8..166ea94cd4 100644 --- a/lib/std/compress/flate/inflate.zig +++ b/lib/std/compress/flate/inflate.zig @@ -403,6 +403,10 @@ pub fn Inflate(comptime container: Container, comptime Lookahead: type) type { }, }; } + + pub fn readable(self: *Self, buffer: []u8) std.io.BufferedReader { + return reader(self).buffered(buffer); + } }; } diff --git a/lib/std/compress/zlib.zig b/lib/std/compress/zlib.zig index 8b0d1d6122..35fede8eeb 100644 --- a/lib/std/compress/zlib.zig +++ b/lib/std/compress/zlib.zig @@ -2,6 +2,11 @@ const std = @import("../std.zig"); const deflate = @import("flate/deflate.zig"); const inflate = @import("flate/inflate.zig"); +/// When decompressing, the output buffer is used as the history window, so +/// less than this may result in failure to decompress streams that were +/// compressed with a larger window. +pub const max_window_len = std.compress.flate.max_window_len; + /// Decompress compressed data from reader and write plain data to the writer. pub fn decompress(reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) !void { try inflate.decompress(.zlib, reader, writer); diff --git a/lib/std/crypto.zig b/lib/std/crypto.zig index 8482502828..5582e91e59 100644 --- a/lib/std/crypto.zig +++ b/lib/std/crypto.zig @@ -1,5 +1,7 @@ //! Cryptography. +const std = @import("std.zig"); +const assert = std.debug.assert; const root = @import("root"); pub const timing_safe = @import("crypto/timing_safe.zig"); @@ -119,7 +121,7 @@ pub const hash = struct { pub const blake2 = @import("crypto/blake2.zig"); pub const Blake3 = @import("crypto/blake3.zig").Blake3; pub const Md5 = @import("crypto/md5.zig").Md5; - pub const Sha1 = @import("crypto/sha1.zig").Sha1; + pub const Sha1 = @import("crypto/Sha1.zig"); pub const sha2 = @import("crypto/sha2.zig"); pub const sha3 = @import("crypto/sha3.zig"); pub const composition = @import("crypto/hash_composition.zig"); @@ -217,8 +219,6 @@ pub const random = @import("crypto/tlcsprng.zig").interface; /// Encoding and decoding pub const codecs = @import("crypto/codecs.zig"); -const std = @import("std.zig"); - pub const errors = @import("crypto/errors.zig"); pub const tls = @import("crypto/tls.zig"); diff --git a/lib/std/crypto/Sha1.zig b/lib/std/crypto/Sha1.zig new file mode 100644 index 0000000000..228a1a74c2 --- /dev/null +++ b/lib/std/crypto/Sha1.zig @@ -0,0 +1,424 @@ +//! The SHA-1 function is now considered cryptographically broken. +//! Namely, it is feasible to find multiple inputs producing the same hash. +//! For a fast-performing, cryptographically secure hash function, see SHA512/256, BLAKE2 or BLAKE3. + +const Sha1 = @This(); +const std = @import("../std.zig"); +const mem = std.mem; +const math = std.math; +const assert = std.debug.assert; + +pub const block_length = 64; +pub const digest_length = 20; +pub const Options = struct {}; + +s: [5]u32, +/// Streaming Cache +buf: [64]u8, +buf_end: u8, +total_len: u64, + +pub fn init(options: Options) Sha1 { + _ = options; + return .{ + .s = [_]u32{ + 0x67452301, + 0xEFCDAB89, + 0x98BADCFE, + 0x10325476, + 0xC3D2E1F0, + }, + .buf = undefined, + .buf_end = 0, + .total_len = 0, + }; +} + +pub fn hash(b: []const u8, out: *[digest_length]u8, options: Options) void { + var d = Sha1.init(options); + d.update(b); + d.final(out); +} + +pub fn update(d: *Sha1, b: []const u8) void { + var off: usize = 0; + + // Partial buffer exists from previous update. Copy into buffer then hash. + const unused_buf = d.buf[d.buf_end..]; + if (unused_buf.len < d.buf.len and b.len >= unused_buf.len) { + @memcpy(unused_buf, b[0..unused_buf.len]); + off += unused_buf.len; + round(&d.s, &d.buf); + d.buf_end = 0; + } + + // Full middle blocks. + while (off + 64 <= b.len) : (off += 64) { + round(&d.s, b[off..][0..64]); + } + + // Copy any remainder for next pass. + const remainder = b[off..]; + @memcpy(d.buf[d.buf_end..][0..remainder.len], remainder); + d.buf_end = @intCast(d.buf_end + remainder.len); + d.total_len += b.len; +} + +pub fn peek(d: Sha1) [digest_length]u8 { + var copy = d; + return copy.finalResult(); +} + +pub fn final(d: *Sha1, out: *[digest_length]u8) void { + // The buffer here will never be completely full. + @memset(d.buf[d.buf_end..], 0); + + // Append padding bits. + d.buf[d.buf_end] = 0x80; + d.buf_end += 1; + + // > 448 mod 512 so need to add an extra round to wrap around. + if (64 - d.buf_end < 8) { + round(&d.s, d.buf[0..]); + @memset(d.buf[0..], 0); + } + + // Append message length. + var i: usize = 1; + var len = d.total_len >> 5; + d.buf[63] = @as(u8, @intCast(d.total_len & 0x1f)) << 3; + while (i < 8) : (i += 1) { + d.buf[63 - i] = @as(u8, @intCast(len & 0xff)); + len >>= 8; + } + + round(&d.s, d.buf[0..]); + + for (d.s, 0..) |s, j| { + mem.writeInt(u32, out[4 * j ..][0..4], s, .big); + } +} + +pub fn finalResult(d: *Sha1) [digest_length]u8 { + var result: [digest_length]u8 = undefined; + d.final(&result); + return result; +} + +pub fn round(d_s: *[5]u32, b: *const [64]u8) void { + var s: [16]u32 = undefined; + + var v: [5]u32 = [_]u32{ + d_s[0], + d_s[1], + d_s[2], + d_s[3], + d_s[4], + }; + + const round0a = comptime [_]RoundParam{ + .abcdei(0, 1, 2, 3, 4, 0), + .abcdei(4, 0, 1, 2, 3, 1), + .abcdei(3, 4, 0, 1, 2, 2), + .abcdei(2, 3, 4, 0, 1, 3), + .abcdei(1, 2, 3, 4, 0, 4), + .abcdei(0, 1, 2, 3, 4, 5), + .abcdei(4, 0, 1, 2, 3, 6), + .abcdei(3, 4, 0, 1, 2, 7), + .abcdei(2, 3, 4, 0, 1, 8), + .abcdei(1, 2, 3, 4, 0, 9), + .abcdei(0, 1, 2, 3, 4, 10), + .abcdei(4, 0, 1, 2, 3, 11), + .abcdei(3, 4, 0, 1, 2, 12), + .abcdei(2, 3, 4, 0, 1, 13), + .abcdei(1, 2, 3, 4, 0, 14), + .abcdei(0, 1, 2, 3, 4, 15), + }; + inline for (round0a) |r| { + s[r.i] = mem.readInt(u32, b[r.i * 4 ..][0..4], .big); + + v[r.e] = v[r.e] +% math.rotl(u32, v[r.a], @as(u32, 5)) +% 0x5A827999 +% s[r.i & 0xf] +% ((v[r.b] & v[r.c]) | (~v[r.b] & v[r.d])); + v[r.b] = math.rotl(u32, v[r.b], @as(u32, 30)); + } + + const round0b = comptime [_]RoundParam{ + .abcdei(4, 0, 1, 2, 3, 16), + .abcdei(3, 4, 0, 1, 2, 17), + .abcdei(2, 3, 4, 0, 1, 18), + .abcdei(1, 2, 3, 4, 0, 19), + }; + inline for (round0b) |r| { + const t = s[(r.i - 3) & 0xf] ^ s[(r.i - 8) & 0xf] ^ s[(r.i - 14) & 0xf] ^ s[(r.i - 16) & 0xf]; + s[r.i & 0xf] = math.rotl(u32, t, @as(u32, 1)); + + v[r.e] = v[r.e] +% math.rotl(u32, v[r.a], @as(u32, 5)) +% 0x5A827999 +% s[r.i & 0xf] +% ((v[r.b] & v[r.c]) | (~v[r.b] & v[r.d])); + v[r.b] = math.rotl(u32, v[r.b], @as(u32, 30)); + } + + const round1 = comptime [_]RoundParam{ + .abcdei(0, 1, 2, 3, 4, 20), + .abcdei(4, 0, 1, 2, 3, 21), + .abcdei(3, 4, 0, 1, 2, 22), + .abcdei(2, 3, 4, 0, 1, 23), + .abcdei(1, 2, 3, 4, 0, 24), + .abcdei(0, 1, 2, 3, 4, 25), + .abcdei(4, 0, 1, 2, 3, 26), + .abcdei(3, 4, 0, 1, 2, 27), + .abcdei(2, 3, 4, 0, 1, 28), + .abcdei(1, 2, 3, 4, 0, 29), + .abcdei(0, 1, 2, 3, 4, 30), + .abcdei(4, 0, 1, 2, 3, 31), + .abcdei(3, 4, 0, 1, 2, 32), + .abcdei(2, 3, 4, 0, 1, 33), + .abcdei(1, 2, 3, 4, 0, 34), + .abcdei(0, 1, 2, 3, 4, 35), + .abcdei(4, 0, 1, 2, 3, 36), + .abcdei(3, 4, 0, 1, 2, 37), + .abcdei(2, 3, 4, 0, 1, 38), + .abcdei(1, 2, 3, 4, 0, 39), + }; + inline for (round1) |r| { + const t = s[(r.i - 3) & 0xf] ^ s[(r.i - 8) & 0xf] ^ s[(r.i - 14) & 0xf] ^ s[(r.i - 16) & 0xf]; + s[r.i & 0xf] = math.rotl(u32, t, @as(u32, 1)); + + v[r.e] = v[r.e] +% math.rotl(u32, v[r.a], @as(u32, 5)) +% 0x6ED9EBA1 +% s[r.i & 0xf] +% (v[r.b] ^ v[r.c] ^ v[r.d]); + v[r.b] = math.rotl(u32, v[r.b], @as(u32, 30)); + } + + const round2 = comptime [_]RoundParam{ + .abcdei(0, 1, 2, 3, 4, 40), + .abcdei(4, 0, 1, 2, 3, 41), + .abcdei(3, 4, 0, 1, 2, 42), + .abcdei(2, 3, 4, 0, 1, 43), + .abcdei(1, 2, 3, 4, 0, 44), + .abcdei(0, 1, 2, 3, 4, 45), + .abcdei(4, 0, 1, 2, 3, 46), + .abcdei(3, 4, 0, 1, 2, 47), + .abcdei(2, 3, 4, 0, 1, 48), + .abcdei(1, 2, 3, 4, 0, 49), + .abcdei(0, 1, 2, 3, 4, 50), + .abcdei(4, 0, 1, 2, 3, 51), + .abcdei(3, 4, 0, 1, 2, 52), + .abcdei(2, 3, 4, 0, 1, 53), + .abcdei(1, 2, 3, 4, 0, 54), + .abcdei(0, 1, 2, 3, 4, 55), + .abcdei(4, 0, 1, 2, 3, 56), + .abcdei(3, 4, 0, 1, 2, 57), + .abcdei(2, 3, 4, 0, 1, 58), + .abcdei(1, 2, 3, 4, 0, 59), + }; + inline for (round2) |r| { + const t = s[(r.i - 3) & 0xf] ^ s[(r.i - 8) & 0xf] ^ s[(r.i - 14) & 0xf] ^ s[(r.i - 16) & 0xf]; + s[r.i & 0xf] = math.rotl(u32, t, @as(u32, 1)); + + v[r.e] = v[r.e] +% math.rotl(u32, v[r.a], @as(u32, 5)) +% 0x8F1BBCDC +% s[r.i & 0xf] +% ((v[r.b] & v[r.c]) ^ (v[r.b] & v[r.d]) ^ (v[r.c] & v[r.d])); + v[r.b] = math.rotl(u32, v[r.b], @as(u32, 30)); + } + + const round3 = comptime [_]RoundParam{ + .abcdei(0, 1, 2, 3, 4, 60), + .abcdei(4, 0, 1, 2, 3, 61), + .abcdei(3, 4, 0, 1, 2, 62), + .abcdei(2, 3, 4, 0, 1, 63), + .abcdei(1, 2, 3, 4, 0, 64), + .abcdei(0, 1, 2, 3, 4, 65), + .abcdei(4, 0, 1, 2, 3, 66), + .abcdei(3, 4, 0, 1, 2, 67), + .abcdei(2, 3, 4, 0, 1, 68), + .abcdei(1, 2, 3, 4, 0, 69), + .abcdei(0, 1, 2, 3, 4, 70), + .abcdei(4, 0, 1, 2, 3, 71), + .abcdei(3, 4, 0, 1, 2, 72), + .abcdei(2, 3, 4, 0, 1, 73), + .abcdei(1, 2, 3, 4, 0, 74), + .abcdei(0, 1, 2, 3, 4, 75), + .abcdei(4, 0, 1, 2, 3, 76), + .abcdei(3, 4, 0, 1, 2, 77), + .abcdei(2, 3, 4, 0, 1, 78), + .abcdei(1, 2, 3, 4, 0, 79), + }; + inline for (round3) |r| { + const t = s[(r.i - 3) & 0xf] ^ s[(r.i - 8) & 0xf] ^ s[(r.i - 14) & 0xf] ^ s[(r.i - 16) & 0xf]; + s[r.i & 0xf] = math.rotl(u32, t, @as(u32, 1)); + + v[r.e] = v[r.e] +% math.rotl(u32, v[r.a], @as(u32, 5)) +% 0xCA62C1D6 +% s[r.i & 0xf] +% (v[r.b] ^ v[r.c] ^ v[r.d]); + v[r.b] = math.rotl(u32, v[r.b], @as(u32, 30)); + } + + d_s[0] +%= v[0]; + d_s[1] +%= v[1]; + d_s[2] +%= v[2]; + d_s[3] +%= v[3]; + d_s[4] +%= v[4]; +} + +pub fn writable(sha1: *Sha1, buffer: []u8) std.io.BufferedWriter { + return .{ + .unbuffered_writer = .{ + .context = sha1, + .vtable = &.{ + .writeSplat = writeSplat, + .writeFile = std.io.Writer.unimplementedWriteFile, + }, + }, + .buffer = buffer, + }; +} + +fn writeSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) std.io.Writer.Error!usize { + const sha1: *Sha1 = @ptrCast(@alignCast(context)); + const start_total = sha1.total_len; + if (sha1.buf_end == 0) { + try writeSplatAligned(sha1, data, splat); + const n: usize = @intCast(sha1.total_len - start_total); + if (n > 0) return n; + } + for (data[0 .. data.len - 1]) |slice| { + const copy_len = @min(slice.len, sha1.buf.len - sha1.buf_end); + @memcpy(sha1.buf[sha1.buf_end..][0..copy_len], slice[0..copy_len]); + sha1.total_len += copy_len; + if (sha1.buf.len - sha1.buf_end - copy_len == 0) { + round(&sha1.s, &sha1.buf); + sha1.buf_end = 0; + return @intCast(sha1.total_len - start_total); + } + sha1.buf_end = @intCast(sha1.buf_end + copy_len); + } + const slice = data[data.len - 1]; + for (0..splat) |_| { + const copy_len = @min(slice.len, sha1.buf.len - sha1.buf_end); + @memcpy(sha1.buf[sha1.buf_end..][0..copy_len], slice[0..copy_len]); + sha1.total_len += copy_len; + if (sha1.buf.len - sha1.buf_end - copy_len == 0) { + round(&sha1.s, &sha1.buf); + sha1.buf_end = 0; + return @intCast(sha1.total_len - start_total); + } + sha1.buf_end = @intCast(sha1.buf_end + copy_len); + } + return @intCast(sha1.total_len - start_total); +} + +fn writeSplatAligned(sha1: *Sha1, data: []const []const u8, splat: usize) std.io.Writer.Error!void { + assert(sha1.buf_end == 0); + for (data[0 .. data.len - 1]) |slice| { + var off: usize = 0; + while (off < slice.len) { + if (off + 64 > slice.len) { + sha1.total_len += off; + return; + } + round(&sha1.s, slice[off..][0..64]); + off += 64; + } + sha1.total_len += off; + } + const last = data[data.len - 1]; + if (last.len * splat < 64) return; + if (last.len == 1) { + @memset(&sha1.buf, last[0]); + for (0..splat / 64) |_| round(&sha1.s, &sha1.buf); + sha1.total_len += (splat / 64) * 64; + return; + } + if (last.len >= 64) { + for (0..splat) |_| { + var off: usize = 0; + while (off < last.len) { + if (off + 64 > last.len) { + sha1.total_len += off; + return; + } + round(&sha1.s, last[off..][0..64]); + off += 64; + } + } + sha1.total_len += last.len * splat; + return; + } + // Opportunity: if last.len is less than 64, we could fill up the buffer + // with the pattern repeated then do rounds. +} + +const RoundParam = struct { + a: usize, + b: usize, + c: usize, + d: usize, + e: usize, + i: u32, + + fn abcdei(a: usize, b: usize, c: usize, d: usize, e: usize, i: u32) RoundParam { + return .{ + .a = a, + .b = b, + .c = c, + .d = d, + .e = e, + .i = i, + }; + } +}; + +const htest = @import("test.zig"); + +test "sha1 single" { + try htest.assertEqualHash(Sha1, "da39a3ee5e6b4b0d3255bfef95601890afd80709", ""); + try htest.assertEqualHash(Sha1, "a9993e364706816aba3e25717850c26c9cd0d89d", "abc"); + try htest.assertEqualHash(Sha1, "a49b2446a02c645bf419f995b67091253a04a259", "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"); +} + +test "sha1 streaming" { + var h = Sha1.init(.{}); + var out: [20]u8 = undefined; + + h.final(&out); + try htest.assertEqual("da39a3ee5e6b4b0d3255bfef95601890afd80709", out[0..]); + + h = Sha1.init(.{}); + h.update("abc"); + h.final(&out); + try htest.assertEqual("a9993e364706816aba3e25717850c26c9cd0d89d", out[0..]); + + h = Sha1.init(.{}); + h.update("a"); + h.update("b"); + h.update("c"); + h.final(&out); + try htest.assertEqual("a9993e364706816aba3e25717850c26c9cd0d89d", out[0..]); +} + +test "sha1 aligned final" { + var block = [_]u8{0} ** Sha1.block_length; + var out: [Sha1.digest_length]u8 = undefined; + + var h = Sha1.init(.{}); + h.update(&block); + h.final(out[0..]); +} + +test "splat" { + var vecs = [_][]const u8{ + "hello", + "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstuyyyyyyyyyyyyyyyyyyyyyyyyy", + "x", + }; + const splat_len = 512; + const update_result = r: { + var sha1: Sha1 = .init(.{}); + sha1.update(vecs[0]); + sha1.update(vecs[1]); + var buffer: [splat_len]u8 = undefined; + @memset(&buffer, vecs[2][0]); + sha1.update(&buffer); + break :r sha1.finalResult(); + }; + const stream_result = r: { + var sha1: Sha1 = .init(.{}); + var bw = sha1.writable(&.{}); + try bw.writeSplatAll(&vecs, splat_len); + try std.testing.expectEqual(vecs[0].len + vecs[1].len + vecs[2].len * splat_len, sha1.total_len); + break :r sha1.finalResult(); + }; + try std.testing.expectEqualSlices(u8, &update_result, &stream_result); +} diff --git a/lib/std/crypto/aegis.zig b/lib/std/crypto/aegis.zig index 2f565ce490..dedd962059 100644 --- a/lib/std/crypto/aegis.zig +++ b/lib/std/crypto/aegis.zig @@ -801,18 +801,6 @@ fn AegisMac(comptime T: type) type { ctx.update(msg); ctx.final(out); } - - pub const Error = error{}; - pub const Writer = std.io.Writer(*Mac, Error, write); - - fn write(self: *Mac, bytes: []const u8) Error!usize { - self.update(bytes); - return bytes.len; - } - - pub fn writer(self: *Mac) Writer { - return .{ .context = self }; - } }; } diff --git a/lib/std/crypto/blake2.zig b/lib/std/crypto/blake2.zig index 1a285080b5..28754b4cae 100644 --- a/lib/std/crypto/blake2.zig +++ b/lib/std/crypto/blake2.zig @@ -185,18 +185,6 @@ pub fn Blake2s(comptime out_bits: usize) type { r.* ^= v[i] ^ v[i + 8]; } } - - pub const Error = error{}; - pub const Writer = std.io.Writer(*Self, Error, write); - - fn write(self: *Self, bytes: []const u8) Error!usize { - self.update(bytes); - return bytes.len; - } - - pub fn writer(self: *Self) Writer { - return .{ .context = self }; - } }; } diff --git a/lib/std/crypto/blake3.zig b/lib/std/crypto/blake3.zig index 585c338417..5fb9f6ee76 100644 --- a/lib/std/crypto/blake3.zig +++ b/lib/std/crypto/blake3.zig @@ -474,18 +474,6 @@ pub const Blake3 = struct { } output.rootOutputBytes(out_slice); } - - pub const Error = error{}; - pub const Writer = std.io.Writer(*Blake3, Error, write); - - fn write(self: *Blake3, bytes: []const u8) Error!usize { - self.update(bytes); - return bytes.len; - } - - pub fn writer(self: *Blake3) Writer { - return .{ .context = self }; - } }; // Use named type declarations to workaround crash with anonymous structs (issue #4373). diff --git a/lib/std/crypto/codecs/asn1.zig b/lib/std/crypto/codecs/asn1.zig index 7921c70118..5ae156ecbd 100644 --- a/lib/std/crypto/codecs/asn1.zig +++ b/lib/std/crypto/codecs/asn1.zig @@ -90,39 +90,32 @@ pub const Tag = struct { }; } - pub fn encode(self: Tag, writer: anytype) @TypeOf(writer).Error!void { - var tag1 = FirstTag{ + pub fn encode(self: Tag, writer: *std.io.BufferedWriter) std.io.Writer.Error!void { + var tag1: FirstTag = .{ .number = undefined, .constructed = self.constructed, .class = self.class, }; - - var buffer: [3]u8 = undefined; - var stream = std.io.fixedBufferStream(&buffer); - var writer2 = stream.writer(); - switch (@intFromEnum(self.number)) { 0...std.math.maxInt(u5) => |n| { tag1.number = @intCast(n); - writer2.writeByte(@bitCast(tag1)) catch unreachable; + try writer.writeByte(@bitCast(tag1)); }, std.math.maxInt(u5) + 1...std.math.maxInt(u7) => |n| { tag1.number = 15; const tag2 = NextTag{ .number = @intCast(n), .continues = false }; - writer2.writeByte(@bitCast(tag1)) catch unreachable; - writer2.writeByte(@bitCast(tag2)) catch unreachable; + try writer.writeByte(@bitCast(tag1)); + try writer.writeByte(@bitCast(tag2)); }, else => |n| { tag1.number = 15; const tag2 = NextTag{ .number = @intCast(n >> 7), .continues = true }; const tag3 = NextTag{ .number = @truncate(n), .continues = false }; - writer2.writeByte(@bitCast(tag1)) catch unreachable; - writer2.writeByte(@bitCast(tag2)) catch unreachable; - writer2.writeByte(@bitCast(tag3)) catch unreachable; + try writer.writeByte(@bitCast(tag1)); + try writer.writeByte(@bitCast(tag2)); + try writer.writeByte(@bitCast(tag3)); }, } - - _ = try writer.write(stream.getWritten()); } const FirstTag = packed struct(u8) { number: u5, constructed: bool, class: Tag.Class }; diff --git a/lib/std/crypto/codecs/asn1/Oid.zig b/lib/std/crypto/codecs/asn1/Oid.zig index edca552050..4c20831c8b 100644 --- a/lib/std/crypto/codecs/asn1/Oid.zig +++ b/lib/std/crypto/codecs/asn1/Oid.zig @@ -4,9 +4,12 @@ //! organizations, or policy documents. encoded: []const u8, -pub const InitError = std.fmt.ParseIntError || error{MissingPrefix} || std.io.FixedBufferStream(u8).WriteError; +pub const EncodeError = error{ + WriteFailed, + MissingPrefix, +}; -pub fn fromDot(dot_notation: []const u8, out: []u8) InitError!Oid { +pub fn encode(dot_notation: []const u8, out: *std.io.BufferedWriter) EncodeError!void { var split = std.mem.splitScalar(u8, dot_notation, '.'); const first_str = split.next() orelse return error.MissingPrefix; const second_str = split.next() orelse return error.MissingPrefix; @@ -14,10 +17,7 @@ pub fn fromDot(dot_notation: []const u8, out: []u8) InitError!Oid { const first = try std.fmt.parseInt(u8, first_str, 10); const second = try std.fmt.parseInt(u8, second_str, 10); - var stream = std.io.fixedBufferStream(out); - var writer = stream.writer(); - - try writer.writeByte(first * 40 + second); + try out.writeByte(first * 40 + second); var i: usize = 1; while (split.next()) |s| { @@ -28,16 +28,26 @@ pub fn fromDot(dot_notation: []const u8, out: []u8) InitError!Oid { const place = std.math.pow(Arc, encoding_base, n_bytes - @as(Arc, @intCast(j))); const digit: u8 = @intCast(@divFloor(parsed, place)); - try writer.writeByte(digit | 0x80); + try out.writeByte(digit | 0x80); parsed -= digit * place; i += 1; } - try writer.writeByte(@intCast(parsed)); + try out.writeByte(@intCast(parsed)); i += 1; } +} - return .{ .encoded = stream.getWritten() }; +pub const InitError = std.fmt.ParseIntError || error{ MissingPrefix, BufferTooSmall }; + +pub fn fromDot(dot_notation: []const u8, out: []u8) InitError!Oid { + var bw: std.io.BufferedWriter = undefined; + bw.initFixed(out); + encode(dot_notation, &bw) catch |err| switch (err) { + error.WriteFailed => return error.BufferTooSmall, + else => |e| return e, + }; + return .{ .encoded = bw.getWritten() }; } test fromDot { @@ -48,7 +58,7 @@ test fromDot { } } -pub fn toDot(self: Oid, writer: anytype) @TypeOf(writer).Error!void { +pub fn toDot(self: Oid, writer: *std.io.BufferedWriter) std.io.Writer.Error!void { const encoded = self.encoded; const first = @divTrunc(encoded[0], 40); const second = encoded[0] - first * 40; @@ -80,9 +90,10 @@ test toDot { var buf: [256]u8 = undefined; for (test_cases) |t| { - var stream = std.io.fixedBufferStream(&buf); - try toDot(Oid{ .encoded = t.encoded }, stream.writer()); - try std.testing.expectEqualStrings(t.dot_notation, stream.getWritten()); + var bw: std.io.BufferedWriter = undefined; + bw.initFixed(&buf); + try toDot(Oid{ .encoded = t.encoded }, &bw); + try std.testing.expectEqualStrings(t.dot_notation, bw.getWritten()); } } diff --git a/lib/std/crypto/phc_encoding.zig b/lib/std/crypto/phc_encoding.zig index 1aa7257bbb..c23702747a 100644 --- a/lib/std/crypto/phc_encoding.zig +++ b/lib/std/crypto/phc_encoding.zig @@ -196,9 +196,11 @@ pub fn serialize(params: anytype, str: []u8) Error![]const u8 { /// Compute the number of bytes required to serialize `params` pub fn calcSize(params: anytype) usize { - var buf = io.countingWriter(io.null_writer); - serializeTo(params, buf.writer()) catch unreachable; - return @as(usize, @intCast(buf.bytes_written)); + var null_writer: std.io.Writer.Null = .{}; + var trash: [128]u8 = undefined; + var bw = null_writer.writable(&trash); + serializeTo(params, &bw) catch unreachable; + return bw.count; } fn serializeTo(params: anytype, out: *std.io.BufferedWriter) !void { diff --git a/lib/std/crypto/sha1.zig b/lib/std/crypto/sha1.zig deleted file mode 100644 index 2968517b68..0000000000 --- a/lib/std/crypto/sha1.zig +++ /dev/null @@ -1,319 +0,0 @@ -const std = @import("../std.zig"); -const mem = std.mem; -const math = std.math; - -const RoundParam = struct { - a: usize, - b: usize, - c: usize, - d: usize, - e: usize, - i: u32, -}; - -fn roundParam(a: usize, b: usize, c: usize, d: usize, e: usize, i: u32) RoundParam { - return RoundParam{ - .a = a, - .b = b, - .c = c, - .d = d, - .e = e, - .i = i, - }; -} - -/// The SHA-1 function is now considered cryptographically broken. -/// Namely, it is feasible to find multiple inputs producing the same hash. -/// For a fast-performing, cryptographically secure hash function, see SHA512/256, BLAKE2 or BLAKE3. -pub const Sha1 = struct { - const Self = @This(); - pub const block_length = 64; - pub const digest_length = 20; - pub const Options = struct {}; - - s: [5]u32, - // Streaming Cache - buf: [64]u8 = undefined, - buf_len: u8 = 0, - total_len: u64 = 0, - - pub fn init(options: Options) Self { - _ = options; - return Self{ - .s = [_]u32{ - 0x67452301, - 0xEFCDAB89, - 0x98BADCFE, - 0x10325476, - 0xC3D2E1F0, - }, - }; - } - - pub fn hash(b: []const u8, out: *[digest_length]u8, options: Options) void { - var d = Sha1.init(options); - d.update(b); - d.final(out); - } - - pub fn update(d: *Self, b: []const u8) void { - var off: usize = 0; - - // Partial buffer exists from previous update. Copy into buffer then hash. - if (d.buf_len != 0 and d.buf_len + b.len >= 64) { - off += 64 - d.buf_len; - @memcpy(d.buf[d.buf_len..][0..off], b[0..off]); - - d.round(d.buf[0..]); - d.buf_len = 0; - } - - // Full middle blocks. - while (off + 64 <= b.len) : (off += 64) { - d.round(b[off..][0..64]); - } - - // Copy any remainder for next pass. - @memcpy(d.buf[d.buf_len..][0 .. b.len - off], b[off..]); - d.buf_len += @as(u8, @intCast(b[off..].len)); - - d.total_len += b.len; - } - - pub fn peek(d: Self) [digest_length]u8 { - var copy = d; - return copy.finalResult(); - } - - pub fn final(d: *Self, out: *[digest_length]u8) void { - // The buffer here will never be completely full. - @memset(d.buf[d.buf_len..], 0); - - // Append padding bits. - d.buf[d.buf_len] = 0x80; - d.buf_len += 1; - - // > 448 mod 512 so need to add an extra round to wrap around. - if (64 - d.buf_len < 8) { - d.round(d.buf[0..]); - @memset(d.buf[0..], 0); - } - - // Append message length. - var i: usize = 1; - var len = d.total_len >> 5; - d.buf[63] = @as(u8, @intCast(d.total_len & 0x1f)) << 3; - while (i < 8) : (i += 1) { - d.buf[63 - i] = @as(u8, @intCast(len & 0xff)); - len >>= 8; - } - - d.round(d.buf[0..]); - - for (d.s, 0..) |s, j| { - mem.writeInt(u32, out[4 * j ..][0..4], s, .big); - } - } - - pub fn finalResult(d: *Self) [digest_length]u8 { - var result: [digest_length]u8 = undefined; - d.final(&result); - return result; - } - - fn round(d: *Self, b: *const [64]u8) void { - var s: [16]u32 = undefined; - - var v: [5]u32 = [_]u32{ - d.s[0], - d.s[1], - d.s[2], - d.s[3], - d.s[4], - }; - - const round0a = comptime [_]RoundParam{ - roundParam(0, 1, 2, 3, 4, 0), - roundParam(4, 0, 1, 2, 3, 1), - roundParam(3, 4, 0, 1, 2, 2), - roundParam(2, 3, 4, 0, 1, 3), - roundParam(1, 2, 3, 4, 0, 4), - roundParam(0, 1, 2, 3, 4, 5), - roundParam(4, 0, 1, 2, 3, 6), - roundParam(3, 4, 0, 1, 2, 7), - roundParam(2, 3, 4, 0, 1, 8), - roundParam(1, 2, 3, 4, 0, 9), - roundParam(0, 1, 2, 3, 4, 10), - roundParam(4, 0, 1, 2, 3, 11), - roundParam(3, 4, 0, 1, 2, 12), - roundParam(2, 3, 4, 0, 1, 13), - roundParam(1, 2, 3, 4, 0, 14), - roundParam(0, 1, 2, 3, 4, 15), - }; - inline for (round0a) |r| { - s[r.i] = mem.readInt(u32, b[r.i * 4 ..][0..4], .big); - - v[r.e] = v[r.e] +% math.rotl(u32, v[r.a], @as(u32, 5)) +% 0x5A827999 +% s[r.i & 0xf] +% ((v[r.b] & v[r.c]) | (~v[r.b] & v[r.d])); - v[r.b] = math.rotl(u32, v[r.b], @as(u32, 30)); - } - - const round0b = comptime [_]RoundParam{ - roundParam(4, 0, 1, 2, 3, 16), - roundParam(3, 4, 0, 1, 2, 17), - roundParam(2, 3, 4, 0, 1, 18), - roundParam(1, 2, 3, 4, 0, 19), - }; - inline for (round0b) |r| { - const t = s[(r.i - 3) & 0xf] ^ s[(r.i - 8) & 0xf] ^ s[(r.i - 14) & 0xf] ^ s[(r.i - 16) & 0xf]; - s[r.i & 0xf] = math.rotl(u32, t, @as(u32, 1)); - - v[r.e] = v[r.e] +% math.rotl(u32, v[r.a], @as(u32, 5)) +% 0x5A827999 +% s[r.i & 0xf] +% ((v[r.b] & v[r.c]) | (~v[r.b] & v[r.d])); - v[r.b] = math.rotl(u32, v[r.b], @as(u32, 30)); - } - - const round1 = comptime [_]RoundParam{ - roundParam(0, 1, 2, 3, 4, 20), - roundParam(4, 0, 1, 2, 3, 21), - roundParam(3, 4, 0, 1, 2, 22), - roundParam(2, 3, 4, 0, 1, 23), - roundParam(1, 2, 3, 4, 0, 24), - roundParam(0, 1, 2, 3, 4, 25), - roundParam(4, 0, 1, 2, 3, 26), - roundParam(3, 4, 0, 1, 2, 27), - roundParam(2, 3, 4, 0, 1, 28), - roundParam(1, 2, 3, 4, 0, 29), - roundParam(0, 1, 2, 3, 4, 30), - roundParam(4, 0, 1, 2, 3, 31), - roundParam(3, 4, 0, 1, 2, 32), - roundParam(2, 3, 4, 0, 1, 33), - roundParam(1, 2, 3, 4, 0, 34), - roundParam(0, 1, 2, 3, 4, 35), - roundParam(4, 0, 1, 2, 3, 36), - roundParam(3, 4, 0, 1, 2, 37), - roundParam(2, 3, 4, 0, 1, 38), - roundParam(1, 2, 3, 4, 0, 39), - }; - inline for (round1) |r| { - const t = s[(r.i - 3) & 0xf] ^ s[(r.i - 8) & 0xf] ^ s[(r.i - 14) & 0xf] ^ s[(r.i - 16) & 0xf]; - s[r.i & 0xf] = math.rotl(u32, t, @as(u32, 1)); - - v[r.e] = v[r.e] +% math.rotl(u32, v[r.a], @as(u32, 5)) +% 0x6ED9EBA1 +% s[r.i & 0xf] +% (v[r.b] ^ v[r.c] ^ v[r.d]); - v[r.b] = math.rotl(u32, v[r.b], @as(u32, 30)); - } - - const round2 = comptime [_]RoundParam{ - roundParam(0, 1, 2, 3, 4, 40), - roundParam(4, 0, 1, 2, 3, 41), - roundParam(3, 4, 0, 1, 2, 42), - roundParam(2, 3, 4, 0, 1, 43), - roundParam(1, 2, 3, 4, 0, 44), - roundParam(0, 1, 2, 3, 4, 45), - roundParam(4, 0, 1, 2, 3, 46), - roundParam(3, 4, 0, 1, 2, 47), - roundParam(2, 3, 4, 0, 1, 48), - roundParam(1, 2, 3, 4, 0, 49), - roundParam(0, 1, 2, 3, 4, 50), - roundParam(4, 0, 1, 2, 3, 51), - roundParam(3, 4, 0, 1, 2, 52), - roundParam(2, 3, 4, 0, 1, 53), - roundParam(1, 2, 3, 4, 0, 54), - roundParam(0, 1, 2, 3, 4, 55), - roundParam(4, 0, 1, 2, 3, 56), - roundParam(3, 4, 0, 1, 2, 57), - roundParam(2, 3, 4, 0, 1, 58), - roundParam(1, 2, 3, 4, 0, 59), - }; - inline for (round2) |r| { - const t = s[(r.i - 3) & 0xf] ^ s[(r.i - 8) & 0xf] ^ s[(r.i - 14) & 0xf] ^ s[(r.i - 16) & 0xf]; - s[r.i & 0xf] = math.rotl(u32, t, @as(u32, 1)); - - v[r.e] = v[r.e] +% math.rotl(u32, v[r.a], @as(u32, 5)) +% 0x8F1BBCDC +% s[r.i & 0xf] +% ((v[r.b] & v[r.c]) ^ (v[r.b] & v[r.d]) ^ (v[r.c] & v[r.d])); - v[r.b] = math.rotl(u32, v[r.b], @as(u32, 30)); - } - - const round3 = comptime [_]RoundParam{ - roundParam(0, 1, 2, 3, 4, 60), - roundParam(4, 0, 1, 2, 3, 61), - roundParam(3, 4, 0, 1, 2, 62), - roundParam(2, 3, 4, 0, 1, 63), - roundParam(1, 2, 3, 4, 0, 64), - roundParam(0, 1, 2, 3, 4, 65), - roundParam(4, 0, 1, 2, 3, 66), - roundParam(3, 4, 0, 1, 2, 67), - roundParam(2, 3, 4, 0, 1, 68), - roundParam(1, 2, 3, 4, 0, 69), - roundParam(0, 1, 2, 3, 4, 70), - roundParam(4, 0, 1, 2, 3, 71), - roundParam(3, 4, 0, 1, 2, 72), - roundParam(2, 3, 4, 0, 1, 73), - roundParam(1, 2, 3, 4, 0, 74), - roundParam(0, 1, 2, 3, 4, 75), - roundParam(4, 0, 1, 2, 3, 76), - roundParam(3, 4, 0, 1, 2, 77), - roundParam(2, 3, 4, 0, 1, 78), - roundParam(1, 2, 3, 4, 0, 79), - }; - inline for (round3) |r| { - const t = s[(r.i - 3) & 0xf] ^ s[(r.i - 8) & 0xf] ^ s[(r.i - 14) & 0xf] ^ s[(r.i - 16) & 0xf]; - s[r.i & 0xf] = math.rotl(u32, t, @as(u32, 1)); - - v[r.e] = v[r.e] +% math.rotl(u32, v[r.a], @as(u32, 5)) +% 0xCA62C1D6 +% s[r.i & 0xf] +% (v[r.b] ^ v[r.c] ^ v[r.d]); - v[r.b] = math.rotl(u32, v[r.b], @as(u32, 30)); - } - - d.s[0] +%= v[0]; - d.s[1] +%= v[1]; - d.s[2] +%= v[2]; - d.s[3] +%= v[3]; - d.s[4] +%= v[4]; - } - - pub const Error = error{}; - pub const Writer = std.io.Writer(*Self, Error, write); - - fn write(self: *Self, bytes: []const u8) Error!usize { - self.update(bytes); - return bytes.len; - } - - pub fn writer(self: *Self) Writer { - return .{ .context = self }; - } -}; - -const htest = @import("test.zig"); - -test "sha1 single" { - try htest.assertEqualHash(Sha1, "da39a3ee5e6b4b0d3255bfef95601890afd80709", ""); - try htest.assertEqualHash(Sha1, "a9993e364706816aba3e25717850c26c9cd0d89d", "abc"); - try htest.assertEqualHash(Sha1, "a49b2446a02c645bf419f995b67091253a04a259", "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu"); -} - -test "sha1 streaming" { - var h = Sha1.init(.{}); - var out: [20]u8 = undefined; - - h.final(&out); - try htest.assertEqual("da39a3ee5e6b4b0d3255bfef95601890afd80709", out[0..]); - - h = Sha1.init(.{}); - h.update("abc"); - h.final(&out); - try htest.assertEqual("a9993e364706816aba3e25717850c26c9cd0d89d", out[0..]); - - h = Sha1.init(.{}); - h.update("a"); - h.update("b"); - h.update("c"); - h.final(&out); - try htest.assertEqual("a9993e364706816aba3e25717850c26c9cd0d89d", out[0..]); -} - -test "sha1 aligned final" { - var block = [_]u8{0} ** Sha1.block_length; - var out: [Sha1.digest_length]u8 = undefined; - - var h = Sha1.init(.{}); - h.update(&block); - h.final(out[0..]); -} diff --git a/lib/std/crypto/sha2.zig b/lib/std/crypto/sha2.zig index edcb2ecb88..9a0cfc0849 100644 --- a/lib/std/crypto/sha2.zig +++ b/lib/std/crypto/sha2.zig @@ -95,14 +95,19 @@ fn Sha2x32(comptime iv: Iv32, digest_bits: comptime_int) type { pub const Options = struct {}; s: [8]u32 align(16), - // Streaming Cache - buf: [64]u8 = undefined, - buf_len: u8 = 0, - total_len: u64 = 0, + /// Streaming Cache + buf: [64]u8, + buf_len: u8, + total_len: u64, pub fn init(options: Options) Self { _ = options; - return Self{ .s = iv }; + return .{ + .s = iv, + .buf = undefined, + .buf_len = 0, + .total_len = 0, + }; } pub fn hash(b: []const u8, out: *[digest_length]u8, options: Options) void { @@ -377,16 +382,25 @@ fn Sha2x32(comptime iv: Iv32, digest_bits: comptime_int) type { for (&d.s, v) |*dv, vv| dv.* +%= vv; } - pub const Error = error{}; - pub const Writer = std.io.Writer(*Self, Error, write); - - fn write(self: *Self, bytes: []const u8) Error!usize { - self.update(bytes); - return bytes.len; + pub fn writable(this: *@This(), buffer: []u8) std.io.BufferedWriter { + return .{ + .unbuffered_writer = .{ + .context = this, + .vtable = &.{ + .writeSplat = writeSplat, + .writeFile = std.io.Writer.unimplementedWriteFile, + }, + }, + .buffer = buffer, + }; } - pub fn writer(self: *Self) Writer { - return .{ .context = self }; + fn writeSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) std.io.Writer.Error!usize { + const this: *@This() = @ptrCast(@alignCast(context)); + const start_total = this.total_len; + for (data[0 .. data.len - 1]) |slice| this.update(slice); + for (0..splat) |_| this.update(data[data.len - 1]); + return @intCast(this.total_len - start_total); } }; } diff --git a/lib/std/crypto/sha3.zig b/lib/std/crypto/sha3.zig index a001538c1d..84cd0c2b0e 100644 --- a/lib/std/crypto/sha3.zig +++ b/lib/std/crypto/sha3.zig @@ -80,18 +80,6 @@ pub fn Keccak(comptime f: u11, comptime output_bits: u11, comptime default_delim self.st.pad(); self.st.squeeze(out[0..]); } - - pub const Error = error{}; - pub const Writer = std.io.Writer(*Self, Error, write); - - fn write(self: *Self, bytes: []const u8) Error!usize { - self.update(bytes); - return bytes.len; - } - - pub fn writer(self: *Self) Writer { - return .{ .context = self }; - } }; } @@ -191,18 +179,6 @@ fn ShakeLike(comptime security_level: u11, comptime default_delim: u8, comptime pub fn fillBlock(self: *Self) void { self.st.fillBlock(); } - - pub const Error = error{}; - pub const Writer = std.io.Writer(*Self, Error, write); - - fn write(self: *Self, bytes: []const u8) Error!usize { - self.update(bytes); - return bytes.len; - } - - pub fn writer(self: *Self) Writer { - return .{ .context = self }; - } }; } @@ -284,18 +260,6 @@ fn CShakeLike(comptime security_level: u11, comptime default_delim: u8, comptime pub fn fillBlock(self: *Self) void { self.shaker.fillBlock(); } - - pub const Error = error{}; - pub const Writer = std.io.Writer(*Self, Error, write); - - fn write(self: *Self, bytes: []const u8) Error!usize { - self.update(bytes); - return bytes.len; - } - - pub fn writer(self: *Self) Writer { - return .{ .context = self }; - } }; } @@ -390,18 +354,6 @@ fn KMacLike(comptime security_level: u11, comptime default_delim: u8, comptime r ctx.update(msg); ctx.final(out); } - - pub const Error = error{}; - pub const Writer = std.io.Writer(*Self, Error, write); - - fn write(self: *Self, bytes: []const u8) Error!usize { - self.update(bytes); - return bytes.len; - } - - pub fn writer(self: *Self) Writer { - return .{ .context = self }; - } }; } @@ -482,18 +434,6 @@ fn TupleHashLike(comptime security_level: u11, comptime default_delim: u8, compt } self.cshaker.squeeze(out); } - - pub const Error = error{}; - pub const Writer = std.io.Writer(*Self, Error, write); - - fn write(self: *Self, bytes: []const u8) Error!usize { - self.update(bytes); - return bytes.len; - } - - pub fn writer(self: *Self) Writer { - return .{ .context = self }; - } }; } diff --git a/lib/std/crypto/siphash.zig b/lib/std/crypto/siphash.zig index a4cd904ba8..cf595327c5 100644 --- a/lib/std/crypto/siphash.zig +++ b/lib/std/crypto/siphash.zig @@ -238,48 +238,6 @@ fn SipHash(comptime T: type, comptime c_rounds: usize, comptime d_rounds: usize) pub fn toInt(msg: []const u8, key: *const [key_length]u8) T { return State.hash(msg, key); } - - pub fn writer(self: *Self) std.io.Writer { - return .{ - .context = self, - .vtable = &.{ - .writeSplat = &writeSplat, - .writeFile = &writeFile, - }, - }; - } - - fn writeSplat(ctx: ?*anyopaque, data: []const []const u8, splat: usize) std.io.Writer.Error!usize { - const self: *Self = @alignCast(@ptrCast(ctx)); - var len: usize = 0; - for (data[0 .. data.len - 1]) |slice| { - self.update(slice); - len += slice.len; - } - { - const slice = data[data.len - 1]; - for (0..splat) |_| self.update(slice); - len += slice.len * splat; - } - return len; - } - - fn writeFile( - ctx: ?*anyopaque, - file: std.fs.File, - offset: std.io.Writer.Offset, - limit: std.io.Writer.Limit, - headers_and_trailers: []const []const u8, - headers_len: usize, - ) std.io.Writer.Error!usize { - _ = ctx; - _ = file; - _ = offset; - _ = limit; - _ = headers_and_trailers; - _ = headers_len; - @panic("TODO"); - } }; } diff --git a/lib/std/debug/Dwarf.zig b/lib/std/debug/Dwarf.zig index 4bca4f1c9d..e81e997948 100644 --- a/lib/std/debug/Dwarf.zig +++ b/lib/std/debug/Dwarf.zig @@ -2176,7 +2176,7 @@ pub const ElfModule = struct { parent_mapped_mem: ?[]align(std.heap.page_size_min) const u8, elf_filename: ?[]const u8, ) LoadError!Dwarf.ElfModule { - if (expected_crc) |crc| if (crc != std.hash.crc.Crc32.hash(mapped_mem)) return error.InvalidDebugInfo; + if (expected_crc) |crc| if (crc != std.hash.Crc32.hash(mapped_mem)) return error.InvalidDebugInfo; const hdr: *const elf.Ehdr = @ptrCast(&mapped_mem[0]); if (!mem.eql(u8, hdr.e_ident[0..4], elf.MAGIC)) return error.InvalidElfMagic; diff --git a/lib/std/fs/File.zig b/lib/std/fs/File.zig index 8161887552..6a3589aae8 100644 --- a/lib/std/fs/File.zig +++ b/lib/std/fs/File.zig @@ -934,6 +934,10 @@ pub const Reader = struct { }; } + pub fn readable(r: *Reader, buffer: []u8) std.io.BufferedReader { + return interface(r).buffered(buffer); + } + pub fn getSize(r: *Reader) GetEndPosError!u64 { return r.size orelse { if (r.size_err) |err| return err; @@ -1228,6 +1232,7 @@ pub const Writer = struct { pos: u64 = 0, sendfile_err: ?SendfileError = null, read_err: ?ReadError = null, + seek_err: ?SeekError = null, pub const Mode = Reader.Mode; @@ -1250,6 +1255,20 @@ pub const Writer = struct { }; } + pub fn writable(w: *Writer, buffer: []u8) std.io.BufferedWriter { + return interface(w).buffered(buffer); + } + + pub fn moveToReader(w: *Writer) Reader { + defer w.* = undefined; + return .{ + .file = w.file, + .mode = w.mode, + .pos = w.pos, + .seek_err = w.seek_err, + }; + } + pub fn writeSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) std.io.Writer.Error!usize { const w: *Writer = @ptrCast(@alignCast(context)); const handle = w.file.handle; @@ -1347,6 +1366,21 @@ pub const Writer = struct { } return error.Unimplemented; } + + pub fn seekTo(w: *Writer, offset: u64) SeekError!void { + if (w.seek_err) |err| return err; + switch (w.mode) { + .positional, .positional_reading => { + w.pos = offset; + }, + .streaming, .streaming_reading => { + posix.lseek_SET(w.file.handle, offset) catch |err| { + w.seek_err = err; + return err; + }; + }, + } + } }; /// Defaults to positional reading; falls back to streaming. diff --git a/lib/std/hash.zig b/lib/std/hash.zig index 157dad67a7..d01cbde6c1 100644 --- a/lib/std/hash.zig +++ b/lib/std/hash.zig @@ -6,9 +6,8 @@ pub const autoHash = auto_hash.autoHash; pub const autoHashStrat = auto_hash.hash; pub const Strategy = auto_hash.HashStrategy; -// pub for polynomials + generic crc32 construction pub const crc = @import("hash/crc.zig"); -pub const Crc32 = crc.Crc32; +pub const Crc32 = crc.Crc32IsoHdlc; const fnv = @import("hash/fnv.zig"); pub const Fnv1a_32 = fnv.Fnv1a_32; diff --git a/lib/std/hash/crc.zig b/lib/std/hash/crc.zig index c0a418a0c2..e48b9eac97 100644 --- a/lib/std/hash/crc.zig +++ b/lib/std/hash/crc.zig @@ -1,19 +1,127 @@ -//! This file is auto-generated by tools/update_crc_catalog.zig. +const std = @import("../std.zig"); -const impl = @import("crc/impl.zig"); +pub fn Generic(comptime W: type, comptime algorithm: Algorithm(W)) type { + return struct { + const Self = @This(); + const I = if (@bitSizeOf(W) < 8) u8 else W; + const lookup_table = blk: { + @setEvalBranchQuota(2500); -pub const Crc = impl.Crc; -pub const Polynomial = impl.Polynomial; -pub const Crc32WithPoly = impl.Crc32WithPoly; -pub const Crc32SmallWithPoly = impl.Crc32SmallWithPoly; + const poly = if (algorithm.reflect_input) + @bitReverse(@as(I, algorithm.polynomial)) >> (@bitSizeOf(I) - @bitSizeOf(W)) + else + @as(I, algorithm.polynomial) << (@bitSizeOf(I) - @bitSizeOf(W)); -pub const Crc32 = Crc32IsoHdlc; + var table: [256]I = undefined; + for (&table, 0..) |*e, i| { + var crc: I = i; + if (algorithm.reflect_input) { + var j: usize = 0; + while (j < 8) : (j += 1) { + crc = (crc >> 1) ^ ((crc & 1) * poly); + } + } else { + crc <<= @bitSizeOf(I) - 8; + var j: usize = 0; + while (j < 8) : (j += 1) { + crc = (crc << 1) ^ (((crc >> (@bitSizeOf(I) - 1)) & 1) * poly); + } + } + e.* = crc; + } + break :blk table; + }; -test { - _ = @import("crc/test.zig"); + crc: I, + + pub fn init() Self { + const initial = if (algorithm.reflect_input) + @bitReverse(@as(I, algorithm.initial)) >> (@bitSizeOf(I) - @bitSizeOf(W)) + else + @as(I, algorithm.initial) << (@bitSizeOf(I) - @bitSizeOf(W)); + return .{ .crc = initial }; + } + + inline fn tableEntry(index: I) I { + return lookup_table[@as(u8, @intCast(index & 0xFF))]; + } + + pub fn updateByte(self: *Self, byte: u8) void { + if (@bitSizeOf(I) <= 8) { + self.crc = tableEntry(self.crc ^ byte); + } else if (algorithm.reflect_input) { + const table_index = self.crc ^ byte; + self.crc = tableEntry(table_index) ^ (self.crc >> 8); + } else { + const table_index = (self.crc >> (@bitSizeOf(I) - 8)) ^ byte; + self.crc = tableEntry(table_index) ^ (self.crc << 8); + } + } + + pub fn update(self: *Self, bytes: []const u8) void { + for (bytes) |byte| updateByte(self, byte); + } + + pub fn final(self: Self) W { + var c = self.crc; + if (algorithm.reflect_input != algorithm.reflect_output) { + c = @bitReverse(c); + } + if (!algorithm.reflect_output) { + c >>= @bitSizeOf(I) - @bitSizeOf(W); + } + return @intCast(c ^ algorithm.xor_output); + } + + pub fn hash(bytes: []const u8) W { + var c = Self.init(); + c.update(bytes); + return c.final(); + } + + pub fn writable(self: *Self, buffer: []u8) std.io.BufferedWriter { + return .{ + .unbuffered_writer = .{ + .context = self, + .vtable = &.{ + .writeSplat = writeSplat, + .writeFile = std.io.Writer.unimplementedWriteFile, + }, + }, + .buffer = buffer, + }; + } + + fn writeSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) std.io.Writer.Error!usize { + const self: *Self = @ptrCast(@alignCast(context)); + var n: usize = 0; + for (data[0 .. data.len - 1]) |slice| { + self.update(slice); + n += slice.len; + } + const last = data[data.len - 1]; + if (last.len == 1) { + for (0..splat) |_| self.updateByte(last[0]); + return n + splat; + } else { + for (0..splat) |_| self.update(last); + return n + last.len * splat; + } + } + }; } -pub const Crc3Gsm = Crc(u3, .{ +pub fn Algorithm(comptime W: type) type { + return struct { + polynomial: W, + initial: W, + reflect_input: bool, + reflect_output: bool, + xor_output: W, + }; +} + +pub const Crc3Gsm = Generic(u3, .{ .polynomial = 0x3, .initial = 0x0, .reflect_input = false, @@ -21,7 +129,7 @@ pub const Crc3Gsm = Crc(u3, .{ .xor_output = 0x7, }); -pub const Crc3Rohc = Crc(u3, .{ +pub const Crc3Rohc = Generic(u3, .{ .polynomial = 0x3, .initial = 0x7, .reflect_input = true, @@ -29,7 +137,7 @@ pub const Crc3Rohc = Crc(u3, .{ .xor_output = 0x0, }); -pub const Crc4G704 = Crc(u4, .{ +pub const Crc4G704 = Generic(u4, .{ .polynomial = 0x3, .initial = 0x0, .reflect_input = true, @@ -37,7 +145,7 @@ pub const Crc4G704 = Crc(u4, .{ .xor_output = 0x0, }); -pub const Crc4Interlaken = Crc(u4, .{ +pub const Crc4Interlaken = Generic(u4, .{ .polynomial = 0x3, .initial = 0xf, .reflect_input = false, @@ -45,7 +153,7 @@ pub const Crc4Interlaken = Crc(u4, .{ .xor_output = 0xf, }); -pub const Crc5EpcC1g2 = Crc(u5, .{ +pub const Crc5EpcC1g2 = Generic(u5, .{ .polynomial = 0x09, .initial = 0x09, .reflect_input = false, @@ -53,7 +161,7 @@ pub const Crc5EpcC1g2 = Crc(u5, .{ .xor_output = 0x00, }); -pub const Crc5G704 = Crc(u5, .{ +pub const Crc5G704 = Generic(u5, .{ .polynomial = 0x15, .initial = 0x00, .reflect_input = true, @@ -61,7 +169,7 @@ pub const Crc5G704 = Crc(u5, .{ .xor_output = 0x00, }); -pub const Crc5Usb = Crc(u5, .{ +pub const Crc5Usb = Generic(u5, .{ .polynomial = 0x05, .initial = 0x1f, .reflect_input = true, @@ -69,7 +177,7 @@ pub const Crc5Usb = Crc(u5, .{ .xor_output = 0x1f, }); -pub const Crc6Cdma2000A = Crc(u6, .{ +pub const Crc6Cdma2000A = Generic(u6, .{ .polynomial = 0x27, .initial = 0x3f, .reflect_input = false, @@ -77,7 +185,7 @@ pub const Crc6Cdma2000A = Crc(u6, .{ .xor_output = 0x00, }); -pub const Crc6Cdma2000B = Crc(u6, .{ +pub const Crc6Cdma2000B = Generic(u6, .{ .polynomial = 0x07, .initial = 0x3f, .reflect_input = false, @@ -85,7 +193,7 @@ pub const Crc6Cdma2000B = Crc(u6, .{ .xor_output = 0x00, }); -pub const Crc6Darc = Crc(u6, .{ +pub const Crc6Darc = Generic(u6, .{ .polynomial = 0x19, .initial = 0x00, .reflect_input = true, @@ -93,7 +201,7 @@ pub const Crc6Darc = Crc(u6, .{ .xor_output = 0x00, }); -pub const Crc6G704 = Crc(u6, .{ +pub const Crc6G704 = Generic(u6, .{ .polynomial = 0x03, .initial = 0x00, .reflect_input = true, @@ -101,7 +209,7 @@ pub const Crc6G704 = Crc(u6, .{ .xor_output = 0x00, }); -pub const Crc6Gsm = Crc(u6, .{ +pub const Crc6Gsm = Generic(u6, .{ .polynomial = 0x2f, .initial = 0x00, .reflect_input = false, @@ -109,7 +217,7 @@ pub const Crc6Gsm = Crc(u6, .{ .xor_output = 0x3f, }); -pub const Crc7Mmc = Crc(u7, .{ +pub const Crc7Mmc = Generic(u7, .{ .polynomial = 0x09, .initial = 0x00, .reflect_input = false, @@ -117,7 +225,7 @@ pub const Crc7Mmc = Crc(u7, .{ .xor_output = 0x00, }); -pub const Crc7Rohc = Crc(u7, .{ +pub const Crc7Rohc = Generic(u7, .{ .polynomial = 0x4f, .initial = 0x7f, .reflect_input = true, @@ -125,7 +233,7 @@ pub const Crc7Rohc = Crc(u7, .{ .xor_output = 0x00, }); -pub const Crc7Umts = Crc(u7, .{ +pub const Crc7Umts = Generic(u7, .{ .polynomial = 0x45, .initial = 0x00, .reflect_input = false, @@ -133,7 +241,7 @@ pub const Crc7Umts = Crc(u7, .{ .xor_output = 0x00, }); -pub const Crc8Autosar = Crc(u8, .{ +pub const Crc8Autosar = Generic(u8, .{ .polynomial = 0x2f, .initial = 0xff, .reflect_input = false, @@ -141,7 +249,7 @@ pub const Crc8Autosar = Crc(u8, .{ .xor_output = 0xff, }); -pub const Crc8Bluetooth = Crc(u8, .{ +pub const Crc8Bluetooth = Generic(u8, .{ .polynomial = 0xa7, .initial = 0x00, .reflect_input = true, @@ -149,7 +257,7 @@ pub const Crc8Bluetooth = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc8Cdma2000 = Crc(u8, .{ +pub const Crc8Cdma2000 = Generic(u8, .{ .polynomial = 0x9b, .initial = 0xff, .reflect_input = false, @@ -157,7 +265,7 @@ pub const Crc8Cdma2000 = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc8Darc = Crc(u8, .{ +pub const Crc8Darc = Generic(u8, .{ .polynomial = 0x39, .initial = 0x00, .reflect_input = true, @@ -165,7 +273,7 @@ pub const Crc8Darc = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc8DvbS2 = Crc(u8, .{ +pub const Crc8DvbS2 = Generic(u8, .{ .polynomial = 0xd5, .initial = 0x00, .reflect_input = false, @@ -173,7 +281,7 @@ pub const Crc8DvbS2 = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc8GsmA = Crc(u8, .{ +pub const Crc8GsmA = Generic(u8, .{ .polynomial = 0x1d, .initial = 0x00, .reflect_input = false, @@ -181,7 +289,7 @@ pub const Crc8GsmA = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc8GsmB = Crc(u8, .{ +pub const Crc8GsmB = Generic(u8, .{ .polynomial = 0x49, .initial = 0x00, .reflect_input = false, @@ -189,7 +297,7 @@ pub const Crc8GsmB = Crc(u8, .{ .xor_output = 0xff, }); -pub const Crc8Hitag = Crc(u8, .{ +pub const Crc8Hitag = Generic(u8, .{ .polynomial = 0x1d, .initial = 0xff, .reflect_input = false, @@ -197,7 +305,7 @@ pub const Crc8Hitag = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc8I4321 = Crc(u8, .{ +pub const Crc8I4321 = Generic(u8, .{ .polynomial = 0x07, .initial = 0x00, .reflect_input = false, @@ -205,7 +313,7 @@ pub const Crc8I4321 = Crc(u8, .{ .xor_output = 0x55, }); -pub const Crc8ICode = Crc(u8, .{ +pub const Crc8ICode = Generic(u8, .{ .polynomial = 0x1d, .initial = 0xfd, .reflect_input = false, @@ -213,7 +321,7 @@ pub const Crc8ICode = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc8Lte = Crc(u8, .{ +pub const Crc8Lte = Generic(u8, .{ .polynomial = 0x9b, .initial = 0x00, .reflect_input = false, @@ -221,7 +329,7 @@ pub const Crc8Lte = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc8MaximDow = Crc(u8, .{ +pub const Crc8MaximDow = Generic(u8, .{ .polynomial = 0x31, .initial = 0x00, .reflect_input = true, @@ -229,7 +337,7 @@ pub const Crc8MaximDow = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc8MifareMad = Crc(u8, .{ +pub const Crc8MifareMad = Generic(u8, .{ .polynomial = 0x1d, .initial = 0xc7, .reflect_input = false, @@ -237,7 +345,7 @@ pub const Crc8MifareMad = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc8Nrsc5 = Crc(u8, .{ +pub const Crc8Nrsc5 = Generic(u8, .{ .polynomial = 0x31, .initial = 0xff, .reflect_input = false, @@ -245,7 +353,7 @@ pub const Crc8Nrsc5 = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc8Opensafety = Crc(u8, .{ +pub const Crc8Opensafety = Generic(u8, .{ .polynomial = 0x2f, .initial = 0x00, .reflect_input = false, @@ -253,7 +361,7 @@ pub const Crc8Opensafety = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc8Rohc = Crc(u8, .{ +pub const Crc8Rohc = Generic(u8, .{ .polynomial = 0x07, .initial = 0xff, .reflect_input = true, @@ -261,7 +369,7 @@ pub const Crc8Rohc = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc8SaeJ1850 = Crc(u8, .{ +pub const Crc8SaeJ1850 = Generic(u8, .{ .polynomial = 0x1d, .initial = 0xff, .reflect_input = false, @@ -269,7 +377,7 @@ pub const Crc8SaeJ1850 = Crc(u8, .{ .xor_output = 0xff, }); -pub const Crc8Smbus = Crc(u8, .{ +pub const Crc8Smbus = Generic(u8, .{ .polynomial = 0x07, .initial = 0x00, .reflect_input = false, @@ -277,7 +385,7 @@ pub const Crc8Smbus = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc8Tech3250 = Crc(u8, .{ +pub const Crc8Tech3250 = Generic(u8, .{ .polynomial = 0x1d, .initial = 0xff, .reflect_input = true, @@ -285,7 +393,7 @@ pub const Crc8Tech3250 = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc8Wcdma = Crc(u8, .{ +pub const Crc8Wcdma = Generic(u8, .{ .polynomial = 0x9b, .initial = 0x00, .reflect_input = true, @@ -293,7 +401,7 @@ pub const Crc8Wcdma = Crc(u8, .{ .xor_output = 0x00, }); -pub const Crc10Atm = Crc(u10, .{ +pub const Crc10Atm = Generic(u10, .{ .polynomial = 0x233, .initial = 0x000, .reflect_input = false, @@ -301,7 +409,7 @@ pub const Crc10Atm = Crc(u10, .{ .xor_output = 0x000, }); -pub const Crc10Cdma2000 = Crc(u10, .{ +pub const Crc10Cdma2000 = Generic(u10, .{ .polynomial = 0x3d9, .initial = 0x3ff, .reflect_input = false, @@ -309,7 +417,7 @@ pub const Crc10Cdma2000 = Crc(u10, .{ .xor_output = 0x000, }); -pub const Crc10Gsm = Crc(u10, .{ +pub const Crc10Gsm = Generic(u10, .{ .polynomial = 0x175, .initial = 0x000, .reflect_input = false, @@ -317,7 +425,7 @@ pub const Crc10Gsm = Crc(u10, .{ .xor_output = 0x3ff, }); -pub const Crc11Flexray = Crc(u11, .{ +pub const Crc11Flexray = Generic(u11, .{ .polynomial = 0x385, .initial = 0x01a, .reflect_input = false, @@ -325,7 +433,7 @@ pub const Crc11Flexray = Crc(u11, .{ .xor_output = 0x000, }); -pub const Crc11Umts = Crc(u11, .{ +pub const Crc11Umts = Generic(u11, .{ .polynomial = 0x307, .initial = 0x000, .reflect_input = false, @@ -333,7 +441,7 @@ pub const Crc11Umts = Crc(u11, .{ .xor_output = 0x000, }); -pub const Crc12Cdma2000 = Crc(u12, .{ +pub const Crc12Cdma2000 = Generic(u12, .{ .polynomial = 0xf13, .initial = 0xfff, .reflect_input = false, @@ -341,7 +449,7 @@ pub const Crc12Cdma2000 = Crc(u12, .{ .xor_output = 0x000, }); -pub const Crc12Dect = Crc(u12, .{ +pub const Crc12Dect = Generic(u12, .{ .polynomial = 0x80f, .initial = 0x000, .reflect_input = false, @@ -349,7 +457,7 @@ pub const Crc12Dect = Crc(u12, .{ .xor_output = 0x000, }); -pub const Crc12Gsm = Crc(u12, .{ +pub const Crc12Gsm = Generic(u12, .{ .polynomial = 0xd31, .initial = 0x000, .reflect_input = false, @@ -357,7 +465,7 @@ pub const Crc12Gsm = Crc(u12, .{ .xor_output = 0xfff, }); -pub const Crc12Umts = Crc(u12, .{ +pub const Crc12Umts = Generic(u12, .{ .polynomial = 0x80f, .initial = 0x000, .reflect_input = false, @@ -365,7 +473,7 @@ pub const Crc12Umts = Crc(u12, .{ .xor_output = 0x000, }); -pub const Crc13Bbc = Crc(u13, .{ +pub const Crc13Bbc = Generic(u13, .{ .polynomial = 0x1cf5, .initial = 0x0000, .reflect_input = false, @@ -373,7 +481,7 @@ pub const Crc13Bbc = Crc(u13, .{ .xor_output = 0x0000, }); -pub const Crc14Darc = Crc(u14, .{ +pub const Crc14Darc = Generic(u14, .{ .polynomial = 0x0805, .initial = 0x0000, .reflect_input = true, @@ -381,7 +489,7 @@ pub const Crc14Darc = Crc(u14, .{ .xor_output = 0x0000, }); -pub const Crc14Gsm = Crc(u14, .{ +pub const Crc14Gsm = Generic(u14, .{ .polynomial = 0x202d, .initial = 0x0000, .reflect_input = false, @@ -389,7 +497,7 @@ pub const Crc14Gsm = Crc(u14, .{ .xor_output = 0x3fff, }); -pub const Crc15Can = Crc(u15, .{ +pub const Crc15Can = Generic(u15, .{ .polynomial = 0x4599, .initial = 0x0000, .reflect_input = false, @@ -397,7 +505,7 @@ pub const Crc15Can = Crc(u15, .{ .xor_output = 0x0000, }); -pub const Crc15Mpt1327 = Crc(u15, .{ +pub const Crc15Mpt1327 = Generic(u15, .{ .polynomial = 0x6815, .initial = 0x0000, .reflect_input = false, @@ -405,7 +513,7 @@ pub const Crc15Mpt1327 = Crc(u15, .{ .xor_output = 0x0001, }); -pub const Crc16Arc = Crc(u16, .{ +pub const Crc16Arc = Generic(u16, .{ .polynomial = 0x8005, .initial = 0x0000, .reflect_input = true, @@ -413,7 +521,7 @@ pub const Crc16Arc = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16Cdma2000 = Crc(u16, .{ +pub const Crc16Cdma2000 = Generic(u16, .{ .polynomial = 0xc867, .initial = 0xffff, .reflect_input = false, @@ -421,7 +529,7 @@ pub const Crc16Cdma2000 = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16Cms = Crc(u16, .{ +pub const Crc16Cms = Generic(u16, .{ .polynomial = 0x8005, .initial = 0xffff, .reflect_input = false, @@ -429,7 +537,7 @@ pub const Crc16Cms = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16Dds110 = Crc(u16, .{ +pub const Crc16Dds110 = Generic(u16, .{ .polynomial = 0x8005, .initial = 0x800d, .reflect_input = false, @@ -437,7 +545,7 @@ pub const Crc16Dds110 = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16DectR = Crc(u16, .{ +pub const Crc16DectR = Generic(u16, .{ .polynomial = 0x0589, .initial = 0x0000, .reflect_input = false, @@ -445,7 +553,7 @@ pub const Crc16DectR = Crc(u16, .{ .xor_output = 0x0001, }); -pub const Crc16DectX = Crc(u16, .{ +pub const Crc16DectX = Generic(u16, .{ .polynomial = 0x0589, .initial = 0x0000, .reflect_input = false, @@ -453,7 +561,7 @@ pub const Crc16DectX = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16Dnp = Crc(u16, .{ +pub const Crc16Dnp = Generic(u16, .{ .polynomial = 0x3d65, .initial = 0x0000, .reflect_input = true, @@ -461,7 +569,7 @@ pub const Crc16Dnp = Crc(u16, .{ .xor_output = 0xffff, }); -pub const Crc16En13757 = Crc(u16, .{ +pub const Crc16En13757 = Generic(u16, .{ .polynomial = 0x3d65, .initial = 0x0000, .reflect_input = false, @@ -469,7 +577,7 @@ pub const Crc16En13757 = Crc(u16, .{ .xor_output = 0xffff, }); -pub const Crc16Genibus = Crc(u16, .{ +pub const Crc16Genibus = Generic(u16, .{ .polynomial = 0x1021, .initial = 0xffff, .reflect_input = false, @@ -477,7 +585,7 @@ pub const Crc16Genibus = Crc(u16, .{ .xor_output = 0xffff, }); -pub const Crc16Gsm = Crc(u16, .{ +pub const Crc16Gsm = Generic(u16, .{ .polynomial = 0x1021, .initial = 0x0000, .reflect_input = false, @@ -485,7 +593,7 @@ pub const Crc16Gsm = Crc(u16, .{ .xor_output = 0xffff, }); -pub const Crc16Ibm3740 = Crc(u16, .{ +pub const Crc16Ibm3740 = Generic(u16, .{ .polynomial = 0x1021, .initial = 0xffff, .reflect_input = false, @@ -493,7 +601,7 @@ pub const Crc16Ibm3740 = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16IbmSdlc = Crc(u16, .{ +pub const Crc16IbmSdlc = Generic(u16, .{ .polynomial = 0x1021, .initial = 0xffff, .reflect_input = true, @@ -501,7 +609,7 @@ pub const Crc16IbmSdlc = Crc(u16, .{ .xor_output = 0xffff, }); -pub const Crc16IsoIec144433A = Crc(u16, .{ +pub const Crc16IsoIec144433A = Generic(u16, .{ .polynomial = 0x1021, .initial = 0xc6c6, .reflect_input = true, @@ -509,7 +617,7 @@ pub const Crc16IsoIec144433A = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16Kermit = Crc(u16, .{ +pub const Crc16Kermit = Generic(u16, .{ .polynomial = 0x1021, .initial = 0x0000, .reflect_input = true, @@ -517,7 +625,7 @@ pub const Crc16Kermit = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16Lj1200 = Crc(u16, .{ +pub const Crc16Lj1200 = Generic(u16, .{ .polynomial = 0x6f63, .initial = 0x0000, .reflect_input = false, @@ -525,7 +633,7 @@ pub const Crc16Lj1200 = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16M17 = Crc(u16, .{ +pub const Crc16M17 = Generic(u16, .{ .polynomial = 0x5935, .initial = 0xffff, .reflect_input = false, @@ -533,7 +641,7 @@ pub const Crc16M17 = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16MaximDow = Crc(u16, .{ +pub const Crc16MaximDow = Generic(u16, .{ .polynomial = 0x8005, .initial = 0x0000, .reflect_input = true, @@ -541,7 +649,7 @@ pub const Crc16MaximDow = Crc(u16, .{ .xor_output = 0xffff, }); -pub const Crc16Mcrf4xx = Crc(u16, .{ +pub const Crc16Mcrf4xx = Generic(u16, .{ .polynomial = 0x1021, .initial = 0xffff, .reflect_input = true, @@ -549,7 +657,7 @@ pub const Crc16Mcrf4xx = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16Modbus = Crc(u16, .{ +pub const Crc16Modbus = Generic(u16, .{ .polynomial = 0x8005, .initial = 0xffff, .reflect_input = true, @@ -557,7 +665,7 @@ pub const Crc16Modbus = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16Nrsc5 = Crc(u16, .{ +pub const Crc16Nrsc5 = Generic(u16, .{ .polynomial = 0x080b, .initial = 0xffff, .reflect_input = true, @@ -565,7 +673,7 @@ pub const Crc16Nrsc5 = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16OpensafetyA = Crc(u16, .{ +pub const Crc16OpensafetyA = Generic(u16, .{ .polynomial = 0x5935, .initial = 0x0000, .reflect_input = false, @@ -573,7 +681,7 @@ pub const Crc16OpensafetyA = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16OpensafetyB = Crc(u16, .{ +pub const Crc16OpensafetyB = Generic(u16, .{ .polynomial = 0x755b, .initial = 0x0000, .reflect_input = false, @@ -581,7 +689,7 @@ pub const Crc16OpensafetyB = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16Profibus = Crc(u16, .{ +pub const Crc16Profibus = Generic(u16, .{ .polynomial = 0x1dcf, .initial = 0xffff, .reflect_input = false, @@ -589,7 +697,7 @@ pub const Crc16Profibus = Crc(u16, .{ .xor_output = 0xffff, }); -pub const Crc16Riello = Crc(u16, .{ +pub const Crc16Riello = Generic(u16, .{ .polynomial = 0x1021, .initial = 0xb2aa, .reflect_input = true, @@ -597,7 +705,7 @@ pub const Crc16Riello = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16SpiFujitsu = Crc(u16, .{ +pub const Crc16SpiFujitsu = Generic(u16, .{ .polynomial = 0x1021, .initial = 0x1d0f, .reflect_input = false, @@ -605,7 +713,7 @@ pub const Crc16SpiFujitsu = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16T10Dif = Crc(u16, .{ +pub const Crc16T10Dif = Generic(u16, .{ .polynomial = 0x8bb7, .initial = 0x0000, .reflect_input = false, @@ -613,7 +721,7 @@ pub const Crc16T10Dif = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16Teledisk = Crc(u16, .{ +pub const Crc16Teledisk = Generic(u16, .{ .polynomial = 0xa097, .initial = 0x0000, .reflect_input = false, @@ -621,7 +729,7 @@ pub const Crc16Teledisk = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16Tms37157 = Crc(u16, .{ +pub const Crc16Tms37157 = Generic(u16, .{ .polynomial = 0x1021, .initial = 0x89ec, .reflect_input = true, @@ -629,7 +737,7 @@ pub const Crc16Tms37157 = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16Umts = Crc(u16, .{ +pub const Crc16Umts = Generic(u16, .{ .polynomial = 0x8005, .initial = 0x0000, .reflect_input = false, @@ -637,7 +745,7 @@ pub const Crc16Umts = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc16Usb = Crc(u16, .{ +pub const Crc16Usb = Generic(u16, .{ .polynomial = 0x8005, .initial = 0xffff, .reflect_input = true, @@ -645,7 +753,7 @@ pub const Crc16Usb = Crc(u16, .{ .xor_output = 0xffff, }); -pub const Crc16Xmodem = Crc(u16, .{ +pub const Crc16Xmodem = Generic(u16, .{ .polynomial = 0x1021, .initial = 0x0000, .reflect_input = false, @@ -653,7 +761,7 @@ pub const Crc16Xmodem = Crc(u16, .{ .xor_output = 0x0000, }); -pub const Crc17CanFd = Crc(u17, .{ +pub const Crc17CanFd = Generic(u17, .{ .polynomial = 0x1685b, .initial = 0x00000, .reflect_input = false, @@ -661,7 +769,7 @@ pub const Crc17CanFd = Crc(u17, .{ .xor_output = 0x00000, }); -pub const Crc21CanFd = Crc(u21, .{ +pub const Crc21CanFd = Generic(u21, .{ .polynomial = 0x102899, .initial = 0x000000, .reflect_input = false, @@ -669,7 +777,7 @@ pub const Crc21CanFd = Crc(u21, .{ .xor_output = 0x000000, }); -pub const Crc24Ble = Crc(u24, .{ +pub const Crc24Ble = Generic(u24, .{ .polynomial = 0x00065b, .initial = 0x555555, .reflect_input = true, @@ -677,7 +785,7 @@ pub const Crc24Ble = Crc(u24, .{ .xor_output = 0x000000, }); -pub const Crc24FlexrayA = Crc(u24, .{ +pub const Crc24FlexrayA = Generic(u24, .{ .polynomial = 0x5d6dcb, .initial = 0xfedcba, .reflect_input = false, @@ -685,7 +793,7 @@ pub const Crc24FlexrayA = Crc(u24, .{ .xor_output = 0x000000, }); -pub const Crc24FlexrayB = Crc(u24, .{ +pub const Crc24FlexrayB = Generic(u24, .{ .polynomial = 0x5d6dcb, .initial = 0xabcdef, .reflect_input = false, @@ -693,7 +801,7 @@ pub const Crc24FlexrayB = Crc(u24, .{ .xor_output = 0x000000, }); -pub const Crc24Interlaken = Crc(u24, .{ +pub const Crc24Interlaken = Generic(u24, .{ .polynomial = 0x328b63, .initial = 0xffffff, .reflect_input = false, @@ -701,7 +809,7 @@ pub const Crc24Interlaken = Crc(u24, .{ .xor_output = 0xffffff, }); -pub const Crc24LteA = Crc(u24, .{ +pub const Crc24LteA = Generic(u24, .{ .polynomial = 0x864cfb, .initial = 0x000000, .reflect_input = false, @@ -709,7 +817,7 @@ pub const Crc24LteA = Crc(u24, .{ .xor_output = 0x000000, }); -pub const Crc24LteB = Crc(u24, .{ +pub const Crc24LteB = Generic(u24, .{ .polynomial = 0x800063, .initial = 0x000000, .reflect_input = false, @@ -717,7 +825,7 @@ pub const Crc24LteB = Crc(u24, .{ .xor_output = 0x000000, }); -pub const Crc24Openpgp = Crc(u24, .{ +pub const Crc24Openpgp = Generic(u24, .{ .polynomial = 0x864cfb, .initial = 0xb704ce, .reflect_input = false, @@ -725,7 +833,7 @@ pub const Crc24Openpgp = Crc(u24, .{ .xor_output = 0x000000, }); -pub const Crc24Os9 = Crc(u24, .{ +pub const Crc24Os9 = Generic(u24, .{ .polynomial = 0x800063, .initial = 0xffffff, .reflect_input = false, @@ -733,7 +841,7 @@ pub const Crc24Os9 = Crc(u24, .{ .xor_output = 0xffffff, }); -pub const Crc30Cdma = Crc(u30, .{ +pub const Crc30Cdma = Generic(u30, .{ .polynomial = 0x2030b9c7, .initial = 0x3fffffff, .reflect_input = false, @@ -741,7 +849,7 @@ pub const Crc30Cdma = Crc(u30, .{ .xor_output = 0x3fffffff, }); -pub const Crc31Philips = Crc(u31, .{ +pub const Crc31Philips = Generic(u31, .{ .polynomial = 0x04c11db7, .initial = 0x7fffffff, .reflect_input = false, @@ -749,7 +857,7 @@ pub const Crc31Philips = Crc(u31, .{ .xor_output = 0x7fffffff, }); -pub const Crc32Aixm = Crc(u32, .{ +pub const Crc32Aixm = Generic(u32, .{ .polynomial = 0x814141ab, .initial = 0x00000000, .reflect_input = false, @@ -757,7 +865,7 @@ pub const Crc32Aixm = Crc(u32, .{ .xor_output = 0x00000000, }); -pub const Crc32Autosar = Crc(u32, .{ +pub const Crc32Autosar = Generic(u32, .{ .polynomial = 0xf4acfb13, .initial = 0xffffffff, .reflect_input = true, @@ -765,7 +873,7 @@ pub const Crc32Autosar = Crc(u32, .{ .xor_output = 0xffffffff, }); -pub const Crc32Base91D = Crc(u32, .{ +pub const Crc32Base91D = Generic(u32, .{ .polynomial = 0xa833982b, .initial = 0xffffffff, .reflect_input = true, @@ -773,7 +881,7 @@ pub const Crc32Base91D = Crc(u32, .{ .xor_output = 0xffffffff, }); -pub const Crc32Bzip2 = Crc(u32, .{ +pub const Crc32Bzip2 = Generic(u32, .{ .polynomial = 0x04c11db7, .initial = 0xffffffff, .reflect_input = false, @@ -781,7 +889,7 @@ pub const Crc32Bzip2 = Crc(u32, .{ .xor_output = 0xffffffff, }); -pub const Crc32CdRomEdc = Crc(u32, .{ +pub const Crc32CdRomEdc = Generic(u32, .{ .polynomial = 0x8001801b, .initial = 0x00000000, .reflect_input = true, @@ -789,7 +897,7 @@ pub const Crc32CdRomEdc = Crc(u32, .{ .xor_output = 0x00000000, }); -pub const Crc32Cksum = Crc(u32, .{ +pub const Crc32Cksum = Generic(u32, .{ .polynomial = 0x04c11db7, .initial = 0x00000000, .reflect_input = false, @@ -797,7 +905,7 @@ pub const Crc32Cksum = Crc(u32, .{ .xor_output = 0xffffffff, }); -pub const Crc32Iscsi = Crc(u32, .{ +pub const Crc32Iscsi = Generic(u32, .{ .polynomial = 0x1edc6f41, .initial = 0xffffffff, .reflect_input = true, @@ -805,7 +913,7 @@ pub const Crc32Iscsi = Crc(u32, .{ .xor_output = 0xffffffff, }); -pub const Crc32IsoHdlc = Crc(u32, .{ +pub const Crc32IsoHdlc = Generic(u32, .{ .polynomial = 0x04c11db7, .initial = 0xffffffff, .reflect_input = true, @@ -813,7 +921,7 @@ pub const Crc32IsoHdlc = Crc(u32, .{ .xor_output = 0xffffffff, }); -pub const Crc32Jamcrc = Crc(u32, .{ +pub const Crc32Jamcrc = Generic(u32, .{ .polynomial = 0x04c11db7, .initial = 0xffffffff, .reflect_input = true, @@ -821,7 +929,7 @@ pub const Crc32Jamcrc = Crc(u32, .{ .xor_output = 0x00000000, }); -pub const Crc32Koopman = Crc(u32, .{ +pub const Crc32Koopman = Generic(u32, .{ .polynomial = 0x741b8cd7, .initial = 0xffffffff, .reflect_input = true, @@ -829,7 +937,7 @@ pub const Crc32Koopman = Crc(u32, .{ .xor_output = 0xffffffff, }); -pub const Crc32Mef = Crc(u32, .{ +pub const Crc32Mef = Generic(u32, .{ .polynomial = 0x741b8cd7, .initial = 0xffffffff, .reflect_input = true, @@ -837,7 +945,7 @@ pub const Crc32Mef = Crc(u32, .{ .xor_output = 0x00000000, }); -pub const Crc32Mpeg2 = Crc(u32, .{ +pub const Crc32Mpeg2 = Generic(u32, .{ .polynomial = 0x04c11db7, .initial = 0xffffffff, .reflect_input = false, @@ -845,7 +953,7 @@ pub const Crc32Mpeg2 = Crc(u32, .{ .xor_output = 0x00000000, }); -pub const Crc32Xfer = Crc(u32, .{ +pub const Crc32Xfer = Generic(u32, .{ .polynomial = 0x000000af, .initial = 0x00000000, .reflect_input = false, @@ -853,7 +961,7 @@ pub const Crc32Xfer = Crc(u32, .{ .xor_output = 0x00000000, }); -pub const Crc40Gsm = Crc(u40, .{ +pub const Crc40Gsm = Generic(u40, .{ .polynomial = 0x0004820009, .initial = 0x0000000000, .reflect_input = false, @@ -861,7 +969,7 @@ pub const Crc40Gsm = Crc(u40, .{ .xor_output = 0xffffffffff, }); -pub const Crc64Ecma182 = Crc(u64, .{ +pub const Crc64Ecma182 = Generic(u64, .{ .polynomial = 0x42f0e1eba9ea3693, .initial = 0x0000000000000000, .reflect_input = false, @@ -869,7 +977,7 @@ pub const Crc64Ecma182 = Crc(u64, .{ .xor_output = 0x0000000000000000, }); -pub const Crc64GoIso = Crc(u64, .{ +pub const Crc64GoIso = Generic(u64, .{ .polynomial = 0x000000000000001b, .initial = 0xffffffffffffffff, .reflect_input = true, @@ -877,7 +985,7 @@ pub const Crc64GoIso = Crc(u64, .{ .xor_output = 0xffffffffffffffff, }); -pub const Crc64Ms = Crc(u64, .{ +pub const Crc64Ms = Generic(u64, .{ .polynomial = 0x259c84cba6426349, .initial = 0xffffffffffffffff, .reflect_input = true, @@ -885,7 +993,7 @@ pub const Crc64Ms = Crc(u64, .{ .xor_output = 0x0000000000000000, }); -pub const Crc64Redis = Crc(u64, .{ +pub const Crc64Redis = Generic(u64, .{ .polynomial = 0xad93d23594c935a9, .initial = 0x0000000000000000, .reflect_input = true, @@ -893,7 +1001,7 @@ pub const Crc64Redis = Crc(u64, .{ .xor_output = 0x0000000000000000, }); -pub const Crc64We = Crc(u64, .{ +pub const Crc64We = Generic(u64, .{ .polynomial = 0x42f0e1eba9ea3693, .initial = 0xffffffffffffffff, .reflect_input = false, @@ -901,7 +1009,7 @@ pub const Crc64We = Crc(u64, .{ .xor_output = 0xffffffffffffffff, }); -pub const Crc64Xz = Crc(u64, .{ +pub const Crc64Xz = Generic(u64, .{ .polynomial = 0x42f0e1eba9ea3693, .initial = 0xffffffffffffffff, .reflect_input = true, @@ -909,10 +1017,14 @@ pub const Crc64Xz = Crc(u64, .{ .xor_output = 0xffffffffffffffff, }); -pub const Crc82Darc = Crc(u82, .{ +pub const Crc82Darc = Generic(u82, .{ .polynomial = 0x0308c0111011401440411, .initial = 0x000000000000000000000, .reflect_input = true, .reflect_output = true, .xor_output = 0x000000000000000000000, }); + +test { + _ = @import("crc/test.zig"); +} diff --git a/lib/std/hash/crc/impl.zig b/lib/std/hash/crc/impl.zig deleted file mode 100644 index 253a7b0a62..0000000000 --- a/lib/std/hash/crc/impl.zig +++ /dev/null @@ -1,112 +0,0 @@ -// There is a generic CRC implementation "Crc()" which can be parameterized via -// the Algorithm struct for a plethora of uses. -// -// The primary interface for all of the standard CRC algorithms is the -// generated file "crc.zig", which uses the implementation code here to define -// many standard CRCs. - -const std = @import("std"); - -pub fn Algorithm(comptime W: type) type { - return struct { - polynomial: W, - initial: W, - reflect_input: bool, - reflect_output: bool, - xor_output: W, - }; -} - -pub fn Crc(comptime W: type, comptime algorithm: Algorithm(W)) type { - return struct { - const Self = @This(); - const I = if (@bitSizeOf(W) < 8) u8 else W; - const lookup_table = blk: { - @setEvalBranchQuota(2500); - - const poly = if (algorithm.reflect_input) - @bitReverse(@as(I, algorithm.polynomial)) >> (@bitSizeOf(I) - @bitSizeOf(W)) - else - @as(I, algorithm.polynomial) << (@bitSizeOf(I) - @bitSizeOf(W)); - - var table: [256]I = undefined; - for (&table, 0..) |*e, i| { - var crc: I = i; - if (algorithm.reflect_input) { - var j: usize = 0; - while (j < 8) : (j += 1) { - crc = (crc >> 1) ^ ((crc & 1) * poly); - } - } else { - crc <<= @bitSizeOf(I) - 8; - var j: usize = 0; - while (j < 8) : (j += 1) { - crc = (crc << 1) ^ (((crc >> (@bitSizeOf(I) - 1)) & 1) * poly); - } - } - e.* = crc; - } - break :blk table; - }; - - crc: I, - - pub fn init() Self { - const initial = if (algorithm.reflect_input) - @bitReverse(@as(I, algorithm.initial)) >> (@bitSizeOf(I) - @bitSizeOf(W)) - else - @as(I, algorithm.initial) << (@bitSizeOf(I) - @bitSizeOf(W)); - return Self{ .crc = initial }; - } - - inline fn tableEntry(index: I) I { - return lookup_table[@as(u8, @intCast(index & 0xFF))]; - } - - pub fn update(self: *Self, bytes: []const u8) void { - var i: usize = 0; - if (@bitSizeOf(I) <= 8) { - while (i < bytes.len) : (i += 1) { - self.crc = tableEntry(self.crc ^ bytes[i]); - } - } else if (algorithm.reflect_input) { - while (i < bytes.len) : (i += 1) { - const table_index = self.crc ^ bytes[i]; - self.crc = tableEntry(table_index) ^ (self.crc >> 8); - } - } else { - while (i < bytes.len) : (i += 1) { - const table_index = (self.crc >> (@bitSizeOf(I) - 8)) ^ bytes[i]; - self.crc = tableEntry(table_index) ^ (self.crc << 8); - } - } - } - - pub fn final(self: Self) W { - var c = self.crc; - if (algorithm.reflect_input != algorithm.reflect_output) { - c = @bitReverse(c); - } - if (!algorithm.reflect_output) { - c >>= @bitSizeOf(I) - @bitSizeOf(W); - } - return @as(W, @intCast(c ^ algorithm.xor_output)); - } - - pub fn hash(bytes: []const u8) W { - var c = Self.init(); - c.update(bytes); - return c.final(); - } - }; -} - -pub const Polynomial = enum(u32) { - IEEE = @compileError("use Crc with algorithm .Crc32IsoHdlc"), - Castagnoli = @compileError("use Crc with algorithm .Crc32Iscsi"), - Koopman = @compileError("use Crc with algorithm .Crc32Koopman"), - _, -}; - -pub const Crc32WithPoly = @compileError("use Crc instead"); -pub const Crc32SmallWithPoly = @compileError("use Crc instead"); diff --git a/lib/std/io/BufferedReader.zig b/lib/std/io/BufferedReader.zig index 52a3d68696..6d0870ddf4 100644 --- a/lib/std/io/BufferedReader.zig +++ b/lib/std/io/BufferedReader.zig @@ -35,6 +35,10 @@ pub fn bufferContents(br: *BufferedReader) []u8 { return br.buffer[br.seek..br.end]; } +pub fn bufferedLen(br: *const BufferedReader) usize { + return br.end - br.seek; +} + /// Although `BufferedReader` can easily satisfy the `Reader` interface, it's /// generally more practical to pass a `BufferedReader` instance itself around, /// since it will result in fewer calls across vtable boundaries. @@ -49,6 +53,10 @@ pub fn reader(br: *BufferedReader) Reader { }; } +pub fn hashed(br: *BufferedReader, hasher: anytype) Reader.Hashed(@TypeOf(hasher)) { + return .{ .in = br, .hasher = hasher }; +} + /// Equivalent semantics to `std.io.Reader.VTable.readVec`. pub fn readVec(br: *BufferedReader, data: []const []u8) Reader.Error!usize { return readVecLimit(br, data, .unlimited); @@ -401,6 +409,30 @@ pub fn readSliceShort(br: *BufferedReader, buffer: []u8) Reader.ShortError!usize } } +/// Fill `buffer` with the next `buffer.len` bytes from the stream, advancing +/// the seek position. +/// +/// Invalidates previously returned values from `peek`. +/// +/// If the provided buffer cannot be filled completely, `error.EndOfStream` is +/// returned instead. +/// +/// The function is inline to avoid the dead code in case `endian` is +/// comptime-known and matches host endianness. +/// +/// See also: +/// * `readSlice` +/// * `readSliceEndianAlloc` +pub inline fn readSliceEndian( + br: *BufferedReader, + comptime Elem: type, + buffer: []Elem, + endian: std.builtin.Endian, +) Reader.Error!void { + try readSlice(br, @ptrCast(buffer)); + if (native_endian != endian) for (buffer) |*elem| std.mem.byteSwapAllFields(Elem, elem); +} + pub const ReadAllocError = Reader.Error || Allocator.Error; /// The function is inline to avoid the dead code in case `endian` is @@ -408,14 +440,14 @@ pub const ReadAllocError = Reader.Error || Allocator.Error; pub inline fn readSliceEndianAlloc( br: *BufferedReader, allocator: Allocator, - Elem: type, + comptime Elem: type, len: usize, endian: std.builtin.Endian, ) ReadAllocError![]Elem { const dest = try allocator.alloc(Elem, len); errdefer allocator.free(dest); try readSlice(br, @ptrCast(dest)); - if (native_endian != endian) std.mem.byteSwapAllFields(Elem, dest); + if (native_endian != endian) for (dest) |*elem| std.mem.byteSwapAllFields(Elem, elem); return dest; } diff --git a/lib/std/io/BufferedWriter.zig b/lib/std/io/BufferedWriter.zig index 30c6ec957d..dc05296fe2 100644 --- a/lib/std/io/BufferedWriter.zig +++ b/lib/std/io/BufferedWriter.zig @@ -58,6 +58,10 @@ pub fn initFixed(bw: *BufferedWriter, buffer: []u8) void { }; } +pub fn hashed(bw: *BufferedWriter, hasher: anytype) Writer.Hashed(@TypeOf(hasher)) { + return .{ .out = bw, .hasher = hasher }; +} + /// This function is available when using `initFixed`. pub fn getWritten(bw: *const BufferedWriter) []u8 { assert(bw.unbuffered_writer.vtable == &fixed_vtable); @@ -157,7 +161,7 @@ pub fn advance(bw: *BufferedWriter, n: usize) void { } /// The `data` parameter is mutable because this function needs to mutate the -/// fields in order to handle partial writes from `Writer.VTable.writeVec`. +/// fields in order to handle partial writes from `Writer.VTable.writeSplat`. pub fn writeVecAll(bw: *BufferedWriter, data: [][]const u8) Writer.Error!void { var index: usize = 0; var truncate: usize = 0; @@ -175,6 +179,36 @@ pub fn writeVecAll(bw: *BufferedWriter, data: [][]const u8) Writer.Error!void { } } +/// The `data` parameter is mutable because this function needs to mutate the +/// fields in order to handle partial writes from `Writer.VTable.writeSplat`. +pub fn writeSplatAll(bw: *BufferedWriter, data: [][]const u8, splat: usize) Writer.Error!void { + var index: usize = 0; + var truncate: usize = 0; + var remaining_splat = splat; + while (index + 1 < data.len) { + { + const untruncated = data[index]; + data[index] = untruncated[truncate..]; + defer data[index] = untruncated; + truncate += try bw.writeSplat(data[index..], remaining_splat); + } + while (truncate >= data[index].len) { + if (index + 1 < data.len) { + truncate -= data[index].len; + index += 1; + } else { + const last = data[data.len - 1]; + remaining_splat -= @divExact(truncate, last.len); + while (remaining_splat > 0) { + const n = try bw.writeSplat(data[data.len - 1 ..][0..1], remaining_splat); + remaining_splat -= @divExact(n, last.len); + } + return; + } + } + } +} + /// If the number of bytes to write based on `data` and `splat` fits inside /// `unusedCapacitySlice`, this function is guaranteed to not fail, not call /// into the underlying writer, and return the full number of bytes. @@ -443,7 +477,7 @@ pub fn splatBytesAll(bw: *BufferedWriter, bytes: []const u8, splat: usize) Write /// Writes the same slice many times, allowing short writes. /// -/// Does maximum of one underlying `Writer.VTable.writeVec`. +/// Does maximum of one underlying `Writer.VTable.writeSplat`. pub fn splatBytes(bw: *BufferedWriter, bytes: []const u8, n: usize) Writer.Error!usize { return passthruWriteSplat(bw, &.{bytes}, n); } @@ -621,7 +655,7 @@ pub const WriteFileOptions = struct { /// size here will save one syscall. limit: Writer.Limit = .unlimited, /// Headers and trailers must be passed together so that in case `len` is - /// zero, they can be forwarded directly to `Writer.VTable.writeVec`. + /// zero, they can be forwarded directly to `Writer.VTable.writeSplat`. /// /// The parameter is mutable because this function needs to mutate the /// fields in order to handle partial writes from `Writer.VTable.writeFile`. diff --git a/lib/std/io/Reader.zig b/lib/std/io/Reader.zig index 343de016ea..b2763ddbfc 100644 --- a/lib/std/io/Reader.zig +++ b/lib/std/io/Reader.zig @@ -331,3 +331,71 @@ test "readAlloc when the backing reader provides one byte at a time" { defer std.testing.allocator.free(res); try std.testing.expectEqualStrings(str, res); } + +/// Provides a `Reader` implementation by passing data from an underlying +/// reader through `Hasher.update`. +/// +/// The underlying reader is best unbuffered. +/// +/// This implementation makes suboptimal buffering decisions due to being +/// generic. A better solution will involve creating a reader for each hash +/// function, where the discard buffer can be tailored to the hash +/// implementation details. +pub fn Hashed(comptime Hasher: type) type { + return struct { + in: *BufferedReader, + hasher: Hasher, + + pub fn readable(this: *@This(), buffer: []u8) BufferedReader { + return .{ + .unbuffered_reader = .{ + .context = this, + .vtable = &.{ + .read = @This().read, + .readVec = @This().readVec, + .discard = @This().discard, + }, + }, + .buffer = buffer, + .end = 0, + .seek = 0, + }; + } + + fn read(context: ?*anyopaque, bw: *BufferedWriter, limit: Limit) RwError!usize { + const this: *@This() = @alignCast(@ptrCast(context)); + const slice = limit.slice(try bw.writableSliceGreedy(1)); + const n = try this.in.readVec(&.{slice}); + this.hasher.update(slice[0..n]); + bw.advance(n); + return n; + } + + fn discard(context: ?*anyopaque, limit: Limit) Error!usize { + const this: *@This() = @alignCast(@ptrCast(context)); + var bw = this.hasher.writable(&.{}); + const n = this.in.read(&bw, limit) catch |err| switch (err) { + error.WriteFailed => unreachable, + else => |e| return e, + }; + return n; + } + + fn readVec(context: ?*anyopaque, data: []const []u8) Error!usize { + const this: *@This() = @alignCast(@ptrCast(context)); + const n = try this.in.readVec(data); + var remaining: usize = n; + for (data) |slice| { + if (remaining < slice.len) { + this.hasher.update(slice[0..remaining]); + return n; + } else { + remaining -= slice.len; + this.hasher.update(slice); + } + } + assert(remaining == 0); + return n; + } + }; +} diff --git a/lib/std/io/Writer.zig b/lib/std/io/Writer.zig index f00ba27daf..e89235c80f 100644 --- a/lib/std/io/Writer.zig +++ b/lib/std/io/Writer.zig @@ -136,8 +136,8 @@ pub fn failingWriteSplat(context: ?*anyopaque, data: []const []const u8, splat: pub fn failingWriteFile( context: ?*anyopaque, file: std.fs.File, - offset: std.io.Writer.Offset, - limit: std.io.Writer.Limit, + offset: Offset, + limit: Limit, headers_and_trailers: []const []const u8, headers_len: usize, ) FileError!usize { @@ -158,11 +158,13 @@ pub const failing: Writer = .{ }, }; +/// For use when the `Writer` implementation can cannot offer a more efficient +/// implementation than a basic read/write loop on the file. pub fn unimplementedWriteFile( context: ?*anyopaque, file: std.fs.File, - offset: std.io.Writer.Offset, - limit: std.io.Writer.Limit, + offset: Offset, + limit: Limit, headers_and_trailers: []const []const u8, headers_len: usize, ) FileError!usize { @@ -175,6 +177,84 @@ pub fn unimplementedWriteFile( return error.Unimplemented; } +/// Provides a `Writer` implementation based on calling `Hasher.update`, sending +/// all data also to an underlying `std.io.BufferedWriter`. +/// +/// When using this, the underlying writer is best unbuffered because all +/// writes are passed on directly to it. +/// +/// This implementation makes suboptimal buffering decisions due to being +/// generic. A better solution will involve creating a writer for each hash +/// function, where the splat buffer can be tailored to the hash implementation +/// details. +pub fn Hashed(comptime Hasher: type) type { + return struct { + out: *std.io.BufferedWriter, + hasher: Hasher, + + pub fn writable(this: *@This(), buffer: []u8) std.io.BufferedWriter { + return .{ + .unbuffered_writer = .{ + .context = this, + .vtable = &.{ + .writeSplat = @This().writeSplat, + .writeFile = Writer.unimplementedWriteFile, + }, + }, + .buffer = buffer, + }; + } + + fn writeSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) Writer.Error!usize { + const this: *@This() = @alignCast(@ptrCast(context)); + const n = try this.out.writeSplat(data, splat); + const short_data = data[0 .. data.len - @intFromBool(splat == 0)]; + var remaining: usize = n; + for (short_data) |slice| { + if (remaining < slice.len) { + this.hasher.update(slice[0..remaining]); + return n; + } else { + remaining -= slice.len; + this.hasher.update(slice); + } + } + const remaining_splat = switch (splat) { + 0, 1 => { + assert(remaining == 0); + return n; + }, + else => splat - 1, + }; + const last = data[data.len - 1]; + assert(remaining == remaining_splat * last.len); + switch (last.len) { + 0 => { + assert(remaining == 0); + return n; + }, + 1 => { + var buffer: [64]u8 = undefined; + @memset(&buffer, last[0]); + while (remaining > 0) { + const update_len = @min(remaining, buffer.len); + this.hasher.update(buffer[0..update_len]); + remaining -= update_len; + } + return n; + }, + else => {}, + } + while (remaining > 0) { + const update_len = @min(remaining, last.len); + this.hasher.update(last[0..update_len]); + remaining -= update_len; + } + return n; + } + }; +} + test { _ = Null; } diff --git a/lib/std/io/Writer/Null.zig b/lib/std/io/Writer/Null.zig index 4ae90844d3..22616bdfd7 100644 --- a/lib/std/io/Writer/Null.zig +++ b/lib/std/io/Writer/Null.zig @@ -19,6 +19,10 @@ pub fn writer(nw: *NullWriter) Writer { }; } +pub fn writable(nw: *NullWriter, buffer: []u8) std.io.BufferedWriter { + return writer(nw).buffered(buffer); +} + fn writeSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) Writer.Error!usize { _ = context; const headers = data[0 .. data.len - 1]; diff --git a/lib/std/mem.zig b/lib/std/mem.zig index 59c26cc887..2e224e5db7 100644 --- a/lib/std/mem.zig +++ b/lib/std/mem.zig @@ -2196,7 +2196,9 @@ pub fn byteSwapAllFields(comptime S: type, ptr: *S) void { } } }, - else => @compileError("byteSwapAllFields expects a struct or array as the first argument"), + else => { + ptr.* = @byteSwap(ptr.*); + }, } } diff --git a/src/Package/Fetch/git.zig b/src/Package/Fetch/git.zig index fe8ee5f667..89ef199996 100644 --- a/src/Package/Fetch/git.zig +++ b/src/Package/Fetch/git.zig @@ -11,6 +11,7 @@ const Allocator = mem.Allocator; const Sha1 = std.crypto.hash.Sha1; const Sha256 = std.crypto.hash.sha2.Sha256; const assert = std.debug.assert; +const zlib = std.compress.zlib; /// The ID of a Git object. pub const Oid = union(Format) { @@ -52,7 +53,6 @@ pub const Oid = union(Format) { }; } - // Must be public for use from HashedReader and HashedWriter. pub fn update(hasher: *Hasher, b: []const u8) void { switch (hasher.*) { inline else => |*inner| inner.update(b), @@ -64,6 +64,12 @@ pub const Oid = union(Format) { inline else => |*inner, tag| @unionInit(Oid, @tagName(tag), inner.finalResult()), }; } + + pub fn writable(hasher: *Hasher, buffer: []u8) std.io.BufferedWriter { + return switch (hasher.*) { + inline else => |*inner| inner.writable(buffer), + }; + } }; pub fn fromBytes(oid_format: Format, bytes: []const u8) Oid { @@ -73,9 +79,18 @@ pub const Oid = union(Format) { }; } - pub fn readBytes(oid_format: Format, reader: anytype) @TypeOf(reader).NoEofError!Oid { + pub fn readBytes(oid_format: Format, reader: *std.io.BufferedReader) std.io.Reader.Error!Oid { return switch (oid_format) { - inline else => |tag| @unionInit(Oid, @tagName(tag), try reader.readBytesNoEof(tag.byteLength())), + .sha1 => { + var result: Oid = .{ .sha1 = undefined }; + try reader.readSlice(&result.sha1); + return result; + }, + .sha256 => { + var result: Oid = .{ .sha256 = undefined }; + try reader.readSlice(&result.sha256); + return result; + }, }; } @@ -167,8 +182,15 @@ pub const Diagnostics = struct { pub const Repository = struct { odb: Odb, - pub fn init(allocator: Allocator, format: Oid.Format, pack_file: std.fs.File, index_file: std.fs.File) !Repository { - return .{ .odb = try Odb.init(allocator, format, pack_file, index_file) }; + pub fn init( + repo: *Repository, + allocator: Allocator, + format: Oid.Format, + pack_file: *std.fs.File.Reader, + index_file: *std.fs.File.Reader, + ) !void { + repo.* = .{ .odb = undefined }; + try repo.odb.init(allocator, format, pack_file, index_file); } pub fn deinit(repository: *Repository) void { @@ -337,24 +359,32 @@ pub const Repository = struct { /// [pack-format](https://git-scm.com/docs/pack-format). const Odb = struct { format: Oid.Format, - pack_file: std.fs.File, + pack_file: *std.fs.File.Reader, index_header: IndexHeader, - index_file: std.fs.File, + index_file: *std.fs.File.Reader, cache: ObjectCache = .{}, allocator: Allocator, /// Initializes the database from open pack and index files. - fn init(allocator: Allocator, format: Oid.Format, pack_file: std.fs.File, index_file: std.fs.File) !Odb { + fn init( + odb: *Odb, + allocator: Allocator, + format: Oid.Format, + pack_file: *std.fs.File.Reader, + index_file: *std.fs.File.Reader, + ) !void { try pack_file.seekTo(0); try index_file.seekTo(0); - const index_header = try IndexHeader.read(index_file.reader()); - return .{ + odb.* = .{ .format = format, .pack_file = pack_file, - .index_header = index_header, + .index_header = undefined, .index_file = index_file, .allocator = allocator, }; + var buffer: [1032]u8 = undefined; + var index_file_br = index_file.readable(&buffer); + try odb.index_header.read(&index_file_br); } fn deinit(odb: *Odb) void { @@ -364,27 +394,30 @@ const Odb = struct { /// Reads the object at the current position in the database. fn readObject(odb: *Odb) !Object { - var base_offset = try odb.pack_file.getPos(); + var pack_read_buffer: [64]u8 = undefined; + var base_offset = odb.pack_file.pos; + var pack_br = odb.pack_file.readable(&pack_read_buffer); var base_header: EntryHeader = undefined; var delta_offsets: std.ArrayListUnmanaged(u64) = .empty; defer delta_offsets.deinit(odb.allocator); const base_object = while (true) { if (odb.cache.get(base_offset)) |base_object| break base_object; - base_header = try EntryHeader.read(odb.format, odb.pack_file.reader()); + base_header = try EntryHeader.read(odb.format, &pack_br); switch (base_header) { .ofs_delta => |ofs_delta| { try delta_offsets.append(odb.allocator, base_offset); base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidFormat; try odb.pack_file.seekTo(base_offset); + pack_br = odb.pack_file.readable(&pack_read_buffer); }, .ref_delta => |ref_delta| { try delta_offsets.append(odb.allocator, base_offset); try odb.seekOid(ref_delta.base_object); - base_offset = try odb.pack_file.getPos(); + base_offset = odb.pack_file.pos - pack_br.bufferedLen(); }, else => { - const base_data = try readObjectRaw(odb.allocator, odb.pack_file.reader(), base_header.uncompressedLength()); + const base_data = try readObjectRaw(odb.allocator, &pack_br, base_header.uncompressedLength()); errdefer odb.allocator.free(base_data); const base_object: Object = .{ .type = base_header.objectType(), .data = base_data }; try odb.cache.put(odb.allocator, base_offset, base_object); @@ -414,7 +447,8 @@ const Odb = struct { const found_index = while (start_index < end_index) { const mid_index = start_index + (end_index - start_index) / 2; try odb.index_file.seekTo(IndexHeader.size + mid_index * oid_length); - const mid_oid = try Oid.readBytes(odb.format, odb.index_file.reader()); + var br = odb.index_file.interface().unbuffered(); + const mid_oid = try Oid.readBytes(odb.format, &br); switch (mem.order(u8, mid_oid.slice(), oid.slice())) { .lt => start_index = mid_index + 1, .gt => end_index = mid_index, @@ -424,13 +458,16 @@ const Odb = struct { const n_objects = odb.index_header.fan_out_table[255]; const offset_values_start = IndexHeader.size + n_objects * (oid_length + 4); + var buffer: [8]u8 = undefined; try odb.index_file.seekTo(offset_values_start + found_index * 4); - const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try odb.index_file.reader().readInt(u32, .big)); + var br = odb.index_file.interface().buffered(&buffer); + const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try br.takeInt(u32, .big)); const pack_offset = pack_offset: { if (l1_offset.big) { const l2_offset_values_start = offset_values_start + n_objects * 4; try odb.index_file.seekTo(l2_offset_values_start + l1_offset.value * 4); - break :pack_offset try odb.index_file.reader().readInt(u64, .big); + br = odb.index_file.interface().buffered(&buffer); + break :pack_offset try br.takeInt(u64, .big); } else { break :pack_offset l1_offset.value; } @@ -556,7 +593,7 @@ const Packet = union(enum) { const max_data_length = 65516; /// Reads a packet in pkt-line format. - fn read(reader: anytype, buf: *[max_data_length]u8) !Packet { + fn read(reader: *std.io.BufferedReader, buf: *[max_data_length]u8) !Packet { const length = std.fmt.parseUnsigned(u16, &try reader.readBytesNoEof(4), 16) catch return error.InvalidPacket; switch (length) { 0 => return .flush, @@ -571,7 +608,7 @@ const Packet = union(enum) { } /// Writes a packet in pkt-line format. - fn write(packet: Packet, writer: anytype) !void { + fn write(packet: Packet, writer: *std.io.BufferedWriter) !void { switch (packet) { .flush => try writer.writeAll("0000"), .delimiter => try writer.writeAll("0001"), @@ -1070,21 +1107,12 @@ const PackHeader = struct { const signature = "PACK"; const supported_version = 2; - fn read(reader: anytype) !PackHeader { - const actual_signature = reader.readBytesNoEof(4) catch |e| switch (e) { - error.EndOfStream => return error.InvalidHeader, - else => |other| return other, - }; - if (!mem.eql(u8, &actual_signature, signature)) return error.InvalidHeader; - const version = reader.readInt(u32, .big) catch |e| switch (e) { - error.EndOfStream => return error.InvalidHeader, - else => |other| return other, - }; + fn read(reader: *std.io.BufferedReader) !PackHeader { + const actual_signature = try reader.take(4); + if (!mem.eql(u8, actual_signature, signature)) return error.InvalidHeader; + const version = try reader.takeInt(u32, .big); if (version != supported_version) return error.UnsupportedVersion; - const total_objects = reader.readInt(u32, .big) catch |e| switch (e) { - error.EndOfStream => return error.InvalidHeader, - else => |other| return other, - }; + const total_objects = try reader.takeInt(u32, .big); return .{ .total_objects = total_objects }; } }; @@ -1133,12 +1161,9 @@ const EntryHeader = union(Type) { }; } - fn read(format: Oid.Format, reader: anytype) !EntryHeader { + fn read(format: Oid.Format, reader: *std.io.BufferedReader) !EntryHeader { const InitialByte = packed struct { len: u4, type: u3, has_next: bool }; - const initial: InitialByte = @bitCast(reader.readByte() catch |e| switch (e) { - error.EndOfStream => return error.InvalidFormat, - else => |other| return other, - }); + const initial: InitialByte = @bitCast(try reader.takeByte()); const rest_len = if (initial.has_next) try readSizeVarInt(reader) else 0; var uncompressed_length: u64 = initial.len; uncompressed_length |= std.math.shlExact(u64, rest_len, 4) catch return error.InvalidFormat; @@ -1162,25 +1187,25 @@ const EntryHeader = union(Type) { } }; -fn readSizeVarInt(r: anytype) !u64 { +fn readSizeVarInt(r: *std.io.BufferedReader) !u64 { const Byte = packed struct { value: u7, has_next: bool }; - var b: Byte = @bitCast(try r.readByte()); + var b: Byte = @bitCast(try r.takeByte()); var value: u64 = b.value; var shift: u6 = 0; while (b.has_next) { - b = @bitCast(try r.readByte()); + b = @bitCast(try r.takeByte()); shift = std.math.add(u6, shift, 7) catch return error.InvalidFormat; value |= @as(u64, b.value) << shift; } return value; } -fn readOffsetVarInt(r: anytype) !u64 { +fn readOffsetVarInt(r: *std.io.BufferedReader) !u64 { const Byte = packed struct { value: u7, has_next: bool }; - var b: Byte = @bitCast(try r.readByte()); + var b: Byte = @bitCast(try r.takeByte()); var value: u64 = b.value; while (b.has_next) { - b = @bitCast(try r.readByte()); + b = @bitCast(try r.takeByte()); value = std.math.shlExact(u64, value + 1, 7) catch return error.InvalidFormat; value |= b.value; } @@ -1194,19 +1219,12 @@ const IndexHeader = struct { const supported_version = 2; const size = 4 + 4 + @sizeOf([256]u32); - fn read(reader: anytype) !IndexHeader { - var header_bytes = try reader.readBytesNoEof(size); - if (!mem.eql(u8, header_bytes[0..4], signature)) return error.InvalidHeader; - const version = mem.readInt(u32, header_bytes[4..8], .big); + fn read(index_header: *IndexHeader, br: *std.io.BufferedReader) !void { + const sig = try br.take(4); + if (!mem.eql(u8, sig, signature)) return error.InvalidHeader; + const version = try br.takeInt(u32, .big); if (version != supported_version) return error.UnsupportedVersion; - - var fan_out_table: [256]u32 = undefined; - var fan_out_table_stream = std.io.fixedBufferStream(header_bytes[8..]); - const fan_out_table_reader = fan_out_table_stream.reader(); - for (&fan_out_table) |*entry| { - entry.* = fan_out_table_reader.readInt(u32, .big) catch unreachable; - } - return .{ .fan_out_table = fan_out_table }; + try br.readSliceEndian(u32, &index_header.fan_out_table, .big); } }; @@ -1217,7 +1235,12 @@ const IndexEntry = struct { /// Writes out a version 2 index for the given packfile, as documented in /// [pack-format](https://git-scm.com/docs/pack-format). -pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, index_writer: anytype) !void { +pub fn indexPack( + allocator: Allocator, + format: Oid.Format, + pack: *std.fs.File.Reader, + index_writer: *std.fs.File.Writer, +) !void { try pack.seekTo(0); var index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry) = .empty; @@ -1270,8 +1293,10 @@ pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, in } @memset(fan_out_table[fan_out_index..], count); - var index_hashed_writer = std.compress.hashedWriter(index_writer, Oid.Hasher.init(format)); - const writer = index_hashed_writer.writer(); + var index_writer_bw = index_writer.writable(&.{}); + var index_hashed_writer = index_writer_bw.hashed(Oid.Hasher.init(format)); + var write_buffer: [256]u8 = undefined; + var writer = index_hashed_writer.writable(&write_buffer); try writer.writeAll(IndexHeader.signature); try writer.writeInt(u32, IndexHeader.supported_version, .big); for (fan_out_table) |fan_out_entry| { @@ -1303,8 +1328,9 @@ pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, in } try writer.writeAll(pack_checksum.slice()); + try writer.flush(); const index_checksum = index_hashed_writer.hasher.finalResult(); - try index_writer.writeAll(index_checksum.slice()); + try index_writer_bw.writeAll(index_checksum.slice()); } /// Performs the first pass over the packfile data for index construction. @@ -1314,50 +1340,46 @@ pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, in fn indexPackFirstPass( allocator: Allocator, format: Oid.Format, - pack: std.fs.File, + pack: *std.fs.File.Reader, index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry), pending_deltas: *std.ArrayListUnmanaged(IndexEntry), ) !Oid { - var pack_buffered_reader = std.io.bufferedReader(pack.reader()); - var pack_counting_reader = std.io.countingReader(pack_buffered_reader.reader()); - var pack_hashed_reader = std.compress.hashedReader(pack_counting_reader.reader(), Oid.Hasher.init(format)); - const pack_reader = pack_hashed_reader.reader(); + var pack_br = pack.readable(&.{}); + var pack_hashed_reader = pack_br.hashed(Oid.Hasher.init(format)); + var pack_buffer: [2048]u8 = undefined; // Reasonably large buffer for file system. + var pack_hashed_br = pack_hashed_reader.readable(&pack_buffer); - const pack_header = try PackHeader.read(pack_reader); + const pack_header = try PackHeader.read(&pack_hashed_br); - var current_entry: u32 = 0; - while (current_entry < pack_header.total_objects) : (current_entry += 1) { - const entry_offset = pack_counting_reader.bytes_read; - var entry_crc32_reader = std.compress.hashedReader(pack_reader, std.hash.Crc32.init()); - const entry_header = try EntryHeader.read(format, entry_crc32_reader.reader()); + for (0..pack_header.total_objects) |_| { + const entry_offset = pack.pos - pack_hashed_br.bufferContents().len; + var entry_crc32_reader = pack_hashed_br.hashed(std.hash.Crc32.init()); + var entry_buffer: [64]u8 = undefined; // Buffer only needed for loading EntryHeader. + var entry_crc32_br = entry_crc32_reader.readable(&entry_buffer); + const entry_header = try EntryHeader.read(format, &entry_crc32_br); + var entry_decompress_stream: zlib.Decompressor = .init(&entry_crc32_br); + // Decompress uses large output buffer; no input buffer needed. + var entry_decompress_br = entry_decompress_stream.readable(&.{}); switch (entry_header) { .commit, .tree, .blob, .tag => |object| { - var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader()); - var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader()); - var entry_hashed_writer = std.compress.hashedWriter(std.io.null_writer, Oid.Hasher.init(format)); - const entry_writer = entry_hashed_writer.writer(); + var oid_hasher = Oid.Hasher.init(format); + var oid_hasher_buffer: [zlib.max_window_len]u8 = undefined; + var oid_hasher_bw = oid_hasher.writable(&oid_hasher_buffer); // The object header is not included in the pack data but is - // part of the object's ID - try entry_writer.print("{s} {}\x00", .{ @tagName(entry_header), object.uncompressed_length }); - var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); - try fifo.pump(entry_counting_reader.reader(), entry_writer); - if (entry_counting_reader.bytes_read != object.uncompressed_length) { - return error.InvalidObject; - } - const oid = entry_hashed_writer.hasher.finalResult(); + // part of the object's ID. + try oid_hasher_bw.print("{s} {d}\x00", .{ @tagName(entry_header), object.uncompressed_length }); + const n = try entry_decompress_br.readRemaining(&oid_hasher_bw); + if (n != object.uncompressed_length) return error.InvalidObject; + try oid_hasher_bw.flush(); + const oid = oid_hasher.finalResult(); try index_entries.put(allocator, oid, .{ .offset = entry_offset, .crc32 = entry_crc32_reader.hasher.final(), }); }, inline .ofs_delta, .ref_delta => |delta| { - var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader()); - var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader()); - var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); - try fifo.pump(entry_counting_reader.reader(), std.io.null_writer); - if (entry_counting_reader.bytes_read != delta.uncompressed_length) { - return error.InvalidObject; - } + const n = try entry_decompress_br.discardRemaining(); + if (n != delta.uncompressed_length) return error.InvalidObject; try pending_deltas.append(allocator, .{ .offset = entry_offset, .crc32 = entry_crc32_reader.hasher.final(), @@ -1367,15 +1389,11 @@ fn indexPackFirstPass( } const pack_checksum = pack_hashed_reader.hasher.finalResult(); - const recorded_checksum = try Oid.readBytes(format, pack_buffered_reader.reader()); + const recorded_checksum = try Oid.readBytes(format, &pack_br); if (!mem.eql(u8, pack_checksum.slice(), recorded_checksum.slice())) { return error.CorruptedPack; } - _ = pack_reader.readByte() catch |e| switch (e) { - error.EndOfStream => return pack_checksum, - else => |other| return other, - }; - return error.InvalidFormat; + return pack_checksum; } /// Attempts to determine the final object ID of the given deltified object. @@ -1384,7 +1402,7 @@ fn indexPackFirstPass( fn indexPackHashDelta( allocator: Allocator, format: Oid.Format, - pack: std.fs.File, + pack: *std.fs.File.Reader, delta: IndexEntry, index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry), cache: *ObjectCache, @@ -1398,7 +1416,9 @@ fn indexPackHashDelta( if (cache.get(base_offset)) |base_object| break base_object; try pack.seekTo(base_offset); - base_header = try EntryHeader.read(format, pack.reader()); + var pack_read_buffer: [64]u8 = undefined; + var pack_br = pack.readable(&pack_read_buffer); + base_header = try EntryHeader.read(format, &pack_br); switch (base_header) { .ofs_delta => |ofs_delta| { try delta_offsets.append(allocator, base_offset); @@ -1409,7 +1429,7 @@ fn indexPackHashDelta( base_offset = (index_entries.get(ref_delta.base_object) orelse return null).offset; }, else => { - const base_data = try readObjectRaw(allocator, pack.reader(), base_header.uncompressedLength()); + const base_data = try readObjectRaw(allocator, &pack_br, base_header.uncompressedLength()); errdefer allocator.free(base_data); const base_object: Object = .{ .type = base_header.objectType(), .data = base_data }; try cache.put(allocator, base_offset, base_object); @@ -1421,9 +1441,12 @@ fn indexPackHashDelta( const base_data = try resolveDeltaChain(allocator, format, pack, base_object, delta_offsets.items, cache); var entry_hasher: Oid.Hasher = .init(format); - var entry_hashed_writer = std.compress.hashedWriter(std.io.null_writer, &entry_hasher); - try entry_hashed_writer.writer().print("{s} {}\x00", .{ @tagName(base_object.type), base_data.len }); - entry_hasher.update(base_data); + var entry_hasher_buffer: [64]u8 = undefined; + var entry_hasher_bw = entry_hasher.writable(&entry_hasher_buffer); + // Writes to hashers cannot fail. + entry_hasher_bw.print("{s} {d}\x00", .{ @tagName(base_object.type), base_data.len }) catch unreachable; + entry_hasher_bw.writeAll(base_data) catch unreachable; + entry_hasher_bw.flush() catch unreachable; return entry_hasher.finalResult(); } @@ -1434,7 +1457,7 @@ fn indexPackHashDelta( fn resolveDeltaChain( allocator: Allocator, format: Oid.Format, - pack: std.fs.File, + pack: *std.fs.File.Reader, base_object: Object, delta_offsets: []const u64, cache: *ObjectCache, @@ -1446,21 +1469,22 @@ fn resolveDeltaChain( const delta_offset = delta_offsets[i]; try pack.seekTo(delta_offset); - const delta_header = try EntryHeader.read(format, pack.reader()); - const delta_data = try readObjectRaw(allocator, pack.reader(), delta_header.uncompressedLength()); - defer allocator.free(delta_data); - var delta_stream = std.io.fixedBufferStream(delta_data); - const delta_reader = delta_stream.reader(); - _ = try readSizeVarInt(delta_reader); // base object size - const expanded_size = try readSizeVarInt(delta_reader); - + var pack_read_buffer: [64]u8 = undefined; + var pack_br = pack.readable(&pack_read_buffer); + const delta_header = try EntryHeader.read(format, &pack_br); + _ = delta_header; + var delta_decompress: zlib.Decompressor = .init(&pack_br); + var delta_decompress_buffer: [zlib.max_window_len]u8 = undefined; + var delta_reader = delta_decompress.readable(&delta_decompress_buffer); + _ = try readSizeVarInt(&delta_reader); // base object size + const expanded_size = try readSizeVarInt(&delta_reader); const expanded_alloc_size = std.math.cast(usize, expanded_size) orelse return error.ObjectTooLarge; const expanded_data = try allocator.alloc(u8, expanded_alloc_size); errdefer allocator.free(expanded_data); - var expanded_delta_stream = std.io.fixedBufferStream(expanded_data); - var base_stream = std.io.fixedBufferStream(base_data); - try expandDelta(&base_stream, delta_reader, expanded_delta_stream.writer()); - if (expanded_delta_stream.pos != expanded_size) return error.InvalidObject; + var expanded_delta_stream: std.io.BufferedWriter = undefined; + expanded_delta_stream.initFixed(expanded_data); + try expandDelta(base_data, &delta_reader, &expanded_delta_stream); + if (expanded_delta_stream.end != expanded_size) return error.InvalidObject; try cache.put(allocator, delta_offset, .{ .type = base_object.type, .data = expanded_data }); base_data = expanded_data; @@ -1468,31 +1492,23 @@ fn resolveDeltaChain( return base_data; } -/// Reads the complete contents of an object from `reader`. This function may -/// read more bytes than required from `reader`, so the reader position after -/// returning is not reliable. -fn readObjectRaw(allocator: Allocator, reader: anytype, size: u64) ![]u8 { +/// Reads the complete contents of an object from `reader`. +fn readObjectRaw(gpa: Allocator, reader: *std.io.BufferedReader, size: u64) ![]u8 { const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge; - var buffered_reader = std.io.bufferedReader(reader); - var decompress_stream = std.compress.zlib.decompressor(buffered_reader.reader()); - const data = try allocator.alloc(u8, alloc_size); - errdefer allocator.free(data); - try decompress_stream.reader().readNoEof(data); - _ = decompress_stream.reader().readByte() catch |e| switch (e) { - error.EndOfStream => return data, - else => |other| return other, - }; - return error.InvalidFormat; + var decompress: zlib.Decompressor = .init(reader); + var buffer: std.ArrayListUnmanaged(u8) = .empty; + defer buffer.deinit(gpa); + try decompress.reader().readRemainingArrayList(gpa, null, &buffer, .limited(alloc_size), zlib.max_window_len); + if (buffer.items.len < size) return error.EndOfStream; + return buffer.toOwnedSlice(gpa); } -/// Expands delta data from `delta_reader` to `writer`. `base_object` must -/// support `reader` and `seekTo` (such as a `std.io.FixedBufferStream`). -/// /// The format of the delta data is documented in /// [pack-format](https://git-scm.com/docs/pack-format). -fn expandDelta(base_object: anytype, delta_reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) !void { +fn expandDelta(base_object: []const u8, delta_reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) !void { + var base_offset: u32 = 0; while (true) { - const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.readByte() catch |e| switch (e) { + const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.takeByte() catch |e| switch (e) { error.EndOfStream => return, else => |other| return other, }); @@ -1507,23 +1523,22 @@ fn expandDelta(base_object: anytype, delta_reader: *std.io.BufferedReader, write size3: bool, } = @bitCast(inst.value); const offset_parts: packed struct { offset1: u8, offset2: u8, offset3: u8, offset4: u8 } = .{ - .offset1 = if (available.offset1) try delta_reader.readByte() else 0, - .offset2 = if (available.offset2) try delta_reader.readByte() else 0, - .offset3 = if (available.offset3) try delta_reader.readByte() else 0, - .offset4 = if (available.offset4) try delta_reader.readByte() else 0, + .offset1 = if (available.offset1) try delta_reader.takeByte() else 0, + .offset2 = if (available.offset2) try delta_reader.takeByte() else 0, + .offset3 = if (available.offset3) try delta_reader.takeByte() else 0, + .offset4 = if (available.offset4) try delta_reader.takeByte() else 0, }; - const offset: u32 = @bitCast(offset_parts); + base_offset = @bitCast(offset_parts); const size_parts: packed struct { size1: u8, size2: u8, size3: u8 } = .{ - .size1 = if (available.size1) try delta_reader.readByte() else 0, - .size2 = if (available.size2) try delta_reader.readByte() else 0, - .size3 = if (available.size3) try delta_reader.readByte() else 0, + .size1 = if (available.size1) try delta_reader.takeByte() else 0, + .size2 = if (available.size2) try delta_reader.takeByte() else 0, + .size3 = if (available.size3) try delta_reader.takeByte() else 0, }; var size: u24 = @bitCast(size_parts); if (size == 0) size = 0x10000; - try base_object.seekTo(offset); - var base_object_br = base_object.reader(); - try base_object_br.readAll(writer, .limited(size)); + try writer.writeAll(base_object[base_offset..][0..size]); + base_offset += size; } else if (inst.value != 0) { try delta_reader.readAll(writer, .limited(inst.value)); } else { @@ -1557,7 +1572,8 @@ fn runRepositoryTest(comptime format: Oid.Format, head_commit: []const u8) !void var index_file = try git_dir.dir.createFile("testrepo.idx", .{ .read = true }); defer index_file.close(); - try indexPack(testing.allocator, format, pack_file, index_file.writer()); + var index_file_writer = index_file.writer(); + try indexPack(testing.allocator, format, pack_file, &index_file_writer); // Arbitrary size limit on files read while checking the repository contents // (all files in the test repo are known to be smaller than this) @@ -1571,7 +1587,8 @@ fn runRepositoryTest(comptime format: Oid.Format, head_commit: []const u8) !void const testrepo_idx = @embedFile("git/testdata/testrepo-" ++ @tagName(format) ++ ".idx"); try testing.expectEqualSlices(u8, testrepo_idx, index_file_data); - var repository = try Repository.init(testing.allocator, format, pack_file, index_file); + var index_file_reader = index_file_writer.moveToReader(); + var repository = try Repository.init(testing.allocator, format, pack_file, &index_file_reader); defer repository.deinit(); var worktree = testing.tmpDir(.{ .iterate = true }); @@ -1652,10 +1669,12 @@ test "SHA-256 packfile indexing and checkout" { /// Checks out a commit of a packfile. Intended for experimenting with and /// benchmarking possible optimizations to the indexing and checkout behavior. pub fn main() !void { - const allocator = std.heap.c_allocator; + var debug_allocator: std.heap.DebugAllocator(.{}) = .init; + defer _ = debug_allocator.deinit(); + const gpa = if (std.debug.runtime_safety) debug_allocator.allocator() else std.heap.smp_allocator; - const args = try std.process.argsAlloc(allocator); - defer std.process.argsFree(allocator, args); + const args = try std.process.argsAlloc(gpa); + defer std.process.argsFree(gpa, args); if (args.len != 5) { return error.InvalidArguments; // Arguments: format packfile commit worktree } @@ -1674,15 +1693,17 @@ pub fn main() !void { std.debug.print("Starting index...\n", .{}); var index_file = try git_dir.createFile("idx", .{ .read = true }); defer index_file.close(); - var index_buffered_writer = std.io.bufferedWriter(index_file.writer()); - try indexPack(allocator, format, pack_file, index_buffered_writer.writer()); - try index_buffered_writer.flush(); + var index_file_writer = index_file.writer(); + var pack_file_reader = pack_file.reader(); + try indexPack(gpa, format, &pack_file_reader, &index_file_writer); try index_file.sync(); std.debug.print("Starting checkout...\n", .{}); - var repository = try Repository.init(allocator, format, pack_file, index_file); + var index_file_reader = index_file_writer.moveToReader(); + var repository: Repository = undefined; + try repository.init(gpa, format, &pack_file_reader, &index_file_reader); defer repository.deinit(); - var diagnostics: Diagnostics = .{ .allocator = allocator }; + var diagnostics: Diagnostics = .{ .allocator = gpa }; defer diagnostics.deinit(); try repository.checkout(worktree, commit, &diagnostics); diff --git a/src/main.zig b/src/main.zig index 561935035f..e93351f2d0 100644 --- a/src/main.zig +++ b/src/main.zig @@ -3330,12 +3330,11 @@ fn buildOutputType( // for the hashing algorithm here and in the cache are the same. // We are providing our own cache key, because this file has nothing // to do with the cache manifest. - var hasher = Cache.Hasher.init("0123456789abcdef"); var file_writer = f.writer(); - var file_writer_bw = file_writer.interface().unbuffered(); - var hasher_writer = hasher.writer(&file_writer_bw); + var file_writer_bw = file_writer.writable(&.{}); + var hasher_writer = file_writer_bw.hashed(Cache.Hasher.init("0123456789abcdef")); var buffer: [1000]u8 = undefined; - var bw = hasher_writer.interface().buffered(&buffer); + var bw = hasher_writer.writable(&buffer); bw.writeFileAll(.stdin(), .{}) catch |err| switch (err) { error.WriteFailed => fatal("failed to write {s}: {s}", .{ dump_path, file_writer.err.? }), else => fatal("failed to pipe stdin to {s}: {s}", .{ dump_path, err }),