mirror/zig
1
0
Fork 0
mirror of https://github.com/ziglang/zig.git synced 2026-02-21 16:54:52 +00:00
Code Issues Packages Projects Releases Wiki Activity

update git fetching logic to new reader/writer API

Browse Source 
- flatten std.crypto.hash.Sha1 and give it a writable interface that
  optimizes splats
- flatten std.hash.crc and give it a writable interface that optimizes
  splats
- remove old writer impls from std.crypto
- add fs.File.Writer.moveToReader
- add fs.File.Writer.seekTo
- add std.io.Reader.Hashed and std.io.Writer.Hashed which are
  passthrough streams. Instead of passing through to null writer, use
  the writable interface implemented directly on hashers which doesn't
  have to account for passing through the data.
- add std.io.BufferedWriter.writeSplatAll
This commit is contained in:
Andrew Kelley 2025-05-09 16:59:13 -07:00
parent d603121dc3
commit 06b44a0afa
28 changed files with 1185 additions and 911 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
lib/std/compress/flate.zig
View File

@ -1,5 +1,10 @@
const std = @import("../std.zig");
/// When decompressing, the output buffer is used as the history window, so
/// less than this may result in failure to decompress streams that were
/// compressed with a larger window.
pub const max_window_len = 1 << 16;
/// Deflate is a lossless data compression file format that uses a combination
/// of LZ77 and Huffman coding.
pub const deflate = @import("flate/deflate.zig");

4
lib/std/compress/flate/inflate.zig
View File

@ -403,6 +403,10 @@ pub fn Inflate(comptime container: Container, comptime Lookahead: type) type {
},
};
}
pub fn readable(self: *Self, buffer: []u8) std.io.BufferedReader {
return reader(self).buffered(buffer);
}
};
}

5
lib/std/compress/zlib.zig
View File

@ -2,6 +2,11 @@ const std = @import("../std.zig");
const deflate = @import("flate/deflate.zig");
const inflate = @import("flate/inflate.zig");
/// When decompressing, the output buffer is used as the history window, so
/// less than this may result in failure to decompress streams that were
/// compressed with a larger window.
pub const max_window_len = std.compress.flate.max_window_len;
/// Decompress compressed data from reader and write plain data to the writer.
pub fn decompress(reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) !void {
try inflate.decompress(.zlib, reader, writer);

6
lib/std/crypto.zig
View File

@ -1,5 +1,7 @@
//! Cryptography.
const std = @import("std.zig");
const assert = std.debug.assert;
const root = @import("root");
pub const timing_safe = @import("crypto/timing_safe.zig");
@ -119,7 +121,7 @@ pub const hash = struct {
pub const blake2 = @import("crypto/blake2.zig");
pub const Blake3 = @import("crypto/blake3.zig").Blake3;
pub const Md5 = @import("crypto/md5.zig").Md5;
pub const Sha1 = @import("crypto/sha1.zig").Sha1;
pub const Sha1 = @import("crypto/Sha1.zig");
pub const sha2 = @import("crypto/sha2.zig");
pub const sha3 = @import("crypto/sha3.zig");
pub const composition = @import("crypto/hash_composition.zig");
@ -217,8 +219,6 @@ pub const random = @import("crypto/tlcsprng.zig").interface;
/// Encoding and decoding
pub const codecs = @import("crypto/codecs.zig");
const std = @import("std.zig");
pub const errors = @import("crypto/errors.zig");
pub const tls = @import("crypto/tls.zig");

424
lib/std/crypto/Sha1.zig Normal file
View File

@ -0,0 +1,424 @@
//! The SHA-1 function is now considered cryptographically broken.
//! Namely, it is feasible to find multiple inputs producing the same hash.
//! For a fast-performing, cryptographically secure hash function, see SHA512/256, BLAKE2 or BLAKE3.
const Sha1 = @This();
const std = @import("../std.zig");
const mem = std.mem;
const math = std.math;
const assert = std.debug.assert;
pub const block_length = 64;
pub const digest_length = 20;
pub const Options = struct {};
s: [5]u32,
/// Streaming Cache
buf: [64]u8,
buf_end: u8,
total_len: u64,
pub fn init(options: Options) Sha1 {
_ = options;
return .{
.s = [_]u32{
0x67452301,
0xEFCDAB89,
0x98BADCFE,
0x10325476,
0xC3D2E1F0,
},
.buf = undefined,
.buf_end = 0,
.total_len = 0,
};
}
pub fn hash(b: []const u8, out: *[digest_length]u8, options: Options) void {
var d = Sha1.init(options);
d.update(b);
d.final(out);
}
pub fn update(d: *Sha1, b: []const u8) void {
var off: usize = 0;
// Partial buffer exists from previous update. Copy into buffer then hash.
const unused_buf = d.buf[d.buf_end..];
if (unused_buf.len < d.buf.len and b.len >= unused_buf.len) {
@memcpy(unused_buf, b[0..unused_buf.len]);
off += unused_buf.len;
round(&d.s, &d.buf);
d.buf_end = 0;
}
// Full middle blocks.
while (off + 64 <= b.len) : (off += 64) {
round(&d.s, b[off..][0..64]);
}
// Copy any remainder for next pass.
const remainder = b[off..];
@memcpy(d.buf[d.buf_end..][0..remainder.len], remainder);
d.buf_end = @intCast(d.buf_end + remainder.len);
d.total_len += b.len;
}
pub fn peek(d: Sha1) [digest_length]u8 {
var copy = d;
return copy.finalResult();
}
pub fn final(d: *Sha1, out: *[digest_length]u8) void {
// The buffer here will never be completely full.
@memset(d.buf[d.buf_end..], 0);
// Append padding bits.
d.buf[d.buf_end] = 0x80;
d.buf_end += 1;
// > 448 mod 512 so need to add an extra round to wrap around.
if (64 - d.buf_end < 8) {
round(&d.s, d.buf[0..]);
@memset(d.buf[0..], 0);
}
// Append message length.
var i: usize = 1;
var len = d.total_len >> 5;
d.buf[63] = @as(u8, @intCast(d.total_len & 0x1f)) << 3;
while (i < 8) : (i += 1) {
d.buf[63 - i] = @as(u8, @intCast(len & 0xff));
len >>= 8;
}
round(&d.s, d.buf[0..]);
for (d.s, 0..) |s, j| {
mem.writeInt(u32, out[4 * j ..][0..4], s, .big);
}
}
pub fn finalResult(d: *Sha1) [digest_length]u8 {
var result: [digest_length]u8 = undefined;
d.final(&result);
return result;
}
pub fn round(d_s: *[5]u32, b: *const [64]u8) void {
var s: [16]u32 = undefined;
var v: [5]u32 = [_]u32{
d_s[0],
d_s[1],
d_s[2],
d_s[3],
d_s[4],
};
const round0a = comptime [_]RoundParam{
.abcdei(0, 1, 2, 3, 4, 0),
.abcdei(4, 0, 1, 2, 3, 1),
.abcdei(3, 4, 0, 1, 2, 2),
.abcdei(2, 3, 4, 0, 1, 3),
.abcdei(1, 2, 3, 4, 0, 4),
.abcdei(0, 1, 2, 3, 4, 5),
.abcdei(4, 0, 1, 2, 3, 6),
.abcdei(3, 4, 0, 1, 2, 7),
.abcdei(2, 3, 4, 0, 1, 8),
.abcdei(1, 2, 3, 4, 0, 9),
.abcdei(0, 1, 2, 3, 4, 10),
.abcdei(4, 0, 1, 2, 3, 11),
.abcdei(3, 4, 0, 1, 2, 12),
.abcdei(2, 3, 4, 0, 1, 13),
.abcdei(1, 2, 3, 4, 0, 14),
.abcdei(0, 1, 2, 3, 4, 15),
};
inline for (round0a) |r| {
s[r.i] = mem.readInt(u32, b[r.i * 4 ..][0..4], .big);
v[r.e] = v[r.e] +% math.rotl(u32, v[r.a], @as(u32, 5)) +% 0x5A827999 +% s[r.i & 0xf] +% ((v[r.b] & v[r.c]) | (~v[r.b] & v[r.d]));
v[r.b] = math.rotl(u32, v[r.b], @as(u32, 30));
}
const round0b = comptime [_]RoundParam{
.abcdei(4, 0, 1, 2, 3, 16),
.abcdei(3, 4, 0, 1, 2, 17),
.abcdei(2, 3, 4, 0, 1, 18),
.abcdei(1, 2, 3, 4, 0, 19),
};
inline for (round0b) |r| {
const t = s[(r.i - 3) & 0xf] ^ s[(r.i - 8) & 0xf] ^ s[(r.i - 14) & 0xf] ^ s[(r.i - 16) & 0xf];
s[r.i & 0xf] = math.rotl(u32, t, @as(u32, 1));
v[r.e] = v[r.e] +% math.rotl(u32, v[r.a], @as(u32, 5)) +% 0x5A827999 +% s[r.i & 0xf] +% ((v[r.b] & v[r.c]) | (~v[r.b] & v[r.d]));
v[r.b] = math.rotl(u32, v[r.b], @as(u32, 30));
}
const round1 = comptime [_]RoundParam{
.abcdei(0, 1, 2, 3, 4, 20),
.abcdei(4, 0, 1, 2, 3, 21),
.abcdei(3, 4, 0, 1, 2, 22),
.abcdei(2, 3, 4, 0, 1, 23),
.abcdei(1, 2, 3, 4, 0, 24),
.abcdei(0, 1, 2, 3, 4, 25),
.abcdei(4, 0, 1, 2, 3, 26),
.abcdei(3, 4, 0, 1, 2, 27),
.abcdei(2, 3, 4, 0, 1, 28),
.abcdei(1, 2, 3, 4, 0, 29),
.abcdei(0, 1, 2, 3, 4, 30),
.abcdei(4, 0, 1, 2, 3, 31),
.abcdei(3, 4, 0, 1, 2, 32),
.abcdei(2, 3, 4, 0, 1, 33),
.abcdei(1, 2, 3, 4, 0, 34),
.abcdei(0, 1, 2, 3, 4, 35),
.abcdei(4, 0, 1, 2, 3, 36),
.abcdei(3, 4, 0, 1, 2, 37),
.abcdei(2, 3, 4, 0, 1, 38),
.abcdei(1, 2, 3, 4, 0, 39),
};
inline for (round1) |r| {
const t = s[(r.i - 3) & 0xf] ^ s[(r.i - 8) & 0xf] ^ s[(r.i - 14) & 0xf] ^ s[(r.i - 16) & 0xf];
s[r.i & 0xf] = math.rotl(u32, t, @as(u32, 1));
v[r.e] = v[r.e] +% math.rotl(u32, v[r.a], @as(u32, 5)) +% 0x6ED9EBA1 +% s[r.i & 0xf] +% (v[r.b] ^ v[r.c] ^ v[r.d]);
v[r.b] = math.rotl(u32, v[r.b], @as(u32, 30));
}
const round2 = comptime [_]RoundParam{
.abcdei(0, 1, 2, 3, 4, 40),
.abcdei(4, 0, 1, 2, 3, 41),
.abcdei(3, 4, 0, 1, 2, 42),
.abcdei(2, 3, 4, 0, 1, 43),
.abcdei(1, 2, 3, 4, 0, 44),
.abcdei(0, 1, 2, 3, 4, 45),
.abcdei(4, 0, 1, 2, 3, 46),
.abcdei(3, 4, 0, 1, 2, 47),
.abcdei(2, 3, 4, 0, 1, 48),
.abcdei(1, 2, 3, 4, 0, 49),
.abcdei(0, 1, 2, 3, 4, 50),
.abcdei(4, 0, 1, 2, 3, 51),
.abcdei(3, 4, 0, 1, 2, 52),
.abcdei(2, 3, 4, 0, 1, 53),
.abcdei(1, 2, 3, 4, 0, 54),
.abcdei(0, 1, 2, 3, 4, 55),
.abcdei(4, 0, 1, 2, 3, 56),
.abcdei(3, 4, 0, 1, 2, 57),
.abcdei(2, 3, 4, 0, 1, 58),
.abcdei(1, 2, 3, 4, 0, 59),
};
inline for (round2) |r| {
const t = s[(r.i - 3) & 0xf] ^ s[(r.i - 8) & 0xf] ^ s[(r.i - 14) & 0xf] ^ s[(r.i - 16) & 0xf];
s[r.i & 0xf] = math.rotl(u32, t, @as(u32, 1));
v[r.e] = v[r.e] +% math.rotl(u32, v[r.a], @as(u32, 5)) +% 0x8F1BBCDC +% s[r.i & 0xf] +% ((v[r.b] & v[r.c]) ^ (v[r.b] & v[r.d]) ^ (v[r.c] & v[r.d]));
v[r.b] = math.rotl(u32, v[r.b], @as(u32, 30));
}
const round3 = comptime [_]RoundParam{
.abcdei(0, 1, 2, 3, 4, 60),
.abcdei(4, 0, 1, 2, 3, 61),
.abcdei(3, 4, 0, 1, 2, 62),
.abcdei(2, 3, 4, 0, 1, 63),
.abcdei(1, 2, 3, 4, 0, 64),
.abcdei(0, 1, 2, 3, 4, 65),
.abcdei(4, 0, 1, 2, 3, 66),
.abcdei(3, 4, 0, 1, 2, 67),
.abcdei(2, 3, 4, 0, 1, 68),
.abcdei(1, 2, 3, 4, 0, 69),
.abcdei(0, 1, 2, 3, 4, 70),
.abcdei(4, 0, 1, 2, 3, 71),
.abcdei(3, 4, 0, 1, 2, 72),
.abcdei(2, 3, 4, 0, 1, 73),
.abcdei(1, 2, 3, 4, 0, 74),
.abcdei(0, 1, 2, 3, 4, 75),
.abcdei(4, 0, 1, 2, 3, 76),
.abcdei(3, 4, 0, 1, 2, 77),
.abcdei(2, 3, 4, 0, 1, 78),
.abcdei(1, 2, 3, 4, 0, 79),
};
inline for (round3) |r| {
const t = s[(r.i - 3) & 0xf] ^ s[(r.i - 8) & 0xf] ^ s[(r.i - 14) & 0xf] ^ s[(r.i - 16) & 0xf];
s[r.i & 0xf] = math.rotl(u32, t, @as(u32, 1));
v[r.e] = v[r.e] +% math.rotl(u32, v[r.a], @as(u32, 5)) +% 0xCA62C1D6 +% s[r.i & 0xf] +% (v[r.b] ^ v[r.c] ^ v[r.d]);
v[r.b] = math.rotl(u32, v[r.b], @as(u32, 30));
}
d_s[0] +%= v[0];
d_s[1] +%= v[1];
d_s[2] +%= v[2];
d_s[3] +%= v[3];
d_s[4] +%= v[4];
}
pub fn writable(sha1: *Sha1, buffer: []u8) std.io.BufferedWriter {
return .{
.unbuffered_writer = .{
.context = sha1,
.vtable = &.{
.writeSplat = writeSplat,
.writeFile = std.io.Writer.unimplementedWriteFile,
},
},
.buffer = buffer,
};
}
fn writeSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) std.io.Writer.Error!usize {
const sha1: *Sha1 = @ptrCast(@alignCast(context));
const start_total = sha1.total_len;
if (sha1.buf_end == 0) {
try writeSplatAligned(sha1, data, splat);
const n: usize = @intCast(sha1.total_len - start_total);
if (n > 0) return n;
}
for (data[0 .. data.len - 1]) |slice| {
const copy_len = @min(slice.len, sha1.buf.len - sha1.buf_end);
@memcpy(sha1.buf[sha1.buf_end..][0..copy_len], slice[0..copy_len]);
sha1.total_len += copy_len;
if (sha1.buf.len - sha1.buf_end - copy_len == 0) {
round(&sha1.s, &sha1.buf);
sha1.buf_end = 0;
return @intCast(sha1.total_len - start_total);
}
sha1.buf_end = @intCast(sha1.buf_end + copy_len);
}
const slice = data[data.len - 1];
for (0..splat) |_| {
const copy_len = @min(slice.len, sha1.buf.len - sha1.buf_end);
@memcpy(sha1.buf[sha1.buf_end..][0..copy_len], slice[0..copy_len]);
sha1.total_len += copy_len;
if (sha1.buf.len - sha1.buf_end - copy_len == 0) {
round(&sha1.s, &sha1.buf);
sha1.buf_end = 0;
return @intCast(sha1.total_len - start_total);
}
sha1.buf_end = @intCast(sha1.buf_end + copy_len);
}
return @intCast(sha1.total_len - start_total);
}
fn writeSplatAligned(sha1: *Sha1, data: []const []const u8, splat: usize) std.io.Writer.Error!void {
assert(sha1.buf_end == 0);
for (data[0 .. data.len - 1]) |slice| {
var off: usize = 0;
while (off < slice.len) {
if (off + 64 > slice.len) {
sha1.total_len += off;
return;
}
round(&sha1.s, slice[off..][0..64]);
off += 64;
}
sha1.total_len += off;
}
const last = data[data.len - 1];
if (last.len * splat < 64) return;
if (last.len == 1) {
@memset(&sha1.buf, last[0]);
for (0..splat / 64) |_| round(&sha1.s, &sha1.buf);
sha1.total_len += (splat / 64) * 64;
return;
}
if (last.len >= 64) {
for (0..splat) |_| {
var off: usize = 0;
while (off < last.len) {
if (off + 64 > last.len) {
sha1.total_len += off;
return;
}
round(&sha1.s, last[off..][0..64]);
off += 64;
}
}
sha1.total_len += last.len * splat;
return;
}
// Opportunity: if last.len is less than 64, we could fill up the buffer
// with the pattern repeated then do rounds.
}
const RoundParam = struct {
a: usize,
b: usize,
c: usize,
d: usize,
e: usize,
i: u32,
fn abcdei(a: usize, b: usize, c: usize, d: usize, e: usize, i: u32) RoundParam {
return .{
.a = a,
.b = b,
.c = c,
.d = d,
.e = e,
.i = i,
};
}
};
const htest = @import("test.zig");
test "sha1 single" {
try htest.assertEqualHash(Sha1, "da39a3ee5e6b4b0d3255bfef95601890afd80709", "");
try htest.assertEqualHash(Sha1, "a9993e364706816aba3e25717850c26c9cd0d89d", "abc");
try htest.assertEqualHash(Sha1, "a49b2446a02c645bf419f995b67091253a04a259", "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu");
}
test "sha1 streaming" {
var h = Sha1.init(.{});
var out: [20]u8 = undefined;
h.final(&out);
try htest.assertEqual("da39a3ee5e6b4b0d3255bfef95601890afd80709", out[0..]);
h = Sha1.init(.{});
h.update("abc");
h.final(&out);
try htest.assertEqual("a9993e364706816aba3e25717850c26c9cd0d89d", out[0..]);
h = Sha1.init(.{});
h.update("a");
h.update("b");
h.update("c");
h.final(&out);
try htest.assertEqual("a9993e364706816aba3e25717850c26c9cd0d89d", out[0..]);
}
test "sha1 aligned final" {
var block = [_]u8{0} ** Sha1.block_length;
var out: [Sha1.digest_length]u8 = undefined;
var h = Sha1.init(.{});
h.update(&block);
h.final(out[0..]);
}
test "splat" {
var vecs = [_][]const u8{
"hello",
"abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstuyyyyyyyyyyyyyyyyyyyyyyyyy",
"x",
};
const splat_len = 512;
const update_result = r: {
var sha1: Sha1 = .init(.{});
sha1.update(vecs[0]);
sha1.update(vecs[1]);
var buffer: [splat_len]u8 = undefined;
@memset(&buffer, vecs[2][0]);
sha1.update(&buffer);
break :r sha1.finalResult();
};
const stream_result = r: {
var sha1: Sha1 = .init(.{});
var bw = sha1.writable(&.{});
try bw.writeSplatAll(&vecs, splat_len);
try std.testing.expectEqual(vecs[0].len + vecs[1].len + vecs[2].len * splat_len, sha1.total_len);
break :r sha1.finalResult();
};
try std.testing.expectEqualSlices(u8, &update_result, &stream_result);
}

12
lib/std/crypto/aegis.zig
View File

@ -801,18 +801,6 @@ fn AegisMac(comptime T: type) type {
ctx.update(msg);
ctx.final(out);
}
pub const Error = error{};
pub const Writer = std.io.Writer(*Mac, Error, write);
fn write(self: *Mac, bytes: []const u8) Error!usize {
self.update(bytes);
return bytes.len;
}
pub fn writer(self: *Mac) Writer {
return .{ .context = self };
}
};
}

12
lib/std/crypto/blake2.zig
View File

@ -185,18 +185,6 @@ pub fn Blake2s(comptime out_bits: usize) type {
r.* ^= v[i] ^ v[i + 8];
}
}
pub const Error = error{};
pub const Writer = std.io.Writer(*Self, Error, write);
fn write(self: *Self, bytes: []const u8) Error!usize {
self.update(bytes);
return bytes.len;
}
pub fn writer(self: *Self) Writer {
return .{ .context = self };
}
};
}

12
lib/std/crypto/blake3.zig
View File

@ -474,18 +474,6 @@ pub const Blake3 = struct {
}
output.rootOutputBytes(out_slice);
}
pub const Error = error{};
pub const Writer = std.io.Writer(*Blake3, Error, write);
fn write(self: *Blake3, bytes: []const u8) Error!usize {
self.update(bytes);
return bytes.len;
}
pub fn writer(self: *Blake3) Writer {
return .{ .context = self };
}
};
// Use named type declarations to workaround crash with anonymous structs (issue #4373).

23
lib/std/crypto/codecs/asn1.zig
View File

@ -90,39 +90,32 @@ pub const Tag = struct {
};
}
pub fn encode(self: Tag, writer: anytype) @TypeOf(writer).Error!void {
var tag1 = FirstTag{
pub fn encode(self: Tag, writer: *std.io.BufferedWriter) std.io.Writer.Error!void {
var tag1: FirstTag = .{
.number = undefined,
.constructed = self.constructed,
.class = self.class,
};
var buffer: [3]u8 = undefined;
var stream = std.io.fixedBufferStream(&buffer);
var writer2 = stream.writer();
switch (@intFromEnum(self.number)) {
0...std.math.maxInt(u5) => |n| {
tag1.number = @intCast(n);
writer2.writeByte(@bitCast(tag1)) catch unreachable;
try writer.writeByte(@bitCast(tag1));
},
std.math.maxInt(u5) + 1...std.math.maxInt(u7) => |n| {
tag1.number = 15;
const tag2 = NextTag{ .number = @intCast(n), .continues = false };
writer2.writeByte(@bitCast(tag1)) catch unreachable;
writer2.writeByte(@bitCast(tag2)) catch unreachable;
try writer.writeByte(@bitCast(tag1));
try writer.writeByte(@bitCast(tag2));
},
else => |n| {
tag1.number = 15;
const tag2 = NextTag{ .number = @intCast(n >> 7), .continues = true };
const tag3 = NextTag{ .number = @truncate(n), .continues = false };
writer2.writeByte(@bitCast(tag1)) catch unreachable;
writer2.writeByte(@bitCast(tag2)) catch unreachable;
writer2.writeByte(@bitCast(tag3)) catch unreachable;
try writer.writeByte(@bitCast(tag1));
try writer.writeByte(@bitCast(tag2));
try writer.writeByte(@bitCast(tag3));
},
}
_ = try writer.write(stream.getWritten());
}
const FirstTag = packed struct(u8) { number: u5, constructed: bool, class: Tag.Class };

37
lib/std/crypto/codecs/asn1/Oid.zig
View File

@ -4,9 +4,12 @@
//! organizations, or policy documents.
encoded: []const u8,
pub const InitError = std.fmt.ParseIntError || error{MissingPrefix} || std.io.FixedBufferStream(u8).WriteError;
pub const EncodeError = error{
WriteFailed,
MissingPrefix,
};
pub fn fromDot(dot_notation: []const u8, out: []u8) InitError!Oid {
pub fn encode(dot_notation: []const u8, out: *std.io.BufferedWriter) EncodeError!void {
var split = std.mem.splitScalar(u8, dot_notation, '.');
const first_str = split.next() orelse return error.MissingPrefix;
const second_str = split.next() orelse return error.MissingPrefix;
@ -14,10 +17,7 @@ pub fn fromDot(dot_notation: []const u8, out: []u8) InitError!Oid {
const first = try std.fmt.parseInt(u8, first_str, 10);
const second = try std.fmt.parseInt(u8, second_str, 10);
var stream = std.io.fixedBufferStream(out);
var writer = stream.writer();
try writer.writeByte(first * 40 + second);
try out.writeByte(first * 40 + second);
var i: usize = 1;
while (split.next()) |s| {
@ -28,16 +28,26 @@ pub fn fromDot(dot_notation: []const u8, out: []u8) InitError!Oid {
const place = std.math.pow(Arc, encoding_base, n_bytes - @as(Arc, @intCast(j)));
const digit: u8 = @intCast(@divFloor(parsed, place));
try writer.writeByte(digit | 0x80);
try out.writeByte(digit | 0x80);
parsed -= digit * place;
i += 1;
}
try writer.writeByte(@intCast(parsed));
try out.writeByte(@intCast(parsed));
i += 1;
}
}
return .{ .encoded = stream.getWritten() };
pub const InitError = std.fmt.ParseIntError || error{ MissingPrefix, BufferTooSmall };
pub fn fromDot(dot_notation: []const u8, out: []u8) InitError!Oid {
var bw: std.io.BufferedWriter = undefined;
bw.initFixed(out);
encode(dot_notation, &bw) catch |err| switch (err) {
error.WriteFailed => return error.BufferTooSmall,
else => |e| return e,
};
return .{ .encoded = bw.getWritten() };
}
test fromDot {
@ -48,7 +58,7 @@ test fromDot {
}
}
pub fn toDot(self: Oid, writer: anytype) @TypeOf(writer).Error!void {
pub fn toDot(self: Oid, writer: *std.io.BufferedWriter) std.io.Writer.Error!void {
const encoded = self.encoded;
const first = @divTrunc(encoded[0], 40);
const second = encoded[0] - first * 40;
@ -80,9 +90,10 @@ test toDot {
var buf: [256]u8 = undefined;
for (test_cases) |t| {
var stream = std.io.fixedBufferStream(&buf);
try toDot(Oid{ .encoded = t.encoded }, stream.writer());
try std.testing.expectEqualStrings(t.dot_notation, stream.getWritten());
var bw: std.io.BufferedWriter = undefined;
bw.initFixed(&buf);
try toDot(Oid{ .encoded = t.encoded }, &bw);
try std.testing.expectEqualStrings(t.dot_notation, bw.getWritten());
}
}

8
lib/std/crypto/phc_encoding.zig
View File

@ -196,9 +196,11 @@ pub fn serialize(params: anytype, str: []u8) Error![]const u8 {
/// Compute the number of bytes required to serialize `params`
pub fn calcSize(params: anytype) usize {
var buf = io.countingWriter(io.null_writer);
serializeTo(params, buf.writer()) catch unreachable;
return @as(usize, @intCast(buf.bytes_written));
var null_writer: std.io.Writer.Null = .{};
var trash: [128]u8 = undefined;
var bw = null_writer.writable(&trash);
serializeTo(params, &bw) catch unreachable;
return bw.count;
}
fn serializeTo(params: anytype, out: *std.io.BufferedWriter) !void {

319
lib/std/crypto/sha1.zig
View File

@ -1,319 +0,0 @@
const std = @import("../std.zig");
const mem = std.mem;
const math = std.math;
const RoundParam = struct {
a: usize,
b: usize,
c: usize,
d: usize,
e: usize,
i: u32,
};
fn roundParam(a: usize, b: usize, c: usize, d: usize, e: usize, i: u32) RoundParam {
return RoundParam{
.a = a,
.b = b,
.c = c,
.d = d,
.e = e,
.i = i,
};
}
/// The SHA-1 function is now considered cryptographically broken.
/// Namely, it is feasible to find multiple inputs producing the same hash.
/// For a fast-performing, cryptographically secure hash function, see SHA512/256, BLAKE2 or BLAKE3.
pub const Sha1 = struct {
const Self = @This();
pub const block_length = 64;
pub const digest_length = 20;
pub const Options = struct {};
s: [5]u32,
// Streaming Cache
buf: [64]u8 = undefined,
buf_len: u8 = 0,
total_len: u64 = 0,
pub fn init(options: Options) Self {
_ = options;
return Self{
.s = [_]u32{
0x67452301,
0xEFCDAB89,
0x98BADCFE,
0x10325476,
0xC3D2E1F0,
},
};
}
pub fn hash(b: []const u8, out: *[digest_length]u8, options: Options) void {
var d = Sha1.init(options);
d.update(b);
d.final(out);
}
pub fn update(d: *Self, b: []const u8) void {
var off: usize = 0;
// Partial buffer exists from previous update. Copy into buffer then hash.
if (d.buf_len != 0 and d.buf_len + b.len >= 64) {
off += 64 - d.buf_len;
@memcpy(d.buf[d.buf_len..][0..off], b[0..off]);
d.round(d.buf[0..]);
d.buf_len = 0;
}
// Full middle blocks.
while (off + 64 <= b.len) : (off += 64) {
d.round(b[off..][0..64]);
}
// Copy any remainder for next pass.
@memcpy(d.buf[d.buf_len..][0 .. b.len - off], b[off..]);
d.buf_len += @as(u8, @intCast(b[off..].len));
d.total_len += b.len;
}
pub fn peek(d: Self) [digest_length]u8 {
var copy = d;
return copy.finalResult();
}
pub fn final(d: *Self, out: *[digest_length]u8) void {
// The buffer here will never be completely full.
@memset(d.buf[d.buf_len..], 0);
// Append padding bits.
d.buf[d.buf_len] = 0x80;
d.buf_len += 1;
// > 448 mod 512 so need to add an extra round to wrap around.
if (64 - d.buf_len < 8) {
d.round(d.buf[0..]);
@memset(d.buf[0..], 0);
}
// Append message length.
var i: usize = 1;
var len = d.total_len >> 5;
d.buf[63] = @as(u8, @intCast(d.total_len & 0x1f)) << 3;
while (i < 8) : (i += 1) {
d.buf[63 - i] = @as(u8, @intCast(len & 0xff));
len >>= 8;
}
d.round(d.buf[0..]);
for (d.s, 0..) |s, j| {
mem.writeInt(u32, out[4 * j ..][0..4], s, .big);
}
}
pub fn finalResult(d: *Self) [digest_length]u8 {
var result: [digest_length]u8 = undefined;
d.final(&result);
return result;
}
fn round(d: *Self, b: *const [64]u8) void {
var s: [16]u32 = undefined;
var v: [5]u32 = [_]u32{
d.s[0],
d.s[1],
d.s[2],
d.s[3],
d.s[4],
};
const round0a = comptime [_]RoundParam{
roundParam(0, 1, 2, 3, 4, 0),
roundParam(4, 0, 1, 2, 3, 1),
roundParam(3, 4, 0, 1, 2, 2),
roundParam(2, 3, 4, 0, 1, 3),
roundParam(1, 2, 3, 4, 0, 4),
roundParam(0, 1, 2, 3, 4, 5),
roundParam(4, 0, 1, 2, 3, 6),
roundParam(3, 4, 0, 1, 2, 7),
roundParam(2, 3, 4, 0, 1, 8),
roundParam(1, 2, 3, 4, 0, 9),
roundParam(0, 1, 2, 3, 4, 10),
roundParam(4, 0, 1, 2, 3, 11),
roundParam(3, 4, 0, 1, 2, 12),
roundParam(2, 3, 4, 0, 1, 13),
roundParam(1, 2, 3, 4, 0, 14),
roundParam(0, 1, 2, 3, 4, 15),
};
inline for (round0a) |r| {
s[r.i] = mem.readInt(u32, b[r.i * 4 ..][0..4], .big);
v[r.e] = v[r.e] +% math.rotl(u32, v[r.a], @as(u32, 5)) +% 0x5A827999 +% s[r.i & 0xf] +% ((v[r.b] & v[r.c]) | (~v[r.b] & v[r.d]));
v[r.b] = math.rotl(u32, v[r.b], @as(u32, 30));
}
const round0b = comptime [_]RoundParam{
roundParam(4, 0, 1, 2, 3, 16),
roundParam(3, 4, 0, 1, 2, 17),
roundParam(2, 3, 4, 0, 1, 18),
roundParam(1, 2, 3, 4, 0, 19),
};
inline for (round0b) |r| {
const t = s[(r.i - 3) & 0xf] ^ s[(r.i - 8) & 0xf] ^ s[(r.i - 14) & 0xf] ^ s[(r.i - 16) & 0xf];
s[r.i & 0xf] = math.rotl(u32, t, @as(u32, 1));
v[r.e] = v[r.e] +% math.rotl(u32, v[r.a], @as(u32, 5)) +% 0x5A827999 +% s[r.i & 0xf] +% ((v[r.b] & v[r.c]) | (~v[r.b] & v[r.d]));
v[r.b] = math.rotl(u32, v[r.b], @as(u32, 30));
}
const round1 = comptime [_]RoundParam{
roundParam(0, 1, 2, 3, 4, 20),
roundParam(4, 0, 1, 2, 3, 21),
roundParam(3, 4, 0, 1, 2, 22),
roundParam(2, 3, 4, 0, 1, 23),
roundParam(1, 2, 3, 4, 0, 24),
roundParam(0, 1, 2, 3, 4, 25),
roundParam(4, 0, 1, 2, 3, 26),
roundParam(3, 4, 0, 1, 2, 27),
roundParam(2, 3, 4, 0, 1, 28),
roundParam(1, 2, 3, 4, 0, 29),
roundParam(0, 1, 2, 3, 4, 30),
roundParam(4, 0, 1, 2, 3, 31),
roundParam(3, 4, 0, 1, 2, 32),
roundParam(2, 3, 4, 0, 1, 33),
roundParam(1, 2, 3, 4, 0, 34),
roundParam(0, 1, 2, 3, 4, 35),
roundParam(4, 0, 1, 2, 3, 36),
roundParam(3, 4, 0, 1, 2, 37),
roundParam(2, 3, 4, 0, 1, 38),
roundParam(1, 2, 3, 4, 0, 39),
};
inline for (round1) |r| {
const t = s[(r.i - 3) & 0xf] ^ s[(r.i - 8) & 0xf] ^ s[(r.i - 14) & 0xf] ^ s[(r.i - 16) & 0xf];
s[r.i & 0xf] = math.rotl(u32, t, @as(u32, 1));
v[r.e] = v[r.e] +% math.rotl(u32, v[r.a], @as(u32, 5)) +% 0x6ED9EBA1 +% s[r.i & 0xf] +% (v[r.b] ^ v[r.c] ^ v[r.d]);
v[r.b] = math.rotl(u32, v[r.b], @as(u32, 30));
}
const round2 = comptime [_]RoundParam{
roundParam(0, 1, 2, 3, 4, 40),
roundParam(4, 0, 1, 2, 3, 41),
roundParam(3, 4, 0, 1, 2, 42),
roundParam(2, 3, 4, 0, 1, 43),
roundParam(1, 2, 3, 4, 0, 44),
roundParam(0, 1, 2, 3, 4, 45),
roundParam(4, 0, 1, 2, 3, 46),
roundParam(3, 4, 0, 1, 2, 47),
roundParam(2, 3, 4, 0, 1, 48),
roundParam(1, 2, 3, 4, 0, 49),
roundParam(0, 1, 2, 3, 4, 50),
roundParam(4, 0, 1, 2, 3, 51),
roundParam(3, 4, 0, 1, 2, 52),
roundParam(2, 3, 4, 0, 1, 53),
roundParam(1, 2, 3, 4, 0, 54),
roundParam(0, 1, 2, 3, 4, 55),
roundParam(4, 0, 1, 2, 3, 56),
roundParam(3, 4, 0, 1, 2, 57),
roundParam(2, 3, 4, 0, 1, 58),
roundParam(1, 2, 3, 4, 0, 59),
};
inline for (round2) |r| {
const t = s[(r.i - 3) & 0xf] ^ s[(r.i - 8) & 0xf] ^ s[(r.i - 14) & 0xf] ^ s[(r.i - 16) & 0xf];
s[r.i & 0xf] = math.rotl(u32, t, @as(u32, 1));
v[r.e] = v[r.e] +% math.rotl(u32, v[r.a], @as(u32, 5)) +% 0x8F1BBCDC +% s[r.i & 0xf] +% ((v[r.b] & v[r.c]) ^ (v[r.b] & v[r.d]) ^ (v[r.c] & v[r.d]));
v[r.b] = math.rotl(u32, v[r.b], @as(u32, 30));
}
const round3 = comptime [_]RoundParam{
roundParam(0, 1, 2, 3, 4, 60),
roundParam(4, 0, 1, 2, 3, 61),
roundParam(3, 4, 0, 1, 2, 62),
roundParam(2, 3, 4, 0, 1, 63),
roundParam(1, 2, 3, 4, 0, 64),
roundParam(0, 1, 2, 3, 4, 65),
roundParam(4, 0, 1, 2, 3, 66),
roundParam(3, 4, 0, 1, 2, 67),
roundParam(2, 3, 4, 0, 1, 68),
roundParam(1, 2, 3, 4, 0, 69),
roundParam(0, 1, 2, 3, 4, 70),
roundParam(4, 0, 1, 2, 3, 71),
roundParam(3, 4, 0, 1, 2, 72),
roundParam(2, 3, 4, 0, 1, 73),
roundParam(1, 2, 3, 4, 0, 74),
roundParam(0, 1, 2, 3, 4, 75),
roundParam(4, 0, 1, 2, 3, 76),
roundParam(3, 4, 0, 1, 2, 77),
roundParam(2, 3, 4, 0, 1, 78),
roundParam(1, 2, 3, 4, 0, 79),
};
inline for (round3) |r| {
const t = s[(r.i - 3) & 0xf] ^ s[(r.i - 8) & 0xf] ^ s[(r.i - 14) & 0xf] ^ s[(r.i - 16) & 0xf];
s[r.i & 0xf] = math.rotl(u32, t, @as(u32, 1));
v[r.e] = v[r.e] +% math.rotl(u32, v[r.a], @as(u32, 5)) +% 0xCA62C1D6 +% s[r.i & 0xf] +% (v[r.b] ^ v[r.c] ^ v[r.d]);
v[r.b] = math.rotl(u32, v[r.b], @as(u32, 30));
}
d.s[0] +%= v[0];
d.s[1] +%= v[1];
d.s[2] +%= v[2];
d.s[3] +%= v[3];
d.s[4] +%= v[4];
}
pub const Error = error{};
pub const Writer = std.io.Writer(*Self, Error, write);
fn write(self: *Self, bytes: []const u8) Error!usize {
self.update(bytes);
return bytes.len;
}
pub fn writer(self: *Self) Writer {
return .{ .context = self };
}
};
const htest = @import("test.zig");
test "sha1 single" {
try htest.assertEqualHash(Sha1, "da39a3ee5e6b4b0d3255bfef95601890afd80709", "");
try htest.assertEqualHash(Sha1, "a9993e364706816aba3e25717850c26c9cd0d89d", "abc");
try htest.assertEqualHash(Sha1, "a49b2446a02c645bf419f995b67091253a04a259", "abcdefghbcdefghicdefghijdefghijkefghijklfghijklmghijklmnhijklmnoijklmnopjklmnopqklmnopqrlmnopqrsmnopqrstnopqrstu");
}
test "sha1 streaming" {
var h = Sha1.init(.{});
var out: [20]u8 = undefined;
h.final(&out);
try htest.assertEqual("da39a3ee5e6b4b0d3255bfef95601890afd80709", out[0..]);
h = Sha1.init(.{});
h.update("abc");
h.final(&out);
try htest.assertEqual("a9993e364706816aba3e25717850c26c9cd0d89d", out[0..]);
h = Sha1.init(.{});
h.update("a");
h.update("b");
h.update("c");
h.final(&out);
try htest.assertEqual("a9993e364706816aba3e25717850c26c9cd0d89d", out[0..]);
}
test "sha1 aligned final" {
var block = [_]u8{0} ** Sha1.block_length;
var out: [Sha1.digest_length]u8 = undefined;
var h = Sha1.init(.{});
h.update(&block);
h.final(out[0..]);
}

40
lib/std/crypto/sha2.zig
View File

@ -95,14 +95,19 @@ fn Sha2x32(comptime iv: Iv32, digest_bits: comptime_int) type {
pub const Options = struct {};
s: [8]u32 align(16),
// Streaming Cache
buf: [64]u8 = undefined,
buf_len: u8 = 0,
total_len: u64 = 0,
/// Streaming Cache
buf: [64]u8,
buf_len: u8,
total_len: u64,
pub fn init(options: Options) Self {
_ = options;
return Self{ .s = iv };
return .{
.s = iv,
.buf = undefined,
.buf_len = 0,
.total_len = 0,
};
}
pub fn hash(b: []const u8, out: *[digest_length]u8, options: Options) void {
@ -377,16 +382,25 @@ fn Sha2x32(comptime iv: Iv32, digest_bits: comptime_int) type {
for (&d.s, v) |*dv, vv| dv.* +%= vv;
}
pub const Error = error{};
pub const Writer = std.io.Writer(*Self, Error, write);
fn write(self: *Self, bytes: []const u8) Error!usize {
self.update(bytes);
return bytes.len;
pub fn writable(this: *@This(), buffer: []u8) std.io.BufferedWriter {
return .{
.unbuffered_writer = .{
.context = this,
.vtable = &.{
.writeSplat = writeSplat,
.writeFile = std.io.Writer.unimplementedWriteFile,
},
},
.buffer = buffer,
};
}
pub fn writer(self: *Self) Writer {
return .{ .context = self };
fn writeSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) std.io.Writer.Error!usize {
const this: *@This() = @ptrCast(@alignCast(context));
const start_total = this.total_len;
for (data[0 .. data.len - 1]) |slice| this.update(slice);
for (0..splat) |_| this.update(data[data.len - 1]);
return @intCast(this.total_len - start_total);
}
};
}

60
lib/std/crypto/sha3.zig
View File

@ -80,18 +80,6 @@ pub fn Keccak(comptime f: u11, comptime output_bits: u11, comptime default_delim
self.st.pad();
self.st.squeeze(out[0..]);
}
pub const Error = error{};
pub const Writer = std.io.Writer(*Self, Error, write);
fn write(self: *Self, bytes: []const u8) Error!usize {
self.update(bytes);
return bytes.len;
}
pub fn writer(self: *Self) Writer {
return .{ .context = self };
}
};
}
@ -191,18 +179,6 @@ fn ShakeLike(comptime security_level: u11, comptime default_delim: u8, comptime
pub fn fillBlock(self: *Self) void {
self.st.fillBlock();
}
pub const Error = error{};
pub const Writer = std.io.Writer(*Self, Error, write);
fn write(self: *Self, bytes: []const u8) Error!usize {
self.update(bytes);
return bytes.len;
}
pub fn writer(self: *Self) Writer {
return .{ .context = self };
}
};
}
@ -284,18 +260,6 @@ fn CShakeLike(comptime security_level: u11, comptime default_delim: u8, comptime
pub fn fillBlock(self: *Self) void {
self.shaker.fillBlock();
}
pub const Error = error{};
pub const Writer = std.io.Writer(*Self, Error, write);
fn write(self: *Self, bytes: []const u8) Error!usize {
self.update(bytes);
return bytes.len;
}
pub fn writer(self: *Self) Writer {
return .{ .context = self };
}
};
}
@ -390,18 +354,6 @@ fn KMacLike(comptime security_level: u11, comptime default_delim: u8, comptime r
ctx.update(msg);
ctx.final(out);
}
pub const Error = error{};
pub const Writer = std.io.Writer(*Self, Error, write);
fn write(self: *Self, bytes: []const u8) Error!usize {
self.update(bytes);
return bytes.len;
}
pub fn writer(self: *Self) Writer {
return .{ .context = self };
}
};
}
@ -482,18 +434,6 @@ fn TupleHashLike(comptime security_level: u11, comptime default_delim: u8, compt
}
self.cshaker.squeeze(out);
}
pub const Error = error{};
pub const Writer = std.io.Writer(*Self, Error, write);
fn write(self: *Self, bytes: []const u8) Error!usize {
self.update(bytes);
return bytes.len;
}
pub fn writer(self: *Self) Writer {
return .{ .context = self };
}
};
}

42
lib/std/crypto/siphash.zig
View File

@ -238,48 +238,6 @@ fn SipHash(comptime T: type, comptime c_rounds: usize, comptime d_rounds: usize)
pub fn toInt(msg: []const u8, key: *const [key_length]u8) T {
return State.hash(msg, key);
}
pub fn writer(self: *Self) std.io.Writer {
return .{
.context = self,
.vtable = &.{
.writeSplat = &writeSplat,
.writeFile = &writeFile,
},
};
}
fn writeSplat(ctx: ?*anyopaque, data: []const []const u8, splat: usize) std.io.Writer.Error!usize {
const self: *Self = @alignCast(@ptrCast(ctx));
var len: usize = 0;
for (data[0 .. data.len - 1]) |slice| {
self.update(slice);
len += slice.len;
}
{
const slice = data[data.len - 1];
for (0..splat) |_| self.update(slice);
len += slice.len * splat;
}
return len;
}
fn writeFile(
ctx: ?*anyopaque,
file: std.fs.File,
offset: std.io.Writer.Offset,
limit: std.io.Writer.Limit,
headers_and_trailers: []const []const u8,
headers_len: usize,
) std.io.Writer.Error!usize {
_ = ctx;
_ = file;
_ = offset;
_ = limit;
_ = headers_and_trailers;
_ = headers_len;
@panic("TODO");
}
};
}

2
lib/std/debug/Dwarf.zig
View File

@ -2176,7 +2176,7 @@ pub const ElfModule = struct {
parent_mapped_mem: ?[]align(std.heap.page_size_min) const u8,
elf_filename: ?[]const u8,
) LoadError!Dwarf.ElfModule {
if (expected_crc) |crc| if (crc != std.hash.crc.Crc32.hash(mapped_mem)) return error.InvalidDebugInfo;
if (expected_crc) |crc| if (crc != std.hash.Crc32.hash(mapped_mem)) return error.InvalidDebugInfo;
const hdr: *const elf.Ehdr = @ptrCast(&mapped_mem[0]);
if (!mem.eql(u8, hdr.e_ident[0..4], elf.MAGIC)) return error.InvalidElfMagic;

34
lib/std/fs/File.zig
View File

@ -934,6 +934,10 @@ pub const Reader = struct {
};
}
pub fn readable(r: *Reader, buffer: []u8) std.io.BufferedReader {
return interface(r).buffered(buffer);
}
pub fn getSize(r: *Reader) GetEndPosError!u64 {
return r.size orelse {
if (r.size_err) |err| return err;
@ -1228,6 +1232,7 @@ pub const Writer = struct {
pos: u64 = 0,
sendfile_err: ?SendfileError = null,
read_err: ?ReadError = null,
seek_err: ?SeekError = null,
pub const Mode = Reader.Mode;
@ -1250,6 +1255,20 @@ pub const Writer = struct {
};
}
pub fn writable(w: *Writer, buffer: []u8) std.io.BufferedWriter {
return interface(w).buffered(buffer);
}
pub fn moveToReader(w: *Writer) Reader {
defer w.* = undefined;
return .{
.file = w.file,
.mode = w.mode,
.pos = w.pos,
.seek_err = w.seek_err,
};
}
pub fn writeSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) std.io.Writer.Error!usize {
const w: *Writer = @ptrCast(@alignCast(context));
const handle = w.file.handle;
@ -1347,6 +1366,21 @@ pub const Writer = struct {
}
return error.Unimplemented;
}
pub fn seekTo(w: *Writer, offset: u64) SeekError!void {
if (w.seek_err) |err| return err;
switch (w.mode) {
.positional, .positional_reading => {
w.pos = offset;
},
.streaming, .streaming_reading => {
posix.lseek_SET(w.file.handle, offset) catch |err| {
w.seek_err = err;
return err;
};
},
}
}
};
/// Defaults to positional reading; falls back to streaming.

3
lib/std/hash.zig
View File

@ -6,9 +6,8 @@ pub const autoHash = auto_hash.autoHash;
pub const autoHashStrat = auto_hash.hash;
pub const Strategy = auto_hash.HashStrategy;
// pub for polynomials + generic crc32 construction
pub const crc = @import("hash/crc.zig");
pub const Crc32 = crc.Crc32;
pub const Crc32 = crc.Crc32IsoHdlc;
const fnv = @import("hash/fnv.zig");
pub const Fnv1a_32 = fnv.Fnv1a_32;

356
lib/std/hash/crc.zig
View File

File diff suppressed because it is too large Load Diff

112
lib/std/hash/crc/impl.zig
View File

@ -1,112 +0,0 @@
// There is a generic CRC implementation "Crc()" which can be parameterized via
// the Algorithm struct for a plethora of uses.
//
// The primary interface for all of the standard CRC algorithms is the
// generated file "crc.zig", which uses the implementation code here to define
// many standard CRCs.
const std = @import("std");
pub fn Algorithm(comptime W: type) type {
return struct {
polynomial: W,
initial: W,
reflect_input: bool,
reflect_output: bool,
xor_output: W,
};
}
pub fn Crc(comptime W: type, comptime algorithm: Algorithm(W)) type {
return struct {
const Self = @This();
const I = if (@bitSizeOf(W) < 8) u8 else W;
const lookup_table = blk: {
@setEvalBranchQuota(2500);
const poly = if (algorithm.reflect_input)
@bitReverse(@as(I, algorithm.polynomial)) >> (@bitSizeOf(I) - @bitSizeOf(W))
else
@as(I, algorithm.polynomial) << (@bitSizeOf(I) - @bitSizeOf(W));
var table: [256]I = undefined;
for (&table, 0..) |*e, i| {
var crc: I = i;
if (algorithm.reflect_input) {
var j: usize = 0;
while (j < 8) : (j += 1) {
crc = (crc >> 1) ^ ((crc & 1) * poly);
}
} else {
crc <<= @bitSizeOf(I) - 8;
var j: usize = 0;
while (j < 8) : (j += 1) {
crc = (crc << 1) ^ (((crc >> (@bitSizeOf(I) - 1)) & 1) * poly);
}
}
e.* = crc;
}
break :blk table;
};
crc: I,
pub fn init() Self {
const initial = if (algorithm.reflect_input)
@bitReverse(@as(I, algorithm.initial)) >> (@bitSizeOf(I) - @bitSizeOf(W))
else
@as(I, algorithm.initial) << (@bitSizeOf(I) - @bitSizeOf(W));
return Self{ .crc = initial };
}
inline fn tableEntry(index: I) I {
return lookup_table[@as(u8, @intCast(index & 0xFF))];
}
pub fn update(self: *Self, bytes: []const u8) void {
var i: usize = 0;
if (@bitSizeOf(I) <= 8) {
while (i < bytes.len) : (i += 1) {
self.crc = tableEntry(self.crc ^ bytes[i]);
}
} else if (algorithm.reflect_input) {
while (i < bytes.len) : (i += 1) {
const table_index = self.crc ^ bytes[i];
self.crc = tableEntry(table_index) ^ (self.crc >> 8);
}
} else {
while (i < bytes.len) : (i += 1) {
const table_index = (self.crc >> (@bitSizeOf(I) - 8)) ^ bytes[i];
self.crc = tableEntry(table_index) ^ (self.crc << 8);
}
}
}
pub fn final(self: Self) W {
var c = self.crc;
if (algorithm.reflect_input != algorithm.reflect_output) {
c = @bitReverse(c);
}
if (!algorithm.reflect_output) {
c >>= @bitSizeOf(I) - @bitSizeOf(W);
}
return @as(W, @intCast(c ^ algorithm.xor_output));
}
pub fn hash(bytes: []const u8) W {
var c = Self.init();
c.update(bytes);
return c.final();
}
};
}
pub const Polynomial = enum(u32) {
IEEE = @compileError("use Crc with algorithm .Crc32IsoHdlc"),
Castagnoli = @compileError("use Crc with algorithm .Crc32Iscsi"),
Koopman = @compileError("use Crc with algorithm .Crc32Koopman"),
_,
};
pub const Crc32WithPoly = @compileError("use Crc instead");
pub const Crc32SmallWithPoly = @compileError("use Crc instead");

36
lib/std/io/BufferedReader.zig
View File

@ -35,6 +35,10 @@ pub fn bufferContents(br: *BufferedReader) []u8 {
return br.buffer[br.seek..br.end];
}
pub fn bufferedLen(br: *const BufferedReader) usize {
return br.end - br.seek;
}
/// Although `BufferedReader` can easily satisfy the `Reader` interface, it's
/// generally more practical to pass a `BufferedReader` instance itself around,
/// since it will result in fewer calls across vtable boundaries.
@ -49,6 +53,10 @@ pub fn reader(br: *BufferedReader) Reader {
};
}
pub fn hashed(br: *BufferedReader, hasher: anytype) Reader.Hashed(@TypeOf(hasher)) {
return .{ .in = br, .hasher = hasher };
}
/// Equivalent semantics to `std.io.Reader.VTable.readVec`.
pub fn readVec(br: *BufferedReader, data: []const []u8) Reader.Error!usize {
return readVecLimit(br, data, .unlimited);
@ -401,6 +409,30 @@ pub fn readSliceShort(br: *BufferedReader, buffer: []u8) Reader.ShortError!usize
}
}
/// Fill `buffer` with the next `buffer.len` bytes from the stream, advancing
/// the seek position.
///
/// Invalidates previously returned values from `peek`.
///
/// If the provided buffer cannot be filled completely, `error.EndOfStream` is
/// returned instead.
///
/// The function is inline to avoid the dead code in case `endian` is
/// comptime-known and matches host endianness.
///
/// See also:
/// * `readSlice`
/// * `readSliceEndianAlloc`
pub inline fn readSliceEndian(
br: *BufferedReader,
comptime Elem: type,
buffer: []Elem,
endian: std.builtin.Endian,
) Reader.Error!void {
try readSlice(br, @ptrCast(buffer));
if (native_endian != endian) for (buffer) |*elem| std.mem.byteSwapAllFields(Elem, elem);
}
pub const ReadAllocError = Reader.Error || Allocator.Error;
/// The function is inline to avoid the dead code in case `endian` is
@ -408,14 +440,14 @@ pub const ReadAllocError = Reader.Error || Allocator.Error;
pub inline fn readSliceEndianAlloc(
br: *BufferedReader,
allocator: Allocator,
Elem: type,
comptime Elem: type,
len: usize,
endian: std.builtin.Endian,
) ReadAllocError![]Elem {
const dest = try allocator.alloc(Elem, len);
errdefer allocator.free(dest);
try readSlice(br, @ptrCast(dest));
if (native_endian != endian) std.mem.byteSwapAllFields(Elem, dest);
if (native_endian != endian) for (dest) |*elem| std.mem.byteSwapAllFields(Elem, elem);
return dest;
}

40
lib/std/io/BufferedWriter.zig
View File

@ -58,6 +58,10 @@ pub fn initFixed(bw: *BufferedWriter, buffer: []u8) void {
};
}
pub fn hashed(bw: *BufferedWriter, hasher: anytype) Writer.Hashed(@TypeOf(hasher)) {
return .{ .out = bw, .hasher = hasher };
}
/// This function is available when using `initFixed`.
pub fn getWritten(bw: *const BufferedWriter) []u8 {
assert(bw.unbuffered_writer.vtable == &fixed_vtable);
@ -157,7 +161,7 @@ pub fn advance(bw: *BufferedWriter, n: usize) void {
}
/// The `data` parameter is mutable because this function needs to mutate the
/// fields in order to handle partial writes from `Writer.VTable.writeVec`.
/// fields in order to handle partial writes from `Writer.VTable.writeSplat`.
pub fn writeVecAll(bw: *BufferedWriter, data: [][]const u8) Writer.Error!void {
var index: usize = 0;
var truncate: usize = 0;
@ -175,6 +179,36 @@ pub fn writeVecAll(bw: *BufferedWriter, data: [][]const u8) Writer.Error!void {
}
}
/// The `data` parameter is mutable because this function needs to mutate the
/// fields in order to handle partial writes from `Writer.VTable.writeSplat`.
pub fn writeSplatAll(bw: *BufferedWriter, data: [][]const u8, splat: usize) Writer.Error!void {
var index: usize = 0;
var truncate: usize = 0;
var remaining_splat = splat;
while (index + 1 < data.len) {
{
const untruncated = data[index];
data[index] = untruncated[truncate..];
defer data[index] = untruncated;
truncate += try bw.writeSplat(data[index..], remaining_splat);
}
while (truncate >= data[index].len) {
if (index + 1 < data.len) {
truncate -= data[index].len;
index += 1;
} else {
const last = data[data.len - 1];
remaining_splat -= @divExact(truncate, last.len);
while (remaining_splat > 0) {
const n = try bw.writeSplat(data[data.len - 1 ..][0..1], remaining_splat);
remaining_splat -= @divExact(n, last.len);
}
return;
}
}
}
}
/// If the number of bytes to write based on `data` and `splat` fits inside
/// `unusedCapacitySlice`, this function is guaranteed to not fail, not call
/// into the underlying writer, and return the full number of bytes.
@ -443,7 +477,7 @@ pub fn splatBytesAll(bw: *BufferedWriter, bytes: []const u8, splat: usize) Write
/// Writes the same slice many times, allowing short writes.
///
/// Does maximum of one underlying `Writer.VTable.writeVec`.
/// Does maximum of one underlying `Writer.VTable.writeSplat`.
pub fn splatBytes(bw: *BufferedWriter, bytes: []const u8, n: usize) Writer.Error!usize {
return passthruWriteSplat(bw, &.{bytes}, n);
}
@ -621,7 +655,7 @@ pub const WriteFileOptions = struct {
/// size here will save one syscall.
limit: Writer.Limit = .unlimited,
/// Headers and trailers must be passed together so that in case `len` is
/// zero, they can be forwarded directly to `Writer.VTable.writeVec`.
/// zero, they can be forwarded directly to `Writer.VTable.writeSplat`.
///
/// The parameter is mutable because this function needs to mutate the
/// fields in order to handle partial writes from `Writer.VTable.writeFile`.

68
lib/std/io/Reader.zig
View File

@ -331,3 +331,71 @@ test "readAlloc when the backing reader provides one byte at a time" {
defer std.testing.allocator.free(res);
try std.testing.expectEqualStrings(str, res);
}
/// Provides a `Reader` implementation by passing data from an underlying
/// reader through `Hasher.update`.
///
/// The underlying reader is best unbuffered.
///
/// This implementation makes suboptimal buffering decisions due to being
/// generic. A better solution will involve creating a reader for each hash
/// function, where the discard buffer can be tailored to the hash
/// implementation details.
pub fn Hashed(comptime Hasher: type) type {
return struct {
in: *BufferedReader,
hasher: Hasher,
pub fn readable(this: *@This(), buffer: []u8) BufferedReader {
return .{
.unbuffered_reader = .{
.context = this,
.vtable = &.{
.read = @This().read,
.readVec = @This().readVec,
.discard = @This().discard,
},
},
.buffer = buffer,
.end = 0,
.seek = 0,
};
}
fn read(context: ?*anyopaque, bw: *BufferedWriter, limit: Limit) RwError!usize {
const this: *@This() = @alignCast(@ptrCast(context));
const slice = limit.slice(try bw.writableSliceGreedy(1));
const n = try this.in.readVec(&.{slice});
this.hasher.update(slice[0..n]);
bw.advance(n);
return n;
}
fn discard(context: ?*anyopaque, limit: Limit) Error!usize {
const this: *@This() = @alignCast(@ptrCast(context));
var bw = this.hasher.writable(&.{});
const n = this.in.read(&bw, limit) catch |err| switch (err) {
error.WriteFailed => unreachable,
else => |e| return e,
};
return n;
}
fn readVec(context: ?*anyopaque, data: []const []u8) Error!usize {
const this: *@This() = @alignCast(@ptrCast(context));
const n = try this.in.readVec(data);
var remaining: usize = n;
for (data) |slice| {
if (remaining < slice.len) {
this.hasher.update(slice[0..remaining]);
return n;
} else {
remaining -= slice.len;
this.hasher.update(slice);
}
}
assert(remaining == 0);
return n;
}
};
}

88
lib/std/io/Writer.zig
View File

@ -136,8 +136,8 @@ pub fn failingWriteSplat(context: ?*anyopaque, data: []const []const u8, splat:
pub fn failingWriteFile(
context: ?*anyopaque,
file: std.fs.File,
offset: std.io.Writer.Offset,
limit: std.io.Writer.Limit,
offset: Offset,
limit: Limit,
headers_and_trailers: []const []const u8,
headers_len: usize,
) FileError!usize {
@ -158,11 +158,13 @@ pub const failing: Writer = .{
},
};
/// For use when the `Writer` implementation can cannot offer a more efficient
/// implementation than a basic read/write loop on the file.
pub fn unimplementedWriteFile(
context: ?*anyopaque,
file: std.fs.File,
offset: std.io.Writer.Offset,
limit: std.io.Writer.Limit,
offset: Offset,
limit: Limit,
headers_and_trailers: []const []const u8,
headers_len: usize,
) FileError!usize {
@ -175,6 +177,84 @@ pub fn unimplementedWriteFile(
return error.Unimplemented;
}
/// Provides a `Writer` implementation based on calling `Hasher.update`, sending
/// all data also to an underlying `std.io.BufferedWriter`.
///
/// When using this, the underlying writer is best unbuffered because all
/// writes are passed on directly to it.
///
/// This implementation makes suboptimal buffering decisions due to being
/// generic. A better solution will involve creating a writer for each hash
/// function, where the splat buffer can be tailored to the hash implementation
/// details.
pub fn Hashed(comptime Hasher: type) type {
return struct {
out: *std.io.BufferedWriter,
hasher: Hasher,
pub fn writable(this: *@This(), buffer: []u8) std.io.BufferedWriter {
return .{
.unbuffered_writer = .{
.context = this,
.vtable = &.{
.writeSplat = @This().writeSplat,
.writeFile = Writer.unimplementedWriteFile,
},
},
.buffer = buffer,
};
}
fn writeSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) Writer.Error!usize {
const this: *@This() = @alignCast(@ptrCast(context));
const n = try this.out.writeSplat(data, splat);
const short_data = data[0 .. data.len - @intFromBool(splat == 0)];
var remaining: usize = n;
for (short_data) |slice| {
if (remaining < slice.len) {
this.hasher.update(slice[0..remaining]);
return n;
} else {
remaining -= slice.len;
this.hasher.update(slice);
}
}
const remaining_splat = switch (splat) {
0, 1 => {
assert(remaining == 0);
return n;
},
else => splat - 1,
};
const last = data[data.len - 1];
assert(remaining == remaining_splat * last.len);
switch (last.len) {
0 => {
assert(remaining == 0);
return n;
},
1 => {
var buffer: [64]u8 = undefined;
@memset(&buffer, last[0]);
while (remaining > 0) {
const update_len = @min(remaining, buffer.len);
this.hasher.update(buffer[0..update_len]);
remaining -= update_len;
}
return n;
},
else => {},
}
while (remaining > 0) {
const update_len = @min(remaining, last.len);
this.hasher.update(last[0..update_len]);
remaining -= update_len;
}
return n;
}
};
}
test {
_ = Null;
}

4
lib/std/io/Writer/Null.zig
View File

@ -19,6 +19,10 @@ pub fn writer(nw: *NullWriter) Writer {
};
}
pub fn writable(nw: *NullWriter, buffer: []u8) std.io.BufferedWriter {
return writer(nw).buffered(buffer);
}
fn writeSplat(context: ?*anyopaque, data: []const []const u8, splat: usize) Writer.Error!usize {
_ = context;
const headers = data[0 .. data.len - 1];

4
lib/std/mem.zig
View File

@ -2196,7 +2196,9 @@ pub fn byteSwapAllFields(comptime S: type, ptr: *S) void {
}
}
},
else => @compileError("byteSwapAllFields expects a struct or array as the first argument"),
else => {
ptr.* = @byteSwap(ptr.*);
},
}
}

333
src/Package/Fetch/git.zig
View File

@ -11,6 +11,7 @@ const Allocator = mem.Allocator;
const Sha1 = std.crypto.hash.Sha1;
const Sha256 = std.crypto.hash.sha2.Sha256;
const assert = std.debug.assert;
const zlib = std.compress.zlib;
/// The ID of a Git object.
pub const Oid = union(Format) {
@ -52,7 +53,6 @@ pub const Oid = union(Format) {
};
}
// Must be public for use from HashedReader and HashedWriter.
pub fn update(hasher: *Hasher, b: []const u8) void {
switch (hasher.*) {
inline else => |*inner| inner.update(b),
@ -64,6 +64,12 @@ pub const Oid = union(Format) {
inline else => |*inner, tag| @unionInit(Oid, @tagName(tag), inner.finalResult()),
};
}
pub fn writable(hasher: *Hasher, buffer: []u8) std.io.BufferedWriter {
return switch (hasher.*) {
inline else => |*inner| inner.writable(buffer),
};
}
};
pub fn fromBytes(oid_format: Format, bytes: []const u8) Oid {
@ -73,9 +79,18 @@ pub const Oid = union(Format) {
};
}
pub fn readBytes(oid_format: Format, reader: anytype) @TypeOf(reader).NoEofError!Oid {
pub fn readBytes(oid_format: Format, reader: *std.io.BufferedReader) std.io.Reader.Error!Oid {
return switch (oid_format) {
inline else => |tag| @unionInit(Oid, @tagName(tag), try reader.readBytesNoEof(tag.byteLength())),
.sha1 => {
var result: Oid = .{ .sha1 = undefined };
try reader.readSlice(&result.sha1);
return result;
},
.sha256 => {
var result: Oid = .{ .sha256 = undefined };
try reader.readSlice(&result.sha256);
return result;
},
};
}
@ -167,8 +182,15 @@ pub const Diagnostics = struct {
pub const Repository = struct {
odb: Odb,
pub fn init(allocator: Allocator, format: Oid.Format, pack_file: std.fs.File, index_file: std.fs.File) !Repository {
return .{ .odb = try Odb.init(allocator, format, pack_file, index_file) };
pub fn init(
repo: *Repository,
allocator: Allocator,
format: Oid.Format,
pack_file: *std.fs.File.Reader,
index_file: *std.fs.File.Reader,
) !void {
repo.* = .{ .odb = undefined };
try repo.odb.init(allocator, format, pack_file, index_file);
}
pub fn deinit(repository: *Repository) void {
@ -337,24 +359,32 @@ pub const Repository = struct {
/// [pack-format](https://git-scm.com/docs/pack-format).
const Odb = struct {
format: Oid.Format,
pack_file: std.fs.File,
pack_file: *std.fs.File.Reader,
index_header: IndexHeader,
index_file: std.fs.File,
index_file: *std.fs.File.Reader,
cache: ObjectCache = .{},
allocator: Allocator,
/// Initializes the database from open pack and index files.
fn init(allocator: Allocator, format: Oid.Format, pack_file: std.fs.File, index_file: std.fs.File) !Odb {
fn init(
odb: *Odb,
allocator: Allocator,
format: Oid.Format,
pack_file: *std.fs.File.Reader,
index_file: *std.fs.File.Reader,
) !void {
try pack_file.seekTo(0);
try index_file.seekTo(0);
const index_header = try IndexHeader.read(index_file.reader());
return .{
odb.* = .{
.format = format,
.pack_file = pack_file,
.index_header = index_header,
.index_header = undefined,
.index_file = index_file,
.allocator = allocator,
};
var buffer: [1032]u8 = undefined;
var index_file_br = index_file.readable(&buffer);
try odb.index_header.read(&index_file_br);
}
fn deinit(odb: *Odb) void {
@ -364,27 +394,30 @@ const Odb = struct {
/// Reads the object at the current position in the database.
fn readObject(odb: *Odb) !Object {
var base_offset = try odb.pack_file.getPos();
var pack_read_buffer: [64]u8 = undefined;
var base_offset = odb.pack_file.pos;
var pack_br = odb.pack_file.readable(&pack_read_buffer);
var base_header: EntryHeader = undefined;
var delta_offsets: std.ArrayListUnmanaged(u64) = .empty;
defer delta_offsets.deinit(odb.allocator);
const base_object = while (true) {
if (odb.cache.get(base_offset)) |base_object| break base_object;
base_header = try EntryHeader.read(odb.format, odb.pack_file.reader());
base_header = try EntryHeader.read(odb.format, &pack_br);
switch (base_header) {
.ofs_delta => |ofs_delta| {
try delta_offsets.append(odb.allocator, base_offset);
base_offset = std.math.sub(u64, base_offset, ofs_delta.offset) catch return error.InvalidFormat;
try odb.pack_file.seekTo(base_offset);
pack_br = odb.pack_file.readable(&pack_read_buffer);
},
.ref_delta => |ref_delta| {
try delta_offsets.append(odb.allocator, base_offset);
try odb.seekOid(ref_delta.base_object);
base_offset = try odb.pack_file.getPos();
base_offset = odb.pack_file.pos - pack_br.bufferedLen();
},
else => {
const base_data = try readObjectRaw(odb.allocator, odb.pack_file.reader(), base_header.uncompressedLength());
const base_data = try readObjectRaw(odb.allocator, &pack_br, base_header.uncompressedLength());
errdefer odb.allocator.free(base_data);
const base_object: Object = .{ .type = base_header.objectType(), .data = base_data };
try odb.cache.put(odb.allocator, base_offset, base_object);
@ -414,7 +447,8 @@ const Odb = struct {
const found_index = while (start_index < end_index) {
const mid_index = start_index + (end_index - start_index) / 2;
try odb.index_file.seekTo(IndexHeader.size + mid_index * oid_length);
const mid_oid = try Oid.readBytes(odb.format, odb.index_file.reader());
var br = odb.index_file.interface().unbuffered();
const mid_oid = try Oid.readBytes(odb.format, &br);
switch (mem.order(u8, mid_oid.slice(), oid.slice())) {
.lt => start_index = mid_index + 1,
.gt => end_index = mid_index,
@ -424,13 +458,16 @@ const Odb = struct {
const n_objects = odb.index_header.fan_out_table[255];
const offset_values_start = IndexHeader.size + n_objects * (oid_length + 4);
var buffer: [8]u8 = undefined;
try odb.index_file.seekTo(offset_values_start + found_index * 4);
const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try odb.index_file.reader().readInt(u32, .big));
var br = odb.index_file.interface().buffered(&buffer);
const l1_offset: packed struct { value: u31, big: bool } = @bitCast(try br.takeInt(u32, .big));
const pack_offset = pack_offset: {
if (l1_offset.big) {
const l2_offset_values_start = offset_values_start + n_objects * 4;
try odb.index_file.seekTo(l2_offset_values_start + l1_offset.value * 4);
break :pack_offset try odb.index_file.reader().readInt(u64, .big);
br = odb.index_file.interface().buffered(&buffer);
break :pack_offset try br.takeInt(u64, .big);
} else {
break :pack_offset l1_offset.value;
}
@ -556,7 +593,7 @@ const Packet = union(enum) {
const max_data_length = 65516;
/// Reads a packet in pkt-line format.
fn read(reader: anytype, buf: *[max_data_length]u8) !Packet {
fn read(reader: *std.io.BufferedReader, buf: *[max_data_length]u8) !Packet {
const length = std.fmt.parseUnsigned(u16, &try reader.readBytesNoEof(4), 16) catch return error.InvalidPacket;
switch (length) {
0 => return .flush,
@ -571,7 +608,7 @@ const Packet = union(enum) {
}
/// Writes a packet in pkt-line format.
fn write(packet: Packet, writer: anytype) !void {
fn write(packet: Packet, writer: *std.io.BufferedWriter) !void {
switch (packet) {
.flush => try writer.writeAll("0000"),
.delimiter => try writer.writeAll("0001"),
@ -1070,21 +1107,12 @@ const PackHeader = struct {
const signature = "PACK";
const supported_version = 2;
fn read(reader: anytype) !PackHeader {
const actual_signature = reader.readBytesNoEof(4) catch |e| switch (e) {
error.EndOfStream => return error.InvalidHeader,
else => |other| return other,
};
if (!mem.eql(u8, &actual_signature, signature)) return error.InvalidHeader;
const version = reader.readInt(u32, .big) catch |e| switch (e) {
error.EndOfStream => return error.InvalidHeader,
else => |other| return other,
};
fn read(reader: *std.io.BufferedReader) !PackHeader {
const actual_signature = try reader.take(4);
if (!mem.eql(u8, actual_signature, signature)) return error.InvalidHeader;
const version = try reader.takeInt(u32, .big);
if (version != supported_version) return error.UnsupportedVersion;
const total_objects = reader.readInt(u32, .big) catch |e| switch (e) {
error.EndOfStream => return error.InvalidHeader,
else => |other| return other,
};
const total_objects = try reader.takeInt(u32, .big);
return .{ .total_objects = total_objects };
}
};
@ -1133,12 +1161,9 @@ const EntryHeader = union(Type) {
};
}
fn read(format: Oid.Format, reader: anytype) !EntryHeader {
fn read(format: Oid.Format, reader: *std.io.BufferedReader) !EntryHeader {
const InitialByte = packed struct { len: u4, type: u3, has_next: bool };
const initial: InitialByte = @bitCast(reader.readByte() catch |e| switch (e) {
error.EndOfStream => return error.InvalidFormat,
else => |other| return other,
});
const initial: InitialByte = @bitCast(try reader.takeByte());
const rest_len = if (initial.has_next) try readSizeVarInt(reader) else 0;
var uncompressed_length: u64 = initial.len;
uncompressed_length |= std.math.shlExact(u64, rest_len, 4) catch return error.InvalidFormat;
@ -1162,25 +1187,25 @@ const EntryHeader = union(Type) {
}
};
fn readSizeVarInt(r: anytype) !u64 {
fn readSizeVarInt(r: *std.io.BufferedReader) !u64 {
const Byte = packed struct { value: u7, has_next: bool };
var b: Byte = @bitCast(try r.readByte());
var b: Byte = @bitCast(try r.takeByte());
var value: u64 = b.value;
var shift: u6 = 0;
while (b.has_next) {
b = @bitCast(try r.readByte());
b = @bitCast(try r.takeByte());
shift = std.math.add(u6, shift, 7) catch return error.InvalidFormat;
value |= @as(u64, b.value) << shift;
}
return value;
}
fn readOffsetVarInt(r: anytype) !u64 {
fn readOffsetVarInt(r: *std.io.BufferedReader) !u64 {
const Byte = packed struct { value: u7, has_next: bool };
var b: Byte = @bitCast(try r.readByte());
var b: Byte = @bitCast(try r.takeByte());
var value: u64 = b.value;
while (b.has_next) {
b = @bitCast(try r.readByte());
b = @bitCast(try r.takeByte());
value = std.math.shlExact(u64, value + 1, 7) catch return error.InvalidFormat;
value |= b.value;
}
@ -1194,19 +1219,12 @@ const IndexHeader = struct {
const supported_version = 2;
const size = 4 + 4 + @sizeOf([256]u32);
fn read(reader: anytype) !IndexHeader {
var header_bytes = try reader.readBytesNoEof(size);
if (!mem.eql(u8, header_bytes[0..4], signature)) return error.InvalidHeader;
const version = mem.readInt(u32, header_bytes[4..8], .big);
fn read(index_header: *IndexHeader, br: *std.io.BufferedReader) !void {
const sig = try br.take(4);
if (!mem.eql(u8, sig, signature)) return error.InvalidHeader;
const version = try br.takeInt(u32, .big);
if (version != supported_version) return error.UnsupportedVersion;
var fan_out_table: [256]u32 = undefined;
var fan_out_table_stream = std.io.fixedBufferStream(header_bytes[8..]);
const fan_out_table_reader = fan_out_table_stream.reader();
for (&fan_out_table) |*entry| {
entry.* = fan_out_table_reader.readInt(u32, .big) catch unreachable;
}
return .{ .fan_out_table = fan_out_table };
try br.readSliceEndian(u32, &index_header.fan_out_table, .big);
}
};
@ -1217,7 +1235,12 @@ const IndexEntry = struct {
/// Writes out a version 2 index for the given packfile, as documented in
/// [pack-format](https://git-scm.com/docs/pack-format).
pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, index_writer: anytype) !void {
pub fn indexPack(
allocator: Allocator,
format: Oid.Format,
pack: *std.fs.File.Reader,
index_writer: *std.fs.File.Writer,
) !void {
try pack.seekTo(0);
var index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry) = .empty;
@ -1270,8 +1293,10 @@ pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, in
}
@memset(fan_out_table[fan_out_index..], count);
var index_hashed_writer = std.compress.hashedWriter(index_writer, Oid.Hasher.init(format));
const writer = index_hashed_writer.writer();
var index_writer_bw = index_writer.writable(&.{});
var index_hashed_writer = index_writer_bw.hashed(Oid.Hasher.init(format));
var write_buffer: [256]u8 = undefined;
var writer = index_hashed_writer.writable(&write_buffer);
try writer.writeAll(IndexHeader.signature);
try writer.writeInt(u32, IndexHeader.supported_version, .big);
for (fan_out_table) |fan_out_entry| {
@ -1303,8 +1328,9 @@ pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, in
}
try writer.writeAll(pack_checksum.slice());
try writer.flush();
const index_checksum = index_hashed_writer.hasher.finalResult();
try index_writer.writeAll(index_checksum.slice());
try index_writer_bw.writeAll(index_checksum.slice());
}
/// Performs the first pass over the packfile data for index construction.
@ -1314,50 +1340,46 @@ pub fn indexPack(allocator: Allocator, format: Oid.Format, pack: std.fs.File, in
fn indexPackFirstPass(
allocator: Allocator,
format: Oid.Format,
pack: std.fs.File,
pack: *std.fs.File.Reader,
index_entries: *std.AutoHashMapUnmanaged(Oid, IndexEntry),
pending_deltas: *std.ArrayListUnmanaged(IndexEntry),
) !Oid {
var pack_buffered_reader = std.io.bufferedReader(pack.reader());
var pack_counting_reader = std.io.countingReader(pack_buffered_reader.reader());
var pack_hashed_reader = std.compress.hashedReader(pack_counting_reader.reader(), Oid.Hasher.init(format));
const pack_reader = pack_hashed_reader.reader();
var pack_br = pack.readable(&.{});
var pack_hashed_reader = pack_br.hashed(Oid.Hasher.init(format));
var pack_buffer: [2048]u8 = undefined; // Reasonably large buffer for file system.
var pack_hashed_br = pack_hashed_reader.readable(&pack_buffer);
const pack_header = try PackHeader.read(pack_reader);
const pack_header = try PackHeader.read(&pack_hashed_br);
var current_entry: u32 = 0;
while (current_entry < pack_header.total_objects) : (current_entry += 1) {
const entry_offset = pack_counting_reader.bytes_read;
var entry_crc32_reader = std.compress.hashedReader(pack_reader, std.hash.Crc32.init());
const entry_header = try EntryHeader.read(format, entry_crc32_reader.reader());
for (0..pack_header.total_objects) |_| {
const entry_offset = pack.pos - pack_hashed_br.bufferContents().len;
var entry_crc32_reader = pack_hashed_br.hashed(std.hash.Crc32.init());
var entry_buffer: [64]u8 = undefined; // Buffer only needed for loading EntryHeader.
var entry_crc32_br = entry_crc32_reader.readable(&entry_buffer);
const entry_header = try EntryHeader.read(format, &entry_crc32_br);
var entry_decompress_stream: zlib.Decompressor = .init(&entry_crc32_br);
// Decompress uses large output buffer; no input buffer needed.
var entry_decompress_br = entry_decompress_stream.readable(&.{});
switch (entry_header) {
.commit, .tree, .blob, .tag => |object| {
var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader());
var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());
var entry_hashed_writer = std.compress.hashedWriter(std.io.null_writer, Oid.Hasher.init(format));
const entry_writer = entry_hashed_writer.writer();
var oid_hasher = Oid.Hasher.init(format);
var oid_hasher_buffer: [zlib.max_window_len]u8 = undefined;
var oid_hasher_bw = oid_hasher.writable(&oid_hasher_buffer);
// The object header is not included in the pack data but is
// part of the object's ID
try entry_writer.print("{s} {}\x00", .{ @tagName(entry_header), object.uncompressed_length });
var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
try fifo.pump(entry_counting_reader.reader(), entry_writer);
if (entry_counting_reader.bytes_read != object.uncompressed_length) {
return error.InvalidObject;
}
const oid = entry_hashed_writer.hasher.finalResult();
// part of the object's ID.
try oid_hasher_bw.print("{s} {d}\x00", .{ @tagName(entry_header), object.uncompressed_length });
const n = try entry_decompress_br.readRemaining(&oid_hasher_bw);
if (n != object.uncompressed_length) return error.InvalidObject;
try oid_hasher_bw.flush();
const oid = oid_hasher.finalResult();
try index_entries.put(allocator, oid, .{
.offset = entry_offset,
.crc32 = entry_crc32_reader.hasher.final(),
});
},
inline .ofs_delta, .ref_delta => |delta| {
var entry_decompress_stream = std.compress.zlib.decompressor(entry_crc32_reader.reader());
var entry_counting_reader = std.io.countingReader(entry_decompress_stream.reader());
var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init();
try fifo.pump(entry_counting_reader.reader(), std.io.null_writer);
if (entry_counting_reader.bytes_read != delta.uncompressed_length) {
return error.InvalidObject;
}
const n = try entry_decompress_br.discardRemaining();
if (n != delta.uncompressed_length) return error.InvalidObject;
try pending_deltas.append(allocator, .{
.offset = entry_offset,
.crc32 = entry_crc32_reader.hasher.final(),
@ -1367,15 +1389,11 @@ fn indexPackFirstPass(
}
const pack_checksum = pack_hashed_reader.hasher.finalResult();
const recorded_checksum = try Oid.readBytes(format, pack_buffered_reader.reader());
const recorded_checksum = try Oid.readBytes(format, &pack_br);
if (!mem.eql(u8, pack_checksum.slice(), recorded_checksum.slice())) {
return error.CorruptedPack;
}
_ = pack_reader.readByte() catch |e| switch (e) {
error.EndOfStream => return pack_checksum,
else => |other| return other,
};
return error.InvalidFormat;
return pack_checksum;
}
/// Attempts to determine the final object ID of the given deltified object.
@ -1384,7 +1402,7 @@ fn indexPackFirstPass(
fn indexPackHashDelta(
allocator: Allocator,
format: Oid.Format,
pack: std.fs.File,
pack: *std.fs.File.Reader,
delta: IndexEntry,
index_entries: std.AutoHashMapUnmanaged(Oid, IndexEntry),
cache: *ObjectCache,
@ -1398,7 +1416,9 @@ fn indexPackHashDelta(
if (cache.get(base_offset)) |base_object| break base_object;
try pack.seekTo(base_offset);
base_header = try EntryHeader.read(format, pack.reader());
var pack_read_buffer: [64]u8 = undefined;
var pack_br = pack.readable(&pack_read_buffer);
base_header = try EntryHeader.read(format, &pack_br);
switch (base_header) {
.ofs_delta => |ofs_delta| {
try delta_offsets.append(allocator, base_offset);
@ -1409,7 +1429,7 @@ fn indexPackHashDelta(
base_offset = (index_entries.get(ref_delta.base_object) orelse return null).offset;
},
else => {
const base_data = try readObjectRaw(allocator, pack.reader(), base_header.uncompressedLength());
const base_data = try readObjectRaw(allocator, &pack_br, base_header.uncompressedLength());
errdefer allocator.free(base_data);
const base_object: Object = .{ .type = base_header.objectType(), .data = base_data };
try cache.put(allocator, base_offset, base_object);
@ -1421,9 +1441,12 @@ fn indexPackHashDelta(
const base_data = try resolveDeltaChain(allocator, format, pack, base_object, delta_offsets.items, cache);
var entry_hasher: Oid.Hasher = .init(format);
var entry_hashed_writer = std.compress.hashedWriter(std.io.null_writer, &entry_hasher);
try entry_hashed_writer.writer().print("{s} {}\x00", .{ @tagName(base_object.type), base_data.len });
entry_hasher.update(base_data);
var entry_hasher_buffer: [64]u8 = undefined;
var entry_hasher_bw = entry_hasher.writable(&entry_hasher_buffer);
// Writes to hashers cannot fail.
entry_hasher_bw.print("{s} {d}\x00", .{ @tagName(base_object.type), base_data.len }) catch unreachable;
entry_hasher_bw.writeAll(base_data) catch unreachable;
entry_hasher_bw.flush() catch unreachable;
return entry_hasher.finalResult();
}
@ -1434,7 +1457,7 @@ fn indexPackHashDelta(
fn resolveDeltaChain(
allocator: Allocator,
format: Oid.Format,
pack: std.fs.File,
pack: *std.fs.File.Reader,
base_object: Object,
delta_offsets: []const u64,
cache: *ObjectCache,
@ -1446,21 +1469,22 @@ fn resolveDeltaChain(
const delta_offset = delta_offsets[i];
try pack.seekTo(delta_offset);
const delta_header = try EntryHeader.read(format, pack.reader());
const delta_data = try readObjectRaw(allocator, pack.reader(), delta_header.uncompressedLength());
defer allocator.free(delta_data);
var delta_stream = std.io.fixedBufferStream(delta_data);
const delta_reader = delta_stream.reader();
_ = try readSizeVarInt(delta_reader); // base object size
const expanded_size = try readSizeVarInt(delta_reader);
var pack_read_buffer: [64]u8 = undefined;
var pack_br = pack.readable(&pack_read_buffer);
const delta_header = try EntryHeader.read(format, &pack_br);
_ = delta_header;
var delta_decompress: zlib.Decompressor = .init(&pack_br);
var delta_decompress_buffer: [zlib.max_window_len]u8 = undefined;
var delta_reader = delta_decompress.readable(&delta_decompress_buffer);
_ = try readSizeVarInt(&delta_reader); // base object size
const expanded_size = try readSizeVarInt(&delta_reader);
const expanded_alloc_size = std.math.cast(usize, expanded_size) orelse return error.ObjectTooLarge;
const expanded_data = try allocator.alloc(u8, expanded_alloc_size);
errdefer allocator.free(expanded_data);
var expanded_delta_stream = std.io.fixedBufferStream(expanded_data);
var base_stream = std.io.fixedBufferStream(base_data);
try expandDelta(&base_stream, delta_reader, expanded_delta_stream.writer());
if (expanded_delta_stream.pos != expanded_size) return error.InvalidObject;
var expanded_delta_stream: std.io.BufferedWriter = undefined;
expanded_delta_stream.initFixed(expanded_data);
try expandDelta(base_data, &delta_reader, &expanded_delta_stream);
if (expanded_delta_stream.end != expanded_size) return error.InvalidObject;
try cache.put(allocator, delta_offset, .{ .type = base_object.type, .data = expanded_data });
base_data = expanded_data;
@ -1468,31 +1492,23 @@ fn resolveDeltaChain(
return base_data;
}
/// Reads the complete contents of an object from `reader`. This function may
/// read more bytes than required from `reader`, so the reader position after
/// returning is not reliable.
fn readObjectRaw(allocator: Allocator, reader: anytype, size: u64) ![]u8 {
/// Reads the complete contents of an object from `reader`.
fn readObjectRaw(gpa: Allocator, reader: *std.io.BufferedReader, size: u64) ![]u8 {
const alloc_size = std.math.cast(usize, size) orelse return error.ObjectTooLarge;
var buffered_reader = std.io.bufferedReader(reader);
var decompress_stream = std.compress.zlib.decompressor(buffered_reader.reader());
const data = try allocator.alloc(u8, alloc_size);
errdefer allocator.free(data);
try decompress_stream.reader().readNoEof(data);
_ = decompress_stream.reader().readByte() catch |e| switch (e) {
error.EndOfStream => return data,
else => |other| return other,
};
return error.InvalidFormat;
var decompress: zlib.Decompressor = .init(reader);
var buffer: std.ArrayListUnmanaged(u8) = .empty;
defer buffer.deinit(gpa);
try decompress.reader().readRemainingArrayList(gpa, null, &buffer, .limited(alloc_size), zlib.max_window_len);
if (buffer.items.len < size) return error.EndOfStream;
return buffer.toOwnedSlice(gpa);
}
/// Expands delta data from `delta_reader` to `writer`. `base_object` must
/// support `reader` and `seekTo` (such as a `std.io.FixedBufferStream`).
///
/// The format of the delta data is documented in
/// [pack-format](https://git-scm.com/docs/pack-format).
fn expandDelta(base_object: anytype, delta_reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) !void {
fn expandDelta(base_object: []const u8, delta_reader: *std.io.BufferedReader, writer: *std.io.BufferedWriter) !void {
var base_offset: u32 = 0;
while (true) {
const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.readByte() catch |e| switch (e) {
const inst: packed struct { value: u7, copy: bool } = @bitCast(delta_reader.takeByte() catch |e| switch (e) {
error.EndOfStream => return,
else => |other| return other,
});
@ -1507,23 +1523,22 @@ fn expandDelta(base_object: anytype, delta_reader: *std.io.BufferedReader, write
size3: bool,
} = @bitCast(inst.value);
const offset_parts: packed struct { offset1: u8, offset2: u8, offset3: u8, offset4: u8 } = .{
.offset1 = if (available.offset1) try delta_reader.readByte() else 0,
.offset2 = if (available.offset2) try delta_reader.readByte() else 0,
.offset3 = if (available.offset3) try delta_reader.readByte() else 0,
.offset4 = if (available.offset4) try delta_reader.readByte() else 0,
.offset1 = if (available.offset1) try delta_reader.takeByte() else 0,
.offset2 = if (available.offset2) try delta_reader.takeByte() else 0,
.offset3 = if (available.offset3) try delta_reader.takeByte() else 0,
.offset4 = if (available.offset4) try delta_reader.takeByte() else 0,
};
const offset: u32 = @bitCast(offset_parts);
base_offset = @bitCast(offset_parts);
const size_parts: packed struct { size1: u8, size2: u8, size3: u8 } = .{
.size1 = if (available.size1) try delta_reader.readByte() else 0,
.size2 = if (available.size2) try delta_reader.readByte() else 0,
.size3 = if (available.size3) try delta_reader.readByte() else 0,
.size1 = if (available.size1) try delta_reader.takeByte() else 0,
.size2 = if (available.size2) try delta_reader.takeByte() else 0,
.size3 = if (available.size3) try delta_reader.takeByte() else 0,
};
var size: u24 = @bitCast(size_parts);
if (size == 0) size = 0x10000;
try base_object.seekTo(offset);
var base_object_br = base_object.reader();
try base_object_br.readAll(writer, .limited(size));
try writer.writeAll(base_object[base_offset..][0..size]);
base_offset += size;
} else if (inst.value != 0) {
try delta_reader.readAll(writer, .limited(inst.value));
} else {
@ -1557,7 +1572,8 @@ fn runRepositoryTest(comptime format: Oid.Format, head_commit: []const u8) !void
var index_file = try git_dir.dir.createFile("testrepo.idx", .{ .read = true });
defer index_file.close();
try indexPack(testing.allocator, format, pack_file, index_file.writer());
var index_file_writer = index_file.writer();
try indexPack(testing.allocator, format, pack_file, &index_file_writer);
// Arbitrary size limit on files read while checking the repository contents
// (all files in the test repo are known to be smaller than this)
@ -1571,7 +1587,8 @@ fn runRepositoryTest(comptime format: Oid.Format, head_commit: []const u8) !void
const testrepo_idx = @embedFile("git/testdata/testrepo-" ++ @tagName(format) ++ ".idx");
try testing.expectEqualSlices(u8, testrepo_idx, index_file_data);
var repository = try Repository.init(testing.allocator, format, pack_file, index_file);
var index_file_reader = index_file_writer.moveToReader();
var repository = try Repository.init(testing.allocator, format, pack_file, &index_file_reader);
defer repository.deinit();
var worktree = testing.tmpDir(.{ .iterate = true });
@ -1652,10 +1669,12 @@ test "SHA-256 packfile indexing and checkout" {
/// Checks out a commit of a packfile. Intended for experimenting with and
/// benchmarking possible optimizations to the indexing and checkout behavior.
pub fn main() !void {
const allocator = std.heap.c_allocator;
var debug_allocator: std.heap.DebugAllocator(.{}) = .init;
defer _ = debug_allocator.deinit();
const gpa = if (std.debug.runtime_safety) debug_allocator.allocator() else std.heap.smp_allocator;
const args = try std.process.argsAlloc(allocator);
defer std.process.argsFree(allocator, args);
const args = try std.process.argsAlloc(gpa);
defer std.process.argsFree(gpa, args);
if (args.len != 5) {
return error.InvalidArguments; // Arguments: format packfile commit worktree
}
@ -1674,15 +1693,17 @@ pub fn main() !void {
std.debug.print("Starting index...\n", .{});
var index_file = try git_dir.createFile("idx", .{ .read = true });
defer index_file.close();
var index_buffered_writer = std.io.bufferedWriter(index_file.writer());
try indexPack(allocator, format, pack_file, index_buffered_writer.writer());
try index_buffered_writer.flush();
var index_file_writer = index_file.writer();
var pack_file_reader = pack_file.reader();
try indexPack(gpa, format, &pack_file_reader, &index_file_writer);
try index_file.sync();
std.debug.print("Starting checkout...\n", .{});
var repository = try Repository.init(allocator, format, pack_file, index_file);
var index_file_reader = index_file_writer.moveToReader();
var repository: Repository = undefined;
try repository.init(gpa, format, &pack_file_reader, &index_file_reader);
defer repository.deinit();
var diagnostics: Diagnostics = .{ .allocator = allocator };
var diagnostics: Diagnostics = .{ .allocator = gpa };
defer diagnostics.deinit();
try repository.checkout(worktree, commit, &diagnostics);

7
src/main.zig
View File

@ -3330,12 +3330,11 @@ fn buildOutputType(
// for the hashing algorithm here and in the cache are the same.
// We are providing our own cache key, because this file has nothing
// to do with the cache manifest.
var hasher = Cache.Hasher.init("0123456789abcdef");
var file_writer = f.writer();
var file_writer_bw = file_writer.interface().unbuffered();
var hasher_writer = hasher.writer(&file_writer_bw);
var file_writer_bw = file_writer.writable(&.{});
var hasher_writer = file_writer_bw.hashed(Cache.Hasher.init("0123456789abcdef"));
var buffer: [1000]u8 = undefined;
var bw = hasher_writer.interface().buffered(&buffer);
var bw = hasher_writer.writable(&buffer);
bw.writeFileAll(.stdin(), .{}) catch |err| switch (err) {
error.WriteFailed => fatal("failed to write {s}: {s}", .{ dump_path, file_writer.err.? }),
else => fatal("failed to pipe stdin to {s}: {s}", .{ dump_path, err }),