Merge pull request #3106 from ziglang/hash-tooling-changes

Hash tooling changes

std/crypto/benchmark.zig

@@ -1,8 +1,10 @@
+// zig run benchmark.zig --release-fast --override-std-dir ..
+
 const builtin = @import("builtin");
-const std = @import("std");
+const std = @import("../std.zig");
 const time = std.time;
 const Timer = time.Timer;
-const crypto = @import("../crypto.zig");
+const crypto = std.crypto;
 
 const KiB = 1024;
 const MiB = 1024 * KiB;
@@ -14,7 +16,7 @@ const Crypto = struct {
     name: []const u8,
 };
 
-const hashes = []Crypto{
+const hashes = [_]Crypto{
     Crypto{ .ty = crypto.Md5, .name = "md5" },
     Crypto{ .ty = crypto.Sha1, .name = "sha1" },
     Crypto{ .ty = crypto.Sha256, .name = "sha256" },
@@ -45,7 +47,7 @@ pub fn benchmarkHash(comptime Hash: var, comptime bytes: comptime_int) !u64 {
     return throughput;
 }
 
-const macs = []Crypto{
+const macs = [_]Crypto{
     Crypto{ .ty = crypto.Poly1305, .name = "poly1305" },
     Crypto{ .ty = crypto.HmacMd5, .name = "hmac-md5" },
     Crypto{ .ty = crypto.HmacSha1, .name = "hmac-sha1" },
@@ -75,7 +77,7 @@ pub fn benchmarkMac(comptime Mac: var, comptime bytes: comptime_int) !u64 {
     return throughput;
 }
 
-const exchanges = []Crypto{Crypto{ .ty = crypto.X25519, .name = "x25519" }};
+const exchanges = [_]Crypto{Crypto{ .ty = crypto.X25519, .name = "x25519" }};
 
 pub fn benchmarkKeyExchange(comptime DhKeyExchange: var, comptime exchange_count: comptime_int) !u64 {
     std.debug.assert(DhKeyExchange.minimum_key_length >= DhKeyExchange.secret_length);
@@ -135,13 +137,16 @@ pub fn main() !void {
 
     var buffer: [1024]u8 = undefined;
     var fixed = std.heap.FixedBufferAllocator.init(buffer[0..]);
-    const args = try std.os.argsAlloc(&fixed.allocator);
+    const args = try std.process.argsAlloc(&fixed.allocator);
 
     var filter: ?[]u8 = "";
 
     var i: usize = 1;
     while (i < args.len) : (i += 1) {
-        if (std.mem.eql(u8, args[i], "--seed")) {
+        if (std.mem.eql(u8, args[i], "--mode")) {
+            try stdout.print("{}\n", builtin.mode);
+            return;
+        } else if (std.mem.eql(u8, args[i], "--seed")) {
             i += 1;
             if (i == args.len) {
                 usage();

std/crypto/blake2.zig

@@ -269,8 +269,8 @@ pub const Blake2b512 = Blake2b(512);
 fn Blake2b(comptime out_len: usize) type {
     return struct {
         const Self = @This();
-        const block_length = 128;
+        pub const block_length = 128;
-        const digest_length = out_len / 8;
+        pub const digest_length = out_len / 8;
 
         const iv = [8]u64{
             0x6a09e667f3bcc908,

std/crypto/sha2.zig

@@ -420,8 +420,8 @@ pub const Sha512 = Sha2_64(Sha512Params);
 fn Sha2_64(comptime params: Sha2Params64) type {
     return struct {
         const Self = @This();
-        const block_length = 128;
+        pub const block_length = 128;
-        const digest_length = params.out_len / 8;
+        pub const digest_length = params.out_len / 8;
 
         s: [8]u64,
         // Streaming Cache

std/hash/benchmark.zig

@@ -0,0 +1,273 @@
+// zig run benchmark.zig --release-fast --override-std-dir ..
+
+const builtin = @import("builtin");
+const std = @import("std");
+const time = std.time;
+const Timer = time.Timer;
+const hash = std.hash;
+
+const KiB = 1024;
+const MiB = 1024 * KiB;
+const GiB = 1024 * MiB;
+
+var prng = std.rand.DefaultPrng.init(0);
+
+const Hash = struct {
+    ty: type,
+    name: []const u8,
+    has_iterative_api: bool = true,
+    init_u8s: ?[]const u8 = null,
+    init_u64: ?u64 = null,
+};
+
+const siphash_key = "0123456789abcdef";
+
+const hashes = [_]Hash{
+    Hash{
+        .ty = hash.Wyhash,
+        .name = "wyhash",
+        .init_u64 = 0,
+    },
+    Hash{
+        .ty = hash.SipHash64(1, 3),
+        .name = "siphash(1,3)",
+        .init_u8s = siphash_key,
+    },
+    Hash{
+        .ty = hash.SipHash64(2, 4),
+        .name = "siphash(2,4)",
+        .init_u8s = siphash_key,
+    },
+    Hash{
+        .ty = hash.Fnv1a_64,
+        .name = "fnv1a",
+    },
+    Hash{
+        .ty = hash.Adler32,
+        .name = "adler32",
+    },
+    Hash{
+        .ty = hash.crc.Crc32WithPoly(.IEEE),
+        .name = "crc32-slicing-by-8",
+    },
+    Hash{
+        .ty = hash.crc.Crc32SmallWithPoly(.IEEE),
+        .name = "crc32-half-byte-lookup",
+    },
+    Hash{
+        .ty = hash.CityHash32,
+        .name = "cityhash-32",
+        .has_iterative_api = false,
+    },
+    Hash{
+        .ty = hash.CityHash64,
+        .name = "cityhash-64",
+        .has_iterative_api = false,
+    },
+    Hash{
+        .ty = hash.Murmur2_32,
+        .name = "murmur2-32",
+        .has_iterative_api = false,
+    },
+    Hash{
+        .ty = hash.Murmur2_64,
+        .name = "murmur2-64",
+        .has_iterative_api = false,
+    },
+    Hash{
+        .ty = hash.Murmur3_32,
+        .name = "murmur3-32",
+        .has_iterative_api = false,
+    },
+};
+
+const Result = struct {
+    hash: u64,
+    throughput: u64,
+};
+
+const block_size: usize = 8192;
+
+pub fn benchmarkHash(comptime H: var, bytes: usize) !Result {
+    var h = blk: {
+        if (H.init_u8s) |init| {
+            break :blk H.ty.init(init);
+        }
+        if (H.init_u64) |init| {
+            break :blk H.ty.init(init);
+        }
+        break :blk H.ty.init();
+    };
+
+    var block: [block_size]u8 = undefined;
+    prng.random.bytes(block[0..]);
+
+    var offset: usize = 0;
+    var timer = try Timer.start();
+    const start = timer.lap();
+    while (offset < bytes) : (offset += block.len) {
+        h.update(block[0..]);
+    }
+    const end = timer.read();
+
+    const elapsed_s = @intToFloat(f64, end - start) / time.ns_per_s;
+    const throughput = @floatToInt(u64, @intToFloat(f64, bytes) / elapsed_s);
+
+    return Result{
+        .hash = h.final(),
+        .throughput = throughput,
+    };
+}
+
+pub fn benchmarkHashSmallKeys(comptime H: var, key_size: usize, bytes: usize) !Result {
+    const key_count = bytes / key_size;
+    var block: [block_size]u8 = undefined;
+    prng.random.bytes(block[0..]);
+
+    var i: usize = 0;
+    var timer = try Timer.start();
+    const start = timer.lap();
+
+    var sum: u64 = 0;
+    while (i < key_count) : (i += 1) {
+        const small_key = block[0..key_size];
+        sum +%= blk: {
+            if (H.init_u8s) |init| {
+                break :blk H.ty.hash(init, small_key);
+            }
+            if (H.init_u64) |init| {
+                break :blk H.ty.hash(init, small_key);
+            }
+            break :blk H.ty.hash(small_key);
+        };
+    }
+    const end = timer.read();
+
+    const elapsed_s = @intToFloat(f64, end - start) / time.ns_per_s;
+    const throughput = @floatToInt(u64, @intToFloat(f64, bytes) / elapsed_s);
+
+    return Result{
+        .hash = sum,
+        .throughput = throughput,
+    };
+}
+
+fn usage() void {
+    std.debug.warn(
+        \\throughput_test [options]
+        \\
+        \\Options:
+        \\  --filter    [test-name]
+        \\  --seed      [int]
+        \\  --count     [int]
+        \\  --key-size  [int]
+        \\  --iterative-only
+        \\  --help
+        \\
+    );
+}
+
+fn mode(comptime x: comptime_int) comptime_int {
+    return if (builtin.mode == builtin.Mode.Debug) x / 64 else x;
+}
+
+// TODO(#1358): Replace with builtin formatted padding when available.
+fn printPad(stdout: var, s: []const u8) !void {
+    var i: usize = 0;
+    while (i < 12 - s.len) : (i += 1) {
+        try stdout.print(" ");
+    }
+    try stdout.print("{}", s);
+}
+
+pub fn main() !void {
+    var stdout_file = try std.io.getStdOut();
+    var stdout_out_stream = stdout_file.outStream();
+    const stdout = &stdout_out_stream.stream;
+
+    var buffer: [1024]u8 = undefined;
+    var fixed = std.heap.FixedBufferAllocator.init(buffer[0..]);
+    const args = try std.process.argsAlloc(&fixed.allocator);
+
+    var filter: ?[]u8 = "";
+    var count: usize = mode(128 * MiB);
+    var key_size: usize = 32;
+    var seed: u32 = 0;
+    var test_iterative_only = false;
+
+    var i: usize = 1;
+    while (i < args.len) : (i += 1) {
+        if (std.mem.eql(u8, args[i], "--mode")) {
+            try stdout.print("{}\n", builtin.mode);
+            return;
+        } else if (std.mem.eql(u8, args[i], "--seed")) {
+            i += 1;
+            if (i == args.len) {
+                usage();
+                std.os.exit(1);
+            }
+
+            seed = try std.fmt.parseUnsigned(u32, args[i], 10);
+            // we seed later
+        } else if (std.mem.eql(u8, args[i], "--filter")) {
+            i += 1;
+            if (i == args.len) {
+                usage();
+                std.os.exit(1);
+            }
+
+            filter = args[i];
+        } else if (std.mem.eql(u8, args[i], "--count")) {
+            i += 1;
+            if (i == args.len) {
+                usage();
+                std.os.exit(1);
+            }
+
+            const c = try std.fmt.parseUnsigned(usize, args[i], 10);
+            count = c * MiB;
+        } else if (std.mem.eql(u8, args[i], "--key-size")) {
+            i += 1;
+            if (i == args.len) {
+                usage();
+                std.os.exit(1);
+            }
+
+            key_size = try std.fmt.parseUnsigned(usize, args[i], 10);
+            if (key_size > block_size) {
+                try stdout.print("key_size cannot exceed block size of {}\n", block_size);
+                std.os.exit(1);
+            }
+        } else if (std.mem.eql(u8, args[i], "--iterative-only")) {
+            test_iterative_only = true;
+        } else if (std.mem.eql(u8, args[i], "--help")) {
+            usage();
+            return;
+        } else {
+            usage();
+            std.os.exit(1);
+        }
+    }
+
+    inline for (hashes) |H| {
+        if (filter == null or std.mem.indexOf(u8, H.name, filter.?) != null) {
+            if (!test_iterative_only or H.has_iterative_api) {
+                try stdout.print("{}\n", H.name);
+
+                // Always reseed prior to every call so we are hashing the same buffer contents.
+                // This allows easier comparison between different implementations.
+                if (H.has_iterative_api) {
+                    prng.seed(seed);
+                    const result = try benchmarkHash(H, count);
+                    try stdout.print("   iterative: {:4} MiB/s [{x:0<16}]\n", result.throughput / (1 * MiB), result.hash);
+                }
+
+                if (!test_iterative_only) {
+                    prng.seed(seed);
+                    const result_small = try benchmarkHashSmallKeys(H, key_size, count);
+                    try stdout.print("  small keys: {:4} MiB/s [{x:0<16}]\n", result_small.throughput / (1 * MiB), result_small.hash);
+                }
+            }
+        }
+    }
+}

std/hash/crc.zig

@@ -9,17 +9,17 @@ const std = @import("../std.zig");
 const debug = std.debug;
 const testing = std.testing;
 
-pub const Polynomial = struct {
+pub const Polynomial = enum(u32) {
-    const IEEE = 0xedb88320;
+    IEEE = 0xedb88320,
-    const Castagnoli = 0x82f63b78;
+    Castagnoli = 0x82f63b78,
-    const Koopman = 0xeb31d82e;
+    Koopman = 0xeb31d82e,
 };
 
 // IEEE is by far the most common CRC and so is aliased by default.
-pub const Crc32 = Crc32WithPoly(Polynomial.IEEE);
+pub const Crc32 = Crc32WithPoly(.IEEE);
 
 // slicing-by-8 crc32 implementation.
-pub fn Crc32WithPoly(comptime poly: u32) type {
+pub fn Crc32WithPoly(comptime poly: Polynomial) type {
     return struct {
         const Self = @This();
         const lookup_tables = comptime block: {
@@ -31,7 +31,7 @@ pub fn Crc32WithPoly(comptime poly: u32) type {
                 var j: usize = 0;
                 while (j < 8) : (j += 1) {
                     if (crc & 1 == 1) {
-                        crc = (crc >> 1) ^ poly;
+                        crc = (crc >> 1) ^ @enumToInt(poly);
                     } else {
                         crc = (crc >> 1);
                     }
@@ -100,7 +100,7 @@ pub fn Crc32WithPoly(comptime poly: u32) type {
 }
 
 test "crc32 ieee" {
-    const Crc32Ieee = Crc32WithPoly(Polynomial.IEEE);
+    const Crc32Ieee = Crc32WithPoly(.IEEE);
 
     testing.expect(Crc32Ieee.hash("") == 0x00000000);
     testing.expect(Crc32Ieee.hash("a") == 0xe8b7be43);
@@ -108,7 +108,7 @@ test "crc32 ieee" {
 }
 
 test "crc32 castagnoli" {
-    const Crc32Castagnoli = Crc32WithPoly(Polynomial.Castagnoli);
+    const Crc32Castagnoli = Crc32WithPoly(.Castagnoli);
 
     testing.expect(Crc32Castagnoli.hash("") == 0x00000000);
     testing.expect(Crc32Castagnoli.hash("a") == 0xc1d04330);
@@ -116,7 +116,7 @@ test "crc32 castagnoli" {
 }
 
 // half-byte lookup table implementation.
-pub fn Crc32SmallWithPoly(comptime poly: u32) type {
+pub fn Crc32SmallWithPoly(comptime poly: Polynomial) type {
     return struct {
         const Self = @This();
         const lookup_table = comptime block: {
@@ -127,7 +127,7 @@ pub fn Crc32SmallWithPoly(comptime poly: u32) type {
                 var j: usize = 0;
                 while (j < 8) : (j += 1) {
                     if (crc & 1 == 1) {
-                        crc = (crc >> 1) ^ poly;
+                        crc = (crc >> 1) ^ @enumToInt(poly);
                     } else {
                         crc = (crc >> 1);
                     }
@@ -164,7 +164,7 @@ pub fn Crc32SmallWithPoly(comptime poly: u32) type {
 }
 
 test "small crc32 ieee" {
-    const Crc32Ieee = Crc32SmallWithPoly(Polynomial.IEEE);
+    const Crc32Ieee = Crc32SmallWithPoly(.IEEE);
 
     testing.expect(Crc32Ieee.hash("") == 0x00000000);
     testing.expect(Crc32Ieee.hash("a") == 0xe8b7be43);
@@ -172,7 +172,7 @@ test "small crc32 ieee" {
 }
 
 test "small crc32 castagnoli" {
-    const Crc32Castagnoli = Crc32SmallWithPoly(Polynomial.Castagnoli);
+    const Crc32Castagnoli = Crc32SmallWithPoly(.Castagnoli);
 
     testing.expect(Crc32Castagnoli.hash("") == 0x00000000);
     testing.expect(Crc32Castagnoli.hash("a") == 0xc1d04330);

std/hash/siphash.zig

@@ -152,8 +152,8 @@ fn SipHash(comptime T: type, comptime c_rounds: usize, comptime d_rounds: usize)
 
         pub fn hash(key: []const u8, input: []const u8) T {
             var c = Self.init(key);
-            c.update(input);
+            @inlineCall(c.update, input);
-            return c.final();
+            return @inlineCall(c.final);
         }
     };
 }

std/hash/throughput_test.zig

@@ -1,148 +0,0 @@
-const builtin = @import("builtin");
-const std = @import("std");
-const time = std.time;
-const Timer = time.Timer;
-const hash = std.hash;
-
-const KiB = 1024;
-const MiB = 1024 * KiB;
-const GiB = 1024 * MiB;
-
-var prng = std.rand.DefaultPrng.init(0);
-
-const Hash = struct {
-    ty: type,
-    name: []const u8,
-    init_u8s: ?[]const u8 = null,
-    init_u64: ?u64 = null,
-};
-
-const siphash_key = "0123456789abcdef";
-
-const hashes = [_]Hash{
-    Hash{ .ty = hash.Wyhash, .name = "wyhash", .init_u64 = 0 },
-    Hash{ .ty = hash.SipHash64(1, 3), .name = "siphash(1,3)", .init_u8s = siphash_key },
-    Hash{ .ty = hash.SipHash64(2, 4), .name = "siphash(2,4)", .init_u8s = siphash_key },
-    Hash{ .ty = hash.Fnv1a_64, .name = "fnv1a" },
-    Hash{ .ty = hash.Crc32, .name = "crc32" },
-};
-
-const Result = struct {
-    hash: u64,
-    throughput: u64,
-};
-
-pub fn benchmarkHash(comptime H: var, bytes: usize) !Result {
-    var h = blk: {
-        if (H.init_u8s) |init| {
-            break :blk H.ty.init(init);
-        }
-        if (H.init_u64) |init| {
-            break :blk H.ty.init(init);
-        }
-        break :blk H.ty.init();
-    };
-
-    var block: [8192]u8 = undefined;
-    prng.random.bytes(block[0..]);
-
-    var offset: usize = 0;
-    var timer = try Timer.start();
-    const start = timer.lap();
-    while (offset < bytes) : (offset += block.len) {
-        h.update(block[0..]);
-    }
-    const end = timer.read();
-
-    const elapsed_s = @intToFloat(f64, end - start) / time.ns_per_s;
-    const throughput = @floatToInt(u64, @intToFloat(f64, bytes) / elapsed_s);
-
-    return Result{
-        .hash = h.final(),
-        .throughput = throughput,
-    };
-}
-
-fn usage() void {
-    std.debug.warn(
-        \\throughput_test [options]
-        \\
-        \\Options:
-        \\  --filter [test-name]
-        \\  --seed   [int]
-        \\  --count  [int]
-        \\  --help
-        \\
-    );
-}
-
-fn mode(comptime x: comptime_int) comptime_int {
-    return if (builtin.mode == builtin.Mode.Debug) x / 64 else x;
-}
-
-// TODO(#1358): Replace with builtin formatted padding when available.
-fn printPad(stdout: var, s: []const u8) !void {
-    var i: usize = 0;
-    while (i < 12 - s.len) : (i += 1) {
-        try stdout.print(" ");
-    }
-    try stdout.print("{}", s);
-}
-
-pub fn main() !void {
-    var stdout_file = try std.io.getStdOut();
-    var stdout_out_stream = stdout_file.outStream();
-    const stdout = &stdout_out_stream.stream;
-
-    var buffer: [1024]u8 = undefined;
-    var fixed = std.heap.FixedBufferAllocator.init(buffer[0..]);
-    const args = try std.process.argsAlloc(&fixed.allocator);
-
-    var filter: ?[]u8 = "";
-    var count: usize = mode(128 * MiB);
-
-    var i: usize = 1;
-    while (i < args.len) : (i += 1) {
-        if (std.mem.eql(u8, args[i], "--seed")) {
-            i += 1;
-            if (i == args.len) {
-                usage();
-                std.os.exit(1);
-            }
-
-            const seed = try std.fmt.parseUnsigned(u32, args[i], 10);
-            prng.seed(seed);
-        } else if (std.mem.eql(u8, args[i], "--filter")) {
-            i += 1;
-            if (i == args.len) {
-                usage();
-                std.os.exit(1);
-            }
-
-            filter = args[i];
-        } else if (std.mem.eql(u8, args[i], "--count")) {
-            i += 1;
-            if (i == args.len) {
-                usage();
-                std.os.exit(1);
-            }
-
-            const c = try std.fmt.parseUnsigned(u32, args[i], 10);
-            count = c * MiB;
-        } else if (std.mem.eql(u8, args[i], "--help")) {
-            usage();
-            return;
-        } else {
-            usage();
-            std.os.exit(1);
-        }
-    }
-
-    inline for (hashes) |H| {
-        if (filter == null or std.mem.indexOf(u8, H.name, filter.?) != null) {
-            const result = try benchmarkHash(H, count);
-            try printPad(stdout, H.name);
-            try stdout.print(": {:4} MiB/s [{:16}]\n", result.throughput / (1 * MiB), result.hash);
-        }
-    }
-}

std/hash/wyhash.zig

@@ -116,8 +116,8 @@ pub const Wyhash = struct {
 
     pub fn hash(seed: u64, input: []const u8) u64 {
         var c = Wyhash.init(seed);
-        c.update(input);
+        @inlineCall(c.update, input);
-        return c.final();
+        return @inlineCall(c.final);
     }
 };