Package: new hash format

legacy format is also supported.

closes #20178

src/Package.zig

@@ -1,8 +1,164 @@
+const std = @import("std");
+const assert = std.debug.assert;
+
 pub const Module = @import("Package/Module.zig");
 pub const Fetch = @import("Package/Fetch.zig");
 pub const build_zig_basename = "build.zig";
 pub const Manifest = @import("Package/Manifest.zig");
 
+pub const multihash_len = 1 + 1 + Hash.Algo.digest_length;
+pub const multihash_hex_digest_len = 2 * multihash_len;
+pub const MultiHashHexDigest = [multihash_hex_digest_len]u8;
+
+/// A user-readable, file system safe hash that identifies an exact package
+/// snapshot, including file contents.
+///
+/// This data structure can be used to store the legacy hash format too. Legacy
+/// hash format is scheduled to be removed after 0.14.0 is tagged.
+pub const Hash = struct {
+    /// Maximum size of a package hash. Unused bytes at the end are
+    /// filled with zeroes.
+    bytes: [max_len]u8,
+
+    pub const Algo = std.crypto.hash.sha2.Sha256;
+    pub const Digest = [Algo.digest_length]u8;
+
+    pub const max_len = 32 + 1 + 32 + 1 + 12;
+
+    pub fn fromSlice(s: []const u8) Hash {
+        assert(s.len <= max_len);
+        var result: Hash = undefined;
+        @memcpy(result.bytes[0..s.len], s);
+        @memset(result.bytes[s.len..], 0);
+        return result;
+    }
+
+    pub fn toSlice(ph: *const Hash) []const u8 {
+        var end: usize = ph.bytes.len;
+        while (true) {
+            end -= 1;
+            if (ph.bytes[end] != 0) return ph.bytes[0 .. end + 1];
+        }
+    }
+
+    pub fn eql(a: *const Hash, b: *const Hash) bool {
+        return std.mem.eql(u8, &a.bytes, &b.bytes);
+    }
+
+    /// Distinguishes whether the legacy multihash format is being stored here.
+    pub fn isOld(h: *const Hash) bool {
+        if (h.bytes.len < 2) return false;
+        const their_multihash_func = std.fmt.parseInt(u8, h.bytes[0..2], 16) catch return false;
+        if (@as(MultihashFunction, @enumFromInt(their_multihash_func)) != multihash_function) return false;
+        if (h.toSlice().len != multihash_hex_digest_len) return false;
+        return std.mem.indexOfScalar(u8, &h.bytes, '-') == null;
+    }
+
+    test isOld {
+        const h: Hash = .fromSlice("1220138f4aba0c01e66b68ed9e1e1e74614c06e4743d88bc58af4f1c3dd0aae5fea7");
+        try std.testing.expect(h.isOld());
+    }
+
+    /// Produces "$name-$semver-$sizedhash".
+    /// * name is the name field from build.zig.zon, truncated at 32 bytes and must
+    ///   be a valid zig identifier
+    /// * semver is the version field from build.zig.zon, truncated at 32 bytes
+    /// * sizedhash is the following 9-byte array, base64 encoded using -_ to make
+    ///   it filesystem safe:
+    ///   - (4 bytes) LE u32 total decompressed size in bytes
+    ///   - (5 bytes) truncated SHA-256 of hashed files of the package
+    ///
+    /// example: "nasm-2.16.1-2-BWdcABvF_jM1"
+    pub fn init(digest: Digest, name: []const u8, ver: []const u8, size: u32) Hash {
+        var result: Hash = undefined;
+        var buf: std.ArrayListUnmanaged(u8) = .initBuffer(&result.bytes);
+        buf.appendSliceAssumeCapacity(name[0..@min(name.len, 32)]);
+        buf.appendAssumeCapacity('-');
+        buf.appendSliceAssumeCapacity(ver[0..@min(ver.len, 32)]);
+        buf.appendAssumeCapacity('-');
+        var sizedhash: [9]u8 = undefined;
+        std.mem.writeInt(u32, sizedhash[0..4], size, .little);
+        sizedhash[4..].* = digest[0..5].*;
+        _ = std.base64.url_safe_no_pad.Encoder.encode(buf.addManyAsArrayAssumeCapacity(12), &sizedhash);
+        @memset(buf.unusedCapacitySlice(), 0);
+        return result;
+    }
+
+    /// Produces "$hashiname-N-$sizedhash". For packages that lack "build.zig.zon" metadata.
+    /// * hashiname is [5..][0..24] bytes of the SHA-256, urlsafe-base64-encoded, for a total of 32 bytes encoded
+    /// * the semver section is replaced with a hardcoded N which stands for
+    ///   "naked". It acts as a version number so that any future updates to the
+    ///   hash format can tell this hash format apart. Note that "N" is an
+    ///   invalid semver.
+    /// * sizedhash is the same as in `init`.
+    ///
+    /// The hash is broken up this way so that "sizedhash" can be calculated
+    /// exactly the same way in both cases, and so that "name" and "hashiname" can
+    /// be used interchangeably in both cases.
+    pub fn initNaked(digest: Digest, size: u32) Hash {
+        var name: [32]u8 = undefined;
+        _ = std.base64.url_safe_no_pad.Encoder.encode(&name, digest[5..][0..24]);
+        return init(digest, &name, "N", size);
+    }
+};
+
+pub const MultihashFunction = enum(u16) {
+    identity = 0x00,
+    sha1 = 0x11,
+    @"sha2-256" = 0x12,
+    @"sha2-512" = 0x13,
+    @"sha3-512" = 0x14,
+    @"sha3-384" = 0x15,
+    @"sha3-256" = 0x16,
+    @"sha3-224" = 0x17,
+    @"sha2-384" = 0x20,
+    @"sha2-256-trunc254-padded" = 0x1012,
+    @"sha2-224" = 0x1013,
+    @"sha2-512-224" = 0x1014,
+    @"sha2-512-256" = 0x1015,
+    @"blake2b-256" = 0xb220,
+    _,
+};
+
+pub const multihash_function: MultihashFunction = switch (Hash.Algo) {
+    std.crypto.hash.sha2.Sha256 => .@"sha2-256",
+    else => @compileError("unreachable"),
+};
+
+pub fn multiHashHexDigest(digest: Hash.Digest) MultiHashHexDigest {
+    const hex_charset = std.fmt.hex_charset;
+
+    var result: MultiHashHexDigest = undefined;
+
+    result[0] = hex_charset[@intFromEnum(multihash_function) >> 4];
+    result[1] = hex_charset[@intFromEnum(multihash_function) & 15];
+
+    result[2] = hex_charset[Hash.Algo.digest_length >> 4];
+    result[3] = hex_charset[Hash.Algo.digest_length & 15];
+
+    for (digest, 0..) |byte, i| {
+        result[4 + i * 2] = hex_charset[byte >> 4];
+        result[5 + i * 2] = hex_charset[byte & 15];
+    }
+    return result;
+}
+
+comptime {
+    // We avoid unnecessary uleb128 code in hexDigest by asserting here the
+    // values are small enough to be contained in the one-byte encoding.
+    assert(@intFromEnum(multihash_function) < 127);
+    assert(Hash.Algo.digest_length < 127);
+}
+
+test Hash {
+    const example_digest: Hash.Digest = .{
+        0xc7, 0xf5, 0x71, 0xb7, 0xb4, 0xe7, 0x6f, 0x3c, 0xdb, 0x87, 0x7a, 0x7f, 0xdd, 0xf9, 0x77, 0x87,
+        0x9d, 0xd3, 0x86, 0xfa, 0x73, 0x57, 0x9a, 0xf7, 0x9d, 0x1e, 0xdb, 0x8f, 0x3a, 0xd9, 0xbd, 0x9f,
+    };
+    const result: Hash = .init(example_digest, "nasm", "2.16.1-2", 10 * 1024 * 1024);
+    try std.testing.expectEqualStrings("nasm-2.16.1-2-AACgAMf1cbe0", result.toSlice());
+}
+
 test {
     _ = Fetch;
 }

src/Package/Fetch.zig

@@ -56,7 +56,7 @@ package_root: Cache.Path,
 error_bundle: ErrorBundle.Wip,
 manifest: ?Manifest,
 manifest_ast: std.zig.Ast,
-actual_hash: Manifest.Digest,
+computed_hash: ComputedHash,
 /// Fetch logic notices whether a package has a build.zig file and sets this flag.
 has_build_zig: bool,
 /// Indicates whether the task aborted due to an out-of-memory condition.
@@ -116,8 +116,8 @@ pub const JobQueue = struct {
     /// as lazy.
     unlazy_set: UnlazySet = .{},
 
-    pub const Table = std.AutoArrayHashMapUnmanaged(Manifest.MultiHashHexDigest, *Fetch);
-    pub const UnlazySet = std.AutoArrayHashMapUnmanaged(Manifest.MultiHashHexDigest, void);
+    pub const Table = std.AutoArrayHashMapUnmanaged(Package.Hash, *Fetch);
+    pub const UnlazySet = std.AutoArrayHashMapUnmanaged(Package.Hash, void);
 
     pub fn deinit(jq: *JobQueue) void {
         if (jq.all_fetches.items.len == 0) return;
@@ -160,22 +160,24 @@ pub const JobQueue = struct {
 
         // Ensure the generated .zig file is deterministic.
         jq.table.sortUnstable(@as(struct {
-            keys: []const Manifest.MultiHashHexDigest,
+            keys: []const Package.Hash,
             pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool {
-                return std.mem.lessThan(u8, &ctx.keys[a_index], &ctx.keys[b_index]);
+                return std.mem.lessThan(u8, &ctx.keys[a_index].bytes, &ctx.keys[b_index].bytes);
             }
         }, .{ .keys = keys }));
 
-        for (keys, jq.table.values()) |hash, fetch| {
+        for (keys, jq.table.values()) |*hash, fetch| {
             if (fetch == jq.all_fetches.items[0]) {
                 // The first one is a dummy package for the current project.
                 continue;
             }
 
+            const hash_slice = hash.toSlice();
+
             try buf.writer().print(
                 \\    pub const {} = struct {{
                 \\
-            , .{std.zig.fmtId(&hash)});
+            , .{std.zig.fmtId(hash_slice)});
 
             lazy: {
                 switch (fetch.lazy_status) {
@@ -207,7 +209,7 @@ pub const JobQueue = struct {
                 try buf.writer().print(
                     \\        pub const build_zig = @import("{}");
                     \\
-                , .{std.zig.fmtEscapes(&hash)});
+                , .{std.zig.fmtEscapes(hash_slice)});
             }
 
             if (fetch.manifest) |*manifest| {
@@ -219,7 +221,7 @@ pub const JobQueue = struct {
                     const h = depDigest(fetch.package_root, jq.global_cache, dep) orelse continue;
                     try buf.writer().print(
                         "            .{{ \"{}\", \"{}\" }},\n",
-                        .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(&h) },
+                        .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(h.toSlice()) },
                     );
                 }
 
@@ -251,7 +253,7 @@ pub const JobQueue = struct {
             const h = depDigest(root_fetch.package_root, jq.global_cache, dep) orelse continue;
             try buf.writer().print(
                 "    .{{ \"{}\", \"{}\" }},\n",
-                .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(&h) },
+                .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(h.toSlice()) },
             );
         }
         try buf.appendSlice("};\n");
@@ -283,7 +285,7 @@ pub const Location = union(enum) {
         url: []const u8,
         /// If this is null it means the user omitted the hash field from a dependency.
         /// It will be an error but the logic should still fetch and print the discovered hash.
-        hash: ?Manifest.MultiHashHexDigest,
+        hash: ?Package.Hash,
     };
 };
 
@@ -325,9 +327,11 @@ pub fn run(f: *Fetch) RunError!void {
                 // "p/$hash/foo", with possibly more directories after "foo".
                 // We want to fail unless the resolved relative path has a
                 // prefix of "p/$hash/".
-                const digest_len = @typeInfo(Manifest.MultiHashHexDigest).array.len;
                 const prefix_len: usize = if (f.job_queue.read_only) 0 else "p/".len;
-                const expected_prefix = f.parent_package_root.sub_path[0 .. prefix_len + digest_len];
+                const parent_sub_path = f.parent_package_root.sub_path;
+                const end = std.mem.indexOfScalarPos(u8, parent_sub_path, prefix_len, fs.path.sep) orelse
+                    parent_sub_path.len;
+                const expected_prefix = parent_sub_path[prefix_len..end];
                 if (!std.mem.startsWith(u8, pkg_root.sub_path, expected_prefix)) {
                     return f.fail(
                         f.location_tok,
@@ -367,9 +371,13 @@ pub fn run(f: *Fetch) RunError!void {
         },
     };
 
-    const s = fs.path.sep_str;
     if (remote.hash) |expected_hash| {
-        const prefixed_pkg_sub_path = "p" ++ s ++ expected_hash;
+        var prefixed_pkg_sub_path_buffer: [100]u8 = undefined;
+        prefixed_pkg_sub_path_buffer[0] = 'p';
+        prefixed_pkg_sub_path_buffer[1] = fs.path.sep;
+        const hash_slice = expected_hash.toSlice();
+        @memcpy(prefixed_pkg_sub_path_buffer[2..][0..hash_slice.len], hash_slice);
+        const prefixed_pkg_sub_path = prefixed_pkg_sub_path_buffer[0 .. 2 + hash_slice.len];
         const prefix_len: usize = if (f.job_queue.read_only) "p/".len else 0;
         const pkg_sub_path = prefixed_pkg_sub_path[prefix_len..];
         if (cache_root.handle.access(pkg_sub_path, .{})) |_| {
@@ -437,7 +445,7 @@ fn runResource(
     f: *Fetch,
     uri_path: []const u8,
     resource: *Resource,
-    remote_hash: ?Manifest.MultiHashHexDigest,
+    remote_hash: ?Package.Hash,
 ) RunError!void {
     defer resource.deinit();
     const arena = f.arena.allocator();
@@ -499,7 +507,7 @@ fn runResource(
         // Empty directories have already been omitted by `unpackResource`.
         // Compute the package hash based on the remaining files in the temporary
         // directory.
-        f.actual_hash = try computeHash(f, pkg_path, filter);
+        f.computed_hash = try computeHash(f, pkg_path, filter);
 
         break :blk if (unpack_result.root_dir.len > 0)
             try fs.path.join(arena, &.{ tmp_dir_sub_path, unpack_result.root_dir })
@@ -507,6 +515,8 @@ fn runResource(
             tmp_dir_sub_path;
     };
 
+    const computed_package_hash = computedPackageHash(f);
+
     // Rename the temporary directory into the global zig package cache
     // directory. If the hash already exists, delete the temporary directory
     // and leave the zig package cache directory untouched as it may be in use
@@ -515,7 +525,7 @@ fn runResource(
 
     f.package_root = .{
         .root_dir = cache_root,
-        .sub_path = try arena.dupe(u8, "p" ++ s ++ Manifest.hexDigest(f.actual_hash)),
+        .sub_path = try std.fmt.allocPrint(arena, "p" ++ s ++ "{s}", .{computed_package_hash.toSlice()}),
     };
     renameTmpIntoCache(cache_root.handle, package_sub_path, f.package_root.sub_path) catch |err| {
         const src = try cache_root.join(arena, &.{tmp_dir_sub_path});
@@ -534,13 +544,22 @@ fn runResource(
     // Validate the computed hash against the expected hash. If invalid, this
     // job is done.
 
-    const actual_hex = Manifest.hexDigest(f.actual_hash);
     if (remote_hash) |declared_hash| {
-        if (!std.mem.eql(u8, &declared_hash, &actual_hex)) {
-            return f.fail(f.hash_tok, try eb.printString(
-                "hash mismatch: manifest declares {s} but the fetched package has {s}",
-                .{ declared_hash, actual_hex },
-            ));
+        if (declared_hash.isOld()) {
+            const actual_hex = Package.multiHashHexDigest(f.computed_hash.digest);
+            if (!std.mem.eql(u8, declared_hash.toSlice(), &actual_hex)) {
+                return f.fail(f.hash_tok, try eb.printString(
+                    "hash mismatch: manifest declares {s} but the fetched package has {s}",
+                    .{ declared_hash.toSlice(), actual_hex },
+                ));
+            }
+        } else {
+            if (!computed_package_hash.eql(&declared_hash)) {
+                return f.fail(f.hash_tok, try eb.printString(
+                    "hash mismatch: manifest declares {s} but the fetched package has {s}",
+                    .{ declared_hash.toSlice(), computed_package_hash.toSlice() },
+                ));
+            }
         }
     } else if (!f.omit_missing_hash_error) {
         const notes_len = 1;
@@ -551,7 +570,7 @@ fn runResource(
         });
         const notes_start = try eb.reserveNotes(notes_len);
         eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{
-            .msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}),
+            .msg = try eb.printString("expected .hash = \"{s}\",", .{computed_package_hash.toSlice()}),
         }));
         return error.FetchFailed;
     }
@@ -562,6 +581,16 @@ fn runResource(
     return queueJobsForDeps(f);
 }
 
+pub fn computedPackageHash(f: *const Fetch) Package.Hash {
+    const saturated_size = std.math.cast(u32, f.computed_hash.total_size) orelse std.math.maxInt(u32);
+    if (f.manifest) |man| {
+        var version_buffer: [32]u8 = undefined;
+        const version: []const u8 = std.fmt.bufPrint(&version_buffer, "{}", .{man.version}) catch &version_buffer;
+        return .init(f.computed_hash.digest, man.name, version, saturated_size);
+    }
+    return .initNaked(f.computed_hash.digest, saturated_size);
+}
+
 /// `computeHash` gets a free check for the existence of `build.zig`, but when
 /// not computing a hash, we need to do a syscall to check for it.
 fn checkBuildFileExistence(f: *Fetch) RunError!void {
@@ -673,9 +702,8 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
                     .url = url,
                     .hash = h: {
                         const h = dep.hash orelse break :h null;
-                        const digest_len = @typeInfo(Manifest.MultiHashHexDigest).array.len;
-                        const multihash_digest = h[0..digest_len].*;
-                        const gop = f.job_queue.table.getOrPutAssumeCapacity(multihash_digest);
+                        const pkg_hash: Package.Hash = .fromSlice(h);
+                        const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash);
                         if (gop.found_existing) {
                             if (!dep.lazy) {
                                 gop.value_ptr.*.lazy_status = .eager;
@@ -683,15 +711,15 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
                             continue;
                         }
                         gop.value_ptr.* = new_fetch;
-                        break :h multihash_digest;
+                        break :h pkg_hash;
                     },
                 } },
                 .path => |rel_path| l: {
                     // This might produce an invalid path, which is checked for
                     // at the beginning of run().
                     const new_root = try f.package_root.resolvePosix(parent_arena, rel_path);
-                    const multihash_digest = relativePathDigest(new_root, cache_root);
-                    const gop = f.job_queue.table.getOrPutAssumeCapacity(multihash_digest);
+                    const pkg_hash = relativePathDigest(new_root, cache_root);
+                    const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash);
                     if (gop.found_existing) {
                         if (!dep.lazy) {
                             gop.value_ptr.*.lazy_status = .eager;
@@ -724,7 +752,7 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
                 .error_bundle = undefined,
                 .manifest = null,
                 .manifest_ast = undefined,
-                .actual_hash = undefined,
+                .computed_hash = undefined,
                 .has_build_zig = false,
                 .oom_flag = false,
                 .latest_commit = null,
@@ -746,11 +774,8 @@ fn queueJobsForDeps(f: *Fetch) RunError!void {
     }
 }
 
-pub fn relativePathDigest(
-    pkg_root: Cache.Path,
-    cache_root: Cache.Directory,
-) Manifest.MultiHashHexDigest {
-    var hasher = Manifest.Hash.init(.{});
+pub fn relativePathDigest(pkg_root: Cache.Path, cache_root: Cache.Directory) Package.Hash {
+    var hasher = Package.Hash.Algo.init(.{});
     // This hash is a tuple of:
     // * whether it relative to the global cache directory or to the root package
     // * the relative file path from there to the build root of the package
@@ -759,7 +784,7 @@ pub fn relativePathDigest(
     else
         &package_hash_prefix_project);
     hasher.update(pkg_root.sub_path);
-    return Manifest.hexDigest(hasher.finalResult());
+    return .fromSlice(&hasher.finalResult());
 }
 
 pub fn workerRun(f: *Fetch, prog_name: []const u8) void {
@@ -1387,11 +1412,7 @@ fn recursiveDirectoryCopy(f: *Fetch, dir: fs.Dir, tmp_dir: fs.Dir) anyerror!void
     }
 }
 
-pub fn renameTmpIntoCache(
-    cache_dir: fs.Dir,
-    tmp_dir_sub_path: []const u8,
-    dest_dir_sub_path: []const u8,
-) !void {
+pub fn renameTmpIntoCache(cache_dir: fs.Dir, tmp_dir_sub_path: []const u8, dest_dir_sub_path: []const u8) !void {
     assert(dest_dir_sub_path[1] == fs.path.sep);
     var handled_missing_dir = false;
     while (true) {
@@ -1417,16 +1438,17 @@ pub fn renameTmpIntoCache(
     }
 }
 
+const ComputedHash = struct {
+    digest: Package.Hash.Digest,
+    total_size: u64,
+};
+
 /// Assumes that files not included in the package have already been filtered
 /// prior to calling this function. This ensures that files not protected by
 /// the hash are not present on the file system. Empty directories are *not
 /// hashed* and must not be present on the file system when calling this
 /// function.
-fn computeHash(
-    f: *Fetch,
-    pkg_path: Cache.Path,
-    filter: Filter,
-) RunError!Manifest.Digest {
+fn computeHash(f: *Fetch, pkg_path: Cache.Path, filter: Filter) RunError!ComputedHash {
     // All the path name strings need to be in memory for sorting.
     const arena = f.arena.allocator();
     const gpa = f.arena.child_allocator;
@@ -1449,6 +1471,9 @@ fn computeHash(
     var walker = try root_dir.walk(gpa);
     defer walker.deinit();
 
+    // Total number of bytes of file contents included in the package.
+    var total_size: u64 = 0;
+
     {
         // The final hash will be a hash of each file hashed independently. This
         // allows hashing in parallel.
@@ -1506,6 +1531,7 @@ fn computeHash(
                 .kind = kind,
                 .hash = undefined, // to be populated by the worker
                 .failure = undefined, // to be populated by the worker
+                .size = undefined, // to be populated by the worker
             };
             thread_pool.spawnWg(&wait_group, workerHashFile, .{ root_dir, hashed_file });
             try all_files.append(hashed_file);
@@ -1544,7 +1570,7 @@ fn computeHash(
 
     std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan);
 
-    var hasher = Manifest.Hash.init(.{});
+    var hasher = Package.Hash.Algo.init(.{});
     var any_failures = false;
     for (all_files.items) |hashed_file| {
         hashed_file.failure catch |err| {
@@ -1556,6 +1582,7 @@ fn computeHash(
             });
         };
         hasher.update(&hashed_file.hash);
+        total_size += hashed_file.size;
     }
     for (deleted_files.items) |deleted_file| {
         deleted_file.failure catch |err| {
@@ -1580,7 +1607,10 @@ fn computeHash(
         };
     }
 
-    return hasher.finalResult();
+    return .{
+        .digest = hasher.finalResult(),
+        .total_size = total_size,
+    };
 }
 
 fn dumpHashInfo(all_files: []const *const HashedFile) !void {
@@ -1609,8 +1639,9 @@ fn workerDeleteFile(dir: fs.Dir, deleted_file: *DeletedFile) void {
 
 fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void {
     var buf: [8000]u8 = undefined;
-    var hasher = Manifest.Hash.init(.{});
+    var hasher = Package.Hash.Algo.init(.{});
     hasher.update(hashed_file.normalized_path);
+    var file_size: u64 = 0;
 
     switch (hashed_file.kind) {
         .file => {
@@ -1622,6 +1653,7 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void
             while (true) {
                 const bytes_read = try file.read(&buf);
                 if (bytes_read == 0) break;
+                file_size += bytes_read;
                 hasher.update(buf[0..bytes_read]);
                 file_header.update(buf[0..bytes_read]);
             }
@@ -1641,6 +1673,7 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void
         },
     }
     hasher.final(&hashed_file.hash);
+    hashed_file.size = file_size;
 }
 
 fn deleteFileFallible(dir: fs.Dir, deleted_file: *DeletedFile) DeletedFile.Error!void {
@@ -1667,9 +1700,10 @@ const DeletedFile = struct {
 const HashedFile = struct {
     fs_path: []const u8,
     normalized_path: []const u8,
-    hash: Manifest.Digest,
+    hash: Package.Hash.Digest,
     failure: Error!void,
     kind: Kind,
+    size: u64,
 
     const Error =
         fs.File.OpenError ||
@@ -1744,12 +1778,8 @@ const Filter = struct {
     }
 };
 
-pub fn depDigest(
-    pkg_root: Cache.Path,
-    cache_root: Cache.Directory,
-    dep: Manifest.Dependency,
-) ?Manifest.MultiHashHexDigest {
-    if (dep.hash) |h| return h[0..Manifest.multihash_hex_digest_len].*;
+pub fn depDigest(pkg_root: Cache.Path, cache_root: Cache.Directory, dep: Manifest.Dependency) ?Package.Hash {
+    if (dep.hash) |h| return .fromSlice(h);
 
     switch (dep.location) {
         .url => return null,
@@ -2137,7 +2167,7 @@ test "tarball with excluded duplicate paths" {
     defer fb.deinit();
     try fetch.run();
 
-    const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash);
+    const hex_digest = Package.multiHashHexDigest(fetch.computed_hash.digest);
     try std.testing.expectEqualStrings(
         "12200bafe035cbb453dd717741b66e9f9d1e6c674069d06121dafa1b2e62eb6b22da",
         &hex_digest,
@@ -2181,7 +2211,7 @@ test "tarball without root folder" {
     defer fb.deinit();
     try fetch.run();
 
-    const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash);
+    const hex_digest = Package.multiHashHexDigest(fetch.computed_hash.digest);
     try std.testing.expectEqualStrings(
         "12209f939bfdcb8b501a61bb4a43124dfa1b2848adc60eec1e4624c560357562b793",
         &hex_digest,
@@ -2222,7 +2252,7 @@ test "set executable bit based on file content" {
     try fetch.run();
     try std.testing.expectEqualStrings(
         "1220fecb4c06a9da8673c87fe8810e15785f1699212f01728eadce094d21effeeef3",
-        &Manifest.hexDigest(fetch.actual_hash),
+        &Package.multiHashHexDigest(fetch.computed_hash.digest),
     );
 
     var out = try fb.packageDir();
@@ -2304,7 +2334,7 @@ const TestFetchBuilder = struct {
             .error_bundle = undefined,
             .manifest = null,
             .manifest_ast = undefined,
-            .actual_hash = undefined,
+            .computed_hash = undefined,
             .has_build_zig = false,
             .oom_flag = false,
             .latest_commit = null,

src/Package/Manifest.zig

@@ -5,15 +5,10 @@ const Allocator = std.mem.Allocator;
 const assert = std.debug.assert;
 const Ast = std.zig.Ast;
 const testing = std.testing;
-const hex_charset = std.fmt.hex_charset;
+const Package = @import("../Package.zig");
 
 pub const max_bytes = 10 * 1024 * 1024;
 pub const basename = "build.zig.zon";
-pub const Hash = std.crypto.hash.sha2.Sha256;
-pub const Digest = [Hash.digest_length]u8;
-pub const multihash_len = 1 + 1 + Hash.digest_length;
-pub const multihash_hex_digest_len = 2 * multihash_len;
-pub const MultiHashHexDigest = [multihash_hex_digest_len]u8;
 
 pub const Dependency = struct {
     location: Location,
@@ -38,35 +33,6 @@ pub const ErrorMessage = struct {
     off: u32,
 };
 
-pub const MultihashFunction = enum(u16) {
-    identity = 0x00,
-    sha1 = 0x11,
-    @"sha2-256" = 0x12,
-    @"sha2-512" = 0x13,
-    @"sha3-512" = 0x14,
-    @"sha3-384" = 0x15,
-    @"sha3-256" = 0x16,
-    @"sha3-224" = 0x17,
-    @"sha2-384" = 0x20,
-    @"sha2-256-trunc254-padded" = 0x1012,
-    @"sha2-224" = 0x1013,
-    @"sha2-512-224" = 0x1014,
-    @"sha2-512-256" = 0x1015,
-    @"blake2b-256" = 0xb220,
-    _,
-};
-
-pub const multihash_function: MultihashFunction = switch (Hash) {
-    std.crypto.hash.sha2.Sha256 => .@"sha2-256",
-    else => @compileError("unreachable"),
-};
-comptime {
-    // We avoid unnecessary uleb128 code in hexDigest by asserting here the
-    // values are small enough to be contained in the one-byte encoding.
-    assert(@intFromEnum(multihash_function) < 127);
-    assert(Hash.digest_length < 127);
-}
-
 name: []const u8,
 version: std.SemanticVersion,
 version_node: Ast.Node.Index,
@@ -164,22 +130,6 @@ pub fn copyErrorsIntoBundle(
     }
 }
 
-pub fn hexDigest(digest: Digest) MultiHashHexDigest {
-    var result: MultiHashHexDigest = undefined;
-
-    result[0] = hex_charset[@intFromEnum(multihash_function) >> 4];
-    result[1] = hex_charset[@intFromEnum(multihash_function) & 15];
-
-    result[2] = hex_charset[Hash.digest_length >> 4];
-    result[3] = hex_charset[Hash.digest_length & 15];
-
-    for (digest, 0..) |byte, i| {
-        result[4 + i * 2] = hex_charset[byte >> 4];
-        result[5 + i * 2] = hex_charset[byte & 15];
-    }
-    return result;
-}
-
 const Parse = struct {
     gpa: Allocator,
     ast: Ast,
@@ -421,21 +371,8 @@ const Parse = struct {
         const tok = main_tokens[node];
         const h = try parseString(p, node);
 
-        if (h.len >= 2) {
-            const their_multihash_func = std.fmt.parseInt(u8, h[0..2], 16) catch |err| {
-                return fail(p, tok, "invalid multihash value: unable to parse hash function: {s}", .{
-                    @errorName(err),
-                });
-            };
-            if (@as(MultihashFunction, @enumFromInt(their_multihash_func)) != multihash_function) {
-                return fail(p, tok, "unsupported hash function: only sha2-256 is supported", .{});
-            }
-        }
-
-        if (h.len != multihash_hex_digest_len) {
-            return fail(p, tok, "wrong hash size. expected: {d}, found: {d}", .{
-                multihash_hex_digest_len, h.len,
-            });
+        if (h.len > Package.Hash.max_len) {
+            return fail(p, tok, "hash length exceeds maximum: {d}", .{h.len});
         }
 
         return h;

src/main.zig

@@ -5197,7 +5197,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
                     .error_bundle = undefined,
                     .manifest = null,
                     .manifest_ast = undefined,
-                    .actual_hash = undefined,
+                    .computed_hash = undefined,
                     .has_build_zig = true,
                     .oom_flag = false,
                     .latest_commit = null,
@@ -5244,13 +5244,14 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
                     const hashes = job_queue.table.keys();
                     const fetches = job_queue.table.values();
                     try deps_mod.deps.ensureUnusedCapacity(arena, @intCast(hashes.len));
-                    for (hashes, fetches) |hash, f| {
+                    for (hashes, fetches) |*hash, f| {
                         if (f == &fetch) {
                             // The first one is a dummy package for the current project.
                             continue;
                         }
                         if (!f.has_build_zig)
                             continue;
+                        const hash_slice = hash.toSlice();
                         const m = try Package.Module.create(arena, .{
                             .global_cache_directory = global_cache_directory,
                             .paths = .{
@@ -5260,7 +5261,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
                             .fully_qualified_name = try std.fmt.allocPrint(
                                 arena,
                                 "root.@dependencies.{s}",
-                                .{&hash},
+                                .{hash_slice},
                             ),
                             .cc_argv = &.{},
                             .inherited = .{},
@@ -5269,7 +5270,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
                             .builtin_mod = builtin_mod,
                             .builtin_modules = null, // `builtin_mod` is specified
                         });
-                        const hash_cloned = try arena.dupe(u8, &hash);
+                        const hash_cloned = try arena.dupe(u8, hash_slice);
                         deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m);
                         f.module = m;
                     }
@@ -5385,23 +5386,22 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void {
                         var any_errors = false;
                         while (it.next()) |hash| {
                             if (hash.len == 0) continue;
-                            const digest_len = @typeInfo(Package.Manifest.MultiHashHexDigest).array.len;
-                            if (hash.len != digest_len) {
-                                std.log.err("invalid digest (length {d} instead of {d}): '{s}'", .{
-                                    hash.len, digest_len, hash,
+                            if (hash.len > Package.Hash.max_len) {
+                                std.log.err("invalid digest (length {d} exceeds maximum): '{s}'", .{
+                                    hash.len, hash,
                                 });
                                 any_errors = true;
                                 continue;
                             }
-                            try unlazy_set.put(arena, hash[0..digest_len].*, {});
+                            try unlazy_set.put(arena, .fromSlice(hash), {});
                         }
                         if (any_errors) process.exit(3);
                         if (system_pkg_dir_path) |p| {
                             // In this mode, the system needs to provide these packages; they
                             // cannot be fetched by Zig.
-                            for (unlazy_set.keys()) |hash| {
+                            for (unlazy_set.keys()) |*hash| {
                                 std.log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{
-                                    p, hash,
+                                    p, hash.toSlice(),
                                 });
                             }
                             std.log.info("remote package fetching disabled due to --system mode", .{});
@@ -7097,7 +7097,7 @@ fn cmdFetch(
         .error_bundle = undefined,
         .manifest = null,
         .manifest_ast = undefined,
-        .actual_hash = undefined,
+        .computed_hash = undefined,
         .has_build_zig = false,
         .oom_flag = false,
         .latest_commit = null,
@@ -7117,14 +7117,15 @@ fn cmdFetch(
         process.exit(1);
     }
 
-    const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash);
+    const package_hash = fetch.computedPackageHash();
+    const package_hash_slice = package_hash.toSlice();
 
     root_prog_node.end();
     root_prog_node = .{ .index = .none };
 
     const name = switch (save) {
         .no => {
-            try io.getStdOut().writeAll(hex_digest ++ "\n");
+            try io.getStdOut().writer().print("{s}\n", .{package_hash_slice});
             return cleanExit();
         },
         .yes, .exact => |name| name: {
@@ -7194,7 +7195,7 @@ fn cmdFetch(
         \\        }}
     , .{
         std.zig.fmtEscapes(saved_path_or_url),
-        std.zig.fmtEscapes(&hex_digest),
+        std.zig.fmtEscapes(package_hash_slice),
     });
 
     const new_node_text = try std.fmt.allocPrint(arena, ".{p_} = {s},\n", .{
@@ -7213,7 +7214,7 @@ fn cmdFetch(
         if (dep.hash) |h| {
             switch (dep.location) {
                 .url => |u| {
-                    if (mem.eql(u8, h, &hex_digest) and mem.eql(u8, u, saved_path_or_url)) {
+                    if (mem.eql(u8, h, package_hash_slice) and mem.eql(u8, u, saved_path_or_url)) {
                         std.log.info("existing dependency named '{s}' is up-to-date", .{name});
                         process.exit(0);
                     }
@@ -7230,7 +7231,7 @@ fn cmdFetch(
         const hash_replace = try std.fmt.allocPrint(
             arena,
             "\"{}\"",
-            .{std.zig.fmtEscapes(&hex_digest)},
+            .{std.zig.fmtEscapes(package_hash_slice)},
         );
 
         warn("overwriting existing dependency named '{s}'", .{name});