diff --git a/src/Package.zig b/src/Package.zig index 61f90727f3..c9f41f6f37 100644 --- a/src/Package.zig +++ b/src/Package.zig @@ -1,8 +1,164 @@ +const std = @import("std"); +const assert = std.debug.assert; + pub const Module = @import("Package/Module.zig"); pub const Fetch = @import("Package/Fetch.zig"); pub const build_zig_basename = "build.zig"; pub const Manifest = @import("Package/Manifest.zig"); +pub const multihash_len = 1 + 1 + Hash.Algo.digest_length; +pub const multihash_hex_digest_len = 2 * multihash_len; +pub const MultiHashHexDigest = [multihash_hex_digest_len]u8; + +/// A user-readable, file system safe hash that identifies an exact package +/// snapshot, including file contents. +/// +/// This data structure can be used to store the legacy hash format too. Legacy +/// hash format is scheduled to be removed after 0.14.0 is tagged. +pub const Hash = struct { + /// Maximum size of a package hash. Unused bytes at the end are + /// filled with zeroes. + bytes: [max_len]u8, + + pub const Algo = std.crypto.hash.sha2.Sha256; + pub const Digest = [Algo.digest_length]u8; + + pub const max_len = 32 + 1 + 32 + 1 + 12; + + pub fn fromSlice(s: []const u8) Hash { + assert(s.len <= max_len); + var result: Hash = undefined; + @memcpy(result.bytes[0..s.len], s); + @memset(result.bytes[s.len..], 0); + return result; + } + + pub fn toSlice(ph: *const Hash) []const u8 { + var end: usize = ph.bytes.len; + while (true) { + end -= 1; + if (ph.bytes[end] != 0) return ph.bytes[0 .. end + 1]; + } + } + + pub fn eql(a: *const Hash, b: *const Hash) bool { + return std.mem.eql(u8, &a.bytes, &b.bytes); + } + + /// Distinguishes whether the legacy multihash format is being stored here. + pub fn isOld(h: *const Hash) bool { + if (h.bytes.len < 2) return false; + const their_multihash_func = std.fmt.parseInt(u8, h.bytes[0..2], 16) catch return false; + if (@as(MultihashFunction, @enumFromInt(their_multihash_func)) != multihash_function) return false; + if (h.toSlice().len != multihash_hex_digest_len) return false; + return std.mem.indexOfScalar(u8, &h.bytes, '-') == null; + } + + test isOld { + const h: Hash = .fromSlice("1220138f4aba0c01e66b68ed9e1e1e74614c06e4743d88bc58af4f1c3dd0aae5fea7"); + try std.testing.expect(h.isOld()); + } + + /// Produces "$name-$semver-$sizedhash". + /// * name is the name field from build.zig.zon, truncated at 32 bytes and must + /// be a valid zig identifier + /// * semver is the version field from build.zig.zon, truncated at 32 bytes + /// * sizedhash is the following 9-byte array, base64 encoded using -_ to make + /// it filesystem safe: + /// - (4 bytes) LE u32 total decompressed size in bytes + /// - (5 bytes) truncated SHA-256 of hashed files of the package + /// + /// example: "nasm-2.16.1-2-BWdcABvF_jM1" + pub fn init(digest: Digest, name: []const u8, ver: []const u8, size: u32) Hash { + var result: Hash = undefined; + var buf: std.ArrayListUnmanaged(u8) = .initBuffer(&result.bytes); + buf.appendSliceAssumeCapacity(name[0..@min(name.len, 32)]); + buf.appendAssumeCapacity('-'); + buf.appendSliceAssumeCapacity(ver[0..@min(ver.len, 32)]); + buf.appendAssumeCapacity('-'); + var sizedhash: [9]u8 = undefined; + std.mem.writeInt(u32, sizedhash[0..4], size, .little); + sizedhash[4..].* = digest[0..5].*; + _ = std.base64.url_safe_no_pad.Encoder.encode(buf.addManyAsArrayAssumeCapacity(12), &sizedhash); + @memset(buf.unusedCapacitySlice(), 0); + return result; + } + + /// Produces "$hashiname-N-$sizedhash". For packages that lack "build.zig.zon" metadata. + /// * hashiname is [5..][0..24] bytes of the SHA-256, urlsafe-base64-encoded, for a total of 32 bytes encoded + /// * the semver section is replaced with a hardcoded N which stands for + /// "naked". It acts as a version number so that any future updates to the + /// hash format can tell this hash format apart. Note that "N" is an + /// invalid semver. + /// * sizedhash is the same as in `init`. + /// + /// The hash is broken up this way so that "sizedhash" can be calculated + /// exactly the same way in both cases, and so that "name" and "hashiname" can + /// be used interchangeably in both cases. + pub fn initNaked(digest: Digest, size: u32) Hash { + var name: [32]u8 = undefined; + _ = std.base64.url_safe_no_pad.Encoder.encode(&name, digest[5..][0..24]); + return init(digest, &name, "N", size); + } +}; + +pub const MultihashFunction = enum(u16) { + identity = 0x00, + sha1 = 0x11, + @"sha2-256" = 0x12, + @"sha2-512" = 0x13, + @"sha3-512" = 0x14, + @"sha3-384" = 0x15, + @"sha3-256" = 0x16, + @"sha3-224" = 0x17, + @"sha2-384" = 0x20, + @"sha2-256-trunc254-padded" = 0x1012, + @"sha2-224" = 0x1013, + @"sha2-512-224" = 0x1014, + @"sha2-512-256" = 0x1015, + @"blake2b-256" = 0xb220, + _, +}; + +pub const multihash_function: MultihashFunction = switch (Hash.Algo) { + std.crypto.hash.sha2.Sha256 => .@"sha2-256", + else => @compileError("unreachable"), +}; + +pub fn multiHashHexDigest(digest: Hash.Digest) MultiHashHexDigest { + const hex_charset = std.fmt.hex_charset; + + var result: MultiHashHexDigest = undefined; + + result[0] = hex_charset[@intFromEnum(multihash_function) >> 4]; + result[1] = hex_charset[@intFromEnum(multihash_function) & 15]; + + result[2] = hex_charset[Hash.Algo.digest_length >> 4]; + result[3] = hex_charset[Hash.Algo.digest_length & 15]; + + for (digest, 0..) |byte, i| { + result[4 + i * 2] = hex_charset[byte >> 4]; + result[5 + i * 2] = hex_charset[byte & 15]; + } + return result; +} + +comptime { + // We avoid unnecessary uleb128 code in hexDigest by asserting here the + // values are small enough to be contained in the one-byte encoding. + assert(@intFromEnum(multihash_function) < 127); + assert(Hash.Algo.digest_length < 127); +} + +test Hash { + const example_digest: Hash.Digest = .{ + 0xc7, 0xf5, 0x71, 0xb7, 0xb4, 0xe7, 0x6f, 0x3c, 0xdb, 0x87, 0x7a, 0x7f, 0xdd, 0xf9, 0x77, 0x87, + 0x9d, 0xd3, 0x86, 0xfa, 0x73, 0x57, 0x9a, 0xf7, 0x9d, 0x1e, 0xdb, 0x8f, 0x3a, 0xd9, 0xbd, 0x9f, + }; + const result: Hash = .init(example_digest, "nasm", "2.16.1-2", 10 * 1024 * 1024); + try std.testing.expectEqualStrings("nasm-2.16.1-2-AACgAMf1cbe0", result.toSlice()); +} + test { _ = Fetch; } diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index 0d6cf55636..11878a12b7 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -56,7 +56,7 @@ package_root: Cache.Path, error_bundle: ErrorBundle.Wip, manifest: ?Manifest, manifest_ast: std.zig.Ast, -actual_hash: Manifest.Digest, +computed_hash: ComputedHash, /// Fetch logic notices whether a package has a build.zig file and sets this flag. has_build_zig: bool, /// Indicates whether the task aborted due to an out-of-memory condition. @@ -116,8 +116,8 @@ pub const JobQueue = struct { /// as lazy. unlazy_set: UnlazySet = .{}, - pub const Table = std.AutoArrayHashMapUnmanaged(Manifest.MultiHashHexDigest, *Fetch); - pub const UnlazySet = std.AutoArrayHashMapUnmanaged(Manifest.MultiHashHexDigest, void); + pub const Table = std.AutoArrayHashMapUnmanaged(Package.Hash, *Fetch); + pub const UnlazySet = std.AutoArrayHashMapUnmanaged(Package.Hash, void); pub fn deinit(jq: *JobQueue) void { if (jq.all_fetches.items.len == 0) return; @@ -160,22 +160,24 @@ pub const JobQueue = struct { // Ensure the generated .zig file is deterministic. jq.table.sortUnstable(@as(struct { - keys: []const Manifest.MultiHashHexDigest, + keys: []const Package.Hash, pub fn lessThan(ctx: @This(), a_index: usize, b_index: usize) bool { - return std.mem.lessThan(u8, &ctx.keys[a_index], &ctx.keys[b_index]); + return std.mem.lessThan(u8, &ctx.keys[a_index].bytes, &ctx.keys[b_index].bytes); } }, .{ .keys = keys })); - for (keys, jq.table.values()) |hash, fetch| { + for (keys, jq.table.values()) |*hash, fetch| { if (fetch == jq.all_fetches.items[0]) { // The first one is a dummy package for the current project. continue; } + const hash_slice = hash.toSlice(); + try buf.writer().print( \\ pub const {} = struct {{ \\ - , .{std.zig.fmtId(&hash)}); + , .{std.zig.fmtId(hash_slice)}); lazy: { switch (fetch.lazy_status) { @@ -207,7 +209,7 @@ pub const JobQueue = struct { try buf.writer().print( \\ pub const build_zig = @import("{}"); \\ - , .{std.zig.fmtEscapes(&hash)}); + , .{std.zig.fmtEscapes(hash_slice)}); } if (fetch.manifest) |*manifest| { @@ -219,7 +221,7 @@ pub const JobQueue = struct { const h = depDigest(fetch.package_root, jq.global_cache, dep) orelse continue; try buf.writer().print( " .{{ \"{}\", \"{}\" }},\n", - .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(&h) }, + .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(h.toSlice()) }, ); } @@ -251,7 +253,7 @@ pub const JobQueue = struct { const h = depDigest(root_fetch.package_root, jq.global_cache, dep) orelse continue; try buf.writer().print( " .{{ \"{}\", \"{}\" }},\n", - .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(&h) }, + .{ std.zig.fmtEscapes(name), std.zig.fmtEscapes(h.toSlice()) }, ); } try buf.appendSlice("};\n"); @@ -283,7 +285,7 @@ pub const Location = union(enum) { url: []const u8, /// If this is null it means the user omitted the hash field from a dependency. /// It will be an error but the logic should still fetch and print the discovered hash. - hash: ?Manifest.MultiHashHexDigest, + hash: ?Package.Hash, }; }; @@ -325,9 +327,11 @@ pub fn run(f: *Fetch) RunError!void { // "p/$hash/foo", with possibly more directories after "foo". // We want to fail unless the resolved relative path has a // prefix of "p/$hash/". - const digest_len = @typeInfo(Manifest.MultiHashHexDigest).array.len; const prefix_len: usize = if (f.job_queue.read_only) 0 else "p/".len; - const expected_prefix = f.parent_package_root.sub_path[0 .. prefix_len + digest_len]; + const parent_sub_path = f.parent_package_root.sub_path; + const end = std.mem.indexOfScalarPos(u8, parent_sub_path, prefix_len, fs.path.sep) orelse + parent_sub_path.len; + const expected_prefix = parent_sub_path[prefix_len..end]; if (!std.mem.startsWith(u8, pkg_root.sub_path, expected_prefix)) { return f.fail( f.location_tok, @@ -367,9 +371,13 @@ pub fn run(f: *Fetch) RunError!void { }, }; - const s = fs.path.sep_str; if (remote.hash) |expected_hash| { - const prefixed_pkg_sub_path = "p" ++ s ++ expected_hash; + var prefixed_pkg_sub_path_buffer: [100]u8 = undefined; + prefixed_pkg_sub_path_buffer[0] = 'p'; + prefixed_pkg_sub_path_buffer[1] = fs.path.sep; + const hash_slice = expected_hash.toSlice(); + @memcpy(prefixed_pkg_sub_path_buffer[2..][0..hash_slice.len], hash_slice); + const prefixed_pkg_sub_path = prefixed_pkg_sub_path_buffer[0 .. 2 + hash_slice.len]; const prefix_len: usize = if (f.job_queue.read_only) "p/".len else 0; const pkg_sub_path = prefixed_pkg_sub_path[prefix_len..]; if (cache_root.handle.access(pkg_sub_path, .{})) |_| { @@ -437,7 +445,7 @@ fn runResource( f: *Fetch, uri_path: []const u8, resource: *Resource, - remote_hash: ?Manifest.MultiHashHexDigest, + remote_hash: ?Package.Hash, ) RunError!void { defer resource.deinit(); const arena = f.arena.allocator(); @@ -499,7 +507,7 @@ fn runResource( // Empty directories have already been omitted by `unpackResource`. // Compute the package hash based on the remaining files in the temporary // directory. - f.actual_hash = try computeHash(f, pkg_path, filter); + f.computed_hash = try computeHash(f, pkg_path, filter); break :blk if (unpack_result.root_dir.len > 0) try fs.path.join(arena, &.{ tmp_dir_sub_path, unpack_result.root_dir }) @@ -507,6 +515,8 @@ fn runResource( tmp_dir_sub_path; }; + const computed_package_hash = computedPackageHash(f); + // Rename the temporary directory into the global zig package cache // directory. If the hash already exists, delete the temporary directory // and leave the zig package cache directory untouched as it may be in use @@ -515,7 +525,7 @@ fn runResource( f.package_root = .{ .root_dir = cache_root, - .sub_path = try arena.dupe(u8, "p" ++ s ++ Manifest.hexDigest(f.actual_hash)), + .sub_path = try std.fmt.allocPrint(arena, "p" ++ s ++ "{s}", .{computed_package_hash.toSlice()}), }; renameTmpIntoCache(cache_root.handle, package_sub_path, f.package_root.sub_path) catch |err| { const src = try cache_root.join(arena, &.{tmp_dir_sub_path}); @@ -534,13 +544,22 @@ fn runResource( // Validate the computed hash against the expected hash. If invalid, this // job is done. - const actual_hex = Manifest.hexDigest(f.actual_hash); if (remote_hash) |declared_hash| { - if (!std.mem.eql(u8, &declared_hash, &actual_hex)) { - return f.fail(f.hash_tok, try eb.printString( - "hash mismatch: manifest declares {s} but the fetched package has {s}", - .{ declared_hash, actual_hex }, - )); + if (declared_hash.isOld()) { + const actual_hex = Package.multiHashHexDigest(f.computed_hash.digest); + if (!std.mem.eql(u8, declared_hash.toSlice(), &actual_hex)) { + return f.fail(f.hash_tok, try eb.printString( + "hash mismatch: manifest declares {s} but the fetched package has {s}", + .{ declared_hash.toSlice(), actual_hex }, + )); + } + } else { + if (!computed_package_hash.eql(&declared_hash)) { + return f.fail(f.hash_tok, try eb.printString( + "hash mismatch: manifest declares {s} but the fetched package has {s}", + .{ declared_hash.toSlice(), computed_package_hash.toSlice() }, + )); + } } } else if (!f.omit_missing_hash_error) { const notes_len = 1; @@ -551,7 +570,7 @@ fn runResource( }); const notes_start = try eb.reserveNotes(notes_len); eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ - .msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}), + .msg = try eb.printString("expected .hash = \"{s}\",", .{computed_package_hash.toSlice()}), })); return error.FetchFailed; } @@ -562,6 +581,16 @@ fn runResource( return queueJobsForDeps(f); } +pub fn computedPackageHash(f: *const Fetch) Package.Hash { + const saturated_size = std.math.cast(u32, f.computed_hash.total_size) orelse std.math.maxInt(u32); + if (f.manifest) |man| { + var version_buffer: [32]u8 = undefined; + const version: []const u8 = std.fmt.bufPrint(&version_buffer, "{}", .{man.version}) catch &version_buffer; + return .init(f.computed_hash.digest, man.name, version, saturated_size); + } + return .initNaked(f.computed_hash.digest, saturated_size); +} + /// `computeHash` gets a free check for the existence of `build.zig`, but when /// not computing a hash, we need to do a syscall to check for it. fn checkBuildFileExistence(f: *Fetch) RunError!void { @@ -673,9 +702,8 @@ fn queueJobsForDeps(f: *Fetch) RunError!void { .url = url, .hash = h: { const h = dep.hash orelse break :h null; - const digest_len = @typeInfo(Manifest.MultiHashHexDigest).array.len; - const multihash_digest = h[0..digest_len].*; - const gop = f.job_queue.table.getOrPutAssumeCapacity(multihash_digest); + const pkg_hash: Package.Hash = .fromSlice(h); + const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash); if (gop.found_existing) { if (!dep.lazy) { gop.value_ptr.*.lazy_status = .eager; @@ -683,15 +711,15 @@ fn queueJobsForDeps(f: *Fetch) RunError!void { continue; } gop.value_ptr.* = new_fetch; - break :h multihash_digest; + break :h pkg_hash; }, } }, .path => |rel_path| l: { // This might produce an invalid path, which is checked for // at the beginning of run(). const new_root = try f.package_root.resolvePosix(parent_arena, rel_path); - const multihash_digest = relativePathDigest(new_root, cache_root); - const gop = f.job_queue.table.getOrPutAssumeCapacity(multihash_digest); + const pkg_hash = relativePathDigest(new_root, cache_root); + const gop = f.job_queue.table.getOrPutAssumeCapacity(pkg_hash); if (gop.found_existing) { if (!dep.lazy) { gop.value_ptr.*.lazy_status = .eager; @@ -724,7 +752,7 @@ fn queueJobsForDeps(f: *Fetch) RunError!void { .error_bundle = undefined, .manifest = null, .manifest_ast = undefined, - .actual_hash = undefined, + .computed_hash = undefined, .has_build_zig = false, .oom_flag = false, .latest_commit = null, @@ -746,11 +774,8 @@ fn queueJobsForDeps(f: *Fetch) RunError!void { } } -pub fn relativePathDigest( - pkg_root: Cache.Path, - cache_root: Cache.Directory, -) Manifest.MultiHashHexDigest { - var hasher = Manifest.Hash.init(.{}); +pub fn relativePathDigest(pkg_root: Cache.Path, cache_root: Cache.Directory) Package.Hash { + var hasher = Package.Hash.Algo.init(.{}); // This hash is a tuple of: // * whether it relative to the global cache directory or to the root package // * the relative file path from there to the build root of the package @@ -759,7 +784,7 @@ pub fn relativePathDigest( else &package_hash_prefix_project); hasher.update(pkg_root.sub_path); - return Manifest.hexDigest(hasher.finalResult()); + return .fromSlice(&hasher.finalResult()); } pub fn workerRun(f: *Fetch, prog_name: []const u8) void { @@ -1387,11 +1412,7 @@ fn recursiveDirectoryCopy(f: *Fetch, dir: fs.Dir, tmp_dir: fs.Dir) anyerror!void } } -pub fn renameTmpIntoCache( - cache_dir: fs.Dir, - tmp_dir_sub_path: []const u8, - dest_dir_sub_path: []const u8, -) !void { +pub fn renameTmpIntoCache(cache_dir: fs.Dir, tmp_dir_sub_path: []const u8, dest_dir_sub_path: []const u8) !void { assert(dest_dir_sub_path[1] == fs.path.sep); var handled_missing_dir = false; while (true) { @@ -1417,16 +1438,17 @@ pub fn renameTmpIntoCache( } } +const ComputedHash = struct { + digest: Package.Hash.Digest, + total_size: u64, +}; + /// Assumes that files not included in the package have already been filtered /// prior to calling this function. This ensures that files not protected by /// the hash are not present on the file system. Empty directories are *not /// hashed* and must not be present on the file system when calling this /// function. -fn computeHash( - f: *Fetch, - pkg_path: Cache.Path, - filter: Filter, -) RunError!Manifest.Digest { +fn computeHash(f: *Fetch, pkg_path: Cache.Path, filter: Filter) RunError!ComputedHash { // All the path name strings need to be in memory for sorting. const arena = f.arena.allocator(); const gpa = f.arena.child_allocator; @@ -1449,6 +1471,9 @@ fn computeHash( var walker = try root_dir.walk(gpa); defer walker.deinit(); + // Total number of bytes of file contents included in the package. + var total_size: u64 = 0; + { // The final hash will be a hash of each file hashed independently. This // allows hashing in parallel. @@ -1506,6 +1531,7 @@ fn computeHash( .kind = kind, .hash = undefined, // to be populated by the worker .failure = undefined, // to be populated by the worker + .size = undefined, // to be populated by the worker }; thread_pool.spawnWg(&wait_group, workerHashFile, .{ root_dir, hashed_file }); try all_files.append(hashed_file); @@ -1544,7 +1570,7 @@ fn computeHash( std.mem.sortUnstable(*HashedFile, all_files.items, {}, HashedFile.lessThan); - var hasher = Manifest.Hash.init(.{}); + var hasher = Package.Hash.Algo.init(.{}); var any_failures = false; for (all_files.items) |hashed_file| { hashed_file.failure catch |err| { @@ -1556,6 +1582,7 @@ fn computeHash( }); }; hasher.update(&hashed_file.hash); + total_size += hashed_file.size; } for (deleted_files.items) |deleted_file| { deleted_file.failure catch |err| { @@ -1580,7 +1607,10 @@ fn computeHash( }; } - return hasher.finalResult(); + return .{ + .digest = hasher.finalResult(), + .total_size = total_size, + }; } fn dumpHashInfo(all_files: []const *const HashedFile) !void { @@ -1609,8 +1639,9 @@ fn workerDeleteFile(dir: fs.Dir, deleted_file: *DeletedFile) void { fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void { var buf: [8000]u8 = undefined; - var hasher = Manifest.Hash.init(.{}); + var hasher = Package.Hash.Algo.init(.{}); hasher.update(hashed_file.normalized_path); + var file_size: u64 = 0; switch (hashed_file.kind) { .file => { @@ -1622,6 +1653,7 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void while (true) { const bytes_read = try file.read(&buf); if (bytes_read == 0) break; + file_size += bytes_read; hasher.update(buf[0..bytes_read]); file_header.update(buf[0..bytes_read]); } @@ -1641,6 +1673,7 @@ fn hashFileFallible(dir: fs.Dir, hashed_file: *HashedFile) HashedFile.Error!void }, } hasher.final(&hashed_file.hash); + hashed_file.size = file_size; } fn deleteFileFallible(dir: fs.Dir, deleted_file: *DeletedFile) DeletedFile.Error!void { @@ -1667,9 +1700,10 @@ const DeletedFile = struct { const HashedFile = struct { fs_path: []const u8, normalized_path: []const u8, - hash: Manifest.Digest, + hash: Package.Hash.Digest, failure: Error!void, kind: Kind, + size: u64, const Error = fs.File.OpenError || @@ -1744,12 +1778,8 @@ const Filter = struct { } }; -pub fn depDigest( - pkg_root: Cache.Path, - cache_root: Cache.Directory, - dep: Manifest.Dependency, -) ?Manifest.MultiHashHexDigest { - if (dep.hash) |h| return h[0..Manifest.multihash_hex_digest_len].*; +pub fn depDigest(pkg_root: Cache.Path, cache_root: Cache.Directory, dep: Manifest.Dependency) ?Package.Hash { + if (dep.hash) |h| return .fromSlice(h); switch (dep.location) { .url => return null, @@ -2137,7 +2167,7 @@ test "tarball with excluded duplicate paths" { defer fb.deinit(); try fetch.run(); - const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash); + const hex_digest = Package.multiHashHexDigest(fetch.computed_hash.digest); try std.testing.expectEqualStrings( "12200bafe035cbb453dd717741b66e9f9d1e6c674069d06121dafa1b2e62eb6b22da", &hex_digest, @@ -2181,7 +2211,7 @@ test "tarball without root folder" { defer fb.deinit(); try fetch.run(); - const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash); + const hex_digest = Package.multiHashHexDigest(fetch.computed_hash.digest); try std.testing.expectEqualStrings( "12209f939bfdcb8b501a61bb4a43124dfa1b2848adc60eec1e4624c560357562b793", &hex_digest, @@ -2222,7 +2252,7 @@ test "set executable bit based on file content" { try fetch.run(); try std.testing.expectEqualStrings( "1220fecb4c06a9da8673c87fe8810e15785f1699212f01728eadce094d21effeeef3", - &Manifest.hexDigest(fetch.actual_hash), + &Package.multiHashHexDigest(fetch.computed_hash.digest), ); var out = try fb.packageDir(); @@ -2304,7 +2334,7 @@ const TestFetchBuilder = struct { .error_bundle = undefined, .manifest = null, .manifest_ast = undefined, - .actual_hash = undefined, + .computed_hash = undefined, .has_build_zig = false, .oom_flag = false, .latest_commit = null, diff --git a/src/Package/Manifest.zig b/src/Package/Manifest.zig index 4eed6cc386..bfc5c813e2 100644 --- a/src/Package/Manifest.zig +++ b/src/Package/Manifest.zig @@ -5,15 +5,10 @@ const Allocator = std.mem.Allocator; const assert = std.debug.assert; const Ast = std.zig.Ast; const testing = std.testing; -const hex_charset = std.fmt.hex_charset; +const Package = @import("../Package.zig"); pub const max_bytes = 10 * 1024 * 1024; pub const basename = "build.zig.zon"; -pub const Hash = std.crypto.hash.sha2.Sha256; -pub const Digest = [Hash.digest_length]u8; -pub const multihash_len = 1 + 1 + Hash.digest_length; -pub const multihash_hex_digest_len = 2 * multihash_len; -pub const MultiHashHexDigest = [multihash_hex_digest_len]u8; pub const Dependency = struct { location: Location, @@ -38,35 +33,6 @@ pub const ErrorMessage = struct { off: u32, }; -pub const MultihashFunction = enum(u16) { - identity = 0x00, - sha1 = 0x11, - @"sha2-256" = 0x12, - @"sha2-512" = 0x13, - @"sha3-512" = 0x14, - @"sha3-384" = 0x15, - @"sha3-256" = 0x16, - @"sha3-224" = 0x17, - @"sha2-384" = 0x20, - @"sha2-256-trunc254-padded" = 0x1012, - @"sha2-224" = 0x1013, - @"sha2-512-224" = 0x1014, - @"sha2-512-256" = 0x1015, - @"blake2b-256" = 0xb220, - _, -}; - -pub const multihash_function: MultihashFunction = switch (Hash) { - std.crypto.hash.sha2.Sha256 => .@"sha2-256", - else => @compileError("unreachable"), -}; -comptime { - // We avoid unnecessary uleb128 code in hexDigest by asserting here the - // values are small enough to be contained in the one-byte encoding. - assert(@intFromEnum(multihash_function) < 127); - assert(Hash.digest_length < 127); -} - name: []const u8, version: std.SemanticVersion, version_node: Ast.Node.Index, @@ -164,22 +130,6 @@ pub fn copyErrorsIntoBundle( } } -pub fn hexDigest(digest: Digest) MultiHashHexDigest { - var result: MultiHashHexDigest = undefined; - - result[0] = hex_charset[@intFromEnum(multihash_function) >> 4]; - result[1] = hex_charset[@intFromEnum(multihash_function) & 15]; - - result[2] = hex_charset[Hash.digest_length >> 4]; - result[3] = hex_charset[Hash.digest_length & 15]; - - for (digest, 0..) |byte, i| { - result[4 + i * 2] = hex_charset[byte >> 4]; - result[5 + i * 2] = hex_charset[byte & 15]; - } - return result; -} - const Parse = struct { gpa: Allocator, ast: Ast, @@ -421,21 +371,8 @@ const Parse = struct { const tok = main_tokens[node]; const h = try parseString(p, node); - if (h.len >= 2) { - const their_multihash_func = std.fmt.parseInt(u8, h[0..2], 16) catch |err| { - return fail(p, tok, "invalid multihash value: unable to parse hash function: {s}", .{ - @errorName(err), - }); - }; - if (@as(MultihashFunction, @enumFromInt(their_multihash_func)) != multihash_function) { - return fail(p, tok, "unsupported hash function: only sha2-256 is supported", .{}); - } - } - - if (h.len != multihash_hex_digest_len) { - return fail(p, tok, "wrong hash size. expected: {d}, found: {d}", .{ - multihash_hex_digest_len, h.len, - }); + if (h.len > Package.Hash.max_len) { + return fail(p, tok, "hash length exceeds maximum: {d}", .{h.len}); } return h; diff --git a/src/main.zig b/src/main.zig index 5e66244484..d6b20f94f9 100644 --- a/src/main.zig +++ b/src/main.zig @@ -5197,7 +5197,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { .error_bundle = undefined, .manifest = null, .manifest_ast = undefined, - .actual_hash = undefined, + .computed_hash = undefined, .has_build_zig = true, .oom_flag = false, .latest_commit = null, @@ -5244,13 +5244,14 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { const hashes = job_queue.table.keys(); const fetches = job_queue.table.values(); try deps_mod.deps.ensureUnusedCapacity(arena, @intCast(hashes.len)); - for (hashes, fetches) |hash, f| { + for (hashes, fetches) |*hash, f| { if (f == &fetch) { // The first one is a dummy package for the current project. continue; } if (!f.has_build_zig) continue; + const hash_slice = hash.toSlice(); const m = try Package.Module.create(arena, .{ .global_cache_directory = global_cache_directory, .paths = .{ @@ -5260,7 +5261,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { .fully_qualified_name = try std.fmt.allocPrint( arena, "root.@dependencies.{s}", - .{&hash}, + .{hash_slice}, ), .cc_argv = &.{}, .inherited = .{}, @@ -5269,7 +5270,7 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { .builtin_mod = builtin_mod, .builtin_modules = null, // `builtin_mod` is specified }); - const hash_cloned = try arena.dupe(u8, &hash); + const hash_cloned = try arena.dupe(u8, hash_slice); deps_mod.deps.putAssumeCapacityNoClobber(hash_cloned, m); f.module = m; } @@ -5385,23 +5386,22 @@ fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !void { var any_errors = false; while (it.next()) |hash| { if (hash.len == 0) continue; - const digest_len = @typeInfo(Package.Manifest.MultiHashHexDigest).array.len; - if (hash.len != digest_len) { - std.log.err("invalid digest (length {d} instead of {d}): '{s}'", .{ - hash.len, digest_len, hash, + if (hash.len > Package.Hash.max_len) { + std.log.err("invalid digest (length {d} exceeds maximum): '{s}'", .{ + hash.len, hash, }); any_errors = true; continue; } - try unlazy_set.put(arena, hash[0..digest_len].*, {}); + try unlazy_set.put(arena, .fromSlice(hash), {}); } if (any_errors) process.exit(3); if (system_pkg_dir_path) |p| { // In this mode, the system needs to provide these packages; they // cannot be fetched by Zig. - for (unlazy_set.keys()) |hash| { + for (unlazy_set.keys()) |*hash| { std.log.err("lazy dependency package not found: {s}" ++ s ++ "{s}", .{ - p, hash, + p, hash.toSlice(), }); } std.log.info("remote package fetching disabled due to --system mode", .{}); @@ -7097,7 +7097,7 @@ fn cmdFetch( .error_bundle = undefined, .manifest = null, .manifest_ast = undefined, - .actual_hash = undefined, + .computed_hash = undefined, .has_build_zig = false, .oom_flag = false, .latest_commit = null, @@ -7117,14 +7117,15 @@ fn cmdFetch( process.exit(1); } - const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash); + const package_hash = fetch.computedPackageHash(); + const package_hash_slice = package_hash.toSlice(); root_prog_node.end(); root_prog_node = .{ .index = .none }; const name = switch (save) { .no => { - try io.getStdOut().writeAll(hex_digest ++ "\n"); + try io.getStdOut().writer().print("{s}\n", .{package_hash_slice}); return cleanExit(); }, .yes, .exact => |name| name: { @@ -7194,7 +7195,7 @@ fn cmdFetch( \\ }} , .{ std.zig.fmtEscapes(saved_path_or_url), - std.zig.fmtEscapes(&hex_digest), + std.zig.fmtEscapes(package_hash_slice), }); const new_node_text = try std.fmt.allocPrint(arena, ".{p_} = {s},\n", .{ @@ -7213,7 +7214,7 @@ fn cmdFetch( if (dep.hash) |h| { switch (dep.location) { .url => |u| { - if (mem.eql(u8, h, &hex_digest) and mem.eql(u8, u, saved_path_or_url)) { + if (mem.eql(u8, h, package_hash_slice) and mem.eql(u8, u, saved_path_or_url)) { std.log.info("existing dependency named '{s}' is up-to-date", .{name}); process.exit(0); } @@ -7230,7 +7231,7 @@ fn cmdFetch( const hash_replace = try std.fmt.allocPrint( arena, "\"{}\"", - .{std.zig.fmtEscapes(&hex_digest)}, + .{std.zig.fmtEscapes(package_hash_slice)}, ); warn("overwriting existing dependency named '{s}'", .{name});