From d0bcc390e8f61ada470b524e3fd203c1af521a99 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Fri, 6 Oct 2023 17:05:58 -0700 Subject: [PATCH] get `zig fetch` working with the new system * start renaming "package" to "module" (see #14307) - build system gains `main_mod_path` and `main_pkg_path` is still there but it is deprecated. * eliminate the object-oriented memory management style of what was previously `*Package`. Now it is `*Package.Module` and all pointers point to externally managed memory. * fixes to get the new Fetch.zig code working. The previous commit was work-in-progress. There are still two commented out code paths, the one that leads to `Compilation.create` and the one for `zig build` that fetches the entire dependency tree and creates the required modules for the build runner. --- build.zig | 2 +- lib/std/Build.zig | 25 +- lib/std/Build/Cache.zig | 11 + lib/std/Build/Step/Compile.zig | 15 +- src/Compilation.zig | 170 +++++---- src/Manifest.zig | 14 +- src/Module.zig | 81 ++--- src/Package.zig | 326 +++++------------ src/Package/Fetch.zig | 645 ++++++++++++++++++++++----------- src/Package/Module.zig | 32 ++ src/crash_report.zig | 22 +- src/main.zig | 308 ++++++++-------- 12 files changed, 892 insertions(+), 759 deletions(-) create mode 100644 src/Package/Module.zig diff --git a/build.zig b/build.zig index 423f80b12a..6f93585872 100644 --- a/build.zig +++ b/build.zig @@ -88,7 +88,7 @@ pub fn build(b: *std.Build) !void { .name = "check-case", .root_source_file = .{ .path = "test/src/Cases.zig" }, .optimize = optimize, - .main_pkg_path = .{ .path = "." }, + .main_mod_path = .{ .path = "." }, }); check_case_exe.stack_size = stack_size; check_case_exe.single_threaded = single_threaded; diff --git a/lib/std/Build.zig b/lib/std/Build.zig index 49afd73778..5c80f4972c 100644 --- a/lib/std/Build.zig +++ b/lib/std/Build.zig @@ -634,6 +634,9 @@ pub const ExecutableOptions = struct { use_llvm: ?bool = null, use_lld: ?bool = null, zig_lib_dir: ?LazyPath = null, + main_mod_path: ?LazyPath = null, + + /// Deprecated; use `main_mod_path`. main_pkg_path: ?LazyPath = null, }; @@ -652,7 +655,7 @@ pub fn addExecutable(b: *Build, options: ExecutableOptions) *Step.Compile { .use_llvm = options.use_llvm, .use_lld = options.use_lld, .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir, - .main_pkg_path = options.main_pkg_path, + .main_mod_path = options.main_mod_path orelse options.main_pkg_path, }); } @@ -667,6 +670,9 @@ pub const ObjectOptions = struct { use_llvm: ?bool = null, use_lld: ?bool = null, zig_lib_dir: ?LazyPath = null, + main_mod_path: ?LazyPath = null, + + /// Deprecated; use `main_mod_path`. main_pkg_path: ?LazyPath = null, }; @@ -683,7 +689,7 @@ pub fn addObject(b: *Build, options: ObjectOptions) *Step.Compile { .use_llvm = options.use_llvm, .use_lld = options.use_lld, .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir, - .main_pkg_path = options.main_pkg_path, + .main_mod_path = options.main_mod_path orelse options.main_pkg_path, }); } @@ -699,6 +705,9 @@ pub const SharedLibraryOptions = struct { use_llvm: ?bool = null, use_lld: ?bool = null, zig_lib_dir: ?LazyPath = null, + main_mod_path: ?LazyPath = null, + + /// Deprecated; use `main_mod_path`. main_pkg_path: ?LazyPath = null, }; @@ -717,7 +726,7 @@ pub fn addSharedLibrary(b: *Build, options: SharedLibraryOptions) *Step.Compile .use_llvm = options.use_llvm, .use_lld = options.use_lld, .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir, - .main_pkg_path = options.main_pkg_path, + .main_mod_path = options.main_mod_path orelse options.main_pkg_path, }); } @@ -733,6 +742,9 @@ pub const StaticLibraryOptions = struct { use_llvm: ?bool = null, use_lld: ?bool = null, zig_lib_dir: ?LazyPath = null, + main_mod_path: ?LazyPath = null, + + /// Deprecated; use `main_mod_path`. main_pkg_path: ?LazyPath = null, }; @@ -751,7 +763,7 @@ pub fn addStaticLibrary(b: *Build, options: StaticLibraryOptions) *Step.Compile .use_llvm = options.use_llvm, .use_lld = options.use_lld, .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir, - .main_pkg_path = options.main_pkg_path, + .main_mod_path = options.main_mod_path orelse options.main_pkg_path, }); } @@ -769,6 +781,9 @@ pub const TestOptions = struct { use_llvm: ?bool = null, use_lld: ?bool = null, zig_lib_dir: ?LazyPath = null, + main_mod_path: ?LazyPath = null, + + /// Deprecated; use `main_mod_path`. main_pkg_path: ?LazyPath = null, }; @@ -787,7 +802,7 @@ pub fn addTest(b: *Build, options: TestOptions) *Step.Compile { .use_llvm = options.use_llvm, .use_lld = options.use_lld, .zig_lib_dir = options.zig_lib_dir orelse b.zig_lib_dir, - .main_pkg_path = options.main_pkg_path, + .main_mod_path = options.main_mod_path orelse options.main_pkg_path, }); } diff --git a/lib/std/Build/Cache.zig b/lib/std/Build/Cache.zig index f76985a01a..bb36bb978f 100644 --- a/lib/std/Build/Cache.zig +++ b/lib/std/Build/Cache.zig @@ -9,6 +9,13 @@ pub const Directory = struct { path: ?[]const u8, handle: fs.Dir, + pub fn cwd() Directory { + return .{ + .path = null, + .handle = fs.cwd(), + }; + } + pub fn join(self: Directory, allocator: Allocator, paths: []const []const u8) ![]u8 { if (self.path) |p| { // TODO clean way to do this with only 1 allocation @@ -53,6 +60,10 @@ pub const Directory = struct { try writer.writeAll(fs.path.sep_str); } } + + pub fn eql(self: Directory, other: Directory) bool { + return self.handle.fd == other.handle.fd; + } }; gpa: Allocator, diff --git a/lib/std/Build/Step/Compile.zig b/lib/std/Build/Step/Compile.zig index 07eb3ded01..9e711b6f52 100644 --- a/lib/std/Build/Step/Compile.zig +++ b/lib/std/Build/Step/Compile.zig @@ -68,7 +68,7 @@ c_std: std.Build.CStd, /// Set via options; intended to be read-only after that. zig_lib_dir: ?LazyPath, /// Set via options; intended to be read-only after that. -main_pkg_path: ?LazyPath, +main_mod_path: ?LazyPath, exec_cmd_args: ?[]const ?[]const u8, filter: ?[]const u8, test_evented_io: bool = false, @@ -316,6 +316,9 @@ pub const Options = struct { use_llvm: ?bool = null, use_lld: ?bool = null, zig_lib_dir: ?LazyPath = null, + main_mod_path: ?LazyPath = null, + + /// deprecated; use `main_mod_path`. main_pkg_path: ?LazyPath = null, }; @@ -480,7 +483,7 @@ pub fn create(owner: *std.Build, options: Options) *Compile { .installed_headers = ArrayList(*Step).init(owner.allocator), .c_std = std.Build.CStd.C99, .zig_lib_dir = null, - .main_pkg_path = null, + .main_mod_path = null, .exec_cmd_args = null, .filter = options.filter, .test_runner = options.test_runner, @@ -515,8 +518,8 @@ pub fn create(owner: *std.Build, options: Options) *Compile { lp.addStepDependencies(&self.step); } - if (options.main_pkg_path) |lp| { - self.main_pkg_path = lp.dupe(self.step.owner); + if (options.main_mod_path orelse options.main_pkg_path) |lp| { + self.main_mod_path = lp.dupe(self.step.owner); lp.addStepDependencies(&self.step); } @@ -1998,8 +2001,8 @@ fn make(step: *Step, prog_node: *std.Progress.Node) !void { try zig_args.append(dir.getPath(b)); } - if (self.main_pkg_path) |dir| { - try zig_args.append("--main-pkg-path"); + if (self.main_mod_path) |dir| { + try zig_args.append("--main-mod-path"); try zig_args.append(dir.getPath(b)); } diff --git a/src/Compilation.zig b/src/Compilation.zig index 28b67ff734..3827789fed 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -273,8 +273,8 @@ const Job = union(enum) { /// The source file containing the Decl has been updated, and so the /// Decl may need its line number information updated in the debug info. update_line_number: Module.Decl.Index, - /// The main source file for the package needs to be analyzed. - analyze_pkg: *Package, + /// The main source file for the module needs to be analyzed. + analyze_mod: *Package.Module, /// one of the glibc static objects glibc_crt_file: glibc.CRTFile, @@ -414,7 +414,7 @@ pub const MiscTask = enum { compiler_rt, libssp, zig_libc, - analyze_pkg, + analyze_mod, @"musl crti.o", @"musl crtn.o", @@ -544,7 +544,7 @@ pub const InitOptions = struct { global_cache_directory: Directory, target: Target, root_name: []const u8, - main_pkg: ?*Package, + main_mod: ?*Package.Module, output_mode: std.builtin.OutputMode, thread_pool: *ThreadPool, dynamic_linker: ?[]const u8 = null, @@ -736,53 +736,53 @@ pub const InitOptions = struct { pdb_out_path: ?[]const u8 = null, }; -fn addPackageTableToCacheHash( +fn addModuleTableToCacheHash( hash: *Cache.HashHelper, arena: *std.heap.ArenaAllocator, - pkg_table: Package.Table, - seen_table: *std.AutoHashMap(*Package, void), + mod_table: Package.Module.Deps, + seen_table: *std.AutoHashMap(*Package.Module, void), hash_type: union(enum) { path_bytes, files: *Cache.Manifest }, ) (error{OutOfMemory} || std.os.GetCwdError)!void { const allocator = arena.allocator(); - const packages = try allocator.alloc(Package.Table.KV, pkg_table.count()); + const modules = try allocator.alloc(Package.Module.Deps.KV, mod_table.count()); { // Copy over the hashmap entries to our slice - var table_it = pkg_table.iterator(); + var table_it = mod_table.iterator(); var idx: usize = 0; while (table_it.next()) |entry| : (idx += 1) { - packages[idx] = .{ + modules[idx] = .{ .key = entry.key_ptr.*, .value = entry.value_ptr.*, }; } } // Sort the slice by package name - mem.sort(Package.Table.KV, packages, {}, struct { - fn lessThan(_: void, lhs: Package.Table.KV, rhs: Package.Table.KV) bool { + mem.sortUnstable(Package.Module.Deps.KV, modules, {}, struct { + fn lessThan(_: void, lhs: Package.Module.Deps.KV, rhs: Package.Module.Deps.KV) bool { return std.mem.lessThan(u8, lhs.key, rhs.key); } }.lessThan); - for (packages) |pkg| { - if ((try seen_table.getOrPut(pkg.value)).found_existing) continue; + for (modules) |mod| { + if ((try seen_table.getOrPut(mod.value)).found_existing) continue; // Finally insert the package name and path to the cache hash. - hash.addBytes(pkg.key); + hash.addBytes(mod.key); switch (hash_type) { .path_bytes => { - hash.addBytes(pkg.value.root_src_path); - hash.addOptionalBytes(pkg.value.root_src_directory.path); + hash.addBytes(mod.value.root_src_path); + hash.addOptionalBytes(mod.value.root_src_directory.path); }, .files => |man| { - const pkg_zig_file = try pkg.value.root_src_directory.join(allocator, &[_][]const u8{ - pkg.value.root_src_path, + const pkg_zig_file = try mod.value.root_src_directory.join(allocator, &[_][]const u8{ + mod.value.root_src_path, }); _ = try man.addFile(pkg_zig_file, null); }, } - // Recurse to handle the package's dependencies - try addPackageTableToCacheHash(hash, arena, pkg.value.table, seen_table, hash_type); + // Recurse to handle the module's dependencies + try addModuleTableToCacheHash(hash, arena, mod.value.deps, seen_table, hash_type); } } @@ -839,7 +839,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { break :blk true; // If we have no zig code to compile, no need for LLVM. - if (options.main_pkg == null) + if (options.main_mod == null) break :blk false; // If LLVM does not support the target, then we can't use it. @@ -869,7 +869,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { // compiler state, the second clause here can be removed so that incremental // cache mode is used for LLVM backend too. We need some fuzz testing before // that can be enabled. - const cache_mode = if ((use_llvm or options.main_pkg == null) and !options.disable_lld_caching) + const cache_mode = if ((use_llvm or options.main_mod == null) and !options.disable_lld_caching) CacheMode.whole else options.cache_mode; @@ -925,7 +925,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { if (use_llvm) { // If stage1 generates an object file, self-hosted linker is not // yet sophisticated enough to handle that. - break :blk options.main_pkg != null; + break :blk options.main_mod != null; } break :blk false; @@ -1210,7 +1210,7 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { if (options.target.os.tag == .wasi) cache.hash.add(wasi_exec_model); // TODO audit this and make sure everything is in it - const module: ?*Module = if (options.main_pkg) |main_pkg| blk: { + const module: ?*Module = if (options.main_mod) |main_mod| blk: { // Options that are specific to zig source files, that cannot be // modified between incremental updates. var hash = cache.hash; @@ -1223,11 +1223,12 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { // do want to namespace different source file names because they are // likely different compilations and therefore this would be likely to // cause cache hits. - hash.addBytes(main_pkg.root_src_path); - hash.addOptionalBytes(main_pkg.root_src_directory.path); + hash.addBytes(main_mod.root_src_path); + hash.addOptionalBytes(main_mod.root.root_dir.path); + hash.addBytes(main_mod.root.sub_path); { - var seen_table = std.AutoHashMap(*Package, void).init(arena); - try addPackageTableToCacheHash(&hash, &arena_allocator, main_pkg.table, &seen_table, .path_bytes); + var seen_table = std.AutoHashMap(*Package.Module, void).init(arena); + try addModuleTableToCacheHash(&hash, &arena_allocator, main_mod.deps, &seen_table, .path_bytes); } }, .whole => { @@ -1283,34 +1284,31 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { .path = try options.local_cache_directory.join(arena, &[_][]const u8{artifact_sub_dir}), }; - const builtin_pkg = try Package.createWithDir( - gpa, - zig_cache_artifact_directory, - null, - "builtin.zig", - ); - errdefer builtin_pkg.destroy(gpa); + const builtin_mod = try Package.Module.create(arena, .{ + .root = .{ .root_dir = zig_cache_artifact_directory }, + .root_src_path = "builtin.zig", + }); - // When you're testing std, the main module is std. In that case, we'll just set the std - // module to the main one, since avoiding the errors caused by duplicating it is more - // effort than it's worth. - const main_pkg_is_std = m: { + // When you're testing std, the main module is std. In that case, + // we'll just set the std module to the main one, since avoiding + // the errors caused by duplicating it is more effort than it's + // worth. + const main_mod_is_std = m: { const std_path = try std.fs.path.resolve(arena, &[_][]const u8{ options.zig_lib_directory.path orelse ".", "std", "std.zig", }); - defer arena.free(std_path); const main_path = try std.fs.path.resolve(arena, &[_][]const u8{ - main_pkg.root_src_directory.path orelse ".", - main_pkg.root_src_path, + main_mod.root.root_dir.path orelse ".", + main_mod.root.sub_path, + main_mod.root_src_path, }); - defer arena.free(main_path); break :m mem.eql(u8, main_path, std_path); }; - const std_pkg = if (main_pkg_is_std) - main_pkg + const std_mod = if (main_mod_is_std) + main_mod else try Package.createWithDir( gpa, @@ -1319,16 +1317,16 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { "std.zig", ); - errdefer if (!main_pkg_is_std) std_pkg.destroy(gpa); + errdefer if (!main_mod_is_std) std_mod.destroy(gpa); - const root_pkg = if (options.is_test) root_pkg: { + const root_mod = if (options.is_test) root_mod: { const test_pkg = if (options.test_runner_path) |test_runner| test_pkg: { const test_dir = std.fs.path.dirname(test_runner); const basename = std.fs.path.basename(test_runner); const pkg = try Package.create(gpa, test_dir, basename); - // copy package table from main_pkg to root_pkg - pkg.table = try main_pkg.table.clone(gpa); + // copy module table from main_mod to root_mod + pkg.deps = try main_mod.deps.clone(gpa); break :test_pkg pkg; } else try Package.createWithDir( gpa, @@ -1338,26 +1336,26 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { ); errdefer test_pkg.destroy(gpa); - break :root_pkg test_pkg; - } else main_pkg; - errdefer if (options.is_test) root_pkg.destroy(gpa); + break :root_mod test_pkg; + } else main_mod; + errdefer if (options.is_test) root_mod.destroy(gpa); - const compiler_rt_pkg = if (include_compiler_rt and options.output_mode == .Obj) compiler_rt_pkg: { - break :compiler_rt_pkg try Package.createWithDir( + const compiler_rt_mod = if (include_compiler_rt and options.output_mode == .Obj) compiler_rt_mod: { + break :compiler_rt_mod try Package.createWithDir( gpa, options.zig_lib_directory, null, "compiler_rt.zig", ); } else null; - errdefer if (compiler_rt_pkg) |p| p.destroy(gpa); + errdefer if (compiler_rt_mod) |p| p.destroy(gpa); - try main_pkg.add(gpa, "builtin", builtin_pkg); - try main_pkg.add(gpa, "root", root_pkg); - try main_pkg.add(gpa, "std", std_pkg); + try main_mod.add(gpa, "builtin", builtin_mod); + try main_mod.add(gpa, "root", root_mod); + try main_mod.add(gpa, "std", std_mod); - if (compiler_rt_pkg) |p| { - try main_pkg.add(gpa, "compiler_rt", p); + if (compiler_rt_mod) |p| { + try main_mod.add(gpa, "compiler_rt", p); } // Pre-open the directory handles for cached ZIR code so that it does not need @@ -1395,8 +1393,8 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { module.* = .{ .gpa = gpa, .comp = comp, - .main_pkg = main_pkg, - .root_pkg = root_pkg, + .main_mod = main_mod, + .root_mod = root_mod, .zig_cache_artifact_directory = zig_cache_artifact_directory, .global_zir_cache = global_zir_cache, .local_zir_cache = local_zir_cache, @@ -2005,8 +2003,8 @@ fn restorePrevZigCacheArtifactDirectory(comp: *Compilation, directory: *Director // This is only for cleanup purposes; Module.deinit calls close // on the handle of zig_cache_artifact_directory. if (comp.bin_file.options.module) |module| { - const builtin_pkg = module.main_pkg.table.get("builtin").?; - module.zig_cache_artifact_directory = builtin_pkg.root_src_directory; + const builtin_mod = module.main_mod.deps.get("builtin").?; + module.zig_cache_artifact_directory = builtin_mod.root_src_directory; } } @@ -2148,8 +2146,8 @@ pub fn update(comp: *Compilation, main_progress_node: *std.Progress.Node) !void // Make sure std.zig is inside the import_table. We unconditionally need // it for start.zig. - const std_pkg = module.main_pkg.table.get("std").?; - _ = try module.importPkg(std_pkg); + const std_mod = module.main_mod.deps.get("std").?; + _ = try module.importPkg(std_mod); // Normally we rely on importing std to in turn import the root source file // in the start code, but when using the stage1 backend that won't happen, @@ -2158,11 +2156,11 @@ pub fn update(comp: *Compilation, main_progress_node: *std.Progress.Node) !void // Likewise, in the case of `zig test`, the test runner is the root source file, // and so there is nothing to import the main file. if (comp.bin_file.options.is_test) { - _ = try module.importPkg(module.main_pkg); + _ = try module.importPkg(module.main_mod); } - if (module.main_pkg.table.get("compiler_rt")) |compiler_rt_pkg| { - _ = try module.importPkg(compiler_rt_pkg); + if (module.main_mod.deps.get("compiler_rt")) |compiler_rt_mod| { + _ = try module.importPkg(compiler_rt_mod); } // Put a work item in for every known source file to detect if @@ -2185,13 +2183,13 @@ pub fn update(comp: *Compilation, main_progress_node: *std.Progress.Node) !void } } - try comp.work_queue.writeItem(.{ .analyze_pkg = std_pkg }); + try comp.work_queue.writeItem(.{ .analyze_mod = std_mod }); if (comp.bin_file.options.is_test) { - try comp.work_queue.writeItem(.{ .analyze_pkg = module.main_pkg }); + try comp.work_queue.writeItem(.{ .analyze_mod = module.main_mod }); } - if (module.main_pkg.table.get("compiler_rt")) |compiler_rt_pkg| { - try comp.work_queue.writeItem(.{ .analyze_pkg = compiler_rt_pkg }); + if (module.main_mod.deps.get("compiler_rt")) |compiler_rt_mod| { + try comp.work_queue.writeItem(.{ .analyze_mod = compiler_rt_mod }); } } @@ -2420,19 +2418,19 @@ fn addNonIncrementalStuffToCacheManifest(comp: *Compilation, man: *Cache.Manifes comptime assert(link_hash_implementation_version == 10); if (comp.bin_file.options.module) |mod| { - const main_zig_file = try mod.main_pkg.root_src_directory.join(arena, &[_][]const u8{ - mod.main_pkg.root_src_path, + const main_zig_file = try mod.main_mod.root_src_directory.join(arena, &[_][]const u8{ + mod.main_mod.root_src_path, }); _ = try man.addFile(main_zig_file, null); { - var seen_table = std.AutoHashMap(*Package, void).init(arena); + var seen_table = std.AutoHashMap(*Package.Module, void).init(arena); // Skip builtin.zig; it is useless as an input, and we don't want to have to // write it before checking for a cache hit. - const builtin_pkg = mod.main_pkg.table.get("builtin").?; - try seen_table.put(builtin_pkg, {}); + const builtin_mod = mod.main_mod.deps.get("builtin").?; + try seen_table.put(builtin_mod, {}); - try addPackageTableToCacheHash(&man.hash, &arena_allocator, mod.main_pkg.table, &seen_table, .{ .files = man }); + try addModuleTableToCacheHash(&man.hash, &arena_allocator, mod.main_mod.deps, &seen_table, .{ .files = man }); } // Synchronize with other matching comments: ZigOnlyHashStuff @@ -3564,8 +3562,8 @@ fn processOneJob(comp: *Compilation, job: Job, prog_node: *std.Progress.Node) !v decl.analysis = .codegen_failure_retryable; }; }, - .analyze_pkg => |pkg| { - const named_frame = tracy.namedFrame("analyze_pkg"); + .analyze_mod => |pkg| { + const named_frame = tracy.namedFrame("analyze_mod"); defer named_frame.end(); const module = comp.bin_file.options.module.?; @@ -6379,11 +6377,11 @@ fn buildOutputFromZig( std.debug.assert(output_mode != .Exe); - var main_pkg: Package = .{ + var main_mod: Package = .{ .root_src_directory = comp.zig_lib_directory, .root_src_path = src_basename, }; - defer main_pkg.deinitTable(comp.gpa); + defer main_mod.deinitTable(comp.gpa); const root_name = src_basename[0 .. src_basename.len - std.fs.path.extension(src_basename).len]; const target = comp.getTarget(); const bin_basename = try std.zig.binNameAlloc(comp.gpa, .{ @@ -6404,7 +6402,7 @@ fn buildOutputFromZig( .cache_mode = .whole, .target = target, .root_name = root_name, - .main_pkg = &main_pkg, + .main_mod = &main_mod, .output_mode = output_mode, .thread_pool = comp.thread_pool, .libc_installation = comp.bin_file.options.libc_installation, @@ -6481,7 +6479,7 @@ pub fn build_crt_file( .cache_mode = .whole, .target = target, .root_name = root_name, - .main_pkg = null, + .main_mod = null, .output_mode = output_mode, .thread_pool = comp.thread_pool, .libc_installation = comp.bin_file.options.libc_installation, diff --git a/src/Manifest.zig b/src/Manifest.zig index ac384b8fcb..4032244d6e 100644 --- a/src/Manifest.zig +++ b/src/Manifest.zig @@ -1,6 +1,10 @@ pub const max_bytes = 10 * 1024 * 1024; pub const basename = "build.zig.zon"; pub const Hash = std.crypto.hash.sha2.Sha256; +pub const Digest = [Hash.digest_length]u8; +pub const multihash_len = 1 + 1 + Hash.digest_length; +pub const multihash_hex_digest_len = 2 * multihash_len; +pub const MultiHashHexDigest = [multihash_hex_digest_len]u8; pub const Dependency = struct { location: union(enum) { @@ -46,7 +50,6 @@ comptime { assert(@intFromEnum(multihash_function) < 127); assert(Hash.digest_length < 127); } -pub const multihash_len = 1 + 1 + Hash.digest_length; name: []const u8, version: std.SemanticVersion, @@ -122,8 +125,8 @@ test hex64 { try std.testing.expectEqualStrings("[00efcdab78563412]", s); } -pub fn hexDigest(digest: [Hash.digest_length]u8) [multihash_len * 2]u8 { - var result: [multihash_len * 2]u8 = undefined; +pub fn hexDigest(digest: Digest) MultiHashHexDigest { + var result: MultiHashHexDigest = undefined; result[0] = hex_charset[@intFromEnum(multihash_function) >> 4]; result[1] = hex_charset[@intFromEnum(multihash_function) & 15]; @@ -339,10 +342,9 @@ const Parse = struct { } } - const hex_multihash_len = 2 * Manifest.multihash_len; - if (h.len != hex_multihash_len) { + if (h.len != multihash_hex_digest_len) { return fail(p, tok, "wrong hash size. expected: {d}, found: {d}", .{ - hex_multihash_len, h.len, + multihash_hex_digest_len, h.len, }); } diff --git a/src/Module.zig b/src/Module.zig index 8c4035cc9a..141ee3c755 100644 --- a/src/Module.zig +++ b/src/Module.zig @@ -55,10 +55,10 @@ comp: *Compilation, /// Where build artifacts and incremental compilation metadata serialization go. zig_cache_artifact_directory: Compilation.Directory, /// Pointer to externally managed resource. -root_pkg: *Package, -/// Normally, `main_pkg` and `root_pkg` are the same. The exception is `zig test`, in which -/// `root_pkg` is the test runner, and `main_pkg` is the user's source file which has the tests. -main_pkg: *Package, +root_mod: *Package.Module, +/// Normally, `main_mod` and `root_mod` are the same. The exception is `zig test`, in which +/// `root_mod` is the test runner, and `main_mod` is the user's source file which has the tests. +main_mod: *Package.Module, sema_prog_node: std.Progress.Node = undefined, /// Used by AstGen worker to load and store ZIR cache. @@ -973,8 +973,8 @@ pub const File = struct { tree: Ast, /// Whether this is populated or not depends on `zir_loaded`. zir: Zir, - /// Package that this file is a part of, managed externally. - pkg: *Package, + /// Module that this file is a part of, managed externally. + mod: *Package.Module, /// Whether this file is a part of multiple packages. This is an error condition which will be reported after AstGen. multi_pkg: bool = false, /// List of references to this file, used for multi-package errors. @@ -1058,14 +1058,9 @@ pub const File = struct { .stat = file.stat, }; - const root_dir_path = file.pkg.root_src_directory.path orelse "."; - log.debug("File.getSource, not cached. pkgdir={s} sub_file_path={s}", .{ - root_dir_path, file.sub_file_path, - }); - // Keep track of inode, file size, mtime, hash so we can detect which files // have been modified when an incremental update is requested. - var f = try file.pkg.root_src_directory.handle.openFile(file.sub_file_path, .{}); + var f = try file.mod.root.openFile(file.sub_file_path, .{}); defer f.close(); const stat = try f.stat(); @@ -1134,14 +1129,12 @@ pub const File = struct { return ip.getOrPutTrailingString(mod.gpa, ip.string_bytes.items.len - start); } - /// Returns the full path to this file relative to its package. pub fn fullPath(file: File, ally: Allocator) ![]u8 { - return file.pkg.root_src_directory.join(ally, &[_][]const u8{file.sub_file_path}); + return file.mod.root.joinString(ally, file.sub_file_path); } - /// Returns the full path to this file relative to its package. pub fn fullPathZ(file: File, ally: Allocator) ![:0]u8 { - return file.pkg.root_src_directory.joinZ(ally, &[_][]const u8{file.sub_file_path}); + return file.mod.root.joinStringZ(ally, file.sub_file_path); } pub fn dumpSrc(file: *File, src: LazySrcLoc) void { @@ -2543,25 +2536,25 @@ pub fn deinit(mod: *Module) void { mod.deletion_set.deinit(gpa); - // The callsite of `Compilation.create` owns the `main_pkg`, however + // The callsite of `Compilation.create` owns the `main_mod`, however // Module owns the builtin and std packages that it adds. - if (mod.main_pkg.table.fetchRemove("builtin")) |kv| { + if (mod.main_mod.table.fetchRemove("builtin")) |kv| { gpa.free(kv.key); kv.value.destroy(gpa); } - if (mod.main_pkg.table.fetchRemove("std")) |kv| { + if (mod.main_mod.table.fetchRemove("std")) |kv| { gpa.free(kv.key); - // It's possible for main_pkg to be std when running 'zig test'! In this case, we must not + // It's possible for main_mod to be std when running 'zig test'! In this case, we must not // destroy it, since it would lead to a double-free. - if (kv.value != mod.main_pkg) { + if (kv.value != mod.main_mod) { kv.value.destroy(gpa); } } - if (mod.main_pkg.table.fetchRemove("root")) |kv| { + if (mod.main_mod.table.fetchRemove("root")) |kv| { gpa.free(kv.key); } - if (mod.root_pkg != mod.main_pkg) { - mod.root_pkg.destroy(gpa); + if (mod.root_mod != mod.main_mod) { + mod.root_mod.destroy(gpa); } mod.compile_log_text.deinit(gpa); @@ -2715,7 +2708,7 @@ pub fn astGenFile(mod: *Module, file: *File) !void { const stat = try source_file.stat(); - const want_local_cache = file.pkg == mod.main_pkg; + const want_local_cache = file.pkg == mod.main_mod; const digest = hash: { var path_hash: Cache.HashHelper = .{}; path_hash.addBytes(build_options.version); @@ -3158,23 +3151,23 @@ pub fn populateBuiltinFile(mod: *Module) !void { comp.mutex.lock(); defer comp.mutex.unlock(); - const builtin_pkg = mod.main_pkg.table.get("builtin").?; - const result = try mod.importPkg(builtin_pkg); + const builtin_mod = mod.main_mod.table.get("builtin").?; + const result = try mod.importPkg(builtin_mod); break :blk .{ .file = result.file, - .pkg = builtin_pkg, + .pkg = builtin_mod, }; }; const file = pkg_and_file.file; - const builtin_pkg = pkg_and_file.pkg; + const builtin_mod = pkg_and_file.pkg; const gpa = mod.gpa; file.source = try comp.generateBuiltinZigSource(gpa); file.source_loaded = true; - if (builtin_pkg.root_src_directory.handle.statFile(builtin_pkg.root_src_path)) |stat| { + if (builtin_mod.root_src_directory.handle.statFile(builtin_mod.root_src_path)) |stat| { if (stat.size != file.source.len) { - const full_path = try builtin_pkg.root_src_directory.join(gpa, &.{ - builtin_pkg.root_src_path, + const full_path = try builtin_mod.root_src_directory.join(gpa, &.{ + builtin_mod.root_src_path, }); defer gpa.free(full_path); @@ -3184,7 +3177,7 @@ pub fn populateBuiltinFile(mod: *Module) !void { .{ full_path, file.source.len, stat.size }, ); - try writeBuiltinFile(file, builtin_pkg); + try writeBuiltinFile(file, builtin_mod); } else { file.stat = .{ .size = stat.size, @@ -3198,7 +3191,7 @@ pub fn populateBuiltinFile(mod: *Module) !void { error.PipeBusy => unreachable, // it's not a pipe error.WouldBlock => unreachable, // not asking for non-blocking I/O - error.FileNotFound => try writeBuiltinFile(file, builtin_pkg), + error.FileNotFound => try writeBuiltinFile(file, builtin_mod), else => |e| return e, } @@ -3212,8 +3205,8 @@ pub fn populateBuiltinFile(mod: *Module) !void { file.status = .success_zir; } -fn writeBuiltinFile(file: *File, builtin_pkg: *Package) !void { - var af = try builtin_pkg.root_src_directory.handle.atomicFile(builtin_pkg.root_src_path, .{}); +fn writeBuiltinFile(file: *File, builtin_mod: *Package.Module) !void { + var af = try builtin_mod.root_src_directory.handle.atomicFile(builtin_mod.root_src_path, .{}); defer af.deinit(); try af.file.writeAll(file.source); try af.finish(); @@ -3748,7 +3741,7 @@ fn semaDecl(mod: *Module, decl_index: Decl.Index) !bool { // TODO: figure out how this works under incremental changes to builtin.zig! const builtin_type_target_index: InternPool.Index = blk: { - const std_mod = mod.main_pkg.table.get("std").?; + const std_mod = mod.main_mod.table.get("std").?; if (decl.getFileScope(mod).pkg != std_mod) break :blk .none; // We're in the std module. const std_file = (try mod.importPkg(std_mod)).file; @@ -4100,13 +4093,13 @@ pub fn importFile( import_string: []const u8, ) !ImportFileResult { if (std.mem.eql(u8, import_string, "std")) { - return mod.importPkg(mod.main_pkg.table.get("std").?); + return mod.importPkg(mod.main_mod.table.get("std").?); } if (std.mem.eql(u8, import_string, "builtin")) { - return mod.importPkg(mod.main_pkg.table.get("builtin").?); + return mod.importPkg(mod.main_mod.table.get("builtin").?); } if (std.mem.eql(u8, import_string, "root")) { - return mod.importPkg(mod.root_pkg); + return mod.importPkg(mod.root_mod); } if (cur_file.pkg.table.get(import_string)) |pkg| { return mod.importPkg(pkg); @@ -4462,14 +4455,14 @@ fn scanDecl(iter: *ScanDeclIter, decl_sub_index: usize, flags: u4) Allocator.Err // test decl with no name. Skip the part where we check against // the test name filter. if (!comp.bin_file.options.is_test) break :blk false; - if (decl_pkg != mod.main_pkg) break :blk false; + if (decl_pkg != mod.main_mod) break :blk false; try mod.test_functions.put(gpa, new_decl_index, {}); break :blk true; }, else => blk: { if (!is_named_test) break :blk false; if (!comp.bin_file.options.is_test) break :blk false; - if (decl_pkg != mod.main_pkg) break :blk false; + if (decl_pkg != mod.main_mod) break :blk false; if (comp.test_filter) |test_filter| { if (mem.indexOf(u8, ip.stringToSlice(decl_name), test_filter) == null) { break :blk false; @@ -5596,8 +5589,8 @@ pub fn populateTestFunctions( ) !void { const gpa = mod.gpa; const ip = &mod.intern_pool; - const builtin_pkg = mod.main_pkg.table.get("builtin").?; - const builtin_file = (mod.importPkg(builtin_pkg) catch unreachable).file; + const builtin_mod = mod.main_mod.table.get("builtin").?; + const builtin_file = (mod.importPkg(builtin_mod) catch unreachable).file; const root_decl = mod.declPtr(builtin_file.root_decl.unwrap().?); const builtin_namespace = mod.namespacePtr(root_decl.src_namespace); const test_functions_str = try ip.getOrPutString(gpa, "test_functions"); diff --git a/src/Package.zig b/src/Package.zig index e5fa24e18d..0249f8dae1 100644 --- a/src/Package.zig +++ b/src/Package.zig @@ -1,251 +1,87 @@ -const Package = @This(); +pub const Module = @import("Package/Module.zig"); +pub const Fetch = @import("Package/Fetch.zig"); +pub const build_zig_basename = "build.zig"; +pub const Manifest = @import("Manifest.zig"); +pub const Path = struct { + root_dir: Cache.Directory, + /// The path, relative to the root dir, that this `Path` represents. + /// Empty string means the root_dir is the path. + sub_path: []const u8 = "", + + pub fn cwd() Path { + return .{ .root_dir = Cache.Directory.cwd() }; + } + + pub fn join(p: Path, allocator: Allocator, sub_path: []const u8) Allocator.Error!Path { + const parts: []const []const u8 = + if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path }; + return .{ + .root_dir = p.root_dir, + .sub_path = try fs.path.join(allocator, parts), + }; + } + + pub fn joinString(p: Path, allocator: Allocator, sub_path: []const u8) Allocator.Error![]u8 { + const parts: []const []const u8 = + if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path }; + return p.root_dir.join(allocator, parts); + } + + pub fn joinStringZ(p: Path, allocator: Allocator, sub_path: []const u8) Allocator.Error![]u8 { + const parts: []const []const u8 = + if (p.sub_path.len == 0) &.{sub_path} else &.{ p.sub_path, sub_path }; + return p.root_dir.joinZ(allocator, parts); + } + + pub fn openFile( + p: Path, + sub_path: []const u8, + flags: fs.File.OpenFlags, + ) fs.File.OpenError!fs.File { + var buf: [fs.MAX_PATH_BYTES]u8 = undefined; + const joined_path = if (p.sub_path.len == 0) sub_path else p: { + break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{ + p.sub_path, sub_path, + }) catch return error.NameTooLong; + }; + return p.root_dir.handle.openFile(joined_path, flags); + } + + pub fn makeOpenPath(p: Path, sub_path: []const u8, opts: fs.OpenDirOptions) !fs.Dir { + var buf: [fs.MAX_PATH_BYTES]u8 = undefined; + const joined_path = if (p.sub_path.len == 0) sub_path else p: { + break :p std.fmt.bufPrint(&buf, "{s}" ++ fs.path.sep_str ++ "{s}", .{ + p.sub_path, sub_path, + }) catch return error.NameTooLong; + }; + return p.root_dir.handle.makeOpenPath(joined_path, opts); + } + + pub fn format( + self: Path, + comptime fmt_string: []const u8, + options: std.fmt.FormatOptions, + writer: anytype, + ) !void { + _ = options; + if (fmt_string.len > 0) + std.fmt.invalidFmtError(fmt_string, self); + if (self.root_dir.path) |p| { + try writer.writeAll(p); + try writer.writeAll(fs.path.sep_str); + } + if (self.sub_path.len > 0) { + try writer.writeAll(self.sub_path); + try writer.writeAll(fs.path.sep_str); + } + } +}; + +const Package = @This(); const builtin = @import("builtin"); const std = @import("std"); const fs = std.fs; -const mem = std.mem; -const Allocator = mem.Allocator; -const ascii = std.ascii; +const Allocator = std.mem.Allocator; const assert = std.debug.assert; -const log = std.log.scoped(.package); -const main = @import("main.zig"); -const ThreadPool = std.Thread.Pool; - -const Compilation = @import("Compilation.zig"); -const Module = @import("Module.zig"); const Cache = std.Build.Cache; -const build_options = @import("build_options"); -const Fetch = @import("Package/Fetch.zig"); - -pub const build_zig_basename = "build.zig"; -pub const Manifest = @import("Manifest.zig"); -pub const Table = std.StringHashMapUnmanaged(*Package); - -root_src_directory: Compilation.Directory, -/// Relative to `root_src_directory`. May contain path separators. -root_src_path: []const u8, -/// The dependency table of this module. Shared dependencies such as 'std', 'builtin', and 'root' -/// are not specified in every dependency table, but instead only in the table of `main_pkg`. -/// `Module.importFile` is responsible for detecting these names and using the correct package. -table: Table = .{}, -/// Whether to free `root_src_directory` on `destroy`. -root_src_directory_owned: bool = false, - -/// Allocate a Package. No references to the slices passed are kept. -pub fn create( - gpa: Allocator, - /// Null indicates the current working directory - root_src_dir_path: ?[]const u8, - /// Relative to root_src_dir_path - root_src_path: []const u8, -) !*Package { - const ptr = try gpa.create(Package); - errdefer gpa.destroy(ptr); - - const owned_dir_path = if (root_src_dir_path) |p| try gpa.dupe(u8, p) else null; - errdefer if (owned_dir_path) |p| gpa.free(p); - - const owned_src_path = try gpa.dupe(u8, root_src_path); - errdefer gpa.free(owned_src_path); - - ptr.* = .{ - .root_src_directory = .{ - .path = owned_dir_path, - .handle = if (owned_dir_path) |p| try fs.cwd().openDir(p, .{}) else fs.cwd(), - }, - .root_src_path = owned_src_path, - .root_src_directory_owned = true, - }; - - return ptr; -} - -pub fn createWithDir( - gpa: Allocator, - directory: Compilation.Directory, - /// Relative to `directory`. If null, means `directory` is the root src dir - /// and is owned externally. - root_src_dir_path: ?[]const u8, - /// Relative to root_src_dir_path - root_src_path: []const u8, -) !*Package { - const ptr = try gpa.create(Package); - errdefer gpa.destroy(ptr); - - const owned_src_path = try gpa.dupe(u8, root_src_path); - errdefer gpa.free(owned_src_path); - - if (root_src_dir_path) |p| { - const owned_dir_path = try directory.join(gpa, &[1][]const u8{p}); - errdefer gpa.free(owned_dir_path); - - ptr.* = .{ - .root_src_directory = .{ - .path = owned_dir_path, - .handle = try directory.handle.openDir(p, .{}), - }, - .root_src_directory_owned = true, - .root_src_path = owned_src_path, - }; - } else { - ptr.* = .{ - .root_src_directory = directory, - .root_src_directory_owned = false, - .root_src_path = owned_src_path, - }; - } - return ptr; -} - -/// Free all memory associated with this package. It does not destroy any packages -/// inside its table; the caller is responsible for calling destroy() on them. -pub fn destroy(pkg: *Package, gpa: Allocator) void { - gpa.free(pkg.root_src_path); - - if (pkg.root_src_directory_owned) { - // If root_src_directory.path is null then the handle is the cwd() - // which shouldn't be closed. - if (pkg.root_src_directory.path) |p| { - gpa.free(p); - pkg.root_src_directory.handle.close(); - } - } - - pkg.deinitTable(gpa); - gpa.destroy(pkg); -} - -/// Only frees memory associated with the table. -pub fn deinitTable(pkg: *Package, gpa: Allocator) void { - pkg.table.deinit(gpa); -} - -pub fn add(pkg: *Package, gpa: Allocator, name: []const u8, package: *Package) !void { - try pkg.table.ensureUnusedCapacity(gpa, 1); - const name_dupe = try gpa.dupe(u8, name); - pkg.table.putAssumeCapacityNoClobber(name_dupe, package); -} - -/// Compute a readable name for the package. The returned name should be freed from gpa. This -/// function is very slow, as it traverses the whole package hierarchy to find a path to this -/// package. It should only be used for error output. -pub fn getName(target: *const Package, gpa: Allocator, mod: Module) ![]const u8 { - // we'll do a breadth-first search from the root module to try and find a short name for this - // module, using a DoublyLinkedList of module/parent pairs. note that the "parent" there is - // just the first-found shortest path - a module may be children of arbitrarily many other - // modules. This path may vary between executions due to hashmap iteration order, but that - // doesn't matter too much. - var node_arena = std.heap.ArenaAllocator.init(gpa); - defer node_arena.deinit(); - const Parented = struct { - parent: ?*const @This(), - mod: *const Package, - }; - const Queue = std.DoublyLinkedList(Parented); - var to_check: Queue = .{}; - - { - const new = try node_arena.allocator().create(Queue.Node); - new.* = .{ .data = .{ .parent = null, .mod = mod.root_pkg } }; - to_check.prepend(new); - } - - if (mod.main_pkg != mod.root_pkg) { - const new = try node_arena.allocator().create(Queue.Node); - // TODO: once #12201 is resolved, we may want a way of indicating a different name for this - new.* = .{ .data = .{ .parent = null, .mod = mod.main_pkg } }; - to_check.prepend(new); - } - - // set of modules we've already checked to prevent loops - var checked = std.AutoHashMap(*const Package, void).init(gpa); - defer checked.deinit(); - - const linked = while (to_check.pop()) |node| { - const check = &node.data; - - if (checked.contains(check.mod)) continue; - try checked.put(check.mod, {}); - - if (check.mod == target) break check; - - var it = check.mod.table.iterator(); - while (it.next()) |kv| { - var new = try node_arena.allocator().create(Queue.Node); - new.* = .{ .data = .{ - .parent = check, - .mod = kv.value_ptr.*, - } }; - to_check.prepend(new); - } - } else { - // this can happen for e.g. @cImport packages - return gpa.dupe(u8, ""); - }; - - // we found a path to the module! unfortunately, we can only traverse *up* it, so we have to put - // all the names into a buffer so we can then print them in order. - var names = std.ArrayList([]const u8).init(gpa); - defer names.deinit(); - - var cur: *const Parented = linked; - while (cur.parent) |parent| : (cur = parent) { - // find cur's name in parent - var it = parent.mod.table.iterator(); - const name = while (it.next()) |kv| { - if (kv.value_ptr.* == cur.mod) { - break kv.key_ptr.*; - } - } else unreachable; - try names.append(name); - } - - // finally, print the names into a buffer! - var buf = std.ArrayList(u8).init(gpa); - defer buf.deinit(); - try buf.writer().writeAll("root"); - var i: usize = names.items.len; - while (i > 0) { - i -= 1; - try buf.writer().print(".{s}", .{names.items[i]}); - } - - return buf.toOwnedSlice(); -} - -pub fn createFilePkg( - gpa: Allocator, - cache_directory: Compilation.Directory, - basename: []const u8, - contents: []const u8, -) !*Package { - const rand_int = std.crypto.random.int(u64); - const tmp_dir_sub_path = "tmp" ++ fs.path.sep_str ++ Manifest.hex64(rand_int); - { - var tmp_dir = try cache_directory.handle.makeOpenPath(tmp_dir_sub_path, .{}); - defer tmp_dir.close(); - try tmp_dir.writeFile(basename, contents); - } - - var hh: Cache.HashHelper = .{}; - hh.addBytes(build_options.version); - hh.addBytes(contents); - const hex_digest = hh.final(); - - const o_dir_sub_path = "o" ++ fs.path.sep_str ++ hex_digest; - try Fetch.renameTmpIntoCache(cache_directory.handle, tmp_dir_sub_path, o_dir_sub_path); - - return createWithDir(gpa, cache_directory, o_dir_sub_path, basename); -} - -const hex_multihash_len = 2 * Manifest.multihash_len; -const MultiHashHexDigest = [hex_multihash_len]u8; - -const DependencyModule = union(enum) { - zig_pkg: *Package, - non_zig_pkg: *Package, -}; -/// This is to avoid creating multiple modules for the same build.zig file. -/// If the value is `null`, the package is a known dependency, but has not yet -/// been fetched. -pub const AllModules = std.AutoHashMapUnmanaged(MultiHashHexDigest, ?DependencyModule); diff --git a/src/Package/Fetch.zig b/src/Package/Fetch.zig index b3b4667e40..55de79f039 100644 --- a/src/Package/Fetch.zig +++ b/src/Package/Fetch.zig @@ -27,59 +27,84 @@ //! All of this must be done with only referring to the state inside this struct //! because this work will be done in a dedicated thread. -/// Try to avoid this as much as possible since arena will have less contention. -gpa: Allocator, arena: std.heap.ArenaAllocator, location: Location, location_tok: std.zig.Ast.TokenIndex, hash_tok: std.zig.Ast.TokenIndex, -global_cache: Cache.Directory, -parent_package_root: Path, +parent_package_root: Package.Path, parent_manifest_ast: ?*const std.zig.Ast, prog_node: *std.Progress.Node, -http_client: *std.http.Client, -thread_pool: *ThreadPool, job_queue: *JobQueue, -wait_group: *WaitGroup, +/// If true, don't add an error for a missing hash. This flag is not passed +/// down to recursive dependencies. It's intended to be used only be the CLI. +omit_missing_hash_error: bool, // Above this are fields provided as inputs to `run`. // Below this are fields populated by `run`. /// This will either be relative to `global_cache`, or to the build root of /// the root package. -package_root: Path, +package_root: Package.Path, error_bundle: std.zig.ErrorBundle.Wip, manifest: ?Manifest, -manifest_ast: ?*std.zig.Ast, -actual_hash: Digest, +manifest_ast: std.zig.Ast, +actual_hash: Manifest.Digest, /// Fetch logic notices whether a package has a build.zig file and sets this flag. has_build_zig: bool, /// Indicates whether the task aborted due to an out-of-memory condition. oom_flag: bool, +/// Contains shared state among all `Fetch` tasks. pub const JobQueue = struct { mutex: std.Thread.Mutex = .{}, -}; + /// Protected by `mutex`. + table: Table = .{}, + /// `table` may be missing some tasks such as ones that failed, so this + /// field contains references to all of them. + /// Protected by `mutex`. + all_fetches: std.ArrayListUnmanaged(*Fetch) = .{}, -pub const Digest = [Manifest.Hash.digest_length]u8; -pub const MultiHashHexDigest = [hex_multihash_len]u8; + http_client: *std.http.Client, + thread_pool: *ThreadPool, + wait_group: WaitGroup = .{}, + global_cache: Cache.Directory, + recursive: bool, + work_around_btrfs_bug: bool, -pub const Path = struct { - root_dir: Cache.Directory, - /// The path, relative to the root dir, that this `Path` represents. - /// Empty string means the root_dir is the path. - sub_path: []const u8 = "", + pub const Table = std.AutoHashMapUnmanaged(Manifest.MultiHashHexDigest, *Fetch); + + pub fn deinit(jq: *JobQueue) void { + if (jq.all_fetches.items.len == 0) return; + const gpa = jq.all_fetches.items[0].arena.child_allocator; + jq.table.deinit(gpa); + // These must be deinitialized in reverse order because subsequent + // `Fetch` instances are allocated in prior ones' arenas. + // Sorry, I know it's a bit weird, but it slightly simplifies the + // critical section. + while (jq.all_fetches.popOrNull()) |f| f.deinit(); + jq.all_fetches.deinit(gpa); + jq.* = undefined; + } }; pub const Location = union(enum) { remote: Remote, + /// A directory found inside the parent package. relative_path: []const u8, + /// Recursive Fetch tasks will never use this Location, but it may be + /// passed in by the CLI. Indicates the file contents here should be copied + /// into the global package cache. It may be a file relative to the cwd or + /// absolute, in which case it should be treated exactly like a `file://` + /// URL, or a directory, in which case it should be treated as an + /// already-unpacked directory (but still needs to be copied into the + /// global package cache and have inclusion rules applied). + path_or_url: []const u8, pub const Remote = struct { url: []const u8, /// If this is null it means the user omitted the hash field from a dependency. /// It will be an error but the logic should still fetch and print the discovered hash. - hash: ?[hex_multihash_len]u8, + hash: ?Manifest.MultiHashHexDigest, }; }; @@ -92,7 +117,11 @@ pub const RunError = error{ pub fn run(f: *Fetch) RunError!void { const eb = &f.error_bundle; - const arena = f.arena_allocator.allocator(); + const arena = f.arena.allocator(); + const gpa = f.arena.child_allocator; + const cache_root = f.job_queue.global_cache; + + try eb.init(gpa); // Check the global zig package cache to see if the hash already exists. If // so, load, parse, and validate the build.zig.zon file therein, and skip @@ -111,43 +140,66 @@ pub fn run(f: *Fetch) RunError!void { ); f.package_root = try f.parent_package_root.join(arena, sub_path); try loadManifest(f, f.package_root); + if (!f.job_queue.recursive) return; // Package hashes are used as unique identifiers for packages, so // we still need one for relative paths. - const hash = h: { + const digest = h: { var hasher = Manifest.Hash.init(.{}); // This hash is a tuple of: // * whether it relative to the global cache directory or to the root package // * the relative file path from there to the build root of the package - hasher.update(if (f.package_root.root_dir.handle == f.global_cache.handle) + hasher.update(if (f.package_root.root_dir.eql(cache_root)) &package_hash_prefix_cached else &package_hash_prefix_project); hasher.update(f.package_root.sub_path); break :h hasher.finalResult(); }; - return queueJobsForDeps(f, hash); + return queueJobsForDeps(f, Manifest.hexDigest(digest)); }, .remote => |remote| remote, + .path_or_url => |path_or_url| { + if (fs.cwd().openIterableDir(path_or_url, .{})) |dir| { + var resource: Resource = .{ .dir = dir }; + return runResource(f, path_or_url, &resource, null); + } else |dir_err| { + const file_err = if (dir_err == error.NotDir) e: { + if (fs.cwd().openFile(path_or_url, .{})) |file| { + var resource: Resource = .{ .file = file }; + return runResource(f, path_or_url, &resource, null); + } else |err| break :e err; + } else dir_err; + + const uri = std.Uri.parse(path_or_url) catch |uri_err| { + return f.fail(0, try eb.printString( + "'{s}' could not be recognized as a file path ({s}) or an URL ({s})", + .{ path_or_url, @errorName(file_err), @errorName(uri_err) }, + )); + }; + var resource = try f.initResource(uri); + return runResource(f, uri.path, &resource, null); + } + }, }; + const s = fs.path.sep_str; if (remote.hash) |expected_hash| { const pkg_sub_path = "p" ++ s ++ expected_hash; - if (f.global_cache.handle.access(pkg_sub_path, .{})) |_| { + if (cache_root.handle.access(pkg_sub_path, .{})) |_| { f.package_root = .{ - .root_dir = f.global_cache, + .root_dir = cache_root, .sub_path = pkg_sub_path, }; try loadManifest(f, f.package_root); + if (!f.job_queue.recursive) return; return queueJobsForDeps(f, expected_hash); } else |err| switch (err) { error.FileNotFound => {}, else => |e| { try eb.addRootErrorMessage(.{ .msg = try eb.printString("unable to open global package cache directory '{s}': {s}", .{ - try f.global_cache.join(arena, .{pkg_sub_path}), @errorName(e), + try cache_root.join(arena, &.{pkg_sub_path}), @errorName(e), }), - .src_loc = .none, - .notes_len = 0, }); return error.FetchFailed; }, @@ -158,22 +210,50 @@ pub fn run(f: *Fetch) RunError!void { const uri = std.Uri.parse(remote.url) catch |err| return f.fail( f.location_tok, - "invalid URI: {s}", - .{@errorName(err)}, + try eb.printString("invalid URI: {s}", .{@errorName(err)}), ); + var resource = try f.initResource(uri); + return runResource(f, uri.path, &resource, remote.hash); +} + +pub fn deinit(f: *Fetch) void { + f.error_bundle.deinit(); + f.arena.deinit(); +} + +/// Consumes `resource`, even if an error is returned. +fn runResource( + f: *Fetch, + uri_path: []const u8, + resource: *Resource, + remote_hash: ?Manifest.MultiHashHexDigest, +) RunError!void { + defer resource.deinit(); + const arena = f.arena.allocator(); + const eb = &f.error_bundle; + const s = fs.path.sep_str; + const cache_root = f.job_queue.global_cache; const rand_int = std.crypto.random.int(u64); const tmp_dir_sub_path = "tmp" ++ s ++ Manifest.hex64(rand_int); + const tmp_directory_path = try cache_root.join(arena, &.{tmp_dir_sub_path}); var tmp_directory: Cache.Directory = .{ - .path = try f.global_cache.join(arena, &.{tmp_dir_sub_path}), - .handle = (try f.global_cache.handle.makeOpenPathIterable(tmp_dir_sub_path, .{})).dir, + .path = tmp_directory_path, + .handle = handle: { + const dir = cache_root.handle.makeOpenPathIterable(tmp_dir_sub_path, .{}) catch |err| { + try eb.addRootErrorMessage(.{ + .msg = try eb.printString("unable to create temporary directory '{s}': {s}", .{ + tmp_directory_path, @errorName(err), + }), + }); + return error.FetchFailed; + }; + break :handle dir.dir; + }, }; defer tmp_directory.handle.close(); - var resource = try f.initResource(uri); - defer resource.deinit(); // releases more than memory - - try f.unpackResource(&resource, uri.path, tmp_directory); + try unpackResource(f, resource, uri_path, tmp_directory); // Load, parse, and validate the unpacked build.zig.zon file. It is allowed // for the file to be missing, in which case this fetched package is @@ -194,15 +274,15 @@ pub fn run(f: *Fetch) RunError!void { // Compute the package hash based on the remaining files in the temporary // directory. - if (builtin.os.tag == .linux and f.work_around_btrfs_bug) { + if (builtin.os.tag == .linux and f.job_queue.work_around_btrfs_bug) { // https://github.com/ziglang/zig/issues/17095 tmp_directory.handle.close(); - const iterable_dir = f.global_cache.handle.makeOpenPathIterable(tmp_dir_sub_path, .{}) catch + const iterable_dir = cache_root.handle.makeOpenPathIterable(tmp_dir_sub_path, .{}) catch @panic("btrfs workaround failed"); tmp_directory.handle = iterable_dir.dir; } - f.actual_hash = try computeHash(f, .{ .dir = tmp_directory.handle }, filter); + f.actual_hash = try computeHash(f, tmp_directory, filter); // Rename the temporary directory into the global zig package cache // directory. If the hash already exists, delete the temporary directory @@ -211,40 +291,54 @@ pub fn run(f: *Fetch) RunError!void { // package with the different hash is used in the future. const dest_pkg_sub_path = "p" ++ s ++ Manifest.hexDigest(f.actual_hash); - try renameTmpIntoCache(f.global_cache.handle, tmp_dir_sub_path, dest_pkg_sub_path); + renameTmpIntoCache(cache_root.handle, tmp_dir_sub_path, dest_pkg_sub_path) catch |err| { + const src = try cache_root.join(arena, &.{tmp_dir_sub_path}); + const dest = try cache_root.join(arena, &.{dest_pkg_sub_path}); + try eb.addRootErrorMessage(.{ .msg = try eb.printString( + "unable to rename temporary directory '{s}' into package cache directory '{s}': {s}", + .{ src, dest, @errorName(err) }, + ) }); + return error.FetchFailed; + }; // Validate the computed hash against the expected hash. If invalid, this // job is done. const actual_hex = Manifest.hexDigest(f.actual_hash); - if (remote.hash) |declared_hash| { - if (!std.mem.eql(u8, declared_hash, &actual_hex)) { - return f.fail(f.hash_tok, "hash mismatch: manifest declares {s} but the fetched package has {s}", .{ - declared_hash, actual_hex, - }); + if (remote_hash) |declared_hash| { + if (!std.mem.eql(u8, &declared_hash, &actual_hex)) { + return f.fail(f.hash_tok, try eb.printString( + "hash mismatch: manifest declares {s} but the fetched package has {s}", + .{ declared_hash, actual_hex }, + )); } - } else { + } else if (!f.omit_missing_hash_error) { const notes_len = 1; - try f.addErrorWithNotes(notes_len, f.location_tok, "dependency is missing hash field"); + try eb.addRootErrorMessage(.{ + .msg = try eb.addString("dependency is missing hash field"), + .src_loc = try f.srcLoc(f.location_tok), + .notes_len = notes_len, + }); const notes_start = try eb.reserveNotes(notes_len); eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ .msg = try eb.printString("expected .hash = \"{s}\",", .{&actual_hex}), })); - return error.PackageFetchFailed; + return error.FetchFailed; } // Spawn a new fetch job for each dependency in the manifest file. Use // a mutex and a hash map so that redundant jobs do not get queued up. - return queueJobsForDeps(f, .{ .hash = f.actual_hash }); + if (!f.job_queue.recursive) return; + return queueJobsForDeps(f, actual_hex); } /// This function populates `f.manifest` or leaves it `null`. -fn loadManifest(f: *Fetch, pkg_root: Path) RunError!void { +fn loadManifest(f: *Fetch, pkg_root: Package.Path) RunError!void { const eb = &f.error_bundle; - const arena = f.arena_allocator.allocator(); - const manifest_bytes = pkg_root.readFileAllocOptions( + const arena = f.arena.allocator(); + const manifest_bytes = pkg_root.root_dir.handle.readFileAllocOptions( arena, - Manifest.basename, + try fs.path.join(arena, &.{ pkg_root.sub_path, Manifest.basename }), Manifest.max_bytes, null, 1, @@ -252,39 +346,39 @@ fn loadManifest(f: *Fetch, pkg_root: Path) RunError!void { ) catch |err| switch (err) { error.FileNotFound => return, else => |e| { - const file_path = try pkg_root.join(arena, .{Manifest.basename}); + const file_path = try pkg_root.join(arena, Manifest.basename); try eb.addRootErrorMessage(.{ - .msg = try eb.printString("unable to load package manifest '{s}': {s}", .{ + .msg = try eb.printString("unable to load package manifest '{}': {s}", .{ file_path, @errorName(e), }), - .src_loc = .none, - .notes_len = 0, }); + return error.FetchFailed; }, }; - var ast = try std.zig.Ast.parse(arena, manifest_bytes, .zon); - f.manifest_ast = ast; + const ast = &f.manifest_ast; + ast.* = try std.zig.Ast.parse(arena, manifest_bytes, .zon); if (ast.errors.len > 0) { - const file_path = try pkg_root.join(arena, .{Manifest.basename}); - try main.putAstErrorsIntoBundle(arena, ast, file_path, eb); - return error.PackageFetchFailed; + const file_path = try std.fmt.allocPrint(arena, "{}" ++ Manifest.basename, .{pkg_root}); + try main.putAstErrorsIntoBundle(arena, ast.*, file_path, eb); + return error.FetchFailed; } - f.manifest = try Manifest.parse(arena, ast); + f.manifest = try Manifest.parse(arena, ast.*); + const manifest = &f.manifest.?; - if (f.manifest.errors.len > 0) { - const file_path = try pkg_root.join(arena, .{Manifest.basename}); + if (manifest.errors.len > 0) { + const src_path = try eb.printString("{}{s}", .{ pkg_root, Manifest.basename }); const token_starts = ast.tokens.items(.start); - for (f.manifest.errors) |msg| { + for (manifest.errors) |msg| { const start_loc = ast.tokenLocation(0, msg.tok); try eb.addRootErrorMessage(.{ .msg = try eb.addString(msg.msg), .src_loc = try eb.addSourceLocation(.{ - .src_path = try eb.addString(file_path), + .src_path = src_path, .span_start = token_starts[msg.tok], .span_end = @intCast(token_starts[msg.tok] + ast.tokenSlice(msg.tok).len), .span_main = token_starts[msg.tok] + msg.off, @@ -292,71 +386,80 @@ fn loadManifest(f: *Fetch, pkg_root: Path) RunError!void { .column = @intCast(start_loc.column), .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), }), - .notes_len = 0, }); } - return error.PackageFetchFailed; + return error.FetchFailed; } } -fn queueJobsForDeps(f: *Fetch, hash: Digest) RunError!void { +fn queueJobsForDeps(f: *Fetch, hash: Manifest.MultiHashHexDigest) RunError!void { + assert(f.job_queue.recursive); + // If the package does not have a build.zig.zon file then there are no dependencies. const manifest = f.manifest orelse return; const new_fetches = nf: { + const deps = manifest.dependencies.values(); + const gpa = f.arena.child_allocator; // Grab the new tasks into a temporary buffer so we can unlock that mutex // as fast as possible. // This overallocates any fetches that get skipped by the `continue` in the // loop below. - const new_fetches = try f.arena.alloc(Fetch, manifest.dependencies.count()); + const new_fetches = try f.arena.allocator().alloc(Fetch, deps.len); var new_fetch_index: usize = 0; - f.job_queue.lock(); - defer f.job_queue.unlock(); + f.job_queue.mutex.lock(); + defer f.job_queue.mutex.unlock(); + + try f.job_queue.all_fetches.ensureUnusedCapacity(gpa, new_fetches.len); + try f.job_queue.table.ensureUnusedCapacity(gpa, @intCast(new_fetches.len + 1)); // It is impossible for there to be a collision here. Consider all three cases: // * Correct hash is provided by manifest. // - Redundant jobs are skipped in the loop below. - // * Incorrect has is provided by manifest. + // * Incorrect hash is provided by manifest. // - Hash mismatch error emitted; `queueJobsForDeps` is not called. // * Hash is not provided by manifest. // - Hash missing error emitted; `queueJobsForDeps` is not called. - try f.job_queue.finish(hash, f, new_fetches.len); + f.job_queue.table.putAssumeCapacityNoClobber(hash, f); - for (manifest.dependencies.values()) |dep| { + for (deps) |dep| { + const new_fetch = &new_fetches[new_fetch_index]; const location: Location = switch (dep.location) { .url => |url| .{ .remote = .{ .url = url, - .hash = if (dep.hash) |h| h[0..hex_multihash_len].* else null, + .hash = h: { + const h = dep.hash orelse break :h null; + const digest_len = @typeInfo(Manifest.MultiHashHexDigest).Array.len; + const multihash_digest = h[0..digest_len].*; + const gop = f.job_queue.table.getOrPutAssumeCapacity(multihash_digest); + if (gop.found_existing) continue; + gop.value_ptr.* = new_fetch; + break :h multihash_digest; + }, } }, .path => |path| .{ .relative_path = path }, }; - const new_fetch = &new_fetches[new_fetch_index]; - const already_done = f.job_queue.add(location, new_fetch); - if (already_done) continue; new_fetch_index += 1; - + f.job_queue.all_fetches.appendAssumeCapacity(new_fetch); new_fetch.* = .{ - .gpa = f.gpa, - .arena = std.heap.ArenaAllocator.init(f.gpa), + .arena = std.heap.ArenaAllocator.init(gpa), .location = location, .location_tok = dep.location_tok, .hash_tok = dep.hash_tok, - .global_cache = f.global_cache, .parent_package_root = f.package_root, - .parent_manifest_ast = f.manifest_ast.?, + .parent_manifest_ast = &f.manifest_ast, .prog_node = f.prog_node, - .http_client = f.http_client, - .thread_pool = f.thread_pool, .job_queue = f.job_queue, - .wait_group = f.wait_group, + .omit_missing_hash_error = false, .package_root = undefined, - .error_bundle = .{}, + .error_bundle = undefined, .manifest = null, - .manifest_ast = null, + .manifest_ast = undefined, .actual_hash = undefined, .has_build_zig = false, + .oom_flag = false, }; } @@ -364,12 +467,14 @@ fn queueJobsForDeps(f: *Fetch, hash: Digest) RunError!void { }; // Now it's time to give tasks to the thread pool. - for (new_fetches) |new_fetch| { - f.wait_group.start(); - f.thread_pool.spawn(workerRun, .{f}) catch |err| switch (err) { + const thread_pool = f.job_queue.thread_pool; + + for (new_fetches) |*new_fetch| { + f.job_queue.wait_group.start(); + thread_pool.spawn(workerRun, .{new_fetch}) catch |err| switch (err) { error.OutOfMemory => { new_fetch.oom_flag = true; - f.wait_group.finish(); + f.job_queue.wait_group.finish(); continue; }, }; @@ -377,43 +482,83 @@ fn queueJobsForDeps(f: *Fetch, hash: Digest) RunError!void { } fn workerRun(f: *Fetch) void { - defer f.wait_group.finish(); + defer f.job_queue.wait_group.finish(); run(f) catch |err| switch (err) { error.OutOfMemory => f.oom_flag = true, - error.FetchFailed => {}, // See `error_bundle`. + error.FetchFailed => { + // Nothing to do because the errors are already reported in `error_bundle`, + // and a reference is kept to the `Fetch` task inside `all_fetches`. + }, }; } -fn fail(f: *Fetch, msg_tok: std.zig.Ast.TokenIndex, msg_str: u32) RunError!void { - const ast = f.parent_manifest_ast; - const token_starts = ast.tokens.items(.start); - const start_loc = ast.tokenLocation(0, msg_tok); +fn srcLoc( + f: *Fetch, + tok: std.zig.Ast.TokenIndex, +) Allocator.Error!std.zig.ErrorBundle.SourceLocationIndex { + const ast = f.parent_manifest_ast orelse return .none; const eb = &f.error_bundle; - const file_path = try f.parent_package_root.join(f.arena, Manifest.basename); + const token_starts = ast.tokens.items(.start); + const start_loc = ast.tokenLocation(0, tok); + const src_path = try eb.printString("{}" ++ Manifest.basename, .{f.parent_package_root}); const msg_off = 0; + return eb.addSourceLocation(.{ + .src_path = src_path, + .span_start = token_starts[tok], + .span_end = @intCast(token_starts[tok] + ast.tokenSlice(tok).len), + .span_main = token_starts[tok] + msg_off, + .line = @intCast(start_loc.line), + .column = @intCast(start_loc.column), + .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), + }); +} +fn fail(f: *Fetch, msg_tok: std.zig.Ast.TokenIndex, msg_str: u32) RunError { + const eb = &f.error_bundle; try eb.addRootErrorMessage(.{ .msg = msg_str, - .src_loc = try eb.addSourceLocation(.{ - .src_path = try eb.addString(file_path), - .span_start = token_starts[msg_tok], - .span_end = @intCast(token_starts[msg_tok] + ast.tokenSlice(msg_tok).len), - .span_main = token_starts[msg_tok] + msg_off, - .line = @intCast(start_loc.line), - .column = @intCast(start_loc.column), - .source_line = try eb.addString(ast.source[start_loc.line_start..start_loc.line_end]), - }), - .notes_len = 0, + .src_loc = try f.srcLoc(msg_tok), }); - return error.FetchFailed; } const Resource = union(enum) { file: fs.File, http_request: std.http.Client.Request, - git_fetch_stream: git.Session.FetchStream, + git: Git, dir: fs.IterableDir, + + const Git = struct { + fetch_stream: git.Session.FetchStream, + want_oid: [git.oid_length]u8, + }; + + fn deinit(resource: *Resource) void { + switch (resource.*) { + .file => |*file| file.close(), + .http_request => |*req| req.deinit(), + .git => |*git_resource| git_resource.fetch_stream.deinit(), + .dir => |*dir| dir.close(), + } + resource.* = undefined; + } + + fn reader(resource: *Resource) std.io.AnyReader { + return .{ + .context = resource, + .readFn = read, + }; + } + + fn read(context: *const anyopaque, buffer: []u8) anyerror!usize { + const resource: *Resource = @constCast(@ptrCast(@alignCast(context))); + switch (resource.*) { + .file => |*f| return f.read(buffer), + .http_request => |*r| return r.read(buffer), + .git => |*g| return g.fetch_stream.read(buffer), + .dir => unreachable, + } + } }; const FileType = enum { @@ -468,30 +613,52 @@ const FileType = enum { }; fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource { - const gpa = f.gpa; - const arena = f.arena_allocator.allocator(); + const gpa = f.arena.child_allocator; + const arena = f.arena.allocator(); const eb = &f.error_bundle; if (ascii.eqlIgnoreCase(uri.scheme, "file")) return .{ - .file = try f.parent_package_root.openFile(uri.path, .{}), + .file = f.parent_package_root.openFile(uri.path, .{}) catch |err| { + return f.fail(f.location_tok, try eb.printString("unable to open '{}{s}': {s}", .{ + f.parent_package_root, uri.path, @errorName(err), + })); + }, }; + const http_client = f.job_queue.http_client; + if (ascii.eqlIgnoreCase(uri.scheme, "http") or ascii.eqlIgnoreCase(uri.scheme, "https")) { var h = std.http.Headers{ .allocator = gpa }; defer h.deinit(); - var req = try f.http_client.request(.GET, uri, h, .{}); + var req = http_client.request(.GET, uri, h, .{}) catch |err| { + return f.fail(f.location_tok, try eb.printString( + "unable to connect to server: {s}", + .{@errorName(err)}, + )); + }; errdefer req.deinit(); // releases more than memory - try req.start(.{}); - try req.wait(); + req.start(.{}) catch |err| { + return f.fail(f.location_tok, try eb.printString( + "HTTP request failed: {s}", + .{@errorName(err)}, + )); + }; + req.wait() catch |err| { + return f.fail(f.location_tok, try eb.printString( + "invalid HTTP response: {s}", + .{@errorName(err)}, + )); + }; if (req.response.status != .ok) { - return f.fail(f.location_tok, "expected response status '200 OK' got '{s} {s}'", .{ - @intFromEnum(req.response.status), req.response.status.phrase() orelse "", - }); + return f.fail(f.location_tok, try eb.printString( + "bad HTTP response code: '{d} {s}'", + .{ @intFromEnum(req.response.status), req.response.status.phrase() orelse "" }, + )); } return .{ .http_request = req }; @@ -503,13 +670,21 @@ fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource { var transport_uri = uri; transport_uri.scheme = uri.scheme["git+".len..]; var redirect_uri: []u8 = undefined; - var session: git.Session = .{ .transport = f.http_client, .uri = transport_uri }; - session.discoverCapabilities(gpa, &redirect_uri) catch |e| switch (e) { + var session: git.Session = .{ .transport = http_client, .uri = transport_uri }; + session.discoverCapabilities(gpa, &redirect_uri) catch |err| switch (err) { error.Redirected => { defer gpa.free(redirect_uri); - return f.fail(f.location_tok, "repository moved to {s}", .{redirect_uri}); + return f.fail(f.location_tok, try eb.printString( + "repository moved to {s}", + .{redirect_uri}, + )); + }, + else => |e| { + return f.fail(f.location_tok, try eb.printString( + "unable to discover remote git server capabilities: {s}", + .{@errorName(e)}, + )); }, - else => |other| return other, }; const want_oid = want_oid: { @@ -519,12 +694,22 @@ fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource { const want_ref_head = try std.fmt.allocPrint(arena, "refs/heads/{s}", .{want_ref}); const want_ref_tag = try std.fmt.allocPrint(arena, "refs/tags/{s}", .{want_ref}); - var ref_iterator = try session.listRefs(gpa, .{ + var ref_iterator = session.listRefs(gpa, .{ .ref_prefixes = &.{ want_ref, want_ref_head, want_ref_tag }, .include_peeled = true, - }); + }) catch |err| { + return f.fail(f.location_tok, try eb.printString( + "unable to list refs: {s}", + .{@errorName(err)}, + )); + }; defer ref_iterator.deinit(); - while (try ref_iterator.next()) |ref| { + while (ref_iterator.next() catch |err| { + return f.fail(f.location_tok, try eb.printString( + "unable to iterate refs: {s}", + .{@errorName(err)}, + )); + }) |ref| { if (std.mem.eql(u8, ref.name, want_ref) or std.mem.eql(u8, ref.name, want_ref_head) or std.mem.eql(u8, ref.name, want_ref_tag)) @@ -532,31 +717,46 @@ fn initResource(f: *Fetch, uri: std.Uri) RunError!Resource { break :want_oid ref.peeled orelse ref.oid; } } - return f.fail(f.location_tok, "ref not found: {s}", .{want_ref}); + return f.fail(f.location_tok, try eb.printString("ref not found: {s}", .{want_ref})); }; if (uri.fragment == null) { const notes_len = 1; - try f.addErrorWithNotes(notes_len, f.location_tok, "url field is missing an explicit ref"); + try eb.addRootErrorMessage(.{ + .msg = try eb.addString("url field is missing an explicit ref"), + .src_loc = try f.srcLoc(f.location_tok), + .notes_len = notes_len, + }); const notes_start = try eb.reserveNotes(notes_len); eb.extra.items[notes_start] = @intFromEnum(try eb.addErrorMessage(.{ .msg = try eb.printString("try .url = \"{+/}#{}\",", .{ uri, std.fmt.fmtSliceHexLower(&want_oid), }), })); - return error.PackageFetchFailed; + return error.FetchFailed; } var want_oid_buf: [git.fmt_oid_length]u8 = undefined; _ = std.fmt.bufPrint(&want_oid_buf, "{}", .{ std.fmt.fmtSliceHexLower(&want_oid), }) catch unreachable; - var fetch_stream = try session.fetch(gpa, &.{&want_oid_buf}); + var fetch_stream = session.fetch(gpa, &.{&want_oid_buf}) catch |err| { + return f.fail(f.location_tok, try eb.printString( + "unable to create fetch stream: {s}", + .{@errorName(err)}, + )); + }; errdefer fetch_stream.deinit(); - return .{ .git_fetch_stream = fetch_stream }; + return .{ .git = .{ + .fetch_stream = fetch_stream, + .want_oid = want_oid, + } }; } - return f.fail(f.location_tok, "unsupported URL scheme: {s}", .{uri.scheme}); + return f.fail(f.location_tok, try eb.printString( + "unsupported URL scheme: {s}", + .{uri.scheme}, + )); } fn unpackResource( @@ -565,52 +765,62 @@ fn unpackResource( uri_path: []const u8, tmp_directory: Cache.Directory, ) RunError!void { + const eb = &f.error_bundle; const file_type = switch (resource.*) { .file => FileType.fromPath(uri_path) orelse - return f.fail(f.location_tok, "unknown file type: '{s}'", .{uri_path}), + return f.fail(f.location_tok, try eb.printString("unknown file type: '{s}'", .{uri_path})), .http_request => |req| ft: { // Content-Type takes first precedence. const content_type = req.response.headers.getFirstValue("Content-Type") orelse - return f.fail(f.location_tok, "missing 'Content-Type' header", .{}); + return f.fail(f.location_tok, try eb.addString("missing 'Content-Type' header")); if (ascii.eqlIgnoreCase(content_type, "application/x-tar")) - return .tar; + break :ft .tar; if (ascii.eqlIgnoreCase(content_type, "application/gzip") or ascii.eqlIgnoreCase(content_type, "application/x-gzip") or ascii.eqlIgnoreCase(content_type, "application/tar+gzip")) { - return .@"tar.gz"; + break :ft .@"tar.gz"; } if (ascii.eqlIgnoreCase(content_type, "application/x-xz")) - return .@"tar.xz"; + break :ft .@"tar.xz"; if (!ascii.eqlIgnoreCase(content_type, "application/octet-stream")) { - return f.fail(f.location_tok, "unrecognized 'Content-Type' header: '{s}'", .{ - content_type, - }); + return f.fail(f.location_tok, try eb.printString( + "unrecognized 'Content-Type' header: '{s}'", + .{content_type}, + )); } // Next, the filename from 'content-disposition: attachment' takes precedence. if (req.response.headers.getFirstValue("Content-Disposition")) |cd_header| { - break :ft FileType.fromContentDisposition(cd_header) orelse - return f.fail( - f.location_tok, - "unsupported Content-Disposition header value: '{s}' for Content-Type=application/octet-stream", - .{cd_header}, - ); + break :ft FileType.fromContentDisposition(cd_header) orelse { + return f.fail(f.location_tok, try eb.printString( + "unsupported Content-Disposition header value: '{s}' for Content-Type=application/octet-stream", + .{cd_header}, + )); + }; } // Finally, the path from the URI is used. - break :ft FileType.fromPath(uri_path) orelse - return f.fail(f.location_tok, "unknown file type: '{s}'", .{uri_path}); + break :ft FileType.fromPath(uri_path) orelse { + return f.fail(f.location_tok, try eb.printString( + "unknown file type: '{s}'", + .{uri_path}, + )); + }; }, - .git_fetch_stream => return .git_pack, - .dir => |dir| { - try f.recursiveDirectoryCopy(dir, tmp_directory.handle); - return; + + .git => .git_pack, + + .dir => |dir| return f.recursiveDirectoryCopy(dir, tmp_directory.handle) catch |err| { + return f.fail(f.location_tok, try eb.printString( + "unable to copy directory '{s}': {s}", + .{ uri_path, @errorName(err) }, + )); }, }; @@ -618,7 +828,14 @@ fn unpackResource( .tar => try unpackTarball(f, tmp_directory.handle, resource.reader()), .@"tar.gz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.gzip), .@"tar.xz" => try unpackTarballCompressed(f, tmp_directory.handle, resource, std.compress.xz), - .git_pack => try unpackGitPack(f, tmp_directory.handle, resource), + .git_pack => unpackGitPack(f, tmp_directory.handle, resource) catch |err| switch (err) { + error.FetchFailed => return error.FetchFailed, + error.OutOfMemory => return error.OutOfMemory, + else => |e| return f.fail(f.location_tok, try eb.printString( + "unable to unpack git files: {s}", + .{@errorName(e)}, + )), + }, } } @@ -628,11 +845,17 @@ fn unpackTarballCompressed( resource: *Resource, comptime Compression: type, ) RunError!void { - const gpa = f.gpa; + const gpa = f.arena.child_allocator; + const eb = &f.error_bundle; const reader = resource.reader(); var br = std.io.bufferedReaderSize(std.crypto.tls.max_ciphertext_record_len, reader); - var decompress = try Compression.decompress(gpa, br.reader()); + var decompress = Compression.decompress(gpa, br.reader()) catch |err| { + return f.fail(f.location_tok, try eb.printString( + "unable to decompress tarball: {s}", + .{@errorName(err)}, + )); + }; defer decompress.deinit(); return unpackTarball(f, out_dir, decompress.reader()); @@ -640,11 +863,12 @@ fn unpackTarballCompressed( fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void { const eb = &f.error_bundle; + const gpa = f.arena.child_allocator; - var diagnostics: std.tar.Options.Diagnostics = .{ .allocator = f.gpa }; + var diagnostics: std.tar.Options.Diagnostics = .{ .allocator = gpa }; defer diagnostics.deinit(); - try std.tar.pipeToFileSystem(out_dir, reader, .{ + std.tar.pipeToFileSystem(out_dir, reader, .{ .diagnostics = &diagnostics, .strip_components = 1, // TODO: we would like to set this to executable_bit_only, but two @@ -653,12 +877,19 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void { // 2. the hashing algorithm here needs to support detecting the is_executable // bit on Windows from the ACLs (see the isExecutable function). .mode_mode = .ignore, - .filter = .{ .exclude_empty_directories = true }, - }); + .exclude_empty_directories = true, + }) catch |err| return f.fail(f.location_tok, try eb.printString( + "unable to unpack tarball to temporary directory: {s}", + .{@errorName(err)}, + )); if (diagnostics.errors.items.len > 0) { const notes_len: u32 = @intCast(diagnostics.errors.items.len); - try f.addErrorWithNotes(notes_len, f.location_tok, "unable to unpack tarball"); + try eb.addRootErrorMessage(.{ + .msg = try eb.addString("unable to unpack tarball"), + .src_loc = try f.srcLoc(f.location_tok), + .notes_len = notes_len, + }); const notes_start = try eb.reserveNotes(notes_len); for (diagnostics.errors.items, notes_start..) |item, note_i| { switch (item) { @@ -678,19 +909,15 @@ fn unpackTarball(f: *Fetch, out_dir: fs.Dir, reader: anytype) RunError!void { }, } } - return error.InvalidTarball; + return error.FetchFailed; } } -fn unpackGitPack( - f: *Fetch, - out_dir: fs.Dir, - resource: *Resource, - want_oid: git.Oid, -) !void { +fn unpackGitPack(f: *Fetch, out_dir: fs.Dir, resource: *Resource) anyerror!void { const eb = &f.error_bundle; - const gpa = f.gpa; - const reader = resource.reader(); + const gpa = f.arena.child_allocator; + const want_oid = resource.git.want_oid; + const reader = resource.git.fetch_stream.reader(); // The .git directory is used to store the packfile and associated index, but // we do not attempt to replicate the exact structure of a real .git // directory, since that isn't relevant for fetching a package. @@ -700,13 +927,13 @@ fn unpackGitPack( var pack_file = try pack_dir.createFile("pkg.pack", .{ .read = true }); defer pack_file.close(); var fifo = std.fifo.LinearFifo(u8, .{ .Static = 4096 }).init(); - try fifo.pump(reader.reader(), pack_file.writer()); + try fifo.pump(reader, pack_file.writer()); try pack_file.sync(); var index_file = try pack_dir.createFile("pkg.idx", .{ .read = true }); defer index_file.close(); { - var index_prog_node = reader.prog_node.start("Index pack", 0); + var index_prog_node = f.prog_node.start("Index pack", 0); defer index_prog_node.end(); index_prog_node.activate(); var index_buffered_writer = std.io.bufferedWriter(index_file.writer()); @@ -716,7 +943,7 @@ fn unpackGitPack( } { - var checkout_prog_node = reader.prog_node.start("Checkout", 0); + var checkout_prog_node = f.prog_node.start("Checkout", 0); defer checkout_prog_node.end(); checkout_prog_node.activate(); var repository = try git.Repository.init(gpa, pack_file, index_file); @@ -727,7 +954,11 @@ fn unpackGitPack( if (diagnostics.errors.items.len > 0) { const notes_len: u32 = @intCast(diagnostics.errors.items.len); - try f.addErrorWithNotes(notes_len, f.location_tok, "unable to unpack packfile"); + try eb.addRootErrorMessage(.{ + .msg = try eb.addString("unable to unpack packfile"), + .src_loc = try f.srcLoc(f.location_tok), + .notes_len = notes_len, + }); const notes_start = try eb.reserveNotes(notes_len); for (diagnostics.errors.items, notes_start..) |item, note_i| { switch (item) { @@ -748,9 +979,10 @@ fn unpackGitPack( try out_dir.deleteTree(".git"); } -fn recursiveDirectoryCopy(f: *Fetch, dir: fs.IterableDir, tmp_dir: fs.Dir) RunError!void { +fn recursiveDirectoryCopy(f: *Fetch, dir: fs.IterableDir, tmp_dir: fs.Dir) anyerror!void { + const gpa = f.arena.child_allocator; // Recursive directory copy. - var it = try dir.walk(f.gpa); + var it = try dir.walk(gpa); defer it.deinit(); while (try it.next()) |entry| { switch (entry.kind) { @@ -816,16 +1048,22 @@ pub fn renameTmpIntoCache( /// the hash are not present on the file system. Empty directories are *not /// hashed* and must not be present on the file system when calling this /// function. -fn computeHash(f: *Fetch, pkg_dir: fs.IterableDir, filter: Filter) RunError!Digest { +fn computeHash( + f: *Fetch, + tmp_directory: Cache.Directory, + filter: Filter, +) RunError!Manifest.Digest { // All the path name strings need to be in memory for sorting. - const arena = f.arena_allocator.allocator(); - const gpa = f.gpa; + const arena = f.arena.allocator(); + const gpa = f.arena.child_allocator; + const eb = &f.error_bundle; + const thread_pool = f.job_queue.thread_pool; // Collect all files, recursively, then sort. var all_files = std.ArrayList(*HashedFile).init(gpa); defer all_files.deinit(); - var walker = try pkg_dir.walk(gpa); + var walker = try @as(fs.IterableDir, .{ .dir = tmp_directory.handle }).walk(gpa); defer walker.deinit(); { @@ -834,19 +1072,28 @@ fn computeHash(f: *Fetch, pkg_dir: fs.IterableDir, filter: Filter) RunError!Dige var wait_group: WaitGroup = .{}; // `computeHash` is called from a worker thread so there must not be // any waiting without working or a deadlock could occur. - defer wait_group.waitAndWork(); + defer thread_pool.waitAndWork(&wait_group); - while (try walker.next()) |entry| { + while (walker.next() catch |err| { + try eb.addRootErrorMessage(.{ .msg = try eb.printString( + "unable to walk temporary directory '{}': {s}", + .{ tmp_directory, @errorName(err) }, + ) }); + return error.FetchFailed; + }) |entry| { _ = filter; // TODO: apply filter rules here const kind: HashedFile.Kind = switch (entry.kind) { .directory => continue, .file => .file, .sym_link => .sym_link, - else => return error.IllegalFileTypeInPackage, + else => return f.fail(f.location_tok, try eb.printString( + "package contains '{s}' which has illegal file type '{s}'", + .{ entry.path, @tagName(entry.kind) }, + )), }; - if (std.mem.eql(u8, entry.path, build_zig_basename)) + if (std.mem.eql(u8, entry.path, Package.build_zig_basename)) f.has_build_zig = true; const hashed_file = try arena.create(HashedFile); @@ -859,7 +1106,9 @@ fn computeHash(f: *Fetch, pkg_dir: fs.IterableDir, filter: Filter) RunError!Dige .failure = undefined, // to be populated by the worker }; wait_group.start(); - try f.thread_pool.spawn(workerHashFile, .{ pkg_dir.dir, hashed_file, &wait_group }); + try thread_pool.spawn(workerHashFile, .{ + tmp_directory.handle, hashed_file, &wait_group, + }); try all_files.append(hashed_file); } @@ -869,19 +1118,13 @@ fn computeHash(f: *Fetch, pkg_dir: fs.IterableDir, filter: Filter) RunError!Dige var hasher = Manifest.Hash.init(.{}); var any_failures = false; - const eb = &f.error_bundle; for (all_files.items) |hashed_file| { hashed_file.failure catch |err| { any_failures = true; try eb.addRootErrorMessage(.{ - .msg = try eb.printString("unable to hash: {s}", .{@errorName(err)}), - .src_loc = try eb.addSourceLocation(.{ - .src_path = try eb.addString(hashed_file.fs_path), - .span_start = 0, - .span_end = 0, - .span_main = 0, + .msg = try eb.printString("unable to hash '{s}': {s}", .{ + hashed_file.fs_path, @errorName(err), }), - .notes_len = 0, }); }; hasher.update(&hashed_file.hash); @@ -934,7 +1177,7 @@ fn isExecutable(file: fs.File) !bool { const HashedFile = struct { fs_path: []const u8, normalized_path: []const u8, - hash: Digest, + hash: Manifest.Digest, failure: Error!void, kind: Kind, @@ -970,7 +1213,7 @@ fn normalizePath(arena: Allocator, fs_path: []const u8) ![]const u8 { return normalized; } -pub const Filter = struct { +const Filter = struct { include_paths: std.StringArrayHashMapUnmanaged(void) = .{}, /// sub_path is relative to the tarball root. @@ -990,12 +1233,9 @@ pub const Filter = struct { } }; -const build_zig_basename = @import("../Package.zig").build_zig_basename; -const hex_multihash_len = 2 * Manifest.multihash_len; - // These are random bytes. -const package_hash_prefix_cached: [8]u8 = &.{ 0x53, 0x7e, 0xfa, 0x94, 0x65, 0xe9, 0xf8, 0x73 }; -const package_hash_prefix_project: [8]u8 = &.{ 0xe1, 0x25, 0xee, 0xfa, 0xa6, 0x17, 0x38, 0xcc }; +const package_hash_prefix_cached = [8]u8{ 0x53, 0x7e, 0xfa, 0x94, 0x65, 0xe9, 0xf8, 0x73 }; +const package_hash_prefix_project = [8]u8{ 0xe1, 0x25, 0xee, 0xfa, 0xa6, 0x17, 0x38, 0xcc }; const builtin = @import("builtin"); const std = @import("std"); @@ -1010,3 +1250,4 @@ const Manifest = @import("../Manifest.zig"); const Fetch = @This(); const main = @import("../main.zig"); const git = @import("../git.zig"); +const Package = @import("../Package.zig"); diff --git a/src/Package/Module.zig b/src/Package/Module.zig new file mode 100644 index 0000000000..f2accc104d --- /dev/null +++ b/src/Package/Module.zig @@ -0,0 +1,32 @@ +//! Corresponds to something that Zig source code can `@import`. +//! Not to be confused with src/Module.zig which should be renamed +//! to something else. https://github.com/ziglang/zig/issues/14307 + +/// Only files inside this directory can be imported. +root: Package.Path, +/// Relative to `root`. May contain path separators. +root_src_path: []const u8, +/// The dependency table of this module. Shared dependencies such as 'std', +/// 'builtin', and 'root' are not specified in every dependency table, but +/// instead only in the table of `main_pkg`. `Module.importFile` is +/// responsible for detecting these names and using the correct package. +deps: Deps = .{}, + +pub const Deps = std.StringHashMapUnmanaged(*Module); + +pub const Tree = struct { + /// Each `Package` exposes a `Module` with build.zig as its root source file. + build_module_table: std.AutoArrayHashMapUnmanaged(MultiHashHexDigest, *Module), +}; + +pub fn create(allocator: Allocator, m: Module) Allocator.Error!*Module { + const new = try allocator.create(Module); + new.* = m; + return new; +} + +const Module = @This(); +const Package = @import("../Package.zig"); +const std = @import("std"); +const Allocator = std.mem.Allocator; +const MultiHashHexDigest = Package.Manifest.MultiHashHexDigest; diff --git a/src/crash_report.zig b/src/crash_report.zig index d4e4b46a53..2b33bd7fa5 100644 --- a/src/crash_report.zig +++ b/src/crash_report.zig @@ -139,18 +139,22 @@ fn dumpStatusReport() !void { var crash_heap: [16 * 4096]u8 = undefined; -fn writeFilePath(file: *Module.File, stream: anytype) !void { - if (file.pkg.root_src_directory.path) |path| { - try stream.writeAll(path); - try stream.writeAll(std.fs.path.sep_str); +fn writeFilePath(file: *Module.File, writer: anytype) !void { + if (file.mod.root.root_dir.path) |path| { + try writer.writeAll(path); + try writer.writeAll(std.fs.path.sep_str); } - try stream.writeAll(file.sub_file_path); + if (file.mod.root.sub_path.len > 0) { + try writer.writeAll(file.mod.root.sub_path); + try writer.writeAll(std.fs.path.sep_str); + } + try writer.writeAll(file.sub_file_path); } -fn writeFullyQualifiedDeclWithFile(mod: *Module, decl: *Decl, stream: anytype) !void { - try writeFilePath(decl.getFileScope(mod), stream); - try stream.writeAll(": "); - try decl.renderFullyQualifiedDebugName(mod, stream); +fn writeFullyQualifiedDeclWithFile(mod: *Module, decl: *Decl, writer: anytype) !void { + try writeFilePath(decl.getFileScope(mod), writer); + try writer.writeAll(": "); + try decl.renderFullyQualifiedDebugName(mod, writer); } pub fn compilerPanic(msg: []const u8, error_return_trace: ?*std.builtin.StackTrace, maybe_ret_addr: ?usize) noreturn { diff --git a/src/main.zig b/src/main.zig index 34ab5ea191..f52a4bf24a 100644 --- a/src/main.zig +++ b/src/main.zig @@ -416,7 +416,7 @@ const usage_build_generic = \\ dep: [[import=]name] \\ --deps [dep],[dep],... Set dependency names for the root package \\ dep: [[import=]name] - \\ --main-pkg-path Set the directory of the root package + \\ --main-mod-path Set the directory of the root module \\ -fPIC Force-enable Position Independent Code \\ -fno-PIC Force-disable Position Independent Code \\ -fPIE Force-enable Position Independent Executable @@ -765,17 +765,11 @@ const Framework = struct { }; const CliModule = struct { - mod: *Package, + mod: *Package.Module, /// still in CLI arg format deps_str: []const u8, }; -fn cleanupModules(modules: *std.StringArrayHashMap(CliModule)) void { - var it = modules.iterator(); - while (it.next()) |kv| kv.value_ptr.mod.destroy(modules.allocator); - modules.deinit(); -} - fn buildOutputType( gpa: Allocator, arena: Allocator, @@ -950,8 +944,7 @@ fn buildOutputType( // Contains every module specified via --mod. The dependencies are added // after argument parsing is completed. We use a StringArrayHashMap to make // error output consistent. - var modules = std.StringArrayHashMap(CliModule).init(gpa); - defer cleanupModules(&modules); + var modules = std.StringArrayHashMap(CliModule).init(arena); // The dependency string for the root package var root_deps_str: ?[]const u8 = null; @@ -1023,32 +1016,37 @@ fn buildOutputType( for ([_][]const u8{ "std", "root", "builtin" }) |name| { if (mem.eql(u8, mod_name, name)) { - fatal("unable to add module '{s}' -> '{s}': conflicts with builtin module", .{ mod_name, root_src }); + fatal("unable to add module '{s}' -> '{s}': conflicts with builtin module", .{ + mod_name, root_src, + }); } } var mod_it = modules.iterator(); while (mod_it.next()) |kv| { if (std.mem.eql(u8, mod_name, kv.key_ptr.*)) { - fatal("unable to add module '{s}' -> '{s}': already exists as '{s}'", .{ mod_name, root_src, kv.value_ptr.mod.root_src_path }); + fatal("unable to add module '{s}' -> '{s}': already exists as '{s}'", .{ + mod_name, root_src, kv.value_ptr.mod.root_src_path, + }); } } - try modules.ensureUnusedCapacity(1); - modules.put(mod_name, .{ - .mod = try Package.create( - gpa, - fs.path.dirname(root_src), - fs.path.basename(root_src), - ), + try modules.put(mod_name, .{ + .mod = try Package.Module.create(arena, .{ + .root = .{ + .root_dir = Cache.Directory.cwd(), + .sub_path = fs.path.dirname(root_src) orelse "", + }, + .root_src_path = fs.path.basename(root_src), + }), .deps_str = deps_str, - }) catch unreachable; + }); } else if (mem.eql(u8, arg, "--deps")) { if (root_deps_str != null) { fatal("only one --deps argument is allowed", .{}); } root_deps_str = args_iter.nextOrFatal(); - } else if (mem.eql(u8, arg, "--main-pkg-path")) { + } else if (mem.eql(u8, arg, "--main-mod-path")) { main_pkg_path = args_iter.nextOrFatal(); } else if (mem.eql(u8, arg, "-cflags")) { extra_cflags.shrinkRetainingCapacity(0); @@ -2461,19 +2459,26 @@ fn buildOutputType( var deps_it = ModuleDepIterator.init(deps_str); while (deps_it.next()) |dep| { if (dep.expose.len == 0) { - fatal("module '{s}' depends on '{s}' with a blank name", .{ kv.key_ptr.*, dep.name }); + fatal("module '{s}' depends on '{s}' with a blank name", .{ + kv.key_ptr.*, dep.name, + }); } for ([_][]const u8{ "std", "root", "builtin" }) |name| { if (mem.eql(u8, dep.expose, name)) { - fatal("unable to add module '{s}' under name '{s}': conflicts with builtin module", .{ dep.name, dep.expose }); + fatal("unable to add module '{s}' under name '{s}': conflicts with builtin module", .{ + dep.name, dep.expose, + }); } } - const dep_mod = modules.get(dep.name) orelse - fatal("module '{s}' depends on module '{s}' which does not exist", .{ kv.key_ptr.*, dep.name }); + const dep_mod = modules.get(dep.name) orelse { + fatal("module '{s}' depends on module '{s}' which does not exist", .{ + kv.key_ptr.*, dep.name, + }); + }; - try kv.value_ptr.mod.add(gpa, dep.expose, dep_mod.mod); + try kv.value_ptr.mod.deps.put(arena, dep.expose, dep_mod.mod); } } } @@ -3229,31 +3234,33 @@ fn buildOutputType( }; defer emit_implib_resolved.deinit(); - const main_pkg: ?*Package = if (root_src_file) |unresolved_src_path| blk: { + const main_mod: ?*Package.Module = if (root_src_file) |unresolved_src_path| blk: { const src_path = try introspect.resolvePath(arena, unresolved_src_path); if (main_pkg_path) |unresolved_main_pkg_path| { const p = try introspect.resolvePath(arena, unresolved_main_pkg_path); - if (p.len == 0) { - break :blk try Package.create(gpa, null, src_path); - } else { - const rel_src_path = try fs.path.relative(arena, p, src_path); - break :blk try Package.create(gpa, p, rel_src_path); - } + break :blk try Package.Module.create(arena, .{ + .root = .{ + .root_dir = Cache.Directory.cwd(), + .sub_path = p, + }, + .root_src_path = if (p.len == 0) + src_path + else + try fs.path.relative(arena, p, src_path), + }); } else { - const root_src_dir_path = fs.path.dirname(src_path); - break :blk Package.create(gpa, root_src_dir_path, fs.path.basename(src_path)) catch |err| { - if (root_src_dir_path) |p| { - fatal("unable to open '{s}': {s}", .{ p, @errorName(err) }); - } else { - return err; - } - }; + break :blk try Package.Module.create(arena, .{ + .root = .{ + .root_dir = Cache.Directory.cwd(), + .sub_path = fs.path.dirname(src_path) orelse "", + }, + .root_src_path = fs.path.basename(src_path), + }); } } else null; - defer if (main_pkg) |p| p.destroy(gpa); // Transfer packages added with --deps to the root package - if (main_pkg) |mod| { + if (main_mod) |mod| { var it = ModuleDepIterator.init(root_deps_str orelse ""); while (it.next()) |dep| { if (dep.expose.len == 0) { @@ -3269,7 +3276,7 @@ fn buildOutputType( const dep_mod = modules.get(dep.name) orelse fatal("root module depends on module '{s}' which does not exist", .{dep.name}); - try mod.add(gpa, dep.expose, dep_mod.mod); + try mod.deps.put(arena, dep.expose, dep_mod.mod); } } @@ -3310,17 +3317,18 @@ fn buildOutputType( if (arg_mode == .run) { break :l global_cache_directory; } - if (main_pkg) |pkg| { + if (main_mod != null) { // search upwards from cwd until we find directory with build.zig const cwd_path = try process.getCwdAlloc(arena); - const build_zig = "build.zig"; const zig_cache = "zig-cache"; var dirname: []const u8 = cwd_path; while (true) { - const joined_path = try fs.path.join(arena, &[_][]const u8{ dirname, build_zig }); + const joined_path = try fs.path.join(arena, &.{ + dirname, Package.build_zig_basename, + }); if (fs.cwd().access(joined_path, .{})) |_| { - const cache_dir_path = try fs.path.join(arena, &[_][]const u8{ dirname, zig_cache }); - const dir = try pkg.root_src_directory.handle.makeOpenPath(cache_dir_path, .{}); + const cache_dir_path = try fs.path.join(arena, &.{ dirname, zig_cache }); + const dir = try fs.cwd().makeOpenPath(cache_dir_path, .{}); cleanup_local_cache_dir = dir; break :l .{ .handle = dir, .path = cache_dir_path }; } else |err| switch (err) { @@ -3378,6 +3386,8 @@ fn buildOutputType( gimmeMoreOfThoseSweetSweetFileDescriptors(); + if (true) @panic("TODO restore Compilation logic"); + const comp = Compilation.create(gpa, .{ .zig_lib_directory = zig_lib_directory, .local_cache_directory = local_cache_directory, @@ -3389,7 +3399,7 @@ fn buildOutputType( .dynamic_linker = target_info.dynamic_linker.get(), .sysroot = sysroot, .output_mode = output_mode, - .main_pkg = main_pkg, + .main_mod = main_mod, .emit_bin = emit_bin_loc, .emit_h = emit_h_resolved.data, .emit_asm = emit_asm_resolved.data, @@ -4799,32 +4809,22 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi try thread_pool.init(.{ .allocator = gpa }); defer thread_pool.deinit(); - var cleanup_build_runner_dir: ?fs.Dir = null; - defer if (cleanup_build_runner_dir) |*dir| dir.close(); - - var main_pkg: Package = if (override_build_runner) |build_runner_path| + var main_mod: Package.Module = if (override_build_runner) |build_runner_path| .{ - .root_src_directory = blk: { - if (std.fs.path.dirname(build_runner_path)) |dirname| { - const dir = fs.cwd().openDir(dirname, .{}) catch |err| { - fatal("unable to open directory to build runner from argument 'build-runner', '{s}': {s}", .{ dirname, @errorName(err) }); - }; - cleanup_build_runner_dir = dir; - break :blk .{ .path = dirname, .handle = dir }; - } - - break :blk .{ .path = null, .handle = fs.cwd() }; + .root = .{ + .root_dir = Cache.Directory.cwd(), + .sub_path = fs.path.dirname(build_runner_path) orelse "", }, - .root_src_path = std.fs.path.basename(build_runner_path), + .root_src_path = fs.path.basename(build_runner_path), } else .{ - .root_src_directory = zig_lib_directory, + .root = .{ .root_dir = zig_lib_directory }, .root_src_path = "build_runner.zig", }; - var build_pkg: Package = .{ - .root_src_directory = build_directory, + var build_mod: Package.Module = .{ + .root = .{ .root_dir = build_directory }, .root_src_path = build_zig_basename, }; if (build_options.only_core_functionality) { @@ -4833,11 +4833,13 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi \\pub const root_deps: []const struct { []const u8, []const u8 } = &.{}; \\ ); - try main_pkg.add(gpa, "@dependencies", deps_pkg); + try main_mod.deps.put(arena, "@dependencies", deps_pkg); } else { var http_client: std.http.Client = .{ .allocator = gpa }; defer http_client.deinit(); + if (true) @panic("TODO restore package fetching logic"); + // Here we provide an import to the build runner that allows using reflection to find // all of the dependencies. Without this, there would be no way to use `@import` to // access dependencies by name, since `@import` requires string literals. @@ -4857,8 +4859,8 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi // Here we borrow main package's table and will replace it with a fresh // one after this process completes. - const fetch_result = build_pkg.fetchAndAddDependencies( - &main_pkg, + const fetch_result = build_mod.fetchAndAddDependencies( + &main_mod, arena, &thread_pool, &http_client, @@ -4886,10 +4888,10 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi dependencies_source.items, ); - mem.swap(Package.Table, &main_pkg.table, &deps_pkg.table); - try main_pkg.add(gpa, "@dependencies", deps_pkg); + mem.swap(Package.Table, &main_mod.table, &deps_pkg.table); + try main_mod.add(gpa, "@dependencies", deps_pkg); } - try main_pkg.add(gpa, "@build", &build_pkg); + try main_mod.add(gpa, "@build", &build_mod); const comp = Compilation.create(gpa, .{ .zig_lib_directory = zig_lib_directory, @@ -4901,7 +4903,7 @@ pub fn cmdBuild(gpa: Allocator, arena: Allocator, args: []const []const u8) !voi .is_native_abi = cross_target.isNativeAbi(), .dynamic_linker = target_info.dynamic_linker.get(), .output_mode = .Exe, - .main_pkg = &main_pkg, + .main_mod = &main_mod, .emit_bin = emit_bin, .emit_h = null, .optimize_mode = .Debug, @@ -5115,12 +5117,14 @@ pub fn cmdFmt(gpa: Allocator, arena: Allocator, args: []const []const u8) !void .tree = tree, .tree_loaded = true, .zir = undefined, - .pkg = undefined, + .mod = undefined, .root_decl = .none, }; - file.pkg = try Package.create(gpa, null, file.sub_file_path); - defer file.pkg.destroy(gpa); + file.mod = try Package.Module.create(arena, .{ + .root = Package.Path.cwd(), + .root_src_path = file.sub_file_path, + }); file.zir = try AstGen.generate(gpa, file.tree); file.zir_loaded = true; @@ -5321,12 +5325,14 @@ fn fmtPathFile( .tree = tree, .tree_loaded = true, .zir = undefined, - .pkg = undefined, + .mod = undefined, .root_decl = .none, }; - file.pkg = try Package.create(gpa, null, file.sub_file_path); - defer file.pkg.destroy(gpa); + file.mod = try Package.Module.create(fmt.arena, .{ + .root = Package.Path.cwd(), + .root_src_path = file.sub_file_path, + }); if (stat.size > max_src_size) return error.FileTooBig; @@ -5387,7 +5393,7 @@ pub fn putAstErrorsIntoBundle( tree: Ast, path: []const u8, wip_errors: *std.zig.ErrorBundle.Wip, -) !void { +) Allocator.Error!void { var file: Module.File = .{ .status = .never_loaded, .source_loaded = true, @@ -5402,12 +5408,15 @@ pub fn putAstErrorsIntoBundle( .tree = tree, .tree_loaded = true, .zir = undefined, - .pkg = undefined, + .mod = undefined, .root_decl = .none, }; - file.pkg = try Package.create(gpa, null, path); - defer file.pkg.destroy(gpa); + file.mod = try Package.Module.create(gpa, .{ + .root = Package.Path.cwd(), + .root_src_path = file.sub_file_path, + }); + defer gpa.destroy(file.mod); file.zir = try AstGen.generate(gpa, file.tree); file.zir_loaded = true; @@ -5933,7 +5942,7 @@ pub fn cmdAstCheck( .stat = undefined, .tree = undefined, .zir = undefined, - .pkg = undefined, + .mod = undefined, .root_decl = .none, }; if (zig_source_file) |file_name| { @@ -5971,8 +5980,10 @@ pub fn cmdAstCheck( file.stat.size = source.len; } - file.pkg = try Package.create(gpa, null, file.sub_file_path); - defer file.pkg.destroy(gpa); + file.mod = try Package.Module.create(arena, .{ + .root = Package.Path.cwd(), + .root_src_path = file.sub_file_path, + }); file.tree = try Ast.parse(gpa, file.source, .zig); file.tree_loaded = true; @@ -6067,7 +6078,7 @@ pub fn cmdDumpZir( .stat = undefined, .tree = undefined, .zir = try Module.loadZirCache(gpa, f), - .pkg = undefined, + .mod = undefined, .root_decl = .none, }; @@ -6136,12 +6147,14 @@ pub fn cmdChangelist( }, .tree = undefined, .zir = undefined, - .pkg = undefined, + .mod = undefined, .root_decl = .none, }; - file.pkg = try Package.create(gpa, null, file.sub_file_path); - defer file.pkg.destroy(gpa); + file.mod = try Package.Module.create(arena, .{ + .root = Package.Path.cwd(), + .root_src_path = file.sub_file_path, + }); const source = try arena.allocSentinel(u8, @as(usize, @intCast(stat.size)), 0); const amt = try f.readAll(source); @@ -6623,8 +6636,11 @@ fn cmdFetch( args: []const []const u8, ) !void { const color: Color = .auto; - var opt_url: ?[]const u8 = null; + const work_around_btrfs_bug = builtin.os.tag == .linux and + std.process.hasEnvVarConstant("ZIG_BTRFS_WORKAROUND"); + var opt_path_or_url: ?[]const u8 = null; var override_global_cache_dir: ?[]const u8 = try optionalStringEnvVar(arena, "ZIG_GLOBAL_CACHE_DIR"); + var recursive = false; { var i: usize = 0; @@ -6640,18 +6656,21 @@ fn cmdFetch( i += 1; override_global_cache_dir = args[i]; continue; + } else if (mem.eql(u8, arg, "--recursive")) { + recursive = true; + continue; } else { fatal("unrecognized parameter: '{s}'", .{arg}); } - } else if (opt_url != null) { + } else if (opt_path_or_url != null) { fatal("unexpected extra parameter: '{s}'", .{arg}); } else { - opt_url = arg; + opt_path_or_url = arg; } } } - const url = opt_url orelse fatal("missing url or path parameter", .{}); + const path_or_url = opt_path_or_url orelse fatal("missing url or path parameter", .{}); var thread_pool: ThreadPool = undefined; try thread_pool.init(.{ .allocator = gpa }); @@ -6664,19 +6683,6 @@ fn cmdFetch( const root_prog_node = progress.start("Fetch", 0); defer root_prog_node.end(); - var wip_errors: std.zig.ErrorBundle.Wip = undefined; - try wip_errors.init(gpa); - defer wip_errors.deinit(); - - var report: Package.Report = .{ - .ast = null, - .directory = .{ - .handle = fs.cwd(), - .path = null, - }, - .error_bundle = &wip_errors, - }; - var global_cache_directory: Compilation.Directory = l: { const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena); break :l .{ @@ -6686,56 +6692,48 @@ fn cmdFetch( }; defer global_cache_directory.handle.close(); - var readable_resource: Package.ReadableResource = rr: { - if (fs.cwd().openIterableDir(url, .{})) |dir| { - break :rr .{ - .path = try gpa.dupe(u8, url), - .resource = .{ .dir = dir }, - }; - } else |dir_err| { - const file_err = if (dir_err == error.NotDir) e: { - if (fs.cwd().openFile(url, .{})) |f| { - break :rr .{ - .path = try gpa.dupe(u8, url), - .resource = .{ .file = f }, - }; - } else |err| break :e err; - } else dir_err; - - const uri = std.Uri.parse(url) catch |uri_err| { - fatal("'{s}' could not be recognized as a file path ({s}) or an URL ({s})", .{ - url, @errorName(file_err), @errorName(uri_err), - }); - }; - const fetch_location = try Package.FetchLocation.initUri(uri, 0, report); - const cwd: Cache.Directory = .{ - .handle = fs.cwd(), - .path = null, - }; - break :rr try fetch_location.fetch(gpa, cwd, &http_client, 0, report); - } + var job_queue: Package.Fetch.JobQueue = .{ + .http_client = &http_client, + .thread_pool = &thread_pool, + .global_cache = global_cache_directory, + .recursive = recursive, + .work_around_btrfs_bug = work_around_btrfs_bug, }; - defer readable_resource.deinit(gpa); + defer job_queue.deinit(); - var package_location = readable_resource.unpack( - gpa, - &thread_pool, - global_cache_directory, - 0, - report, - root_prog_node, - ) catch |err| { - if (wip_errors.root_list.items.len > 0) { - var errors = try wip_errors.toOwnedBundle(""); - defer errors.deinit(gpa); - errors.renderToStdErr(renderOptions(color)); - process.exit(1); - } - fatal("unable to unpack '{s}': {s}", .{ url, @errorName(err) }); + var fetch: Package.Fetch = .{ + .arena = std.heap.ArenaAllocator.init(gpa), + .location = .{ .path_or_url = path_or_url }, + .location_tok = 0, + .hash_tok = 0, + .parent_package_root = undefined, + .parent_manifest_ast = null, + .prog_node = root_prog_node, + .job_queue = &job_queue, + .omit_missing_hash_error = true, + + .package_root = undefined, + .error_bundle = undefined, + .manifest = null, + .manifest_ast = undefined, + .actual_hash = undefined, + .has_build_zig = false, + .oom_flag = false, }; - defer package_location.deinit(gpa); + defer fetch.deinit(); - const hex_digest = Package.Manifest.hexDigest(package_location.hash); + fetch.run() catch |err| switch (err) { + error.OutOfMemory => fatal("out of memory", .{}), + error.FetchFailed => {}, // error bundle checked below + }; + + if (fetch.error_bundle.root_list.items.len > 0) { + var errors = try fetch.error_bundle.toOwnedBundle(""); + errors.renderToStdErr(renderOptions(color)); + process.exit(1); + } + + const hex_digest = Package.Manifest.hexDigest(fetch.actual_hash); progress.done = true; progress.refresh();