From dacd36ca9b193444e9e2ca3f132fccf3e08fb4f9 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 22 Sep 2020 14:06:57 -0700 Subject: [PATCH] stage2: implement using the global cache dir --- BRANCH_TODO | 13 ++++------ src/Compilation.zig | 31 ++++++++++++----------- src/glibc.zig | 21 +++++++++------- src/introspect.zig | 7 ------ src/libcxx.zig | 8 +++--- src/libunwind.zig | 4 +-- src/main.zig | 60 ++++++++++++++++++++++++++++++++++++++------- src/test.zig | 3 ++- 8 files changed, 93 insertions(+), 54 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index c84dc397bb..4a55c8192f 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -14,26 +14,23 @@ * -fno-emit-asm (default) do not output .s (assembly code)\n" * -femit-llvm-ir produce a .ll file with LLVM IR\n" * -fno-emit-llvm-ir (default) do not produce a .ll file with LLVM IR\n" - * --cache-dir [path] override the local cache directory - * --global-cache-dir [path] override the global cache directory - * implement proper parsing of LLD stderr/stdout and exposing compile errors - * implement proper parsing of clang stderr/stdout and exposing compile errors * support rpaths in ELF linker code * add CLI support for a way to pass extra flags to c source files * musl * mingw-w64 - * use global zig-cache dir for crt files - * use global zig-cache dir for `zig run` executables but not `zig test` * MachO LLD linking * COFF LLD linking * WASM LLD linking - * support cross compiling stage2 with `zig build` * --main-pkg-path * audit the CLI options for stage2 - * restore error messages for stage2_add_link_lib * audit the base cache hash + * implement proper parsing of LLD stderr/stdout and exposing compile errors + * implement proper parsing of clang stderr/stdout and exposing compile errors * On operating systems that support it, do an execve for `zig test` and `zig run` rather than child process. + * restore error messages for stage2_add_link_lib + * update zig build to use new CLI + * support cross compiling stage2 with `zig build` * implement proper compile errors for failing to build glibc crt files and shared libs * implement -fno-emit-bin * improve the stage2 tests to support testing with LLVM extensions enabled diff --git a/src/Compilation.zig b/src/Compilation.zig index 70492b70be..9d2d59b98a 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -64,7 +64,8 @@ cache_parent: *Cache, /// Path to own executable for invoking `zig clang`. self_exe_path: ?[]const u8, zig_lib_directory: Directory, -zig_cache_directory: Directory, +local_cache_directory: Directory, +global_cache_directory: Directory, libc_include_dir_list: []const []const u8, rand: *std.rand.Random, @@ -267,7 +268,8 @@ pub const EmitLoc = struct { pub const InitOptions = struct { zig_lib_directory: Directory, - zig_cache_directory: Directory, + local_cache_directory: Directory, + global_cache_directory: Directory, target: Target, root_name: []const u8, root_pkg: ?*Package, @@ -523,7 +525,7 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { const cache = try arena.create(Cache); cache.* = .{ .gpa = gpa, - .manifest_dir = try options.zig_cache_directory.handle.makeOpenPath("h", .{}), + .manifest_dir = try options.local_cache_directory.handle.makeOpenPath("h", .{}), }; errdefer cache.manifest_dir.close(); @@ -569,11 +571,11 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { const digest = hash.final(); const artifact_sub_dir = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest }); - var artifact_dir = try options.zig_cache_directory.handle.makeOpenPath(artifact_sub_dir, .{}); + var artifact_dir = try options.local_cache_directory.handle.makeOpenPath(artifact_sub_dir, .{}); errdefer artifact_dir.close(); const zig_cache_artifact_directory: Directory = .{ .handle = artifact_dir, - .path = if (options.zig_cache_directory.path) |p| + .path = if (options.local_cache_directory.path) |p| try std.fs.path.join(arena, &[_][]const u8{ p, artifact_sub_dir }) else artifact_sub_dir, @@ -647,11 +649,11 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { const digest = hash.final(); const artifact_sub_dir = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest }); - var artifact_dir = try options.zig_cache_directory.handle.makeOpenPath(artifact_sub_dir, .{}); + var artifact_dir = try options.local_cache_directory.handle.makeOpenPath(artifact_sub_dir, .{}); owned_link_dir = artifact_dir; const link_artifact_directory: Directory = .{ .handle = artifact_dir, - .path = try options.zig_cache_directory.join(arena, &[_][]const u8{artifact_sub_dir}), + .path = try options.local_cache_directory.join(arena, &[_][]const u8{artifact_sub_dir}), }; break :blk link_artifact_directory; }; @@ -715,7 +717,8 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { .gpa = gpa, .arena_state = arena_allocator.state, .zig_lib_directory = options.zig_lib_directory, - .zig_cache_directory = options.zig_cache_directory, + .local_cache_directory = options.local_cache_directory, + .global_cache_directory = options.global_cache_directory, .bin_file = bin_file, .work_queue = std.fifo.LinearFifo(Job, .Dynamic).init(gpa), .keep_source_files_loaded = options.keep_source_files_loaded, @@ -1230,7 +1233,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { // We can't know the digest until we do the C compiler invocation, so we need a temporary filename. const out_obj_path = try comp.tmpFilePath(arena, o_basename); - var zig_cache_tmp_dir = try comp.zig_cache_directory.handle.makeOpenPath("tmp", .{}); + var zig_cache_tmp_dir = try comp.local_cache_directory.handle.makeOpenPath("tmp", .{}); defer zig_cache_tmp_dir.close(); try argv.appendSlice(&[_][]const u8{ self_exe_path, "clang", "-c" }); @@ -1319,7 +1322,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { // Rename into place. const digest = ch.final(); const o_sub_path = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest }); - var o_dir = try comp.zig_cache_directory.handle.makeOpenPath(o_sub_path, .{}); + var o_dir = try comp.local_cache_directory.handle.makeOpenPath(o_sub_path, .{}); defer o_dir.close(); // TODO https://github.com/ziglang/zig/issues/6344 const tmp_basename = std.fs.path.basename(out_obj_path); @@ -1331,7 +1334,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { break :blk digest; }; - const components = if (comp.zig_cache_directory.path) |p| + const components = if (comp.local_cache_directory.path) |p| &[_][]const u8{ p, "o", &digest, o_basename } else &[_][]const u8{ "o", &digest, o_basename }; @@ -1347,7 +1350,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject) !void { fn tmpFilePath(comp: *Compilation, arena: *Allocator, suffix: []const u8) error{OutOfMemory}![]const u8 { const s = std.fs.path.sep_str; const rand_int = comp.rand.int(u64); - if (comp.zig_cache_directory.path) |p| { + if (comp.local_cache_directory.path) |p| { return std.fmt.allocPrint(arena, "{}" ++ s ++ "tmp" ++ s ++ "{x}-{s}", .{ p, rand_int, suffix }); } else { return std.fmt.allocPrint(arena, "tmp" ++ s ++ "{x}-{s}", .{ rand_int, suffix }); @@ -2116,8 +2119,8 @@ fn buildStaticLibFromZig(comp: *Compilation, basename: []const u8, out: *?CRTFil } }; const sub_compilation = try Compilation.create(comp.gpa, .{ - // TODO use the global cache directory here - .zig_cache_directory = comp.zig_cache_directory, + .global_cache_directory = comp.global_cache_directory, + .local_cache_directory = comp.global_cache_directory, .zig_lib_directory = comp.zig_lib_directory, .target = comp.getTarget(), .root_name = mem.split(basename, ".").next().?, diff --git a/src/glibc.zig b/src/glibc.zig index 22f060f514..c7c02a4470 100644 --- a/src/glibc.zig +++ b/src/glibc.zig @@ -691,8 +691,8 @@ fn build_crt_file( .basename = basename, }; const sub_compilation = try Compilation.create(comp.gpa, .{ - // TODO use the global cache directory here - .zig_cache_directory = comp.zig_cache_directory, + .local_cache_directory = comp.global_cache_directory, + .global_cache_directory = comp.global_cache_directory, .zig_lib_directory = comp.zig_lib_directory, .target = comp.getTarget(), .root_name = mem.split(basename, ".").next().?, @@ -769,11 +769,13 @@ pub fn buildSharedObjects(comp: *Compilation) !void { const target = comp.getTarget(); const target_version = target.os.version_range.linux.glibc; - // TODO use the global cache directory here + // Use the global cache directory. var cache_parent: Cache = .{ .gpa = comp.gpa, - .manifest_dir = comp.cache_parent.manifest_dir, + .manifest_dir = try comp.global_cache_directory.handle.makeOpenPath("h", .{}), }; + defer cache_parent.manifest_dir.close(); + var cache = cache_parent.obtain(); defer cache.deinit(); cache.hash.addBytes(build_options.version); @@ -790,8 +792,8 @@ pub fn buildSharedObjects(comp: *Compilation) !void { // We use the presence of an "ok" file to determine if it is a true hit. var o_directory: Compilation.Directory = .{ - .handle = try comp.zig_cache_directory.handle.makeOpenPath(o_sub_path, .{}), - .path = try path.join(arena, &[_][]const u8{ comp.zig_cache_directory.path.?, o_sub_path }), + .handle = try comp.global_cache_directory.handle.makeOpenPath(o_sub_path, .{}), + .path = try path.join(arena, &[_][]const u8{ comp.global_cache_directory.path.?, o_sub_path }), }; defer o_directory.handle.close(); @@ -931,7 +933,7 @@ pub fn buildSharedObjects(comp: *Compilation) !void { const asm_file_basename = std.fmt.bufPrint(&lib_name_buf, "{s}.s", .{lib.name}) catch unreachable; try o_directory.handle.writeFile(asm_file_basename, zig_body.items); - try buildSharedLib(comp, arena, comp.zig_cache_directory, o_directory, asm_file_basename, lib); + try buildSharedLib(comp, arena, comp.global_cache_directory, o_directory, asm_file_basename, lib); } // No need to write the manifest because there are no file inputs associated with this cache hash. // However we do need to write the ok file now. @@ -945,7 +947,7 @@ pub fn buildSharedObjects(comp: *Compilation) !void { assert(comp.glibc_so_files == null); comp.glibc_so_files = BuiltSharedObjects{ .lock = cache.toOwnedLock(), - .dir_path = try path.join(comp.gpa, &[_][]const u8{ comp.zig_cache_directory.path.?, o_sub_path }), + .dir_path = try path.join(comp.gpa, &[_][]const u8{ comp.global_cache_directory.path.?, o_sub_path }), }; } @@ -976,7 +978,8 @@ fn buildSharedLib( }, }; const sub_compilation = try Compilation.create(comp.gpa, .{ - .zig_cache_directory = zig_cache_directory, + .local_cache_directory = zig_cache_directory, + .global_cache_directory = comp.global_cache_directory, .zig_lib_directory = comp.zig_lib_directory, .target = comp.getTarget(), .root_name = lib.name, diff --git a/src/introspect.zig b/src/introspect.zig index 067326ebb6..b75bf8f4b8 100644 --- a/src/introspect.zig +++ b/src/introspect.zig @@ -73,10 +73,3 @@ pub fn resolveGlobalCacheDir(allocator: *mem.Allocator) ![]u8 { return fs.getAppDataDir(allocator, appname); } - -pub fn openGlobalCacheDir() !fs.Dir { - var buf: [fs.MAX_PATH_BYTES]u8 = undefined; - var fba = std.heap.FixedBufferAllocator.init(&buf); - const path_name = try resolveGlobalCacheDir(&fba.allocator); - return fs.cwd().makeOpenPath(path_name, .{}); -} diff --git a/src/libcxx.zig b/src/libcxx.zig index e6ed35c560..4fbca42875 100644 --- a/src/libcxx.zig +++ b/src/libcxx.zig @@ -146,8 +146,8 @@ pub fn buildLibCXX(comp: *Compilation) !void { } const sub_compilation = try Compilation.create(comp.gpa, .{ - // TODO use the global cache directory here - .zig_cache_directory = comp.zig_cache_directory, + .local_cache_directory = comp.global_cache_directory, + .global_cache_directory = comp.global_cache_directory, .zig_lib_directory = comp.zig_lib_directory, .target = target, .root_name = root_name, @@ -255,8 +255,8 @@ pub fn buildLibCXXABI(comp: *Compilation) !void { } const sub_compilation = try Compilation.create(comp.gpa, .{ - // TODO use the global cache directory here - .zig_cache_directory = comp.zig_cache_directory, + .local_cache_directory = comp.global_cache_directory, + .global_cache_directory = comp.global_cache_directory, .zig_lib_directory = comp.zig_lib_directory, .target = target, .root_name = root_name, diff --git a/src/libunwind.zig b/src/libunwind.zig index e7562670e4..28445b7284 100644 --- a/src/libunwind.zig +++ b/src/libunwind.zig @@ -86,8 +86,8 @@ pub fn buildStaticLib(comp: *Compilation) !void { }; } const sub_compilation = try Compilation.create(comp.gpa, .{ - // TODO use the global cache directory here - .zig_cache_directory = comp.zig_cache_directory, + .local_cache_directory = comp.global_cache_directory, + .global_cache_directory = comp.global_cache_directory, .zig_lib_directory = comp.zig_lib_directory, .target = target, .root_name = root_name, diff --git a/src/main.zig b/src/main.zig index 3faa8b62a1..e88343e124 100644 --- a/src/main.zig +++ b/src/main.zig @@ -193,6 +193,8 @@ const usage_build_generic = \\ -femit-bin[=path] (default) output machine code \\ -fno-emit-bin Do not output machine code \\ --show-builtin Output the source of @import("builtin") then exit + \\ --cache-dir [path] Override the local cache directory + \\ --global-cache-dir [path] Override the global cache directory \\ \\Compile Options: \\ -target [name] -- see the targets command @@ -353,6 +355,8 @@ pub fn buildOutputType( var runtime_args_start: ?usize = null; var test_filter: ?[]const u8 = null; var test_name_prefix: ?[]const u8 = null; + var override_local_cache_dir: ?[]const u8 = null; + var override_global_cache_dir: ?[]const u8 = null; var system_libs = std.ArrayList([]const u8).init(gpa); defer system_libs.deinit(); @@ -535,6 +539,14 @@ pub fn buildOutputType( if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); i += 1; try test_exec_args.append(args[i]); + } else if (mem.eql(u8, arg, "--cache-dir")) { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); + i += 1; + override_local_cache_dir = args[i]; + } else if (mem.eql(u8, arg, "--global-cache-dir")) { + if (i + 1 >= args.len) fatal("expected parameter after {}", .{arg}); + i += 1; + override_global_cache_dir = args[i]; } else if (mem.eql(u8, arg, "--test-cmd-bin")) { try test_exec_args.append(null); } else if (mem.eql(u8, arg, "--test-evented-io")) { @@ -1093,7 +1105,10 @@ pub fn buildOutputType( const emit_bin_loc: ?Compilation.EmitLoc = switch (emit_bin) { .no => null, .yes_default_path => Compilation.EmitLoc{ - .directory = .{ .path = null, .handle = fs.cwd() }, + .directory = switch (arg_mode) { + .run, .zig_test => null, + else => .{ .path = null, .handle = fs.cwd() }, + }, .basename = try std.zig.binNameAlloc( arena, root_name, @@ -1198,26 +1213,53 @@ pub fn buildOutputType( }; } - const cache_parent_dir = if (root_pkg) |pkg| pkg.root_src_directory.handle else fs.cwd(); - var cache_dir = try cache_parent_dir.makeOpenPath("zig-cache", .{}); - defer cache_dir.close(); - const zig_cache_directory: Compilation.Directory = .{ - .handle = cache_dir, - .path = blk: { + var global_cache_directory: Compilation.Directory = l: { + const p = override_global_cache_dir orelse try introspect.resolveGlobalCacheDir(arena); + break :l .{ + .handle = try fs.cwd().makeOpenPath(p, .{}), + .path = p, + }; + }; + defer global_cache_directory.handle.close(); + + var cleanup_local_cache_dir: ?fs.Dir = null; + defer if (cleanup_local_cache_dir) |*dir| dir.close(); + + var local_cache_directory: Compilation.Directory = l: { + if (override_local_cache_dir) |local_cache_dir_path| { + const dir = try fs.cwd().makeOpenPath(local_cache_dir_path, .{}); + cleanup_local_cache_dir = dir; + break :l .{ + .handle = dir, + .path = local_cache_dir_path, + }; + } + if (arg_mode == .run) { + break :l global_cache_directory; + } + const cache_dir_path = blk: { if (root_pkg) |pkg| { if (pkg.root_src_directory.path) |p| { break :blk try fs.path.join(arena, &[_][]const u8{ p, "zig-cache" }); } } break :blk "zig-cache"; - }, + }; + const cache_parent_dir = if (root_pkg) |pkg| pkg.root_src_directory.handle else fs.cwd(); + const dir = try cache_parent_dir.makeOpenPath("zig-cache", .{}); + cleanup_local_cache_dir = dir; + break :l .{ + .handle = dir, + .path = cache_dir_path, + }; }; gimmeMoreOfThoseSweetSweetFileDescriptors(); const comp = Compilation.create(gpa, .{ .zig_lib_directory = zig_lib_directory, - .zig_cache_directory = zig_cache_directory, + .local_cache_directory = local_cache_directory, + .global_cache_directory = global_cache_directory, .root_name = root_name, .target = target_info.target, .is_native_os = cross_target.isNativeOs(), diff --git a/src/test.zig b/src/test.zig index 0f63c3f6a6..558fe3e95d 100644 --- a/src/test.zig +++ b/src/test.zig @@ -480,7 +480,8 @@ pub const TestContext = struct { .basename = bin_name, }; const comp = try Compilation.create(allocator, .{ - .zig_cache_directory = zig_cache_directory, + .local_cache_directory = zig_cache_directory, + .global_cache_directory = zig_cache_directory, .zig_lib_directory = zig_lib_directory, .rand = rand, .root_name = "test_case",