From 10132126972604c4636b148df27ab7fe9e50136b Mon Sep 17 00:00:00 2001 From: Eric Joldasov Date: Sat, 1 Oct 2022 22:58:09 +0600 Subject: [PATCH 1/5] cmake: separate installing and building (again) --- CMakeLists.txt | 21 ++++++++++----------- cmake/install.cmake | 24 ++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 11 deletions(-) create mode 100644 cmake/install.cmake diff --git a/CMakeLists.txt b/CMakeLists.txt index 7de04a9492..16e6776010 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1017,9 +1017,9 @@ if("${ZIG_EXECUTABLE}" STREQUAL "") WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" ) if (WIN32) - set(ZIG_EXECUTABLE "${zig2_BINARY_DIR}/zig2.exe") + set(ZIG_EXECUTABLE "${CMAKE_BINARY_DIR}/zig2.exe") else() - set(ZIG_EXECUTABLE "${zig2_BINARY_DIR}/zig2") + set(ZIG_EXECUTABLE "${CMAKE_BINARY_DIR}/zig2") endif() else() add_custom_command( @@ -1045,15 +1045,8 @@ elseif(MINGW) target_link_libraries(zig2 ntdll) endif() -# Dummy install command so that the "install" target is not missing. -# This is redundant from the "stage3" custom target below. -if(NOT ZIG_SKIP_INSTALL_LIB_FILES) - install(FILES "lib/compiler_rt.zig" DESTINATION "lib/zig") -endif() - -set(ZIG_INSTALL_ARGS "build" +set(ZIG_BUILD_ARGS "build" --zig-lib-dir "${CMAKE_SOURCE_DIR}/lib" - --prefix "${CMAKE_INSTALL_PREFIX}" "-Dconfig_h=${ZIG_CONFIG_H_OUT}" "-Denable-llvm" "-Denable-stage1" @@ -1067,8 +1060,14 @@ set(ZIG_INSTALL_ARGS "build" ) add_custom_target(stage3 ALL - COMMAND zig2 ${ZIG_INSTALL_ARGS} + COMMAND zig2 ${ZIG_BUILD_ARGS} DEPENDS zig2 COMMENT STATUS "Building stage3" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" ) + +install(CODE "set(ZIG_EXECUTABLE \"${ZIG_EXECUTABLE}\")") +install(CODE "set(ZIG_BUILD_ARGS \"${ZIG_BUILD_ARGS}\")") +install(CODE "set(CMAKE_INSTALL_PREFIX \"${CMAKE_INSTALL_PREFIX}\")") +install(CODE "set(CMAKE_SOURCE_DIR \"${CMAKE_SOURCE_DIR}\")") +install(SCRIPT "${CMAKE_SOURCE_DIR}/cmake/install.cmake") diff --git a/cmake/install.cmake b/cmake/install.cmake new file mode 100644 index 0000000000..f7b14c471e --- /dev/null +++ b/cmake/install.cmake @@ -0,0 +1,24 @@ +set(ZIG_INSTALL_ARGS ${ZIG_BUILD_ARGS} --prefix "${CMAKE_INSTALL_PREFIX}") +execute_process(COMMAND "${ZIG_EXECUTABLE}" ${ZIG_INSTALL_ARGS} WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" RESULT_VARIABLE _result) + +if(_result) + message("::") + message(":: ERROR: ${_result}") + message(":: (execute_process)") + + list(JOIN ZIG_INSTALL_ARGS " " s_INSTALL_LIBSTAGE2_ARGS) + message("::") + message(":: argv: ${ZIG_EXECUTABLE} ${s_INSTALL_LIBSTAGE2_ARGS}") + + set(_args ${ZIG_EXECUTABLE} ${ZIG_INSTALL_ARGS}) + list(LENGTH _args _len) + math(EXPR _len "${_len} - 1") + message("::") + foreach(_i RANGE 0 ${_len}) + list(GET _args ${_i} _arg) + message(":: argv[${_i}]: ${_arg}") + endforeach() + + message("::") + message(FATAL_ERROR) +endif() From caddbbc315e834241c6ae7e22434b574c589e5fc Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Sun, 16 Oct 2022 12:46:39 -0700 Subject: [PATCH 2/5] build: avoid compiling self-hosted twice build.zig: add a 'compile' step to compile the self-hosted compiler without installing it. Compilation: set cache mode to whole when using the LLVM backend and --enable-cache is passed. This makes `zig build` act the same as it does with stage1. Upside is that a second invocation of `zig build` on an unmodified source tree will avoid redoing the compilation again. Downside is that it will proliferate more garbage in the project-local cache (same as stage1). This can eventually be fixed when Zig's incremental compilation is more robust; we can go back to having LLVM use CacheMode.incremental and rely on it detecting no changes and avoiding doing the flush() step. --- CMakeLists.txt | 4 ++-- build.zig | 4 ++++ cmake/install.cmake | 7 +++++-- src/Compilation.zig | 13 ++++++++----- src/link/Elf.zig | 2 +- 5 files changed, 20 insertions(+), 10 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 16e6776010..dcc7e03d6b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1045,7 +1045,7 @@ elseif(MINGW) target_link_libraries(zig2 ntdll) endif() -set(ZIG_BUILD_ARGS "build" +set(ZIG_BUILD_ARGS --zig-lib-dir "${CMAKE_SOURCE_DIR}/lib" "-Dconfig_h=${ZIG_CONFIG_H_OUT}" "-Denable-llvm" @@ -1060,7 +1060,7 @@ set(ZIG_BUILD_ARGS "build" ) add_custom_target(stage3 ALL - COMMAND zig2 ${ZIG_BUILD_ARGS} + COMMAND zig2 build compile ${ZIG_BUILD_ARGS} DEPENDS zig2 COMMENT STATUS "Building stage3" WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" diff --git a/build.zig b/build.zig index aec623f642..2dd047a1e3 100644 --- a/build.zig +++ b/build.zig @@ -142,6 +142,10 @@ pub fn build(b: *Builder) !void { }; const exe = b.addExecutable("zig", main_file); + + const compile_step = b.step("compile", "Build the self-hosted compiler"); + compile_step.dependOn(&exe.step); + exe.stack_size = stack_size; exe.strip = strip; exe.sanitize_thread = sanitize_thread; diff --git a/cmake/install.cmake b/cmake/install.cmake index f7b14c471e..4a8356313e 100644 --- a/cmake/install.cmake +++ b/cmake/install.cmake @@ -1,5 +1,8 @@ -set(ZIG_INSTALL_ARGS ${ZIG_BUILD_ARGS} --prefix "${CMAKE_INSTALL_PREFIX}") -execute_process(COMMAND "${ZIG_EXECUTABLE}" ${ZIG_INSTALL_ARGS} WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" RESULT_VARIABLE _result) +set(ZIG_INSTALL_ARGS build ${ZIG_BUILD_ARGS} --prefix "${CMAKE_INSTALL_PREFIX}") +execute_process( + COMMAND "${ZIG_EXECUTABLE}" ${ZIG_INSTALL_ARGS} + WORKING_DIRECTORY "${CMAKE_SOURCE_DIR}" + RESULT_VARIABLE _result) if(_result) message("::") diff --git a/src/Compilation.zig b/src/Compilation.zig index 0ff9481875..659cfda9bd 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -1109,11 +1109,6 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { const use_stage1 = options.use_stage1 orelse false; - const cache_mode = if (use_stage1 and !options.disable_lld_caching) - CacheMode.whole - else - options.cache_mode; - // Make a decision on whether to use LLVM or our own backend. const use_llvm = build_options.have_llvm and blk: { if (options.use_llvm) |explicit| @@ -1154,6 +1149,14 @@ pub fn create(gpa: Allocator, options: InitOptions) !*Compilation { } } + // TODO: once we support incremental compilation for the LLVM backend via + // saving the LLVM module into a bitcode file and restoring it, along with + // compiler state, the second clause here can be removed so that incremental + // cache mode is used for LLVM backend too. We need some fuzz testing before + // that can be enabled. + const cache_mode = if ((use_stage1 and !options.disable_lld_caching) or + (use_llvm and !options.disable_lld_caching)) CacheMode.whole else options.cache_mode; + const tsan = options.want_tsan orelse false; // TSAN is implemented in C++ so it requires linking libc++. const link_libcpp = options.link_libcpp or tsan; diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 4e67c095c0..1a722c1dde 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -1282,7 +1282,7 @@ fn linkWithLLD(self: *Elf, comp: *Compilation, prog_node: *std.Progress.Node) !v // linked are in the hash that namespaces the directory we are outputting to. Therefore, // we must hash those now, and the resulting digest will form the "id" of the linking // job we are about to perform. - // After a successful link, we store the id in the metadata of a symlink named "id.txt" in + // After a successful link, we store the id in the metadata of a symlink named "lld.id" in // the artifact directory. So, now, we check if this symlink exists, and if it matches // our digest. If so, we can skip linking. Otherwise, we proceed with invoking LLD. const id_symlink_basename = "lld.id"; From c7772dd69455f6abf0ff2594f8b049acb078c921 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 17 Oct 2022 14:22:38 -0700 Subject: [PATCH 3/5] std.zig.system.NativePaths: avoid calling std.os.getenv on Windows --- lib/std/zig/system/NativePaths.zig | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/lib/std/zig/system/NativePaths.zig b/lib/std/zig/system/NativePaths.zig index b28e5e7602..2c4db3ec85 100644 --- a/lib/std/zig/system/NativePaths.zig +++ b/lib/std/zig/system/NativePaths.zig @@ -96,7 +96,7 @@ pub fn detect(allocator: Allocator, native_info: NativeTargetInfo) !NativePaths return self; } - if (comptime native_target.os.tag == .solaris) { + if (builtin.os.tag == .solaris) { try self.addLibDir("/usr/lib/64"); try self.addLibDir("/usr/local/lib/64"); try self.addLibDir("/lib/64"); @@ -107,7 +107,7 @@ pub fn detect(allocator: Allocator, native_info: NativeTargetInfo) !NativePaths return self; } - if (native_target.os.tag != .windows) { + if (builtin.os.tag != .windows) { const triple = try native_target.linuxTriple(allocator); defer allocator.free(triple); @@ -136,11 +136,10 @@ pub fn detect(allocator: Allocator, native_info: NativeTargetInfo) !NativePaths // libz.so.1 is in /lib/x86_64-linux-gnu (added here) try self.addLibDirFmt("/lib/{s}", .{triple}); - // NOTE: distro like guix doesn't use FHS, so it relies on envorinment - // variables (C_INCLUDE_PATH, CPLUS_INCLUDE_PATH and LIBRARY_PATH) to - // search for headers and libraries - // NOTE: we use os.getenv here since this part won't be executed on - // windows, to get rid of unnecessary error handling + // Distros like guix don't use FHS, so they rely on environment + // variables to search for headers and libraries. + // We use os.getenv here since this part won't be executed on + // windows, to get rid of unnecessary error handling. if (std.os.getenv("C_INCLUDE_PATH")) |c_include_path| { var it = mem.tokenize(u8, c_include_path, ":"); while (it.next()) |dir| { From 9ee4530b9b38805f88a1e3df4b4b74db240f05df Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 17 Oct 2022 16:53:17 -0700 Subject: [PATCH 4/5] std.os.windows.OpenFile: handle INVALID_HANDLE ntstatus --- lib/std/os/windows.zig | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/std/os/windows.zig b/lib/std/os/windows.zig index 081bff845c..d68a66ed87 100644 --- a/lib/std/os/windows.zig +++ b/lib/std/os/windows.zig @@ -134,6 +134,7 @@ pub fn OpenFile(sub_path_w: []const u16, options: OpenFileOptions) OpenError!HAN .OBJECT_NAME_COLLISION => return error.PathAlreadyExists, .FILE_IS_A_DIRECTORY => return error.IsDir, .NOT_A_DIRECTORY => return error.NotDir, + .INVALID_HANDLE => unreachable, else => return unexpectedStatus(rc), } } From cb635e084b0c36e3c7a6bf63f49f7e7e9918532d Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 17 Oct 2022 16:54:52 -0700 Subject: [PATCH 5/5] stage2: better handling of CacheMode.whole on Windows Windows gives AccessDenied if you delete a directory which contains open file handles. This could be triggered when using CacheMode.whole when cross compiling macho test binaries. --- src/Compilation.zig | 16 ++++++++++++++-- src/link.zig | 14 ++++++-------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index 659cfda9bd..52acf1f031 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -2390,9 +2390,21 @@ pub fn update(comp: *Compilation) !void { const o_sub_path = try std.fs.path.join(comp.gpa, &[_][]const u8{ "o", &digest }); defer comp.gpa.free(o_sub_path); + // Work around windows `AccessDenied` if any files within this directory are open + // by doing the makeExecutable/makeWritable dance. + const need_writable_dance = builtin.os.tag == .windows and comp.bin_file.file != null; + if (need_writable_dance) { + try comp.bin_file.makeExecutable(); + } + try comp.bin_file.renameTmpIntoCache(comp.local_cache_directory, tmp_dir_sub_path, o_sub_path); comp.wholeCacheModeSetBinFilePath(&digest); + // Has to be after the `wholeCacheModeSetBinFilePath` above. + if (need_writable_dance) { + try comp.bin_file.makeWritable(); + } + // This is intentionally sandwiched between renameTmpIntoCache() and writeManifest(). if (comp.bin_file.options.module) |module| { // We need to set the zig_cache_artifact_directory for -femit-asm, -femit-llvm-ir, @@ -3207,8 +3219,8 @@ fn processOneJob(comp: *Compilation, job: Job) !void { // TODO Surface more error details. comp.lockAndSetMiscFailure( .mingw_crt_file, - "unable to build mingw-w64 CRT file: {s}", - .{@errorName(err)}, + "unable to build mingw-w64 CRT file {s}: {s}", + .{ @tagName(crt_file), @errorName(err) }, ); }; }, diff --git a/src/link.zig b/src/link.zig index 0b6ffa8bf4..1b5ce2246b 100644 --- a/src/link.zig +++ b/src/link.zig @@ -403,10 +403,8 @@ pub const File = struct { try emit.directory.handle.copyFile(emit.sub_path, emit.directory.handle, emit.sub_path, .{}); } } - if (base.intermediary_basename == null) { - f.close(); - base.file = null; - } + f.close(); + base.file = null; }, .coff, .elf, .plan9, .wasm => if (base.file) |f| { if (base.intermediary_basename != null) { @@ -777,7 +775,7 @@ pub const File = struct { _ = base; while (true) { if (builtin.os.tag == .windows) { - // workaround windows `renameW` can't fail with `PathAlreadyExists` + // Work around windows `renameW` can't fail with `PathAlreadyExists` // See https://github.com/ziglang/zig/issues/8362 if (cache_directory.handle.access(o_sub_path, .{})) |_| { try cache_directory.handle.deleteTree(o_sub_path); @@ -791,9 +789,9 @@ pub const File = struct { tmp_dir_sub_path, cache_directory.handle, o_sub_path, - ) catch |err| switch (err) { - error.AccessDenied => unreachable, // We are most likely trying to move a dir with open handles to its resources - else => |e| return e, + ) catch |err| { + log.err("unable to rename cache dir {s} to {s}: {s}", .{ tmp_dir_sub_path, o_sub_path, @errorName(err) }); + return err; }; break; } else {