From ada19c498d6cb52dd8f0de71d42baf845cfadc21 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Mon, 28 Sep 2020 19:20:58 -0700 Subject: [PATCH] stage2: building DLL import lib files --- BRANCH_TODO | 2 +- src/Compilation.zig | 19 ++++ src/llvm.zig | 63 ++++++++++++ src/mingw.zig | 229 ++++++++++++++++++++++++++++++++++++++++++++ src/target.zig | 57 +++++++++++ src/zig_llvm.cpp | 2 +- src/zig_llvm.h | 4 +- 7 files changed, 372 insertions(+), 4 deletions(-) diff --git a/BRANCH_TODO b/BRANCH_TODO index d134b5c733..3f725e208c 100644 --- a/BRANCH_TODO +++ b/BRANCH_TODO @@ -1,7 +1,6 @@ * the have_foo flags that we get from stage1 have to be stored in the cache otherwise we get a different result for subsystem when we have a cached stage1 execution result. same deal with extern "foo" libraries used - * add jobs to build import libs for windows DLLs for explicitly linked libs * add jobs to build import libs for windows DLLs for extern "foo" functions used * MachO LLD linking * WASM LLD linking @@ -53,3 +52,4 @@ * make proposal about log levels * proposal for changing fs Z/W functions to be native paths and have a way to do native path string literals * proposal for block { break x; } + * generally look for the "TODO surface this as a real compile error message" and fix all that stuff diff --git a/src/Compilation.zig b/src/Compilation.zig index 00eb8c374d..3095ffab81 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -168,6 +168,9 @@ const Job = union(enum) { generate_builtin_zig: void, /// Use stage1 C++ code to compile zig code into an object file. stage1_module: void, + + /// The value is the index into `link.File.Options.system_libs`. + windows_import_lib: usize, }; pub const CObject = struct { @@ -868,6 +871,15 @@ pub fn create(gpa: *Allocator, options: InitOptions) !*Compilation { comp.work_queue.writeAssumeCapacity(&static_lib_jobs); comp.work_queue.writeItemAssumeCapacity(crt_job); } + // Generate Windows import libs. + if (comp.getTarget().os.tag == .windows) { + const count = comp.bin_file.options.system_libs.count(); + try comp.work_queue.ensureUnusedCapacity(count); + var i: usize = 0; + while (i < count) : (i += 1) { + comp.work_queue.writeItemAssumeCapacity(.{ .windows_import_lib = i }); + } + } if (comp.wantBuildLibUnwindFromSource()) { try comp.work_queue.writeItem(.{ .libunwind = {} }); } @@ -1233,6 +1245,13 @@ pub fn performAllTheWork(self: *Compilation) error{OutOfMemory}!void { fatal("unable to build mingw-w64 CRT file: {}", .{@errorName(err)}); }; }, + .windows_import_lib => |index| { + const link_lib = self.bin_file.options.system_libs.items()[index].key; + mingw.buildImportLib(self, link_lib) catch |err| { + // TODO Expose this as a normal compile error rather than crashing here. + fatal("unable to generate DLL import .lib file: {}", .{@errorName(err)}); + }; + }, .libunwind => { libunwind.buildStaticLib(self) catch |err| { // TODO Expose this as a normal compile error rather than crashing here. diff --git a/src/llvm.zig b/src/llvm.zig index ceefb62c5d..3aebf46b81 100644 --- a/src/llvm.zig +++ b/src/llvm.zig @@ -73,5 +73,68 @@ pub const OSType = extern enum(c_int) { Emscripten = 35, }; +pub const ArchType = extern enum(c_int) { + UnknownArch = 0, + arm = 1, + armeb = 2, + aarch64 = 3, + aarch64_be = 4, + aarch64_32 = 5, + arc = 6, + avr = 7, + bpfel = 8, + bpfeb = 9, + hexagon = 10, + mips = 11, + mipsel = 12, + mips64 = 13, + mips64el = 14, + msp430 = 15, + ppc = 16, + ppc64 = 17, + ppc64le = 18, + r600 = 19, + amdgcn = 20, + riscv32 = 21, + riscv64 = 22, + sparc = 23, + sparcv9 = 24, + sparcel = 25, + systemz = 26, + tce = 27, + tcele = 28, + thumb = 29, + thumbeb = 30, + x86 = 31, + x86_64 = 32, + xcore = 33, + nvptx = 34, + nvptx64 = 35, + le32 = 36, + le64 = 37, + amdil = 38, + amdil64 = 39, + hsail = 40, + hsail64 = 41, + spir = 42, + spir64 = 43, + kalimba = 44, + shave = 45, + lanai = 46, + wasm32 = 47, + wasm64 = 48, + renderscript32 = 49, + renderscript64 = 50, + ve = 51, +}; + pub const ParseCommandLineOptions = ZigLLVMParseCommandLineOptions; extern fn ZigLLVMParseCommandLineOptions(argc: usize, argv: [*]const [*:0]const u8) void; + +pub const WriteImportLibrary = ZigLLVMWriteImportLibrary; +extern fn ZigLLVMWriteImportLibrary( + def_path: [*:0]const u8, + arch: ArchType, + output_lib_path: [*c]const u8, + kill_at: bool, +) bool; diff --git a/src/mingw.zig b/src/mingw.zig index 2a2879de05..41a328c6f4 100644 --- a/src/mingw.zig +++ b/src/mingw.zig @@ -3,10 +3,12 @@ const Allocator = std.mem.Allocator; const mem = std.mem; const path = std.fs.path; const assert = std.debug.assert; +const log = std.log.scoped(.mingw); const target_util = @import("target.zig"); const Compilation = @import("Compilation.zig"); const build_options = @import("build_options"); +const Cache = @import("Cache.zig"); pub const CRTFile = enum { crt2_o, @@ -277,6 +279,233 @@ fn add_cc_args( }); } +pub fn buildImportLib(comp: *Compilation, lib_name: []const u8) !void { + var arena_allocator = std.heap.ArenaAllocator.init(comp.gpa); + defer arena_allocator.deinit(); + const arena = &arena_allocator.allocator; + + const def_file_path = findDef(comp, arena, lib_name) catch |err| switch (err) { + error.FileNotFound => { + log.debug("no {s}.def file available to make a DLL import {s}.lib", .{ lib_name, lib_name }); + // In this case we will end up putting foo.lib onto the linker line and letting the linker + // use its library paths to look for libraries and report any problems. + return; + }, + else => |e| return e, + }; + + // We need to invoke `zig clang` to use the preprocessor. + if (!build_options.have_llvm) return error.ZigCompilerNotBuiltWithLLVMExtensions; + const self_exe_path = comp.self_exe_path orelse return error.PreprocessorDisabled; + + const target = comp.getTarget(); + + var cache: Cache = .{ + .gpa = comp.gpa, + .manifest_dir = comp.cache_parent.manifest_dir, + }; + cache.hash.addBytes(build_options.version); + cache.hash.addOptionalBytes(comp.zig_lib_directory.path); + cache.hash.add(target.cpu.arch); + + var man = cache.obtain(); + defer man.deinit(); + + _ = try man.addFile(def_file_path, null); + + const final_lib_basename = try std.fmt.allocPrint(comp.gpa, "{s}.lib", .{lib_name}); + errdefer comp.gpa.free(final_lib_basename); + + if (try man.hit()) { + const digest = man.final(); + + try comp.crt_files.ensureCapacity(comp.gpa, comp.crt_files.count() + 1); + comp.crt_files.putAssumeCapacityNoClobber(final_lib_basename, .{ + .full_object_path = try comp.global_cache_directory.join(comp.gpa, &[_][]const u8{ + "o", &digest, final_lib_basename, + }), + .lock = man.toOwnedLock(), + }); + return; + } + + const digest = man.final(); + const o_sub_path = try std.fs.path.join(arena, &[_][]const u8{ "o", &digest }); + var o_dir = try comp.global_cache_directory.handle.makeOpenPath(o_sub_path, .{}); + defer o_dir.close(); + + const final_def_basename = try std.fmt.allocPrint(arena, "{s}.def", .{lib_name}); + const def_final_path = try comp.global_cache_directory.join(arena, &[_][]const u8{ + "o", &digest, final_def_basename, + }); + + const target_def_arg = switch (target.cpu.arch) { + .i386 => "-DDEF_I386", + .x86_64 => "-DDEF_X64", + .arm, .armeb => switch (target.cpu.arch.ptrBitWidth()) { + 32 => "-DDEF_ARM32", + 64 => "-DDEF_ARM64", + else => unreachable, + }, + else => unreachable, + }; + + const args = [_][]const u8{ + self_exe_path, + "clang", + "-x", + "c", + def_file_path, + "-Wp,-w", + "-undef", + "-P", + "-I", + try comp.zig_lib_directory.join(arena, &[_][]const u8{ "libc", "mingw", "def-include" }), + target_def_arg, + "-E", + "-o", + def_final_path, + }; + + if (comp.verbose_cc) { + Compilation.dump_argv(&args); + } + + const child = try std.ChildProcess.init(&args, arena); + defer child.deinit(); + + child.stdin_behavior = .Ignore; + child.stdout_behavior = .Pipe; + child.stderr_behavior = .Pipe; + + try child.spawn(); + + const stdout_reader = child.stdout.?.reader(); + const stderr_reader = child.stderr.?.reader(); + + // TODO https://github.com/ziglang/zig/issues/6343 + const stdout = try stdout_reader.readAllAlloc(arena, std.math.maxInt(u32)); + const stderr = try stderr_reader.readAllAlloc(arena, 10 * 1024 * 1024); + + const term = child.wait() catch |err| { + // TODO surface a proper error here + log.err("unable to spawn {}: {}", .{ args[0], @errorName(err) }); + return error.ClangPreprocessorFailed; + }; + + switch (term) { + .Exited => |code| { + if (code != 0) { + // TODO surface a proper error here + log.err("clang exited with code {d} and stderr: {s}", .{ code, stderr }); + return error.ClangPreprocessorFailed; + } + }, + else => { + // TODO surface a proper error here + log.err("clang terminated unexpectedly with stderr: {}", .{stderr}); + return error.ClangPreprocessorFailed; + }, + } + + const lib_final_path = try comp.global_cache_directory.join(comp.gpa, &[_][]const u8{ + "o", &digest, final_lib_basename, + }); + errdefer comp.gpa.free(lib_final_path); + + const llvm = @import("llvm.zig"); + const arch_type = @import("target.zig").archToLLVM(target.cpu.arch); + const def_final_path_z = try arena.dupeZ(u8, def_final_path); + const lib_final_path_z = try arena.dupeZ(u8, lib_final_path); + if (llvm.WriteImportLibrary(def_final_path_z.ptr, arch_type, lib_final_path_z.ptr, true)) { + // TODO surface a proper error here + log.err("unable to turn {s}.def into {s}.lib", .{ lib_name, lib_name }); + return error.WritingImportLibFailed; + } + + man.writeManifest() catch |err| { + log.warn("failed to write cache manifest for DLL import {s}.lib: {s}", .{ lib_name, @errorName(err) }); + }; + + try comp.crt_files.putNoClobber(comp.gpa, final_lib_basename, .{ + .full_object_path = lib_final_path, + .lock = man.toOwnedLock(), + }); +} + +/// This function body is verbose but all it does is test 3 different paths and see if a .def file exists. +fn findDef(comp: *Compilation, allocator: *Allocator, lib_name: []const u8) ![]u8 { + const target = comp.getTarget(); + + const lib_path = switch (target.cpu.arch) { + .i386 => "lib32", + .x86_64 => "lib64", + .arm, .armeb => switch (target.cpu.arch.ptrBitWidth()) { + 32 => "libarm32", + 64 => "libarm64", + else => unreachable, + }, + else => unreachable, + }; + + var override_path = std.ArrayList(u8).init(allocator); + defer override_path.deinit(); + + const s = path.sep_str; + + { + // Try the archtecture-specific path first. + const fmt_path = "libc" ++ s ++ "mingw" ++ s ++ "{s}" ++ s ++ "{s}.def"; + if (comp.zig_lib_directory.path) |p| { + try override_path.writer().print("{s}" ++ s ++ fmt_path, .{ p, lib_path, lib_name }); + } else { + try override_path.writer().print(fmt_path, .{ lib_path, lib_name }); + } + if (std.fs.cwd().access(override_path.items, .{})) |_| { + return override_path.toOwnedSlice(); + } else |err| switch (err) { + error.FileNotFound => {}, + else => |e| return e, + } + } + + { + // Try the generic version. + override_path.shrinkRetainingCapacity(0); + const fmt_path = "libc" ++ s ++ "mingw" ++ s ++ "lib-common" ++ s ++ "{s}.def"; + if (comp.zig_lib_directory.path) |p| { + try override_path.writer().print("{s}" ++ s ++ fmt_path, .{ p, lib_name }); + } else { + try override_path.writer().print(fmt_path, .{lib_name}); + } + if (std.fs.cwd().access(override_path.items, .{})) |_| { + return override_path.toOwnedSlice(); + } else |err| switch (err) { + error.FileNotFound => {}, + else => |e| return e, + } + } + + { + // Try the generic version and preprocess it. + override_path.shrinkRetainingCapacity(0); + const fmt_path = "libc" ++ s ++ "mingw" ++ s ++ "lib-common" ++ s ++ "{s}.def.in"; + if (comp.zig_lib_directory.path) |p| { + try override_path.writer().print("{s}" ++ s ++ fmt_path, .{ p, lib_name }); + } else { + try override_path.writer().print(fmt_path, .{lib_name}); + } + if (std.fs.cwd().access(override_path.items, .{})) |_| { + return override_path.toOwnedSlice(); + } else |err| switch (err) { + error.FileNotFound => {}, + else => |e| return e, + } + } + + return error.FileNotFound; +} + const mingw32_lib_deps = [_][]const u8{ "crt0_c.c", "dll_argv.c", diff --git a/src/target.zig b/src/target.zig index 9b5ea2a366..f6d0f41112 100644 --- a/src/target.zig +++ b/src/target.zig @@ -224,6 +224,63 @@ pub fn osToLLVM(os_tag: std.Target.Os.Tag) llvm.OSType { }; } +pub fn archToLLVM(arch_tag: std.Target.Cpu.Arch) llvm.ArchType { + return switch (arch_tag) { + .arm => .arm, + .armeb => .armeb, + .aarch64 => .aarch64, + .aarch64_be => .aarch64_be, + .aarch64_32 => .aarch64_32, + .arc => .arc, + .avr => .avr, + .bpfel => .bpfel, + .bpfeb => .bpfeb, + .hexagon => .hexagon, + .mips => .mips, + .mipsel => .mipsel, + .mips64 => .mips64, + .mips64el => .mips64el, + .msp430 => .msp430, + .powerpc => .ppc, + .powerpc64 => .ppc64, + .powerpc64le => .ppc64le, + .r600 => .r600, + .amdgcn => .amdgcn, + .riscv32 => .riscv32, + .riscv64 => .riscv64, + .sparc => .sparc, + .sparcv9 => .sparcv9, + .sparcel => .sparcel, + .s390x => .systemz, + .tce => .tce, + .tcele => .tcele, + .thumb => .thumb, + .thumbeb => .thumbeb, + .i386 => .x86, + .x86_64 => .x86_64, + .xcore => .xcore, + .nvptx => .nvptx, + .nvptx64 => .nvptx64, + .le32 => .le32, + .le64 => .le64, + .amdil => .amdil, + .amdil64 => .amdil64, + .hsail => .hsail, + .hsail64 => .hsail64, + .spir => .spir, + .spir64 => .spir64, + .kalimba => .kalimba, + .shave => .shave, + .lanai => .lanai, + .wasm32 => .wasm32, + .wasm64 => .wasm64, + .renderscript32 => .renderscript32, + .renderscript64 => .renderscript64, + .ve => .ve, + .spu_2 => .UnknownArch, + }; +} + fn eqlIgnoreCase(ignore_case: bool, a: []const u8, b: []const u8) bool { if (ignore_case) { return std.ascii.eqlIgnoreCase(a, b); diff --git a/src/zig_llvm.cpp b/src/zig_llvm.cpp index e5b9df625c..08823050ad 100644 --- a/src/zig_llvm.cpp +++ b/src/zig_llvm.cpp @@ -927,7 +927,7 @@ class MyOStream: public raw_ostream { }; bool ZigLLVMWriteImportLibrary(const char *def_path, const ZigLLVM_ArchType arch, - const char *output_lib_path, const bool kill_at) + const char *output_lib_path, bool kill_at) { COFF::MachineTypes machine = COFF::IMAGE_FILE_MACHINE_UNKNOWN; diff --git a/src/zig_llvm.h b/src/zig_llvm.h index 4de048a3d8..007d8afc1f 100644 --- a/src/zig_llvm.h +++ b/src/zig_llvm.h @@ -500,8 +500,8 @@ ZIG_EXTERN_C bool ZigLLDLink(enum ZigLLVM_ObjectFormatType oformat, const char * ZIG_EXTERN_C bool ZigLLVMWriteArchive(const char *archive_name, const char **file_names, size_t file_name_count, enum ZigLLVM_OSType os_type); -bool ZigLLVMWriteImportLibrary(const char *def_path, const enum ZigLLVM_ArchType arch, - const char *output_lib_path, const bool kill_at); +ZIG_EXTERN_C bool ZigLLVMWriteImportLibrary(const char *def_path, const enum ZigLLVM_ArchType arch, + const char *output_lib_path, bool kill_at); ZIG_EXTERN_C void ZigLLVMGetNativeTarget(enum ZigLLVM_ArchType *arch_type, enum ZigLLVM_VendorType *vendor_type, enum ZigLLVM_OSType *os_type, enum ZigLLVM_EnvironmentType *environ_type,