From 72f6c6e6345030392a3f8cac79862d58359f1e76 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 8 Dec 2020 19:57:15 -0700 Subject: [PATCH 1/3] invoke LLD as a child process rather than a library Closes #3825 --- src/link/Coff.zig | 95 ++++++++++++++++++++------------------------- src/link/Elf.zig | 96 ++++++++++++++++++++------------------------- src/link/MachO.zig | 97 +++++++++++++++++++++------------------------- src/link/Wasm.zig | 95 ++++++++++++++++++++------------------------- src/llvm.zig | 18 ++++----- src/main.zig | 39 +++++++++++++++++++ src/zig_llvm.cpp | 45 +++++++-------------- src/zig_llvm.h | 7 ++-- 8 files changed, 240 insertions(+), 252 deletions(-) diff --git a/src/link/Coff.zig b/src/link/Coff.zig index e89ace95c0..0d620ba12f 100644 --- a/src/link/Coff.zig +++ b/src/link/Coff.zig @@ -907,11 +907,10 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void { // Create an LLD command line and invoke it. var argv = std.ArrayList([]const u8).init(self.base.allocator); defer argv.deinit(); - // The first argument is ignored as LLD is called as a library, set it - // anyway to the correct LLD driver name for this target so that it's - // correctly printed when `verbose_link` is true. This is needed for some - // tools such as CMake when Zig is used as C compiler. - try argv.append("lld-link"); + // We will invoke ourselves as a child process to gain access to LLD. + // This is necessary because LLD does not behave properly as a library - + // it calls exit() and does not reset all global data between invocations. + try argv.appendSlice(&[_][]const u8{ comp.self_exe_path.?, "lld-link" }); try argv.append("-ERRORLIMIT:0"); try argv.append("-NOLOGO"); @@ -1149,45 +1148,51 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void { } if (self.base.options.verbose_link) { - Compilation.dump_argv(argv.items); + // Skip over our own name so that the LLD linker name is the first argv item. + Compilation.dump_argv(argv.items[1..]); } - const new_argv = try arena.allocSentinel(?[*:0]const u8, argv.items.len, null); - for (argv.items) |arg, i| { - new_argv[i] = try arena.dupeZ(u8, arg); + // Sadly, we must run LLD as a child process because it does not behave + // properly as a library. One exception is if we are running in passthrough + // mode, which means Clang / LLD should inherit stdio and are allowed to + // crash zig directly. + if (comp.clang_passthrough_mode) { + return @import("../main.zig").punt_to_lld(arena, argv.items); } - var stderr_context: LLDContext = .{ - .coff = self, - .data = std.ArrayList(u8).init(self.base.allocator), + const child = try std.ChildProcess.init(argv.items, arena); + defer child.deinit(); + + child.stdin_behavior = .Ignore; + child.stdout_behavior = .Ignore; + child.stderr_behavior = .Pipe; + + try child.spawn(); + + const stderr = try child.stderr.?.reader().readAllAlloc(arena, 10 * 1024 * 1024); + + const term = child.wait() catch |err| { + log.err("unable to spawn {s}: {s}", .{ argv.items[0], @errorName(err) }); + return error.UnableToSpawnSelf; }; - defer stderr_context.data.deinit(); - var stdout_context: LLDContext = .{ - .coff = self, - .data = std.ArrayList(u8).init(self.base.allocator), - }; - defer stdout_context.data.deinit(); - const llvm = @import("../llvm.zig"); - const ok = llvm.Link( - .COFF, - new_argv.ptr, - new_argv.len, - append_diagnostic, - @ptrToInt(&stdout_context), - @ptrToInt(&stderr_context), - ); - if (stderr_context.oom or stdout_context.oom) return error.OutOfMemory; - if (stdout_context.data.items.len != 0) { - std.log.warn("unexpected LLD stdout: {}", .{stdout_context.data.items}); + + switch (term) { + .Exited => |code| { + if (code != 0) { + // TODO parse this output and surface with the Compilation API rather than + // directly outputting to stderr here. + std.debug.print("{s}", .{stderr}); + return error.LLDReportedFailure; + } + }, + else => { + log.err("{s} terminated with stderr:\n{s}", .{ argv.items[0], stderr }); + return error.LLDCrashed; + }, } - if (!ok) { - // TODO parse this output and surface with the Compilation API rather than - // directly outputting to stderr here. - std.debug.print("{}", .{stderr_context.data.items}); - return error.LLDReportedFailure; - } - if (stderr_context.data.items.len != 0) { - std.log.warn("unexpected LLD stderr: {}", .{stderr_context.data.items}); + + if (stderr.len != 0) { + std.log.warn("unexpected LLD stderr:\n{s}", .{stderr}); } } @@ -1207,20 +1212,6 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void { } } -const LLDContext = struct { - data: std.ArrayList(u8), - coff: *Coff, - oom: bool = false, -}; - -fn append_diagnostic(context: usize, ptr: [*]const u8, len: usize) callconv(.C) void { - const lld_context = @intToPtr(*LLDContext, context); - const msg = ptr[0..len]; - lld_context.data.appendSlice(msg) catch |err| switch (err) { - error.OutOfMemory => lld_context.oom = true, - }; -} - pub fn getDeclVAddr(self: *Coff, decl: *const Module.Decl) u64 { return self.text_section_virtual_address + decl.link.coff.text_offset; } diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 1e95f3fa71..24219d3c44 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -1360,11 +1360,10 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { // Create an LLD command line and invoke it. var argv = std.ArrayList([]const u8).init(self.base.allocator); defer argv.deinit(); - // The first argument is ignored as LLD is called as a library, set it - // anyway to the correct LLD driver name for this target so that it's - // correctly printed when `verbose_link` is true. This is needed for some - // tools such as CMake when Zig is used as C compiler. - try argv.append("ld.lld"); + // We will invoke ourselves as a child process to gain access to LLD. + // This is necessary because LLD does not behave properly as a library - + // it calls exit() and does not reset all global data between invocations. + try argv.appendSlice(&[_][]const u8{ comp.self_exe_path.?, "ld.lld" }); if (is_obj) { try argv.append("-r"); } @@ -1628,46 +1627,51 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { } if (self.base.options.verbose_link) { - Compilation.dump_argv(argv.items); + // Skip over our own name so that the LLD linker name is the first argv item. + Compilation.dump_argv(argv.items[1..]); } - // Oh, snapplesauce! We need null terminated argv. - const new_argv = try arena.allocSentinel(?[*:0]const u8, argv.items.len, null); - for (argv.items) |arg, i| { - new_argv[i] = try arena.dupeZ(u8, arg); + // Sadly, we must run LLD as a child process because it does not behave + // properly as a library. One exception is if we are running in passthrough + // mode, which means Clang / LLD should inherit stdio and are allowed to + // crash zig directly. + if (comp.clang_passthrough_mode) { + return @import("../main.zig").punt_to_lld(arena, argv.items); } - var stderr_context: LLDContext = .{ - .elf = self, - .data = std.ArrayList(u8).init(self.base.allocator), + const child = try std.ChildProcess.init(argv.items, arena); + defer child.deinit(); + + child.stdin_behavior = .Ignore; + child.stdout_behavior = .Ignore; + child.stderr_behavior = .Pipe; + + try child.spawn(); + + const stderr = try child.stderr.?.reader().readAllAlloc(arena, 10 * 1024 * 1024); + + const term = child.wait() catch |err| { + log.err("unable to spawn {s}: {s}", .{ argv.items[0], @errorName(err) }); + return error.UnableToSpawnSelf; }; - defer stderr_context.data.deinit(); - var stdout_context: LLDContext = .{ - .elf = self, - .data = std.ArrayList(u8).init(self.base.allocator), - }; - defer stdout_context.data.deinit(); - const llvm = @import("../llvm.zig"); - const ok = llvm.Link( - .ELF, - new_argv.ptr, - new_argv.len, - append_diagnostic, - @ptrToInt(&stdout_context), - @ptrToInt(&stderr_context), - ); - if (stderr_context.oom or stdout_context.oom) return error.OutOfMemory; - if (stdout_context.data.items.len != 0) { - std.log.warn("unexpected LLD stdout: {}", .{stdout_context.data.items}); + + switch (term) { + .Exited => |code| { + if (code != 0) { + // TODO parse this output and surface with the Compilation API rather than + // directly outputting to stderr here. + std.debug.print("{s}", .{stderr}); + return error.LLDReportedFailure; + } + }, + else => { + log.err("{s} terminated with stderr:\n{s}", .{ argv.items[0], stderr }); + return error.LLDCrashed; + }, } - if (!ok) { - // TODO parse this output and surface with the Compilation API rather than - // directly outputting to stderr here. - std.debug.print("{}", .{stderr_context.data.items}); - return error.LLDReportedFailure; - } - if (stderr_context.data.items.len != 0) { - std.log.warn("unexpected LLD stderr: {}", .{stderr_context.data.items}); + + if (stderr.len != 0) { + std.log.warn("unexpected LLD stderr:\n{s}", .{stderr}); } if (!self.base.options.disable_lld_caching) { @@ -1686,20 +1690,6 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { } } -const LLDContext = struct { - data: std.ArrayList(u8), - elf: *Elf, - oom: bool = false, -}; - -fn append_diagnostic(context: usize, ptr: [*]const u8, len: usize) callconv(.C) void { - const lld_context = @intToPtr(*LLDContext, context); - const msg = ptr[0..len]; - lld_context.data.appendSlice(msg) catch |err| switch (err) { - error.OutOfMemory => lld_context.oom = true, - }; -} - fn writeDwarfAddrAssumeCapacity(self: *Elf, buf: *std.ArrayList(u8), addr: u64) void { const target_endian = self.base.options.target.cpu.arch.endian(); switch (self.ptr_width) { diff --git a/src/link/MachO.zig b/src/link/MachO.zig index 1904b2cf5e..35a6a7d24a 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -489,12 +489,10 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void { if (self.base.options.system_linker_hack) { try argv.append("ld"); } else { - // The first argument is ignored as LLD is called as a library, set - // it anyway to the correct LLD driver name for this target so that - // it's correctly printed when `verbose_link` is true. This is - // needed for some tools such as CMake when Zig is used as C - // compiler. - try argv.append("ld64"); + // We will invoke ourselves as a child process to gain access to LLD. + // This is necessary because LLD does not behave properly as a library - + // it calls exit() and does not reset all global data between invocations. + try argv.appendSlice(&[_][]const u8{ comp.self_exe_path.?, "ld64.lld" }); try argv.append("-error-limit"); try argv.append("0"); @@ -660,7 +658,9 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void { } if (self.base.options.verbose_link) { - Compilation.dump_argv(argv.items); + // Potentially skip over our own name so that the LLD linker name is the first argv item. + const adjusted_argv = if (self.base.options.system_linker_hack) argv.items else argv.items[1..]; + Compilation.dump_argv(adjusted_argv); } // TODO https://github.com/ziglang/zig/issues/6971 @@ -685,42 +685,47 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void { return error.LDReportedFailure; } } else { - const new_argv = try arena.allocSentinel(?[*:0]const u8, argv.items.len, null); - for (argv.items) |arg, i| { - new_argv[i] = try arena.dupeZ(u8, arg); + // Sadly, we must run LLD as a child process because it does not behave + // properly as a library. One exception is if we are running in passthrough + // mode, which means Clang / LLD should inherit stdio and are allowed to + // crash zig directly. + if (comp.clang_passthrough_mode) { + return @import("../main.zig").punt_to_lld(arena, argv.items); } - var stderr_context: LLDContext = .{ - .macho = self, - .data = std.ArrayList(u8).init(self.base.allocator), + const child = try std.ChildProcess.init(argv.items, arena); + defer child.deinit(); + + child.stdin_behavior = .Ignore; + child.stdout_behavior = .Ignore; + child.stderr_behavior = .Pipe; + + try child.spawn(); + + const stderr = try child.stderr.?.reader().readAllAlloc(arena, 10 * 1024 * 1024); + + const term = child.wait() catch |err| { + log.err("unable to spawn {s}: {s}", .{ argv.items[0], @errorName(err) }); + return error.UnableToSpawnSelf; }; - defer stderr_context.data.deinit(); - var stdout_context: LLDContext = .{ - .macho = self, - .data = std.ArrayList(u8).init(self.base.allocator), - }; - defer stdout_context.data.deinit(); - const llvm = @import("../llvm.zig"); - const ok = llvm.Link( - .MachO, - new_argv.ptr, - new_argv.len, - append_diagnostic, - @ptrToInt(&stdout_context), - @ptrToInt(&stderr_context), - ); - if (stderr_context.oom or stdout_context.oom) return error.OutOfMemory; - if (stdout_context.data.items.len != 0) { - std.log.warn("unexpected LLD stdout: {}", .{stdout_context.data.items}); + + switch (term) { + .Exited => |code| { + if (code != 0) { + // TODO parse this output and surface with the Compilation API rather than + // directly outputting to stderr here. + std.debug.print("{s}", .{stderr}); + return error.LLDReportedFailure; + } + }, + else => { + log.err("{s} terminated with stderr:\n{s}", .{ argv.items[0], stderr }); + return error.LLDCrashed; + }, } - if (!ok) { - // TODO parse this output and surface with the Compilation API rather than - // directly outputting to stderr here. - std.log.err("{}", .{stderr_context.data.items}); - return error.LLDReportedFailure; - } - if (stderr_context.data.items.len != 0) { - std.log.warn("unexpected LLD stderr: {}", .{stderr_context.data.items}); + + if (stderr.len != 0) { + std.log.warn("unexpected LLD stderr:\n{s}", .{stderr}); } // At this stage, LLD has done its job. It is time to patch the resultant @@ -785,20 +790,6 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void { } } -const LLDContext = struct { - data: std.ArrayList(u8), - macho: *MachO, - oom: bool = false, -}; - -fn append_diagnostic(context: usize, ptr: [*]const u8, len: usize) callconv(.C) void { - const lld_context = @intToPtr(*LLDContext, context); - const msg = ptr[0..len]; - lld_context.data.appendSlice(msg) catch |err| switch (err) { - error.OutOfMemory => lld_context.oom = true, - }; -} - fn darwinArchString(arch: std.Target.Cpu.Arch) []const u8 { return switch (arch) { .aarch64, .aarch64_be, .aarch64_32 => "arm64", diff --git a/src/link/Wasm.zig b/src/link/Wasm.zig index 51535d47ed..41f43a08fd 100644 --- a/src/link/Wasm.zig +++ b/src/link/Wasm.zig @@ -345,11 +345,10 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void { // Create an LLD command line and invoke it. var argv = std.ArrayList([]const u8).init(self.base.allocator); defer argv.deinit(); - // The first argument is ignored as LLD is called as a library, set it - // anyway to the correct LLD driver name for this target so that it's - // correctly printed when `verbose_link` is true. This is needed for some - // tools such as CMake when Zig is used as C compiler. - try argv.append("ld-wasm"); + // We will invoke ourselves as a child process to gain access to LLD. + // This is necessary because LLD does not behave properly as a library - + // it calls exit() and does not reset all global data between invocations. + try argv.appendSlice(&[_][]const u8{ comp.self_exe_path.?, "wasm-ld" }); if (is_obj) { try argv.append("-r"); } @@ -399,45 +398,51 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void { } if (self.base.options.verbose_link) { - Compilation.dump_argv(argv.items); + // Skip over our own name so that the LLD linker name is the first argv item. + Compilation.dump_argv(argv.items[1..]); } - const new_argv = try arena.allocSentinel(?[*:0]const u8, argv.items.len, null); - for (argv.items) |arg, i| { - new_argv[i] = try arena.dupeZ(u8, arg); + // Sadly, we must run LLD as a child process because it does not behave + // properly as a library. One exception is if we are running in passthrough + // mode, which means Clang / LLD should inherit stdio and are allowed to + // crash zig directly. + if (comp.clang_passthrough_mode) { + return @import("../main.zig").punt_to_lld(arena, argv.items); } - var stderr_context: LLDContext = .{ - .wasm = self, - .data = std.ArrayList(u8).init(self.base.allocator), + const child = try std.ChildProcess.init(argv.items, arena); + defer child.deinit(); + + child.stdin_behavior = .Ignore; + child.stdout_behavior = .Ignore; + child.stderr_behavior = .Pipe; + + try child.spawn(); + + const stderr = try child.stderr.?.reader().readAllAlloc(arena, 10 * 1024 * 1024); + + const term = child.wait() catch |err| { + log.err("unable to spawn {s}: {s}", .{ argv.items[0], @errorName(err) }); + return error.UnableToSpawnSelf; }; - defer stderr_context.data.deinit(); - var stdout_context: LLDContext = .{ - .wasm = self, - .data = std.ArrayList(u8).init(self.base.allocator), - }; - defer stdout_context.data.deinit(); - const llvm = @import("../llvm.zig"); - const ok = llvm.Link( - .Wasm, - new_argv.ptr, - new_argv.len, - append_diagnostic, - @ptrToInt(&stdout_context), - @ptrToInt(&stderr_context), - ); - if (stderr_context.oom or stdout_context.oom) return error.OutOfMemory; - if (stdout_context.data.items.len != 0) { - std.log.warn("unexpected LLD stdout: {}", .{stdout_context.data.items}); + + switch (term) { + .Exited => |code| { + if (code != 0) { + // TODO parse this output and surface with the Compilation API rather than + // directly outputting to stderr here. + std.debug.print("{s}", .{stderr}); + return error.LLDReportedFailure; + } + }, + else => { + log.err("{s} terminated with stderr:\n{s}", .{ argv.items[0], stderr }); + return error.LLDCrashed; + }, } - if (!ok) { - // TODO parse this output and surface with the Compilation API rather than - // directly outputting to stderr here. - std.debug.print("{}", .{stderr_context.data.items}); - return error.LLDReportedFailure; - } - if (stderr_context.data.items.len != 0) { - std.log.warn("unexpected LLD stderr: {}", .{stderr_context.data.items}); + + if (stderr.len != 0) { + std.log.warn("unexpected LLD stderr:\n{s}", .{stderr}); } if (!self.base.options.disable_lld_caching) { @@ -456,20 +461,6 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void { } } -const LLDContext = struct { - data: std.ArrayList(u8), - wasm: *Wasm, - oom: bool = false, -}; - -fn append_diagnostic(context: usize, ptr: [*]const u8, len: usize) callconv(.C) void { - const lld_context = @intToPtr(*LLDContext, context); - const msg = ptr[0..len]; - lld_context.data.appendSlice(msg) catch |err| switch (err) { - error.OutOfMemory => lld_context.oom = true, - }; -} - /// Get the current index of a given Decl in the function list /// TODO: we could maintain a hash map to potentially make this fn getFuncidx(self: Wasm, decl: *Module.Decl) ?u32 { diff --git a/src/llvm.zig b/src/llvm.zig index 3aebf46b81..865e989c89 100644 --- a/src/llvm.zig +++ b/src/llvm.zig @@ -1,15 +1,15 @@ //! We do this instead of @cImport because the self-hosted compiler is easier //! to bootstrap if it does not depend on translate-c. -pub const Link = ZigLLDLink; -extern fn ZigLLDLink( - oformat: ObjectFormatType, - args: [*:null]const ?[*:0]const u8, - arg_count: usize, - append_diagnostic: fn (context: usize, ptr: [*]const u8, len: usize) callconv(.C) void, - context_stdout: usize, - context_stderr: usize, -) bool; +extern fn ZigLLDLinkCOFF(argc: c_int, argv: [*:null]const ?[*:0]const u8, can_exit_early: bool) c_int; +extern fn ZigLLDLinkELF(argc: c_int, argv: [*:null]const ?[*:0]const u8, can_exit_early: bool) c_int; +extern fn ZigLLDLinkMachO(argc: c_int, argv: [*:null]const ?[*:0]const u8, can_exit_early: bool) c_int; +extern fn ZigLLDLinkWasm(argc: c_int, argv: [*:null]const ?[*:0]const u8, can_exit_early: bool) c_int; + +pub const LinkCOFF = ZigLLDLinkCOFF; +pub const LinkELF = ZigLLDLinkELF; +pub const LinkMachO = ZigLLDLinkMachO; +pub const LinkWasm = ZigLLDLinkWasm; pub const ObjectFormatType = extern enum(c_int) { Unknown, diff --git a/src/main.zig b/src/main.zig index 48232e6816..62732d252b 100644 --- a/src/main.zig +++ b/src/main.zig @@ -176,6 +176,12 @@ pub fn mainArgs(gpa: *Allocator, arena: *Allocator, args: []const []const u8) !v mem.eql(u8, cmd, "-cc1") or mem.eql(u8, cmd, "-cc1as")) { return punt_to_clang(arena, args); + } else if (mem.eql(u8, cmd, "ld.lld") or + mem.eql(u8, cmd, "ld64.lld") or + mem.eql(u8, cmd, "lld-link") or + mem.eql(u8, cmd, "wasm-ld")) + { + return punt_to_lld(arena, args); } else if (mem.eql(u8, cmd, "build")) { return cmdBuild(gpa, arena, cmd_args); } else if (mem.eql(u8, cmd, "fmt")) { @@ -2819,6 +2825,39 @@ fn punt_to_clang(arena: *Allocator, args: []const []const u8) error{OutOfMemory} process.exit(@bitCast(u8, @truncate(i8, exit_code))); } +/// The first argument determines which backend is invoked. The options are: +/// * `ld.lld` - ELF +/// * `ld64.lld` - Mach-O +/// * `lld-link` - COFF +/// * `wasm-ld` - WebAssembly +/// TODO https://github.com/ziglang/zig/issues/3257 +pub fn punt_to_lld(arena: *Allocator, args: []const []const u8) error{OutOfMemory} { + if (!build_options.have_llvm) + fatal("`zig {s}` unavailable: compiler built without LLVM extensions", .{args[0]}); + // Convert the args to the format LLD expects. + // We subtract 1 to shave off the zig binary from args[0]. + const argv = try arena.allocSentinel(?[*:0]const u8, args.len - 1, null); + for (args[1..]) |arg, i| { + argv[i] = try arena.dupeZ(u8, arg); // TODO If there was an argsAllocZ we could avoid this allocation. + } + const exit_code = rc: { + const llvm = @import("llvm.zig"); + const argc = @intCast(c_int, argv.len); + if (mem.eql(u8, args[1], "ld.lld")) { + break :rc llvm.LinkELF(argc, argv.ptr, true); + } else if (mem.eql(u8, args[1], "ld64.lld")) { + break :rc llvm.LinkMachO(argc, argv.ptr, true); + } else if (mem.eql(u8, args[1], "lld-link")) { + break :rc llvm.LinkCOFF(argc, argv.ptr, true); + } else if (mem.eql(u8, args[1], "wasm-ld")) { + break :rc llvm.LinkWasm(argc, argv.ptr, true); + } else { + unreachable; + } + }; + process.exit(@bitCast(u8, @truncate(i8, exit_code))); +} + const clang_args = @import("clang_options.zig").list; pub const ClangArgIterator = struct { diff --git a/src/zig_llvm.cpp b/src/zig_llvm.cpp index d8eeaf7fae..9b1ab71e9a 100644 --- a/src/zig_llvm.cpp +++ b/src/zig_llvm.cpp @@ -1056,39 +1056,24 @@ bool ZigLLVMWriteArchive(const char *archive_name, const char **file_names, size return false; } +int ZigLLDLinkCOFF(int argc, const char **argv, bool can_exit_early) { + std::vector args(argv, argv + argc); + return lld::coff::link(args, can_exit_early, llvm::outs(), llvm::errs()); +} -bool ZigLLDLink(ZigLLVM_ObjectFormatType oformat, const char **args, size_t arg_count, - void (*append_diagnostic)(void *, const char *, size_t), - void *context_stdout, void *context_stderr) -{ - ArrayRef array_ref_args(args, arg_count); +int ZigLLDLinkELF(int argc, const char **argv, bool can_exit_early) { + std::vector args(argv, argv + argc); + return lld::elf::link(args, can_exit_early, llvm::outs(), llvm::errs()); +} - MyOStream diag_stdout(append_diagnostic, context_stdout); - MyOStream diag_stderr(append_diagnostic, context_stderr); +int ZigLLDLinkMachO(int argc, const char **argv, bool can_exit_early) { + std::vector args(argv, argv + argc); + return lld::mach_o::link(args, can_exit_early, llvm::outs(), llvm::errs()); +} - switch (oformat) { - case ZigLLVM_UnknownObjectFormat: - case ZigLLVM_XCOFF: - assert(false); // unreachable - break; - - case ZigLLVM_COFF: - return lld::coff::link(array_ref_args, false, diag_stdout, diag_stderr); - - case ZigLLVM_ELF: - return lld::elf::link(array_ref_args, false, diag_stdout, diag_stderr); - - case ZigLLVM_MachO: - return lld::mach_o::link(array_ref_args, false, diag_stdout, diag_stderr); - - case ZigLLVM_Wasm: - return lld::wasm::link(array_ref_args, false, diag_stdout, diag_stderr); - - default: - break; - } - assert(false); // unreachable - abort(); +int ZigLLDLinkWasm(int argc, const char **argv, bool can_exit_early) { + std::vector args(argv, argv + argc); + return lld::wasm::link(args, can_exit_early, llvm::outs(), llvm::errs()); } static AtomicRMWInst::BinOp toLLVMRMWBinOp(enum ZigLLVM_AtomicRMWBinOp BinOp) { diff --git a/src/zig_llvm.h b/src/zig_llvm.h index 24ea533954..4e57af7933 100644 --- a/src/zig_llvm.h +++ b/src/zig_llvm.h @@ -507,9 +507,10 @@ ZIG_EXTERN_C const char *ZigLLVMGetVendorTypeName(enum ZigLLVM_VendorType vendor ZIG_EXTERN_C const char *ZigLLVMGetOSTypeName(enum ZigLLVM_OSType os); ZIG_EXTERN_C const char *ZigLLVMGetEnvironmentTypeName(enum ZigLLVM_EnvironmentType abi); -ZIG_EXTERN_C bool ZigLLDLink(enum ZigLLVM_ObjectFormatType oformat, const char **args, size_t arg_count, - void (*append_diagnostic)(void *, const char *, size_t), - void *context_stdout, void *context_stderr); +ZIG_EXTERN_C int ZigLLDLinkCOFF(int argc, const char **argv, bool can_exit_early); +ZIG_EXTERN_C int ZigLLDLinkELF(int argc, const char **argv, bool can_exit_early); +ZIG_EXTERN_C int ZigLLDLinkMachO(int argc, const char **argv, bool can_exit_early); +ZIG_EXTERN_C int ZigLLDLinkWasm(int argc, const char **argv, bool can_exit_early); ZIG_EXTERN_C bool ZigLLVMWriteArchive(const char *archive_name, const char **file_names, size_t file_name_count, enum ZigLLVM_OSType os_type); From 4592fd26b937d8c7ff91295408d8377c1c13cf53 Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 8 Dec 2020 21:54:46 -0700 Subject: [PATCH 2/3] add std.testing.expectStringEndsWith --- lib/std/testing.zig | 21 +++++++++++++++++++++ test/cli.zig | 4 ++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/lib/std/testing.zig b/lib/std/testing.zig index 8ab4e802ab..69df01190d 100644 --- a/lib/std/testing.zig +++ b/lib/std/testing.zig @@ -247,6 +247,7 @@ test "expectWithinEpsilon" { /// This function is intended to be used only in tests. When the two slices are not /// equal, prints diagnostics to stderr to show exactly how they are not equal, /// then aborts. +/// If your inputs are UTF-8 encoded strings, consider calling `expectEqualStrings` instead. pub fn expectEqualSlices(comptime T: type, expected: []const T, actual: []const T) void { // TODO better printing of the difference // If the arrays are small enough we could print the whole thing @@ -368,6 +369,26 @@ pub fn expectEqualStrings(expected: []const u8, actual: []const u8) void { } } +pub fn expectStringEndsWith(actual: []const u8, expected_ends_with: []const u8) void { + if (std.mem.endsWith(u8, actual, expected_ends_with)) + return; + + const shortened_actual = if (actual.len >= expected_ends_with.len) + actual[0..expected_ends_with.len] + else + actual; + + print("\n====== expected to end with: =========\n", .{}); + printWithVisibleNewlines(expected_ends_with); + print("\n====== instead ended with: ===========\n", .{}); + printWithVisibleNewlines(shortened_actual); + print("\n========= full output: ==============\n", .{}); + printWithVisibleNewlines(actual); + print("\n======================================\n", .{}); + + @panic("test failure"); +} + fn printIndicatorLine(source: []const u8, indicator_index: usize) void { const line_begin_index = if (std.mem.lastIndexOfScalar(u8, source[0..indicator_index], '\n')) |line_begin| line_begin + 1 diff --git a/test/cli.zig b/test/cli.zig index 284a184dc6..33dbc2d62b 100644 --- a/test/cli.zig +++ b/test/cli.zig @@ -92,13 +92,13 @@ fn exec(cwd: []const u8, expect_0: bool, argv: []const []const u8) !ChildProcess fn testZigInitLib(zig_exe: []const u8, dir_path: []const u8) !void { _ = try exec(dir_path, true, &[_][]const u8{ zig_exe, "init-lib" }); const test_result = try exec(dir_path, true, &[_][]const u8{ zig_exe, "build", "test" }); - testing.expect(std.mem.endsWith(u8, test_result.stderr, "All 1 tests passed.\n")); + testing.expectStringEndsWith(test_result.stderr, "All 1 tests passed.\n"); } fn testZigInitExe(zig_exe: []const u8, dir_path: []const u8) !void { _ = try exec(dir_path, true, &[_][]const u8{ zig_exe, "init-exe" }); const run_result = try exec(dir_path, true, &[_][]const u8{ zig_exe, "build", "run" }); - testing.expect(std.mem.eql(u8, run_result.stderr, "info: All your codebase are belong to us.\n")); + testing.expectEqualStrings("info: All your codebase are belong to us.\n", run_result.stderr); } fn testGodboltApi(zig_exe: []const u8, dir_path: []const u8) anyerror!void { From 7dd4afb224f4ca747b8eb462c28337ce9a63d38c Mon Sep 17 00:00:00 2001 From: Andrew Kelley Date: Tue, 8 Dec 2020 22:37:01 -0700 Subject: [PATCH 3/3] stage2: link: properly implement passthrough mode for LLD child proc passthrough mode does not mean always exit - it just means to pass through stdio and exit if the child process exits, without doing any special error reporting. --- src/Compilation.zig | 2 +- src/link/Coff.zig | 78 ++++++++++++++++++++++++++------------------- src/link/Elf.zig | 78 ++++++++++++++++++++++++++------------------- src/link/MachO.zig | 78 ++++++++++++++++++++++++++------------------- src/link/Wasm.zig | 78 ++++++++++++++++++++++++++------------------- 5 files changed, 185 insertions(+), 129 deletions(-) diff --git a/src/Compilation.zig b/src/Compilation.zig index fbccb21557..572dc1e0d7 100644 --- a/src/Compilation.zig +++ b/src/Compilation.zig @@ -1804,7 +1804,7 @@ fn updateCObject(comp: *Compilation, c_object: *CObject, c_comp_progress_node: * if (comp.clang_preprocessor_mode == .stdout) std.process.exit(0); }, - else => std.process.exit(1), + else => std.process.abort(), } } else { child.stdin_behavior = .Ignore; diff --git a/src/link/Coff.zig b/src/link/Coff.zig index 0d620ba12f..b5eb645e5c 100644 --- a/src/link/Coff.zig +++ b/src/link/Coff.zig @@ -1153,46 +1153,60 @@ fn linkWithLLD(self: *Coff, comp: *Compilation) !void { } // Sadly, we must run LLD as a child process because it does not behave - // properly as a library. One exception is if we are running in passthrough - // mode, which means Clang / LLD should inherit stdio and are allowed to - // crash zig directly. - if (comp.clang_passthrough_mode) { - return @import("../main.zig").punt_to_lld(arena, argv.items); - } - + // properly as a library. const child = try std.ChildProcess.init(argv.items, arena); defer child.deinit(); - child.stdin_behavior = .Ignore; - child.stdout_behavior = .Ignore; - child.stderr_behavior = .Pipe; + if (comp.clang_passthrough_mode) { + child.stdin_behavior = .Inherit; + child.stdout_behavior = .Inherit; + child.stderr_behavior = .Inherit; - try child.spawn(); + const term = child.spawnAndWait() catch |err| { + log.err("unable to spawn {s}: {s}", .{ argv.items[0], @errorName(err) }); + return error.UnableToSpawnSelf; + }; + switch (term) { + .Exited => |code| { + if (code != 0) { + // TODO https://github.com/ziglang/zig/issues/6342 + std.process.exit(1); + } + }, + else => std.process.abort(), + } + } else { + child.stdin_behavior = .Ignore; + child.stdout_behavior = .Ignore; + child.stderr_behavior = .Pipe; - const stderr = try child.stderr.?.reader().readAllAlloc(arena, 10 * 1024 * 1024); + try child.spawn(); - const term = child.wait() catch |err| { - log.err("unable to spawn {s}: {s}", .{ argv.items[0], @errorName(err) }); - return error.UnableToSpawnSelf; - }; + const stderr = try child.stderr.?.reader().readAllAlloc(arena, 10 * 1024 * 1024); - switch (term) { - .Exited => |code| { - if (code != 0) { - // TODO parse this output and surface with the Compilation API rather than - // directly outputting to stderr here. - std.debug.print("{s}", .{stderr}); - return error.LLDReportedFailure; - } - }, - else => { - log.err("{s} terminated with stderr:\n{s}", .{ argv.items[0], stderr }); - return error.LLDCrashed; - }, - } + const term = child.wait() catch |err| { + log.err("unable to spawn {s}: {s}", .{ argv.items[0], @errorName(err) }); + return error.UnableToSpawnSelf; + }; - if (stderr.len != 0) { - std.log.warn("unexpected LLD stderr:\n{s}", .{stderr}); + switch (term) { + .Exited => |code| { + if (code != 0) { + // TODO parse this output and surface with the Compilation API rather than + // directly outputting to stderr here. + std.debug.print("{s}", .{stderr}); + return error.LLDReportedFailure; + } + }, + else => { + log.err("{s} terminated with stderr:\n{s}", .{ argv.items[0], stderr }); + return error.LLDCrashed; + }, + } + + if (stderr.len != 0) { + std.log.warn("unexpected LLD stderr:\n{s}", .{stderr}); + } } } diff --git a/src/link/Elf.zig b/src/link/Elf.zig index 24219d3c44..ba1a9f824d 100644 --- a/src/link/Elf.zig +++ b/src/link/Elf.zig @@ -1632,46 +1632,60 @@ fn linkWithLLD(self: *Elf, comp: *Compilation) !void { } // Sadly, we must run LLD as a child process because it does not behave - // properly as a library. One exception is if we are running in passthrough - // mode, which means Clang / LLD should inherit stdio and are allowed to - // crash zig directly. - if (comp.clang_passthrough_mode) { - return @import("../main.zig").punt_to_lld(arena, argv.items); - } - + // properly as a library. const child = try std.ChildProcess.init(argv.items, arena); defer child.deinit(); - child.stdin_behavior = .Ignore; - child.stdout_behavior = .Ignore; - child.stderr_behavior = .Pipe; + if (comp.clang_passthrough_mode) { + child.stdin_behavior = .Inherit; + child.stdout_behavior = .Inherit; + child.stderr_behavior = .Inherit; - try child.spawn(); + const term = child.spawnAndWait() catch |err| { + log.err("unable to spawn {s}: {s}", .{ argv.items[0], @errorName(err) }); + return error.UnableToSpawnSelf; + }; + switch (term) { + .Exited => |code| { + if (code != 0) { + // TODO https://github.com/ziglang/zig/issues/6342 + std.process.exit(1); + } + }, + else => std.process.abort(), + } + } else { + child.stdin_behavior = .Ignore; + child.stdout_behavior = .Ignore; + child.stderr_behavior = .Pipe; - const stderr = try child.stderr.?.reader().readAllAlloc(arena, 10 * 1024 * 1024); + try child.spawn(); - const term = child.wait() catch |err| { - log.err("unable to spawn {s}: {s}", .{ argv.items[0], @errorName(err) }); - return error.UnableToSpawnSelf; - }; + const stderr = try child.stderr.?.reader().readAllAlloc(arena, 10 * 1024 * 1024); - switch (term) { - .Exited => |code| { - if (code != 0) { - // TODO parse this output and surface with the Compilation API rather than - // directly outputting to stderr here. - std.debug.print("{s}", .{stderr}); - return error.LLDReportedFailure; - } - }, - else => { - log.err("{s} terminated with stderr:\n{s}", .{ argv.items[0], stderr }); - return error.LLDCrashed; - }, - } + const term = child.wait() catch |err| { + log.err("unable to spawn {s}: {s}", .{ argv.items[0], @errorName(err) }); + return error.UnableToSpawnSelf; + }; - if (stderr.len != 0) { - std.log.warn("unexpected LLD stderr:\n{s}", .{stderr}); + switch (term) { + .Exited => |code| { + if (code != 0) { + // TODO parse this output and surface with the Compilation API rather than + // directly outputting to stderr here. + std.debug.print("{s}", .{stderr}); + return error.LLDReportedFailure; + } + }, + else => { + log.err("{s} terminated with stderr:\n{s}", .{ argv.items[0], stderr }); + return error.LLDCrashed; + }, + } + + if (stderr.len != 0) { + std.log.warn("unexpected LLD stderr:\n{s}", .{stderr}); + } } if (!self.base.options.disable_lld_caching) { diff --git a/src/link/MachO.zig b/src/link/MachO.zig index 35a6a7d24a..0767ad149f 100644 --- a/src/link/MachO.zig +++ b/src/link/MachO.zig @@ -686,46 +686,60 @@ fn linkWithLLD(self: *MachO, comp: *Compilation) !void { } } else { // Sadly, we must run LLD as a child process because it does not behave - // properly as a library. One exception is if we are running in passthrough - // mode, which means Clang / LLD should inherit stdio and are allowed to - // crash zig directly. - if (comp.clang_passthrough_mode) { - return @import("../main.zig").punt_to_lld(arena, argv.items); - } - + // properly as a library. const child = try std.ChildProcess.init(argv.items, arena); defer child.deinit(); - child.stdin_behavior = .Ignore; - child.stdout_behavior = .Ignore; - child.stderr_behavior = .Pipe; + if (comp.clang_passthrough_mode) { + child.stdin_behavior = .Inherit; + child.stdout_behavior = .Inherit; + child.stderr_behavior = .Inherit; - try child.spawn(); + const term = child.spawnAndWait() catch |err| { + log.err("unable to spawn {s}: {s}", .{ argv.items[0], @errorName(err) }); + return error.UnableToSpawnSelf; + }; + switch (term) { + .Exited => |code| { + if (code != 0) { + // TODO https://github.com/ziglang/zig/issues/6342 + std.process.exit(1); + } + }, + else => std.process.abort(), + } + } else { + child.stdin_behavior = .Ignore; + child.stdout_behavior = .Ignore; + child.stderr_behavior = .Pipe; - const stderr = try child.stderr.?.reader().readAllAlloc(arena, 10 * 1024 * 1024); + try child.spawn(); - const term = child.wait() catch |err| { - log.err("unable to spawn {s}: {s}", .{ argv.items[0], @errorName(err) }); - return error.UnableToSpawnSelf; - }; + const stderr = try child.stderr.?.reader().readAllAlloc(arena, 10 * 1024 * 1024); - switch (term) { - .Exited => |code| { - if (code != 0) { - // TODO parse this output and surface with the Compilation API rather than - // directly outputting to stderr here. - std.debug.print("{s}", .{stderr}); - return error.LLDReportedFailure; - } - }, - else => { - log.err("{s} terminated with stderr:\n{s}", .{ argv.items[0], stderr }); - return error.LLDCrashed; - }, - } + const term = child.wait() catch |err| { + log.err("unable to spawn {s}: {s}", .{ argv.items[0], @errorName(err) }); + return error.UnableToSpawnSelf; + }; - if (stderr.len != 0) { - std.log.warn("unexpected LLD stderr:\n{s}", .{stderr}); + switch (term) { + .Exited => |code| { + if (code != 0) { + // TODO parse this output and surface with the Compilation API rather than + // directly outputting to stderr here. + std.debug.print("{s}", .{stderr}); + return error.LLDReportedFailure; + } + }, + else => { + log.err("{s} terminated with stderr:\n{s}", .{ argv.items[0], stderr }); + return error.LLDCrashed; + }, + } + + if (stderr.len != 0) { + std.log.warn("unexpected LLD stderr:\n{s}", .{stderr}); + } } // At this stage, LLD has done its job. It is time to patch the resultant diff --git a/src/link/Wasm.zig b/src/link/Wasm.zig index 41f43a08fd..cc57aee03d 100644 --- a/src/link/Wasm.zig +++ b/src/link/Wasm.zig @@ -403,46 +403,60 @@ fn linkWithLLD(self: *Wasm, comp: *Compilation) !void { } // Sadly, we must run LLD as a child process because it does not behave - // properly as a library. One exception is if we are running in passthrough - // mode, which means Clang / LLD should inherit stdio and are allowed to - // crash zig directly. - if (comp.clang_passthrough_mode) { - return @import("../main.zig").punt_to_lld(arena, argv.items); - } - + // properly as a library. const child = try std.ChildProcess.init(argv.items, arena); defer child.deinit(); - child.stdin_behavior = .Ignore; - child.stdout_behavior = .Ignore; - child.stderr_behavior = .Pipe; + if (comp.clang_passthrough_mode) { + child.stdin_behavior = .Inherit; + child.stdout_behavior = .Inherit; + child.stderr_behavior = .Inherit; - try child.spawn(); + const term = child.spawnAndWait() catch |err| { + log.err("unable to spawn {s}: {s}", .{ argv.items[0], @errorName(err) }); + return error.UnableToSpawnSelf; + }; + switch (term) { + .Exited => |code| { + if (code != 0) { + // TODO https://github.com/ziglang/zig/issues/6342 + std.process.exit(1); + } + }, + else => std.process.abort(), + } + } else { + child.stdin_behavior = .Ignore; + child.stdout_behavior = .Ignore; + child.stderr_behavior = .Pipe; - const stderr = try child.stderr.?.reader().readAllAlloc(arena, 10 * 1024 * 1024); + try child.spawn(); - const term = child.wait() catch |err| { - log.err("unable to spawn {s}: {s}", .{ argv.items[0], @errorName(err) }); - return error.UnableToSpawnSelf; - }; + const stderr = try child.stderr.?.reader().readAllAlloc(arena, 10 * 1024 * 1024); - switch (term) { - .Exited => |code| { - if (code != 0) { - // TODO parse this output and surface with the Compilation API rather than - // directly outputting to stderr here. - std.debug.print("{s}", .{stderr}); - return error.LLDReportedFailure; - } - }, - else => { - log.err("{s} terminated with stderr:\n{s}", .{ argv.items[0], stderr }); - return error.LLDCrashed; - }, - } + const term = child.wait() catch |err| { + log.err("unable to spawn {s}: {s}", .{ argv.items[0], @errorName(err) }); + return error.UnableToSpawnSelf; + }; - if (stderr.len != 0) { - std.log.warn("unexpected LLD stderr:\n{s}", .{stderr}); + switch (term) { + .Exited => |code| { + if (code != 0) { + // TODO parse this output and surface with the Compilation API rather than + // directly outputting to stderr here. + std.debug.print("{s}", .{stderr}); + return error.LLDReportedFailure; + } + }, + else => { + log.err("{s} terminated with stderr:\n{s}", .{ argv.items[0], stderr }); + return error.LLDCrashed; + }, + } + + if (stderr.len != 0) { + std.log.warn("unexpected LLD stderr:\n{s}", .{stderr}); + } } if (!self.base.options.disable_lld_caching) {